From 2c858368c8c4b7e82c8d134786026a62a72d2676 Mon Sep 17 00:00:00 2001 From: Randy West Date: Mon, 18 Dec 2017 18:22:03 -0500 Subject: Compute test accuracy in batches to avoid OOM on GPUs. Reported here: https://github.com/tensorflow/tensorflow/issues/136 Alternative to this for mnist_deep.py: https://github.com/tensorflow/tensorflow/pull/157 --- tensorflow/examples/tutorials/mnist/mnist_deep.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tensorflow/examples/tutorials/mnist/mnist_deep.py b/tensorflow/examples/tutorials/mnist/mnist_deep.py index 1e0294db27..2699738735 100644 --- a/tensorflow/examples/tutorials/mnist/mnist_deep.py +++ b/tensorflow/examples/tutorials/mnist/mnist_deep.py @@ -34,6 +34,8 @@ from tensorflow.examples.tutorials.mnist import input_data import tensorflow as tf +import numpy + FLAGS = None @@ -164,8 +166,13 @@ def main(_): print('step %d, training accuracy %g' % (i, train_accuracy)) train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5}) - print('test accuracy %g' % accuracy.eval(feed_dict={ - x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})) + # compute in batches to avoid OOM on GPUs + accuracy_l = [] + for i in range(50): + batch = mnist.test.next_batch(500, shuffle=False) + accuracy_l.append(accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0})) + print('test accuracy %g' % numpy.mean(accuracy_l)) + if __name__ == '__main__': parser = argparse.ArgumentParser() -- cgit v1.2.3 From 3f18817317940253e6ec0e6b412492c5add5927b Mon Sep 17 00:00:00 2001 From: Randy West Date: Mon, 18 Dec 2017 23:18:30 -0500 Subject: Fix basic arithmetic fail + make loop pythonic --- tensorflow/examples/tutorials/mnist/mnist_deep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/tutorials/mnist/mnist_deep.py b/tensorflow/examples/tutorials/mnist/mnist_deep.py index 2699738735..47d2777813 100644 --- a/tensorflow/examples/tutorials/mnist/mnist_deep.py +++ b/tensorflow/examples/tutorials/mnist/mnist_deep.py @@ -168,7 +168,7 @@ def main(_): # compute in batches to avoid OOM on GPUs accuracy_l = [] - for i in range(50): + for _ in range(20): batch = mnist.test.next_batch(500, shuffle=False) accuracy_l.append(accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0})) print('test accuracy %g' % numpy.mean(accuracy_l)) -- cgit v1.2.3 From 5eb246cb79e37b6a7006b6dead99219ffd25de31 Mon Sep 17 00:00:00 2001 From: DavidNorman Date: Wed, 16 May 2018 17:05:24 +0100 Subject: Don't do int64 tests for devices which do not support int64 --- tensorflow/compiler/tests/binary_ops_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index 1e4dd32916..64eeed8312 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -686,11 +686,12 @@ class BinaryOpsTest(XLATestCase): np.array([[10], [7], [2]], dtype=np.float32), np.float32(7), expected=np.array([[False], [False], [True]], dtype=np.bool)) - self._testBinary( - less_op, - np.array([[10], [7], [2], [-1]], dtype=np.int64), - np.int64(7), - expected=np.array([[False], [False], [True], [True]], dtype=np.bool)) + if np.int64 in self.numeric_types: + self._testBinary( + less_op, + np.array([[10], [7], [2], [-1]], dtype=np.int64), + np.int64(7), + expected=np.array([[False], [False], [True], [True]], dtype=np.bool)) for less_equal_op in [math_ops.less_equal, (lambda x, y: x <= y)]: self._testBinary( -- cgit v1.2.3 From f2e22502fd58e8d81c9e080b9242375fbf2bc772 Mon Sep 17 00:00:00 2001 From: Jesse Date: Tue, 5 Jun 2018 14:35:38 +0200 Subject: Updated line for creating global step + grammar tf.train.get_global_step() returns None if there is no global step, preventing the pruning from working. Therefore, tf.train.get_or_create_global_step() is a safer option. --- tensorflow/contrib/model_pruning/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/model_pruning/README.md b/tensorflow/contrib/model_pruning/README.md index 86f4fd6adf..50e7e5d7cd 100644 --- a/tensorflow/contrib/model_pruning/README.md +++ b/tensorflow/contrib/model_pruning/README.md @@ -66,10 +66,10 @@ is the sparsity_function_begin_step. In this equation, the sparsity_function_exponent is set to 3. ### Adding pruning ops to the training graph -The final step involves adding ops to the training graph that monitors the -distribution of the layer's weight magnitudes and determines the layer threshold -such masking all the weights below this threshold achieves the sparsity level -desired for the current training step. This can be achieved as follows: +The final step involves adding ops to the training graph that monitor the +distribution of the layer's weight magnitudes and determine the layer threshold, +such that masking all the weights below this threshold achieves the sparsity +level desired for the current training step. This can be achieved as follows: ```python tf.app.flags.DEFINE_string( @@ -79,7 +79,7 @@ tf.app.flags.DEFINE_string( with tf.graph.as_default(): # Create global step variable - global_step = tf.train.get_global_step() + global_step = tf.train.get_or_create_global_step() # Parse pruning hyperparameters pruning_hparams = pruning.get_pruning_hparams().parse(FLAGS.pruning_hparams) -- cgit v1.2.3 From f9c7fe82cb930ee26d281e4bf21211ed352d176e Mon Sep 17 00:00:00 2001 From: Jesse Date: Tue, 5 Jun 2018 14:49:04 +0200 Subject: Put some emphasis on incrementing global step Pruning will not work if the global step is not incremented --- tensorflow/contrib/model_pruning/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/model_pruning/README.md b/tensorflow/contrib/model_pruning/README.md index 50e7e5d7cd..9143d082bf 100644 --- a/tensorflow/contrib/model_pruning/README.md +++ b/tensorflow/contrib/model_pruning/README.md @@ -103,6 +103,7 @@ with tf.graph.as_default(): mon_sess.run(mask_update_op) ``` +Ensure that `global_step` is being [incremented](https://www.tensorflow.org/api_docs/python/tf/train/Optimizer#minimize), otherwise pruning will not work! ## Example: Pruning and training deep CNNs on the cifar10 dataset -- cgit v1.2.3 From e106a458dd26db58c7d5abbd4afef60f8ce33252 Mon Sep 17 00:00:00 2001 From: Jesse Date: Tue, 5 Jun 2018 15:22:07 +0200 Subject: Prevent redundant ":0" in summary names Take identical approach as is done with thresholds: using tf.Variable.op.name instead of tf.Variable.name, to prevent TensorFlow saying summary names are illegal (due to ":") --- tensorflow/contrib/model_pruning/python/pruning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/model_pruning/python/pruning.py b/tensorflow/contrib/model_pruning/python/pruning.py index 4b7af18b33..e6f9acc139 100644 --- a/tensorflow/contrib/model_pruning/python/pruning.py +++ b/tensorflow/contrib/model_pruning/python/pruning.py @@ -520,7 +520,7 @@ class Pruning(object): thresholds = get_thresholds() for index, mask in enumerate(masks): if not self._exists_in_do_not_prune_list(mask.name): - summary.scalar(mask.name + '/sparsity', nn_impl.zero_fraction(mask)) + summary.scalar(mask.op.name + '/sparsity', nn_impl.zero_fraction(mask)) summary.scalar(thresholds[index].op.name + '/threshold', thresholds[index]) -- cgit v1.2.3 From 90b28b7316edb644b71b01edaaa8553d5913fc19 Mon Sep 17 00:00:00 2001 From: Jesse Date: Wed, 6 Jun 2018 16:07:20 +0200 Subject: Removed redundant use of enumeration Since every mask has an accompanying threshold, zip(masks, thresholds) can be used instead of enumerate(masks) and calling thresholds by index. --- tensorflow/contrib/model_pruning/python/pruning.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/model_pruning/python/pruning.py b/tensorflow/contrib/model_pruning/python/pruning.py index e6f9acc139..d843fa26d5 100644 --- a/tensorflow/contrib/model_pruning/python/pruning.py +++ b/tensorflow/contrib/model_pruning/python/pruning.py @@ -518,11 +518,10 @@ class Pruning(object): summary.scalar('last_mask_update_step', self._last_update_step) masks = get_masks() thresholds = get_thresholds() - for index, mask in enumerate(masks): + for mask, threshold in zip(masks, thresholds): if not self._exists_in_do_not_prune_list(mask.name): summary.scalar(mask.op.name + '/sparsity', nn_impl.zero_fraction(mask)) - summary.scalar(thresholds[index].op.name + '/threshold', - thresholds[index]) + summary.scalar(threshold.op.name + '/threshold', threshold) def print_hparams(self): logging.info(self._spec.to_json()) -- cgit v1.2.3 From 02b7fa3dfe3e82ca61581bf3365788c8acaa2b19 Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Wed, 6 Jun 2018 14:04:40 -0700 Subject: Adding a constraint for the setuptools version. --- tensorflow/tools/pip_package/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 78d955c637..97f625e7e9 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -54,6 +54,7 @@ REQUIRED_PACKAGES = [ 'numpy >= 1.13.3', 'six >= 1.10.0', 'protobuf >= 3.4.0', + 'setuptools <= 39.1.0', 'tensorboard >= 1.8.0, < 1.9.0', 'termcolor >= 1.1.0', ] -- cgit v1.2.3 From da3f4f86267a42f1a7780222143d79b167a75eb1 Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Wed, 6 Jun 2018 14:27:59 -0700 Subject: Removing the force downgrade install. --- tensorflow/tools/ci_build/builds/pip.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensorflow/tools/ci_build/builds/pip.sh b/tensorflow/tools/ci_build/builds/pip.sh index 883bb93647..5fa75e1d61 100755 --- a/tensorflow/tools/ci_build/builds/pip.sh +++ b/tensorflow/tools/ci_build/builds/pip.sh @@ -322,10 +322,6 @@ create_activate_virtualenv_and_install_tensorflow() { pip install -v ${PIP_FLAGS} ${WHL_PATH} || \ die "pip install (forcing to reinstall tensorflow) FAILED" echo "Successfully installed pip package ${TF_WHEEL_PATH}" - - # Force downgrade setuptools. - pip install --upgrade setuptools==39.1.0 - } ################################################################################ -- cgit v1.2.3 From 60cb7f88afda606df2b700ce0bb662f22e1a7709 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 7 Jun 2018 12:53:11 -0700 Subject: Consolidate `tf.data` release notes. --- RELEASE.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index c1ed69bd45..8f76e7efb4 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -14,8 +14,13 @@ ## Bug Fixes and Other Changes * `tf.data`: - * The `DatasetBase::DebugString()` method is now `const`. - * Added the `tf.contrib.data.sample_from_datasets()` API for randomly sampling from multiple datasets. + * `Dataset.from_generator()` now accepts an `args` list, in order to create nested generators. + * `Dataset.list_files()` now produces determinstic results when `shuffle=False` or a `seed` is passed. + * `tf.contrib.data.sample_from_datasets()` and `tf.contrib.data.choose_from_datasets()` make it easier to sample or deterministically choose elements from multiple datasets. + * `tf.contrib.data.make_csv_dataset()` now supports line breaks in quoted strings, and two infrequently used arguments removed. + * (C++) `DatasetBase::DebugString()` is now `const`. + * (C++) `DatasetBase::MakeIterator()` has been renamed to `DatasetBase::MakeIteratorInternal()`. + * (C++) `IteratorBase::Initialize()` method was added to support raising errors during iterator construction. * Eager Execution: * `tf.keras`: * Move Keras code out of _impl folder and remove API files. @@ -24,8 +29,6 @@ * Accelerated Linear Algebra (XLA): * TensorFlow Debugger (tfdbg) CLI: * `tf.contrib`: - * Add `tf.contrib.data.choose_from_datasets()`. - * `tf.contrib.data.make_csv_dataset()` now supports line breaks in quoted strings. Two arguments were removed from `make_csv_dataset`. * `tf.contrib.framework.zero_initializer` supports ResourceVariable. * Adding "constrained_optimization" to tensorflow/contrib. * Other: @@ -35,7 +38,6 @@ * More consistent GcsFileSystem behavior for certain reads past EOF. * Update benchmark for tf.scan to match ranges across eager and graph modes. * Fixed bug in `tf.reduce_prod gradient` for complex dtypes. - * Add optional `args` argument to `Dataset.from_generator()`. * Allow the use of '.' in variables (e.g. "hparams.parse('a.b=1.0')"), which would previously raise an error. This will correspond to an attribute name with an embedded '.' symbol (e.g. 'a.b'), which can only be accessed indirectly (e.g. through getattr and setattr). To set this up the user will first need to explicitly add the variable to the hparam object (e.g. "hparams.add_hparam(name='a.b', value=0.0)"). * Benchmark for tf.scan in graph and eager modes. * Added complex128 support to FFT, FFT2D, FFT3D, IFFT, IFFT2D, and IFFT3D. @@ -45,7 +47,6 @@ * LinearOperator[1D,2D,3D]Circulant added to `tensorflow.linalg`. * Conv3D, Conv3DBackpropInput, Conv3DBackpropFilter now supports arbitrary. * Added `tf.train.Checkpoint` for reading/writing object-based checkpoints. - * `Dataset.list_files()` now produces determinstic results when `shuffle=False` or a `seed` is passed. * Added LinearOperatorKronecker, a dense-free implementation of the Kronecker Product. * Allow LinearOperator to broadcast. * SavedModelBuilder will now deduplicate asset names that point to files with the same basename and the same contents. Note that this may result in new asset files included in SavedModels in cases where assets with the same name but different contents were previously overwriting each other. -- cgit v1.2.3 From d3b482dadfa1b59ec04ee668ebd899e6bcb4b7b8 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 8 Jun 2018 14:55:26 -0400 Subject: Update RELEASE.md (r1.9) for tfdbg and XLA --- RELEASE.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 8f76e7efb4..879ce6e440 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -26,8 +26,7 @@ * Move Keras code out of _impl folder and remove API files. * `tf.keras.Model.save_weights` now saves in TensorFlow format by default. * Enable dataset iterators to be passed to `tf.keras.Model` training/eval methods. -* Accelerated Linear Algebra (XLA): -* TensorFlow Debugger (tfdbg) CLI: +* TensorFlow Debugger (tfdbg) CLI: fix an issue in which the TensorBoard Debugger Plugin could not handle total source file size exceeding gRPC message size limit (4 MB). * `tf.contrib`: * `tf.contrib.framework.zero_initializer` supports ResourceVariable. * Adding "constrained_optimization" to tensorflow/contrib. -- cgit v1.2.3 From a08c8a79f3d0ea5a7fac74d8f5e9da5def89170b Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 4 Jun 2018 11:11:06 -0700 Subject: Fix visibility for tf.keras.__version__ PiperOrigin-RevId: 199161696 --- tensorflow/python/keras/__init__.py | 4 ++++ tensorflow/python/keras/integration_test.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/tensorflow/python/keras/__init__.py b/tensorflow/python/keras/__init__.py index 197f306097..3493069a5b 100644 --- a/tensorflow/python/keras/__init__.py +++ b/tensorflow/python/keras/__init__.py @@ -41,8 +41,12 @@ from tensorflow.python.keras.layers import Input from tensorflow.python.keras.models import Model from tensorflow.python.keras.models import Sequential +from tensorflow.python.util.tf_export import tf_export + __version__ = '2.1.6-tf' +tf_export('keras.__version__').export_constant(__name__, '__version__') + del absolute_import del division del print_function diff --git a/tensorflow/python/keras/integration_test.py b/tensorflow/python/keras/integration_test.py index 2e83544d97..2a05699407 100644 --- a/tensorflow/python/keras/integration_test.py +++ b/tensorflow/python/keras/integration_test.py @@ -29,6 +29,9 @@ from tensorflow.python.platform import test class KerasIntegrationTest(test.TestCase): + def test_version(self): + self.assertTrue(keras.__version__.endswith('-tf')) + def test_vector_classification_sequential(self): with self.test_session(): np.random.seed(1337) -- cgit v1.2.3 From 0eac1ebafc1e16e6440658d6b431998f3e682bbc Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Mon, 4 Jun 2018 14:46:38 -0700 Subject: Add various missing aliases for symbols in tf.keras submodules. PiperOrigin-RevId: 199198086 --- tensorflow/python/keras/losses.py | 35 ++++++++++++++--- tensorflow/python/ops/init_ops.py | 21 +++++++---- .../tensorflow.keras.initializers.constant.pbtxt | 18 +++++++++ .../tensorflow.keras.initializers.identity.pbtxt | 18 +++++++++ .../tensorflow.keras.initializers.normal.pbtxt | 18 +++++++++ .../tensorflow.keras.initializers.ones.pbtxt | 18 +++++++++ .../tensorflow.keras.initializers.orthogonal.pbtxt | 18 +++++++++ .../api/golden/tensorflow.keras.initializers.pbtxt | 40 ++++++++++++++++++++ ...nsorflow.keras.initializers.random_normal.pbtxt | 18 +++++++++ ...sorflow.keras.initializers.random_uniform.pbtxt | 18 +++++++++ ...rflow.keras.initializers.truncated_normal.pbtxt | 18 +++++++++ .../tensorflow.keras.initializers.uniform.pbtxt | 18 +++++++++ .../tensorflow.keras.initializers.zeros.pbtxt | 18 +++++++++ .../tools/api/golden/tensorflow.keras.losses.pbtxt | 44 ++++++++++++++++++++++ .../api/golden/tensorflow.keras.metrics.pbtxt | 44 ++++++++++++++++++++++ 15 files changed, 350 insertions(+), 14 deletions(-) create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.initializers.constant.pbtxt create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.initializers.identity.pbtxt create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.initializers.normal.pbtxt create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.initializers.ones.pbtxt create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.initializers.orthogonal.pbtxt create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.initializers.random_normal.pbtxt create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.initializers.random_uniform.pbtxt create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.initializers.truncated_normal.pbtxt create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.initializers.uniform.pbtxt create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.initializers.zeros.pbtxt diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py index d82ebd9c31..9f548bfe04 100644 --- a/tensorflow/python/keras/losses.py +++ b/tensorflow/python/keras/losses.py @@ -30,19 +30,31 @@ from tensorflow.python.util.tf_export import tf_export @tf_export('keras.metrics.mean_squared_error', - 'keras.losses.mean_squared_error') + 'keras.metrics.mse', + 'keras.metrics.MSE', + 'keras.losses.mean_squared_error', + 'keras.losses.mse', + 'keras.losses.MSE') def mean_squared_error(y_true, y_pred): return K.mean(math_ops.square(y_pred - y_true), axis=-1) @tf_export('keras.metrics.mean_absolute_error', - 'keras.losses.mean_absolute_error') + 'keras.metrics.mae', + 'keras.metrics.MAE', + 'keras.losses.mean_absolute_error', + 'keras.losses.mae', + 'keras.losses.MAE') def mean_absolute_error(y_true, y_pred): return K.mean(math_ops.abs(y_pred - y_true), axis=-1) @tf_export('keras.metrics.mean_absolute_percentage_error', - 'keras.losses.mean_absolute_percentage_error') + 'keras.metrics.mape', + 'keras.metrics.MAPE', + 'keras.losses.mean_absolute_percentage_error', + 'keras.losses.mape', + 'keras.losses.MAPE') def mean_absolute_percentage_error(y_true, y_pred): diff = math_ops.abs( (y_true - y_pred) / K.clip(math_ops.abs(y_true), K.epsilon(), None)) @@ -50,7 +62,11 @@ def mean_absolute_percentage_error(y_true, y_pred): @tf_export('keras.metrics.mean_squared_logarithmic_error', - 'keras.losses.mean_squared_logarithmic_error') + 'keras.metrics.msle', + 'keras.metrics.MSLE', + 'keras.losses.mean_squared_logarithmic_error', + 'keras.losses.msle', + 'keras.losses.MSLE') def mean_squared_logarithmic_error(y_true, y_pred): first_log = math_ops.log(K.clip(y_pred, K.epsilon(), None) + 1.) second_log = math_ops.log(K.clip(y_true, K.epsilon(), None) + 1.) @@ -117,7 +133,11 @@ def binary_crossentropy(y_true, y_pred): @tf_export('keras.metrics.kullback_leibler_divergence', - 'keras.losses.kullback_leibler_divergence') + 'keras.metrics.kld', + 'keras.metrics.KLD', + 'keras.losses.kullback_leibler_divergence', + 'keras.losses.kld', + 'keras.losses.KLD') def kullback_leibler_divergence(y_true, y_pred): y_true = K.clip(y_true, K.epsilon(), 1) y_pred = K.clip(y_pred, K.epsilon(), 1) @@ -129,7 +149,10 @@ def poisson(y_true, y_pred): return K.mean(y_pred - y_true * math_ops.log(y_pred + K.epsilon()), axis=-1) -@tf_export('keras.metrics.cosine_proximity', 'keras.losses.cosine_proximity') +@tf_export('keras.metrics.cosine_proximity', + 'keras.metrics.cosine', + 'keras.losses.cosine_proximity', + 'keras.losses.cosine') def cosine_proximity(y_true, y_pred): y_true = nn.l2_normalize(y_true, axis=-1) y_pred = nn.l2_normalize(y_pred, axis=-1) diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py index 1f8d8dc4f3..2df230d470 100644 --- a/tensorflow/python/ops/init_ops.py +++ b/tensorflow/python/ops/init_ops.py @@ -86,7 +86,7 @@ class Initializer(object): @tf_export("keras.initializers.Zeros", "initializers.zeros", - "zeros_initializer") + "zeros_initializer", "keras.initializers.zeros") class Zeros(Initializer): """Initializer that generates tensors initialized to 0.""" @@ -102,7 +102,8 @@ class Zeros(Initializer): return {"dtype": self.dtype.name} -@tf_export("keras.initializers.Ones", "initializers.ones", "ones_initializer") +@tf_export("keras.initializers.Ones", "initializers.ones", "ones_initializer", + "keras.initializers.ones") class Ones(Initializer): """Initializer that generates tensors initialized to 1.""" @@ -119,7 +120,7 @@ class Ones(Initializer): @tf_export("keras.initializers.Constant", "initializers.constant", - "constant_initializer") + "constant_initializer", "keras.initializers.constant") class Constant(Initializer): """Initializer that generates tensors with constant values. @@ -225,7 +226,8 @@ class Constant(Initializer): @tf_export("keras.initializers.RandomUniform", "initializers.random_uniform", - "random_uniform_initializer") + "random_uniform_initializer", "keras.initializers.uniform", + "keras.initializers.random_uniform") class RandomUniform(Initializer): """Initializer that generates tensors with a uniform distribution. @@ -262,7 +264,8 @@ class RandomUniform(Initializer): @tf_export("keras.initializers.RandomNormal", "initializers.random_normal", - "random_normal_initializer") + "random_normal_initializer", "keras.initializers.normal", + "keras.initializers.random_normal") class RandomNormal(Initializer): """Initializer that generates tensors with a normal distribution. @@ -299,7 +302,8 @@ class RandomNormal(Initializer): @tf_export("keras.initializers.TruncatedNormal", - "initializers.truncated_normal", "truncated_normal_initializer") + "initializers.truncated_normal", "truncated_normal_initializer", + "keras.initializers.truncated_normal") class TruncatedNormal(Initializer): """Initializer that generates a truncated normal distribution. @@ -482,7 +486,7 @@ class VarianceScaling(Initializer): @tf_export("keras.initializers.Orthogonal", "initializers.orthogonal", - "orthogonal_initializer") + "orthogonal_initializer", "keras.initializers.orthogonal") class Orthogonal(Initializer): """Initializer that generates an orthogonal matrix. @@ -1062,7 +1066,8 @@ class ConvolutionOrthogonal3D(ConvolutionOrthogonal): return self._dict_to_tensor(p, ksize, ksize, ksize) -@tf_export("keras.initializers.Identity", "initializers.identity") +@tf_export("keras.initializers.Identity", "initializers.identity", + "keras.initializers.identity") class Identity(Initializer): """Initializer that generates the identity matrix. diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.constant.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.constant.pbtxt new file mode 100644 index 0000000000..bddc37b907 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.constant.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.keras.initializers.constant" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'value\', \'dtype\', \'verify_shape\'], varargs=None, keywords=None, defaults=[\'0\', \"\", \'False\'], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.identity.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.identity.pbtxt new file mode 100644 index 0000000000..a4c5a61490 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.identity.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.keras.initializers.identity" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'gain\', \'dtype\'], varargs=None, keywords=None, defaults=[\'1.0\', \"\"], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.normal.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.normal.pbtxt new file mode 100644 index 0000000000..7485772784 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.normal.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.keras.initializers.normal" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'mean\', \'stddev\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \'None\', \"\"], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.ones.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.ones.pbtxt new file mode 100644 index 0000000000..a89f78d1e1 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.ones.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.keras.initializers.ones" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'dtype\'], varargs=None, keywords=None, defaults=[\"\"], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.orthogonal.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.orthogonal.pbtxt new file mode 100644 index 0000000000..ee1e9bbae2 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.orthogonal.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.keras.initializers.orthogonal" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'gain\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=[\'1.0\', \'None\', \"\"], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.pbtxt index 093c56595b..14a667870d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.initializers.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.pbtxt @@ -40,6 +40,46 @@ tf_module { name: "Zeros" mtype: "" } + member { + name: "constant" + mtype: "" + } + member { + name: "identity" + mtype: "" + } + member { + name: "normal" + mtype: "" + } + member { + name: "ones" + mtype: "" + } + member { + name: "orthogonal" + mtype: "" + } + member { + name: "random_normal" + mtype: "" + } + member { + name: "random_uniform" + mtype: "" + } + member { + name: "truncated_normal" + mtype: "" + } + member { + name: "uniform" + mtype: "" + } + member { + name: "zeros" + mtype: "" + } member_method { name: "deserialize" argspec: "args=[\'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.random_normal.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.random_normal.pbtxt new file mode 100644 index 0000000000..a6df1e87a3 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.random_normal.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.keras.initializers.random_normal" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'mean\', \'stddev\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \'None\', \"\"], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.random_uniform.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.random_uniform.pbtxt new file mode 100644 index 0000000000..37a0fa0d55 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.random_uniform.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.keras.initializers.random_uniform" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'minval\', \'maxval\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0\', \'None\', \'None\', \"\"], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.truncated_normal.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.truncated_normal.pbtxt new file mode 100644 index 0000000000..f97e93f0b7 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.truncated_normal.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.keras.initializers.truncated_normal" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'mean\', \'stddev\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0.0\', \'1.0\', \'None\', \"\"], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.uniform.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.uniform.pbtxt new file mode 100644 index 0000000000..58186b1383 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.uniform.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.keras.initializers.uniform" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'minval\', \'maxval\', \'seed\', \'dtype\'], varargs=None, keywords=None, defaults=[\'0\', \'None\', \'None\', \"\"], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.keras.initializers.zeros.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.initializers.zeros.pbtxt new file mode 100644 index 0000000000..a262390687 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.keras.initializers.zeros.pbtxt @@ -0,0 +1,18 @@ +path: "tensorflow.keras.initializers.zeros" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + member_method { + name: "__init__" + argspec: "args=[\'self\', \'dtype\'], varargs=None, keywords=None, defaults=[\"\"], " + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.keras.losses.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.losses.pbtxt index ae5f6305b7..eca6b91538 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.losses.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.losses.pbtxt @@ -1,5 +1,25 @@ path: "tensorflow.keras.losses" tf_module { + member_method { + name: "KLD" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "MAE" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "MAPE" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "MSE" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "MSLE" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "binary_crossentropy" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -12,6 +32,10 @@ tf_module { name: "categorical_hinge" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "cosine" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "cosine_proximity" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -28,6 +52,10 @@ tf_module { name: "hinge" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "kld" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "kullback_leibler_divergence" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -36,6 +64,14 @@ tf_module { name: "logcosh" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "mae" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "mape" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "mean_absolute_error" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -52,6 +88,14 @@ tf_module { name: "mean_squared_logarithmic_error" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "mse" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "msle" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "poisson" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.metrics.pbtxt index 42729e4237..a97a9b5758 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.metrics.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.metrics.pbtxt @@ -1,5 +1,25 @@ path: "tensorflow.keras.metrics" tf_module { + member_method { + name: "KLD" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "MAE" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "MAPE" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "MSE" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "MSLE" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "binary_accuracy" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -16,6 +36,10 @@ tf_module { name: "categorical_crossentropy" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "cosine" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "cosine_proximity" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -32,10 +56,22 @@ tf_module { name: "hinge" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "kld" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "kullback_leibler_divergence" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "mae" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "mape" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "mean_absolute_error" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" @@ -52,6 +88,14 @@ tf_module { name: "mean_squared_logarithmic_error" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "mse" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "msle" + argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "poisson" argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None" -- cgit v1.2.3 From 7c33a7751d77cfd70a5c441da369440f4f6b633a Mon Sep 17 00:00:00 2001 From: Pavithra Vijay Date: Thu, 7 Jun 2018 09:20:57 -0700 Subject: Fix bug due to incorrect nesting of return statement in eager iterator evaluation. PiperOrigin-RevId: 199645638 --- tensorflow/python/keras/engine/training_eager.py | 10 ++-- .../python/keras/engine/training_eager_test.py | 54 ++++++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py index 081e46aa66..a70b488f25 100644 --- a/tensorflow/python/keras/engine/training_eager.py +++ b/tensorflow/python/keras/engine/training_eager.py @@ -501,11 +501,11 @@ def iterator_test_loop(model, inputs, steps, verbose=0): if verbose == 1: progbar.update(step_index + 1) - for i in range(len(outs)): - outs[i] /= num_samples - if len(outs) == 1: - return outs[0] - return outs + for i in range(len(outs)): + outs[i] /= num_samples + if len(outs) == 1: + return outs[0] + return outs def batch_test_loop(model, diff --git a/tensorflow/python/keras/engine/training_eager_test.py b/tensorflow/python/keras/engine/training_eager_test.py index d9446fd437..7906d208eb 100644 --- a/tensorflow/python/keras/engine/training_eager_test.py +++ b/tensorflow/python/keras/engine/training_eager_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np +from tensorflow.python.data.ops import dataset_ops from tensorflow.python import keras from tensorflow.python.framework import ops from tensorflow.python.framework import test_util as tf_test_util @@ -670,6 +671,59 @@ class CorrectnessTest(test.TestCase): outs = model.evaluate(x, y) self.assertEqual(outs[1], 0.) + @tf_test_util.run_in_graph_and_eager_modes() + def test_loss_correctness_with_iterator(self): + # Test that training loss is the same in eager and graph + # (by comparing it to a reference value in a deterministic case) + model = keras.Sequential() + model.add( + keras.layers.Dense( + 3, activation='relu', input_dim=4, kernel_initializer='ones')) + model.add( + keras.layers.Dense(2, activation='softmax', kernel_initializer='ones')) + model.compile( + loss='sparse_categorical_crossentropy', + optimizer=RMSPropOptimizer(learning_rate=0.001)) + x = np.ones((100, 4), dtype=np.float32) + np.random.seed(123) + y = np.random.randint(0, 1, size=(100, 1)) + dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + iterator = dataset.make_one_shot_iterator() + history = model.fit(iterator, epochs=1, steps_per_epoch=10) + self.assertEqual(np.around(history.history['loss'][-1], decimals=4), 0.6173) + + @tf_test_util.run_in_graph_and_eager_modes() + def test_metrics_correctness_with_iterator(self): + model = keras.Sequential() + model.add( + keras.layers.Dense( + 8, activation='relu', input_dim=4, kernel_initializer='ones')) + model.add( + keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')) + model.compile( + loss='binary_crossentropy', + metrics=['accuracy'], + optimizer=RMSPropOptimizer(learning_rate=0.001)) + np.random.seed(123) + x = np.random.randint(10, size=(100, 4)).astype(np.float32) + y = np.random.randint(2, size=(100, 1)).astype(np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) + dataset = dataset.batch(10) + iterator = dataset.make_one_shot_iterator() + outs = model.evaluate(iterator, steps=10) + self.assertEqual(np.around(outs[1], decimals=1), 0.5) + + y = np.zeros((100, 1), dtype=np.float32) + dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) + dataset = dataset.repeat(100) + dataset = dataset.batch(10) + iterator = dataset.make_one_shot_iterator() + outs = model.evaluate(iterator, steps=10) + self.assertEqual(outs[1], 0.) + + if __name__ == '__main__': ops.enable_eager_execution() test.main() -- cgit v1.2.3 From 5177fd2f9acb9b46b9182ad782bb8b7b9386baeb Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 5 Jun 2018 15:59:21 -0700 Subject: Only calls compare function if values were read from event file PiperOrigin-RevId: 199373169 --- tensorflow/python/estimator/exporter.py | 7 +++--- tensorflow/python/estimator/exporter_test.py | 34 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/estimator/exporter.py b/tensorflow/python/estimator/exporter.py index a7212bb83e..766ea23f2a 100644 --- a/tensorflow/python/estimator/exporter.py +++ b/tensorflow/python/estimator/exporter.py @@ -360,9 +360,10 @@ class BestExporter(Exporter): for value in event.summary.value: if value.HasField('simple_value'): event_eval_result[value.tag] = value.simple_value - if best_eval_result is None or self._compare_fn( - best_eval_result, event_eval_result): - best_eval_result = event_eval_result + if event_eval_result: + if best_eval_result is None or self._compare_fn( + best_eval_result, event_eval_result): + best_eval_result = event_eval_result return best_eval_result diff --git a/tensorflow/python/estimator/exporter_test.py b/tensorflow/python/estimator/exporter_test.py index 4cb4bffc8d..c4b006955c 100644 --- a/tensorflow/python/estimator/exporter_test.py +++ b/tensorflow/python/estimator/exporter_test.py @@ -148,6 +148,40 @@ class BestExporterTest(test.TestCase): "checkpoint_path", {"loss": 20}, False) self.assertEqual(None, export_result) + def test_best_exporter_with_empty_event(self): + + def _serving_input_receiver_fn(): + pass + + export_dir_base = tempfile.mkdtemp() + gfile.MkDir(export_dir_base) + gfile.MkDir(export_dir_base + "/export") + gfile.MkDir(export_dir_base + "/eval") + + eval_dir_base = os.path.join(export_dir_base, "eval_continuous") + estimator_lib._write_dict_to_summary(eval_dir_base, {}, 1) + estimator_lib._write_dict_to_summary(eval_dir_base, {"loss": 60}, 2) + + exporter = exporter_lib.BestExporter( + name="best_exporter", + serving_input_receiver_fn=_serving_input_receiver_fn, + event_file_pattern="eval_continuous/*.tfevents.*", + assets_extra={"from/path": "to/path"}, + as_text=False, + exports_to_keep=1) + + estimator = test.mock.Mock(spec=estimator_lib.Estimator) + estimator.model_dir = export_dir_base + estimator.export_savedmodel.return_value = "export_result_path" + + export_result = exporter.export(estimator, export_dir_base, + "checkpoint_path", {"loss": 100}, False) + self.assertEqual(None, export_result) + + export_result = exporter.export(estimator, export_dir_base, + "checkpoint_path", {"loss": 10}, False) + self.assertEqual("export_result_path", export_result) + def test_garbage_collect_exports(self): export_dir_base = tempfile.mkdtemp() gfile.MkDir(export_dir_base) -- cgit v1.2.3 From 4fe8d4a14936dc38558a858283574993909c9895 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sun, 27 May 2018 10:49:12 -0700 Subject: TPUEstimator.export_savedmodel() saves a SavedModel with both TPU and CPU graphs. PiperOrigin-RevId: 198229550 --- tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 4465833f88..c8c08a5a63 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -1807,7 +1807,7 @@ class TPUEstimator(estimator_lib.Estimator): export_outputs['classes'] = export_output_lib.ClassificationOutput(classes=classes) - tpu.outside_compilation(host_call, logits) + tpu.outside_compilation(host_call, [logits]) ... ``` @@ -1969,7 +1969,7 @@ class TPUEstimator(estimator_lib.Estimator): input_receiver_fn_map[mode]} export_tags = [tag_constants.SERVING, tag_constants.TPU] mode = _REWRITE_FOR_INFERENCE_MODE - try: + if self._export_to_tpu: (super(TPUEstimator, self). _add_meta_graph_for_mode(builder, input_receiver_fn_map, @@ -1978,9 +1978,6 @@ class TPUEstimator(estimator_lib.Estimator): save_variables=False, mode=mode, export_tags=export_tags)) - except Exception as error: # pylint: disable=broad-except - logging.warning('Saving meta graph for TPU failed: {}.' - .format(str(error))) def _call_model_fn(self, features, labels, mode, config): if mode == _REWRITE_FOR_INFERENCE_MODE: -- cgit v1.2.3 -- cgit v1.2.3 From 982f3e3038f8d07964b2c58843a51bd9745a8990 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Jun 2018 16:32:20 -0700 Subject: Allow user to opt out of saving metagraph for TPU with TPUEstimator.export_output(). PiperOrigin-RevId: 198944144 --- tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index c8c08a5a63..7c770912b4 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -1830,6 +1830,7 @@ class TPUEstimator(estimator_lib.Estimator): predict_batch_size=None, batch_axis=None, eval_on_tpu=True, + export_to_tpu=True, warm_start_from=None): """Constructs an `TPUEstimator` instance. @@ -1872,6 +1873,8 @@ class TPUEstimator(estimator_lib.Estimator): False or `PER_HOST_V2`, batch_axis is ignored. eval_on_tpu: If False, evaluation runs on CPU or GPU. In this case, the model_fn must return `EstimatorSpec` when called with `mode` as `EVAL`. + export_to_tpu: If True, `export_savedmodel()` exports a metagraph for + serving on TPU besides the one on CPU. warm_start_from: Optional string filepath to a checkpoint or SavedModel to warm-start from, or a `tf.estimator.WarmStartSettings` object to fully configure warm-starting. If the string @@ -1943,6 +1946,8 @@ class TPUEstimator(estimator_lib.Estimator): use_tpu, eval_on_tpu) + self._export_to_tpu = export_to_tpu + self._is_input_fn_invoked = None def _add_meta_graph_for_mode(self, @@ -1965,11 +1970,11 @@ class TPUEstimator(estimator_lib.Estimator): save_variables, mode=mode) - input_receiver_fn_map = {_REWRITE_FOR_INFERENCE_MODE: - input_receiver_fn_map[mode]} - export_tags = [tag_constants.SERVING, tag_constants.TPU] - mode = _REWRITE_FOR_INFERENCE_MODE if self._export_to_tpu: + input_receiver_fn_map = {_REWRITE_FOR_INFERENCE_MODE: + input_receiver_fn_map[mode]} + export_tags = [tag_constants.SERVING, tag_constants.TPU] + mode = _REWRITE_FOR_INFERENCE_MODE (super(TPUEstimator, self). _add_meta_graph_for_mode(builder, input_receiver_fn_map, -- cgit v1.2.3 From 6cc2741eb1c9b19742b32b8edda39090afbf5abf Mon Sep 17 00:00:00 2001 From: DavidNorman Date: Tue, 12 Jun 2018 09:32:53 +0100 Subject: Fix python lint errors --- tensorflow/compiler/tests/binary_ops_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index 64eeed8312..823afbbbdc 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -687,11 +687,12 @@ class BinaryOpsTest(XLATestCase): np.float32(7), expected=np.array([[False], [False], [True]], dtype=np.bool)) if np.int64 in self.numeric_types: - self._testBinary( - less_op, - np.array([[10], [7], [2], [-1]], dtype=np.int64), - np.int64(7), - expected=np.array([[False], [False], [True], [True]], dtype=np.bool)) + self._testBinary( + less_op, + np.array([[10], [7], [2], [-1]], dtype=np.int64), + np.int64(7), + expected=np.array( + [[False], [False], [True], [True]], dtype=np.bool)) for less_equal_op in [math_ops.less_equal, (lambda x, y: x <= y)]: self._testBinary( -- cgit v1.2.3 From fd44596bc4b3ea8c67838b728b450a44e35c1b89 Mon Sep 17 00:00:00 2001 From: Anna R Date: Mon, 11 Jun 2018 17:21:06 -0700 Subject: Merging --- tensorflow/tools/api/generator/BUILD | 24 +++++++ .../tools/api/generator/create_python_api.py | 54 +++++++++++++-- tensorflow/tools/api/generator/doc_srcs.py | 65 ++++++++++++++++++ tensorflow/tools/api/generator/doc_srcs_test.py | 80 ++++++++++++++++++++++ 4 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 tensorflow/tools/api/generator/doc_srcs.py create mode 100644 tensorflow/tools/api/generator/doc_srcs_test.py diff --git a/tensorflow/tools/api/generator/BUILD b/tensorflow/tools/api/generator/BUILD index f0c5877a90..3a28153e52 100644 --- a/tensorflow/tools/api/generator/BUILD +++ b/tensorflow/tools/api/generator/BUILD @@ -5,12 +5,21 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) +load("//tensorflow/tools/api/generator:api_gen.bzl", "TENSORFLOW_API_INIT_FILES") + +py_library( + name = "doc_srcs", + srcs = ["doc_srcs.py"], + srcs_version = "PY2AND3", +) + py_binary( name = "create_python_api", srcs = ["create_python_api.py"], srcs_version = "PY2AND3", visibility = ["//visibility:public"], deps = [ + ":doc_srcs", "//tensorflow/python:no_contrib", ], ) @@ -24,3 +33,18 @@ py_test( "//tensorflow/python:client_testlib", ], ) + +py_test( + name = "tensorflow_doc_srcs_test", + srcs = ["doc_srcs_test.py"], + args = [ + "--package=tensorflow.python", + ] + TENSORFLOW_API_INIT_FILES, + main = "doc_srcs_test.py", + srcs_version = "PY2AND3", + deps = [ + ":doc_srcs", + "//tensorflow/python:client_testlib", + "//tensorflow/python:no_contrib", + ], +) diff --git a/tensorflow/tools/api/generator/create_python_api.py b/tensorflow/tools/api/generator/create_python_api.py index 9f210ad42b..31f287b7fe 100644 --- a/tensorflow/tools/api/generator/create_python_api.py +++ b/tensorflow/tools/api/generator/create_python_api.py @@ -25,6 +25,8 @@ import os import sys from tensorflow.python.util import tf_decorator +from tensorflow.python.util import tf_export +from tensorflow.tools.api.generator import doc_srcs _API_CONSTANTS_ATTR = '_tf_api_constants' @@ -36,10 +38,9 @@ _SYMBOLS_TO_SKIP_EXPLICITLY = { # would have side effects. 'tensorflow.python.platform.flags.FLAGS' } -_GENERATED_FILE_HEADER = """\"\"\"Imports for Python API. - -This file is MACHINE GENERATED! Do not edit. -Generated by: tensorflow/tools/api/generator/create_python_api.py script. +_GENERATED_FILE_HEADER = """# This file is MACHINE GENERATED! Do not edit. +# Generated by: tensorflow/tools/api/generator/create_python_api.py script. +\"\"\"%s \"\"\" from __future__ import print_function @@ -254,6 +255,44 @@ def get_module(dir_path, relative_to_dir): return dir_path.replace('/', '.').strip('.') +def get_module_docstring(module_name, package): + """Get docstring for the given module. + + This method looks for docstring in the following order: + 1. Checks if module has a docstring specified in doc_srcs. + 2. Checks if module has a docstring source module specified + in doc_srcs. If it does, gets docstring from that module. + 3. Checks if module with module_name exists under base package. + If it does, gets docstring from that module. + 4. Returns a default docstring. + + Args: + module_name: module name relative to tensorflow + (excluding 'tensorflow.' prefix) to get a docstring for. + package: Base python package containing python with target tf_export + decorators. + + Returns: + One-line docstring to describe the module. + """ + # Module under base package to get a docstring from. + docstring_module_name = module_name + + if module_name in doc_srcs.TENSORFLOW_DOC_SOURCES: + docsrc = doc_srcs.TENSORFLOW_DOC_SOURCES[module_name] + if docsrc.docstring: + return docsrc.docstring + if docsrc.docstring_module_name: + docstring_module_name = docsrc.docstring_module_name + + docstring_module_name = package + '.' + docstring_module_name + if (docstring_module_name in sys.modules and + sys.modules[docstring_module_name].__doc__): + return sys.modules[docstring_module_name].__doc__ + + return 'Public API for tf.%s namespace.' % module_name + + def create_api_files( output_files, package, root_init_template, output_dir): """Creates __init__.py files for the Python API. @@ -296,7 +335,10 @@ def create_api_files( continue contents = '' if module or not root_init_template: - contents = _GENERATED_FILE_HEADER + text + _GENERATED_FILE_FOOTER + contents = ( + _GENERATED_FILE_HEADER % + get_module_docstring(module, package) + text + + _GENERATED_FILE_FOOTER) else: # Read base init file with open(root_init_template, 'r') as root_init_template_file: @@ -309,7 +351,7 @@ def create_api_files( raise ValueError( 'Missing outputs for python_api_gen genrule:\n%s.' 'Make sure all required outputs are in the ' - 'tensorflow/tools/api/generator/BUILD file.' % + 'tensorflow/tools/api/generator/api_gen.bzl file.' % ',\n'.join(sorted(missing_output_files))) diff --git a/tensorflow/tools/api/generator/doc_srcs.py b/tensorflow/tools/api/generator/doc_srcs.py new file mode 100644 index 0000000000..74f6db98fd --- /dev/null +++ b/tensorflow/tools/api/generator/doc_srcs.py @@ -0,0 +1,65 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Specifies sources of doc strings for API modules.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections + + +# Specifies docstring source for a module. +# Only one of docstring or docstring_module_name should be set. +# * If docstring is set, then we will use this docstring when +# for the module. +# * If docstring_module_name is set, then we will copy the docstring +# from docstring source module. +DocSource = collections.namedtuple( + 'DocSource', ['docstring', 'docstring_module_name']) +# Each attribute of DocSource is optional. +DocSource.__new__.__defaults__ = (None,) * len(DocSource._fields) + +TENSORFLOW_DOC_SOURCES = { + 'app': DocSource(docstring_module_name='platform.app'), + 'compat': DocSource(docstring_module_name='util.compat'), + 'distributions': DocSource( + docstring_module_name='ops.distributions.distributions'), + 'bitwise': DocSource(docstring_module_name='ops.bitwise_ops'), + 'errors': DocSource(docstring_module_name='framework.errors'), + 'gfile': DocSource(docstring_module_name='platform.gfile'), + 'graph_util': DocSource(docstring_module_name='framework.graph_util'), + 'image': DocSource(docstring_module_name='ops.image_ops'), + 'keras.estimator': DocSource(docstring_module_name='estimator.keras'), + 'linalg': DocSource(docstring_module_name='ops.linalg_ops'), + 'logging': DocSource(docstring_module_name='ops.logging_ops'), + 'losses': DocSource(docstring_module_name='ops.losses.losses'), + 'manip': DocSource(docstring_module_name='ops.manip_ops'), + 'math': DocSource(docstring_module_name='ops.math_ops'), + 'metrics': DocSource(docstring_module_name='ops.metrics'), + 'nn': DocSource(docstring_module_name='ops.nn_ops'), + 'nn.rnn_cell': DocSource(docstring_module_name='ops.rnn_cell'), + 'python_io': DocSource(docstring_module_name='lib.io.python_io'), + 'resource_loader': DocSource( + docstring_module_name='platform.resource_loader'), + 'sets': DocSource(docstring_module_name='ops.sets'), + 'sparse': DocSource(docstring_module_name='ops.sparse_ops'), + 'spectral': DocSource(docstring_module_name='ops.spectral_ops'), + 'strings': DocSource(docstring_module_name='ops.string_ops'), + 'sysconfig': DocSource(docstring_module_name='platform.sysconfig'), + 'test': DocSource(docstring_module_name='platform.test'), + 'train': DocSource(docstring_module_name='training.training'), + 'train.queue_runner': DocSource( + docstring_module_name='training.queue_runner'), +} diff --git a/tensorflow/tools/api/generator/doc_srcs_test.py b/tensorflow/tools/api/generator/doc_srcs_test.py new file mode 100644 index 0000000000..9ba95a3439 --- /dev/null +++ b/tensorflow/tools/api/generator/doc_srcs_test.py @@ -0,0 +1,80 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Tests for tensorflow.tools.api.generator.doc_srcs.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import importlib +import sys + +from tensorflow.python.platform import test +from tensorflow.tools.api.generator import doc_srcs + + +FLAGS = None + + +class DocSrcsTest(test.TestCase): + + def testModulesAreValidAPIModules(self): + for module_name in doc_srcs.TENSORFLOW_DOC_SOURCES: + # Convert module_name to corresponding __init__.py file path. + file_path = module_name.replace('.', '/') + if file_path: + file_path += '/' + file_path += '__init__.py' + + if file_path not in FLAGS.outputs: + self.assertFalse('%s is not a valid API module' % module_name) + + def testHaveDocstringOrDocstringModule(self): + for module_name, docsrc in doc_srcs.TENSORFLOW_DOC_SOURCES.items(): + if docsrc.docstring and docsrc.docstring_module_name: + self.assertFalse( + '%s contains DocSource has both a docstring and a ' + 'docstring_module_name. ' + 'Only one of "docstring" or "docstring_module_name" should be set.' + % (module_name)) + + def testDocstringModulesAreValidModules(self): + for _, docsrc in doc_srcs.TENSORFLOW_DOC_SOURCES.items(): + if docsrc.docstring_module_name: + doc_module_name = '.'.join([ + FLAGS.package, docsrc.docstring_module_name]) + if doc_module_name not in sys.modules: + sys.assertFalse( + 'docsources_module %s is not a valid module under %s.' % + (docsrc.docstring_module_name, FLAGS.package)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + 'outputs', metavar='O', type=str, nargs='+', + help='create_python_api output files.') + parser.add_argument( + '--package', type=str, + help='Base package that imports modules containing the target tf_export ' + 'decorators.') + FLAGS, unparsed = parser.parse_known_args() + + importlib.import_module(FLAGS.package) + + # Now update argv, so that unittest library does not get confused. + sys.argv = [sys.argv[0]] + unparsed + test.main() -- cgit v1.2.3 From e042e3e051d3bd6bfb63dfd4ad407a82f7d1dacc Mon Sep 17 00:00:00 2001 From: Anna R Date: Tue, 12 Jun 2018 17:47:58 -0700 Subject: Remove unused tf_export import --- tensorflow/tools/api/generator/create_python_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/tools/api/generator/create_python_api.py b/tensorflow/tools/api/generator/create_python_api.py index 31f287b7fe..e3ab056efc 100644 --- a/tensorflow/tools/api/generator/create_python_api.py +++ b/tensorflow/tools/api/generator/create_python_api.py @@ -25,7 +25,6 @@ import os import sys from tensorflow.python.util import tf_decorator -from tensorflow.python.util import tf_export from tensorflow.tools.api.generator import doc_srcs -- cgit v1.2.3 From f055a9f2f21154140785b9da7c3b2eae88e65623 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Tue, 12 Jun 2018 18:09:35 -0700 Subject: Check to ensure the Cloud TPU is ready before resolving. Cherry picking this into the TF 1.9 release. PiperOrigin-RevId: 200095692 Previous commit: 32c8013f0ab3feb139648ae759e2d0168fb5dc95 --- .../python/training/tpu_cluster_resolver.py | 3 ++ .../python/training/tpu_cluster_resolver_test.py | 44 ++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py index 880fca4ea6..935ad5ff37 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py +++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py @@ -255,6 +255,9 @@ class TPUClusterResolver(ClusterResolver): request = self._service.projects().locations().nodes().get(name=full_name) response = request.execute() + if 'state' in response and response['state'] != 'READY': + raise RuntimeError('TPU "%s" is not yet ready; state: "%s"' % + (self._tpu, response['state'])) if 'health' in response and response['health'] != 'HEALTHY': raise RuntimeError('TPU "%s" is unhealthy: "%s"' % (self._tpu, response['health'])) diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py index 5fac55fd02..7e002cc72f 100644 --- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py +++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py @@ -157,6 +157,50 @@ class TPUClusterResolverTest(test.TestCase): job { name: 'worker' tasks { key: 0 value: '10.1.2.3:8470' } } """ self._verifyClusterSpecEquality(actual_cluster_spec, expected_proto) + + @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata', + mock_request_compute_metadata) + def testUnhealthyCloudTpu(self): + tpu_map = { + 'projects/test-project/locations/us-central1-c/nodes/test-tpu-1': { + 'ipAddress': '10.1.2.3', + 'port': '8470', + 'health': 'UNHEALTHY' + } + } + + tpu_cluster_resolver = TPUClusterResolver( + project=None, + zone=None, + tpu='test-tpu-1', + coordinator_name=None, + credentials=None, + service=self.mock_service_client(tpu_map=tpu_map)) + + with self.assertRaises(RuntimeError): + tpu_cluster_resolver.cluster_spec() + + @mock.patch.object(TPUClusterResolver, '_requestComputeMetadata', + mock_request_compute_metadata) + def testNotReadyCloudTpu(self): + tpu_map = { + 'projects/test-project/locations/us-central1-c/nodes/test-tpu-1': { + 'ipAddress': '10.1.2.3', + 'port': '8470', + 'state': 'CREATING' + } + } + + tpu_cluster_resolver = TPUClusterResolver( + project=None, + zone=None, + tpu='test-tpu-1', + coordinator_name=None, + credentials=None, + service=self.mock_service_client(tpu_map=tpu_map)) + + with self.assertRaises(RuntimeError): + tpu_cluster_resolver.cluster_spec() def testSimpleSuccessfulRetrieval(self): tpu_map = { -- cgit v1.2.3 From 9a087a42293be8342570039d2c6d329a0589b773 Mon Sep 17 00:00:00 2001 From: Nick Felt Date: Wed, 13 Jun 2018 00:30:09 -0700 Subject: Update tensorboard dependency to 1.9.x --- tensorflow/tools/pip_package/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 97f625e7e9..92a1465cea 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -55,7 +55,7 @@ REQUIRED_PACKAGES = [ 'six >= 1.10.0', 'protobuf >= 3.4.0', 'setuptools <= 39.1.0', - 'tensorboard >= 1.8.0, < 1.9.0', + 'tensorboard >= 1.9.0, < 1.10.0', 'termcolor >= 1.1.0', ] -- cgit v1.2.3 From b1d0048f2be83d6c6f7e1be996ef9c8358922aa6 Mon Sep 17 00:00:00 2001 From: Pete Warden Date: Wed, 13 Jun 2018 01:06:50 -0700 Subject: Documentation for Raspberry Pi installation --- tensorflow/docs_src/install/index.md | 2 + tensorflow/docs_src/install/install_raspbian.md | 317 ++++++++++++++++++++++++ 2 files changed, 319 insertions(+) create mode 100644 tensorflow/docs_src/install/install_raspbian.md diff --git a/tensorflow/docs_src/install/index.md b/tensorflow/docs_src/install/index.md index 4f85383925..c2e5a991d4 100644 --- a/tensorflow/docs_src/install/index.md +++ b/tensorflow/docs_src/install/index.md @@ -6,6 +6,7 @@ operating systems: * macOS 10.12.6 (Sierra) or later. * Ubuntu 16.04 or later * Windows 7 or later. + * Raspbian 9.0 or later. Although you might be able to install TensorFlow on other laptop or desktop systems, we only support (and only fix issues in) the preceding configurations. @@ -16,6 +17,7 @@ that enables you to write applications in Python: * @{$install_linux$Installing TensorFlow on Ubuntu} * @{$install_mac$Installing TensorFlow on macOS} * @{$install_windows$Installing TensorFlow on Windows} + * @{$install_raspbian$Installing TensorFlow on a Raspberry Pi} * @{$install_sources$Installing TensorFlow from Sources} Many aspects of the Python TensorFlow API changed from version 0.n to 1.0. diff --git a/tensorflow/docs_src/install/install_raspbian.md b/tensorflow/docs_src/install/install_raspbian.md new file mode 100644 index 0000000000..2f425162a1 --- /dev/null +++ b/tensorflow/docs_src/install/install_raspbian.md @@ -0,0 +1,317 @@ +# Installing TensorFlow on Raspbian + +This guide explains how to install TensorFlow on a Raspberry Pi running +Raspbian. Although these instructions might also work on other Pi variants, we +have only tested (and we only support) these instructions on machines meeting +the following requirements: + +* Raspberry Pi devices running Raspbian 9.0 or higher + +## Determine how to install TensorFlow + +You must pick the mechanism by which you install TensorFlow. The supported +choices are as follows: + +* "Native" pip. +* Cross-compiling from sources. + +**We recommend pip installation.** + +## Installing with native pip + +We have uploaded the TensorFlow binaries to piwheels.org. Therefore, you can +install TensorFlow through pip. + +The [REQUIRED_PACKAGES section of +setup.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/pip_package/setup.py) +lists the packages that pip will install or upgrade. + +### Prerequisite: Python + +In order to install TensorFlow, your system must contain one of the following +Python versions: + +* Python 2.7 +* Python 3.4+ + +If your system does not already have one of the preceding Python versions, +[install](https://wiki.python.org/moin/BeginnersGuide/Download) it now. It +should already be included when Raspbian was installed though, so no extra steps +should be needed. + +### Prerequisite: pip + +[Pip](https://en.wikipedia.org/wiki/Pip_\(package_manager\)) installs and +manages software packages written in Python. If you intend to install with +native pip, then one of the following flavors of pip must be installed on your +system: + +* `pip3`, for Python 3.n (preferred). +* `pip`, for Python 2.7. + +`pip` or `pip3` was probably installed on your system when you installed Python. +To determine whether pip or pip3 is actually installed on your system, issue one +of the following commands: + +
$ pip3 -V # for Python 3.n
+$ pip -V  # for Python 2.7
+ +If it gives the error "Command not found", then the package has not been +installed yet. To install if for the first time, run: + +
$ sudo apt-get install python3-pip # for Python 3.n
+sudo apt-get install python-pip # for Python 2.7
+ +You can find more help on installing and upgrading pip in +[the Raspberry Pi documentation](https://www.raspberrypi.org/documentation/linux/software/python.md). + +### Prerequisite: Atlas + +[Atlas](http://math-atlas.sourceforge.net/) is a linear algebra library that +numpy depends on, and so needs to be installed before TensorFlow. To add it to +your system, run the following command: + +
$ sudo apt install libatlas-base-dev
+ +### Install TensorFlow + +Assuming the prerequisite software is installed on your Pi, install TensorFlow +by invoking **one** of the following commands: + + 
 $ pip3 install tensorflow     # Python 3.n
+     $ pip install tensorflow      # Python 2.7
+ +This can take some time on certain platforms like the Pi Zero, where some Python +packages like scipy that TensorFlow depends on need to be compiled before the +installation can complete. The Python 3 version will typically be faster to +install because piwheels.org has pre-built versions of the dependencies +available, so this is our recommended option. + +### Next Steps + +After installing TensorFlow, [validate your +installation](#ValidateYourInstallation) to confirm that the installation worked +properly. + +### Uninstalling TensorFlow + +To uninstall TensorFlow, issue one of following commands: + +
$ pip uninstall tensorflow
+$ pip3 uninstall tensorflow
+ +## Cross-compiling from sources + +Cross-compilation means building on a different machine than than you'll be +deploying on. Since Raspberry Pi's only have limited RAM and comparatively slow +processors, and TensorFlow has a large amount of source code to compile, it's +easier to use a MacOS or Linux desktop or laptop to handle the build process. +Because it can take over 24 hours to build on a Pi, and requires external swap +space to cope with the memory shortage, we recommend using cross-compilation if +you do need to compile TensorFlow from source. To make the dependency management +process easier, we also recommend using Docker to help simplify building. + +Note that we provide well-tested, pre-built TensorFlow binaries for Raspbian +systems. So, don't build a TensorFlow binary yourself unless you are very +comfortable building complex packages from source and dealing with the +inevitable aftermath should things not go exactly as documented + +### Prerequisite: Docker + +Install Docker on your machine as described in the [Docker +documentation](https://docs.docker.com/engine/installation/#/on-macos-and-windows). + +### Clone the TensorFlow repository + +Start the process of building TensorFlow by cloning a TensorFlow repository. + +To clone **the latest** TensorFlow repository, issue the following command: + +
$ git clone https://github.com/tensorflow/tensorflow
+ +The preceding git clone command creates a subdirectory named +`tensorflow`. After cloning, you may optionally build a **specific branch** +(such as a release branch) by invoking the following commands: + +
+$ cd tensorflow
+$ git checkout Branch # where Branch is the desired branch
+
+ +For example, to work with the `r1.0` release instead of the master release, +issue the following command: + +
$ git checkout r1.0
+ +### Build from source + +To compile TensorFlow and produce a binary pip can install, do the following: + +1. Start a terminal. +2. Navigate to the directory containing the tensorflow source code. +3. Run a command to cross-compile the library, for example: + +
$ CI_DOCKER_EXTRA_PARAMS="-e CI_BUILD_PYTHON=python3 -e CROSSTOOL_PYTHON_INCLUDE_PATH=/usr/include/python3.4" \
+tensorflow/tools/ci_build/ci_build.sh PI-PYTHON3 tensorflow/tools/ci_build/pi/build_raspberry_pi.sh
+
+ +This will build a pip .whl file for Python 3.4, with Arm v7 instructions that +will only work on the Pi models 2 or 3. These NEON instructions are required for +the fastest operation on those devices, but you can build a library that will +run across all Pi devices by passing `PI_ONE` at the end of the command line. +You can also target Python 2.7 by omitting the initial docker parameters. Here's +an example of building for Python 2.7 and Raspberry Pi model Zero or One +devices: + +
$ tensorflow/tools/ci_build/ci_build.sh PI tensorflow/tools/ci_build/pi/build_raspberry_pi.sh PI_ONE
+ +This will take some time to complete, typically twenty or thirty minutes, and +should produce a .whl file in an output-artifacts sub-folder inside your source +tree at the end. This wheel file can be installed through pip or pip3 (depending +on your Python version) by copying it to a Raspberry Pi and running a terminal +command like this (with the name of your actual file substituted): + +
$ pip3 install tensorflow-1.9.0-cp34-none-linux_armv7l.whl
+ +### Troubleshooting the build + +The build script uses Docker internally to create a Linux virtual machine to +handle the compilation. If you do have problems running the script, first check +that you're able to run Docker tests like `docker run hello-world` on your +system. + +If you're building from the latest development branch, try syncing to an older +version that's known to work, for example release 1.9, with a command like this: + +
$ git checkout r1.0
+ + + +## Validate your installation + +To validate your TensorFlow installation, do the following: + +1. Ensure that your environment is prepared to run TensorFlow programs. +2. Run a short TensorFlow program. + +### Prepare your environment + +If you installed on native pip, Virtualenv, or Anaconda, then do the following: + +1. Start a terminal. +2. If you installed TensorFlow source code, navigate to any directory *except* + one containing TensorFlow source code. + +### Run a short TensorFlow program + +Invoke python from your shell as follows: + +
$ python
+ +Enter the following short program inside the python interactive shell: + +```python +# Python +import tensorflow as tf +hello = tf.constant('Hello, TensorFlow!') +sess = tf.Session() +print(sess.run(hello)) +``` + +If the system outputs the following, then you are ready to begin writing +TensorFlow programs: + +
Hello, TensorFlow!
+ +If you're running with Python 3.5, you may see a warning when you first import +TensorFlow. This is not an error, and TensorFlow should continue to run with no +problems, despite the log message. + +If the system outputs an error message instead of a greeting, see [Common +installation problems](#common_installation_problems). + +If you are new to machine learning, we recommend the [Machine Learning Crash +Course](https://developers.google.com/machine-learning/crash-course). + +If you are experienced with machine learning but new to TensorFlow, see +@{$get_started/eager}. + +## Common installation problems + +We are relying on Stack Overflow to document TensorFlow installation problems +and their remedies. The following table contains links to Stack Overflow answers +for some common installation problems. If you encounter an error message or +other installation problem not listed in the following table, search for it on +Stack Overflow. If Stack Overflow doesn't show the error message, ask a new +question about it on Stack Overflow and specify the `tensorflow` tag. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Stack Overflow Link Error Message
42006320
ImportError: Traceback (most recent call last):
+File ".../tensorflow/core/framework/graph_pb2.py", line 6, in 
+from google.protobuf import descriptor as _descriptor
+ImportError: cannot import name 'descriptor'
+
33623453
IOError: [Errno 2] No such file or directory:
+  '/tmp/pip-o6Tpui-build/setup.py'
+
35190574 
SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify
+  failed
42009190
+  Installing collected packages: setuptools, protobuf, wheel, numpy, tensorflow
+  Found existing installation: setuptools 1.1.6
+  Uninstalling setuptools-1.1.6:
+  Exception:
+  ...
+  [Errno 1] Operation not permitted:
+  '/tmp/pip-a1DXRT-uninstall/.../lib/python/_markerlib'
33622019
ImportError: No module named copyreg
37810228During a pip install operation, the system returns: + 
OSError: [Errno 1] Operation not permitted
+
33622842An import tensorflow statement triggers an error such as the + following:
Traceback (most recent call last):
+  File "", line 1, in 
+  File "/usr/local/lib/python2.7/site-packages/tensorflow/__init__.py",
+    line 4, in 
+    from tensorflow.python import *
+    ...
+  File "/usr/local/lib/python2.7/site-packages/tensorflow/core/framework/tensor_shape_pb2.py",
+    line 22, in 
+    serialized_pb=_b('\n,tensorflow/core/framework/tensor_shape.proto\x12\ntensorflow\"d\n\x10TensorShapeProto\x12-\n\x03\x64im\x18\x02
+      \x03(\x0b\x32
+      .tensorflow.TensorShapeProto.Dim\x1a!\n\x03\x44im\x12\x0c\n\x04size\x18\x01
+      \x01(\x03\x12\x0c\n\x04name\x18\x02 \x01(\tb\x06proto3')
+  TypeError: __init__() got an unexpected keyword argument 'syntax'
+
-- cgit v1.2.3 From 76b8b01740233ff289d70a0d516c6e0ac0e6b042 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Mon, 11 Jun 2018 11:55:34 -0700 Subject: Use the Keras session for saving/loading in TensorFlow format Fixes issues when there's no default session PiperOrigin-RevId: 200088574 --- tensorflow/python/keras/engine/network.py | 10 ++++-- tensorflow/python/keras/engine/saving_test.py | 52 ++++++++++++++++++--------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py index 9dbf94a276..3d567b8378 100644 --- a/tensorflow/python/keras/engine/network.py +++ b/tensorflow/python/keras/engine/network.py @@ -20,6 +20,7 @@ from __future__ import division from __future__ import print_function import copy +import functools import json import os import weakref @@ -1264,7 +1265,11 @@ class Network(base_layer.Layer): with h5py.File(filepath, 'w') as f: saving.save_weights_to_hdf5_group(f, self.layers) else: - self._checkpointable_saver.save(filepath) + if context.executing_eagerly(): + session = None + else: + session = backend.get_session() + self._checkpointable_saver.save(filepath, session=session) def load_weights(self, filepath, by_name=False): """Loads all layer weights, either from a TensorFlow or an HDF5 weight file. @@ -1324,7 +1329,8 @@ class Network(base_layer.Layer): 'loading TensorFlow-formatted weights (got by_name=True to ' 'load_weights).') if not context.executing_eagerly(): - finalizer = status.run_restore_ops + session = backend.get_session() + finalizer = functools.partial(status.run_restore_ops, session=session) if self.built: finalizer() else: diff --git a/tensorflow/python/keras/engine/saving_test.py b/tensorflow/python/keras/engine/saving_test.py index 30bcd3d185..b5448a9be1 100644 --- a/tensorflow/python/keras/engine/saving_test.py +++ b/tensorflow/python/keras/engine/saving_test.py @@ -404,26 +404,27 @@ class TestWholeModelSaving(test.TestCase): os.remove(fname) def test_saving_lambda_numpy_array_arguments(self): - if h5py is None: - self.skipTest('h5py required to run this test') + with self.test_session(): + if h5py is None: + self.skipTest('h5py required to run this test') - mean = np.random.random((4, 2, 3)) - std = np.abs(np.random.random((4, 2, 3))) + 1e-5 - inputs = keras.layers.Input(shape=(4, 2, 3)) - output = keras.layers.Lambda(lambda image, mu, std: (image - mu) / std, - arguments={'mu': mean, 'std': std})(inputs) - model = keras.models.Model(inputs, output) - model.compile(loss='mse', optimizer='sgd', metrics=['acc']) + mean = np.random.random((4, 2, 3)) + std = np.abs(np.random.random((4, 2, 3))) + 1e-5 + inputs = keras.layers.Input(shape=(4, 2, 3)) + output = keras.layers.Lambda(lambda image, mu, std: (image - mu) / std, + arguments={'mu': mean, 'std': std})(inputs) + model = keras.models.Model(inputs, output) + model.compile(loss='mse', optimizer='sgd', metrics=['acc']) - fd, fname = tempfile.mkstemp('.h5') - keras.models.save_model(model, fname) + fd, fname = tempfile.mkstemp('.h5') + keras.models.save_model(model, fname) - model = keras.models.load_model(fname) - os.close(fd) - os.remove(fname) + model = keras.models.load_model(fname) + os.close(fd) + os.remove(fname) - self.assertAllClose(mean, model.layers[1].arguments['mu']) - self.assertAllClose(std, model.layers[1].arguments['std']) + self.assertAllClose(mean, model.layers[1].arguments['mu']) + self.assertAllClose(std, model.layers[1].arguments['std']) def test_saving_model_with_long_layer_names(self): if h5py is None: @@ -580,6 +581,25 @@ class TestWeightSavingAndLoadingTFFormat(test.TestCase): # Indirectly tests that the user is prompted model.save_weights(prefix, save_format='tensorflow', overwrite=False) + def test_no_default_session(self): + with ops.Graph().as_default(): + self.assertFalse(ops.get_default_session()) + data = np.random.random((1000, 32)).astype(np.float32) + labels = np.random.random((1000, 10)).astype(np.float32) + + model = keras.models.Sequential([ + keras.layers.Dense(10, activation='softmax'), + keras.layers.Dense(10, activation='softmax')]) + + model.compile(optimizer=training_module.RMSPropOptimizer(0.001), + loss='categorical_crossentropy', + metrics=['accuracy']) + + model.fit(data, labels) + fname = os.path.join(self.get_temp_dir(), 'weights', 'ckpt') + model.save_weights(fname) + model.load_weights(fname) + def test_no_graph_pollution(self): with context.graph_mode(): graph = ops.Graph() -- cgit v1.2.3 From 50ba6dd3a182c9578bc10cb2a21d7914a1e7bac1 Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Mon, 11 Jun 2018 10:42:15 -0700 Subject: Don't call back into python during insert (which will leave the set in a broken condition if the runtime decides to let another thread run). Thank you for finding the bug. The watched_variables_ set should not really require a lock since all our functions hold the GIL (verified by looking at the generated SWIG). The reason that there was a concurrent access to the set is that the insert was calling back into python (which might release the GIL and let another thread run, which will also attempt to insert a variable and break the set). I included the lock to be safe though, since its non-trivial to verify without looking at the generated swig wrappers that the GIL is held. PiperOrigin-RevId: 200074843 --- tensorflow/python/eager/pywrap_tfe_src.cc | 82 ++++++++++++++++--------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index e3ce0ef9d0..52b3268903 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -873,22 +873,6 @@ static tensorflow::DataType FastTensorDtype(PyObject* tensor) { return static_cast(id); } -static tensorflow::int64 FastHandleId(PyObject* variable) { - PyObject* handle = PyObject_GetAttrString(variable, "handle"); - if (handle == nullptr) { - return -1; - } - tensorflow::int64 id = FastTensorId(handle); - Py_DECREF(handle); - return id; -} - -struct CompareByHandleId { - bool operator()(PyObject* lhs, PyObject* rhs) { - return FastHandleId(lhs) < FastHandleId(rhs); - } -}; - class GradientTape : public tensorflow::eager::GradientTape { public: @@ -897,35 +881,63 @@ class GradientTape persistent) {} virtual ~GradientTape() { - for (PyObject* v : watched_variables_) { - Py_DECREF(v); + for (const IdAndVariable& v : watched_variables_) { + Py_DECREF(v.variable); } } void WatchVariable(PyObject* v) { - auto insert_result = watched_variables_.insert(v); - if (insert_result.second) { - // Only increment the reference count if we aren't already watching this - // variable. - Py_INCREF(v); - } - PyObject* handle = PyObject_GetAttrString(v, "handle"); + tensorflow::Safe_PyObjectPtr handle(PyObject_GetAttrString(v, "handle")); if (handle == nullptr) { return; } - tensorflow::int64 id = FastTensorId(handle); - Py_DECREF(handle); + tensorflow::int64 id = FastTensorId(handle.get()); + if (!PyErr_Occurred()) { this->Watch(id); } + + tensorflow::mutex_lock l(watched_variables_mu_); + auto insert_result = watched_variables_.emplace(id, v); + + if (insert_result.second) { + // Only increment the reference count if we aren't already watching this + // variable. + Py_INCREF(v); + } } - const std::set WatchedVariables() { - return watched_variables_; + PyObject* GetVariablesAsPyTuple() { + tensorflow::mutex_lock l(watched_variables_mu_); + PyObject* result = PyTuple_New(watched_variables_.size()); + Py_ssize_t pos = 0; + for (const IdAndVariable& id_and_variable : watched_variables_) { + PyTuple_SET_ITEM(result, pos++, id_and_variable.variable); + Py_INCREF(id_and_variable.variable); + } + return result; } private: - std::set watched_variables_; + // We store an IdAndVariable in the map since the map needs to be locked + // during insert, but should not call back into python during insert to avoid + // deadlocking with the GIL. + struct IdAndVariable { + tensorflow::int64 id; + PyObject* variable; + + IdAndVariable(tensorflow::int64 id, PyObject* variable) + : id(id), variable(variable) {} + }; + struct CompareById { + bool operator()(const IdAndVariable& lhs, const IdAndVariable& rhs) { + return lhs.id < rhs.id; + } + }; + + tensorflow::mutex watched_variables_mu_; + std::set watched_variables_ + GUARDED_BY(watched_variables_mu_); }; typedef struct { @@ -1217,15 +1229,7 @@ void TFE_Py_TapeSetWatchVariable(PyObject* variable) { } PyObject* TFE_Py_TapeWatchedVariables(PyObject* tape) { - const auto& watched_variables = - reinterpret_cast(tape)->tape->WatchedVariables(); - PyObject* result = PyTuple_New(watched_variables.size()); - Py_ssize_t pos = 0; - for (PyObject* variable : watched_variables) { - PyTuple_SET_ITEM(result, pos++, variable); - Py_INCREF(variable); - } - return result; + return reinterpret_cast(tape)->tape->GetVariablesAsPyTuple(); } namespace { -- cgit v1.2.3 From ec769c7ec368adf90aaa0b6d2a97525da14e1a37 Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Mon, 11 Jun 2018 16:27:12 -0700 Subject: Remove memory leak in read variable call, and record gradient call. Fix #19385 PiperOrigin-RevId: 200132949 --- tensorflow/python/eager/pywrap_tfe_src.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index 52b3268903..6c9481c3af 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -1873,6 +1873,8 @@ PyObject* RecordGradient(PyObject* op_name, PyObject* inputs, PyObject* attrs, delete backward_function; }); + Py_DECREF(num_inputs); + Py_RETURN_NONE; } @@ -1931,8 +1933,10 @@ bool ReadVariableOp(const FastPathOpExecInfo& parent_op_exec_info, Py_INCREF(output->get()); // stay alive after since tuple steals. PyTuple_SET_ITEM(outputs.get(), 0, output->get()); - if (!RecordGradient(GetPythonObjectFromString("ReadVariableOp"), - inputs.get(), Py_None, outputs.get(), Py_None)) { + tensorflow::Safe_PyObjectPtr op_string( + GetPythonObjectFromString("ReadVariableOp")); + if (!RecordGradient(op_string.get(), inputs.get(), Py_None, outputs.get(), + Py_None)) { return false; } } -- cgit v1.2.3 From c77fead531bc3756d765ba90e2e549abd7adf320 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Wed, 13 Jun 2018 15:46:12 -0700 Subject: Make GCS ops work in open source --- tensorflow/contrib/cloud/__init__.py | 5 +++-- tensorflow/contrib/cloud/kernels/BUILD | 1 + tensorflow/core/platform/cloud/gcs_file_system.cc | 4 +++- tensorflow/core/platform/default/build_config.bzl | 2 ++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/cloud/__init__.py b/tensorflow/contrib/cloud/__init__.py index a6e13ea3ae..ef7aa7624c 100644 --- a/tensorflow/contrib/cloud/__init__.py +++ b/tensorflow/contrib/cloud/__init__.py @@ -27,8 +27,9 @@ from tensorflow.python.util.all_util import remove_undocumented _allowed_symbols = [ 'BigQueryReader', - 'ConfigureColabSession', - 'ConfigureGcs', + 'BlockCacheParams', + 'configure_colab_session', + 'configure_gcs', 'ConfigureGcsHook', ] remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/cloud/kernels/BUILD b/tensorflow/contrib/cloud/kernels/BUILD index 40160706f7..1311063ec0 100644 --- a/tensorflow/contrib/cloud/kernels/BUILD +++ b/tensorflow/contrib/cloud/kernels/BUILD @@ -79,6 +79,7 @@ tf_kernel_library( srcs = ["gcs_config_ops.cc"], visibility = ["//tensorflow:internal"], deps = [ + "//tensorflow/contrib/cloud:gcs_config_ops_op_lib", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core/platform/cloud:curl_http_request", diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index 22ae6121e0..803b08f1a3 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -804,7 +804,9 @@ void GcsFileSystem::ResetFileBlockCache(size_t block_size_bytes, mutex_lock l(block_cache_lock_); file_block_cache_ = MakeFileBlockCache(block_size_bytes, max_bytes, max_staleness_secs); - stats_->Configure(this, &throttle_, file_block_cache_.get()); + if (stats_) { + stats_->Configure(this, &throttle_, file_block_cache_.get()); + } } // A helper function to build a FileBlockCache for GcsFileSystem. diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index 9e52ba344a..f12732b434 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -633,6 +633,7 @@ def tf_additional_cloud_op_deps(): "//tensorflow:with_gcp_support_ios_override": [], "//tensorflow:with_gcp_support": [ "//tensorflow/contrib/cloud:bigquery_reader_ops_op_lib", + "//tensorflow/contrib/cloud:gcs_config_ops_op_lib", ], "//conditions:default": [], }) @@ -645,6 +646,7 @@ def tf_additional_cloud_kernel_deps(): "//tensorflow:with_gcp_support_ios_override": [], "//tensorflow:with_gcp_support": [ "//tensorflow/contrib/cloud/kernels:bigquery_reader_ops", + "//tensorflow/contrib/cloud/kernels:gcs_config_ops", ], "//conditions:default": [], }) -- cgit v1.2.3 From f9a44a69c35dcf7f1c0f42e1ae9971bae0148099 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Wed, 13 Jun 2018 18:05:39 -0700 Subject: Update the docs and api_def. --- tensorflow/contrib/cloud/ops/gcs_config_ops.cc | 42 ++-------------------- .../base_api/api_def_GcsConfigureBlockCache.pbtxt | 9 +++++ .../base_api/api_def_GcsConfigureCredentials.pbtxt | 33 +++++++++++++++++ 3 files changed, 44 insertions(+), 40 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_GcsConfigureBlockCache.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_GcsConfigureCredentials.pbtxt diff --git a/tensorflow/contrib/cloud/ops/gcs_config_ops.cc b/tensorflow/contrib/cloud/ops/gcs_config_ops.cc index 9cf85f5f18..5e31a15498 100644 --- a/tensorflow/contrib/cloud/ops/gcs_config_ops.cc +++ b/tensorflow/contrib/cloud/ops/gcs_config_ops.cc @@ -21,50 +21,12 @@ namespace tensorflow { REGISTER_OP("GcsConfigureCredentials") .Input("json: string") - .SetShapeFn(shape_inference::NoOutputs) - .Doc(R"doc( -Configures the credentials used by the GCS client of the local TF runtime. - -The json input can be of the format: - -1. Refresh Token: -{ - "client_id": "", - "client_secret": "", - "refresh_token: "", - "type": "authorized_user", -} - -2. Service Account: -{ - "type": "service_account", - "project_id": "", - "private_key_id": "", - "private_key": "------BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY------\n", - "client_email": "@.iam.gserviceaccount.com", - "client_id": "", - # Some additional fields elided -} - -Note the credentials established through this method are shared across all -sessions run on this runtime. - -Note be sure to feed the inputs to this op to ensure the credentials are not -stored in a constant op within the graph that might accidentally be checkpointed -or in other ways be persisted or exfiltrated. -)doc"); + .SetShapeFn(shape_inference::NoOutputs); REGISTER_OP("GcsConfigureBlockCache") .Input("max_cache_size: uint64") .Input("block_size: uint64") .Input("max_staleness: uint64") - .SetShapeFn(shape_inference::NoOutputs) - .Doc(R"doc( -Re-configures the GCS block cache with the new configuration values. - -If the values are the same as already configured values, this op is a no-op. If -they are different, the current contents of the block cache is dropped, and a -new block cache is created fresh. -)doc"); + .SetShapeFn(shape_inference::NoOutputs); } // namespace tensorflow diff --git a/tensorflow/core/api_def/base_api/api_def_GcsConfigureBlockCache.pbtxt b/tensorflow/core/api_def/base_api/api_def_GcsConfigureBlockCache.pbtxt new file mode 100644 index 0000000000..9d32940c64 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_GcsConfigureBlockCache.pbtxt @@ -0,0 +1,9 @@ +op { + graph_op_name: "GcsConfigureBlockCache" + summary: "Re-configures the GCS block cache with the new configuration values." + description: <", + "client_secret": "", + "refresh_token: "", + "type": "authorized_user", +} + +2. Service Account: +{ + "type": "service_account", + "project_id": "", + "private_key_id": "", + "private_key": "------BEGIN PRIVATE KEY-----\n\n-----END PRIVATE KEY------\n", + "client_email": "@.iam.gserviceaccount.com", + "client_id": "", + # Some additional fields elided +} + +Note the credentials established through this method are shared across all +sessions run on this runtime. + +Note be sure to feed the inputs to this op to ensure the credentials are not +stored in a constant op within the graph that might accidentally be checkpointed +or in other ways be persisted or exfiltrated. +END0 +} -- cgit v1.2.3 From ea3bdbc7ea72e488566326aeb446681a557f4334 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Thu, 14 Jun 2018 06:17:00 -0700 Subject: Update version strings for 1.9.0-rc1. --- tensorflow/core/public/version.h | 2 +- tensorflow/docs_src/install/install_c.md | 2 +- tensorflow/docs_src/install/install_go.md | 2 +- tensorflow/docs_src/install/install_java.md | 22 +++++++++++----------- tensorflow/docs_src/install/install_linux.md | 18 +++++++++--------- tensorflow/docs_src/install/install_mac.md | 10 +++++----- tensorflow/docs_src/install/install_sources.md | 4 ++-- tensorflow/tools/pip_package/setup.py | 2 +- 8 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index cb1fd09dbb..9e5e747557 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -24,7 +24,7 @@ limitations under the License. // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") -#define TF_VERSION_SUFFIX "-rc0" +#define TF_VERSION_SUFFIX "-rc1" #define TF_STR_HELPER(x) #x #define TF_STR(x) TF_STR_HELPER(x) diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md index 2901848745..2f81ae0c40 100644 --- a/tensorflow/docs_src/install/install_c.md +++ b/tensorflow/docs_src/install/install_c.md @@ -38,7 +38,7 @@ enable TensorFlow for C: OS="linux" # Change to "darwin" for macOS TARGET_DIRECTORY="/usr/local" curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.9.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.9.0-rc1.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index 55bc0f64e7..1c03dd223e 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go: TF_TYPE="cpu" # Change to "gpu" for GPU support TARGET_DIRECTORY='/usr/local' curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.9.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.9.0-rc1.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index b3b739212e..c73e2f4281 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs: org.tensorflow tensorflow - 1.9.0-rc0 + 1.9.0-rc1 ``` @@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow: org.tensorflow tensorflow - 1.9.0-rc0 + 1.9.0-rc1 @@ -124,12 +124,12 @@ instead: org.tensorflow libtensorflow - 1.9.0-rc0 + 1.9.0-rc1 org.tensorflow libtensorflow_jni_gpu - 1.9.0-rc0 + 1.9.0-rc1 ``` @@ -148,7 +148,7 @@ refer to the simpler instructions above instead. Take the following steps to install TensorFlow for Java on Linux or macOS: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc0.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc1.jar), which is the TensorFlow Java Archive (JAR). 2. Decide whether you will run TensorFlow for Java on CPU(s) only or with @@ -167,7 +167,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: OS=$(uname -s | tr '[:upper:]' '[:lower:]') mkdir -p ./jni curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.9.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.9.0-rc1.tar.gz" | tar -xz -C ./jni ### Install on Windows @@ -175,10 +175,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: Take the following steps to install TensorFlow for Java on Windows: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc0.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc1.jar), which is the TensorFlow Java Archive (JAR). 2. Download the following Java Native Interface (JNI) file appropriate for - [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.9.0-rc0.zip). + [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.9.0-rc1.zip). 3. Extract this .zip file. @@ -227,7 +227,7 @@ must be part of your `classpath`. For example, you can include the downloaded `.jar` in your `classpath` by using the `-cp` compilation flag as follows: -
javac -cp libtensorflow-1.9.0-rc0.jar HelloTF.java
+
javac -cp libtensorflow-1.9.0-rc1.jar HelloTF.java
### Running @@ -241,11 +241,11 @@ two files are available to the JVM: For example, the following command line executes the `HelloTF` program on Linux and macOS X: -
java -cp libtensorflow-1.9.0-rc0.jar:. -Djava.library.path=./jni HelloTF
+
java -cp libtensorflow-1.9.0-rc1.jar:. -Djava.library.path=./jni HelloTF
And the following command line executes the `HelloTF` program on Windows: -
java -cp libtensorflow-1.9.0-rc0.jar;. -Djava.library.path=jni HelloTF
+
java -cp libtensorflow-1.9.0-rc1.jar;. -Djava.library.path=jni HelloTF
If the program prints Hello from version, you've successfully installed TensorFlow for Java and are ready to use the API. If the program diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 2ecab808c4..9baf6870be 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -438,7 +438,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: 
      (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc1-cp34-cp34m-linux_x86_64.whl ## Validate your installation @@ -684,14 +684,14 @@ This section documents the relevant values for Linux installations. CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc1-cp27-none-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc1-cp27-none-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -703,14 +703,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc1-cp34-cp34m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc1-cp34-cp34m-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -722,14 +722,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc1-cp35-cp35m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc1-cp35-cp35m-linux_x86_64.whl
@@ -741,14 +741,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc1-cp36-cp36m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc1-cp36-cp36m-linux_x86_64.whl
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index 9d01271c5a..693254f876 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv: TensorFlow in the active Virtualenv is as follows: 
 $ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py3-none-any.whl If you encounter installation problems, see [Common Installation Problems](#common-installation-problems). @@ -242,7 +242,7 @@ take the following steps: issue the following command: 
 $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py3-none-any.whl If the preceding command fails, see [installation problems](#common-installation-problems). @@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: TensorFlow for Python 2.7: 
 (targetDirectory)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py2-none-any.whl @@ -522,7 +522,7 @@ The value you specify depends on your Python version. 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py2-none-any.whl
@@ -530,5 +530,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py2-none-a 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py3-none-any.whl
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index d25e641cee..70e97cf556 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -328,10 +328,10 @@ Invoke `pip install` to install that pip package. The filename of the `.whl` file depends on your platform. For example, the following command will install the pip package -for TensorFlow 1.9.0rc0 on Linux: +for TensorFlow 1.9.0rc1 on Linux: 
-$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.9.0rc0-py2-none-any.whl
+$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.9.0rc1-py2-none-any.whl
## Validate your installation diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 92a1465cea..eb2e359ee5 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -45,7 +45,7 @@ DOCLINES = __doc__.split('\n') # This version string is semver compatible, but incompatible with pip. # For pip, we will remove all '-' characters from this string, and use the # result for pip. -_VERSION = '1.9.0-rc0' +_VERSION = '1.9.0-rc1' REQUIRED_PACKAGES = [ 'absl-py >= 0.1.6', -- cgit v1.2.3 From f5ee4df50af4041dc0063d0adc31c7a6eebdbcd3 Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Fri, 8 Jun 2018 15:47:19 -0700 Subject: Copy edits to Keras guide, formatting, moving some things around. Make the right TOC nav more useful. PiperOrigin-RevId: 199863216 --- tensorflow/docs_src/programmers_guide/keras.md | 870 +++++++++++-------------- 1 file changed, 389 insertions(+), 481 deletions(-) diff --git a/tensorflow/docs_src/programmers_guide/keras.md b/tensorflow/docs_src/programmers_guide/keras.md index 6a9df12a25..c6aca7ebf4 100644 --- a/tensorflow/docs_src/programmers_guide/keras.md +++ b/tensorflow/docs_src/programmers_guide/keras.md @@ -1,334 +1,304 @@ # Keras -## What's Keras? - -Keras is a high-level API specification for building and training deep learning -models, suitable for fast prototyping, advanced research, and production. -It offers three key advantages: - -- **User friendliness.** Keras follows best practices for reducing - cognitive load: it offers consistent & simple interfaces, - it minimizes the number of user actions required for common use cases, - and it provides clear and actionable feedback upon user error. -- **Modularity and composability.** A Keras model is composed of - fully-configurable building blocks that can be plugged together - with as few restrictions as possible -- like Lego bricks. -- **Easy extensibility.** You can easily write your own building blocks - (such as new layers, new loss functions, new models where you write - the forward pass from scratch). This allows for total expressiveness, - making Keras suitable for advanced research. - - -## What's tf.keras? - -`tf.keras` is TensorFlow's implementation of the Keras API specification, that -serves as the TensorFlow high-level API: it's how you build models in TensorFlow. -`tf.keras` seamlessly integrates with the rest of the TensorFlow API -(such as `tf.data` input pipelines), bringing you the full power and flexibility -of TensorFlow through an easy-to-use interface. - -You can import `tf.keras` via: +Keras is a high-level API to build and train deep learning models. It's used for +fast prototyping, advanced research, and production, with three key advantages: + +- *User friendly*
+ Keras has a simple, consistent interface optimized for common use cases. It + provides clear and actionable feedback for user errors. +- *Modular and composable*
+ Keras models are made by connecting configurable building blocks together, + with few restrictions. +- *Easy to extend*
Write custom building blocks to express new ideas for + research. Create new layers, loss functions, and develop state-of-the-art + models. + +## Import tf.keras + +`tf.keras` is TensorFlow's implementation of the +[Keras API specification](https://keras.io){:.external}. This is a high-level +API to build and train models that includes first-class support for +TensorFlow-specific functionality, such as [eager execution](#eager_execution), +`tf.data` pipelines, and [Estimators](/programmers_guide/estimators). +`tf.keras` makes TensorFlow easier to use without sacrificing flexibility and +performance. + +To get started, import `tf.keras` as part of your TensorFlow program setup: ```python +import tensorflow as tf from tensorflow import keras ``` -What follows is a quick introduction to the basics of `tf.keras`. +`tf.keras` can run any Keras-compatible code, but keep in mind: +* The `tf.keras` version in the latest TensorFlow release might not be the same + as the latest `keras` version from PyPI. Check `tf.keras.__version__`. +* When [saving a model's weights](#weights_only), `tf.keras` defaults to the + [checkpoint format](/get_started/checkpoints). Pass `save_format='h5'` to use + HDF5. -## Table of contents +## Build a simple model -- [Getting started: the Sequential model](#getting-started-the-sequential-model) -- [Configuring layers](#configuring-layers) -- [Configuring training](#configuring-training) -- [Training and evaluation](#training-and-evaluation) -- [Building advanced models: the functional API](#building-advanced-models-the-functional-api) -- [Building fully-customizable research models: the Model subclassing API](#building-fully-customizable-research-models-the-model-subclassing-api) -- [Callbacks](#callbacks) -- [Saving and serialization](#saving-and-serialization) -- [Developing custom layers](#developing-custom-layers) -- [Eager execution](#eager-execution) -- [Further reading](#further-reading) -- [FAQ](#faq) +### Sequential model +In Keras, you assemble *layers* to build *models*. A model is (usually) a graph +of layers. The most common type of model is a stack of layers: the +`tf.keras.Sequential` model. ---- - -## Getting started: the Sequential model - -In `tf.keras`, you're assembling together **layers** to build **models**. -A model is generally a graph of layers. -The most common type of model is just a stack of layers: the `Sequential` class. - -Here's how to build a simple fully-connected network (multi-layer perceptron): +To build a simple, fully-connected network (i.e. multi-layer perceptron): ```python -from tensorflow import keras -from tensorflow.keras import layers - model = keras.Sequential() -# This adds to the model a densely-connected layer with 64 units: -model.add(Dense(64, activation='relu')) -# Another one: -model.add(Dense(64, activation='relu')) -# This adds a softmax layer with 10 output units: -model.add(Dense(10, activation='softmax')) +# Adds a densely-connected layer with 64 units to the model: +model.add(keras.layers.Dense(64, activation='relu')) +# Add another: +model.add(keras.layers.Dense(64, activation='relu')) +# Add a softmax layer with 10 output units: +model.add(keras.layers.Dense(10, activation='softmax')) ``` ---- - -## Configuring layers - -Each layer may have unique constructor arguments, but some common arguments include: +### Configure the layers -- `activation`: the activation function to be used. - It could be specified by name, as a string (for built-in functions) - or as a callable object. By default, no activation is applied. -- `kernel_initializer` and `bias_initializer`: the initialization schemes to use - to create the layer's weights (kernel and bias). - Likewise, they may be passed either by name or by specifying a callable. - By default, the "Glorot uniform" initializer is used. -- `kernel_regularizer` and `bias_regularizer`: the regularization schemes to - apply to the layer's weights (kernel and bias), such as L1 - or L2 regularization. By default, no regularization is applied. +There are many `tf.keras.layers` available with some common constructor +parameters: +* `activation`: Set the activation function for the layer. This parameter is + specified by the name of a built-in function or as a callable object. By + default, no activation is applied. +* `kernel_initializer` and `bias_initializer`: The initialization schemes + that create the layer's weights (kernel and bias). This parameter is a name or + a callable object. This defaults to the `"Glorot uniform"` initializer. +* `kernel_regularizer` and `bias_regularizer`: The regularization schemes + that apply the layer's weights (kernel and bias), such as L1 or L2 + regularization. By default, no regularization is applied. -### Examples +The following instantiates `tf.keras.layers.Dense` layers using constructor +arguments: ```python -import tensorflow as tf -from tensorflow.keras.layers import Dense -from tensorflow.keras import regularizers -from tensorflow.keras import initializers - -# A sigmoid layer: -Dense(64, activation='sigmoid') -# Another way to define the same sigmoid layer: -Dense(64, activation=tf.sigmoid) - -# A linear layer with L1 regularization of factor 0.01 -# applied to the kernel matrix: -Dense(64, kernel_regularizer=regularizers.l1(0.01)) -# A linear layer with L2 regularization of factor 0.01 -# applied to the bias vector: -Dense(64, bias_regularizer=regularizers.l2(0.01)) +# Create a sigmoid layer: +layers.Dense(64, activation='sigmoid') +# Or: +layers.Dense(64, activation=tf.sigmoid) + +# A linear layer with L1 regularization of factor 0.01 applied to the kernel matrix: +layers.Dense(64, kernel_regularizer=keras.regularizers.l1(0.01)) +# A linear layer with L2 regularization of factor 0.01 applied to the bias vector: +layers.Dense(64, bias_regularizer=keras.regularizers.l2(0.01)) # A linear layer with a kernel initialized to a random orthogonal matrix: -Dense(64, kernel_initializer='orthogonal') +layers.Dense(64, kernel_initializer='orthogonal') # A linear layer with a bias vector initialized to 2.0s: -Dense(64, bias_initializer=initializers.constant(2.0)) +layers.Dense(64, bias_initializer=keras.initializers.constant(2.0)) ``` ---- +## Train and evaluate -## Configuring training +### Set up training -Once your model looks good, configure its learning process by calling `compile`: +After the model is constructed, configure its learning process by calling the +`compile` method: ```python -import tensorflow as tf - model.compile(optimizer=tf.train.AdamOptimizer(0.001), loss='categorical_crossentropy', metrics=['accuracy']) ``` -There are three key arguments that you need to specify: +`tf.keras.Model.compile` takes three important arguments: -- An `optimizer`: this object specifies the training procedure. - We recommend that you pass instances of optimizers from the `tf.train` module - (such as [`AdamOptimizer`](https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer), - [`RMSPropOptimizer`](https://www.tensorflow.org/api_docs/python/tf/train/RMSPropOptimizer), - or [`GradientDescentOptimizer`](https://www.tensorflow.org/api_docs/python/tf/train/GradientDescentOptimizer)). -- A `loss` function to minimize: this specifies the optimization objective. - Common choices include mean square error (`mse`), `categorical_crossentropy` - and `binary_crossentropy`. Loss functions may be specified by name - or by passing a callable (e.g. from the `tf.keras.losses` module). -- Some `metrics` to monitor during training: again, you can pass these as either - string names or callables (e.g. from the `tf.keras.metrics` module). +* `optimizer`: This object specifies the training procedure. Pass it optimizer + instances from the `tf.train` module, such as + [`AdamOptimizer`](/api_docs/python/tf/train/AdamOptimizer), + [`RMSPropOptimizer`](/api_docs/python/tf/train/RMSPropOptimizer), or + [`GradientDescentOptimizer`](/api_docs/python/tf/train/GradientDescentOptimizer). +* `loss`: The function to minimize during optimization. Common choices include + mean square error (`mse`), `categorical_crossentropy`, and + `binary_crossentropy`. Loss functions are specified by name or by + passing a callable object from the `tf.keras.losses` module. +* `metrics`: Used to monitor training. These are string names or callables from + the `tf.keras.metrics` module. - -### Examples +The following shows a few examples of configuring a model for training: ```python -# Configures a model to do mean-squared error regression. +# Configure a model for mean-squared error regression. model.compile(optimizer=tf.train.AdamOptimizer(0.01), - loss='mse', # mean squared error + loss='mse', # mean squared error metrics=['mae']) # mean absolute error -``` -```python -# Configures a model to do categorical classification. + +# Configure a model for categorical classification. model.compile(optimizer=tf.train.RMSPropOptimizer(0.01), - loss=tf.keras.losses.categorical_crossentropy, - metrics=[tf.keras.metrics.categorical_accuracy]) + loss=keras.losses.categorical_crossentropy, + metrics=[keras.metrics.categorical_accuracy]) ``` ---- - -## Training and evaluation +### Input NumPy data -### From Numpy data - -When running locally on small datasets, the easiest way to do training and -evaluation is to pass data to your model as Numpy arrays of inputs and targets. -You can "fit" your model to some training data using the `model.fit()` method: +For small datasets, use in-memory [NumPy](https://www.numpy.org/){:.external} +arrays to train and evaluate a model. The model is "fit" to the training data +using the `fit` method: ```python import numpy as np -data = np.random.random(shape=(1000, 32)) -targets = np.random.random(shape=(1000, 10)) +data = np.random.random((1000, 32)) +labels = np.random.random((1000, 10)) -model.fit(data, targets, epochs=10, batch_size=32) +model.fit(data, labels, epochs=10, batch_size=32) ``` -Here are some key arguments you can pass to the `fit` method: - -- `epochs`: Training is structured into **epochs**. An epoch is one iteration - over the entire input data (which is done in smaller batches). -- `batch_size`: when passing Numpy data, the model will slice the data into - smaller batches and iterate over these batches during training. - This integer specifies the size of each batch - (the last batch may be smaller if the total number of samples is not - divisible by the batch size). -- `validation_data`: when prototyping a model, you want to be able to quickly - monitor its performance on some validation data. - When you pass this argument (it expects a tuple of inputs and targets), - the model will display the loss and metrics in inference mode on the data - you passed, at the end of each epoch. +`tf.keras.Model.fit` takes three important arguments: + +* `epochs`: Training is structured into *epochs*. An epoch is one iteration over + the entire input data (this is done in smaller batches). +* `batch_size`: When passed NumPy data, the model slices the data into smaller + batches and iterates over these batches during training. This integer + specifies the size of each batch. Be aware that the last batch may be smaller + if the total number of samples is not divisible by the batch size. +* `validation_data`: When prototyping a model, you want to easily monitor its + performance on some validation data. Passing this argument—a tuple of inputs + and labels—allows the model to display the loss and metrics in inference mode + for the passed data, at the end of each epoch. Here's an example using `validation_data`: ```python import numpy as np -data = np.random.random(shape=(1000, 32)) -targets = np.random.random(shape=(1000, 10)) +data = np.random.random((1000, 32)) +labels = np.random.random((1000, 10)) -val_data = np.random.random(shape=(100, 32)) -val_targets = np.random.random(shape=(100, 10)) +val_data = np.random.random((100, 32)) +val_labels = np.random.random((100, 10)) -model.fit(data, targets, epochs=10, batch_size=32, - validation_data=(val_data, val_targets)) +model.fit(data, labels, epochs=10, batch_size=32, + validation_data=(val_data, val_labels)) ``` -### From tf.data datasets +### Input tf.data datasets -When you need to scale to large datasets or multi-device training, -training from Numpy arrays in memory will not be ideal. -In such cases, you should use [the `tf.data` API](https://www.tensorflow.org/programmers_guide/datasets). -You can pass a `tf.data.Dataset` instance to the `fit` method: +Use the [Datasets API](/programmers_guide/datasets) to scale to large datasets +or multi-device training. Pass a `tf.data.Dataset` instance to the `fit` +method: ```python -import tensorflow as tf - # Instantiates a toy dataset instance: -dataset = tf.data.Dataset.from_tensor_slices((data, targets)).batch(32) +dataset = tf.data.Dataset.from_tensor_slices((data, labels)) +dataset = dataset.batch(32) +dataset = dataset.repeat() # Don't forget to specify `steps_per_epoch` when calling `fit` on a dataset. model.fit(dataset, epochs=10, steps_per_epoch=30) ``` -When doing so, the dataset itself will yield batches of data, -so the model does not need to be passed `batch_size` information. -Instead, the model needs to know for how many steps (or batches of data) -it should run at each epoch. -You specify this with the `steps_per_epoch` argument: it's the number of -training steps the model will run before moving on the next epoch. +Here, the `fit` method uses the `steps_per_epoch` argument—this is the number of +training steps the model runs before it moves to the next epoch. Since the +`Dataset` yields batches of data, this snippet does not require a `batch_size`. -You can also pass datasets for validation: +Datasets can also be used for validation: ```python -dataset = tf.data.Dataset.from_tensor_slices((data, targets)).batch(32) -val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_targets)).batch(32) +dataset = tf.data.Dataset.from_tensor_slices((data, labels)) +dataset = dataset.batch(32).repeat() -model.fit(dataset, epochs=10, steps_per_epoch=30, validation_data=val_dataset, validation_steps=3) +val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_labels)) +val_dataset = val_dataset.batch(32).repeat() + +model.fit(dataset, epochs=10, steps_per_epoch=30, + validation_data=val_dataset, + validation_steps=3) ``` ### Evaluate and predict -In addition, you get access to the following methods -(both with Numpy data and dataset instances): +The `tf.keras.Model.evaluate` and `tf.keras.Model.predict` methods can use NumPy +data and a `tf.data.Dataset`. -- `model.evaluate(x, y, batch_size=32)` or `model.evaluate(dataset, steps=30)` - will return the inference-mode loss and metrics for the data provided. -- `model.predict(x, y, batch_size=32)` or `model.predict(dataset, steps=30)` - will return the output(s) of the last layer(s) in inference on the data - provided, as Numpy array(s). +To *evaluate* the inference-mode loss and metrics for the data provided: ---- +```python +model.evaluate(x, y, batch_size=32) -## Building advanced models: the functional API +model.evaluate(dataset, steps=30 +``` -The `Sequential` model cannot represent arbitrary models -- only simple stacks -of layers. If you need to use more complex model topologies, -such as multi-input models, multi-output models, -models with a same layer called several times (shared layers), -or models with non-sequential data flows (e.g. residual connections), -you can use the 'functional API'. +And to *predict* the output of the last layer in inference for the data provided, +as a NumPy array: -Here's how it works: +``` +model.predict(x, batch_size=32) -- A layer instance is callable (on a tensor), and it returns a tensor. -- Input tensor(s) and output tensor(s) can then be used to define a `Model` instance. -- Such a model can be trained just like the `Sequential` model. +model.predict(dataset, steps=30) +``` -Here's a basic example showing the same model we previously defined, -built using the functional API: +## Build advanced models -```python -from tensorflow import keras -from tensorflow.keras import layers +### Functional API -# This returns a placeholder tensor: -inputs = keras.Input(shape=(784,)) +The `tf.keras.Sequential` model is a simple stack of layers that cannot +represent arbitrary models. Use the +[Keras functional API](https://keras.io/getting-started/functional-api-guide/){:.external} +to build complex model topologies such as: + +* Multi-input models, +* Multi-output models, +* Models with shared layers (the same layer called several times), +* Models with non-sequential data flows (e.g. residual connections). + +Building a model with the functional API works like this: + +1. A layer instance is callable and returns a tensor. +2. Input tensors and output tensors are used to define a `tf.keras.Model` + instance. +3. This model is trained just like the `Sequential` model. + +The following example uses the functional API to build a simple, fully-connected +network: + +```python +inputs = keras.Input(shape=(32,)) # Returns a placeholder tensor # A layer instance is callable on a tensor, and returns a tensor. -x = layers.Dense(64, activation='relu')(inputs) -x = layers.Dense(64, activation='relu')(x) -predictions = layers.Dense(10, activation='softmax')(x) +x = keras.layers.Dense(64, activation='relu')(inputs) +x = keras.layers.Dense(64, activation='relu')(x) +predictions = keras.layers.Dense(10, activation='softmax')(x) -# Instantiates the model given inputs and outputs. +# Instantiate the model given inputs and outputs. model = keras.Model(inputs=inputs, outputs=predictions) -# The "compile" step specifies the training configuration. -model.compile(optimizer='rmsprop', +# The compile step specifies the training configuration. +model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), loss='categorical_crossentropy', metrics=['accuracy']) -# Trains for 5 epochs. +# Trains for 5 epochs model.fit(data, labels, batch_size=32, epochs=5) ``` -This API enables you to create models with multiple inputs and outputs, -and to "share" layers across different inputs -(i.e. to reuse a same instance multiple times). -For examples of these use cases, -please see [this guide to the functional API in Keras](https://keras.io/getting-started/functional-api-guide/). +### Model subclassing ---- +Build a fully-customizable model by subclassing `tf.keras.Model` and defining +your own forward pass. Create layers in the `__init__` method and set them as +attributes of the class instance. Define the forward pass in the `call` method. -## Building fully-customizable research models: the Model subclassing API +Model subclassing is particularly useful when +[eager execution](/programmers_guide/eager) is enabled since the forward pass +can be written imperatively. -Besides `Sequential` and the functional API, one last, more flexible way to -define models is to directly subclass the `Model` class and define your own -forward pass manually. +Key Point: Use the right API for the job. While model subclassing offers +flexibility, it comes at a cost of greater complexity and more opportunities for +user errors. If possible, prefer the functional API. -In this API, you instante layers in `__init__` and set them as attribute of the -class instance. Then you specify the forward pass in `call`. -This API is particularly valuable when using TensorFlow with [eager execution](https://www.tensorflow.org/programmers_guide/eager), -since eager execution allows you to write your forward pass in an -imperative fashion (as if you were writing Numpy code, for instance). +The following example shows a subclassed `tf.keras.Model` using a custom forward +pass: ```python -import tensorflow as tf -from tensorflow import keras - - class MyModel(keras.Model): - def __init__(self, num_classes=2): + def __init__(self, num_classes=10): super(MyModel, self).__init__(name='my_model') self.num_classes = num_classes # Define your layers here. @@ -351,10 +321,10 @@ class MyModel(keras.Model): # Instantiates the subclassed model. -model = MyModel(num_classes=2) +model = MyModel(num_classes=10) -# The "compile" step specifies the training configuration. -model.compile(optimizer='rmsprop', +# The compile step specifies the training configuration. +model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), loss='categorical_crossentropy', metrics=['accuracy']) @@ -362,353 +332,291 @@ model.compile(optimizer='rmsprop', model.fit(data, labels, batch_size=32, epochs=5) ``` -**Remember:** use the right API for the right job. -Using the `Model` subclassing API offers more flexibility, -but at the cost of greater complexity and a larger potential user error surface. -Prefer using the functional API when possible. ---- +### Custom layers -## Callbacks +Create a custom layer by subclassing `tf.keras.layers.Layer` and implementing +the following methods: -Callbacks are objects that you can pass to your model that customize and extend -its behavior during training. -There are callbacks for saving checkpoints of your model at regular intervals -(`tf.keras.callbacks.ModelCheckpoint`), -to dynamically change the learning rate (`tf.keras.callbacks.LearningRateScheduler`) -or to interrupt training when validation performance has stopped improving -(`tf.keras.callbacks.EarlyStopping`). -You can also use a callback to monitor your model's behavior using -[TensorBoard](https://www.tensorflow.org/programmers_guide/summaries_and_tensorboard) -(`tf.keras.callbacks.TensorBoard`). -You can also write your own custom callbacks. - -Different built-in callback are found in `tf.keras.callbacks`. -You use them by passing a `Callback` instance to `fit`: +* `build`: Create the weights of the layer. Add weights with the `add_weight` + method. +* `call`: Define the forward pass. +* `compute_output_shape`: Specify how to compute the output shape of the layer + given the input shape. +* Optionally, a layer can be serialized by implementing the `get_config` method + and the `from_config` class method. + +Here's an example of a custom layer that implements a `matmul` of an input with +a kernel matrix: ```python -from tensorflow import keras +class MyLayer(keras.layers.Layer): + + def __init__(self, output_dim, **kwargs): + self.output_dim = output_dim + super(MyLayer, self).__init__(**kwargs) + + def build(self, input_shape): + shape = tf.TensorShape((input_shape[1], self.output_dim)) + # Create a trainable weight variable for this layer. + self.kernel = self.add_weight(name='kernel', + shape=shape, + initializer='uniform', + trainable=True) + # Be sure to call this at the end + super(MyLayer, self).build(input_shape) -callbacks = [ - # Interrupt training if `val_loss` stops improving for over 2 epochs - keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'), - # Write TensorBoard logs to `./logs` directory - keras.callbacks.TensorBoard(log_dir='./logs') -] -model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks) -``` + def call(self, inputs): + return tf.matmul(inputs, self.kernel) ---- + def compute_output_shape(self, input_shape): + shape = tf.TensorShape(input_shape).as_list() + shape[-1] = self.output_dim + return tf.TensorShape(shape) -## Saving and serialization + def get_config(self): + base_config = super(MyLayer, self).get_config() + base_config['output_dim'] = self.output_dim -### Weights-only saving + @classmethod + def from_config(cls, config): + return cls(**config) -You can save the weight values of a model via `model.save_weights(filepath)`: -```python -# Saves weights to a SavedModel file. -model.save_weights('my_model') +# Create a model using the custom layer +model = keras.Sequential([MyLayer(10), + keras.layers.Activation('softmax')]) -# Restores the model's state -# (this requires a model that has the same architecture). -model.load_weights('my_model') +# The compile step specifies the training configuration +model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), + loss='categorical_crossentropy', + metrics=['accuracy']) + +# Trains for 5 epochs. +model.fit(data, targets, batch_size=32, epochs=5) ``` -By default, this saves the weight in the TensorFlow -[`SavedModel`](https://www.tensorflow.org/programmers_guide/saved_model) format. -You could also save them in the Keras HDF5 format -(which is the default in the multi-backend implementation of Keras): -```python -# Saves weights to a HDF5 file. -model.save_weights('my_model.h5', format='h5') +## Callbacks -# Restores the model's state. -model.load_weights('my_model.h5') -``` +A callback is an object passed to a model to customize and extend its behavior +during training. You can write your own custom callback, or use the built-in +`tf.keras.callbacks` that include: -### Configuration-only saving (serialization) +* `tf.keras.callbacks.ModelCheckpoint`: Save checkpoints of your model at + regular intervals. +* `tf.keras.callbacks.LearningRateScheduler`: Dynamically change the learning + rate. +* `tf.keras.callbacks.EarlyStopping`: Interrupt training when validation + performance has stopped improving. +* `tf.keras.callbacks.TensorBoard`: Monitor the model's behavior using + [TensorBoard](/programmers_guide/summaries_and_tensorboard). -You can also save the model's configuration -(its architecture, without any weight values), -which allows you to recreate the same model later (freshly initialized) even if -you don't have the code that defined it anymore. -Two possible serialization formats are JSON and YAML: +To use a `tf.keras.callbacks.Callback`, pass it to the model's `fit` method: ```python -from tensorflow.keras import models - -# Serializes a model to JSON. -json_string = model.to_json() -# Recreates the model (freshly initialized). -fresh_model = models.from_json(json_string) - -# Serializes a model to YAML. -yaml_string = model.to_yaml() -# Recreates the model. -fresh_model = models.from_yaml(yaml_string) +callbacks = [ + # Interrupt training if `val_loss` stops improving for over 2 epochs + keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'), + # Write TensorBoard logs to `./logs` directory + keras.callbacks.TensorBoard(log_dir='./logs') +] +model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks, + validation_data=(val_data, val_targets)) ``` -Note that this feature is not available with subclassed models, -because they are simply not serializable: -their architecture is defined as Python code -(the body of the `call` method of the model). -### Whole-model saving +## Save and restore -Finally, you can also save a model wholesale, to a file that will contain both -the weight values, the model's configuration, -and even the optimizer's configuration. -The allows you to checkpoint a model and resume training later -- -from the exact same state -- even if you don't have access to the original code. +### Weights only -```python -from tensorflow.keras import models +Save and load the weights of a model using `tf.keras.Model.save_weights`: -model.save('my_model.h5') +```python +# Save weights to a TensorFlow Checkpoint file +model.save_weights('./my_model') -# Recreates the exact same model, complete with weights and optimizer. -model = models.load_model('my_model.h5') +# Restore the model's state, +# this requires a model with the same architecture. +model.load_weights('my_model') ``` ---- - -## Developing custom layers - -You can write your own custom layers by subclassing the class -`tf.keras.layers.Layer`. You will need to implement the following three methods: - -- `build`: Creates the weights of the layer. - Weights should be added via the `add_weight` method. -- `call`: Specifies the forward pass. -- `compute_output_shape`: Specifies how to compute the output shape of the layer - given the input shape. - -Optionally, you may also implement the method `get_config()` and the -class method `from_config()` if you want your layer to be serializable. - -Here's a simple example of a custom layer that implements a `matmul` -of an input with a kernel matrix: +By default, this saves the model's weights in the +[TensorFlow checkpoint](/get_started/checkpoints) file format. Weights can also +be saved to the Keras HDF5 format (the default for the multi-backend +implementation of Keras): ```python -import tensorflow as tf -from tensorflow.keras import layers - -class MyLayer(layers.Layer): - - def __init__(self, output_dim, **kwargs): - self.output_dim = output_dim - super(MyLayer, self).__init__(**kwargs) - - def build(self, input_shape): - # Create a trainable weight variable for this layer. - self.kernel = self.add_weight(name='kernel', - shape=(input_shape[1], self.output_dim), - initializer='uniform', - trainable=True) - # Be sure to call this at the end - super(MyLayer, self).build(input_shape) - - def call(self, inputs): - return tf.matmul(inputs, self.kernel) - - def compute_output_shape(self, input_shape): - shape = tf.TensorShape(input_shape).as_list() - shape[-1] = self.output_dim - return tf.TensorShape(shape) - - def get_config(self): - base_config = super(MyLayer, self).get_config() - base_config['output_dim'] = self.output_dim - - @classmethod - def from_config(cls, config): - return cls(**config) -``` +# Save weights to a HDF5 file +model.save_weights('my_model.h5', save_format='h5') ---- - -## Eager execution +# Restore the model's state +model.load_weights('my_model.h5') +``` -[Eager execution](https://www.tensorflow.org/programmers_guide/eager) -is a way to write TensorFlow code imperatively. -All three `tf.keras` model-building APIs -(`Sequential`, the functional API `Model(inputs, outputs)`, -and the subclassing API `MyModel(Model)`) are compatible with eager execution. -When using `Sequential` or the functional API, it makes no difference to the -user experience whether the model is executing eagerly or not. -Eager execution is most beneficial when used with the `Model` subclassing API, -or when prototyping a custom layer -- that is to say, in APIs that require you -to *write a forward pass as code*, rather than in APIs that allow you to create -models by assembling together existing layers. +### Configuration only -While the same training and evaluating APIs presented in this guide work -as usual with eager execution, you can in addition -write custom training loops using the eager `GradientTape` -and define-by-run autodifferentiation: +A model's configuration can be saved—this serializes the model architecture +without any weights. A saved configuration can recreate and initialize the same +model, even without the code that defined the original model. Keras supports +JSON and YAML serialization formats: ```python -import tensorflow as tf -from tensorflow.contrib import eager as tfe - -# This call begins the eager execution session. -tf.enable_eager_execution() - -model = ... # Defines a Keras model (we recommend Model subclassing in this case). -dataset = ... # Defines a `tf.data` dataset. +# Serialize a model to JSON format +json_string = model.to_json() -optimizer = tf.train.AdamOptimizer(0.01) +# Recreate the model (freshly initialized) +fresh_model = keras.models.from_json(json_string) -for data, labels in dataset: - # Runs the forward pass and loss computation under a `GradientTape` scope, - # which will record all operations in order to prepare for the backward pass. - with tfe.GradientTape() as tape: - predictions = model(data) - loss = loss_function(labels, predictions) +# Serializes a model to YAML format +yaml_string = model.to_yaml() - # Runs the backward pass manually using the operations recorded - # by the gradient tape. - grads = tape.gradient(loss, model.trainable_weights) - optimizer.apply_gradients(zip(grads, model.trainable_weights), - global_step=tf.train.get_or_create_global_step()) +# Recreate the model +fresh_model = keras.models.from_yaml(yaml_string) ``` ---- +Caution: Subclassed models are not serializable because their architecture is +defined by the Python code in the body of the `call` method. -## Further reading -### Documentation +### Entire model -- [tf.keras documentation](https://www.tensorflow.org/api_docs/python/tf/keras) -- [keras.io](https://keras.io/) +The entire model can be saved to a file that contains the weight values, the +model's configuration, and even the optimizer's configuration. This allows you +to checkpoint a model and resume training later—from the exact same +state—without access to the original code. -### tf.keras tutorials and examples - -- [Fashion-MNIST with tf.Keras](https://medium.com/tensorflow/hello-deep-learning-fashion-mnist-with-keras-50fcff8cd74a) -- [Predicting the price of wine with the Keras Functional API and TensorFlow]( - https://medium.com/tensorflow/predicting-the-price-of-wine-with-the-keras-functional-api-and-tensorflow-a95d1c2c1b03) +```python +# Create a trivial model +model = keras.Sequential([ + keras.layers.Dense(10, activation='softmax', input_shape=(32,)), + keras.layers.Dense(10, activation='softmax') +]) +model.compile(optimizer='rmsprop', + loss='categorical_crossentropy', + metrics=['accuracy']) +model.fit(data, targets, batch_size=32, epochs=5) ---- +# Save entire model to a HDF5 file +model.save('my_model.h5') -## FAQ +# Recreate the exact same model, including weights and optimizer. +model = keras.models.load_model('my_model.h5') +``` -### What are the differences between tf.keras and the multi-backend Keras implementation? -`tf.keras` includes first-class support for important TensorFlow-specific -functionality not found in other Keras implementations, in particular: +## Eager execution -- Support for eager execution. -- Support for the `tf.data` API. -- Integration with the - [`tf.estimator` API](https://www.tensorflow.org/programmers_guide/estimators), - via `tf.keras.estimator.model_to_estimator`. +[Eager execution](/programmers_guide/eager) is an imperative programming +environment that evaluates operations immediately. This is not required for +Keras, but is supported by `tf.keras` and useful for inspecting your program and +debugging. -In terms of API differences: `tf.keras` is a full implementation of the -Keras API, so any code targeting the Keras API will run on `tf.keras`. -However, keep in mind that: +All of the `tf.keras` model-building APIs are compatible with eager execution. +And while the `Sequential` and functional APIs can be used, eager execution +especially benefits *model subclassing* and building *custom layers*—the APIs +that require you to write the forward pass as code (instead of the APIs that +create models by assembling existing layers). -- The `tf.keras` API version in the latest TensorFlow release might not be the - same as the latest `keras` version from PyPI. - Check out `tf.keras.__version__` if in doubt. -- In `tf.keras`, the default file format saved by `model.save_weights` is the - TensorFlow `SavedModel` format. - To use HDF5, you can pass the `format='h5'` argument. +See the [eager execution guide](/programmers_guide/eager#build_a_model) for +examples of using Keras models with custom training loops and `tf.GradientTape`. -### What is the relationship between tf.keras and tf.estimator? +## Distribution -The [`tf.estimator` API](https://www.tensorflow.org/programmers_guide/estimators) -is a high-level TensorFlow API for training "estimator" models, -in particular in distributed settings. -This API targets industry use cases, such as distributed training -on large datasets with a focus on eventually exporting a production model. +### Estimators -If you have a `tf.keras` model that would like to train with the `tf.estimator` -API, you can convert your model to an `Estimator` object via the -`model_to_estimator` utility](https://www.tensorflow.org/programmers_guide/estimators#creating_estimators_from_keras_models): +The [Estimators](/programmers_guide/estimators) API is used for training models +for distributed environments. This targets industry use cases such as +distributed training on large datasets that can export a model for production. +A `tf.keras.Model` can be trained with the `tf.estimator` API by converting the +model to an `tf.estimator.Estimator` object with +`tf.keras.estimator.model_to_estimator`. See +[Creating Estimators from Keras models](/programmers_guide/estimators#creating_estimators_from_keras_models). ```python -estimator = tf.keras.estimator.model_to_estimator(model) -``` +model = keras.Sequential([layers.Dense(10,activation='softmax'), + layers.Dense(10,activation='softmax')]) -When using `model_to_estimator`, enabling eager execution is helpful for -developing and debugging your `input_fn` -(as it allows you to easily print your data). +model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), + loss='categorical_crossentropy', + metrics=['accuracy']) + +estimator = keras.estimator.model_to_estimator(model) +``` +Note: Enable [eager execution](/programmers_guide/eager) for debugging +[Estimator input functions](/programmers_guide/premade_estimators#create_input_functions) +and inspecting data. -### How can I run tf.keras models on multiple GPUs? +### Multiple GPUs -You can run tf.keras models on multiple GPUs using the -[`DistributionStrategy API`](https://www.tensorflow.org/versions/master/api_docs/python/tf/contrib/distribute/DistributionStrategy). -The `DistributionStrategy` API allow you to distribute training on multiple GPUs -with almost no changes to your existing code. +`tf.keras` models can run on multiple GPUs using +`tf.contrib.distribute.DistributionStrategy`. This API provides distributed +training on multiple GPUs with almost no changes to existing code. -Currently [`MirroredStrategy`](https://www.tensorflow.org/versions/master/api_docs/python/tf/contrib/distribute/MirroredStrategy) -is the only supported strategy. -`MirroredStrategy` allows you to do in-graph replication with synchronous -training using all-reduce on a single machine. -To use `DistributionStrategy` with a `tf.keras` model, -you can use the `model_to_estimator` utility to convert a `tf.keras` model to -an `Estimator` and then train the estimator. +Currently, `tf.contrib.distribute.MirroredStrategy` is the only supported +distribution strategy. `MirroredStrategy` does in-graph replication with +synchronous training using all-reduce on a single machine. To use +`DistributionStrategy` with Keras, convert the `tf.keras.Model` to a +`tf.estimator.Estimator` with `tf.keras.estimator.model_to_estimator`, then +train the estimator -Here is a simple example of distributing a `tf.keras` model across multiple GPUs -on a single machine. +The following example distributes a `tf.keras.Model` across multiple GPUs on a +single machine. -Let's first define a simple model: +First, define a simple model: ```python -model = tf.keras.Sequential() -model.add(tf.keras.layers.Dense(16, activation='relu', input_shape=(10,))) -model.add(tf.keras.layers.Dense(1, activation='sigmoid')) +model = keras.Sequential() +model.add(keras.layers.Dense(16, activation='relu', input_shape=(10,))) +model.add(keras.layers.Dense(1, activation='sigmoid')) + optimizer = tf.train.GradientDescentOptimizer(0.2) + model.compile(loss='binary_crossentropy', optimizer=optimizer) model.summary() ``` -Let's use `model_to_estimator` to create an `Estimator` instance from the -`tf.keras` model defined above. +Convert the Keras model to a `tf.estimator.Estimator` instance: ```python -keras_estimator = tf.keras.estimator.model_to_estimator( - keras_model=model, - config=config, - model_dir='/tmp/model_dir') +keras_estimator = keras.estimator.model_to_estimator( + keras_model=model, + config=config, + model_dir='/tmp/model_dir') ``` -We'll use `tf.data.Datasets` to define our input pipeline. -Our `input_fn` returns a `tf.data.Dataset` object that we then use to distribute -the data across multiple devices with each device processing +Define an *input pipeline*. The `input_fn` returns a `tf.data.Dataset` object +used to distribute the data across multiple devices—with each device processing a slice of the input batch. ```python def input_fn(): - x = np.random.random((1024, 10)) - y = np.random.randint(2, size=(1024, 1)) - x = tf.cast(x, tf.float32) - dataset = tf.data.Dataset.from_tensor_slices((x, y)) - dataset = dataset.repeat(10) - dataset = dataset.batch(32) - return dataset + x = np.random.random((1024, 10)) + y = np.random.randint(2, size=(1024, 1)) + x = tf.cast(x, tf.float32) + dataset = tf.data.Dataset.from_tensor_slices((x, y)) + dataset = dataset.repeat(10) + dataset = dataset.batch(32) + return dataset ``` -The next step is to create a `RunConfig` and set the train_distribute argument -to the new `MirroredStrategy` instance. -You can specify a list of devices or the `num_gpus` argument when creating -a `MirroredStrategy` instance. -Not specifying any arguments defaults to using all the available GPUs like we do -in this example. +Next, create a `tf.estimator.RunConfig` and set the `train_distribute` argument +to the `tf.contrib.distribute.MirroredStrategy` instance. When creating +`MirroredStrategy`, you can specify a list of devices or set the `num_gpus` +argument. The default uses all available GPUs, like the following: ```python strategy = tf.contrib.distribute.MirroredStrategy() config = tf.estimator.RunConfig(train_distribute=strategy) ``` -Call train on the `Estimator` instance providing the `input_fn` and `steps` -arguments as input: +Finally, train the `Estimator` instance by providing the `input_fn` and `steps` +arguments: ```python keras_estimator.train(input_fn=input_fn, steps=10) -- cgit v1.2.3 From 7e859ebc65bf7d77ed89f736c7fd6fede0a93c92 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Mon, 18 Jun 2018 11:07:48 -0700 Subject: Add missing Eager relnotes for TensorFlow 1.9. (#20101) --- RELEASE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 879ce6e440..510eca5467 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -22,6 +22,8 @@ * (C++) `DatasetBase::MakeIterator()` has been renamed to `DatasetBase::MakeIteratorInternal()`. * (C++) `IteratorBase::Initialize()` method was added to support raising errors during iterator construction. * Eager Execution: + * Added the ability to pause recording operations for gradient computation via `tf.GradientTape.stop_recording`. + * Updated documentation, introductory notebooks. * `tf.keras`: * Move Keras code out of _impl folder and remove API files. * `tf.keras.Model.save_weights` now saves in TensorFlow format by default. -- cgit v1.2.3 From 86a6b0d7efbe5a3fa1f511237b85c926a6aef3a5 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Tue, 19 Jun 2018 17:47:37 -0700 Subject: [GCS] Typo in ConfigureGcsHook. This commit fixes a typo on ConfigureGcsHook that prevented its correct operation. --- tensorflow/contrib/cloud/python/ops/gcs_config_ops.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/cloud/python/ops/gcs_config_ops.py b/tensorflow/contrib/cloud/python/ops/gcs_config_ops.py index 8c8c5acb31..4f7300fd1f 100644 --- a/tensorflow/contrib/cloud/python/ops/gcs_config_ops.py +++ b/tensorflow/contrib/cloud/python/ops/gcs_config_ops.py @@ -120,13 +120,17 @@ class ConfigureGcsHook(training.SessionRunHook): def begin(self): if self._credentials: self._credentials_placeholder = array_ops.placeholder(dtypes.string) - self._credentials_ops = gen_gcs_config_ops.gcs_configure_credentials( + self._credentials_op = gen_gcs_config_ops.gcs_configure_credentials( self._credentials_placeholder) + else: + self._credentials_op = None if self._block_cache: self._block_cache_op = gen_gcs_config_ops.gcs_configure_block_cache( max_cache_size=self._block_cache.max_bytes, block_size=self._block_cache.block_size, max_staleness=self._block_cache.max_staleness) + else: + self._block_cache_op = None def after_create_session(self, session, coord): del coord -- cgit v1.2.3 From fdbb80f217d3a153b4eda66c766df921b3f73ab4 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Wed, 20 Jun 2018 14:08:57 -0700 Subject: Move external/ directory in pip package. Moving external/ directory in the pip packages (which is currently installed directly into site-packages directory). Moving the directory to tensorflow/include/external/. Also, removing all python files from external (since it should really only contain headers and license files.) --- tensorflow/tools/pip_package/build_pip_package.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh index f7e42ce536..9e41514cfa 100755 --- a/tensorflow/tools/pip_package/build_pip_package.sh +++ b/tensorflow/tools/pip_package/build_pip_package.sh @@ -24,9 +24,15 @@ function real_path() { function cp_external() { local src_dir=$1 local dest_dir=$2 - for f in `find "$src_dir" -maxdepth 1 -mindepth 1 ! -name '*local_config_cuda*' ! -name '*local_config_tensorrt*' ! -name '*org_tensorflow*'`; do - cp -R "$f" "$dest_dir" + + pushd . + cd "$src_dir" + for f in `find . ! -type d ! -name '*.py' ! -name '*local_config_cuda*' ! -name '*local_config_tensorrt*' ! -name '*org_tensorflow*'`; do + mkdir -p "${dest_dir}/$(dirname ${f})" + cp "${f}" "${dest_dir}/$(dirname ${f})/" done + popd + mkdir -p "${dest_dir}/local_config_cuda/cuda/cuda/" cp "${src_dir}/local_config_cuda/cuda/cuda/cuda_config.h" "${dest_dir}/local_config_cuda/cuda/cuda/" } @@ -49,6 +55,8 @@ function prepare_src() { TMPDIR="$1" mkdir -p "$TMPDIR" + EXTERNAL_INCLUDES="${TMPDIR}/tensorflow/include/external" + echo $(date) : "=== Preparing sources in dir: ${TMPDIR}" if [ ! -d bazel-bin/tensorflow ]; then @@ -66,10 +74,9 @@ function prepare_src() { cp -R \ bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles/org_tensorflow/tensorflow \ "${TMPDIR}" - mkdir "${TMPDIR}/external" cp_external \ bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles \ - "${TMPDIR}/external" + "${EXTERNAL_INCLUDES}/" RUNFILES=bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles/org_tensorflow else RUNFILES=bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow @@ -78,10 +85,9 @@ function prepare_src() { cp -R \ bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/tensorflow \ "${TMPDIR}" - mkdir "${TMPDIR}/external" cp_external \ bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/external \ - "${TMPDIR}/external" + "${EXTERNAL_INCLUDES}" # Copy MKL libs over so they can be loaded at runtime so_lib_dir=$(ls $RUNFILES | grep solib) || true if [ -n "${so_lib_dir}" ]; then @@ -96,10 +102,9 @@ function prepare_src() { cp -R \ bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/tensorflow \ "${TMPDIR}" - mkdir "${TMPDIR}/external" cp_external \ bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles \ - "${TMPDIR}/external" + "${EXTERNAL_INCLUDES}" # Copy MKL libs over so they can be loaded at runtime so_lib_dir=$(ls $RUNFILES | grep solib) || true if [ -n "${so_lib_dir}" ]; then -- cgit v1.2.3 From 1adbc5aa6927d1a5d7151c31aea1da6e73a1b53c Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Thu, 31 May 2018 19:03:21 -0700 Subject: Add a single positional argument mode for shape inference in subclassed Models. Allows fit() when call's signature looks something like call(x, training=True). Calling conventions are "inputs", single positional, and multiple positional. Right now the distinction between "inputs" and single positional calling conventions is the text of one error message. Both support shape inference (which just hasn't been implemented for multiple positional input arguments yet). PiperOrigin-RevId: 198815483 --- tensorflow/python/keras/engine/base_layer.py | 45 ++++++++++++++++---- tensorflow/python/keras/engine/network.py | 50 +++++++++++++++++++---- tensorflow/python/keras/engine/training.py | 27 +++++++----- tensorflow/python/keras/model_subclassing_test.py | 4 +- 4 files changed, 98 insertions(+), 28 deletions(-) diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py index 24716cfbe4..4814275fd5 100644 --- a/tensorflow/python/keras/engine/base_layer.py +++ b/tensorflow/python/keras/engine/base_layer.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function import collections +import enum # pylint: disable=g-bad-import-order import inspect # Necessary supplement to tf_inspect to deal with variadic args. import numpy as np @@ -50,6 +51,20 @@ from tensorflow.python.util import tf_inspect from tensorflow.python.util.tf_export import tf_export +class CallConvention(enum.Enum): + """Calling conventions for passing `Layer` inputs to `Layer.call`.""" + # The Layer takes inputs as its first argument, named "inputs" for + # compatibility with the signature of Layer.__call__. This is the mode assumed + # for Layers which are not subclassed Models. + EXPLICIT_INPUTS_ARGUMENT = 1 + # The Layer takes a single positional argument, not named "inputs". It's + # treated like an "inputs" argument. + SINGLE_POSITIONAL_ARGUMENT = 2 + # The Layer has multiple positional arguments to which its inputs should be + # bound. + POSITIONAL_ARGUMENTS_ARE_INPUTS = 3 + + @tf_export('keras.layers.Layer') class Layer(checkpointable.CheckpointableBase): """Base layer class. @@ -149,7 +164,7 @@ class Layer(checkpointable.CheckpointableBase): self._call_fn_args = function_utils.fn_args(self.call) self._compute_previous_mask = ('mask' in self._call_fn_args or hasattr(self, 'compute_mask')) - self._uses_inputs_arg = True + self._call_convention = CallConvention.EXPLICIT_INPUTS_ARGUMENT # These lists will be filled via successive calls # to self._add_inbound_node(). @@ -793,12 +808,22 @@ class Layer(checkpointable.CheckpointableBase): pass # C type such as dict. Masking not supported in this case. def _set_connectivity_metadata_(self, inputs, outputs, args, kwargs): - if args and getattr(self, '_uses_inputs_arg', True): - raise TypeError( - 'This Layer takes an `inputs` argument to call(), and only the ' - '`inputs` argument may be specified as a positional argument. ' - 'Pass everything else as a keyword argument (those arguments will' - ' not be tracked as inputs to the Layer).') + call_convention = getattr(self, '_call_convention', + CallConvention.EXPLICIT_INPUTS_ARGUMENT) + if args: + if call_convention == CallConvention.EXPLICIT_INPUTS_ARGUMENT: + raise TypeError( + 'This Layer takes an `inputs` argument to call(), and only the ' + '`inputs` argument may be specified as a positional argument. ' + 'Pass everything else as a keyword argument (those arguments will' + ' not be tracked as inputs to the Layer).') + elif call_convention == CallConvention.SINGLE_POSITIONAL_ARGUMENT: + raise TypeError( + 'This Layer takes a single positional argument to call(), which is ' + 'by convention the inputs argument, and only this argument may be ' + 'specified as a positional argument. Pass everything else as a ' + 'keyword argument (those arguments will not be tracked as inputs ' + 'to the Layer).') # If the layer returns tensors from its inputs, unmodified, # we copy them to avoid loss of tensor metadata. @@ -834,7 +859,11 @@ class Layer(checkpointable.CheckpointableBase): A tuple of (inputs, non_input_kwargs). These may be the same objects as were passed in (call_args and call_kwargs). """ - if getattr(self, '_uses_inputs_arg', True): + call_convention = getattr(self, '_call_convention', + CallConvention.EXPLICIT_INPUTS_ARGUMENT) + if (call_convention in ( + CallConvention.EXPLICIT_INPUTS_ARGUMENT, + CallConvention.SINGLE_POSITIONAL_ARGUMENT)): assert len(call_args) == 1 # TypeError raised earlier in __call__. return call_args[0], call_kwargs else: diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py index 3d567b8378..6f27eea1e7 100644 --- a/tensorflow/python/keras/engine/network.py +++ b/tensorflow/python/keras/engine/network.py @@ -135,7 +135,7 @@ class Network(base_layer.Layer): self._in_progress_restore_finalizer = None def _init_graph_network(self, inputs, outputs, name=None): - self._uses_inputs_arg = True + self._call_convention = base_layer.CallConvention.EXPLICIT_INPUTS_ARGUMENT # Normalize and set self.inputs, self.outputs. if isinstance(inputs, (list, tuple)): self.inputs = list(inputs) # Tensor or list of tensors. @@ -295,19 +295,55 @@ class Network(base_layer.Layer): def _init_subclassed_network(self, name=None): self._base_init(name=name) self._is_graph_network = False - call_args = tf_inspect.getargspec(self.call).args - if 'training' in call_args: + call_argspec = tf_inspect.getargspec(self.call) + if 'training' in call_argspec.args: self._expects_training_arg = True else: self._expects_training_arg = False - if 'inputs' in call_args: - self._uses_inputs_arg = True - else: - self._uses_inputs_arg = False + self._call_convention = self._determine_call_convention(call_argspec) self.outputs = None self.inputs = None self.built = False + def _determine_call_convention(self, call_argspec): + """Decides how `self.call()` is invoked. See base_layer.CallConvention.""" + if call_argspec.varargs: + may_take_single_argument = False + else: + try: + # Note: tf_inspect doesn't raise a TypeError when regular inspect would, + # so we need to keep in mind that "getcallargs" may have returned + # something even though we under-specified positional arguments. + all_args = tf_inspect.getcallargs(self.call, None) + self_args = set() + for arg_name, obj in all_args.items(): + if obj is self: + self_args.add(arg_name) + may_take_single_argument = True + except TypeError: + may_take_single_argument = False + if may_take_single_argument: + # A single positional argument (plus "self") is considered equivalent to + # an "inputs" argument. + all_positional_args = len(call_argspec.args) + if call_argspec.defaults is not None: + all_positional_args -= len(call_argspec.defaults) + non_self_positional_args = all_positional_args + for positional_arg_name in call_argspec.args[:all_positional_args]: + if positional_arg_name in self_args: + non_self_positional_args -= 1 + if non_self_positional_args == 1: + if 'inputs' in call_argspec.args[all_positional_args:]: + raise TypeError( + "Model.call() takes a single positional argument (to which " + "inputs are passed by convention) and a separate 'inputs' " + "argument. Unable to determine which arguments are inputs.") + return base_layer.CallConvention.SINGLE_POSITIONAL_ARGUMENT + if 'inputs' in call_argspec.args: + return base_layer.CallConvention.EXPLICIT_INPUTS_ARGUMENT + else: + return base_layer.CallConvention.POSITIONAL_ARGUMENTS_ARE_INPUTS + def _track_layers(self, layers): """Add Checkpointable dependencies on a list of Layers.""" weight_layer_index = 0 diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index 6d625f16c2..04a2aa7664 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -31,12 +31,11 @@ from tensorflow.python.keras import backend as K from tensorflow.python.keras import losses from tensorflow.python.keras import metrics as metrics_module from tensorflow.python.keras import optimizers +from tensorflow.python.keras.engine import base_layer from tensorflow.python.keras.engine import training_arrays from tensorflow.python.keras.engine import training_eager from tensorflow.python.keras.engine import training_generator from tensorflow.python.keras.engine import training_utils -from tensorflow.python.keras.engine.base_layer import DeferredTensor -from tensorflow.python.keras.engine.base_layer import Layer from tensorflow.python.keras.engine.network import Network from tensorflow.python.keras.utils.generic_utils import slice_arrays from tensorflow.python.ops import array_ops @@ -523,7 +522,7 @@ class Model(Network): # Keep track of state updates created by # stateful metrics (i.e. metrics layers). - if isinstance(metric_fn, Layer) and metric_fn.stateful: + if isinstance(metric_fn, base_layer.Layer) and metric_fn.stateful: self.stateful_metric_names.append(metric_name) self.stateful_metric_functions.append(metric_fn) self.metrics_updates += metric_fn.updates @@ -959,11 +958,17 @@ class Model(Network): whether to build the model's graph in inference mode (False), training mode (True), or using the Keras learning phase (None). """ - if not getattr(self, '_uses_inputs_arg', True): + call_convention = getattr( + self, + '_call_convention', + base_layer.CallConvention.EXPLICIT_INPUTS_ARGUMENT) + if call_convention not in ( + base_layer.CallConvention.EXPLICIT_INPUTS_ARGUMENT, + base_layer.CallConvention.SINGLE_POSITIONAL_ARGUMENT): raise NotImplementedError( - 'Subclassed Models without "inputs" in their call() signatures do ' - 'not yet support shape inference. File a feature request if this ' - 'limitation bothers you.') + 'Subclassed Models without "inputs" (or single positional arguments) ' + 'in their call() signatures do not yet support shape inference. File ' + 'a feature request if this limitation bothers you.') if self.__class__.__name__ == 'Sequential': # Note: we can't test whether the model is `Sequential` via `isinstance` # since `Sequential` depends on `Model`. @@ -1020,11 +1025,11 @@ class Model(Network): else: dummy_output_values = [dummy_output_values] self.outputs = [ - DeferredTensor(shape=(None for _ in v.shape), - dtype=v.dtype) for v in dummy_output_values] + base_layer.DeferredTensor(shape=(None for _ in v.shape), + dtype=v.dtype) for v in dummy_output_values] self.inputs = [ - DeferredTensor(shape=(None for _ in v.shape), - dtype=v.dtype) for v in dummy_input_values] + base_layer.DeferredTensor(shape=(None for _ in v.shape), + dtype=v.dtype) for v in dummy_input_values] self.input_names = [ 'input_%d' % (i + 1) for i in range(len(dummy_input_values))] self.output_names = [ diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py index 86f7e20bec..8fb957da43 100644 --- a/tensorflow/python/keras/model_subclassing_test.py +++ b/tensorflow/python/keras/model_subclassing_test.py @@ -56,8 +56,8 @@ class SimpleTestModel(keras.Model): if self.use_bn: self.bn = keras.layers.BatchNormalization(axis=-1) - def call(self, inputs): - x = self.dense1(inputs) + def call(self, x): + x = self.dense1(x) if self.use_dp: x = self.dp(x) if self.use_bn: -- cgit v1.2.3 From 6fb75293ec2cb5cd8d815cf98ec33aa953442b34 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Wed, 20 Jun 2018 10:10:55 -0700 Subject: [tf.data] Properly export `tf.contrib.data.choose_from_datasets()` PiperOrigin-RevId: 201371642 --- tensorflow/contrib/data/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py index 1af1ed08b5..9c6a13333e 100644 --- a/tensorflow/contrib/data/__init__.py +++ b/tensorflow/contrib/data/__init__.py @@ -72,6 +72,7 @@ from tensorflow.contrib.data.python.ops.error_ops import ignore_errors from tensorflow.contrib.data.python.ops.get_single_element import get_single_element from tensorflow.contrib.data.python.ops.grouping import bucket_by_sequence_length from tensorflow.contrib.data.python.ops.grouping import group_by_window +from tensorflow.contrib.data.python.ops.interleave_ops import choose_from_datasets from tensorflow.contrib.data.python.ops.interleave_ops import parallel_interleave from tensorflow.contrib.data.python.ops.interleave_ops import sample_from_datasets from tensorflow.contrib.data.python.ops.interleave_ops import sloppy_interleave -- cgit v1.2.3 From 579b598862b14b7a8e242cb1b094221f7e08b499 Mon Sep 17 00:00:00 2001 From: Yu Yi Date: Fri, 22 Jun 2018 11:46:33 -0400 Subject: update the py/py3 toolchain config - the python3 are installed in /opt/python3.6 in the base toolchain container: https://console.cloud.google.com/launcher/details/google/rbe-ubuntu16-04 --- third_party/toolchains/cpus/py/BUILD | 242 ++++++++++++++++------------------ third_party/toolchains/cpus/py3/BUILD | 234 ++++++++++++++++---------------- 2 files changed, 231 insertions(+), 245 deletions(-) diff --git a/third_party/toolchains/cpus/py/BUILD b/third_party/toolchains/cpus/py/BUILD index c175742cbf..10184e215b 100644 --- a/third_party/toolchains/cpus/py/BUILD +++ b/third_party/toolchains/cpus/py/BUILD @@ -6,20 +6,26 @@ licenses(["restricted"]) package(default_visibility = ["//visibility:public"]) +# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib +# See https://docs.python.org/3/extending/windows.html +cc_import( + name = "python_lib", + interface_library = select({ + ":windows": ":python_import_lib", + # A placeholder for Unix platforms which makes --no_build happy. + "//conditions:default": "not-existing.lib", + }), + system_provided = 1, +) + cc_library( name = "python_headers", hdrs = [":python_include"], - data = select({ - ":windows": [":python_import_lib"], + deps = select({ + ":windows": [":python_lib"], "//conditions:default": [], }), includes = ["python_include"], - linkopts = select({ - # TODO(pcloudy): Ideally, this should just go into deps after resolving - # https://github.com/bazelbuild/bazel/issues/3237, - ":windows": ["$(locations :python_import_lib)"], - "//conditions:default": [], - }), ) cc_library( @@ -37,161 +43,135 @@ config_setting( genrule( name = "python_include", outs = [ + "python_include/Python-ast.h", + "python_include/Python.h", + "python_include/abstract.h", + "python_include/asdl.h", + "python_include/ast.h", + "python_include/bitset.h", + "python_include/boolobject.h", + "python_include/bufferobject.h", + "python_include/bytearrayobject.h", + "python_include/bytes_methods.h", + "python_include/bytesobject.h", + "python_include/cStringIO.h", + "python_include/cellobject.h", + "python_include/ceval.h", + "python_include/classobject.h", + "python_include/cobject.h", "python_include/code.h", + "python_include/codecs.h", + "python_include/compile.h", + "python_include/complexobject.h", + "python_include/datetime.h", + "python_include/descrobject.h", + "python_include/dictobject.h", "python_include/dtoa.h", - "python_include/tupleobject.h", - "python_include/object.h", - "python_include/ast.h", - "python_include/pymacconfig.h", + "python_include/enumobject.h", "python_include/errcode.h", + "python_include/eval.h", + "python_include/fileobject.h", + "python_include/floatobject.h", "python_include/frameobject.h", - "python_include/pgenheaders.h", - "python_include/cellobject.h", + "python_include/funcobject.h", + "python_include/genobject.h", + "python_include/graminit.h", + "python_include/grammar.h", + "python_include/import.h", "python_include/intobject.h", - "python_include/pythread.h", - "python_include/cStringIO.h", - "python_include/boolobject.h", + "python_include/intrcheck.h", + "python_include/iterobject.h", + "python_include/listobject.h", + "python_include/longintrepr.h", + "python_include/longobject.h", + "python_include/marshal.h", + "python_include/memoryobject.h", + "python_include/metagrammar.h", + "python_include/methodobject.h", "python_include/modsupport.h", - "python_include/import.h", - "python_include/pymath.h", + "python_include/moduleobject.h", "python_include/node.h", - "python_include/funcobject.h", - "python_include/eval.h", - "python_include/longintrepr.h", - "python_include/floatobject.h", - "python_include/rangeobject.h", - "python_include/pyfpe.h", - "python_include/pystrcmp.h", - "python_include/dictobject.h", - "python_include/pyarena.h", + "python_include/object.h", "python_include/objimpl.h", - "python_include/bitset.h", - "python_include/memoryobject.h", - "python_include/bytearrayobject.h", + "python_include/opcode.h", + "python_include/osdefs.h", + "python_include/parsetok.h", + "python_include/patchlevel.h", + "python_include/pgen.h", + "python_include/pgenheaders.h", + "python_include/py_curses.h", + "python_include/pyarena.h", + "python_include/pycapsule.h", + "python_include/pyconfig.h", + "python_include/pyctype.h", "python_include/pydebug.h", "python_include/pyerrors.h", - "python_include/weakrefobject.h", - "python_include/grammar.h", - "python_include/symtable.h", - "python_include/longobject.h", - "python_include/structmember.h", - "python_include/enumobject.h", - "python_include/classobject.h", - "python_include/unicodeobject.h", - "python_include/sliceobject.h", - "python_include/pystrtod.h", - "python_include/genobject.h", - "python_include/pymactoolbox.h", - "python_include/compile.h", "python_include/pyexpat.h", - "python_include/asdl.h", - "python_include/codecs.h", - "python_include/pyctype.h", - "python_include/sysmodule.h", - "python_include/methodobject.h", - "python_include/graminit.h", - "python_include/cobject.h", - "python_include/intrcheck.h", - "python_include/pyport.h", - "python_include/warnings.h", - "python_include/osdefs.h", - "python_include/fileobject.h", - "python_include/stringobject.h", - "python_include/timefuncs.h", - "python_include/traceback.h", - "python_include/ceval.h", - "python_include/bytes_methods.h", - "python_include/pyconfig.h", - "python_include/Python.h", - "python_include/moduleobject.h", - "python_include/pystate.h", - "python_include/descrobject.h", - "python_include/ucnhash.h", + "python_include/pyfpe.h", "python_include/pygetopt.h", + "python_include/pymacconfig.h", + "python_include/pymactoolbox.h", + "python_include/pymath.h", "python_include/pymem.h", - "python_include/complexobject.h", - "python_include/structseq.h", - "python_include/datetime.h", + "python_include/pyport.h", + "python_include/pystate.h", + "python_include/pystrcmp.h", + "python_include/pystrtod.h", "python_include/pythonrun.h", - "python_include/numpy/oldnumeric.h", - "python_include/numpy/npy_1_7_deprecated_api.h", - "python_include/numpy/ufunc_api.txt", - "python_include/numpy/multiarray_api.txt", - "python_include/numpy/halffloat.h", - "python_include/numpy/npy_common.h", - "python_include/numpy/utils.h", - "python_include/numpy/npy_interrupt.h", - "python_include/numpy/npy_endian.h", - "python_include/numpy/__ufunc_api.h", - "python_include/numpy/_neighborhood_iterator_imp.h", - "python_include/numpy/ufuncobject.h", - "python_include/numpy/ndarraytypes.h", - "python_include/numpy/npy_math.h", - "python_include/numpy/noprefix.h", - "python_include/numpy/npy_3kcompat.h", - "python_include/numpy/arrayscalars.h", - "python_include/numpy/npy_os.h", - "python_include/numpy/ndarrayobject.h", - "python_include/numpy/npy_no_deprecated_api.h", - "python_include/numpy/arrayobject.h", - "python_include/numpy/_numpyconfig.h", - "python_include/numpy/__multiarray_api.h", - "python_include/numpy/npy_cpu.h", - "python_include/numpy/old_defines.h", - "python_include/numpy/numpyconfig.h", - "python_include/pycapsule.h", + "python_include/pythread.h", + "python_include/rangeobject.h", "python_include/setobject.h", - "python_include/listobject.h", - "python_include/bytesobject.h", - "python_include/pgen.h", - "python_include/patchlevel.h", - "python_include/opcode.h", - "python_include/parsetok.h", - "python_include/marshal.h", + "python_include/sliceobject.h", + "python_include/stringobject.h", + "python_include/structmember.h", + "python_include/structseq.h", + "python_include/symtable.h", + "python_include/sysmodule.h", + "python_include/timefuncs.h", "python_include/token.h", - "python_include/iterobject.h", - "python_include/abstract.h", - "python_include/py_curses.h", - "python_include/metagrammar.h", - "python_include/bufferobject.h", - "python_include/Python-ast.h", + "python_include/traceback.h", + "python_include/tupleobject.h", + "python_include/ucnhash.h", + "python_include/unicodeobject.h", + "python_include/warnings.h", + "python_include/weakrefobject.h", ], cmd = """ -cp "/usr/include/python2.7/code.h" "$(@D)/python_include/code.h" && cp "/usr/include/python2.7/dtoa.h" "$(@D)/python_include/dtoa.h" && cp "/usr/include/python2.7/tupleobject.h" "$(@D)/python_include/tupleobject.h" && cp "/usr/include/python2.7/object.h" "$(@D)/python_include/object.h" && cp "/usr/include/python2.7/ast.h" "$(@D)/python_include/ast.h" && cp "/usr/include/python2.7/pymacconfig.h" "$(@D)/python_include/pymacconfig.h" && cp "/usr/include/python2.7/errcode.h" "$(@D)/python_include/errcode.h" && cp "/usr/include/python2.7/frameobject.h" "$(@D)/python_include/frameobject.h" && cp "/usr/include/python2.7/pgenheaders.h" "$(@D)/python_include/pgenheaders.h" && cp "/usr/include/python2.7/cellobject.h" "$(@D)/python_include/cellobject.h" && cp "/usr/include/python2.7/intobject.h" "$(@D)/python_include/intobject.h" && cp "/usr/include/python2.7/pythread.h" "$(@D)/python_include/pythread.h" && cp "/usr/include/python2.7/cStringIO.h" "$(@D)/python_include/cStringIO.h" && cp "/usr/include/python2.7/boolobject.h" "$(@D)/python_include/boolobject.h" && cp "/usr/include/python2.7/modsupport.h" "$(@D)/python_include/modsupport.h" && cp "/usr/include/python2.7/import.h" "$(@D)/python_include/import.h" && cp "/usr/include/python2.7/pymath.h" "$(@D)/python_include/pymath.h" && cp "/usr/include/python2.7/node.h" "$(@D)/python_include/node.h" && cp "/usr/include/python2.7/funcobject.h" "$(@D)/python_include/funcobject.h" && cp "/usr/include/python2.7/eval.h" "$(@D)/python_include/eval.h" && cp "/usr/include/python2.7/longintrepr.h" "$(@D)/python_include/longintrepr.h" && cp "/usr/include/python2.7/floatobject.h" "$(@D)/python_include/floatobject.h" && cp "/usr/include/python2.7/rangeobject.h" "$(@D)/python_include/rangeobject.h" && cp "/usr/include/python2.7/pyfpe.h" "$(@D)/python_include/pyfpe.h" && cp "/usr/include/python2.7/pystrcmp.h" "$(@D)/python_include/pystrcmp.h" && cp "/usr/include/python2.7/dictobject.h" "$(@D)/python_include/dictobject.h" && cp "/usr/include/python2.7/pyarena.h" "$(@D)/python_include/pyarena.h" && cp "/usr/include/python2.7/objimpl.h" "$(@D)/python_include/objimpl.h" && cp "/usr/include/python2.7/bitset.h" "$(@D)/python_include/bitset.h" && cp "/usr/include/python2.7/memoryobject.h" "$(@D)/python_include/memoryobject.h" && cp "/usr/include/python2.7/bytearrayobject.h" "$(@D)/python_include/bytearrayobject.h" && cp "/usr/include/python2.7/pydebug.h" "$(@D)/python_include/pydebug.h" && cp "/usr/include/python2.7/pyerrors.h" "$(@D)/python_include/pyerrors.h" && cp "/usr/include/python2.7/weakrefobject.h" "$(@D)/python_include/weakrefobject.h" && cp "/usr/include/python2.7/grammar.h" "$(@D)/python_include/grammar.h" && cp "/usr/include/python2.7/symtable.h" "$(@D)/python_include/symtable.h" && cp "/usr/include/python2.7/longobject.h" "$(@D)/python_include/longobject.h" && cp "/usr/include/python2.7/structmember.h" "$(@D)/python_include/structmember.h" && cp "/usr/include/python2.7/enumobject.h" "$(@D)/python_include/enumobject.h" && cp "/usr/include/python2.7/classobject.h" "$(@D)/python_include/classobject.h" && cp "/usr/include/python2.7/unicodeobject.h" "$(@D)/python_include/unicodeobject.h" && cp "/usr/include/python2.7/sliceobject.h" "$(@D)/python_include/sliceobject.h" && cp "/usr/include/python2.7/pystrtod.h" "$(@D)/python_include/pystrtod.h" && cp "/usr/include/python2.7/genobject.h" "$(@D)/python_include/genobject.h" && cp "/usr/include/python2.7/pymactoolbox.h" "$(@D)/python_include/pymactoolbox.h" && cp "/usr/include/python2.7/compile.h" "$(@D)/python_include/compile.h" && cp "/usr/include/python2.7/pyexpat.h" "$(@D)/python_include/pyexpat.h" && cp "/usr/include/python2.7/asdl.h" "$(@D)/python_include/asdl.h" && cp "/usr/include/python2.7/codecs.h" "$(@D)/python_include/codecs.h" && cp "/usr/include/python2.7/pyctype.h" "$(@D)/python_include/pyctype.h" && cp "/usr/include/python2.7/sysmodule.h" "$(@D)/python_include/sysmodule.h" && cp "/usr/include/python2.7/methodobject.h" "$(@D)/python_include/methodobject.h" && cp "/usr/include/python2.7/graminit.h" "$(@D)/python_include/graminit.h" && cp "/usr/include/python2.7/cobject.h" "$(@D)/python_include/cobject.h" && cp "/usr/include/python2.7/intrcheck.h" "$(@D)/python_include/intrcheck.h" && cp "/usr/include/python2.7/pyport.h" "$(@D)/python_include/pyport.h" && cp "/usr/include/python2.7/warnings.h" "$(@D)/python_include/warnings.h" && cp "/usr/include/python2.7/osdefs.h" "$(@D)/python_include/osdefs.h" && cp "/usr/include/python2.7/fileobject.h" "$(@D)/python_include/fileobject.h" && cp "/usr/include/python2.7/stringobject.h" "$(@D)/python_include/stringobject.h" && cp "/usr/include/python2.7/timefuncs.h" "$(@D)/python_include/timefuncs.h" && cp "/usr/include/python2.7/traceback.h" "$(@D)/python_include/traceback.h" && cp "/usr/include/python2.7/ceval.h" "$(@D)/python_include/ceval.h" && cp "/usr/include/python2.7/bytes_methods.h" "$(@D)/python_include/bytes_methods.h" && cp "/usr/include/python2.7/pyconfig.h" "$(@D)/python_include/pyconfig.h" && cp "/usr/include/python2.7/Python.h" "$(@D)/python_include/Python.h" && cp "/usr/include/python2.7/moduleobject.h" "$(@D)/python_include/moduleobject.h" && cp "/usr/include/python2.7/pystate.h" "$(@D)/python_include/pystate.h" && cp "/usr/include/python2.7/descrobject.h" "$(@D)/python_include/descrobject.h" && cp "/usr/include/python2.7/ucnhash.h" "$(@D)/python_include/ucnhash.h" && cp "/usr/include/python2.7/pygetopt.h" "$(@D)/python_include/pygetopt.h" && cp "/usr/include/python2.7/pymem.h" "$(@D)/python_include/pymem.h" && cp "/usr/include/python2.7/complexobject.h" "$(@D)/python_include/complexobject.h" && cp "/usr/include/python2.7/structseq.h" "$(@D)/python_include/structseq.h" && cp "/usr/include/python2.7/datetime.h" "$(@D)/python_include/datetime.h" && cp "/usr/include/python2.7/pythonrun.h" "$(@D)/python_include/pythonrun.h" && cp "/usr/include/python2.7/numpy/oldnumeric.h" "$(@D)/python_include/numpy/oldnumeric.h" && cp "/usr/include/python2.7/numpy/npy_1_7_deprecated_api.h" "$(@D)/python_include/numpy/npy_1_7_deprecated_api.h" && cp "/usr/include/python2.7/numpy/ufunc_api.txt" "$(@D)/python_include/numpy/ufunc_api.txt" && cp "/usr/include/python2.7/numpy/multiarray_api.txt" "$(@D)/python_include/numpy/multiarray_api.txt" && cp "/usr/include/python2.7/numpy/halffloat.h" "$(@D)/python_include/numpy/halffloat.h" && cp "/usr/include/python2.7/numpy/npy_common.h" "$(@D)/python_include/numpy/npy_common.h" && cp "/usr/include/python2.7/numpy/utils.h" "$(@D)/python_include/numpy/utils.h" && cp "/usr/include/python2.7/numpy/npy_interrupt.h" "$(@D)/python_include/numpy/npy_interrupt.h" && cp "/usr/include/python2.7/numpy/npy_endian.h" "$(@D)/python_include/numpy/npy_endian.h" && cp "/usr/include/python2.7/numpy/__ufunc_api.h" "$(@D)/python_include/numpy/__ufunc_api.h" && cp "/usr/include/python2.7/numpy/_neighborhood_iterator_imp.h" "$(@D)/python_include/numpy/_neighborhood_iterator_imp.h" && cp "/usr/include/python2.7/numpy/ufuncobject.h" "$(@D)/python_include/numpy/ufuncobject.h" && cp "/usr/include/python2.7/numpy/ndarraytypes.h" "$(@D)/python_include/numpy/ndarraytypes.h" && cp "/usr/include/python2.7/numpy/npy_math.h" "$(@D)/python_include/numpy/npy_math.h" && cp "/usr/include/python2.7/numpy/noprefix.h" "$(@D)/python_include/numpy/noprefix.h" && cp "/usr/include/python2.7/numpy/npy_3kcompat.h" "$(@D)/python_include/numpy/npy_3kcompat.h" && cp "/usr/include/python2.7/numpy/arrayscalars.h" "$(@D)/python_include/numpy/arrayscalars.h" && cp "/usr/include/python2.7/numpy/npy_os.h" "$(@D)/python_include/numpy/npy_os.h" && cp "/usr/include/python2.7/numpy/ndarrayobject.h" "$(@D)/python_include/numpy/ndarrayobject.h" && cp "/usr/include/python2.7/numpy/npy_no_deprecated_api.h" "$(@D)/python_include/numpy/npy_no_deprecated_api.h" && cp "/usr/include/python2.7/numpy/arrayobject.h" "$(@D)/python_include/numpy/arrayobject.h" && cp "/usr/include/python2.7/numpy/_numpyconfig.h" "$(@D)/python_include/numpy/_numpyconfig.h" && cp "/usr/include/python2.7/numpy/__multiarray_api.h" "$(@D)/python_include/numpy/__multiarray_api.h" && cp "/usr/include/python2.7/numpy/npy_cpu.h" "$(@D)/python_include/numpy/npy_cpu.h" && cp "/usr/include/python2.7/numpy/old_defines.h" "$(@D)/python_include/numpy/old_defines.h" && cp "/usr/include/python2.7/numpy/numpyconfig.h" "$(@D)/python_include/numpy/numpyconfig.h" && cp "/usr/include/python2.7/pycapsule.h" "$(@D)/python_include/pycapsule.h" && cp "/usr/include/python2.7/setobject.h" "$(@D)/python_include/setobject.h" && cp "/usr/include/python2.7/listobject.h" "$(@D)/python_include/listobject.h" && cp "/usr/include/python2.7/bytesobject.h" "$(@D)/python_include/bytesobject.h" && cp "/usr/include/python2.7/pgen.h" "$(@D)/python_include/pgen.h" && cp "/usr/include/python2.7/patchlevel.h" "$(@D)/python_include/patchlevel.h" && cp "/usr/include/python2.7/opcode.h" "$(@D)/python_include/opcode.h" && cp "/usr/include/python2.7/parsetok.h" "$(@D)/python_include/parsetok.h" && cp "/usr/include/python2.7/marshal.h" "$(@D)/python_include/marshal.h" && cp "/usr/include/python2.7/token.h" "$(@D)/python_include/token.h" && cp "/usr/include/python2.7/iterobject.h" "$(@D)/python_include/iterobject.h" && cp "/usr/include/python2.7/abstract.h" "$(@D)/python_include/abstract.h" && cp "/usr/include/python2.7/py_curses.h" "$(@D)/python_include/py_curses.h" && cp "/usr/include/python2.7/metagrammar.h" "$(@D)/python_include/metagrammar.h" && cp "/usr/include/python2.7/bufferobject.h" "$(@D)/python_include/bufferobject.h" && cp "/usr/include/python2.7/Python-ast.h" "$(@D)/python_include/Python-ast.h" +cp "/usr/include/python2.7/Python-ast.h" "$(@D)/python_include/Python-ast.h" && cp "/usr/include/python2.7/Python.h" "$(@D)/python_include/Python.h" && cp "/usr/include/python2.7/abstract.h" "$(@D)/python_include/abstract.h" && cp "/usr/include/python2.7/asdl.h" "$(@D)/python_include/asdl.h" && cp "/usr/include/python2.7/ast.h" "$(@D)/python_include/ast.h" && cp "/usr/include/python2.7/bitset.h" "$(@D)/python_include/bitset.h" && cp "/usr/include/python2.7/boolobject.h" "$(@D)/python_include/boolobject.h" && cp "/usr/include/python2.7/bufferobject.h" "$(@D)/python_include/bufferobject.h" && cp "/usr/include/python2.7/bytearrayobject.h" "$(@D)/python_include/bytearrayobject.h" && cp "/usr/include/python2.7/bytes_methods.h" "$(@D)/python_include/bytes_methods.h" && cp "/usr/include/python2.7/bytesobject.h" "$(@D)/python_include/bytesobject.h" && cp "/usr/include/python2.7/cStringIO.h" "$(@D)/python_include/cStringIO.h" && cp "/usr/include/python2.7/cellobject.h" "$(@D)/python_include/cellobject.h" && cp "/usr/include/python2.7/ceval.h" "$(@D)/python_include/ceval.h" && cp "/usr/include/python2.7/classobject.h" "$(@D)/python_include/classobject.h" && cp "/usr/include/python2.7/cobject.h" "$(@D)/python_include/cobject.h" && cp "/usr/include/python2.7/code.h" "$(@D)/python_include/code.h" && cp "/usr/include/python2.7/codecs.h" "$(@D)/python_include/codecs.h" && cp "/usr/include/python2.7/compile.h" "$(@D)/python_include/compile.h" && cp "/usr/include/python2.7/complexobject.h" "$(@D)/python_include/complexobject.h" && cp "/usr/include/python2.7/datetime.h" "$(@D)/python_include/datetime.h" && cp "/usr/include/python2.7/descrobject.h" "$(@D)/python_include/descrobject.h" && cp "/usr/include/python2.7/dictobject.h" "$(@D)/python_include/dictobject.h" && cp "/usr/include/python2.7/dtoa.h" "$(@D)/python_include/dtoa.h" && cp "/usr/include/python2.7/enumobject.h" "$(@D)/python_include/enumobject.h" && cp "/usr/include/python2.7/errcode.h" "$(@D)/python_include/errcode.h" && cp "/usr/include/python2.7/eval.h" "$(@D)/python_include/eval.h" && cp "/usr/include/python2.7/fileobject.h" "$(@D)/python_include/fileobject.h" && cp "/usr/include/python2.7/floatobject.h" "$(@D)/python_include/floatobject.h" && cp "/usr/include/python2.7/frameobject.h" "$(@D)/python_include/frameobject.h" && cp "/usr/include/python2.7/funcobject.h" "$(@D)/python_include/funcobject.h" && cp "/usr/include/python2.7/genobject.h" "$(@D)/python_include/genobject.h" && cp "/usr/include/python2.7/graminit.h" "$(@D)/python_include/graminit.h" && cp "/usr/include/python2.7/grammar.h" "$(@D)/python_include/grammar.h" && cp "/usr/include/python2.7/import.h" "$(@D)/python_include/import.h" && cp "/usr/include/python2.7/intobject.h" "$(@D)/python_include/intobject.h" && cp "/usr/include/python2.7/intrcheck.h" "$(@D)/python_include/intrcheck.h" && cp "/usr/include/python2.7/iterobject.h" "$(@D)/python_include/iterobject.h" && cp "/usr/include/python2.7/listobject.h" "$(@D)/python_include/listobject.h" && cp "/usr/include/python2.7/longintrepr.h" "$(@D)/python_include/longintrepr.h" && cp "/usr/include/python2.7/longobject.h" "$(@D)/python_include/longobject.h" && cp "/usr/include/python2.7/marshal.h" "$(@D)/python_include/marshal.h" && cp "/usr/include/python2.7/memoryobject.h" "$(@D)/python_include/memoryobject.h" && cp "/usr/include/python2.7/metagrammar.h" "$(@D)/python_include/metagrammar.h" && cp "/usr/include/python2.7/methodobject.h" "$(@D)/python_include/methodobject.h" && cp "/usr/include/python2.7/modsupport.h" "$(@D)/python_include/modsupport.h" && cp "/usr/include/python2.7/moduleobject.h" "$(@D)/python_include/moduleobject.h" && cp "/usr/include/python2.7/node.h" "$(@D)/python_include/node.h" && cp "/usr/include/python2.7/object.h" "$(@D)/python_include/object.h" && cp "/usr/include/python2.7/objimpl.h" "$(@D)/python_include/objimpl.h" && cp "/usr/include/python2.7/opcode.h" "$(@D)/python_include/opcode.h" && cp "/usr/include/python2.7/osdefs.h" "$(@D)/python_include/osdefs.h" && cp "/usr/include/python2.7/parsetok.h" "$(@D)/python_include/parsetok.h" && cp "/usr/include/python2.7/patchlevel.h" "$(@D)/python_include/patchlevel.h" && cp "/usr/include/python2.7/pgen.h" "$(@D)/python_include/pgen.h" && cp "/usr/include/python2.7/pgenheaders.h" "$(@D)/python_include/pgenheaders.h" && cp "/usr/include/python2.7/py_curses.h" "$(@D)/python_include/py_curses.h" && cp "/usr/include/python2.7/pyarena.h" "$(@D)/python_include/pyarena.h" && cp "/usr/include/python2.7/pycapsule.h" "$(@D)/python_include/pycapsule.h" && cp "/usr/include/python2.7/pyconfig.h" "$(@D)/python_include/pyconfig.h" && cp "/usr/include/python2.7/pyctype.h" "$(@D)/python_include/pyctype.h" && cp "/usr/include/python2.7/pydebug.h" "$(@D)/python_include/pydebug.h" && cp "/usr/include/python2.7/pyerrors.h" "$(@D)/python_include/pyerrors.h" && cp "/usr/include/python2.7/pyexpat.h" "$(@D)/python_include/pyexpat.h" && cp "/usr/include/python2.7/pyfpe.h" "$(@D)/python_include/pyfpe.h" && cp "/usr/include/python2.7/pygetopt.h" "$(@D)/python_include/pygetopt.h" && cp "/usr/include/python2.7/pymacconfig.h" "$(@D)/python_include/pymacconfig.h" && cp "/usr/include/python2.7/pymactoolbox.h" "$(@D)/python_include/pymactoolbox.h" && cp "/usr/include/python2.7/pymath.h" "$(@D)/python_include/pymath.h" && cp "/usr/include/python2.7/pymem.h" "$(@D)/python_include/pymem.h" && cp "/usr/include/python2.7/pyport.h" "$(@D)/python_include/pyport.h" && cp "/usr/include/python2.7/pystate.h" "$(@D)/python_include/pystate.h" && cp "/usr/include/python2.7/pystrcmp.h" "$(@D)/python_include/pystrcmp.h" && cp "/usr/include/python2.7/pystrtod.h" "$(@D)/python_include/pystrtod.h" && cp "/usr/include/python2.7/pythonrun.h" "$(@D)/python_include/pythonrun.h" && cp "/usr/include/python2.7/pythread.h" "$(@D)/python_include/pythread.h" && cp "/usr/include/python2.7/rangeobject.h" "$(@D)/python_include/rangeobject.h" && cp "/usr/include/python2.7/setobject.h" "$(@D)/python_include/setobject.h" && cp "/usr/include/python2.7/sliceobject.h" "$(@D)/python_include/sliceobject.h" && cp "/usr/include/python2.7/stringobject.h" "$(@D)/python_include/stringobject.h" && cp "/usr/include/python2.7/structmember.h" "$(@D)/python_include/structmember.h" && cp "/usr/include/python2.7/structseq.h" "$(@D)/python_include/structseq.h" && cp "/usr/include/python2.7/symtable.h" "$(@D)/python_include/symtable.h" && cp "/usr/include/python2.7/sysmodule.h" "$(@D)/python_include/sysmodule.h" && cp "/usr/include/python2.7/timefuncs.h" "$(@D)/python_include/timefuncs.h" && cp "/usr/include/python2.7/token.h" "$(@D)/python_include/token.h" && cp "/usr/include/python2.7/traceback.h" "$(@D)/python_include/traceback.h" && cp "/usr/include/python2.7/tupleobject.h" "$(@D)/python_include/tupleobject.h" && cp "/usr/include/python2.7/ucnhash.h" "$(@D)/python_include/ucnhash.h" && cp "/usr/include/python2.7/unicodeobject.h" "$(@D)/python_include/unicodeobject.h" && cp "/usr/include/python2.7/warnings.h" "$(@D)/python_include/warnings.h" && cp "/usr/include/python2.7/weakrefobject.h" "$(@D)/python_include/weakrefobject.h" """, ) genrule( name = "numpy_include", outs = [ - "numpy_include/numpy/oldnumeric.h", - "numpy_include/numpy/npy_1_7_deprecated_api.h", - "numpy_include/numpy/ufunc_api.txt", - "numpy_include/numpy/multiarray_api.txt", - "numpy_include/numpy/halffloat.h", - "numpy_include/numpy/npy_common.h", - "numpy_include/numpy/utils.h", - "numpy_include/numpy/npy_interrupt.h", - "numpy_include/numpy/npy_endian.h", + "numpy_include/numpy/__multiarray_api.h", "numpy_include/numpy/__ufunc_api.h", "numpy_include/numpy/_neighborhood_iterator_imp.h", - "numpy_include/numpy/ufuncobject.h", + "numpy_include/numpy/_numpyconfig.h", + "numpy_include/numpy/arrayobject.h", + "numpy_include/numpy/arrayscalars.h", + "numpy_include/numpy/halffloat.h", + "numpy_include/numpy/multiarray_api.txt", + "numpy_include/numpy/ndarrayobject.h", "numpy_include/numpy/ndarraytypes.h", - "numpy_include/numpy/npy_math.h", "numpy_include/numpy/noprefix.h", + "numpy_include/numpy/npy_1_7_deprecated_api.h", "numpy_include/numpy/npy_3kcompat.h", - "numpy_include/numpy/arrayscalars.h", - "numpy_include/numpy/npy_os.h", - "numpy_include/numpy/ndarrayobject.h", - "numpy_include/numpy/npy_no_deprecated_api.h", - "numpy_include/numpy/arrayobject.h", - "numpy_include/numpy/_numpyconfig.h", - "numpy_include/numpy/__multiarray_api.h", + "numpy_include/numpy/npy_common.h", "numpy_include/numpy/npy_cpu.h", - "numpy_include/numpy/old_defines.h", + "numpy_include/numpy/npy_endian.h", + "numpy_include/numpy/npy_interrupt.h", + "numpy_include/numpy/npy_math.h", + "numpy_include/numpy/npy_no_deprecated_api.h", + "numpy_include/numpy/npy_os.h", "numpy_include/numpy/numpyconfig.h", + "numpy_include/numpy/old_defines.h", + "numpy_include/numpy/oldnumeric.h", + "numpy_include/numpy/ufunc_api.txt", + "numpy_include/numpy/ufuncobject.h", + "numpy_include/numpy/utils.h", ], cmd = """ -cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/oldnumeric.h" "$(@D)/numpy_include/numpy/oldnumeric.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_1_7_deprecated_api.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/ufunc_api.txt" "$(@D)/numpy_include/numpy/ufunc_api.txt" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/multiarray_api.txt" "$(@D)/numpy_include/numpy/multiarray_api.txt" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/halffloat.h" "$(@D)/numpy_include/numpy/halffloat.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_common.h" "$(@D)/numpy_include/numpy/npy_common.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/utils.h" "$(@D)/numpy_include/numpy/utils.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_interrupt.h" "$(@D)/numpy_include/numpy/npy_interrupt.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_endian.h" "$(@D)/numpy_include/numpy/npy_endian.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/__ufunc_api.h" "$(@D)/numpy_include/numpy/__ufunc_api.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/_neighborhood_iterator_imp.h" "$(@D)/numpy_include/numpy/_neighborhood_iterator_imp.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/ufuncobject.h" "$(@D)/numpy_include/numpy/ufuncobject.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/ndarraytypes.h" "$(@D)/numpy_include/numpy/ndarraytypes.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_math.h" "$(@D)/numpy_include/numpy/npy_math.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/noprefix.h" "$(@D)/numpy_include/numpy/noprefix.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_3kcompat.h" "$(@D)/numpy_include/numpy/npy_3kcompat.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/arrayscalars.h" "$(@D)/numpy_include/numpy/arrayscalars.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_os.h" "$(@D)/numpy_include/numpy/npy_os.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/ndarrayobject.h" "$(@D)/numpy_include/numpy/ndarrayobject.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_no_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_no_deprecated_api.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/arrayobject.h" "$(@D)/numpy_include/numpy/arrayobject.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/_numpyconfig.h" "$(@D)/numpy_include/numpy/_numpyconfig.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/__multiarray_api.h" "$(@D)/numpy_include/numpy/__multiarray_api.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_cpu.h" "$(@D)/numpy_include/numpy/npy_cpu.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/old_defines.h" "$(@D)/numpy_include/numpy/old_defines.h" && cp "/usr/lib/python2.7/dist-packages/numpy/core/include/numpy/numpyconfig.h" "$(@D)/numpy_include/numpy/numpyconfig.h" +cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/__multiarray_api.h" "$(@D)/numpy_include/numpy/__multiarray_api.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/__ufunc_api.h" "$(@D)/numpy_include/numpy/__ufunc_api.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/_neighborhood_iterator_imp.h" "$(@D)/numpy_include/numpy/_neighborhood_iterator_imp.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/_numpyconfig.h" "$(@D)/numpy_include/numpy/_numpyconfig.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/arrayobject.h" "$(@D)/numpy_include/numpy/arrayobject.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/arrayscalars.h" "$(@D)/numpy_include/numpy/arrayscalars.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/halffloat.h" "$(@D)/numpy_include/numpy/halffloat.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/multiarray_api.txt" "$(@D)/numpy_include/numpy/multiarray_api.txt" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/ndarrayobject.h" "$(@D)/numpy_include/numpy/ndarrayobject.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/ndarraytypes.h" "$(@D)/numpy_include/numpy/ndarraytypes.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/noprefix.h" "$(@D)/numpy_include/numpy/noprefix.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_1_7_deprecated_api.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_3kcompat.h" "$(@D)/numpy_include/numpy/npy_3kcompat.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_common.h" "$(@D)/numpy_include/numpy/npy_common.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_cpu.h" "$(@D)/numpy_include/numpy/npy_cpu.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_endian.h" "$(@D)/numpy_include/numpy/npy_endian.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_interrupt.h" "$(@D)/numpy_include/numpy/npy_interrupt.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_math.h" "$(@D)/numpy_include/numpy/npy_math.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_no_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_no_deprecated_api.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/npy_os.h" "$(@D)/numpy_include/numpy/npy_os.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/numpyconfig.h" "$(@D)/numpy_include/numpy/numpyconfig.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/old_defines.h" "$(@D)/numpy_include/numpy/old_defines.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/oldnumeric.h" "$(@D)/numpy_include/numpy/oldnumeric.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/ufunc_api.txt" "$(@D)/numpy_include/numpy/ufunc_api.txt" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/ufuncobject.h" "$(@D)/numpy_include/numpy/ufuncobject.h" && cp "/usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/utils.h" "$(@D)/numpy_include/numpy/utils.h" """, ) diff --git a/third_party/toolchains/cpus/py3/BUILD b/third_party/toolchains/cpus/py3/BUILD index 932a25239f..28712a7cb1 100644 --- a/third_party/toolchains/cpus/py3/BUILD +++ b/third_party/toolchains/cpus/py3/BUILD @@ -6,20 +6,26 @@ licenses(["restricted"]) package(default_visibility = ["//visibility:public"]) +# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib +# See https://docs.python.org/3/extending/windows.html +cc_import( + name = "python_lib", + interface_library = select({ + ":windows": ":python_import_lib", + # A placeholder for Unix platforms which makes --no_build happy. + "//conditions:default": "not-existing.lib", + }), + system_provided = 1, +) + cc_library( name = "python_headers", hdrs = [":python_include"], - data = select({ - ":windows": [":python_import_lib"], + deps = select({ + ":windows": [":python_lib"], "//conditions:default": [], }), includes = ["python_include"], - linkopts = select({ - # TODO(pcloudy): Ideally, this should just go into deps after resolving - # https://github.com/bazelbuild/bazel/issues/3237, - ":windows": ["$(locations :python_import_lib)"], - "//conditions:default": [], - }), ) cc_library( @@ -37,143 +43,143 @@ config_setting( genrule( name = "python_include", outs = [ - "python_include/code.h", - "python_include/dtoa.h", - "python_include/tupleobject.h", - "python_include/object.h", - "python_include/ast.h", - "python_include/pymacconfig.h", - "python_include/errcode.h", - "python_include/frameobject.h", - "python_include/typeslots.h", - "python_include/pgenheaders.h", - "python_include/cellobject.h", - "python_include/pythread.h", - "python_include/boolobject.h", + "python_include/Python-ast.h", + "python_include/Python.h", + "python_include/abstract.h", "python_include/accu.h", - "python_include/modsupport.h", - "python_include/import.h", - "python_include/pymath.h", - "python_include/node.h", - "python_include/funcobject.h", - "python_include/eval.h", - "python_include/pyatomic.h", - "python_include/longintrepr.h", - "python_include/floatobject.h", - "python_include/rangeobject.h", - "python_include/pyfpe.h", - "python_include/pystrcmp.h", - "python_include/fileutils.h", - "python_include/dictobject.h", - "python_include/pyarena.h", - "python_include/osmodule.h", - "python_include/objimpl.h", + "python_include/asdl.h", + "python_include/ast.h", "python_include/bitset.h", - "python_include/memoryobject.h", + "python_include/bltinmodule.h", + "python_include/boolobject.h", "python_include/bytearrayobject.h", - "python_include/pydebug.h", - "python_include/pyerrors.h", - "python_include/weakrefobject.h", - "python_include/grammar.h", - "python_include/symtable.h", - "python_include/longobject.h", - "python_include/structmember.h", - "python_include/enumobject.h", - "python_include/pymacro.h", + "python_include/bytes_methods.h", + "python_include/bytesobject.h", + "python_include/cellobject.h", + "python_include/ceval.h", "python_include/classobject.h", - "python_include/unicodeobject.h", - "python_include/sliceobject.h", - "python_include/pystrtod.h", - "python_include/genobject.h", - "python_include/compile.h", - "python_include/pyexpat.h", - "python_include/asdl.h", + "python_include/code.h", "python_include/codecs.h", + "python_include/compile.h", + "python_include/complexobject.h", + "python_include/datetime.h", + "python_include/descrobject.h", + "python_include/dictobject.h", + "python_include/dtoa.h", "python_include/dynamic_annotations.h", - "python_include/pyctype.h", - "python_include/sysmodule.h", - "python_include/methodobject.h", + "python_include/enumobject.h", + "python_include/errcode.h", + "python_include/eval.h", + "python_include/fileobject.h", + "python_include/fileutils.h", + "python_include/floatobject.h", + "python_include/frameobject.h", + "python_include/funcobject.h", + "python_include/genobject.h", "python_include/graminit.h", - "python_include/bltinmodule.h", + "python_include/grammar.h", + "python_include/import.h", "python_include/intrcheck.h", - "python_include/pyport.h", - "python_include/warnings.h", - "python_include/osdefs.h", - "python_include/pydtrace.h", - "python_include/pylifecycle.h", - "python_include/fileobject.h", - "python_include/pytime.h", - "python_include/traceback.h", - "python_include/ceval.h", - "python_include/bytes_methods.h", - "python_include/namespaceobject.h", - "python_include/pyconfig.h", - "python_include/Python.h", + "python_include/iterobject.h", + "python_include/listobject.h", + "python_include/longintrepr.h", + "python_include/longobject.h", + "python_include/marshal.h", + "python_include/memoryobject.h", + "python_include/metagrammar.h", + "python_include/methodobject.h", + "python_include/modsupport.h", "python_include/moduleobject.h", - "python_include/pystate.h", - "python_include/descrobject.h", + "python_include/namespaceobject.h", + "python_include/node.h", + "python_include/object.h", + "python_include/objimpl.h", "python_include/odictobject.h", - "python_include/ucnhash.h", + "python_include/opcode.h", + "python_include/osdefs.h", + "python_include/osmodule.h", + "python_include/parsetok.h", + "python_include/patchlevel.h", + "python_include/pgen.h", + "python_include/pgenheaders.h", + "python_include/py_curses.h", + "python_include/pyarena.h", + "python_include/pyatomic.h", + "python_include/pycapsule.h", + "python_include/pyconfig.h", + "python_include/pyctype.h", + "python_include/pydebug.h", + "python_include/pydtrace.h", + "python_include/pyerrors.h", + "python_include/pyexpat.h", + "python_include/pyfpe.h", "python_include/pygetopt.h", + "python_include/pyhash.h", + "python_include/pylifecycle.h", + "python_include/pymacconfig.h", + "python_include/pymacro.h", + "python_include/pymath.h", "python_include/pymem.h", - "python_include/complexobject.h", - "python_include/structseq.h", - "python_include/datetime.h", + "python_include/pyport.h", + "python_include/pystate.h", + "python_include/pystrcmp.h", + "python_include/pystrhex.h", + "python_include/pystrtod.h", "python_include/pythonrun.h", - "python_include/pyhash.h", - "python_include/pycapsule.h", + "python_include/pythread.h", + "python_include/pytime.h", + "python_include/rangeobject.h", "python_include/setobject.h", - "python_include/listobject.h", - "python_include/bytesobject.h", - "python_include/pgen.h", - "python_include/patchlevel.h", - "python_include/opcode.h", - "python_include/parsetok.h", - "python_include/pystrhex.h", - "python_include/marshal.h", + "python_include/sliceobject.h", + "python_include/structmember.h", + "python_include/structseq.h", + "python_include/symtable.h", + "python_include/sysmodule.h", "python_include/token.h", - "python_include/iterobject.h", - "python_include/abstract.h", - "python_include/py_curses.h", - "python_include/metagrammar.h", - "python_include/Python-ast.h", + "python_include/traceback.h", + "python_include/tupleobject.h", + "python_include/typeslots.h", + "python_include/ucnhash.h", + "python_include/unicodeobject.h", + "python_include/warnings.h", + "python_include/weakrefobject.h", ], cmd = """ -cp "/opt/python3.6/include/python3.6m/code.h" "$(@D)/python_include/code.h" && cp "/opt/python3.6/include/python3.6m/dtoa.h" "$(@D)/python_include/dtoa.h" && cp "/opt/python3.6/include/python3.6m/tupleobject.h" "$(@D)/python_include/tupleobject.h" && cp "/opt/python3.6/include/python3.6m/object.h" "$(@D)/python_include/object.h" && cp "/opt/python3.6/include/python3.6m/ast.h" "$(@D)/python_include/ast.h" && cp "/opt/python3.6/include/python3.6m/pymacconfig.h" "$(@D)/python_include/pymacconfig.h" && cp "/opt/python3.6/include/python3.6m/errcode.h" "$(@D)/python_include/errcode.h" && cp "/opt/python3.6/include/python3.6m/frameobject.h" "$(@D)/python_include/frameobject.h" && cp "/opt/python3.6/include/python3.6m/typeslots.h" "$(@D)/python_include/typeslots.h" && cp "/opt/python3.6/include/python3.6m/pgenheaders.h" "$(@D)/python_include/pgenheaders.h" && cp "/opt/python3.6/include/python3.6m/cellobject.h" "$(@D)/python_include/cellobject.h" && cp "/opt/python3.6/include/python3.6m/pythread.h" "$(@D)/python_include/pythread.h" && cp "/opt/python3.6/include/python3.6m/boolobject.h" "$(@D)/python_include/boolobject.h" && cp "/opt/python3.6/include/python3.6m/accu.h" "$(@D)/python_include/accu.h" && cp "/opt/python3.6/include/python3.6m/modsupport.h" "$(@D)/python_include/modsupport.h" && cp "/opt/python3.6/include/python3.6m/import.h" "$(@D)/python_include/import.h" && cp "/opt/python3.6/include/python3.6m/pymath.h" "$(@D)/python_include/pymath.h" && cp "/opt/python3.6/include/python3.6m/node.h" "$(@D)/python_include/node.h" && cp "/opt/python3.6/include/python3.6m/funcobject.h" "$(@D)/python_include/funcobject.h" && cp "/opt/python3.6/include/python3.6m/eval.h" "$(@D)/python_include/eval.h" && cp "/opt/python3.6/include/python3.6m/pyatomic.h" "$(@D)/python_include/pyatomic.h" && cp "/opt/python3.6/include/python3.6m/longintrepr.h" "$(@D)/python_include/longintrepr.h" && cp "/opt/python3.6/include/python3.6m/floatobject.h" "$(@D)/python_include/floatobject.h" && cp "/opt/python3.6/include/python3.6m/rangeobject.h" "$(@D)/python_include/rangeobject.h" && cp "/opt/python3.6/include/python3.6m/pyfpe.h" "$(@D)/python_include/pyfpe.h" && cp "/opt/python3.6/include/python3.6m/pystrcmp.h" "$(@D)/python_include/pystrcmp.h" && cp "/opt/python3.6/include/python3.6m/fileutils.h" "$(@D)/python_include/fileutils.h" && cp "/opt/python3.6/include/python3.6m/dictobject.h" "$(@D)/python_include/dictobject.h" && cp "/opt/python3.6/include/python3.6m/pyarena.h" "$(@D)/python_include/pyarena.h" && cp "/opt/python3.6/include/python3.6m/osmodule.h" "$(@D)/python_include/osmodule.h" && cp "/opt/python3.6/include/python3.6m/objimpl.h" "$(@D)/python_include/objimpl.h" && cp "/opt/python3.6/include/python3.6m/bitset.h" "$(@D)/python_include/bitset.h" && cp "/opt/python3.6/include/python3.6m/memoryobject.h" "$(@D)/python_include/memoryobject.h" && cp "/opt/python3.6/include/python3.6m/bytearrayobject.h" "$(@D)/python_include/bytearrayobject.h" && cp "/opt/python3.6/include/python3.6m/pydebug.h" "$(@D)/python_include/pydebug.h" && cp "/opt/python3.6/include/python3.6m/pyerrors.h" "$(@D)/python_include/pyerrors.h" && cp "/opt/python3.6/include/python3.6m/weakrefobject.h" "$(@D)/python_include/weakrefobject.h" && cp "/opt/python3.6/include/python3.6m/grammar.h" "$(@D)/python_include/grammar.h" && cp "/opt/python3.6/include/python3.6m/symtable.h" "$(@D)/python_include/symtable.h" && cp "/opt/python3.6/include/python3.6m/longobject.h" "$(@D)/python_include/longobject.h" && cp "/opt/python3.6/include/python3.6m/structmember.h" "$(@D)/python_include/structmember.h" && cp "/opt/python3.6/include/python3.6m/enumobject.h" "$(@D)/python_include/enumobject.h" && cp "/opt/python3.6/include/python3.6m/pymacro.h" "$(@D)/python_include/pymacro.h" && cp "/opt/python3.6/include/python3.6m/classobject.h" "$(@D)/python_include/classobject.h" && cp "/opt/python3.6/include/python3.6m/unicodeobject.h" "$(@D)/python_include/unicodeobject.h" && cp "/opt/python3.6/include/python3.6m/sliceobject.h" "$(@D)/python_include/sliceobject.h" && cp "/opt/python3.6/include/python3.6m/pystrtod.h" "$(@D)/python_include/pystrtod.h" && cp "/opt/python3.6/include/python3.6m/genobject.h" "$(@D)/python_include/genobject.h" && cp "/opt/python3.6/include/python3.6m/compile.h" "$(@D)/python_include/compile.h" && cp "/opt/python3.6/include/python3.6m/pyexpat.h" "$(@D)/python_include/pyexpat.h" && cp "/opt/python3.6/include/python3.6m/asdl.h" "$(@D)/python_include/asdl.h" && cp "/opt/python3.6/include/python3.6m/codecs.h" "$(@D)/python_include/codecs.h" && cp "/opt/python3.6/include/python3.6m/dynamic_annotations.h" "$(@D)/python_include/dynamic_annotations.h" && cp "/opt/python3.6/include/python3.6m/pyctype.h" "$(@D)/python_include/pyctype.h" && cp "/opt/python3.6/include/python3.6m/sysmodule.h" "$(@D)/python_include/sysmodule.h" && cp "/opt/python3.6/include/python3.6m/methodobject.h" "$(@D)/python_include/methodobject.h" && cp "/opt/python3.6/include/python3.6m/graminit.h" "$(@D)/python_include/graminit.h" && cp "/opt/python3.6/include/python3.6m/bltinmodule.h" "$(@D)/python_include/bltinmodule.h" && cp "/opt/python3.6/include/python3.6m/intrcheck.h" "$(@D)/python_include/intrcheck.h" && cp "/opt/python3.6/include/python3.6m/pyport.h" "$(@D)/python_include/pyport.h" && cp "/opt/python3.6/include/python3.6m/warnings.h" "$(@D)/python_include/warnings.h" && cp "/opt/python3.6/include/python3.6m/osdefs.h" "$(@D)/python_include/osdefs.h" && cp "/opt/python3.6/include/python3.6m/pydtrace.h" "$(@D)/python_include/pydtrace.h" && cp "/opt/python3.6/include/python3.6m/pylifecycle.h" "$(@D)/python_include/pylifecycle.h" && cp "/opt/python3.6/include/python3.6m/fileobject.h" "$(@D)/python_include/fileobject.h" && cp "/opt/python3.6/include/python3.6m/pytime.h" "$(@D)/python_include/pytime.h" && cp "/opt/python3.6/include/python3.6m/traceback.h" "$(@D)/python_include/traceback.h" && cp "/opt/python3.6/include/python3.6m/ceval.h" "$(@D)/python_include/ceval.h" && cp "/opt/python3.6/include/python3.6m/bytes_methods.h" "$(@D)/python_include/bytes_methods.h" && cp "/opt/python3.6/include/python3.6m/namespaceobject.h" "$(@D)/python_include/namespaceobject.h" && cp "/opt/python3.6/include/python3.6m/pyconfig.h" "$(@D)/python_include/pyconfig.h" && cp "/opt/python3.6/include/python3.6m/Python.h" "$(@D)/python_include/Python.h" && cp "/opt/python3.6/include/python3.6m/moduleobject.h" "$(@D)/python_include/moduleobject.h" && cp "/opt/python3.6/include/python3.6m/pystate.h" "$(@D)/python_include/pystate.h" && cp "/opt/python3.6/include/python3.6m/descrobject.h" "$(@D)/python_include/descrobject.h" && cp "/opt/python3.6/include/python3.6m/odictobject.h" "$(@D)/python_include/odictobject.h" && cp "/opt/python3.6/include/python3.6m/ucnhash.h" "$(@D)/python_include/ucnhash.h" && cp "/opt/python3.6/include/python3.6m/pygetopt.h" "$(@D)/python_include/pygetopt.h" && cp "/opt/python3.6/include/python3.6m/pymem.h" "$(@D)/python_include/pymem.h" && cp "/opt/python3.6/include/python3.6m/complexobject.h" "$(@D)/python_include/complexobject.h" && cp "/opt/python3.6/include/python3.6m/structseq.h" "$(@D)/python_include/structseq.h" && cp "/opt/python3.6/include/python3.6m/datetime.h" "$(@D)/python_include/datetime.h" && cp "/opt/python3.6/include/python3.6m/pythonrun.h" "$(@D)/python_include/pythonrun.h" && cp "/opt/python3.6/include/python3.6m/pyhash.h" "$(@D)/python_include/pyhash.h" && cp "/opt/python3.6/include/python3.6m/pycapsule.h" "$(@D)/python_include/pycapsule.h" && cp "/opt/python3.6/include/python3.6m/setobject.h" "$(@D)/python_include/setobject.h" && cp "/opt/python3.6/include/python3.6m/listobject.h" "$(@D)/python_include/listobject.h" && cp "/opt/python3.6/include/python3.6m/bytesobject.h" "$(@D)/python_include/bytesobject.h" && cp "/opt/python3.6/include/python3.6m/pgen.h" "$(@D)/python_include/pgen.h" && cp "/opt/python3.6/include/python3.6m/patchlevel.h" "$(@D)/python_include/patchlevel.h" && cp "/opt/python3.6/include/python3.6m/opcode.h" "$(@D)/python_include/opcode.h" && cp "/opt/python3.6/include/python3.6m/parsetok.h" "$(@D)/python_include/parsetok.h" && cp "/opt/python3.6/include/python3.6m/pystrhex.h" "$(@D)/python_include/pystrhex.h" && cp "/opt/python3.6/include/python3.6m/marshal.h" "$(@D)/python_include/marshal.h" && cp "/opt/python3.6/include/python3.6m/token.h" "$(@D)/python_include/token.h" && cp "/opt/python3.6/include/python3.6m/iterobject.h" "$(@D)/python_include/iterobject.h" && cp "/opt/python3.6/include/python3.6m/abstract.h" "$(@D)/python_include/abstract.h" && cp "/opt/python3.6/include/python3.6m/py_curses.h" "$(@D)/python_include/py_curses.h" && cp "/opt/python3.6/include/python3.6m/metagrammar.h" "$(@D)/python_include/metagrammar.h" && cp "/opt/python3.6/include/python3.6m/Python-ast.h" "$(@D)/python_include/Python-ast.h" +cp "/opt/python3.6/include/python3.6m/Python-ast.h" "$(@D)/python_include/Python-ast.h" && cp "/opt/python3.6/include/python3.6m/Python.h" "$(@D)/python_include/Python.h" && cp "/opt/python3.6/include/python3.6m/abstract.h" "$(@D)/python_include/abstract.h" && cp "/opt/python3.6/include/python3.6m/accu.h" "$(@D)/python_include/accu.h" && cp "/opt/python3.6/include/python3.6m/asdl.h" "$(@D)/python_include/asdl.h" && cp "/opt/python3.6/include/python3.6m/ast.h" "$(@D)/python_include/ast.h" && cp "/opt/python3.6/include/python3.6m/bitset.h" "$(@D)/python_include/bitset.h" && cp "/opt/python3.6/include/python3.6m/bltinmodule.h" "$(@D)/python_include/bltinmodule.h" && cp "/opt/python3.6/include/python3.6m/boolobject.h" "$(@D)/python_include/boolobject.h" && cp "/opt/python3.6/include/python3.6m/bytearrayobject.h" "$(@D)/python_include/bytearrayobject.h" && cp "/opt/python3.6/include/python3.6m/bytes_methods.h" "$(@D)/python_include/bytes_methods.h" && cp "/opt/python3.6/include/python3.6m/bytesobject.h" "$(@D)/python_include/bytesobject.h" && cp "/opt/python3.6/include/python3.6m/cellobject.h" "$(@D)/python_include/cellobject.h" && cp "/opt/python3.6/include/python3.6m/ceval.h" "$(@D)/python_include/ceval.h" && cp "/opt/python3.6/include/python3.6m/classobject.h" "$(@D)/python_include/classobject.h" && cp "/opt/python3.6/include/python3.6m/code.h" "$(@D)/python_include/code.h" && cp "/opt/python3.6/include/python3.6m/codecs.h" "$(@D)/python_include/codecs.h" && cp "/opt/python3.6/include/python3.6m/compile.h" "$(@D)/python_include/compile.h" && cp "/opt/python3.6/include/python3.6m/complexobject.h" "$(@D)/python_include/complexobject.h" && cp "/opt/python3.6/include/python3.6m/datetime.h" "$(@D)/python_include/datetime.h" && cp "/opt/python3.6/include/python3.6m/descrobject.h" "$(@D)/python_include/descrobject.h" && cp "/opt/python3.6/include/python3.6m/dictobject.h" "$(@D)/python_include/dictobject.h" && cp "/opt/python3.6/include/python3.6m/dtoa.h" "$(@D)/python_include/dtoa.h" && cp "/opt/python3.6/include/python3.6m/dynamic_annotations.h" "$(@D)/python_include/dynamic_annotations.h" && cp "/opt/python3.6/include/python3.6m/enumobject.h" "$(@D)/python_include/enumobject.h" && cp "/opt/python3.6/include/python3.6m/errcode.h" "$(@D)/python_include/errcode.h" && cp "/opt/python3.6/include/python3.6m/eval.h" "$(@D)/python_include/eval.h" && cp "/opt/python3.6/include/python3.6m/fileobject.h" "$(@D)/python_include/fileobject.h" && cp "/opt/python3.6/include/python3.6m/fileutils.h" "$(@D)/python_include/fileutils.h" && cp "/opt/python3.6/include/python3.6m/floatobject.h" "$(@D)/python_include/floatobject.h" && cp "/opt/python3.6/include/python3.6m/frameobject.h" "$(@D)/python_include/frameobject.h" && cp "/opt/python3.6/include/python3.6m/funcobject.h" "$(@D)/python_include/funcobject.h" && cp "/opt/python3.6/include/python3.6m/genobject.h" "$(@D)/python_include/genobject.h" && cp "/opt/python3.6/include/python3.6m/graminit.h" "$(@D)/python_include/graminit.h" && cp "/opt/python3.6/include/python3.6m/grammar.h" "$(@D)/python_include/grammar.h" && cp "/opt/python3.6/include/python3.6m/import.h" "$(@D)/python_include/import.h" && cp "/opt/python3.6/include/python3.6m/intrcheck.h" "$(@D)/python_include/intrcheck.h" && cp "/opt/python3.6/include/python3.6m/iterobject.h" "$(@D)/python_include/iterobject.h" && cp "/opt/python3.6/include/python3.6m/listobject.h" "$(@D)/python_include/listobject.h" && cp "/opt/python3.6/include/python3.6m/longintrepr.h" "$(@D)/python_include/longintrepr.h" && cp "/opt/python3.6/include/python3.6m/longobject.h" "$(@D)/python_include/longobject.h" && cp "/opt/python3.6/include/python3.6m/marshal.h" "$(@D)/python_include/marshal.h" && cp "/opt/python3.6/include/python3.6m/memoryobject.h" "$(@D)/python_include/memoryobject.h" && cp "/opt/python3.6/include/python3.6m/metagrammar.h" "$(@D)/python_include/metagrammar.h" && cp "/opt/python3.6/include/python3.6m/methodobject.h" "$(@D)/python_include/methodobject.h" && cp "/opt/python3.6/include/python3.6m/modsupport.h" "$(@D)/python_include/modsupport.h" && cp "/opt/python3.6/include/python3.6m/moduleobject.h" "$(@D)/python_include/moduleobject.h" && cp "/opt/python3.6/include/python3.6m/namespaceobject.h" "$(@D)/python_include/namespaceobject.h" && cp "/opt/python3.6/include/python3.6m/node.h" "$(@D)/python_include/node.h" && cp "/opt/python3.6/include/python3.6m/object.h" "$(@D)/python_include/object.h" && cp "/opt/python3.6/include/python3.6m/objimpl.h" "$(@D)/python_include/objimpl.h" && cp "/opt/python3.6/include/python3.6m/odictobject.h" "$(@D)/python_include/odictobject.h" && cp "/opt/python3.6/include/python3.6m/opcode.h" "$(@D)/python_include/opcode.h" && cp "/opt/python3.6/include/python3.6m/osdefs.h" "$(@D)/python_include/osdefs.h" && cp "/opt/python3.6/include/python3.6m/osmodule.h" "$(@D)/python_include/osmodule.h" && cp "/opt/python3.6/include/python3.6m/parsetok.h" "$(@D)/python_include/parsetok.h" && cp "/opt/python3.6/include/python3.6m/patchlevel.h" "$(@D)/python_include/patchlevel.h" && cp "/opt/python3.6/include/python3.6m/pgen.h" "$(@D)/python_include/pgen.h" && cp "/opt/python3.6/include/python3.6m/pgenheaders.h" "$(@D)/python_include/pgenheaders.h" && cp "/opt/python3.6/include/python3.6m/py_curses.h" "$(@D)/python_include/py_curses.h" && cp "/opt/python3.6/include/python3.6m/pyarena.h" "$(@D)/python_include/pyarena.h" && cp "/opt/python3.6/include/python3.6m/pyatomic.h" "$(@D)/python_include/pyatomic.h" && cp "/opt/python3.6/include/python3.6m/pycapsule.h" "$(@D)/python_include/pycapsule.h" && cp "/opt/python3.6/include/python3.6m/pyconfig.h" "$(@D)/python_include/pyconfig.h" && cp "/opt/python3.6/include/python3.6m/pyctype.h" "$(@D)/python_include/pyctype.h" && cp "/opt/python3.6/include/python3.6m/pydebug.h" "$(@D)/python_include/pydebug.h" && cp "/opt/python3.6/include/python3.6m/pydtrace.h" "$(@D)/python_include/pydtrace.h" && cp "/opt/python3.6/include/python3.6m/pyerrors.h" "$(@D)/python_include/pyerrors.h" && cp "/opt/python3.6/include/python3.6m/pyexpat.h" "$(@D)/python_include/pyexpat.h" && cp "/opt/python3.6/include/python3.6m/pyfpe.h" "$(@D)/python_include/pyfpe.h" && cp "/opt/python3.6/include/python3.6m/pygetopt.h" "$(@D)/python_include/pygetopt.h" && cp "/opt/python3.6/include/python3.6m/pyhash.h" "$(@D)/python_include/pyhash.h" && cp "/opt/python3.6/include/python3.6m/pylifecycle.h" "$(@D)/python_include/pylifecycle.h" && cp "/opt/python3.6/include/python3.6m/pymacconfig.h" "$(@D)/python_include/pymacconfig.h" && cp "/opt/python3.6/include/python3.6m/pymacro.h" "$(@D)/python_include/pymacro.h" && cp "/opt/python3.6/include/python3.6m/pymath.h" "$(@D)/python_include/pymath.h" && cp "/opt/python3.6/include/python3.6m/pymem.h" "$(@D)/python_include/pymem.h" && cp "/opt/python3.6/include/python3.6m/pyport.h" "$(@D)/python_include/pyport.h" && cp "/opt/python3.6/include/python3.6m/pystate.h" "$(@D)/python_include/pystate.h" && cp "/opt/python3.6/include/python3.6m/pystrcmp.h" "$(@D)/python_include/pystrcmp.h" && cp "/opt/python3.6/include/python3.6m/pystrhex.h" "$(@D)/python_include/pystrhex.h" && cp "/opt/python3.6/include/python3.6m/pystrtod.h" "$(@D)/python_include/pystrtod.h" && cp "/opt/python3.6/include/python3.6m/pythonrun.h" "$(@D)/python_include/pythonrun.h" && cp "/opt/python3.6/include/python3.6m/pythread.h" "$(@D)/python_include/pythread.h" && cp "/opt/python3.6/include/python3.6m/pytime.h" "$(@D)/python_include/pytime.h" && cp "/opt/python3.6/include/python3.6m/rangeobject.h" "$(@D)/python_include/rangeobject.h" && cp "/opt/python3.6/include/python3.6m/setobject.h" "$(@D)/python_include/setobject.h" && cp "/opt/python3.6/include/python3.6m/sliceobject.h" "$(@D)/python_include/sliceobject.h" && cp "/opt/python3.6/include/python3.6m/structmember.h" "$(@D)/python_include/structmember.h" && cp "/opt/python3.6/include/python3.6m/structseq.h" "$(@D)/python_include/structseq.h" && cp "/opt/python3.6/include/python3.6m/symtable.h" "$(@D)/python_include/symtable.h" && cp "/opt/python3.6/include/python3.6m/sysmodule.h" "$(@D)/python_include/sysmodule.h" && cp "/opt/python3.6/include/python3.6m/token.h" "$(@D)/python_include/token.h" && cp "/opt/python3.6/include/python3.6m/traceback.h" "$(@D)/python_include/traceback.h" && cp "/opt/python3.6/include/python3.6m/tupleobject.h" "$(@D)/python_include/tupleobject.h" && cp "/opt/python3.6/include/python3.6m/typeslots.h" "$(@D)/python_include/typeslots.h" && cp "/opt/python3.6/include/python3.6m/ucnhash.h" "$(@D)/python_include/ucnhash.h" && cp "/opt/python3.6/include/python3.6m/unicodeobject.h" "$(@D)/python_include/unicodeobject.h" && cp "/opt/python3.6/include/python3.6m/warnings.h" "$(@D)/python_include/warnings.h" && cp "/opt/python3.6/include/python3.6m/weakrefobject.h" "$(@D)/python_include/weakrefobject.h" """, ) genrule( name = "numpy_include", outs = [ - "numpy_include/numpy/oldnumeric.h", - "numpy_include/numpy/npy_1_7_deprecated_api.h", - "numpy_include/numpy/ufunc_api.txt", - "numpy_include/numpy/multiarray_api.txt", - "numpy_include/numpy/halffloat.h", - "numpy_include/numpy/npy_common.h", - "numpy_include/numpy/utils.h", - "numpy_include/numpy/npy_interrupt.h", - "numpy_include/numpy/npy_endian.h", + "numpy_include/numpy/__multiarray_api.h", "numpy_include/numpy/__ufunc_api.h", "numpy_include/numpy/_neighborhood_iterator_imp.h", - "numpy_include/numpy/ufuncobject.h", + "numpy_include/numpy/_numpyconfig.h", + "numpy_include/numpy/arrayobject.h", + "numpy_include/numpy/arrayscalars.h", + "numpy_include/numpy/halffloat.h", + "numpy_include/numpy/multiarray_api.txt", + "numpy_include/numpy/ndarrayobject.h", "numpy_include/numpy/ndarraytypes.h", - "numpy_include/numpy/npy_math.h", "numpy_include/numpy/noprefix.h", + "numpy_include/numpy/npy_1_7_deprecated_api.h", "numpy_include/numpy/npy_3kcompat.h", - "numpy_include/numpy/arrayscalars.h", - "numpy_include/numpy/npy_os.h", - "numpy_include/numpy/ndarrayobject.h", - "numpy_include/numpy/npy_no_deprecated_api.h", - "numpy_include/numpy/arrayobject.h", - "numpy_include/numpy/_numpyconfig.h", - "numpy_include/numpy/__multiarray_api.h", + "numpy_include/numpy/npy_common.h", "numpy_include/numpy/npy_cpu.h", - "numpy_include/numpy/old_defines.h", + "numpy_include/numpy/npy_endian.h", + "numpy_include/numpy/npy_interrupt.h", + "numpy_include/numpy/npy_math.h", + "numpy_include/numpy/npy_no_deprecated_api.h", + "numpy_include/numpy/npy_os.h", "numpy_include/numpy/numpyconfig.h", + "numpy_include/numpy/old_defines.h", + "numpy_include/numpy/oldnumeric.h", + "numpy_include/numpy/ufunc_api.txt", + "numpy_include/numpy/ufuncobject.h", + "numpy_include/numpy/utils.h", ], cmd = """ -cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/oldnumeric.h" "$(@D)/numpy_include/numpy/oldnumeric.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_1_7_deprecated_api.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/ufunc_api.txt" "$(@D)/numpy_include/numpy/ufunc_api.txt" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/multiarray_api.txt" "$(@D)/numpy_include/numpy/multiarray_api.txt" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/halffloat.h" "$(@D)/numpy_include/numpy/halffloat.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_common.h" "$(@D)/numpy_include/numpy/npy_common.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/utils.h" "$(@D)/numpy_include/numpy/utils.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_interrupt.h" "$(@D)/numpy_include/numpy/npy_interrupt.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_endian.h" "$(@D)/numpy_include/numpy/npy_endian.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/__ufunc_api.h" "$(@D)/numpy_include/numpy/__ufunc_api.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/_neighborhood_iterator_imp.h" "$(@D)/numpy_include/numpy/_neighborhood_iterator_imp.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/ufuncobject.h" "$(@D)/numpy_include/numpy/ufuncobject.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/ndarraytypes.h" "$(@D)/numpy_include/numpy/ndarraytypes.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_math.h" "$(@D)/numpy_include/numpy/npy_math.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/noprefix.h" "$(@D)/numpy_include/numpy/noprefix.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_3kcompat.h" "$(@D)/numpy_include/numpy/npy_3kcompat.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/arrayscalars.h" "$(@D)/numpy_include/numpy/arrayscalars.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_os.h" "$(@D)/numpy_include/numpy/npy_os.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/ndarrayobject.h" "$(@D)/numpy_include/numpy/ndarrayobject.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_no_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_no_deprecated_api.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/arrayobject.h" "$(@D)/numpy_include/numpy/arrayobject.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/_numpyconfig.h" "$(@D)/numpy_include/numpy/_numpyconfig.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/__multiarray_api.h" "$(@D)/numpy_include/numpy/__multiarray_api.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_cpu.h" "$(@D)/numpy_include/numpy/npy_cpu.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/old_defines.h" "$(@D)/numpy_include/numpy/old_defines.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/numpyconfig.h" "$(@D)/numpy_include/numpy/numpyconfig.h" +cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/__multiarray_api.h" "$(@D)/numpy_include/numpy/__multiarray_api.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/__ufunc_api.h" "$(@D)/numpy_include/numpy/__ufunc_api.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/_neighborhood_iterator_imp.h" "$(@D)/numpy_include/numpy/_neighborhood_iterator_imp.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/_numpyconfig.h" "$(@D)/numpy_include/numpy/_numpyconfig.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/arrayobject.h" "$(@D)/numpy_include/numpy/arrayobject.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/arrayscalars.h" "$(@D)/numpy_include/numpy/arrayscalars.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/halffloat.h" "$(@D)/numpy_include/numpy/halffloat.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/multiarray_api.txt" "$(@D)/numpy_include/numpy/multiarray_api.txt" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/ndarrayobject.h" "$(@D)/numpy_include/numpy/ndarrayobject.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/ndarraytypes.h" "$(@D)/numpy_include/numpy/ndarraytypes.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/noprefix.h" "$(@D)/numpy_include/numpy/noprefix.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_1_7_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_1_7_deprecated_api.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_3kcompat.h" "$(@D)/numpy_include/numpy/npy_3kcompat.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_common.h" "$(@D)/numpy_include/numpy/npy_common.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_cpu.h" "$(@D)/numpy_include/numpy/npy_cpu.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_endian.h" "$(@D)/numpy_include/numpy/npy_endian.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_interrupt.h" "$(@D)/numpy_include/numpy/npy_interrupt.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_math.h" "$(@D)/numpy_include/numpy/npy_math.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_no_deprecated_api.h" "$(@D)/numpy_include/numpy/npy_no_deprecated_api.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/npy_os.h" "$(@D)/numpy_include/numpy/npy_os.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/numpyconfig.h" "$(@D)/numpy_include/numpy/numpyconfig.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/old_defines.h" "$(@D)/numpy_include/numpy/old_defines.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/oldnumeric.h" "$(@D)/numpy_include/numpy/oldnumeric.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/ufunc_api.txt" "$(@D)/numpy_include/numpy/ufunc_api.txt" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/ufuncobject.h" "$(@D)/numpy_include/numpy/ufuncobject.h" && cp "/opt/python3.6/lib/python3.6/site-packages/numpy/core/include/numpy/utils.h" "$(@D)/numpy_include/numpy/utils.h" """, ) -- cgit v1.2.3 From f283e65a1bdb797070be9b84a69ef323268f7c3c Mon Sep 17 00:00:00 2001 From: Tom Hennigan Date: Tue, 5 Jun 2018 03:56:47 -0700 Subject: Handle scalar input to assert_equal in eager. PiperOrigin-RevId: 199274329 --- tensorflow/python/kernel_tests/check_ops_test.py | 7 +++++++ tensorflow/python/ops/check_ops.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/kernel_tests/check_ops_test.py b/tensorflow/python/kernel_tests/check_ops_test.py index 5a83ec8d30..7ef841c96b 100644 --- a/tensorflow/python/kernel_tests/check_ops_test.py +++ b/tensorflow/python/kernel_tests/check_ops_test.py @@ -88,6 +88,13 @@ class AssertEqualTest(test.TestCase): out = array_ops.identity(small) self.evaluate(out) + @test_util.run_in_graph_and_eager_modes() + def test_scalar_comparison(self): + const_true = constant_op.constant(True, name="true") + const_false = constant_op.constant(False, name="false") + with self.assertRaisesRegexp(errors.InvalidArgumentError, "fail"): + check_ops.assert_equal(const_true, const_false, message="fail") + def test_returns_none_with_eager(self): with context.eager_mode(): small = constant_op.constant([1, 2], name="small") diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py index cabc1e724c..375a5ec2c3 100644 --- a/tensorflow/python/ops/check_ops.py +++ b/tensorflow/python/ops/check_ops.py @@ -341,8 +341,8 @@ def assert_equal(x, y, data=None, summarize=None, message=None, name=None): y_sum, y_np[:y_sum])) index_and_values_str = '' - if x.shape == y.shape: - # If the shapes of x and y are the same, + if x.shape == y.shape and x.shape.as_list(): + # If the shapes of x and y are the same (and not scalars), # Get the values that actually differed and their indices. # If shapes are different this information is more confusing # than useful. -- cgit v1.2.3 From e71f9b863097086c91b2a3f5aea1e081f275ceca Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 21 Jun 2018 18:53:05 -0700 Subject: Update Eigen version to commit e5e305a158a029f5b5f837bf821411a51439a970. PiperOrigin-RevId: 201624024 --- .../kernel_tests/distributions/dirichlet_multinomial_test.py | 6 +++--- tensorflow/workspace.bzl | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py b/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py index 7922fb0606..daea699514 100644 --- a/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py +++ b/tensorflow/python/kernel_tests/distributions/dirichlet_multinomial_test.py @@ -250,9 +250,9 @@ class DirichletMultinomialTest(test.TestCase): dist.variance(), dist.stddev(), ]) - self.assertAllClose(sample_mean_, analytic_mean, atol=0., rtol=0.04) - self.assertAllClose(sample_cov_, analytic_cov, atol=0., rtol=0.05) - self.assertAllClose(sample_var_, analytic_var, atol=0., rtol=0.05) + self.assertAllClose(sample_mean_, analytic_mean, atol=0., rtol=0.06) + self.assertAllClose(sample_cov_, analytic_cov, atol=0., rtol=0.07) + self.assertAllClose(sample_var_, analytic_var, atol=0., rtol=0.07) self.assertAllClose(sample_stddev_, analytic_stddev, atol=0., rtol=0.02) def testCovariance(self): diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 50a69598a1..43152c88cf 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -107,11 +107,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): tf_http_archive( name = "eigen_archive", urls = [ - "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/6913f0cf7d06.tar.gz", - "https://bitbucket.org/eigen/eigen/get/6913f0cf7d06.tar.gz", + "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/e5e305a158a0.tar.gz", + "https://bitbucket.org/eigen/eigen/get/e5e305a158a0.tar.gz", ], - sha256 = "791b836cacd03e20bae5bdd25f1c4a5505a0a9975ba94a61eb4e2631fbd1d53a", - strip_prefix = "eigen-eigen-6913f0cf7d06", + sha256 = "8bbe676d69e7f59070c83a949454b8b6344034e0ebbf686b337528e5dc04c7de", + strip_prefix = "eigen-eigen-e5e305a158a0", build_file = clean_dep("//third_party:eigen.BUILD"), patch_file = clean_dep("//third_party:eigen_fix_cuda_compilation.patch") ) -- cgit v1.2.3 From 5c450d2e1d0d3a1abae4997df0da1b8d73684e01 Mon Sep 17 00:00:00 2001 From: Rasmus Munk Larsen Date: Fri, 22 Jun 2018 13:36:57 -0700 Subject: Update workspace.bzl --- tensorflow/workspace.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 43152c88cf..3c657c4a5b 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -113,7 +113,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "8bbe676d69e7f59070c83a949454b8b6344034e0ebbf686b337528e5dc04c7de", strip_prefix = "eigen-eigen-e5e305a158a0", build_file = clean_dep("//third_party:eigen.BUILD"), - patch_file = clean_dep("//third_party:eigen_fix_cuda_compilation.patch") ) tf_http_archive( -- cgit v1.2.3 From df2c8315211895afab0d7ba1ff64e831d9d3ce3b Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Tue, 19 Jun 2018 23:11:00 -0700 Subject: Get started landing page. Move "Datasets Quickstart" to "Datasets for Estimators" under guide. PiperOrigin-RevId: 201301717 --- tensorflow/docs_src/get_started/_index.yaml | 255 ++++++++++++++ .../docs_src/get_started/basic_classification.md | 3 + .../docs_src/get_started/basic_regression.md | 3 + .../get_started/basic_text_classification.md | 3 + .../docs_src/get_started/datasets_quickstart.md | 387 --------------------- tensorflow/docs_src/get_started/eager.md | 2 +- tensorflow/docs_src/get_started/index.md | 29 -- tensorflow/docs_src/get_started/leftnav_files | 12 +- tensorflow/docs_src/get_started/next_steps.md | 36 ++ .../docs_src/get_started/overfit_and_underfit.md | 3 + .../get_started/save_and_restore_models.md | 3 + tensorflow/docs_src/install/install_linux.md | 8 +- tensorflow/docs_src/install/install_mac.md | 6 +- tensorflow/docs_src/install/install_raspbian.md | 6 +- tensorflow/docs_src/install/install_sources.md | 2 +- tensorflow/docs_src/install/install_windows.md | 7 +- .../programmers_guide/datasets_for_estimators.md | 387 +++++++++++++++++++++ tensorflow/docs_src/programmers_guide/index.md | 1 + .../docs_src/programmers_guide/leftnav_files | 1 + .../programmers_guide/premade_estimators.md | 8 +- tensorflow/docs_src/tutorials/index.md | 5 +- 21 files changed, 715 insertions(+), 452 deletions(-) create mode 100644 tensorflow/docs_src/get_started/_index.yaml create mode 100644 tensorflow/docs_src/get_started/basic_classification.md create mode 100644 tensorflow/docs_src/get_started/basic_regression.md create mode 100644 tensorflow/docs_src/get_started/basic_text_classification.md delete mode 100644 tensorflow/docs_src/get_started/datasets_quickstart.md delete mode 100644 tensorflow/docs_src/get_started/index.md create mode 100644 tensorflow/docs_src/get_started/next_steps.md create mode 100644 tensorflow/docs_src/get_started/overfit_and_underfit.md create mode 100644 tensorflow/docs_src/get_started/save_and_restore_models.md create mode 100644 tensorflow/docs_src/programmers_guide/datasets_for_estimators.md diff --git a/tensorflow/docs_src/get_started/_index.yaml b/tensorflow/docs_src/get_started/_index.yaml new file mode 100644 index 0000000000..af255a482d --- /dev/null +++ b/tensorflow/docs_src/get_started/_index.yaml @@ -0,0 +1,255 @@ +project_path: /_project.yaml +book_path: /_book.yaml +description: +landing_page: + show_side_navs: True + rows: + - description: > +

Get Started with TensorFlow

+

+ TensorFlow is an open-source machine learning library for research and + production. TensorFlow offers APIs for beginners and experts to develop + for desktop, mobile, web, and cloud. See the sections below to get + started. +

+ items: + - custom_html: > + +
+ +

Learn and use ML

+ +
+

+ The high-level Keras API provides building blocks to create and + train deep learning models. Start with these beginner-friendly + notebook examples, then read the + TensorFlow Keras guide. +

+
    +
  1. Basic classification
    2. +
  2. Text classification
    3. +
  3. Regression
    4. +
  4. Overfitting and underfitting
    5. +
  5. Save and load
    6. +
+
+
+ Read the Keras guide +
+
+ - classname: tfo-landing-row-item-code-block + code_block: | + 
+        import tensorflow as tf
+        mnist = tf.keras.datasets.mnist
+
+        (x_train, y_train),(x_test, y_test) = mnist.load_data()
+        x_train, x_test = x_train / 255.0, x_test / 255.0
+
+        model = tf.keras.models.Sequential([
+          tf.keras.layers.Flatten(),
+          tf.keras.layers.Dense(512, activation=tf.nn.relu),
+          tf.keras.layers.Dropout(0.2),
+          tf.keras.layers.Dense(10, activation=tf.nn.softmax)
+        ])
+        model.compile(optimizer='adam',
+                      loss='sparse_categorical_crossentropy',
+                      metrics=['accuracy'])
+
+        model.fit(x_train, y_train, epochs=5)
+        model.evaluate(x_test, y_test)
+
+ {% dynamic if request.tld != 'cn' %} + Run in a Notebook + {% dynamic endif %} + + - items: + - custom_html: > +
+ +

Research and experimentation

+ +
+

+ Eager execution provides an imperative, define-by-run interface for advanced operations. Write custom layers, forward passes, and training loops with auto‑differentiation. Start with + these notebooks, then read the eager execution guide. +

+
    +
  1. + {% dynamic if request.tld == 'cn' %} + Eager execution basics + {% dynamic else %} + Eager execution basics + {% dynamic endif %} +
    2. +
  2. + {% dynamic if request.tld == 'cn' %} + Automatic differentiation and gradient tapes + {% dynamic else %} + Automatic differentiation and gradient tapes + {% dynamic endif %} +
    3. +
  3. + {% dynamic if request.tld == 'cn' %} + Variables, models, and training + {% dynamic else %} + Variables, models, and training + {% dynamic endif %} +
    4. +
  4. + {% dynamic if request.tld == 'cn' %} + Custom layers + {% dynamic else %} + Custom layers + {% dynamic endif %} +
    5. +
  5. Custom training walkthrough
    6. +
  6. + {% dynamic if request.tld == 'cn' %} + Example: Neural machine translation w/ attention + {% dynamic else %} + Example: Neural machine translation w/ attention + {% dynamic endif %} +
    7. +
+
+
+ Read the eager execution guide +
+
+ - custom_html: > +
+ +

ML at production scale

+ +
+

+ Estimators can train large models on multiple machines in a + production environment. Try the examples below and read the + Estimators guide. +

+
    +
  1. How to build a simple text classifier with TF-Hub
    2. +
  2. Classifying Higgs boson processes
    3. +
  3. Wide and deep learning using estimators
    4. +
+
+
+ Read the Estimators guide +
+
+ + - description: > +

Google Colab: An easy way to learn and use TensorFlow

+

+ Colaboratory + is a Google research project created to help disseminate machine learning + education and research. It's a Jupyter notebook environment that requires + no setup to use and runs entirely in the cloud. + Read the blog post. +

+ + - description: > +

Build your first ML app

+

Create and deploy TensorFlow models on web and mobile.

+ background: grey + items: + - custom_html: > +
+ +

Web developers

+ +
+ TensorFlow.js is a WebGL accelerated, JavaScript library to train and + deploy ML models in the browser and for Node.js. +
+
+ - custom_html: > +
+ +

Mobile developers

+ +
+ TensorFlow Lite is lightweight solution for mobile and embedded devices. +
+
+ + - description: > +

Videos and updates

+

+ Subscribe to the TensorFlow + YouTube channel + and blog for + the latest videos and updates. +

+ items: + - description: > +

Get started with TensorFlow's High-Level APIs

+ youtube_id: tjsHSIG8I08 + buttons: + - label: Watch the video + path: https://www.youtube.com/watch?v=tjsHSIG8I08 + - description: > +

Eager execution

+ youtube_id: T8AW0fKP0Hs + background: grey + buttons: + - label: Watch the video + path: https://www.youtube.com/watch?v=T8AW0fKP0Hs + - description: > +

tf.data: Fast, flexible, and easy-to-use input pipelines

+ youtube_id: uIcqeP7MFH0 + buttons: + - label: Watch the video + path: https://www.youtube.com/watch?v=uIcqeP7MFH0 diff --git a/tensorflow/docs_src/get_started/basic_classification.md b/tensorflow/docs_src/get_started/basic_classification.md new file mode 100644 index 0000000000..91bbd85b24 --- /dev/null +++ b/tensorflow/docs_src/get_started/basic_classification.md @@ -0,0 +1,3 @@ +# Basic Classification + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_classification.ipynb) diff --git a/tensorflow/docs_src/get_started/basic_regression.md b/tensorflow/docs_src/get_started/basic_regression.md new file mode 100644 index 0000000000..a535f22f5a --- /dev/null +++ b/tensorflow/docs_src/get_started/basic_regression.md @@ -0,0 +1,3 @@ +# Basic Regression + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_regression.ipynb) diff --git a/tensorflow/docs_src/get_started/basic_text_classification.md b/tensorflow/docs_src/get_started/basic_text_classification.md new file mode 100644 index 0000000000..7c5d4f7896 --- /dev/null +++ b/tensorflow/docs_src/get_started/basic_text_classification.md @@ -0,0 +1,3 @@ +# Basic Text Classification + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_text_classification.ipynb) diff --git a/tensorflow/docs_src/get_started/datasets_quickstart.md b/tensorflow/docs_src/get_started/datasets_quickstart.md deleted file mode 100644 index 020e40dd3b..0000000000 --- a/tensorflow/docs_src/get_started/datasets_quickstart.md +++ /dev/null @@ -1,387 +0,0 @@ -# Datasets Quick Start - -The @{tf.data} module contains a collection of classes that allows you to -easily load data, manipulate it, and pipe it into your model. This document -introduces the API by walking through two simple examples: - -* Reading in-memory data from numpy arrays. -* Reading lines from a csv file. - - - -## Basic input - -Taking slices from an array is the simplest way to get started with `tf.data`. - -The @{$premade_estimators$Premade Estimators} chapter describes -the following `train_input_fn`, from -[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py), -to pipe the data into the Estimator: - -``` python -def train_input_fn(features, labels, batch_size): - """An input function for training""" - # Convert the inputs to a Dataset. - dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) - - # Shuffle, repeat, and batch the examples. - dataset = dataset.shuffle(1000).repeat().batch(batch_size) - - # Return the dataset. - return dataset -``` - -Let's look at this more closely. - -### Arguments - -This function expects three arguments. Arguments expecting an "array" can -accept nearly anything that can be converted to an array with `numpy.array`. -One exception is -[`tuple`](https://docs.python.org/3/tutorial/datastructures.html#tuples-and-sequences) -which, as we will see, has special meaning for `Datasets`. - -* `features`: A `{'feature_name':array}` dictionary (or - [`DataFrame`](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)) - containing the raw input features. -* `labels` : An array containing the - [label](https://developers.google.com/machine-learning/glossary/#label) - for each example. -* `batch_size` : An integer indicating the desired batch size. - -In [`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py) -we retrieved the Iris data using the `iris_data.load_data()` function. -You can run it, and unpack the results as follows: - -``` python -import iris_data - -# Fetch the data -train, test = iris_data.load_data() -features, labels = train -``` - -Then we passed this data to the input function, with a line similar to this: - -``` python -batch_size=100 -iris_data.train_input_fn(features, labels, batch_size) -``` - -Let's walk through the `train_input_fn()`. - -### Slices - -The function starts by using the @{tf.data.Dataset.from_tensor_slices} function -to create a @{tf.data.Dataset} representing slices of the array. The array is -sliced across the first dimension. For example, an array containing the -@{$tutorials/layers$mnist training data} has a shape of `(60000, 28, 28)`. -Passing this to `from_tensor_slices` returns a `Dataset` object containing -60000 slices, each one a 28x28 image. - -The code that returns this `Dataset` is as follows: - -``` python -train, test = tf.keras.datasets.mnist.load_data() -mnist_x, mnist_y = train - -mnist_ds = tf.data.Dataset.from_tensor_slices(mnist_x) -print(mnist_ds) -``` - -This will print the following line, showing the -@{$programmers_guide/tensors#shapes$shapes} and -@{$programmers_guide/tensors#data_types$types} of the items in -the dataset. Note that a `Dataset` does not know how many items it contains. - -``` None - -``` - -The `Dataset` above represents a simple collection of arrays, but datasets are -much more powerful than this. A `Dataset` can transparently handle any nested -combination of dictionaries or tuples (or -[`namedtuple`](https://docs.python.org/2/library/collections.html#collections.namedtuple) -). - -For example after converting the iris `features` -to a standard python dictionary, you can then convert the dictionary of arrays -to a `Dataset` of dictionaries as follows: - -``` python -dataset = tf.data.Dataset.from_tensor_slices(dict(features)) -print(dataset) -``` -``` None - -``` - -Here we see that when a `Dataset` contains structured elements, the `shapes` -and `types` of the `Dataset` take on the same structure. This dataset contains -dictionaries of @{$programmers_guide/tensors#rank$scalars}, all of type -`tf.float64`. - -The first line of the iris `train_input_fn` uses the same functionality, but -adds another level of structure. It creates a dataset containing -`(features_dict, label)` pairs. - -The following code shows that the label is a scalar with type `int64`: - -``` python -# Convert the inputs to a Dataset. -dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) -print(dataset) -``` -``` - -``` - -### Manipulation - -Currently the `Dataset` would iterate over the data once, in a fixed order, and -only produce a single element at a time. It needs further processing before it -can be used for training. Fortunately, the `tf.data.Dataset` class provides -methods to better prepare the data for training. The next line of the input -function takes advantage of several of these methods: - -``` python -# Shuffle, repeat, and batch the examples. -dataset = dataset.shuffle(1000).repeat().batch(batch_size) -``` - -The @{tf.data.Dataset.shuffle$`shuffle`} method uses a fixed-size buffer to -shuffle the items as they pass through. In this case the `buffer_size` is -greater than the number of examples in the `Dataset`, ensuring that the data is -completely shuffled (The Iris data set only contains 150 examples). - -The @{tf.data.Dataset.repeat$`repeat`} method restarts the `Dataset` when -it reaches the end. To limit the number of epochs, set the `count` argument. - -The @{tf.data.Dataset.batch$`batch`} method collects a number of examples and -stacks them, to create batches. This adds a dimension to their shape. The new -dimension is added as the first dimension. The following code uses -the `batch` method on the MNIST `Dataset`, from earlier. This results in a -`Dataset` containing 3D arrays representing stacks of `(28,28)` images: - -``` python -print(mnist_ds.batch(100)) -``` - -``` none - -``` -Note that the dataset has an unknown batch size because the last batch will -have fewer elements. - -In `train_input_fn`, after batching the `Dataset` contains 1D vectors of -elements where each scalar was previously: - -```python -print(dataset) -``` -``` - -``` - - -### Return - -At this point the `Dataset` contains `(features_dict, labels)` pairs. -This is the format expected by the `train` and `evaluate` methods, so the -`input_fn` returns the dataset. - -The `labels` can/should be omitted when using the `predict` method. - - - - -## Reading a CSV File - -The most common real-world use case for the `Dataset` class is to stream data -from files on disk. The @{tf.data} module includes a variety of -file readers. Let's see how parsing the Iris dataset from the csv file looks -using a `Dataset`. - -The following call to the `iris_data.maybe_download` function downloads the -data if necessary, and returns the pathnames of the resulting files: - -``` python -import iris_data -train_path, test_path = iris_data.maybe_download() -``` - -The [`iris_data.csv_input_fn`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py) -function contains an alternative implementation that parses the csv files using -a `Dataset`. - -Let's look at how to build an Estimator-compatible input function that reads -from the local files. - -### Build the `Dataset` - -We start by building a @{tf.data.TextLineDataset$`TextLineDataset`} object to -read the file one line at a time. Then, we call the -@{tf.data.Dataset.skip$`skip`} method to skip over the first line of the file, which contains a header, not an example: - -``` python -ds = tf.data.TextLineDataset(train_path).skip(1) -``` - -### Build a csv line parser - -We will start by building a function to parse a single line. - -The following `iris_data.parse_line` function accomplishes this task using the -@{tf.decode_csv} function, and some simple python code: - -We must parse each of the lines in the dataset in order to generate the -necessary `(features, label)` pairs. The following `_parse_line` function -calls @{tf.decode_csv} to parse a single line into its features -and the label. Since Estimators require that features be represented as a -dictionary, we rely on Python's built-in `dict` and `zip` functions to build -that dictionary. The feature names are the keys of that dictionary. -We then call the dictionary's `pop` method to remove the label field from -the features dictionary: - -``` python -# Metadata describing the text columns -COLUMNS = ['SepalLength', 'SepalWidth', - 'PetalLength', 'PetalWidth', - 'label'] -FIELD_DEFAULTS = [[0.0], [0.0], [0.0], [0.0], [0]] -def _parse_line(line): - # Decode the line into its fields - fields = tf.decode_csv(line, FIELD_DEFAULTS) - - # Pack the result into a dictionary - features = dict(zip(COLUMNS,fields)) - - # Separate the label from the features - label = features.pop('label') - - return features, label -``` - -### Parse the lines - -Datasets have many methods for manipulating the data while it is being piped -to a model. The most heavily-used method is @{tf.data.Dataset.map$`map`}, which -applies a transformation to each element of the `Dataset`. - -The `map` method takes a `map_func` argument that describes how each item in the -`Dataset` should be transformed. - -
- -
-
-The @{tf.data.Dataset.map$`map`} method applies the `map_func` to -transform each item in the Dataset. -
- -So to parse the lines as they are streamed out of the csv file, we pass our -`_parse_line` function to the `map` method: - -``` python -ds = ds.map(_parse_line) -print(ds) -``` -``` None - -``` - -Now instead of simple scalar strings, the dataset contains `(features, label)` -pairs. - -the remainder of the `iris_data.csv_input_fn` function is identical -to `iris_data.train_input_fn` which was covered in the in the -[Basic input](#basic_input) section. - -### Try it out - -This function can be used as a replacement for -`iris_data.train_input_fn`. It can be used to feed an estimator as follows: - -``` python -train_path, test_path = iris_data.maybe_download() - -# All the inputs are numeric -feature_columns = [ - tf.feature_column.numeric_column(name) - for name in iris_data.CSV_COLUMN_NAMES[:-1]] - -# Build the estimator -est = tf.estimator.LinearClassifier(feature_columns, - n_classes=3) -# Train the estimator -batch_size = 100 -est.train( - steps=1000, - input_fn=lambda : iris_data.csv_input_fn(train_path, batch_size)) -``` - -Estimators expect an `input_fn` to take no arguments. To work around this -restriction, we use `lambda` to capture the arguments and provide the expected -interface. - -## Summary - -The `tf.data` module provides a collection of classes and functions for easily -reading data from a variety of sources. Furthermore, `tf.data` has simple -powerful methods for applying a wide variety of standard and custom -transformations. - -Now you have the basic idea of how to efficiently load data into an -Estimator. Consider the following documents next: - - -* @{$custom_estimators}, which demonstrates how to build your own - custom `Estimator` model. -* The @{$low_level_intro#datasets$Low Level Introduction}, which demonstrates - how to experiment directly with `tf.data.Datasets` using TensorFlow's low - level APIs. -* @{$programmers_guide/datasets} which goes into great detail about additional - functionality of `Datasets`. - diff --git a/tensorflow/docs_src/get_started/eager.md b/tensorflow/docs_src/get_started/eager.md index bbb25e20c6..ddf239485a 100644 --- a/tensorflow/docs_src/get_started/eager.md +++ b/tensorflow/docs_src/get_started/eager.md @@ -1,3 +1,3 @@ -# Get Started with Eager Execution +# Custom Training Walkthrough [Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/r1.9.0/samples/core/get_started/eager.ipynb) diff --git a/tensorflow/docs_src/get_started/index.md b/tensorflow/docs_src/get_started/index.md deleted file mode 100644 index 232d2f1547..0000000000 --- a/tensorflow/docs_src/get_started/index.md +++ /dev/null @@ -1,29 +0,0 @@ -# Get Started - -If you are new to machine learning, we recommend taking the following online -course prior to diving into TensorFlow documentation: - - * [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/), - which introduces machine learning concepts and encourages experimentation - with existing TensorFlow code. - -TensorFlow is a tool for machine learning. While it contains a wide range of -functionality, TensorFlow is mainly designed for deep neural network models. - -The easiest way to get started with TensorFlow is by using Eager Execution. - - * @{$get_started/eager}, is for anyone new to machine learning or TensorFlow. - -TensorFlow provides many APIs. The remainder of this section focuses on the -Estimator API which provide scalable, high-performance models. See the -@{$estimators} guide. - -For more advanced users: - - * The @{$low_level_intro$Low Level Introduction} demonstrates how to use - TensorFlow outside of the Estimator framework, for debugging and - experimentation. - * The @{$programmers_guide$Programmer's Guide} details major - TensorFlow components. - * The @{$tutorials$Tutorials} provide walkthroughs of a variety of - TensorFlow models. diff --git a/tensorflow/docs_src/get_started/leftnav_files b/tensorflow/docs_src/get_started/leftnav_files index e6cc8d5658..9a60496cb5 100644 --- a/tensorflow/docs_src/get_started/leftnav_files +++ b/tensorflow/docs_src/get_started/leftnav_files @@ -1,4 +1,10 @@ -index.md +### Learn and use ML +basic_classification.md +basic_text_classification.md +basic_regression.md +overfit_and_underfit.md +save_and_restore_models.md +next_steps.md -eager.md -datasets_quickstart.md +### Research and experimentation +custom_training_walkthrough.md diff --git a/tensorflow/docs_src/get_started/next_steps.md b/tensorflow/docs_src/get_started/next_steps.md new file mode 100644 index 0000000000..79c0ef3346 --- /dev/null +++ b/tensorflow/docs_src/get_started/next_steps.md @@ -0,0 +1,36 @@ +# Next Steps + +## Learn more about TensorFlow + +* The [TensorFlow Guide](/programmers_guide) includes usage guides for the + high-level APIs, as well as advanced TensorFlow operations. +* [Premade Estimators](/programmers_guide/premade_estimators) are designed to + get results out of the box. Use TensorFlow without building your own models. +* [TensorFlow.js](https://js.tensorflow.org/) allows web developers to train and + deploy ML models in the browser and using Node.js. +* [TFLite](/mobile/tflite) allows mobile developers to do inference efficiently + on mobile devices. +* [TensorFlow Serving](/serving) is an open-source project that can put + TensorFlow models in production quickly. +* The [ecosystem](/ecosystem) contains more projects, including + [Magenta](https://magenta.tensorflow.org/), [TFX](/tfx), + [Swift for TensorFlow](https://github.com/tensorflow/swift), and more. + +## Learn more about machine learning + +Recommended resources include: + +* [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/), + a course from Google that introduces machine learning concepts. +* [CS 20: Tensorflow for Deep Learning Research](http://web.stanford.edu/class/cs20si/), + notes from an intro course from Stanford. +* [CS231n: Convolutional Neural Networks for Visual Recognition](http://cs231n.stanford.edu/), + a course that teaches how convolutional networks work. +* [Machine Learning Recipes](https://www.youtube.com/watch?v=cKxRvEZd3Mw&list=PLOU2XLYxmsIIuiBfYad6rFYQU_jL2ryal), + a video series that introduces basic machine learning concepts with few prerequisites. +* [Deep Learning with Python](https://www.manning.com/books/deep-learning-with-python), + a book by Francois Chollet about the Keras API, as well as an excellent hands on intro to Deep Learning. +* [Hands-on Machine Learning with Scikit-Learn and TensorFlow](https://github.com/ageron/handson-ml), + a book by Aurélien Geron's that is a clear getting-started guide to data science and deep learning. +* [Deep Learning](https://www.deeplearningbook.org/), a book by Ian Goodfellow et al. + that provides a technical dive into learning machine learning. diff --git a/tensorflow/docs_src/get_started/overfit_and_underfit.md b/tensorflow/docs_src/get_started/overfit_and_underfit.md new file mode 100644 index 0000000000..e5b5ae7b5a --- /dev/null +++ b/tensorflow/docs_src/get_started/overfit_and_underfit.md @@ -0,0 +1,3 @@ +# Overfitting and Underfitting + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/overfit_and_underfit.ipynb) diff --git a/tensorflow/docs_src/get_started/save_and_restore_models.md b/tensorflow/docs_src/get_started/save_and_restore_models.md new file mode 100644 index 0000000000..44b3772945 --- /dev/null +++ b/tensorflow/docs_src/get_started/save_and_restore_models.md @@ -0,0 +1,3 @@ +# Save and restore Models + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/save_and_restore_models.ipynb) diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 9baf6870be..41619ca230 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -491,13 +491,7 @@ TensorFlow programs: If the system outputs an error message instead of a greeting, see [Common installation problems](#common_installation_problems). -If you are new to machine learning, we recommend the following: - -* [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course) -* @{$get_started/eager} - -If you are experienced with machine learning but new to TensorFlow, see -@{$get_started/eager}. +To learn more, see [Get Started with TensorFlow](https://www.tensorflow.org/get_started). ## TensorFlow GPU support diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index 693254f876..eeca389617 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -403,11 +403,7 @@ writing TensorFlow programs: If the system outputs an error message instead of a greeting, see [Common installation problems](#common_installation_problems). -If you are new to machine learning, we recommend the -[Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course). - -If you are experienced with machine learning but new to TensorFlow, see -@{$get_started/eager}. +To learn more, see [Get Started with TensorFlow](https://www.tensorflow.org/get_started). ## Common installation problems diff --git a/tensorflow/docs_src/install/install_raspbian.md b/tensorflow/docs_src/install/install_raspbian.md index 2f425162a1..0caab6d335 100644 --- a/tensorflow/docs_src/install/install_raspbian.md +++ b/tensorflow/docs_src/install/install_raspbian.md @@ -230,11 +230,7 @@ problems, despite the log message. If the system outputs an error message instead of a greeting, see [Common installation problems](#common_installation_problems). -If you are new to machine learning, we recommend the [Machine Learning Crash -Course](https://developers.google.com/machine-learning/crash-course). - -If you are experienced with machine learning but new to TensorFlow, see -@{$get_started/eager}. +To learn more, see [Get Started with TensorFlow](https://www.tensorflow.org/get_started). ## Common installation problems diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index 70e97cf556..7afcd340aa 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -362,7 +362,7 @@ TensorFlow programs: 
Hello, TensorFlow!
-If you are new to TensorFlow, see @{$get_started/eager}. +To learn more, see [Get Started with TensorFlow](https://www.tensorflow.org/get_started). If the system outputs an error message instead of a greeting, see [Common installation problems](#common_installation_problems). diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md index 6c4f5b85ab..7fe94f0bc3 100644 --- a/tensorflow/docs_src/install/install_windows.md +++ b/tensorflow/docs_src/install/install_windows.md @@ -157,12 +157,7 @@ TensorFlow programs: If the system outputs an error message instead of a greeting, see [Common installation problems](#common_installation_problems). -If you are new to machine learning, we recommend the -[Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course). - -If you are experienced with machine learning but new to TensorFlow, see -@{$get_started/eager}. - +To learn more, see [Get Started with TensorFlow](https://www.tensorflow.org/get_started). ## Common installation problems diff --git a/tensorflow/docs_src/programmers_guide/datasets_for_estimators.md b/tensorflow/docs_src/programmers_guide/datasets_for_estimators.md new file mode 100644 index 0000000000..345a31b985 --- /dev/null +++ b/tensorflow/docs_src/programmers_guide/datasets_for_estimators.md @@ -0,0 +1,387 @@ +# Datasets for Estimators + +The @{tf.data} module contains a collection of classes that allows you to +easily load data, manipulate it, and pipe it into your model. This document +introduces the API by walking through two simple examples: + +* Reading in-memory data from numpy arrays. +* Reading lines from a csv file. + + + +## Basic input + +Taking slices from an array is the simplest way to get started with `tf.data`. + +The @{$premade_estimators$Premade Estimators} chapter describes +the following `train_input_fn`, from +[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py), +to pipe the data into the Estimator: + +``` python +def train_input_fn(features, labels, batch_size): + """An input function for training""" + # Convert the inputs to a Dataset. + dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) + + # Shuffle, repeat, and batch the examples. + dataset = dataset.shuffle(1000).repeat().batch(batch_size) + + # Return the dataset. + return dataset +``` + +Let's look at this more closely. + +### Arguments + +This function expects three arguments. Arguments expecting an "array" can +accept nearly anything that can be converted to an array with `numpy.array`. +One exception is +[`tuple`](https://docs.python.org/3/tutorial/datastructures.html#tuples-and-sequences) +which, as we will see, has special meaning for `Datasets`. + +* `features`: A `{'feature_name':array}` dictionary (or + [`DataFrame`](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)) + containing the raw input features. +* `labels` : An array containing the + [label](https://developers.google.com/machine-learning/glossary/#label) + for each example. +* `batch_size` : An integer indicating the desired batch size. + +In [`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py) +we retrieved the Iris data using the `iris_data.load_data()` function. +You can run it, and unpack the results as follows: + +``` python +import iris_data + +# Fetch the data +train, test = iris_data.load_data() +features, labels = train +``` + +Then we passed this data to the input function, with a line similar to this: + +``` python +batch_size=100 +iris_data.train_input_fn(features, labels, batch_size) +``` + +Let's walk through the `train_input_fn()`. + +### Slices + +The function starts by using the @{tf.data.Dataset.from_tensor_slices} function +to create a @{tf.data.Dataset} representing slices of the array. The array is +sliced across the first dimension. For example, an array containing the +@{$tutorials/layers$mnist training data} has a shape of `(60000, 28, 28)`. +Passing this to `from_tensor_slices` returns a `Dataset` object containing +60000 slices, each one a 28x28 image. + +The code that returns this `Dataset` is as follows: + +``` python +train, test = tf.keras.datasets.mnist.load_data() +mnist_x, mnist_y = train + +mnist_ds = tf.data.Dataset.from_tensor_slices(mnist_x) +print(mnist_ds) +``` + +This will print the following line, showing the +@{$programmers_guide/tensors#shapes$shapes} and +@{$programmers_guide/tensors#data_types$types} of the items in +the dataset. Note that a `Dataset` does not know how many items it contains. + +``` None + +``` + +The `Dataset` above represents a simple collection of arrays, but datasets are +much more powerful than this. A `Dataset` can transparently handle any nested +combination of dictionaries or tuples (or +[`namedtuple`](https://docs.python.org/2/library/collections.html#collections.namedtuple) +). + +For example after converting the iris `features` +to a standard python dictionary, you can then convert the dictionary of arrays +to a `Dataset` of dictionaries as follows: + +``` python +dataset = tf.data.Dataset.from_tensor_slices(dict(features)) +print(dataset) +``` +``` None + +``` + +Here we see that when a `Dataset` contains structured elements, the `shapes` +and `types` of the `Dataset` take on the same structure. This dataset contains +dictionaries of @{$programmers_guide/tensors#rank$scalars}, all of type +`tf.float64`. + +The first line of the iris `train_input_fn` uses the same functionality, but +adds another level of structure. It creates a dataset containing +`(features_dict, label)` pairs. + +The following code shows that the label is a scalar with type `int64`: + +``` python +# Convert the inputs to a Dataset. +dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) +print(dataset) +``` +``` + +``` + +### Manipulation + +Currently the `Dataset` would iterate over the data once, in a fixed order, and +only produce a single element at a time. It needs further processing before it +can be used for training. Fortunately, the `tf.data.Dataset` class provides +methods to better prepare the data for training. The next line of the input +function takes advantage of several of these methods: + +``` python +# Shuffle, repeat, and batch the examples. +dataset = dataset.shuffle(1000).repeat().batch(batch_size) +``` + +The @{tf.data.Dataset.shuffle$`shuffle`} method uses a fixed-size buffer to +shuffle the items as they pass through. In this case the `buffer_size` is +greater than the number of examples in the `Dataset`, ensuring that the data is +completely shuffled (The Iris data set only contains 150 examples). + +The @{tf.data.Dataset.repeat$`repeat`} method restarts the `Dataset` when +it reaches the end. To limit the number of epochs, set the `count` argument. + +The @{tf.data.Dataset.batch$`batch`} method collects a number of examples and +stacks them, to create batches. This adds a dimension to their shape. The new +dimension is added as the first dimension. The following code uses +the `batch` method on the MNIST `Dataset`, from earlier. This results in a +`Dataset` containing 3D arrays representing stacks of `(28,28)` images: + +``` python +print(mnist_ds.batch(100)) +``` + +``` none + +``` +Note that the dataset has an unknown batch size because the last batch will +have fewer elements. + +In `train_input_fn`, after batching the `Dataset` contains 1D vectors of +elements where each scalar was previously: + +```python +print(dataset) +``` +``` + +``` + + +### Return + +At this point the `Dataset` contains `(features_dict, labels)` pairs. +This is the format expected by the `train` and `evaluate` methods, so the +`input_fn` returns the dataset. + +The `labels` can/should be omitted when using the `predict` method. + + + + +## Reading a CSV File + +The most common real-world use case for the `Dataset` class is to stream data +from files on disk. The @{tf.data} module includes a variety of +file readers. Let's see how parsing the Iris dataset from the csv file looks +using a `Dataset`. + +The following call to the `iris_data.maybe_download` function downloads the +data if necessary, and returns the pathnames of the resulting files: + +``` python +import iris_data +train_path, test_path = iris_data.maybe_download() +``` + +The [`iris_data.csv_input_fn`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py) +function contains an alternative implementation that parses the csv files using +a `Dataset`. + +Let's look at how to build an Estimator-compatible input function that reads +from the local files. + +### Build the `Dataset` + +We start by building a @{tf.data.TextLineDataset$`TextLineDataset`} object to +read the file one line at a time. Then, we call the +@{tf.data.Dataset.skip$`skip`} method to skip over the first line of the file, which contains a header, not an example: + +``` python +ds = tf.data.TextLineDataset(train_path).skip(1) +``` + +### Build a csv line parser + +We will start by building a function to parse a single line. + +The following `iris_data.parse_line` function accomplishes this task using the +@{tf.decode_csv} function, and some simple python code: + +We must parse each of the lines in the dataset in order to generate the +necessary `(features, label)` pairs. The following `_parse_line` function +calls @{tf.decode_csv} to parse a single line into its features +and the label. Since Estimators require that features be represented as a +dictionary, we rely on Python's built-in `dict` and `zip` functions to build +that dictionary. The feature names are the keys of that dictionary. +We then call the dictionary's `pop` method to remove the label field from +the features dictionary: + +``` python +# Metadata describing the text columns +COLUMNS = ['SepalLength', 'SepalWidth', + 'PetalLength', 'PetalWidth', + 'label'] +FIELD_DEFAULTS = [[0.0], [0.0], [0.0], [0.0], [0]] +def _parse_line(line): + # Decode the line into its fields + fields = tf.decode_csv(line, FIELD_DEFAULTS) + + # Pack the result into a dictionary + features = dict(zip(COLUMNS,fields)) + + # Separate the label from the features + label = features.pop('label') + + return features, label +``` + +### Parse the lines + +Datasets have many methods for manipulating the data while it is being piped +to a model. The most heavily-used method is @{tf.data.Dataset.map$`map`}, which +applies a transformation to each element of the `Dataset`. + +The `map` method takes a `map_func` argument that describes how each item in the +`Dataset` should be transformed. + +
+ +
+
+The @{tf.data.Dataset.map$`map`} method applies the `map_func` to +transform each item in the Dataset. +
+ +So to parse the lines as they are streamed out of the csv file, we pass our +`_parse_line` function to the `map` method: + +``` python +ds = ds.map(_parse_line) +print(ds) +``` +``` None + +``` + +Now instead of simple scalar strings, the dataset contains `(features, label)` +pairs. + +the remainder of the `iris_data.csv_input_fn` function is identical +to `iris_data.train_input_fn` which was covered in the in the +[Basic input](#basic_input) section. + +### Try it out + +This function can be used as a replacement for +`iris_data.train_input_fn`. It can be used to feed an estimator as follows: + +``` python +train_path, test_path = iris_data.maybe_download() + +# All the inputs are numeric +feature_columns = [ + tf.feature_column.numeric_column(name) + for name in iris_data.CSV_COLUMN_NAMES[:-1]] + +# Build the estimator +est = tf.estimator.LinearClassifier(feature_columns, + n_classes=3) +# Train the estimator +batch_size = 100 +est.train( + steps=1000, + input_fn=lambda : iris_data.csv_input_fn(train_path, batch_size)) +``` + +Estimators expect an `input_fn` to take no arguments. To work around this +restriction, we use `lambda` to capture the arguments and provide the expected +interface. + +## Summary + +The `tf.data` module provides a collection of classes and functions for easily +reading data from a variety of sources. Furthermore, `tf.data` has simple +powerful methods for applying a wide variety of standard and custom +transformations. + +Now you have the basic idea of how to efficiently load data into an +Estimator. Consider the following documents next: + + +* @{$custom_estimators}, which demonstrates how to build your own + custom `Estimator` model. +* The @{$low_level_intro#datasets$Low Level Introduction}, which demonstrates + how to experiment directly with `tf.data.Datasets` using TensorFlow's low + level APIs. +* @{$programmers_guide/datasets} which goes into great detail about additional + functionality of `Datasets`. + diff --git a/tensorflow/docs_src/programmers_guide/index.md b/tensorflow/docs_src/programmers_guide/index.md index 0c2d4afb11..9c58a3b45e 100644 --- a/tensorflow/docs_src/programmers_guide/index.md +++ b/tensorflow/docs_src/programmers_guide/index.md @@ -22,6 +22,7 @@ works. The units are as follows: design yourself. * @{$feature_columns}, which shows how an Estimator can handle a variety of input data types without changes to the model. +* @{$datasets_for_estimators} describes using tf.data with estimators. * @{$checkpoints}, which explains how to save training progress and resume where you left off. diff --git a/tensorflow/docs_src/programmers_guide/leftnav_files b/tensorflow/docs_src/programmers_guide/leftnav_files index 3bcf864e13..357a2a1cb9 100644 --- a/tensorflow/docs_src/programmers_guide/leftnav_files +++ b/tensorflow/docs_src/programmers_guide/leftnav_files @@ -10,6 +10,7 @@ estimators.md: Introduction to Estimators premade_estimators.md custom_estimators.md feature_columns.md +datasets_for_estimators.md checkpoints.md ### Accelerators diff --git a/tensorflow/docs_src/programmers_guide/premade_estimators.md b/tensorflow/docs_src/programmers_guide/premade_estimators.md index f6dd75eaca..02e2caf64b 100644 --- a/tensorflow/docs_src/programmers_guide/premade_estimators.md +++ b/tensorflow/docs_src/programmers_guide/premade_estimators.md @@ -81,7 +81,7 @@ We strongly recommend writing TensorFlow programs with the following APIs: * @{$programmers_guide/estimators$Estimators}, which represent a complete model. The Estimator API provides methods to train the model, to judge the model's accuracy, and to generate predictions. -* @{$get_started/datasets_quickstart$Datasets}, which build a data input +* @{$programmers_guide/datasets_for_estimators}, which build a data input pipeline. The Dataset API has methods to load and manipulate data, and feed it into your model. The Dataset API meshes well with the Estimators API. @@ -424,9 +424,7 @@ Now that you've gotten started writing TensorFlow programs, consider the following material: * @{$checkpoints$Checkpoints} to learn how to save and restore models. -* @{$get_started/datasets_quickstart$Datasets} to learn more about importing - data into your - model. +* @{$programmers_guide/datasets_for_estimators} to learn more about importing + data into your model. * @{$custom_estimators$Creating Custom Estimators} to learn how to write your own Estimator, customized for a particular problem. - diff --git a/tensorflow/docs_src/tutorials/index.md b/tensorflow/docs_src/tutorials/index.md index af01d3eaa1..6bd3a3a897 100644 --- a/tensorflow/docs_src/tutorials/index.md +++ b/tensorflow/docs_src/tutorials/index.md @@ -2,9 +2,8 @@ This section contains tutorials demonstrating how to do specific tasks -in TensorFlow. If you are new to TensorFlow, we recommend reading the -documents in the "@{$get_started$Get Started}" section before reading -these tutorials. +in TensorFlow. If you are new to TensorFlow, we recommend reading +[Get Started with TensorFlow](/get_started/). ## Images -- cgit v1.2.3 From d9e006e80990e54913c25de70a1f8e7db2f22bc8 Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Wed, 20 Jun 2018 10:02:25 -0700 Subject: Fix eager path in get_started leftnav PiperOrigin-RevId: 201370156 --- tensorflow/docs_src/get_started/leftnav_files | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/get_started/leftnav_files b/tensorflow/docs_src/get_started/leftnav_files index 9a60496cb5..5c400a67f0 100644 --- a/tensorflow/docs_src/get_started/leftnav_files +++ b/tensorflow/docs_src/get_started/leftnav_files @@ -7,4 +7,4 @@ save_and_restore_models.md next_steps.md ### Research and experimentation -custom_training_walkthrough.md +eager.md -- cgit v1.2.3 From 4e0b1612e0a71b0e14da2bc37c49e3d65744342c Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Fri, 22 Jun 2018 15:37:58 -0700 Subject: Add Install Raspbian to leftnav. PiperOrigin-RevId: 201752380 --- tensorflow/docs_src/install/leftnav_files | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/docs_src/install/leftnav_files b/tensorflow/docs_src/install/leftnav_files index e523e06f67..ace275c0e8 100644 --- a/tensorflow/docs_src/install/leftnav_files +++ b/tensorflow/docs_src/install/leftnav_files @@ -4,6 +4,7 @@ index.md install_linux.md: Ubuntu install_mac.md: MacOS install_windows.md: Windows +install_raspbian.md: Raspbian install_sources.md: From source >>> migration.md -- cgit v1.2.3 From 2897538b938dcd6d9c63a97f0870232ac9e4819e Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 25 Jun 2018 12:48:40 -0700 Subject: Update r1.9 release notes. - link to new get_started. - Add keras CuDNN layers. - Links for gradient boosted estimators. - Added new contrib-estimators and string-processing. - Bumped some minor sounding things down from "Major" to "Bugfix+Other" --- RELEASE.md | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 510eca5467..bfe0da8739 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,18 +1,37 @@ # Release 1.9.0 ## Major Features And Improvements -* Update tf.keras to the Keras 2.1.6 API. +* New `tf.keras` based [get_started](http://tensorflow.org/versions/r1.9/get_started) +* Update `tf.keras` to the Keras 2.1.6 API. +* Added [`tf.keras.layers.CuDNNGRU`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/keras/layers/CuDNNGRU) and [`tf.keras.layers.CuDNNLSTM`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/keras/layers/CuDNNLSTM) layers. [Try it](https://colab.sandbox.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb?linkId=53292082). +* Adding support of core [feature columns](https://www.tensorflow.org/get_started/feature_columns) and [losses](https://www.tensorflow.org/api_docs/python/tf/losses) to [gradient boosted trees estimators](https://github.com/tensorflow/models/tree/master/official/boosted_trees). +* The [python interface](https://tensorflow-dot-devsite.googleplex.com/versions/r1.9/api_docs/python/tf/contrib/lite) + for the [TFLite Optimizing Converter](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/toco/README.md) + has been expanded, and the command line interface (AKA: `toco`, `tflite_convert`) is once again + included in the standard `pip` installation. +* Improved data-loading and text processing with: + * [`tf.decode_compressed`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/decode_compressed) + * [`tf.string_strip`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/string_strip) + * [`tf.strings.regex_full_match`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/strings/regex_full_match) +* Added experimental support for new pre-made Estimators: + * [`tf.contrib.estimator.BaselineEstimator`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/contrib/estimator/BaselineEstimator) + * [`tf.contrib.estimator.RNNClassifier`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/contrib/estimator/RNNEstimator) + * [`tf.contrib.estimator.RNNEstimator`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/contrib/estimator/RNNClassifier) +* The [distributions.Bijector](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/contrib/distributions/bijectors/Bijector) + API supports broadcasting for Bijectors with new API changes. + +## Breaking Chances + * If you're opening empty variable scopes; replace `variable_scope('', ...)` by + `variable_scope(tf.get_variable_scope(), ...)`. + +## Bug Fixes and Other Changes + * `tfe.Network` is deprecated. Please inherit from `tf.keras.Model`. -* Adding support of core feature columns and losses to gradient boosted trees estimators. -* The distributions.Bijector API supports broadcasting for Bijectors with new API changes. See [here](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/distributions/bijectors/Bijector) for more details. * Layered variable names have changed in the following conditions: * Using `tf.keras.layers` with custom variable scopes. - * Using `tf.layers` in a subclassed `tf.keras.Model` class. See [here](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/layers) for more details + * Using `tf.layers` in a subclassed `tf.keras.Model` class. See + [here](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/layers) for more details -## Breaking Chances - * If you're opening empty variable scopes; replace `variable_scope`('', ...) by `variable_scope`(`tf.get_variable_scope()`, ...). - -## Bug Fixes and Other Changes * `tf.data`: * `Dataset.from_generator()` now accepts an `args` list, in order to create nested generators. * `Dataset.list_files()` now produces determinstic results when `shuffle=False` or a `seed` is passed. -- cgit v1.2.3 From 56fba15b868145f87109bd5cb155527b0c0640d1 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 25 Jun 2018 13:16:52 -0700 Subject: Update RELEASE.md --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index bfe0da8739..f6a52a2951 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,7 +1,7 @@ # Release 1.9.0 ## Major Features And Improvements -* New `tf.keras` based [get_started](http://tensorflow.org/versions/r1.9/get_started) +* New `tf.keras` based [get_started](http://tensorflow.org/versions/r1.9/get_started), and [programmers_guide](http://tensorflow.org/versions/r1.9/programmers_guide/keras). * Update `tf.keras` to the Keras 2.1.6 API. * Added [`tf.keras.layers.CuDNNGRU`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/keras/layers/CuDNNGRU) and [`tf.keras.layers.CuDNNLSTM`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/keras/layers/CuDNNLSTM) layers. [Try it](https://colab.sandbox.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb?linkId=53292082). * Adding support of core [feature columns](https://www.tensorflow.org/get_started/feature_columns) and [losses](https://www.tensorflow.org/api_docs/python/tf/losses) to [gradient boosted trees estimators](https://github.com/tensorflow/models/tree/master/official/boosted_trees). -- cgit v1.2.3 From ce03a10d70884d2b6d8134b30ad3c5d181877403 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 25 Jun 2018 13:22:14 -0700 Subject: Update RELEASE.md --- RELEASE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index f6a52a2951..5c79ebec34 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,7 +1,8 @@ # Release 1.9.0 ## Major Features And Improvements -* New `tf.keras` based [get_started](http://tensorflow.org/versions/r1.9/get_started), and [programmers_guide](http://tensorflow.org/versions/r1.9/programmers_guide/keras). +* Updated docs for `tf.keras`: New Keras-based [get started](http://tensorflow.org/versions/r1.9/get_started), + and [programmers guide page](http://tensorflow.org/versions/r1.9/programmers_guide/keras). * Update `tf.keras` to the Keras 2.1.6 API. * Added [`tf.keras.layers.CuDNNGRU`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/keras/layers/CuDNNGRU) and [`tf.keras.layers.CuDNNLSTM`](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/keras/layers/CuDNNLSTM) layers. [Try it](https://colab.sandbox.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb?linkId=53292082). * Adding support of core [feature columns](https://www.tensorflow.org/get_started/feature_columns) and [losses](https://www.tensorflow.org/api_docs/python/tf/losses) to [gradient boosted trees estimators](https://github.com/tensorflow/models/tree/master/official/boosted_trees). -- cgit v1.2.3 From b0f2eee339a041de4e7837b68a9ff4fc77ca7c4a Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Fri, 22 Jun 2018 17:40:27 -0700 Subject: Rename programmers_guide/ directory to guide/. Update references in source files and docs in tensorflow and related projects. PiperOrigin-RevId: 201766994 --- README.md | 2 +- RELEASE.md | 6 +- tensorflow/contrib/autograph/README.md | 2 +- tensorflow/contrib/data/__init__.py | 2 +- tensorflow/contrib/eager/README.md | 2 +- .../python/examples/notebooks/3_datasets.ipynb | 6 +- tensorflow/contrib/eager/python/g3doc/guide.md | 4 +- tensorflow/contrib/lite/toco/README.md | 2 +- tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 2 +- tensorflow/core/protobuf/config.proto | 6 +- tensorflow/docs_src/api_guides/python/client.md | 2 +- .../docs_src/api_guides/python/input_dataset.md | 3 +- .../docs_src/api_guides/python/reading_data.md | 8 +- tensorflow/docs_src/deploy/distributed.md | 2 +- tensorflow/docs_src/extend/architecture.md | 5 +- tensorflow/docs_src/get_started/_index.yaml | 12 +- tensorflow/docs_src/get_started/next_steps.md | 4 +- tensorflow/docs_src/guide/checkpoints.md | 238 +++++ tensorflow/docs_src/guide/custom_estimators.md | 602 +++++++++++++ tensorflow/docs_src/guide/datasets.md | 823 +++++++++++++++++ .../docs_src/guide/datasets_for_estimators.md | 387 ++++++++ tensorflow/docs_src/guide/debugger.md | 804 +++++++++++++++++ tensorflow/docs_src/guide/eager.md | 849 +++++++++++++++++ tensorflow/docs_src/guide/embedding.md | 262 ++++++ tensorflow/docs_src/guide/estimators.md | 193 ++++ tensorflow/docs_src/guide/faq.md | 297 ++++++ tensorflow/docs_src/guide/feature_columns.md | 572 ++++++++++++ tensorflow/docs_src/guide/graph_viz.md | 316 +++++++ tensorflow/docs_src/guide/graphs.md | 558 ++++++++++++ tensorflow/docs_src/guide/index.md | 86 ++ tensorflow/docs_src/guide/keras.md | 623 +++++++++++++ tensorflow/docs_src/guide/leftnav_files | 40 + tensorflow/docs_src/guide/low_level_intro.md | 604 +++++++++++++ tensorflow/docs_src/guide/premade_estimators.md | 430 +++++++++ tensorflow/docs_src/guide/saved_model.md | 999 +++++++++++++++++++++ .../docs_src/guide/summaries_and_tensorboard.md | 225 +++++ .../docs_src/guide/tensorboard_histograms.md | 245 +++++ tensorflow/docs_src/guide/tensors.md | 330 +++++++ tensorflow/docs_src/guide/using_gpu.md | 215 +++++ tensorflow/docs_src/guide/using_tpu.md | 395 ++++++++ tensorflow/docs_src/guide/variables.md | 319 +++++++ tensorflow/docs_src/guide/version_compat.md | 319 +++++++ tensorflow/docs_src/install/install_go.md | 2 +- tensorflow/docs_src/install/install_java.md | 2 +- .../docs_src/programmers_guide/checkpoints.md | 240 ----- .../programmers_guide/custom_estimators.md | 602 ------------- tensorflow/docs_src/programmers_guide/datasets.md | 823 ----------------- .../programmers_guide/datasets_for_estimators.md | 387 -------- tensorflow/docs_src/programmers_guide/debugger.md | 804 ----------------- tensorflow/docs_src/programmers_guide/eager.md | 849 ----------------- tensorflow/docs_src/programmers_guide/embedding.md | 262 ------ .../docs_src/programmers_guide/estimators.md | 193 ---- tensorflow/docs_src/programmers_guide/faq.md | 297 ------ .../docs_src/programmers_guide/feature_columns.md | 572 ------------ tensorflow/docs_src/programmers_guide/graph_viz.md | 316 ------- tensorflow/docs_src/programmers_guide/graphs.md | 558 ------------ tensorflow/docs_src/programmers_guide/index.md | 86 -- tensorflow/docs_src/programmers_guide/keras.md | 623 ------------- .../docs_src/programmers_guide/leftnav_files | 40 - .../docs_src/programmers_guide/low_level_intro.md | 604 ------------- .../programmers_guide/premade_estimators.md | 430 --------- .../docs_src/programmers_guide/saved_model.md | 999 --------------------- .../programmers_guide/summaries_and_tensorboard.md | 225 ----- .../programmers_guide/tensorboard_histograms.md | 245 ----- tensorflow/docs_src/programmers_guide/tensors.md | 330 ------- tensorflow/docs_src/programmers_guide/using_gpu.md | 215 ----- tensorflow/docs_src/programmers_guide/using_tpu.md | 395 -------- tensorflow/docs_src/programmers_guide/variables.md | 319 ------- .../docs_src/programmers_guide/version_compat.md | 319 ------- tensorflow/docs_src/tutorials/deep_cnn.md | 2 +- tensorflow/docs_src/tutorials/layers.md | 2 +- .../how_tos/reading_data/fully_connected_reader.py | 2 +- tensorflow/java/README.md | 5 +- .../src/main/java/org/tensorflow/package-info.java | 2 +- tensorflow/python/data/__init__.py | 2 +- tensorflow/python/data/ops/dataset_ops.py | 14 + tensorflow/python/debug/BUILD | 2 +- tensorflow/python/debug/README.md | 4 +- tensorflow/python/debug/examples/README.md | 4 +- tensorflow/python/estimator/keras.py | 2 +- tensorflow/python/ops/script_ops.py | 2 +- tensorflow/python/tools/saved_model_cli.py | 4 +- third_party/examples/eager/spinn/README.md | 2 +- 83 files changed, 10798 insertions(+), 10789 deletions(-) create mode 100644 tensorflow/docs_src/guide/checkpoints.md create mode 100644 tensorflow/docs_src/guide/custom_estimators.md create mode 100644 tensorflow/docs_src/guide/datasets.md create mode 100644 tensorflow/docs_src/guide/datasets_for_estimators.md create mode 100644 tensorflow/docs_src/guide/debugger.md create mode 100644 tensorflow/docs_src/guide/eager.md create mode 100644 tensorflow/docs_src/guide/embedding.md create mode 100644 tensorflow/docs_src/guide/estimators.md create mode 100644 tensorflow/docs_src/guide/faq.md create mode 100644 tensorflow/docs_src/guide/feature_columns.md create mode 100644 tensorflow/docs_src/guide/graph_viz.md create mode 100644 tensorflow/docs_src/guide/graphs.md create mode 100644 tensorflow/docs_src/guide/index.md create mode 100644 tensorflow/docs_src/guide/keras.md create mode 100644 tensorflow/docs_src/guide/leftnav_files create mode 100644 tensorflow/docs_src/guide/low_level_intro.md create mode 100644 tensorflow/docs_src/guide/premade_estimators.md create mode 100644 tensorflow/docs_src/guide/saved_model.md create mode 100644 tensorflow/docs_src/guide/summaries_and_tensorboard.md create mode 100644 tensorflow/docs_src/guide/tensorboard_histograms.md create mode 100644 tensorflow/docs_src/guide/tensors.md create mode 100644 tensorflow/docs_src/guide/using_gpu.md create mode 100644 tensorflow/docs_src/guide/using_tpu.md create mode 100644 tensorflow/docs_src/guide/variables.md create mode 100644 tensorflow/docs_src/guide/version_compat.md delete mode 100644 tensorflow/docs_src/programmers_guide/checkpoints.md delete mode 100644 tensorflow/docs_src/programmers_guide/custom_estimators.md delete mode 100644 tensorflow/docs_src/programmers_guide/datasets.md delete mode 100644 tensorflow/docs_src/programmers_guide/datasets_for_estimators.md delete mode 100644 tensorflow/docs_src/programmers_guide/debugger.md delete mode 100644 tensorflow/docs_src/programmers_guide/eager.md delete mode 100644 tensorflow/docs_src/programmers_guide/embedding.md delete mode 100644 tensorflow/docs_src/programmers_guide/estimators.md delete mode 100644 tensorflow/docs_src/programmers_guide/faq.md delete mode 100644 tensorflow/docs_src/programmers_guide/feature_columns.md delete mode 100644 tensorflow/docs_src/programmers_guide/graph_viz.md delete mode 100644 tensorflow/docs_src/programmers_guide/graphs.md delete mode 100644 tensorflow/docs_src/programmers_guide/index.md delete mode 100644 tensorflow/docs_src/programmers_guide/keras.md delete mode 100644 tensorflow/docs_src/programmers_guide/leftnav_files delete mode 100644 tensorflow/docs_src/programmers_guide/low_level_intro.md delete mode 100644 tensorflow/docs_src/programmers_guide/premade_estimators.md delete mode 100644 tensorflow/docs_src/programmers_guide/saved_model.md delete mode 100644 tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md delete mode 100644 tensorflow/docs_src/programmers_guide/tensorboard_histograms.md delete mode 100644 tensorflow/docs_src/programmers_guide/tensors.md delete mode 100644 tensorflow/docs_src/programmers_guide/using_gpu.md delete mode 100644 tensorflow/docs_src/programmers_guide/using_tpu.md delete mode 100644 tensorflow/docs_src/programmers_guide/variables.md delete mode 100644 tensorflow/docs_src/programmers_guide/version_compat.md diff --git a/README.md b/README.md index 6fb4486d0d..4e4d139bd1 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ data flow graphs. The graph nodes represent mathematical operations, while the graph edges represent the multidimensional data arrays (tensors) that flow between them. This flexible architecture enables you to deploy computation to one or more CPUs or GPUs in a desktop, server, or mobile device without rewriting -code. TensorFlow also includes [TensorBoard](https://www.tensorflow.org/programmers_guide/summaries_and_tensorboard), a data visualization toolkit. +code. TensorFlow also includes [TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard), a data visualization toolkit. TensorFlow was originally developed by researchers and engineers working on the Google Brain team within Google's Machine Intelligence Research diff --git a/RELEASE.md b/RELEASE.md index 510eca5467..5fec61af7e 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -467,7 +467,7 @@ answered questions, and were part of inspiring discussions. ## Major Features And Improvements * `tf.keras` is now part of the core TensorFlow API. -* [`tf.data`](http://tensorflow.org/programmers_guide/datasets) is now part of +* [`tf.data`](http://tensorflow.org/guide/datasets) is now part of the core TensorFlow API. * The API is now subject to backwards compatibility guarantees. @@ -495,7 +495,7 @@ answered questions, and were part of inspiring discussions. * TensorFlow Debugger (tfdbg): * Add `eval` command to allow evaluation of arbitrary Python/numpy expressions in tfdbg command-line interface. See - [Debugging TensorFlow Programs](https://www.tensorflow.org/programmers_guide/debugger) + [Debugging TensorFlow Programs](https://www.tensorflow.org/guide/debugger) for more details. * Usability improvement: The frequently used tensor filter `has_inf_or_nan` is now added to `Session` wrappers and hooks by default. So there is no need @@ -782,7 +782,7 @@ answered questions, and were part of inspiring discussions. * Support client-provided ClusterSpec's and propagate them to all workers to enable the creation of dynamic TensorFlow clusters. * TensorFlow C library now available for Windows. * We released a new open-source version of TensorBoard. -* [`SavedModel CLI`](https://www.tensorflow.org/versions/master/programmers_guide/saved_model_cli) tool available to inspect and execute MetaGraph in SavedModel +* [`SavedModel CLI`](https://www.tensorflow.org/versions/master/guide/saved_model_cli) tool available to inspect and execute MetaGraph in SavedModel * Android releases of TensorFlow are now pushed to jcenter for easier integration into apps. See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/android/README.md diff --git a/tensorflow/contrib/autograph/README.md b/tensorflow/contrib/autograph/README.md index 674859bed4..47b1d4a99a 100644 --- a/tensorflow/contrib/autograph/README.md +++ b/tensorflow/contrib/autograph/README.md @@ -4,7 +4,7 @@ IMPORTANT: AutoGraph is alpha software, and under active development. Expect rou AutoGraph is a Python to TensorFlow compiler. -With AutoGraph, you can write [Eager style](https://www.tensorflow.org/programmers_guide/eager) code in a concise manner, and run it as a TensorFlow graph. AutoGraph uses source code transformation and partial evaluation to generate Python code that builds an equivalent TensorFlow subgraph. The result is code that behaves like ops and can be freely combined with other TensorFlow ops. +With AutoGraph, you can write [Eager style](https://www.tensorflow.org/guide/eager) code in a concise manner, and run it as a TensorFlow graph. AutoGraph uses source code transformation and partial evaluation to generate Python code that builds an equivalent TensorFlow subgraph. The result is code that behaves like ops and can be freely combined with other TensorFlow ops. For example, this Python function: diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py index 9c6a13333e..3510e7b1ad 100644 --- a/tensorflow/contrib/data/__init__.py +++ b/tensorflow/contrib/data/__init__.py @@ -20,7 +20,7 @@ be used in conjunction with the @{tf.data.Dataset} API. Note that the guarantees as `tf.data`, but we will provide deprecation advice in advance of removing existing functionality. -See the @{$datasets$Importing Data} Programmer's Guide for an overview. +See @{$guide/datasets$Importing Data} for an overview. @@Counter @@CheckpointInputPipelineHook diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md index 4384431e7b..86d203452e 100644 --- a/tensorflow/contrib/eager/README.md +++ b/tensorflow/contrib/eager/README.md @@ -44,7 +44,7 @@ Installation instructions at https://www.tensorflow.org/install/ For an introduction to eager execution in TensorFlow, see: -- [User Guide](https://www.tensorflow.org/programmers_guide/eager) ([source](../../docs_src/programmers_guide/eager.md)) +- [User Guide](https://www.tensorflow.org/guide/eager) ([source](../../docs_src/guide/eager.md)) - Notebook: [Basic Usage](python/examples/notebooks/1_basics.ipynb) - Notebook: [Gradients](python/examples/notebooks/2_gradients.ipynb) - Notebook: [Importing Data](python/examples/notebooks/3_datasets.ipynb) diff --git a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb index bfcc7feb07..d268cbcd91 100644 --- a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb +++ b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb @@ -9,7 +9,7 @@ "source": [ "# Eager Execution Tutorial: Importing Data\n", "\n", - "This notebook demonstrates the use of the [`tf.data.Dataset` API](https://www.tensorflow.org/programmers_guide/datasets) to build pipelines to feed data to your program. It covers:\n", + "This notebook demonstrates the use of the [`tf.data.Dataset` API](https://www.tensorflow.org/guide/datasets) to build pipelines to feed data to your program. It covers:\n", "\n", "* Creating a `Dataset`.\n", "* Iteration over a `Dataset` with eager execution enabled.\n", @@ -18,7 +18,7 @@ "\n", "If you're familiar with TensorFlow graphs, the API for constructing the `Dataset` object remains exactly the same when eager execution is enabled, but the process of iterating over elements of the dataset is slightly simpler.\n", "You can use Python iteration over the `tf.data.Dataset` object and do not need to explicitly create an `tf.data.Iterator` object.\n", - "As a result, the discussion on iterators in the [Programmer's Guide](https://www.tensorflow.org/programmers_guide/datasets) is not relevant when eager execution is enabled." + "As a result, the discussion on iterators in the [TensorFlow Guide](https://www.tensorflow.org/guide/datasets) is not relevant when eager execution is enabled." ] }, { @@ -63,7 +63,7 @@ "source": [ "# Step 1: Create a source `Dataset`\n", "\n", - "Create a _source_ dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensor_slices) or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset). See the [Programmer's Guide](https://www.google.com/url?sa=D\u0026q=https%3A%2F%2Fwww.tensorflow.org%2Fprogrammers_guide%2Fdatasets%23reading_input_data) for more information." + "Create a _source_ dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensor_slices) or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset). See the [TensorFlow Guide](https://www.tensorflow.org/guide/datasets#reading_input_data) for more information." ] }, { diff --git a/tensorflow/contrib/eager/python/g3doc/guide.md b/tensorflow/contrib/eager/python/g3doc/guide.md index 2d2aba6908..23f33d0230 100644 --- a/tensorflow/contrib/eager/python/g3doc/guide.md +++ b/tensorflow/contrib/eager/python/g3doc/guide.md @@ -4,8 +4,8 @@ Eager execution is a feature that makes TensorFlow execute operations immediately: concrete values are returned, instead of creating a computational graph that is executed later. -A user guide is available: https://www.tensorflow.org/programmers_guide/eager -([source file](../../../../docs_src/programmers_guide/eager.md)) +A user guide is available: https://www.tensorflow.org/guide/eager +([source file](../../../../docs_src/guide/eager.md)) We welcome feedback through [GitHub issues](https://github.com/tensorflow/tensorflow/labels/comp:eager). diff --git a/tensorflow/contrib/lite/toco/README.md b/tensorflow/contrib/lite/toco/README.md index 522e260ad2..ee83c7a6e3 100644 --- a/tensorflow/contrib/lite/toco/README.md +++ b/tensorflow/contrib/lite/toco/README.md @@ -17,7 +17,7 @@ Usage information is given in these documents: Once an application developer has a trained TensorFlow model, TOCO will accept that model and generate a TensorFlow Lite [FlatBuffer](https://google.github.io/flatbuffers/) file. TOCO currently supports -[SavedModels](https://www.tensorflow.org/programmers_guide/saved_model#using_savedmodel_with_estimators) +[SavedModels](https://www.tensorflow.org/guide/saved_model#using_savedmodel_with_estimators) and frozen graphs (models generated via [freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py)). The TensorFlow Lite FlatBuffer file can be shipped to client devices, generally diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 7c770912b4..c57acd0a2d 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -1103,7 +1103,7 @@ class _InputPipeline(object): err_msg = ('Input pipeline contains one or more QueueRunners. ' 'It could be slow and not scalable. Please consider ' 'converting your input pipeline to use `tf.data` instead (see ' - 'https://www.tensorflow.org/programmers_guide/datasets for ' + 'https://www.tensorflow.org/guide/datasets for ' 'instructions.') if _WRAP_INPUT_FN_INTO_WHILE_LOOP: raise RuntimeError(err_msg) diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto index 9a48f43a63..d83215d5c2 100644 --- a/tensorflow/core/protobuf/config.proto +++ b/tensorflow/core/protobuf/config.proto @@ -147,7 +147,7 @@ message GPUOptions { // Everything inside experimental is subject to change and is not subject // to API stability guarantees in - // https://www.tensorflow.org/programmers_guide/version_compat. + // https://www.tensorflow.org/guide/version_compat. Experimental experimental = 9; }; @@ -381,7 +381,7 @@ message ConfigProto { // Everything inside Experimental is subject to change and is not subject // to API stability guarantees in - // https://www.tensorflow.org/programmers_guide/version_compat. + // https://www.tensorflow.org/guide/version_compat. message Experimental { // Task name for group resolution. string collective_group_leader = 1; @@ -426,7 +426,7 @@ message RunOptions { // Everything inside Experimental is subject to change and is not subject // to API stability guarantees in - // https://www.tensorflow.org/programmers_guide/version_compat. + // https://www.tensorflow.org/guide/version_compat. message Experimental { // If non-zero, declares that this graph is going to use collective // ops and must synchronize step_ids with any other graph with this diff --git a/tensorflow/docs_src/api_guides/python/client.md b/tensorflow/docs_src/api_guides/python/client.md index eef23696db..27fc8610bf 100644 --- a/tensorflow/docs_src/api_guides/python/client.md +++ b/tensorflow/docs_src/api_guides/python/client.md @@ -3,7 +3,7 @@ This library contains classes for launching graphs and executing operations. -@{$programmers_guide/low_level_intro$This guide} has examples of how a graph +@{$guide/low_level_intro$This guide} has examples of how a graph is launched in a @{tf.Session}. ## Session management diff --git a/tensorflow/docs_src/api_guides/python/input_dataset.md b/tensorflow/docs_src/api_guides/python/input_dataset.md index a6e2fc48e0..a6612d1bf7 100644 --- a/tensorflow/docs_src/api_guides/python/input_dataset.md +++ b/tensorflow/docs_src/api_guides/python/input_dataset.md @@ -2,8 +2,7 @@ [TOC] @{tf.data.Dataset} allows you to build complex input pipelines. See the -@{$datasets$programmer's guide} for an in-depth explanation of how to use this -API. +@{$guide/datasets} for an in-depth explanation of how to use this API. ## Reader classes diff --git a/tensorflow/docs_src/api_guides/python/reading_data.md b/tensorflow/docs_src/api_guides/python/reading_data.md index 5bbbfd3216..d7d0904ae2 100644 --- a/tensorflow/docs_src/api_guides/python/reading_data.md +++ b/tensorflow/docs_src/api_guides/python/reading_data.md @@ -16,8 +16,8 @@ There are four methods of getting data into a TensorFlow program: ## `tf.data` API -See the @{$datasets$programmer's guide} for an in-depth explanation of -@{tf.data.Dataset}. The `tf.data` API enables you to extract and preprocess data +See the @{$guide/datasets} for an in-depth explanation of @{tf.data.Dataset}. +The `tf.data` API enables you to extract and preprocess data from different input/file formats, and apply transformations such as batching, shuffling, and mapping functions over the dataset. This is an improved version of the old input methods---feeding and `QueueRunner`---which are described @@ -511,8 +511,8 @@ You can have the train and eval in the same graph in the same process, and share their trained variables or layers. See @{$variables$the shared variables tutorial}. To support the single-graph approach -@{$programmers_guide/datasets$`tf.data`} also supplies -@{$programmers_guide/datasets#creating_an_iterator$advanced iterator types} that +@{$guide/datasets$`tf.data`} also supplies +@{$guide/datasets#creating_an_iterator$advanced iterator types} that that allow the user to change the input pipeline without rebuilding the graph or session. diff --git a/tensorflow/docs_src/deploy/distributed.md b/tensorflow/docs_src/deploy/distributed.md index d7ed6b1deb..8e2c818e39 100644 --- a/tensorflow/docs_src/deploy/distributed.md +++ b/tensorflow/docs_src/deploy/distributed.md @@ -2,7 +2,7 @@ This document shows how to create a cluster of TensorFlow servers, and how to distribute a computation graph across that cluster. We assume that you are -familiar with the @{$programmers_guide/low_level_intro$basic concepts} of +familiar with the @{$guide/low_level_intro$basic concepts} of writing low level TensorFlow programs. ## Hello distributed TensorFlow! diff --git a/tensorflow/docs_src/extend/architecture.md b/tensorflow/docs_src/extend/architecture.md index c8f522a03a..84435a57f2 100644 --- a/tensorflow/docs_src/extend/architecture.md +++ b/tensorflow/docs_src/extend/architecture.md @@ -7,9 +7,8 @@ learning models and system-level optimizations. This document describes the system architecture that makes this combination of scale and flexibility possible. It assumes that you have basic familiarity with TensorFlow programming concepts such as the computation graph, operations, -and sessions. See @{$programmers_guide/low_level_intro$this document} -for an introduction to these topics. Some familiarity -with @{$distributed$distributed TensorFlow} +and sessions. See @{$guide/low_level_intro$this document} for an introduction to +these topics. Some familiarity with @{$distributed$distributed TensorFlow} will also be helpful. This document is for developers who want to extend TensorFlow in some way not diff --git a/tensorflow/docs_src/get_started/_index.yaml b/tensorflow/docs_src/get_started/_index.yaml index af255a482d..277fc852fb 100644 --- a/tensorflow/docs_src/get_started/_index.yaml +++ b/tensorflow/docs_src/get_started/_index.yaml @@ -74,7 +74,7 @@ landing_page: The high-level Keras API provides building blocks to create and train deep learning models. Start with these beginner-friendly notebook examples, then read the - TensorFlow Keras guide. + TensorFlow Keras guide.

  1. Basic classification
    2. @@ -85,7 +85,7 @@ landing_page:
- Read the Keras guide + Read the Keras guide
- classname: tfo-landing-row-item-code-block @@ -123,7 +123,7 @@ landing_page:

Eager execution provides an imperative, define-by-run interface for advanced operations. Write custom layers, forward passes, and training loops with auto‑differentiation. Start with - these notebooks, then read the eager execution guide. + these notebooks, then read the eager execution guide.

  1. @@ -165,7 +165,7 @@ landing_page:
- Read the eager execution guide + Read the eager execution guide
- custom_html: > @@ -177,7 +177,7 @@ landing_page:

Estimators can train large models on multiple machines in a production environment. Try the examples below and read the - Estimators guide. + Estimators guide.

  1. How to build a simple text classifier with TF-Hub
    2. @@ -186,7 +186,7 @@ landing_page:
- Read the Estimators guide + Read the Estimators guide
diff --git a/tensorflow/docs_src/get_started/next_steps.md b/tensorflow/docs_src/get_started/next_steps.md index 79c0ef3346..6318a39c6c 100644 --- a/tensorflow/docs_src/get_started/next_steps.md +++ b/tensorflow/docs_src/get_started/next_steps.md @@ -2,9 +2,9 @@ ## Learn more about TensorFlow -* The [TensorFlow Guide](/programmers_guide) includes usage guides for the +* The [TensorFlow Guide](/guide) includes usage guides for the high-level APIs, as well as advanced TensorFlow operations. -* [Premade Estimators](/programmers_guide/premade_estimators) are designed to +* [Premade Estimators](/guide/premade_estimators) are designed to get results out of the box. Use TensorFlow without building your own models. * [TensorFlow.js](https://js.tensorflow.org/) allows web developers to train and deploy ML models in the browser and using Node.js. diff --git a/tensorflow/docs_src/guide/checkpoints.md b/tensorflow/docs_src/guide/checkpoints.md new file mode 100644 index 0000000000..dfb2626b86 --- /dev/null +++ b/tensorflow/docs_src/guide/checkpoints.md @@ -0,0 +1,238 @@ +# Checkpoints + +This document examines how to save and restore TensorFlow models built with +Estimators. TensorFlow provides two model formats: + +* checkpoints, which is a format dependent on the code that created + the model. +* SavedModel, which is a format independent of the code that created + the model. + +This document focuses on checkpoints. For details on `SavedModel`, see the +@{$saved_model$Saving and Restoring} guide. + + +## Sample code + +This document relies on the same +[Iris classification example](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py) detailed in @{$premade_estimators$Getting Started with TensorFlow}. +To download and access the example, invoke the following two commands: + +```shell +git clone https://github.com/tensorflow/models/ +cd models/samples/core/get_started +``` + +Most of the code snippets in this document are minor variations +on `premade_estimator.py`. + + +## Saving partially-trained models + +Estimators automatically write the following to disk: + +* **checkpoints**, which are versions of the model created during training. +* **event files**, which contain information that + [TensorBoard](https://developers.google.com/machine-learning/glossary/#TensorBoard) + uses to create visualizations. + +To specify the top-level directory in which the Estimator stores its +information, assign a value to the optional `model_dir` argument of *any* +`Estimator`'s constructor. +Taking `DNNClassifier` as an example, +the following code sets the `model_dir` +argument to the `models/iris` directory: + +```python +classifier = tf.estimator.DNNClassifier( + feature_columns=my_feature_columns, + hidden_units=[10, 10], + n_classes=3, + model_dir='models/iris') +``` + +Suppose you call the Estimator's `train` method. For example: + + +```python +classifier.train( + input_fn=lambda:train_input_fn(train_x, train_y, batch_size=100), + steps=200) +``` + +As suggested by the following diagrams, the first call to `train` +adds checkpoints and other files to the `model_dir` directory: + +
+ +
+
+The first call to train(). +
+ + +To see the objects in the created `model_dir` directory on a +UNIX-based system, just call `ls` as follows: + +```none +$ ls -1 models/iris +checkpoint +events.out.tfevents.timestamp.hostname +graph.pbtxt +model.ckpt-1.data-00000-of-00001 +model.ckpt-1.index +model.ckpt-1.meta +model.ckpt-200.data-00000-of-00001 +model.ckpt-200.index +model.ckpt-200.meta +``` + +The preceding `ls` command shows that the Estimator created checkpoints +at steps 1 (the start of training) and 200 (the end of training). + + +### Default checkpoint directory + +If you don't specify `model_dir` in an Estimator's constructor, the Estimator +writes checkpoint files to a temporary directory chosen by Python's +[tempfile.mkdtemp](https://docs.python.org/3/library/tempfile.html#tempfile.mkdtemp) +function. For example, the following Estimator constructor does *not* specify +the `model_dir` argument: + +```python +classifier = tf.estimator.DNNClassifier( + feature_columns=my_feature_columns, + hidden_units=[10, 10], + n_classes=3) + +print(classifier.model_dir) +``` + +The `tempfile.mkdtemp` function picks a secure, temporary directory +appropriate for your operating system. For example, a typical temporary +directory on macOS might be something like the following: + +```None +/var/folders/0s/5q9kfzfj3gx2knj0vj8p68yc00dhcr/T/tmpYm1Rwa +``` + +### Checkpointing Frequency + +By default, the Estimator saves +[checkpoints](https://developers.google.com/machine-learning/glossary/#checkpoint) +in the `model_dir` according to the following schedule: + +* Writes a checkpoint every 10 minutes (600 seconds). +* Writes a checkpoint when the `train` method starts (first iteration) + and completes (final iteration). +* Retains only the 5 most recent checkpoints in the directory. + +You may alter the default schedule by taking the following steps: + +1. Create a @{tf.estimator.RunConfig$`RunConfig`} object that defines the + desired schedule. +2. When instantiating the Estimator, pass that `RunConfig` object to the + Estimator's `config` argument. + +For example, the following code changes the checkpointing schedule to every +20 minutes and retains the 10 most recent checkpoints: + +```python +my_checkpointing_config = tf.estimator.RunConfig( + save_checkpoints_secs = 20*60, # Save checkpoints every 20 minutes. + keep_checkpoint_max = 10, # Retain the 10 most recent checkpoints. +) + +classifier = tf.estimator.DNNClassifier( + feature_columns=my_feature_columns, + hidden_units=[10, 10], + n_classes=3, + model_dir='models/iris', + config=my_checkpointing_config) +``` + +## Restoring your model + +The first time you call an Estimator's `train` method, TensorFlow saves a +checkpoint to the `model_dir`. Each subsequent call to the Estimator's +`train`, `evaluate`, or `predict` method causes the following: + +1. The Estimator builds the model's + [graph](https://developers.google.com/machine-learning/glossary/#graph) + by running the `model_fn()`. (For details on the `model_fn()`, see + @{$custom_estimators$Creating Custom Estimators.}) +2. The Estimator initializes the weights of the new model from the data + stored in the most recent checkpoint. + +In other words, as the following illustration suggests, once checkpoints +exist, TensorFlow rebuilds the model each time you call `train()`, +`evaluate()`, or `predict()`. + +
+ +
+
+Subsequent calls to train(), evaluate(), or predict() +
+ + +### Avoiding a bad restoration + +Restoring a model's state from a checkpoint only works if the model +and checkpoint are compatible. For example, suppose you trained a +`DNNClassifier` Estimator containing two hidden layers, +each having 10 nodes: + +```python +classifier = tf.estimator.DNNClassifier( + feature_columns=feature_columns, + hidden_units=[10, 10], + n_classes=3, + model_dir='models/iris') + +classifier.train( + input_fn=lambda:train_input_fn(train_x, train_y, batch_size=100), + steps=200) +``` + +After training (and, therefore, after creating checkpoints in `models/iris`), +imagine that you changed the number of neurons in each hidden layer from 10 to +20 and then attempted to retrain the model: + +``` python +classifier2 = tf.estimator.DNNClassifier( + feature_columns=my_feature_columns, + hidden_units=[20, 20], # Change the number of neurons in the model. + n_classes=3, + model_dir='models/iris') + +classifier.train( + input_fn=lambda:train_input_fn(train_x, train_y, batch_size=100), + steps=200) +``` + +Since the state in the checkpoint is incompatible with the model described +in `classifier2`, retraining fails with the following error: + +```None +... +InvalidArgumentError (see above for traceback): tensor_name = +dnn/hiddenlayer_1/bias/t_0/Adagrad; shape in shape_and_slice spec [10] +does not match the shape stored in checkpoint: [20] +``` + +To run experiments in which you train and compare slightly different +versions of a model, save a copy of the code that created each +`model_dir`, possibly by creating a separate git branch for each version. +This separation will keep your checkpoints recoverable. + +## Summary + +Checkpoints provide an easy automatic mechanism for saving and restoring +models created by Estimators. + +See the @{$saved_model$Saving and Restoring} guide for details about: + +* Saving and restoring models using low-level TensorFlow APIs. +* Exporting and importing models in the SavedModel format, which is a + language-neutral, recoverable, serialization format. diff --git a/tensorflow/docs_src/guide/custom_estimators.md b/tensorflow/docs_src/guide/custom_estimators.md new file mode 100644 index 0000000000..fb20b35c12 --- /dev/null +++ b/tensorflow/docs_src/guide/custom_estimators.md @@ -0,0 +1,602 @@ + +# Creating Custom Estimators + +This document introduces custom Estimators. In particular, this document +demonstrates how to create a custom @{tf.estimator.Estimator$Estimator} that +mimics the behavior of the pre-made Estimator +@{tf.estimator.DNNClassifier$`DNNClassifier`} in solving the Iris problem. See +the @{$premade_estimators$Pre-Made Estimators chapter} for details +on the Iris problem. + +To download and access the example code invoke the following two commands: + +```shell +git clone https://github.com/tensorflow/models/ +cd models/samples/core/get_started +``` + +In this document we will be looking at +[`custom_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/custom_estimator.py). +You can run it with the following command: + +```bsh +python custom_estimator.py +``` + +If you are feeling impatient, feel free to compare and contrast +[`custom_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/custom_estimator.py) +with +[`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py). +(which is in the same directory). + + + +## Pre-made vs. custom + +As the following figure shows, pre-made Estimators are subclasses of the +@{tf.estimator.Estimator} base class, while custom Estimators are an instance +of tf.estimator.Estimator: + +
+Premade estimators are sub-classes of `Estimator`. Custom Estimators are usually (direct) instances of `Estimator` +
+
+Pre-made and custom Estimators are all Estimators. +
+ +Pre-made Estimators are fully baked. Sometimes though, you need more control +over an Estimator's behavior. That's where custom Estimators come in. You can +create a custom Estimator to do just about anything. If you want hidden layers +connected in some unusual fashion, write a custom Estimator. If you want to +calculate a unique +[metric](https://developers.google.com/machine-learning/glossary/#metric) +for your model, write a custom Estimator. Basically, if you want an Estimator +optimized for your specific problem, write a custom Estimator. + +A model function (or `model_fn`) implements the ML algorithm. The +only difference between working with pre-made Estimators and custom Estimators +is: + +* With pre-made Estimators, someone already wrote the model function for you. +* With custom Estimators, you must write the model function. + +Your model function could implement a wide range of algorithms, defining all +sorts of hidden layers and metrics. Like input functions, all model functions +must accept a standard group of input parameters and return a standard group of +output values. Just as input functions can leverage the Dataset API, model +functions can leverage the Layers API and the Metrics API. + +Let's see how to solve the Iris problem with a custom Estimator. A quick +reminder--here's the organization of the Iris model that we're trying to mimic: + +
+A diagram of the network architecture: Inputs, 2 hidden layers, and outputs +
+
+Our implementation of Iris contains four features, two hidden layers, +and a logits output layer. +
+ +## Write an Input function + +Our custom Estimator implementation uses the same input function as our +@{$premade_estimators$pre-made Estimator implementation}, from +[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py). +Namely: + +```python +def train_input_fn(features, labels, batch_size): + """An input function for training""" + # Convert the inputs to a Dataset. + dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) + + # Shuffle, repeat, and batch the examples. + dataset = dataset.shuffle(1000).repeat().batch(batch_size) + + # Return the read end of the pipeline. + return dataset.make_one_shot_iterator().get_next() +``` + +This input function builds an input pipeline that yields batches of +`(features, labels)` pairs, where `features` is a dictionary features. + +## Create feature columns + +As detailed in the @{$premade_estimators$Premade Estimators} and +@{$feature_columns$Feature Columns} chapters, you must define +your model's feature columns to specify how the model should use each feature. +Whether working with pre-made Estimators or custom Estimators, you define +feature columns in the same fashion. + +The following code creates a simple `numeric_column` for each input feature, +indicating that the value of the input feature should be used directly as an +input to the model: + +```python +# Feature columns describe how to use the input. +my_feature_columns = [] +for key in train_x.keys(): + my_feature_columns.append(tf.feature_column.numeric_column(key=key)) +``` + +## Write a model function + +The model function we'll use has the following call signature: + +```python +def my_model_fn( + features, # This is batch_features from input_fn + labels, # This is batch_labels from input_fn + mode, # An instance of tf.estimator.ModeKeys + params): # Additional configuration +``` + +The first two arguments are the batches of features and labels returned from +the input function; that is, `features` and `labels` are the handles to the +data your model will use. The `mode` argument indicates whether the caller is +requesting training, predicting, or evaluation. + +The caller may pass `params` to an Estimator's constructor. Any `params` passed +to the constructor are in turn passed on to the `model_fn`. In +[`custom_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/custom_estimator.py) +the following lines create the estimator and set the params to configure the +model. This configuration step is similar to how we configured the @{tf.estimator.DNNClassifier} in +@{$premade_estimators}. + +```python +classifier = tf.estimator.Estimator( + model_fn=my_model, + params={ + 'feature_columns': my_feature_columns, + # Two hidden layers of 10 nodes each. + 'hidden_units': [10, 10], + # The model must choose between 3 classes. + 'n_classes': 3, + }) +``` + +To implement a typical model function, you must do the following: + +* [Define the model](#define_the_model). +* Specify additional calculations for each of + the [three different modes](#modes): + * [Predict](#predict) + * [Evaluate](#evaluate) + * [Train](#train) + +## Define the model + +The basic deep neural network model must define the following three sections: + +* An [input layer](https://developers.google.com/machine-learning/glossary/#input_layer) +* One or more [hidden layers](https://developers.google.com/machine-learning/glossary/#hidden_layer) +* An [output layer](https://developers.google.com/machine-learning/glossary/#output_layer) + +### Define the input layer + +The first line of the `model_fn` calls @{tf.feature_column.input_layer} to +convert the feature dictionary and `feature_columns` into input for your model, +as follows: + +```python + # Use `input_layer` to apply the feature columns. + net = tf.feature_column.input_layer(features, params['feature_columns']) +``` + +The preceding line applies the transformations defined by your feature columns, +creating the model's input layer. + +
+A diagram of the input layer, in this case a 1:1 mapping from raw-inputs to features. +
+ + +### Hidden Layers + +If you are creating a deep neural network, you must define one or more hidden +layers. The Layers API provides a rich set of functions to define all types of +hidden layers, including convolutional, pooling, and dropout layers. For Iris, +we're simply going to call @{tf.layers.dense} to create hidden layers, with +dimensions defined by `params['hidden_layers']`. In a `dense` layer each node +is connected to every node in the preceding layer. Here's the relevant code: + +``` python + # Build the hidden layers, sized according to the 'hidden_units' param. + for units in params['hidden_units']: + net = tf.layers.dense(net, units=units, activation=tf.nn.relu) +``` + +* The `units` parameter defines the number of output neurons in a given layer. +* The `activation` parameter defines the [activation function](https://developers.google.com/machine-learning/glossary/#activation_function) — + [Relu](https://developers.google.com/machine-learning/glossary/#ReLU) in this + case. + +The variable `net` here signifies the current top layer of the network. During +the first iteration, `net` signifies the input layer. On each loop iteration +`tf.layers.dense` creates a new layer, which takes the previous layer's output +as its input, using the variable `net`. + +After creating two hidden layers, our network looks as follows. For +simplicity, the figure does not show all the units in each layer. + +
+The input layer with two hidden layers added. +
+ +Note that @{tf.layers.dense} provides many additional capabilities, including +the ability to set a multitude of regularization parameters. For the sake of +simplicity, though, we're going to simply accept the default values of the +other parameters. + +### Output Layer + +We'll define the output layer by calling @{tf.layers.dense} yet again, this +time without an activation function: + +```python + # Compute logits (1 per class). + logits = tf.layers.dense(net, params['n_classes'], activation=None) +``` + +Here, `net` signifies the final hidden layer. Therefore, the full set of layers +is now connected as follows: + +
+A logit output layer connected to the top hidden layer +
+
+The final hidden layer feeds into the output layer. +
+ +When defining an output layer, the `units` parameter specifies the number of +outputs. So, by setting `units` to `params['n_classes']`, the model produces +one output value per class. Each element of the output vector will contain the +score, or "logit", calculated for the associated class of Iris: Setosa, +Versicolor, or Virginica, respectively. + +Later on, these logits will be transformed into probabilities by the +@{tf.nn.softmax} function. + +## Implement training, evaluation, and prediction {#modes} + +The final step in creating a model function is to write branching code that +implements prediction, evaluation, and training. + +The model function gets invoked whenever someone calls the Estimator's `train`, +`evaluate`, or `predict` methods. Recall that the signature for the model +function looks like this: + +``` python +def my_model_fn( + features, # This is batch_features from input_fn + labels, # This is batch_labels from input_fn + mode, # An instance of tf.estimator.ModeKeys, see below + params): # Additional configuration +``` + +Focus on that third argument, mode. As the following table shows, when someone +calls `train`, `evaluate`, or `predict`, the Estimator framework invokes your model +function with the mode parameter set as follows: + +| Estimator method | Estimator Mode | +|:---------------------------------|:------------------| +|@{tf.estimator.Estimator.train$`train()`} |@{tf.estimator.ModeKeys.TRAIN$`ModeKeys.TRAIN`} | +|@{tf.estimator.Estimator.evaluate$`evaluate()`} |@{tf.estimator.ModeKeys.EVAL$`ModeKeys.EVAL`} | +|@{tf.estimator.Estimator.predict$`predict()`}|@{tf.estimator.ModeKeys.PREDICT$`ModeKeys.PREDICT`} | + +For example, suppose you instantiate a custom Estimator to generate an object +named `classifier`. Then, you make the following call: + +``` python +classifier = tf.estimator.Estimator(...) +classifier.train(input_fn=lambda: my_input_fn(FILE_TRAIN, True, 500)) +``` +The Estimator framework then calls your model function with mode set to +`ModeKeys.TRAIN`. + +Your model function must provide code to handle all three of the mode values. +For each mode value, your code must return an instance of +`tf.estimator.EstimatorSpec`, which contains the information the caller +requires. Let's examine each mode. + +### Predict + +When the Estimator's `predict` method is called, the `model_fn` receives +`mode = ModeKeys.PREDICT`. In this case, the model function must return a +`tf.estimator.EstimatorSpec` containing the prediction. + +The model must have been trained prior to making a prediction. The trained model +is stored on disk in the `model_dir` directory established when you +instantiated the Estimator. + +The code to generate the prediction for this model looks as follows: + +```python +# Compute predictions. +predicted_classes = tf.argmax(logits, 1) +if mode == tf.estimator.ModeKeys.PREDICT: + predictions = { + 'class_ids': predicted_classes[:, tf.newaxis], + 'probabilities': tf.nn.softmax(logits), + 'logits': logits, + } + return tf.estimator.EstimatorSpec(mode, predictions=predictions) +``` +The prediction dictionary contains everything that your model returns when run +in prediction mode. + +
+Additional outputs added to the output layer. +
+ +The `predictions` holds the following three key/value pairs: + +* `class_ids` holds the class id (0, 1, or 2) representing the model's + prediction of the most likely species for this example. +* `probabilities` holds the three probabilities (in this example, 0.02, 0.95, + and 0.03) +* `logit` holds the raw logit values (in this example, -1.3, 2.6, and -0.9) + +We return that dictionary to the caller via the `predictions` parameter of the +@{tf.estimator.EstimatorSpec}. The Estimator's +@{tf.estimator.Estimator.predict$`predict`} method will yield these +dictionaries. + +### Calculate the loss + +For both [training](#train) and [evaluation](#evaluate) we need to calculate the +model's loss. This is the +[objective](https://developers.google.com/machine-learning/glossary/#objective) +that will be optimized. + +We can calculate the loss by calling @{tf.losses.sparse_softmax_cross_entropy}. +The value returned by this function will be lowest, approximately 0, +probability of the correct class (at index `label`) is near 1.0. The loss value +returned is progressively larger as the probability of the correct class +decreases. + +This function returns the average over the whole batch. + +```python +# Compute loss. +loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) +``` + +### Evaluate + +When the Estimator's `evaluate` method is called, the `model_fn` receives +`mode = ModeKeys.EVAL`. In this case, the model function must return a +`tf.estimator.EstimatorSpec` containing the model's loss and optionally one +or more metrics. + +Although returning metrics is optional, most custom Estimators do return at +least one metric. TensorFlow provides a Metrics module @{tf.metrics} to +calculate common metrics. For brevity's sake, we'll only return accuracy. The +@{tf.metrics.accuracy} function compares our predictions against the +true values, that is, against the labels provided by the input function. The +@{tf.metrics.accuracy} function requires the labels and predictions to have the +same shape. Here's the call to @{tf.metrics.accuracy}: + +``` python +# Compute evaluation metrics. +accuracy = tf.metrics.accuracy(labels=labels, + predictions=predicted_classes, + name='acc_op') +``` + +The @{tf.estimator.EstimatorSpec$`EstimatorSpec`} returned for evaluation +typically contains the following information: + +* `loss`, which is the model's loss +* `eval_metric_ops`, which is an optional dictionary of metrics. + +So, we'll create a dictionary containing our sole metric. If we had calculated +other metrics, we would have added them as additional key/value pairs to that +same dictionary. Then, we'll pass that dictionary in the `eval_metric_ops` +argument of `tf.estimator.EstimatorSpec`. Here's the code: + +```python +metrics = {'accuracy': accuracy} +tf.summary.scalar('accuracy', accuracy[1]) + +if mode == tf.estimator.ModeKeys.EVAL: + return tf.estimator.EstimatorSpec( + mode, loss=loss, eval_metric_ops=metrics) +``` + +The @{tf.summary.scalar} will make accuracy available to TensorBoard +in both `TRAIN` and `EVAL` modes. (More on this later). + +### Train + +When the Estimator's `train` method is called, the `model_fn` is called +with `mode = ModeKeys.TRAIN`. In this case, the model function must return an +`EstimatorSpec` that contains the loss and a training operation. + +Building the training operation will require an optimizer. We will use +@{tf.train.AdagradOptimizer} because we're mimicking the `DNNClassifier`, which +also uses `Adagrad` by default. The `tf.train` package provides many other +optimizers—feel free to experiment with them. + +Here is the code that builds the optimizer: + +``` python +optimizer = tf.train.AdagradOptimizer(learning_rate=0.1) +``` + +Next, we build the training operation using the optimizer's +@{tf.train.Optimizer.minimize$`minimize`} method on the loss we calculated +earlier. + +The `minimize` method also takes a `global_step` parameter. TensorFlow uses this +parameter to count the number of training steps that have been processed +(to know when to end a training run). Furthermore, the `global_step` is +essential for TensorBoard graphs to work correctly. Simply call +@{tf.train.get_global_step} and pass the result to the `global_step` +argument of `minimize`. + +Here's the code to train the model: + +``` python +train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) +``` + +The @{tf.estimator.EstimatorSpec$`EstimatorSpec`} returned for training +must have the following fields set: + +* `loss`, which contains the value of the loss function. +* `train_op`, which executes a training step. + +Here's our code to call `EstimatorSpec`: + +```python +return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op) +``` + +The model function is now complete. + +## The custom Estimator + +Instantiate the custom Estimator through the Estimator base class as follows: + +```python + # Build 2 hidden layer DNN with 10, 10 units respectively. + classifier = tf.estimator.Estimator( + model_fn=my_model, + params={ + 'feature_columns': my_feature_columns, + # Two hidden layers of 10 nodes each. + 'hidden_units': [10, 10], + # The model must choose between 3 classes. + 'n_classes': 3, + }) +``` +Here the `params` dictionary serves the same purpose as the key-word +arguments of `DNNClassifier`; that is, the `params` dictionary lets you +configure your Estimator without modifying the code in the `model_fn`. + +The rest of the code to train, evaluate, and generate predictions using our +Estimator is the same as in the +@{$premade_estimators$Premade Estimators} chapter. For +example, the following line will train the model: + +```python +# Train the Model. +classifier.train( + input_fn=lambda:iris_data.train_input_fn(train_x, train_y, args.batch_size), + steps=args.train_steps) +``` + +## TensorBoard + +You can view training results for your custom Estimator in TensorBoard. To see +this reporting, start TensorBoard from your command line as follows: + +```bsh +# Replace PATH with the actual path passed as model_dir +tensorboard --logdir=PATH +``` + +Then, open TensorBoard by browsing to: [http://localhost:6006](http://localhost:6006) + +All the pre-made Estimators automatically log a lot of information to +TensorBoard. With custom Estimators, however, TensorBoard only provides one +default log (a graph of the loss) plus the information you explicitly tell +TensorBoard to log. For the custom Estimator you just created, TensorBoard +generates the following: + +
+ +Accuracy, 'scalar' graph from tensorboard + +loss 'scalar' graph from tensorboard + +steps/second 'scalar' graph from tensorboard +
+ +
+TensorBoard displays three graphs. +
+ + +In brief, here's what the three graphs tell you: + +* global_step/sec: A performance indicator showing how many batches (gradient + updates) we processed per second as the model trains. + +* loss: The loss reported. + +* accuracy: The accuracy is recorded by the following two lines: + + * `eval_metric_ops={'my_accuracy': accuracy}`, during evaluation. + * `tf.summary.scalar('accuracy', accuracy[1])`, during training. + +These tensorboard graphs are one of the main reasons it's important to pass a +`global_step` to your optimizer's `minimize` method. The model can't record +the x-coordinate for these graphs without it. + +Note the following in the `my_accuracy` and `loss` graphs: + +* The orange line represents training. +* The blue dot represents evaluation. + +During training, summaries (the orange line) are recorded periodically as +batches are processed, which is why it becomes a graph spanning x-axis range. + +By contrast, evaluation produces only a single point on the graph for each call +to `evaluate`. This point contains the average over the entire evaluation call. +This has no width on the graph as it is evaluated entirely from the model state +at a particular training step (from a single checkpoint). + +As suggested in the following figure, you may see and also selectively +disable/enable the reporting using the controls on the left side. + +
+Check-boxes allowing the user to select which runs are shown. +
+
+Enable or disable reporting. +
+ + +## Summary + +Although pre-made Estimators can be an effective way to quickly create new +models, you will often need the additional flexibility that custom Estimators +provide. Fortunately, pre-made and custom Estimators follow the same +programming model. The only practical difference is that you must write a model +function for custom Estimators; everything else is the same. + +For more details, be sure to check out: + +* The + [official TensorFlow implementation of MNIST](https://github.com/tensorflow/models/tree/master/official/mnist), + which uses a custom estimator. +* The TensorFlow + [official models repository](https://github.com/tensorflow/models/tree/master/official), + which contains more curated examples using custom estimators. +* This [TensorBoard video](https://youtu.be/eBbEDRsCmv4), which introduces + TensorBoard. +* The @{$low_level_intro$Low Level Introduction}, which demonstrates + how to experiment directly with TensorFlow's low level APIs, making debugging + easier. diff --git a/tensorflow/docs_src/guide/datasets.md b/tensorflow/docs_src/guide/datasets.md new file mode 100644 index 0000000000..8b69860a68 --- /dev/null +++ b/tensorflow/docs_src/guide/datasets.md @@ -0,0 +1,823 @@ +# Importing Data + +The @{tf.data} API enables you to build complex input pipelines from +simple, reusable pieces. For example, the pipeline for an image model might +aggregate data from files in a distributed file system, apply random +perturbations to each image, and merge randomly selected images into a batch +for training. The pipeline for a text model might involve extracting symbols +from raw text data, converting them to embedding identifiers with a lookup +table, and batching together sequences of different lengths. The `tf.data` API +makes it easy to deal with large amounts of data, different data formats, and +complicated transformations. + +The `tf.data` API introduces two new abstractions to TensorFlow: + +* A `tf.data.Dataset` represents a sequence of elements, in which + each element contains one or more `Tensor` objects. For example, in an image + pipeline, an element might be a single training example, with a pair of + tensors representing the image data and a label. There are two distinct + ways to create a dataset: + + * Creating a **source** (e.g. `Dataset.from_tensor_slices()`) constructs a + dataset from + one or more `tf.Tensor` objects. + + * Applying a **transformation** (e.g. `Dataset.batch()`) constructs a dataset + from one or more `tf.data.Dataset` objects. + +* A `tf.data.Iterator` provides the main way to extract elements from a + dataset. The operation returned by `Iterator.get_next()` yields the next + element of a `Dataset` when executed, and typically acts as the interface + between input pipeline code and your model. The simplest iterator is a + "one-shot iterator", which is associated with a particular `Dataset` and + iterates through it once. For more sophisticated uses, the + `Iterator.initializer` operation enables you to reinitialize and parameterize + an iterator with different datasets, so that you can, for example, iterate + over training and validation data multiple times in the same program. + +## Basic mechanics + +This section of the guide describes the fundamentals of creating different kinds +of `Dataset` and `Iterator` objects, and how to extract data from them. + +To start an input pipeline, you must define a *source*. For example, to +construct a `Dataset` from some tensors in memory, you can use +`tf.data.Dataset.from_tensors()` or +`tf.data.Dataset.from_tensor_slices()`. Alternatively, if your input +data are on disk in the recommended TFRecord format, you can construct a +`tf.data.TFRecordDataset`. + +Once you have a `Dataset` object, you can *transform* it into a new `Dataset` by +chaining method calls on the `tf.data.Dataset` object. For example, you +can apply per-element transformations such as `Dataset.map()` (to apply a +function to each element), and multi-element transformations such as +`Dataset.batch()`. See the documentation for @{tf.data.Dataset} +for a complete list of transformations. + +The most common way to consume values from a `Dataset` is to make an +**iterator** object that provides access to one element of the dataset at a time +(for example, by calling `Dataset.make_one_shot_iterator()`). A +`tf.data.Iterator` provides two operations: `Iterator.initializer`, +which enables you to (re)initialize the iterator's state; and +`Iterator.get_next()`, which returns `tf.Tensor` objects that correspond to the +symbolic next element. Depending on your use case, you might choose a different +type of iterator, and the options are outlined below. + +### Dataset structure + +A dataset comprises elements that each have the same structure. An element +contains one or more `tf.Tensor` objects, called *components*. Each component +has a `tf.DType` representing the type of elements in the tensor, and a +`tf.TensorShape` representing the (possibly partially specified) static shape of +each element. The `Dataset.output_types` and `Dataset.output_shapes` properties +allow you to inspect the inferred types and shapes of each component of a +dataset element. The *nested structure* of these properties map to the structure +of an element, which may be a single tensor, a tuple of tensors, or a nested +tuple of tensors. For example: + +```python +dataset1 = tf.data.Dataset.from_tensor_slices(tf.random_uniform([4, 10])) +print(dataset1.output_types) # ==> "tf.float32" +print(dataset1.output_shapes) # ==> "(10,)" + +dataset2 = tf.data.Dataset.from_tensor_slices( + (tf.random_uniform([4]), + tf.random_uniform([4, 100], maxval=100, dtype=tf.int32))) +print(dataset2.output_types) # ==> "(tf.float32, tf.int32)" +print(dataset2.output_shapes) # ==> "((), (100,))" + +dataset3 = tf.data.Dataset.zip((dataset1, dataset2)) +print(dataset3.output_types) # ==> (tf.float32, (tf.float32, tf.int32)) +print(dataset3.output_shapes) # ==> "(10, ((), (100,)))" +``` + +It is often convenient to give names to each component of an element, for +example if they represent different features of a training example. In addition +to tuples, you can use `collections.namedtuple` or a dictionary mapping strings +to tensors to represent a single element of a `Dataset`. + +```python +dataset = tf.data.Dataset.from_tensor_slices( + {"a": tf.random_uniform([4]), + "b": tf.random_uniform([4, 100], maxval=100, dtype=tf.int32)}) +print(dataset.output_types) # ==> "{'a': tf.float32, 'b': tf.int32}" +print(dataset.output_shapes) # ==> "{'a': (), 'b': (100,)}" +``` + +The `Dataset` transformations support datasets of any structure. When using the +`Dataset.map()`, `Dataset.flat_map()`, and `Dataset.filter()` transformations, +which apply a function to each element, the element structure determines the +arguments of the function: + +```python +dataset1 = dataset1.map(lambda x: ...) + +dataset2 = dataset2.flat_map(lambda x, y: ...) + +# Note: Argument destructuring is not available in Python 3. +dataset3 = dataset3.filter(lambda x, (y, z): ...) +``` + +### Creating an iterator + +Once you have built a `Dataset` to represent your input data, the next step is to +create an `Iterator` to access elements from that dataset. The `tf.data` API +currently supports the following iterators, in increasing level of +sophistication: + +* **one-shot**, +* **initializable**, +* **reinitializable**, and +* **feedable**. + +A **one-shot** iterator is the simplest form of iterator, which only supports +iterating once through a dataset, with no need for explicit initialization. +One-shot iterators handle almost all of the cases that the existing queue-based +input pipelines support, but they do not support parameterization. Using the +example of `Dataset.range()`: + +```python +dataset = tf.data.Dataset.range(100) +iterator = dataset.make_one_shot_iterator() +next_element = iterator.get_next() + +for i in range(100): + value = sess.run(next_element) + assert i == value +``` + +Note: Currently, one-shot iterators are the only type that is easily usable +with an `Estimator`. + +An **initializable** iterator requires you to run an explicit +`iterator.initializer` operation before using it. In exchange for this +inconvenience, it enables you to *parameterize* the definition of the dataset, +using one or more `tf.placeholder()` tensors that can be fed when you +initialize the iterator. Continuing the `Dataset.range()` example: + +```python +max_value = tf.placeholder(tf.int64, shape=[]) +dataset = tf.data.Dataset.range(max_value) +iterator = dataset.make_initializable_iterator() +next_element = iterator.get_next() + +# Initialize an iterator over a dataset with 10 elements. +sess.run(iterator.initializer, feed_dict={max_value: 10}) +for i in range(10): + value = sess.run(next_element) + assert i == value + +# Initialize the same iterator over a dataset with 100 elements. +sess.run(iterator.initializer, feed_dict={max_value: 100}) +for i in range(100): + value = sess.run(next_element) + assert i == value +``` + +A **reinitializable** iterator can be initialized from multiple different +`Dataset` objects. For example, you might have a training input pipeline that +uses random perturbations to the input images to improve generalization, and +a validation input pipeline that evaluates predictions on unmodified data. These +pipelines will typically use different `Dataset` objects that have the same +structure (i.e. the same types and compatible shapes for each component). + +```python +# Define training and validation datasets with the same structure. +training_dataset = tf.data.Dataset.range(100).map( + lambda x: x + tf.random_uniform([], -10, 10, tf.int64)) +validation_dataset = tf.data.Dataset.range(50) + +# A reinitializable iterator is defined by its structure. We could use the +# `output_types` and `output_shapes` properties of either `training_dataset` +# or `validation_dataset` here, because they are compatible. +iterator = tf.data.Iterator.from_structure(training_dataset.output_types, + training_dataset.output_shapes) +next_element = iterator.get_next() + +training_init_op = iterator.make_initializer(training_dataset) +validation_init_op = iterator.make_initializer(validation_dataset) + +# Run 20 epochs in which the training dataset is traversed, followed by the +# validation dataset. +for _ in range(20): + # Initialize an iterator over the training dataset. + sess.run(training_init_op) + for _ in range(100): + sess.run(next_element) + + # Initialize an iterator over the validation dataset. + sess.run(validation_init_op) + for _ in range(50): + sess.run(next_element) +``` + +A **feedable** iterator can be used together with @{tf.placeholder} to select +what `Iterator` to use in each call to @{tf.Session.run}, via the familiar +`feed_dict` mechanism. It offers the same functionality as a reinitializable +iterator, but it does not require you to initialize the iterator from the start +of a dataset when you switch between iterators. For example, using the same +training and validation example from above, you can use +@{tf.data.Iterator.from_string_handle} to define a feedable iterator +that allows you to switch between the two datasets: + +```python +# Define training and validation datasets with the same structure. +training_dataset = tf.data.Dataset.range(100).map( + lambda x: x + tf.random_uniform([], -10, 10, tf.int64)).repeat() +validation_dataset = tf.data.Dataset.range(50) + +# A feedable iterator is defined by a handle placeholder and its structure. We +# could use the `output_types` and `output_shapes` properties of either +# `training_dataset` or `validation_dataset` here, because they have +# identical structure. +handle = tf.placeholder(tf.string, shape=[]) +iterator = tf.data.Iterator.from_string_handle( + handle, training_dataset.output_types, training_dataset.output_shapes) +next_element = iterator.get_next() + +# You can use feedable iterators with a variety of different kinds of iterator +# (such as one-shot and initializable iterators). +training_iterator = training_dataset.make_one_shot_iterator() +validation_iterator = validation_dataset.make_initializable_iterator() + +# The `Iterator.string_handle()` method returns a tensor that can be evaluated +# and used to feed the `handle` placeholder. +training_handle = sess.run(training_iterator.string_handle()) +validation_handle = sess.run(validation_iterator.string_handle()) + +# Loop forever, alternating between training and validation. +while True: + # Run 200 steps using the training dataset. Note that the training dataset is + # infinite, and we resume from where we left off in the previous `while` loop + # iteration. + for _ in range(200): + sess.run(next_element, feed_dict={handle: training_handle}) + + # Run one pass over the validation dataset. + sess.run(validation_iterator.initializer) + for _ in range(50): + sess.run(next_element, feed_dict={handle: validation_handle}) +``` + +### Consuming values from an iterator + +The `Iterator.get_next()` method returns one or more `tf.Tensor` objects that +correspond to the symbolic next element of an iterator. Each time these tensors +are evaluated, they take the value of the next element in the underlying +dataset. (Note that, like other stateful objects in TensorFlow, calling +`Iterator.get_next()` does not immediately advance the iterator. Instead you +must use the returned `tf.Tensor` objects in a TensorFlow expression, and pass +the result of that expression to `tf.Session.run()` to get the next elements and +advance the iterator.) + +If the iterator reaches the end of the dataset, executing +the `Iterator.get_next()` operation will raise a `tf.errors.OutOfRangeError`. +After this point the iterator will be in an unusable state, and you must +initialize it again if you want to use it further. + +```python +dataset = tf.data.Dataset.range(5) +iterator = dataset.make_initializable_iterator() +next_element = iterator.get_next() + +# Typically `result` will be the output of a model, or an optimizer's +# training operation. +result = tf.add(next_element, next_element) + +sess.run(iterator.initializer) +print(sess.run(result)) # ==> "0" +print(sess.run(result)) # ==> "2" +print(sess.run(result)) # ==> "4" +print(sess.run(result)) # ==> "6" +print(sess.run(result)) # ==> "8" +try: + sess.run(result) +except tf.errors.OutOfRangeError: + print("End of dataset") # ==> "End of dataset" +``` + +A common pattern is to wrap the "training loop" in a `try`-`except` block: + +```python +sess.run(iterator.initializer) +while True: + try: + sess.run(result) + except tf.errors.OutOfRangeError: + break +``` + +If each element of the dataset has a nested structure, the return value of +`Iterator.get_next()` will be one or more `tf.Tensor` objects in the same +nested structure: + +```python +dataset1 = tf.data.Dataset.from_tensor_slices(tf.random_uniform([4, 10])) +dataset2 = tf.data.Dataset.from_tensor_slices((tf.random_uniform([4]), tf.random_uniform([4, 100]))) +dataset3 = tf.data.Dataset.zip((dataset1, dataset2)) + +iterator = dataset3.make_initializable_iterator() + +sess.run(iterator.initializer) +next1, (next2, next3) = iterator.get_next() +``` + +Note that `next1`, `next2`, and `next3` are tensors produced by the +same op/node (created by `Iterator.get_next()`). Therefore, evaluating *any* of +these tensors will advance the iterator for all components. A typical consumer +of an iterator will include all components in a single expression. + +### Saving iterator state + +The @{tf.contrib.data.make_saveable_from_iterator} function creates a +`SaveableObject` from an iterator, which can be used to save and +restore the current state of the iterator (and, effectively, the whole input +pipeline). A saveable object thus created can be added to @{tf.train.Saver} +variables list or the `tf.GraphKeys.SAVEABLE_OBJECTS` collection for saving and +restoring in the same manner as a @{tf.Variable}. Refer to +@{$saved_model$Saving and Restoring} for details on how to save and restore +variables. + +```python +# Create saveable object from iterator. +saveable = tf.contrib.data.make_saveable_from_iterator(iterator) + +# Save the iterator state by adding it to the saveable objects collection. +tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, saveable) +saver = tf.train.Saver() + +with tf.Session() as sess: + + if should_checkpoint: + saver.save(path_to_checkpoint) + +# Restore the iterator state. +with tf.Session() as sess: + saver.restore(sess, path_to_checkpoint) +``` + +## Reading input data + +### Consuming NumPy arrays + +If all of your input data fit in memory, the simplest way to create a `Dataset` +from them is to convert them to `tf.Tensor` objects and use +`Dataset.from_tensor_slices()`. + +```python +# Load the training data into two NumPy arrays, for example using `np.load()`. +with np.load("/var/data/training_data.npy") as data: + features = data["features"] + labels = data["labels"] + +# Assume that each row of `features` corresponds to the same row as `labels`. +assert features.shape[0] == labels.shape[0] + +dataset = tf.data.Dataset.from_tensor_slices((features, labels)) +``` + +Note that the above code snippet will embed the `features` and `labels` arrays +in your TensorFlow graph as `tf.constant()` operations. This works well for a +small dataset, but wastes memory---because the contents of the array will be +copied multiple times---and can run into the 2GB limit for the `tf.GraphDef` +protocol buffer. + +As an alternative, you can define the `Dataset` in terms of `tf.placeholder()` +tensors, and *feed* the NumPy arrays when you initialize an `Iterator` over the +dataset. + +```python +# Load the training data into two NumPy arrays, for example using `np.load()`. +with np.load("/var/data/training_data.npy") as data: + features = data["features"] + labels = data["labels"] + +# Assume that each row of `features` corresponds to the same row as `labels`. +assert features.shape[0] == labels.shape[0] + +features_placeholder = tf.placeholder(features.dtype, features.shape) +labels_placeholder = tf.placeholder(labels.dtype, labels.shape) + +dataset = tf.data.Dataset.from_tensor_slices((features_placeholder, labels_placeholder)) +# [Other transformations on `dataset`...] +dataset = ... +iterator = dataset.make_initializable_iterator() + +sess.run(iterator.initializer, feed_dict={features_placeholder: features, + labels_placeholder: labels}) +``` + +### Consuming TFRecord data + +The `tf.data` API supports a variety of file formats so that you can process +large datasets that do not fit in memory. For example, the TFRecord file format +is a simple record-oriented binary format that many TensorFlow applications use +for training data. The `tf.data.TFRecordDataset` class enables you to +stream over the contents of one or more TFRecord files as part of an input +pipeline. + +```python +# Creates a dataset that reads all of the examples from two files. +filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] +dataset = tf.data.TFRecordDataset(filenames) +``` + +The `filenames` argument to the `TFRecordDataset` initializer can either be a +string, a list of strings, or a `tf.Tensor` of strings. Therefore if you have +two sets of files for training and validation purposes, you can use a +`tf.placeholder(tf.string)` to represent the filenames, and initialize an +iterator from the appropriate filenames: + +```python +filenames = tf.placeholder(tf.string, shape=[None]) +dataset = tf.data.TFRecordDataset(filenames) +dataset = dataset.map(...) # Parse the record into tensors. +dataset = dataset.repeat() # Repeat the input indefinitely. +dataset = dataset.batch(32) +iterator = dataset.make_initializable_iterator() + +# You can feed the initializer with the appropriate filenames for the current +# phase of execution, e.g. training vs. validation. + +# Initialize `iterator` with training data. +training_filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] +sess.run(iterator.initializer, feed_dict={filenames: training_filenames}) + +# Initialize `iterator` with validation data. +validation_filenames = ["/var/data/validation1.tfrecord", ...] +sess.run(iterator.initializer, feed_dict={filenames: validation_filenames}) +``` + +### Consuming text data + +Many datasets are distributed as one or more text files. The +`tf.data.TextLineDataset` provides an easy way to extract lines from +one or more text files. Given one or more filenames, a `TextLineDataset` will +produce one string-valued element per line of those files. Like a +`TFRecordDataset`, `TextLineDataset` accepts `filenames` as a `tf.Tensor`, so +you can parameterize it by passing a `tf.placeholder(tf.string)`. + +```python +filenames = ["/var/data/file1.txt", "/var/data/file2.txt"] +dataset = tf.data.TextLineDataset(filenames) +``` + +By default, a `TextLineDataset` yields *every* line of each file, which may +not be desirable, for example if the file starts with a header line, or contains +comments. These lines can be removed using the `Dataset.skip()` and +`Dataset.filter()` transformations. To apply these transformations to each +file separately, we use `Dataset.flat_map()` to create a nested `Dataset` for +each file. + +```python +filenames = ["/var/data/file1.txt", "/var/data/file2.txt"] + +dataset = tf.data.Dataset.from_tensor_slices(filenames) + +# Use `Dataset.flat_map()` to transform each file as a separate nested dataset, +# and then concatenate their contents sequentially into a single "flat" dataset. +# * Skip the first line (header row). +# * Filter out lines beginning with "#" (comments). +dataset = dataset.flat_map( + lambda filename: ( + tf.data.TextLineDataset(filename) + .skip(1) + .filter(lambda line: tf.not_equal(tf.substr(line, 0, 1), "#")))) +``` + +### Consuming CSV data + +The CSV file format is a popular format for storing tabular data in plain text. +The @{tf.contrib.data.CsvDataset} class provides a way to extract records from +one or more CSV files that comply with [RFC 4180](https://tools.ietf.org/html/rfc4180). +Given one or more filenames and a list of defaults, a `CsvDataset` will produce +a tuple of elements whose types correspond to the types of the defaults +provided, per CSV record. Like `TFRecordDataset` and `TextLineDataset`, +`CsvDataset` accepts `filenames` as a `tf.Tensor`, so you can parameterize it +by passing a `tf.placeholder(tf.string)`. + +``` +# Creates a dataset that reads all of the records from two CSV files, each with +# eight float columns +filenames = ["/var/data/file1.csv", "/var/data/file2.csv"] +record_defaults = [tf.float32] * 8 # Eight required float columns +dataset = tf.contrib.data.CsvDataset(filenames, record_defaults) +``` + +If some columns are empty, you can provide defaults instead of types. + +``` +# Creates a dataset that reads all of the records from two CSV files, each with +# four float columns which may have missing values +record_defaults = [[0.0]] * 8 +dataset = tf.contrib.data.CsvDataset(filenames, record_defaults) +``` + +By default, a `CsvDataset` yields *every* column of *every* line of the file, +which may not be desirable, for example if the file starts with a header line +that should be ignored, or if some columns are not required in the input. +These lines and fields can be removed with the `header` and `select_cols` +arguments respectively. + +``` +# Creates a dataset that reads all of the records from two CSV files with +# headers, extracting float data from columns 2 and 4. +record_defaults = [[0.0]] * 2 # Only provide defaults for the selected columns +dataset = tf.contrib.data.CsvDataset(filenames, record_defaults, header=True, select_cols=[2,4]) +``` + + +## Preprocessing data with `Dataset.map()` + +The `Dataset.map(f)` transformation produces a new dataset by applying a given +function `f` to each element of the input dataset. It is based on +the +[`map()` function](https://en.wikipedia.org/wiki/Map_(higher-order_function)) +that is commonly applied to lists (and other structures) in functional +programming languages. The function `f` takes the `tf.Tensor` objects that +represent a single element in the input, and returns the `tf.Tensor` objects +that will represent a single element in the new dataset. Its implementation uses +standard TensorFlow operations to transform one element into another. + +This section covers common examples of how to use `Dataset.map()`. + +### Parsing `tf.Example` protocol buffer messages + +Many input pipelines extract `tf.train.Example` protocol buffer messages from a +TFRecord-format file (written, for example, using +`tf.python_io.TFRecordWriter`). Each `tf.train.Example` record contains one or +more "features", and the input pipeline typically converts these features into +tensors. + +```python +# Transforms a scalar string `example_proto` into a pair of a scalar string and +# a scalar integer, representing an image and its label, respectively. +def _parse_function(example_proto): + features = {"image": tf.FixedLenFeature((), tf.string, default_value=""), + "label": tf.FixedLenFeature((), tf.int32, default_value=0)} + parsed_features = tf.parse_single_example(example_proto, features) + return parsed_features["image"], parsed_features["label"] + +# Creates a dataset that reads all of the examples from two files, and extracts +# the image and label features. +filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] +dataset = tf.data.TFRecordDataset(filenames) +dataset = dataset.map(_parse_function) +``` + +### Decoding image data and resizing it + +When training a neural network on real-world image data, it is often necessary +to convert images of different sizes to a common size, so that they may be +batched into a fixed size. + +```python +# Reads an image from a file, decodes it into a dense tensor, and resizes it +# to a fixed shape. +def _parse_function(filename, label): + image_string = tf.read_file(filename) + image_decoded = tf.image.decode_jpeg(image_string) + image_resized = tf.image.resize_images(image_decoded, [28, 28]) + return image_resized, label + +# A vector of filenames. +filenames = tf.constant(["/var/data/image1.jpg", "/var/data/image2.jpg", ...]) + +# `labels[i]` is the label for the image in `filenames[i]. +labels = tf.constant([0, 37, ...]) + +dataset = tf.data.Dataset.from_tensor_slices((filenames, labels)) +dataset = dataset.map(_parse_function) +``` + +### Applying arbitrary Python logic with `tf.py_func()` + +For performance reasons, we encourage you to use TensorFlow operations for +preprocessing your data whenever possible. However, it is sometimes useful to +call upon external Python libraries when parsing your input data. To do so, +invoke, the `tf.py_func()` operation in a `Dataset.map()` transformation. + +```python +import cv2 + +# Use a custom OpenCV function to read the image, instead of the standard +# TensorFlow `tf.read_file()` operation. +def _read_py_function(filename, label): + image_decoded = cv2.imread(filename.decode(), cv2.IMREAD_GRAYSCALE) + return image_decoded, label + +# Use standard TensorFlow operations to resize the image to a fixed shape. +def _resize_function(image_decoded, label): + image_decoded.set_shape([None, None, None]) + image_resized = tf.image.resize_images(image_decoded, [28, 28]) + return image_resized, label + +filenames = ["/var/data/image1.jpg", "/var/data/image2.jpg", ...] +labels = [0, 37, 29, 1, ...] + +dataset = tf.data.Dataset.from_tensor_slices((filenames, labels)) +dataset = dataset.map( + lambda filename, label: tuple(tf.py_func( + _read_py_function, [filename, label], [tf.uint8, label.dtype]))) +dataset = dataset.map(_resize_function) +``` + + + +## Batching dataset elements + +### Simple batching + +The simplest form of batching stacks `n` consecutive elements of a dataset into +a single element. The `Dataset.batch()` transformation does exactly this, with +the same constraints as the `tf.stack()` operator, applied to each component +of the elements: i.e. for each component *i*, all elements must have a tensor +of the exact same shape. + +```python +inc_dataset = tf.data.Dataset.range(100) +dec_dataset = tf.data.Dataset.range(0, -100, -1) +dataset = tf.data.Dataset.zip((inc_dataset, dec_dataset)) +batched_dataset = dataset.batch(4) + +iterator = batched_dataset.make_one_shot_iterator() +next_element = iterator.get_next() + +print(sess.run(next_element)) # ==> ([0, 1, 2, 3], [ 0, -1, -2, -3]) +print(sess.run(next_element)) # ==> ([4, 5, 6, 7], [-4, -5, -6, -7]) +print(sess.run(next_element)) # ==> ([8, 9, 10, 11], [-8, -9, -10, -11]) +``` + +### Batching tensors with padding + +The above recipe works for tensors that all have the same size. However, many +models (e.g. sequence models) work with input data that can have varying size +(e.g. sequences of different lengths). To handle this case, the +`Dataset.padded_batch()` transformation enables you to batch tensors of +different shape by specifying one or more dimensions in which they may be +padded. + +```python +dataset = tf.data.Dataset.range(100) +dataset = dataset.map(lambda x: tf.fill([tf.cast(x, tf.int32)], x)) +dataset = dataset.padded_batch(4, padded_shapes=[None]) + +iterator = dataset.make_one_shot_iterator() +next_element = iterator.get_next() + +print(sess.run(next_element)) # ==> [[0, 0, 0], [1, 0, 0], [2, 2, 0], [3, 3, 3]] +print(sess.run(next_element)) # ==> [[4, 4, 4, 4, 0, 0, 0], + # [5, 5, 5, 5, 5, 0, 0], + # [6, 6, 6, 6, 6, 6, 0], + # [7, 7, 7, 7, 7, 7, 7]] +``` + +The `Dataset.padded_batch()` transformation allows you to set different padding +for each dimension of each component, and it may be variable-length (signified +by `None` in the example above) or constant-length. It is also possible to +override the padding value, which defaults to 0. + + + +## Training workflows + +### Processing multiple epochs + +The `tf.data` API offers two main ways to process multiple epochs of the same +data. + +The simplest way to iterate over a dataset in multiple epochs is to use the +`Dataset.repeat()` transformation. For example, to create a dataset that repeats +its input for 10 epochs: + +```python +filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] +dataset = tf.data.TFRecordDataset(filenames) +dataset = dataset.map(...) +dataset = dataset.repeat(10) +dataset = dataset.batch(32) +``` + +Applying the `Dataset.repeat()` transformation with no arguments will repeat +the input indefinitely. The `Dataset.repeat()` transformation concatenates its +arguments without signaling the end of one epoch and the beginning of the next +epoch. + +If you want to receive a signal at the end of each epoch, you can write a +training loop that catches the `tf.errors.OutOfRangeError` at the end of a +dataset. At that point you might collect some statistics (e.g. the validation +error) for the epoch. + +```python +filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] +dataset = tf.data.TFRecordDataset(filenames) +dataset = dataset.map(...) +dataset = dataset.batch(32) +iterator = dataset.make_initializable_iterator() +next_element = iterator.get_next() + +# Compute for 100 epochs. +for _ in range(100): + sess.run(iterator.initializer) + while True: + try: + sess.run(next_element) + except tf.errors.OutOfRangeError: + break + + # [Perform end-of-epoch calculations here.] +``` + +### Randomly shuffling input data + +The `Dataset.shuffle()` transformation randomly shuffles the input dataset +using a similar algorithm to `tf.RandomShuffleQueue`: it maintains a fixed-size +buffer and chooses the next element uniformly at random from that buffer. + +```python +filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] +dataset = tf.data.TFRecordDataset(filenames) +dataset = dataset.map(...) +dataset = dataset.shuffle(buffer_size=10000) +dataset = dataset.batch(32) +dataset = dataset.repeat() +``` + +### Using high-level APIs + +The @{tf.train.MonitoredTrainingSession} API simplifies many aspects of running +TensorFlow in a distributed setting. `MonitoredTrainingSession` uses the +@{tf.errors.OutOfRangeError} to signal that training has completed, so to use it +with the `tf.data` API, we recommend using +`Dataset.make_one_shot_iterator()`. For example: + +```python +filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] +dataset = tf.data.TFRecordDataset(filenames) +dataset = dataset.map(...) +dataset = dataset.shuffle(buffer_size=10000) +dataset = dataset.batch(32) +dataset = dataset.repeat(num_epochs) +iterator = dataset.make_one_shot_iterator() + +next_example, next_label = iterator.get_next() +loss = model_function(next_example, next_label) + +training_op = tf.train.AdagradOptimizer(...).minimize(loss) + +with tf.train.MonitoredTrainingSession(...) as sess: + while not sess.should_stop(): + sess.run(training_op) +``` + +To use a `Dataset` in the `input_fn` of a @{tf.estimator.Estimator}, we also +recommend using `Dataset.make_one_shot_iterator()`. For example: + +```python +def dataset_input_fn(): + filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] + dataset = tf.data.TFRecordDataset(filenames) + + # Use `tf.parse_single_example()` to extract data from a `tf.Example` + # protocol buffer, and perform any additional per-record preprocessing. + def parser(record): + keys_to_features = { + "image_data": tf.FixedLenFeature((), tf.string, default_value=""), + "date_time": tf.FixedLenFeature((), tf.int64, default_value=""), + "label": tf.FixedLenFeature((), tf.int64, + default_value=tf.zeros([], dtype=tf.int64)), + } + parsed = tf.parse_single_example(record, keys_to_features) + + # Perform additional preprocessing on the parsed data. + image = tf.image.decode_jpeg(parsed["image_data"]) + image = tf.reshape(image, [299, 299, 1]) + label = tf.cast(parsed["label"], tf.int32) + + return {"image_data": image, "date_time": parsed["date_time"]}, label + + # Use `Dataset.map()` to build a pair of a feature dictionary and a label + # tensor for each example. + dataset = dataset.map(parser) + dataset = dataset.shuffle(buffer_size=10000) + dataset = dataset.batch(32) + dataset = dataset.repeat(num_epochs) + iterator = dataset.make_one_shot_iterator() + + # `features` is a dictionary in which each value is a batch of values for + # that feature; `labels` is a batch of labels. + features, labels = iterator.get_next() + return features, labels +``` diff --git a/tensorflow/docs_src/guide/datasets_for_estimators.md b/tensorflow/docs_src/guide/datasets_for_estimators.md new file mode 100644 index 0000000000..b04af78cd8 --- /dev/null +++ b/tensorflow/docs_src/guide/datasets_for_estimators.md @@ -0,0 +1,387 @@ +# Datasets for Estimators + +The @{tf.data} module contains a collection of classes that allows you to +easily load data, manipulate it, and pipe it into your model. This document +introduces the API by walking through two simple examples: + +* Reading in-memory data from numpy arrays. +* Reading lines from a csv file. + + + +## Basic input + +Taking slices from an array is the simplest way to get started with `tf.data`. + +The @{$premade_estimators$Premade Estimators} chapter describes +the following `train_input_fn`, from +[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py), +to pipe the data into the Estimator: + +``` python +def train_input_fn(features, labels, batch_size): + """An input function for training""" + # Convert the inputs to a Dataset. + dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) + + # Shuffle, repeat, and batch the examples. + dataset = dataset.shuffle(1000).repeat().batch(batch_size) + + # Return the dataset. + return dataset +``` + +Let's look at this more closely. + +### Arguments + +This function expects three arguments. Arguments expecting an "array" can +accept nearly anything that can be converted to an array with `numpy.array`. +One exception is +[`tuple`](https://docs.python.org/3/tutorial/datastructures.html#tuples-and-sequences) +which, as we will see, has special meaning for `Datasets`. + +* `features`: A `{'feature_name':array}` dictionary (or + [`DataFrame`](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)) + containing the raw input features. +* `labels` : An array containing the + [label](https://developers.google.com/machine-learning/glossary/#label) + for each example. +* `batch_size` : An integer indicating the desired batch size. + +In [`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py) +we retrieved the Iris data using the `iris_data.load_data()` function. +You can run it, and unpack the results as follows: + +``` python +import iris_data + +# Fetch the data +train, test = iris_data.load_data() +features, labels = train +``` + +Then we passed this data to the input function, with a line similar to this: + +``` python +batch_size=100 +iris_data.train_input_fn(features, labels, batch_size) +``` + +Let's walk through the `train_input_fn()`. + +### Slices + +The function starts by using the @{tf.data.Dataset.from_tensor_slices} function +to create a @{tf.data.Dataset} representing slices of the array. The array is +sliced across the first dimension. For example, an array containing the +@{$tutorials/layers$mnist training data} has a shape of `(60000, 28, 28)`. +Passing this to `from_tensor_slices` returns a `Dataset` object containing +60000 slices, each one a 28x28 image. + +The code that returns this `Dataset` is as follows: + +``` python +train, test = tf.keras.datasets.mnist.load_data() +mnist_x, mnist_y = train + +mnist_ds = tf.data.Dataset.from_tensor_slices(mnist_x) +print(mnist_ds) +``` + +This will print the following line, showing the +@{$guide/tensors#shapes$shapes} and +@{$guide/tensors#data_types$types} of the items in +the dataset. Note that a `Dataset` does not know how many items it contains. + +``` None + +``` + +The `Dataset` above represents a simple collection of arrays, but datasets are +much more powerful than this. A `Dataset` can transparently handle any nested +combination of dictionaries or tuples (or +[`namedtuple`](https://docs.python.org/2/library/collections.html#collections.namedtuple) +). + +For example after converting the iris `features` +to a standard python dictionary, you can then convert the dictionary of arrays +to a `Dataset` of dictionaries as follows: + +``` python +dataset = tf.data.Dataset.from_tensor_slices(dict(features)) +print(dataset) +``` +``` None + +``` + +Here we see that when a `Dataset` contains structured elements, the `shapes` +and `types` of the `Dataset` take on the same structure. This dataset contains +dictionaries of @{$guide/tensors#rank$scalars}, all of type +`tf.float64`. + +The first line of the iris `train_input_fn` uses the same functionality, but +adds another level of structure. It creates a dataset containing +`(features_dict, label)` pairs. + +The following code shows that the label is a scalar with type `int64`: + +``` python +# Convert the inputs to a Dataset. +dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) +print(dataset) +``` +``` + +``` + +### Manipulation + +Currently the `Dataset` would iterate over the data once, in a fixed order, and +only produce a single element at a time. It needs further processing before it +can be used for training. Fortunately, the `tf.data.Dataset` class provides +methods to better prepare the data for training. The next line of the input +function takes advantage of several of these methods: + +``` python +# Shuffle, repeat, and batch the examples. +dataset = dataset.shuffle(1000).repeat().batch(batch_size) +``` + +The @{tf.data.Dataset.shuffle$`shuffle`} method uses a fixed-size buffer to +shuffle the items as they pass through. In this case the `buffer_size` is +greater than the number of examples in the `Dataset`, ensuring that the data is +completely shuffled (The Iris data set only contains 150 examples). + +The @{tf.data.Dataset.repeat$`repeat`} method restarts the `Dataset` when +it reaches the end. To limit the number of epochs, set the `count` argument. + +The @{tf.data.Dataset.batch$`batch`} method collects a number of examples and +stacks them, to create batches. This adds a dimension to their shape. The new +dimension is added as the first dimension. The following code uses +the `batch` method on the MNIST `Dataset`, from earlier. This results in a +`Dataset` containing 3D arrays representing stacks of `(28,28)` images: + +``` python +print(mnist_ds.batch(100)) +``` + +``` none + +``` +Note that the dataset has an unknown batch size because the last batch will +have fewer elements. + +In `train_input_fn`, after batching the `Dataset` contains 1D vectors of +elements where each scalar was previously: + +```python +print(dataset) +``` +``` + +``` + + +### Return + +At this point the `Dataset` contains `(features_dict, labels)` pairs. +This is the format expected by the `train` and `evaluate` methods, so the +`input_fn` returns the dataset. + +The `labels` can/should be omitted when using the `predict` method. + + + + +## Reading a CSV File + +The most common real-world use case for the `Dataset` class is to stream data +from files on disk. The @{tf.data} module includes a variety of +file readers. Let's see how parsing the Iris dataset from the csv file looks +using a `Dataset`. + +The following call to the `iris_data.maybe_download` function downloads the +data if necessary, and returns the pathnames of the resulting files: + +``` python +import iris_data +train_path, test_path = iris_data.maybe_download() +``` + +The [`iris_data.csv_input_fn`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py) +function contains an alternative implementation that parses the csv files using +a `Dataset`. + +Let's look at how to build an Estimator-compatible input function that reads +from the local files. + +### Build the `Dataset` + +We start by building a @{tf.data.TextLineDataset$`TextLineDataset`} object to +read the file one line at a time. Then, we call the +@{tf.data.Dataset.skip$`skip`} method to skip over the first line of the file, which contains a header, not an example: + +``` python +ds = tf.data.TextLineDataset(train_path).skip(1) +``` + +### Build a csv line parser + +We will start by building a function to parse a single line. + +The following `iris_data.parse_line` function accomplishes this task using the +@{tf.decode_csv} function, and some simple python code: + +We must parse each of the lines in the dataset in order to generate the +necessary `(features, label)` pairs. The following `_parse_line` function +calls @{tf.decode_csv} to parse a single line into its features +and the label. Since Estimators require that features be represented as a +dictionary, we rely on Python's built-in `dict` and `zip` functions to build +that dictionary. The feature names are the keys of that dictionary. +We then call the dictionary's `pop` method to remove the label field from +the features dictionary: + +``` python +# Metadata describing the text columns +COLUMNS = ['SepalLength', 'SepalWidth', + 'PetalLength', 'PetalWidth', + 'label'] +FIELD_DEFAULTS = [[0.0], [0.0], [0.0], [0.0], [0]] +def _parse_line(line): + # Decode the line into its fields + fields = tf.decode_csv(line, FIELD_DEFAULTS) + + # Pack the result into a dictionary + features = dict(zip(COLUMNS,fields)) + + # Separate the label from the features + label = features.pop('label') + + return features, label +``` + +### Parse the lines + +Datasets have many methods for manipulating the data while it is being piped +to a model. The most heavily-used method is @{tf.data.Dataset.map$`map`}, which +applies a transformation to each element of the `Dataset`. + +The `map` method takes a `map_func` argument that describes how each item in the +`Dataset` should be transformed. + +
+ +
+
+The @{tf.data.Dataset.map$`map`} method applies the `map_func` to +transform each item in the Dataset. +
+ +So to parse the lines as they are streamed out of the csv file, we pass our +`_parse_line` function to the `map` method: + +``` python +ds = ds.map(_parse_line) +print(ds) +``` +``` None + +``` + +Now instead of simple scalar strings, the dataset contains `(features, label)` +pairs. + +the remainder of the `iris_data.csv_input_fn` function is identical +to `iris_data.train_input_fn` which was covered in the in the +[Basic input](#basic_input) section. + +### Try it out + +This function can be used as a replacement for +`iris_data.train_input_fn`. It can be used to feed an estimator as follows: + +``` python +train_path, test_path = iris_data.maybe_download() + +# All the inputs are numeric +feature_columns = [ + tf.feature_column.numeric_column(name) + for name in iris_data.CSV_COLUMN_NAMES[:-1]] + +# Build the estimator +est = tf.estimator.LinearClassifier(feature_columns, + n_classes=3) +# Train the estimator +batch_size = 100 +est.train( + steps=1000, + input_fn=lambda : iris_data.csv_input_fn(train_path, batch_size)) +``` + +Estimators expect an `input_fn` to take no arguments. To work around this +restriction, we use `lambda` to capture the arguments and provide the expected +interface. + +## Summary + +The `tf.data` module provides a collection of classes and functions for easily +reading data from a variety of sources. Furthermore, `tf.data` has simple +powerful methods for applying a wide variety of standard and custom +transformations. + +Now you have the basic idea of how to efficiently load data into an +Estimator. Consider the following documents next: + + +* @{$custom_estimators}, which demonstrates how to build your own + custom `Estimator` model. +* The @{$low_level_intro#datasets$Low Level Introduction}, which demonstrates + how to experiment directly with `tf.data.Datasets` using TensorFlow's low + level APIs. +* @{$guide/datasets} which goes into great detail about additional + functionality of `Datasets`. + diff --git a/tensorflow/docs_src/guide/debugger.md b/tensorflow/docs_src/guide/debugger.md new file mode 100644 index 0000000000..6bd941886d --- /dev/null +++ b/tensorflow/docs_src/guide/debugger.md @@ -0,0 +1,804 @@ +# TensorFlow Debugger + + + +[TOC] + +`tfdbg` is a specialized debugger for TensorFlow. It lets you view the internal +structure and states of running TensorFlow graphs during training and inference, +which is difficult to debug with general-purpose debuggers such as Python's `pdb` +due to TensorFlow's computation-graph paradigm. + +This guide focuses on the command-line interface (CLI) of `tfdbg`. For guide on +how to use the graphical user interface (GUI) of tfdbg, i.e., the +**TensorBoard Debugger Plugin**, please visit +[its README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md). + +Note: The TensorFlow debugger uses a +[curses](https://en.wikipedia.org/wiki/Curses_\(programming_library\))-based text +user interface. On Mac OS X, the `ncurses` library is required and can be +installed with `brew install homebrew/dupes/ncurses`. On Windows, curses isn't as +well supported, so a [readline](https://en.wikipedia.org/wiki/GNU_Readline)-based +interface can be used with tfdbg by installing `pyreadline` with `pip`. If you +use Anaconda3, you can install it with a command such as +`"C:\Program Files\Anaconda3\Scripts\pip.exe" install pyreadline`. Unofficial +Windows curses packages can be downloaded +[here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#curses), then subsequently +installed using `pip install .whl`, however curses on Windows may +not work as reliably as curses on Linux or Mac. + +This tutorial demonstrates how to use the **tfdbg** CLI to debug the appearance +of [`nan`s](https://en.wikipedia.org/wiki/NaN) +and [`inf`s](https://en.wikipedia.org/wiki/Infinity), a frequently-encountered +type of bug in TensorFlow model development. +The following example is for users who use the low-level +[`Session`](https://www.tensorflow.org/api_docs/python/tf/Session) API of +TensorFlow. A later section of this document describes how to use **tfdbg** +with a higher-level API, namely `Estimator`s. +To *observe* such an issue, run the following command without the debugger (the +source code can be found +[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/debug/examples/debug_mnist.py)): + +```none +python -m tensorflow.python.debug.examples.debug_mnist +``` + +This code trains a simple neural network for MNIST digit image recognition. +Notice that the accuracy increases slightly after the first training step, but +then gets stuck at a low (near-chance) level: + +```none +Accuracy at step 0: 0.1113 +Accuracy at step 1: 0.3183 +Accuracy at step 2: 0.098 +Accuracy at step 3: 0.098 +Accuracy at step 4: 0.098 +``` + +Wondering what might have gone wrong, you suspect that certain nodes in the +training graph generated bad numeric values such as `inf`s and `nan`s, because +this is a common cause of this type of training failure. +Let's use tfdbg to debug this issue and pinpoint the exact graph node where this +numeric problem first surfaced. + +## Wrapping TensorFlow Sessions with tfdbg + +To add support for tfdbg in our example, all that is needed is to add the +following lines of code and wrap the Session object with a debugger wrapper. +This code is already added in +[debug_mnist.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/debug/examples/debug_mnist.py), +so you can activate tfdbg CLI with the `--debug` flag at the command line. + +```python +# Let your BUILD target depend on "//tensorflow/python/debug:debug_py" +# (You don't need to worry about the BUILD dependency if you are using a pip +# install of open-source TensorFlow.) +from tensorflow.python import debug as tf_debug + +sess = tf_debug.LocalCLIDebugWrapperSession(sess) +``` + +This wrapper has the same interface as Session, so enabling debugging requires +no other changes to the code. The wrapper provides additional features, +including: + +* Bringing up a CLI before and after `Session.run()` calls, to let you +control the execution and inspect the graph's internal state. +* Allowing you to register special `filters` for tensor values, to facilitate +the diagnosis of issues. + +In this example, we have already registered a tensor filter called +@{tfdbg.has_inf_or_nan}, +which simply determines if there are any `nan` or `inf` values in any +intermediate tensors (tensors that are neither inputs or outputs of the +`Session.run()` call, but are in the path leading from the inputs to the +outputs). This filter is for `nan`s and `inf`s is a common enough use case that +we ship it with the +@{$python/tfdbg#Classes_for_debug_dump_data_and_directories$`debug_data`} +module. + +Note: You can also write your own custom filters. See +the @{tfdbg.DebugDumpDir.find$API documentation} +of `DebugDumpDir.find()` for additional information. + +## Debugging Model Training with tfdbg + + +Let's try training the model again, but with the `--debug` flag added this time: + +```none +python -m tensorflow.python.debug.examples.debug_mnist --debug +``` + +The debug wrapper session will prompt you when it is about to execute the first +`Session.run()` call, with information regarding the fetched tensor and feed +dictionaries displayed on the screen. + +![tfdbg run-start UI](https://www.tensorflow.org/images/tfdbg_screenshot_run_start.png) + +This is what we refer to as the *run-start CLI*. It lists the feeds and fetches +to the current `Session.run` call, before executing anything. + +If the screen size is too small to display the content of the message in its +entirety, you can resize it. + +Use the **PageUp** / **PageDown** / **Home** / **End** keys to navigate the +screen output. On most keyboards lacking those keys **Fn + Up** / +**Fn + Down** / **Fn + Right** / **Fn + Left** will work. + +Enter the `run` command (or just `r`) at the command prompt: + +``` +tfdbg> run +``` + +The `run` command causes tfdbg to execute until the end of the next +`Session.run()` call, which calculates the model's accuracy using a test data +set. tfdbg augments the runtime Graph to dump all intermediate tensors. +After the run ends, tfdbg displays all the dumped tensors values in the +*run-end CLI*. For example: + +![tfdbg run-end UI: accuracy](https://www.tensorflow.org/images/tfdbg_screenshot_run_end_accuracy.png) + +This list of tensors can also be obtained by running the command `lt` after you +executed `run`. + +### tfdbg CLI Frequently-Used Commands + +Try the following commands at the `tfdbg>` prompt (referencing the code at +`tensorflow/python/debug/examples/debug_mnist.py`): + +| Command | Syntax or Option | Explanation | Example | +|:-------------------|:---------------- |:------------ |:------------------------- | +| **`lt`** | | **List dumped tensors.** | `lt` | +| | `-n ` | List dumped tensors with names matching given regular-expression pattern. | `lt -n Softmax.*` | +| | `-t ` | List dumped tensors with op types matching given regular-expression pattern. | `lt -t MatMul` | +| | `-f ` | List only the tensors that pass a registered tensor filter. | `lt -f has_inf_or_nan` | +| | `-f -fenn ` | List only the tensors that pass a registered tensor filter, excluding nodes with names matching the regular expression. | `lt -f has_inf_or_nan` `-fenn .*Sqrt.*` | +| | `-s ` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` | +| | `-r` | Sort in reverse order. | `lt -r -s dump_size` | +| **`pt`** | | **Print value of a dumped tensor.** | | +| | `pt ` | Print tensor value. | `pt hidden/Relu:0` | +| | `pt [slicing]` | Print a subarray of tensor, using [numpy](http://www.numpy.org/)-style array slicing. | `pt hidden/Relu:0[0:50,:]` | +| | `-a` | Print the entirety of a large tensor, without using ellipses. (May take a long time for large tensors.) | `pt -a hidden/Relu:0[0:50,:]` | +| | `-r ` | Highlight elements falling into specified numerical range. Multiple ranges can be used in conjunction. | `pt hidden/Relu:0 -a -r [[-inf,-1],[1,inf]]` | +| | `-n ` | Print dump corresponding to specified 0-based dump number. Required for tensors with multiple dumps. | `pt -n 0 hidden/Relu:0` | +| | `-s` | Include a summary of the numeric values of the tensor (applicable only to non-empty tensors with Boolean and numeric types such as `int*` and `float*`.) | `pt -s hidden/Relu:0[0:50,:]` | +| | `-w` | Write the value of the tensor (possibly sliced) to a Numpy file using [`numpy.save()`](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.save.html) | `pt -s hidden/Relu:0 -w /tmp/relu.npy` | +| **`@[coordinates]`** | | Navigate to specified element in `pt` output. | `@[10,0]` or `@10,0` | +| **`/regex`** | | [less](https://linux.die.net/man/1/less)-style search for given regular expression. | `/inf` | +| **`/`** | | Scroll to the next line with matches to the searched regex (if any). | `/` | +| **`pf`** | | **Print a value in the feed_dict to `Session.run`.** | | +| | `pf ` | Print the value of the feed. Also note that the `pf` command has the `-a`, `-r` and `-s` flags (not listed below), which have the same syntax and semantics as the identically-named flags of `pt`. | `pf input_xs:0` | +| **eval** | | **Evaluate arbitrary Python and numpy expression.** | | +| | `eval ` | Evaluate a Python / numpy expression, with numpy available as `np` and debug tensor names enclosed in backticks. | ``eval "np.matmul((`output/Identity:0` / `Softmax:0`).T, `Softmax:0`)"`` | +| | `-a` | Print a large-sized evaluation result in its entirety, i.e., without using ellipses. | ``eval -a 'np.sum(`Softmax:0`, axis=1)'`` | +| | `-w` | Write the result of the evaluation to a Numpy file using [`numpy.save()`](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.save.html) | ``eval -a 'np.sum(`Softmax:0`, axis=1)' -w /tmp/softmax_sum.npy`` | +| **`ni`** | | **Display node information.** | | +| | `-a` | Include node attributes in the output. | `ni -a hidden/Relu` | +| | `-d` | List the debug dumps available from the node. | `ni -d hidden/Relu` | +| | `-t` | Display the Python stack trace of the node's creation. | `ni -t hidden/Relu` | +| **`li`** | | **List inputs to node** | | +| | `-r` | List the inputs to node, recursively (the input tree.) | `li -r hidden/Relu:0` | +| | `-d ` | Limit recursion depth under the `-r` mode. | `li -r -d 3 hidden/Relu:0` | +| | `-c` | Include control inputs. | `li -c -r hidden/Relu:0` | +| | `-t` | Show op types of input nodes. | `li -t -r hidden/Relu:0` | +| **`lo`** | | **List output recipients of node** | | +| | `-r` | List the output recipients of node, recursively (the output tree.) | `lo -r hidden/Relu:0` | +| | `-d ` | Limit recursion depth under the `-r` mode. | `lo -r -d 3 hidden/Relu:0` | +| | `-c` | Include recipients via control edges. | `lo -c -r hidden/Relu:0` | +| | `-t` | Show op types of recipient nodes. | `lo -t -r hidden/Relu:0` | +| **`ls`** | | **List Python source files involved in node creation.** | | +| | `-p ` | Limit output to source files matching given regular-expression path pattern. | `ls -p .*debug_mnist.*` | +| | `-n` | Limit output to node names matching given regular-expression pattern. | `ls -n Softmax.*` | +| **`ps`** | | **Print Python source file.** | | +| | `ps ` | Print given Python source file source.py, with the lines annotated with the nodes created at each of them (if any). | `ps /path/to/source.py` | +| | `-t` | Perform annotation with respect to Tensors, instead of the default, nodes. | `ps -t /path/to/source.py` | +| | `-b ` | Annotate source.py beginning at given line. | `ps -b 30 /path/to/source.py` | +| | `-m ` | Limit the number of elements in the annotation for each line. | `ps -m 100 /path/to/source.py` | +| **`run`** | | **Proceed to the next Session.run()** | `run` | +| | `-n` | Execute through the next `Session.run` without debugging, and drop to CLI right before the run after that. | `run -n` | +| | `-t ` | Execute `Session.run` `T - 1` times without debugging, followed by a run with debugging. Then drop to CLI right after the debugged run. | `run -t 10` | +| | `-f ` | Continue executing `Session.run` until any intermediate tensor triggers the specified Tensor filter (causes the filter to return `True`). | `run -f has_inf_or_nan` | +| | `-f -fenn ` | Continue executing `Session.run` until any intermediate tensor whose node names doesn't match the regular expression triggers the specified Tensor filter (causes the filter to return `True`). | `run -f has_inf_or_nan -fenn .*Sqrt.*` | +| | `--node_name_filter ` | Execute the next `Session.run`, watching only nodes with names matching the given regular-expression pattern. | `run --node_name_filter Softmax.*` | +| | `--op_type_filter ` | Execute the next `Session.run`, watching only nodes with op types matching the given regular-expression pattern. | `run --op_type_filter Variable.*` | +| | `--tensor_dtype_filter ` | Execute the next `Session.run`, dumping only Tensors with data types (`dtype`s) matching the given regular-expression pattern. | `run --tensor_dtype_filter int.*` | +| | `-p` | Execute the next `Session.run` call in profiling mode. | `run -p` | +| **`ri`** | | **Display information about the run the current run, including fetches and feeds.** | `ri` | +| **`config`** | | **Set or show persistent TFDBG UI configuration.** | | +| | `set` | Set the value of a config item: {`graph_recursion_depth`, `mouse_mode`}. | `config set graph_recursion_depth 3` | +| | `show` | Show current persistent UI configuration. | `config show` | +| **`help`** | | **Print general help information** | `help` | +| | `help ` | Print help for given command. | `help lt` | + +Note that each time you enter a command, a new screen output +will appear. This is somewhat analogous to web pages in a browser. You can +navigate between these screens by clicking the `<--` and +`-->` text arrows near the top-left corner of the CLI. + +### Other Features of the tfdbg CLI + +In addition to the commands listed above, the tfdbg CLI provides the following +additional features: + +* To navigate through previous tfdbg commands, type in a few characters + followed by the Up or Down arrow keys. tfdbg will show you the history of + commands that started with those characters. +* To navigate through the history of screen outputs, do either of the + following: + * Use the `prev` and `next` commands. + * Click underlined `<--` and `-->` links near the top left corner of the + screen. +* Tab completion of commands and some command arguments. +* To redirect the screen output to a file instead of the screen, end the + command with bash-style redirection. For example, the following command + redirects the output of the pt command to the `/tmp/xent_value_slices.txt` + file: + + ```none + tfdbg> pt cross_entropy/Log:0[:, 0:10] > /tmp/xent_value_slices.txt + ``` + +### Finding `nan`s and `inf`s + +In this first `Session.run()` call, there happen to be no problematic numerical +values. You can move on to the next run by using the command `run` or its +shorthand `r`. + +> TIP: If you enter `run` or `r` repeatedly, you will be able to move through +> the `Session.run()` calls in a sequential manner. +> +> You can also use the `-t` flag to move ahead a number of `Session.run()` calls +> at a time, for example: +> +> ``` +> tfdbg> run -t 10 +> ``` + +Instead of entering `run` repeatedly and manually searching for `nan`s and +`inf`s in the run-end UI after every `Session.run()` call (for example, by using +the `pt` command shown in the table above) , you can use the following +command to let the debugger repeatedly execute `Session.run()` calls without +stopping at the run-start or run-end prompt, until the first `nan` or `inf` +value shows up in the graph. This is analogous to *conditional breakpoints* in +some procedural-language debuggers: + +```none +tfdbg> run -f has_inf_or_nan +``` + +> NOTE: The preceding command works properly because a tensor filter called +> `has_inf_or_nan` has been registered for you when the wrapped session is +> created. This filter detects `nan`s and `inf`s (as explained previously). +> If you have registered any other filters, you can +> use "run -f" to have tfdbg run until any tensor triggers that filter (cause +> the filter to return True). +> +> ``` python +> def my_filter_callable(datum, tensor): +> # A filter that detects zero-valued scalars. +> return len(tensor.shape) == 0 and tensor == 0.0 +> +> sess.add_tensor_filter('my_filter', my_filter_callable) +> ``` +> +> Then at the tfdbg run-start prompt run until your filter is triggered: +> +> ``` +> tfdbg> run -f my_filter +> ``` + +See [this API document](https://www.tensorflow.org/api_docs/python/tfdbg/DebugDumpDir#find) +for more information on the expected signature and return value of the predicate +`Callable` used with `add_tensor_filter()`. + +![tfdbg run-end UI: infs and nans](https://www.tensorflow.org/images/tfdbg_screenshot_run_end_inf_nan.png) + +As the screen display indicates on the first line, the `has_inf_or_nan` filter is first triggered +during the fourth `Session.run()` call: an +[Adam optimizer](https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer) +forward-backward training pass on the graph. In this run, 36 (out of the total +95) intermediate tensors contain `nan` or `inf` values. These tensors are listed +in chronological order, with their timestamps displayed on the left. At the top +of the list, you can see the first tensor in which the bad numerical values +first surfaced: `cross_entropy/Log:0`. + +To view the value of the tensor, click the underlined tensor name +`cross_entropy/Log:0` or enter the equivalent command: + +```none +tfdbg> pt cross_entropy/Log:0 +``` + +Scroll down a little and you will notice some scattered `inf` values. If the +instances of `inf` and `nan` are difficult to spot by eye, you can use the +following command to perform a regex search and highlight the output: + +```none +tfdbg> /inf +``` + +Or, alternatively: + +```none +tfdbg> /(inf|nan) +``` + +You can also use the `-s` or `--numeric_summary` command to get a quick summary +of the types of numeric values in the tensor: + +``` none +tfdbg> pt -s cross_entropy/Log:0 +``` + +From the summary, you can see that several of the 1000 elements of the +`cross_entropy/Log:0` tensor are `-inf`s (negative infinities). + +Why did these infinities appear? To further debug, display more information +about the node `cross_entropy/Log` by clicking the underlined `node_info` menu +item on the top or entering the equivalent node_info (`ni`) command: + +```none +tfdbg> ni cross_entropy/Log +``` + +![tfdbg run-end UI: infs and nans](https://www.tensorflow.org/images/tfdbg_screenshot_run_end_node_info.png) + +You can see that this node has the op type `Log` +and that its input is the node `Softmax`. Run the following command to +take a closer look at the input tensor: + +```none +tfdbg> pt Softmax:0 +``` + +Examine the values in the input tensor, searching for zeros: + +```none +tfdbg> /0\.000 +``` + +Indeed, there are zeros. Now it is clear that the origin of the bad numerical +values is the node `cross_entropy/Log` taking logs of zeros. To find out the +culprit line in the Python source code, use the `-t` flag of the `ni` command +to show the traceback of the node's construction: + +```none +tfdbg> ni -t cross_entropy/Log +``` + +If you click "node_info" at the top of the screen, tfdbg automatically shows the +traceback of the node's construction. + +From the traceback, you can see that the op is constructed at the following +line: +[`debug_mnist.py`](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_mnist.py): + +```python +diff = y_ * tf.log(y) +``` + +**tfdbg** has a feature that makes it easy to trace Tensors and ops back to +lines in Python source files. It can annotate lines of a Python file with +the ops or Tensors created by them. To use this feature, +simply click the underlined line numbers in the stack trace output of the +`ni -t ` commands, or use the `ps` (or `print_source`) command such as: +`ps /path/to/source.py`. For example, the following screenshot shows the output +of a `ps` command. + +![tfdbg run-end UI: annotated Python source file](https://www.tensorflow.org/images/tfdbg_screenshot_run_end_annotated_source.png) + +### Fixing the problem + +To fix the problem, edit `debug_mnist.py`, changing the original line: + +```python +diff = -(y_ * tf.log(y)) +``` + +to the built-in, numerically-stable implementation of softmax cross-entropy: + +```python +diff = tf.losses.softmax_cross_entropy(labels=y_, logits=logits) +``` + +Rerun with the `--debug` flag as follows: + +```none +python -m tensorflow.python.debug.examples.debug_mnist --debug +``` + +At the `tfdbg>` prompt, enter the following command: + +```none +run -f has_inf_or_nan` +``` + +Confirm that no tensors are flagged as containing `nan` or `inf` values, and +accuracy now continues to rise rather than getting stuck. Success! + +## Debugging TensorFlow Estimators + +This section explains how to debug TensorFlow programs that use the `Estimator` +APIs. Part of the convenience provided by these APIs is that +they manage `Session`s internally. This makes the `LocalCLIDebugWrapperSession` +described in the preceding sections inapplicable. Fortunately, you can still +debug them by using special `hook`s provided by `tfdbg`. + +`tfdbg` can debug the +@{tf.estimator.Estimator.train$`train()`}, +@{tf.estimator.Estimator.evaluate$`evaluate()`} and +@{tf.estimator.Estimator.predict$`predict()`} +methods of tf-learn `Estimator`s. To debug `Estimator.train()`, +create a `LocalCLIDebugHook` and supply it in the `hooks` argument. For example: + +```python +# First, let your BUILD target depend on "//tensorflow/python/debug:debug_py" +# (You don't need to worry about the BUILD dependency if you are using a pip +# install of open-source TensorFlow.) +from tensorflow.python import debug as tf_debug + +# Create a LocalCLIDebugHook and use it as a monitor when calling fit(). +hooks = [tf_debug.LocalCLIDebugHook()] + +# To debug `train`: +classifier.train(input_fn, + steps=1000, + hooks=hooks) +``` + +Similarly, to debug `Estimator.evaluate()` and `Estimator.predict()`, assign +hooks to the `hooks` parameter, as in the following example: + +```python +# To debug `evaluate`: +accuracy_score = classifier.evaluate(eval_input_fn, + hooks=hooks)["accuracy"] + +# To debug `predict`: +predict_results = classifier.predict(predict_input_fn, hooks=hooks) +``` + +[debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_tflearn_iris.py), +based on [tf-learn's iris tutorial](https://www.tensorflow.org/versions/r1.8/get_started/tflearn), +contains a full example of how to use the tfdbg with `Estimator`s. +To run this example, do: + +```none +python -m tensorflow.python.debug.examples.debug_tflearn_iris --debug +``` + +The `LocalCLIDebugHook` also allows you to configure a `watch_fn` that can be +used to flexibly specify what `Tensor`s to watch on different `Session.run()` +calls, as a function of the `fetches` and `feed_dict` and other states. See +@{tfdbg.DumpingDebugWrapperSession.__init__$this API doc} +for more details. + +## Debugging Keras Models with TFDBG + +To use TFDBG with [Keras](https://keras.io/), let the Keras backend use +a TFDBG-wrapped Session object. For example, to use the CLI wrapper: + +``` python +import tensorflow as tf +from keras import backend as keras_backend +from tensorflow.python import debug as tf_debug + +keras_backend.set_session(tf_debug.LocalCLIDebugWrapperSession(tf.Session())) + +# Define your keras model, called "model". +model.fit(...) # This will break into the TFDBG CLI. +``` + +## Debugging tf-slim with TFDBG + +TFDBG supports debugging of training and evaluation with +[tf-slim](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/slim). +As detailed below, training and evaluation require slightly different debugging +workflows. + +### Debugging training in tf-slim +To debug the training process, provide `LocalCLIDebugWrapperSession` to the +`session_wrapper` argument of `slim.learning.train()`. For example: + +``` python +import tensorflow as tf +from tensorflow.python import debug as tf_debug + +# ... Code that creates the graph and the train_op ... +tf.contrib.slim.learning.train( + train_op, + logdir, + number_of_steps=10, + session_wrapper=tf_debug.LocalCLIDebugWrapperSession) +``` + +### Debugging evaluation in tf-slim +To debug the evaluation process, provide `LocalCLIDebugHook` to the +`hooks` argument of `slim.evaluation.evaluate_once()`. For example: + +``` python +import tensorflow as tf +from tensorflow.python import debug as tf_debug + +# ... Code that creates the graph and the eval and final ops ... +tf.contrib.slim.evaluation.evaluate_once( + '', + checkpoint_path, + logdir, + eval_op=my_eval_op, + final_op=my_value_op, + hooks=[tf_debug.LocalCLIDebugHook()]) +``` + +## Offline Debugging of Remotely-Running Sessions + +Often, your model is running on a remote machine or a process that you don't +have terminal access to. To perform model debugging in such cases, you can use +the `offline_analyzer` binary of `tfdbg` (described below). It operates on +dumped data directories. This can be done to both the lower-level `Session` API +and the higher-level `Estimator` API. + +### Debugging Remote tf.Sessions + +If you interact directly with the `tf.Session` API in `python`, you can +configure the `RunOptions` proto that you call your `Session.run()` method +with, by using the method @{tfdbg.watch_graph}. +This will cause the intermediate tensors and runtime graphs to be dumped to a +shared storage location of your choice when the `Session.run()` call occurs +(at the cost of slower performance). For example: + +```python +from tensorflow.python import debug as tf_debug + +# ... Code where your session and graph are set up... + +run_options = tf.RunOptions() +tf_debug.watch_graph( + run_options, + session.graph, + debug_urls=["file:///shared/storage/location/tfdbg_dumps_1"]) +# Be sure to specify different directories for different run() calls. + +session.run(fetches, feed_dict=feeds, options=run_options) +``` + +Later, in an environment that you have terminal access to (for example, a local +computer that can access the shared storage location specified in the code +above), you can load and inspect the data in the dump directory on the shared +storage by using the `offline_analyzer` binary of `tfdbg`. For example: + +```none +python -m tensorflow.python.debug.cli.offline_analyzer \ + --dump_dir=/shared/storage/location/tfdbg_dumps_1 +``` + +The `Session` wrapper `DumpingDebugWrapperSession` offers an easier and more +flexible way to generate file-system dumps that can be analyzed offline. +To use it, simply wrap your session in a `tf_debug.DumpingDebugWrapperSession`. +For example: + +```python +# Let your BUILD target depend on "//tensorflow/python/debug:debug_py +# (You don't need to worry about the BUILD dependency if you are using a pip +# install of open-source TensorFlow.) +from tensorflow.python import debug as tf_debug + +sess = tf_debug.DumpingDebugWrapperSession( + sess, "/shared/storage/location/tfdbg_dumps_1/", watch_fn=my_watch_fn) +``` + +The `watch_fn` argument accepts a `Callable` that allows you to configure what +`tensor`s to watch on different `Session.run()` calls, as a function of the +`fetches` and `feed_dict` to the `run()` call and other states. + +### C++ and other languages + +If your model code is written in C++ or other languages, you can also +modify the `debug_options` field of `RunOptions` to generate debug dumps that +can be inspected offline. See +[the proto definition](https://www.tensorflow.org/code/tensorflow/core/protobuf/debug.proto) +for more details. + +### Debugging Remotely-Running Estimators + +If your remote TensorFlow server runs `Estimator`s, +you can use the non-interactive `DumpingDebugHook`. For example: + +```python +# Let your BUILD target depend on "//tensorflow/python/debug:debug_py +# (You don't need to worry about the BUILD dependency if you are using a pip +# install of open-source TensorFlow.) +from tensorflow.python import debug as tf_debug + +hooks = [tf_debug.DumpingDebugHook("/shared/storage/location/tfdbg_dumps_1")] +``` + +Then this `hook` can be used in the same way as the `LocalCLIDebugHook` examples +described earlier in this document. +As the training, evalution or prediction happens with `Estimator`, +tfdbg creates directories having the following name pattern: +`/shared/storage/location/tfdbg_dumps_1/run__`. +Each directory corresponds to a `Session.run()` call that underlies +the `fit()` or `evaluate()` call. You can load these directories and inspect +them in a command-line interface in an offline manner using the +`offline_analyzer` offered by tfdbg. For example: + +```bash +python -m tensorflow.python.debug.cli.offline_analyzer \ + --dump_dir="/shared/storage/location/tfdbg_dumps_1/run__" +``` + +## Frequently Asked Questions + +**Q**: _Do the timestamps on the left side of the `lt` output reflect actual + performance in a non-debugging session?_ + +**A**: No. The debugger inserts additional special-purpose debug nodes to the + graph to record the values of intermediate tensors. These nodes + slow down the graph execution. If you are interested in profiling your + model, check out + + 1. The profiling mode of tfdbg: `tfdbg> run -p`. + 2. [tfprof](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core/profiler) + and other profiling tools for TensorFlow. + +**Q**: _How do I link tfdbg against my `Session` in Bazel? Why do I see an + error such as "ImportError: cannot import name debug"?_ + +**A**: In your BUILD rule, declare dependencies: + `"//tensorflow:tensorflow_py"` and `"//tensorflow/python/debug:debug_py"`. + The first is the dependency that you include to use TensorFlow even + without debugger support; the second enables the debugger. + Then, In your Python file, add: + +```python +from tensorflow.python import debug as tf_debug + +# Then wrap your TensorFlow Session with the local-CLI wrapper. +sess = tf_debug.LocalCLIDebugWrapperSession(sess) +``` + +**Q**: _Does tfdbg help debug runtime errors such as shape mismatches?_ + +**A**: Yes. tfdbg intercepts errors generated by ops during runtime and presents + the errors with some debug instructions to the user in the CLI. + See examples: + +```none +# Debugging shape mismatch during matrix multiplication. +python -m tensorflow.python.debug.examples.debug_errors \ + --error shape_mismatch --debug + +# Debugging uninitialized variable. +python -m tensorflow.python.debug.examples.debug_errors \ + --error uninitialized_variable --debug +``` + +**Q**: _How can I let my tfdbg-wrapped Sessions or Hooks run the debug mode +only from the main thread?_ + +**A**: +This is a common use case, in which the `Session` object is used from multiple +threads concurrently. Typically, the child threads take care of background tasks +such as running enqueue operations. Often, you want to debug only the main +thread (or less frequently, only one of the child threads). You can use the +`thread_name_filter` keyword argument of `LocalCLIDebugWrapperSession` to +achieve this type of thread-selective debugging. For example, to debug from the +main thread only, construct a wrapped `Session` as follows: + +```python +sess = tf_debug.LocalCLIDebugWrapperSession(sess, thread_name_filter="MainThread$") +``` + +The above example relies on the fact that main threads in Python have the +default name `MainThread`. + +**Q**: _The model I am debugging is very large. The data dumped by tfdbg +fills up the free space of my disk. What can I do?_ + +**A**: +You might encounter this problem in any of the following situations: + +* models with many intermediate tensors +* very large intermediate tensors +* many @{tf.while_loop} iterations + +There are three possible workarounds or solutions: + +* The constructors of `LocalCLIDebugWrapperSession` and `LocalCLIDebugHook` + provide a keyword argument, `dump_root`, to specify the path + to which tfdbg dumps the debug data. You can use it to let tfdbg dump the + debug data on a disk with larger free space. For example: + +```python +# For LocalCLIDebugWrapperSession +sess = tf_debug.LocalCLIDebugWrapperSession(dump_root="/with/lots/of/space") + +# For LocalCLIDebugHook +hooks = [tf_debug.LocalCLIDebugHook(dump_root="/with/lots/of/space")] +``` + Make sure that the directory pointed to by dump_root is empty or nonexistent. + `tfdbg` cleans up the dump directories before exiting. + +* Reduce the batch size used during the runs. +* Use the filtering options of tfdbg's `run` command to watch only specific + nodes in the graph. For example: + + ``` + tfdbg> run --node_name_filter .*hidden.* + tfdbg> run --op_type_filter Variable.* + tfdbg> run --tensor_dtype_filter int.* + ``` + + The first command above watches only nodes whose name match the + regular-expression pattern `.*hidden.*`. The second command watches only + operations whose name match the pattern `Variable.*`. The third one watches + only the tensors whose dtype match the pattern `int.*` (e.g., `int32`). + + +**Q**: _Why can't I select text in the tfdbg CLI?_ + +**A**: This is because the tfdbg CLI enables mouse events in the terminal by + default. This [mouse-mask](https://linux.die.net/man/3/mousemask) mode + overrides default terminal interactions, including text selection. You + can re-enable text selection by using the command `mouse off` or + `m off`. + +**Q**: _Why does the tfdbg CLI show no dumped tensors when I debug code like the following?_ + +``` python +a = tf.ones([10], name="a") +b = tf.add(a, a, name="b") +sess = tf.Session() +sess = tf_debug.LocalCLIDebugWrapperSession(sess) +sess.run(b) +``` + +**A**: The reason why you see no data dumped is because every node in the + executed TensorFlow graph is constant-folded by the TensorFlow runtime. + In this exapmle, `a` is a constant tensor; therefore, the fetched + tensor `b` is effectively also a constant tensor. TensorFlow's graph + optimization folds the graph that contains `a` and `b` into a single + node to speed up future runs of the graph, which is why `tfdbg` does + not generate any intermediate tensor dumps. However, if `a` were a + @{tf.Variable}, as in the following example: + +``` python +import numpy as np + +a = tf.Variable(np.ones[10], name="a") +b = tf.add(a, a, name="b") +sess = tf.Session() +sess.run(tf.global_variables_initializer()) +sess = tf_debug.LocalCLIDebugWrapperSession(sess) +sess.run(b) +``` + +the constant-folding would not occur and `tfdbg` should show the intermediate +tensor dumps. + + +**Q**: I am debugging a model that generates unwanted infinities or NaNs. But + there are some nodes in my model that are known to generate infinities + or NaNs in their output tensors even under completely normal conditions. + How can I skip those nodes during my `run -f has_inf_or_nan` actions? + +**A**: Use the `--filter_exclude_node_names` (`-fenn` for short) flag. For + example, if you known you have a node with name matching the regular + expression `.*Sqrt.*` that generates infinities or NaNs regardless + of whether the model is behaving correctly, you can exclude the nodes + from the infinity/NaN-finding runs with the command + `run -f has_inf_or_nan -fenn .*Sqrt.*`. + + +**Q**: Is there a GUI for tfdbg? + +**A**: Yes, the **TensorBoard Debugger Plugin** is the GUI of tfdbg. + It offers features such as inspection of the computation graph, + real-time visualization of tensor values, continuation to tensor + and conditional breakpoints, and tying tensors to their + graph-construction source code, all in the browser environment. + To get started, please visit + [its README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md). diff --git a/tensorflow/docs_src/guide/eager.md b/tensorflow/docs_src/guide/eager.md new file mode 100644 index 0000000000..00d02b4455 --- /dev/null +++ b/tensorflow/docs_src/guide/eager.md @@ -0,0 +1,849 @@ +# Eager Execution + +TensorFlow's eager execution is an imperative programming environment that +evaluates operations immediately, without building graphs: operations return +concrete values instead of constructing a computational graph to run later. This +makes it easy to get started with TensorFlow and debug models, and it +reduces boilerplate as well. To follow along with this guide, run the code +samples below in an interactive `python` interpreter. + +Eager execution is a flexible machine learning platform for research and +experimentation, providing: + +* *An intuitive interface*—Structure your code naturally and use Python data + structures. Quickly iterate on small models and small data. +* *Easier debugging*—Call ops directly to inspect running models and test + changes. Use standard Python debugging tools for immediate error reporting. +* *Natural control flow*—Use Python control flow instead of graph control + flow, simplifying the specification of dynamic models. + +Eager execution supports most TensorFlow operations and GPU acceleration. For a +collection of examples running in eager execution, see: +[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples). + +Note: Some models may experience increased overhead with eager execution +enabled. Performance improvements are ongoing, but please +[file a bug](https://github.com/tensorflow/tensorflow/issues) if you find a +problem and share your benchmarks. + +## Setup and basic usage + +Upgrade to the latest version of TensorFlow: + +``` +$ pip install --upgrade tensorflow +``` + +To start eager execution, add `tf.enable_eager_execution()` to the beginning of +the program or console session. Do not add this operation to other modules that +the program calls. + +```py +from __future__ import absolute_import, division, print_function + +import tensorflow as tf + +tf.enable_eager_execution() +``` + +Now you can run TensorFlow operations and the results will return immediately: + +```py +tf.executing_eagerly() # => True + +x = [[2.]] +m = tf.matmul(x, x) +print("hello, {}".format(m)) # => "hello, [[4.]]" +``` + +Enabling eager execution changes how TensorFlow operations behave—now they +immediately evaluate and return their values to Python. `tf.Tensor` objects +reference concrete values instead of symbolic handles to nodes in a computational +graph. Since there isn't a computational graph to build and run later in a +session, it's easy to inspect results using `print()` or a debugger. Evaluating, +printing, and checking tensor values does not break the flow for computing +gradients. + +Eager execution works nicely with [NumPy](http://www.numpy.org/). NumPy +operations accept `tf.Tensor` arguments. TensorFlow +[math operations](https://www.tensorflow.org/api_guides/python/math_ops) convert +Python objects and NumPy arrays to `tf.Tensor` objects. The +`tf.Tensor.numpy` method returns the object's value as a NumPy `ndarray`. + +```py +a = tf.constant([[1, 2], + [3, 4]]) +print(a) +# => tf.Tensor([[1 2] +# [3 4]], shape=(2, 2), dtype=int32) + +# Broadcasting support +b = tf.add(a, 1) +print(b) +# => tf.Tensor([[2 3] +# [4 5]], shape=(2, 2), dtype=int32) + +# Operator overloading is supported +print(a * b) +# => tf.Tensor([[ 2 6] +# [12 20]], shape=(2, 2), dtype=int32) + +# Use NumPy values +import numpy as np + +c = np.multiply(a, b) +print(c) +# => [[ 2 6] +# [12 20]] + +# Obtain numpy value from a tensor: +print(a.numpy()) +# => [[1 2] +# [3 4]] +``` + +The `tf.contrib.eager` module contains symbols available to both eager and graph execution +environments and is useful for writing code to [work with graphs](#work_with_graphs): + +```py +tfe = tf.contrib.eager +``` + +## Dynamic control flow + +A major benefit of eager execution is that all the functionality of the host +language is available while your model is executing. So, for example, +it is easy to write [fizzbuzz](https://en.wikipedia.org/wiki/Fizz_buzz): + +```py +def fizzbuzz(max_num): + counter = tf.constant(0) + max_num = tf.convert_to_tensor(max_num) + for num in range(max_num.numpy()): + num = tf.constant(num) + if int(num % 3) == 0 and int(num % 5) == 0: + print('FizzBuzz') + elif int(num % 3) == 0: + print('Fizz') + elif int(num % 5) == 0: + print('Buzz') + else: + print(num) + counter += 1 + return counter +``` + +This has conditionals that depend on tensor values and it prints these values +at runtime. + +## Build a model + +Many machine learning models are represented by composing layers. When +using TensorFlow with eager execution you can either write your own layers or +use a layer provided in the `tf.keras.layers` package. + +While you can use any Python object to represent a layer, +TensorFlow has `tf.keras.layers.Layer` as a convenient base class. Inherit from +it to implement your own layer: + +```py +class MySimpleLayer(tf.keras.layers.Layer): + def __init__(self, output_units): + self.output_units = output_units + + def build(self, input): + # The build method gets called the first time your layer is used. + # Creating variables on build() allows you to make their shape depend + # on the input shape and hence remove the need for the user to specify + # full shapes. It is possible to create variables during __init__() if + # you already know their full shapes. + self.kernel = self.add_variable( + "kernel", [input.shape[-1], self.output_units]) + + def call(self, input): + # Override call() instead of __call__ so we can perform some bookkeeping. + return tf.matmul(input, self.kernel) +``` + +Use `tf.keras.layers.Dense` layer instead of `MySimpleLayer` above as it has +a superset of its functionality (it can also add a bias). + +When composing layers into models you can use `tf.keras.Sequential` to represent +models which are a linear stack of layers. It is easy to use for basic models: + +```py +model = tf.keras.Sequential([ + tf.keras.layers.Dense(10, input_shape=(784,)), # must declare input shape + tf.keras.layers.Dense(10) +]) +``` + +Alternatively, organize models in classes by inheriting from `tf.keras.Model`. +This is a container for layers that is a layer itself, allowing `tf.keras.Model` +objects to contain other `tf.keras.Model` objects. + +```py +class MNISTModel(tf.keras.Model): + def __init__(self): + super(MNISTModel, self).__init__() + self.dense1 = tf.keras.layers.Dense(units=10) + self.dense2 = tf.keras.layers.Dense(units=10) + + def call(self, input): + """Run the model.""" + result = self.dense1(input) + result = self.dense2(result) + result = self.dense2(result) # reuse variables from dense2 layer + return result + +model = MNISTModel() +``` + +It's not required to set an input shape for the `tf.keras.Model` class since +the parameters are set the first time input is passed to the layer. + +`tf.keras.layers` classes create and contain their own model variables that +are tied to the lifetime of their layer objects. To share layer variables, share +their objects. + + +## Eager training + +### Computing gradients + +[Automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) +is useful for implementing machine learning algorithms such as +[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) for training +neural networks. During eager execution, use `tf.GradientTape` to trace +operations for computing gradients later. + +`tf.GradientTape` is an opt-in feature to provide maximal performance when +not tracing. Since different operations can occur during each call, all +forward-pass operations get recorded to a "tape". To compute the gradient, play +the tape backwards and then discard. A particular `tf.GradientTape` can only +compute one gradient; subsequent calls throw a runtime error. + +```py +w = tfe.Variable([[1.0]]) +with tf.GradientTape() as tape: + loss = w * w + +grad = tape.gradient(loss, w) +print(grad) # => tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32) +``` + +Here's an example of `tf.GradientTape` that records forward-pass operations +to train a simple model: + +```py +# A toy dataset of points around 3 * x + 2 +NUM_EXAMPLES = 1000 +training_inputs = tf.random_normal([NUM_EXAMPLES]) +noise = tf.random_normal([NUM_EXAMPLES]) +training_outputs = training_inputs * 3 + 2 + noise + +def prediction(input, weight, bias): + return input * weight + bias + +# A loss function using mean-squared error +def loss(weights, biases): + error = prediction(training_inputs, weights, biases) - training_outputs + return tf.reduce_mean(tf.square(error)) + +# Return the derivative of loss with respect to weight and bias +def grad(weights, biases): + with tf.GradientTape() as tape: + loss_value = loss(weights, biases) + return tape.gradient(loss_value, [weights, biases]) + +train_steps = 200 +learning_rate = 0.01 +# Start with arbitrary values for W and B on the same batch of data +W = tfe.Variable(5.) +B = tfe.Variable(10.) + +print("Initial loss: {:.3f}".format(loss(W, B))) + +for i in range(train_steps): + dW, dB = grad(W, B) + W.assign_sub(dW * learning_rate) + B.assign_sub(dB * learning_rate) + if i % 20 == 0: + print("Loss at step {:03d}: {:.3f}".format(i, loss(W, B))) + +print("Final loss: {:.3f}".format(loss(W, B))) +print("W = {}, B = {}".format(W.numpy(), B.numpy())) +``` + +Output (exact numbers may vary): + +``` +Initial loss: 71.204 +Loss at step 000: 68.333 +Loss at step 020: 30.222 +Loss at step 040: 13.691 +Loss at step 060: 6.508 +Loss at step 080: 3.382 +Loss at step 100: 2.018 +Loss at step 120: 1.422 +Loss at step 140: 1.161 +Loss at step 160: 1.046 +Loss at step 180: 0.996 +Final loss: 0.974 +W = 3.01582956314, B = 2.1191945076 +``` + +Replay the `tf.GradientTape` to compute the gradients and apply them in a +training loop. This is demonstrated in an excerpt from the +[mnist_eager.py](https://github.com/tensorflow/models/blob/master/official/mnist/mnist_eager.py) +example: + +```py +dataset = tf.data.Dataset.from_tensor_slices((data.train.images, + data.train.labels)) +... +for (batch, (images, labels)) in enumerate(dataset): + ... + with tf.GradientTape() as tape: + logits = model(images, training=True) + loss_value = loss(logits, labels) + ... + grads = tape.gradient(loss_value, model.variables) + optimizer.apply_gradients(zip(grads, model.variables), + global_step=tf.train.get_or_create_global_step()) +``` + + +The following example creates a multi-layer model that classifies the standard +[MNIST handwritten digits](https://www.tensorflow.org/tutorials/layers). It +demonstrates the optimizer and layer APIs to build trainable graphs in an eager +execution environment. + +### Train a model + +Even without training, call the model and inspect the output in eager execution: + +```py +# Create a tensor representing a blank image +batch = tf.zeros([1, 1, 784]) +print(batch.shape) # => (1, 1, 784) + +result = model(batch) +# => tf.Tensor([[[ 0. 0., ..., 0.]]], shape=(1, 1, 10), dtype=float32) +``` + +This example uses the +[dataset.py module](https://github.com/tensorflow/models/blob/master/official/mnist/dataset.py) +from the +[TensorFlow MNIST example](https://github.com/tensorflow/models/tree/master/official/mnist); +download this file to your local directory. Run the following to download the +MNIST data files to your working directory and prepare a `tf.data.Dataset` +for training: + +```py +import dataset # download dataset.py file +dataset_train = dataset.train('./datasets').shuffle(60000).repeat(4).batch(32) +``` + +To train a model, define a loss function to optimize and then calculate +gradients. Use an optimizer to update the variables: + +```py +def loss(model, x, y): + prediction = model(x) + return tf.losses.sparse_softmax_cross_entropy(labels=y, logits=prediction) + +def grad(model, inputs, targets): + with tf.GradientTape() as tape: + loss_value = loss(model, inputs, targets) + return tape.gradient(loss_value, model.variables) + +optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) + +x, y = iter(dataset_train).next() +print("Initial loss: {:.3f}".format(loss(model, x, y))) + +# Training loop +for (i, (x, y)) in enumerate(dataset_train): + # Calculate derivatives of the input function with respect to its parameters. + grads = grad(model, x, y) + # Apply the gradient to the model + optimizer.apply_gradients(zip(grads, model.variables), + global_step=tf.train.get_or_create_global_step()) + if i % 200 == 0: + print("Loss at step {:04d}: {:.3f}".format(i, loss(model, x, y))) + +print("Final loss: {:.3f}".format(loss(model, x, y))) +``` + +Output (exact numbers may vary): + +``` +Initial loss: 2.674 +Loss at step 0000: 2.593 +Loss at step 0200: 2.143 +Loss at step 0400: 2.009 +Loss at step 0600: 2.103 +Loss at step 0800: 1.621 +Loss at step 1000: 1.695 +... +Loss at step 6600: 0.602 +Loss at step 6800: 0.557 +Loss at step 7000: 0.499 +Loss at step 7200: 0.744 +Loss at step 7400: 0.681 +Final loss: 0.670 +``` + +And for faster training, move the computation to a GPU: + +```py +with tf.device("/gpu:0"): + for (i, (x, y)) in enumerate(dataset_train): + # minimize() is equivalent to the grad() and apply_gradients() calls. + optimizer.minimize(lambda: loss(model, x, y), + global_step=tf.train.get_or_create_global_step()) +``` + +### Variables and optimizers + +`tfe.Variable` objects store mutable `tf.Tensor` values accessed during +training to make automatic differentiation easier. The parameters of a model can +be encapsulated in classes as variables. + +Better encapsulate model parameters by using `tfe.Variable` with +`tf.GradientTape`. For example, the automatic differentiation example above +can be rewritten: + +```py +class Model(tf.keras.Model): + def __init__(self): + super(Model, self).__init__() + self.W = tfe.Variable(5., name='weight') + self.B = tfe.Variable(10., name='bias') + def predict(self, inputs): + return inputs * self.W + self.B + +# A toy dataset of points around 3 * x + 2 +NUM_EXAMPLES = 2000 +training_inputs = tf.random_normal([NUM_EXAMPLES]) +noise = tf.random_normal([NUM_EXAMPLES]) +training_outputs = training_inputs * 3 + 2 + noise + +# The loss function to be optimized +def loss(model, inputs, targets): + error = model.predict(inputs) - targets + return tf.reduce_mean(tf.square(error)) + +def grad(model, inputs, targets): + with tf.GradientTape() as tape: + loss_value = loss(model, inputs, targets) + return tape.gradient(loss_value, [model.W, model.B]) + +# Define: +# 1. A model. +# 2. Derivatives of a loss function with respect to model parameters. +# 3. A strategy for updating the variables based on the derivatives. +model = Model() +optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) + +print("Initial loss: {:.3f}".format(loss(model, training_inputs, training_outputs))) + +# Training loop +for i in range(300): + grads = grad(model, training_inputs, training_outputs) + optimizer.apply_gradients(zip(grads, [model.W, model.B]), + global_step=tf.train.get_or_create_global_step()) + if i % 20 == 0: + print("Loss at step {:03d}: {:.3f}".format(i, loss(model, training_inputs, training_outputs))) + +print("Final loss: {:.3f}".format(loss(model, training_inputs, training_outputs))) +print("W = {}, B = {}".format(model.W.numpy(), model.B.numpy())) +``` + +Output (exact numbers may vary): + +``` +Initial loss: 69.066 +Loss at step 000: 66.368 +Loss at step 020: 30.107 +Loss at step 040: 13.959 +Loss at step 060: 6.769 +Loss at step 080: 3.567 +Loss at step 100: 2.141 +Loss at step 120: 1.506 +Loss at step 140: 1.223 +Loss at step 160: 1.097 +Loss at step 180: 1.041 +Loss at step 200: 1.016 +Loss at step 220: 1.005 +Loss at step 240: 1.000 +Loss at step 260: 0.998 +Loss at step 280: 0.997 +Final loss: 0.996 +W = 2.99431324005, B = 2.02129220963 +``` + +## Use objects for state during eager execution + +With graph execution, program state (such as the variables) is stored in global +collections and their lifetime is managed by the `tf.Session` object. In +contrast, during eager execution the lifetime of state objects is determined by +the lifetime of their corresponding Python object. + +### Variables are objects + +During eager execution, variables persist until the last reference to the object +is removed, and is then deleted. + +```py +with tf.device("gpu:0"): + v = tfe.Variable(tf.random_normal([1000, 1000])) + v = None # v no longer takes up GPU memory +``` + +### Object-based saving + +`tfe.Checkpoint` can save and restore `tfe.Variable`s to and from +checkpoints: + +```py +x = tfe.Variable(10.) + +checkpoint = tfe.Checkpoint(x=x) # save as "x" + +x.assign(2.) # Assign a new value to the variables and save. +save_path = checkpoint.save('./ckpt/') + +x.assign(11.) # Change the variable after saving. + +# Restore values from the checkpoint +checkpoint.restore(save_path) + +print(x) # => 2.0 +``` + +To save and load models, `tfe.Checkpoint` stores the internal state of objects, +without requiring hidden variables. To record the state of a `model`, +an `optimizer`, and a global step, pass them to a `tfe.Checkpoint`: + +```py +model = MyModel() +optimizer = tf.train.AdamOptimizer(learning_rate=0.001) +checkpoint_dir = ‘/path/to/model_dir’ +checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt") +root = tfe.Checkpoint(optimizer=optimizer, + model=model, + optimizer_step=tf.train.get_or_create_global_step()) + +root.save(file_prefix=checkpoint_prefix) +# or +root.restore(tf.train.latest_checkpoint(checkpoint_dir)) +``` + +### Object-oriented metrics + +`tfe.metrics` are stored as objects. Update a metric by passing the new data to +the callable, and retrieve the result using the `tfe.metrics.result` method, +for example: + +```py +m = tfe.metrics.Mean("loss") +m(0) +m(5) +m.result() # => 2.5 +m([8, 9]) +m.result() # => 5.5 +``` + +#### Summaries and TensorBoard + +@{$summaries_and_tensorboard$TensorBoard} is a visualization tool for +understanding, debugging and optimizing the model training process. It uses +summary events that are written while executing the program. + +`tf.contrib.summary` is compatible with both eager and graph execution +environments. Summary operations, such as `tf.contrib.summary.scalar`, are +inserted during model construction. For example, to record summaries once every +100 global steps: + +```py +writer = tf.contrib.summary.create_file_writer(logdir) +global_step=tf.train.get_or_create_global_step() # return global step var + +writer.set_as_default() + +for _ in range(iterations): + global_step.assign_add(1) + # Must include a record_summaries method + with tf.contrib.summary.record_summaries_every_n_global_steps(100): + # your model code goes here + tf.contrib.summary.scalar('loss', loss) + ... +``` + +## Advanced automatic differentiation topics + +### Dynamic models + +`tf.GradientTape` can also be used in dynamic models. This example for a +[backtracking line search](https://wikipedia.org/wiki/Backtracking_line_search) +algorithm looks like normal NumPy code, except there are gradients and is +differentiable, despite the complex control flow: + +```py +def line_search_step(fn, init_x, rate=1.0): + with tf.GradientTape() as tape: + # Variables are automatically recorded, but manually watch a tensor + tape.watch(init_x) + value = fn(init_x) + grad = tape.gradient(value, init_x) + grad_norm = tf.reduce_sum(grad * grad) + init_value = value + while value > init_value - rate * grad_norm: + x = init_x - rate * grad + value = fn(x) + rate /= 2.0 + return x, value +``` + +### Additional functions to compute gradients + +`tf.GradientTape` is a powerful interface for computing gradients, but there +is another [Autograd](https://github.com/HIPS/autograd)-style API available for +automatic differentiation. These functions are useful if writing math code with +only tensors and gradient functions, and without `tfe.Variables`: + +* `tfe.gradients_function` —Returns a function that computes the derivatives + of its input function parameter with respect to its arguments. The input + function parameter must return a scalar value. When the returned function is + invoked, it returns a list of `tf.Tensor` objects: one element for each + argument of the input function. Since anything of interest must be passed as a + function parameter, this becomes unwieldy if there's a dependency on many + trainable parameters. +* `tfe.value_and_gradients_function` —Similar to + `tfe.gradients_function`, but when the returned function is invoked, it + returns the value from the input function in addition to the list of + derivatives of the input function with respect to its arguments. + +In the following example, `tfe.gradients_function` takes the `square` +function as an argument and returns a function that computes the partial +derivatives of `square` with respect to its inputs. To calculate the derivative +of `square` at `3`, `grad(3.0)` returns `6`. + +```py +def square(x): + return tf.multiply(x, x) + +grad = tfe.gradients_function(square) + +square(3.) # => 9.0 +grad(3.) # => [6.0] + +# The second-order derivative of square: +gradgrad = tfe.gradients_function(lambda x: grad(x)[0]) +gradgrad(3.) # => [2.0] + +# The third-order derivative is None: +gradgradgrad = tfe.gradients_function(lambda x: gradgrad(x)[0]) +gradgradgrad(3.) # => [None] + + +# With flow control: +def abs(x): + return x if x > 0. else -x + +grad = tfe.gradients_function(abs) + +grad(3.) # => [1.0] +grad(-3.) # => [-1.0] +``` + +### Custom gradients + +Custom gradients are an easy way to override gradients in eager and graph +execution. Within the forward function, define the gradient with respect to the +inputs, outputs, or intermediate results. For example, here's an easy way to clip +the norm of the gradients in the backward pass: + +```py +@tf.custom_gradient +def clip_gradient_by_norm(x, norm): + y = tf.identity(x) + def grad_fn(dresult): + return [tf.clip_by_norm(dresult, norm), None] + return y, grad_fn +``` + +Custom gradients are commonly used to provide a numerically stable gradient for a +sequence of operations: + +```py +def log1pexp(x): + return tf.log(1 + tf.exp(x)) +grad_log1pexp = tfe.gradients_function(log1pexp) + +# The gradient computation works fine at x = 0. +grad_log1pexp(0.) # => [0.5] + +# However, x = 100 fails because of numerical instability. +grad_log1pexp(100.) # => [nan] +``` + +Here, the `log1pexp` function can be analytically simplified with a custom +gradient. The implementation below reuses the value for `tf.exp(x)` that is +computed during the forward pass—making it more efficient by eliminating +redundant calculations: + +```py +@tf.custom_gradient +def log1pexp(x): + e = tf.exp(x) + def grad(dy): + return dy * (1 - 1 / (1 + e)) + return tf.log(1 + e), grad + +grad_log1pexp = tfe.gradients_function(log1pexp) + +# As before, the gradient computation works fine at x = 0. +grad_log1pexp(0.) # => [0.5] + +# And the gradient computation also works at x = 100. +grad_log1pexp(100.) # => [1.0] +``` + +## Performance + +Computation is automatically offloaded to GPUs during eager execution. If you +want control over where a computation runs you can enclose it in a +`tf.device('/gpu:0')` block (or the CPU equivalent): + +```py +import time + +def measure(x, steps): + # TensorFlow initializes a GPU the first time it's used, exclude from timing. + tf.matmul(x, x) + start = time.time() + for i in range(steps): + x = tf.matmul(x, x) + _ = x.numpy() # Make sure to execute op and not just enqueue it + end = time.time() + return end - start + +shape = (1000, 1000) +steps = 200 +print("Time to multiply a {} matrix by itself {} times:".format(shape, steps)) + +# Run on CPU: +with tf.device("/cpu:0"): + print("CPU: {} secs".format(measure(tf.random_normal(shape), steps))) + +# Run on GPU, if available: +if tfe.num_gpus() > 0: + with tf.device("/gpu:0"): + print("GPU: {} secs".format(measure(tf.random_normal(shape), steps))) +else: + print("GPU: not found") +``` + +Output (exact numbers depend on hardware): + +``` +Time to multiply a (1000, 1000) matrix by itself 200 times: +CPU: 4.614904403686523 secs +GPU: 0.5581181049346924 secs +``` + +A `tf.Tensor` object can be copied to a different device to execute its +operations: + +```py +x = tf.random_normal([10, 10]) + +x_gpu0 = x.gpu() +x_cpu = x.cpu() + +_ = tf.matmul(x_cpu, x_cpu) # Runs on CPU +_ = tf.matmul(x_gpu0, x_gpu0) # Runs on GPU:0 + +if tfe.num_gpus() > 1: + x_gpu1 = x.gpu(1) + _ = tf.matmul(x_gpu1, x_gpu1) # Runs on GPU:1 +``` + +### Benchmarks + +For compute-heavy models, such as +[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50) +training on a GPU, eager execution performance is comparable to graph execution. +But this gap grows larger for models with less computation and there is work to +be done for optimizing hot code paths for models with lots of small operations. + + +## Work with graphs + +While eager execution makes development and debugging more interactive, +TensorFlow graph execution has advantages for distributed training, performance +optimizations, and production deployment. However, writing graph code can feel +different than writing regular Python code and more difficult to debug. + +For building and training graph-constructed models, the Python program first +builds a graph representing the computation, then invokes `Session.run` to send +the graph for execution on the C++-based runtime. This provides: + +* Automatic differentiation using static autodiff. +* Simple deployment to a platform independent server. +* Graph-based optimizations (common subexpression elimination, constant-folding, etc.). +* Compilation and kernel fusion. +* Automatic distribution and replication (placing nodes on the distributed system). + +Deploying code written for eager execution is more difficult: either generate a +graph from the model, or run the Python runtime and code directly on the server. + +### Write compatible code + +The same code written for eager execution will also build a graph during graph +execution. Do this by simply running the same code in a new Python session where +eager execution is not enabled. + +Most TensorFlow operations work during eager execution, but there are some things +to keep in mind: + +* Use `tf.data` for input processing instead of queues. It's faster and easier. +* Use object-oriented layer APIs—like `tf.keras.layers` and + `tf.keras.Model`—since they have explicit storage for variables. +* Most model code works the same during eager and graph execution, but there are + exceptions. (For example, dynamic models using Python control flow to change the + computation based on inputs.) +* Once eager execution is enabled with `tf.enable_eager_execution`, it + cannot be turned off. Start a new Python session to return to graph execution. + +It's best to write code for both eager execution *and* graph execution. This +gives you eager's interactive experimentation and debuggability with the +distributed performance benefits of graph execution. + +Write, debug, and iterate in eager execution, then import the model graph for +production deployment. Use `tfe.Checkpoint` to save and restore model +variables, this allows movement between eager and graph execution environments. +See the examples in: +[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples). + +### Use eager execution in a graph environment + +Selectively enable eager execution in a TensorFlow graph environment using +`tfe.py_func`. This is used when `tf.enable_eager_execution()` has *not* +been called. + +```py +def my_py_func(x): + x = tf.matmul(x, x) # You can use tf ops + print(x) # but it's eager! + return x + +with tf.Session() as sess: + x = tf.placeholder(dtype=tf.float32) + # Call eager function in graph! + pf = tfe.py_func(my_py_func, [x], tf.float32) + sess.run(pf, feed_dict={x: [[2.0]]}) # [[4.0]] +``` diff --git a/tensorflow/docs_src/guide/embedding.md b/tensorflow/docs_src/guide/embedding.md new file mode 100644 index 0000000000..8a98367dfb --- /dev/null +++ b/tensorflow/docs_src/guide/embedding.md @@ -0,0 +1,262 @@ +# Embeddings + +This document introduces the concept of embeddings, gives a simple example of +how to train an embedding in TensorFlow, and explains how to view embeddings +with the TensorBoard Embedding Projector +([live example](http://projector.tensorflow.org)). The first two parts target +newcomers to machine learning or TensorFlow, and the Embedding Projector how-to +is for users at all levels. + +An alternative tutorial on these concepts is available in the +[Embeddings section of Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/embeddings/video-lecture). + +[TOC] + +An **embedding** is a mapping from discrete objects, such as words, to vectors +of real numbers. For example, a 300-dimensional embedding for English words +could include: + +``` +blue: (0.01359, 0.00075997, 0.24608, ..., -0.2524, 1.0048, 0.06259) +blues: (0.01396, 0.11887, -0.48963, ..., 0.033483, -0.10007, 0.1158) +orange: (-0.24776, -0.12359, 0.20986, ..., 0.079717, 0.23865, -0.014213) +oranges: (-0.35609, 0.21854, 0.080944, ..., -0.35413, 0.38511, -0.070976) +``` + +The individual dimensions in these vectors typically have no inherent meaning. +Instead, it's the overall patterns of location and distance between vectors +that machine learning takes advantage of. + +Embeddings are important for input to machine learning. Classifiers, and neural +networks more generally, work on vectors of real numbers. They train best on +dense vectors, where all values contribute to define an object. However, many +important inputs to machine learning, such as words of text, do not have a +natural vector representation. Embedding functions are the standard and +effective way to transform such discrete input objects into useful +continuous vectors. + +Embeddings are also valuable as outputs of machine learning. Because embeddings +map objects to vectors, applications can use similarity in vector space (for +instance, Euclidean distance or the angle between vectors) as a robust and +flexible measure of object similarity. One common use is to find nearest +neighbors. Using the same word embeddings as above, for instance, here are the +three nearest neighbors for each word and the corresponding angles: + +``` +blue: (red, 47.6°), (yellow, 51.9°), (purple, 52.4°) +blues: (jazz, 53.3°), (folk, 59.1°), (bluegrass, 60.6°) +orange: (yellow, 53.5°), (colored, 58.0°), (bright, 59.9°) +oranges: (apples, 45.3°), (lemons, 48.3°), (mangoes, 50.4°) +``` + +This would tell an application that apples and oranges are in some way more +similar (45.3° apart) than lemons and oranges (48.3° apart). + +## Embeddings in TensorFlow + +To create word embeddings in TensorFlow, we first split the text into words +and then assign an integer to every word in the vocabulary. Let us assume that +this has already been done, and that `word_ids` is a vector of these integers. +For example, the sentence “I have a cat.” could be split into +`[“I”, “have”, “a”, “cat”, “.”]` and then the corresponding `word_ids` tensor +would have shape `[5]` and consist of 5 integers. To map these word ids +to vectors, we need to create the embedding variable and use the +`tf.nn.embedding_lookup` function as follows: + +``` +word_embeddings = tf.get_variable(“word_embeddings”, + [vocabulary_size, embedding_size]) +embedded_word_ids = tf.nn.embedding_lookup(word_embeddings, word_ids) +``` + +After this, the tensor `embedded_word_ids` will have shape `[5, embedding_size]` +in our example and contain the embeddings (dense vectors) for each of the 5 +words. At the end of training, `word_embeddings` will contain the embeddings +for all words in the vocabulary. + +Embeddings can be trained in many network types, and with various loss +functions and data sets. For example, one could use a recurrent neural network +to predict the next word from the previous one given a large corpus of +sentences, or one could train two networks to do multi-lingual translation. +These methods are described in the @{$word2vec$Vector Representations of Words} +tutorial. + +## Visualizing Embeddings + +TensorBoard includes the **Embedding Projector**, a tool that lets you +interactively visualize embeddings. This tool can read embeddings from your +model and render them in two or three dimensions. + +The Embedding Projector has three panels: + +- *Data panel* on the top left, where you can choose the run, the embedding + variable and data columns to color and label points by. +- *Projections panel* on the bottom left, where you can choose the type of + projection. +- *Inspector panel* on the right side, where you can search for particular + points and see a list of nearest neighbors. + +### Projections +The Embedding Projector provides three ways to reduce the dimensionality of a +data set. + +- *[t-SNE](https://en.wikipedia.org/wiki/T-distributed_stochastic_neighbor_embedding)*: + a nonlinear nondeterministic algorithm (T-distributed stochastic neighbor + embedding) that tries to preserve local neighborhoods in the data, often at + the expense of distorting global structure. You can choose whether to compute + two- or three-dimensional projections. + +- *[PCA](https://en.wikipedia.org/wiki/Principal_component_analysis)*: + a linear deterministic algorithm (principal component analysis) that tries to + capture as much of the data variability in as few dimensions as possible. PCA + tends to highlight large-scale structure in the data, but can distort local + neighborhoods. The Embedding Projector computes the top 10 principal + components, from which you can choose two or three to view. + +- *Custom*: a linear projection onto horizontal and vertical axes that you + specify using labels in the data. You define the horizontal axis, for + instance, by giving text patterns for "Left" and "Right". The Embedding + Projector finds all points whose label matches the "Left" pattern and + computes the centroid of that set; similarly for "Right". The line passing + through these two centroids defines the horizontal axis. The vertical axis is + likewise computed from the centroids for points matching the "Up" and "Down" + text patterns. + +Further useful articles are +[How to Use t-SNE Effectively](https://distill.pub/2016/misread-tsne/) and +[Principal Component Analysis Explained Visually](http://setosa.io/ev/principal-component-analysis/). + +### Exploration + +You can explore visually by zooming, rotating, and panning using natural +click-and-drag gestures. Hovering your mouse over a point will show any +[metadata](#metadata) for that point. You can also inspect nearest-neighbor +subsets. Clicking on a point causes the right pane to list the nearest +neighbors, along with distances to the current point. The nearest-neighbor +points are also highlighted in the projection. + +It is sometimes useful to restrict the view to a subset of points and perform +projections only on those points. To do so, you can select points in multiple +ways: + +- After clicking on a point, its nearest neighbors are also selected. +- After a search, the points matching the query are selected. +- Enabling selection, clicking on a point and dragging defines a selection + sphere. + +Then click the "Isolate *nnn* points" button at the top of the Inspector pane +on the right hand side. The following image shows 101 points selected and ready +for the user to click "Isolate 101 points": + +![Selection of nearest neighbors](https://www.tensorflow.org/images/embedding-nearest-points.png "Selection of nearest neighbors") + +*Selection of the nearest neighbors of “important” in a word embedding dataset.* + +Advanced tip: filtering with custom projection can be powerful. Below, we +filtered the 100 nearest neighbors of “politics” and projected them onto the +“worst” - “best” vector as an x axis. The y axis is random. As a result, one +finds on the right side “ideas”, “science”, “perspective”, “journalism” but on +the left “crisis”, “violence” and “conflict”. + + + + + + + + + + +
+ Custom controls panel + + Custom projection +
+ Custom projection controls. + + Custom projection of neighbors of "politics" onto "best" - "worst" vector. +
+ +To share your findings, you can use the bookmark panel in the bottom right +corner and save the current state (including computed coordinates of any +projection) as a small file. The Projector can then be pointed to a set of one +or more of these files, producing the panel below. Other users can then walk +through a sequence of bookmarks. + +Bookmark panel + +### Metadata + +If you are working with an embedding, you'll probably want to attach +labels/images to the data points. You can do this by generating a metadata file +containing the labels for each point and clicking "Load data" in the data panel +of the Embedding Projector. + +The metadata can be either labels or images, which are +stored in a separate file. For labels, the format should +be a [TSV file](https://en.wikipedia.org/wiki/Tab-separated_values) +(tab characters shown in red) whose first line contains column headers +(shown in bold) and subsequent lines contain the metadata values. For example: + + +Word\tFrequency
+ Airplane\t345
+ Car\t241
+ ... + + +The order of lines in the metadata file is assumed to match the order of +vectors in the embedding variable, except for the header. Consequently, the +(i+1)-th line in the metadata file corresponds to the i-th row of the embedding +variable. If the TSV metadata file has only a single column, then we don’t +expect a header row, and assume each row is the label of the embedding. We +include this exception because it matches the commonly-used "vocab file" +format. + +To use images as metadata, you must produce a single +[sprite image](https://www.google.com/webhp#q=what+is+a+sprite+image), +consisting of small thumbnails, one for each vector in the embedding. The +sprite should store thumbnails in row-first order: the first data point placed +in the top left and the last data point in the bottom right, though the last +row doesn't have to be filled, as shown below. + + + + + + + + + + + + + + + + + +
012
345
67
+ +Follow [this link](https://www.tensorflow.org/images/embedding-mnist.mp4) +to see a fun example of thumbnail images in the Embedding Projector. + + +## Mini-FAQ + +**Is "embedding" an action or a thing?** +Both. People talk about embedding words in a vector space (action) and about +producing word embeddings (things). Common to both is the notion of embedding +as a mapping from discrete objects to vectors. Creating or applying that +mapping is an action, but the mapping itself is a thing. + +**Are embeddings high-dimensional or low-dimensional?** +It depends. A 300-dimensional vector space of words and phrases, for instance, +is often called low-dimensional (and dense) when compared to the millions of +words and phrases it can contain. But mathematically it is high-dimensional, +displaying many properties that are dramatically different from what our human +intuition has learned about 2- and 3-dimensional spaces. + +**Is an embedding the same as an embedding layer?** +No. An *embedding layer* is a part of neural network, but an *embedding* is a more +general concept. diff --git a/tensorflow/docs_src/guide/estimators.md b/tensorflow/docs_src/guide/estimators.md new file mode 100644 index 0000000000..78b30c3040 --- /dev/null +++ b/tensorflow/docs_src/guide/estimators.md @@ -0,0 +1,193 @@ +# Estimators + +This document introduces @{tf.estimator$**Estimators**}--a high-level TensorFlow +API that greatly simplifies machine learning programming. Estimators encapsulate +the following actions: + +* training +* evaluation +* prediction +* export for serving + +You may either use the pre-made Estimators we provide or write your +own custom Estimators. All Estimators--whether pre-made or custom--are +classes based on the @{tf.estimator.Estimator} class. + +Note: TensorFlow also includes a deprecated `Estimator` class at +@{tf.contrib.learn.Estimator}, which you should not use. + + +## Advantages of Estimators + +Estimators provide the following benefits: + +* You can run Estimator-based models on a local host or on a + distributed multi-server environment without changing your model. + Furthermore, you can run Estimator-based models on CPUs, GPUs, + or TPUs without recoding your model. +* Estimators simplify sharing implementations between model developers. +* You can develop a state of the art model with high-level intuitive code. + In short, it is generally much easier to create models with Estimators + than with the low-level TensorFlow APIs. +* Estimators are themselves built on @{tf.layers}, which + simplifies customization. +* Estimators build the graph for you. +* Estimators provide a safe distributed training loop that controls how and + when to: + * build the graph + * initialize variables + * start queues + * handle exceptions + * create checkpoint files and recover from failures + * save summaries for TensorBoard + +When writing an application with Estimators, you must separate the data input +pipeline from the model. This separation simplifies experiments with +different data sets. + + +## Pre-made Estimators + +Pre-made Estimators enable you to work at a much higher conceptual level +than the base TensorFlow APIs. You no longer have to worry about creating +the computational graph or sessions since Estimators handle all +the "plumbing" for you. That is, pre-made Estimators create and manage +@{tf.Graph$`Graph`} and @{tf.Session$`Session`} objects for you. Furthermore, +pre-made Estimators let you experiment with different model architectures by +making only minimal code changes. @{tf.estimator.DNNClassifier$`DNNClassifier`}, +for example, is a pre-made Estimator class that trains classification models +based on dense, feed-forward neural networks. + + +### Structure of a pre-made Estimators program + +A TensorFlow program relying on a pre-made Estimator typically consists +of the following four steps: + +1. **Write one or more dataset importing functions.** For example, you might + create one function to import the training set and another function to + import the test set. Each dataset importing function must return two + objects: + + * a dictionary in which the keys are feature names and the + values are Tensors (or SparseTensors) containing the corresponding + feature data + * a Tensor containing one or more labels + + For example, the following code illustrates the basic skeleton for + an input function: + + def input_fn(dataset): + ... # manipulate dataset, extracting the feature dict and the label + return feature_dict, label + + (See @{$guide/datasets} for full details.) + +2. **Define the feature columns.** Each @{tf.feature_column} + identifies a feature name, its type, and any input pre-processing. + For example, the following snippet creates three feature + columns that hold integer or floating-point data. The first two + feature columns simply identify the feature's name and type. The + third feature column also specifies a lambda the program will invoke + to scale the raw data: + + # Define three numeric feature columns. + population = tf.feature_column.numeric_column('population') + crime_rate = tf.feature_column.numeric_column('crime_rate') + median_education = tf.feature_column.numeric_column('median_education', + normalizer_fn=lambda x: x - global_education_mean) + +3. **Instantiate the relevant pre-made Estimator.** For example, here's + a sample instantiation of a pre-made Estimator named `LinearClassifier`: + + # Instantiate an estimator, passing the feature columns. + estimator = tf.estimator.LinearClassifier( + feature_columns=[population, crime_rate, median_education], + ) + +4. **Call a training, evaluation, or inference method.** + For example, all Estimators provide a `train` method, which trains a model. + + # my_training_set is the function created in Step 1 + estimator.train(input_fn=my_training_set, steps=2000) + + +### Benefits of pre-made Estimators + +Pre-made Estimators encode best practices, providing the following benefits: + +* Best practices for determining where different parts of the computational + graph should run, implementing strategies on a single machine or on a + cluster. +* Best practices for event (summary) writing and universally useful + summaries. + +If you don't use pre-made Estimators, you must implement the preceding +features yourself. + + +## Custom Estimators + +The heart of every Estimator--whether pre-made or custom--is its +**model function**, which is a method that builds graphs for training, +evaluation, and prediction. When you are using a pre-made Estimator, +someone else has already implemented the model function. When relying +on a custom Estimator, you must write the model function yourself. A +@{$custom_estimators$companion document} +explains how to write the model function. + + +## Recommended workflow + +We recommend the following workflow: + +1. Assuming a suitable pre-made Estimator exists, use it to build your + first model and use its results to establish a baseline. +2. Build and test your overall pipeline, including the integrity and + reliability of your data with this pre-made Estimator. +3. If suitable alternative pre-made Estimators are available, run + experiments to determine which pre-made Estimator produces the + best results. +4. Possibly, further improve your model by building your own custom Estimator. + + +## Creating Estimators from Keras models + +You can convert existing Keras models to Estimators. Doing so enables your Keras +model to access Estimator's strengths, such as distributed training. Call +@{tf.keras.estimator.model_to_estimator} as in the +following sample: + +```python +# Instantiate a Keras inception v3 model. +keras_inception_v3 = tf.keras.applications.inception_v3.InceptionV3(weights=None) +# Compile model with the optimizer, loss, and metrics you'd like to train with. +keras_inception_v3.compile(optimizer=tf.keras.optimizers.SGD(lr=0.0001, momentum=0.9), + loss='categorical_crossentropy', + metric='accuracy') +# Create an Estimator from the compiled Keras model. Note the initial model +# state of the keras model is preserved in the created Estimator. +est_inception_v3 = tf.keras.estimator.model_to_estimator(keras_model=keras_inception_v3) + +# Treat the derived Estimator as you would with any other Estimator. +# First, recover the input name(s) of Keras model, so we can use them as the +# feature column name(s) of the Estimator input function: +keras_inception_v3.input_names # print out: ['input_1'] +# Once we have the input name(s), we can create the input function, for example, +# for input(s) in the format of numpy ndarray: +train_input_fn = tf.estimator.inputs.numpy_input_fn( + x={"input_1": train_data}, + y=train_labels, + num_epochs=1, + shuffle=False) +# To train, we call Estimator's train function: +est_inception_v3.train(input_fn=train_input_fn, steps=2000) +``` +Note that the names of feature columns and labels of a keras estimator come from +the corresponding compiled keras model. For example, the input key names for +`train_input_fn` above can be obtained from `keras_inception_v3.input_names`, +and similarly, the predicted output names can be obtained from +`keras_inception_v3.output_names`. + +For more details, please refer to the documentation for +@{tf.keras.estimator.model_to_estimator}. diff --git a/tensorflow/docs_src/guide/faq.md b/tensorflow/docs_src/guide/faq.md new file mode 100644 index 0000000000..b6291a9ffa --- /dev/null +++ b/tensorflow/docs_src/guide/faq.md @@ -0,0 +1,297 @@ +# Frequently Asked Questions + +This document provides answers to some of the frequently asked questions about +TensorFlow. If you have a question that is not covered here, you might find an +answer on one of the TensorFlow @{$about$community resources}. + +[TOC] + +## Features and Compatibility + +#### Can I run distributed training on multiple computers? + +Yes! TensorFlow gained +@{$distributed$support for distributed computation} in +version 0.8. TensorFlow now supports multiple devices (CPUs and GPUs) in one or +more computers. + +#### Does TensorFlow work with Python 3? + +As of the 0.6.0 release timeframe (Early December 2015), we do support Python +3.3+. + +## Building a TensorFlow graph + +See also the +@{$python/framework$API documentation on building graphs}. + +#### Why does `c = tf.matmul(a, b)` not execute the matrix multiplication immediately? + +In the TensorFlow Python API, `a`, `b`, and `c` are +@{tf.Tensor} objects. A `Tensor` object is +a symbolic handle to the result of an operation, but does not actually hold the +values of the operation's output. Instead, TensorFlow encourages users to build +up complicated expressions (such as entire neural networks and its gradients) as +a dataflow graph. You then offload the computation of the entire dataflow graph +(or a subgraph of it) to a TensorFlow +@{tf.Session}, which is able to execute the +whole computation much more efficiently than executing the operations +one-by-one. + +#### How are devices named? + +The supported device names are `"/device:CPU:0"` (or `"/cpu:0"`) for the CPU +device, and `"/device:GPU:i"` (or `"/gpu:i"`) for the *i*th GPU device. + +#### How do I place operations on a particular device? + +To place a group of operations on a device, create them within a +@{tf.device$`with tf.device(name):`} context. See +the how-to documentation on +@{$using_gpu$using GPUs with TensorFlow} for details of how +TensorFlow assigns operations to devices, and the +@{$deep_cnn$CIFAR-10 tutorial} for an example model that +uses multiple GPUs. + + +## Running a TensorFlow computation + +See also the +@{$python/client$API documentation on running graphs}. + +#### What's the deal with feeding and placeholders? + +Feeding is a mechanism in the TensorFlow Session API that allows you to +substitute different values for one or more tensors at run time. The `feed_dict` +argument to @{tf.Session.run} is a +dictionary that maps @{tf.Tensor} objects to +numpy arrays (and some other types), which will be used as the values of those +tensors in the execution of a step. + +#### What is the difference between `Session.run()` and `Tensor.eval()`? + +If `t` is a @{tf.Tensor} object, +@{tf.Tensor.eval} is shorthand for +@{tf.Session.run}, where `sess` is the +current @{tf.get_default_session}. The +two following snippets of code are equivalent: + +```python +# Using `Session.run()`. +sess = tf.Session() +c = tf.constant(5.0) +print(sess.run(c)) + +# Using `Tensor.eval()`. +c = tf.constant(5.0) +with tf.Session(): + print(c.eval()) +``` + +In the second example, the session acts as a +[context manager](https://docs.python.org/2.7/reference/compound_stmts.html#with), +which has the effect of installing it as the default session for the lifetime of +the `with` block. The context manager approach can lead to more concise code for +simple use cases (like unit tests); if your code deals with multiple graphs and +sessions, it may be more straightforward to make explicit calls to +`Session.run()`. + +#### Do Sessions have a lifetime? What about intermediate tensors? + +Sessions can own resources, such as +@{tf.Variable}, +@{tf.QueueBase}, and +@{tf.ReaderBase}. These resources can sometimes use +a significant amount of memory, and can be released when the session is closed by calling +@{tf.Session.close}. + +The intermediate tensors that are created as part of a call to +@{$python/client$`Session.run()`} will be freed at or before the +end of the call. + +#### Does the runtime parallelize parts of graph execution? + +The TensorFlow runtime parallelizes graph execution across many different +dimensions: + +* The individual ops have parallel implementations, using multiple cores in a + CPU, or multiple threads in a GPU. +* Independent nodes in a TensorFlow graph can run in parallel on multiple + devices, which makes it possible to speed up + @{$deep_cnn$CIFAR-10 training using multiple GPUs}. +* The Session API allows multiple concurrent steps (i.e. calls to + @{tf.Session.run} in parallel). This + enables the runtime to get higher throughput, if a single step does not use + all of the resources in your computer. + +#### Which client languages are supported in TensorFlow? + +TensorFlow is designed to support multiple client languages. +Currently, the best-supported client language is [Python](../api_docs/python/index.md). Experimental interfaces for +executing and constructing graphs are also available for +[C++](../api_docs/cc/index.md), [Java](../api_docs/java/reference/org/tensorflow/package-summary.html) and [Go](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go). + +TensorFlow also has a +[C-based client API](https://www.tensorflow.org/code/tensorflow/c/c_api.h) +to help build support for more client languages. We invite contributions of new +language bindings. + +Bindings for various other languages (such as [C#](https://github.com/migueldeicaza/TensorFlowSharp), [Julia](https://github.com/malmaud/TensorFlow.jl), [Ruby](https://github.com/somaticio/tensorflow.rb) and [Scala](https://github.com/eaplatanios/tensorflow_scala)) created and supported by the open source community build on top of the C API supported by the TensorFlow maintainers. + +#### Does TensorFlow make use of all the devices (GPUs and CPUs) available on my machine? + +TensorFlow supports multiple GPUs and CPUs. See the how-to documentation on +@{$using_gpu$using GPUs with TensorFlow} for details of how +TensorFlow assigns operations to devices, and the +@{$deep_cnn$CIFAR-10 tutorial} for an example model that +uses multiple GPUs. + +Note that TensorFlow only uses GPU devices with a compute capability greater +than 3.5. + +#### Why does `Session.run()` hang when using a reader or a queue? + +The @{tf.ReaderBase} and +@{tf.QueueBase} classes provide special operations that +can *block* until input (or free space in a bounded queue) becomes +available. These operations allow you to build sophisticated +@{$reading_data$input pipelines}, at the cost of making the +TensorFlow computation somewhat more complicated. See the how-to documentation +for +@{$reading_data#creating_threads_to_prefetch_using_queuerunner_objects$using `QueueRunner` objects to drive queues and readers} +for more information on how to use them. + +## Variables + +See also the how-to documentation on @{$variables$variables} and +@{$python/state_ops$the API documentation for variables}. + +#### What is the lifetime of a variable? + +A variable is created when you first run the +@{tf.Variable.initializer} +operation for that variable in a session. It is destroyed when that +@{tf.Session.close}. + +#### How do variables behave when they are concurrently accessed? + +Variables allow concurrent read and write operations. The value read from a +variable may change if it is concurrently updated. By default, concurrent +assignment operations to a variable are allowed to run with no mutual exclusion. +To acquire a lock when assigning to a variable, pass `use_locking=True` to +@{tf.Variable.assign}. + +## Tensor shapes + +See also the +@{tf.TensorShape}. + +#### How can I determine the shape of a tensor in Python? + +In TensorFlow, a tensor has both a static (inferred) shape and a dynamic (true) +shape. The static shape can be read using the +@{tf.Tensor.get_shape} +method: this shape is inferred from the operations that were used to create the +tensor, and may be +@{tf.TensorShape$partially complete}. If the static +shape is not fully defined, the dynamic shape of a `Tensor` `t` can be +determined by evaluating @{tf.shape$`tf.shape(t)`}. + +#### What is the difference between `x.set_shape()` and `x = tf.reshape(x)`? + +The @{tf.Tensor.set_shape} method updates +the static shape of a `Tensor` object, and it is typically used to provide +additional shape information when this cannot be inferred directly. It does not +change the dynamic shape of the tensor. + +The @{tf.reshape} operation creates +a new tensor with a different dynamic shape. + +#### How do I build a graph that works with variable batch sizes? + +It is often useful to build a graph that works with variable batch sizes +so that the same code can be used for (mini-)batch training, and +single-instance inference. The resulting graph can be +@{tf.Graph.as_graph_def$saved as a protocol buffer} +and +@{tf.import_graph_def$imported into another program}. + +When building a variable-size graph, the most important thing to remember is not +to encode the batch size as a Python constant, but instead to use a symbolic +`Tensor` to represent it. The following tips may be useful: + +* Use [`batch_size = tf.shape(input)[0]`](../api_docs/python/array_ops.md#shape) + to extract the batch dimension from a `Tensor` called `input`, and store it in + a `Tensor` called `batch_size`. + +* Use @{tf.reduce_mean} instead + of `tf.reduce_sum(...) / batch_size`. + + +## TensorBoard + +#### How can I visualize a TensorFlow graph? + +See the @{$graph_viz$graph visualization tutorial}. + +#### What is the simplest way to send data to TensorBoard? + +Add summary ops to your TensorFlow graph, and write +these summaries to a log directory. Then, start TensorBoard using + + python tensorflow/tensorboard/tensorboard.py --logdir=path/to/log-directory + +For more details, see the +@{$summaries_and_tensorboard$Summaries and TensorBoard tutorial}. + +#### Every time I launch TensorBoard, I get a network security popup! + +You can change TensorBoard to serve on localhost rather than '0.0.0.0' by +the flag --host=localhost. This should quiet any security warnings. + +## Extending TensorFlow + +See the how-to documentation for +@{$adding_an_op$adding a new operation to TensorFlow}. + +#### My data is in a custom format. How do I read it using TensorFlow? + +There are three main options for dealing with data in a custom format. + +The easiest option is to write parsing code in Python that transforms the data +into a numpy array. Then, use @{tf.data.Dataset.from_tensor_slices} to +create an input pipeline from the in-memory data. + +If your data doesn't fit in memory, try doing the parsing in the Dataset +pipeline. Start with an appropriate file reader, like +@{tf.data.TextLineDataset}. Then convert the dataset by mapping +@{tf.data.Dataset.map$mapping} appropriate operations over it. +Prefer predefined TensorFlow operations such as @{tf.decode_raw}, +@{tf.decode_csv}, @{tf.parse_example}, or @{tf.image.decode_png}. + +If your data is not easily parsable with the built-in TensorFlow operations, +consider converting it, offline, to a format that is easily parsable, such +as @{tf.python_io.TFRecordWriter$`TFRecord`} format. + +The most efficient method to customize the parsing behavior is to +@{$adding_an_op$add a new op written in C++} that parses your +data format. The @{$new_data_formats$guide to handling new data formats} has +more information about the steps for doing this. + + +## Miscellaneous + +#### What is TensorFlow's coding style convention? + +The TensorFlow Python API adheres to the +[PEP8](https://www.python.org/dev/peps/pep-0008/) conventions.* In +particular, we use `CamelCase` names for classes, and `snake_case` names for +functions, methods, and properties. We also adhere to the +[Google Python style guide](https://google.github.io/styleguide/pyguide.html). + +The TensorFlow C++ code base adheres to the +[Google C++ style guide](https://google.github.io/styleguide/cppguide.html). + +(* With one exception: we use 2-space indentation instead of 4-space +indentation.) + diff --git a/tensorflow/docs_src/guide/feature_columns.md b/tensorflow/docs_src/guide/feature_columns.md new file mode 100644 index 0000000000..1013ec910c --- /dev/null +++ b/tensorflow/docs_src/guide/feature_columns.md @@ -0,0 +1,572 @@ +# Feature Columns + +This document details feature columns. Think of **feature columns** as the +intermediaries between raw data and Estimators. Feature columns are very rich, +enabling you to transform a diverse range of raw data into formats that +Estimators can use, allowing easy experimentation. + +In @{$premade_estimators$Premade Estimators}, we used the premade +Estimator, @{tf.estimator.DNNClassifier$`DNNClassifier`} to train a model to +predict different types of Iris flowers from four input features. That example +created only numerical feature columns (of type +@{tf.feature_column.numeric_column}). Although numerical feature columns model +the lengths of petals and sepals effectively, real world data sets contain all +kinds of features, many of which are non-numerical. + +
+ +
+
+Some real-world features (such as, longitude) are numerical, but many are not. +
+ +## Input to a Deep Neural Network + +What kind of data can a deep neural network operate on? The answer +is, of course, numbers (for example, `tf.float32`). After all, every neuron in +a neural network performs multiplication and addition operations on weights and +input data. Real-life input data, however, often contains non-numerical +(categorical) data. For example, consider a `product_class` feature that can +contain the following three non-numerical values: + +* `kitchenware` +* `electronics` +* `sports` + +ML models generally represent categorical values as simple vectors in which a +1 represents the presence of a value and a 0 represents the absence of a value. +For example, when `product_class` is set to `sports`, an ML model would usually +represent `product_class` as `[0, 0, 1]`, meaning: + +* `0`: `kitchenware` is absent +* `0`: `electronics` is absent +* `1`: `sports` is present + +So, although raw data can be numerical or categorical, an ML model represents +all features as numbers. + +## Feature Columns + +As the following figure suggests, you specify the input to a model through the +`feature_columns` argument of an Estimator (`DNNClassifier` for Iris). +Feature Columns bridge input data (as returned by `input_fn`) with your model. + +
+ +
+
+Feature columns bridge raw data with the data your model needs. +
+ +To create feature columns, call functions from the +@{tf.feature_column} module. This document explains nine of the functions in +that module. As the following figure shows, all nine functions return either a +Categorical-Column or a Dense-Column object, except `bucketized_column`, which +inherits from both classes: + +
+ +
+
+Feature column methods fall into two main categories and one hybrid category. +
+ +Let's look at these functions in more detail. + +### Numeric column + +The Iris classifier calls the @{tf.feature_column.numeric_column} function for +all input features: + + * `SepalLength` + * `SepalWidth` + * `PetalLength` + * `PetalWidth` + +Although `tf.numeric_column` provides optional arguments, calling +`tf.numeric_column` without any arguments, as follows, is a fine way to specify +a numerical value with the default data type (`tf.float32`) as input to your +model: + +```python +# Defaults to a tf.float32 scalar. +numeric_feature_column = tf.feature_column.numeric_column(key="SepalLength") +``` + +To specify a non-default numerical data type, use the `dtype` argument. For +example: + +``` python +# Represent a tf.float64 scalar. +numeric_feature_column = tf.feature_column.numeric_column(key="SepalLength", + dtype=tf.float64) +``` + +By default, a numeric column creates a single value (scalar). Use the shape +argument to specify another shape. For example: + + +```python +# Represent a 10-element vector in which each cell contains a tf.float32. +vector_feature_column = tf.feature_column.numeric_column(key="Bowling", + shape=10) + +# Represent a 10x5 matrix in which each cell contains a tf.float32. +matrix_feature_column = tf.feature_column.numeric_column(key="MyMatrix", + shape=[10,5]) +``` +### Bucketized column + +Often, you don't want to feed a number directly into the model, but instead +split its value into different categories based on numerical ranges. To do so, +create a @{tf.feature_column.bucketized_column$bucketized column}. For +example, consider raw data that represents the year a house was built. Instead +of representing that year as a scalar numeric column, we could split the year +into the following four buckets: + +
+ +
+
+Dividing year data into four buckets. +
+ +The model will represent the buckets as follows: + +|Date Range |Represented as... | +|:----------|:-----------------| +|< 1960 | [1, 0, 0, 0] | +|>= 1960 but < 1980 | [0, 1, 0, 0] | +|>= 1980 but < 2000 | [0, 0, 1, 0] | +|>= 2000 | [0, 0, 0, 1] | + +Why would you want to split a number—a perfectly valid input to your +model—into a categorical value? Well, notice that the categorization splits a +single input number into a four-element vector. Therefore, the model now can +learn _four individual weights_ rather than just one; four weights creates a +richer model than one weight. More importantly, bucketizing enables the model +to clearly distinguish between different year categories since only one of the +elements is set (1) and the other three elements are cleared (0). For example, +when we just use a single number (a year) as input, a linear model can only +learn a linear relationship. So, bucketing provides the model with additional +flexibility that the model can use to learn. + +The following code demonstrates how to create a bucketized feature: + + +```python +# First, convert the raw input to a numeric column. +numeric_feature_column = tf.feature_column.numeric_column("Year") + +# Then, bucketize the numeric column on the years 1960, 1980, and 2000. +bucketized_feature_column = tf.feature_column.bucketized_column( + source_column = numeric_feature_column, + boundaries = [1960, 1980, 2000]) +``` +Note that specifying a _three_-element boundaries vector creates a +_four_-element bucketized vector. + + +### Categorical identity column + +**Categorical identity columns** can be seen as a special case of bucketized +columns. In traditional bucketized columns, each bucket represents a range of +values (for example, from 1960 to 1979). In a categorical identity column, each +bucket represents a single, unique integer. For example, let's say you want to +represent the integer range `[0, 4)`. That is, you want to represent the +integers 0, 1, 2, or 3. In this case, the categorical identity mapping looks +like this: + +
+ +
+
+A categorical identity column mapping. Note that this is a one-hot +encoding, not a binary numerical encoding. +
+ +As with bucketized columns, a model can learn a separate weight for each class +in a categorical identity column. For example, instead of using a string to +represent the `product_class`, let's represent each class with a unique integer +value. That is: + +* `0="kitchenware"` +* `1="electronics"` +* `2="sport"` + +Call @{tf.feature_column.categorical_column_with_identity} to implement a +categorical identity column. For example: + +``` python +# Create categorical output for an integer feature named "my_feature_b", +# The values of my_feature_b must be >= 0 and < num_buckets +identity_feature_column = tf.feature_column.categorical_column_with_identity( + key='my_feature_b', + num_buckets=4) # Values [0, 4) + +# In order for the preceding call to work, the input_fn() must return +# a dictionary containing 'my_feature_b' as a key. Furthermore, the values +# assigned to 'my_feature_b' must belong to the set [0, 4). +def input_fn(): + ... + return ({ 'my_feature_a':[7, 9, 5, 2], 'my_feature_b':[3, 1, 2, 2] }, + [Label_values]) +``` + +### Categorical vocabulary column + +We cannot input strings directly to a model. Instead, we must first map strings +to numeric or categorical values. Categorical vocabulary columns provide a good +way to represent strings as a one-hot vector. For example: + +
+ +
+
+Mapping string values to vocabulary columns. +
+ +As you can see, categorical vocabulary columns are kind of an enum version of +categorical identity columns. TensorFlow provides two different functions to +create categorical vocabulary columns: + +* @{tf.feature_column.categorical_column_with_vocabulary_list} +* @{tf.feature_column.categorical_column_with_vocabulary_file} + +`categorical_column_with_vocabulary_list` maps each string to an integer based +on an explicit vocabulary list. For example: + +```python +# Given input "feature_name_from_input_fn" which is a string, +# create a categorical feature by mapping the input to one of +# the elements in the vocabulary list. +vocabulary_feature_column = + tf.feature_column.categorical_column_with_vocabulary_list( + key=feature_name_from_input_fn, + vocabulary_list=["kitchenware", "electronics", "sports"]) +``` + +The preceding function is pretty straightforward, but it has a significant +drawback. Namely, there's way too much typing when the vocabulary list is long. +For these cases, call +`tf.feature_column.categorical_column_with_vocabulary_file` instead, which lets +you place the vocabulary words in a separate file. For example: + +```python + +# Given input "feature_name_from_input_fn" which is a string, +# create a categorical feature to our model by mapping the input to one of +# the elements in the vocabulary file +vocabulary_feature_column = + tf.feature_column.categorical_column_with_vocabulary_file( + key=feature_name_from_input_fn, + vocabulary_file="product_class.txt", + vocabulary_size=3) +``` + +`product_class.txt` should contain one line for each vocabulary element. In our +case: + +```None +kitchenware +electronics +sports +``` + +### Hashed Column + +So far, we've worked with a naively small number of categories. For example, +our product_class example has only 3 categories. Often though, the number of +categories can be so big that it's not possible to have individual categories +for each vocabulary word or integer because that would consume too much memory. +For these cases, we can instead turn the question around and ask, "How many +categories am I willing to have for my input?" In fact, the +@{tf.feature_column.categorical_column_with_hash_bucket} function enables you +to specify the number of categories. For this type of feature column the model +calculates a hash value of the input, then puts it into one of +the `hash_bucket_size` categories using the modulo operator, as in the following +pseudocode: + +```python +# pseudocode +feature_id = hash(raw_feature) % hash_buckets_size +``` + +The code to create the `feature_column` might look something like this: + +``` python +hashed_feature_column = + tf.feature_column.categorical_column_with_hash_bucket( + key = "some_feature", + hash_buckets_size = 100) # The number of categories +``` +At this point, you might rightfully think: "This is crazy!" After all, we are +forcing the different input values to a smaller set of categories. This means +that two probably unrelated inputs will be mapped to the same +category, and consequently mean the same thing to the neural network. The +following figure illustrates this dilemma, showing that kitchenware and sports +both get assigned to category (hash bucket) 12: + +
+ +
+
+Representing data with hash buckets. +
+ +As with many counterintuitive phenomena in machine learning, it turns out that +hashing often works well in practice. That's because hash categories provide +the model with some separation. The model can use additional features to further +separate kitchenware from sports. + +### Crossed column + +Combining features into a single feature, better known as +[feature crosses](https://developers.google.com/machine-learning/glossary/#feature_cross), +enables the model to learn separate weights for each combination of +features. + +More concretely, suppose we want our model to calculate real estate prices in +Atlanta, GA. Real-estate prices within this city vary greatly depending on +location. Representing latitude and longitude as separate features isn't very +useful in identifying real-estate location dependencies; however, crossing +latitude and longitude into a single feature can pinpoint locations. Suppose we +represent Atlanta as a grid of 100x100 rectangular sections, identifying each +of the 10,000 sections by a feature cross of latitude and longitude. This +feature cross enables the model to train on pricing conditions related to each +individual section, which is a much stronger signal than latitude and longitude +alone. + +The following figure shows our plan, with the latitude & longitude values for +the corners of the city in red text: + +
+ +
+
+Map of Atlanta. Imagine this map divided into 10,000 sections of +equal size. +
+ +For the solution, we used a combination of the `bucketized_column` we looked at +earlier, with the @{tf.feature_column.crossed_column} function. + + + +``` python +def make_dataset(latitude, longitude, labels): + assert latitude.shape == longitude.shape == labels.shape + + features = {'latitude': latitude.flatten(), + 'longitude': longitude.flatten()} + labels=labels.flatten() + + return tf.data.Dataset.from_tensor_slices((features, labels)) + + +# Bucketize the latitude and longitude using the `edges` +latitude_bucket_fc = tf.feature_column.bucketized_column( + tf.feature_column.numeric_column('latitude'), + list(atlanta.latitude.edges)) + +longitude_bucket_fc = tf.feature_column.bucketized_column( + tf.feature_column.numeric_column('longitude'), + list(atlanta.longitude.edges)) + +# Cross the bucketized columns, using 5000 hash bins. +crossed_lat_lon_fc = tf.feature_column.crossed_column( + [latitude_bucket_fc, longitude_bucket_fc], 5000) + +fc = [ + latitude_bucket_fc, + longitude_bucket_fc, + crossed_lat_lon_fc] + +# Build and train the Estimator. +est = tf.estimator.LinearRegressor(fc, ...) +``` + +You may create a feature cross from either of the following: + +* Feature names; that is, names from the `dict` returned from `input_fn`. +* Any categorical column, except `categorical_column_with_hash_bucket` + (since `crossed_column` hashes the input). + +When the feature columns `latitude_bucket_fc` and `longitude_bucket_fc` are +crossed, TensorFlow will create `(latitude_fc, longitude_fc)` pairs for each +example. This would produce a full grid of possibilities as follows: + +``` None + (0,0), (0,1)... (0,99) + (1,0), (1,1)... (1,99) + ... ... ... +(99,0), (99,1)...(99, 99) +``` + +Except that a full grid would only be tractable for inputs with limited +vocabularies. Instead of building this, potentially huge, table of inputs, +the `crossed_column` only builds the number requested by the `hash_bucket_size` +argument. The feature column assigns an example to a index by running a hash +function on the tuple of inputs, followed by a modulo operation with +`hash_bucket_size`. + +As discussed earlier, performing the +hash and modulo function limits the number of categories, but can cause category +collisions; that is, multiple (latitude, longitude) feature crosses will end +up in the same hash bucket. In practice though, performing feature crosses +still adds significant value to the learning capability of your models. + +Somewhat counterintuitively, when creating feature crosses, you typically still +should include the original (uncrossed) features in your model (as in the +preceding code snippet). The independent latitude and longitude features help the +model distinguish between examples where a hash collision has occurred in the +crossed feature. + +## Indicator and embedding columns + +Indicator columns and embedding columns never work on features directly, but +instead take categorical columns as input. + +When using an indicator column, we're telling TensorFlow to do exactly what +we've seen in our categorical product_class example. That is, an +**indicator column** treats each category as an element in a one-hot vector, +where the matching category has value 1 and the rest have 0s: + +
+ +
+
+Representing data in indicator columns. +
+ +Here's how you create an indicator column by calling +@{tf.feature_column.indicator_column}: + +``` python +categorical_column = ... # Create any type of categorical column. + +# Represent the categorical column as an indicator column. +indicator_column = tf.feature_column.indicator_column(categorical_column) +``` + +Now, suppose instead of having just three possible classes, we have a million. +Or maybe a billion. For a number of reasons, as the number of categories grow +large, it becomes infeasible to train a neural network using indicator columns. + +We can use an embedding column to overcome this limitation. Instead of +representing the data as a one-hot vector of many dimensions, an +**embedding column** represents that data as a lower-dimensional, ordinary +vector in which each cell can contain any number, not just 0 or 1. By +permitting a richer palette of numbers for every cell, an embedding column +contains far fewer cells than an indicator column. + +Let's look at an example comparing indicator and embedding columns. Suppose our +input examples consist of different words from a limited palette of only 81 +words. Further suppose that the data set provides the following input +words in 4 separate examples: + +* `"dog"` +* `"spoon"` +* `"scissors"` +* `"guitar"` + +In that case, the following figure illustrates the processing path for +embedding columns or indicator columns. + +
+ +
+
+An embedding column stores categorical data in a lower-dimensional +vector than an indicator column. (We just placed random numbers into the +embedding vectors; training determines the actual numbers.) +
+ +When an example is processed, one of the `categorical_column_with...` functions +maps the example string to a numerical categorical value. For example, a +function maps "spoon" to `[32]`. (The 32 comes from our imagination—the actual +values depend on the mapping function.) You may then represent these numerical +categorical values in either of the following two ways: + +* As an indicator column. A function converts each numeric categorical value + into an 81-element vector (because our palette consists of 81 words), placing + a 1 in the index of the categorical value (0, 32, 79, 80) and a 0 in all the + other positions. + +* As an embedding column. A function uses the numerical categorical values + `(0, 32, 79, 80)` as indices to a lookup table. Each slot in that lookup table + contains a 3-element vector. + +How do the values in the embeddings vectors magically get assigned? Actually, +the assignments happen during training. That is, the model learns the best way +to map your input numeric categorical values to the embeddings vector value in +order to solve your problem. Embedding columns increase your model's +capabilities, since an embeddings vector learns new relationships between +categories from the training data. + +Why is the embedding vector size 3 in our example? Well, the following "formula" +provides a general rule of thumb about the number of embedding dimensions: + +```python +embedding_dimensions = number_of_categories**0.25 +``` + +That is, the embedding vector dimension should be the 4th root of the number of +categories. Since our vocabulary size in this example is 81, the recommended +number of dimensions is 3: + +``` python +3 = 81**0.25 +``` +Note that this is just a general guideline; you can set the number of embedding +dimensions as you please. + +Call @{tf.feature_column.embedding_column} to create an `embedding_column` as +suggested by the following snippet: + +``` python +categorical_column = ... # Create any categorical column + +# Represent the categorical column as an embedding column. +# This means creating an embedding vector lookup table with one element for each category. +embedding_column = tf.feature_column.embedding_column( + categorical_column=categorical_column, + dimension=embedding_dimensions) +``` + +@{$guide/embedding$Embeddings} is a significant topic within machine +learning. This information was just to get you started using them as feature +columns. + +## Passing feature columns to Estimators + +As the following list indicates, not all Estimators permit all types of +`feature_columns` argument(s): + +* @{tf.estimator.LinearClassifier$`LinearClassifier`} and + @{tf.estimator.LinearRegressor$`LinearRegressor`}: Accept all types of + feature column. +* @{tf.estimator.DNNClassifier$`DNNClassifier`} and + @{tf.estimator.DNNRegressor$`DNNRegressor`}: Only accept dense columns. Other + column types must be wrapped in either an `indicator_column` or + `embedding_column`. +* @{tf.estimator.DNNLinearCombinedClassifier$`DNNLinearCombinedClassifier`} and + @{tf.estimator.DNNLinearCombinedRegressor$`DNNLinearCombinedRegressor`}: + * The `linear_feature_columns` argument accepts any feature column type. + * The `dnn_feature_columns` argument only accepts dense columns. + +## Other Sources + +For more examples on feature columns, view the following: + +* The @{$low_level_intro#feature_columns$Low Level Introduction} demonstrates how + experiment directly with `feature_columns` using TensorFlow's low level APIs. +* The @{$wide$wide} and @{$wide_and_deep$Wide & Deep} Tutorials solve a + binary classification problem using `feature_columns` on a variety of input + data types. + +To learn more about embeddings, see the following: + +* [Deep Learning, NLP, and representations](http://colah.github.io/posts/2014-07-NLP-RNNs-Representations/) + (Chris Olah's blog) +* The TensorFlow [Embedding Projector](http://projector.tensorflow.org) diff --git a/tensorflow/docs_src/guide/graph_viz.md b/tensorflow/docs_src/guide/graph_viz.md new file mode 100644 index 0000000000..f581ae56da --- /dev/null +++ b/tensorflow/docs_src/guide/graph_viz.md @@ -0,0 +1,316 @@ +# TensorBoard: Graph Visualization + +TensorFlow computation graphs are powerful but complicated. The graph visualization can help you understand and debug them. Here's an example of the visualization at work. + +![Visualization of a TensorFlow graph](https://www.tensorflow.org/images/graph_vis_animation.gif "Visualization of a TensorFlow graph") +*Visualization of a TensorFlow graph.* + +To see your own graph, run TensorBoard pointing it to the log directory of the job, click on the graph tab on the top pane and select the appropriate run using the menu at the upper left corner. For in depth information on how to run TensorBoard and make sure you are logging all the necessary information, see @{$summaries_and_tensorboard$TensorBoard: Visualizing Learning}. + +## Name scoping and nodes + +Typical TensorFlow graphs can have many thousands of nodes--far too many to see +easily all at once, or even to lay out using standard graph tools. To simplify, +variable names can be scoped and the visualization uses this information to +define a hierarchy on the nodes in the graph. By default, only the top of this +hierarchy is shown. Here is an example that defines three operations under the +`hidden` name scope using +@{tf.name_scope}: + +```python +import tensorflow as tf + +with tf.name_scope('hidden') as scope: + a = tf.constant(5, name='alpha') + W = tf.Variable(tf.random_uniform([1, 2], -1.0, 1.0), name='weights') + b = tf.Variable(tf.zeros([1]), name='biases') +``` + +This results in the following three op names: + +* `hidden/alpha` +* `hidden/weights` +* `hidden/biases` + +By default, the visualization will collapse all three into a node labeled `hidden`. +The extra detail isn't lost. You can double-click, or click +on the orange `+` sign in the top right to expand the node, and then you'll see +three subnodes for `alpha`, `weights` and `biases`. + +Here's a real-life example of a more complicated node in its initial and +expanded states. + + + + + + + + + + +
+ Unexpanded name scope + + Expanded name scope +
+ Initial view of top-level name scope pool_1. Clicking on the orange + button on the top right or double-clicking on the node itself will expand it. + + Expanded view of pool_1 name scope. Clicking on the orange - button on the top right or double-clicking on the node itself will collapse the name scope. +
+ +Grouping nodes by name scopes is critical to making a legible graph. If you're +building a model, name scopes give you control over the resulting visualization. +**The better your name scopes, the better your visualization.** + +The figure above illustrates a second aspect of the visualization. TensorFlow +graphs have two kinds of connections: data dependencies and control +dependencies. Data dependencies show the flow of tensors between two ops and +are shown as solid arrows, while control dependencies use dotted lines. In the +expanded view (right side of the figure above) all the connections are data +dependencies with the exception of the dotted line connecting `CheckNumerics` +and `control_dependency`. + +There's a second trick to simplifying the layout. Most TensorFlow graphs have a +few nodes with many connections to other nodes. For example, many nodes might +have a control dependency on an initialization step. Drawing all edges between +the `init` node and its dependencies would create a very cluttered view. + +To reduce clutter, the visualization separates out all high-degree nodes to an +*auxiliary* area on the right and doesn't draw lines to represent their edges. +Instead of lines, we draw small *node icons* to indicate the connections. +Separating out the auxiliary nodes typically doesn't remove critical +information since these nodes are usually related to bookkeeping functions. +See [Interaction](#interaction) for how to move nodes between the main graph +and the auxiliary area. + + + + + + + + + + +
+ conv_1 is part of the main graph + + save is extracted as auxiliary node +
+ Node conv_1 is connected to save. Note the little save node icon on its right. + + save has a high degree, and will appear as an auxiliary node. The connection with conv_1 is shown as a node icon on its left. To further reduce clutter, since save has a lot of connections, we show the first 5 and abbreviate the others as ... 12 more. +
+ +One last structural simplification is *series collapsing*. Sequential +motifs--that is, nodes whose names differ by a number at the end and have +isomorphic structures--are collapsed into a single *stack* of nodes, as shown +below. For networks with long sequences, this greatly simplifies the view. As +with hierarchical nodes, double-clicking expands the series. See +[Interaction](#interaction) for how to disable/enable series collapsing for a +specific set of nodes. + + + + + + + + + + +
+ Sequence of nodes + + Expanded sequence of nodes +
+ A collapsed view of a node sequence. + + A small piece of the expanded view, after double-click. +
+ +Finally, as one last aid to legibility, the visualization uses special icons +for constants and summary nodes. To summarize, here's a table of node symbols: + +Symbol | Meaning +--- | --- +![Name scope](https://www.tensorflow.org/images/namespace_node.png "Name scope") | *High-level* node representing a name scope. Double-click to expand a high-level node. +![Sequence of unconnected nodes](https://www.tensorflow.org/images/horizontal_stack.png "Sequence of unconnected nodes") | Sequence of numbered nodes that are not connected to each other. +![Sequence of connected nodes](https://www.tensorflow.org/images/vertical_stack.png "Sequence of connected nodes") | Sequence of numbered nodes that are connected to each other. +![Operation node](https://www.tensorflow.org/images/op_node.png "Operation node") | An individual operation node. +![Constant node](https://www.tensorflow.org/images/constant.png "Constant node") | A constant. +![Summary node](https://www.tensorflow.org/images/summary.png "Summary node") | A summary node. +![Data flow edge](https://www.tensorflow.org/images/dataflow_edge.png "Data flow edge") | Edge showing the data flow between operations. +![Control dependency edge](https://www.tensorflow.org/images/control_edge.png "Control dependency edge") | Edge showing the control dependency between operations. +![Reference edge](https://www.tensorflow.org/images/reference_edge.png "Reference edge") | A reference edge showing that the outgoing operation node can mutate the incoming tensor. + +## Interaction {#interaction} + +Navigate the graph by panning and zooming. Click and drag to pan, and use a +scroll gesture to zoom. Double-click on a node, or click on its `+` button, to +expand a name scope that represents a group of operations. To easily keep +track of the current viewpoint when zooming and panning, there is a minimap in +the bottom right corner. + +To close an open node, double-click it again or click its `-` button. You can +also click once to select a node. It will turn a darker color, and details +about it and the nodes it connects to will appear in the info card at upper +right corner of the visualization. + + + + + + + + + + +
+ Info card of a name scope + + Info card of operation node +
+ Info card showing detailed information for the conv2 name scope. The inputs and outputs are combined from the inputs and outputs of the operation nodes inside the name scope. For name scopes no attributes are shown. + + Info card showing detailed information for the DecodeRaw operation node. In addition to inputs and outputs, the card shows the device and the attributes associated with the current operation. +
+ +TensorBoard provides several ways to change the visual layout of the graph. This +doesn't change the graph's computational semantics, but it can bring some +clarity to the network's structure. By right clicking on a node or pressing +buttons on the bottom of that node's info card, you can make the following +changes to its layout: + +* Nodes can be moved between the main graph and the auxiliary area. +* A series of nodes can be ungrouped so that the nodes in the series do not +appear grouped together. Ungrouped series can likewise be regrouped. + +Selection can also be helpful in understanding high-degree nodes. Select any +high-degree node, and the corresponding node icons for its other connections +will be selected as well. This makes it easy, for example, to see which nodes +are being saved--and which aren't. + +Clicking on a node name in the info card will select it. If necessary, the +viewpoint will automatically pan so that the node is visible. + +Finally, you can choose two color schemes for your graph, using the color menu +above the legend. The default *Structure View* shows structure: when two +high-level nodes have the same structure, they appear in the same color of the +rainbow. Uniquely structured nodes are gray. There's a second view, which shows +what device the different operations run on. Name scopes are colored +proportionally to the fraction of devices for the operations inside them. + +The images below give an illustration for a piece of a real-life graph. + + + + + + + + + + +
+ Color by structure + + Color by device +
+ Structure view: The gray nodes have unique structure. The orange conv1 and conv2 nodes have the same structure, and analogously for nodes with other colors. + + Device view: Name scopes are colored proportionally to the fraction of devices of the operation nodes inside them. Here, purple means GPU and the green is CPU. +
+ +## Tensor shape information + +When the serialized `GraphDef` includes tensor shapes, the graph visualizer +labels edges with tensor dimensions, and edge thickness reflects total tensor +size. To include tensor shapes in the `GraphDef` pass the actual graph object +(as in `sess.graph`) to the `FileWriter` when serializing the graph. +The images below show the CIFAR-10 model with tensor shape information: + + + + + + + +
+ CIFAR-10 model with tensor shape information +
+ CIFAR-10 model with tensor shape information. +
+ +## Runtime statistics + +Often it is useful to collect runtime metadata for a run, such as total memory +usage, total compute time, and tensor shapes for nodes. The code example below +is a snippet from the train and test section of a modification of the +@{$layers$simple MNIST tutorial}, in which we have recorded summaries and +runtime statistics. See the +@{$summaries_and_tensorboard#serializing-the-data$Summaries Tutorial} +for details on how to record summaries. +Full source is [here](https://www.tensorflow.org/code/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py). + +```python + # Train the model, and also write summaries. + # Every 10th step, measure test-set accuracy, and write test summaries + # All other steps, run train_step on training data, & add training summaries + + def feed_dict(train): + """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" + if train or FLAGS.fake_data: + xs, ys = mnist.train.next_batch(100, fake_data=FLAGS.fake_data) + k = FLAGS.dropout + else: + xs, ys = mnist.test.images, mnist.test.labels + k = 1.0 + return {x: xs, y_: ys, keep_prob: k} + + for i in range(FLAGS.max_steps): + if i % 10 == 0: # Record summaries and test-set accuracy + summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False)) + test_writer.add_summary(summary, i) + print('Accuracy at step %s: %s' % (i, acc)) + else: # Record train set summaries, and train + if i % 100 == 99: # Record execution stats + run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) + run_metadata = tf.RunMetadata() + summary, _ = sess.run([merged, train_step], + feed_dict=feed_dict(True), + options=run_options, + run_metadata=run_metadata) + train_writer.add_run_metadata(run_metadata, 'step%d' % i) + train_writer.add_summary(summary, i) + print('Adding run metadata for', i) + else: # Record a summary + summary, _ = sess.run([merged, train_step], feed_dict=feed_dict(True)) + train_writer.add_summary(summary, i) +``` + +This code will emit runtime statistics for every 100th step starting at step99. + +When you launch tensorboard and go to the Graph tab, you will now see options +under "Session runs" which correspond to the steps where run metadata was added. +Selecting one of these runs will show you the snapshot of the network at that +step, fading out unused nodes. In the controls on the left hand side, you will +be able to color the nodes by total memory or total compute time. Additionally, +clicking on a node will display the exact total memory, compute time, and +tensor output sizes. + + + + + + + + +
+ Color by compute time + + Run metadata graph + + Run metadata info card +
diff --git a/tensorflow/docs_src/guide/graphs.md b/tensorflow/docs_src/guide/graphs.md new file mode 100644 index 0000000000..e6246ef148 --- /dev/null +++ b/tensorflow/docs_src/guide/graphs.md @@ -0,0 +1,558 @@ +# Graphs and Sessions + +TensorFlow uses a **dataflow graph** to represent your computation in terms of +the dependencies between individual operations. This leads to a low-level +programming model in which you first define the dataflow graph, then create a +TensorFlow **session** to run parts of the graph across a set of local and +remote devices. + +This guide will be most useful if you intend to use the low-level programming +model directly. Higher-level APIs such as @{tf.estimator.Estimator} and Keras +hide the details of graphs and sessions from the end user, but this guide may +also be useful if you want to understand how these APIs are implemented. + +## Why dataflow graphs? + +![](../images/tensors_flowing.gif) + +[Dataflow](https://en.wikipedia.org/wiki/Dataflow_programming) is a common +programming model for parallel computing. In a dataflow graph, the nodes +represent units of computation, and the edges represent the data consumed or +produced by a computation. For example, in a TensorFlow graph, the @{tf.matmul} +operation would correspond to a single node with two incoming edges (the +matrices to be multiplied) and one outgoing edge (the result of the +multiplication). + + + +Dataflow has several advantages that TensorFlow leverages when executing your +programs: + +* **Parallelism.** By using explicit edges to represent dependencies between + operations, it is easy for the system to identify operations that can execute + in parallel. + +* **Distributed execution.** By using explicit edges to represent the values + that flow between operations, it is possible for TensorFlow to partition your + program across multiple devices (CPUs, GPUs, and TPUs) attached to different + machines. TensorFlow inserts the necessary communication and coordination + between devices. + +* **Compilation.** TensorFlow's @{$performance/xla$XLA compiler} can + use the information in your dataflow graph to generate faster code, for + example, by fusing together adjacent operations. + +* **Portability.** The dataflow graph is a language-independent representation + of the code in your model. You can build a dataflow graph in Python, store it + in a @{$saved_model$SavedModel}, and restore it in a C++ program for + low-latency inference. + + +## What is a @{tf.Graph}? + +A @{tf.Graph} contains two relevant kinds of information: + +* **Graph structure.** The nodes and edges of the graph, indicating how + individual operations are composed together, but not prescribing how they + should be used. The graph structure is like assembly code: inspecting it can + convey some useful information, but it does not contain all of the useful + context that source code conveys. + +* **Graph collections.** TensorFlow provides a general mechanism for storing + collections of metadata in a @{tf.Graph}. The @{tf.add_to_collection} function + enables you to associate a list of objects with a key (where @{tf.GraphKeys} + defines some of the standard keys), and @{tf.get_collection} enables you to + look up all objects associated with a key. Many parts of the TensorFlow + library use this facility: for example, when you create a @{tf.Variable}, it + is added by default to collections representing "global variables" and + "trainable variables". When you later come to create a @{tf.train.Saver} or + @{tf.train.Optimizer}, the variables in these collections are used as the + default arguments. + + +## Building a @{tf.Graph} + +Most TensorFlow programs start with a dataflow graph construction phase. In this +phase, you invoke TensorFlow API functions that construct new @{tf.Operation} +(node) and @{tf.Tensor} (edge) objects and add them to a @{tf.Graph} +instance. TensorFlow provides a **default graph** that is an implicit argument +to all API functions in the same context. For example: + +* Calling `tf.constant(42.0)` creates a single @{tf.Operation} that produces the + value `42.0`, adds it to the default graph, and returns a @{tf.Tensor} that + represents the value of the constant. + +* Calling `tf.matmul(x, y)` creates a single @{tf.Operation} that multiplies + the values of @{tf.Tensor} objects `x` and `y`, adds it to the default graph, + and returns a @{tf.Tensor} that represents the result of the multiplication. + +* Executing `v = tf.Variable(0)` adds to the graph a @{tf.Operation} that will + store a writeable tensor value that persists between @{tf.Session.run} calls. + The @{tf.Variable} object wraps this operation, and can be used [like a + tensor](#tensor-like_objects), which will read the current value of the + stored value. The @{tf.Variable} object also has methods such as + @{tf.Variable.assign$`assign`} and @{tf.Variable.assign_add$`assign_add`} that + create @{tf.Operation} objects that, when executed, update the stored value. + (See @{$guide/variables} for more information about variables.) + +* Calling @{tf.train.Optimizer.minimize} will add operations and tensors to the + default graph that calculates gradients, and return a @{tf.Operation} that, + when run, will apply those gradients to a set of variables. + +Most programs rely solely on the default graph. However, +see [Dealing with multiple graphs](#programming_with_multiple_graphs) for more +advanced use cases. High-level APIs such as the @{tf.estimator.Estimator} API +manage the default graph on your behalf, and--for example--may create different +graphs for training and evaluation. + +Note: Calling most functions in the TensorFlow API merely adds operations +and tensors to the default graph, but **does not** perform the actual +computation. Instead, you compose these functions until you have a @{tf.Tensor} +or @{tf.Operation} that represents the overall computation--such as performing +one step of gradient descent--and then pass that object to a @{tf.Session} to +perform the computation. See the section "Executing a graph in a @{tf.Session}" +for more details. + +## Naming operations + +A @{tf.Graph} object defines a **namespace** for the @{tf.Operation} objects it +contains. TensorFlow automatically chooses a unique name for each operation in +your graph, but giving operations descriptive names can make your program easier +to read and debug. The TensorFlow API provides two ways to override the name of +an operation: + +* Each API function that creates a new @{tf.Operation} or returns a new + @{tf.Tensor} accepts an optional `name` argument. For example, + `tf.constant(42.0, name="answer")` creates a new @{tf.Operation} named + `"answer"` and returns a @{tf.Tensor} named `"answer:0"`. If the default graph + already contains an operation named `"answer"`, then TensorFlow would append + `"_1"`, `"_2"`, and so on to the name, in order to make it unique. + +* The @{tf.name_scope} function makes it possible to add a **name scope** prefix + to all operations created in a particular context. The current name scope + prefix is a `"/"`-delimited list of the names of all active @{tf.name_scope} + context managers. If a name scope has already been used in the current + context, TensorFlow appends `"_1"`, `"_2"`, and so on. For example: + + ```python + c_0 = tf.constant(0, name="c") # => operation named "c" + + # Already-used names will be "uniquified". + c_1 = tf.constant(2, name="c") # => operation named "c_1" + + # Name scopes add a prefix to all operations created in the same context. + with tf.name_scope("outer"): + c_2 = tf.constant(2, name="c") # => operation named "outer/c" + + # Name scopes nest like paths in a hierarchical file system. + with tf.name_scope("inner"): + c_3 = tf.constant(3, name="c") # => operation named "outer/inner/c" + + # Exiting a name scope context will return to the previous prefix. + c_4 = tf.constant(4, name="c") # => operation named "outer/c_1" + + # Already-used name scopes will be "uniquified". + with tf.name_scope("inner"): + c_5 = tf.constant(5, name="c") # => operation named "outer/inner_1/c" + ``` + +The graph visualizer uses name scopes to group operations and reduce the visual +complexity of a graph. See [Visualizing your graph](#visualizing-your-graph) for +more information. + +Note that @{tf.Tensor} objects are implicitly named after the @{tf.Operation} +that produces the tensor as output. A tensor name has the form `":"` +where: + +* `""` is the name of the operation that produces it. +* `""` is an integer representing the index of that tensor among the + operation's outputs. + +## Placing operations on different devices + +If you want your TensorFlow program to use multiple different devices, the +@{tf.device} function provides a convenient way to request that all operations +created in a particular context are placed on the same device (or type of +device). + +A **device specification** has the following form: + +``` +/job:/task:/device:: +``` + +where: + +* `` is an alpha-numeric string that does not start with a number. +* `` is a registered device type (such as `GPU` or `CPU`). +* `` is a non-negative integer representing the index of the task + in the job named ``. See @{tf.train.ClusterSpec} for an explanation + of jobs and tasks. +* `` is a non-negative integer representing the index of the + device, for example, to distinguish between different GPU devices used in the + same process. + +You do not need to specify every part of a device specification. For example, +if you are running in a single-machine configuration with a single GPU, you +might use @{tf.device} to pin some operations to the CPU and GPU: + +```python +# Operations created outside either context will run on the "best possible" +# device. For example, if you have a GPU and a CPU available, and the operation +# has a GPU implementation, TensorFlow will choose the GPU. +weights = tf.random_normal(...) + +with tf.device("/device:CPU:0"): + # Operations created in this context will be pinned to the CPU. + img = tf.decode_jpeg(tf.read_file("img.jpg")) + +with tf.device("/device:GPU:0"): + # Operations created in this context will be pinned to the GPU. + result = tf.matmul(weights, img) +``` +If you are deploying TensorFlow in a @{$distributed$typical distributed configuration}, +you might specify the job name and task ID to place variables on +a task in the parameter server job (`"/job:ps"`), and the other operations on +task in the worker job (`"/job:worker"`): + +```python +with tf.device("/job:ps/task:0"): + weights_1 = tf.Variable(tf.truncated_normal([784, 100])) + biases_1 = tf.Variable(tf.zeroes([100])) + +with tf.device("/job:ps/task:1"): + weights_2 = tf.Variable(tf.truncated_normal([100, 10])) + biases_2 = tf.Variable(tf.zeroes([10])) + +with tf.device("/job:worker"): + layer_1 = tf.matmul(train_batch, weights_1) + biases_1 + layer_2 = tf.matmul(train_batch, weights_2) + biases_2 +``` + +@{tf.device} gives you a lot of flexibility to choose placements for individual +operations or broad regions of a TensorFlow graph. In many cases, there are +simple heuristics that work well. For example, the +@{tf.train.replica_device_setter} API can be used with @{tf.device} to place +operations for **data-parallel distributed training**. For example, the +following code fragment shows how @{tf.train.replica_device_setter} applies +different placement policies to @{tf.Variable} objects and other operations: + +```python +with tf.device(tf.train.replica_device_setter(ps_tasks=3)): + # tf.Variable objects are, by default, placed on tasks in "/job:ps" in a + # round-robin fashion. + w_0 = tf.Variable(...) # placed on "/job:ps/task:0" + b_0 = tf.Variable(...) # placed on "/job:ps/task:1" + w_1 = tf.Variable(...) # placed on "/job:ps/task:2" + b_1 = tf.Variable(...) # placed on "/job:ps/task:0" + + input_data = tf.placeholder(tf.float32) # placed on "/job:worker" + layer_0 = tf.matmul(input_data, w_0) + b_0 # placed on "/job:worker" + layer_1 = tf.matmul(layer_0, w_1) + b_1 # placed on "/job:worker" +``` + +## Tensor-like objects + +Many TensorFlow operations take one or more @{tf.Tensor} objects as arguments. +For example, @{tf.matmul} takes two @{tf.Tensor} objects, and @{tf.add_n} takes +a list of `n` @{tf.Tensor} objects. For convenience, these functions will accept +a **tensor-like object** in place of a @{tf.Tensor}, and implicitly convert it +to a @{tf.Tensor} using the @{tf.convert_to_tensor} method. Tensor-like objects +include elements of the following types: + +* @{tf.Tensor} +* @{tf.Variable} +* [`numpy.ndarray`](https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html) +* `list` (and lists of tensor-like objects) +* Scalar Python types: `bool`, `float`, `int`, `str` + +You can register additional tensor-like types using +@{tf.register_tensor_conversion_function}. + +Note: By default, TensorFlow will create a new @{tf.Tensor} each time you use +the same tensor-like object. If the tensor-like object is large (e.g. a +`numpy.ndarray` containing a set of training examples) and you use it multiple +times, you may run out of memory. To avoid this, manually call +@{tf.convert_to_tensor} on the tensor-like object once and use the returned +@{tf.Tensor} instead. + +## Executing a graph in a @{tf.Session} + +TensorFlow uses the @{tf.Session} class to represent a connection between the +client program---typically a Python program, although a similar interface is +available in other languages---and the C++ runtime. A @{tf.Session} object +provides access to devices in the local machine, and remote devices using the +distributed TensorFlow runtime. It also caches information about your +@{tf.Graph} so that you can efficiently run the same computation multiple times. + +### Creating a @{tf.Session} + +If you are using the low-level TensorFlow API, you can create a @{tf.Session} +for the current default graph as follows: + +```python +# Create a default in-process session. +with tf.Session() as sess: + # ... + +# Create a remote session. +with tf.Session("grpc://example.org:2222"): + # ... +``` + +Since a @{tf.Session} owns physical resources (such as GPUs and +network connections), it is typically used as a context manager (in a `with` +block) that automatically closes the session when you exit the block. It is +also possible to create a session without using a `with` block, but you should +explicitly call @{tf.Session.close} when you are finished with it to free the +resources. + +Note: Higher-level APIs such as @{tf.train.MonitoredTrainingSession} or +@{tf.estimator.Estimator} will create and manage a @{tf.Session} for you. These +APIs accept optional `target` and `config` arguments (either directly, or as +part of a @{tf.estimator.RunConfig} object), with the same meaning as +described below. + +@{tf.Session.__init__} accepts three optional arguments: + +* **`target`.** If this argument is left empty (the default), the session will + only use devices in the local machine. However, you may also specify a + `grpc://` URL to specify the address of a TensorFlow server, which gives the + session access to all devices on machines that this server controls. See + @{tf.train.Server} for details of how to create a TensorFlow + server. For example, in the common **between-graph replication** + configuration, the @{tf.Session} connects to a @{tf.train.Server} in the same + process as the client. The [distributed TensorFlow](../deploy/distributed.md) + deployment guide describes other common scenarios. + +* **`graph`.** By default, a new @{tf.Session} will be bound to---and only able + to run operations in---the current default graph. If you are using multiple + graphs in your program (see [Programming with multiple + graphs](#programming_with_multiple_graphs) for more details), you can specify + an explicit @{tf.Graph} when you construct the session. + +* **`config`.** This argument allows you to specify a @{tf.ConfigProto} that + controls the behavior of the session. For example, some of the configuration + options include: + + * `allow_soft_placement`. Set this to `True` to enable a "soft" device + placement algorithm, which ignores @{tf.device} annotations that attempt + to place CPU-only operations on a GPU device, and places them on the CPU + instead. + + * `cluster_def`. When using distributed TensorFlow, this option allows you + to specify what machines to use in the computation, and provide a mapping + between job names, task indices, and network addresses. See + @{tf.train.ClusterSpec.as_cluster_def} for details. + + * `graph_options.optimizer_options`. Provides control over the optimizations + that TensorFlow performs on your graph before executing it. + + * `gpu_options.allow_growth`. Set this to `True` to change the GPU memory + allocator so that it gradually increases the amount of memory allocated, + rather than allocating most of the memory at startup. + + +### Using @{tf.Session.run} to execute operations + +The @{tf.Session.run} method is the main mechanism for running a @{tf.Operation} +or evaluating a @{tf.Tensor}. You can pass one or more @{tf.Operation} or +@{tf.Tensor} objects to @{tf.Session.run}, and TensorFlow will execute the +operations that are needed to compute the result. + +@{tf.Session.run} requires you to specify a list of **fetches**, which determine +the return values, and may be a @{tf.Operation}, a @{tf.Tensor}, or +a [tensor-like type](#tensor-like_objects) such as @{tf.Variable}. These fetches +determine what **subgraph** of the overall @{tf.Graph} must be executed to +produce the result: this is the subgraph that contains all operations named in +the fetch list, plus all operations whose outputs are used to compute the value +of the fetches. For example, the following code fragment shows how different +arguments to @{tf.Session.run} cause different subgraphs to be executed: + +```python +x = tf.constant([[37.0, -23.0], [1.0, 4.0]]) +w = tf.Variable(tf.random_uniform([2, 2])) +y = tf.matmul(x, w) +output = tf.nn.softmax(y) +init_op = w.initializer + +with tf.Session() as sess: + # Run the initializer on `w`. + sess.run(init_op) + + # Evaluate `output`. `sess.run(output)` will return a NumPy array containing + # the result of the computation. + print(sess.run(output)) + + # Evaluate `y` and `output`. Note that `y` will only be computed once, and its + # result used both to return `y_val` and as an input to the `tf.nn.softmax()` + # op. Both `y_val` and `output_val` will be NumPy arrays. + y_val, output_val = sess.run([y, output]) +``` + +@{tf.Session.run} also optionally takes a dictionary of **feeds**, which is a +mapping from @{tf.Tensor} objects (typically @{tf.placeholder} tensors) to +values (typically Python scalars, lists, or NumPy arrays) that will be +substituted for those tensors in the execution. For example: + +```python +# Define a placeholder that expects a vector of three floating-point values, +# and a computation that depends on it. +x = tf.placeholder(tf.float32, shape=[3]) +y = tf.square(x) + +with tf.Session() as sess: + # Feeding a value changes the result that is returned when you evaluate `y`. + print(sess.run(y, {x: [1.0, 2.0, 3.0]})) # => "[1.0, 4.0, 9.0]" + print(sess.run(y, {x: [0.0, 0.0, 5.0]})) # => "[0.0, 0.0, 25.0]" + + # Raises `tf.errors.InvalidArgumentError`, because you must feed a value for + # a `tf.placeholder()` when evaluating a tensor that depends on it. + sess.run(y) + + # Raises `ValueError`, because the shape of `37.0` does not match the shape + # of placeholder `x`. + sess.run(y, {x: 37.0}) +``` + +@{tf.Session.run} also accepts an optional `options` argument that enables you +to specify options about the call, and an optional `run_metadata` argument that +enables you to collect metadata about the execution. For example, you can use +these options together to collect tracing information about the execution: + +``` +y = tf.matmul([[37.0, -23.0], [1.0, 4.0]], tf.random_uniform([2, 2])) + +with tf.Session() as sess: + # Define options for the `sess.run()` call. + options = tf.RunOptions() + options.output_partition_graphs = True + options.trace_level = tf.RunOptions.FULL_TRACE + + # Define a container for the returned metadata. + metadata = tf.RunMetadata() + + sess.run(y, options=options, run_metadata=metadata) + + # Print the subgraphs that executed on each device. + print(metadata.partition_graphs) + + # Print the timings of each operation that executed. + print(metadata.step_stats) +``` + + +## Visualizing your graph + +TensorFlow includes tools that can help you to understand the code in a graph. +The **graph visualizer** is a component of TensorBoard that renders the +structure of your graph visually in a browser. The easiest way to create a +visualization is to pass a @{tf.Graph} when creating the +@{tf.summary.FileWriter}: + +```python +# Build your graph. +x = tf.constant([[37.0, -23.0], [1.0, 4.0]]) +w = tf.Variable(tf.random_uniform([2, 2])) +y = tf.matmul(x, w) +# ... +loss = ... +train_op = tf.train.AdagradOptimizer(0.01).minimize(loss) + +with tf.Session() as sess: + # `sess.graph` provides access to the graph used in a `tf.Session`. + writer = tf.summary.FileWriter("/tmp/log/...", sess.graph) + + # Perform your computation... + for i in range(1000): + sess.run(train_op) + # ... + + writer.close() +``` + +Note: If you are using a @{tf.estimator.Estimator}, the graph (and any +summaries) will be logged automatically to the `model_dir` that you specified +when creating the estimator. + +You can then open the log in `tensorboard`, navigate to the "Graph" tab, and +see a high-level visualization of your graph's structure. Note that a typical +TensorFlow graph---especially training graphs with automatically computed +gradients---has too many nodes to visualize at once. The graph visualizer makes +use of name scopes to group related operations into "super" nodes. You can +click on the orange "+" button on any of these super nodes to expand the +subgraph inside. + +![](../images/mnist_deep.png) + +For more information about visualizing your TensorFlow application with +TensorBoard, see the [TensorBoard tutorial](../get_started/summaries_and_tensorboard.md). + +## Programming with multiple graphs + +Note: When training a model, a common way of organizing your code is to use one +graph for training your model, and a separate graph for evaluating or performing +inference with a trained model. In many cases, the inference graph will be +different from the training graph: for example, techniques like dropout and +batch normalization use different operations in each case. Furthermore, by +default utilities like @{tf.train.Saver} use the names of @{tf.Variable} objects +(which have names based on an underlying @{tf.Operation}) to identify each +variable in a saved checkpoint. When programming this way, you can either use +completely separate Python processes to build and execute the graphs, or you can +use multiple graphs in the same process. This section describes how to use +multiple graphs in the same process. + +As noted above, TensorFlow provides a "default graph" that is implicitly passed +to all API functions in the same context. For many applications, a single graph +is sufficient. However, TensorFlow also provides methods for manipulating +the default graph, which can be useful in more advanced use cases. For example: + +* A @{tf.Graph} defines the namespace for @{tf.Operation} objects: each + operation in a single graph must have a unique name. TensorFlow will + "uniquify" the names of operations by appending `"_1"`, `"_2"`, and so on to + their names if the requested name is already taken. Using multiple explicitly + created graphs gives you more control over what name is given to each + operation. + +* The default graph stores information about every @{tf.Operation} and + @{tf.Tensor} that was ever added to it. If your program creates a large number + of unconnected subgraphs, it may be more efficient to use a different + @{tf.Graph} to build each subgraph, so that unrelated state can be garbage + collected. + +You can install a different @{tf.Graph} as the default graph, using the +@{tf.Graph.as_default} context manager: + +```python +g_1 = tf.Graph() +with g_1.as_default(): + # Operations created in this scope will be added to `g_1`. + c = tf.constant("Node in g_1") + + # Sessions created in this scope will run operations from `g_1`. + sess_1 = tf.Session() + +g_2 = tf.Graph() +with g_2.as_default(): + # Operations created in this scope will be added to `g_2`. + d = tf.constant("Node in g_2") + +# Alternatively, you can pass a graph when constructing a `tf.Session`: +# `sess_2` will run operations from `g_2`. +sess_2 = tf.Session(graph=g_2) + +assert c.graph is g_1 +assert sess_1.graph is g_1 + +assert d.graph is g_2 +assert sess_2.graph is g_2 +``` + +To inspect the current default graph, call @{tf.get_default_graph}, which +returns a @{tf.Graph} object: + +```python +# Print all of the operations in the default graph. +g = tf.get_default_graph() +print(g.get_operations()) +``` diff --git a/tensorflow/docs_src/guide/index.md b/tensorflow/docs_src/guide/index.md new file mode 100644 index 0000000000..eefdb9ceae --- /dev/null +++ b/tensorflow/docs_src/guide/index.md @@ -0,0 +1,86 @@ +# TensorFlow Guide + +The documents in this unit dive into the details of how TensorFlow +works. The units are as follows: + +## High Level APIs + + * @{$guide/keras}, TensorFlow's high-level API for building and + training deep learning models. + * @{$guide/eager}, an API for writing TensorFlow code + imperatively, like you would use Numpy. + * @{$guide/estimators}, a high-level API that provides + fully-packaged models ready for large-scale training and production. + * @{$guide/datasets}, easy input pipelines to bring your data into + your TensorFlow program. + +## Estimators + +* @{$estimators} provides an introduction. +* @{$premade_estimators}, introduces Estimators for machine learning. +* @{$custom_estimators}, which demonstrates how to build and train models you + design yourself. +* @{$feature_columns}, which shows how an Estimator can handle a variety of input + data types without changes to the model. +* @{$datasets_for_estimators} describes using tf.data with estimators. +* @{$checkpoints}, which explains how to save training progress and resume where + you left off. + +## Accelerators + + * @{$using_gpu} explains how TensorFlow assigns operations to + devices and how you can change the arrangement manually. + * @{$using_tpu} explains how to modify `Estimator` programs to run on a TPU. + +## Low Level APIs + + * @{$guide/low_level_intro}, which introduces the + basics of how you can use TensorFlow outside of the high Level APIs. + * @{$guide/tensors}, which explains how to create, + manipulate, and access Tensors--the fundamental object in TensorFlow. + * @{$guide/variables}, which details how + to represent shared, persistent state in your program. + * @{$guide/graphs}, which explains: + * dataflow graphs, which are TensorFlow's representation of computations + as dependencies between operations. + * sessions, which are TensorFlow's mechanism for running dataflow graphs + across one or more local or remote devices. + If you are programming with the low-level TensorFlow API, this unit + is essential. If you are programming with a high-level TensorFlow API + such as Estimators or Keras, the high-level API creates and manages + graphs and sessions for you, but understanding graphs and sessions + can still be helpful. + * @{$guide/saved_model}, which + explains how to save and restore variables and models. + +## ML Concepts + + * @{$guide/embedding}, which introduces the concept + of embeddings, provides a simple example of training an embedding in + TensorFlow, and explains how to view embeddings with the TensorBoard + Embedding Projector. + +## Debugging + + * @{$guide/debugger}, which + explains how to use the TensorFlow debugger (tfdbg). + +## TensorBoard + +TensorBoard is a utility to visualize different aspects of machine learning. +The following guides explain how to use TensorBoard: + + * @{$guide/summaries_and_tensorboard}, + which introduces TensorBoard. + * @{$guide/graph_viz}, which + explains how to visualize the computational graph. + * @{$guide/tensorboard_histograms} which demonstrates the how to + use TensorBoard's histogram dashboard. + + +## Misc + + * @{$guide/version_compat}, + which explains backward compatibility guarantees and non-guarantees. + * @{$guide/faq}, which contains frequently asked + questions about TensorFlow. diff --git a/tensorflow/docs_src/guide/keras.md b/tensorflow/docs_src/guide/keras.md new file mode 100644 index 0000000000..83172dab7f --- /dev/null +++ b/tensorflow/docs_src/guide/keras.md @@ -0,0 +1,623 @@ +# Keras + +Keras is a high-level API to build and train deep learning models. It's used for +fast prototyping, advanced research, and production, with three key advantages: + +- *User friendly*
+ Keras has a simple, consistent interface optimized for common use cases. It + provides clear and actionable feedback for user errors. +- *Modular and composable*
+ Keras models are made by connecting configurable building blocks together, + with few restrictions. +- *Easy to extend*
Write custom building blocks to express new ideas for + research. Create new layers, loss functions, and develop state-of-the-art + models. + +## Import tf.keras + +`tf.keras` is TensorFlow's implementation of the +[Keras API specification](https://keras.io){:.external}. This is a high-level +API to build and train models that includes first-class support for +TensorFlow-specific functionality, such as [eager execution](#eager_execution), +`tf.data` pipelines, and [Estimators](./estimators.md). +`tf.keras` makes TensorFlow easier to use without sacrificing flexibility and +performance. + +To get started, import `tf.keras` as part of your TensorFlow program setup: + +```python +import tensorflow as tf +from tensorflow import keras +``` + +`tf.keras` can run any Keras-compatible code, but keep in mind: + +* The `tf.keras` version in the latest TensorFlow release might not be the same + as the latest `keras` version from PyPI. Check `tf.keras.__version__`. +* When [saving a model's weights](#weights_only), `tf.keras` defaults to the + [checkpoint format](../get_started/checkpoints.md). Pass `save_format='h5'` to + use HDF5. + +## Build a simple model + +### Sequential model + +In Keras, you assemble *layers* to build *models*. A model is (usually) a graph +of layers. The most common type of model is a stack of layers: the +`tf.keras.Sequential` model. + +To build a simple, fully-connected network (i.e. multi-layer perceptron): + +```python +model = keras.Sequential() +# Adds a densely-connected layer with 64 units to the model: +model.add(keras.layers.Dense(64, activation='relu')) +# Add another: +model.add(keras.layers.Dense(64, activation='relu')) +# Add a softmax layer with 10 output units: +model.add(keras.layers.Dense(10, activation='softmax')) +``` + +### Configure the layers + +There are many `tf.keras.layers` available with some common constructor +parameters: + +* `activation`: Set the activation function for the layer. This parameter is + specified by the name of a built-in function or as a callable object. By + default, no activation is applied. +* `kernel_initializer` and `bias_initializer`: The initialization schemes + that create the layer's weights (kernel and bias). This parameter is a name or + a callable object. This defaults to the `"Glorot uniform"` initializer. +* `kernel_regularizer` and `bias_regularizer`: The regularization schemes + that apply the layer's weights (kernel and bias), such as L1 or L2 + regularization. By default, no regularization is applied. + +The following instantiates `tf.keras.layers.Dense` layers using constructor +arguments: + +```python +# Create a sigmoid layer: +layers.Dense(64, activation='sigmoid') +# Or: +layers.Dense(64, activation=tf.sigmoid) + +# A linear layer with L1 regularization of factor 0.01 applied to the kernel matrix: +layers.Dense(64, kernel_regularizer=keras.regularizers.l1(0.01)) +# A linear layer with L2 regularization of factor 0.01 applied to the bias vector: +layers.Dense(64, bias_regularizer=keras.regularizers.l2(0.01)) + +# A linear layer with a kernel initialized to a random orthogonal matrix: +layers.Dense(64, kernel_initializer='orthogonal') +# A linear layer with a bias vector initialized to 2.0s: +layers.Dense(64, bias_initializer=keras.initializers.constant(2.0)) +``` + +## Train and evaluate + +### Set up training + +After the model is constructed, configure its learning process by calling the +`compile` method: + +```python +model.compile(optimizer=tf.train.AdamOptimizer(0.001), + loss='categorical_crossentropy', + metrics=['accuracy']) +``` + +`tf.keras.Model.compile` takes three important arguments: + +* `optimizer`: This object specifies the training procedure. Pass it optimizer + instances from the `tf.train` module, such as + [`AdamOptimizer`](/api_docs/python/tf/train/AdamOptimizer), + [`RMSPropOptimizer`](/api_docs/python/tf/train/RMSPropOptimizer), or + [`GradientDescentOptimizer`](/api_docs/python/tf/train/GradientDescentOptimizer). +* `loss`: The function to minimize during optimization. Common choices include + mean square error (`mse`), `categorical_crossentropy`, and + `binary_crossentropy`. Loss functions are specified by name or by + passing a callable object from the `tf.keras.losses` module. +* `metrics`: Used to monitor training. These are string names or callables from + the `tf.keras.metrics` module. + +The following shows a few examples of configuring a model for training: + +```python +# Configure a model for mean-squared error regression. +model.compile(optimizer=tf.train.AdamOptimizer(0.01), + loss='mse', # mean squared error + metrics=['mae']) # mean absolute error + +# Configure a model for categorical classification. +model.compile(optimizer=tf.train.RMSPropOptimizer(0.01), + loss=keras.losses.categorical_crossentropy, + metrics=[keras.metrics.categorical_accuracy]) +``` + +### Input NumPy data + +For small datasets, use in-memory [NumPy](https://www.numpy.org/){:.external} +arrays to train and evaluate a model. The model is "fit" to the training data +using the `fit` method: + +```python +import numpy as np + +data = np.random.random((1000, 32)) +labels = np.random.random((1000, 10)) + +model.fit(data, labels, epochs=10, batch_size=32) +``` + +`tf.keras.Model.fit` takes three important arguments: + +* `epochs`: Training is structured into *epochs*. An epoch is one iteration over + the entire input data (this is done in smaller batches). +* `batch_size`: When passed NumPy data, the model slices the data into smaller + batches and iterates over these batches during training. This integer + specifies the size of each batch. Be aware that the last batch may be smaller + if the total number of samples is not divisible by the batch size. +* `validation_data`: When prototyping a model, you want to easily monitor its + performance on some validation data. Passing this argument—a tuple of inputs + and labels—allows the model to display the loss and metrics in inference mode + for the passed data, at the end of each epoch. + +Here's an example using `validation_data`: + +```python +import numpy as np + +data = np.random.random((1000, 32)) +labels = np.random.random((1000, 10)) + +val_data = np.random.random((100, 32)) +val_labels = np.random.random((100, 10)) + +model.fit(data, labels, epochs=10, batch_size=32, + validation_data=(val_data, val_labels)) +``` + +### Input tf.data datasets + +Use the [Datasets API](./datasets.md) to scale to large datasets +or multi-device training. Pass a `tf.data.Dataset` instance to the `fit` +method: + +```python +# Instantiates a toy dataset instance: +dataset = tf.data.Dataset.from_tensor_slices((data, labels)) +dataset = dataset.batch(32) +dataset = dataset.repeat() + +# Don't forget to specify `steps_per_epoch` when calling `fit` on a dataset. +model.fit(dataset, epochs=10, steps_per_epoch=30) +``` + +Here, the `fit` method uses the `steps_per_epoch` argument—this is the number of +training steps the model runs before it moves to the next epoch. Since the +`Dataset` yields batches of data, this snippet does not require a `batch_size`. + +Datasets can also be used for validation: + +```python +dataset = tf.data.Dataset.from_tensor_slices((data, labels)) +dataset = dataset.batch(32).repeat() + +val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_labels)) +val_dataset = val_dataset.batch(32).repeat() + +model.fit(dataset, epochs=10, steps_per_epoch=30, + validation_data=val_dataset, + validation_steps=3) +``` + +### Evaluate and predict + +The `tf.keras.Model.evaluate` and `tf.keras.Model.predict` methods can use NumPy +data and a `tf.data.Dataset`. + +To *evaluate* the inference-mode loss and metrics for the data provided: + +```python +model.evaluate(x, y, batch_size=32) + +model.evaluate(dataset, steps=30 +``` + +And to *predict* the output of the last layer in inference for the data provided, +as a NumPy array: + +``` +model.predict(x, batch_size=32) + +model.predict(dataset, steps=30) +``` + + +## Build advanced models + +### Functional API + +The `tf.keras.Sequential` model is a simple stack of layers that cannot +represent arbitrary models. Use the +[Keras functional API](https://keras.io/getting-started/functional-api-guide/){:.external} +to build complex model topologies such as: + +* Multi-input models, +* Multi-output models, +* Models with shared layers (the same layer called several times), +* Models with non-sequential data flows (e.g. residual connections). + +Building a model with the functional API works like this: + +1. A layer instance is callable and returns a tensor. +2. Input tensors and output tensors are used to define a `tf.keras.Model` + instance. +3. This model is trained just like the `Sequential` model. + +The following example uses the functional API to build a simple, fully-connected +network: + +```python +inputs = keras.Input(shape=(32,)) # Returns a placeholder tensor + +# A layer instance is callable on a tensor, and returns a tensor. +x = keras.layers.Dense(64, activation='relu')(inputs) +x = keras.layers.Dense(64, activation='relu')(x) +predictions = keras.layers.Dense(10, activation='softmax')(x) + +# Instantiate the model given inputs and outputs. +model = keras.Model(inputs=inputs, outputs=predictions) + +# The compile step specifies the training configuration. +model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), + loss='categorical_crossentropy', + metrics=['accuracy']) + +# Trains for 5 epochs +model.fit(data, labels, batch_size=32, epochs=5) +``` + +### Model subclassing + +Build a fully-customizable model by subclassing `tf.keras.Model` and defining +your own forward pass. Create layers in the `__init__` method and set them as +attributes of the class instance. Define the forward pass in the `call` method. + +Model subclassing is particularly useful when +[eager execution](./eager.md) is enabled since the forward pass +can be written imperatively. + +Key Point: Use the right API for the job. While model subclassing offers +flexibility, it comes at a cost of greater complexity and more opportunities for +user errors. If possible, prefer the functional API. + +The following example shows a subclassed `tf.keras.Model` using a custom forward +pass: + +```python +class MyModel(keras.Model): + + def __init__(self, num_classes=10): + super(MyModel, self).__init__(name='my_model') + self.num_classes = num_classes + # Define your layers here. + self.dense_1 = keras.layers.Dense(32, activation='relu') + self.dense_2 = keras.layers.Dense(num_classes, activation='sigmoid') + + def call(self, inputs): + # Define your forward pass here, + # using layers you previously defined (in `__init__`). + x = self.dense_1(inputs) + return self.dense_2(x) + + def compute_output_shape(self, input_shape): + # You need to override this function if you want to use the subclassed model + # as part of a functional-style model. + # Otherwise, this method is optional. + shape = tf.TensorShape(input_shape).as_list() + shape[-1] = self.num_classes + return tf.TensorShape(shape) + + +# Instantiates the subclassed model. +model = MyModel(num_classes=10) + +# The compile step specifies the training configuration. +model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), + loss='categorical_crossentropy', + metrics=['accuracy']) + +# Trains for 5 epochs. +model.fit(data, labels, batch_size=32, epochs=5) +``` + + +### Custom layers + +Create a custom layer by subclassing `tf.keras.layers.Layer` and implementing +the following methods: + +* `build`: Create the weights of the layer. Add weights with the `add_weight` + method. +* `call`: Define the forward pass. +* `compute_output_shape`: Specify how to compute the output shape of the layer + given the input shape. +* Optionally, a layer can be serialized by implementing the `get_config` method + and the `from_config` class method. + +Here's an example of a custom layer that implements a `matmul` of an input with +a kernel matrix: + +```python +class MyLayer(keras.layers.Layer): + + def __init__(self, output_dim, **kwargs): + self.output_dim = output_dim + super(MyLayer, self).__init__(**kwargs) + + def build(self, input_shape): + shape = tf.TensorShape((input_shape[1], self.output_dim)) + # Create a trainable weight variable for this layer. + self.kernel = self.add_weight(name='kernel', + shape=shape, + initializer='uniform', + trainable=True) + # Be sure to call this at the end + super(MyLayer, self).build(input_shape) + + def call(self, inputs): + return tf.matmul(inputs, self.kernel) + + def compute_output_shape(self, input_shape): + shape = tf.TensorShape(input_shape).as_list() + shape[-1] = self.output_dim + return tf.TensorShape(shape) + + def get_config(self): + base_config = super(MyLayer, self).get_config() + base_config['output_dim'] = self.output_dim + + @classmethod + def from_config(cls, config): + return cls(**config) + + +# Create a model using the custom layer +model = keras.Sequential([MyLayer(10), + keras.layers.Activation('softmax')]) + +# The compile step specifies the training configuration +model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), + loss='categorical_crossentropy', + metrics=['accuracy']) + +# Trains for 5 epochs. +model.fit(data, targets, batch_size=32, epochs=5) +``` + + +## Callbacks + +A callback is an object passed to a model to customize and extend its behavior +during training. You can write your own custom callback, or use the built-in +`tf.keras.callbacks` that include: + +* `tf.keras.callbacks.ModelCheckpoint`: Save checkpoints of your model at + regular intervals. +* `tf.keras.callbacks.LearningRateScheduler`: Dynamically change the learning + rate. +* `tf.keras.callbacks.EarlyStopping`: Interrupt training when validation + performance has stopped improving. +* `tf.keras.callbacks.TensorBoard`: Monitor the model's behavior using + [TensorBoard](./summaries_and_tensorboard.md). + +To use a `tf.keras.callbacks.Callback`, pass it to the model's `fit` method: + +```python +callbacks = [ + # Interrupt training if `val_loss` stops improving for over 2 epochs + keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'), + # Write TensorBoard logs to `./logs` directory + keras.callbacks.TensorBoard(log_dir='./logs') +] +model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks, + validation_data=(val_data, val_targets)) +``` + + +## Save and restore + +### Weights only + +Save and load the weights of a model using `tf.keras.Model.save_weights`: + +```python +# Save weights to a TensorFlow Checkpoint file +model.save_weights('./my_model') + +# Restore the model's state, +# this requires a model with the same architecture. +model.load_weights('my_model') +``` + +By default, this saves the model's weights in the +[TensorFlow checkpoint](../get_started/checkpoints.md) file format. Weights can +also be saved to the Keras HDF5 format (the default for the multi-backend +implementation of Keras): + +```python +# Save weights to a HDF5 file +model.save_weights('my_model.h5', save_format='h5') + +# Restore the model's state +model.load_weights('my_model.h5') +``` + + +### Configuration only + +A model's configuration can be saved—this serializes the model architecture +without any weights. A saved configuration can recreate and initialize the same +model, even without the code that defined the original model. Keras supports +JSON and YAML serialization formats: + +```python +# Serialize a model to JSON format +json_string = model.to_json() + +# Recreate the model (freshly initialized) +fresh_model = keras.models.from_json(json_string) + +# Serializes a model to YAML format +yaml_string = model.to_yaml() + +# Recreate the model +fresh_model = keras.models.from_yaml(yaml_string) +``` + +Caution: Subclassed models are not serializable because their architecture is +defined by the Python code in the body of the `call` method. + + +### Entire model + +The entire model can be saved to a file that contains the weight values, the +model's configuration, and even the optimizer's configuration. This allows you +to checkpoint a model and resume training later—from the exact same +state—without access to the original code. + +```python +# Create a trivial model +model = keras.Sequential([ + keras.layers.Dense(10, activation='softmax', input_shape=(32,)), + keras.layers.Dense(10, activation='softmax') +]) +model.compile(optimizer='rmsprop', + loss='categorical_crossentropy', + metrics=['accuracy']) +model.fit(data, targets, batch_size=32, epochs=5) + + +# Save entire model to a HDF5 file +model.save('my_model.h5') + +# Recreate the exact same model, including weights and optimizer. +model = keras.models.load_model('my_model.h5') +``` + + +## Eager execution + +[Eager execution](./eager.md) is an imperative programming +environment that evaluates operations immediately. This is not required for +Keras, but is supported by `tf.keras` and useful for inspecting your program and +debugging. + +All of the `tf.keras` model-building APIs are compatible with eager execution. +And while the `Sequential` and functional APIs can be used, eager execution +especially benefits *model subclassing* and building *custom layers*—the APIs +that require you to write the forward pass as code (instead of the APIs that +create models by assembling existing layers). + +See the [eager execution guide](./eager.md#build_a_model) for +examples of using Keras models with custom training loops and `tf.GradientTape`. + + +## Distribution + +### Estimators + +The [Estimators](./estimators.md) API is used for training models +for distributed environments. This targets industry use cases such as +distributed training on large datasets that can export a model for production. + +A `tf.keras.Model` can be trained with the `tf.estimator` API by converting the +model to an `tf.estimator.Estimator` object with +`tf.keras.estimator.model_to_estimator`. See +[Creating Estimators from Keras models](./estimators.md#creating_estimators_from_keras_models). + +```python +model = keras.Sequential([layers.Dense(10,activation='softmax'), + layers.Dense(10,activation='softmax')]) + +model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), + loss='categorical_crossentropy', + metrics=['accuracy']) + +estimator = keras.estimator.model_to_estimator(model) +``` + +Note: Enable [eager execution](./eager.md) for debugging +[Estimator input functions](./premade_estimators.md#create_input_functions) +and inspecting data. + +### Multiple GPUs + +`tf.keras` models can run on multiple GPUs using +`tf.contrib.distribute.DistributionStrategy`. This API provides distributed +training on multiple GPUs with almost no changes to existing code. + +Currently, `tf.contrib.distribute.MirroredStrategy` is the only supported +distribution strategy. `MirroredStrategy` does in-graph replication with +synchronous training using all-reduce on a single machine. To use +`DistributionStrategy` with Keras, convert the `tf.keras.Model` to a +`tf.estimator.Estimator` with `tf.keras.estimator.model_to_estimator`, then +train the estimator + +The following example distributes a `tf.keras.Model` across multiple GPUs on a +single machine. + +First, define a simple model: + +```python +model = keras.Sequential() +model.add(keras.layers.Dense(16, activation='relu', input_shape=(10,))) +model.add(keras.layers.Dense(1, activation='sigmoid')) + +optimizer = tf.train.GradientDescentOptimizer(0.2) + +model.compile(loss='binary_crossentropy', optimizer=optimizer) +model.summary() +``` + +Convert the Keras model to a `tf.estimator.Estimator` instance: + +```python +keras_estimator = keras.estimator.model_to_estimator( + keras_model=model, + config=config, + model_dir='/tmp/model_dir') +``` + +Define an *input pipeline*. The `input_fn` returns a `tf.data.Dataset` object +used to distribute the data across multiple devices—with each device processing +a slice of the input batch. + +```python +def input_fn(): + x = np.random.random((1024, 10)) + y = np.random.randint(2, size=(1024, 1)) + x = tf.cast(x, tf.float32) + dataset = tf.data.Dataset.from_tensor_slices((x, y)) + dataset = dataset.repeat(10) + dataset = dataset.batch(32) + return dataset +``` + +Next, create a `tf.estimator.RunConfig` and set the `train_distribute` argument +to the `tf.contrib.distribute.MirroredStrategy` instance. When creating +`MirroredStrategy`, you can specify a list of devices or set the `num_gpus` +argument. The default uses all available GPUs, like the following: + +```python +strategy = tf.contrib.distribute.MirroredStrategy() +config = tf.estimator.RunConfig(train_distribute=strategy) +``` + +Finally, train the `Estimator` instance by providing the `input_fn` and `steps` +arguments: + +```python +keras_estimator.train(input_fn=input_fn, steps=10) +``` diff --git a/tensorflow/docs_src/guide/leftnav_files b/tensorflow/docs_src/guide/leftnav_files new file mode 100644 index 0000000000..357a2a1cb9 --- /dev/null +++ b/tensorflow/docs_src/guide/leftnav_files @@ -0,0 +1,40 @@ +index.md + +### High Level APIs +keras.md +eager.md +datasets.md + +### Estimators +estimators.md: Introduction to Estimators +premade_estimators.md +custom_estimators.md +feature_columns.md +datasets_for_estimators.md +checkpoints.md + +### Accelerators +using_gpu.md +using_tpu.md + +### Low Level APIs +low_level_intro.md +tensors.md +variables.md +graphs.md +saved_model.md + +### ML Concepts +embedding.md + +### Debugging +debugger.md + +### TensorBoard +summaries_and_tensorboard.md: Visualizing Learning +graph_viz.md: Graphs +tensorboard_histograms.md: Histograms + +### Misc +version_compat.md +faq.md diff --git a/tensorflow/docs_src/guide/low_level_intro.md b/tensorflow/docs_src/guide/low_level_intro.md new file mode 100644 index 0000000000..665a5568b4 --- /dev/null +++ b/tensorflow/docs_src/guide/low_level_intro.md @@ -0,0 +1,604 @@ +# Introduction + +This guide gets you started programming in the low-level TensorFlow APIs +(TensorFlow Core), showing you how to: + + * Manage your own TensorFlow program (a `tf.Graph`) and TensorFlow + runtime (a `tf.Session`), instead of relying on Estimators to manage them. + * Run TensorFlow operations, using a `tf.Session`. + * Use high level components ([datasets](#datasets), [layers](#layers), and + [feature_columns](#feature_columns)) in this low level environment. + * Build your own training loop, instead of using the one + @{$premade_estimators$provided by Estimators}. + +We recommend using the higher level APIs to build models when possible. +Knowing TensorFlow Core is valuable for the following reasons: + + * Experimentation and debugging are both more straight forward + when you can use low level TensorFlow operations directly. + * It gives you a mental model of how things work internally when + using the higher level APIs. + +## Setup + +Before using this guide, @{$install$install TensorFlow}. + +To get the most out of this guide, you should know the following: + +* How to program in Python. +* At least a little bit about arrays. +* Ideally, something about machine learning. + +Feel free to launch `python` and follow along with this walkthrough. +Run the following lines to set up your Python environment: + +```python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +``` + +## Tensor Values + +The central unit of data in TensorFlow is the **tensor**. A tensor consists of a +set of primitive values shaped into an array of any number of dimensions. A +tensor's **rank** is its number of dimensions, while its **shape** is a tuple +of integers specifying the array's length along each dimension. Here are some +examples of tensor values: + +```python +3. # a rank 0 tensor; a scalar with shape [], +[1., 2., 3.] # a rank 1 tensor; a vector with shape [3] +[[1., 2., 3.], [4., 5., 6.]] # a rank 2 tensor; a matrix with shape [2, 3] +[[[1., 2., 3.]], [[7., 8., 9.]]] # a rank 3 tensor with shape [2, 1, 3] +``` + +TensorFlow uses numpy arrays to represent tensor **values**. + +## TensorFlow Core Walkthrough + +You might think of TensorFlow Core programs as consisting of two discrete +sections: + +1. Building the computational graph (a @{tf.Graph}). +2. Running the computational graph (using a @{tf.Session}). + +### Graph + +A **computational graph** is a series of TensorFlow operations arranged into a +graph. The graph is composed of two types of objects. + + * @{tf.Operation$Operations} (or "ops"): The nodes of the graph. + Operations describe calculations that consume and produce tensors. + * @{tf.Tensor$Tensors}: The edges in the graph. These represent the values + that will flow through the graph. Most TensorFlow functions return + `tf.Tensors`. + +Important: `tf.Tensors` do not have values, they are just handles to elements +in the computation graph. + +Let's build a simple computational graph. The most basic operation is a +constant. The Python function that builds the operation takes a tensor value as +input. The resulting operation takes no inputs. When run, it outputs the +value that was passed to the constructor. We can create two floating point +constants `a` and `b` as follows: + +```python +a = tf.constant(3.0, dtype=tf.float32) +b = tf.constant(4.0) # also tf.float32 implicitly +total = a + b +print(a) +print(b) +print(total) +``` + +The print statements produce: + +``` +Tensor("Const:0", shape=(), dtype=float32) +Tensor("Const_1:0", shape=(), dtype=float32) +Tensor("add:0", shape=(), dtype=float32) +``` + +Notice that printing the tensors does not output the values `3.0`, `4.0`, and +`7.0` as you might expect. The above statements only build the computation +graph. These `tf.Tensor` objects just represent the results of the operations +that will be run. + +Each operation in a graph is given a unique name. This name is independent of +the names the objects are assigned to in Python. Tensors are named after the +operation that produces them followed by an output index, as in +`"add:0"` above. + +### TensorBoard + +TensorFlow provides a utility called TensorBoard. One of TensorBoard's many +capabilities is visualizing a computation graph. You can easily do this with +a few simple commands. + +First you save the computation graph to a TensorBoard summary file as +follows: + +``` +writer = tf.summary.FileWriter('.') +writer.add_graph(tf.get_default_graph()) +``` + +This will produce an `event` file in the current directory with a name in the +following format: + +``` +events.out.tfevents.{timestamp}.{hostname} +``` + +Now, in a new terminal, launch TensorBoard with the following shell command: + +```bsh +tensorboard --logdir . +``` + +Then open TensorBoard's [graphs page](http://localhost:6006/#graphs) in your +browser, and you should see a graph similar to the following: + +![TensorBoard screenshot](https://www.tensorflow.org/images/getting_started_add.png) + +For more about TensorBoard's graph visualization tools see @{$graph_viz}. + +### Session + +To evaluate tensors, instantiate a @{tf.Session} object, informally known as a +**session**. A session encapsulates the state of the TensorFlow runtime, and +runs TensorFlow operations. If a `tf.Graph` is like a `.py` file, a `tf.Session` +is like the `python` executable. + +The following code creates a `tf.Session` object and then invokes its `run` +method to evaluate the `total` tensor we created above: + +```python +sess = tf.Session() +print(sess.run(total)) +``` + +When you request the output of a node with `Session.run` TensorFlow backtracks +through the graph and runs all the nodes that provide input to the requested +output node. So this prints the expected value of 7.0: + +``` +7.0 +``` + +You can pass multiple tensors to `tf.Session.run`. The `run` method +transparently handles any combination of tuples or dictionaries, as in the +following example: + +```python +print(sess.run({'ab':(a, b), 'total':total})) +``` + +which returns the results in a structure of the same layout: + +``` None +{'total': 7.0, 'ab': (3.0, 4.0)} +``` + +During a call to `tf.Session.run` any `tf.Tensor` only has a single value. +For example, the following code calls `tf.random_uniform` to produce a +`tf.Tensor` that generates a random 3-element vector (with values in `[0,1)`): + +```python +vec = tf.random_uniform(shape=(3,)) +out1 = vec + 1 +out2 = vec + 2 +print(sess.run(vec)) +print(sess.run(vec)) +print(sess.run((out1, out2))) +``` + +The result shows a different random value on each call to `run`, but +a consistent value during a single `run` (`out1` and `out2` receive the same +random input): + +``` +[ 0.52917576 0.64076328 0.68353939] +[ 0.66192627 0.89126778 0.06254101] +( + array([ 1.88408756, 1.87149239, 1.84057522], dtype=float32), + array([ 2.88408756, 2.87149239, 2.84057522], dtype=float32) +) +``` + +Some TensorFlow functions return `tf.Operations` instead of `tf.Tensors`. +The result of calling `run` on an Operation is `None`. You run an operation +to cause a side-effect, not to retrieve a value. Examples of this include the +[initialization](#Initializing Layers), and [training](#Training) ops +demonstrated later. + +### Feeding + +As it stands, this graph is not especially interesting because it always +produces a constant result. A graph can be parameterized to accept external +inputs, known as **placeholders**. A **placeholder** is a promise to provide a +value later, like a function argument. + +```python +x = tf.placeholder(tf.float32) +y = tf.placeholder(tf.float32) +z = x + y +``` + +The preceding three lines are a bit like a function in which we +define two input parameters (`x` and `y`) and then an operation on them. We can +evaluate this graph with multiple inputs by using the `feed_dict` argument of +the @{tf.Session.run$run method} to feed concrete values to the placeholders: + +```python +print(sess.run(z, feed_dict={x: 3, y: 4.5})) +print(sess.run(z, feed_dict={x: [1, 3], y: [2, 4]})) +``` +This results in the following output: + +``` +7.5 +[ 3. 7.] +``` + +Also note that the `feed_dict` argument can be used to overwrite any tensor in +the graph. The only difference between placeholders and other `tf.Tensors` is +that placeholders throw an error if no value is fed to them. + +## Datasets + +Placeholders work for simple experiments, but @{tf.data$Datasets} are the +preferred method of streaming data into a model. + +To get a runnable `tf.Tensor` from a Dataset you must first convert it to a +@{tf.data.Iterator}, and then call the Iterator's +@{tf.data.Iterator.get_next$`get_next`} method. + +The simplest way to create an Iterator is with the +@{tf.data.Dataset.make_one_shot_iterator$`make_one_shot_iterator`} method. +For example, in the following code the `next_item` tensor will return a row from +the `my_data` array on each `run` call: + +``` python +my_data = [ + [0, 1,], + [2, 3,], + [4, 5,], + [6, 7,], +] +slices = tf.data.Dataset.from_tensor_slices(my_data) +next_item = slices.make_one_shot_iterator().get_next() +``` + +Reaching the end of the data stream causes `Dataset` to throw an +@{tf.errors.OutOfRangeError$`OutOfRangeError`}. For example, the following code +reads the `next_item` until there is no more data to read: + +``` python +while True: + try: + print(sess.run(next_item)) + except tf.errors.OutOfRangeError: + break +``` + +If the `Dataset` depends on stateful operations you may need to +initialize the iterator before using it, as shown below: + +``` python +r = tf.random_normal([10,3]) +dataset = tf.data.Dataset.from_tensor_slices(r) +iterator = dataset.make_initializable_iterator() +next_row = iterator.get_next() + +sess.run(iterator.initializer) +while True: + try: + print(sess.run(next_row)) + except tf.errors.OutOfRangeError: + break +``` + +For more details on Datasets and Iterators see: @{$guide/datasets}. + +## Layers + +A trainable model must modify the values in the graph to get new outputs with +the same input. @{tf.layers$Layers} are the preferred way to add trainable +parameters to a graph. + +Layers package together both the variables and the operations that act +on them. For example a +[densely-connected layer](https://developers.google.com/machine-learning/glossary/#fully_connected_layer) +performs a weighted sum across all inputs +for each output and applies an optional +[activation function](https://developers.google.com/machine-learning/glossary/#activation_function). +The connection weights and biases are managed by the layer object. + +### Creating Layers + +The following code creates a @{tf.layers.Dense$`Dense`} layer that takes a +batch of input vectors, and produces a single output value for each. To apply a +layer to an input, call the layer as if it were a function. For example: + +```python +x = tf.placeholder(tf.float32, shape=[None, 3]) +linear_model = tf.layers.Dense(units=1) +y = linear_model(x) +``` + +The layer inspects its input to determine sizes for its internal variables. So +here we must set the shape of the `x` placeholder so that the layer can +build a weight matrix of the correct size. + +Now that we have defined the calculation of the output, `y`, there is one more +detail we need to take care of before we run the calculation. + +### Initializing Layers + +The layer contains variables that must be **initialized** before they can be +used. While it is possible to initialize variables individually, you can easily +initialize all the variables in a TensorFlow graph as follows: + +```python +init = tf.global_variables_initializer() +sess.run(init) +``` + +Important: Calling `tf.global_variables_initializer` only +creates and returns a handle to a TensorFlow operation. That op +will initialize all the global variables when we run it with `tf.Session.run`. + +Also note that this `global_variables_initializer` only initializes variables +that existed in the graph when the initializer was created. So the initializer +should be one of the last things added during graph construction. + +### Executing Layers + +Now that the layer is initialized, we can evaluate the `linear_model`'s output +tensor as we would any other tensor. For example, the following code: + +```python +print(sess.run(y, {x: [[1, 2, 3],[4, 5, 6]]})) +``` + +will generate a two-element output vector such as the following: + +``` +[[-3.41378999] + [-9.14999008]] +``` + +### Layer Function shortcuts + +For each layer class (like @{tf.layers.Dense}) TensorFlow also supplies a +shortcut function (like @{tf.layers.dense}). The only difference is that the +shortcut function versions create and run the layer in a single call. For +example, the following code is equivalent to the earlier version: + +```python +x = tf.placeholder(tf.float32, shape=[None, 3]) +y = tf.layers.dense(x, units=1) + +init = tf.global_variables_initializer() +sess.run(init) + +print(sess.run(y, {x: [[1, 2, 3], [4, 5, 6]]})) +``` + +While convenient, this approach allows no access to the @{tf.layers.Layer} +object. This makes introspection and debugging more difficult, +and layer reuse impossible. + +## Feature columns + +The easiest way to experiment with feature columns is using the +@{tf.feature_column.input_layer} function. This function only accepts +@{$feature_columns$dense columns} as inputs, so to view the result +of a categorical column you must wrap it in an +@{tf.feature_column.indicator_column}. For example: + +``` python +features = { + 'sales' : [[5], [10], [8], [9]], + 'department': ['sports', 'sports', 'gardening', 'gardening']} + +department_column = tf.feature_column.categorical_column_with_vocabulary_list( + 'department', ['sports', 'gardening']) +department_column = tf.feature_column.indicator_column(department_column) + +columns = [ + tf.feature_column.numeric_column('sales'), + department_column +] + +inputs = tf.feature_column.input_layer(features, columns) +``` + +Running the `inputs` tensor will parse the `features` into a batch of vectors. + +Feature columns can have internal state, like layers, so they often need to be +initialized. Categorical columns use @{tf.contrib.lookup$lookup tables} +internally and these require a separate initialization op, +@{tf.tables_initializer}. + +``` python +var_init = tf.global_variables_initializer() +table_init = tf.tables_initializer() +sess = tf.Session() +sess.run((var_init, table_init)) +``` + +Once the internal state has been initialized you can run `inputs` like any +other `tf.Tensor`: + +```python +print(sess.run(inputs)) +``` + +This shows how the feature columns have packed the input vectors, with the +one-hot "department" as the first two indices and "sales" as the third. + +```None +[[ 1. 0. 5.] + [ 1. 0. 10.] + [ 0. 1. 8.] + [ 0. 1. 9.]] +``` + +## Training + +Now that you're familiar with the basics of core TensorFlow, let's train a +small regression model manually. + +### Define the data + +First let's define some inputs, `x`, and the expected output for each input, +`y_true`: + +```python +x = tf.constant([[1], [2], [3], [4]], dtype=tf.float32) +y_true = tf.constant([[0], [-1], [-2], [-3]], dtype=tf.float32) +``` + +### Define the model + +Next, build a simple linear model, with 1 output: + +``` python +linear_model = tf.layers.Dense(units=1) + +y_pred = linear_model(x) +``` + +You can evaluate the predictions as follows: + +``` python +sess = tf.Session() +init = tf.global_variables_initializer() +sess.run(init) + +print(sess.run(y_pred)) +``` + +The model hasn't yet been trained, so the four "predicted" values aren't very +good. Here's what we got; your own output will almost certainly differ: + +``` None +[[ 0.02631879] + [ 0.05263758] + [ 0.07895637] + [ 0.10527515]] +``` + +### Loss + +To optimize a model, you first need to define the loss. We'll use the mean +square error, a standard loss for regression problems. + +While you could do this manually with lower level math operations, +the @{tf.losses} module provides a set of common loss functions. You can use it +to calculate the mean square error as follows: + +``` python +loss = tf.losses.mean_squared_error(labels=y_true, predictions=y_pred) + +print(sess.run(loss)) +``` +This will produce a loss value, something like: + +``` None +2.23962 +``` + +### Training + +TensorFlow provides +[**optimizers**](https://developers.google.com/machine-learning/glossary/#optimizer) +implementing standard optimization algorithms. These are implemented as +sub-classes of @{tf.train.Optimizer}. They incrementally change each +variable in order to minimize the loss. The simplest optimization algorithm is +[**gradient descent**](https://developers.google.com/machine-learning/glossary/#gradient_descent), +implemented by @{tf.train.GradientDescentOptimizer}. It modifies each +variable according to the magnitude of the derivative of loss with respect to +that variable. For example: + +```python +optimizer = tf.train.GradientDescentOptimizer(0.01) +train = optimizer.minimize(loss) +``` + +This code builds all the graph components necessary for the optimization, and +returns a training operation. When run, the training op will update variables +in the graph. You might run it as follows: + +```python +for i in range(100): + _, loss_value = sess.run((train, loss)) + print(loss_value) +``` + +Since `train` is an op, not a tensor, it doesn't return a value when run. +To see the progression of the loss during training, we run the loss tensor at +the same time, producing output like the following: + +``` None +1.35659 +1.00412 +0.759167 +0.588829 +0.470264 +0.387626 +0.329918 +0.289511 +0.261112 +0.241046 +... +``` + +### Complete program + +```python +x = tf.constant([[1], [2], [3], [4]], dtype=tf.float32) +y_true = tf.constant([[0], [-1], [-2], [-3]], dtype=tf.float32) + +linear_model = tf.layers.Dense(units=1) + +y_pred = linear_model(x) +loss = tf.losses.mean_squared_error(labels=y_true, predictions=y_pred) + +optimizer = tf.train.GradientDescentOptimizer(0.01) +train = optimizer.minimize(loss) + +init = tf.global_variables_initializer() + +sess = tf.Session() +sess.run(init) +for i in range(100): + _, loss_value = sess.run((train, loss)) + print(loss_value) + +print(sess.run(y_pred)) +``` + +## Next steps + +To learn more about building models with TensorFlow consider the following: + +* @{$custom_estimators$Custom Estimators}, to learn how to build + customized models with TensorFlow. Your knowledge of TensorFlow Core will + help you understand and debug your own models. + +If you want to learn more about the inner workings of TensorFlow consider the +following documents, which go into more depth on many of the topics discussed +here: + +* @{$graphs} +* @{$tensors} +* @{$variables} + + diff --git a/tensorflow/docs_src/guide/premade_estimators.md b/tensorflow/docs_src/guide/premade_estimators.md new file mode 100644 index 0000000000..3e910c1fe2 --- /dev/null +++ b/tensorflow/docs_src/guide/premade_estimators.md @@ -0,0 +1,430 @@ +# Premade Estimators + +This document introduces the TensorFlow programming environment and shows you +how to solve the Iris classification problem in TensorFlow. + +## Prerequisites + +Prior to using the sample code in this document, you'll need to do the +following: + +* @{$install$Install TensorFlow}. +* If you installed TensorFlow with virtualenv or Anaconda, activate your + TensorFlow environment. +* Install or upgrade pandas by issuing the following command: + + pip install pandas + +## Getting the sample code + +Take the following steps to get the sample code we'll be going through: + +1. Clone the TensorFlow Models repository from GitHub by entering the following + command: + + git clone https://github.com/tensorflow/models + +1. Change directory within that branch to the location containing the examples + used in this document: + + cd models/samples/core/get_started/ + +The program described in this document is +[`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py). +This program uses +[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py) +to fetch its training data. + +### Running the program + +You run TensorFlow programs as you would run any Python program. For example: + +``` bsh +python premade_estimator.py +``` + +The program should output training logs followed by some predictions against +the test set. For example, the first line in the following output shows that +the model thinks there is a 99.6% chance that the first example in the test +set is a Setosa. Since the test set expected Setosa, this appears to be +a good prediction. + +``` None +... +Prediction is "Setosa" (99.6%), expected "Setosa" + +Prediction is "Versicolor" (99.8%), expected "Versicolor" + +Prediction is "Virginica" (97.9%), expected "Virginica" +``` + +If the program generates errors instead of answers, ask yourself the following +questions: + +* Did you install TensorFlow properly? +* Are you using the correct version of TensorFlow? +* Did you activate the environment you installed TensorFlow in? (This is + only relevant in certain installation mechanisms.) + +## The programming stack + +Before getting into the details of the program itself, let's investigate the +programming environment. As the following illustration shows, TensorFlow +provides a programming stack consisting of multiple API layers: + +
+ +
+ +We strongly recommend writing TensorFlow programs with the following APIs: + +* @{$guide/estimators$Estimators}, which represent a complete model. + The Estimator API provides methods to train the model, to judge the model's + accuracy, and to generate predictions. +* @{$guide/datasets_for_estimators}, which build a data input + pipeline. The Dataset API has methods to load and manipulate data, and feed + it into your model. The Dataset API meshes well with the Estimators API. + +## Classifying irises: an overview + +The sample program in this document builds and tests a model that +classifies Iris flowers into three different species based on the size of their +[sepals](https://en.wikipedia.org/wiki/Sepal) and +[petals](https://en.wikipedia.org/wiki/Petal). + +
+Petal geometry compared for three iris species: Iris setosa, Iris virginica, and Iris versicolor +
+ +**From left to right, +[*Iris setosa*](https://commons.wikimedia.org/w/index.php?curid=170298) (by +[Radomil](https://commons.wikimedia.org/wiki/User:Radomil), CC BY-SA 3.0), +[*Iris versicolor*](https://commons.wikimedia.org/w/index.php?curid=248095) (by +[Dlanglois](https://commons.wikimedia.org/wiki/User:Dlanglois), CC BY-SA 3.0), +and [*Iris virginica*](https://www.flickr.com/photos/33397993@N05/3352169862) +(by [Frank Mayfield](https://www.flickr.com/photos/33397993@N05), CC BY-SA +2.0).** + +### The data set + +The Iris data set contains four features and one +[label](https://developers.google.com/machine-learning/glossary/#label). +The four features identify the following botanical characteristics of +individual Iris flowers: + +* sepal length +* sepal width +* petal length +* petal width + +Our model will represent these features as `float32` numerical data. + +The label identifies the Iris species, which must be one of the following: + +* Iris setosa (0) +* Iris versicolor (1) +* Iris virginica (2) + +Our model will represent the label as `int32` categorical data. + +The following table shows three examples in the data set: + +|sepal length | sepal width | petal length | petal width| species (label) | +|------------:|------------:|-------------:|-----------:|:---------------:| +| 5.1 | 3.3 | 1.7 | 0.5 | 0 (Setosa) | +| 5.0 | 2.3 | 3.3 | 1.0 | 1 (versicolor)| +| 6.4 | 2.8 | 5.6 | 2.2 | 2 (virginica) | + +### The algorithm + +The program trains a Deep Neural Network classifier model having the following +topology: + +* 2 hidden layers. +* Each hidden layer contains 10 nodes. + +The following figure illustrates the features, hidden layers, and predictions +(not all of the nodes in the hidden layers are shown): + +
+A diagram of the network architecture: Inputs, 2 hidden layers, and outputs +
+ +### Inference + +Running the trained model on an unlabeled example yields three predictions, +namely, the likelihood that this flower is the given Iris species. The sum of +those output predictions will be 1.0. For example, the prediction on an +unlabeled example might be something like the following: + +* 0.03 for Iris Setosa +* 0.95 for Iris Versicolor +* 0.02 for Iris Virginica + +The preceding prediction indicates a 95% probability that the given unlabeled +example is an Iris Versicolor. + +## Overview of programming with Estimators + +An Estimator is TensorFlow's high-level representation of a complete model. It +handles the details of initialization, logging, saving and restoring, and many +other features so you can concentrate on your model. For more details see +@{$guide/estimators}. + +An Estimator is any class derived from @{tf.estimator.Estimator}. TensorFlow +provides a collection of +@{tf.estimator$pre-made Estimators} +(for example, `LinearRegressor`) to implement common ML algorithms. Beyond +those, you may write your own +@{$custom_estimators$custom Estimators}. +We recommend using pre-made Estimators when just getting started. + +To write a TensorFlow program based on pre-made Estimators, you must perform the +following tasks: + +* Create one or more input functions. +* Define the model's feature columns. +* Instantiate an Estimator, specifying the feature columns and various + hyperparameters. +* Call one or more methods on the Estimator object, passing the appropriate + input function as the source of the data. + +Let's see how those tasks are implemented for Iris classification. + +## Create input functions + +You must create input functions to supply data for training, +evaluating, and prediction. + +An **input function** is a function that returns a @{tf.data.Dataset} object +which outputs the following two-element tuple: + +* [`features`](https://developers.google.com/machine-learning/glossary/#feature) - A Python dictionary in which: + * Each key is the name of a feature. + * Each value is an array containing all of that feature's values. +* `label` - An array containing the values of the + [label](https://developers.google.com/machine-learning/glossary/#label) for + every example. + +Just to demonstrate the format of the input function, here's a simple +implementation: + +```python +def input_evaluation_set(): + features = {'SepalLength': np.array([6.4, 5.0]), + 'SepalWidth': np.array([2.8, 2.3]), + 'PetalLength': np.array([5.6, 3.3]), + 'PetalWidth': np.array([2.2, 1.0])} + labels = np.array([2, 1]) + return features, labels +``` + +Your input function may generate the `features` dictionary and `label` list any +way you like. However, we recommend using TensorFlow's Dataset API, which can +parse all sorts of data. At a high level, the Dataset API consists of the +following classes: + +
+A diagram showing subclasses of the Dataset class +
+ +Where the individual members are: + +* `Dataset` - Base class containing methods to create and transform + datasets. Also allows you to initialize a dataset from data in memory, or from + a Python generator. +* `TextLineDataset` - Reads lines from text files. +* `TFRecordDataset` - Reads records from TFRecord files. +* `FixedLengthRecordDataset` - Reads fixed size records from binary files. +* `Iterator` - Provides a way to access one data set element at a time. + +The Dataset API can handle a lot of common cases for you. For example, +using the Dataset API, you can easily read in records from a large collection +of files in parallel and join them into a single stream. + +To keep things simple in this example we are going to load the data with +[pandas](https://pandas.pydata.org/), and build our input pipeline from this +in-memory data. + +Here is the input function used for training in this program, which is available +in [`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py): + +``` python +def train_input_fn(features, labels, batch_size): + """An input function for training""" + # Convert the inputs to a Dataset. + dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) + + # Shuffle, repeat, and batch the examples. + return dataset.shuffle(1000).repeat().batch(batch_size) +``` + +## Define the feature columns + +A [**feature column**](https://developers.google.com/machine-learning/glossary/#feature_columns) +is an object describing how the model should use raw input data from the +features dictionary. When you build an Estimator model, you pass it a list of +feature columns that describes each of the features you want the model to use. +The @{tf.feature_column} module provides many options for representing data +to the model. + +For Iris, the 4 raw features are numeric values, so we'll build a list of +feature columns to tell the Estimator model to represent each of the four +features as 32-bit floating-point values. Therefore, the code to create the +feature column is: + +```python +# Feature columns describe how to use the input. +my_feature_columns = [] +for key in train_x.keys(): + my_feature_columns.append(tf.feature_column.numeric_column(key=key)) +``` + +Feature columns can be far more sophisticated than those we're showing here. We +detail feature columns @{$feature_columns$later on} in our Getting +Started guide. + +Now that we have the description of how we want the model to represent the raw +features, we can build the estimator. + + +## Instantiate an estimator + +The Iris problem is a classic classification problem. Fortunately, TensorFlow +provides several pre-made classifier Estimators, including: + +* @{tf.estimator.DNNClassifier} for deep models that perform multi-class + classification. +* @{tf.estimator.DNNLinearCombinedClassifier} for wide & deep models. +* @{tf.estimator.LinearClassifier} for classifiers based on linear models. + +For the Iris problem, `tf.estimator.DNNClassifier` seems like the best choice. +Here's how we instantiated this Estimator: + +```python +# Build a DNN with 2 hidden layers and 10 nodes in each hidden layer. +classifier = tf.estimator.DNNClassifier( + feature_columns=my_feature_columns, + # Two hidden layers of 10 nodes each. + hidden_units=[10, 10], + # The model must choose between 3 classes. + n_classes=3) +``` + +## Train, Evaluate, and Predict + +Now that we have an Estimator object, we can call methods to do the following: + +* Train the model. +* Evaluate the trained model. +* Use the trained model to make predictions. + +### Train the model + +Train the model by calling the Estimator's `train` method as follows: + +```python +# Train the Model. +classifier.train( + input_fn=lambda:iris_data.train_input_fn(train_x, train_y, args.batch_size), + steps=args.train_steps) +``` + +Here we wrap up our `input_fn` call in a +[`lambda`](https://docs.python.org/3/tutorial/controlflow.html) +to capture the arguments while providing an input function that takes no +arguments, as expected by the Estimator. The `steps` argument tells the method +to stop training after a number of training steps. + +### Evaluate the trained model + +Now that the model has been trained, we can get some statistics on its +performance. The following code block evaluates the accuracy of the trained +model on the test data: + +```python +# Evaluate the model. +eval_result = classifier.evaluate( + input_fn=lambda:iris_data.eval_input_fn(test_x, test_y, args.batch_size)) + +print('\nTest set accuracy: {accuracy:0.3f}\n'.format(**eval_result)) +``` + +Unlike our call to the `train` method, we did not pass the `steps` +argument to evaluate. Our `eval_input_fn` only yields a single +[epoch](https://developers.google.com/machine-learning/glossary/#epoch) of data. + +Running this code yields the following output (or something similar): + +```none +Test set accuracy: 0.967 +``` + +### Making predictions (inferring) from the trained model + +We now have a trained model that produces good evaluation results. +We can now use the trained model to predict the species of an Iris flower +based on some unlabeled measurements. As with training and evaluation, we make +predictions using a single function call: + +```python +# Generate predictions from the model +expected = ['Setosa', 'Versicolor', 'Virginica'] +predict_x = { + 'SepalLength': [5.1, 5.9, 6.9], + 'SepalWidth': [3.3, 3.0, 3.1], + 'PetalLength': [1.7, 4.2, 5.4], + 'PetalWidth': [0.5, 1.5, 2.1], +} + +predictions = classifier.predict( + input_fn=lambda:iris_data.eval_input_fn(predict_x, + batch_size=args.batch_size)) +``` + +The `predict` method returns a Python iterable, yielding a dictionary of +prediction results for each example. The following code prints a few +predictions and their probabilities: + + +``` python +template = ('\nPrediction is "{}" ({:.1f}%), expected "{}"') + +for pred_dict, expec in zip(predictions, expected): + class_id = pred_dict['class_ids'][0] + probability = pred_dict['probabilities'][class_id] + + print(template.format(iris_data.SPECIES[class_id], + 100 * probability, expec)) +``` + +Running the preceding code yields the following output: + +``` None +... +Prediction is "Setosa" (99.6%), expected "Setosa" + +Prediction is "Versicolor" (99.8%), expected "Versicolor" + +Prediction is "Virginica" (97.9%), expected "Virginica" +``` + + +## Summary + +Pre-made Estimators are an effective way to quickly create standard models. + +Now that you've gotten started writing TensorFlow programs, consider the +following material: + +* @{$checkpoints$Checkpoints} to learn how to save and restore models. +* @{$guide/datasets_for_estimators} to learn more about importing + data into your model. +* @{$custom_estimators$Creating Custom Estimators} to learn how to + write your own Estimator, customized for a particular problem. diff --git a/tensorflow/docs_src/guide/saved_model.md b/tensorflow/docs_src/guide/saved_model.md new file mode 100644 index 0000000000..27ef7bb0da --- /dev/null +++ b/tensorflow/docs_src/guide/saved_model.md @@ -0,0 +1,999 @@ +# Save and Restore + +The @{tf.train.Saver} class provides methods to save and restore models. The +@{tf.saved_model.simple_save} function is an easy way to build a +@{tf.saved_model$saved model} suitable for serving. +[Estimators](@{$guide/estimators}) automatically save and restore +variables in the `model_dir`. + +## Save and restore variables + +TensorFlow @{$variables} are the best way to represent shared, persistent state +manipulated by your program. The `tf.train.Saver` constructor adds `save` and +`restore` ops to the graph for all, or a specified list, of the variables in the +graph. The `Saver` object provides methods to run these ops, specifying paths +for the checkpoint files to write to or read from. + +`Saver` restores all variables already defined in your model. If you're +loading a model without knowing how to build its graph (for example, if you're +writing a generic program to load models), then read the +[Overview of saving and restoring models](#models) section +later in this document. + +TensorFlow saves variables in binary *checkpoint files* that map variable +names to tensor values. + +Caution: TensorFlow model files are code. Be careful with untrusted code. +See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) +for details. + +### Save variables + +Create a `Saver` with `tf.train.Saver()` to manage all variables in the +model. For example, the following snippet demonstrates how to call the +`tf.train.Saver.save` method to save variables to checkpoint files: + +```python +# Create some variables. +v1 = tf.get_variable("v1", shape=[3], initializer = tf.zeros_initializer) +v2 = tf.get_variable("v2", shape=[5], initializer = tf.zeros_initializer) + +inc_v1 = v1.assign(v1+1) +dec_v2 = v2.assign(v2-1) + +# Add an op to initialize the variables. +init_op = tf.global_variables_initializer() + +# Add ops to save and restore all the variables. +saver = tf.train.Saver() + +# Later, launch the model, initialize the variables, do some work, and save the +# variables to disk. +with tf.Session() as sess: + sess.run(init_op) + # Do some work with the model. + inc_v1.op.run() + dec_v2.op.run() + # Save the variables to disk. + save_path = saver.save(sess, "/tmp/model.ckpt") + print("Model saved in path: %s" % save_path) +``` + +### Restore variables + +The `tf.train.Saver` object not only saves variables to checkpoint files, it +also restores variables. Note that when you restore variables you do not have +to initialize them beforehand. For example, the following snippet demonstrates +how to call the `tf.train.Saver.restore` method to restore variables from the +checkpoint files: + +```python +tf.reset_default_graph() + +# Create some variables. +v1 = tf.get_variable("v1", shape=[3]) +v2 = tf.get_variable("v2", shape=[5]) + +# Add ops to save and restore all the variables. +saver = tf.train.Saver() + +# Later, launch the model, use the saver to restore variables from disk, and +# do some work with the model. +with tf.Session() as sess: + # Restore variables from disk. + saver.restore(sess, "/tmp/model.ckpt") + print("Model restored.") + # Check the values of the variables + print("v1 : %s" % v1.eval()) + print("v2 : %s" % v2.eval()) +``` + +Note: There is not a physical file called `/tmp/model.ckpt`. It is the *prefix* of +filenames created for the checkpoint. Users only interact with the prefix +instead of physical checkpoint files. + +### Choose variables to save and restore + +If you do not pass any arguments to `tf.train.Saver()`, the saver handles all +variables in the graph. Each variable is saved under the name that was passed +when the variable was created. + +It is sometimes useful to explicitly specify names for variables in the +checkpoint files. For example, you may have trained a model with a variable +named `"weights"` whose value you want to restore into a variable named +`"params"`. + +It is also sometimes useful to only save or restore a subset of the variables +used by a model. For example, you may have trained a neural net with five +layers, and you now want to train a new model with six layers that reuses the +existing weights of the five trained layers. You can use the saver to restore +the weights of just the first five layers. + +You can easily specify the names and variables to save or load by passing to the +`tf.train.Saver()` constructor either of the following: + +* A list of variables (which will be stored under their own names). +* A Python dictionary in which keys are the names to use and the values are the +variables to manage. + +Continuing from the save/restore examples shown earlier: + +```python +tf.reset_default_graph() +# Create some variables. +v1 = tf.get_variable("v1", [3], initializer = tf.zeros_initializer) +v2 = tf.get_variable("v2", [5], initializer = tf.zeros_initializer) + +# Add ops to save and restore only `v2` using the name "v2" +saver = tf.train.Saver({"v2": v2}) + +# Use the saver object normally after that. +with tf.Session() as sess: + # Initialize v1 since the saver will not. + v1.initializer.run() + saver.restore(sess, "/tmp/model.ckpt") + + print("v1 : %s" % v1.eval()) + print("v2 : %s" % v2.eval()) +``` + +Notes: + +* You can create as many `Saver` objects as you want if you need to save and + restore different subsets of the model variables. The same variable can be + listed in multiple saver objects; its value is only changed when the + `Saver.restore()` method is run. + +* If you only restore a subset of the model variables at the start of a + session, you have to run an initialize op for the other variables. See + @{tf.variables_initializer} for more information. + +* To inspect the variables in a checkpoint, you can use the + [`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py) + library, particularly the `print_tensors_in_checkpoint_file` function. + +* By default, `Saver` uses the value of the @{tf.Variable.name} property + for each variable. However, when you create a `Saver` object, you may + optionally choose names for the variables in the checkpoint files. + + +### Inspect variables in a checkpoint + +We can quickly inspect variables in a checkpoint with the +[`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py) library. + +Continuing from the save/restore examples shown earlier: + +```python +# import the inspect_checkpoint library +from tensorflow.python.tools import inspect_checkpoint as chkp + +# print all tensors in checkpoint file +chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='', all_tensors=True) + +# tensor_name: v1 +# [ 1. 1. 1.] +# tensor_name: v2 +# [-1. -1. -1. -1. -1.] + +# print only tensor v1 in checkpoint file +chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='v1', all_tensors=False) + +# tensor_name: v1 +# [ 1. 1. 1.] + +# print only tensor v2 in checkpoint file +chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='v2', all_tensors=False) + +# tensor_name: v2 +# [-1. -1. -1. -1. -1.] +``` + + + +## Save and restore models + +Use `SavedModel` to save and load your model—variables, the graph, and the +graph's metadata. This is a language-neutral, recoverable, hermetic +serialization format that enables higher-level systems and tools to produce, +consume, and transform TensorFlow models. TensorFlow provides several ways to +interact with `SavedModel`, including the @{tf.saved_model} APIs, +@{tf.estimator.Estimator}, and a command-line interface. + + +## Build and load a SavedModel + +### Simple save + +The easiest way to create a `SavedModel` is to use the @{tf.saved_model.simple_save} +function: + +```python +simple_save(session, + export_dir, + inputs={"x": x, "y": y}, + outputs={"z": z}) +``` + +This configures the `SavedModel` so it can be loaded by +[TensorFlow serving](/serving/serving_basic) and supports the +[Predict API](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/predict.proto). +To access the classify, regress, or multi-inference APIs, use the manual +`SavedModel` builder APIs or an @{tf.estimator.Estimator}. + +### Manually build a SavedModel + +If your use case isn't covered by @{tf.saved_model.simple_save}, use the manual +@{tf.saved_model.builder$builder APIs} to create a `SavedModel`. + +The @{tf.saved_model.builder.SavedModelBuilder} class provides functionality to +save multiple `MetaGraphDef`s. A **MetaGraph** is a dataflow graph, plus +its associated variables, assets, and signatures. A **`MetaGraphDef`** +is the protocol buffer representation of a MetaGraph. A **signature** is +the set of inputs to and outputs from a graph. + +If assets need to be saved and written or copied to disk, they can be provided +when the first `MetaGraphDef` is added. If multiple `MetaGraphDef`s are +associated with an asset of the same name, only the first version is retained. + +Each `MetaGraphDef` added to the SavedModel must be annotated with +user-specified tags. The tags provide a means to identify the specific +`MetaGraphDef` to load and restore, along with the shared set of variables +and assets. These tags +typically annotate a `MetaGraphDef` with its functionality (for example, +serving or training), and optionally with hardware-specific aspects (for +example, GPU). + +For example, the following code suggests a typical way to use +`SavedModelBuilder` to build a SavedModel: + +```python +export_dir = ... +... +builder = tf.saved_model.builder.SavedModelBuilder(export_dir) +with tf.Session(graph=tf.Graph()) as sess: + ... + builder.add_meta_graph_and_variables(sess, + [tag_constants.TRAINING], + signature_def_map=foo_signatures, + assets_collection=foo_assets, + strip_default_attrs=True) +... +# Add a second MetaGraphDef for inference. +with tf.Session(graph=tf.Graph()) as sess: + ... + builder.add_meta_graph([tag_constants.SERVING], strip_default_attrs=True) +... +builder.save() +``` + + +#### Forward compatibility via `strip_default_attrs=True` + +Following the guidance below gives you forward compatibility only if the set of +Ops has not changed. + +The @{tf.saved_model.builder.SavedModelBuilder$`SavedModelBuilder`} class allows +users to control whether default-valued attributes must be stripped from the +@{$extend/tool_developers#nodes$`NodeDefs`} +while adding a meta graph to the SavedModel bundle. Both +@{tf.saved_model.builder.SavedModelBuilder.add_meta_graph_and_variables$`SavedModelBuilder.add_meta_graph_and_variables`} +and @{tf.saved_model.builder.SavedModelBuilder.add_meta_graph$`SavedModelBuilder.add_meta_graph`} +methods accept a Boolean flag `strip_default_attrs` that controls this behavior. + +If `strip_default_attrs` is `False`, the exported @{tf.MetaGraphDef} will have +the default valued attributes in all its @{tf.NodeDef} instances. +This can break forward compatibility with a sequence of events such as the +following: + +* An existing Op (`Foo`) is updated to include a new attribute (`T`) with a + default (`bool`) at version 101. +* A model producer such as a "trainer binary" picks up this change (version 101) + to the `OpDef` and re-exports an existing model that uses Op `Foo`. +* A model consumer (such as [Tensorflow Serving](/serving)) running an older + binary (version 100) doesn't have attribute `T` for Op `Foo`, but tries to + import this model. The model consumer doesn't recognize attribute `T` in a + `NodeDef` that uses Op `Foo` and therefore fails to load the model. +* By setting `strip_default_attrs` to True, the model producers can strip away + any default valued attributes in the `NodeDefs`. This helps ensure that newly + added attributes with defaults don't cause older model consumers to fail + loading models regenerated with newer training binaries. + +See [compatibility guidance](./version_compat.md) +for more information. + +### Loading a SavedModel in Python + +The Python version of the SavedModel +@{tf.saved_model.loader$loader} +provides load and restore capability for a SavedModel. The `load` operation +requires the following information: + +* The session in which to restore the graph definition and variables. +* The tags used to identify the MetaGraphDef to load. +* The location (directory) of the SavedModel. + +Upon a load, the subset of variables, assets, and signatures supplied as part of +the specific MetaGraphDef will be restored into the supplied session. + + +```python +export_dir = ... +... +with tf.Session(graph=tf.Graph()) as sess: + tf.saved_model.loader.load(sess, [tag_constants.TRAINING], export_dir) + ... +``` + + +### Load a SavedModel in C++ + +The C++ version of the SavedModel +[loader](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/loader.h) +provides an API to load a SavedModel from a path, while allowing +`SessionOptions` and `RunOptions`. +You have to specify the tags associated with the graph to be loaded. +The loaded version of SavedModel is referred to as `SavedModelBundle` +and contains the MetaGraphDef and the session within which it is loaded. + +```c++ +const string export_dir = ... +SavedModelBundle bundle; +... +LoadSavedModel(session_options, run_options, export_dir, {kSavedModelTagTrain}, + &bundle); +``` + +### Load and serve a SavedModel in TensorFlow serving + +You can easily load and serve a SavedModel with the TensorFlow Serving Model +Server binary. See [instructions](https://www.tensorflow.org/serving/setup#installing_using_apt-get) +on how to install the server, or build it if you wish. + +Once you have the Model Server, run it with: +``` +tensorflow_model_server --port=port-numbers --model_name=your-model-name --model_base_path=your_model_base_path +``` +Set the port and model_name flags to values of your choosing. The +model_base_path flag expects to be to a base directory, with each version of +your model residing in a numerically named subdirectory. If you only have a +single version of your model, simply place it in a subdirectory like so: +* Place the model in /tmp/model/0001 +* Set model_base_path to /tmp/model + +Store different versions of your model in numerically named subdirectories of a +common base directory. For example, suppose the base directory is `/tmp/model`. +If you have only one version of your model, store it in `/tmp/model/0001`. If +you have two versions of your model, store the second version in +`/tmp/model/0002`, and so on. Set the `--model-base_path` flag to the base +directory (`/tmp/model`, in this example). TensorFlow Model Server will serve +the model in the highest numbered subdirectory of that base directory. + +### Standard constants + +SavedModel offers the flexibility to build and load TensorFlow graphs for a +variety of use-cases. For the most common use-cases, SavedModel's APIs +provide a set of constants in Python and C++ that are easy to +reuse and share across tools consistently. + +#### Standard MetaGraphDef tags + +You may use sets of tags to uniquely identify a `MetaGraphDef` saved in a +SavedModel. A subset of commonly used tags is specified in: + +* [Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py) +* [C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/tag_constants.h) + + +#### Standard SignatureDef constants + +A [**SignatureDef**](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto) +is a protocol buffer that defines the signature of a computation +supported by a graph. +Commonly used input keys, output keys, and method names are +defined in: + +* [Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) +* [C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/signature_constants.h) + +## Using SavedModel with Estimators + +After training an `Estimator` model, you may want to create a service +from that model that takes requests and returns a result. You can run such a +service locally on your machine or deploy it in the cloud. + +To prepare a trained Estimator for serving, you must export it in the standard +SavedModel format. This section explains how to: + +* Specify the output nodes and the corresponding + [APIs](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto) + that can be served (Classify, Regress, or Predict). +* Export your model to the SavedModel format. +* Serve the model from a local server and request predictions. + + +### Prepare serving inputs + +During training, an @{$premade_estimators#input_fn$`input_fn()`} ingests data +and prepares it for use by the model. At serving time, similarly, a +`serving_input_receiver_fn()` accepts inference requests and prepares them for +the model. This function has the following purposes: + +* To add placeholders to the graph that the serving system will feed + with inference requests. +* To add any additional ops needed to convert data from the input format + into the feature `Tensor`s expected by the model. + +The function returns a @{tf.estimator.export.ServingInputReceiver} object, +which packages the placeholders and the resulting feature `Tensor`s together. + +A typical pattern is that inference requests arrive in the form of serialized +`tf.Example`s, so the `serving_input_receiver_fn()` creates a single string +placeholder to receive them. The `serving_input_receiver_fn()` is then also +responsible for parsing the `tf.Example`s by adding a @{tf.parse_example} op to +the graph. + +When writing such a `serving_input_receiver_fn()`, you must pass a parsing +specification to @{tf.parse_example} to tell the parser what feature names to +expect and how to map them to `Tensor`s. A parsing specification takes the +form of a dict from feature names to @{tf.FixedLenFeature}, @{tf.VarLenFeature}, +and @{tf.SparseFeature}. Note this parsing specification should not include +any label or weight columns, since those will not be available at serving +time—in contrast to a parsing specification used in the `input_fn()` at +training time. + +In combination, then: + +```py +feature_spec = {'foo': tf.FixedLenFeature(...), + 'bar': tf.VarLenFeature(...)} + +def serving_input_receiver_fn(): + """An input receiver that expects a serialized tf.Example.""" + serialized_tf_example = tf.placeholder(dtype=tf.string, + shape=[default_batch_size], + name='input_example_tensor') + receiver_tensors = {'examples': serialized_tf_example} + features = tf.parse_example(serialized_tf_example, feature_spec) + return tf.estimator.export.ServingInputReceiver(features, receiver_tensors) +``` + +The @{tf.estimator.export.build_parsing_serving_input_receiver_fn} utility +function provides that input receiver for the common case. + +> Note: when training a model to be served using the Predict API with a local +> server, the parsing step is not needed because the model will receive raw +> feature data. + +Even if you require no parsing or other input processing—that is, if the +serving system will feed feature `Tensor`s directly—you must still provide +a `serving_input_receiver_fn()` that creates placeholders for the feature +`Tensor`s and passes them through. The +@{tf.estimator.export.build_raw_serving_input_receiver_fn} utility provides for +this. + +If these utilities do not meet your needs, you are free to write your own +`serving_input_receiver_fn()`. One case where this may be needed is if your +training `input_fn()` incorporates some preprocessing logic that must be +recapitulated at serving time. To reduce the risk of training-serving skew, we +recommend encapsulating such processing in a function which is then called +from both `input_fn()` and `serving_input_receiver_fn()`. + +Note that the `serving_input_receiver_fn()` also determines the *input* +portion of the signature. That is, when writing a +`serving_input_receiver_fn()`, you must tell the parser what signatures +to expect and how to map them to your model's expected inputs. +By contrast, the *output* portion of the signature is determined by the model. + + +### Specify the outputs of a custom model + +When writing a custom `model_fn`, you must populate the `export_outputs` element +of the @{tf.estimator.EstimatorSpec} return value. This is a dict of +`{name: output}` describing the output signatures to be exported and used during +serving. + +In the usual case of making a single prediction, this dict contains +one element, and the `name` is immaterial. In a multi-headed model, each head +is represented by an entry in this dict. In this case the `name` is a string +of your choice that can be used to request a specific head at serving time. + +Each `output` value must be an `ExportOutput` object such as +@{tf.estimator.export.ClassificationOutput}, +@{tf.estimator.export.RegressionOutput}, or +@{tf.estimator.export.PredictOutput}. + +These output types map straightforwardly to the +[TensorFlow Serving APIs](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto), +and so determine which request types will be honored. + +Note: In the multi-headed case, a `SignatureDef` will be generated for each +element of the `export_outputs` dict returned from the model_fn, named using +the same keys. These `SignatureDef`s differ only in their outputs, as +provided by the corresponding `ExportOutput` entry. The inputs are always +those provided by the `serving_input_receiver_fn`. +An inference request may specify the head by name. One head must be named +using [`signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY`](https://www.tensorflow.org/code/tensorflow/python/saved_model/signature_constants.py) +indicating which `SignatureDef` will be served when an inference request +does not specify one. + + +### Perform the export + +To export your trained Estimator, call +@{tf.estimator.Estimator.export_savedmodel} with the export base path and +the `serving_input_receiver_fn`. + +```py +estimator.export_savedmodel(export_dir_base, serving_input_receiver_fn, + strip_default_attrs=True) +``` + +This method builds a new graph by first calling the +`serving_input_receiver_fn()` to obtain feature `Tensor`s, and then calling +this `Estimator`'s `model_fn()` to generate the model graph based on those +features. It starts a fresh `Session`, and, by default, restores the most recent +checkpoint into it. (A different checkpoint may be passed, if needed.) +Finally it creates a time-stamped export directory below the given +`export_dir_base` (i.e., `export_dir_base/`), and writes a +SavedModel into it containing a single `MetaGraphDef` saved from this +Session. + +> Note: It is your responsibility to garbage-collect old exports. +> Otherwise, successive exports will accumulate under `export_dir_base`. + +### Serve the exported model locally + +For local deployment, you can serve your model using +[TensorFlow Serving](https://github.com/tensorflow/serving), an open-source project that loads a +SavedModel and exposes it as a [gRPC](https://www.grpc.io/) service. + +First, [install TensorFlow Serving](https://github.com/tensorflow/serving). + +Then build and run the local model server, substituting `$export_dir_base` with +the path to the SavedModel you exported above: + +```sh +bazel build //tensorflow_serving/model_servers:tensorflow_model_server +bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_base_path=$export_dir_base +``` + +Now you have a server listening for inference requests via gRPC on port 9000! + + +### Request predictions from a local server + +The server responds to gRPC requests according to the +[PredictionService](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto#L15) +gRPC API service definition. (The nested protocol buffers are defined in +various [neighboring files](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis)). + +From the API service definition, the gRPC framework generates client libraries +in various languages providing remote access to the API. In a project using the +Bazel build tool, these libraries are built automatically and provided via +dependencies like these (using Python for example): + +```build + deps = [ + "//tensorflow_serving/apis:classification_proto_py_pb2", + "//tensorflow_serving/apis:regression_proto_py_pb2", + "//tensorflow_serving/apis:predict_proto_py_pb2", + "//tensorflow_serving/apis:prediction_service_proto_py_pb2" + ] +``` + +Python client code can then import the libraries thus: + +```py +from tensorflow_serving.apis import classification_pb2 +from tensorflow_serving.apis import regression_pb2 +from tensorflow_serving.apis import predict_pb2 +from tensorflow_serving.apis import prediction_service_pb2 +``` + +> Note: `prediction_service_pb2` defines the service as a whole and so +> is always required. However a typical client will need only one of +> `classification_pb2`, `regression_pb2`, and `predict_pb2`, depending on the +> type of requests being made. + +Sending a gRPC request is then accomplished by assembling a protocol buffer +containing the request data and passing it to the service stub. Note how the +request protocol buffer is created empty and then populated via the +[generated protocol buffer API](https://developers.google.com/protocol-buffers/docs/reference/python-generated). + +```py +from grpc.beta import implementations + +channel = implementations.insecure_channel(host, int(port)) +stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) + +request = classification_pb2.ClassificationRequest() +example = request.input.example_list.examples.add() +example.features.feature['x'].float_list.value.extend(image[0].astype(float)) + +result = stub.Classify(request, 10.0) # 10 secs timeout +``` + +The returned result in this example is a `ClassificationResponse` protocol +buffer. + +This is a skeletal example; please see the @{$deploy$Tensorflow Serving} +documentation and [examples](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example) +for more details. + +> Note: `ClassificationRequest` and `RegressionRequest` contain a +> `tensorflow.serving.Input` protocol buffer, which in turn contains a list of +> `tensorflow.Example` protocol buffers. `PredictRequest`, by contrast, +> contains a mapping from feature names to values encoded via `TensorProto`. +> Correspondingly: When using the `Classify` and `Regress` APIs, TensorFlow +> Serving feeds serialized `tf.Example`s to the graph, so your +> `serving_input_receiver_fn()` should include a `tf.parse_example()` Op. +> When using the generic `Predict` API, however, TensorFlow Serving feeds raw +> feature data to the graph, so a pass through `serving_input_receiver_fn()` +> should be used. + + + + + + + + + +## CLI to inspect and execute SavedModel + +You can use the SavedModel Command Line Interface (CLI) to inspect and +execute a SavedModel. +For example, you can use the CLI to inspect the model's `SignatureDef`s. +The CLI enables you to quickly confirm that the input +@{$tensors$Tensor dtype and shape} match the model. Moreover, if you +want to test your model, you can use the CLI to do a sanity check by +passing in sample inputs in various formats (for example, Python +expressions) and then fetching the output. + + +### Install the SavedModel CLI + +Broadly speaking, you can install TensorFlow in either of the following +two ways: + +* By installing a pre-built TensorFlow binary. +* By building TensorFlow from source code. + +If you installed TensorFlow through a pre-built TensorFlow binary, +then the SavedModel CLI is already installed on your system +at pathname `bin\saved_model_cli`. + +If you built TensorFlow from source code, you must run the following +additional command to build `saved_model_cli`: + +``` +$ bazel build tensorflow/python/tools:saved_model_cli +``` + +### Overview of commands + +The SavedModel CLI supports the following two commands on a +`MetaGraphDef` in a SavedModel: + +* `show`, which shows a computation on a `MetaGraphDef` in a SavedModel. +* `run`, which runs a computation on a `MetaGraphDef`. + + +### `show` command + +A SavedModel contains one or more `MetaGraphDef`s, identified by their tag-sets. +To serve a model, you +might wonder what kind of `SignatureDef`s are in each model, and what are their +inputs and outputs. The `show` command let you examine the contents of the +SavedModel in hierarchical order. Here's the syntax: + +``` +usage: saved_model_cli show [-h] --dir DIR [--all] +[--tag_set TAG_SET] [--signature_def SIGNATURE_DEF_KEY] +``` + +For example, the following command shows all available +MetaGraphDef tag-sets in the SavedModel: + +``` +$ saved_model_cli show --dir /tmp/saved_model_dir +The given SavedModel contains the following tag-sets: +serve +serve, gpu +``` + +The following command shows all available `SignatureDef` keys in +a `MetaGraphDef`: + +``` +$ saved_model_cli show --dir /tmp/saved_model_dir --tag_set serve +The given SavedModel `MetaGraphDef` contains `SignatureDefs` with the +following keys: +SignatureDef key: "classify_x2_to_y3" +SignatureDef key: "classify_x_to_y" +SignatureDef key: "regress_x2_to_y3" +SignatureDef key: "regress_x_to_y" +SignatureDef key: "regress_x_to_y2" +SignatureDef key: "serving_default" +``` + +If a `MetaGraphDef` has *multiple* tags in the tag-set, you must specify +all tags, each tag separated by a comma. For example: + +```none +$ saved_model_cli show --dir /tmp/saved_model_dir --tag_set serve,gpu +``` + +To show all inputs and outputs TensorInfo for a specific `SignatureDef`, pass in +the `SignatureDef` key to `signature_def` option. This is very useful when you +want to know the tensor key value, dtype and shape of the input tensors for +executing the computation graph later. For example: + +``` +$ saved_model_cli show --dir \ +/tmp/saved_model_dir --tag_set serve --signature_def serving_default +The given SavedModel SignatureDef contains the following input(s): + inputs['x'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1) + name: x:0 +The given SavedModel SignatureDef contains the following output(s): + outputs['y'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1) + name: y:0 +Method name is: tensorflow/serving/predict +``` + +To show all available information in the SavedModel, use the `--all` option. +For example: + +```none +$ saved_model_cli show --dir /tmp/saved_model_dir --all +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['classify_x2_to_y3']: + The given SavedModel SignatureDef contains the following input(s): + inputs['inputs'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1) + name: x2:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['scores'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1) + name: y3:0 + Method name is: tensorflow/serving/classify + +... + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['x'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1) + name: x:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['y'] tensor_info: + dtype: DT_FLOAT + shape: (-1, 1) + name: y:0 + Method name is: tensorflow/serving/predict +``` + + +### `run` command + +Invoke the `run` command to run a graph computation, passing +inputs and then displaying (and optionally saving) the outputs. +Here's the syntax: + +``` +usage: saved_model_cli run [-h] --dir DIR --tag_set TAG_SET --signature_def + SIGNATURE_DEF_KEY [--inputs INPUTS] + [--input_exprs INPUT_EXPRS] [--outdir OUTDIR] + [--overwrite] [--tf_debug] +``` + +The `run` command provides the following two ways to pass inputs to the model: + +* `--inputs` option enables you to pass numpy ndarray in files. +* `--input_exprs` option enables you to pass Python expressions. +* `--input_examples` option enables you to pass `tf.train.Example`. + + +#### `--inputs` + +To pass input data in files, specify the `--inputs` option, which takes the +following general format: + +```bsh +--inputs +``` + +where *INPUTS* is either of the following formats: + +* `=` +* `=[]` + +You may pass multiple *INPUTS*. If you do pass multiple inputs, use a semicolon +to separate each of the *INPUTS*. + +`saved_model_cli` uses `numpy.load` to load the *filename*. +The *filename* may be in any of the following formats: + +* `.npy` +* `.npz` +* pickle format + +A `.npy` file always contains a numpy ndarray. Therefore, when loading from +a `.npy` file, the content will be directly assigned to the specified input +tensor. If you specify a *variable_name* with that `.npy` file, the +*variable_name* will be ignored and a warning will be issued. + +When loading from a `.npz` (zip) file, you may optionally specify a +*variable_name* to identify the variable within the zip file to load for +the input tensor key. If you don't specify a *variable_name*, the SavedModel +CLI will check that only one file is included in the zip file and load it +for the specified input tensor key. + +When loading from a pickle file, if no `variable_name` is specified in the +square brackets, whatever that is inside the pickle file will be passed to the +specified input tensor key. Otherwise, the SavedModel CLI will assume a +dictionary is stored in the pickle file and the value corresponding to +the *variable_name* will be used. + + +#### `--inputs_exprs` + +To pass inputs through Python expressions, specify the `--input_exprs` option. +This can be useful for when you don't have data +files lying around, but still want to sanity check the model with some simple +inputs that match the dtype and shape of the model's `SignatureDef`s. +For example: + +```bsh +`=[[1],[2],[3]]` +``` + +In addition to Python expressions, you may also pass numpy functions. For +example: + +```bsh +`=np.ones((32,32,3))` +``` + +(Note that the `numpy` module is already available to you as `np`.) + + +#### `--inputs_examples` + +To pass `tf.train.Example` as inputs, specify the `--input_examples` option. +For each input key, it takes a list of dictionary, where each dictionary is an +instance of `tf.train.Example`. The dictionary keys are the features and the +values are the value lists for each feature. +For example: + +```bsh +`=[{"age":[22,24],"education":["BS","MS"]}]` +``` + +#### Save output + +By default, the SavedModel CLI writes output to stdout. If a directory is +passed to `--outdir` option, the outputs will be saved as npy files named after +output tensor keys under the given directory. + +Use `--overwrite` to overwrite existing output files. + + +#### TensorFlow debugger (tfdbg) integration + +If `--tf_debug` option is set, the SavedModel CLI will use the +TensorFlow Debugger (tfdbg) to watch the intermediate Tensors and runtime +graphs or subgraphs while running the SavedModel. + + +#### Full examples of `run` + +Given: + +* Your model simply adds `x1` and `x2` to get output `y`. +* All tensors in the model have shape `(-1, 1)`. +* You have two `npy` files: + * `/tmp/my_data1.npy`, which contains a numpy ndarray `[[1], [2], [3]]`. + * `/tmp/my_data2.npy`, which contains another numpy + ndarray `[[0.5], [0.5], [0.5]]`. + +To run these two `npy` files through the model to get output `y`, issue +the following command: + +``` +$ saved_model_cli run --dir /tmp/saved_model_dir --tag_set serve \ +--signature_def x1_x2_to_y --inputs x1=/tmp/my_data1.npy;x2=/tmp/my_data2.npy \ +--outdir /tmp/out +Result for output key y: +[[ 1.5] + [ 2.5] + [ 3.5]] +``` + +Let's change the preceding example slightly. This time, instead of two +`.npy` files, you now have an `.npz` file and a pickle file. Furthermore, +you want to overwrite any existing output file. Here's the command: + +``` +$ saved_model_cli run --dir /tmp/saved_model_dir --tag_set serve \ +--signature_def x1_x2_to_y \ +--inputs x1=/tmp/my_data1.npz[x];x2=/tmp/my_data2.pkl --outdir /tmp/out \ +--overwrite +Result for output key y: +[[ 1.5] + [ 2.5] + [ 3.5]] +``` + +You may specify python expression instead of an input file. For example, +the following command replaces input `x2` with a Python expression: + +``` +$ saved_model_cli run --dir /tmp/saved_model_dir --tag_set serve \ +--signature_def x1_x2_to_y --inputs x1=/tmp/my_data1.npz[x] \ +--input_exprs 'x2=np.ones((3,1))' +Result for output key y: +[[ 2] + [ 3] + [ 4]] +``` + +To run the model with the TensorFlow Debugger on, issue the +following command: + +``` +$ saved_model_cli run --dir /tmp/saved_model_dir --tag_set serve \ +--signature_def serving_default --inputs x=/tmp/data.npz[x] --tf_debug +``` + + + +## Structure of a SavedModel directory + +When you save a model in SavedModel format, TensorFlow creates +a SavedModel directory consisting of the following subdirectories +and files: + +```bsh +assets/ +assets.extra/ +variables/ + variables.data-?????-of-????? + variables.index +saved_model.pb|saved_model.pbtxt +``` + +where: + +* `assets` is a subfolder containing auxiliary (external) files, + such as vocabularies. Assets are copied to the SavedModel location + and can be read when loading a specific `MetaGraphDef`. +* `assets.extra` is a subfolder where higher-level libraries and users can + add their own assets that co-exist with the model, but are not loaded by + the graph. This subfolder is not managed by the SavedModel libraries. +* `variables` is a subfolder that includes output from + `tf.train.Saver`. +* `saved_model.pb` or `saved_model.pbtxt` is the SavedModel protocol buffer. + It includes the graph definitions as `MetaGraphDef` protocol buffers. + +A single SavedModel can represent multiple graphs. In this case, all the +graphs in the SavedModel share a *single* set of checkpoints (variables) +and assets. For example, the following diagram shows one SavedModel +containing three `MetaGraphDef`s, all three of which share the same set +of checkpoints and assets: + +![SavedModel represents checkpoints, assets, and one or more MetaGraphDefs](../images/SavedModel.svg) + +Each graph is associated with a specific set of tags, which enables +identification during a load or restore operation. diff --git a/tensorflow/docs_src/guide/summaries_and_tensorboard.md b/tensorflow/docs_src/guide/summaries_and_tensorboard.md new file mode 100644 index 0000000000..fadfa03e78 --- /dev/null +++ b/tensorflow/docs_src/guide/summaries_and_tensorboard.md @@ -0,0 +1,225 @@ +# TensorBoard: Visualizing Learning + +The computations you'll use TensorFlow for - like training a massive +deep neural network - can be complex and confusing. To make it easier to +understand, debug, and optimize TensorFlow programs, we've included a suite of +visualization tools called TensorBoard. You can use TensorBoard to visualize +your TensorFlow graph, plot quantitative metrics about the execution of your +graph, and show additional data like images that pass through it. When +TensorBoard is fully configured, it looks like this: + +![MNIST TensorBoard](https://www.tensorflow.org/images/mnist_tensorboard.png "MNIST TensorBoard") + +
+ +
+ +This 30-minute tutorial is intended to get you started with simple TensorBoard +usage. It assumes a basic understanding of TensorFlow. + +There are other resources available as well! The [TensorBoard GitHub](https://github.com/tensorflow/tensorboard) +has a lot more information on using individual dashboards within TensorBoard +including tips & tricks and debugging information. + +## Setup + +[Install TensorFlow](https://www.tensorflow.org/install/). Installing TensorFlow +via pip should also automatically install TensorBoard. + +## Serializing the data + +TensorBoard operates by reading TensorFlow events files, which contain summary +data that you can generate when running TensorFlow. Here's the general +lifecycle for summary data within TensorBoard. + +First, create the TensorFlow graph that you'd like to collect summary +data from, and decide which nodes you would like to annotate with +@{$python/summary$summary operations}. + +For example, suppose you are training a convolutional neural network for +recognizing MNIST digits. You'd like to record how the learning rate +varies over time, and how the objective function is changing. Collect these by +attaching @{tf.summary.scalar} ops +to the nodes that output the learning rate and loss respectively. Then, give +each `scalar_summary` a meaningful `tag`, like `'learning rate'` or `'loss +function'`. + +Perhaps you'd also like to visualize the distributions of activations coming +off a particular layer, or the distribution of gradients or weights. Collect +this data by attaching +@{tf.summary.histogram} ops to +the gradient outputs and to the variable that holds your weights, respectively. + +For details on all of the summary operations available, check out the docs on +@{$python/summary$summary operations}. + +Operations in TensorFlow don't do anything until you run them, or an op that +depends on their output. And the summary nodes that we've just created are +peripheral to your graph: none of the ops you are currently running depend on +them. So, to generate summaries, we need to run all of these summary nodes. +Managing them by hand would be tedious, so use +@{tf.summary.merge_all} +to combine them into a single op that generates all the summary data. + +Then, you can just run the merged summary op, which will generate a serialized +`Summary` protobuf object with all of your summary data at a given step. +Finally, to write this summary data to disk, pass the summary protobuf to a +@{tf.summary.FileWriter}. + +The `FileWriter` takes a logdir in its constructor - this logdir is quite +important, it's the directory where all of the events will be written out. +Also, the `FileWriter` can optionally take a `Graph` in its constructor. +If it receives a `Graph` object, then TensorBoard will visualize your graph +along with tensor shape information. This will give you a much better sense of +what flows through the graph: see +@{$graph_viz#tensor-shape-information$Tensor shape information}. + +Now that you've modified your graph and have a `FileWriter`, you're ready to +start running your network! If you want, you could run the merged summary op +every single step, and record a ton of training data. That's likely to be more +data than you need, though. Instead, consider running the merged summary op +every `n` steps. + +The code example below is a modification of the +[simple MNIST tutorial](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/mnist/mnist.py), +in which we have added some summary ops, and run them every ten steps. If you +run this and then launch `tensorboard --logdir=/tmp/tensorflow/mnist`, you'll be able +to visualize statistics, such as how the weights or accuracy varied during +training. The code below is an excerpt; full source is +[here](https://www.tensorflow.org/code/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py). + +```python +def variable_summaries(var): + """Attach a lot of summaries to a Tensor (for TensorBoard visualization).""" + with tf.name_scope('summaries'): + mean = tf.reduce_mean(var) + tf.summary.scalar('mean', mean) + with tf.name_scope('stddev'): + stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) + tf.summary.scalar('stddev', stddev) + tf.summary.scalar('max', tf.reduce_max(var)) + tf.summary.scalar('min', tf.reduce_min(var)) + tf.summary.histogram('histogram', var) + +def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu): + """Reusable code for making a simple neural net layer. + + It does a matrix multiply, bias add, and then uses relu to nonlinearize. + It also sets up name scoping so that the resultant graph is easy to read, + and adds a number of summary ops. + """ + # Adding a name scope ensures logical grouping of the layers in the graph. + with tf.name_scope(layer_name): + # This Variable will hold the state of the weights for the layer + with tf.name_scope('weights'): + weights = weight_variable([input_dim, output_dim]) + variable_summaries(weights) + with tf.name_scope('biases'): + biases = bias_variable([output_dim]) + variable_summaries(biases) + with tf.name_scope('Wx_plus_b'): + preactivate = tf.matmul(input_tensor, weights) + biases + tf.summary.histogram('pre_activations', preactivate) + activations = act(preactivate, name='activation') + tf.summary.histogram('activations', activations) + return activations + +hidden1 = nn_layer(x, 784, 500, 'layer1') + +with tf.name_scope('dropout'): + keep_prob = tf.placeholder(tf.float32) + tf.summary.scalar('dropout_keep_probability', keep_prob) + dropped = tf.nn.dropout(hidden1, keep_prob) + +# Do not apply softmax activation yet, see below. +y = nn_layer(dropped, 500, 10, 'layer2', act=tf.identity) + +with tf.name_scope('cross_entropy'): + # The raw formulation of cross-entropy, + # + # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.softmax(y)), + # reduction_indices=[1])) + # + # can be numerically unstable. + # + # So here we use tf.losses.sparse_softmax_cross_entropy on the + # raw logit outputs of the nn_layer above. + with tf.name_scope('total'): + cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_, logits=y) +tf.summary.scalar('cross_entropy', cross_entropy) + +with tf.name_scope('train'): + train_step = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize( + cross_entropy) + +with tf.name_scope('accuracy'): + with tf.name_scope('correct_prediction'): + correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) + with tf.name_scope('accuracy'): + accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) +tf.summary.scalar('accuracy', accuracy) + +# Merge all the summaries and write them out to /tmp/mnist_logs (by default) +merged = tf.summary.merge_all() +train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train', + sess.graph) +test_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/test') +tf.global_variables_initializer().run() +``` + +After we've initialized the `FileWriters`, we have to add summaries to the +`FileWriters` as we train and test the model. + +```python +# Train the model, and also write summaries. +# Every 10th step, measure test-set accuracy, and write test summaries +# All other steps, run train_step on training data, & add training summaries + +def feed_dict(train): + """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" + if train or FLAGS.fake_data: + xs, ys = mnist.train.next_batch(100, fake_data=FLAGS.fake_data) + k = FLAGS.dropout + else: + xs, ys = mnist.test.images, mnist.test.labels + k = 1.0 + return {x: xs, y_: ys, keep_prob: k} + +for i in range(FLAGS.max_steps): + if i % 10 == 0: # Record summaries and test-set accuracy + summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False)) + test_writer.add_summary(summary, i) + print('Accuracy at step %s: %s' % (i, acc)) + else: # Record train set summaries, and train + summary, _ = sess.run([merged, train_step], feed_dict=feed_dict(True)) + train_writer.add_summary(summary, i) +``` + +You're now all set to visualize this data using TensorBoard. + + +## Launching TensorBoard + +To run TensorBoard, use the following command (alternatively `python -m +tensorboard.main`) + +```bash +tensorboard --logdir=path/to/log-directory +``` + +where `logdir` points to the directory where the `FileWriter` serialized its +data. If this `logdir` directory contains subdirectories which contain +serialized data from separate runs, then TensorBoard will visualize the data +from all of those runs. Once TensorBoard is running, navigate your web browser +to `localhost:6006` to view the TensorBoard. + +When looking at TensorBoard, you will see the navigation tabs in the top right +corner. Each tab represents a set of serialized data that can be visualized. + +For in depth information on how to use the *graph* tab to visualize your graph, +see @{$graph_viz$TensorBoard: Graph Visualization}. + +For more usage information on TensorBoard in general, see the +[TensorBoard GitHub](https://github.com/tensorflow/tensorboard). diff --git a/tensorflow/docs_src/guide/tensorboard_histograms.md b/tensorflow/docs_src/guide/tensorboard_histograms.md new file mode 100644 index 0000000000..918deda190 --- /dev/null +++ b/tensorflow/docs_src/guide/tensorboard_histograms.md @@ -0,0 +1,245 @@ +# TensorBoard Histogram Dashboard + +The TensorBoard Histogram Dashboard displays how the distribution of some +`Tensor` in your TensorFlow graph has changed over time. It does this by showing +many histograms visualizations of your tensor at different points in time. + +## A Basic Example + +Let's start with a simple case: a normally-distributed variable, where the mean +shifts over time. +TensorFlow has an op +[`tf.random_normal`](https://www.tensorflow.org/api_docs/python/tf/random_normal) +which is perfect for this purpose. As is usually the case with TensorBoard, we +will ingest data using a summary op; in this case, +['tf.summary.histogram'](https://www.tensorflow.org/api_docs/python/tf/summary/histogram). +For a primer on how summaries work, please see the general +[TensorBoard tutorial](https://www.tensorflow.org/get_started/summaries_and_tensorboard). + +Here is a code snippet that will generate some histogram summaries containing +normally distributed data, where the mean of the distribution increases over +time. + +```python +import tensorflow as tf + +k = tf.placeholder(tf.float32) + +# Make a normal distribution, with a shifting mean +mean_moving_normal = tf.random_normal(shape=[1000], mean=(5*k), stddev=1) +# Record that distribution into a histogram summary +tf.summary.histogram("normal/moving_mean", mean_moving_normal) + +# Setup a session and summary writer +sess = tf.Session() +writer = tf.summary.FileWriter("/tmp/histogram_example") + +summaries = tf.summary.merge_all() + +# Setup a loop and write the summaries to disk +N = 400 +for step in range(N): + k_val = step/float(N) + summ = sess.run(summaries, feed_dict={k: k_val}) + writer.add_summary(summ, global_step=step) +``` + +Once that code runs, we can load the data into TensorBoard via the command line: + + +```sh +tensorboard --logdir=/tmp/histogram_example +``` + +Once TensorBoard is running, load it in Chrome or Firefox and navigate to the +Histogram Dashboard. Then we can see a histogram visualization for our normally +distributed data. + +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/1_moving_mean.png) + +`tf.summary.histogram` takes an arbitrarily sized and shaped Tensor, and +compresses it into a histogram data structure consisting of many bins with +widths and counts. For example, let's say we want to organize the numbers +`[0.5, 1.1, 1.3, 2.2, 2.9, 2.99]` into bins. We could make three bins: +* a bin +containing everything from 0 to 1 (it would contain one element, 0.5), +* a bin +containing everything from 1-2 (it would contain two elements, 1.1 and 1.3), +* a bin containing everything from 2-3 (it would contain three elements: 2.2, +2.9 and 2.99). + +TensorFlow uses a similar approach to create bins, but unlike in our example, it +doesn't create integer bins. For large, sparse datasets, that might result in +many thousands of bins. +Instead, [the bins are exponentially distributed, with many bins close to 0 and +comparatively few bins for very large numbers.](https://github.com/tensorflow/tensorflow/blob/c8b59c046895fa5b6d79f73e0b5817330fcfbfc1/tensorflow/core/lib/histogram/histogram.cc#L28) +However, visualizing exponentially-distributed bins is tricky; if height is used +to encode count, then wider bins take more space, even if they have the same +number of elements. Conversely, encoding count in the area makes height +comparisons impossible. Instead, the histograms [resample the data](https://github.com/tensorflow/tensorflow/blob/17c47804b86e340203d451125a721310033710f1/tensorflow/tensorboard/components/tf_backend/backend.ts#L400) +into uniform bins. This can lead to unfortunate artifacts in some cases. + +Each slice in the histogram visualizer displays a single histogram. +The slices are organized by step; +older slices (e.g. step 0) are further "back" and darker, while newer slices +(e.g. step 400) are close to the foreground, and lighter in color. +The y-axis on the right shows the step number. + +You can mouse over the histogram to see tooltips with some more detailed +information. For example, in the following image we can see that the histogram +at timestep 176 has a bin centered at 2.25 with 177 elements in that bin. + +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/2_moving_mean_tooltip.png) + +Also, you may note that the histogram slices are not always evenly spaced in +step count or time. This is because TensorBoard uses +[reservoir sampling](https://en.wikipedia.org/wiki/Reservoir_sampling) to keep a +subset of all the histograms, to save on memory. Reservoir sampling guarantees +that every sample has an equal likelihood of being included, but because it is +a randomized algorithm, the samples chosen don't occur at even steps. + +## Overlay Mode + +There is a control on the left of the dashboard that allows you to toggle the +histogram mode from "offset" to "overlay": + +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/3_overlay_offset.png) + +In "offset" mode, the visualization rotates 45 degrees, so that the individual +histogram slices are no longer spread out in time, but instead are all plotted +on the same y-axis. + +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/4_overlay.png) +Now, each slice is a separate line on the chart, and the y-axis shows the item +count within each bucket. Darker lines are older, earlier steps, and lighter +lines are more recent, later steps. Once again, you can mouse over the chart to +see some additional information. + +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/5_overlay_tooltips.png) + +In general, the overlay visualization is useful if you want to directly compare +the counts of different histograms. + +## Multimodal Distributions + +The Histogram Dashboard is great for visualizing multimodal +distributions. Let's construct a simple bimodal distribution by concatenating +the outputs from two different normal distributions. The code will look like +this: + +```python +import tensorflow as tf + +k = tf.placeholder(tf.float32) + +# Make a normal distribution, with a shifting mean +mean_moving_normal = tf.random_normal(shape=[1000], mean=(5*k), stddev=1) +# Record that distribution into a histogram summary +tf.summary.histogram("normal/moving_mean", mean_moving_normal) + +# Make a normal distribution with shrinking variance +variance_shrinking_normal = tf.random_normal(shape=[1000], mean=0, stddev=1-(k)) +# Record that distribution too +tf.summary.histogram("normal/shrinking_variance", variance_shrinking_normal) + +# Let's combine both of those distributions into one dataset +normal_combined = tf.concat([mean_moving_normal, variance_shrinking_normal], 0) +# We add another histogram summary to record the combined distribution +tf.summary.histogram("normal/bimodal", normal_combined) + +summaries = tf.summary.merge_all() + +# Setup a session and summary writer +sess = tf.Session() +writer = tf.summary.FileWriter("/tmp/histogram_example") + +# Setup a loop and write the summaries to disk +N = 400 +for step in range(N): + k_val = step/float(N) + summ = sess.run(summaries, feed_dict={k: k_val}) + writer.add_summary(summ, global_step=step) +``` + +You already remember our "moving mean" normal distribution from the example +above. Now we also have a "shrinking variance" distribution. Side-by-side, they +look like this: +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/6_two_distributions.png) + +When we concatenate them, we get a chart that clearly reveals the divergent, +bimodal structure: +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/7_bimodal.png) + +## Some more distributions + +Just for fun, let's generate and visualize a few more distributions, and then +combine them all into one chart. Here's the code we'll use: + +```python +import tensorflow as tf + +k = tf.placeholder(tf.float32) + +# Make a normal distribution, with a shifting mean +mean_moving_normal = tf.random_normal(shape=[1000], mean=(5*k), stddev=1) +# Record that distribution into a histogram summary +tf.summary.histogram("normal/moving_mean", mean_moving_normal) + +# Make a normal distribution with shrinking variance +variance_shrinking_normal = tf.random_normal(shape=[1000], mean=0, stddev=1-(k)) +# Record that distribution too +tf.summary.histogram("normal/shrinking_variance", variance_shrinking_normal) + +# Let's combine both of those distributions into one dataset +normal_combined = tf.concat([mean_moving_normal, variance_shrinking_normal], 0) +# We add another histogram summary to record the combined distribution +tf.summary.histogram("normal/bimodal", normal_combined) + +# Add a gamma distribution +gamma = tf.random_gamma(shape=[1000], alpha=k) +tf.summary.histogram("gamma", gamma) + +# And a poisson distribution +poisson = tf.random_poisson(shape=[1000], lam=k) +tf.summary.histogram("poisson", poisson) + +# And a uniform distribution +uniform = tf.random_uniform(shape=[1000], maxval=k*10) +tf.summary.histogram("uniform", uniform) + +# Finally, combine everything together! +all_distributions = [mean_moving_normal, variance_shrinking_normal, + gamma, poisson, uniform] +all_combined = tf.concat(all_distributions, 0) +tf.summary.histogram("all_combined", all_combined) + +summaries = tf.summary.merge_all() + +# Setup a session and summary writer +sess = tf.Session() +writer = tf.summary.FileWriter("/tmp/histogram_example") + +# Setup a loop and write the summaries to disk +N = 400 +for step in range(N): + k_val = step/float(N) + summ = sess.run(summaries, feed_dict={k: k_val}) + writer.add_summary(summ, global_step=step) +``` +### Gamma Distribution +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/8_gamma.png) + +### Uniform Distribution +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/9_uniform.png) + +### Poisson Distribution +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/10_poisson.png) +The poisson distribution is defined over the integers. So, all of the values +being generated are perfect integers. The histogram compression moves the data +into floating-point bins, causing the visualization to show little +bumps over the integer values rather than perfect spikes. + +### All Together Now +Finally, we can concatenate all of the data into one funny-looking curve. +![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/11_all_combined.png) + diff --git a/tensorflow/docs_src/guide/tensors.md b/tensorflow/docs_src/guide/tensors.md new file mode 100644 index 0000000000..7227260f1a --- /dev/null +++ b/tensorflow/docs_src/guide/tensors.md @@ -0,0 +1,330 @@ +# Tensors + +TensorFlow, as the name indicates, is a framework to define and run computations +involving tensors. A **tensor** is a generalization of vectors and matrices to +potentially higher dimensions. Internally, TensorFlow represents tensors as +n-dimensional arrays of base datatypes. + +When writing a TensorFlow program, the main object you manipulate and pass +around is the `tf.Tensor`. A `tf.Tensor` object represents a partially defined +computation that will eventually produce a value. TensorFlow programs work by +first building a graph of `tf.Tensor` objects, detailing how each tensor is +computed based on the other available tensors and then by running parts of this +graph to achieve the desired results. + +A `tf.Tensor` has the following properties: + + * a data type (`float32`, `int32`, or `string`, for example) + * a shape + + +Each element in the Tensor has the same data type, and the data type is always +known. The shape (that is, the number of dimensions it has and the size of each +dimension) might be only partially known. Most operations produce tensors of +fully-known shapes if the shapes of their inputs are also fully known, but in +some cases it's only possible to find the shape of a tensor at graph execution +time. + +Some types of tensors are special, and these will be covered in other +units of the TensorFlow guide. The main ones are: + + * `tf.Variable` + * `tf.constant` + * `tf.placeholder` + * `tf.SparseTensor` + +With the exception of `tf.Variable`, the value of a tensor is immutable, which +means that in the context of a single execution tensors only have a single +value. However, evaluating the same tensor twice can return different values; +for example that tensor can be the result of reading data from disk, or +generating a random number. + +## Rank + +The **rank** of a `tf.Tensor` object is its number of dimensions. Synonyms for +rank include **order** or **degree** or **n-dimension**. +Note that rank in TensorFlow is not the same as matrix rank in mathematics. +As the following table shows, each rank in TensorFlow corresponds to a +different mathematical entity: + +Rank | Math entity +--- | --- +0 | Scalar (magnitude only) +1 | Vector (magnitude and direction) +2 | Matrix (table of numbers) +3 | 3-Tensor (cube of numbers) +n | n-Tensor (you get the idea) + + +### Rank 0 + +The following snippet demonstrates creating a few rank 0 variables: + +```python +mammal = tf.Variable("Elephant", tf.string) +ignition = tf.Variable(451, tf.int16) +floating = tf.Variable(3.14159265359, tf.float64) +its_complicated = tf.Variable(12.3 - 4.85j, tf.complex64) +``` + +Note: A string is treated as a single item in TensorFlow, not as a sequence of +characters. It is possible to have scalar strings, vectors of strings, etc. + +### Rank 1 + +To create a rank 1 `tf.Tensor` object, you can pass a list of items as the +initial value. For example: + +```python +mystr = tf.Variable(["Hello"], tf.string) +cool_numbers = tf.Variable([3.14159, 2.71828], tf.float32) +first_primes = tf.Variable([2, 3, 5, 7, 11], tf.int32) +its_very_complicated = tf.Variable([12.3 - 4.85j, 7.5 - 6.23j], tf.complex64) +``` + + +### Higher ranks + +A rank 2 `tf.Tensor` object consists of at least one row and at least +one column: + +```python +mymat = tf.Variable([[7],[11]], tf.int16) +myxor = tf.Variable([[False, True],[True, False]], tf.bool) +linear_squares = tf.Variable([[4], [9], [16], [25]], tf.int32) +squarish_squares = tf.Variable([ [4, 9], [16, 25] ], tf.int32) +rank_of_squares = tf.rank(squarish_squares) +mymatC = tf.Variable([[7],[11]], tf.int32) +``` + +Higher-rank Tensors, similarly, consist of an n-dimensional array. For example, +during image processing, many tensors of rank 4 are used, with dimensions +corresponding to example-in-batch, image width, image height, and color channel. + +``` python +my_image = tf.zeros([10, 299, 299, 3]) # batch x height x width x color +``` + +### Getting a `tf.Tensor` object's rank + +To determine the rank of a `tf.Tensor` object, call the `tf.rank` method. +For example, the following method programmatically determines the rank +of the `tf.Tensor` defined in the previous section: + +```python +r = tf.rank(my_image) +# After the graph runs, r will hold the value 4. +``` + +### Referring to `tf.Tensor` slices + +Since a `tf.Tensor` is an n-dimensional array of cells, to access a single cell +in a `tf.Tensor` you need to specify n indices. + +For a rank 0 tensor (a scalar), no indices are necessary, since it is already a +single number. + +For a rank 1 tensor (a vector), passing a single index allows you to access a +number: + +```python +my_scalar = my_vector[2] +``` + +Note that the index passed inside the `[]` can itself be a scalar `tf.Tensor`, if +you want to dynamically choose an element from the vector. + +For tensors of rank 2 or higher, the situation is more interesting. For a +`tf.Tensor` of rank 2, passing two numbers returns a scalar, as expected: + + +```python +my_scalar = my_matrix[1, 2] +``` + + +Passing a single number, however, returns a subvector of a matrix, as follows: + + +```python +my_row_vector = my_matrix[2] +my_column_vector = my_matrix[:, 3] +``` + +The `:` notation is python slicing syntax for "leave this dimension alone". This +is useful in higher-rank Tensors, as it allows you to access its subvectors, +submatrices, and even other subtensors. + + +## Shape + +The **shape** of a tensor is the number of elements in each dimension. +TensorFlow automatically infers shapes during graph construction. These inferred +shapes might have known or unknown rank. If the rank is known, the sizes of each +dimension might be known or unknown. + +The TensorFlow documentation uses three notational conventions to describe +tensor dimensionality: rank, shape, and dimension number. The following table +shows how these relate to one another: + +Rank | Shape | Dimension number | Example +--- | --- | --- | --- +0 | [] | 0-D | A 0-D tensor. A scalar. +1 | [D0] | 1-D | A 1-D tensor with shape [5]. +2 | [D0, D1] | 2-D | A 2-D tensor with shape [3, 4]. +3 | [D0, D1, D2] | 3-D | A 3-D tensor with shape [1, 4, 3]. +n | [D0, D1, ... Dn-1] | n-D | A tensor with shape [D0, D1, ... Dn-1]. + +Shapes can be represented via Python lists / tuples of ints, or with the +@{tf.TensorShape}. + +### Getting a `tf.Tensor` object's shape + +There are two ways of accessing the shape of a `tf.Tensor`. While building the +graph, it is often useful to ask what is already known about a tensor's +shape. This can be done by reading the `shape` property of a `tf.Tensor` object. +This method returns a `TensorShape` object, which is a convenient way of +representing partially-specified shapes (since, when building the graph, not all +shapes will be fully known). + +It is also possible to get a `tf.Tensor` that will represent the fully-defined +shape of another `tf.Tensor` at runtime. This is done by calling the `tf.shape` +operation. This way, you can build a graph that manipulates the shapes of +tensors by building other tensors that depend on the dynamic shape of the input +`tf.Tensor`. + +For example, here is how to make a vector of zeros with the same size as the +number of columns in a given matrix: + +``` python +zeros = tf.zeros(my_matrix.shape[1]) +``` + +### Changing the shape of a `tf.Tensor` + +The **number of elements** of a tensor is the product of the sizes of all its +shapes. The number of elements of a scalar is always `1`. Since there are often +many different shapes that have the same number of elements, it's often +convenient to be able to change the shape of a `tf.Tensor`, keeping its elements +fixed. This can be done with `tf.reshape`. + +The following examples demonstrate how to reshape tensors: + +```python +rank_three_tensor = tf.ones([3, 4, 5]) +matrix = tf.reshape(rank_three_tensor, [6, 10]) # Reshape existing content into + # a 6x10 matrix +matrixB = tf.reshape(matrix, [3, -1]) # Reshape existing content into a 3x20 + # matrix. -1 tells reshape to calculate + # the size of this dimension. +matrixAlt = tf.reshape(matrixB, [4, 3, -1]) # Reshape existing content into a + #4x3x5 tensor + +# Note that the number of elements of the reshaped Tensors has to match the +# original number of elements. Therefore, the following example generates an +# error because no possible value for the last dimension will match the number +# of elements. +yet_another = tf.reshape(matrixAlt, [13, 2, -1]) # ERROR! +``` + +## Data types + +In addition to dimensionality, Tensors have a data type. Refer to the +`tf.DType` page for a complete list of the data types. + +It is not possible to have a `tf.Tensor` with more than one data type. It is +possible, however, to serialize arbitrary data structures as `string`s and store +those in `tf.Tensor`s. + +It is possible to cast `tf.Tensor`s from one datatype to another using +`tf.cast`: + +``` python +# Cast a constant integer tensor into floating point. +float_tensor = tf.cast(tf.constant([1, 2, 3]), dtype=tf.float32) +``` + +To inspect a `tf.Tensor`'s data type use the `Tensor.dtype` property. + +When creating a `tf.Tensor` from a python object you may optionally specify the +datatype. If you don't, TensorFlow chooses a datatype that can represent your +data. TensorFlow converts Python integers to `tf.int32` and python floating +point numbers to `tf.float32`. Otherwise TensorFlow uses the same rules numpy +uses when converting to arrays. + +## Evaluating Tensors + +Once the computation graph has been built, you can run the computation that +produces a particular `tf.Tensor` and fetch the value assigned to it. This is +often useful for debugging as well as being required for much of TensorFlow to +work. + +The simplest way to evaluate a Tensor is using the `Tensor.eval` method. For +example: + +```python +constant = tf.constant([1, 2, 3]) +tensor = constant * constant +print(tensor.eval()) +``` + +The `eval` method only works when a default `tf.Session` is active (see +Graphs and Sessions for more information). + +`Tensor.eval` returns a numpy array with the same contents as the tensor. + +Sometimes it is not possible to evaluate a `tf.Tensor` with no context because +its value might depend on dynamic information that is not available. For +example, tensors that depend on `placeholder`s can't be evaluated without +providing a value for the `placeholder`. + +``` python +p = tf.placeholder(tf.float32) +t = p + 1.0 +t.eval() # This will fail, since the placeholder did not get a value. +t.eval(feed_dict={p:2.0}) # This will succeed because we're feeding a value + # to the placeholder. +``` + +Note that it is possible to feed any `tf.Tensor`, not just placeholders. + +Other model constructs might make evaluating a `tf.Tensor` +complicated. TensorFlow can't directly evaluate `tf.Tensor`s defined inside +functions or inside control flow constructs. If a `tf.Tensor` depends on a value +from a queue, evaluating the `tf.Tensor` will only work once something has been +enqueued; otherwise, evaluating it will hang. When working with queues, remember +to call `tf.train.start_queue_runners` before evaluating any `tf.Tensor`s. + +## Printing Tensors + +For debugging purposes you might want to print the value of a `tf.Tensor`. While + @{$debugger$tfdbg} provides advanced debugging support, TensorFlow also has an + operation to directly print the value of a `tf.Tensor`. + +Note that you rarely want to use the following pattern when printing a +`tf.Tensor`: + +``` python +t = <> +print(t) # This will print the symbolic tensor when the graph is being built. + # This tensor does not have a value in this context. +``` + +This code prints the `tf.Tensor` object (which represents deferred computation) +and not its value. Instead, TensorFlow provides the `tf.Print` operation, which +returns its first tensor argument unchanged while printing the set of +`tf.Tensor`s it is passed as the second argument. + +To correctly use `tf.Print` its return value must be used. See the example below + +``` python +t = <> +tf.Print(t, [t]) # This does nothing +t = tf.Print(t, [t]) # Here we are using the value returned by tf.Print +result = t + 1 # Now when result is evaluated the value of `t` will be printed. +``` + +When you evaluate `result` you will evaluate everything `result` depends +upon. Since `result` depends upon `t`, and evaluating `t` has the side effect of +printing its input (the old value of `t`), `t` gets printed. + diff --git a/tensorflow/docs_src/guide/using_gpu.md b/tensorflow/docs_src/guide/using_gpu.md new file mode 100644 index 0000000000..c429ca4750 --- /dev/null +++ b/tensorflow/docs_src/guide/using_gpu.md @@ -0,0 +1,215 @@ +# Using GPUs + +## Supported devices + +On a typical system, there are multiple computing devices. In TensorFlow, the +supported device types are `CPU` and `GPU`. They are represented as `strings`. +For example: + +* `"/cpu:0"`: The CPU of your machine. +* `"/device:GPU:0"`: The GPU of your machine, if you have one. +* `"/device:GPU:1"`: The second GPU of your machine, etc. + +If a TensorFlow operation has both CPU and GPU implementations, the GPU devices +will be given priority when the operation is assigned to a device. For example, +`matmul` has both CPU and GPU kernels. On a system with devices `cpu:0` and +`gpu:0`, `gpu:0` will be selected to run `matmul`. + +## Logging Device placement + +To find out which devices your operations and tensors are assigned to, create +the session with `log_device_placement` configuration option set to `True`. + +```python +# Creates a graph. +a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') +b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') +c = tf.matmul(a, b) +# Creates a session with log_device_placement set to True. +sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) +# Runs the op. +print(sess.run(c)) +``` + +You should see the following output: + +``` +Device mapping: +/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: Tesla K40c, pci bus +id: 0000:05:00.0 +b: /job:localhost/replica:0/task:0/device:GPU:0 +a: /job:localhost/replica:0/task:0/device:GPU:0 +MatMul: /job:localhost/replica:0/task:0/device:GPU:0 +[[ 22. 28.] + [ 49. 64.]] + +``` + +## Manual device placement + +If you would like a particular operation to run on a device of your choice +instead of what's automatically selected for you, you can use `with tf.device` +to create a device context such that all the operations within that context will +have the same device assignment. + +```python +# Creates a graph. +with tf.device('/cpu:0'): + a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') + b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') +c = tf.matmul(a, b) +# Creates a session with log_device_placement set to True. +sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) +# Runs the op. +print(sess.run(c)) +``` + +You will see that now `a` and `b` are assigned to `cpu:0`. Since a device was +not explicitly specified for the `MatMul` operation, the TensorFlow runtime will +choose one based on the operation and available devices (`gpu:0` in this +example) and automatically copy tensors between devices if required. + +``` +Device mapping: +/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: Tesla K40c, pci bus +id: 0000:05:00.0 +b: /job:localhost/replica:0/task:0/cpu:0 +a: /job:localhost/replica:0/task:0/cpu:0 +MatMul: /job:localhost/replica:0/task:0/device:GPU:0 +[[ 22. 28.] + [ 49. 64.]] +``` + +## Allowing GPU memory growth + +By default, TensorFlow maps nearly all of the GPU memory of all GPUs (subject to +[`CUDA_VISIBLE_DEVICES`](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars)) +visible to the process. This is done to more efficiently use the relatively +precious GPU memory resources on the devices by reducing [memory +fragmentation](https://en.wikipedia.org/wiki/Fragmentation_\(computing\)). + +In some cases it is desirable for the process to only allocate a subset of the +available memory, or to only grow the memory usage as is needed by the process. +TensorFlow provides two Config options on the Session to control this. + +The first is the `allow_growth` option, which attempts to allocate only as much +GPU memory based on runtime allocations: it starts out allocating very little +memory, and as Sessions get run and more GPU memory is needed, we extend the GPU +memory region needed by the TensorFlow process. Note that we do not release +memory, since that can lead to even worse memory fragmentation. To turn this +option on, set the option in the ConfigProto by: + +```python +config = tf.ConfigProto() +config.gpu_options.allow_growth = True +session = tf.Session(config=config, ...) +``` + +The second method is the `per_process_gpu_memory_fraction` option, which +determines the fraction of the overall amount of memory that each visible GPU +should be allocated. For example, you can tell TensorFlow to only allocate 40% +of the total memory of each GPU by: + +```python +config = tf.ConfigProto() +config.gpu_options.per_process_gpu_memory_fraction = 0.4 +session = tf.Session(config=config, ...) +``` + +This is useful if you want to truly bound the amount of GPU memory available to +the TensorFlow process. + +## Using a single GPU on a multi-GPU system + +If you have more than one GPU in your system, the GPU with the lowest ID will be +selected by default. If you would like to run on a different GPU, you will need +to specify the preference explicitly: + +```python +# Creates a graph. +with tf.device('/device:GPU:2'): + a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') + b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') + c = tf.matmul(a, b) +# Creates a session with log_device_placement set to True. +sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) +# Runs the op. +print(sess.run(c)) +``` + +If the device you have specified does not exist, you will get +`InvalidArgumentError`: + +``` +InvalidArgumentError: Invalid argument: Cannot assign a device to node 'b': +Could not satisfy explicit device specification '/device:GPU:2' + [[Node: b = Const[dtype=DT_FLOAT, value=Tensor, _device="/device:GPU:2"]()]] +``` + +If you would like TensorFlow to automatically choose an existing and supported +device to run the operations in case the specified one doesn't exist, you can +set `allow_soft_placement` to `True` in the configuration option when creating +the session. + +```python +# Creates a graph. +with tf.device('/device:GPU:2'): + a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') + b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') + c = tf.matmul(a, b) +# Creates a session with allow_soft_placement and log_device_placement set +# to True. +sess = tf.Session(config=tf.ConfigProto( + allow_soft_placement=True, log_device_placement=True)) +# Runs the op. +print(sess.run(c)) +``` + +## Using multiple GPUs + +If you would like to run TensorFlow on multiple GPUs, you can construct your +model in a multi-tower fashion where each tower is assigned to a different GPU. +For example: + +``` python +# Creates a graph. +c = [] +for d in ['/device:GPU:2', '/device:GPU:3']: + with tf.device(d): + a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3]) + b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2]) + c.append(tf.matmul(a, b)) +with tf.device('/cpu:0'): + sum = tf.add_n(c) +# Creates a session with log_device_placement set to True. +sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) +# Runs the op. +print(sess.run(sum)) +``` + +You will see the following output. + +``` +Device mapping: +/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: Tesla K20m, pci bus +id: 0000:02:00.0 +/job:localhost/replica:0/task:0/device:GPU:1 -> device: 1, name: Tesla K20m, pci bus +id: 0000:03:00.0 +/job:localhost/replica:0/task:0/device:GPU:2 -> device: 2, name: Tesla K20m, pci bus +id: 0000:83:00.0 +/job:localhost/replica:0/task:0/device:GPU:3 -> device: 3, name: Tesla K20m, pci bus +id: 0000:84:00.0 +Const_3: /job:localhost/replica:0/task:0/device:GPU:3 +Const_2: /job:localhost/replica:0/task:0/device:GPU:3 +MatMul_1: /job:localhost/replica:0/task:0/device:GPU:3 +Const_1: /job:localhost/replica:0/task:0/device:GPU:2 +Const: /job:localhost/replica:0/task:0/device:GPU:2 +MatMul: /job:localhost/replica:0/task:0/device:GPU:2 +AddN: /job:localhost/replica:0/task:0/cpu:0 +[[ 44. 56.] + [ 98. 128.]] +``` + +The @{$deep_cnn$cifar10 tutorial} is a good example +demonstrating how to do training with multiple GPUs. diff --git a/tensorflow/docs_src/guide/using_tpu.md b/tensorflow/docs_src/guide/using_tpu.md new file mode 100644 index 0000000000..41d80d9d60 --- /dev/null +++ b/tensorflow/docs_src/guide/using_tpu.md @@ -0,0 +1,395 @@ +# Using TPUs + +This document walks through the principal TensorFlow APIs necessary to make +effective use of a [Cloud TPU](https://cloud.google.com/tpu/), and highlights +the differences between regular TensorFlow usage, and usage on a TPU. + +This doc is aimed at users who: + +* Are familiar with TensorFlow's `Estimator` and `Dataset` APIs +* Have maybe [tried out a Cloud TPU](https://cloud.google.com/tpu/docs/quickstart) + using an existing model. +* Have, perhaps, skimmed the code of an example TPU model + [[1]](https://github.com/tensorflow/models/blob/master/official/mnist/mnist_tpu.py) + [[2]](https://github.com/tensorflow/tpu/tree/master/models). +* Are interested in porting an existing `Estimator` model to + run on Cloud TPUs + +## TPUEstimator + +@{tf.estimator.Estimator$Estimators} are TensorFlow's model-level abstraction. +Standard `Estimators` can drive models on CPU and GPUs. You must use +@{tf.contrib.tpu.TPUEstimator} to drive a model on TPUs. + +Refer to TensorFlow's Getting Started section for an introduction to the basics +of using a @{$premade_estimators$pre-made `Estimator`}, and +@{$custom_estimators$custom `Estimator`s}. + +The `TPUEstimator` class differs somewhat from the `Estimator` class. + +The simplest way to maintain a model that can be run both on CPU/GPU or on a +Cloud TPU is to define the model's inference phase (from inputs to predictions) +outside of the `model_fn`. Then maintain separate implementations of the +`Estimator` setup and `model_fn`, both wrapping this inference step. For an +example of this pattern compare the `mnist.py` and `mnist_tpu.py` implementation in +[tensorflow/models](https://github.com/tensorflow/models/tree/master/official/mnist). + +### Running a `TPUEstimator` locally + +To create a standard `Estimator` you call the constructor, and pass it a +`model_fn`, for example: + +``` +my_estimator = tf.estimator.Estimator( + model_fn=my_model_fn) +``` + +The changes required to use a @{tf.contrib.tpu.TPUEstimator} on your local +machine are relatively minor. The constructor requires two additional arguments. +You should set the `use_tpu` argument to `False`, and pass a +@{tf.contrib.tpu.RunConfig} as the `config` argument, as shown below: + +``` python +my_tpu_estimator = tf.contrib.tpu.TPUEstimator( + model_fn=my_model_fn, + config=tf.contrib.tpu.RunConfig() + use_tpu=False) +``` + +Just this simple change will allow you to run a `TPUEstimator` locally. +The majority of example TPU models can be run in this local mode, +by setting the command line flags as follows: + + +``` +$> python mnist_tpu.py --use_tpu=false --master='' +``` + +Note: This `use_tpu=False` argument is useful for trying out the `TPUEstimator` +API. It is not meant to be a complete TPU compatibility test. Successfully +running a model locally in a `TPUEstimator` does not guarantee that it will +work on a TPU. + + +### Building a `tpu.RunConfig` + +While the default `RunConfig` is sufficient for local training, these settings +cannot be ignored in real usage. + +A more typical setup for a `RunConfig`, that can be switched to use a Cloud +TPU, might be as follows: + +``` python +import tempfile +import subprocess + +class FLAGS(object): + use_tpu=False + tpu_name=None + # Use a local temporary path for the `model_dir` + model_dir = tempfile.mkdtemp() + # Number of training steps to run on the Cloud TPU before returning control. + iterations = 50 + # A single Cloud TPU has 8 shards. + num_shards = 8 + +if FLAGS.use_tpu: + my_project_name = subprocess.check_output([ + 'gcloud','config','get-value','project']) + my_zone = subprocess.check_output([ + 'gcloud','config','get-value','compute/zone']) + cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( + tpu_names=[FLAGS.tpu_name], + zone=my_zone, + project=my_project) + master = tpu_cluster_resolver.get_master() +else: + master = '' + +my_tpu_run_config = tf.contrib.tpu.RunConfig( + master=master, + evaluation_master=master, + model_dir=FLAGS.model_dir, + session_config=tf.ConfigProto( + allow_soft_placement=True, log_device_placement=True), + tpu_config=tf.contrib.tpu.TPUConfig(FLAGS.iterations, + FLAGS.num_shards), +) +``` + +Then you must pass the @{tf.contrib.tpu.RunConfig} to the constructor: + +``` python +my_tpu_estimator = tf.contrib.tpu.TPUEstimator( + model_fn=my_model_fn, + config = my_tpu_run_config, + use_tpu=FLAGS.use_tpu) +``` + +Typically the `FLAGS` would be set by command line arguments. To switch from +training locally to training on a cloud TPU you would need to: + +* Set `FLAGS.use_tpu` to `True` +* Set `FLAGS.tpu_name` so the `tf.contrib.cluster_resolver.TPUClusterResolver` can find it +* Set `FLAGS.model_dir` to a Google Cloud Storage bucket url (`gs://`). + + +## Optimizer + +When training on a cloud TPU you **must** wrap the optimizer in a +@{tf.contrib.tpu.CrossShardOptimizer}, which uses an `allreduce` to aggregate +gradients and broadcast the result to each shard (each TPU core). + +The `CrossShardOptimizer` is not compatible with local training. So, to have +the same code run both locally and on a Cloud TPU, add lines like the following: + +``` python +optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate) +if FLAGS.use_tpu: + optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) +``` + +If you prefer to avoid a global `FLAGS` variable in your model code, one +approach is to set the optimizer as one of the `Estimator`'s params, +as follows: + +``` python +my_tpu_estimator = tf.contrib.tpu.TPUEstimator( + model_fn=my_model_fn, + config = my_tpu_run_config, + use_tpu=FLAGS.use_tpu, + params={'optimizer':optimizer}) +``` + +## Model Function + +This section details the changes you must make to the model function +(`model_fn()`) to make it `TPUEstimator` compatible. + +### Static shapes + +During regular usage TensorFlow attempts to determine the shapes of each +`tf.Tensor` during graph construction. During execution any unknown shape +dimensions are determined dynamically, +see @{$guide/tensors#shape$Tensor Shapes} for more details. + +To run on Cloud TPUs TensorFlow models are compiled using @{$xla$XLA}. +XLA uses a similar system for determining shapes at compile time. XLA requires +that all tensor dimensions be statically defined at compile time. All shapes +must evaluate to a constant, and not depend on external data, or stateful +operations like variables or a random number generator. + + +### Summaries + +Remove any use of `tf.summary` from your model. + +@{$summaries_and_tensorboard$TensorBoard summaries} are a great way see inside +your model. A minimal set of basic summaries are automatically recorded by the +`TPUEstimator`, to `event` files in the `model_dir`. Custom summaries, however, +are currently unsupported when training on a Cloud TPU. So while the +`TPUEstimator` will still run locally with summaries, it will fail if used on a +TPU. + +### Metrics + +Build your evaluation metrics dictionary in a stand-alone `metric_fn`. + + + +Evaluation metrics are an essential part of training a model. These are fully +supported on Cloud TPUs, but with a slightly different syntax. + +A standard @{tf.metrics} returns two tensors. The first returns the running +average of the metric value, while the second updates the running average and +returns the value for this batch: + +``` +running_average, current_batch = tf.metrics.accuracy(labels, predictions) +``` + +In a standard `Estimator` you create a dictionary of these pairs, and return it +as part of the `EstimatorSpec`. + +```python +my_metrics = {'accuracy': tf.metrics.accuracy(labels, predictions)} + +return tf.estimator.EstimatorSpec( + ... + eval_metric_ops=my_metrics +) +``` + +In a `TPUEstimator` you instead pass a function (which returns a metrics +dictionary) and a list of argument tensors, as shown below: + +```python +def my_metric_fn(labels, predictions): + return {'accuracy': tf.metrics.accuracy(labels, predictions)} + +return tf.contrib.tpu.TPUEstimatorSpec( + ... + eval_metrics=(my_metric_fn, [labels, predictions]) +) +``` + +### Use `TPUEstimatorSpec` + +`TPUEstimatorSpec` do not support hooks, and require function wrappers for +some fields. + +An `Estimator`'s `model_fn` must return an `EstimatorSpec`. An `EstimatorSpec` +is a simple structure of named fields containing all the `tf.Tensors` of the +model that the `Estimator` may need to interact with. + +`TPUEstimators` use a @{tf.contrib.tpu.TPUEstimatorSpec}. There are a few +differences between it and a standard @{tf.estimator.EstimatorSpec}: + + +* The `eval_metric_ops` must be wrapped into a `metrics_fn`, this field is + renamed `eval_metrics` ([see above](#metrics)). +* The @{tf.train.SessionRunHook$hooks} are unsupported, so these fields are + omitted. +* The @{tf.train.Scaffold$`scaffold`}, if used, must also be wrapped in a + function. This field is renamed to `scaffold_fn`. + +`Scaffold` and `Hooks` are for advanced usage, and can typically be omitted. + +## Input functions + +Input functions work mainly unchanged as they run on the host computer, not the +Cloud TPU itself. This section explains the two necessary adjustments. + +### Params argument + + + +The `input_fn` for a standard `Estimator` _can_ include a +`params` argument; the `input_fn` for a `TPUEstimator` *must* include a +`params` argument. This is necessary to allow the estimator to set the batch +size for each replica of the input stream. So the minimum signature for an +`input_fn` for a `TPUEstimator` is: + +``` +def my_input_fn(params): + pass +``` + +Where `params['batch-size']` will contain the batch size. + +### Static shapes and batch size + +The input pipeline generated by your `input_fn` is run on CPU. So it is mostly +free from the strict static shape requirements imposed by the XLA/TPU environment. +The one requirement is that the batches of data fed from your input pipeline to +the TPU have a static shape, as determined by the standard TensorFlow shape +inference algorithm. Intermediate tensors are free to have a dynamic shapes. +If shape inference has failed, but the shape is known it is possible to +impose the correct shape using `tf.set_shape()`. + +In the example below the shape +inference algorithm fails, but it is correctly using `set_shape`: + +``` +>>> x = tf.zeros(tf.constant([1,2,3])+1) +>>> x.shape + +TensorShape([Dimension(None), Dimension(None), Dimension(None)]) + +>>> x.set_shape([2,3,4]) +``` + +In many cases the batch size is the only unknown dimension. + +A typical input pipeline, using `tf.data`, will usually produce batches of a +fixed size. The last batch of a finite `Dataset`, however, is typically smaller, +containing just the remaining elements. Since a `Dataset` does not know its own +length or finiteness, the standard @{tf.data.Dataset.batch$`batch`} method +cannot determine if all batches will have a fixed size batch on its own: + +``` +>>> params = {'batch_size':32} +>>> ds = tf.data.Dataset.from_tensors([0, 1, 2]) +>>> ds = ds.repeat().batch(params['batch-size']) +>>> ds + + +``` + +The most straightforward fix is to +@{tf.data.Dataset.apply$apply} @{tf.contrib.data.batch_and_drop_remainder} +as follows: + +``` +>>> params = {'batch_size':32} +>>> ds = tf.data.Dataset.from_tensors([0, 1, 2]) +>>> ds = ds.repeat().apply( +... tf.contrib.data.batch_and_drop_remainder(params['batch-size'])) +>>> ds + + <_RestructuredDataset shapes: (32, 3), types: tf.int32> +``` + +The one downside to this approach is that, as the name implies, this batching +method throws out any fractional batch at the end of the dataset. This is fine +for an infinitely repeating dataset being used for training, but could be a +problem if you want to train for an exact number of epochs. + +To do an exact 1-epoch of _evaluation_ you can work around this by manually +padding the length of the batches, and setting the padding entries to have zero +weight when creating your `tf.metrics`. + +## Datasets + +Efficient use of the `tf.data.Dataset` API is critical when using a Cloud +TPU, as it is impossible to use the Cloud TPU's unless you can feed it data +quickly enough. See @{$datasets_performance} for details on dataset performance. + +For all but the simplest experimentation (using +@{tf.data.Dataset.from_tensor_slices} or other in-graph data) you will need to +store all data files read by the `TPUEstimator`'s `Dataset` in Google Cloud +Storage Buckets. + + + +For most use-cases, we recommend converting your data into `TFRecord` +format and using a @{tf.data.TFRecordDataset} to read it. This, however, is not +a hard requirement and you can use other dataset readers +(`FixedLengthRecordDataset` or `TextLineDataset`) if you prefer. + +Small datasets can be loaded entirely into memory using +@{tf.data.Dataset.cache}. + +Regardless of the data format used, it is strongly recommended that you +@{$performance_guide#use_large_files$use large files}, on the order of +100MB. This is especially important in this networked setting as the overhead +of opening a file is significantly higher. + +It is also important, regardless of the type of reader used, to enable buffering +using the `buffer_size` argument to the constructor. This argument is specified +in bytes. A minimum of a few MB (`buffer_size=8*1024*1024`) is recommended so +that data is available when needed. + +The TPU-demos repo includes +[a script](https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py) +for downloading the imagenet dataset and converting it to an appropriate format. +This together with the imagenet +[models](https://github.com/tensorflow/tpu/tree/master/models) +included in the repo demonstrate all of these best-practices. + + +## What Next + +For details on how to actually set up and run a Cloud TPU see: + + * [Google Cloud TPU Documentation](https://cloud.google.com/tpu/docs/) + +This document is by no means exhaustive. The best source of more detail on how +to make a Cloud TPU compatible model are the example models published in: + + * The [TPU Demos Repository.](https://github.com/tensorflow/tpu) + +For more information about tuning TensorFlow code for performance see: + + * The @{$performance$Performance Section.} + diff --git a/tensorflow/docs_src/guide/variables.md b/tensorflow/docs_src/guide/variables.md new file mode 100644 index 0000000000..cd8c4b5b9a --- /dev/null +++ b/tensorflow/docs_src/guide/variables.md @@ -0,0 +1,319 @@ +# Variables + +A TensorFlow **variable** is the best way to represent shared, persistent state +manipulated by your program. + +Variables are manipulated via the `tf.Variable` class. A `tf.Variable` +represents a tensor whose value can be changed by running ops on it. Unlike +`tf.Tensor` objects, a `tf.Variable` exists outside the context of a single +`session.run` call. + +Internally, a `tf.Variable` stores a persistent tensor. Specific ops allow you +to read and modify the values of this tensor. These modifications are visible +across multiple `tf.Session`s, so multiple workers can see the same values for a +`tf.Variable`. + +## Creating a Variable + +The best way to create a variable is to call the `tf.get_variable` +function. This function requires you to specify the Variable's name. This name +will be used by other replicas to access the same variable, as well as to name +this variable's value when checkpointing and exporting models. `tf.get_variable` +also allows you to reuse a previously created variable of the same name, making it +easy to define models which reuse layers. + +To create a variable with `tf.get_variable`, simply provide the name and shape + +``` python +my_variable = tf.get_variable("my_variable", [1, 2, 3]) +``` + +This creates a variable named "my_variable" which is a three-dimensional tensor +with shape `[1, 2, 3]`. This variable will, by default, have the `dtype` +`tf.float32` and its initial value will be randomized via +`tf.glorot_uniform_initializer`. + +You may optionally specify the `dtype` and initializer to `tf.get_variable`. For +example: + +``` python +my_int_variable = tf.get_variable("my_int_variable", [1, 2, 3], dtype=tf.int32, + initializer=tf.zeros_initializer) +``` + +TensorFlow provides many convenient initializers. Alternatively, you may +initialize a `tf.Variable` to have the value of a `tf.Tensor`. For example: + +``` python +other_variable = tf.get_variable("other_variable", dtype=tf.int32, + initializer=tf.constant([23, 42])) +``` + +Note that when the initializer is a `tf.Tensor` you should not specify the +variable's shape, as the shape of the initializer tensor will be used. + + + +### Variable collections + +Because disconnected parts of a TensorFlow program might want to create +variables, it is sometimes useful to have a single way to access all of +them. For this reason TensorFlow provides **collections**, which are named lists +of tensors or other objects, such as `tf.Variable` instances. + +By default every `tf.Variable` gets placed in the following two collections: + + * `tf.GraphKeys.GLOBAL_VARIABLES` --- variables that can be shared across + multiple devices, + * `tf.GraphKeys.TRAINABLE_VARIABLES` --- variables for which TensorFlow will + calculate gradients. + +If you don't want a variable to be trainable, add it to the +`tf.GraphKeys.LOCAL_VARIABLES` collection instead. For example, the following +snippet demonstrates how to add a variable named `my_local` to this collection: + +``` python +my_local = tf.get_variable("my_local", shape=(), +collections=[tf.GraphKeys.LOCAL_VARIABLES]) +``` + +Alternatively, you can specify `trainable=False` as an argument to +`tf.get_variable`: + +``` python +my_non_trainable = tf.get_variable("my_non_trainable", + shape=(), + trainable=False) +``` + + +You can also use your own collections. Any string is a valid collection name, +and there is no need to explicitly create a collection. To add a variable (or +any other object) to a collection after creating the variable, call +`tf.add_to_collection`. For example, the following code adds an existing +variable named `my_local` to a collection named `my_collection_name`: + +``` python +tf.add_to_collection("my_collection_name", my_local) +``` + +And to retrieve a list of all the variables (or other objects) you've placed in +a collection you can use: + +``` python +tf.get_collection("my_collection_name") +``` + +### Device placement + +Just like any other TensorFlow operation, you can place variables on particular +devices. For example, the following snippet creates a variable named `v` and +places it on the second GPU device: + +``` python +with tf.device("/device:GPU:1"): + v = tf.get_variable("v", [1]) +``` + +It is particularly important for variables to be in the correct device in +distributed settings. Accidentally putting variables on workers instead of +parameter servers, for example, can severely slow down training or, in the worst +case, let each worker blithely forge ahead with its own independent copy of each +variable. For this reason we provide @{tf.train.replica_device_setter}, which +can automatically place variables in parameter servers. For example: + +``` python +cluster_spec = { + "ps": ["ps0:2222", "ps1:2222"], + "worker": ["worker0:2222", "worker1:2222", "worker2:2222"]} +with tf.device(tf.train.replica_device_setter(cluster=cluster_spec)): + v = tf.get_variable("v", shape=[20, 20]) # this variable is placed + # in the parameter server + # by the replica_device_setter +``` + +## Initializing variables + +Before you can use a variable, it must be initialized. If you are programming in +the low-level TensorFlow API (that is, you are explicitly creating your own +graphs and sessions), you must explicitly initialize the variables. Most +high-level frameworks such as `tf.contrib.slim`, `tf.estimator.Estimator` and +`Keras` automatically initialize variables for you before training a model. + +Explicit initialization is otherwise useful because it allows you not to rerun +potentially expensive initializers when reloading a model from a checkpoint as +well as allowing determinism when randomly-initialized variables are shared in a +distributed setting. + +To initialize all trainable variables in one go, before training starts, call +`tf.global_variables_initializer()`. This function returns a single operation +responsible for initializing all variables in the +`tf.GraphKeys.GLOBAL_VARIABLES` collection. Running this operation initializes +all variables. For example: + +``` python +session.run(tf.global_variables_initializer()) +# Now all variables are initialized. +``` + +If you do need to initialize variables yourself, you can run the variable's +initializer operation. For example: + +``` python +session.run(my_variable.initializer) +``` + + +You can also ask which variables have still not been initialized. For example, +the following code prints the names of all variables which have not yet been +initialized: + +``` python +print(session.run(tf.report_uninitialized_variables())) +``` + + +Note that by default `tf.global_variables_initializer` does not specify the +order in which variables are initialized. Therefore, if the initial value of a +variable depends on another variable's value, it's likely that you'll get an +error. Any time you use the value of a variable in a context in which not all +variables are initialized (say, if you use a variable's value while initializing +another variable), it is best to use `variable.initialized_value()` instead of +`variable`: + +``` python +v = tf.get_variable("v", shape=(), initializer=tf.zeros_initializer()) +w = tf.get_variable("w", initializer=v.initialized_value() + 1) +``` + +## Using variables + +To use the value of a `tf.Variable` in a TensorFlow graph, simply treat it like +a normal `tf.Tensor`: + +``` python +v = tf.get_variable("v", shape=(), initializer=tf.zeros_initializer()) +w = v + 1 # w is a tf.Tensor which is computed based on the value of v. + # Any time a variable is used in an expression it gets automatically + # converted to a tf.Tensor representing its value. +``` + +To assign a value to a variable, use the methods `assign`, `assign_add`, and +friends in the `tf.Variable` class. For example, here is how you can call these +methods: + +``` python +v = tf.get_variable("v", shape=(), initializer=tf.zeros_initializer()) +assignment = v.assign_add(1) +tf.global_variables_initializer().run() +sess.run(assignment) # or assignment.op.run(), or assignment.eval() +``` + +Most TensorFlow optimizers have specialized ops that efficiently update the +values of variables according to some gradient descent-like algorithm. See +@{tf.train.Optimizer} for an explanation of how to use optimizers. + +Because variables are mutable it's sometimes useful to know what version of a +variable's value is being used at any point in time. To force a re-read of the +value of a variable after something has happened, you can use +`tf.Variable.read_value`. For example: + +``` python +v = tf.get_variable("v", shape=(), initializer=tf.zeros_initializer()) +assignment = v.assign_add(1) +with tf.control_dependencies([assignment]): + w = v.read_value() # w is guaranteed to reflect v's value after the + # assign_add operation. +``` + + +## Sharing variables + +TensorFlow supports two ways of sharing variables: + + * Explicitly passing `tf.Variable` objects around. + * Implicitly wrapping `tf.Variable` objects within `tf.variable_scope` objects. + +While code which explicitly passes variables around is very clear, it is +sometimes convenient to write TensorFlow functions that implicitly use +variables in their implementations. Most of the functional layers from +`tf.layers` use this approach, as well as all `tf.metrics`, and a few other +library utilities. + +Variable scopes allow you to control variable reuse when calling functions which +implicitly create and use variables. They also allow you to name your variables +in a hierarchical and understandable way. + +For example, let's say we write a function to create a convolutional / relu +layer: + +```python +def conv_relu(input, kernel_shape, bias_shape): + # Create variable named "weights". + weights = tf.get_variable("weights", kernel_shape, + initializer=tf.random_normal_initializer()) + # Create variable named "biases". + biases = tf.get_variable("biases", bias_shape, + initializer=tf.constant_initializer(0.0)) + conv = tf.nn.conv2d(input, weights, + strides=[1, 1, 1, 1], padding='SAME') + return tf.nn.relu(conv + biases) +``` + +This function uses short names `weights` and `biases`, which is good for +clarity. In a real model, however, we want many such convolutional layers, and +calling this function repeatedly would not work: + +``` python +input1 = tf.random_normal([1,10,10,32]) +input2 = tf.random_normal([1,20,20,32]) +x = conv_relu(input1, kernel_shape=[5, 5, 32, 32], bias_shape=[32]) +x = conv_relu(x, kernel_shape=[5, 5, 32, 32], bias_shape = [32]) # This fails. +``` + +Since the desired behavior is unclear (create new variables or reuse the +existing ones?) TensorFlow will fail. Calling `conv_relu` in different scopes, +however, clarifies that we want to create new variables: + +```python +def my_image_filter(input_images): + with tf.variable_scope("conv1"): + # Variables created here will be named "conv1/weights", "conv1/biases". + relu1 = conv_relu(input_images, [5, 5, 32, 32], [32]) + with tf.variable_scope("conv2"): + # Variables created here will be named "conv2/weights", "conv2/biases". + return conv_relu(relu1, [5, 5, 32, 32], [32]) +``` + +If you do want the variables to be shared, you have two options. First, you can +create a scope with the same name using `reuse=True`: + +``` python +with tf.variable_scope("model"): + output1 = my_image_filter(input1) +with tf.variable_scope("model", reuse=True): + output2 = my_image_filter(input2) + +``` + +You can also call `scope.reuse_variables()` to trigger a reuse: + +``` python +with tf.variable_scope("model") as scope: + output1 = my_image_filter(input1) + scope.reuse_variables() + output2 = my_image_filter(input2) + +``` + +Since depending on exact string names of scopes can feel dangerous, it's also +possible to initialize a variable scope based on another one: + +``` python +with tf.variable_scope("model") as scope: + output1 = my_image_filter(input1) +with tf.variable_scope(scope, reuse=True): + output2 = my_image_filter(input2) + +``` + diff --git a/tensorflow/docs_src/guide/version_compat.md b/tensorflow/docs_src/guide/version_compat.md new file mode 100644 index 0000000000..72e427c5f8 --- /dev/null +++ b/tensorflow/docs_src/guide/version_compat.md @@ -0,0 +1,319 @@ +# TensorFlow Version Compatibility + +This document is for users who need backwards compatibility across different +versions of TensorFlow (either for code or data), and for developers who want +to modify TensorFlow while preserving compatibility. + +## Semantic Versioning 2.0 + +TensorFlow follows Semantic Versioning 2.0 ([semver](http://semver.org)) for its +public API. Each release version of TensorFlow has the form `MAJOR.MINOR.PATCH`. +For example, TensorFlow version 1.2.3 has `MAJOR` version 1, `MINOR` version 2, +and `PATCH` version 3. Changes to each number have the following meaning: + +* **MAJOR**: Potentially backwards incompatible changes. Code and data that + worked with a previous major release will not necessarily work with the new + release. However, in some cases existing TensorFlow graphs and checkpoints + may be migratable to the newer release; see + [Compatibility of graphs and checkpoints](#compatibility_of_graphs_and_checkpoints) + for details on data compatibility. + +* **MINOR**: Backwards compatible features, speed improvements, etc. Code and + data that worked with a previous minor release *and* which depends only on the + public API will continue to work unchanged. For details on what is and is + not the public API, see [What is covered](#what_is_covered). + +* **PATCH**: Backwards compatible bug fixes. + +For example, release 1.0.0 introduced backwards *incompatible* changes from +release 0.12.1. However, release 1.1.1 was backwards *compatible* with release +1.0.0. + +## What is covered + +Only the public APIs of TensorFlow are backwards compatible across minor and +patch versions. The public APIs consist of + +* All the documented [Python](../api_docs/python) functions and classes in the + `tensorflow` module and its submodules, except for + * functions and classes in `tf.contrib` + * functions and classes whose names start with `_` (as these are private) + Note that the code in the `examples/` and `tools/` directories is not + reachable through the `tensorflow` Python module and is thus not covered by + the compatibility guarantee. + + If a symbol is available through the `tensorflow` Python module or its + submodules, but is not documented, then it is **not** considered part of the + public API. + +* The [C API](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/c/c_api.h). + +* The following protocol buffer files: + * [`attr_value`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto) + * [`config`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto) + * [`event`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/util/event.proto) + * [`graph`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/graph.proto) + * [`op_def`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def.proto) + * [`reader_base`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/reader_base.proto) + * [`summary`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto) + * [`tensor`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto) + * [`tensor_shape`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.proto) + * [`types`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.proto) + + +## What is *not* covered + +Some API functions are explicitly marked as "experimental" and can change in +backward incompatible ways between minor releases. These include: + +* **Experimental APIs**: The @{tf.contrib} module and its submodules in Python + and any functions in the C API or fields in protocol buffers that are + explicitly commented as being experimental. In particular, any field in a + protocol buffer which is called "experimental" and all its fields and + submessages can change at any time. + +* **Other languages**: TensorFlow APIs in languages other than Python and C, + such as: + + - @{$cc/guide$C++} (exposed through header files in + [`tensorflow/cc`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/cc)). + - [Java](../api_docs/java/reference/org/tensorflow/package-summary), + - [Go](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go) + +* **Details of composite ops:** Many public functions in Python expand to + several primitive ops in the graph, and these details will be part of any + graphs saved to disk as `GraphDef`s. These details may change for + minor releases. In particular, regressions tests that check for exact + matching between graphs are likely to break across minor releases, even + though the behavior of the graph should be unchanged and existing + checkpoints will still work. + +* **Floating point numerical details:** The specific floating point values + computed by ops may change at any time. Users should rely only on + approximate accuracy and numerical stability, not on the specific bits + computed. Changes to numerical formulas in minor and patch releases should + result in comparable or improved accuracy, with the caveat that in machine + learning improved accuracy of specific formulas may result in decreased + accuracy for the overall system. + +* **Random numbers:** The specific random numbers computed by the + @{$python/constant_op#Random_Tensors$random ops} may change at any time. + Users should rely only on approximately correct distributions and + statistical strength, not the specific bits computed. However, we will make + changes to random bits rarely (or perhaps never) for patch releases. We + will, of course, document all such changes. + +* **Version skew in distributed Tensorflow:** Running two different versions + of TensorFlow in a single cluster is unsupported. There are no guarantees + about backwards compatibility of the wire protocol. + +* **Bugs:** We reserve the right to make backwards incompatible behavior + (though not API) changes if the current implementation is clearly broken, + that is, if it contradicts the documentation or if a well-known and + well-defined intended behavior is not properly implemented due to a bug. + For example, if an optimizer claims to implement a well-known optimization + algorithm but does not match that algorithm due to a bug, then we will fix + the optimizer. Our fix may break code relying on the wrong behavior for + convergence. We will note such changes in the release notes. + +* **Error messages:** We reserve the right to change the text of error + messages. In addition, the type of an error may change unless the type is + specified in the documentation. For example, a function documented to + raise an `InvalidArgument` exception will continue to + raise `InvalidArgument`, but the human-readable message contents can change. + +## Compatibility of graphs and checkpoints + +You'll sometimes need to preserve graphs and checkpoints. +Graphs describe the data flow of ops to be run during training and +inference, and checkpoints contain the saved tensor values of variables in a +graph. + +Many TensorFlow users save graphs and trained models to disk for +later evaluation or additional training, but end up running their saved graphs +or models on a later release. In compliance with semver, any graph or checkpoint +written out with one version of TensorFlow can be loaded and evaluated with a +later version of TensorFlow with the same major release. However, we will +endeavor to preserve backwards compatibility even across major releases when +possible, so that the serialized files are usable over long periods of time. + + +Graphs are serialized via the `GraphDef` protocol buffer. To facilitate (rare) +backwards incompatible changes to graphs, each `GraphDef` has a version number +separate from the TensorFlow version. For example, `GraphDef` version 17 +deprecated the `inv` op in favor of `reciprocal`. The semantics are: + +* Each version of TensorFlow supports an interval of `GraphDef` versions. This + interval will be constant across patch releases, and will only grow across + minor releases. Dropping support for a `GraphDef` version will only occur + for a major release of TensorFlow. + +* Newly created graphs are assigned the latest `GraphDef` version number. + +* If a given version of TensorFlow supports the `GraphDef` version of a graph, + it will load and evaluate with the same behavior as the TensorFlow version + used to generate it (except for floating point numerical details and random + numbers), regardless of the major version of TensorFlow. In particular, all + checkpoint files will be compatible. + +* If the `GraphDef` *upper* bound is increased to X in a (minor) release, there + will be at least six months before the *lower* bound is increased to X. For + example (we're using hypothetical version numbers here): + * TensorFlow 1.2 might support `GraphDef` versions 4 to 7. + * TensorFlow 1.3 could add `GraphDef` version 8 and support versions 4 to 8. + * At least six months later, TensorFlow 2.0.0 could drop support for + versions 4 to 7, leaving version 8 only. + +Finally, when support for a `GraphDef` version is dropped, we will attempt to +provide tools for automatically converting graphs to a newer supported +`GraphDef` version. + +## Graph and checkpoint compatibility when extending TensorFlow + +This section is relevant only when making incompatible changes to the `GraphDef` +format, such as when adding ops, removing ops, or changing the functionality +of existing ops. The previous section should suffice for most users. + +### Backward and partial forward compatibility + +Our versioning scheme has three requirements: + +* **Backward compatibility** to support loading graphs and checkpoints + created with older versions of TensorFlow. +* **Forward compatibility** to support scenarios where the producer of a + graph or checkpoint is upgraded to a newer version of TensorFlow before + the consumer. +* Enable evolving TensorFlow in incompatible ways. For example, removing ops, + adding attributes, and removing attributes. + +Note that while the `GraphDef` version mechanism is separate from the TensorFlow +version, backwards incompatible changes to the `GraphDef` format are still +restricted by Semantic Versioning. This means functionality can only be removed +or changed between `MAJOR` versions of TensorFlow (such as `1.7` to `2.0`). +Additionally, forward compatibility is enforced within Patch releases (`1.x.1` +to `1.x.2` for example). + +To achieve backward and forward compatibility and to know when to enforce changes +in formats, graphs and checkpoints have metadata that describes when they +were produced. The sections below detail the TensorFlow implementation and +guidelines for evolving `GraphDef` versions. + +### Independent data version schemes + +There are different data versions for graphs and checkpoints. The two data +formats evolve at different rates from each other and also at different rates +from TensorFlow. Both versioning systems are defined in +[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h). +Whenever a new version is added, a note is added to the header detailing what +changed and the date. + +### Data, producers, and consumers + +We distinguish between the following kinds of data version information: +* **producers**: binaries that produce data. Producers have a version + (`producer`) and a minimum consumer version that they are compatible with + (`min_consumer`). +* **consumers**: binaries that consume data. Consumers have a version + (`consumer`) and a minimum producer version that they are compatible with + (`min_producer`). + +Each piece of versioned data has a [`VersionDef +versions`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/versions.proto) +field which records the `producer` that made the data, the `min_consumer` +that it is compatible with, and a list of `bad_consumers` versions that are +disallowed. + +By default, when a producer makes some data, the data inherits the producer's +`producer` and `min_consumer` versions. `bad_consumers` can be set if specific +consumer versions are known to contain bugs and must be avoided. A consumer can +accept a piece of data if the following are all true: + +* `consumer` >= data's `min_consumer` +* data's `producer` >= consumer's `min_producer` +* `consumer` not in data's `bad_consumers` + +Since both producers and consumers come from the same TensorFlow code base, +[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h) +contains a main data version which is treated as either `producer` or +`consumer` depending on context and both `min_consumer` and `min_producer` +(needed by producers and consumers, respectively). Specifically, + +* For `GraphDef` versions, we have `TF_GRAPH_DEF_VERSION`, + `TF_GRAPH_DEF_VERSION_MIN_CONSUMER`, and + `TF_GRAPH_DEF_VERSION_MIN_PRODUCER`. +* For checkpoint versions, we have `TF_CHECKPOINT_VERSION`, + `TF_CHECKPOINT_VERSION_MIN_CONSUMER`, and + `TF_CHECKPOINT_VERSION_MIN_PRODUCER`. + +### Add a new attribute with default to an existing op + +Following the guidance below gives you forward compatibility only if the set of +ops has not changed: + +1. If forward compatibility is desired, set `strip_default_attrs` to `True` + while exporting the model using either the + @{tf.saved_model.builder.SavedModelBuilder.add_meta_graph_and_variables$`add_meta_graph_and_variables`} + and @{tf.saved_model.builder.SavedModelBuilder.add_meta_graph$`add_meta_graph`} + methods of the `SavedModelBuilder` class, or + @{tf.estimator.Estimator.export_savedmodel$`Estimator.export_savedmodel`} +2. This strips off the default valued attributes at the time of + producing/exporting the models. This makes sure that the exported + @{tf.MetaGraphDef} does not contain the new op-attribute when the default + value is used. +3. Having this control could allow out-of-date consumers (for example, serving + binaries that lag behind training binaries) to continue loading the models + and prevent interruptions in model serving. + +### Evolving GraphDef versions + +This section explains how to use this versioning mechanism to make different +types of changes to the `GraphDef` format. + +#### Add an op + +Add the new op to both consumers and producers at the same time, and do not +change any `GraphDef` versions. This type of change is automatically +backward compatible, and does not impact forward compatibility plan since +existing producer scripts will not suddenly use the new functionality. + +#### Add an op and switch existing Python wrappers to use it + +1. Implement new consumer functionality and increment the `GraphDef` version. +2. If it is possible to make the wrappers use the new functionality only in + cases that did not work before, the wrappers can be updated now. +3. Change Python wrappers to use the new functionality. Do not increment + `min_consumer`, since models that do not use this op should not break. + +#### Remove or restrict an op's functionality + +1. Fix all producer scripts (not TensorFlow itself) to not use the banned op or + functionality. +2. Increment the `GraphDef` version and implement new consumer functionality + that bans the removed op or functionality for GraphDefs at the new version + and above. If possible, make TensorFlow stop producing `GraphDefs` with the + banned functionality. To do so, add the + [`REGISTER_OP(...).Deprecated(deprecated_at_version, + message)`](https://github.com/tensorflow/tensorflow/blob/b289bc7a50fc0254970c60aaeba01c33de61a728/tensorflow/core/ops/array_ops.cc#L1009). +3. Wait for a major release for backward compatibility purposes. +4. Increase `min_producer` to the GraphDef version from (2) and remove the + functionality entirely. + +#### Change an op's functionality + +1. Add a new similar op named `SomethingV2` or similar and go through the + process of adding it and switching existing Python wrappers to use it, which + may take three weeks if forward compatibility is desired. +2. Remove the old op (Can only take place with a major version change due to + backward compatibility). +3. Increase `min_consumer` to rule out consumers with the old op, add back the + old op as an alias for `SomethingV2`, and go through the process to switch + existing Python wrappers to use it. +4. Go through the process to remove `SomethingV2`. + +#### Ban a single unsafe consumer version + +1. Bump the `GraphDef` version and add the bad version to `bad_consumers` for + all new GraphDefs. If possible, add to `bad_consumers` only for GraphDefs + which contain a certain op or similar. +2. If existing consumers have the bad version, push them out as soon as + possible. diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index 1c03dd223e..5451e1b319 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -6,7 +6,7 @@ a Go application. This guide explains how to install and set up the [TensorFlow Go package](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go). Warning: The TensorFlow Go API is *not* covered by the TensorFlow -[API stability guarantees](https://www.tensorflow.org/programmers_guide/version_semantics). +[API stability guarantees](../guide/version_semantics.md). ## Supported Platforms diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index c73e2f4281..ad3544b595 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -7,7 +7,7 @@ Java application. This guide explains how to install and use it in a Java application. Warning: The TensorFlow Java API is *not* covered by the TensorFlow -[API stability guarantees](https://www.tensorflow.org/programmers_guide/version_semantics). +[API stability guarantees](../guide/version_semantics.md). ## Supported Platforms diff --git a/tensorflow/docs_src/programmers_guide/checkpoints.md b/tensorflow/docs_src/programmers_guide/checkpoints.md deleted file mode 100644 index 8dfd91e3c8..0000000000 --- a/tensorflow/docs_src/programmers_guide/checkpoints.md +++ /dev/null @@ -1,240 +0,0 @@ -# Checkpoints - -This document examines how to save and restore TensorFlow models built with -Estimators. TensorFlow provides two model formats: - -* checkpoints, which is a format dependent on the code that created - the model. -* SavedModel, which is a format independent of the code that created - the model. - -This document focuses on checkpoints. For details on SavedModel, see the -@{$saved_model$Saving and Restoring} chapter of the -*TensorFlow Programmer's Guide*. - - -## Sample code - -This document relies on the same -[Iris classification example](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py) detailed in @{$premade_estimators$Getting Started with TensorFlow}. -To download and access the example, invoke the following two commands: - -```shell -git clone https://github.com/tensorflow/models/ -cd models/samples/core/get_started -``` - -Most of the code snippets in this document are minor variations -on `premade_estimator.py`. - - -## Saving partially-trained models - -Estimators automatically write the following to disk: - -* **checkpoints**, which are versions of the model created during training. -* **event files**, which contain information that - [TensorBoard](https://developers.google.com/machine-learning/glossary/#TensorBoard) - uses to create visualizations. - -To specify the top-level directory in which the Estimator stores its -information, assign a value to the optional `model_dir` argument of *any* -`Estimator`'s constructor. -Taking `DNNClassifier` as an example, -the following code sets the `model_dir` -argument to the `models/iris` directory: - -```python -classifier = tf.estimator.DNNClassifier( - feature_columns=my_feature_columns, - hidden_units=[10, 10], - n_classes=3, - model_dir='models/iris') -``` - -Suppose you call the Estimator's `train` method. For example: - - -```python -classifier.train( - input_fn=lambda:train_input_fn(train_x, train_y, batch_size=100), - steps=200) -``` - -As suggested by the following diagrams, the first call to `train` -adds checkpoints and other files to the `model_dir` directory: - -
- -
-
-The first call to train(). -
- - -To see the objects in the created `model_dir` directory on a -UNIX-based system, just call `ls` as follows: - -```none -$ ls -1 models/iris -checkpoint -events.out.tfevents.timestamp.hostname -graph.pbtxt -model.ckpt-1.data-00000-of-00001 -model.ckpt-1.index -model.ckpt-1.meta -model.ckpt-200.data-00000-of-00001 -model.ckpt-200.index -model.ckpt-200.meta -``` - -The preceding `ls` command shows that the Estimator created checkpoints -at steps 1 (the start of training) and 200 (the end of training). - - -### Default checkpoint directory - -If you don't specify `model_dir` in an Estimator's constructor, the Estimator -writes checkpoint files to a temporary directory chosen by Python's -[tempfile.mkdtemp](https://docs.python.org/3/library/tempfile.html#tempfile.mkdtemp) -function. For example, the following Estimator constructor does *not* specify -the `model_dir` argument: - -```python -classifier = tf.estimator.DNNClassifier( - feature_columns=my_feature_columns, - hidden_units=[10, 10], - n_classes=3) - -print(classifier.model_dir) -``` - -The `tempfile.mkdtemp` function picks a secure, temporary directory -appropriate for your operating system. For example, a typical temporary -directory on macOS might be something like the following: - -```None -/var/folders/0s/5q9kfzfj3gx2knj0vj8p68yc00dhcr/T/tmpYm1Rwa -``` - -### Checkpointing Frequency - -By default, the Estimator saves -[checkpoints](https://developers.google.com/machine-learning/glossary/#checkpoint) -in the `model_dir` according to the following schedule: - -* Writes a checkpoint every 10 minutes (600 seconds). -* Writes a checkpoint when the `train` method starts (first iteration) - and completes (final iteration). -* Retains only the 5 most recent checkpoints in the directory. - -You may alter the default schedule by taking the following steps: - -1. Create a @{tf.estimator.RunConfig$`RunConfig`} object that defines the - desired schedule. -2. When instantiating the Estimator, pass that `RunConfig` object to the - Estimator's `config` argument. - -For example, the following code changes the checkpointing schedule to every -20 minutes and retains the 10 most recent checkpoints: - -```python -my_checkpointing_config = tf.estimator.RunConfig( - save_checkpoints_secs = 20*60, # Save checkpoints every 20 minutes. - keep_checkpoint_max = 10, # Retain the 10 most recent checkpoints. -) - -classifier = tf.estimator.DNNClassifier( - feature_columns=my_feature_columns, - hidden_units=[10, 10], - n_classes=3, - model_dir='models/iris', - config=my_checkpointing_config) -``` - -## Restoring your model - -The first time you call an Estimator's `train` method, TensorFlow saves a -checkpoint to the `model_dir`. Each subsequent call to the Estimator's -`train`, `evaluate`, or `predict` method causes the following: - -1. The Estimator builds the model's - [graph](https://developers.google.com/machine-learning/glossary/#graph) - by running the `model_fn()`. (For details on the `model_fn()`, see - @{$custom_estimators$Creating Custom Estimators.}) -2. The Estimator initializes the weights of the new model from the data - stored in the most recent checkpoint. - -In other words, as the following illustration suggests, once checkpoints -exist, TensorFlow rebuilds the model each time you call `train()`, -`evaluate()`, or `predict()`. - -
- -
-
-Subsequent calls to train(), evaluate(), or predict() -
- - -### Avoiding a bad restoration - -Restoring a model's state from a checkpoint only works if the model -and checkpoint are compatible. For example, suppose you trained a -`DNNClassifier` Estimator containing two hidden layers, -each having 10 nodes: - -```python -classifier = tf.estimator.DNNClassifier( - feature_columns=feature_columns, - hidden_units=[10, 10], - n_classes=3, - model_dir='models/iris') - -classifier.train( - input_fn=lambda:train_input_fn(train_x, train_y, batch_size=100), - steps=200) -``` - -After training (and, therefore, after creating checkpoints in `models/iris`), -imagine that you changed the number of neurons in each hidden layer from 10 to -20 and then attempted to retrain the model: - -``` python -classifier2 = tf.estimator.DNNClassifier( - feature_columns=my_feature_columns, - hidden_units=[20, 20], # Change the number of neurons in the model. - n_classes=3, - model_dir='models/iris') - -classifier.train( - input_fn=lambda:train_input_fn(train_x, train_y, batch_size=100), - steps=200) -``` - -Since the state in the checkpoint is incompatible with the model described -in `classifier2`, retraining fails with the following error: - -```None -... -InvalidArgumentError (see above for traceback): tensor_name = -dnn/hiddenlayer_1/bias/t_0/Adagrad; shape in shape_and_slice spec [10] -does not match the shape stored in checkpoint: [20] -``` - -To run experiments in which you train and compare slightly different -versions of a model, save a copy of the code that created each -`model_dir`, possibly by creating a separate git branch for each version. -This separation will keep your checkpoints recoverable. - -## Summary - -Checkpoints provide an easy automatic mechanism for saving and restoring -models created by Estimators. - -See the @{$saved_model$Saving and Restoring} -chapter of the *TensorFlow Programmer's Guide* for details on: - -* Saving and restoring models using low-level TensorFlow APIs. -* Exporting and importing models in the SavedModel format, which is a - language-neutral, recoverable, serialization format. diff --git a/tensorflow/docs_src/programmers_guide/custom_estimators.md b/tensorflow/docs_src/programmers_guide/custom_estimators.md deleted file mode 100644 index fb20b35c12..0000000000 --- a/tensorflow/docs_src/programmers_guide/custom_estimators.md +++ /dev/null @@ -1,602 +0,0 @@ - -# Creating Custom Estimators - -This document introduces custom Estimators. In particular, this document -demonstrates how to create a custom @{tf.estimator.Estimator$Estimator} that -mimics the behavior of the pre-made Estimator -@{tf.estimator.DNNClassifier$`DNNClassifier`} in solving the Iris problem. See -the @{$premade_estimators$Pre-Made Estimators chapter} for details -on the Iris problem. - -To download and access the example code invoke the following two commands: - -```shell -git clone https://github.com/tensorflow/models/ -cd models/samples/core/get_started -``` - -In this document we will be looking at -[`custom_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/custom_estimator.py). -You can run it with the following command: - -```bsh -python custom_estimator.py -``` - -If you are feeling impatient, feel free to compare and contrast -[`custom_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/custom_estimator.py) -with -[`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py). -(which is in the same directory). - - - -## Pre-made vs. custom - -As the following figure shows, pre-made Estimators are subclasses of the -@{tf.estimator.Estimator} base class, while custom Estimators are an instance -of tf.estimator.Estimator: - -
-Premade estimators are sub-classes of `Estimator`. Custom Estimators are usually (direct) instances of `Estimator` -
-
-Pre-made and custom Estimators are all Estimators. -
- -Pre-made Estimators are fully baked. Sometimes though, you need more control -over an Estimator's behavior. That's where custom Estimators come in. You can -create a custom Estimator to do just about anything. If you want hidden layers -connected in some unusual fashion, write a custom Estimator. If you want to -calculate a unique -[metric](https://developers.google.com/machine-learning/glossary/#metric) -for your model, write a custom Estimator. Basically, if you want an Estimator -optimized for your specific problem, write a custom Estimator. - -A model function (or `model_fn`) implements the ML algorithm. The -only difference between working with pre-made Estimators and custom Estimators -is: - -* With pre-made Estimators, someone already wrote the model function for you. -* With custom Estimators, you must write the model function. - -Your model function could implement a wide range of algorithms, defining all -sorts of hidden layers and metrics. Like input functions, all model functions -must accept a standard group of input parameters and return a standard group of -output values. Just as input functions can leverage the Dataset API, model -functions can leverage the Layers API and the Metrics API. - -Let's see how to solve the Iris problem with a custom Estimator. A quick -reminder--here's the organization of the Iris model that we're trying to mimic: - -
-A diagram of the network architecture: Inputs, 2 hidden layers, and outputs -
-
-Our implementation of Iris contains four features, two hidden layers, -and a logits output layer. -
- -## Write an Input function - -Our custom Estimator implementation uses the same input function as our -@{$premade_estimators$pre-made Estimator implementation}, from -[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py). -Namely: - -```python -def train_input_fn(features, labels, batch_size): - """An input function for training""" - # Convert the inputs to a Dataset. - dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) - - # Shuffle, repeat, and batch the examples. - dataset = dataset.shuffle(1000).repeat().batch(batch_size) - - # Return the read end of the pipeline. - return dataset.make_one_shot_iterator().get_next() -``` - -This input function builds an input pipeline that yields batches of -`(features, labels)` pairs, where `features` is a dictionary features. - -## Create feature columns - -As detailed in the @{$premade_estimators$Premade Estimators} and -@{$feature_columns$Feature Columns} chapters, you must define -your model's feature columns to specify how the model should use each feature. -Whether working with pre-made Estimators or custom Estimators, you define -feature columns in the same fashion. - -The following code creates a simple `numeric_column` for each input feature, -indicating that the value of the input feature should be used directly as an -input to the model: - -```python -# Feature columns describe how to use the input. -my_feature_columns = [] -for key in train_x.keys(): - my_feature_columns.append(tf.feature_column.numeric_column(key=key)) -``` - -## Write a model function - -The model function we'll use has the following call signature: - -```python -def my_model_fn( - features, # This is batch_features from input_fn - labels, # This is batch_labels from input_fn - mode, # An instance of tf.estimator.ModeKeys - params): # Additional configuration -``` - -The first two arguments are the batches of features and labels returned from -the input function; that is, `features` and `labels` are the handles to the -data your model will use. The `mode` argument indicates whether the caller is -requesting training, predicting, or evaluation. - -The caller may pass `params` to an Estimator's constructor. Any `params` passed -to the constructor are in turn passed on to the `model_fn`. In -[`custom_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/custom_estimator.py) -the following lines create the estimator and set the params to configure the -model. This configuration step is similar to how we configured the @{tf.estimator.DNNClassifier} in -@{$premade_estimators}. - -```python -classifier = tf.estimator.Estimator( - model_fn=my_model, - params={ - 'feature_columns': my_feature_columns, - # Two hidden layers of 10 nodes each. - 'hidden_units': [10, 10], - # The model must choose between 3 classes. - 'n_classes': 3, - }) -``` - -To implement a typical model function, you must do the following: - -* [Define the model](#define_the_model). -* Specify additional calculations for each of - the [three different modes](#modes): - * [Predict](#predict) - * [Evaluate](#evaluate) - * [Train](#train) - -## Define the model - -The basic deep neural network model must define the following three sections: - -* An [input layer](https://developers.google.com/machine-learning/glossary/#input_layer) -* One or more [hidden layers](https://developers.google.com/machine-learning/glossary/#hidden_layer) -* An [output layer](https://developers.google.com/machine-learning/glossary/#output_layer) - -### Define the input layer - -The first line of the `model_fn` calls @{tf.feature_column.input_layer} to -convert the feature dictionary and `feature_columns` into input for your model, -as follows: - -```python - # Use `input_layer` to apply the feature columns. - net = tf.feature_column.input_layer(features, params['feature_columns']) -``` - -The preceding line applies the transformations defined by your feature columns, -creating the model's input layer. - -
-A diagram of the input layer, in this case a 1:1 mapping from raw-inputs to features. -
- - -### Hidden Layers - -If you are creating a deep neural network, you must define one or more hidden -layers. The Layers API provides a rich set of functions to define all types of -hidden layers, including convolutional, pooling, and dropout layers. For Iris, -we're simply going to call @{tf.layers.dense} to create hidden layers, with -dimensions defined by `params['hidden_layers']`. In a `dense` layer each node -is connected to every node in the preceding layer. Here's the relevant code: - -``` python - # Build the hidden layers, sized according to the 'hidden_units' param. - for units in params['hidden_units']: - net = tf.layers.dense(net, units=units, activation=tf.nn.relu) -``` - -* The `units` parameter defines the number of output neurons in a given layer. -* The `activation` parameter defines the [activation function](https://developers.google.com/machine-learning/glossary/#activation_function) — - [Relu](https://developers.google.com/machine-learning/glossary/#ReLU) in this - case. - -The variable `net` here signifies the current top layer of the network. During -the first iteration, `net` signifies the input layer. On each loop iteration -`tf.layers.dense` creates a new layer, which takes the previous layer's output -as its input, using the variable `net`. - -After creating two hidden layers, our network looks as follows. For -simplicity, the figure does not show all the units in each layer. - -
-The input layer with two hidden layers added. -
- -Note that @{tf.layers.dense} provides many additional capabilities, including -the ability to set a multitude of regularization parameters. For the sake of -simplicity, though, we're going to simply accept the default values of the -other parameters. - -### Output Layer - -We'll define the output layer by calling @{tf.layers.dense} yet again, this -time without an activation function: - -```python - # Compute logits (1 per class). - logits = tf.layers.dense(net, params['n_classes'], activation=None) -``` - -Here, `net` signifies the final hidden layer. Therefore, the full set of layers -is now connected as follows: - -
-A logit output layer connected to the top hidden layer -
-
-The final hidden layer feeds into the output layer. -
- -When defining an output layer, the `units` parameter specifies the number of -outputs. So, by setting `units` to `params['n_classes']`, the model produces -one output value per class. Each element of the output vector will contain the -score, or "logit", calculated for the associated class of Iris: Setosa, -Versicolor, or Virginica, respectively. - -Later on, these logits will be transformed into probabilities by the -@{tf.nn.softmax} function. - -## Implement training, evaluation, and prediction {#modes} - -The final step in creating a model function is to write branching code that -implements prediction, evaluation, and training. - -The model function gets invoked whenever someone calls the Estimator's `train`, -`evaluate`, or `predict` methods. Recall that the signature for the model -function looks like this: - -``` python -def my_model_fn( - features, # This is batch_features from input_fn - labels, # This is batch_labels from input_fn - mode, # An instance of tf.estimator.ModeKeys, see below - params): # Additional configuration -``` - -Focus on that third argument, mode. As the following table shows, when someone -calls `train`, `evaluate`, or `predict`, the Estimator framework invokes your model -function with the mode parameter set as follows: - -| Estimator method | Estimator Mode | -|:---------------------------------|:------------------| -|@{tf.estimator.Estimator.train$`train()`} |@{tf.estimator.ModeKeys.TRAIN$`ModeKeys.TRAIN`} | -|@{tf.estimator.Estimator.evaluate$`evaluate()`} |@{tf.estimator.ModeKeys.EVAL$`ModeKeys.EVAL`} | -|@{tf.estimator.Estimator.predict$`predict()`}|@{tf.estimator.ModeKeys.PREDICT$`ModeKeys.PREDICT`} | - -For example, suppose you instantiate a custom Estimator to generate an object -named `classifier`. Then, you make the following call: - -``` python -classifier = tf.estimator.Estimator(...) -classifier.train(input_fn=lambda: my_input_fn(FILE_TRAIN, True, 500)) -``` -The Estimator framework then calls your model function with mode set to -`ModeKeys.TRAIN`. - -Your model function must provide code to handle all three of the mode values. -For each mode value, your code must return an instance of -`tf.estimator.EstimatorSpec`, which contains the information the caller -requires. Let's examine each mode. - -### Predict - -When the Estimator's `predict` method is called, the `model_fn` receives -`mode = ModeKeys.PREDICT`. In this case, the model function must return a -`tf.estimator.EstimatorSpec` containing the prediction. - -The model must have been trained prior to making a prediction. The trained model -is stored on disk in the `model_dir` directory established when you -instantiated the Estimator. - -The code to generate the prediction for this model looks as follows: - -```python -# Compute predictions. -predicted_classes = tf.argmax(logits, 1) -if mode == tf.estimator.ModeKeys.PREDICT: - predictions = { - 'class_ids': predicted_classes[:, tf.newaxis], - 'probabilities': tf.nn.softmax(logits), - 'logits': logits, - } - return tf.estimator.EstimatorSpec(mode, predictions=predictions) -``` -The prediction dictionary contains everything that your model returns when run -in prediction mode. - -
-Additional outputs added to the output layer. -
- -The `predictions` holds the following three key/value pairs: - -* `class_ids` holds the class id (0, 1, or 2) representing the model's - prediction of the most likely species for this example. -* `probabilities` holds the three probabilities (in this example, 0.02, 0.95, - and 0.03) -* `logit` holds the raw logit values (in this example, -1.3, 2.6, and -0.9) - -We return that dictionary to the caller via the `predictions` parameter of the -@{tf.estimator.EstimatorSpec}. The Estimator's -@{tf.estimator.Estimator.predict$`predict`} method will yield these -dictionaries. - -### Calculate the loss - -For both [training](#train) and [evaluation](#evaluate) we need to calculate the -model's loss. This is the -[objective](https://developers.google.com/machine-learning/glossary/#objective) -that will be optimized. - -We can calculate the loss by calling @{tf.losses.sparse_softmax_cross_entropy}. -The value returned by this function will be lowest, approximately 0, -probability of the correct class (at index `label`) is near 1.0. The loss value -returned is progressively larger as the probability of the correct class -decreases. - -This function returns the average over the whole batch. - -```python -# Compute loss. -loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) -``` - -### Evaluate - -When the Estimator's `evaluate` method is called, the `model_fn` receives -`mode = ModeKeys.EVAL`. In this case, the model function must return a -`tf.estimator.EstimatorSpec` containing the model's loss and optionally one -or more metrics. - -Although returning metrics is optional, most custom Estimators do return at -least one metric. TensorFlow provides a Metrics module @{tf.metrics} to -calculate common metrics. For brevity's sake, we'll only return accuracy. The -@{tf.metrics.accuracy} function compares our predictions against the -true values, that is, against the labels provided by the input function. The -@{tf.metrics.accuracy} function requires the labels and predictions to have the -same shape. Here's the call to @{tf.metrics.accuracy}: - -``` python -# Compute evaluation metrics. -accuracy = tf.metrics.accuracy(labels=labels, - predictions=predicted_classes, - name='acc_op') -``` - -The @{tf.estimator.EstimatorSpec$`EstimatorSpec`} returned for evaluation -typically contains the following information: - -* `loss`, which is the model's loss -* `eval_metric_ops`, which is an optional dictionary of metrics. - -So, we'll create a dictionary containing our sole metric. If we had calculated -other metrics, we would have added them as additional key/value pairs to that -same dictionary. Then, we'll pass that dictionary in the `eval_metric_ops` -argument of `tf.estimator.EstimatorSpec`. Here's the code: - -```python -metrics = {'accuracy': accuracy} -tf.summary.scalar('accuracy', accuracy[1]) - -if mode == tf.estimator.ModeKeys.EVAL: - return tf.estimator.EstimatorSpec( - mode, loss=loss, eval_metric_ops=metrics) -``` - -The @{tf.summary.scalar} will make accuracy available to TensorBoard -in both `TRAIN` and `EVAL` modes. (More on this later). - -### Train - -When the Estimator's `train` method is called, the `model_fn` is called -with `mode = ModeKeys.TRAIN`. In this case, the model function must return an -`EstimatorSpec` that contains the loss and a training operation. - -Building the training operation will require an optimizer. We will use -@{tf.train.AdagradOptimizer} because we're mimicking the `DNNClassifier`, which -also uses `Adagrad` by default. The `tf.train` package provides many other -optimizers—feel free to experiment with them. - -Here is the code that builds the optimizer: - -``` python -optimizer = tf.train.AdagradOptimizer(learning_rate=0.1) -``` - -Next, we build the training operation using the optimizer's -@{tf.train.Optimizer.minimize$`minimize`} method on the loss we calculated -earlier. - -The `minimize` method also takes a `global_step` parameter. TensorFlow uses this -parameter to count the number of training steps that have been processed -(to know when to end a training run). Furthermore, the `global_step` is -essential for TensorBoard graphs to work correctly. Simply call -@{tf.train.get_global_step} and pass the result to the `global_step` -argument of `minimize`. - -Here's the code to train the model: - -``` python -train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) -``` - -The @{tf.estimator.EstimatorSpec$`EstimatorSpec`} returned for training -must have the following fields set: - -* `loss`, which contains the value of the loss function. -* `train_op`, which executes a training step. - -Here's our code to call `EstimatorSpec`: - -```python -return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op) -``` - -The model function is now complete. - -## The custom Estimator - -Instantiate the custom Estimator through the Estimator base class as follows: - -```python - # Build 2 hidden layer DNN with 10, 10 units respectively. - classifier = tf.estimator.Estimator( - model_fn=my_model, - params={ - 'feature_columns': my_feature_columns, - # Two hidden layers of 10 nodes each. - 'hidden_units': [10, 10], - # The model must choose between 3 classes. - 'n_classes': 3, - }) -``` -Here the `params` dictionary serves the same purpose as the key-word -arguments of `DNNClassifier`; that is, the `params` dictionary lets you -configure your Estimator without modifying the code in the `model_fn`. - -The rest of the code to train, evaluate, and generate predictions using our -Estimator is the same as in the -@{$premade_estimators$Premade Estimators} chapter. For -example, the following line will train the model: - -```python -# Train the Model. -classifier.train( - input_fn=lambda:iris_data.train_input_fn(train_x, train_y, args.batch_size), - steps=args.train_steps) -``` - -## TensorBoard - -You can view training results for your custom Estimator in TensorBoard. To see -this reporting, start TensorBoard from your command line as follows: - -```bsh -# Replace PATH with the actual path passed as model_dir -tensorboard --logdir=PATH -``` - -Then, open TensorBoard by browsing to: [http://localhost:6006](http://localhost:6006) - -All the pre-made Estimators automatically log a lot of information to -TensorBoard. With custom Estimators, however, TensorBoard only provides one -default log (a graph of the loss) plus the information you explicitly tell -TensorBoard to log. For the custom Estimator you just created, TensorBoard -generates the following: - -
- -Accuracy, 'scalar' graph from tensorboard - -loss 'scalar' graph from tensorboard - -steps/second 'scalar' graph from tensorboard -
- -
-TensorBoard displays three graphs. -
- - -In brief, here's what the three graphs tell you: - -* global_step/sec: A performance indicator showing how many batches (gradient - updates) we processed per second as the model trains. - -* loss: The loss reported. - -* accuracy: The accuracy is recorded by the following two lines: - - * `eval_metric_ops={'my_accuracy': accuracy}`, during evaluation. - * `tf.summary.scalar('accuracy', accuracy[1])`, during training. - -These tensorboard graphs are one of the main reasons it's important to pass a -`global_step` to your optimizer's `minimize` method. The model can't record -the x-coordinate for these graphs without it. - -Note the following in the `my_accuracy` and `loss` graphs: - -* The orange line represents training. -* The blue dot represents evaluation. - -During training, summaries (the orange line) are recorded periodically as -batches are processed, which is why it becomes a graph spanning x-axis range. - -By contrast, evaluation produces only a single point on the graph for each call -to `evaluate`. This point contains the average over the entire evaluation call. -This has no width on the graph as it is evaluated entirely from the model state -at a particular training step (from a single checkpoint). - -As suggested in the following figure, you may see and also selectively -disable/enable the reporting using the controls on the left side. - -
-Check-boxes allowing the user to select which runs are shown. -
-
-Enable or disable reporting. -
- - -## Summary - -Although pre-made Estimators can be an effective way to quickly create new -models, you will often need the additional flexibility that custom Estimators -provide. Fortunately, pre-made and custom Estimators follow the same -programming model. The only practical difference is that you must write a model -function for custom Estimators; everything else is the same. - -For more details, be sure to check out: - -* The - [official TensorFlow implementation of MNIST](https://github.com/tensorflow/models/tree/master/official/mnist), - which uses a custom estimator. -* The TensorFlow - [official models repository](https://github.com/tensorflow/models/tree/master/official), - which contains more curated examples using custom estimators. -* This [TensorBoard video](https://youtu.be/eBbEDRsCmv4), which introduces - TensorBoard. -* The @{$low_level_intro$Low Level Introduction}, which demonstrates - how to experiment directly with TensorFlow's low level APIs, making debugging - easier. diff --git a/tensorflow/docs_src/programmers_guide/datasets.md b/tensorflow/docs_src/programmers_guide/datasets.md deleted file mode 100644 index 8b69860a68..0000000000 --- a/tensorflow/docs_src/programmers_guide/datasets.md +++ /dev/null @@ -1,823 +0,0 @@ -# Importing Data - -The @{tf.data} API enables you to build complex input pipelines from -simple, reusable pieces. For example, the pipeline for an image model might -aggregate data from files in a distributed file system, apply random -perturbations to each image, and merge randomly selected images into a batch -for training. The pipeline for a text model might involve extracting symbols -from raw text data, converting them to embedding identifiers with a lookup -table, and batching together sequences of different lengths. The `tf.data` API -makes it easy to deal with large amounts of data, different data formats, and -complicated transformations. - -The `tf.data` API introduces two new abstractions to TensorFlow: - -* A `tf.data.Dataset` represents a sequence of elements, in which - each element contains one or more `Tensor` objects. For example, in an image - pipeline, an element might be a single training example, with a pair of - tensors representing the image data and a label. There are two distinct - ways to create a dataset: - - * Creating a **source** (e.g. `Dataset.from_tensor_slices()`) constructs a - dataset from - one or more `tf.Tensor` objects. - - * Applying a **transformation** (e.g. `Dataset.batch()`) constructs a dataset - from one or more `tf.data.Dataset` objects. - -* A `tf.data.Iterator` provides the main way to extract elements from a - dataset. The operation returned by `Iterator.get_next()` yields the next - element of a `Dataset` when executed, and typically acts as the interface - between input pipeline code and your model. The simplest iterator is a - "one-shot iterator", which is associated with a particular `Dataset` and - iterates through it once. For more sophisticated uses, the - `Iterator.initializer` operation enables you to reinitialize and parameterize - an iterator with different datasets, so that you can, for example, iterate - over training and validation data multiple times in the same program. - -## Basic mechanics - -This section of the guide describes the fundamentals of creating different kinds -of `Dataset` and `Iterator` objects, and how to extract data from them. - -To start an input pipeline, you must define a *source*. For example, to -construct a `Dataset` from some tensors in memory, you can use -`tf.data.Dataset.from_tensors()` or -`tf.data.Dataset.from_tensor_slices()`. Alternatively, if your input -data are on disk in the recommended TFRecord format, you can construct a -`tf.data.TFRecordDataset`. - -Once you have a `Dataset` object, you can *transform* it into a new `Dataset` by -chaining method calls on the `tf.data.Dataset` object. For example, you -can apply per-element transformations such as `Dataset.map()` (to apply a -function to each element), and multi-element transformations such as -`Dataset.batch()`. See the documentation for @{tf.data.Dataset} -for a complete list of transformations. - -The most common way to consume values from a `Dataset` is to make an -**iterator** object that provides access to one element of the dataset at a time -(for example, by calling `Dataset.make_one_shot_iterator()`). A -`tf.data.Iterator` provides two operations: `Iterator.initializer`, -which enables you to (re)initialize the iterator's state; and -`Iterator.get_next()`, which returns `tf.Tensor` objects that correspond to the -symbolic next element. Depending on your use case, you might choose a different -type of iterator, and the options are outlined below. - -### Dataset structure - -A dataset comprises elements that each have the same structure. An element -contains one or more `tf.Tensor` objects, called *components*. Each component -has a `tf.DType` representing the type of elements in the tensor, and a -`tf.TensorShape` representing the (possibly partially specified) static shape of -each element. The `Dataset.output_types` and `Dataset.output_shapes` properties -allow you to inspect the inferred types and shapes of each component of a -dataset element. The *nested structure* of these properties map to the structure -of an element, which may be a single tensor, a tuple of tensors, or a nested -tuple of tensors. For example: - -```python -dataset1 = tf.data.Dataset.from_tensor_slices(tf.random_uniform([4, 10])) -print(dataset1.output_types) # ==> "tf.float32" -print(dataset1.output_shapes) # ==> "(10,)" - -dataset2 = tf.data.Dataset.from_tensor_slices( - (tf.random_uniform([4]), - tf.random_uniform([4, 100], maxval=100, dtype=tf.int32))) -print(dataset2.output_types) # ==> "(tf.float32, tf.int32)" -print(dataset2.output_shapes) # ==> "((), (100,))" - -dataset3 = tf.data.Dataset.zip((dataset1, dataset2)) -print(dataset3.output_types) # ==> (tf.float32, (tf.float32, tf.int32)) -print(dataset3.output_shapes) # ==> "(10, ((), (100,)))" -``` - -It is often convenient to give names to each component of an element, for -example if they represent different features of a training example. In addition -to tuples, you can use `collections.namedtuple` or a dictionary mapping strings -to tensors to represent a single element of a `Dataset`. - -```python -dataset = tf.data.Dataset.from_tensor_slices( - {"a": tf.random_uniform([4]), - "b": tf.random_uniform([4, 100], maxval=100, dtype=tf.int32)}) -print(dataset.output_types) # ==> "{'a': tf.float32, 'b': tf.int32}" -print(dataset.output_shapes) # ==> "{'a': (), 'b': (100,)}" -``` - -The `Dataset` transformations support datasets of any structure. When using the -`Dataset.map()`, `Dataset.flat_map()`, and `Dataset.filter()` transformations, -which apply a function to each element, the element structure determines the -arguments of the function: - -```python -dataset1 = dataset1.map(lambda x: ...) - -dataset2 = dataset2.flat_map(lambda x, y: ...) - -# Note: Argument destructuring is not available in Python 3. -dataset3 = dataset3.filter(lambda x, (y, z): ...) -``` - -### Creating an iterator - -Once you have built a `Dataset` to represent your input data, the next step is to -create an `Iterator` to access elements from that dataset. The `tf.data` API -currently supports the following iterators, in increasing level of -sophistication: - -* **one-shot**, -* **initializable**, -* **reinitializable**, and -* **feedable**. - -A **one-shot** iterator is the simplest form of iterator, which only supports -iterating once through a dataset, with no need for explicit initialization. -One-shot iterators handle almost all of the cases that the existing queue-based -input pipelines support, but they do not support parameterization. Using the -example of `Dataset.range()`: - -```python -dataset = tf.data.Dataset.range(100) -iterator = dataset.make_one_shot_iterator() -next_element = iterator.get_next() - -for i in range(100): - value = sess.run(next_element) - assert i == value -``` - -Note: Currently, one-shot iterators are the only type that is easily usable -with an `Estimator`. - -An **initializable** iterator requires you to run an explicit -`iterator.initializer` operation before using it. In exchange for this -inconvenience, it enables you to *parameterize* the definition of the dataset, -using one or more `tf.placeholder()` tensors that can be fed when you -initialize the iterator. Continuing the `Dataset.range()` example: - -```python -max_value = tf.placeholder(tf.int64, shape=[]) -dataset = tf.data.Dataset.range(max_value) -iterator = dataset.make_initializable_iterator() -next_element = iterator.get_next() - -# Initialize an iterator over a dataset with 10 elements. -sess.run(iterator.initializer, feed_dict={max_value: 10}) -for i in range(10): - value = sess.run(next_element) - assert i == value - -# Initialize the same iterator over a dataset with 100 elements. -sess.run(iterator.initializer, feed_dict={max_value: 100}) -for i in range(100): - value = sess.run(next_element) - assert i == value -``` - -A **reinitializable** iterator can be initialized from multiple different -`Dataset` objects. For example, you might have a training input pipeline that -uses random perturbations to the input images to improve generalization, and -a validation input pipeline that evaluates predictions on unmodified data. These -pipelines will typically use different `Dataset` objects that have the same -structure (i.e. the same types and compatible shapes for each component). - -```python -# Define training and validation datasets with the same structure. -training_dataset = tf.data.Dataset.range(100).map( - lambda x: x + tf.random_uniform([], -10, 10, tf.int64)) -validation_dataset = tf.data.Dataset.range(50) - -# A reinitializable iterator is defined by its structure. We could use the -# `output_types` and `output_shapes` properties of either `training_dataset` -# or `validation_dataset` here, because they are compatible. -iterator = tf.data.Iterator.from_structure(training_dataset.output_types, - training_dataset.output_shapes) -next_element = iterator.get_next() - -training_init_op = iterator.make_initializer(training_dataset) -validation_init_op = iterator.make_initializer(validation_dataset) - -# Run 20 epochs in which the training dataset is traversed, followed by the -# validation dataset. -for _ in range(20): - # Initialize an iterator over the training dataset. - sess.run(training_init_op) - for _ in range(100): - sess.run(next_element) - - # Initialize an iterator over the validation dataset. - sess.run(validation_init_op) - for _ in range(50): - sess.run(next_element) -``` - -A **feedable** iterator can be used together with @{tf.placeholder} to select -what `Iterator` to use in each call to @{tf.Session.run}, via the familiar -`feed_dict` mechanism. It offers the same functionality as a reinitializable -iterator, but it does not require you to initialize the iterator from the start -of a dataset when you switch between iterators. For example, using the same -training and validation example from above, you can use -@{tf.data.Iterator.from_string_handle} to define a feedable iterator -that allows you to switch between the two datasets: - -```python -# Define training and validation datasets with the same structure. -training_dataset = tf.data.Dataset.range(100).map( - lambda x: x + tf.random_uniform([], -10, 10, tf.int64)).repeat() -validation_dataset = tf.data.Dataset.range(50) - -# A feedable iterator is defined by a handle placeholder and its structure. We -# could use the `output_types` and `output_shapes` properties of either -# `training_dataset` or `validation_dataset` here, because they have -# identical structure. -handle = tf.placeholder(tf.string, shape=[]) -iterator = tf.data.Iterator.from_string_handle( - handle, training_dataset.output_types, training_dataset.output_shapes) -next_element = iterator.get_next() - -# You can use feedable iterators with a variety of different kinds of iterator -# (such as one-shot and initializable iterators). -training_iterator = training_dataset.make_one_shot_iterator() -validation_iterator = validation_dataset.make_initializable_iterator() - -# The `Iterator.string_handle()` method returns a tensor that can be evaluated -# and used to feed the `handle` placeholder. -training_handle = sess.run(training_iterator.string_handle()) -validation_handle = sess.run(validation_iterator.string_handle()) - -# Loop forever, alternating between training and validation. -while True: - # Run 200 steps using the training dataset. Note that the training dataset is - # infinite, and we resume from where we left off in the previous `while` loop - # iteration. - for _ in range(200): - sess.run(next_element, feed_dict={handle: training_handle}) - - # Run one pass over the validation dataset. - sess.run(validation_iterator.initializer) - for _ in range(50): - sess.run(next_element, feed_dict={handle: validation_handle}) -``` - -### Consuming values from an iterator - -The `Iterator.get_next()` method returns one or more `tf.Tensor` objects that -correspond to the symbolic next element of an iterator. Each time these tensors -are evaluated, they take the value of the next element in the underlying -dataset. (Note that, like other stateful objects in TensorFlow, calling -`Iterator.get_next()` does not immediately advance the iterator. Instead you -must use the returned `tf.Tensor` objects in a TensorFlow expression, and pass -the result of that expression to `tf.Session.run()` to get the next elements and -advance the iterator.) - -If the iterator reaches the end of the dataset, executing -the `Iterator.get_next()` operation will raise a `tf.errors.OutOfRangeError`. -After this point the iterator will be in an unusable state, and you must -initialize it again if you want to use it further. - -```python -dataset = tf.data.Dataset.range(5) -iterator = dataset.make_initializable_iterator() -next_element = iterator.get_next() - -# Typically `result` will be the output of a model, or an optimizer's -# training operation. -result = tf.add(next_element, next_element) - -sess.run(iterator.initializer) -print(sess.run(result)) # ==> "0" -print(sess.run(result)) # ==> "2" -print(sess.run(result)) # ==> "4" -print(sess.run(result)) # ==> "6" -print(sess.run(result)) # ==> "8" -try: - sess.run(result) -except tf.errors.OutOfRangeError: - print("End of dataset") # ==> "End of dataset" -``` - -A common pattern is to wrap the "training loop" in a `try`-`except` block: - -```python -sess.run(iterator.initializer) -while True: - try: - sess.run(result) - except tf.errors.OutOfRangeError: - break -``` - -If each element of the dataset has a nested structure, the return value of -`Iterator.get_next()` will be one or more `tf.Tensor` objects in the same -nested structure: - -```python -dataset1 = tf.data.Dataset.from_tensor_slices(tf.random_uniform([4, 10])) -dataset2 = tf.data.Dataset.from_tensor_slices((tf.random_uniform([4]), tf.random_uniform([4, 100]))) -dataset3 = tf.data.Dataset.zip((dataset1, dataset2)) - -iterator = dataset3.make_initializable_iterator() - -sess.run(iterator.initializer) -next1, (next2, next3) = iterator.get_next() -``` - -Note that `next1`, `next2`, and `next3` are tensors produced by the -same op/node (created by `Iterator.get_next()`). Therefore, evaluating *any* of -these tensors will advance the iterator for all components. A typical consumer -of an iterator will include all components in a single expression. - -### Saving iterator state - -The @{tf.contrib.data.make_saveable_from_iterator} function creates a -`SaveableObject` from an iterator, which can be used to save and -restore the current state of the iterator (and, effectively, the whole input -pipeline). A saveable object thus created can be added to @{tf.train.Saver} -variables list or the `tf.GraphKeys.SAVEABLE_OBJECTS` collection for saving and -restoring in the same manner as a @{tf.Variable}. Refer to -@{$saved_model$Saving and Restoring} for details on how to save and restore -variables. - -```python -# Create saveable object from iterator. -saveable = tf.contrib.data.make_saveable_from_iterator(iterator) - -# Save the iterator state by adding it to the saveable objects collection. -tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, saveable) -saver = tf.train.Saver() - -with tf.Session() as sess: - - if should_checkpoint: - saver.save(path_to_checkpoint) - -# Restore the iterator state. -with tf.Session() as sess: - saver.restore(sess, path_to_checkpoint) -``` - -## Reading input data - -### Consuming NumPy arrays - -If all of your input data fit in memory, the simplest way to create a `Dataset` -from them is to convert them to `tf.Tensor` objects and use -`Dataset.from_tensor_slices()`. - -```python -# Load the training data into two NumPy arrays, for example using `np.load()`. -with np.load("/var/data/training_data.npy") as data: - features = data["features"] - labels = data["labels"] - -# Assume that each row of `features` corresponds to the same row as `labels`. -assert features.shape[0] == labels.shape[0] - -dataset = tf.data.Dataset.from_tensor_slices((features, labels)) -``` - -Note that the above code snippet will embed the `features` and `labels` arrays -in your TensorFlow graph as `tf.constant()` operations. This works well for a -small dataset, but wastes memory---because the contents of the array will be -copied multiple times---and can run into the 2GB limit for the `tf.GraphDef` -protocol buffer. - -As an alternative, you can define the `Dataset` in terms of `tf.placeholder()` -tensors, and *feed* the NumPy arrays when you initialize an `Iterator` over the -dataset. - -```python -# Load the training data into two NumPy arrays, for example using `np.load()`. -with np.load("/var/data/training_data.npy") as data: - features = data["features"] - labels = data["labels"] - -# Assume that each row of `features` corresponds to the same row as `labels`. -assert features.shape[0] == labels.shape[0] - -features_placeholder = tf.placeholder(features.dtype, features.shape) -labels_placeholder = tf.placeholder(labels.dtype, labels.shape) - -dataset = tf.data.Dataset.from_tensor_slices((features_placeholder, labels_placeholder)) -# [Other transformations on `dataset`...] -dataset = ... -iterator = dataset.make_initializable_iterator() - -sess.run(iterator.initializer, feed_dict={features_placeholder: features, - labels_placeholder: labels}) -``` - -### Consuming TFRecord data - -The `tf.data` API supports a variety of file formats so that you can process -large datasets that do not fit in memory. For example, the TFRecord file format -is a simple record-oriented binary format that many TensorFlow applications use -for training data. The `tf.data.TFRecordDataset` class enables you to -stream over the contents of one or more TFRecord files as part of an input -pipeline. - -```python -# Creates a dataset that reads all of the examples from two files. -filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] -dataset = tf.data.TFRecordDataset(filenames) -``` - -The `filenames` argument to the `TFRecordDataset` initializer can either be a -string, a list of strings, or a `tf.Tensor` of strings. Therefore if you have -two sets of files for training and validation purposes, you can use a -`tf.placeholder(tf.string)` to represent the filenames, and initialize an -iterator from the appropriate filenames: - -```python -filenames = tf.placeholder(tf.string, shape=[None]) -dataset = tf.data.TFRecordDataset(filenames) -dataset = dataset.map(...) # Parse the record into tensors. -dataset = dataset.repeat() # Repeat the input indefinitely. -dataset = dataset.batch(32) -iterator = dataset.make_initializable_iterator() - -# You can feed the initializer with the appropriate filenames for the current -# phase of execution, e.g. training vs. validation. - -# Initialize `iterator` with training data. -training_filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] -sess.run(iterator.initializer, feed_dict={filenames: training_filenames}) - -# Initialize `iterator` with validation data. -validation_filenames = ["/var/data/validation1.tfrecord", ...] -sess.run(iterator.initializer, feed_dict={filenames: validation_filenames}) -``` - -### Consuming text data - -Many datasets are distributed as one or more text files. The -`tf.data.TextLineDataset` provides an easy way to extract lines from -one or more text files. Given one or more filenames, a `TextLineDataset` will -produce one string-valued element per line of those files. Like a -`TFRecordDataset`, `TextLineDataset` accepts `filenames` as a `tf.Tensor`, so -you can parameterize it by passing a `tf.placeholder(tf.string)`. - -```python -filenames = ["/var/data/file1.txt", "/var/data/file2.txt"] -dataset = tf.data.TextLineDataset(filenames) -``` - -By default, a `TextLineDataset` yields *every* line of each file, which may -not be desirable, for example if the file starts with a header line, or contains -comments. These lines can be removed using the `Dataset.skip()` and -`Dataset.filter()` transformations. To apply these transformations to each -file separately, we use `Dataset.flat_map()` to create a nested `Dataset` for -each file. - -```python -filenames = ["/var/data/file1.txt", "/var/data/file2.txt"] - -dataset = tf.data.Dataset.from_tensor_slices(filenames) - -# Use `Dataset.flat_map()` to transform each file as a separate nested dataset, -# and then concatenate their contents sequentially into a single "flat" dataset. -# * Skip the first line (header row). -# * Filter out lines beginning with "#" (comments). -dataset = dataset.flat_map( - lambda filename: ( - tf.data.TextLineDataset(filename) - .skip(1) - .filter(lambda line: tf.not_equal(tf.substr(line, 0, 1), "#")))) -``` - -### Consuming CSV data - -The CSV file format is a popular format for storing tabular data in plain text. -The @{tf.contrib.data.CsvDataset} class provides a way to extract records from -one or more CSV files that comply with [RFC 4180](https://tools.ietf.org/html/rfc4180). -Given one or more filenames and a list of defaults, a `CsvDataset` will produce -a tuple of elements whose types correspond to the types of the defaults -provided, per CSV record. Like `TFRecordDataset` and `TextLineDataset`, -`CsvDataset` accepts `filenames` as a `tf.Tensor`, so you can parameterize it -by passing a `tf.placeholder(tf.string)`. - -``` -# Creates a dataset that reads all of the records from two CSV files, each with -# eight float columns -filenames = ["/var/data/file1.csv", "/var/data/file2.csv"] -record_defaults = [tf.float32] * 8 # Eight required float columns -dataset = tf.contrib.data.CsvDataset(filenames, record_defaults) -``` - -If some columns are empty, you can provide defaults instead of types. - -``` -# Creates a dataset that reads all of the records from two CSV files, each with -# four float columns which may have missing values -record_defaults = [[0.0]] * 8 -dataset = tf.contrib.data.CsvDataset(filenames, record_defaults) -``` - -By default, a `CsvDataset` yields *every* column of *every* line of the file, -which may not be desirable, for example if the file starts with a header line -that should be ignored, or if some columns are not required in the input. -These lines and fields can be removed with the `header` and `select_cols` -arguments respectively. - -``` -# Creates a dataset that reads all of the records from two CSV files with -# headers, extracting float data from columns 2 and 4. -record_defaults = [[0.0]] * 2 # Only provide defaults for the selected columns -dataset = tf.contrib.data.CsvDataset(filenames, record_defaults, header=True, select_cols=[2,4]) -``` - - -## Preprocessing data with `Dataset.map()` - -The `Dataset.map(f)` transformation produces a new dataset by applying a given -function `f` to each element of the input dataset. It is based on -the -[`map()` function](https://en.wikipedia.org/wiki/Map_(higher-order_function)) -that is commonly applied to lists (and other structures) in functional -programming languages. The function `f` takes the `tf.Tensor` objects that -represent a single element in the input, and returns the `tf.Tensor` objects -that will represent a single element in the new dataset. Its implementation uses -standard TensorFlow operations to transform one element into another. - -This section covers common examples of how to use `Dataset.map()`. - -### Parsing `tf.Example` protocol buffer messages - -Many input pipelines extract `tf.train.Example` protocol buffer messages from a -TFRecord-format file (written, for example, using -`tf.python_io.TFRecordWriter`). Each `tf.train.Example` record contains one or -more "features", and the input pipeline typically converts these features into -tensors. - -```python -# Transforms a scalar string `example_proto` into a pair of a scalar string and -# a scalar integer, representing an image and its label, respectively. -def _parse_function(example_proto): - features = {"image": tf.FixedLenFeature((), tf.string, default_value=""), - "label": tf.FixedLenFeature((), tf.int32, default_value=0)} - parsed_features = tf.parse_single_example(example_proto, features) - return parsed_features["image"], parsed_features["label"] - -# Creates a dataset that reads all of the examples from two files, and extracts -# the image and label features. -filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] -dataset = tf.data.TFRecordDataset(filenames) -dataset = dataset.map(_parse_function) -``` - -### Decoding image data and resizing it - -When training a neural network on real-world image data, it is often necessary -to convert images of different sizes to a common size, so that they may be -batched into a fixed size. - -```python -# Reads an image from a file, decodes it into a dense tensor, and resizes it -# to a fixed shape. -def _parse_function(filename, label): - image_string = tf.read_file(filename) - image_decoded = tf.image.decode_jpeg(image_string) - image_resized = tf.image.resize_images(image_decoded, [28, 28]) - return image_resized, label - -# A vector of filenames. -filenames = tf.constant(["/var/data/image1.jpg", "/var/data/image2.jpg", ...]) - -# `labels[i]` is the label for the image in `filenames[i]. -labels = tf.constant([0, 37, ...]) - -dataset = tf.data.Dataset.from_tensor_slices((filenames, labels)) -dataset = dataset.map(_parse_function) -``` - -### Applying arbitrary Python logic with `tf.py_func()` - -For performance reasons, we encourage you to use TensorFlow operations for -preprocessing your data whenever possible. However, it is sometimes useful to -call upon external Python libraries when parsing your input data. To do so, -invoke, the `tf.py_func()` operation in a `Dataset.map()` transformation. - -```python -import cv2 - -# Use a custom OpenCV function to read the image, instead of the standard -# TensorFlow `tf.read_file()` operation. -def _read_py_function(filename, label): - image_decoded = cv2.imread(filename.decode(), cv2.IMREAD_GRAYSCALE) - return image_decoded, label - -# Use standard TensorFlow operations to resize the image to a fixed shape. -def _resize_function(image_decoded, label): - image_decoded.set_shape([None, None, None]) - image_resized = tf.image.resize_images(image_decoded, [28, 28]) - return image_resized, label - -filenames = ["/var/data/image1.jpg", "/var/data/image2.jpg", ...] -labels = [0, 37, 29, 1, ...] - -dataset = tf.data.Dataset.from_tensor_slices((filenames, labels)) -dataset = dataset.map( - lambda filename, label: tuple(tf.py_func( - _read_py_function, [filename, label], [tf.uint8, label.dtype]))) -dataset = dataset.map(_resize_function) -``` - - - -## Batching dataset elements - -### Simple batching - -The simplest form of batching stacks `n` consecutive elements of a dataset into -a single element. The `Dataset.batch()` transformation does exactly this, with -the same constraints as the `tf.stack()` operator, applied to each component -of the elements: i.e. for each component *i*, all elements must have a tensor -of the exact same shape. - -```python -inc_dataset = tf.data.Dataset.range(100) -dec_dataset = tf.data.Dataset.range(0, -100, -1) -dataset = tf.data.Dataset.zip((inc_dataset, dec_dataset)) -batched_dataset = dataset.batch(4) - -iterator = batched_dataset.make_one_shot_iterator() -next_element = iterator.get_next() - -print(sess.run(next_element)) # ==> ([0, 1, 2, 3], [ 0, -1, -2, -3]) -print(sess.run(next_element)) # ==> ([4, 5, 6, 7], [-4, -5, -6, -7]) -print(sess.run(next_element)) # ==> ([8, 9, 10, 11], [-8, -9, -10, -11]) -``` - -### Batching tensors with padding - -The above recipe works for tensors that all have the same size. However, many -models (e.g. sequence models) work with input data that can have varying size -(e.g. sequences of different lengths). To handle this case, the -`Dataset.padded_batch()` transformation enables you to batch tensors of -different shape by specifying one or more dimensions in which they may be -padded. - -```python -dataset = tf.data.Dataset.range(100) -dataset = dataset.map(lambda x: tf.fill([tf.cast(x, tf.int32)], x)) -dataset = dataset.padded_batch(4, padded_shapes=[None]) - -iterator = dataset.make_one_shot_iterator() -next_element = iterator.get_next() - -print(sess.run(next_element)) # ==> [[0, 0, 0], [1, 0, 0], [2, 2, 0], [3, 3, 3]] -print(sess.run(next_element)) # ==> [[4, 4, 4, 4, 0, 0, 0], - # [5, 5, 5, 5, 5, 0, 0], - # [6, 6, 6, 6, 6, 6, 0], - # [7, 7, 7, 7, 7, 7, 7]] -``` - -The `Dataset.padded_batch()` transformation allows you to set different padding -for each dimension of each component, and it may be variable-length (signified -by `None` in the example above) or constant-length. It is also possible to -override the padding value, which defaults to 0. - - - -## Training workflows - -### Processing multiple epochs - -The `tf.data` API offers two main ways to process multiple epochs of the same -data. - -The simplest way to iterate over a dataset in multiple epochs is to use the -`Dataset.repeat()` transformation. For example, to create a dataset that repeats -its input for 10 epochs: - -```python -filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] -dataset = tf.data.TFRecordDataset(filenames) -dataset = dataset.map(...) -dataset = dataset.repeat(10) -dataset = dataset.batch(32) -``` - -Applying the `Dataset.repeat()` transformation with no arguments will repeat -the input indefinitely. The `Dataset.repeat()` transformation concatenates its -arguments without signaling the end of one epoch and the beginning of the next -epoch. - -If you want to receive a signal at the end of each epoch, you can write a -training loop that catches the `tf.errors.OutOfRangeError` at the end of a -dataset. At that point you might collect some statistics (e.g. the validation -error) for the epoch. - -```python -filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] -dataset = tf.data.TFRecordDataset(filenames) -dataset = dataset.map(...) -dataset = dataset.batch(32) -iterator = dataset.make_initializable_iterator() -next_element = iterator.get_next() - -# Compute for 100 epochs. -for _ in range(100): - sess.run(iterator.initializer) - while True: - try: - sess.run(next_element) - except tf.errors.OutOfRangeError: - break - - # [Perform end-of-epoch calculations here.] -``` - -### Randomly shuffling input data - -The `Dataset.shuffle()` transformation randomly shuffles the input dataset -using a similar algorithm to `tf.RandomShuffleQueue`: it maintains a fixed-size -buffer and chooses the next element uniformly at random from that buffer. - -```python -filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] -dataset = tf.data.TFRecordDataset(filenames) -dataset = dataset.map(...) -dataset = dataset.shuffle(buffer_size=10000) -dataset = dataset.batch(32) -dataset = dataset.repeat() -``` - -### Using high-level APIs - -The @{tf.train.MonitoredTrainingSession} API simplifies many aspects of running -TensorFlow in a distributed setting. `MonitoredTrainingSession` uses the -@{tf.errors.OutOfRangeError} to signal that training has completed, so to use it -with the `tf.data` API, we recommend using -`Dataset.make_one_shot_iterator()`. For example: - -```python -filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] -dataset = tf.data.TFRecordDataset(filenames) -dataset = dataset.map(...) -dataset = dataset.shuffle(buffer_size=10000) -dataset = dataset.batch(32) -dataset = dataset.repeat(num_epochs) -iterator = dataset.make_one_shot_iterator() - -next_example, next_label = iterator.get_next() -loss = model_function(next_example, next_label) - -training_op = tf.train.AdagradOptimizer(...).minimize(loss) - -with tf.train.MonitoredTrainingSession(...) as sess: - while not sess.should_stop(): - sess.run(training_op) -``` - -To use a `Dataset` in the `input_fn` of a @{tf.estimator.Estimator}, we also -recommend using `Dataset.make_one_shot_iterator()`. For example: - -```python -def dataset_input_fn(): - filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] - dataset = tf.data.TFRecordDataset(filenames) - - # Use `tf.parse_single_example()` to extract data from a `tf.Example` - # protocol buffer, and perform any additional per-record preprocessing. - def parser(record): - keys_to_features = { - "image_data": tf.FixedLenFeature((), tf.string, default_value=""), - "date_time": tf.FixedLenFeature((), tf.int64, default_value=""), - "label": tf.FixedLenFeature((), tf.int64, - default_value=tf.zeros([], dtype=tf.int64)), - } - parsed = tf.parse_single_example(record, keys_to_features) - - # Perform additional preprocessing on the parsed data. - image = tf.image.decode_jpeg(parsed["image_data"]) - image = tf.reshape(image, [299, 299, 1]) - label = tf.cast(parsed["label"], tf.int32) - - return {"image_data": image, "date_time": parsed["date_time"]}, label - - # Use `Dataset.map()` to build a pair of a feature dictionary and a label - # tensor for each example. - dataset = dataset.map(parser) - dataset = dataset.shuffle(buffer_size=10000) - dataset = dataset.batch(32) - dataset = dataset.repeat(num_epochs) - iterator = dataset.make_one_shot_iterator() - - # `features` is a dictionary in which each value is a batch of values for - # that feature; `labels` is a batch of labels. - features, labels = iterator.get_next() - return features, labels -``` diff --git a/tensorflow/docs_src/programmers_guide/datasets_for_estimators.md b/tensorflow/docs_src/programmers_guide/datasets_for_estimators.md deleted file mode 100644 index 345a31b985..0000000000 --- a/tensorflow/docs_src/programmers_guide/datasets_for_estimators.md +++ /dev/null @@ -1,387 +0,0 @@ -# Datasets for Estimators - -The @{tf.data} module contains a collection of classes that allows you to -easily load data, manipulate it, and pipe it into your model. This document -introduces the API by walking through two simple examples: - -* Reading in-memory data from numpy arrays. -* Reading lines from a csv file. - - - -## Basic input - -Taking slices from an array is the simplest way to get started with `tf.data`. - -The @{$premade_estimators$Premade Estimators} chapter describes -the following `train_input_fn`, from -[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py), -to pipe the data into the Estimator: - -``` python -def train_input_fn(features, labels, batch_size): - """An input function for training""" - # Convert the inputs to a Dataset. - dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) - - # Shuffle, repeat, and batch the examples. - dataset = dataset.shuffle(1000).repeat().batch(batch_size) - - # Return the dataset. - return dataset -``` - -Let's look at this more closely. - -### Arguments - -This function expects three arguments. Arguments expecting an "array" can -accept nearly anything that can be converted to an array with `numpy.array`. -One exception is -[`tuple`](https://docs.python.org/3/tutorial/datastructures.html#tuples-and-sequences) -which, as we will see, has special meaning for `Datasets`. - -* `features`: A `{'feature_name':array}` dictionary (or - [`DataFrame`](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)) - containing the raw input features. -* `labels` : An array containing the - [label](https://developers.google.com/machine-learning/glossary/#label) - for each example. -* `batch_size` : An integer indicating the desired batch size. - -In [`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py) -we retrieved the Iris data using the `iris_data.load_data()` function. -You can run it, and unpack the results as follows: - -``` python -import iris_data - -# Fetch the data -train, test = iris_data.load_data() -features, labels = train -``` - -Then we passed this data to the input function, with a line similar to this: - -``` python -batch_size=100 -iris_data.train_input_fn(features, labels, batch_size) -``` - -Let's walk through the `train_input_fn()`. - -### Slices - -The function starts by using the @{tf.data.Dataset.from_tensor_slices} function -to create a @{tf.data.Dataset} representing slices of the array. The array is -sliced across the first dimension. For example, an array containing the -@{$tutorials/layers$mnist training data} has a shape of `(60000, 28, 28)`. -Passing this to `from_tensor_slices` returns a `Dataset` object containing -60000 slices, each one a 28x28 image. - -The code that returns this `Dataset` is as follows: - -``` python -train, test = tf.keras.datasets.mnist.load_data() -mnist_x, mnist_y = train - -mnist_ds = tf.data.Dataset.from_tensor_slices(mnist_x) -print(mnist_ds) -``` - -This will print the following line, showing the -@{$programmers_guide/tensors#shapes$shapes} and -@{$programmers_guide/tensors#data_types$types} of the items in -the dataset. Note that a `Dataset` does not know how many items it contains. - -``` None - -``` - -The `Dataset` above represents a simple collection of arrays, but datasets are -much more powerful than this. A `Dataset` can transparently handle any nested -combination of dictionaries or tuples (or -[`namedtuple`](https://docs.python.org/2/library/collections.html#collections.namedtuple) -). - -For example after converting the iris `features` -to a standard python dictionary, you can then convert the dictionary of arrays -to a `Dataset` of dictionaries as follows: - -``` python -dataset = tf.data.Dataset.from_tensor_slices(dict(features)) -print(dataset) -``` -``` None - -``` - -Here we see that when a `Dataset` contains structured elements, the `shapes` -and `types` of the `Dataset` take on the same structure. This dataset contains -dictionaries of @{$programmers_guide/tensors#rank$scalars}, all of type -`tf.float64`. - -The first line of the iris `train_input_fn` uses the same functionality, but -adds another level of structure. It creates a dataset containing -`(features_dict, label)` pairs. - -The following code shows that the label is a scalar with type `int64`: - -``` python -# Convert the inputs to a Dataset. -dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) -print(dataset) -``` -``` - -``` - -### Manipulation - -Currently the `Dataset` would iterate over the data once, in a fixed order, and -only produce a single element at a time. It needs further processing before it -can be used for training. Fortunately, the `tf.data.Dataset` class provides -methods to better prepare the data for training. The next line of the input -function takes advantage of several of these methods: - -``` python -# Shuffle, repeat, and batch the examples. -dataset = dataset.shuffle(1000).repeat().batch(batch_size) -``` - -The @{tf.data.Dataset.shuffle$`shuffle`} method uses a fixed-size buffer to -shuffle the items as they pass through. In this case the `buffer_size` is -greater than the number of examples in the `Dataset`, ensuring that the data is -completely shuffled (The Iris data set only contains 150 examples). - -The @{tf.data.Dataset.repeat$`repeat`} method restarts the `Dataset` when -it reaches the end. To limit the number of epochs, set the `count` argument. - -The @{tf.data.Dataset.batch$`batch`} method collects a number of examples and -stacks them, to create batches. This adds a dimension to their shape. The new -dimension is added as the first dimension. The following code uses -the `batch` method on the MNIST `Dataset`, from earlier. This results in a -`Dataset` containing 3D arrays representing stacks of `(28,28)` images: - -``` python -print(mnist_ds.batch(100)) -``` - -``` none - -``` -Note that the dataset has an unknown batch size because the last batch will -have fewer elements. - -In `train_input_fn`, after batching the `Dataset` contains 1D vectors of -elements where each scalar was previously: - -```python -print(dataset) -``` -``` - -``` - - -### Return - -At this point the `Dataset` contains `(features_dict, labels)` pairs. -This is the format expected by the `train` and `evaluate` methods, so the -`input_fn` returns the dataset. - -The `labels` can/should be omitted when using the `predict` method. - - - - -## Reading a CSV File - -The most common real-world use case for the `Dataset` class is to stream data -from files on disk. The @{tf.data} module includes a variety of -file readers. Let's see how parsing the Iris dataset from the csv file looks -using a `Dataset`. - -The following call to the `iris_data.maybe_download` function downloads the -data if necessary, and returns the pathnames of the resulting files: - -``` python -import iris_data -train_path, test_path = iris_data.maybe_download() -``` - -The [`iris_data.csv_input_fn`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py) -function contains an alternative implementation that parses the csv files using -a `Dataset`. - -Let's look at how to build an Estimator-compatible input function that reads -from the local files. - -### Build the `Dataset` - -We start by building a @{tf.data.TextLineDataset$`TextLineDataset`} object to -read the file one line at a time. Then, we call the -@{tf.data.Dataset.skip$`skip`} method to skip over the first line of the file, which contains a header, not an example: - -``` python -ds = tf.data.TextLineDataset(train_path).skip(1) -``` - -### Build a csv line parser - -We will start by building a function to parse a single line. - -The following `iris_data.parse_line` function accomplishes this task using the -@{tf.decode_csv} function, and some simple python code: - -We must parse each of the lines in the dataset in order to generate the -necessary `(features, label)` pairs. The following `_parse_line` function -calls @{tf.decode_csv} to parse a single line into its features -and the label. Since Estimators require that features be represented as a -dictionary, we rely on Python's built-in `dict` and `zip` functions to build -that dictionary. The feature names are the keys of that dictionary. -We then call the dictionary's `pop` method to remove the label field from -the features dictionary: - -``` python -# Metadata describing the text columns -COLUMNS = ['SepalLength', 'SepalWidth', - 'PetalLength', 'PetalWidth', - 'label'] -FIELD_DEFAULTS = [[0.0], [0.0], [0.0], [0.0], [0]] -def _parse_line(line): - # Decode the line into its fields - fields = tf.decode_csv(line, FIELD_DEFAULTS) - - # Pack the result into a dictionary - features = dict(zip(COLUMNS,fields)) - - # Separate the label from the features - label = features.pop('label') - - return features, label -``` - -### Parse the lines - -Datasets have many methods for manipulating the data while it is being piped -to a model. The most heavily-used method is @{tf.data.Dataset.map$`map`}, which -applies a transformation to each element of the `Dataset`. - -The `map` method takes a `map_func` argument that describes how each item in the -`Dataset` should be transformed. - -
- -
-
-The @{tf.data.Dataset.map$`map`} method applies the `map_func` to -transform each item in the Dataset. -
- -So to parse the lines as they are streamed out of the csv file, we pass our -`_parse_line` function to the `map` method: - -``` python -ds = ds.map(_parse_line) -print(ds) -``` -``` None - -``` - -Now instead of simple scalar strings, the dataset contains `(features, label)` -pairs. - -the remainder of the `iris_data.csv_input_fn` function is identical -to `iris_data.train_input_fn` which was covered in the in the -[Basic input](#basic_input) section. - -### Try it out - -This function can be used as a replacement for -`iris_data.train_input_fn`. It can be used to feed an estimator as follows: - -``` python -train_path, test_path = iris_data.maybe_download() - -# All the inputs are numeric -feature_columns = [ - tf.feature_column.numeric_column(name) - for name in iris_data.CSV_COLUMN_NAMES[:-1]] - -# Build the estimator -est = tf.estimator.LinearClassifier(feature_columns, - n_classes=3) -# Train the estimator -batch_size = 100 -est.train( - steps=1000, - input_fn=lambda : iris_data.csv_input_fn(train_path, batch_size)) -``` - -Estimators expect an `input_fn` to take no arguments. To work around this -restriction, we use `lambda` to capture the arguments and provide the expected -interface. - -## Summary - -The `tf.data` module provides a collection of classes and functions for easily -reading data from a variety of sources. Furthermore, `tf.data` has simple -powerful methods for applying a wide variety of standard and custom -transformations. - -Now you have the basic idea of how to efficiently load data into an -Estimator. Consider the following documents next: - - -* @{$custom_estimators}, which demonstrates how to build your own - custom `Estimator` model. -* The @{$low_level_intro#datasets$Low Level Introduction}, which demonstrates - how to experiment directly with `tf.data.Datasets` using TensorFlow's low - level APIs. -* @{$programmers_guide/datasets} which goes into great detail about additional - functionality of `Datasets`. - diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md deleted file mode 100644 index 6bd941886d..0000000000 --- a/tensorflow/docs_src/programmers_guide/debugger.md +++ /dev/null @@ -1,804 +0,0 @@ -# TensorFlow Debugger - - - -[TOC] - -`tfdbg` is a specialized debugger for TensorFlow. It lets you view the internal -structure and states of running TensorFlow graphs during training and inference, -which is difficult to debug with general-purpose debuggers such as Python's `pdb` -due to TensorFlow's computation-graph paradigm. - -This guide focuses on the command-line interface (CLI) of `tfdbg`. For guide on -how to use the graphical user interface (GUI) of tfdbg, i.e., the -**TensorBoard Debugger Plugin**, please visit -[its README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md). - -Note: The TensorFlow debugger uses a -[curses](https://en.wikipedia.org/wiki/Curses_\(programming_library\))-based text -user interface. On Mac OS X, the `ncurses` library is required and can be -installed with `brew install homebrew/dupes/ncurses`. On Windows, curses isn't as -well supported, so a [readline](https://en.wikipedia.org/wiki/GNU_Readline)-based -interface can be used with tfdbg by installing `pyreadline` with `pip`. If you -use Anaconda3, you can install it with a command such as -`"C:\Program Files\Anaconda3\Scripts\pip.exe" install pyreadline`. Unofficial -Windows curses packages can be downloaded -[here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#curses), then subsequently -installed using `pip install .whl`, however curses on Windows may -not work as reliably as curses on Linux or Mac. - -This tutorial demonstrates how to use the **tfdbg** CLI to debug the appearance -of [`nan`s](https://en.wikipedia.org/wiki/NaN) -and [`inf`s](https://en.wikipedia.org/wiki/Infinity), a frequently-encountered -type of bug in TensorFlow model development. -The following example is for users who use the low-level -[`Session`](https://www.tensorflow.org/api_docs/python/tf/Session) API of -TensorFlow. A later section of this document describes how to use **tfdbg** -with a higher-level API, namely `Estimator`s. -To *observe* such an issue, run the following command without the debugger (the -source code can be found -[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/debug/examples/debug_mnist.py)): - -```none -python -m tensorflow.python.debug.examples.debug_mnist -``` - -This code trains a simple neural network for MNIST digit image recognition. -Notice that the accuracy increases slightly after the first training step, but -then gets stuck at a low (near-chance) level: - -```none -Accuracy at step 0: 0.1113 -Accuracy at step 1: 0.3183 -Accuracy at step 2: 0.098 -Accuracy at step 3: 0.098 -Accuracy at step 4: 0.098 -``` - -Wondering what might have gone wrong, you suspect that certain nodes in the -training graph generated bad numeric values such as `inf`s and `nan`s, because -this is a common cause of this type of training failure. -Let's use tfdbg to debug this issue and pinpoint the exact graph node where this -numeric problem first surfaced. - -## Wrapping TensorFlow Sessions with tfdbg - -To add support for tfdbg in our example, all that is needed is to add the -following lines of code and wrap the Session object with a debugger wrapper. -This code is already added in -[debug_mnist.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/debug/examples/debug_mnist.py), -so you can activate tfdbg CLI with the `--debug` flag at the command line. - -```python -# Let your BUILD target depend on "//tensorflow/python/debug:debug_py" -# (You don't need to worry about the BUILD dependency if you are using a pip -# install of open-source TensorFlow.) -from tensorflow.python import debug as tf_debug - -sess = tf_debug.LocalCLIDebugWrapperSession(sess) -``` - -This wrapper has the same interface as Session, so enabling debugging requires -no other changes to the code. The wrapper provides additional features, -including: - -* Bringing up a CLI before and after `Session.run()` calls, to let you -control the execution and inspect the graph's internal state. -* Allowing you to register special `filters` for tensor values, to facilitate -the diagnosis of issues. - -In this example, we have already registered a tensor filter called -@{tfdbg.has_inf_or_nan}, -which simply determines if there are any `nan` or `inf` values in any -intermediate tensors (tensors that are neither inputs or outputs of the -`Session.run()` call, but are in the path leading from the inputs to the -outputs). This filter is for `nan`s and `inf`s is a common enough use case that -we ship it with the -@{$python/tfdbg#Classes_for_debug_dump_data_and_directories$`debug_data`} -module. - -Note: You can also write your own custom filters. See -the @{tfdbg.DebugDumpDir.find$API documentation} -of `DebugDumpDir.find()` for additional information. - -## Debugging Model Training with tfdbg - - -Let's try training the model again, but with the `--debug` flag added this time: - -```none -python -m tensorflow.python.debug.examples.debug_mnist --debug -``` - -The debug wrapper session will prompt you when it is about to execute the first -`Session.run()` call, with information regarding the fetched tensor and feed -dictionaries displayed on the screen. - -![tfdbg run-start UI](https://www.tensorflow.org/images/tfdbg_screenshot_run_start.png) - -This is what we refer to as the *run-start CLI*. It lists the feeds and fetches -to the current `Session.run` call, before executing anything. - -If the screen size is too small to display the content of the message in its -entirety, you can resize it. - -Use the **PageUp** / **PageDown** / **Home** / **End** keys to navigate the -screen output. On most keyboards lacking those keys **Fn + Up** / -**Fn + Down** / **Fn + Right** / **Fn + Left** will work. - -Enter the `run` command (or just `r`) at the command prompt: - -``` -tfdbg> run -``` - -The `run` command causes tfdbg to execute until the end of the next -`Session.run()` call, which calculates the model's accuracy using a test data -set. tfdbg augments the runtime Graph to dump all intermediate tensors. -After the run ends, tfdbg displays all the dumped tensors values in the -*run-end CLI*. For example: - -![tfdbg run-end UI: accuracy](https://www.tensorflow.org/images/tfdbg_screenshot_run_end_accuracy.png) - -This list of tensors can also be obtained by running the command `lt` after you -executed `run`. - -### tfdbg CLI Frequently-Used Commands - -Try the following commands at the `tfdbg>` prompt (referencing the code at -`tensorflow/python/debug/examples/debug_mnist.py`): - -| Command | Syntax or Option | Explanation | Example | -|:-------------------|:---------------- |:------------ |:------------------------- | -| **`lt`** | | **List dumped tensors.** | `lt` | -| | `-n ` | List dumped tensors with names matching given regular-expression pattern. | `lt -n Softmax.*` | -| | `-t ` | List dumped tensors with op types matching given regular-expression pattern. | `lt -t MatMul` | -| | `-f ` | List only the tensors that pass a registered tensor filter. | `lt -f has_inf_or_nan` | -| | `-f -fenn ` | List only the tensors that pass a registered tensor filter, excluding nodes with names matching the regular expression. | `lt -f has_inf_or_nan` `-fenn .*Sqrt.*` | -| | `-s ` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` | -| | `-r` | Sort in reverse order. | `lt -r -s dump_size` | -| **`pt`** | | **Print value of a dumped tensor.** | | -| | `pt ` | Print tensor value. | `pt hidden/Relu:0` | -| | `pt [slicing]` | Print a subarray of tensor, using [numpy](http://www.numpy.org/)-style array slicing. | `pt hidden/Relu:0[0:50,:]` | -| | `-a` | Print the entirety of a large tensor, without using ellipses. (May take a long time for large tensors.) | `pt -a hidden/Relu:0[0:50,:]` | -| | `-r ` | Highlight elements falling into specified numerical range. Multiple ranges can be used in conjunction. | `pt hidden/Relu:0 -a -r [[-inf,-1],[1,inf]]` | -| | `-n ` | Print dump corresponding to specified 0-based dump number. Required for tensors with multiple dumps. | `pt -n 0 hidden/Relu:0` | -| | `-s` | Include a summary of the numeric values of the tensor (applicable only to non-empty tensors with Boolean and numeric types such as `int*` and `float*`.) | `pt -s hidden/Relu:0[0:50,:]` | -| | `-w` | Write the value of the tensor (possibly sliced) to a Numpy file using [`numpy.save()`](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.save.html) | `pt -s hidden/Relu:0 -w /tmp/relu.npy` | -| **`@[coordinates]`** | | Navigate to specified element in `pt` output. | `@[10,0]` or `@10,0` | -| **`/regex`** | | [less](https://linux.die.net/man/1/less)-style search for given regular expression. | `/inf` | -| **`/`** | | Scroll to the next line with matches to the searched regex (if any). | `/` | -| **`pf`** | | **Print a value in the feed_dict to `Session.run`.** | | -| | `pf ` | Print the value of the feed. Also note that the `pf` command has the `-a`, `-r` and `-s` flags (not listed below), which have the same syntax and semantics as the identically-named flags of `pt`. | `pf input_xs:0` | -| **eval** | | **Evaluate arbitrary Python and numpy expression.** | | -| | `eval ` | Evaluate a Python / numpy expression, with numpy available as `np` and debug tensor names enclosed in backticks. | ``eval "np.matmul((`output/Identity:0` / `Softmax:0`).T, `Softmax:0`)"`` | -| | `-a` | Print a large-sized evaluation result in its entirety, i.e., without using ellipses. | ``eval -a 'np.sum(`Softmax:0`, axis=1)'`` | -| | `-w` | Write the result of the evaluation to a Numpy file using [`numpy.save()`](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.save.html) | ``eval -a 'np.sum(`Softmax:0`, axis=1)' -w /tmp/softmax_sum.npy`` | -| **`ni`** | | **Display node information.** | | -| | `-a` | Include node attributes in the output. | `ni -a hidden/Relu` | -| | `-d` | List the debug dumps available from the node. | `ni -d hidden/Relu` | -| | `-t` | Display the Python stack trace of the node's creation. | `ni -t hidden/Relu` | -| **`li`** | | **List inputs to node** | | -| | `-r` | List the inputs to node, recursively (the input tree.) | `li -r hidden/Relu:0` | -| | `-d ` | Limit recursion depth under the `-r` mode. | `li -r -d 3 hidden/Relu:0` | -| | `-c` | Include control inputs. | `li -c -r hidden/Relu:0` | -| | `-t` | Show op types of input nodes. | `li -t -r hidden/Relu:0` | -| **`lo`** | | **List output recipients of node** | | -| | `-r` | List the output recipients of node, recursively (the output tree.) | `lo -r hidden/Relu:0` | -| | `-d ` | Limit recursion depth under the `-r` mode. | `lo -r -d 3 hidden/Relu:0` | -| | `-c` | Include recipients via control edges. | `lo -c -r hidden/Relu:0` | -| | `-t` | Show op types of recipient nodes. | `lo -t -r hidden/Relu:0` | -| **`ls`** | | **List Python source files involved in node creation.** | | -| | `-p ` | Limit output to source files matching given regular-expression path pattern. | `ls -p .*debug_mnist.*` | -| | `-n` | Limit output to node names matching given regular-expression pattern. | `ls -n Softmax.*` | -| **`ps`** | | **Print Python source file.** | | -| | `ps ` | Print given Python source file source.py, with the lines annotated with the nodes created at each of them (if any). | `ps /path/to/source.py` | -| | `-t` | Perform annotation with respect to Tensors, instead of the default, nodes. | `ps -t /path/to/source.py` | -| | `-b ` | Annotate source.py beginning at given line. | `ps -b 30 /path/to/source.py` | -| | `-m ` | Limit the number of elements in the annotation for each line. | `ps -m 100 /path/to/source.py` | -| **`run`** | | **Proceed to the next Session.run()** | `run` | -| | `-n` | Execute through the next `Session.run` without debugging, and drop to CLI right before the run after that. | `run -n` | -| | `-t ` | Execute `Session.run` `T - 1` times without debugging, followed by a run with debugging. Then drop to CLI right after the debugged run. | `run -t 10` | -| | `-f ` | Continue executing `Session.run` until any intermediate tensor triggers the specified Tensor filter (causes the filter to return `True`). | `run -f has_inf_or_nan` | -| | `-f -fenn ` | Continue executing `Session.run` until any intermediate tensor whose node names doesn't match the regular expression triggers the specified Tensor filter (causes the filter to return `True`). | `run -f has_inf_or_nan -fenn .*Sqrt.*` | -| | `--node_name_filter ` | Execute the next `Session.run`, watching only nodes with names matching the given regular-expression pattern. | `run --node_name_filter Softmax.*` | -| | `--op_type_filter ` | Execute the next `Session.run`, watching only nodes with op types matching the given regular-expression pattern. | `run --op_type_filter Variable.*` | -| | `--tensor_dtype_filter ` | Execute the next `Session.run`, dumping only Tensors with data types (`dtype`s) matching the given regular-expression pattern. | `run --tensor_dtype_filter int.*` | -| | `-p` | Execute the next `Session.run` call in profiling mode. | `run -p` | -| **`ri`** | | **Display information about the run the current run, including fetches and feeds.** | `ri` | -| **`config`** | | **Set or show persistent TFDBG UI configuration.** | | -| | `set` | Set the value of a config item: {`graph_recursion_depth`, `mouse_mode`}. | `config set graph_recursion_depth 3` | -| | `show` | Show current persistent UI configuration. | `config show` | -| **`help`** | | **Print general help information** | `help` | -| | `help ` | Print help for given command. | `help lt` | - -Note that each time you enter a command, a new screen output -will appear. This is somewhat analogous to web pages in a browser. You can -navigate between these screens by clicking the `<--` and -`-->` text arrows near the top-left corner of the CLI. - -### Other Features of the tfdbg CLI - -In addition to the commands listed above, the tfdbg CLI provides the following -additional features: - -* To navigate through previous tfdbg commands, type in a few characters - followed by the Up or Down arrow keys. tfdbg will show you the history of - commands that started with those characters. -* To navigate through the history of screen outputs, do either of the - following: - * Use the `prev` and `next` commands. - * Click underlined `<--` and `-->` links near the top left corner of the - screen. -* Tab completion of commands and some command arguments. -* To redirect the screen output to a file instead of the screen, end the - command with bash-style redirection. For example, the following command - redirects the output of the pt command to the `/tmp/xent_value_slices.txt` - file: - - ```none - tfdbg> pt cross_entropy/Log:0[:, 0:10] > /tmp/xent_value_slices.txt - ``` - -### Finding `nan`s and `inf`s - -In this first `Session.run()` call, there happen to be no problematic numerical -values. You can move on to the next run by using the command `run` or its -shorthand `r`. - -> TIP: If you enter `run` or `r` repeatedly, you will be able to move through -> the `Session.run()` calls in a sequential manner. -> -> You can also use the `-t` flag to move ahead a number of `Session.run()` calls -> at a time, for example: -> -> ``` -> tfdbg> run -t 10 -> ``` - -Instead of entering `run` repeatedly and manually searching for `nan`s and -`inf`s in the run-end UI after every `Session.run()` call (for example, by using -the `pt` command shown in the table above) , you can use the following -command to let the debugger repeatedly execute `Session.run()` calls without -stopping at the run-start or run-end prompt, until the first `nan` or `inf` -value shows up in the graph. This is analogous to *conditional breakpoints* in -some procedural-language debuggers: - -```none -tfdbg> run -f has_inf_or_nan -``` - -> NOTE: The preceding command works properly because a tensor filter called -> `has_inf_or_nan` has been registered for you when the wrapped session is -> created. This filter detects `nan`s and `inf`s (as explained previously). -> If you have registered any other filters, you can -> use "run -f" to have tfdbg run until any tensor triggers that filter (cause -> the filter to return True). -> -> ``` python -> def my_filter_callable(datum, tensor): -> # A filter that detects zero-valued scalars. -> return len(tensor.shape) == 0 and tensor == 0.0 -> -> sess.add_tensor_filter('my_filter', my_filter_callable) -> ``` -> -> Then at the tfdbg run-start prompt run until your filter is triggered: -> -> ``` -> tfdbg> run -f my_filter -> ``` - -See [this API document](https://www.tensorflow.org/api_docs/python/tfdbg/DebugDumpDir#find) -for more information on the expected signature and return value of the predicate -`Callable` used with `add_tensor_filter()`. - -![tfdbg run-end UI: infs and nans](https://www.tensorflow.org/images/tfdbg_screenshot_run_end_inf_nan.png) - -As the screen display indicates on the first line, the `has_inf_or_nan` filter is first triggered -during the fourth `Session.run()` call: an -[Adam optimizer](https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer) -forward-backward training pass on the graph. In this run, 36 (out of the total -95) intermediate tensors contain `nan` or `inf` values. These tensors are listed -in chronological order, with their timestamps displayed on the left. At the top -of the list, you can see the first tensor in which the bad numerical values -first surfaced: `cross_entropy/Log:0`. - -To view the value of the tensor, click the underlined tensor name -`cross_entropy/Log:0` or enter the equivalent command: - -```none -tfdbg> pt cross_entropy/Log:0 -``` - -Scroll down a little and you will notice some scattered `inf` values. If the -instances of `inf` and `nan` are difficult to spot by eye, you can use the -following command to perform a regex search and highlight the output: - -```none -tfdbg> /inf -``` - -Or, alternatively: - -```none -tfdbg> /(inf|nan) -``` - -You can also use the `-s` or `--numeric_summary` command to get a quick summary -of the types of numeric values in the tensor: - -``` none -tfdbg> pt -s cross_entropy/Log:0 -``` - -From the summary, you can see that several of the 1000 elements of the -`cross_entropy/Log:0` tensor are `-inf`s (negative infinities). - -Why did these infinities appear? To further debug, display more information -about the node `cross_entropy/Log` by clicking the underlined `node_info` menu -item on the top or entering the equivalent node_info (`ni`) command: - -```none -tfdbg> ni cross_entropy/Log -``` - -![tfdbg run-end UI: infs and nans](https://www.tensorflow.org/images/tfdbg_screenshot_run_end_node_info.png) - -You can see that this node has the op type `Log` -and that its input is the node `Softmax`. Run the following command to -take a closer look at the input tensor: - -```none -tfdbg> pt Softmax:0 -``` - -Examine the values in the input tensor, searching for zeros: - -```none -tfdbg> /0\.000 -``` - -Indeed, there are zeros. Now it is clear that the origin of the bad numerical -values is the node `cross_entropy/Log` taking logs of zeros. To find out the -culprit line in the Python source code, use the `-t` flag of the `ni` command -to show the traceback of the node's construction: - -```none -tfdbg> ni -t cross_entropy/Log -``` - -If you click "node_info" at the top of the screen, tfdbg automatically shows the -traceback of the node's construction. - -From the traceback, you can see that the op is constructed at the following -line: -[`debug_mnist.py`](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_mnist.py): - -```python -diff = y_ * tf.log(y) -``` - -**tfdbg** has a feature that makes it easy to trace Tensors and ops back to -lines in Python source files. It can annotate lines of a Python file with -the ops or Tensors created by them. To use this feature, -simply click the underlined line numbers in the stack trace output of the -`ni -t ` commands, or use the `ps` (or `print_source`) command such as: -`ps /path/to/source.py`. For example, the following screenshot shows the output -of a `ps` command. - -![tfdbg run-end UI: annotated Python source file](https://www.tensorflow.org/images/tfdbg_screenshot_run_end_annotated_source.png) - -### Fixing the problem - -To fix the problem, edit `debug_mnist.py`, changing the original line: - -```python -diff = -(y_ * tf.log(y)) -``` - -to the built-in, numerically-stable implementation of softmax cross-entropy: - -```python -diff = tf.losses.softmax_cross_entropy(labels=y_, logits=logits) -``` - -Rerun with the `--debug` flag as follows: - -```none -python -m tensorflow.python.debug.examples.debug_mnist --debug -``` - -At the `tfdbg>` prompt, enter the following command: - -```none -run -f has_inf_or_nan` -``` - -Confirm that no tensors are flagged as containing `nan` or `inf` values, and -accuracy now continues to rise rather than getting stuck. Success! - -## Debugging TensorFlow Estimators - -This section explains how to debug TensorFlow programs that use the `Estimator` -APIs. Part of the convenience provided by these APIs is that -they manage `Session`s internally. This makes the `LocalCLIDebugWrapperSession` -described in the preceding sections inapplicable. Fortunately, you can still -debug them by using special `hook`s provided by `tfdbg`. - -`tfdbg` can debug the -@{tf.estimator.Estimator.train$`train()`}, -@{tf.estimator.Estimator.evaluate$`evaluate()`} and -@{tf.estimator.Estimator.predict$`predict()`} -methods of tf-learn `Estimator`s. To debug `Estimator.train()`, -create a `LocalCLIDebugHook` and supply it in the `hooks` argument. For example: - -```python -# First, let your BUILD target depend on "//tensorflow/python/debug:debug_py" -# (You don't need to worry about the BUILD dependency if you are using a pip -# install of open-source TensorFlow.) -from tensorflow.python import debug as tf_debug - -# Create a LocalCLIDebugHook and use it as a monitor when calling fit(). -hooks = [tf_debug.LocalCLIDebugHook()] - -# To debug `train`: -classifier.train(input_fn, - steps=1000, - hooks=hooks) -``` - -Similarly, to debug `Estimator.evaluate()` and `Estimator.predict()`, assign -hooks to the `hooks` parameter, as in the following example: - -```python -# To debug `evaluate`: -accuracy_score = classifier.evaluate(eval_input_fn, - hooks=hooks)["accuracy"] - -# To debug `predict`: -predict_results = classifier.predict(predict_input_fn, hooks=hooks) -``` - -[debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_tflearn_iris.py), -based on [tf-learn's iris tutorial](https://www.tensorflow.org/versions/r1.8/get_started/tflearn), -contains a full example of how to use the tfdbg with `Estimator`s. -To run this example, do: - -```none -python -m tensorflow.python.debug.examples.debug_tflearn_iris --debug -``` - -The `LocalCLIDebugHook` also allows you to configure a `watch_fn` that can be -used to flexibly specify what `Tensor`s to watch on different `Session.run()` -calls, as a function of the `fetches` and `feed_dict` and other states. See -@{tfdbg.DumpingDebugWrapperSession.__init__$this API doc} -for more details. - -## Debugging Keras Models with TFDBG - -To use TFDBG with [Keras](https://keras.io/), let the Keras backend use -a TFDBG-wrapped Session object. For example, to use the CLI wrapper: - -``` python -import tensorflow as tf -from keras import backend as keras_backend -from tensorflow.python import debug as tf_debug - -keras_backend.set_session(tf_debug.LocalCLIDebugWrapperSession(tf.Session())) - -# Define your keras model, called "model". -model.fit(...) # This will break into the TFDBG CLI. -``` - -## Debugging tf-slim with TFDBG - -TFDBG supports debugging of training and evaluation with -[tf-slim](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/slim). -As detailed below, training and evaluation require slightly different debugging -workflows. - -### Debugging training in tf-slim -To debug the training process, provide `LocalCLIDebugWrapperSession` to the -`session_wrapper` argument of `slim.learning.train()`. For example: - -``` python -import tensorflow as tf -from tensorflow.python import debug as tf_debug - -# ... Code that creates the graph and the train_op ... -tf.contrib.slim.learning.train( - train_op, - logdir, - number_of_steps=10, - session_wrapper=tf_debug.LocalCLIDebugWrapperSession) -``` - -### Debugging evaluation in tf-slim -To debug the evaluation process, provide `LocalCLIDebugHook` to the -`hooks` argument of `slim.evaluation.evaluate_once()`. For example: - -``` python -import tensorflow as tf -from tensorflow.python import debug as tf_debug - -# ... Code that creates the graph and the eval and final ops ... -tf.contrib.slim.evaluation.evaluate_once( - '', - checkpoint_path, - logdir, - eval_op=my_eval_op, - final_op=my_value_op, - hooks=[tf_debug.LocalCLIDebugHook()]) -``` - -## Offline Debugging of Remotely-Running Sessions - -Often, your model is running on a remote machine or a process that you don't -have terminal access to. To perform model debugging in such cases, you can use -the `offline_analyzer` binary of `tfdbg` (described below). It operates on -dumped data directories. This can be done to both the lower-level `Session` API -and the higher-level `Estimator` API. - -### Debugging Remote tf.Sessions - -If you interact directly with the `tf.Session` API in `python`, you can -configure the `RunOptions` proto that you call your `Session.run()` method -with, by using the method @{tfdbg.watch_graph}. -This will cause the intermediate tensors and runtime graphs to be dumped to a -shared storage location of your choice when the `Session.run()` call occurs -(at the cost of slower performance). For example: - -```python -from tensorflow.python import debug as tf_debug - -# ... Code where your session and graph are set up... - -run_options = tf.RunOptions() -tf_debug.watch_graph( - run_options, - session.graph, - debug_urls=["file:///shared/storage/location/tfdbg_dumps_1"]) -# Be sure to specify different directories for different run() calls. - -session.run(fetches, feed_dict=feeds, options=run_options) -``` - -Later, in an environment that you have terminal access to (for example, a local -computer that can access the shared storage location specified in the code -above), you can load and inspect the data in the dump directory on the shared -storage by using the `offline_analyzer` binary of `tfdbg`. For example: - -```none -python -m tensorflow.python.debug.cli.offline_analyzer \ - --dump_dir=/shared/storage/location/tfdbg_dumps_1 -``` - -The `Session` wrapper `DumpingDebugWrapperSession` offers an easier and more -flexible way to generate file-system dumps that can be analyzed offline. -To use it, simply wrap your session in a `tf_debug.DumpingDebugWrapperSession`. -For example: - -```python -# Let your BUILD target depend on "//tensorflow/python/debug:debug_py -# (You don't need to worry about the BUILD dependency if you are using a pip -# install of open-source TensorFlow.) -from tensorflow.python import debug as tf_debug - -sess = tf_debug.DumpingDebugWrapperSession( - sess, "/shared/storage/location/tfdbg_dumps_1/", watch_fn=my_watch_fn) -``` - -The `watch_fn` argument accepts a `Callable` that allows you to configure what -`tensor`s to watch on different `Session.run()` calls, as a function of the -`fetches` and `feed_dict` to the `run()` call and other states. - -### C++ and other languages - -If your model code is written in C++ or other languages, you can also -modify the `debug_options` field of `RunOptions` to generate debug dumps that -can be inspected offline. See -[the proto definition](https://www.tensorflow.org/code/tensorflow/core/protobuf/debug.proto) -for more details. - -### Debugging Remotely-Running Estimators - -If your remote TensorFlow server runs `Estimator`s, -you can use the non-interactive `DumpingDebugHook`. For example: - -```python -# Let your BUILD target depend on "//tensorflow/python/debug:debug_py -# (You don't need to worry about the BUILD dependency if you are using a pip -# install of open-source TensorFlow.) -from tensorflow.python import debug as tf_debug - -hooks = [tf_debug.DumpingDebugHook("/shared/storage/location/tfdbg_dumps_1")] -``` - -Then this `hook` can be used in the same way as the `LocalCLIDebugHook` examples -described earlier in this document. -As the training, evalution or prediction happens with `Estimator`, -tfdbg creates directories having the following name pattern: -`/shared/storage/location/tfdbg_dumps_1/run__`. -Each directory corresponds to a `Session.run()` call that underlies -the `fit()` or `evaluate()` call. You can load these directories and inspect -them in a command-line interface in an offline manner using the -`offline_analyzer` offered by tfdbg. For example: - -```bash -python -m tensorflow.python.debug.cli.offline_analyzer \ - --dump_dir="/shared/storage/location/tfdbg_dumps_1/run__" -``` - -## Frequently Asked Questions - -**Q**: _Do the timestamps on the left side of the `lt` output reflect actual - performance in a non-debugging session?_ - -**A**: No. The debugger inserts additional special-purpose debug nodes to the - graph to record the values of intermediate tensors. These nodes - slow down the graph execution. If you are interested in profiling your - model, check out - - 1. The profiling mode of tfdbg: `tfdbg> run -p`. - 2. [tfprof](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core/profiler) - and other profiling tools for TensorFlow. - -**Q**: _How do I link tfdbg against my `Session` in Bazel? Why do I see an - error such as "ImportError: cannot import name debug"?_ - -**A**: In your BUILD rule, declare dependencies: - `"//tensorflow:tensorflow_py"` and `"//tensorflow/python/debug:debug_py"`. - The first is the dependency that you include to use TensorFlow even - without debugger support; the second enables the debugger. - Then, In your Python file, add: - -```python -from tensorflow.python import debug as tf_debug - -# Then wrap your TensorFlow Session with the local-CLI wrapper. -sess = tf_debug.LocalCLIDebugWrapperSession(sess) -``` - -**Q**: _Does tfdbg help debug runtime errors such as shape mismatches?_ - -**A**: Yes. tfdbg intercepts errors generated by ops during runtime and presents - the errors with some debug instructions to the user in the CLI. - See examples: - -```none -# Debugging shape mismatch during matrix multiplication. -python -m tensorflow.python.debug.examples.debug_errors \ - --error shape_mismatch --debug - -# Debugging uninitialized variable. -python -m tensorflow.python.debug.examples.debug_errors \ - --error uninitialized_variable --debug -``` - -**Q**: _How can I let my tfdbg-wrapped Sessions or Hooks run the debug mode -only from the main thread?_ - -**A**: -This is a common use case, in which the `Session` object is used from multiple -threads concurrently. Typically, the child threads take care of background tasks -such as running enqueue operations. Often, you want to debug only the main -thread (or less frequently, only one of the child threads). You can use the -`thread_name_filter` keyword argument of `LocalCLIDebugWrapperSession` to -achieve this type of thread-selective debugging. For example, to debug from the -main thread only, construct a wrapped `Session` as follows: - -```python -sess = tf_debug.LocalCLIDebugWrapperSession(sess, thread_name_filter="MainThread$") -``` - -The above example relies on the fact that main threads in Python have the -default name `MainThread`. - -**Q**: _The model I am debugging is very large. The data dumped by tfdbg -fills up the free space of my disk. What can I do?_ - -**A**: -You might encounter this problem in any of the following situations: - -* models with many intermediate tensors -* very large intermediate tensors -* many @{tf.while_loop} iterations - -There are three possible workarounds or solutions: - -* The constructors of `LocalCLIDebugWrapperSession` and `LocalCLIDebugHook` - provide a keyword argument, `dump_root`, to specify the path - to which tfdbg dumps the debug data. You can use it to let tfdbg dump the - debug data on a disk with larger free space. For example: - -```python -# For LocalCLIDebugWrapperSession -sess = tf_debug.LocalCLIDebugWrapperSession(dump_root="/with/lots/of/space") - -# For LocalCLIDebugHook -hooks = [tf_debug.LocalCLIDebugHook(dump_root="/with/lots/of/space")] -``` - Make sure that the directory pointed to by dump_root is empty or nonexistent. - `tfdbg` cleans up the dump directories before exiting. - -* Reduce the batch size used during the runs. -* Use the filtering options of tfdbg's `run` command to watch only specific - nodes in the graph. For example: - - ``` - tfdbg> run --node_name_filter .*hidden.* - tfdbg> run --op_type_filter Variable.* - tfdbg> run --tensor_dtype_filter int.* - ``` - - The first command above watches only nodes whose name match the - regular-expression pattern `.*hidden.*`. The second command watches only - operations whose name match the pattern `Variable.*`. The third one watches - only the tensors whose dtype match the pattern `int.*` (e.g., `int32`). - - -**Q**: _Why can't I select text in the tfdbg CLI?_ - -**A**: This is because the tfdbg CLI enables mouse events in the terminal by - default. This [mouse-mask](https://linux.die.net/man/3/mousemask) mode - overrides default terminal interactions, including text selection. You - can re-enable text selection by using the command `mouse off` or - `m off`. - -**Q**: _Why does the tfdbg CLI show no dumped tensors when I debug code like the following?_ - -``` python -a = tf.ones([10], name="a") -b = tf.add(a, a, name="b") -sess = tf.Session() -sess = tf_debug.LocalCLIDebugWrapperSession(sess) -sess.run(b) -``` - -**A**: The reason why you see no data dumped is because every node in the - executed TensorFlow graph is constant-folded by the TensorFlow runtime. - In this exapmle, `a` is a constant tensor; therefore, the fetched - tensor `b` is effectively also a constant tensor. TensorFlow's graph - optimization folds the graph that contains `a` and `b` into a single - node to speed up future runs of the graph, which is why `tfdbg` does - not generate any intermediate tensor dumps. However, if `a` were a - @{tf.Variable}, as in the following example: - -``` python -import numpy as np - -a = tf.Variable(np.ones[10], name="a") -b = tf.add(a, a, name="b") -sess = tf.Session() -sess.run(tf.global_variables_initializer()) -sess = tf_debug.LocalCLIDebugWrapperSession(sess) -sess.run(b) -``` - -the constant-folding would not occur and `tfdbg` should show the intermediate -tensor dumps. - - -**Q**: I am debugging a model that generates unwanted infinities or NaNs. But - there are some nodes in my model that are known to generate infinities - or NaNs in their output tensors even under completely normal conditions. - How can I skip those nodes during my `run -f has_inf_or_nan` actions? - -**A**: Use the `--filter_exclude_node_names` (`-fenn` for short) flag. For - example, if you known you have a node with name matching the regular - expression `.*Sqrt.*` that generates infinities or NaNs regardless - of whether the model is behaving correctly, you can exclude the nodes - from the infinity/NaN-finding runs with the command - `run -f has_inf_or_nan -fenn .*Sqrt.*`. - - -**Q**: Is there a GUI for tfdbg? - -**A**: Yes, the **TensorBoard Debugger Plugin** is the GUI of tfdbg. - It offers features such as inspection of the computation graph, - real-time visualization of tensor values, continuation to tensor - and conditional breakpoints, and tying tensors to their - graph-construction source code, all in the browser environment. - To get started, please visit - [its README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md). diff --git a/tensorflow/docs_src/programmers_guide/eager.md b/tensorflow/docs_src/programmers_guide/eager.md deleted file mode 100644 index 00d02b4455..0000000000 --- a/tensorflow/docs_src/programmers_guide/eager.md +++ /dev/null @@ -1,849 +0,0 @@ -# Eager Execution - -TensorFlow's eager execution is an imperative programming environment that -evaluates operations immediately, without building graphs: operations return -concrete values instead of constructing a computational graph to run later. This -makes it easy to get started with TensorFlow and debug models, and it -reduces boilerplate as well. To follow along with this guide, run the code -samples below in an interactive `python` interpreter. - -Eager execution is a flexible machine learning platform for research and -experimentation, providing: - -* *An intuitive interface*—Structure your code naturally and use Python data - structures. Quickly iterate on small models and small data. -* *Easier debugging*—Call ops directly to inspect running models and test - changes. Use standard Python debugging tools for immediate error reporting. -* *Natural control flow*—Use Python control flow instead of graph control - flow, simplifying the specification of dynamic models. - -Eager execution supports most TensorFlow operations and GPU acceleration. For a -collection of examples running in eager execution, see: -[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples). - -Note: Some models may experience increased overhead with eager execution -enabled. Performance improvements are ongoing, but please -[file a bug](https://github.com/tensorflow/tensorflow/issues) if you find a -problem and share your benchmarks. - -## Setup and basic usage - -Upgrade to the latest version of TensorFlow: - -``` -$ pip install --upgrade tensorflow -``` - -To start eager execution, add `tf.enable_eager_execution()` to the beginning of -the program or console session. Do not add this operation to other modules that -the program calls. - -```py -from __future__ import absolute_import, division, print_function - -import tensorflow as tf - -tf.enable_eager_execution() -``` - -Now you can run TensorFlow operations and the results will return immediately: - -```py -tf.executing_eagerly() # => True - -x = [[2.]] -m = tf.matmul(x, x) -print("hello, {}".format(m)) # => "hello, [[4.]]" -``` - -Enabling eager execution changes how TensorFlow operations behave—now they -immediately evaluate and return their values to Python. `tf.Tensor` objects -reference concrete values instead of symbolic handles to nodes in a computational -graph. Since there isn't a computational graph to build and run later in a -session, it's easy to inspect results using `print()` or a debugger. Evaluating, -printing, and checking tensor values does not break the flow for computing -gradients. - -Eager execution works nicely with [NumPy](http://www.numpy.org/). NumPy -operations accept `tf.Tensor` arguments. TensorFlow -[math operations](https://www.tensorflow.org/api_guides/python/math_ops) convert -Python objects and NumPy arrays to `tf.Tensor` objects. The -`tf.Tensor.numpy` method returns the object's value as a NumPy `ndarray`. - -```py -a = tf.constant([[1, 2], - [3, 4]]) -print(a) -# => tf.Tensor([[1 2] -# [3 4]], shape=(2, 2), dtype=int32) - -# Broadcasting support -b = tf.add(a, 1) -print(b) -# => tf.Tensor([[2 3] -# [4 5]], shape=(2, 2), dtype=int32) - -# Operator overloading is supported -print(a * b) -# => tf.Tensor([[ 2 6] -# [12 20]], shape=(2, 2), dtype=int32) - -# Use NumPy values -import numpy as np - -c = np.multiply(a, b) -print(c) -# => [[ 2 6] -# [12 20]] - -# Obtain numpy value from a tensor: -print(a.numpy()) -# => [[1 2] -# [3 4]] -``` - -The `tf.contrib.eager` module contains symbols available to both eager and graph execution -environments and is useful for writing code to [work with graphs](#work_with_graphs): - -```py -tfe = tf.contrib.eager -``` - -## Dynamic control flow - -A major benefit of eager execution is that all the functionality of the host -language is available while your model is executing. So, for example, -it is easy to write [fizzbuzz](https://en.wikipedia.org/wiki/Fizz_buzz): - -```py -def fizzbuzz(max_num): - counter = tf.constant(0) - max_num = tf.convert_to_tensor(max_num) - for num in range(max_num.numpy()): - num = tf.constant(num) - if int(num % 3) == 0 and int(num % 5) == 0: - print('FizzBuzz') - elif int(num % 3) == 0: - print('Fizz') - elif int(num % 5) == 0: - print('Buzz') - else: - print(num) - counter += 1 - return counter -``` - -This has conditionals that depend on tensor values and it prints these values -at runtime. - -## Build a model - -Many machine learning models are represented by composing layers. When -using TensorFlow with eager execution you can either write your own layers or -use a layer provided in the `tf.keras.layers` package. - -While you can use any Python object to represent a layer, -TensorFlow has `tf.keras.layers.Layer` as a convenient base class. Inherit from -it to implement your own layer: - -```py -class MySimpleLayer(tf.keras.layers.Layer): - def __init__(self, output_units): - self.output_units = output_units - - def build(self, input): - # The build method gets called the first time your layer is used. - # Creating variables on build() allows you to make their shape depend - # on the input shape and hence remove the need for the user to specify - # full shapes. It is possible to create variables during __init__() if - # you already know their full shapes. - self.kernel = self.add_variable( - "kernel", [input.shape[-1], self.output_units]) - - def call(self, input): - # Override call() instead of __call__ so we can perform some bookkeeping. - return tf.matmul(input, self.kernel) -``` - -Use `tf.keras.layers.Dense` layer instead of `MySimpleLayer` above as it has -a superset of its functionality (it can also add a bias). - -When composing layers into models you can use `tf.keras.Sequential` to represent -models which are a linear stack of layers. It is easy to use for basic models: - -```py -model = tf.keras.Sequential([ - tf.keras.layers.Dense(10, input_shape=(784,)), # must declare input shape - tf.keras.layers.Dense(10) -]) -``` - -Alternatively, organize models in classes by inheriting from `tf.keras.Model`. -This is a container for layers that is a layer itself, allowing `tf.keras.Model` -objects to contain other `tf.keras.Model` objects. - -```py -class MNISTModel(tf.keras.Model): - def __init__(self): - super(MNISTModel, self).__init__() - self.dense1 = tf.keras.layers.Dense(units=10) - self.dense2 = tf.keras.layers.Dense(units=10) - - def call(self, input): - """Run the model.""" - result = self.dense1(input) - result = self.dense2(result) - result = self.dense2(result) # reuse variables from dense2 layer - return result - -model = MNISTModel() -``` - -It's not required to set an input shape for the `tf.keras.Model` class since -the parameters are set the first time input is passed to the layer. - -`tf.keras.layers` classes create and contain their own model variables that -are tied to the lifetime of their layer objects. To share layer variables, share -their objects. - - -## Eager training - -### Computing gradients - -[Automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) -is useful for implementing machine learning algorithms such as -[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) for training -neural networks. During eager execution, use `tf.GradientTape` to trace -operations for computing gradients later. - -`tf.GradientTape` is an opt-in feature to provide maximal performance when -not tracing. Since different operations can occur during each call, all -forward-pass operations get recorded to a "tape". To compute the gradient, play -the tape backwards and then discard. A particular `tf.GradientTape` can only -compute one gradient; subsequent calls throw a runtime error. - -```py -w = tfe.Variable([[1.0]]) -with tf.GradientTape() as tape: - loss = w * w - -grad = tape.gradient(loss, w) -print(grad) # => tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32) -``` - -Here's an example of `tf.GradientTape` that records forward-pass operations -to train a simple model: - -```py -# A toy dataset of points around 3 * x + 2 -NUM_EXAMPLES = 1000 -training_inputs = tf.random_normal([NUM_EXAMPLES]) -noise = tf.random_normal([NUM_EXAMPLES]) -training_outputs = training_inputs * 3 + 2 + noise - -def prediction(input, weight, bias): - return input * weight + bias - -# A loss function using mean-squared error -def loss(weights, biases): - error = prediction(training_inputs, weights, biases) - training_outputs - return tf.reduce_mean(tf.square(error)) - -# Return the derivative of loss with respect to weight and bias -def grad(weights, biases): - with tf.GradientTape() as tape: - loss_value = loss(weights, biases) - return tape.gradient(loss_value, [weights, biases]) - -train_steps = 200 -learning_rate = 0.01 -# Start with arbitrary values for W and B on the same batch of data -W = tfe.Variable(5.) -B = tfe.Variable(10.) - -print("Initial loss: {:.3f}".format(loss(W, B))) - -for i in range(train_steps): - dW, dB = grad(W, B) - W.assign_sub(dW * learning_rate) - B.assign_sub(dB * learning_rate) - if i % 20 == 0: - print("Loss at step {:03d}: {:.3f}".format(i, loss(W, B))) - -print("Final loss: {:.3f}".format(loss(W, B))) -print("W = {}, B = {}".format(W.numpy(), B.numpy())) -``` - -Output (exact numbers may vary): - -``` -Initial loss: 71.204 -Loss at step 000: 68.333 -Loss at step 020: 30.222 -Loss at step 040: 13.691 -Loss at step 060: 6.508 -Loss at step 080: 3.382 -Loss at step 100: 2.018 -Loss at step 120: 1.422 -Loss at step 140: 1.161 -Loss at step 160: 1.046 -Loss at step 180: 0.996 -Final loss: 0.974 -W = 3.01582956314, B = 2.1191945076 -``` - -Replay the `tf.GradientTape` to compute the gradients and apply them in a -training loop. This is demonstrated in an excerpt from the -[mnist_eager.py](https://github.com/tensorflow/models/blob/master/official/mnist/mnist_eager.py) -example: - -```py -dataset = tf.data.Dataset.from_tensor_slices((data.train.images, - data.train.labels)) -... -for (batch, (images, labels)) in enumerate(dataset): - ... - with tf.GradientTape() as tape: - logits = model(images, training=True) - loss_value = loss(logits, labels) - ... - grads = tape.gradient(loss_value, model.variables) - optimizer.apply_gradients(zip(grads, model.variables), - global_step=tf.train.get_or_create_global_step()) -``` - - -The following example creates a multi-layer model that classifies the standard -[MNIST handwritten digits](https://www.tensorflow.org/tutorials/layers). It -demonstrates the optimizer and layer APIs to build trainable graphs in an eager -execution environment. - -### Train a model - -Even without training, call the model and inspect the output in eager execution: - -```py -# Create a tensor representing a blank image -batch = tf.zeros([1, 1, 784]) -print(batch.shape) # => (1, 1, 784) - -result = model(batch) -# => tf.Tensor([[[ 0. 0., ..., 0.]]], shape=(1, 1, 10), dtype=float32) -``` - -This example uses the -[dataset.py module](https://github.com/tensorflow/models/blob/master/official/mnist/dataset.py) -from the -[TensorFlow MNIST example](https://github.com/tensorflow/models/tree/master/official/mnist); -download this file to your local directory. Run the following to download the -MNIST data files to your working directory and prepare a `tf.data.Dataset` -for training: - -```py -import dataset # download dataset.py file -dataset_train = dataset.train('./datasets').shuffle(60000).repeat(4).batch(32) -``` - -To train a model, define a loss function to optimize and then calculate -gradients. Use an optimizer to update the variables: - -```py -def loss(model, x, y): - prediction = model(x) - return tf.losses.sparse_softmax_cross_entropy(labels=y, logits=prediction) - -def grad(model, inputs, targets): - with tf.GradientTape() as tape: - loss_value = loss(model, inputs, targets) - return tape.gradient(loss_value, model.variables) - -optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) - -x, y = iter(dataset_train).next() -print("Initial loss: {:.3f}".format(loss(model, x, y))) - -# Training loop -for (i, (x, y)) in enumerate(dataset_train): - # Calculate derivatives of the input function with respect to its parameters. - grads = grad(model, x, y) - # Apply the gradient to the model - optimizer.apply_gradients(zip(grads, model.variables), - global_step=tf.train.get_or_create_global_step()) - if i % 200 == 0: - print("Loss at step {:04d}: {:.3f}".format(i, loss(model, x, y))) - -print("Final loss: {:.3f}".format(loss(model, x, y))) -``` - -Output (exact numbers may vary): - -``` -Initial loss: 2.674 -Loss at step 0000: 2.593 -Loss at step 0200: 2.143 -Loss at step 0400: 2.009 -Loss at step 0600: 2.103 -Loss at step 0800: 1.621 -Loss at step 1000: 1.695 -... -Loss at step 6600: 0.602 -Loss at step 6800: 0.557 -Loss at step 7000: 0.499 -Loss at step 7200: 0.744 -Loss at step 7400: 0.681 -Final loss: 0.670 -``` - -And for faster training, move the computation to a GPU: - -```py -with tf.device("/gpu:0"): - for (i, (x, y)) in enumerate(dataset_train): - # minimize() is equivalent to the grad() and apply_gradients() calls. - optimizer.minimize(lambda: loss(model, x, y), - global_step=tf.train.get_or_create_global_step()) -``` - -### Variables and optimizers - -`tfe.Variable` objects store mutable `tf.Tensor` values accessed during -training to make automatic differentiation easier. The parameters of a model can -be encapsulated in classes as variables. - -Better encapsulate model parameters by using `tfe.Variable` with -`tf.GradientTape`. For example, the automatic differentiation example above -can be rewritten: - -```py -class Model(tf.keras.Model): - def __init__(self): - super(Model, self).__init__() - self.W = tfe.Variable(5., name='weight') - self.B = tfe.Variable(10., name='bias') - def predict(self, inputs): - return inputs * self.W + self.B - -# A toy dataset of points around 3 * x + 2 -NUM_EXAMPLES = 2000 -training_inputs = tf.random_normal([NUM_EXAMPLES]) -noise = tf.random_normal([NUM_EXAMPLES]) -training_outputs = training_inputs * 3 + 2 + noise - -# The loss function to be optimized -def loss(model, inputs, targets): - error = model.predict(inputs) - targets - return tf.reduce_mean(tf.square(error)) - -def grad(model, inputs, targets): - with tf.GradientTape() as tape: - loss_value = loss(model, inputs, targets) - return tape.gradient(loss_value, [model.W, model.B]) - -# Define: -# 1. A model. -# 2. Derivatives of a loss function with respect to model parameters. -# 3. A strategy for updating the variables based on the derivatives. -model = Model() -optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) - -print("Initial loss: {:.3f}".format(loss(model, training_inputs, training_outputs))) - -# Training loop -for i in range(300): - grads = grad(model, training_inputs, training_outputs) - optimizer.apply_gradients(zip(grads, [model.W, model.B]), - global_step=tf.train.get_or_create_global_step()) - if i % 20 == 0: - print("Loss at step {:03d}: {:.3f}".format(i, loss(model, training_inputs, training_outputs))) - -print("Final loss: {:.3f}".format(loss(model, training_inputs, training_outputs))) -print("W = {}, B = {}".format(model.W.numpy(), model.B.numpy())) -``` - -Output (exact numbers may vary): - -``` -Initial loss: 69.066 -Loss at step 000: 66.368 -Loss at step 020: 30.107 -Loss at step 040: 13.959 -Loss at step 060: 6.769 -Loss at step 080: 3.567 -Loss at step 100: 2.141 -Loss at step 120: 1.506 -Loss at step 140: 1.223 -Loss at step 160: 1.097 -Loss at step 180: 1.041 -Loss at step 200: 1.016 -Loss at step 220: 1.005 -Loss at step 240: 1.000 -Loss at step 260: 0.998 -Loss at step 280: 0.997 -Final loss: 0.996 -W = 2.99431324005, B = 2.02129220963 -``` - -## Use objects for state during eager execution - -With graph execution, program state (such as the variables) is stored in global -collections and their lifetime is managed by the `tf.Session` object. In -contrast, during eager execution the lifetime of state objects is determined by -the lifetime of their corresponding Python object. - -### Variables are objects - -During eager execution, variables persist until the last reference to the object -is removed, and is then deleted. - -```py -with tf.device("gpu:0"): - v = tfe.Variable(tf.random_normal([1000, 1000])) - v = None # v no longer takes up GPU memory -``` - -### Object-based saving - -`tfe.Checkpoint` can save and restore `tfe.Variable`s to and from -checkpoints: - -```py -x = tfe.Variable(10.) - -checkpoint = tfe.Checkpoint(x=x) # save as "x" - -x.assign(2.) # Assign a new value to the variables and save. -save_path = checkpoint.save('./ckpt/') - -x.assign(11.) # Change the variable after saving. - -# Restore values from the checkpoint -checkpoint.restore(save_path) - -print(x) # => 2.0 -``` - -To save and load models, `tfe.Checkpoint` stores the internal state of objects, -without requiring hidden variables. To record the state of a `model`, -an `optimizer`, and a global step, pass them to a `tfe.Checkpoint`: - -```py -model = MyModel() -optimizer = tf.train.AdamOptimizer(learning_rate=0.001) -checkpoint_dir = ‘/path/to/model_dir’ -checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt") -root = tfe.Checkpoint(optimizer=optimizer, - model=model, - optimizer_step=tf.train.get_or_create_global_step()) - -root.save(file_prefix=checkpoint_prefix) -# or -root.restore(tf.train.latest_checkpoint(checkpoint_dir)) -``` - -### Object-oriented metrics - -`tfe.metrics` are stored as objects. Update a metric by passing the new data to -the callable, and retrieve the result using the `tfe.metrics.result` method, -for example: - -```py -m = tfe.metrics.Mean("loss") -m(0) -m(5) -m.result() # => 2.5 -m([8, 9]) -m.result() # => 5.5 -``` - -#### Summaries and TensorBoard - -@{$summaries_and_tensorboard$TensorBoard} is a visualization tool for -understanding, debugging and optimizing the model training process. It uses -summary events that are written while executing the program. - -`tf.contrib.summary` is compatible with both eager and graph execution -environments. Summary operations, such as `tf.contrib.summary.scalar`, are -inserted during model construction. For example, to record summaries once every -100 global steps: - -```py -writer = tf.contrib.summary.create_file_writer(logdir) -global_step=tf.train.get_or_create_global_step() # return global step var - -writer.set_as_default() - -for _ in range(iterations): - global_step.assign_add(1) - # Must include a record_summaries method - with tf.contrib.summary.record_summaries_every_n_global_steps(100): - # your model code goes here - tf.contrib.summary.scalar('loss', loss) - ... -``` - -## Advanced automatic differentiation topics - -### Dynamic models - -`tf.GradientTape` can also be used in dynamic models. This example for a -[backtracking line search](https://wikipedia.org/wiki/Backtracking_line_search) -algorithm looks like normal NumPy code, except there are gradients and is -differentiable, despite the complex control flow: - -```py -def line_search_step(fn, init_x, rate=1.0): - with tf.GradientTape() as tape: - # Variables are automatically recorded, but manually watch a tensor - tape.watch(init_x) - value = fn(init_x) - grad = tape.gradient(value, init_x) - grad_norm = tf.reduce_sum(grad * grad) - init_value = value - while value > init_value - rate * grad_norm: - x = init_x - rate * grad - value = fn(x) - rate /= 2.0 - return x, value -``` - -### Additional functions to compute gradients - -`tf.GradientTape` is a powerful interface for computing gradients, but there -is another [Autograd](https://github.com/HIPS/autograd)-style API available for -automatic differentiation. These functions are useful if writing math code with -only tensors and gradient functions, and without `tfe.Variables`: - -* `tfe.gradients_function` —Returns a function that computes the derivatives - of its input function parameter with respect to its arguments. The input - function parameter must return a scalar value. When the returned function is - invoked, it returns a list of `tf.Tensor` objects: one element for each - argument of the input function. Since anything of interest must be passed as a - function parameter, this becomes unwieldy if there's a dependency on many - trainable parameters. -* `tfe.value_and_gradients_function` —Similar to - `tfe.gradients_function`, but when the returned function is invoked, it - returns the value from the input function in addition to the list of - derivatives of the input function with respect to its arguments. - -In the following example, `tfe.gradients_function` takes the `square` -function as an argument and returns a function that computes the partial -derivatives of `square` with respect to its inputs. To calculate the derivative -of `square` at `3`, `grad(3.0)` returns `6`. - -```py -def square(x): - return tf.multiply(x, x) - -grad = tfe.gradients_function(square) - -square(3.) # => 9.0 -grad(3.) # => [6.0] - -# The second-order derivative of square: -gradgrad = tfe.gradients_function(lambda x: grad(x)[0]) -gradgrad(3.) # => [2.0] - -# The third-order derivative is None: -gradgradgrad = tfe.gradients_function(lambda x: gradgrad(x)[0]) -gradgradgrad(3.) # => [None] - - -# With flow control: -def abs(x): - return x if x > 0. else -x - -grad = tfe.gradients_function(abs) - -grad(3.) # => [1.0] -grad(-3.) # => [-1.0] -``` - -### Custom gradients - -Custom gradients are an easy way to override gradients in eager and graph -execution. Within the forward function, define the gradient with respect to the -inputs, outputs, or intermediate results. For example, here's an easy way to clip -the norm of the gradients in the backward pass: - -```py -@tf.custom_gradient -def clip_gradient_by_norm(x, norm): - y = tf.identity(x) - def grad_fn(dresult): - return [tf.clip_by_norm(dresult, norm), None] - return y, grad_fn -``` - -Custom gradients are commonly used to provide a numerically stable gradient for a -sequence of operations: - -```py -def log1pexp(x): - return tf.log(1 + tf.exp(x)) -grad_log1pexp = tfe.gradients_function(log1pexp) - -# The gradient computation works fine at x = 0. -grad_log1pexp(0.) # => [0.5] - -# However, x = 100 fails because of numerical instability. -grad_log1pexp(100.) # => [nan] -``` - -Here, the `log1pexp` function can be analytically simplified with a custom -gradient. The implementation below reuses the value for `tf.exp(x)` that is -computed during the forward pass—making it more efficient by eliminating -redundant calculations: - -```py -@tf.custom_gradient -def log1pexp(x): - e = tf.exp(x) - def grad(dy): - return dy * (1 - 1 / (1 + e)) - return tf.log(1 + e), grad - -grad_log1pexp = tfe.gradients_function(log1pexp) - -# As before, the gradient computation works fine at x = 0. -grad_log1pexp(0.) # => [0.5] - -# And the gradient computation also works at x = 100. -grad_log1pexp(100.) # => [1.0] -``` - -## Performance - -Computation is automatically offloaded to GPUs during eager execution. If you -want control over where a computation runs you can enclose it in a -`tf.device('/gpu:0')` block (or the CPU equivalent): - -```py -import time - -def measure(x, steps): - # TensorFlow initializes a GPU the first time it's used, exclude from timing. - tf.matmul(x, x) - start = time.time() - for i in range(steps): - x = tf.matmul(x, x) - _ = x.numpy() # Make sure to execute op and not just enqueue it - end = time.time() - return end - start - -shape = (1000, 1000) -steps = 200 -print("Time to multiply a {} matrix by itself {} times:".format(shape, steps)) - -# Run on CPU: -with tf.device("/cpu:0"): - print("CPU: {} secs".format(measure(tf.random_normal(shape), steps))) - -# Run on GPU, if available: -if tfe.num_gpus() > 0: - with tf.device("/gpu:0"): - print("GPU: {} secs".format(measure(tf.random_normal(shape), steps))) -else: - print("GPU: not found") -``` - -Output (exact numbers depend on hardware): - -``` -Time to multiply a (1000, 1000) matrix by itself 200 times: -CPU: 4.614904403686523 secs -GPU: 0.5581181049346924 secs -``` - -A `tf.Tensor` object can be copied to a different device to execute its -operations: - -```py -x = tf.random_normal([10, 10]) - -x_gpu0 = x.gpu() -x_cpu = x.cpu() - -_ = tf.matmul(x_cpu, x_cpu) # Runs on CPU -_ = tf.matmul(x_gpu0, x_gpu0) # Runs on GPU:0 - -if tfe.num_gpus() > 1: - x_gpu1 = x.gpu(1) - _ = tf.matmul(x_gpu1, x_gpu1) # Runs on GPU:1 -``` - -### Benchmarks - -For compute-heavy models, such as -[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50) -training on a GPU, eager execution performance is comparable to graph execution. -But this gap grows larger for models with less computation and there is work to -be done for optimizing hot code paths for models with lots of small operations. - - -## Work with graphs - -While eager execution makes development and debugging more interactive, -TensorFlow graph execution has advantages for distributed training, performance -optimizations, and production deployment. However, writing graph code can feel -different than writing regular Python code and more difficult to debug. - -For building and training graph-constructed models, the Python program first -builds a graph representing the computation, then invokes `Session.run` to send -the graph for execution on the C++-based runtime. This provides: - -* Automatic differentiation using static autodiff. -* Simple deployment to a platform independent server. -* Graph-based optimizations (common subexpression elimination, constant-folding, etc.). -* Compilation and kernel fusion. -* Automatic distribution and replication (placing nodes on the distributed system). - -Deploying code written for eager execution is more difficult: either generate a -graph from the model, or run the Python runtime and code directly on the server. - -### Write compatible code - -The same code written for eager execution will also build a graph during graph -execution. Do this by simply running the same code in a new Python session where -eager execution is not enabled. - -Most TensorFlow operations work during eager execution, but there are some things -to keep in mind: - -* Use `tf.data` for input processing instead of queues. It's faster and easier. -* Use object-oriented layer APIs—like `tf.keras.layers` and - `tf.keras.Model`—since they have explicit storage for variables. -* Most model code works the same during eager and graph execution, but there are - exceptions. (For example, dynamic models using Python control flow to change the - computation based on inputs.) -* Once eager execution is enabled with `tf.enable_eager_execution`, it - cannot be turned off. Start a new Python session to return to graph execution. - -It's best to write code for both eager execution *and* graph execution. This -gives you eager's interactive experimentation and debuggability with the -distributed performance benefits of graph execution. - -Write, debug, and iterate in eager execution, then import the model graph for -production deployment. Use `tfe.Checkpoint` to save and restore model -variables, this allows movement between eager and graph execution environments. -See the examples in: -[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples). - -### Use eager execution in a graph environment - -Selectively enable eager execution in a TensorFlow graph environment using -`tfe.py_func`. This is used when `tf.enable_eager_execution()` has *not* -been called. - -```py -def my_py_func(x): - x = tf.matmul(x, x) # You can use tf ops - print(x) # but it's eager! - return x - -with tf.Session() as sess: - x = tf.placeholder(dtype=tf.float32) - # Call eager function in graph! - pf = tfe.py_func(my_py_func, [x], tf.float32) - sess.run(pf, feed_dict={x: [[2.0]]}) # [[4.0]] -``` diff --git a/tensorflow/docs_src/programmers_guide/embedding.md b/tensorflow/docs_src/programmers_guide/embedding.md deleted file mode 100644 index 8a98367dfb..0000000000 --- a/tensorflow/docs_src/programmers_guide/embedding.md +++ /dev/null @@ -1,262 +0,0 @@ -# Embeddings - -This document introduces the concept of embeddings, gives a simple example of -how to train an embedding in TensorFlow, and explains how to view embeddings -with the TensorBoard Embedding Projector -([live example](http://projector.tensorflow.org)). The first two parts target -newcomers to machine learning or TensorFlow, and the Embedding Projector how-to -is for users at all levels. - -An alternative tutorial on these concepts is available in the -[Embeddings section of Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/embeddings/video-lecture). - -[TOC] - -An **embedding** is a mapping from discrete objects, such as words, to vectors -of real numbers. For example, a 300-dimensional embedding for English words -could include: - -``` -blue: (0.01359, 0.00075997, 0.24608, ..., -0.2524, 1.0048, 0.06259) -blues: (0.01396, 0.11887, -0.48963, ..., 0.033483, -0.10007, 0.1158) -orange: (-0.24776, -0.12359, 0.20986, ..., 0.079717, 0.23865, -0.014213) -oranges: (-0.35609, 0.21854, 0.080944, ..., -0.35413, 0.38511, -0.070976) -``` - -The individual dimensions in these vectors typically have no inherent meaning. -Instead, it's the overall patterns of location and distance between vectors -that machine learning takes advantage of. - -Embeddings are important for input to machine learning. Classifiers, and neural -networks more generally, work on vectors of real numbers. They train best on -dense vectors, where all values contribute to define an object. However, many -important inputs to machine learning, such as words of text, do not have a -natural vector representation. Embedding functions are the standard and -effective way to transform such discrete input objects into useful -continuous vectors. - -Embeddings are also valuable as outputs of machine learning. Because embeddings -map objects to vectors, applications can use similarity in vector space (for -instance, Euclidean distance or the angle between vectors) as a robust and -flexible measure of object similarity. One common use is to find nearest -neighbors. Using the same word embeddings as above, for instance, here are the -three nearest neighbors for each word and the corresponding angles: - -``` -blue: (red, 47.6°), (yellow, 51.9°), (purple, 52.4°) -blues: (jazz, 53.3°), (folk, 59.1°), (bluegrass, 60.6°) -orange: (yellow, 53.5°), (colored, 58.0°), (bright, 59.9°) -oranges: (apples, 45.3°), (lemons, 48.3°), (mangoes, 50.4°) -``` - -This would tell an application that apples and oranges are in some way more -similar (45.3° apart) than lemons and oranges (48.3° apart). - -## Embeddings in TensorFlow - -To create word embeddings in TensorFlow, we first split the text into words -and then assign an integer to every word in the vocabulary. Let us assume that -this has already been done, and that `word_ids` is a vector of these integers. -For example, the sentence “I have a cat.” could be split into -`[“I”, “have”, “a”, “cat”, “.”]` and then the corresponding `word_ids` tensor -would have shape `[5]` and consist of 5 integers. To map these word ids -to vectors, we need to create the embedding variable and use the -`tf.nn.embedding_lookup` function as follows: - -``` -word_embeddings = tf.get_variable(“word_embeddings”, - [vocabulary_size, embedding_size]) -embedded_word_ids = tf.nn.embedding_lookup(word_embeddings, word_ids) -``` - -After this, the tensor `embedded_word_ids` will have shape `[5, embedding_size]` -in our example and contain the embeddings (dense vectors) for each of the 5 -words. At the end of training, `word_embeddings` will contain the embeddings -for all words in the vocabulary. - -Embeddings can be trained in many network types, and with various loss -functions and data sets. For example, one could use a recurrent neural network -to predict the next word from the previous one given a large corpus of -sentences, or one could train two networks to do multi-lingual translation. -These methods are described in the @{$word2vec$Vector Representations of Words} -tutorial. - -## Visualizing Embeddings - -TensorBoard includes the **Embedding Projector**, a tool that lets you -interactively visualize embeddings. This tool can read embeddings from your -model and render them in two or three dimensions. - -The Embedding Projector has three panels: - -- *Data panel* on the top left, where you can choose the run, the embedding - variable and data columns to color and label points by. -- *Projections panel* on the bottom left, where you can choose the type of - projection. -- *Inspector panel* on the right side, where you can search for particular - points and see a list of nearest neighbors. - -### Projections -The Embedding Projector provides three ways to reduce the dimensionality of a -data set. - -- *[t-SNE](https://en.wikipedia.org/wiki/T-distributed_stochastic_neighbor_embedding)*: - a nonlinear nondeterministic algorithm (T-distributed stochastic neighbor - embedding) that tries to preserve local neighborhoods in the data, often at - the expense of distorting global structure. You can choose whether to compute - two- or three-dimensional projections. - -- *[PCA](https://en.wikipedia.org/wiki/Principal_component_analysis)*: - a linear deterministic algorithm (principal component analysis) that tries to - capture as much of the data variability in as few dimensions as possible. PCA - tends to highlight large-scale structure in the data, but can distort local - neighborhoods. The Embedding Projector computes the top 10 principal - components, from which you can choose two or three to view. - -- *Custom*: a linear projection onto horizontal and vertical axes that you - specify using labels in the data. You define the horizontal axis, for - instance, by giving text patterns for "Left" and "Right". The Embedding - Projector finds all points whose label matches the "Left" pattern and - computes the centroid of that set; similarly for "Right". The line passing - through these two centroids defines the horizontal axis. The vertical axis is - likewise computed from the centroids for points matching the "Up" and "Down" - text patterns. - -Further useful articles are -[How to Use t-SNE Effectively](https://distill.pub/2016/misread-tsne/) and -[Principal Component Analysis Explained Visually](http://setosa.io/ev/principal-component-analysis/). - -### Exploration - -You can explore visually by zooming, rotating, and panning using natural -click-and-drag gestures. Hovering your mouse over a point will show any -[metadata](#metadata) for that point. You can also inspect nearest-neighbor -subsets. Clicking on a point causes the right pane to list the nearest -neighbors, along with distances to the current point. The nearest-neighbor -points are also highlighted in the projection. - -It is sometimes useful to restrict the view to a subset of points and perform -projections only on those points. To do so, you can select points in multiple -ways: - -- After clicking on a point, its nearest neighbors are also selected. -- After a search, the points matching the query are selected. -- Enabling selection, clicking on a point and dragging defines a selection - sphere. - -Then click the "Isolate *nnn* points" button at the top of the Inspector pane -on the right hand side. The following image shows 101 points selected and ready -for the user to click "Isolate 101 points": - -![Selection of nearest neighbors](https://www.tensorflow.org/images/embedding-nearest-points.png "Selection of nearest neighbors") - -*Selection of the nearest neighbors of “important” in a word embedding dataset.* - -Advanced tip: filtering with custom projection can be powerful. Below, we -filtered the 100 nearest neighbors of “politics” and projected them onto the -“worst” - “best” vector as an x axis. The y axis is random. As a result, one -finds on the right side “ideas”, “science”, “perspective”, “journalism” but on -the left “crisis”, “violence” and “conflict”. - - - - - - - - - - -
- Custom controls panel - - Custom projection -
- Custom projection controls. - - Custom projection of neighbors of "politics" onto "best" - "worst" vector. -
- -To share your findings, you can use the bookmark panel in the bottom right -corner and save the current state (including computed coordinates of any -projection) as a small file. The Projector can then be pointed to a set of one -or more of these files, producing the panel below. Other users can then walk -through a sequence of bookmarks. - -Bookmark panel - -### Metadata - -If you are working with an embedding, you'll probably want to attach -labels/images to the data points. You can do this by generating a metadata file -containing the labels for each point and clicking "Load data" in the data panel -of the Embedding Projector. - -The metadata can be either labels or images, which are -stored in a separate file. For labels, the format should -be a [TSV file](https://en.wikipedia.org/wiki/Tab-separated_values) -(tab characters shown in red) whose first line contains column headers -(shown in bold) and subsequent lines contain the metadata values. For example: - - -Word\tFrequency
- Airplane\t345
- Car\t241
- ... - - -The order of lines in the metadata file is assumed to match the order of -vectors in the embedding variable, except for the header. Consequently, the -(i+1)-th line in the metadata file corresponds to the i-th row of the embedding -variable. If the TSV metadata file has only a single column, then we don’t -expect a header row, and assume each row is the label of the embedding. We -include this exception because it matches the commonly-used "vocab file" -format. - -To use images as metadata, you must produce a single -[sprite image](https://www.google.com/webhp#q=what+is+a+sprite+image), -consisting of small thumbnails, one for each vector in the embedding. The -sprite should store thumbnails in row-first order: the first data point placed -in the top left and the last data point in the bottom right, though the last -row doesn't have to be filled, as shown below. - - - - - - - - - - - - - - - - - -
012
345
67
- -Follow [this link](https://www.tensorflow.org/images/embedding-mnist.mp4) -to see a fun example of thumbnail images in the Embedding Projector. - - -## Mini-FAQ - -**Is "embedding" an action or a thing?** -Both. People talk about embedding words in a vector space (action) and about -producing word embeddings (things). Common to both is the notion of embedding -as a mapping from discrete objects to vectors. Creating or applying that -mapping is an action, but the mapping itself is a thing. - -**Are embeddings high-dimensional or low-dimensional?** -It depends. A 300-dimensional vector space of words and phrases, for instance, -is often called low-dimensional (and dense) when compared to the millions of -words and phrases it can contain. But mathematically it is high-dimensional, -displaying many properties that are dramatically different from what our human -intuition has learned about 2- and 3-dimensional spaces. - -**Is an embedding the same as an embedding layer?** -No. An *embedding layer* is a part of neural network, but an *embedding* is a more -general concept. diff --git a/tensorflow/docs_src/programmers_guide/estimators.md b/tensorflow/docs_src/programmers_guide/estimators.md deleted file mode 100644 index b13b47184d..0000000000 --- a/tensorflow/docs_src/programmers_guide/estimators.md +++ /dev/null @@ -1,193 +0,0 @@ -# Estimators - -This document introduces @{tf.estimator$**Estimators**}--a high-level TensorFlow -API that greatly simplifies machine learning programming. Estimators encapsulate -the following actions: - -* training -* evaluation -* prediction -* export for serving - -You may either use the pre-made Estimators we provide or write your -own custom Estimators. All Estimators--whether pre-made or custom--are -classes based on the @{tf.estimator.Estimator} class. - -Note: TensorFlow also includes a deprecated `Estimator` class at -@{tf.contrib.learn.Estimator}, which you should not use. - - -## Advantages of Estimators - -Estimators provide the following benefits: - -* You can run Estimator-based models on a local host or on a - distributed multi-server environment without changing your model. - Furthermore, you can run Estimator-based models on CPUs, GPUs, - or TPUs without recoding your model. -* Estimators simplify sharing implementations between model developers. -* You can develop a state of the art model with high-level intuitive code. - In short, it is generally much easier to create models with Estimators - than with the low-level TensorFlow APIs. -* Estimators are themselves built on @{tf.layers}, which - simplifies customization. -* Estimators build the graph for you. -* Estimators provide a safe distributed training loop that controls how and - when to: - * build the graph - * initialize variables - * start queues - * handle exceptions - * create checkpoint files and recover from failures - * save summaries for TensorBoard - -When writing an application with Estimators, you must separate the data input -pipeline from the model. This separation simplifies experiments with -different data sets. - - -## Pre-made Estimators - -Pre-made Estimators enable you to work at a much higher conceptual level -than the base TensorFlow APIs. You no longer have to worry about creating -the computational graph or sessions since Estimators handle all -the "plumbing" for you. That is, pre-made Estimators create and manage -@{tf.Graph$`Graph`} and @{tf.Session$`Session`} objects for you. Furthermore, -pre-made Estimators let you experiment with different model architectures by -making only minimal code changes. @{tf.estimator.DNNClassifier$`DNNClassifier`}, -for example, is a pre-made Estimator class that trains classification models -based on dense, feed-forward neural networks. - - -### Structure of a pre-made Estimators program - -A TensorFlow program relying on a pre-made Estimator typically consists -of the following four steps: - -1. **Write one or more dataset importing functions.** For example, you might - create one function to import the training set and another function to - import the test set. Each dataset importing function must return two - objects: - - * a dictionary in which the keys are feature names and the - values are Tensors (or SparseTensors) containing the corresponding - feature data - * a Tensor containing one or more labels - - For example, the following code illustrates the basic skeleton for - an input function: - - def input_fn(dataset): - ... # manipulate dataset, extracting the feature dict and the label - return feature_dict, label - - (See @{$programmers_guide/datasets} for full details.) - -2. **Define the feature columns.** Each @{tf.feature_column} - identifies a feature name, its type, and any input pre-processing. - For example, the following snippet creates three feature - columns that hold integer or floating-point data. The first two - feature columns simply identify the feature's name and type. The - third feature column also specifies a lambda the program will invoke - to scale the raw data: - - # Define three numeric feature columns. - population = tf.feature_column.numeric_column('population') - crime_rate = tf.feature_column.numeric_column('crime_rate') - median_education = tf.feature_column.numeric_column('median_education', - normalizer_fn=lambda x: x - global_education_mean) - -3. **Instantiate the relevant pre-made Estimator.** For example, here's - a sample instantiation of a pre-made Estimator named `LinearClassifier`: - - # Instantiate an estimator, passing the feature columns. - estimator = tf.estimator.LinearClassifier( - feature_columns=[population, crime_rate, median_education], - ) - -4. **Call a training, evaluation, or inference method.** - For example, all Estimators provide a `train` method, which trains a model. - - # my_training_set is the function created in Step 1 - estimator.train(input_fn=my_training_set, steps=2000) - - -### Benefits of pre-made Estimators - -Pre-made Estimators encode best practices, providing the following benefits: - -* Best practices for determining where different parts of the computational - graph should run, implementing strategies on a single machine or on a - cluster. -* Best practices for event (summary) writing and universally useful - summaries. - -If you don't use pre-made Estimators, you must implement the preceding -features yourself. - - -## Custom Estimators - -The heart of every Estimator--whether pre-made or custom--is its -**model function**, which is a method that builds graphs for training, -evaluation, and prediction. When you are using a pre-made Estimator, -someone else has already implemented the model function. When relying -on a custom Estimator, you must write the model function yourself. A -@{$custom_estimators$companion document} -explains how to write the model function. - - -## Recommended workflow - -We recommend the following workflow: - -1. Assuming a suitable pre-made Estimator exists, use it to build your - first model and use its results to establish a baseline. -2. Build and test your overall pipeline, including the integrity and - reliability of your data with this pre-made Estimator. -3. If suitable alternative pre-made Estimators are available, run - experiments to determine which pre-made Estimator produces the - best results. -4. Possibly, further improve your model by building your own custom Estimator. - - -## Creating Estimators from Keras models - -You can convert existing Keras models to Estimators. Doing so enables your Keras -model to access Estimator's strengths, such as distributed training. Call -@{tf.keras.estimator.model_to_estimator} as in the -following sample: - -```python -# Instantiate a Keras inception v3 model. -keras_inception_v3 = tf.keras.applications.inception_v3.InceptionV3(weights=None) -# Compile model with the optimizer, loss, and metrics you'd like to train with. -keras_inception_v3.compile(optimizer=tf.keras.optimizers.SGD(lr=0.0001, momentum=0.9), - loss='categorical_crossentropy', - metric='accuracy') -# Create an Estimator from the compiled Keras model. Note the initial model -# state of the keras model is preserved in the created Estimator. -est_inception_v3 = tf.keras.estimator.model_to_estimator(keras_model=keras_inception_v3) - -# Treat the derived Estimator as you would with any other Estimator. -# First, recover the input name(s) of Keras model, so we can use them as the -# feature column name(s) of the Estimator input function: -keras_inception_v3.input_names # print out: ['input_1'] -# Once we have the input name(s), we can create the input function, for example, -# for input(s) in the format of numpy ndarray: -train_input_fn = tf.estimator.inputs.numpy_input_fn( - x={"input_1": train_data}, - y=train_labels, - num_epochs=1, - shuffle=False) -# To train, we call Estimator's train function: -est_inception_v3.train(input_fn=train_input_fn, steps=2000) -``` -Note that the names of feature columns and labels of a keras estimator come from -the corresponding compiled keras model. For example, the input key names for -`train_input_fn` above can be obtained from `keras_inception_v3.input_names`, -and similarly, the predicted output names can be obtained from -`keras_inception_v3.output_names`. - -For more details, please refer to the documentation for -@{tf.keras.estimator.model_to_estimator}. diff --git a/tensorflow/docs_src/programmers_guide/faq.md b/tensorflow/docs_src/programmers_guide/faq.md deleted file mode 100644 index b6291a9ffa..0000000000 --- a/tensorflow/docs_src/programmers_guide/faq.md +++ /dev/null @@ -1,297 +0,0 @@ -# Frequently Asked Questions - -This document provides answers to some of the frequently asked questions about -TensorFlow. If you have a question that is not covered here, you might find an -answer on one of the TensorFlow @{$about$community resources}. - -[TOC] - -## Features and Compatibility - -#### Can I run distributed training on multiple computers? - -Yes! TensorFlow gained -@{$distributed$support for distributed computation} in -version 0.8. TensorFlow now supports multiple devices (CPUs and GPUs) in one or -more computers. - -#### Does TensorFlow work with Python 3? - -As of the 0.6.0 release timeframe (Early December 2015), we do support Python -3.3+. - -## Building a TensorFlow graph - -See also the -@{$python/framework$API documentation on building graphs}. - -#### Why does `c = tf.matmul(a, b)` not execute the matrix multiplication immediately? - -In the TensorFlow Python API, `a`, `b`, and `c` are -@{tf.Tensor} objects. A `Tensor` object is -a symbolic handle to the result of an operation, but does not actually hold the -values of the operation's output. Instead, TensorFlow encourages users to build -up complicated expressions (such as entire neural networks and its gradients) as -a dataflow graph. You then offload the computation of the entire dataflow graph -(or a subgraph of it) to a TensorFlow -@{tf.Session}, which is able to execute the -whole computation much more efficiently than executing the operations -one-by-one. - -#### How are devices named? - -The supported device names are `"/device:CPU:0"` (or `"/cpu:0"`) for the CPU -device, and `"/device:GPU:i"` (or `"/gpu:i"`) for the *i*th GPU device. - -#### How do I place operations on a particular device? - -To place a group of operations on a device, create them within a -@{tf.device$`with tf.device(name):`} context. See -the how-to documentation on -@{$using_gpu$using GPUs with TensorFlow} for details of how -TensorFlow assigns operations to devices, and the -@{$deep_cnn$CIFAR-10 tutorial} for an example model that -uses multiple GPUs. - - -## Running a TensorFlow computation - -See also the -@{$python/client$API documentation on running graphs}. - -#### What's the deal with feeding and placeholders? - -Feeding is a mechanism in the TensorFlow Session API that allows you to -substitute different values for one or more tensors at run time. The `feed_dict` -argument to @{tf.Session.run} is a -dictionary that maps @{tf.Tensor} objects to -numpy arrays (and some other types), which will be used as the values of those -tensors in the execution of a step. - -#### What is the difference between `Session.run()` and `Tensor.eval()`? - -If `t` is a @{tf.Tensor} object, -@{tf.Tensor.eval} is shorthand for -@{tf.Session.run}, where `sess` is the -current @{tf.get_default_session}. The -two following snippets of code are equivalent: - -```python -# Using `Session.run()`. -sess = tf.Session() -c = tf.constant(5.0) -print(sess.run(c)) - -# Using `Tensor.eval()`. -c = tf.constant(5.0) -with tf.Session(): - print(c.eval()) -``` - -In the second example, the session acts as a -[context manager](https://docs.python.org/2.7/reference/compound_stmts.html#with), -which has the effect of installing it as the default session for the lifetime of -the `with` block. The context manager approach can lead to more concise code for -simple use cases (like unit tests); if your code deals with multiple graphs and -sessions, it may be more straightforward to make explicit calls to -`Session.run()`. - -#### Do Sessions have a lifetime? What about intermediate tensors? - -Sessions can own resources, such as -@{tf.Variable}, -@{tf.QueueBase}, and -@{tf.ReaderBase}. These resources can sometimes use -a significant amount of memory, and can be released when the session is closed by calling -@{tf.Session.close}. - -The intermediate tensors that are created as part of a call to -@{$python/client$`Session.run()`} will be freed at or before the -end of the call. - -#### Does the runtime parallelize parts of graph execution? - -The TensorFlow runtime parallelizes graph execution across many different -dimensions: - -* The individual ops have parallel implementations, using multiple cores in a - CPU, or multiple threads in a GPU. -* Independent nodes in a TensorFlow graph can run in parallel on multiple - devices, which makes it possible to speed up - @{$deep_cnn$CIFAR-10 training using multiple GPUs}. -* The Session API allows multiple concurrent steps (i.e. calls to - @{tf.Session.run} in parallel). This - enables the runtime to get higher throughput, if a single step does not use - all of the resources in your computer. - -#### Which client languages are supported in TensorFlow? - -TensorFlow is designed to support multiple client languages. -Currently, the best-supported client language is [Python](../api_docs/python/index.md). Experimental interfaces for -executing and constructing graphs are also available for -[C++](../api_docs/cc/index.md), [Java](../api_docs/java/reference/org/tensorflow/package-summary.html) and [Go](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go). - -TensorFlow also has a -[C-based client API](https://www.tensorflow.org/code/tensorflow/c/c_api.h) -to help build support for more client languages. We invite contributions of new -language bindings. - -Bindings for various other languages (such as [C#](https://github.com/migueldeicaza/TensorFlowSharp), [Julia](https://github.com/malmaud/TensorFlow.jl), [Ruby](https://github.com/somaticio/tensorflow.rb) and [Scala](https://github.com/eaplatanios/tensorflow_scala)) created and supported by the open source community build on top of the C API supported by the TensorFlow maintainers. - -#### Does TensorFlow make use of all the devices (GPUs and CPUs) available on my machine? - -TensorFlow supports multiple GPUs and CPUs. See the how-to documentation on -@{$using_gpu$using GPUs with TensorFlow} for details of how -TensorFlow assigns operations to devices, and the -@{$deep_cnn$CIFAR-10 tutorial} for an example model that -uses multiple GPUs. - -Note that TensorFlow only uses GPU devices with a compute capability greater -than 3.5. - -#### Why does `Session.run()` hang when using a reader or a queue? - -The @{tf.ReaderBase} and -@{tf.QueueBase} classes provide special operations that -can *block* until input (or free space in a bounded queue) becomes -available. These operations allow you to build sophisticated -@{$reading_data$input pipelines}, at the cost of making the -TensorFlow computation somewhat more complicated. See the how-to documentation -for -@{$reading_data#creating_threads_to_prefetch_using_queuerunner_objects$using `QueueRunner` objects to drive queues and readers} -for more information on how to use them. - -## Variables - -See also the how-to documentation on @{$variables$variables} and -@{$python/state_ops$the API documentation for variables}. - -#### What is the lifetime of a variable? - -A variable is created when you first run the -@{tf.Variable.initializer} -operation for that variable in a session. It is destroyed when that -@{tf.Session.close}. - -#### How do variables behave when they are concurrently accessed? - -Variables allow concurrent read and write operations. The value read from a -variable may change if it is concurrently updated. By default, concurrent -assignment operations to a variable are allowed to run with no mutual exclusion. -To acquire a lock when assigning to a variable, pass `use_locking=True` to -@{tf.Variable.assign}. - -## Tensor shapes - -See also the -@{tf.TensorShape}. - -#### How can I determine the shape of a tensor in Python? - -In TensorFlow, a tensor has both a static (inferred) shape and a dynamic (true) -shape. The static shape can be read using the -@{tf.Tensor.get_shape} -method: this shape is inferred from the operations that were used to create the -tensor, and may be -@{tf.TensorShape$partially complete}. If the static -shape is not fully defined, the dynamic shape of a `Tensor` `t` can be -determined by evaluating @{tf.shape$`tf.shape(t)`}. - -#### What is the difference between `x.set_shape()` and `x = tf.reshape(x)`? - -The @{tf.Tensor.set_shape} method updates -the static shape of a `Tensor` object, and it is typically used to provide -additional shape information when this cannot be inferred directly. It does not -change the dynamic shape of the tensor. - -The @{tf.reshape} operation creates -a new tensor with a different dynamic shape. - -#### How do I build a graph that works with variable batch sizes? - -It is often useful to build a graph that works with variable batch sizes -so that the same code can be used for (mini-)batch training, and -single-instance inference. The resulting graph can be -@{tf.Graph.as_graph_def$saved as a protocol buffer} -and -@{tf.import_graph_def$imported into another program}. - -When building a variable-size graph, the most important thing to remember is not -to encode the batch size as a Python constant, but instead to use a symbolic -`Tensor` to represent it. The following tips may be useful: - -* Use [`batch_size = tf.shape(input)[0]`](../api_docs/python/array_ops.md#shape) - to extract the batch dimension from a `Tensor` called `input`, and store it in - a `Tensor` called `batch_size`. - -* Use @{tf.reduce_mean} instead - of `tf.reduce_sum(...) / batch_size`. - - -## TensorBoard - -#### How can I visualize a TensorFlow graph? - -See the @{$graph_viz$graph visualization tutorial}. - -#### What is the simplest way to send data to TensorBoard? - -Add summary ops to your TensorFlow graph, and write -these summaries to a log directory. Then, start TensorBoard using - - python tensorflow/tensorboard/tensorboard.py --logdir=path/to/log-directory - -For more details, see the -@{$summaries_and_tensorboard$Summaries and TensorBoard tutorial}. - -#### Every time I launch TensorBoard, I get a network security popup! - -You can change TensorBoard to serve on localhost rather than '0.0.0.0' by -the flag --host=localhost. This should quiet any security warnings. - -## Extending TensorFlow - -See the how-to documentation for -@{$adding_an_op$adding a new operation to TensorFlow}. - -#### My data is in a custom format. How do I read it using TensorFlow? - -There are three main options for dealing with data in a custom format. - -The easiest option is to write parsing code in Python that transforms the data -into a numpy array. Then, use @{tf.data.Dataset.from_tensor_slices} to -create an input pipeline from the in-memory data. - -If your data doesn't fit in memory, try doing the parsing in the Dataset -pipeline. Start with an appropriate file reader, like -@{tf.data.TextLineDataset}. Then convert the dataset by mapping -@{tf.data.Dataset.map$mapping} appropriate operations over it. -Prefer predefined TensorFlow operations such as @{tf.decode_raw}, -@{tf.decode_csv}, @{tf.parse_example}, or @{tf.image.decode_png}. - -If your data is not easily parsable with the built-in TensorFlow operations, -consider converting it, offline, to a format that is easily parsable, such -as @{tf.python_io.TFRecordWriter$`TFRecord`} format. - -The most efficient method to customize the parsing behavior is to -@{$adding_an_op$add a new op written in C++} that parses your -data format. The @{$new_data_formats$guide to handling new data formats} has -more information about the steps for doing this. - - -## Miscellaneous - -#### What is TensorFlow's coding style convention? - -The TensorFlow Python API adheres to the -[PEP8](https://www.python.org/dev/peps/pep-0008/) conventions.* In -particular, we use `CamelCase` names for classes, and `snake_case` names for -functions, methods, and properties. We also adhere to the -[Google Python style guide](https://google.github.io/styleguide/pyguide.html). - -The TensorFlow C++ code base adheres to the -[Google C++ style guide](https://google.github.io/styleguide/cppguide.html). - -(* With one exception: we use 2-space indentation instead of 4-space -indentation.) - diff --git a/tensorflow/docs_src/programmers_guide/feature_columns.md b/tensorflow/docs_src/programmers_guide/feature_columns.md deleted file mode 100644 index 90f5c53a17..0000000000 --- a/tensorflow/docs_src/programmers_guide/feature_columns.md +++ /dev/null @@ -1,572 +0,0 @@ -# Feature Columns - -This document details feature columns. Think of **feature columns** as the -intermediaries between raw data and Estimators. Feature columns are very rich, -enabling you to transform a diverse range of raw data into formats that -Estimators can use, allowing easy experimentation. - -In @{$premade_estimators$Premade Estimators}, we used the premade -Estimator, @{tf.estimator.DNNClassifier$`DNNClassifier`} to train a model to -predict different types of Iris flowers from four input features. That example -created only numerical feature columns (of type -@{tf.feature_column.numeric_column}). Although numerical feature columns model -the lengths of petals and sepals effectively, real world data sets contain all -kinds of features, many of which are non-numerical. - -
- -
-
-Some real-world features (such as, longitude) are numerical, but many are not. -
- -## Input to a Deep Neural Network - -What kind of data can a deep neural network operate on? The answer -is, of course, numbers (for example, `tf.float32`). After all, every neuron in -a neural network performs multiplication and addition operations on weights and -input data. Real-life input data, however, often contains non-numerical -(categorical) data. For example, consider a `product_class` feature that can -contain the following three non-numerical values: - -* `kitchenware` -* `electronics` -* `sports` - -ML models generally represent categorical values as simple vectors in which a -1 represents the presence of a value and a 0 represents the absence of a value. -For example, when `product_class` is set to `sports`, an ML model would usually -represent `product_class` as `[0, 0, 1]`, meaning: - -* `0`: `kitchenware` is absent -* `0`: `electronics` is absent -* `1`: `sports` is present - -So, although raw data can be numerical or categorical, an ML model represents -all features as numbers. - -## Feature Columns - -As the following figure suggests, you specify the input to a model through the -`feature_columns` argument of an Estimator (`DNNClassifier` for Iris). -Feature Columns bridge input data (as returned by `input_fn`) with your model. - -
- -
-
-Feature columns bridge raw data with the data your model needs. -
- -To create feature columns, call functions from the -@{tf.feature_column} module. This document explains nine of the functions in -that module. As the following figure shows, all nine functions return either a -Categorical-Column or a Dense-Column object, except `bucketized_column`, which -inherits from both classes: - -
- -
-
-Feature column methods fall into two main categories and one hybrid category. -
- -Let's look at these functions in more detail. - -### Numeric column - -The Iris classifier calls the @{tf.feature_column.numeric_column} function for -all input features: - - * `SepalLength` - * `SepalWidth` - * `PetalLength` - * `PetalWidth` - -Although `tf.numeric_column` provides optional arguments, calling -`tf.numeric_column` without any arguments, as follows, is a fine way to specify -a numerical value with the default data type (`tf.float32`) as input to your -model: - -```python -# Defaults to a tf.float32 scalar. -numeric_feature_column = tf.feature_column.numeric_column(key="SepalLength") -``` - -To specify a non-default numerical data type, use the `dtype` argument. For -example: - -``` python -# Represent a tf.float64 scalar. -numeric_feature_column = tf.feature_column.numeric_column(key="SepalLength", - dtype=tf.float64) -``` - -By default, a numeric column creates a single value (scalar). Use the shape -argument to specify another shape. For example: - - -```python -# Represent a 10-element vector in which each cell contains a tf.float32. -vector_feature_column = tf.feature_column.numeric_column(key="Bowling", - shape=10) - -# Represent a 10x5 matrix in which each cell contains a tf.float32. -matrix_feature_column = tf.feature_column.numeric_column(key="MyMatrix", - shape=[10,5]) -``` -### Bucketized column - -Often, you don't want to feed a number directly into the model, but instead -split its value into different categories based on numerical ranges. To do so, -create a @{tf.feature_column.bucketized_column$bucketized column}. For -example, consider raw data that represents the year a house was built. Instead -of representing that year as a scalar numeric column, we could split the year -into the following four buckets: - -
- -
-
-Dividing year data into four buckets. -
- -The model will represent the buckets as follows: - -|Date Range |Represented as... | -|:----------|:-----------------| -|< 1960 | [1, 0, 0, 0] | -|>= 1960 but < 1980 | [0, 1, 0, 0] | -|>= 1980 but < 2000 | [0, 0, 1, 0] | -|>= 2000 | [0, 0, 0, 1] | - -Why would you want to split a number—a perfectly valid input to your -model—into a categorical value? Well, notice that the categorization splits a -single input number into a four-element vector. Therefore, the model now can -learn _four individual weights_ rather than just one; four weights creates a -richer model than one weight. More importantly, bucketizing enables the model -to clearly distinguish between different year categories since only one of the -elements is set (1) and the other three elements are cleared (0). For example, -when we just use a single number (a year) as input, a linear model can only -learn a linear relationship. So, bucketing provides the model with additional -flexibility that the model can use to learn. - -The following code demonstrates how to create a bucketized feature: - - -```python -# First, convert the raw input to a numeric column. -numeric_feature_column = tf.feature_column.numeric_column("Year") - -# Then, bucketize the numeric column on the years 1960, 1980, and 2000. -bucketized_feature_column = tf.feature_column.bucketized_column( - source_column = numeric_feature_column, - boundaries = [1960, 1980, 2000]) -``` -Note that specifying a _three_-element boundaries vector creates a -_four_-element bucketized vector. - - -### Categorical identity column - -**Categorical identity columns** can be seen as a special case of bucketized -columns. In traditional bucketized columns, each bucket represents a range of -values (for example, from 1960 to 1979). In a categorical identity column, each -bucket represents a single, unique integer. For example, let's say you want to -represent the integer range `[0, 4)`. That is, you want to represent the -integers 0, 1, 2, or 3. In this case, the categorical identity mapping looks -like this: - -
- -
-
-A categorical identity column mapping. Note that this is a one-hot -encoding, not a binary numerical encoding. -
- -As with bucketized columns, a model can learn a separate weight for each class -in a categorical identity column. For example, instead of using a string to -represent the `product_class`, let's represent each class with a unique integer -value. That is: - -* `0="kitchenware"` -* `1="electronics"` -* `2="sport"` - -Call @{tf.feature_column.categorical_column_with_identity} to implement a -categorical identity column. For example: - -``` python -# Create categorical output for an integer feature named "my_feature_b", -# The values of my_feature_b must be >= 0 and < num_buckets -identity_feature_column = tf.feature_column.categorical_column_with_identity( - key='my_feature_b', - num_buckets=4) # Values [0, 4) - -# In order for the preceding call to work, the input_fn() must return -# a dictionary containing 'my_feature_b' as a key. Furthermore, the values -# assigned to 'my_feature_b' must belong to the set [0, 4). -def input_fn(): - ... - return ({ 'my_feature_a':[7, 9, 5, 2], 'my_feature_b':[3, 1, 2, 2] }, - [Label_values]) -``` - -### Categorical vocabulary column - -We cannot input strings directly to a model. Instead, we must first map strings -to numeric or categorical values. Categorical vocabulary columns provide a good -way to represent strings as a one-hot vector. For example: - -
- -
-
-Mapping string values to vocabulary columns. -
- -As you can see, categorical vocabulary columns are kind of an enum version of -categorical identity columns. TensorFlow provides two different functions to -create categorical vocabulary columns: - -* @{tf.feature_column.categorical_column_with_vocabulary_list} -* @{tf.feature_column.categorical_column_with_vocabulary_file} - -`categorical_column_with_vocabulary_list` maps each string to an integer based -on an explicit vocabulary list. For example: - -```python -# Given input "feature_name_from_input_fn" which is a string, -# create a categorical feature by mapping the input to one of -# the elements in the vocabulary list. -vocabulary_feature_column = - tf.feature_column.categorical_column_with_vocabulary_list( - key=feature_name_from_input_fn, - vocabulary_list=["kitchenware", "electronics", "sports"]) -``` - -The preceding function is pretty straightforward, but it has a significant -drawback. Namely, there's way too much typing when the vocabulary list is long. -For these cases, call -`tf.feature_column.categorical_column_with_vocabulary_file` instead, which lets -you place the vocabulary words in a separate file. For example: - -```python - -# Given input "feature_name_from_input_fn" which is a string, -# create a categorical feature to our model by mapping the input to one of -# the elements in the vocabulary file -vocabulary_feature_column = - tf.feature_column.categorical_column_with_vocabulary_file( - key=feature_name_from_input_fn, - vocabulary_file="product_class.txt", - vocabulary_size=3) -``` - -`product_class.txt` should contain one line for each vocabulary element. In our -case: - -```None -kitchenware -electronics -sports -``` - -### Hashed Column - -So far, we've worked with a naively small number of categories. For example, -our product_class example has only 3 categories. Often though, the number of -categories can be so big that it's not possible to have individual categories -for each vocabulary word or integer because that would consume too much memory. -For these cases, we can instead turn the question around and ask, "How many -categories am I willing to have for my input?" In fact, the -@{tf.feature_column.categorical_column_with_hash_bucket} function enables you -to specify the number of categories. For this type of feature column the model -calculates a hash value of the input, then puts it into one of -the `hash_bucket_size` categories using the modulo operator, as in the following -pseudocode: - -```python -# pseudocode -feature_id = hash(raw_feature) % hash_buckets_size -``` - -The code to create the `feature_column` might look something like this: - -``` python -hashed_feature_column = - tf.feature_column.categorical_column_with_hash_bucket( - key = "some_feature", - hash_buckets_size = 100) # The number of categories -``` -At this point, you might rightfully think: "This is crazy!" After all, we are -forcing the different input values to a smaller set of categories. This means -that two probably unrelated inputs will be mapped to the same -category, and consequently mean the same thing to the neural network. The -following figure illustrates this dilemma, showing that kitchenware and sports -both get assigned to category (hash bucket) 12: - -
- -
-
-Representing data with hash buckets. -
- -As with many counterintuitive phenomena in machine learning, it turns out that -hashing often works well in practice. That's because hash categories provide -the model with some separation. The model can use additional features to further -separate kitchenware from sports. - -### Crossed column - -Combining features into a single feature, better known as -[feature crosses](https://developers.google.com/machine-learning/glossary/#feature_cross), -enables the model to learn separate weights for each combination of -features. - -More concretely, suppose we want our model to calculate real estate prices in -Atlanta, GA. Real-estate prices within this city vary greatly depending on -location. Representing latitude and longitude as separate features isn't very -useful in identifying real-estate location dependencies; however, crossing -latitude and longitude into a single feature can pinpoint locations. Suppose we -represent Atlanta as a grid of 100x100 rectangular sections, identifying each -of the 10,000 sections by a feature cross of latitude and longitude. This -feature cross enables the model to train on pricing conditions related to each -individual section, which is a much stronger signal than latitude and longitude -alone. - -The following figure shows our plan, with the latitude & longitude values for -the corners of the city in red text: - -
- -
-
-Map of Atlanta. Imagine this map divided into 10,000 sections of -equal size. -
- -For the solution, we used a combination of the `bucketized_column` we looked at -earlier, with the @{tf.feature_column.crossed_column} function. - - - -``` python -def make_dataset(latitude, longitude, labels): - assert latitude.shape == longitude.shape == labels.shape - - features = {'latitude': latitude.flatten(), - 'longitude': longitude.flatten()} - labels=labels.flatten() - - return tf.data.Dataset.from_tensor_slices((features, labels)) - - -# Bucketize the latitude and longitude using the `edges` -latitude_bucket_fc = tf.feature_column.bucketized_column( - tf.feature_column.numeric_column('latitude'), - list(atlanta.latitude.edges)) - -longitude_bucket_fc = tf.feature_column.bucketized_column( - tf.feature_column.numeric_column('longitude'), - list(atlanta.longitude.edges)) - -# Cross the bucketized columns, using 5000 hash bins. -crossed_lat_lon_fc = tf.feature_column.crossed_column( - [latitude_bucket_fc, longitude_bucket_fc], 5000) - -fc = [ - latitude_bucket_fc, - longitude_bucket_fc, - crossed_lat_lon_fc] - -# Build and train the Estimator. -est = tf.estimator.LinearRegressor(fc, ...) -``` - -You may create a feature cross from either of the following: - -* Feature names; that is, names from the `dict` returned from `input_fn`. -* Any categorical column, except `categorical_column_with_hash_bucket` - (since `crossed_column` hashes the input). - -When the feature columns `latitude_bucket_fc` and `longitude_bucket_fc` are -crossed, TensorFlow will create `(latitude_fc, longitude_fc)` pairs for each -example. This would produce a full grid of possibilities as follows: - -``` None - (0,0), (0,1)... (0,99) - (1,0), (1,1)... (1,99) - ... ... ... -(99,0), (99,1)...(99, 99) -``` - -Except that a full grid would only be tractable for inputs with limited -vocabularies. Instead of building this, potentially huge, table of inputs, -the `crossed_column` only builds the number requested by the `hash_bucket_size` -argument. The feature column assigns an example to a index by running a hash -function on the tuple of inputs, followed by a modulo operation with -`hash_bucket_size`. - -As discussed earlier, performing the -hash and modulo function limits the number of categories, but can cause category -collisions; that is, multiple (latitude, longitude) feature crosses will end -up in the same hash bucket. In practice though, performing feature crosses -still adds significant value to the learning capability of your models. - -Somewhat counterintuitively, when creating feature crosses, you typically still -should include the original (uncrossed) features in your model (as in the -preceding code snippet). The independent latitude and longitude features help the -model distinguish between examples where a hash collision has occurred in the -crossed feature. - -## Indicator and embedding columns - -Indicator columns and embedding columns never work on features directly, but -instead take categorical columns as input. - -When using an indicator column, we're telling TensorFlow to do exactly what -we've seen in our categorical product_class example. That is, an -**indicator column** treats each category as an element in a one-hot vector, -where the matching category has value 1 and the rest have 0s: - -
- -
-
-Representing data in indicator columns. -
- -Here's how you create an indicator column by calling -@{tf.feature_column.indicator_column}: - -``` python -categorical_column = ... # Create any type of categorical column. - -# Represent the categorical column as an indicator column. -indicator_column = tf.feature_column.indicator_column(categorical_column) -``` - -Now, suppose instead of having just three possible classes, we have a million. -Or maybe a billion. For a number of reasons, as the number of categories grow -large, it becomes infeasible to train a neural network using indicator columns. - -We can use an embedding column to overcome this limitation. Instead of -representing the data as a one-hot vector of many dimensions, an -**embedding column** represents that data as a lower-dimensional, ordinary -vector in which each cell can contain any number, not just 0 or 1. By -permitting a richer palette of numbers for every cell, an embedding column -contains far fewer cells than an indicator column. - -Let's look at an example comparing indicator and embedding columns. Suppose our -input examples consist of different words from a limited palette of only 81 -words. Further suppose that the data set provides the following input -words in 4 separate examples: - -* `"dog"` -* `"spoon"` -* `"scissors"` -* `"guitar"` - -In that case, the following figure illustrates the processing path for -embedding columns or indicator columns. - -
- -
-
-An embedding column stores categorical data in a lower-dimensional -vector than an indicator column. (We just placed random numbers into the -embedding vectors; training determines the actual numbers.) -
- -When an example is processed, one of the `categorical_column_with...` functions -maps the example string to a numerical categorical value. For example, a -function maps "spoon" to `[32]`. (The 32 comes from our imagination—the actual -values depend on the mapping function.) You may then represent these numerical -categorical values in either of the following two ways: - -* As an indicator column. A function converts each numeric categorical value - into an 81-element vector (because our palette consists of 81 words), placing - a 1 in the index of the categorical value (0, 32, 79, 80) and a 0 in all the - other positions. - -* As an embedding column. A function uses the numerical categorical values - `(0, 32, 79, 80)` as indices to a lookup table. Each slot in that lookup table - contains a 3-element vector. - -How do the values in the embeddings vectors magically get assigned? Actually, -the assignments happen during training. That is, the model learns the best way -to map your input numeric categorical values to the embeddings vector value in -order to solve your problem. Embedding columns increase your model's -capabilities, since an embeddings vector learns new relationships between -categories from the training data. - -Why is the embedding vector size 3 in our example? Well, the following "formula" -provides a general rule of thumb about the number of embedding dimensions: - -```python -embedding_dimensions = number_of_categories**0.25 -``` - -That is, the embedding vector dimension should be the 4th root of the number of -categories. Since our vocabulary size in this example is 81, the recommended -number of dimensions is 3: - -``` python -3 = 81**0.25 -``` -Note that this is just a general guideline; you can set the number of embedding -dimensions as you please. - -Call @{tf.feature_column.embedding_column} to create an `embedding_column` as -suggested by the following snippet: - -``` python -categorical_column = ... # Create any categorical column - -# Represent the categorical column as an embedding column. -# This means creating an embedding vector lookup table with one element for each category. -embedding_column = tf.feature_column.embedding_column( - categorical_column=categorical_column, - dimension=embedding_dimensions) -``` - -@{$programmers_guide/embedding$Embeddings} is a significant topic within machine -learning. This information was just to get you started using them as feature -columns. - -## Passing feature columns to Estimators - -As the following list indicates, not all Estimators permit all types of -`feature_columns` argument(s): - -* @{tf.estimator.LinearClassifier$`LinearClassifier`} and - @{tf.estimator.LinearRegressor$`LinearRegressor`}: Accept all types of - feature column. -* @{tf.estimator.DNNClassifier$`DNNClassifier`} and - @{tf.estimator.DNNRegressor$`DNNRegressor`}: Only accept dense columns. Other - column types must be wrapped in either an `indicator_column` or - `embedding_column`. -* @{tf.estimator.DNNLinearCombinedClassifier$`DNNLinearCombinedClassifier`} and - @{tf.estimator.DNNLinearCombinedRegressor$`DNNLinearCombinedRegressor`}: - * The `linear_feature_columns` argument accepts any feature column type. - * The `dnn_feature_columns` argument only accepts dense columns. - -## Other Sources - -For more examples on feature columns, view the following: - -* The @{$low_level_intro#feature_columns$Low Level Introduction} demonstrates how - experiment directly with `feature_columns` using TensorFlow's low level APIs. -* The @{$wide$wide} and @{$wide_and_deep$Wide & Deep} Tutorials solve a - binary classification problem using `feature_columns` on a variety of input - data types. - -To learn more about embeddings, see the following: - -* [Deep Learning, NLP, and representations](http://colah.github.io/posts/2014-07-NLP-RNNs-Representations/) - (Chris Olah's blog) -* The TensorFlow [Embedding Projector](http://projector.tensorflow.org) diff --git a/tensorflow/docs_src/programmers_guide/graph_viz.md b/tensorflow/docs_src/programmers_guide/graph_viz.md deleted file mode 100644 index f581ae56da..0000000000 --- a/tensorflow/docs_src/programmers_guide/graph_viz.md +++ /dev/null @@ -1,316 +0,0 @@ -# TensorBoard: Graph Visualization - -TensorFlow computation graphs are powerful but complicated. The graph visualization can help you understand and debug them. Here's an example of the visualization at work. - -![Visualization of a TensorFlow graph](https://www.tensorflow.org/images/graph_vis_animation.gif "Visualization of a TensorFlow graph") -*Visualization of a TensorFlow graph.* - -To see your own graph, run TensorBoard pointing it to the log directory of the job, click on the graph tab on the top pane and select the appropriate run using the menu at the upper left corner. For in depth information on how to run TensorBoard and make sure you are logging all the necessary information, see @{$summaries_and_tensorboard$TensorBoard: Visualizing Learning}. - -## Name scoping and nodes - -Typical TensorFlow graphs can have many thousands of nodes--far too many to see -easily all at once, or even to lay out using standard graph tools. To simplify, -variable names can be scoped and the visualization uses this information to -define a hierarchy on the nodes in the graph. By default, only the top of this -hierarchy is shown. Here is an example that defines three operations under the -`hidden` name scope using -@{tf.name_scope}: - -```python -import tensorflow as tf - -with tf.name_scope('hidden') as scope: - a = tf.constant(5, name='alpha') - W = tf.Variable(tf.random_uniform([1, 2], -1.0, 1.0), name='weights') - b = tf.Variable(tf.zeros([1]), name='biases') -``` - -This results in the following three op names: - -* `hidden/alpha` -* `hidden/weights` -* `hidden/biases` - -By default, the visualization will collapse all three into a node labeled `hidden`. -The extra detail isn't lost. You can double-click, or click -on the orange `+` sign in the top right to expand the node, and then you'll see -three subnodes for `alpha`, `weights` and `biases`. - -Here's a real-life example of a more complicated node in its initial and -expanded states. - - - - - - - - - - -
- Unexpanded name scope - - Expanded name scope -
- Initial view of top-level name scope pool_1. Clicking on the orange + button on the top right or double-clicking on the node itself will expand it. - - Expanded view of pool_1 name scope. Clicking on the orange - button on the top right or double-clicking on the node itself will collapse the name scope. -
- -Grouping nodes by name scopes is critical to making a legible graph. If you're -building a model, name scopes give you control over the resulting visualization. -**The better your name scopes, the better your visualization.** - -The figure above illustrates a second aspect of the visualization. TensorFlow -graphs have two kinds of connections: data dependencies and control -dependencies. Data dependencies show the flow of tensors between two ops and -are shown as solid arrows, while control dependencies use dotted lines. In the -expanded view (right side of the figure above) all the connections are data -dependencies with the exception of the dotted line connecting `CheckNumerics` -and `control_dependency`. - -There's a second trick to simplifying the layout. Most TensorFlow graphs have a -few nodes with many connections to other nodes. For example, many nodes might -have a control dependency on an initialization step. Drawing all edges between -the `init` node and its dependencies would create a very cluttered view. - -To reduce clutter, the visualization separates out all high-degree nodes to an -*auxiliary* area on the right and doesn't draw lines to represent their edges. -Instead of lines, we draw small *node icons* to indicate the connections. -Separating out the auxiliary nodes typically doesn't remove critical -information since these nodes are usually related to bookkeeping functions. -See [Interaction](#interaction) for how to move nodes between the main graph -and the auxiliary area. - - - - - - - - - - -
- conv_1 is part of the main graph - - save is extracted as auxiliary node -
- Node conv_1 is connected to save. Note the little save node icon on its right. - - save has a high degree, and will appear as an auxiliary node. The connection with conv_1 is shown as a node icon on its left. To further reduce clutter, since save has a lot of connections, we show the first 5 and abbreviate the others as ... 12 more. -
- -One last structural simplification is *series collapsing*. Sequential -motifs--that is, nodes whose names differ by a number at the end and have -isomorphic structures--are collapsed into a single *stack* of nodes, as shown -below. For networks with long sequences, this greatly simplifies the view. As -with hierarchical nodes, double-clicking expands the series. See -[Interaction](#interaction) for how to disable/enable series collapsing for a -specific set of nodes. - - - - - - - - - - -
- Sequence of nodes - - Expanded sequence of nodes -
- A collapsed view of a node sequence. - - A small piece of the expanded view, after double-click. -
- -Finally, as one last aid to legibility, the visualization uses special icons -for constants and summary nodes. To summarize, here's a table of node symbols: - -Symbol | Meaning ---- | --- -![Name scope](https://www.tensorflow.org/images/namespace_node.png "Name scope") | *High-level* node representing a name scope. Double-click to expand a high-level node. -![Sequence of unconnected nodes](https://www.tensorflow.org/images/horizontal_stack.png "Sequence of unconnected nodes") | Sequence of numbered nodes that are not connected to each other. -![Sequence of connected nodes](https://www.tensorflow.org/images/vertical_stack.png "Sequence of connected nodes") | Sequence of numbered nodes that are connected to each other. -![Operation node](https://www.tensorflow.org/images/op_node.png "Operation node") | An individual operation node. -![Constant node](https://www.tensorflow.org/images/constant.png "Constant node") | A constant. -![Summary node](https://www.tensorflow.org/images/summary.png "Summary node") | A summary node. -![Data flow edge](https://www.tensorflow.org/images/dataflow_edge.png "Data flow edge") | Edge showing the data flow between operations. -![Control dependency edge](https://www.tensorflow.org/images/control_edge.png "Control dependency edge") | Edge showing the control dependency between operations. -![Reference edge](https://www.tensorflow.org/images/reference_edge.png "Reference edge") | A reference edge showing that the outgoing operation node can mutate the incoming tensor. - -## Interaction {#interaction} - -Navigate the graph by panning and zooming. Click and drag to pan, and use a -scroll gesture to zoom. Double-click on a node, or click on its `+` button, to -expand a name scope that represents a group of operations. To easily keep -track of the current viewpoint when zooming and panning, there is a minimap in -the bottom right corner. - -To close an open node, double-click it again or click its `-` button. You can -also click once to select a node. It will turn a darker color, and details -about it and the nodes it connects to will appear in the info card at upper -right corner of the visualization. - - - - - - - - - - -
- Info card of a name scope - - Info card of operation node -
- Info card showing detailed information for the conv2 name scope. The inputs and outputs are combined from the inputs and outputs of the operation nodes inside the name scope. For name scopes no attributes are shown. - - Info card showing detailed information for the DecodeRaw operation node. In addition to inputs and outputs, the card shows the device and the attributes associated with the current operation. -
- -TensorBoard provides several ways to change the visual layout of the graph. This -doesn't change the graph's computational semantics, but it can bring some -clarity to the network's structure. By right clicking on a node or pressing -buttons on the bottom of that node's info card, you can make the following -changes to its layout: - -* Nodes can be moved between the main graph and the auxiliary area. -* A series of nodes can be ungrouped so that the nodes in the series do not -appear grouped together. Ungrouped series can likewise be regrouped. - -Selection can also be helpful in understanding high-degree nodes. Select any -high-degree node, and the corresponding node icons for its other connections -will be selected as well. This makes it easy, for example, to see which nodes -are being saved--and which aren't. - -Clicking on a node name in the info card will select it. If necessary, the -viewpoint will automatically pan so that the node is visible. - -Finally, you can choose two color schemes for your graph, using the color menu -above the legend. The default *Structure View* shows structure: when two -high-level nodes have the same structure, they appear in the same color of the -rainbow. Uniquely structured nodes are gray. There's a second view, which shows -what device the different operations run on. Name scopes are colored -proportionally to the fraction of devices for the operations inside them. - -The images below give an illustration for a piece of a real-life graph. - - - - - - - - - - -
- Color by structure - - Color by device -
- Structure view: The gray nodes have unique structure. The orange conv1 and conv2 nodes have the same structure, and analogously for nodes with other colors. - - Device view: Name scopes are colored proportionally to the fraction of devices of the operation nodes inside them. Here, purple means GPU and the green is CPU. -
- -## Tensor shape information - -When the serialized `GraphDef` includes tensor shapes, the graph visualizer -labels edges with tensor dimensions, and edge thickness reflects total tensor -size. To include tensor shapes in the `GraphDef` pass the actual graph object -(as in `sess.graph`) to the `FileWriter` when serializing the graph. -The images below show the CIFAR-10 model with tensor shape information: - - - - - - - -
- CIFAR-10 model with tensor shape information -
- CIFAR-10 model with tensor shape information. -
- -## Runtime statistics - -Often it is useful to collect runtime metadata for a run, such as total memory -usage, total compute time, and tensor shapes for nodes. The code example below -is a snippet from the train and test section of a modification of the -@{$layers$simple MNIST tutorial}, in which we have recorded summaries and -runtime statistics. See the -@{$summaries_and_tensorboard#serializing-the-data$Summaries Tutorial} -for details on how to record summaries. -Full source is [here](https://www.tensorflow.org/code/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py). - -```python - # Train the model, and also write summaries. - # Every 10th step, measure test-set accuracy, and write test summaries - # All other steps, run train_step on training data, & add training summaries - - def feed_dict(train): - """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" - if train or FLAGS.fake_data: - xs, ys = mnist.train.next_batch(100, fake_data=FLAGS.fake_data) - k = FLAGS.dropout - else: - xs, ys = mnist.test.images, mnist.test.labels - k = 1.0 - return {x: xs, y_: ys, keep_prob: k} - - for i in range(FLAGS.max_steps): - if i % 10 == 0: # Record summaries and test-set accuracy - summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False)) - test_writer.add_summary(summary, i) - print('Accuracy at step %s: %s' % (i, acc)) - else: # Record train set summaries, and train - if i % 100 == 99: # Record execution stats - run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) - run_metadata = tf.RunMetadata() - summary, _ = sess.run([merged, train_step], - feed_dict=feed_dict(True), - options=run_options, - run_metadata=run_metadata) - train_writer.add_run_metadata(run_metadata, 'step%d' % i) - train_writer.add_summary(summary, i) - print('Adding run metadata for', i) - else: # Record a summary - summary, _ = sess.run([merged, train_step], feed_dict=feed_dict(True)) - train_writer.add_summary(summary, i) -``` - -This code will emit runtime statistics for every 100th step starting at step99. - -When you launch tensorboard and go to the Graph tab, you will now see options -under "Session runs" which correspond to the steps where run metadata was added. -Selecting one of these runs will show you the snapshot of the network at that -step, fading out unused nodes. In the controls on the left hand side, you will -be able to color the nodes by total memory or total compute time. Additionally, -clicking on a node will display the exact total memory, compute time, and -tensor output sizes. - - - - - - - - -
- Color by compute time - - Run metadata graph - - Run metadata info card -
diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md deleted file mode 100644 index f0dd8def17..0000000000 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ /dev/null @@ -1,558 +0,0 @@ -# Graphs and Sessions - -TensorFlow uses a **dataflow graph** to represent your computation in terms of -the dependencies between individual operations. This leads to a low-level -programming model in which you first define the dataflow graph, then create a -TensorFlow **session** to run parts of the graph across a set of local and -remote devices. - -This guide will be most useful if you intend to use the low-level programming -model directly. Higher-level APIs such as @{tf.estimator.Estimator} and Keras -hide the details of graphs and sessions from the end user, but this guide may -also be useful if you want to understand how these APIs are implemented. - -## Why dataflow graphs? - -![](../images/tensors_flowing.gif) - -[Dataflow](https://en.wikipedia.org/wiki/Dataflow_programming) is a common -programming model for parallel computing. In a dataflow graph, the nodes -represent units of computation, and the edges represent the data consumed or -produced by a computation. For example, in a TensorFlow graph, the @{tf.matmul} -operation would correspond to a single node with two incoming edges (the -matrices to be multiplied) and one outgoing edge (the result of the -multiplication). - - - -Dataflow has several advantages that TensorFlow leverages when executing your -programs: - -* **Parallelism.** By using explicit edges to represent dependencies between - operations, it is easy for the system to identify operations that can execute - in parallel. - -* **Distributed execution.** By using explicit edges to represent the values - that flow between operations, it is possible for TensorFlow to partition your - program across multiple devices (CPUs, GPUs, and TPUs) attached to different - machines. TensorFlow inserts the necessary communication and coordination - between devices. - -* **Compilation.** TensorFlow's @{$performance/xla$XLA compiler} can - use the information in your dataflow graph to generate faster code, for - example, by fusing together adjacent operations. - -* **Portability.** The dataflow graph is a language-independent representation - of the code in your model. You can build a dataflow graph in Python, store it - in a @{$saved_model$SavedModel}, and restore it in a C++ program for - low-latency inference. - - -## What is a @{tf.Graph}? - -A @{tf.Graph} contains two relevant kinds of information: - -* **Graph structure.** The nodes and edges of the graph, indicating how - individual operations are composed together, but not prescribing how they - should be used. The graph structure is like assembly code: inspecting it can - convey some useful information, but it does not contain all of the useful - context that source code conveys. - -* **Graph collections.** TensorFlow provides a general mechanism for storing - collections of metadata in a @{tf.Graph}. The @{tf.add_to_collection} function - enables you to associate a list of objects with a key (where @{tf.GraphKeys} - defines some of the standard keys), and @{tf.get_collection} enables you to - look up all objects associated with a key. Many parts of the TensorFlow - library use this facility: for example, when you create a @{tf.Variable}, it - is added by default to collections representing "global variables" and - "trainable variables". When you later come to create a @{tf.train.Saver} or - @{tf.train.Optimizer}, the variables in these collections are used as the - default arguments. - - -## Building a @{tf.Graph} - -Most TensorFlow programs start with a dataflow graph construction phase. In this -phase, you invoke TensorFlow API functions that construct new @{tf.Operation} -(node) and @{tf.Tensor} (edge) objects and add them to a @{tf.Graph} -instance. TensorFlow provides a **default graph** that is an implicit argument -to all API functions in the same context. For example: - -* Calling `tf.constant(42.0)` creates a single @{tf.Operation} that produces the - value `42.0`, adds it to the default graph, and returns a @{tf.Tensor} that - represents the value of the constant. - -* Calling `tf.matmul(x, y)` creates a single @{tf.Operation} that multiplies - the values of @{tf.Tensor} objects `x` and `y`, adds it to the default graph, - and returns a @{tf.Tensor} that represents the result of the multiplication. - -* Executing `v = tf.Variable(0)` adds to the graph a @{tf.Operation} that will - store a writeable tensor value that persists between @{tf.Session.run} calls. - The @{tf.Variable} object wraps this operation, and can be used [like a - tensor](#tensor-like_objects), which will read the current value of the - stored value. The @{tf.Variable} object also has methods such as - @{tf.Variable.assign$`assign`} and @{tf.Variable.assign_add$`assign_add`} that - create @{tf.Operation} objects that, when executed, update the stored value. - (See @{$programmers_guide/variables} for more information about variables.) - -* Calling @{tf.train.Optimizer.minimize} will add operations and tensors to the - default graph that calculates gradients, and return a @{tf.Operation} that, - when run, will apply those gradients to a set of variables. - -Most programs rely solely on the default graph. However, -see [Dealing with multiple graphs](#programming_with_multiple_graphs) for more -advanced use cases. High-level APIs such as the @{tf.estimator.Estimator} API -manage the default graph on your behalf, and--for example--may create different -graphs for training and evaluation. - -Note: Calling most functions in the TensorFlow API merely adds operations -and tensors to the default graph, but **does not** perform the actual -computation. Instead, you compose these functions until you have a @{tf.Tensor} -or @{tf.Operation} that represents the overall computation--such as performing -one step of gradient descent--and then pass that object to a @{tf.Session} to -perform the computation. See the section "Executing a graph in a @{tf.Session}" -for more details. - -## Naming operations - -A @{tf.Graph} object defines a **namespace** for the @{tf.Operation} objects it -contains. TensorFlow automatically chooses a unique name for each operation in -your graph, but giving operations descriptive names can make your program easier -to read and debug. The TensorFlow API provides two ways to override the name of -an operation: - -* Each API function that creates a new @{tf.Operation} or returns a new - @{tf.Tensor} accepts an optional `name` argument. For example, - `tf.constant(42.0, name="answer")` creates a new @{tf.Operation} named - `"answer"` and returns a @{tf.Tensor} named `"answer:0"`. If the default graph - already contains an operation named `"answer"`, then TensorFlow would append - `"_1"`, `"_2"`, and so on to the name, in order to make it unique. - -* The @{tf.name_scope} function makes it possible to add a **name scope** prefix - to all operations created in a particular context. The current name scope - prefix is a `"/"`-delimited list of the names of all active @{tf.name_scope} - context managers. If a name scope has already been used in the current - context, TensorFlow appends `"_1"`, `"_2"`, and so on. For example: - - ```python - c_0 = tf.constant(0, name="c") # => operation named "c" - - # Already-used names will be "uniquified". - c_1 = tf.constant(2, name="c") # => operation named "c_1" - - # Name scopes add a prefix to all operations created in the same context. - with tf.name_scope("outer"): - c_2 = tf.constant(2, name="c") # => operation named "outer/c" - - # Name scopes nest like paths in a hierarchical file system. - with tf.name_scope("inner"): - c_3 = tf.constant(3, name="c") # => operation named "outer/inner/c" - - # Exiting a name scope context will return to the previous prefix. - c_4 = tf.constant(4, name="c") # => operation named "outer/c_1" - - # Already-used name scopes will be "uniquified". - with tf.name_scope("inner"): - c_5 = tf.constant(5, name="c") # => operation named "outer/inner_1/c" - ``` - -The graph visualizer uses name scopes to group operations and reduce the visual -complexity of a graph. See [Visualizing your graph](#visualizing-your-graph) for -more information. - -Note that @{tf.Tensor} objects are implicitly named after the @{tf.Operation} -that produces the tensor as output. A tensor name has the form `":"` -where: - -* `""` is the name of the operation that produces it. -* `""` is an integer representing the index of that tensor among the - operation's outputs. - -## Placing operations on different devices - -If you want your TensorFlow program to use multiple different devices, the -@{tf.device} function provides a convenient way to request that all operations -created in a particular context are placed on the same device (or type of -device). - -A **device specification** has the following form: - -``` -/job:/task:/device:: -``` - -where: - -* `` is an alpha-numeric string that does not start with a number. -* `` is a registered device type (such as `GPU` or `CPU`). -* `` is a non-negative integer representing the index of the task - in the job named ``. See @{tf.train.ClusterSpec} for an explanation - of jobs and tasks. -* `` is a non-negative integer representing the index of the - device, for example, to distinguish between different GPU devices used in the - same process. - -You do not need to specify every part of a device specification. For example, -if you are running in a single-machine configuration with a single GPU, you -might use @{tf.device} to pin some operations to the CPU and GPU: - -```python -# Operations created outside either context will run on the "best possible" -# device. For example, if you have a GPU and a CPU available, and the operation -# has a GPU implementation, TensorFlow will choose the GPU. -weights = tf.random_normal(...) - -with tf.device("/device:CPU:0"): - # Operations created in this context will be pinned to the CPU. - img = tf.decode_jpeg(tf.read_file("img.jpg")) - -with tf.device("/device:GPU:0"): - # Operations created in this context will be pinned to the GPU. - result = tf.matmul(weights, img) -``` -If you are deploying TensorFlow in a @{$distributed$typical distributed configuration}, -you might specify the job name and task ID to place variables on -a task in the parameter server job (`"/job:ps"`), and the other operations on -task in the worker job (`"/job:worker"`): - -```python -with tf.device("/job:ps/task:0"): - weights_1 = tf.Variable(tf.truncated_normal([784, 100])) - biases_1 = tf.Variable(tf.zeroes([100])) - -with tf.device("/job:ps/task:1"): - weights_2 = tf.Variable(tf.truncated_normal([100, 10])) - biases_2 = tf.Variable(tf.zeroes([10])) - -with tf.device("/job:worker"): - layer_1 = tf.matmul(train_batch, weights_1) + biases_1 - layer_2 = tf.matmul(train_batch, weights_2) + biases_2 -``` - -@{tf.device} gives you a lot of flexibility to choose placements for individual -operations or broad regions of a TensorFlow graph. In many cases, there are -simple heuristics that work well. For example, the -@{tf.train.replica_device_setter} API can be used with @{tf.device} to place -operations for **data-parallel distributed training**. For example, the -following code fragment shows how @{tf.train.replica_device_setter} applies -different placement policies to @{tf.Variable} objects and other operations: - -```python -with tf.device(tf.train.replica_device_setter(ps_tasks=3)): - # tf.Variable objects are, by default, placed on tasks in "/job:ps" in a - # round-robin fashion. - w_0 = tf.Variable(...) # placed on "/job:ps/task:0" - b_0 = tf.Variable(...) # placed on "/job:ps/task:1" - w_1 = tf.Variable(...) # placed on "/job:ps/task:2" - b_1 = tf.Variable(...) # placed on "/job:ps/task:0" - - input_data = tf.placeholder(tf.float32) # placed on "/job:worker" - layer_0 = tf.matmul(input_data, w_0) + b_0 # placed on "/job:worker" - layer_1 = tf.matmul(layer_0, w_1) + b_1 # placed on "/job:worker" -``` - -## Tensor-like objects - -Many TensorFlow operations take one or more @{tf.Tensor} objects as arguments. -For example, @{tf.matmul} takes two @{tf.Tensor} objects, and @{tf.add_n} takes -a list of `n` @{tf.Tensor} objects. For convenience, these functions will accept -a **tensor-like object** in place of a @{tf.Tensor}, and implicitly convert it -to a @{tf.Tensor} using the @{tf.convert_to_tensor} method. Tensor-like objects -include elements of the following types: - -* @{tf.Tensor} -* @{tf.Variable} -* [`numpy.ndarray`](https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.html) -* `list` (and lists of tensor-like objects) -* Scalar Python types: `bool`, `float`, `int`, `str` - -You can register additional tensor-like types using -@{tf.register_tensor_conversion_function}. - -Note: By default, TensorFlow will create a new @{tf.Tensor} each time you use -the same tensor-like object. If the tensor-like object is large (e.g. a -`numpy.ndarray` containing a set of training examples) and you use it multiple -times, you may run out of memory. To avoid this, manually call -@{tf.convert_to_tensor} on the tensor-like object once and use the returned -@{tf.Tensor} instead. - -## Executing a graph in a @{tf.Session} - -TensorFlow uses the @{tf.Session} class to represent a connection between the -client program---typically a Python program, although a similar interface is -available in other languages---and the C++ runtime. A @{tf.Session} object -provides access to devices in the local machine, and remote devices using the -distributed TensorFlow runtime. It also caches information about your -@{tf.Graph} so that you can efficiently run the same computation multiple times. - -### Creating a @{tf.Session} - -If you are using the low-level TensorFlow API, you can create a @{tf.Session} -for the current default graph as follows: - -```python -# Create a default in-process session. -with tf.Session() as sess: - # ... - -# Create a remote session. -with tf.Session("grpc://example.org:2222"): - # ... -``` - -Since a @{tf.Session} owns physical resources (such as GPUs and -network connections), it is typically used as a context manager (in a `with` -block) that automatically closes the session when you exit the block. It is -also possible to create a session without using a `with` block, but you should -explicitly call @{tf.Session.close} when you are finished with it to free the -resources. - -Note: Higher-level APIs such as @{tf.train.MonitoredTrainingSession} or -@{tf.estimator.Estimator} will create and manage a @{tf.Session} for you. These -APIs accept optional `target` and `config` arguments (either directly, or as -part of a @{tf.estimator.RunConfig} object), with the same meaning as -described below. - -@{tf.Session.__init__} accepts three optional arguments: - -* **`target`.** If this argument is left empty (the default), the session will - only use devices in the local machine. However, you may also specify a - `grpc://` URL to specify the address of a TensorFlow server, which gives the - session access to all devices on machines that this server controls. See - @{tf.train.Server} for details of how to create a TensorFlow - server. For example, in the common **between-graph replication** - configuration, the @{tf.Session} connects to a @{tf.train.Server} in the same - process as the client. The [distributed TensorFlow](../deploy/distributed.md) - deployment guide describes other common scenarios. - -* **`graph`.** By default, a new @{tf.Session} will be bound to---and only able - to run operations in---the current default graph. If you are using multiple - graphs in your program (see [Programming with multiple - graphs](#programming_with_multiple_graphs) for more details), you can specify - an explicit @{tf.Graph} when you construct the session. - -* **`config`.** This argument allows you to specify a @{tf.ConfigProto} that - controls the behavior of the session. For example, some of the configuration - options include: - - * `allow_soft_placement`. Set this to `True` to enable a "soft" device - placement algorithm, which ignores @{tf.device} annotations that attempt - to place CPU-only operations on a GPU device, and places them on the CPU - instead. - - * `cluster_def`. When using distributed TensorFlow, this option allows you - to specify what machines to use in the computation, and provide a mapping - between job names, task indices, and network addresses. See - @{tf.train.ClusterSpec.as_cluster_def} for details. - - * `graph_options.optimizer_options`. Provides control over the optimizations - that TensorFlow performs on your graph before executing it. - - * `gpu_options.allow_growth`. Set this to `True` to change the GPU memory - allocator so that it gradually increases the amount of memory allocated, - rather than allocating most of the memory at startup. - - -### Using @{tf.Session.run} to execute operations - -The @{tf.Session.run} method is the main mechanism for running a @{tf.Operation} -or evaluating a @{tf.Tensor}. You can pass one or more @{tf.Operation} or -@{tf.Tensor} objects to @{tf.Session.run}, and TensorFlow will execute the -operations that are needed to compute the result. - -@{tf.Session.run} requires you to specify a list of **fetches**, which determine -the return values, and may be a @{tf.Operation}, a @{tf.Tensor}, or -a [tensor-like type](#tensor-like_objects) such as @{tf.Variable}. These fetches -determine what **subgraph** of the overall @{tf.Graph} must be executed to -produce the result: this is the subgraph that contains all operations named in -the fetch list, plus all operations whose outputs are used to compute the value -of the fetches. For example, the following code fragment shows how different -arguments to @{tf.Session.run} cause different subgraphs to be executed: - -```python -x = tf.constant([[37.0, -23.0], [1.0, 4.0]]) -w = tf.Variable(tf.random_uniform([2, 2])) -y = tf.matmul(x, w) -output = tf.nn.softmax(y) -init_op = w.initializer - -with tf.Session() as sess: - # Run the initializer on `w`. - sess.run(init_op) - - # Evaluate `output`. `sess.run(output)` will return a NumPy array containing - # the result of the computation. - print(sess.run(output)) - - # Evaluate `y` and `output`. Note that `y` will only be computed once, and its - # result used both to return `y_val` and as an input to the `tf.nn.softmax()` - # op. Both `y_val` and `output_val` will be NumPy arrays. - y_val, output_val = sess.run([y, output]) -``` - -@{tf.Session.run} also optionally takes a dictionary of **feeds**, which is a -mapping from @{tf.Tensor} objects (typically @{tf.placeholder} tensors) to -values (typically Python scalars, lists, or NumPy arrays) that will be -substituted for those tensors in the execution. For example: - -```python -# Define a placeholder that expects a vector of three floating-point values, -# and a computation that depends on it. -x = tf.placeholder(tf.float32, shape=[3]) -y = tf.square(x) - -with tf.Session() as sess: - # Feeding a value changes the result that is returned when you evaluate `y`. - print(sess.run(y, {x: [1.0, 2.0, 3.0]})) # => "[1.0, 4.0, 9.0]" - print(sess.run(y, {x: [0.0, 0.0, 5.0]})) # => "[0.0, 0.0, 25.0]" - - # Raises `tf.errors.InvalidArgumentError`, because you must feed a value for - # a `tf.placeholder()` when evaluating a tensor that depends on it. - sess.run(y) - - # Raises `ValueError`, because the shape of `37.0` does not match the shape - # of placeholder `x`. - sess.run(y, {x: 37.0}) -``` - -@{tf.Session.run} also accepts an optional `options` argument that enables you -to specify options about the call, and an optional `run_metadata` argument that -enables you to collect metadata about the execution. For example, you can use -these options together to collect tracing information about the execution: - -``` -y = tf.matmul([[37.0, -23.0], [1.0, 4.0]], tf.random_uniform([2, 2])) - -with tf.Session() as sess: - # Define options for the `sess.run()` call. - options = tf.RunOptions() - options.output_partition_graphs = True - options.trace_level = tf.RunOptions.FULL_TRACE - - # Define a container for the returned metadata. - metadata = tf.RunMetadata() - - sess.run(y, options=options, run_metadata=metadata) - - # Print the subgraphs that executed on each device. - print(metadata.partition_graphs) - - # Print the timings of each operation that executed. - print(metadata.step_stats) -``` - - -## Visualizing your graph - -TensorFlow includes tools that can help you to understand the code in a graph. -The **graph visualizer** is a component of TensorBoard that renders the -structure of your graph visually in a browser. The easiest way to create a -visualization is to pass a @{tf.Graph} when creating the -@{tf.summary.FileWriter}: - -```python -# Build your graph. -x = tf.constant([[37.0, -23.0], [1.0, 4.0]]) -w = tf.Variable(tf.random_uniform([2, 2])) -y = tf.matmul(x, w) -# ... -loss = ... -train_op = tf.train.AdagradOptimizer(0.01).minimize(loss) - -with tf.Session() as sess: - # `sess.graph` provides access to the graph used in a `tf.Session`. - writer = tf.summary.FileWriter("/tmp/log/...", sess.graph) - - # Perform your computation... - for i in range(1000): - sess.run(train_op) - # ... - - writer.close() -``` - -Note: If you are using a @{tf.estimator.Estimator}, the graph (and any -summaries) will be logged automatically to the `model_dir` that you specified -when creating the estimator. - -You can then open the log in `tensorboard`, navigate to the "Graph" tab, and -see a high-level visualization of your graph's structure. Note that a typical -TensorFlow graph---especially training graphs with automatically computed -gradients---has too many nodes to visualize at once. The graph visualizer makes -use of name scopes to group related operations into "super" nodes. You can -click on the orange "+" button on any of these super nodes to expand the -subgraph inside. - -![](../images/mnist_deep.png) - -For more information about visualizing your TensorFlow application with -TensorBoard, see the [TensorBoard tutorial](../get_started/summaries_and_tensorboard.md). - -## Programming with multiple graphs - -Note: When training a model, a common way of organizing your code is to use one -graph for training your model, and a separate graph for evaluating or performing -inference with a trained model. In many cases, the inference graph will be -different from the training graph: for example, techniques like dropout and -batch normalization use different operations in each case. Furthermore, by -default utilities like @{tf.train.Saver} use the names of @{tf.Variable} objects -(which have names based on an underlying @{tf.Operation}) to identify each -variable in a saved checkpoint. When programming this way, you can either use -completely separate Python processes to build and execute the graphs, or you can -use multiple graphs in the same process. This section describes how to use -multiple graphs in the same process. - -As noted above, TensorFlow provides a "default graph" that is implicitly passed -to all API functions in the same context. For many applications, a single graph -is sufficient. However, TensorFlow also provides methods for manipulating -the default graph, which can be useful in more advanced use cases. For example: - -* A @{tf.Graph} defines the namespace for @{tf.Operation} objects: each - operation in a single graph must have a unique name. TensorFlow will - "uniquify" the names of operations by appending `"_1"`, `"_2"`, and so on to - their names if the requested name is already taken. Using multiple explicitly - created graphs gives you more control over what name is given to each - operation. - -* The default graph stores information about every @{tf.Operation} and - @{tf.Tensor} that was ever added to it. If your program creates a large number - of unconnected subgraphs, it may be more efficient to use a different - @{tf.Graph} to build each subgraph, so that unrelated state can be garbage - collected. - -You can install a different @{tf.Graph} as the default graph, using the -@{tf.Graph.as_default} context manager: - -```python -g_1 = tf.Graph() -with g_1.as_default(): - # Operations created in this scope will be added to `g_1`. - c = tf.constant("Node in g_1") - - # Sessions created in this scope will run operations from `g_1`. - sess_1 = tf.Session() - -g_2 = tf.Graph() -with g_2.as_default(): - # Operations created in this scope will be added to `g_2`. - d = tf.constant("Node in g_2") - -# Alternatively, you can pass a graph when constructing a `tf.Session`: -# `sess_2` will run operations from `g_2`. -sess_2 = tf.Session(graph=g_2) - -assert c.graph is g_1 -assert sess_1.graph is g_1 - -assert d.graph is g_2 -assert sess_2.graph is g_2 -``` - -To inspect the current default graph, call @{tf.get_default_graph}, which -returns a @{tf.Graph} object: - -```python -# Print all of the operations in the default graph. -g = tf.get_default_graph() -print(g.get_operations()) -``` diff --git a/tensorflow/docs_src/programmers_guide/index.md b/tensorflow/docs_src/programmers_guide/index.md deleted file mode 100644 index 9c58a3b45e..0000000000 --- a/tensorflow/docs_src/programmers_guide/index.md +++ /dev/null @@ -1,86 +0,0 @@ -# Programmer's Guide - -The documents in this unit dive into the details of how TensorFlow -works. The units are as follows: - -## High Level APIs - - * @{$programmers_guide/keras}, TensorFlow's high-level API for building and - training deep learning models. - * @{$programmers_guide/eager}, an API for writing TensorFlow code - imperatively, like you would use Numpy. - * @{$programmers_guide/estimators}, a high-level API that provides - fully-packaged models ready for large-scale training and production. - * @{$programmers_guide/datasets}, easy input pipelines to bring your data into - your TensorFlow program. - -## Estimators - -* @{$estimators} provides an introduction. -* @{$premade_estimators}, introduces Estimators for machine learning. -* @{$custom_estimators}, which demonstrates how to build and train models you - design yourself. -* @{$feature_columns}, which shows how an Estimator can handle a variety of input - data types without changes to the model. -* @{$datasets_for_estimators} describes using tf.data with estimators. -* @{$checkpoints}, which explains how to save training progress and resume where - you left off. - -## Accelerators - - * @{$using_gpu} explains how TensorFlow assigns operations to - devices and how you can change the arrangement manually. - * @{$using_tpu} explains how to modify `Estimator` programs to run on a TPU. - -## Low Level APIs - - * @{$programmers_guide/low_level_intro}, which introduces the - basics of how you can use TensorFlow outside of the high Level APIs. - * @{$programmers_guide/tensors}, which explains how to create, - manipulate, and access Tensors--the fundamental object in TensorFlow. - * @{$programmers_guide/variables}, which details how - to represent shared, persistent state in your program. - * @{$programmers_guide/graphs}, which explains: - * dataflow graphs, which are TensorFlow's representation of computations - as dependencies between operations. - * sessions, which are TensorFlow's mechanism for running dataflow graphs - across one or more local or remote devices. - If you are programming with the low-level TensorFlow API, this unit - is essential. If you are programming with a high-level TensorFlow API - such as Estimators or Keras, the high-level API creates and manages - graphs and sessions for you, but understanding graphs and sessions - can still be helpful. - * @{$programmers_guide/saved_model}, which - explains how to save and restore variables and models. - -## ML Concepts - - * @{$programmers_guide/embedding}, which introduces the concept - of embeddings, provides a simple example of training an embedding in - TensorFlow, and explains how to view embeddings with the TensorBoard - Embedding Projector. - -## Debugging - - * @{$programmers_guide/debugger}, which - explains how to use the TensorFlow debugger (tfdbg). - -## TensorBoard - -TensorBoard is a utility to visualize different aspects of machine learning. -The following guides explain how to use TensorBoard: - - * @{$programmers_guide/summaries_and_tensorboard}, - which introduces TensorBoard. - * @{$programmers_guide/graph_viz}, which - explains how to visualize the computational graph. - * @{$programmers_guide/tensorboard_histograms} which demonstrates the how to - use TensorBoard's histogram dashboard. - - -## Misc - - * @{$programmers_guide/version_compat}, - which explains backward compatibility guarantees and non-guarantees. - * @{$programmers_guide/faq}, which contains frequently asked - questions about TensorFlow. diff --git a/tensorflow/docs_src/programmers_guide/keras.md b/tensorflow/docs_src/programmers_guide/keras.md deleted file mode 100644 index c6aca7ebf4..0000000000 --- a/tensorflow/docs_src/programmers_guide/keras.md +++ /dev/null @@ -1,623 +0,0 @@ -# Keras - -Keras is a high-level API to build and train deep learning models. It's used for -fast prototyping, advanced research, and production, with three key advantages: - -- *User friendly*
- Keras has a simple, consistent interface optimized for common use cases. It - provides clear and actionable feedback for user errors. -- *Modular and composable*
- Keras models are made by connecting configurable building blocks together, - with few restrictions. -- *Easy to extend*
Write custom building blocks to express new ideas for - research. Create new layers, loss functions, and develop state-of-the-art - models. - -## Import tf.keras - -`tf.keras` is TensorFlow's implementation of the -[Keras API specification](https://keras.io){:.external}. This is a high-level -API to build and train models that includes first-class support for -TensorFlow-specific functionality, such as [eager execution](#eager_execution), -`tf.data` pipelines, and [Estimators](/programmers_guide/estimators). -`tf.keras` makes TensorFlow easier to use without sacrificing flexibility and -performance. - -To get started, import `tf.keras` as part of your TensorFlow program setup: - -```python -import tensorflow as tf -from tensorflow import keras -``` - -`tf.keras` can run any Keras-compatible code, but keep in mind: - -* The `tf.keras` version in the latest TensorFlow release might not be the same - as the latest `keras` version from PyPI. Check `tf.keras.__version__`. -* When [saving a model's weights](#weights_only), `tf.keras` defaults to the - [checkpoint format](/get_started/checkpoints). Pass `save_format='h5'` to use - HDF5. - -## Build a simple model - -### Sequential model - -In Keras, you assemble *layers* to build *models*. A model is (usually) a graph -of layers. The most common type of model is a stack of layers: the -`tf.keras.Sequential` model. - -To build a simple, fully-connected network (i.e. multi-layer perceptron): - -```python -model = keras.Sequential() -# Adds a densely-connected layer with 64 units to the model: -model.add(keras.layers.Dense(64, activation='relu')) -# Add another: -model.add(keras.layers.Dense(64, activation='relu')) -# Add a softmax layer with 10 output units: -model.add(keras.layers.Dense(10, activation='softmax')) -``` - -### Configure the layers - -There are many `tf.keras.layers` available with some common constructor -parameters: - -* `activation`: Set the activation function for the layer. This parameter is - specified by the name of a built-in function or as a callable object. By - default, no activation is applied. -* `kernel_initializer` and `bias_initializer`: The initialization schemes - that create the layer's weights (kernel and bias). This parameter is a name or - a callable object. This defaults to the `"Glorot uniform"` initializer. -* `kernel_regularizer` and `bias_regularizer`: The regularization schemes - that apply the layer's weights (kernel and bias), such as L1 or L2 - regularization. By default, no regularization is applied. - -The following instantiates `tf.keras.layers.Dense` layers using constructor -arguments: - -```python -# Create a sigmoid layer: -layers.Dense(64, activation='sigmoid') -# Or: -layers.Dense(64, activation=tf.sigmoid) - -# A linear layer with L1 regularization of factor 0.01 applied to the kernel matrix: -layers.Dense(64, kernel_regularizer=keras.regularizers.l1(0.01)) -# A linear layer with L2 regularization of factor 0.01 applied to the bias vector: -layers.Dense(64, bias_regularizer=keras.regularizers.l2(0.01)) - -# A linear layer with a kernel initialized to a random orthogonal matrix: -layers.Dense(64, kernel_initializer='orthogonal') -# A linear layer with a bias vector initialized to 2.0s: -layers.Dense(64, bias_initializer=keras.initializers.constant(2.0)) -``` - -## Train and evaluate - -### Set up training - -After the model is constructed, configure its learning process by calling the -`compile` method: - -```python -model.compile(optimizer=tf.train.AdamOptimizer(0.001), - loss='categorical_crossentropy', - metrics=['accuracy']) -``` - -`tf.keras.Model.compile` takes three important arguments: - -* `optimizer`: This object specifies the training procedure. Pass it optimizer - instances from the `tf.train` module, such as - [`AdamOptimizer`](/api_docs/python/tf/train/AdamOptimizer), - [`RMSPropOptimizer`](/api_docs/python/tf/train/RMSPropOptimizer), or - [`GradientDescentOptimizer`](/api_docs/python/tf/train/GradientDescentOptimizer). -* `loss`: The function to minimize during optimization. Common choices include - mean square error (`mse`), `categorical_crossentropy`, and - `binary_crossentropy`. Loss functions are specified by name or by - passing a callable object from the `tf.keras.losses` module. -* `metrics`: Used to monitor training. These are string names or callables from - the `tf.keras.metrics` module. - -The following shows a few examples of configuring a model for training: - -```python -# Configure a model for mean-squared error regression. -model.compile(optimizer=tf.train.AdamOptimizer(0.01), - loss='mse', # mean squared error - metrics=['mae']) # mean absolute error - -# Configure a model for categorical classification. -model.compile(optimizer=tf.train.RMSPropOptimizer(0.01), - loss=keras.losses.categorical_crossentropy, - metrics=[keras.metrics.categorical_accuracy]) -``` - -### Input NumPy data - -For small datasets, use in-memory [NumPy](https://www.numpy.org/){:.external} -arrays to train and evaluate a model. The model is "fit" to the training data -using the `fit` method: - -```python -import numpy as np - -data = np.random.random((1000, 32)) -labels = np.random.random((1000, 10)) - -model.fit(data, labels, epochs=10, batch_size=32) -``` - -`tf.keras.Model.fit` takes three important arguments: - -* `epochs`: Training is structured into *epochs*. An epoch is one iteration over - the entire input data (this is done in smaller batches). -* `batch_size`: When passed NumPy data, the model slices the data into smaller - batches and iterates over these batches during training. This integer - specifies the size of each batch. Be aware that the last batch may be smaller - if the total number of samples is not divisible by the batch size. -* `validation_data`: When prototyping a model, you want to easily monitor its - performance on some validation data. Passing this argument—a tuple of inputs - and labels—allows the model to display the loss and metrics in inference mode - for the passed data, at the end of each epoch. - -Here's an example using `validation_data`: - -```python -import numpy as np - -data = np.random.random((1000, 32)) -labels = np.random.random((1000, 10)) - -val_data = np.random.random((100, 32)) -val_labels = np.random.random((100, 10)) - -model.fit(data, labels, epochs=10, batch_size=32, - validation_data=(val_data, val_labels)) -``` - -### Input tf.data datasets - -Use the [Datasets API](/programmers_guide/datasets) to scale to large datasets -or multi-device training. Pass a `tf.data.Dataset` instance to the `fit` -method: - -```python -# Instantiates a toy dataset instance: -dataset = tf.data.Dataset.from_tensor_slices((data, labels)) -dataset = dataset.batch(32) -dataset = dataset.repeat() - -# Don't forget to specify `steps_per_epoch` when calling `fit` on a dataset. -model.fit(dataset, epochs=10, steps_per_epoch=30) -``` - -Here, the `fit` method uses the `steps_per_epoch` argument—this is the number of -training steps the model runs before it moves to the next epoch. Since the -`Dataset` yields batches of data, this snippet does not require a `batch_size`. - -Datasets can also be used for validation: - -```python -dataset = tf.data.Dataset.from_tensor_slices((data, labels)) -dataset = dataset.batch(32).repeat() - -val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_labels)) -val_dataset = val_dataset.batch(32).repeat() - -model.fit(dataset, epochs=10, steps_per_epoch=30, - validation_data=val_dataset, - validation_steps=3) -``` - -### Evaluate and predict - -The `tf.keras.Model.evaluate` and `tf.keras.Model.predict` methods can use NumPy -data and a `tf.data.Dataset`. - -To *evaluate* the inference-mode loss and metrics for the data provided: - -```python -model.evaluate(x, y, batch_size=32) - -model.evaluate(dataset, steps=30 -``` - -And to *predict* the output of the last layer in inference for the data provided, -as a NumPy array: - -``` -model.predict(x, batch_size=32) - -model.predict(dataset, steps=30) -``` - - -## Build advanced models - -### Functional API - -The `tf.keras.Sequential` model is a simple stack of layers that cannot -represent arbitrary models. Use the -[Keras functional API](https://keras.io/getting-started/functional-api-guide/){:.external} -to build complex model topologies such as: - -* Multi-input models, -* Multi-output models, -* Models with shared layers (the same layer called several times), -* Models with non-sequential data flows (e.g. residual connections). - -Building a model with the functional API works like this: - -1. A layer instance is callable and returns a tensor. -2. Input tensors and output tensors are used to define a `tf.keras.Model` - instance. -3. This model is trained just like the `Sequential` model. - -The following example uses the functional API to build a simple, fully-connected -network: - -```python -inputs = keras.Input(shape=(32,)) # Returns a placeholder tensor - -# A layer instance is callable on a tensor, and returns a tensor. -x = keras.layers.Dense(64, activation='relu')(inputs) -x = keras.layers.Dense(64, activation='relu')(x) -predictions = keras.layers.Dense(10, activation='softmax')(x) - -# Instantiate the model given inputs and outputs. -model = keras.Model(inputs=inputs, outputs=predictions) - -# The compile step specifies the training configuration. -model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), - loss='categorical_crossentropy', - metrics=['accuracy']) - -# Trains for 5 epochs -model.fit(data, labels, batch_size=32, epochs=5) -``` - -### Model subclassing - -Build a fully-customizable model by subclassing `tf.keras.Model` and defining -your own forward pass. Create layers in the `__init__` method and set them as -attributes of the class instance. Define the forward pass in the `call` method. - -Model subclassing is particularly useful when -[eager execution](/programmers_guide/eager) is enabled since the forward pass -can be written imperatively. - -Key Point: Use the right API for the job. While model subclassing offers -flexibility, it comes at a cost of greater complexity and more opportunities for -user errors. If possible, prefer the functional API. - -The following example shows a subclassed `tf.keras.Model` using a custom forward -pass: - -```python -class MyModel(keras.Model): - - def __init__(self, num_classes=10): - super(MyModel, self).__init__(name='my_model') - self.num_classes = num_classes - # Define your layers here. - self.dense_1 = keras.layers.Dense(32, activation='relu') - self.dense_2 = keras.layers.Dense(num_classes, activation='sigmoid') - - def call(self, inputs): - # Define your forward pass here, - # using layers you previously defined (in `__init__`). - x = self.dense_1(inputs) - return self.dense_2(x) - - def compute_output_shape(self, input_shape): - # You need to override this function if you want to use the subclassed model - # as part of a functional-style model. - # Otherwise, this method is optional. - shape = tf.TensorShape(input_shape).as_list() - shape[-1] = self.num_classes - return tf.TensorShape(shape) - - -# Instantiates the subclassed model. -model = MyModel(num_classes=10) - -# The compile step specifies the training configuration. -model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), - loss='categorical_crossentropy', - metrics=['accuracy']) - -# Trains for 5 epochs. -model.fit(data, labels, batch_size=32, epochs=5) -``` - - -### Custom layers - -Create a custom layer by subclassing `tf.keras.layers.Layer` and implementing -the following methods: - -* `build`: Create the weights of the layer. Add weights with the `add_weight` - method. -* `call`: Define the forward pass. -* `compute_output_shape`: Specify how to compute the output shape of the layer - given the input shape. -* Optionally, a layer can be serialized by implementing the `get_config` method - and the `from_config` class method. - -Here's an example of a custom layer that implements a `matmul` of an input with -a kernel matrix: - -```python -class MyLayer(keras.layers.Layer): - - def __init__(self, output_dim, **kwargs): - self.output_dim = output_dim - super(MyLayer, self).__init__(**kwargs) - - def build(self, input_shape): - shape = tf.TensorShape((input_shape[1], self.output_dim)) - # Create a trainable weight variable for this layer. - self.kernel = self.add_weight(name='kernel', - shape=shape, - initializer='uniform', - trainable=True) - # Be sure to call this at the end - super(MyLayer, self).build(input_shape) - - def call(self, inputs): - return tf.matmul(inputs, self.kernel) - - def compute_output_shape(self, input_shape): - shape = tf.TensorShape(input_shape).as_list() - shape[-1] = self.output_dim - return tf.TensorShape(shape) - - def get_config(self): - base_config = super(MyLayer, self).get_config() - base_config['output_dim'] = self.output_dim - - @classmethod - def from_config(cls, config): - return cls(**config) - - -# Create a model using the custom layer -model = keras.Sequential([MyLayer(10), - keras.layers.Activation('softmax')]) - -# The compile step specifies the training configuration -model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), - loss='categorical_crossentropy', - metrics=['accuracy']) - -# Trains for 5 epochs. -model.fit(data, targets, batch_size=32, epochs=5) -``` - - -## Callbacks - -A callback is an object passed to a model to customize and extend its behavior -during training. You can write your own custom callback, or use the built-in -`tf.keras.callbacks` that include: - -* `tf.keras.callbacks.ModelCheckpoint`: Save checkpoints of your model at - regular intervals. -* `tf.keras.callbacks.LearningRateScheduler`: Dynamically change the learning - rate. -* `tf.keras.callbacks.EarlyStopping`: Interrupt training when validation - performance has stopped improving. -* `tf.keras.callbacks.TensorBoard`: Monitor the model's behavior using - [TensorBoard](/programmers_guide/summaries_and_tensorboard). - -To use a `tf.keras.callbacks.Callback`, pass it to the model's `fit` method: - -```python -callbacks = [ - # Interrupt training if `val_loss` stops improving for over 2 epochs - keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'), - # Write TensorBoard logs to `./logs` directory - keras.callbacks.TensorBoard(log_dir='./logs') -] -model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks, - validation_data=(val_data, val_targets)) -``` - - -## Save and restore - -### Weights only - -Save and load the weights of a model using `tf.keras.Model.save_weights`: - -```python -# Save weights to a TensorFlow Checkpoint file -model.save_weights('./my_model') - -# Restore the model's state, -# this requires a model with the same architecture. -model.load_weights('my_model') -``` - -By default, this saves the model's weights in the -[TensorFlow checkpoint](/get_started/checkpoints) file format. Weights can also -be saved to the Keras HDF5 format (the default for the multi-backend -implementation of Keras): - -```python -# Save weights to a HDF5 file -model.save_weights('my_model.h5', save_format='h5') - -# Restore the model's state -model.load_weights('my_model.h5') -``` - - -### Configuration only - -A model's configuration can be saved—this serializes the model architecture -without any weights. A saved configuration can recreate and initialize the same -model, even without the code that defined the original model. Keras supports -JSON and YAML serialization formats: - -```python -# Serialize a model to JSON format -json_string = model.to_json() - -# Recreate the model (freshly initialized) -fresh_model = keras.models.from_json(json_string) - -# Serializes a model to YAML format -yaml_string = model.to_yaml() - -# Recreate the model -fresh_model = keras.models.from_yaml(yaml_string) -``` - -Caution: Subclassed models are not serializable because their architecture is -defined by the Python code in the body of the `call` method. - - -### Entire model - -The entire model can be saved to a file that contains the weight values, the -model's configuration, and even the optimizer's configuration. This allows you -to checkpoint a model and resume training later—from the exact same -state—without access to the original code. - -```python -# Create a trivial model -model = keras.Sequential([ - keras.layers.Dense(10, activation='softmax', input_shape=(32,)), - keras.layers.Dense(10, activation='softmax') -]) -model.compile(optimizer='rmsprop', - loss='categorical_crossentropy', - metrics=['accuracy']) -model.fit(data, targets, batch_size=32, epochs=5) - - -# Save entire model to a HDF5 file -model.save('my_model.h5') - -# Recreate the exact same model, including weights and optimizer. -model = keras.models.load_model('my_model.h5') -``` - - -## Eager execution - -[Eager execution](/programmers_guide/eager) is an imperative programming -environment that evaluates operations immediately. This is not required for -Keras, but is supported by `tf.keras` and useful for inspecting your program and -debugging. - -All of the `tf.keras` model-building APIs are compatible with eager execution. -And while the `Sequential` and functional APIs can be used, eager execution -especially benefits *model subclassing* and building *custom layers*—the APIs -that require you to write the forward pass as code (instead of the APIs that -create models by assembling existing layers). - -See the [eager execution guide](/programmers_guide/eager#build_a_model) for -examples of using Keras models with custom training loops and `tf.GradientTape`. - - -## Distribution - -### Estimators - -The [Estimators](/programmers_guide/estimators) API is used for training models -for distributed environments. This targets industry use cases such as -distributed training on large datasets that can export a model for production. - -A `tf.keras.Model` can be trained with the `tf.estimator` API by converting the -model to an `tf.estimator.Estimator` object with -`tf.keras.estimator.model_to_estimator`. See -[Creating Estimators from Keras models](/programmers_guide/estimators#creating_estimators_from_keras_models). - -```python -model = keras.Sequential([layers.Dense(10,activation='softmax'), - layers.Dense(10,activation='softmax')]) - -model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), - loss='categorical_crossentropy', - metrics=['accuracy']) - -estimator = keras.estimator.model_to_estimator(model) -``` - -Note: Enable [eager execution](/programmers_guide/eager) for debugging -[Estimator input functions](/programmers_guide/premade_estimators#create_input_functions) -and inspecting data. - -### Multiple GPUs - -`tf.keras` models can run on multiple GPUs using -`tf.contrib.distribute.DistributionStrategy`. This API provides distributed -training on multiple GPUs with almost no changes to existing code. - -Currently, `tf.contrib.distribute.MirroredStrategy` is the only supported -distribution strategy. `MirroredStrategy` does in-graph replication with -synchronous training using all-reduce on a single machine. To use -`DistributionStrategy` with Keras, convert the `tf.keras.Model` to a -`tf.estimator.Estimator` with `tf.keras.estimator.model_to_estimator`, then -train the estimator - -The following example distributes a `tf.keras.Model` across multiple GPUs on a -single machine. - -First, define a simple model: - -```python -model = keras.Sequential() -model.add(keras.layers.Dense(16, activation='relu', input_shape=(10,))) -model.add(keras.layers.Dense(1, activation='sigmoid')) - -optimizer = tf.train.GradientDescentOptimizer(0.2) - -model.compile(loss='binary_crossentropy', optimizer=optimizer) -model.summary() -``` - -Convert the Keras model to a `tf.estimator.Estimator` instance: - -```python -keras_estimator = keras.estimator.model_to_estimator( - keras_model=model, - config=config, - model_dir='/tmp/model_dir') -``` - -Define an *input pipeline*. The `input_fn` returns a `tf.data.Dataset` object -used to distribute the data across multiple devices—with each device processing -a slice of the input batch. - -```python -def input_fn(): - x = np.random.random((1024, 10)) - y = np.random.randint(2, size=(1024, 1)) - x = tf.cast(x, tf.float32) - dataset = tf.data.Dataset.from_tensor_slices((x, y)) - dataset = dataset.repeat(10) - dataset = dataset.batch(32) - return dataset -``` - -Next, create a `tf.estimator.RunConfig` and set the `train_distribute` argument -to the `tf.contrib.distribute.MirroredStrategy` instance. When creating -`MirroredStrategy`, you can specify a list of devices or set the `num_gpus` -argument. The default uses all available GPUs, like the following: - -```python -strategy = tf.contrib.distribute.MirroredStrategy() -config = tf.estimator.RunConfig(train_distribute=strategy) -``` - -Finally, train the `Estimator` instance by providing the `input_fn` and `steps` -arguments: - -```python -keras_estimator.train(input_fn=input_fn, steps=10) -``` diff --git a/tensorflow/docs_src/programmers_guide/leftnav_files b/tensorflow/docs_src/programmers_guide/leftnav_files deleted file mode 100644 index 357a2a1cb9..0000000000 --- a/tensorflow/docs_src/programmers_guide/leftnav_files +++ /dev/null @@ -1,40 +0,0 @@ -index.md - -### High Level APIs -keras.md -eager.md -datasets.md - -### Estimators -estimators.md: Introduction to Estimators -premade_estimators.md -custom_estimators.md -feature_columns.md -datasets_for_estimators.md -checkpoints.md - -### Accelerators -using_gpu.md -using_tpu.md - -### Low Level APIs -low_level_intro.md -tensors.md -variables.md -graphs.md -saved_model.md - -### ML Concepts -embedding.md - -### Debugging -debugger.md - -### TensorBoard -summaries_and_tensorboard.md: Visualizing Learning -graph_viz.md: Graphs -tensorboard_histograms.md: Histograms - -### Misc -version_compat.md -faq.md diff --git a/tensorflow/docs_src/programmers_guide/low_level_intro.md b/tensorflow/docs_src/programmers_guide/low_level_intro.md deleted file mode 100644 index 478e2bb70b..0000000000 --- a/tensorflow/docs_src/programmers_guide/low_level_intro.md +++ /dev/null @@ -1,604 +0,0 @@ -# Introduction - -This guide gets you started programming in the low-level TensorFlow APIs -(TensorFlow Core), showing you how to: - - * Manage your own TensorFlow program (a `tf.Graph`) and TensorFlow - runtime (a `tf.Session`), instead of relying on Estimators to manage them. - * Run TensorFlow operations, using a `tf.Session`. - * Use high level components ([datasets](#datasets), [layers](#layers), and - [feature_columns](#feature_columns)) in this low level environment. - * Build your own training loop, instead of using the one - @{$premade_estimators$provided by Estimators}. - -We recommend using the higher level APIs to build models when possible. -Knowing TensorFlow Core is valuable for the following reasons: - - * Experimentation and debugging are both more straight forward - when you can use low level TensorFlow operations directly. - * It gives you a mental model of how things work internally when - using the higher level APIs. - -## Setup - -Before using this guide, @{$install$install TensorFlow}. - -To get the most out of this guide, you should know the following: - -* How to program in Python. -* At least a little bit about arrays. -* Ideally, something about machine learning. - -Feel free to launch `python` and follow along with this walkthrough. -Run the following lines to set up your Python environment: - -```python -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import tensorflow as tf -``` - -## Tensor Values - -The central unit of data in TensorFlow is the **tensor**. A tensor consists of a -set of primitive values shaped into an array of any number of dimensions. A -tensor's **rank** is its number of dimensions, while its **shape** is a tuple -of integers specifying the array's length along each dimension. Here are some -examples of tensor values: - -```python -3. # a rank 0 tensor; a scalar with shape [], -[1., 2., 3.] # a rank 1 tensor; a vector with shape [3] -[[1., 2., 3.], [4., 5., 6.]] # a rank 2 tensor; a matrix with shape [2, 3] -[[[1., 2., 3.]], [[7., 8., 9.]]] # a rank 3 tensor with shape [2, 1, 3] -``` - -TensorFlow uses numpy arrays to represent tensor **values**. - -## TensorFlow Core Walkthrough - -You might think of TensorFlow Core programs as consisting of two discrete -sections: - -1. Building the computational graph (a @{tf.Graph}). -2. Running the computational graph (using a @{tf.Session}). - -### Graph - -A **computational graph** is a series of TensorFlow operations arranged into a -graph. The graph is composed of two types of objects. - - * @{tf.Operation$Operations} (or "ops"): The nodes of the graph. - Operations describe calculations that consume and produce tensors. - * @{tf.Tensor$Tensors}: The edges in the graph. These represent the values - that will flow through the graph. Most TensorFlow functions return - `tf.Tensors`. - -Important: `tf.Tensors` do not have values, they are just handles to elements -in the computation graph. - -Let's build a simple computational graph. The most basic operation is a -constant. The Python function that builds the operation takes a tensor value as -input. The resulting operation takes no inputs. When run, it outputs the -value that was passed to the constructor. We can create two floating point -constants `a` and `b` as follows: - -```python -a = tf.constant(3.0, dtype=tf.float32) -b = tf.constant(4.0) # also tf.float32 implicitly -total = a + b -print(a) -print(b) -print(total) -``` - -The print statements produce: - -``` -Tensor("Const:0", shape=(), dtype=float32) -Tensor("Const_1:0", shape=(), dtype=float32) -Tensor("add:0", shape=(), dtype=float32) -``` - -Notice that printing the tensors does not output the values `3.0`, `4.0`, and -`7.0` as you might expect. The above statements only build the computation -graph. These `tf.Tensor` objects just represent the results of the operations -that will be run. - -Each operation in a graph is given a unique name. This name is independent of -the names the objects are assigned to in Python. Tensors are named after the -operation that produces them followed by an output index, as in -`"add:0"` above. - -### TensorBoard - -TensorFlow provides a utility called TensorBoard. One of TensorBoard's many -capabilities is visualizing a computation graph. You can easily do this with -a few simple commands. - -First you save the computation graph to a TensorBoard summary file as -follows: - -``` -writer = tf.summary.FileWriter('.') -writer.add_graph(tf.get_default_graph()) -``` - -This will produce an `event` file in the current directory with a name in the -following format: - -``` -events.out.tfevents.{timestamp}.{hostname} -``` - -Now, in a new terminal, launch TensorBoard with the following shell command: - -```bsh -tensorboard --logdir . -``` - -Then open TensorBoard's [graphs page](http://localhost:6006/#graphs) in your -browser, and you should see a graph similar to the following: - -![TensorBoard screenshot](https://www.tensorflow.org/images/getting_started_add.png) - -For more about TensorBoard's graph visualization tools see @{$graph_viz}. - -### Session - -To evaluate tensors, instantiate a @{tf.Session} object, informally known as a -**session**. A session encapsulates the state of the TensorFlow runtime, and -runs TensorFlow operations. If a `tf.Graph` is like a `.py` file, a `tf.Session` -is like the `python` executable. - -The following code creates a `tf.Session` object and then invokes its `run` -method to evaluate the `total` tensor we created above: - -```python -sess = tf.Session() -print(sess.run(total)) -``` - -When you request the output of a node with `Session.run` TensorFlow backtracks -through the graph and runs all the nodes that provide input to the requested -output node. So this prints the expected value of 7.0: - -``` -7.0 -``` - -You can pass multiple tensors to `tf.Session.run`. The `run` method -transparently handles any combination of tuples or dictionaries, as in the -following example: - -```python -print(sess.run({'ab':(a, b), 'total':total})) -``` - -which returns the results in a structure of the same layout: - -``` None -{'total': 7.0, 'ab': (3.0, 4.0)} -``` - -During a call to `tf.Session.run` any `tf.Tensor` only has a single value. -For example, the following code calls `tf.random_uniform` to produce a -`tf.Tensor` that generates a random 3-element vector (with values in `[0,1)`): - -```python -vec = tf.random_uniform(shape=(3,)) -out1 = vec + 1 -out2 = vec + 2 -print(sess.run(vec)) -print(sess.run(vec)) -print(sess.run((out1, out2))) -``` - -The result shows a different random value on each call to `run`, but -a consistent value during a single `run` (`out1` and `out2` receive the same -random input): - -``` -[ 0.52917576 0.64076328 0.68353939] -[ 0.66192627 0.89126778 0.06254101] -( - array([ 1.88408756, 1.87149239, 1.84057522], dtype=float32), - array([ 2.88408756, 2.87149239, 2.84057522], dtype=float32) -) -``` - -Some TensorFlow functions return `tf.Operations` instead of `tf.Tensors`. -The result of calling `run` on an Operation is `None`. You run an operation -to cause a side-effect, not to retrieve a value. Examples of this include the -[initialization](#Initializing Layers), and [training](#Training) ops -demonstrated later. - -### Feeding - -As it stands, this graph is not especially interesting because it always -produces a constant result. A graph can be parameterized to accept external -inputs, known as **placeholders**. A **placeholder** is a promise to provide a -value later, like a function argument. - -```python -x = tf.placeholder(tf.float32) -y = tf.placeholder(tf.float32) -z = x + y -``` - -The preceding three lines are a bit like a function in which we -define two input parameters (`x` and `y`) and then an operation on them. We can -evaluate this graph with multiple inputs by using the `feed_dict` argument of -the @{tf.Session.run$run method} to feed concrete values to the placeholders: - -```python -print(sess.run(z, feed_dict={x: 3, y: 4.5})) -print(sess.run(z, feed_dict={x: [1, 3], y: [2, 4]})) -``` -This results in the following output: - -``` -7.5 -[ 3. 7.] -``` - -Also note that the `feed_dict` argument can be used to overwrite any tensor in -the graph. The only difference between placeholders and other `tf.Tensors` is -that placeholders throw an error if no value is fed to them. - -## Datasets - -Placeholders work for simple experiments, but @{tf.data$Datasets} are the -preferred method of streaming data into a model. - -To get a runnable `tf.Tensor` from a Dataset you must first convert it to a -@{tf.data.Iterator}, and then call the Iterator's -@{tf.data.Iterator.get_next$`get_next`} method. - -The simplest way to create an Iterator is with the -@{tf.data.Dataset.make_one_shot_iterator$`make_one_shot_iterator`} method. -For example, in the following code the `next_item` tensor will return a row from -the `my_data` array on each `run` call: - -``` python -my_data = [ - [0, 1,], - [2, 3,], - [4, 5,], - [6, 7,], -] -slices = tf.data.Dataset.from_tensor_slices(my_data) -next_item = slices.make_one_shot_iterator().get_next() -``` - -Reaching the end of the data stream causes `Dataset` to throw an -@{tf.errors.OutOfRangeError$`OutOfRangeError`}. For example, the following code -reads the `next_item` until there is no more data to read: - -``` python -while True: - try: - print(sess.run(next_item)) - except tf.errors.OutOfRangeError: - break -``` - -If the `Dataset` depends on stateful operations you may need to -initialize the iterator before using it, as shown below: - -``` python -r = tf.random_normal([10,3]) -dataset = tf.data.Dataset.from_tensor_slices(r) -iterator = dataset.make_initializable_iterator() -next_row = iterator.get_next() - -sess.run(iterator.initializer) -while True: - try: - print(sess.run(next_row)) - except tf.errors.OutOfRangeError: - break -``` - -For more details on Datasets and Iterators see: @{$programmers_guide/datasets}. - -## Layers - -A trainable model must modify the values in the graph to get new outputs with -the same input. @{tf.layers$Layers} are the preferred way to add trainable -parameters to a graph. - -Layers package together both the variables and the operations that act -on them. For example a -[densely-connected layer](https://developers.google.com/machine-learning/glossary/#fully_connected_layer) -performs a weighted sum across all inputs -for each output and applies an optional -[activation function](https://developers.google.com/machine-learning/glossary/#activation_function). -The connection weights and biases are managed by the layer object. - -### Creating Layers - -The following code creates a @{tf.layers.Dense$`Dense`} layer that takes a -batch of input vectors, and produces a single output value for each. To apply a -layer to an input, call the layer as if it were a function. For example: - -```python -x = tf.placeholder(tf.float32, shape=[None, 3]) -linear_model = tf.layers.Dense(units=1) -y = linear_model(x) -``` - -The layer inspects its input to determine sizes for its internal variables. So -here we must set the shape of the `x` placeholder so that the layer can -build a weight matrix of the correct size. - -Now that we have defined the calculation of the output, `y`, there is one more -detail we need to take care of before we run the calculation. - -### Initializing Layers - -The layer contains variables that must be **initialized** before they can be -used. While it is possible to initialize variables individually, you can easily -initialize all the variables in a TensorFlow graph as follows: - -```python -init = tf.global_variables_initializer() -sess.run(init) -``` - -Important: Calling `tf.global_variables_initializer` only -creates and returns a handle to a TensorFlow operation. That op -will initialize all the global variables when we run it with `tf.Session.run`. - -Also note that this `global_variables_initializer` only initializes variables -that existed in the graph when the initializer was created. So the initializer -should be one of the last things added during graph construction. - -### Executing Layers - -Now that the layer is initialized, we can evaluate the `linear_model`'s output -tensor as we would any other tensor. For example, the following code: - -```python -print(sess.run(y, {x: [[1, 2, 3],[4, 5, 6]]})) -``` - -will generate a two-element output vector such as the following: - -``` -[[-3.41378999] - [-9.14999008]] -``` - -### Layer Function shortcuts - -For each layer class (like @{tf.layers.Dense}) TensorFlow also supplies a -shortcut function (like @{tf.layers.dense}). The only difference is that the -shortcut function versions create and run the layer in a single call. For -example, the following code is equivalent to the earlier version: - -```python -x = tf.placeholder(tf.float32, shape=[None, 3]) -y = tf.layers.dense(x, units=1) - -init = tf.global_variables_initializer() -sess.run(init) - -print(sess.run(y, {x: [[1, 2, 3], [4, 5, 6]]})) -``` - -While convenient, this approach allows no access to the @{tf.layers.Layer} -object. This makes introspection and debugging more difficult, -and layer reuse impossible. - -## Feature columns - -The easiest way to experiment with feature columns is using the -@{tf.feature_column.input_layer} function. This function only accepts -@{$feature_columns$dense columns} as inputs, so to view the result -of a categorical column you must wrap it in an -@{tf.feature_column.indicator_column}. For example: - -``` python -features = { - 'sales' : [[5], [10], [8], [9]], - 'department': ['sports', 'sports', 'gardening', 'gardening']} - -department_column = tf.feature_column.categorical_column_with_vocabulary_list( - 'department', ['sports', 'gardening']) -department_column = tf.feature_column.indicator_column(department_column) - -columns = [ - tf.feature_column.numeric_column('sales'), - department_column -] - -inputs = tf.feature_column.input_layer(features, columns) -``` - -Running the `inputs` tensor will parse the `features` into a batch of vectors. - -Feature columns can have internal state, like layers, so they often need to be -initialized. Categorical columns use @{tf.contrib.lookup$lookup tables} -internally and these require a separate initialization op, -@{tf.tables_initializer}. - -``` python -var_init = tf.global_variables_initializer() -table_init = tf.tables_initializer() -sess = tf.Session() -sess.run((var_init, table_init)) -``` - -Once the internal state has been initialized you can run `inputs` like any -other `tf.Tensor`: - -```python -print(sess.run(inputs)) -``` - -This shows how the feature columns have packed the input vectors, with the -one-hot "department" as the first two indices and "sales" as the third. - -```None -[[ 1. 0. 5.] - [ 1. 0. 10.] - [ 0. 1. 8.] - [ 0. 1. 9.]] -``` - -## Training - -Now that you're familiar with the basics of core TensorFlow, let's train a -small regression model manually. - -### Define the data - -First let's define some inputs, `x`, and the expected output for each input, -`y_true`: - -```python -x = tf.constant([[1], [2], [3], [4]], dtype=tf.float32) -y_true = tf.constant([[0], [-1], [-2], [-3]], dtype=tf.float32) -``` - -### Define the model - -Next, build a simple linear model, with 1 output: - -``` python -linear_model = tf.layers.Dense(units=1) - -y_pred = linear_model(x) -``` - -You can evaluate the predictions as follows: - -``` python -sess = tf.Session() -init = tf.global_variables_initializer() -sess.run(init) - -print(sess.run(y_pred)) -``` - -The model hasn't yet been trained, so the four "predicted" values aren't very -good. Here's what we got; your own output will almost certainly differ: - -``` None -[[ 0.02631879] - [ 0.05263758] - [ 0.07895637] - [ 0.10527515]] -``` - -### Loss - -To optimize a model, you first need to define the loss. We'll use the mean -square error, a standard loss for regression problems. - -While you could do this manually with lower level math operations, -the @{tf.losses} module provides a set of common loss functions. You can use it -to calculate the mean square error as follows: - -``` python -loss = tf.losses.mean_squared_error(labels=y_true, predictions=y_pred) - -print(sess.run(loss)) -``` -This will produce a loss value, something like: - -``` None -2.23962 -``` - -### Training - -TensorFlow provides -[**optimizers**](https://developers.google.com/machine-learning/glossary/#optimizer) -implementing standard optimization algorithms. These are implemented as -sub-classes of @{tf.train.Optimizer}. They incrementally change each -variable in order to minimize the loss. The simplest optimization algorithm is -[**gradient descent**](https://developers.google.com/machine-learning/glossary/#gradient_descent), -implemented by @{tf.train.GradientDescentOptimizer}. It modifies each -variable according to the magnitude of the derivative of loss with respect to -that variable. For example: - -```python -optimizer = tf.train.GradientDescentOptimizer(0.01) -train = optimizer.minimize(loss) -``` - -This code builds all the graph components necessary for the optimization, and -returns a training operation. When run, the training op will update variables -in the graph. You might run it as follows: - -```python -for i in range(100): - _, loss_value = sess.run((train, loss)) - print(loss_value) -``` - -Since `train` is an op, not a tensor, it doesn't return a value when run. -To see the progression of the loss during training, we run the loss tensor at -the same time, producing output like the following: - -``` None -1.35659 -1.00412 -0.759167 -0.588829 -0.470264 -0.387626 -0.329918 -0.289511 -0.261112 -0.241046 -... -``` - -### Complete program - -```python -x = tf.constant([[1], [2], [3], [4]], dtype=tf.float32) -y_true = tf.constant([[0], [-1], [-2], [-3]], dtype=tf.float32) - -linear_model = tf.layers.Dense(units=1) - -y_pred = linear_model(x) -loss = tf.losses.mean_squared_error(labels=y_true, predictions=y_pred) - -optimizer = tf.train.GradientDescentOptimizer(0.01) -train = optimizer.minimize(loss) - -init = tf.global_variables_initializer() - -sess = tf.Session() -sess.run(init) -for i in range(100): - _, loss_value = sess.run((train, loss)) - print(loss_value) - -print(sess.run(y_pred)) -``` - -## Next steps - -To learn more about building models with TensorFlow consider the following: - -* @{$custom_estimators$Custom Estimators}, to learn how to build - customized models with TensorFlow. Your knowledge of TensorFlow Core will - help you understand and debug your own models. - -If you want to learn more about the inner workings of TensorFlow consider the -following documents, which go into more depth on many of the topics discussed -here: - -* @{$graphs} -* @{$tensors} -* @{$variables} - - diff --git a/tensorflow/docs_src/programmers_guide/premade_estimators.md b/tensorflow/docs_src/programmers_guide/premade_estimators.md deleted file mode 100644 index 02e2caf64b..0000000000 --- a/tensorflow/docs_src/programmers_guide/premade_estimators.md +++ /dev/null @@ -1,430 +0,0 @@ -# Premade Estimators - -This document introduces the TensorFlow programming environment and shows you -how to solve the Iris classification problem in TensorFlow. - -## Prerequisites - -Prior to using the sample code in this document, you'll need to do the -following: - -* @{$install$Install TensorFlow}. -* If you installed TensorFlow with virtualenv or Anaconda, activate your - TensorFlow environment. -* Install or upgrade pandas by issuing the following command: - - pip install pandas - -## Getting the sample code - -Take the following steps to get the sample code we'll be going through: - -1. Clone the TensorFlow Models repository from GitHub by entering the following - command: - - git clone https://github.com/tensorflow/models - -1. Change directory within that branch to the location containing the examples - used in this document: - - cd models/samples/core/get_started/ - -The program described in this document is -[`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py). -This program uses -[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py) -to fetch its training data. - -### Running the program - -You run TensorFlow programs as you would run any Python program. For example: - -``` bsh -python premade_estimator.py -``` - -The program should output training logs followed by some predictions against -the test set. For example, the first line in the following output shows that -the model thinks there is a 99.6% chance that the first example in the test -set is a Setosa. Since the test set expected Setosa, this appears to be -a good prediction. - -``` None -... -Prediction is "Setosa" (99.6%), expected "Setosa" - -Prediction is "Versicolor" (99.8%), expected "Versicolor" - -Prediction is "Virginica" (97.9%), expected "Virginica" -``` - -If the program generates errors instead of answers, ask yourself the following -questions: - -* Did you install TensorFlow properly? -* Are you using the correct version of TensorFlow? -* Did you activate the environment you installed TensorFlow in? (This is - only relevant in certain installation mechanisms.) - -## The programming stack - -Before getting into the details of the program itself, let's investigate the -programming environment. As the following illustration shows, TensorFlow -provides a programming stack consisting of multiple API layers: - -
- -
- -We strongly recommend writing TensorFlow programs with the following APIs: - -* @{$programmers_guide/estimators$Estimators}, which represent a complete model. - The Estimator API provides methods to train the model, to judge the model's - accuracy, and to generate predictions. -* @{$programmers_guide/datasets_for_estimators}, which build a data input - pipeline. The Dataset API has methods to load and manipulate data, and feed - it into your model. The Dataset API meshes well with the Estimators API. - -## Classifying irises: an overview - -The sample program in this document builds and tests a model that -classifies Iris flowers into three different species based on the size of their -[sepals](https://en.wikipedia.org/wiki/Sepal) and -[petals](https://en.wikipedia.org/wiki/Petal). - -
-Petal geometry compared for three iris species: Iris setosa, Iris virginica, and Iris versicolor -
- -**From left to right, -[*Iris setosa*](https://commons.wikimedia.org/w/index.php?curid=170298) (by -[Radomil](https://commons.wikimedia.org/wiki/User:Radomil), CC BY-SA 3.0), -[*Iris versicolor*](https://commons.wikimedia.org/w/index.php?curid=248095) (by -[Dlanglois](https://commons.wikimedia.org/wiki/User:Dlanglois), CC BY-SA 3.0), -and [*Iris virginica*](https://www.flickr.com/photos/33397993@N05/3352169862) -(by [Frank Mayfield](https://www.flickr.com/photos/33397993@N05), CC BY-SA -2.0).** - -### The data set - -The Iris data set contains four features and one -[label](https://developers.google.com/machine-learning/glossary/#label). -The four features identify the following botanical characteristics of -individual Iris flowers: - -* sepal length -* sepal width -* petal length -* petal width - -Our model will represent these features as `float32` numerical data. - -The label identifies the Iris species, which must be one of the following: - -* Iris setosa (0) -* Iris versicolor (1) -* Iris virginica (2) - -Our model will represent the label as `int32` categorical data. - -The following table shows three examples in the data set: - -|sepal length | sepal width | petal length | petal width| species (label) | -|------------:|------------:|-------------:|-----------:|:---------------:| -| 5.1 | 3.3 | 1.7 | 0.5 | 0 (Setosa) | -| 5.0 | 2.3 | 3.3 | 1.0 | 1 (versicolor)| -| 6.4 | 2.8 | 5.6 | 2.2 | 2 (virginica) | - -### The algorithm - -The program trains a Deep Neural Network classifier model having the following -topology: - -* 2 hidden layers. -* Each hidden layer contains 10 nodes. - -The following figure illustrates the features, hidden layers, and predictions -(not all of the nodes in the hidden layers are shown): - -
-A diagram of the network architecture: Inputs, 2 hidden layers, and outputs -
- -### Inference - -Running the trained model on an unlabeled example yields three predictions, -namely, the likelihood that this flower is the given Iris species. The sum of -those output predictions will be 1.0. For example, the prediction on an -unlabeled example might be something like the following: - -* 0.03 for Iris Setosa -* 0.95 for Iris Versicolor -* 0.02 for Iris Virginica - -The preceding prediction indicates a 95% probability that the given unlabeled -example is an Iris Versicolor. - -## Overview of programming with Estimators - -An Estimator is TensorFlow's high-level representation of a complete model. It -handles the details of initialization, logging, saving and restoring, and many -other features so you can concentrate on your model. For more details see -@{$programmers_guide/estimators}. - -An Estimator is any class derived from @{tf.estimator.Estimator}. TensorFlow -provides a collection of -@{tf.estimator$pre-made Estimators} -(for example, `LinearRegressor`) to implement common ML algorithms. Beyond -those, you may write your own -@{$custom_estimators$custom Estimators}. -We recommend using pre-made Estimators when just getting started. - -To write a TensorFlow program based on pre-made Estimators, you must perform the -following tasks: - -* Create one or more input functions. -* Define the model's feature columns. -* Instantiate an Estimator, specifying the feature columns and various - hyperparameters. -* Call one or more methods on the Estimator object, passing the appropriate - input function as the source of the data. - -Let's see how those tasks are implemented for Iris classification. - -## Create input functions - -You must create input functions to supply data for training, -evaluating, and prediction. - -An **input function** is a function that returns a @{tf.data.Dataset} object -which outputs the following two-element tuple: - -* [`features`](https://developers.google.com/machine-learning/glossary/#feature) - A Python dictionary in which: - * Each key is the name of a feature. - * Each value is an array containing all of that feature's values. -* `label` - An array containing the values of the - [label](https://developers.google.com/machine-learning/glossary/#label) for - every example. - -Just to demonstrate the format of the input function, here's a simple -implementation: - -```python -def input_evaluation_set(): - features = {'SepalLength': np.array([6.4, 5.0]), - 'SepalWidth': np.array([2.8, 2.3]), - 'PetalLength': np.array([5.6, 3.3]), - 'PetalWidth': np.array([2.2, 1.0])} - labels = np.array([2, 1]) - return features, labels -``` - -Your input function may generate the `features` dictionary and `label` list any -way you like. However, we recommend using TensorFlow's Dataset API, which can -parse all sorts of data. At a high level, the Dataset API consists of the -following classes: - -
-A diagram showing subclasses of the Dataset class -
- -Where the individual members are: - -* `Dataset` - Base class containing methods to create and transform - datasets. Also allows you to initialize a dataset from data in memory, or from - a Python generator. -* `TextLineDataset` - Reads lines from text files. -* `TFRecordDataset` - Reads records from TFRecord files. -* `FixedLengthRecordDataset` - Reads fixed size records from binary files. -* `Iterator` - Provides a way to access one data set element at a time. - -The Dataset API can handle a lot of common cases for you. For example, -using the Dataset API, you can easily read in records from a large collection -of files in parallel and join them into a single stream. - -To keep things simple in this example we are going to load the data with -[pandas](https://pandas.pydata.org/), and build our input pipeline from this -in-memory data. - -Here is the input function used for training in this program, which is available -in [`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py): - -``` python -def train_input_fn(features, labels, batch_size): - """An input function for training""" - # Convert the inputs to a Dataset. - dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels)) - - # Shuffle, repeat, and batch the examples. - return dataset.shuffle(1000).repeat().batch(batch_size) -``` - -## Define the feature columns - -A [**feature column**](https://developers.google.com/machine-learning/glossary/#feature_columns) -is an object describing how the model should use raw input data from the -features dictionary. When you build an Estimator model, you pass it a list of -feature columns that describes each of the features you want the model to use. -The @{tf.feature_column} module provides many options for representing data -to the model. - -For Iris, the 4 raw features are numeric values, so we'll build a list of -feature columns to tell the Estimator model to represent each of the four -features as 32-bit floating-point values. Therefore, the code to create the -feature column is: - -```python -# Feature columns describe how to use the input. -my_feature_columns = [] -for key in train_x.keys(): - my_feature_columns.append(tf.feature_column.numeric_column(key=key)) -``` - -Feature columns can be far more sophisticated than those we're showing here. We -detail feature columns @{$feature_columns$later on} in our Getting -Started guide. - -Now that we have the description of how we want the model to represent the raw -features, we can build the estimator. - - -## Instantiate an estimator - -The Iris problem is a classic classification problem. Fortunately, TensorFlow -provides several pre-made classifier Estimators, including: - -* @{tf.estimator.DNNClassifier} for deep models that perform multi-class - classification. -* @{tf.estimator.DNNLinearCombinedClassifier} for wide & deep models. -* @{tf.estimator.LinearClassifier} for classifiers based on linear models. - -For the Iris problem, `tf.estimator.DNNClassifier` seems like the best choice. -Here's how we instantiated this Estimator: - -```python -# Build a DNN with 2 hidden layers and 10 nodes in each hidden layer. -classifier = tf.estimator.DNNClassifier( - feature_columns=my_feature_columns, - # Two hidden layers of 10 nodes each. - hidden_units=[10, 10], - # The model must choose between 3 classes. - n_classes=3) -``` - -## Train, Evaluate, and Predict - -Now that we have an Estimator object, we can call methods to do the following: - -* Train the model. -* Evaluate the trained model. -* Use the trained model to make predictions. - -### Train the model - -Train the model by calling the Estimator's `train` method as follows: - -```python -# Train the Model. -classifier.train( - input_fn=lambda:iris_data.train_input_fn(train_x, train_y, args.batch_size), - steps=args.train_steps) -``` - -Here we wrap up our `input_fn` call in a -[`lambda`](https://docs.python.org/3/tutorial/controlflow.html) -to capture the arguments while providing an input function that takes no -arguments, as expected by the Estimator. The `steps` argument tells the method -to stop training after a number of training steps. - -### Evaluate the trained model - -Now that the model has been trained, we can get some statistics on its -performance. The following code block evaluates the accuracy of the trained -model on the test data: - -```python -# Evaluate the model. -eval_result = classifier.evaluate( - input_fn=lambda:iris_data.eval_input_fn(test_x, test_y, args.batch_size)) - -print('\nTest set accuracy: {accuracy:0.3f}\n'.format(**eval_result)) -``` - -Unlike our call to the `train` method, we did not pass the `steps` -argument to evaluate. Our `eval_input_fn` only yields a single -[epoch](https://developers.google.com/machine-learning/glossary/#epoch) of data. - -Running this code yields the following output (or something similar): - -```none -Test set accuracy: 0.967 -``` - -### Making predictions (inferring) from the trained model - -We now have a trained model that produces good evaluation results. -We can now use the trained model to predict the species of an Iris flower -based on some unlabeled measurements. As with training and evaluation, we make -predictions using a single function call: - -```python -# Generate predictions from the model -expected = ['Setosa', 'Versicolor', 'Virginica'] -predict_x = { - 'SepalLength': [5.1, 5.9, 6.9], - 'SepalWidth': [3.3, 3.0, 3.1], - 'PetalLength': [1.7, 4.2, 5.4], - 'PetalWidth': [0.5, 1.5, 2.1], -} - -predictions = classifier.predict( - input_fn=lambda:iris_data.eval_input_fn(predict_x, - batch_size=args.batch_size)) -``` - -The `predict` method returns a Python iterable, yielding a dictionary of -prediction results for each example. The following code prints a few -predictions and their probabilities: - - -``` python -template = ('\nPrediction is "{}" ({:.1f}%), expected "{}"') - -for pred_dict, expec in zip(predictions, expected): - class_id = pred_dict['class_ids'][0] - probability = pred_dict['probabilities'][class_id] - - print(template.format(iris_data.SPECIES[class_id], - 100 * probability, expec)) -``` - -Running the preceding code yields the following output: - -``` None -... -Prediction is "Setosa" (99.6%), expected "Setosa" - -Prediction is "Versicolor" (99.8%), expected "Versicolor" - -Prediction is "Virginica" (97.9%), expected "Virginica" -``` - - -## Summary - -Pre-made Estimators are an effective way to quickly create standard models. - -Now that you've gotten started writing TensorFlow programs, consider the -following material: - -* @{$checkpoints$Checkpoints} to learn how to save and restore models. -* @{$programmers_guide/datasets_for_estimators} to learn more about importing - data into your model. -* @{$custom_estimators$Creating Custom Estimators} to learn how to - write your own Estimator, customized for a particular problem. diff --git a/tensorflow/docs_src/programmers_guide/saved_model.md b/tensorflow/docs_src/programmers_guide/saved_model.md deleted file mode 100644 index c6ef87c54a..0000000000 --- a/tensorflow/docs_src/programmers_guide/saved_model.md +++ /dev/null @@ -1,999 +0,0 @@ -# Save and Restore - -The @{tf.train.Saver} class provides methods to save and restore models. The -@{tf.saved_model.simple_save} function is an easy way to build a -@{tf.saved_model$saved model} suitable for serving. -[Estimators](@{$programmers_guide/estimators}) automatically save and restore -variables in the `model_dir`. - -## Save and restore variables - -TensorFlow @{$variables} are the best way to represent shared, persistent state -manipulated by your program. The `tf.train.Saver` constructor adds `save` and -`restore` ops to the graph for all, or a specified list, of the variables in the -graph. The `Saver` object provides methods to run these ops, specifying paths -for the checkpoint files to write to or read from. - -`Saver` restores all variables already defined in your model. If you're -loading a model without knowing how to build its graph (for example, if you're -writing a generic program to load models), then read the -[Overview of saving and restoring models](#models) section -later in this document. - -TensorFlow saves variables in binary *checkpoint files* that map variable -names to tensor values. - -Caution: TensorFlow model files are code. Be careful with untrusted code. -See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) -for details. - -### Save variables - -Create a `Saver` with `tf.train.Saver()` to manage all variables in the -model. For example, the following snippet demonstrates how to call the -`tf.train.Saver.save` method to save variables to checkpoint files: - -```python -# Create some variables. -v1 = tf.get_variable("v1", shape=[3], initializer = tf.zeros_initializer) -v2 = tf.get_variable("v2", shape=[5], initializer = tf.zeros_initializer) - -inc_v1 = v1.assign(v1+1) -dec_v2 = v2.assign(v2-1) - -# Add an op to initialize the variables. -init_op = tf.global_variables_initializer() - -# Add ops to save and restore all the variables. -saver = tf.train.Saver() - -# Later, launch the model, initialize the variables, do some work, and save the -# variables to disk. -with tf.Session() as sess: - sess.run(init_op) - # Do some work with the model. - inc_v1.op.run() - dec_v2.op.run() - # Save the variables to disk. - save_path = saver.save(sess, "/tmp/model.ckpt") - print("Model saved in path: %s" % save_path) -``` - -### Restore variables - -The `tf.train.Saver` object not only saves variables to checkpoint files, it -also restores variables. Note that when you restore variables you do not have -to initialize them beforehand. For example, the following snippet demonstrates -how to call the `tf.train.Saver.restore` method to restore variables from the -checkpoint files: - -```python -tf.reset_default_graph() - -# Create some variables. -v1 = tf.get_variable("v1", shape=[3]) -v2 = tf.get_variable("v2", shape=[5]) - -# Add ops to save and restore all the variables. -saver = tf.train.Saver() - -# Later, launch the model, use the saver to restore variables from disk, and -# do some work with the model. -with tf.Session() as sess: - # Restore variables from disk. - saver.restore(sess, "/tmp/model.ckpt") - print("Model restored.") - # Check the values of the variables - print("v1 : %s" % v1.eval()) - print("v2 : %s" % v2.eval()) -``` - -Note: There is not a physical file called `/tmp/model.ckpt`. It is the *prefix* of -filenames created for the checkpoint. Users only interact with the prefix -instead of physical checkpoint files. - -### Choose variables to save and restore - -If you do not pass any arguments to `tf.train.Saver()`, the saver handles all -variables in the graph. Each variable is saved under the name that was passed -when the variable was created. - -It is sometimes useful to explicitly specify names for variables in the -checkpoint files. For example, you may have trained a model with a variable -named `"weights"` whose value you want to restore into a variable named -`"params"`. - -It is also sometimes useful to only save or restore a subset of the variables -used by a model. For example, you may have trained a neural net with five -layers, and you now want to train a new model with six layers that reuses the -existing weights of the five trained layers. You can use the saver to restore -the weights of just the first five layers. - -You can easily specify the names and variables to save or load by passing to the -`tf.train.Saver()` constructor either of the following: - -* A list of variables (which will be stored under their own names). -* A Python dictionary in which keys are the names to use and the values are the -variables to manage. - -Continuing from the save/restore examples shown earlier: - -```python -tf.reset_default_graph() -# Create some variables. -v1 = tf.get_variable("v1", [3], initializer = tf.zeros_initializer) -v2 = tf.get_variable("v2", [5], initializer = tf.zeros_initializer) - -# Add ops to save and restore only `v2` using the name "v2" -saver = tf.train.Saver({"v2": v2}) - -# Use the saver object normally after that. -with tf.Session() as sess: - # Initialize v1 since the saver will not. - v1.initializer.run() - saver.restore(sess, "/tmp/model.ckpt") - - print("v1 : %s" % v1.eval()) - print("v2 : %s" % v2.eval()) -``` - -Notes: - -* You can create as many `Saver` objects as you want if you need to save and - restore different subsets of the model variables. The same variable can be - listed in multiple saver objects; its value is only changed when the - `Saver.restore()` method is run. - -* If you only restore a subset of the model variables at the start of a - session, you have to run an initialize op for the other variables. See - @{tf.variables_initializer} for more information. - -* To inspect the variables in a checkpoint, you can use the - [`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py) - library, particularly the `print_tensors_in_checkpoint_file` function. - -* By default, `Saver` uses the value of the @{tf.Variable.name} property - for each variable. However, when you create a `Saver` object, you may - optionally choose names for the variables in the checkpoint files. - - -### Inspect variables in a checkpoint - -We can quickly inspect variables in a checkpoint with the -[`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py) library. - -Continuing from the save/restore examples shown earlier: - -```python -# import the inspect_checkpoint library -from tensorflow.python.tools import inspect_checkpoint as chkp - -# print all tensors in checkpoint file -chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='', all_tensors=True) - -# tensor_name: v1 -# [ 1. 1. 1.] -# tensor_name: v2 -# [-1. -1. -1. -1. -1.] - -# print only tensor v1 in checkpoint file -chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='v1', all_tensors=False) - -# tensor_name: v1 -# [ 1. 1. 1.] - -# print only tensor v2 in checkpoint file -chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='v2', all_tensors=False) - -# tensor_name: v2 -# [-1. -1. -1. -1. -1.] -``` - - - -## Save and restore models - -Use `SavedModel` to save and load your model—variables, the graph, and the -graph's metadata. This is a language-neutral, recoverable, hermetic -serialization format that enables higher-level systems and tools to produce, -consume, and transform TensorFlow models. TensorFlow provides several ways to -interact with `SavedModel`, including the @{tf.saved_model} APIs, -@{tf.estimator.Estimator}, and a command-line interface. - - -## Build and load a SavedModel - -### Simple save - -The easiest way to create a `SavedModel` is to use the @{tf.saved_model.simple_save} -function: - -```python -simple_save(session, - export_dir, - inputs={"x": x, "y": y}, - outputs={"z": z}) -``` - -This configures the `SavedModel` so it can be loaded by -[TensorFlow serving](/serving/serving_basic) and supports the -[Predict API](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/predict.proto). -To access the classify, regress, or multi-inference APIs, use the manual -`SavedModel` builder APIs or an @{tf.estimator.Estimator}. - -### Manually build a SavedModel - -If your use case isn't covered by @{tf.saved_model.simple_save}, use the manual -@{tf.saved_model.builder$builder APIs} to create a `SavedModel`. - -The @{tf.saved_model.builder.SavedModelBuilder} class provides functionality to -save multiple `MetaGraphDef`s. A **MetaGraph** is a dataflow graph, plus -its associated variables, assets, and signatures. A **`MetaGraphDef`** -is the protocol buffer representation of a MetaGraph. A **signature** is -the set of inputs to and outputs from a graph. - -If assets need to be saved and written or copied to disk, they can be provided -when the first `MetaGraphDef` is added. If multiple `MetaGraphDef`s are -associated with an asset of the same name, only the first version is retained. - -Each `MetaGraphDef` added to the SavedModel must be annotated with -user-specified tags. The tags provide a means to identify the specific -`MetaGraphDef` to load and restore, along with the shared set of variables -and assets. These tags -typically annotate a `MetaGraphDef` with its functionality (for example, -serving or training), and optionally with hardware-specific aspects (for -example, GPU). - -For example, the following code suggests a typical way to use -`SavedModelBuilder` to build a SavedModel: - -```python -export_dir = ... -... -builder = tf.saved_model.builder.SavedModelBuilder(export_dir) -with tf.Session(graph=tf.Graph()) as sess: - ... - builder.add_meta_graph_and_variables(sess, - [tag_constants.TRAINING], - signature_def_map=foo_signatures, - assets_collection=foo_assets, - strip_default_attrs=True) -... -# Add a second MetaGraphDef for inference. -with tf.Session(graph=tf.Graph()) as sess: - ... - builder.add_meta_graph([tag_constants.SERVING], strip_default_attrs=True) -... -builder.save() -``` - - -#### Forward compatibility via `strip_default_attrs=True` - -Following the guidance below gives you forward compatibility only if the set of -Ops has not changed. - -The @{tf.saved_model.builder.SavedModelBuilder$`SavedModelBuilder`} class allows -users to control whether default-valued attributes must be stripped from the -@{$extend/tool_developers#nodes$`NodeDefs`} -while adding a meta graph to the SavedModel bundle. Both -@{tf.saved_model.builder.SavedModelBuilder.add_meta_graph_and_variables$`SavedModelBuilder.add_meta_graph_and_variables`} -and @{tf.saved_model.builder.SavedModelBuilder.add_meta_graph$`SavedModelBuilder.add_meta_graph`} -methods accept a Boolean flag `strip_default_attrs` that controls this behavior. - -If `strip_default_attrs` is `False`, the exported @{tf.MetaGraphDef} will have -the default valued attributes in all its @{tf.NodeDef} instances. -This can break forward compatibility with a sequence of events such as the -following: - -* An existing Op (`Foo`) is updated to include a new attribute (`T`) with a - default (`bool`) at version 101. -* A model producer such as a "trainer binary" picks up this change (version 101) - to the `OpDef` and re-exports an existing model that uses Op `Foo`. -* A model consumer (such as [Tensorflow Serving](/serving)) running an older - binary (version 100) doesn't have attribute `T` for Op `Foo`, but tries to - import this model. The model consumer doesn't recognize attribute `T` in a - `NodeDef` that uses Op `Foo` and therefore fails to load the model. -* By setting `strip_default_attrs` to True, the model producers can strip away - any default valued attributes in the `NodeDefs`. This helps ensure that newly - added attributes with defaults don't cause older model consumers to fail - loading models regenerated with newer training binaries. - -See [compatibility guidance](https://www.tensorflow.org/programmers_guide/version_compat) -for more information. - -### Loading a SavedModel in Python - -The Python version of the SavedModel -@{tf.saved_model.loader$loader} -provides load and restore capability for a SavedModel. The `load` operation -requires the following information: - -* The session in which to restore the graph definition and variables. -* The tags used to identify the MetaGraphDef to load. -* The location (directory) of the SavedModel. - -Upon a load, the subset of variables, assets, and signatures supplied as part of -the specific MetaGraphDef will be restored into the supplied session. - - -```python -export_dir = ... -... -with tf.Session(graph=tf.Graph()) as sess: - tf.saved_model.loader.load(sess, [tag_constants.TRAINING], export_dir) - ... -``` - - -### Load a SavedModel in C++ - -The C++ version of the SavedModel -[loader](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/loader.h) -provides an API to load a SavedModel from a path, while allowing -`SessionOptions` and `RunOptions`. -You have to specify the tags associated with the graph to be loaded. -The loaded version of SavedModel is referred to as `SavedModelBundle` -and contains the MetaGraphDef and the session within which it is loaded. - -```c++ -const string export_dir = ... -SavedModelBundle bundle; -... -LoadSavedModel(session_options, run_options, export_dir, {kSavedModelTagTrain}, - &bundle); -``` - -### Load and serve a SavedModel in TensorFlow serving - -You can easily load and serve a SavedModel with the TensorFlow Serving Model -Server binary. See [instructions](https://www.tensorflow.org/serving/setup#installing_using_apt-get) -on how to install the server, or build it if you wish. - -Once you have the Model Server, run it with: -``` -tensorflow_model_server --port=port-numbers --model_name=your-model-name --model_base_path=your_model_base_path -``` -Set the port and model_name flags to values of your choosing. The -model_base_path flag expects to be to a base directory, with each version of -your model residing in a numerically named subdirectory. If you only have a -single version of your model, simply place it in a subdirectory like so: -* Place the model in /tmp/model/0001 -* Set model_base_path to /tmp/model - -Store different versions of your model in numerically named subdirectories of a -common base directory. For example, suppose the base directory is `/tmp/model`. -If you have only one version of your model, store it in `/tmp/model/0001`. If -you have two versions of your model, store the second version in -`/tmp/model/0002`, and so on. Set the `--model-base_path` flag to the base -directory (`/tmp/model`, in this example). TensorFlow Model Server will serve -the model in the highest numbered subdirectory of that base directory. - -### Standard constants - -SavedModel offers the flexibility to build and load TensorFlow graphs for a -variety of use-cases. For the most common use-cases, SavedModel's APIs -provide a set of constants in Python and C++ that are easy to -reuse and share across tools consistently. - -#### Standard MetaGraphDef tags - -You may use sets of tags to uniquely identify a `MetaGraphDef` saved in a -SavedModel. A subset of commonly used tags is specified in: - -* [Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py) -* [C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/tag_constants.h) - - -#### Standard SignatureDef constants - -A [**SignatureDef**](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto) -is a protocol buffer that defines the signature of a computation -supported by a graph. -Commonly used input keys, output keys, and method names are -defined in: - -* [Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) -* [C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/signature_constants.h) - -## Using SavedModel with Estimators - -After training an `Estimator` model, you may want to create a service -from that model that takes requests and returns a result. You can run such a -service locally on your machine or deploy it in the cloud. - -To prepare a trained Estimator for serving, you must export it in the standard -SavedModel format. This section explains how to: - -* Specify the output nodes and the corresponding - [APIs](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto) - that can be served (Classify, Regress, or Predict). -* Export your model to the SavedModel format. -* Serve the model from a local server and request predictions. - - -### Prepare serving inputs - -During training, an @{$premade_estimators#input_fn$`input_fn()`} ingests data -and prepares it for use by the model. At serving time, similarly, a -`serving_input_receiver_fn()` accepts inference requests and prepares them for -the model. This function has the following purposes: - -* To add placeholders to the graph that the serving system will feed - with inference requests. -* To add any additional ops needed to convert data from the input format - into the feature `Tensor`s expected by the model. - -The function returns a @{tf.estimator.export.ServingInputReceiver} object, -which packages the placeholders and the resulting feature `Tensor`s together. - -A typical pattern is that inference requests arrive in the form of serialized -`tf.Example`s, so the `serving_input_receiver_fn()` creates a single string -placeholder to receive them. The `serving_input_receiver_fn()` is then also -responsible for parsing the `tf.Example`s by adding a @{tf.parse_example} op to -the graph. - -When writing such a `serving_input_receiver_fn()`, you must pass a parsing -specification to @{tf.parse_example} to tell the parser what feature names to -expect and how to map them to `Tensor`s. A parsing specification takes the -form of a dict from feature names to @{tf.FixedLenFeature}, @{tf.VarLenFeature}, -and @{tf.SparseFeature}. Note this parsing specification should not include -any label or weight columns, since those will not be available at serving -time—in contrast to a parsing specification used in the `input_fn()` at -training time. - -In combination, then: - -```py -feature_spec = {'foo': tf.FixedLenFeature(...), - 'bar': tf.VarLenFeature(...)} - -def serving_input_receiver_fn(): - """An input receiver that expects a serialized tf.Example.""" - serialized_tf_example = tf.placeholder(dtype=tf.string, - shape=[default_batch_size], - name='input_example_tensor') - receiver_tensors = {'examples': serialized_tf_example} - features = tf.parse_example(serialized_tf_example, feature_spec) - return tf.estimator.export.ServingInputReceiver(features, receiver_tensors) -``` - -The @{tf.estimator.export.build_parsing_serving_input_receiver_fn} utility -function provides that input receiver for the common case. - -> Note: when training a model to be served using the Predict API with a local -> server, the parsing step is not needed because the model will receive raw -> feature data. - -Even if you require no parsing or other input processing—that is, if the -serving system will feed feature `Tensor`s directly—you must still provide -a `serving_input_receiver_fn()` that creates placeholders for the feature -`Tensor`s and passes them through. The -@{tf.estimator.export.build_raw_serving_input_receiver_fn} utility provides for -this. - -If these utilities do not meet your needs, you are free to write your own -`serving_input_receiver_fn()`. One case where this may be needed is if your -training `input_fn()` incorporates some preprocessing logic that must be -recapitulated at serving time. To reduce the risk of training-serving skew, we -recommend encapsulating such processing in a function which is then called -from both `input_fn()` and `serving_input_receiver_fn()`. - -Note that the `serving_input_receiver_fn()` also determines the *input* -portion of the signature. That is, when writing a -`serving_input_receiver_fn()`, you must tell the parser what signatures -to expect and how to map them to your model's expected inputs. -By contrast, the *output* portion of the signature is determined by the model. - - -### Specify the outputs of a custom model - -When writing a custom `model_fn`, you must populate the `export_outputs` element -of the @{tf.estimator.EstimatorSpec} return value. This is a dict of -`{name: output}` describing the output signatures to be exported and used during -serving. - -In the usual case of making a single prediction, this dict contains -one element, and the `name` is immaterial. In a multi-headed model, each head -is represented by an entry in this dict. In this case the `name` is a string -of your choice that can be used to request a specific head at serving time. - -Each `output` value must be an `ExportOutput` object such as -@{tf.estimator.export.ClassificationOutput}, -@{tf.estimator.export.RegressionOutput}, or -@{tf.estimator.export.PredictOutput}. - -These output types map straightforwardly to the -[TensorFlow Serving APIs](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto), -and so determine which request types will be honored. - -Note: In the multi-headed case, a `SignatureDef` will be generated for each -element of the `export_outputs` dict returned from the model_fn, named using -the same keys. These `SignatureDef`s differ only in their outputs, as -provided by the corresponding `ExportOutput` entry. The inputs are always -those provided by the `serving_input_receiver_fn`. -An inference request may specify the head by name. One head must be named -using [`signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY`](https://www.tensorflow.org/code/tensorflow/python/saved_model/signature_constants.py) -indicating which `SignatureDef` will be served when an inference request -does not specify one. - - -### Perform the export - -To export your trained Estimator, call -@{tf.estimator.Estimator.export_savedmodel} with the export base path and -the `serving_input_receiver_fn`. - -```py -estimator.export_savedmodel(export_dir_base, serving_input_receiver_fn, - strip_default_attrs=True) -``` - -This method builds a new graph by first calling the -`serving_input_receiver_fn()` to obtain feature `Tensor`s, and then calling -this `Estimator`'s `model_fn()` to generate the model graph based on those -features. It starts a fresh `Session`, and, by default, restores the most recent -checkpoint into it. (A different checkpoint may be passed, if needed.) -Finally it creates a time-stamped export directory below the given -`export_dir_base` (i.e., `export_dir_base/`), and writes a -SavedModel into it containing a single `MetaGraphDef` saved from this -Session. - -> Note: It is your responsibility to garbage-collect old exports. -> Otherwise, successive exports will accumulate under `export_dir_base`. - -### Serve the exported model locally - -For local deployment, you can serve your model using -[TensorFlow Serving](https://github.com/tensorflow/serving), an open-source project that loads a -SavedModel and exposes it as a [gRPC](https://www.grpc.io/) service. - -First, [install TensorFlow Serving](https://github.com/tensorflow/serving). - -Then build and run the local model server, substituting `$export_dir_base` with -the path to the SavedModel you exported above: - -```sh -bazel build //tensorflow_serving/model_servers:tensorflow_model_server -bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_base_path=$export_dir_base -``` - -Now you have a server listening for inference requests via gRPC on port 9000! - - -### Request predictions from a local server - -The server responds to gRPC requests according to the -[PredictionService](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto#L15) -gRPC API service definition. (The nested protocol buffers are defined in -various [neighboring files](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis)). - -From the API service definition, the gRPC framework generates client libraries -in various languages providing remote access to the API. In a project using the -Bazel build tool, these libraries are built automatically and provided via -dependencies like these (using Python for example): - -```build - deps = [ - "//tensorflow_serving/apis:classification_proto_py_pb2", - "//tensorflow_serving/apis:regression_proto_py_pb2", - "//tensorflow_serving/apis:predict_proto_py_pb2", - "//tensorflow_serving/apis:prediction_service_proto_py_pb2" - ] -``` - -Python client code can then import the libraries thus: - -```py -from tensorflow_serving.apis import classification_pb2 -from tensorflow_serving.apis import regression_pb2 -from tensorflow_serving.apis import predict_pb2 -from tensorflow_serving.apis import prediction_service_pb2 -``` - -> Note: `prediction_service_pb2` defines the service as a whole and so -> is always required. However a typical client will need only one of -> `classification_pb2`, `regression_pb2`, and `predict_pb2`, depending on the -> type of requests being made. - -Sending a gRPC request is then accomplished by assembling a protocol buffer -containing the request data and passing it to the service stub. Note how the -request protocol buffer is created empty and then populated via the -[generated protocol buffer API](https://developers.google.com/protocol-buffers/docs/reference/python-generated). - -```py -from grpc.beta import implementations - -channel = implementations.insecure_channel(host, int(port)) -stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) - -request = classification_pb2.ClassificationRequest() -example = request.input.example_list.examples.add() -example.features.feature['x'].float_list.value.extend(image[0].astype(float)) - -result = stub.Classify(request, 10.0) # 10 secs timeout -``` - -The returned result in this example is a `ClassificationResponse` protocol -buffer. - -This is a skeletal example; please see the @{$deploy$Tensorflow Serving} -documentation and [examples](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example) -for more details. - -> Note: `ClassificationRequest` and `RegressionRequest` contain a -> `tensorflow.serving.Input` protocol buffer, which in turn contains a list of -> `tensorflow.Example` protocol buffers. `PredictRequest`, by contrast, -> contains a mapping from feature names to values encoded via `TensorProto`. -> Correspondingly: When using the `Classify` and `Regress` APIs, TensorFlow -> Serving feeds serialized `tf.Example`s to the graph, so your -> `serving_input_receiver_fn()` should include a `tf.parse_example()` Op. -> When using the generic `Predict` API, however, TensorFlow Serving feeds raw -> feature data to the graph, so a pass through `serving_input_receiver_fn()` -> should be used. - - - - - - - - - -## CLI to inspect and execute SavedModel - -You can use the SavedModel Command Line Interface (CLI) to inspect and -execute a SavedModel. -For example, you can use the CLI to inspect the model's `SignatureDef`s. -The CLI enables you to quickly confirm that the input -@{$tensors$Tensor dtype and shape} match the model. Moreover, if you -want to test your model, you can use the CLI to do a sanity check by -passing in sample inputs in various formats (for example, Python -expressions) and then fetching the output. - - -### Install the SavedModel CLI - -Broadly speaking, you can install TensorFlow in either of the following -two ways: - -* By installing a pre-built TensorFlow binary. -* By building TensorFlow from source code. - -If you installed TensorFlow through a pre-built TensorFlow binary, -then the SavedModel CLI is already installed on your system -at pathname `bin\saved_model_cli`. - -If you built TensorFlow from source code, you must run the following -additional command to build `saved_model_cli`: - -``` -$ bazel build tensorflow/python/tools:saved_model_cli -``` - -### Overview of commands - -The SavedModel CLI supports the following two commands on a -`MetaGraphDef` in a SavedModel: - -* `show`, which shows a computation on a `MetaGraphDef` in a SavedModel. -* `run`, which runs a computation on a `MetaGraphDef`. - - -### `show` command - -A SavedModel contains one or more `MetaGraphDef`s, identified by their tag-sets. -To serve a model, you -might wonder what kind of `SignatureDef`s are in each model, and what are their -inputs and outputs. The `show` command let you examine the contents of the -SavedModel in hierarchical order. Here's the syntax: - -``` -usage: saved_model_cli show [-h] --dir DIR [--all] -[--tag_set TAG_SET] [--signature_def SIGNATURE_DEF_KEY] -``` - -For example, the following command shows all available -MetaGraphDef tag-sets in the SavedModel: - -``` -$ saved_model_cli show --dir /tmp/saved_model_dir -The given SavedModel contains the following tag-sets: -serve -serve, gpu -``` - -The following command shows all available `SignatureDef` keys in -a `MetaGraphDef`: - -``` -$ saved_model_cli show --dir /tmp/saved_model_dir --tag_set serve -The given SavedModel `MetaGraphDef` contains `SignatureDefs` with the -following keys: -SignatureDef key: "classify_x2_to_y3" -SignatureDef key: "classify_x_to_y" -SignatureDef key: "regress_x2_to_y3" -SignatureDef key: "regress_x_to_y" -SignatureDef key: "regress_x_to_y2" -SignatureDef key: "serving_default" -``` - -If a `MetaGraphDef` has *multiple* tags in the tag-set, you must specify -all tags, each tag separated by a comma. For example: - -```none -$ saved_model_cli show --dir /tmp/saved_model_dir --tag_set serve,gpu -``` - -To show all inputs and outputs TensorInfo for a specific `SignatureDef`, pass in -the `SignatureDef` key to `signature_def` option. This is very useful when you -want to know the tensor key value, dtype and shape of the input tensors for -executing the computation graph later. For example: - -``` -$ saved_model_cli show --dir \ -/tmp/saved_model_dir --tag_set serve --signature_def serving_default -The given SavedModel SignatureDef contains the following input(s): - inputs['x'] tensor_info: - dtype: DT_FLOAT - shape: (-1, 1) - name: x:0 -The given SavedModel SignatureDef contains the following output(s): - outputs['y'] tensor_info: - dtype: DT_FLOAT - shape: (-1, 1) - name: y:0 -Method name is: tensorflow/serving/predict -``` - -To show all available information in the SavedModel, use the `--all` option. -For example: - -```none -$ saved_model_cli show --dir /tmp/saved_model_dir --all -MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: - -signature_def['classify_x2_to_y3']: - The given SavedModel SignatureDef contains the following input(s): - inputs['inputs'] tensor_info: - dtype: DT_FLOAT - shape: (-1, 1) - name: x2:0 - The given SavedModel SignatureDef contains the following output(s): - outputs['scores'] tensor_info: - dtype: DT_FLOAT - shape: (-1, 1) - name: y3:0 - Method name is: tensorflow/serving/classify - -... - -signature_def['serving_default']: - The given SavedModel SignatureDef contains the following input(s): - inputs['x'] tensor_info: - dtype: DT_FLOAT - shape: (-1, 1) - name: x:0 - The given SavedModel SignatureDef contains the following output(s): - outputs['y'] tensor_info: - dtype: DT_FLOAT - shape: (-1, 1) - name: y:0 - Method name is: tensorflow/serving/predict -``` - - -### `run` command - -Invoke the `run` command to run a graph computation, passing -inputs and then displaying (and optionally saving) the outputs. -Here's the syntax: - -``` -usage: saved_model_cli run [-h] --dir DIR --tag_set TAG_SET --signature_def - SIGNATURE_DEF_KEY [--inputs INPUTS] - [--input_exprs INPUT_EXPRS] [--outdir OUTDIR] - [--overwrite] [--tf_debug] -``` - -The `run` command provides the following two ways to pass inputs to the model: - -* `--inputs` option enables you to pass numpy ndarray in files. -* `--input_exprs` option enables you to pass Python expressions. -* `--input_examples` option enables you to pass `tf.train.Example`. - - -#### `--inputs` - -To pass input data in files, specify the `--inputs` option, which takes the -following general format: - -```bsh ---inputs -``` - -where *INPUTS* is either of the following formats: - -* `=` -* `=[]` - -You may pass multiple *INPUTS*. If you do pass multiple inputs, use a semicolon -to separate each of the *INPUTS*. - -`saved_model_cli` uses `numpy.load` to load the *filename*. -The *filename* may be in any of the following formats: - -* `.npy` -* `.npz` -* pickle format - -A `.npy` file always contains a numpy ndarray. Therefore, when loading from -a `.npy` file, the content will be directly assigned to the specified input -tensor. If you specify a *variable_name* with that `.npy` file, the -*variable_name* will be ignored and a warning will be issued. - -When loading from a `.npz` (zip) file, you may optionally specify a -*variable_name* to identify the variable within the zip file to load for -the input tensor key. If you don't specify a *variable_name*, the SavedModel -CLI will check that only one file is included in the zip file and load it -for the specified input tensor key. - -When loading from a pickle file, if no `variable_name` is specified in the -square brackets, whatever that is inside the pickle file will be passed to the -specified input tensor key. Otherwise, the SavedModel CLI will assume a -dictionary is stored in the pickle file and the value corresponding to -the *variable_name* will be used. - - -#### `--inputs_exprs` - -To pass inputs through Python expressions, specify the `--input_exprs` option. -This can be useful for when you don't have data -files lying around, but still want to sanity check the model with some simple -inputs that match the dtype and shape of the model's `SignatureDef`s. -For example: - -```bsh -`=[[1],[2],[3]]` -``` - -In addition to Python expressions, you may also pass numpy functions. For -example: - -```bsh -`=np.ones((32,32,3))` -``` - -(Note that the `numpy` module is already available to you as `np`.) - - -#### `--inputs_examples` - -To pass `tf.train.Example` as inputs, specify the `--input_examples` option. -For each input key, it takes a list of dictionary, where each dictionary is an -instance of `tf.train.Example`. The dictionary keys are the features and the -values are the value lists for each feature. -For example: - -```bsh -`=[{"age":[22,24],"education":["BS","MS"]}]` -``` - -#### Save output - -By default, the SavedModel CLI writes output to stdout. If a directory is -passed to `--outdir` option, the outputs will be saved as npy files named after -output tensor keys under the given directory. - -Use `--overwrite` to overwrite existing output files. - - -#### TensorFlow debugger (tfdbg) integration - -If `--tf_debug` option is set, the SavedModel CLI will use the -TensorFlow Debugger (tfdbg) to watch the intermediate Tensors and runtime -graphs or subgraphs while running the SavedModel. - - -#### Full examples of `run` - -Given: - -* Your model simply adds `x1` and `x2` to get output `y`. -* All tensors in the model have shape `(-1, 1)`. -* You have two `npy` files: - * `/tmp/my_data1.npy`, which contains a numpy ndarray `[[1], [2], [3]]`. - * `/tmp/my_data2.npy`, which contains another numpy - ndarray `[[0.5], [0.5], [0.5]]`. - -To run these two `npy` files through the model to get output `y`, issue -the following command: - -``` -$ saved_model_cli run --dir /tmp/saved_model_dir --tag_set serve \ ---signature_def x1_x2_to_y --inputs x1=/tmp/my_data1.npy;x2=/tmp/my_data2.npy \ ---outdir /tmp/out -Result for output key y: -[[ 1.5] - [ 2.5] - [ 3.5]] -``` - -Let's change the preceding example slightly. This time, instead of two -`.npy` files, you now have an `.npz` file and a pickle file. Furthermore, -you want to overwrite any existing output file. Here's the command: - -``` -$ saved_model_cli run --dir /tmp/saved_model_dir --tag_set serve \ ---signature_def x1_x2_to_y \ ---inputs x1=/tmp/my_data1.npz[x];x2=/tmp/my_data2.pkl --outdir /tmp/out \ ---overwrite -Result for output key y: -[[ 1.5] - [ 2.5] - [ 3.5]] -``` - -You may specify python expression instead of an input file. For example, -the following command replaces input `x2` with a Python expression: - -``` -$ saved_model_cli run --dir /tmp/saved_model_dir --tag_set serve \ ---signature_def x1_x2_to_y --inputs x1=/tmp/my_data1.npz[x] \ ---input_exprs 'x2=np.ones((3,1))' -Result for output key y: -[[ 2] - [ 3] - [ 4]] -``` - -To run the model with the TensorFlow Debugger on, issue the -following command: - -``` -$ saved_model_cli run --dir /tmp/saved_model_dir --tag_set serve \ ---signature_def serving_default --inputs x=/tmp/data.npz[x] --tf_debug -``` - - - -## Structure of a SavedModel directory - -When you save a model in SavedModel format, TensorFlow creates -a SavedModel directory consisting of the following subdirectories -and files: - -```bsh -assets/ -assets.extra/ -variables/ - variables.data-?????-of-????? - variables.index -saved_model.pb|saved_model.pbtxt -``` - -where: - -* `assets` is a subfolder containing auxiliary (external) files, - such as vocabularies. Assets are copied to the SavedModel location - and can be read when loading a specific `MetaGraphDef`. -* `assets.extra` is a subfolder where higher-level libraries and users can - add their own assets that co-exist with the model, but are not loaded by - the graph. This subfolder is not managed by the SavedModel libraries. -* `variables` is a subfolder that includes output from - `tf.train.Saver`. -* `saved_model.pb` or `saved_model.pbtxt` is the SavedModel protocol buffer. - It includes the graph definitions as `MetaGraphDef` protocol buffers. - -A single SavedModel can represent multiple graphs. In this case, all the -graphs in the SavedModel share a *single* set of checkpoints (variables) -and assets. For example, the following diagram shows one SavedModel -containing three `MetaGraphDef`s, all three of which share the same set -of checkpoints and assets: - -![SavedModel represents checkpoints, assets, and one or more MetaGraphDefs](../images/SavedModel.svg) - -Each graph is associated with a specific set of tags, which enables -identification during a load or restore operation. diff --git a/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md b/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md deleted file mode 100644 index fadfa03e78..0000000000 --- a/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md +++ /dev/null @@ -1,225 +0,0 @@ -# TensorBoard: Visualizing Learning - -The computations you'll use TensorFlow for - like training a massive -deep neural network - can be complex and confusing. To make it easier to -understand, debug, and optimize TensorFlow programs, we've included a suite of -visualization tools called TensorBoard. You can use TensorBoard to visualize -your TensorFlow graph, plot quantitative metrics about the execution of your -graph, and show additional data like images that pass through it. When -TensorBoard is fully configured, it looks like this: - -![MNIST TensorBoard](https://www.tensorflow.org/images/mnist_tensorboard.png "MNIST TensorBoard") - -
- -
- -This 30-minute tutorial is intended to get you started with simple TensorBoard -usage. It assumes a basic understanding of TensorFlow. - -There are other resources available as well! The [TensorBoard GitHub](https://github.com/tensorflow/tensorboard) -has a lot more information on using individual dashboards within TensorBoard -including tips & tricks and debugging information. - -## Setup - -[Install TensorFlow](https://www.tensorflow.org/install/). Installing TensorFlow -via pip should also automatically install TensorBoard. - -## Serializing the data - -TensorBoard operates by reading TensorFlow events files, which contain summary -data that you can generate when running TensorFlow. Here's the general -lifecycle for summary data within TensorBoard. - -First, create the TensorFlow graph that you'd like to collect summary -data from, and decide which nodes you would like to annotate with -@{$python/summary$summary operations}. - -For example, suppose you are training a convolutional neural network for -recognizing MNIST digits. You'd like to record how the learning rate -varies over time, and how the objective function is changing. Collect these by -attaching @{tf.summary.scalar} ops -to the nodes that output the learning rate and loss respectively. Then, give -each `scalar_summary` a meaningful `tag`, like `'learning rate'` or `'loss -function'`. - -Perhaps you'd also like to visualize the distributions of activations coming -off a particular layer, or the distribution of gradients or weights. Collect -this data by attaching -@{tf.summary.histogram} ops to -the gradient outputs and to the variable that holds your weights, respectively. - -For details on all of the summary operations available, check out the docs on -@{$python/summary$summary operations}. - -Operations in TensorFlow don't do anything until you run them, or an op that -depends on their output. And the summary nodes that we've just created are -peripheral to your graph: none of the ops you are currently running depend on -them. So, to generate summaries, we need to run all of these summary nodes. -Managing them by hand would be tedious, so use -@{tf.summary.merge_all} -to combine them into a single op that generates all the summary data. - -Then, you can just run the merged summary op, which will generate a serialized -`Summary` protobuf object with all of your summary data at a given step. -Finally, to write this summary data to disk, pass the summary protobuf to a -@{tf.summary.FileWriter}. - -The `FileWriter` takes a logdir in its constructor - this logdir is quite -important, it's the directory where all of the events will be written out. -Also, the `FileWriter` can optionally take a `Graph` in its constructor. -If it receives a `Graph` object, then TensorBoard will visualize your graph -along with tensor shape information. This will give you a much better sense of -what flows through the graph: see -@{$graph_viz#tensor-shape-information$Tensor shape information}. - -Now that you've modified your graph and have a `FileWriter`, you're ready to -start running your network! If you want, you could run the merged summary op -every single step, and record a ton of training data. That's likely to be more -data than you need, though. Instead, consider running the merged summary op -every `n` steps. - -The code example below is a modification of the -[simple MNIST tutorial](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/mnist/mnist.py), -in which we have added some summary ops, and run them every ten steps. If you -run this and then launch `tensorboard --logdir=/tmp/tensorflow/mnist`, you'll be able -to visualize statistics, such as how the weights or accuracy varied during -training. The code below is an excerpt; full source is -[here](https://www.tensorflow.org/code/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py). - -```python -def variable_summaries(var): - """Attach a lot of summaries to a Tensor (for TensorBoard visualization).""" - with tf.name_scope('summaries'): - mean = tf.reduce_mean(var) - tf.summary.scalar('mean', mean) - with tf.name_scope('stddev'): - stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) - tf.summary.scalar('stddev', stddev) - tf.summary.scalar('max', tf.reduce_max(var)) - tf.summary.scalar('min', tf.reduce_min(var)) - tf.summary.histogram('histogram', var) - -def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu): - """Reusable code for making a simple neural net layer. - - It does a matrix multiply, bias add, and then uses relu to nonlinearize. - It also sets up name scoping so that the resultant graph is easy to read, - and adds a number of summary ops. - """ - # Adding a name scope ensures logical grouping of the layers in the graph. - with tf.name_scope(layer_name): - # This Variable will hold the state of the weights for the layer - with tf.name_scope('weights'): - weights = weight_variable([input_dim, output_dim]) - variable_summaries(weights) - with tf.name_scope('biases'): - biases = bias_variable([output_dim]) - variable_summaries(biases) - with tf.name_scope('Wx_plus_b'): - preactivate = tf.matmul(input_tensor, weights) + biases - tf.summary.histogram('pre_activations', preactivate) - activations = act(preactivate, name='activation') - tf.summary.histogram('activations', activations) - return activations - -hidden1 = nn_layer(x, 784, 500, 'layer1') - -with tf.name_scope('dropout'): - keep_prob = tf.placeholder(tf.float32) - tf.summary.scalar('dropout_keep_probability', keep_prob) - dropped = tf.nn.dropout(hidden1, keep_prob) - -# Do not apply softmax activation yet, see below. -y = nn_layer(dropped, 500, 10, 'layer2', act=tf.identity) - -with tf.name_scope('cross_entropy'): - # The raw formulation of cross-entropy, - # - # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.softmax(y)), - # reduction_indices=[1])) - # - # can be numerically unstable. - # - # So here we use tf.losses.sparse_softmax_cross_entropy on the - # raw logit outputs of the nn_layer above. - with tf.name_scope('total'): - cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_, logits=y) -tf.summary.scalar('cross_entropy', cross_entropy) - -with tf.name_scope('train'): - train_step = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize( - cross_entropy) - -with tf.name_scope('accuracy'): - with tf.name_scope('correct_prediction'): - correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) - with tf.name_scope('accuracy'): - accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) -tf.summary.scalar('accuracy', accuracy) - -# Merge all the summaries and write them out to /tmp/mnist_logs (by default) -merged = tf.summary.merge_all() -train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train', - sess.graph) -test_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/test') -tf.global_variables_initializer().run() -``` - -After we've initialized the `FileWriters`, we have to add summaries to the -`FileWriters` as we train and test the model. - -```python -# Train the model, and also write summaries. -# Every 10th step, measure test-set accuracy, and write test summaries -# All other steps, run train_step on training data, & add training summaries - -def feed_dict(train): - """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" - if train or FLAGS.fake_data: - xs, ys = mnist.train.next_batch(100, fake_data=FLAGS.fake_data) - k = FLAGS.dropout - else: - xs, ys = mnist.test.images, mnist.test.labels - k = 1.0 - return {x: xs, y_: ys, keep_prob: k} - -for i in range(FLAGS.max_steps): - if i % 10 == 0: # Record summaries and test-set accuracy - summary, acc = sess.run([merged, accuracy], feed_dict=feed_dict(False)) - test_writer.add_summary(summary, i) - print('Accuracy at step %s: %s' % (i, acc)) - else: # Record train set summaries, and train - summary, _ = sess.run([merged, train_step], feed_dict=feed_dict(True)) - train_writer.add_summary(summary, i) -``` - -You're now all set to visualize this data using TensorBoard. - - -## Launching TensorBoard - -To run TensorBoard, use the following command (alternatively `python -m -tensorboard.main`) - -```bash -tensorboard --logdir=path/to/log-directory -``` - -where `logdir` points to the directory where the `FileWriter` serialized its -data. If this `logdir` directory contains subdirectories which contain -serialized data from separate runs, then TensorBoard will visualize the data -from all of those runs. Once TensorBoard is running, navigate your web browser -to `localhost:6006` to view the TensorBoard. - -When looking at TensorBoard, you will see the navigation tabs in the top right -corner. Each tab represents a set of serialized data that can be visualized. - -For in depth information on how to use the *graph* tab to visualize your graph, -see @{$graph_viz$TensorBoard: Graph Visualization}. - -For more usage information on TensorBoard in general, see the -[TensorBoard GitHub](https://github.com/tensorflow/tensorboard). diff --git a/tensorflow/docs_src/programmers_guide/tensorboard_histograms.md b/tensorflow/docs_src/programmers_guide/tensorboard_histograms.md deleted file mode 100644 index 918deda190..0000000000 --- a/tensorflow/docs_src/programmers_guide/tensorboard_histograms.md +++ /dev/null @@ -1,245 +0,0 @@ -# TensorBoard Histogram Dashboard - -The TensorBoard Histogram Dashboard displays how the distribution of some -`Tensor` in your TensorFlow graph has changed over time. It does this by showing -many histograms visualizations of your tensor at different points in time. - -## A Basic Example - -Let's start with a simple case: a normally-distributed variable, where the mean -shifts over time. -TensorFlow has an op -[`tf.random_normal`](https://www.tensorflow.org/api_docs/python/tf/random_normal) -which is perfect for this purpose. As is usually the case with TensorBoard, we -will ingest data using a summary op; in this case, -['tf.summary.histogram'](https://www.tensorflow.org/api_docs/python/tf/summary/histogram). -For a primer on how summaries work, please see the general -[TensorBoard tutorial](https://www.tensorflow.org/get_started/summaries_and_tensorboard). - -Here is a code snippet that will generate some histogram summaries containing -normally distributed data, where the mean of the distribution increases over -time. - -```python -import tensorflow as tf - -k = tf.placeholder(tf.float32) - -# Make a normal distribution, with a shifting mean -mean_moving_normal = tf.random_normal(shape=[1000], mean=(5*k), stddev=1) -# Record that distribution into a histogram summary -tf.summary.histogram("normal/moving_mean", mean_moving_normal) - -# Setup a session and summary writer -sess = tf.Session() -writer = tf.summary.FileWriter("/tmp/histogram_example") - -summaries = tf.summary.merge_all() - -# Setup a loop and write the summaries to disk -N = 400 -for step in range(N): - k_val = step/float(N) - summ = sess.run(summaries, feed_dict={k: k_val}) - writer.add_summary(summ, global_step=step) -``` - -Once that code runs, we can load the data into TensorBoard via the command line: - - -```sh -tensorboard --logdir=/tmp/histogram_example -``` - -Once TensorBoard is running, load it in Chrome or Firefox and navigate to the -Histogram Dashboard. Then we can see a histogram visualization for our normally -distributed data. - -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/1_moving_mean.png) - -`tf.summary.histogram` takes an arbitrarily sized and shaped Tensor, and -compresses it into a histogram data structure consisting of many bins with -widths and counts. For example, let's say we want to organize the numbers -`[0.5, 1.1, 1.3, 2.2, 2.9, 2.99]` into bins. We could make three bins: -* a bin -containing everything from 0 to 1 (it would contain one element, 0.5), -* a bin -containing everything from 1-2 (it would contain two elements, 1.1 and 1.3), -* a bin containing everything from 2-3 (it would contain three elements: 2.2, -2.9 and 2.99). - -TensorFlow uses a similar approach to create bins, but unlike in our example, it -doesn't create integer bins. For large, sparse datasets, that might result in -many thousands of bins. -Instead, [the bins are exponentially distributed, with many bins close to 0 and -comparatively few bins for very large numbers.](https://github.com/tensorflow/tensorflow/blob/c8b59c046895fa5b6d79f73e0b5817330fcfbfc1/tensorflow/core/lib/histogram/histogram.cc#L28) -However, visualizing exponentially-distributed bins is tricky; if height is used -to encode count, then wider bins take more space, even if they have the same -number of elements. Conversely, encoding count in the area makes height -comparisons impossible. Instead, the histograms [resample the data](https://github.com/tensorflow/tensorflow/blob/17c47804b86e340203d451125a721310033710f1/tensorflow/tensorboard/components/tf_backend/backend.ts#L400) -into uniform bins. This can lead to unfortunate artifacts in some cases. - -Each slice in the histogram visualizer displays a single histogram. -The slices are organized by step; -older slices (e.g. step 0) are further "back" and darker, while newer slices -(e.g. step 400) are close to the foreground, and lighter in color. -The y-axis on the right shows the step number. - -You can mouse over the histogram to see tooltips with some more detailed -information. For example, in the following image we can see that the histogram -at timestep 176 has a bin centered at 2.25 with 177 elements in that bin. - -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/2_moving_mean_tooltip.png) - -Also, you may note that the histogram slices are not always evenly spaced in -step count or time. This is because TensorBoard uses -[reservoir sampling](https://en.wikipedia.org/wiki/Reservoir_sampling) to keep a -subset of all the histograms, to save on memory. Reservoir sampling guarantees -that every sample has an equal likelihood of being included, but because it is -a randomized algorithm, the samples chosen don't occur at even steps. - -## Overlay Mode - -There is a control on the left of the dashboard that allows you to toggle the -histogram mode from "offset" to "overlay": - -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/3_overlay_offset.png) - -In "offset" mode, the visualization rotates 45 degrees, so that the individual -histogram slices are no longer spread out in time, but instead are all plotted -on the same y-axis. - -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/4_overlay.png) -Now, each slice is a separate line on the chart, and the y-axis shows the item -count within each bucket. Darker lines are older, earlier steps, and lighter -lines are more recent, later steps. Once again, you can mouse over the chart to -see some additional information. - -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/5_overlay_tooltips.png) - -In general, the overlay visualization is useful if you want to directly compare -the counts of different histograms. - -## Multimodal Distributions - -The Histogram Dashboard is great for visualizing multimodal -distributions. Let's construct a simple bimodal distribution by concatenating -the outputs from two different normal distributions. The code will look like -this: - -```python -import tensorflow as tf - -k = tf.placeholder(tf.float32) - -# Make a normal distribution, with a shifting mean -mean_moving_normal = tf.random_normal(shape=[1000], mean=(5*k), stddev=1) -# Record that distribution into a histogram summary -tf.summary.histogram("normal/moving_mean", mean_moving_normal) - -# Make a normal distribution with shrinking variance -variance_shrinking_normal = tf.random_normal(shape=[1000], mean=0, stddev=1-(k)) -# Record that distribution too -tf.summary.histogram("normal/shrinking_variance", variance_shrinking_normal) - -# Let's combine both of those distributions into one dataset -normal_combined = tf.concat([mean_moving_normal, variance_shrinking_normal], 0) -# We add another histogram summary to record the combined distribution -tf.summary.histogram("normal/bimodal", normal_combined) - -summaries = tf.summary.merge_all() - -# Setup a session and summary writer -sess = tf.Session() -writer = tf.summary.FileWriter("/tmp/histogram_example") - -# Setup a loop and write the summaries to disk -N = 400 -for step in range(N): - k_val = step/float(N) - summ = sess.run(summaries, feed_dict={k: k_val}) - writer.add_summary(summ, global_step=step) -``` - -You already remember our "moving mean" normal distribution from the example -above. Now we also have a "shrinking variance" distribution. Side-by-side, they -look like this: -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/6_two_distributions.png) - -When we concatenate them, we get a chart that clearly reveals the divergent, -bimodal structure: -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/7_bimodal.png) - -## Some more distributions - -Just for fun, let's generate and visualize a few more distributions, and then -combine them all into one chart. Here's the code we'll use: - -```python -import tensorflow as tf - -k = tf.placeholder(tf.float32) - -# Make a normal distribution, with a shifting mean -mean_moving_normal = tf.random_normal(shape=[1000], mean=(5*k), stddev=1) -# Record that distribution into a histogram summary -tf.summary.histogram("normal/moving_mean", mean_moving_normal) - -# Make a normal distribution with shrinking variance -variance_shrinking_normal = tf.random_normal(shape=[1000], mean=0, stddev=1-(k)) -# Record that distribution too -tf.summary.histogram("normal/shrinking_variance", variance_shrinking_normal) - -# Let's combine both of those distributions into one dataset -normal_combined = tf.concat([mean_moving_normal, variance_shrinking_normal], 0) -# We add another histogram summary to record the combined distribution -tf.summary.histogram("normal/bimodal", normal_combined) - -# Add a gamma distribution -gamma = tf.random_gamma(shape=[1000], alpha=k) -tf.summary.histogram("gamma", gamma) - -# And a poisson distribution -poisson = tf.random_poisson(shape=[1000], lam=k) -tf.summary.histogram("poisson", poisson) - -# And a uniform distribution -uniform = tf.random_uniform(shape=[1000], maxval=k*10) -tf.summary.histogram("uniform", uniform) - -# Finally, combine everything together! -all_distributions = [mean_moving_normal, variance_shrinking_normal, - gamma, poisson, uniform] -all_combined = tf.concat(all_distributions, 0) -tf.summary.histogram("all_combined", all_combined) - -summaries = tf.summary.merge_all() - -# Setup a session and summary writer -sess = tf.Session() -writer = tf.summary.FileWriter("/tmp/histogram_example") - -# Setup a loop and write the summaries to disk -N = 400 -for step in range(N): - k_val = step/float(N) - summ = sess.run(summaries, feed_dict={k: k_val}) - writer.add_summary(summ, global_step=step) -``` -### Gamma Distribution -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/8_gamma.png) - -### Uniform Distribution -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/9_uniform.png) - -### Poisson Distribution -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/10_poisson.png) -The poisson distribution is defined over the integers. So, all of the values -being generated are perfect integers. The histogram compression moves the data -into floating-point bins, causing the visualization to show little -bumps over the integer values rather than perfect spikes. - -### All Together Now -Finally, we can concatenate all of the data into one funny-looking curve. -![](https://www.tensorflow.org/images/tensorboard/histogram_dashboard/11_all_combined.png) - diff --git a/tensorflow/docs_src/programmers_guide/tensors.md b/tensorflow/docs_src/programmers_guide/tensors.md deleted file mode 100644 index 1248c3cabe..0000000000 --- a/tensorflow/docs_src/programmers_guide/tensors.md +++ /dev/null @@ -1,330 +0,0 @@ -# Tensors - -TensorFlow, as the name indicates, is a framework to define and run computations -involving tensors. A **tensor** is a generalization of vectors and matrices to -potentially higher dimensions. Internally, TensorFlow represents tensors as -n-dimensional arrays of base datatypes. - -When writing a TensorFlow program, the main object you manipulate and pass -around is the `tf.Tensor`. A `tf.Tensor` object represents a partially defined -computation that will eventually produce a value. TensorFlow programs work by -first building a graph of `tf.Tensor` objects, detailing how each tensor is -computed based on the other available tensors and then by running parts of this -graph to achieve the desired results. - -A `tf.Tensor` has the following properties: - - * a data type (`float32`, `int32`, or `string`, for example) - * a shape - - -Each element in the Tensor has the same data type, and the data type is always -known. The shape (that is, the number of dimensions it has and the size of each -dimension) might be only partially known. Most operations produce tensors of -fully-known shapes if the shapes of their inputs are also fully known, but in -some cases it's only possible to find the shape of a tensor at graph execution -time. - -Some types of tensors are special, and these will be covered in other -units of the Programmer's guide. The main ones are: - - * `tf.Variable` - * `tf.constant` - * `tf.placeholder` - * `tf.SparseTensor` - -With the exception of `tf.Variable`, the value of a tensor is immutable, which -means that in the context of a single execution tensors only have a single -value. However, evaluating the same tensor twice can return different values; -for example that tensor can be the result of reading data from disk, or -generating a random number. - -## Rank - -The **rank** of a `tf.Tensor` object is its number of dimensions. Synonyms for -rank include **order** or **degree** or **n-dimension**. -Note that rank in TensorFlow is not the same as matrix rank in mathematics. -As the following table shows, each rank in TensorFlow corresponds to a -different mathematical entity: - -Rank | Math entity ---- | --- -0 | Scalar (magnitude only) -1 | Vector (magnitude and direction) -2 | Matrix (table of numbers) -3 | 3-Tensor (cube of numbers) -n | n-Tensor (you get the idea) - - -### Rank 0 - -The following snippet demonstrates creating a few rank 0 variables: - -```python -mammal = tf.Variable("Elephant", tf.string) -ignition = tf.Variable(451, tf.int16) -floating = tf.Variable(3.14159265359, tf.float64) -its_complicated = tf.Variable(12.3 - 4.85j, tf.complex64) -``` - -Note: A string is treated as a single item in TensorFlow, not as a sequence of -characters. It is possible to have scalar strings, vectors of strings, etc. - -### Rank 1 - -To create a rank 1 `tf.Tensor` object, you can pass a list of items as the -initial value. For example: - -```python -mystr = tf.Variable(["Hello"], tf.string) -cool_numbers = tf.Variable([3.14159, 2.71828], tf.float32) -first_primes = tf.Variable([2, 3, 5, 7, 11], tf.int32) -its_very_complicated = tf.Variable([12.3 - 4.85j, 7.5 - 6.23j], tf.complex64) -``` - - -### Higher ranks - -A rank 2 `tf.Tensor` object consists of at least one row and at least -one column: - -```python -mymat = tf.Variable([[7],[11]], tf.int16) -myxor = tf.Variable([[False, True],[True, False]], tf.bool) -linear_squares = tf.Variable([[4], [9], [16], [25]], tf.int32) -squarish_squares = tf.Variable([ [4, 9], [16, 25] ], tf.int32) -rank_of_squares = tf.rank(squarish_squares) -mymatC = tf.Variable([[7],[11]], tf.int32) -``` - -Higher-rank Tensors, similarly, consist of an n-dimensional array. For example, -during image processing, many tensors of rank 4 are used, with dimensions -corresponding to example-in-batch, image width, image height, and color channel. - -``` python -my_image = tf.zeros([10, 299, 299, 3]) # batch x height x width x color -``` - -### Getting a `tf.Tensor` object's rank - -To determine the rank of a `tf.Tensor` object, call the `tf.rank` method. -For example, the following method programmatically determines the rank -of the `tf.Tensor` defined in the previous section: - -```python -r = tf.rank(my_image) -# After the graph runs, r will hold the value 4. -``` - -### Referring to `tf.Tensor` slices - -Since a `tf.Tensor` is an n-dimensional array of cells, to access a single cell -in a `tf.Tensor` you need to specify n indices. - -For a rank 0 tensor (a scalar), no indices are necessary, since it is already a -single number. - -For a rank 1 tensor (a vector), passing a single index allows you to access a -number: - -```python -my_scalar = my_vector[2] -``` - -Note that the index passed inside the `[]` can itself be a scalar `tf.Tensor`, if -you want to dynamically choose an element from the vector. - -For tensors of rank 2 or higher, the situation is more interesting. For a -`tf.Tensor` of rank 2, passing two numbers returns a scalar, as expected: - - -```python -my_scalar = my_matrix[1, 2] -``` - - -Passing a single number, however, returns a subvector of a matrix, as follows: - - -```python -my_row_vector = my_matrix[2] -my_column_vector = my_matrix[:, 3] -``` - -The `:` notation is python slicing syntax for "leave this dimension alone". This -is useful in higher-rank Tensors, as it allows you to access its subvectors, -submatrices, and even other subtensors. - - -## Shape - -The **shape** of a tensor is the number of elements in each dimension. -TensorFlow automatically infers shapes during graph construction. These inferred -shapes might have known or unknown rank. If the rank is known, the sizes of each -dimension might be known or unknown. - -The TensorFlow documentation uses three notational conventions to describe -tensor dimensionality: rank, shape, and dimension number. The following table -shows how these relate to one another: - -Rank | Shape | Dimension number | Example ---- | --- | --- | --- -0 | [] | 0-D | A 0-D tensor. A scalar. -1 | [D0] | 1-D | A 1-D tensor with shape [5]. -2 | [D0, D1] | 2-D | A 2-D tensor with shape [3, 4]. -3 | [D0, D1, D2] | 3-D | A 3-D tensor with shape [1, 4, 3]. -n | [D0, D1, ... Dn-1] | n-D | A tensor with shape [D0, D1, ... Dn-1]. - -Shapes can be represented via Python lists / tuples of ints, or with the -@{tf.TensorShape}. - -### Getting a `tf.Tensor` object's shape - -There are two ways of accessing the shape of a `tf.Tensor`. While building the -graph, it is often useful to ask what is already known about a tensor's -shape. This can be done by reading the `shape` property of a `tf.Tensor` object. -This method returns a `TensorShape` object, which is a convenient way of -representing partially-specified shapes (since, when building the graph, not all -shapes will be fully known). - -It is also possible to get a `tf.Tensor` that will represent the fully-defined -shape of another `tf.Tensor` at runtime. This is done by calling the `tf.shape` -operation. This way, you can build a graph that manipulates the shapes of -tensors by building other tensors that depend on the dynamic shape of the input -`tf.Tensor`. - -For example, here is how to make a vector of zeros with the same size as the -number of columns in a given matrix: - -``` python -zeros = tf.zeros(my_matrix.shape[1]) -``` - -### Changing the shape of a `tf.Tensor` - -The **number of elements** of a tensor is the product of the sizes of all its -shapes. The number of elements of a scalar is always `1`. Since there are often -many different shapes that have the same number of elements, it's often -convenient to be able to change the shape of a `tf.Tensor`, keeping its elements -fixed. This can be done with `tf.reshape`. - -The following examples demonstrate how to reshape tensors: - -```python -rank_three_tensor = tf.ones([3, 4, 5]) -matrix = tf.reshape(rank_three_tensor, [6, 10]) # Reshape existing content into - # a 6x10 matrix -matrixB = tf.reshape(matrix, [3, -1]) # Reshape existing content into a 3x20 - # matrix. -1 tells reshape to calculate - # the size of this dimension. -matrixAlt = tf.reshape(matrixB, [4, 3, -1]) # Reshape existing content into a - #4x3x5 tensor - -# Note that the number of elements of the reshaped Tensors has to match the -# original number of elements. Therefore, the following example generates an -# error because no possible value for the last dimension will match the number -# of elements. -yet_another = tf.reshape(matrixAlt, [13, 2, -1]) # ERROR! -``` - -## Data types - -In addition to dimensionality, Tensors have a data type. Refer to the -`tf.DataType` page in the programmer's guide for a full list of the data types. - -It is not possible to have a `tf.Tensor` with more than one data type. It is -possible, however, to serialize arbitrary data structures as `string`s and store -those in `tf.Tensor`s. - -It is possible to cast `tf.Tensor`s from one datatype to another using -`tf.cast`: - -``` python -# Cast a constant integer tensor into floating point. -float_tensor = tf.cast(tf.constant([1, 2, 3]), dtype=tf.float32) -``` - -To inspect a `tf.Tensor`'s data type use the `Tensor.dtype` property. - -When creating a `tf.Tensor` from a python object you may optionally specify the -datatype. If you don't, TensorFlow chooses a datatype that can represent your -data. TensorFlow converts Python integers to `tf.int32` and python floating -point numbers to `tf.float32`. Otherwise TensorFlow uses the same rules numpy -uses when converting to arrays. - -## Evaluating Tensors - -Once the computation graph has been built, you can run the computation that -produces a particular `tf.Tensor` and fetch the value assigned to it. This is -often useful for debugging as well as being required for much of TensorFlow to -work. - -The simplest way to evaluate a Tensor is using the `Tensor.eval` method. For -example: - -```python -constant = tf.constant([1, 2, 3]) -tensor = constant * constant -print(tensor.eval()) -``` - -The `eval` method only works when a default `tf.Session` is active (see -Graphs and Sessions for more information). - -`Tensor.eval` returns a numpy array with the same contents as the tensor. - -Sometimes it is not possible to evaluate a `tf.Tensor` with no context because -its value might depend on dynamic information that is not available. For -example, tensors that depend on `placeholder`s can't be evaluated without -providing a value for the `placeholder`. - -``` python -p = tf.placeholder(tf.float32) -t = p + 1.0 -t.eval() # This will fail, since the placeholder did not get a value. -t.eval(feed_dict={p:2.0}) # This will succeed because we're feeding a value - # to the placeholder. -``` - -Note that it is possible to feed any `tf.Tensor`, not just placeholders. - -Other model constructs might make evaluating a `tf.Tensor` -complicated. TensorFlow can't directly evaluate `tf.Tensor`s defined inside -functions or inside control flow constructs. If a `tf.Tensor` depends on a value -from a queue, evaluating the `tf.Tensor` will only work once something has been -enqueued; otherwise, evaluating it will hang. When working with queues, remember -to call `tf.train.start_queue_runners` before evaluating any `tf.Tensor`s. - -## Printing Tensors - -For debugging purposes you might want to print the value of a `tf.Tensor`. While - @{$debugger$tfdbg} provides advanced debugging support, TensorFlow also has an - operation to directly print the value of a `tf.Tensor`. - -Note that you rarely want to use the following pattern when printing a -`tf.Tensor`: - -``` python -t = <> -print(t) # This will print the symbolic tensor when the graph is being built. - # This tensor does not have a value in this context. -``` - -This code prints the `tf.Tensor` object (which represents deferred computation) -and not its value. Instead, TensorFlow provides the `tf.Print` operation, which -returns its first tensor argument unchanged while printing the set of -`tf.Tensor`s it is passed as the second argument. - -To correctly use `tf.Print` its return value must be used. See the example below - -``` python -t = <> -tf.Print(t, [t]) # This does nothing -t = tf.Print(t, [t]) # Here we are using the value returned by tf.Print -result = t + 1 # Now when result is evaluated the value of `t` will be printed. -``` - -When you evaluate `result` you will evaluate everything `result` depends -upon. Since `result` depends upon `t`, and evaluating `t` has the side effect of -printing its input (the old value of `t`), `t` gets printed. - diff --git a/tensorflow/docs_src/programmers_guide/using_gpu.md b/tensorflow/docs_src/programmers_guide/using_gpu.md deleted file mode 100644 index c429ca4750..0000000000 --- a/tensorflow/docs_src/programmers_guide/using_gpu.md +++ /dev/null @@ -1,215 +0,0 @@ -# Using GPUs - -## Supported devices - -On a typical system, there are multiple computing devices. In TensorFlow, the -supported device types are `CPU` and `GPU`. They are represented as `strings`. -For example: - -* `"/cpu:0"`: The CPU of your machine. -* `"/device:GPU:0"`: The GPU of your machine, if you have one. -* `"/device:GPU:1"`: The second GPU of your machine, etc. - -If a TensorFlow operation has both CPU and GPU implementations, the GPU devices -will be given priority when the operation is assigned to a device. For example, -`matmul` has both CPU and GPU kernels. On a system with devices `cpu:0` and -`gpu:0`, `gpu:0` will be selected to run `matmul`. - -## Logging Device placement - -To find out which devices your operations and tensors are assigned to, create -the session with `log_device_placement` configuration option set to `True`. - -```python -# Creates a graph. -a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') -b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') -c = tf.matmul(a, b) -# Creates a session with log_device_placement set to True. -sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) -# Runs the op. -print(sess.run(c)) -``` - -You should see the following output: - -``` -Device mapping: -/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: Tesla K40c, pci bus -id: 0000:05:00.0 -b: /job:localhost/replica:0/task:0/device:GPU:0 -a: /job:localhost/replica:0/task:0/device:GPU:0 -MatMul: /job:localhost/replica:0/task:0/device:GPU:0 -[[ 22. 28.] - [ 49. 64.]] - -``` - -## Manual device placement - -If you would like a particular operation to run on a device of your choice -instead of what's automatically selected for you, you can use `with tf.device` -to create a device context such that all the operations within that context will -have the same device assignment. - -```python -# Creates a graph. -with tf.device('/cpu:0'): - a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') - b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') -c = tf.matmul(a, b) -# Creates a session with log_device_placement set to True. -sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) -# Runs the op. -print(sess.run(c)) -``` - -You will see that now `a` and `b` are assigned to `cpu:0`. Since a device was -not explicitly specified for the `MatMul` operation, the TensorFlow runtime will -choose one based on the operation and available devices (`gpu:0` in this -example) and automatically copy tensors between devices if required. - -``` -Device mapping: -/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: Tesla K40c, pci bus -id: 0000:05:00.0 -b: /job:localhost/replica:0/task:0/cpu:0 -a: /job:localhost/replica:0/task:0/cpu:0 -MatMul: /job:localhost/replica:0/task:0/device:GPU:0 -[[ 22. 28.] - [ 49. 64.]] -``` - -## Allowing GPU memory growth - -By default, TensorFlow maps nearly all of the GPU memory of all GPUs (subject to -[`CUDA_VISIBLE_DEVICES`](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars)) -visible to the process. This is done to more efficiently use the relatively -precious GPU memory resources on the devices by reducing [memory -fragmentation](https://en.wikipedia.org/wiki/Fragmentation_\(computing\)). - -In some cases it is desirable for the process to only allocate a subset of the -available memory, or to only grow the memory usage as is needed by the process. -TensorFlow provides two Config options on the Session to control this. - -The first is the `allow_growth` option, which attempts to allocate only as much -GPU memory based on runtime allocations: it starts out allocating very little -memory, and as Sessions get run and more GPU memory is needed, we extend the GPU -memory region needed by the TensorFlow process. Note that we do not release -memory, since that can lead to even worse memory fragmentation. To turn this -option on, set the option in the ConfigProto by: - -```python -config = tf.ConfigProto() -config.gpu_options.allow_growth = True -session = tf.Session(config=config, ...) -``` - -The second method is the `per_process_gpu_memory_fraction` option, which -determines the fraction of the overall amount of memory that each visible GPU -should be allocated. For example, you can tell TensorFlow to only allocate 40% -of the total memory of each GPU by: - -```python -config = tf.ConfigProto() -config.gpu_options.per_process_gpu_memory_fraction = 0.4 -session = tf.Session(config=config, ...) -``` - -This is useful if you want to truly bound the amount of GPU memory available to -the TensorFlow process. - -## Using a single GPU on a multi-GPU system - -If you have more than one GPU in your system, the GPU with the lowest ID will be -selected by default. If you would like to run on a different GPU, you will need -to specify the preference explicitly: - -```python -# Creates a graph. -with tf.device('/device:GPU:2'): - a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') - b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') - c = tf.matmul(a, b) -# Creates a session with log_device_placement set to True. -sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) -# Runs the op. -print(sess.run(c)) -``` - -If the device you have specified does not exist, you will get -`InvalidArgumentError`: - -``` -InvalidArgumentError: Invalid argument: Cannot assign a device to node 'b': -Could not satisfy explicit device specification '/device:GPU:2' - [[Node: b = Const[dtype=DT_FLOAT, value=Tensor, _device="/device:GPU:2"]()]] -``` - -If you would like TensorFlow to automatically choose an existing and supported -device to run the operations in case the specified one doesn't exist, you can -set `allow_soft_placement` to `True` in the configuration option when creating -the session. - -```python -# Creates a graph. -with tf.device('/device:GPU:2'): - a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3], name='a') - b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2], name='b') - c = tf.matmul(a, b) -# Creates a session with allow_soft_placement and log_device_placement set -# to True. -sess = tf.Session(config=tf.ConfigProto( - allow_soft_placement=True, log_device_placement=True)) -# Runs the op. -print(sess.run(c)) -``` - -## Using multiple GPUs - -If you would like to run TensorFlow on multiple GPUs, you can construct your -model in a multi-tower fashion where each tower is assigned to a different GPU. -For example: - -``` python -# Creates a graph. -c = [] -for d in ['/device:GPU:2', '/device:GPU:3']: - with tf.device(d): - a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[2, 3]) - b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape=[3, 2]) - c.append(tf.matmul(a, b)) -with tf.device('/cpu:0'): - sum = tf.add_n(c) -# Creates a session with log_device_placement set to True. -sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) -# Runs the op. -print(sess.run(sum)) -``` - -You will see the following output. - -``` -Device mapping: -/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: Tesla K20m, pci bus -id: 0000:02:00.0 -/job:localhost/replica:0/task:0/device:GPU:1 -> device: 1, name: Tesla K20m, pci bus -id: 0000:03:00.0 -/job:localhost/replica:0/task:0/device:GPU:2 -> device: 2, name: Tesla K20m, pci bus -id: 0000:83:00.0 -/job:localhost/replica:0/task:0/device:GPU:3 -> device: 3, name: Tesla K20m, pci bus -id: 0000:84:00.0 -Const_3: /job:localhost/replica:0/task:0/device:GPU:3 -Const_2: /job:localhost/replica:0/task:0/device:GPU:3 -MatMul_1: /job:localhost/replica:0/task:0/device:GPU:3 -Const_1: /job:localhost/replica:0/task:0/device:GPU:2 -Const: /job:localhost/replica:0/task:0/device:GPU:2 -MatMul: /job:localhost/replica:0/task:0/device:GPU:2 -AddN: /job:localhost/replica:0/task:0/cpu:0 -[[ 44. 56.] - [ 98. 128.]] -``` - -The @{$deep_cnn$cifar10 tutorial} is a good example -demonstrating how to do training with multiple GPUs. diff --git a/tensorflow/docs_src/programmers_guide/using_tpu.md b/tensorflow/docs_src/programmers_guide/using_tpu.md deleted file mode 100644 index 44aabf0557..0000000000 --- a/tensorflow/docs_src/programmers_guide/using_tpu.md +++ /dev/null @@ -1,395 +0,0 @@ -# Using TPUs - -This document walks through the principal TensorFlow APIs necessary to make -effective use of a [Cloud TPU](https://cloud.google.com/tpu/), and highlights -the differences between regular TensorFlow usage, and usage on a TPU. - -This doc is aimed at users who: - -* Are familiar with TensorFlow's `Estimator` and `Dataset` APIs -* Have maybe [tried out a Cloud TPU](https://cloud.google.com/tpu/docs/quickstart) - using an existing model. -* Have, perhaps, skimmed the code of an example TPU model - [[1]](https://github.com/tensorflow/models/blob/master/official/mnist/mnist_tpu.py) - [[2]](https://github.com/tensorflow/tpu/tree/master/models). -* Are interested in porting an existing `Estimator` model to - run on Cloud TPUs - -## TPUEstimator - -@{tf.estimator.Estimator$Estimators} are TensorFlow's model-level abstraction. -Standard `Estimators` can drive models on CPU and GPUs. You must use -@{tf.contrib.tpu.TPUEstimator} to drive a model on TPUs. - -Refer to TensorFlow's Getting Started section for an introduction to the basics -of using a @{$premade_estimators$pre-made `Estimator`}, and -@{$custom_estimators$custom `Estimator`s}. - -The `TPUEstimator` class differs somewhat from the `Estimator` class. - -The simplest way to maintain a model that can be run both on CPU/GPU or on a -Cloud TPU is to define the model's inference phase (from inputs to predictions) -outside of the `model_fn`. Then maintain separate implementations of the -`Estimator` setup and `model_fn`, both wrapping this inference step. For an -example of this pattern compare the `mnist.py` and `mnist_tpu.py` implementation in -[tensorflow/models](https://github.com/tensorflow/models/tree/master/official/mnist). - -### Running a `TPUEstimator` locally - -To create a standard `Estimator` you call the constructor, and pass it a -`model_fn`, for example: - -``` -my_estimator = tf.estimator.Estimator( - model_fn=my_model_fn) -``` - -The changes required to use a @{tf.contrib.tpu.TPUEstimator} on your local -machine are relatively minor. The constructor requires two additional arguments. -You should set the `use_tpu` argument to `False`, and pass a -@{tf.contrib.tpu.RunConfig} as the `config` argument, as shown below: - -``` python -my_tpu_estimator = tf.contrib.tpu.TPUEstimator( - model_fn=my_model_fn, - config=tf.contrib.tpu.RunConfig() - use_tpu=False) -``` - -Just this simple change will allow you to run a `TPUEstimator` locally. -The majority of example TPU models can be run in this local mode, -by setting the command line flags as follows: - - -``` -$> python mnist_tpu.py --use_tpu=false --master='' -``` - -Note: This `use_tpu=False` argument is useful for trying out the `TPUEstimator` -API. It is not meant to be a complete TPU compatibility test. Successfully -running a model locally in a `TPUEstimator` does not guarantee that it will -work on a TPU. - - -### Building a `tpu.RunConfig` - -While the default `RunConfig` is sufficient for local training, these settings -cannot be ignored in real usage. - -A more typical setup for a `RunConfig`, that can be switched to use a Cloud -TPU, might be as follows: - -``` python -import tempfile -import subprocess - -class FLAGS(object): - use_tpu=False - tpu_name=None - # Use a local temporary path for the `model_dir` - model_dir = tempfile.mkdtemp() - # Number of training steps to run on the Cloud TPU before returning control. - iterations = 50 - # A single Cloud TPU has 8 shards. - num_shards = 8 - -if FLAGS.use_tpu: - my_project_name = subprocess.check_output([ - 'gcloud','config','get-value','project']) - my_zone = subprocess.check_output([ - 'gcloud','config','get-value','compute/zone']) - cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( - tpu_names=[FLAGS.tpu_name], - zone=my_zone, - project=my_project) - master = tpu_cluster_resolver.get_master() -else: - master = '' - -my_tpu_run_config = tf.contrib.tpu.RunConfig( - master=master, - evaluation_master=master, - model_dir=FLAGS.model_dir, - session_config=tf.ConfigProto( - allow_soft_placement=True, log_device_placement=True), - tpu_config=tf.contrib.tpu.TPUConfig(FLAGS.iterations, - FLAGS.num_shards), -) -``` - -Then you must pass the @{tf.contrib.tpu.RunConfig} to the constructor: - -``` python -my_tpu_estimator = tf.contrib.tpu.TPUEstimator( - model_fn=my_model_fn, - config = my_tpu_run_config, - use_tpu=FLAGS.use_tpu) -``` - -Typically the `FLAGS` would be set by command line arguments. To switch from -training locally to training on a cloud TPU you would need to: - -* Set `FLAGS.use_tpu` to `True` -* Set `FLAGS.tpu_name` so the `tf.contrib.cluster_resolver.TPUClusterResolver` can find it -* Set `FLAGS.model_dir` to a Google Cloud Storage bucket url (`gs://`). - - -## Optimizer - -When training on a cloud TPU you **must** wrap the optimizer in a -@{tf.contrib.tpu.CrossShardOptimizer}, which uses an `allreduce` to aggregate -gradients and broadcast the result to each shard (each TPU core). - -The `CrossShardOptimizer` is not compatible with local training. So, to have -the same code run both locally and on a Cloud TPU, add lines like the following: - -``` python -optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate) -if FLAGS.use_tpu: - optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) -``` - -If you prefer to avoid a global `FLAGS` variable in your model code, one -approach is to set the optimizer as one of the `Estimator`'s params, -as follows: - -``` python -my_tpu_estimator = tf.contrib.tpu.TPUEstimator( - model_fn=my_model_fn, - config = my_tpu_run_config, - use_tpu=FLAGS.use_tpu, - params={'optimizer':optimizer}) -``` - -## Model Function - -This section details the changes you must make to the model function -(`model_fn()`) to make it `TPUEstimator` compatible. - -### Static shapes - -During regular usage TensorFlow attempts to determine the shapes of each -`tf.Tensor` during graph construction. During execution any unknown shape -dimensions are determined dynamically, -see @{$programmers_guide/tensors#shape$Tensor Shapes} for more details. - -To run on Cloud TPUs TensorFlow models are compiled using @{$xla$XLA}. -XLA uses a similar system for determining shapes at compile time. XLA requires -that all tensor dimensions be statically defined at compile time. All shapes -must evaluate to a constant, and not depend on external data, or stateful -operations like variables or a random number generator. - - -### Summaries - -Remove any use of `tf.summary` from your model. - -@{$summaries_and_tensorboard$TensorBoard summaries} are a great way see inside -your model. A minimal set of basic summaries are automatically recorded by the -`TPUEstimator`, to `event` files in the `model_dir`. Custom summaries, however, -are currently unsupported when training on a Cloud TPU. So while the -`TPUEstimator` will still run locally with summaries, it will fail if used on a -TPU. - -### Metrics - -Build your evaluation metrics dictionary in a stand-alone `metric_fn`. - - - -Evaluation metrics are an essential part of training a model. These are fully -supported on Cloud TPUs, but with a slightly different syntax. - -A standard @{tf.metrics} returns two tensors. The first returns the running -average of the metric value, while the second updates the running average and -returns the value for this batch: - -``` -running_average, current_batch = tf.metrics.accuracy(labels, predictions) -``` - -In a standard `Estimator` you create a dictionary of these pairs, and return it -as part of the `EstimatorSpec`. - -```python -my_metrics = {'accuracy': tf.metrics.accuracy(labels, predictions)} - -return tf.estimator.EstimatorSpec( - ... - eval_metric_ops=my_metrics -) -``` - -In a `TPUEstimator` you instead pass a function (which returns a metrics -dictionary) and a list of argument tensors, as shown below: - -```python -def my_metric_fn(labels, predictions): - return {'accuracy': tf.metrics.accuracy(labels, predictions)} - -return tf.contrib.tpu.TPUEstimatorSpec( - ... - eval_metrics=(my_metric_fn, [labels, predictions]) -) -``` - -### Use `TPUEstimatorSpec` - -`TPUEstimatorSpec` do not support hooks, and require function wrappers for -some fields. - -An `Estimator`'s `model_fn` must return an `EstimatorSpec`. An `EstimatorSpec` -is a simple structure of named fields containing all the `tf.Tensors` of the -model that the `Estimator` may need to interact with. - -`TPUEstimators` use a @{tf.contrib.tpu.TPUEstimatorSpec}. There are a few -differences between it and a standard @{tf.estimator.EstimatorSpec}: - - -* The `eval_metric_ops` must be wrapped into a `metrics_fn`, this field is - renamed `eval_metrics` ([see above](#metrics)). -* The @{tf.train.SessionRunHook$hooks} are unsupported, so these fields are - omitted. -* The @{tf.train.Scaffold$`scaffold`}, if used, must also be wrapped in a - function. This field is renamed to `scaffold_fn`. - -`Scaffold` and `Hooks` are for advanced usage, and can typically be omitted. - -## Input functions - -Input functions work mainly unchanged as they run on the host computer, not the -Cloud TPU itself. This section explains the two necessary adjustments. - -### Params argument - - - -The `input_fn` for a standard `Estimator` _can_ include a -`params` argument; the `input_fn` for a `TPUEstimator` *must* include a -`params` argument. This is necessary to allow the estimator to set the batch -size for each replica of the input stream. So the minimum signature for an -`input_fn` for a `TPUEstimator` is: - -``` -def my_input_fn(params): - pass -``` - -Where `params['batch-size']` will contain the batch size. - -### Static shapes and batch size - -The input pipeline generated by your `input_fn` is run on CPU. So it is mostly -free from the strict static shape requirements imposed by the XLA/TPU environment. -The one requirement is that the batches of data fed from your input pipeline to -the TPU have a static shape, as determined by the standard TensorFlow shape -inference algorithm. Intermediate tensors are free to have a dynamic shapes. -If shape inference has failed, but the shape is known it is possible to -impose the correct shape using `tf.set_shape()`. - -In the example below the shape -inference algorithm fails, but it is correctly using `set_shape`: - -``` ->>> x = tf.zeros(tf.constant([1,2,3])+1) ->>> x.shape - -TensorShape([Dimension(None), Dimension(None), Dimension(None)]) - ->>> x.set_shape([2,3,4]) -``` - -In many cases the batch size is the only unknown dimension. - -A typical input pipeline, using `tf.data`, will usually produce batches of a -fixed size. The last batch of a finite `Dataset`, however, is typically smaller, -containing just the remaining elements. Since a `Dataset` does not know its own -length or finiteness, the standard @{tf.data.Dataset.batch$`batch`} method -cannot determine if all batches will have a fixed size batch on its own: - -``` ->>> params = {'batch_size':32} ->>> ds = tf.data.Dataset.from_tensors([0, 1, 2]) ->>> ds = ds.repeat().batch(params['batch-size']) ->>> ds - - -``` - -The most straightforward fix is to -@{tf.data.Dataset.apply$apply} @{tf.contrib.data.batch_and_drop_remainder} -as follows: - -``` ->>> params = {'batch_size':32} ->>> ds = tf.data.Dataset.from_tensors([0, 1, 2]) ->>> ds = ds.repeat().apply( -... tf.contrib.data.batch_and_drop_remainder(params['batch-size'])) ->>> ds - - <_RestructuredDataset shapes: (32, 3), types: tf.int32> -``` - -The one downside to this approach is that, as the name implies, this batching -method throws out any fractional batch at the end of the dataset. This is fine -for an infinitely repeating dataset being used for training, but could be a -problem if you want to train for an exact number of epochs. - -To do an exact 1-epoch of _evaluation_ you can work around this by manually -padding the length of the batches, and setting the padding entries to have zero -weight when creating your `tf.metrics`. - -## Datasets - -Efficient use of the `tf.data.Dataset` API is critical when using a Cloud -TPU, as it is impossible to use the Cloud TPU's unless you can feed it data -quickly enough. See @{$datasets_performance} for details on dataset performance. - -For all but the simplest experimentation (using -@{tf.data.Dataset.from_tensor_slices} or other in-graph data) you will need to -store all data files read by the `TPUEstimator`'s `Dataset` in Google Cloud -Storage Buckets. - - - -For most use-cases, we recommend converting your data into `TFRecord` -format and using a @{tf.data.TFRecordDataset} to read it. This, however, is not -a hard requirement and you can use other dataset readers -(`FixedLengthRecordDataset` or `TextLineDataset`) if you prefer. - -Small datasets can be loaded entirely into memory using -@{tf.data.Dataset.cache}. - -Regardless of the data format used, it is strongly recommended that you -@{$performance_guide#use_large_files$use large files}, on the order of -100MB. This is especially important in this networked setting as the overhead -of opening a file is significantly higher. - -It is also important, regardless of the type of reader used, to enable buffering -using the `buffer_size` argument to the constructor. This argument is specified -in bytes. A minimum of a few MB (`buffer_size=8*1024*1024`) is recommended so -that data is available when needed. - -The TPU-demos repo includes -[a script](https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py) -for downloading the imagenet dataset and converting it to an appropriate format. -This together with the imagenet -[models](https://github.com/tensorflow/tpu/tree/master/models) -included in the repo demonstrate all of these best-practices. - - -## What Next - -For details on how to actually set up and run a Cloud TPU see: - - * [Google Cloud TPU Documentation](https://cloud.google.com/tpu/docs/) - -This document is by no means exhaustive. The best source of more detail on how -to make a Cloud TPU compatible model are the example models published in: - - * The [TPU Demos Repository.](https://github.com/tensorflow/tpu) - -For more information about tuning TensorFlow code for performance see: - - * The @{$performance$Performance Section.} - diff --git a/tensorflow/docs_src/programmers_guide/variables.md b/tensorflow/docs_src/programmers_guide/variables.md deleted file mode 100644 index cd8c4b5b9a..0000000000 --- a/tensorflow/docs_src/programmers_guide/variables.md +++ /dev/null @@ -1,319 +0,0 @@ -# Variables - -A TensorFlow **variable** is the best way to represent shared, persistent state -manipulated by your program. - -Variables are manipulated via the `tf.Variable` class. A `tf.Variable` -represents a tensor whose value can be changed by running ops on it. Unlike -`tf.Tensor` objects, a `tf.Variable` exists outside the context of a single -`session.run` call. - -Internally, a `tf.Variable` stores a persistent tensor. Specific ops allow you -to read and modify the values of this tensor. These modifications are visible -across multiple `tf.Session`s, so multiple workers can see the same values for a -`tf.Variable`. - -## Creating a Variable - -The best way to create a variable is to call the `tf.get_variable` -function. This function requires you to specify the Variable's name. This name -will be used by other replicas to access the same variable, as well as to name -this variable's value when checkpointing and exporting models. `tf.get_variable` -also allows you to reuse a previously created variable of the same name, making it -easy to define models which reuse layers. - -To create a variable with `tf.get_variable`, simply provide the name and shape - -``` python -my_variable = tf.get_variable("my_variable", [1, 2, 3]) -``` - -This creates a variable named "my_variable" which is a three-dimensional tensor -with shape `[1, 2, 3]`. This variable will, by default, have the `dtype` -`tf.float32` and its initial value will be randomized via -`tf.glorot_uniform_initializer`. - -You may optionally specify the `dtype` and initializer to `tf.get_variable`. For -example: - -``` python -my_int_variable = tf.get_variable("my_int_variable", [1, 2, 3], dtype=tf.int32, - initializer=tf.zeros_initializer) -``` - -TensorFlow provides many convenient initializers. Alternatively, you may -initialize a `tf.Variable` to have the value of a `tf.Tensor`. For example: - -``` python -other_variable = tf.get_variable("other_variable", dtype=tf.int32, - initializer=tf.constant([23, 42])) -``` - -Note that when the initializer is a `tf.Tensor` you should not specify the -variable's shape, as the shape of the initializer tensor will be used. - - - -### Variable collections - -Because disconnected parts of a TensorFlow program might want to create -variables, it is sometimes useful to have a single way to access all of -them. For this reason TensorFlow provides **collections**, which are named lists -of tensors or other objects, such as `tf.Variable` instances. - -By default every `tf.Variable` gets placed in the following two collections: - - * `tf.GraphKeys.GLOBAL_VARIABLES` --- variables that can be shared across - multiple devices, - * `tf.GraphKeys.TRAINABLE_VARIABLES` --- variables for which TensorFlow will - calculate gradients. - -If you don't want a variable to be trainable, add it to the -`tf.GraphKeys.LOCAL_VARIABLES` collection instead. For example, the following -snippet demonstrates how to add a variable named `my_local` to this collection: - -``` python -my_local = tf.get_variable("my_local", shape=(), -collections=[tf.GraphKeys.LOCAL_VARIABLES]) -``` - -Alternatively, you can specify `trainable=False` as an argument to -`tf.get_variable`: - -``` python -my_non_trainable = tf.get_variable("my_non_trainable", - shape=(), - trainable=False) -``` - - -You can also use your own collections. Any string is a valid collection name, -and there is no need to explicitly create a collection. To add a variable (or -any other object) to a collection after creating the variable, call -`tf.add_to_collection`. For example, the following code adds an existing -variable named `my_local` to a collection named `my_collection_name`: - -``` python -tf.add_to_collection("my_collection_name", my_local) -``` - -And to retrieve a list of all the variables (or other objects) you've placed in -a collection you can use: - -``` python -tf.get_collection("my_collection_name") -``` - -### Device placement - -Just like any other TensorFlow operation, you can place variables on particular -devices. For example, the following snippet creates a variable named `v` and -places it on the second GPU device: - -``` python -with tf.device("/device:GPU:1"): - v = tf.get_variable("v", [1]) -``` - -It is particularly important for variables to be in the correct device in -distributed settings. Accidentally putting variables on workers instead of -parameter servers, for example, can severely slow down training or, in the worst -case, let each worker blithely forge ahead with its own independent copy of each -variable. For this reason we provide @{tf.train.replica_device_setter}, which -can automatically place variables in parameter servers. For example: - -``` python -cluster_spec = { - "ps": ["ps0:2222", "ps1:2222"], - "worker": ["worker0:2222", "worker1:2222", "worker2:2222"]} -with tf.device(tf.train.replica_device_setter(cluster=cluster_spec)): - v = tf.get_variable("v", shape=[20, 20]) # this variable is placed - # in the parameter server - # by the replica_device_setter -``` - -## Initializing variables - -Before you can use a variable, it must be initialized. If you are programming in -the low-level TensorFlow API (that is, you are explicitly creating your own -graphs and sessions), you must explicitly initialize the variables. Most -high-level frameworks such as `tf.contrib.slim`, `tf.estimator.Estimator` and -`Keras` automatically initialize variables for you before training a model. - -Explicit initialization is otherwise useful because it allows you not to rerun -potentially expensive initializers when reloading a model from a checkpoint as -well as allowing determinism when randomly-initialized variables are shared in a -distributed setting. - -To initialize all trainable variables in one go, before training starts, call -`tf.global_variables_initializer()`. This function returns a single operation -responsible for initializing all variables in the -`tf.GraphKeys.GLOBAL_VARIABLES` collection. Running this operation initializes -all variables. For example: - -``` python -session.run(tf.global_variables_initializer()) -# Now all variables are initialized. -``` - -If you do need to initialize variables yourself, you can run the variable's -initializer operation. For example: - -``` python -session.run(my_variable.initializer) -``` - - -You can also ask which variables have still not been initialized. For example, -the following code prints the names of all variables which have not yet been -initialized: - -``` python -print(session.run(tf.report_uninitialized_variables())) -``` - - -Note that by default `tf.global_variables_initializer` does not specify the -order in which variables are initialized. Therefore, if the initial value of a -variable depends on another variable's value, it's likely that you'll get an -error. Any time you use the value of a variable in a context in which not all -variables are initialized (say, if you use a variable's value while initializing -another variable), it is best to use `variable.initialized_value()` instead of -`variable`: - -``` python -v = tf.get_variable("v", shape=(), initializer=tf.zeros_initializer()) -w = tf.get_variable("w", initializer=v.initialized_value() + 1) -``` - -## Using variables - -To use the value of a `tf.Variable` in a TensorFlow graph, simply treat it like -a normal `tf.Tensor`: - -``` python -v = tf.get_variable("v", shape=(), initializer=tf.zeros_initializer()) -w = v + 1 # w is a tf.Tensor which is computed based on the value of v. - # Any time a variable is used in an expression it gets automatically - # converted to a tf.Tensor representing its value. -``` - -To assign a value to a variable, use the methods `assign`, `assign_add`, and -friends in the `tf.Variable` class. For example, here is how you can call these -methods: - -``` python -v = tf.get_variable("v", shape=(), initializer=tf.zeros_initializer()) -assignment = v.assign_add(1) -tf.global_variables_initializer().run() -sess.run(assignment) # or assignment.op.run(), or assignment.eval() -``` - -Most TensorFlow optimizers have specialized ops that efficiently update the -values of variables according to some gradient descent-like algorithm. See -@{tf.train.Optimizer} for an explanation of how to use optimizers. - -Because variables are mutable it's sometimes useful to know what version of a -variable's value is being used at any point in time. To force a re-read of the -value of a variable after something has happened, you can use -`tf.Variable.read_value`. For example: - -``` python -v = tf.get_variable("v", shape=(), initializer=tf.zeros_initializer()) -assignment = v.assign_add(1) -with tf.control_dependencies([assignment]): - w = v.read_value() # w is guaranteed to reflect v's value after the - # assign_add operation. -``` - - -## Sharing variables - -TensorFlow supports two ways of sharing variables: - - * Explicitly passing `tf.Variable` objects around. - * Implicitly wrapping `tf.Variable` objects within `tf.variable_scope` objects. - -While code which explicitly passes variables around is very clear, it is -sometimes convenient to write TensorFlow functions that implicitly use -variables in their implementations. Most of the functional layers from -`tf.layers` use this approach, as well as all `tf.metrics`, and a few other -library utilities. - -Variable scopes allow you to control variable reuse when calling functions which -implicitly create and use variables. They also allow you to name your variables -in a hierarchical and understandable way. - -For example, let's say we write a function to create a convolutional / relu -layer: - -```python -def conv_relu(input, kernel_shape, bias_shape): - # Create variable named "weights". - weights = tf.get_variable("weights", kernel_shape, - initializer=tf.random_normal_initializer()) - # Create variable named "biases". - biases = tf.get_variable("biases", bias_shape, - initializer=tf.constant_initializer(0.0)) - conv = tf.nn.conv2d(input, weights, - strides=[1, 1, 1, 1], padding='SAME') - return tf.nn.relu(conv + biases) -``` - -This function uses short names `weights` and `biases`, which is good for -clarity. In a real model, however, we want many such convolutional layers, and -calling this function repeatedly would not work: - -``` python -input1 = tf.random_normal([1,10,10,32]) -input2 = tf.random_normal([1,20,20,32]) -x = conv_relu(input1, kernel_shape=[5, 5, 32, 32], bias_shape=[32]) -x = conv_relu(x, kernel_shape=[5, 5, 32, 32], bias_shape = [32]) # This fails. -``` - -Since the desired behavior is unclear (create new variables or reuse the -existing ones?) TensorFlow will fail. Calling `conv_relu` in different scopes, -however, clarifies that we want to create new variables: - -```python -def my_image_filter(input_images): - with tf.variable_scope("conv1"): - # Variables created here will be named "conv1/weights", "conv1/biases". - relu1 = conv_relu(input_images, [5, 5, 32, 32], [32]) - with tf.variable_scope("conv2"): - # Variables created here will be named "conv2/weights", "conv2/biases". - return conv_relu(relu1, [5, 5, 32, 32], [32]) -``` - -If you do want the variables to be shared, you have two options. First, you can -create a scope with the same name using `reuse=True`: - -``` python -with tf.variable_scope("model"): - output1 = my_image_filter(input1) -with tf.variable_scope("model", reuse=True): - output2 = my_image_filter(input2) - -``` - -You can also call `scope.reuse_variables()` to trigger a reuse: - -``` python -with tf.variable_scope("model") as scope: - output1 = my_image_filter(input1) - scope.reuse_variables() - output2 = my_image_filter(input2) - -``` - -Since depending on exact string names of scopes can feel dangerous, it's also -possible to initialize a variable scope based on another one: - -``` python -with tf.variable_scope("model") as scope: - output1 = my_image_filter(input1) -with tf.variable_scope(scope, reuse=True): - output2 = my_image_filter(input2) - -``` - diff --git a/tensorflow/docs_src/programmers_guide/version_compat.md b/tensorflow/docs_src/programmers_guide/version_compat.md deleted file mode 100644 index 72e427c5f8..0000000000 --- a/tensorflow/docs_src/programmers_guide/version_compat.md +++ /dev/null @@ -1,319 +0,0 @@ -# TensorFlow Version Compatibility - -This document is for users who need backwards compatibility across different -versions of TensorFlow (either for code or data), and for developers who want -to modify TensorFlow while preserving compatibility. - -## Semantic Versioning 2.0 - -TensorFlow follows Semantic Versioning 2.0 ([semver](http://semver.org)) for its -public API. Each release version of TensorFlow has the form `MAJOR.MINOR.PATCH`. -For example, TensorFlow version 1.2.3 has `MAJOR` version 1, `MINOR` version 2, -and `PATCH` version 3. Changes to each number have the following meaning: - -* **MAJOR**: Potentially backwards incompatible changes. Code and data that - worked with a previous major release will not necessarily work with the new - release. However, in some cases existing TensorFlow graphs and checkpoints - may be migratable to the newer release; see - [Compatibility of graphs and checkpoints](#compatibility_of_graphs_and_checkpoints) - for details on data compatibility. - -* **MINOR**: Backwards compatible features, speed improvements, etc. Code and - data that worked with a previous minor release *and* which depends only on the - public API will continue to work unchanged. For details on what is and is - not the public API, see [What is covered](#what_is_covered). - -* **PATCH**: Backwards compatible bug fixes. - -For example, release 1.0.0 introduced backwards *incompatible* changes from -release 0.12.1. However, release 1.1.1 was backwards *compatible* with release -1.0.0. - -## What is covered - -Only the public APIs of TensorFlow are backwards compatible across minor and -patch versions. The public APIs consist of - -* All the documented [Python](../api_docs/python) functions and classes in the - `tensorflow` module and its submodules, except for - * functions and classes in `tf.contrib` - * functions and classes whose names start with `_` (as these are private) - Note that the code in the `examples/` and `tools/` directories is not - reachable through the `tensorflow` Python module and is thus not covered by - the compatibility guarantee. - - If a symbol is available through the `tensorflow` Python module or its - submodules, but is not documented, then it is **not** considered part of the - public API. - -* The [C API](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/c/c_api.h). - -* The following protocol buffer files: - * [`attr_value`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto) - * [`config`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto) - * [`event`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/util/event.proto) - * [`graph`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/graph.proto) - * [`op_def`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def.proto) - * [`reader_base`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/reader_base.proto) - * [`summary`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto) - * [`tensor`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto) - * [`tensor_shape`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.proto) - * [`types`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.proto) - - -## What is *not* covered - -Some API functions are explicitly marked as "experimental" and can change in -backward incompatible ways between minor releases. These include: - -* **Experimental APIs**: The @{tf.contrib} module and its submodules in Python - and any functions in the C API or fields in protocol buffers that are - explicitly commented as being experimental. In particular, any field in a - protocol buffer which is called "experimental" and all its fields and - submessages can change at any time. - -* **Other languages**: TensorFlow APIs in languages other than Python and C, - such as: - - - @{$cc/guide$C++} (exposed through header files in - [`tensorflow/cc`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/cc)). - - [Java](../api_docs/java/reference/org/tensorflow/package-summary), - - [Go](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go) - -* **Details of composite ops:** Many public functions in Python expand to - several primitive ops in the graph, and these details will be part of any - graphs saved to disk as `GraphDef`s. These details may change for - minor releases. In particular, regressions tests that check for exact - matching between graphs are likely to break across minor releases, even - though the behavior of the graph should be unchanged and existing - checkpoints will still work. - -* **Floating point numerical details:** The specific floating point values - computed by ops may change at any time. Users should rely only on - approximate accuracy and numerical stability, not on the specific bits - computed. Changes to numerical formulas in minor and patch releases should - result in comparable or improved accuracy, with the caveat that in machine - learning improved accuracy of specific formulas may result in decreased - accuracy for the overall system. - -* **Random numbers:** The specific random numbers computed by the - @{$python/constant_op#Random_Tensors$random ops} may change at any time. - Users should rely only on approximately correct distributions and - statistical strength, not the specific bits computed. However, we will make - changes to random bits rarely (or perhaps never) for patch releases. We - will, of course, document all such changes. - -* **Version skew in distributed Tensorflow:** Running two different versions - of TensorFlow in a single cluster is unsupported. There are no guarantees - about backwards compatibility of the wire protocol. - -* **Bugs:** We reserve the right to make backwards incompatible behavior - (though not API) changes if the current implementation is clearly broken, - that is, if it contradicts the documentation or if a well-known and - well-defined intended behavior is not properly implemented due to a bug. - For example, if an optimizer claims to implement a well-known optimization - algorithm but does not match that algorithm due to a bug, then we will fix - the optimizer. Our fix may break code relying on the wrong behavior for - convergence. We will note such changes in the release notes. - -* **Error messages:** We reserve the right to change the text of error - messages. In addition, the type of an error may change unless the type is - specified in the documentation. For example, a function documented to - raise an `InvalidArgument` exception will continue to - raise `InvalidArgument`, but the human-readable message contents can change. - -## Compatibility of graphs and checkpoints - -You'll sometimes need to preserve graphs and checkpoints. -Graphs describe the data flow of ops to be run during training and -inference, and checkpoints contain the saved tensor values of variables in a -graph. - -Many TensorFlow users save graphs and trained models to disk for -later evaluation or additional training, but end up running their saved graphs -or models on a later release. In compliance with semver, any graph or checkpoint -written out with one version of TensorFlow can be loaded and evaluated with a -later version of TensorFlow with the same major release. However, we will -endeavor to preserve backwards compatibility even across major releases when -possible, so that the serialized files are usable over long periods of time. - - -Graphs are serialized via the `GraphDef` protocol buffer. To facilitate (rare) -backwards incompatible changes to graphs, each `GraphDef` has a version number -separate from the TensorFlow version. For example, `GraphDef` version 17 -deprecated the `inv` op in favor of `reciprocal`. The semantics are: - -* Each version of TensorFlow supports an interval of `GraphDef` versions. This - interval will be constant across patch releases, and will only grow across - minor releases. Dropping support for a `GraphDef` version will only occur - for a major release of TensorFlow. - -* Newly created graphs are assigned the latest `GraphDef` version number. - -* If a given version of TensorFlow supports the `GraphDef` version of a graph, - it will load and evaluate with the same behavior as the TensorFlow version - used to generate it (except for floating point numerical details and random - numbers), regardless of the major version of TensorFlow. In particular, all - checkpoint files will be compatible. - -* If the `GraphDef` *upper* bound is increased to X in a (minor) release, there - will be at least six months before the *lower* bound is increased to X. For - example (we're using hypothetical version numbers here): - * TensorFlow 1.2 might support `GraphDef` versions 4 to 7. - * TensorFlow 1.3 could add `GraphDef` version 8 and support versions 4 to 8. - * At least six months later, TensorFlow 2.0.0 could drop support for - versions 4 to 7, leaving version 8 only. - -Finally, when support for a `GraphDef` version is dropped, we will attempt to -provide tools for automatically converting graphs to a newer supported -`GraphDef` version. - -## Graph and checkpoint compatibility when extending TensorFlow - -This section is relevant only when making incompatible changes to the `GraphDef` -format, such as when adding ops, removing ops, or changing the functionality -of existing ops. The previous section should suffice for most users. - -### Backward and partial forward compatibility - -Our versioning scheme has three requirements: - -* **Backward compatibility** to support loading graphs and checkpoints - created with older versions of TensorFlow. -* **Forward compatibility** to support scenarios where the producer of a - graph or checkpoint is upgraded to a newer version of TensorFlow before - the consumer. -* Enable evolving TensorFlow in incompatible ways. For example, removing ops, - adding attributes, and removing attributes. - -Note that while the `GraphDef` version mechanism is separate from the TensorFlow -version, backwards incompatible changes to the `GraphDef` format are still -restricted by Semantic Versioning. This means functionality can only be removed -or changed between `MAJOR` versions of TensorFlow (such as `1.7` to `2.0`). -Additionally, forward compatibility is enforced within Patch releases (`1.x.1` -to `1.x.2` for example). - -To achieve backward and forward compatibility and to know when to enforce changes -in formats, graphs and checkpoints have metadata that describes when they -were produced. The sections below detail the TensorFlow implementation and -guidelines for evolving `GraphDef` versions. - -### Independent data version schemes - -There are different data versions for graphs and checkpoints. The two data -formats evolve at different rates from each other and also at different rates -from TensorFlow. Both versioning systems are defined in -[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h). -Whenever a new version is added, a note is added to the header detailing what -changed and the date. - -### Data, producers, and consumers - -We distinguish between the following kinds of data version information: -* **producers**: binaries that produce data. Producers have a version - (`producer`) and a minimum consumer version that they are compatible with - (`min_consumer`). -* **consumers**: binaries that consume data. Consumers have a version - (`consumer`) and a minimum producer version that they are compatible with - (`min_producer`). - -Each piece of versioned data has a [`VersionDef -versions`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/versions.proto) -field which records the `producer` that made the data, the `min_consumer` -that it is compatible with, and a list of `bad_consumers` versions that are -disallowed. - -By default, when a producer makes some data, the data inherits the producer's -`producer` and `min_consumer` versions. `bad_consumers` can be set if specific -consumer versions are known to contain bugs and must be avoided. A consumer can -accept a piece of data if the following are all true: - -* `consumer` >= data's `min_consumer` -* data's `producer` >= consumer's `min_producer` -* `consumer` not in data's `bad_consumers` - -Since both producers and consumers come from the same TensorFlow code base, -[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h) -contains a main data version which is treated as either `producer` or -`consumer` depending on context and both `min_consumer` and `min_producer` -(needed by producers and consumers, respectively). Specifically, - -* For `GraphDef` versions, we have `TF_GRAPH_DEF_VERSION`, - `TF_GRAPH_DEF_VERSION_MIN_CONSUMER`, and - `TF_GRAPH_DEF_VERSION_MIN_PRODUCER`. -* For checkpoint versions, we have `TF_CHECKPOINT_VERSION`, - `TF_CHECKPOINT_VERSION_MIN_CONSUMER`, and - `TF_CHECKPOINT_VERSION_MIN_PRODUCER`. - -### Add a new attribute with default to an existing op - -Following the guidance below gives you forward compatibility only if the set of -ops has not changed: - -1. If forward compatibility is desired, set `strip_default_attrs` to `True` - while exporting the model using either the - @{tf.saved_model.builder.SavedModelBuilder.add_meta_graph_and_variables$`add_meta_graph_and_variables`} - and @{tf.saved_model.builder.SavedModelBuilder.add_meta_graph$`add_meta_graph`} - methods of the `SavedModelBuilder` class, or - @{tf.estimator.Estimator.export_savedmodel$`Estimator.export_savedmodel`} -2. This strips off the default valued attributes at the time of - producing/exporting the models. This makes sure that the exported - @{tf.MetaGraphDef} does not contain the new op-attribute when the default - value is used. -3. Having this control could allow out-of-date consumers (for example, serving - binaries that lag behind training binaries) to continue loading the models - and prevent interruptions in model serving. - -### Evolving GraphDef versions - -This section explains how to use this versioning mechanism to make different -types of changes to the `GraphDef` format. - -#### Add an op - -Add the new op to both consumers and producers at the same time, and do not -change any `GraphDef` versions. This type of change is automatically -backward compatible, and does not impact forward compatibility plan since -existing producer scripts will not suddenly use the new functionality. - -#### Add an op and switch existing Python wrappers to use it - -1. Implement new consumer functionality and increment the `GraphDef` version. -2. If it is possible to make the wrappers use the new functionality only in - cases that did not work before, the wrappers can be updated now. -3. Change Python wrappers to use the new functionality. Do not increment - `min_consumer`, since models that do not use this op should not break. - -#### Remove or restrict an op's functionality - -1. Fix all producer scripts (not TensorFlow itself) to not use the banned op or - functionality. -2. Increment the `GraphDef` version and implement new consumer functionality - that bans the removed op or functionality for GraphDefs at the new version - and above. If possible, make TensorFlow stop producing `GraphDefs` with the - banned functionality. To do so, add the - [`REGISTER_OP(...).Deprecated(deprecated_at_version, - message)`](https://github.com/tensorflow/tensorflow/blob/b289bc7a50fc0254970c60aaeba01c33de61a728/tensorflow/core/ops/array_ops.cc#L1009). -3. Wait for a major release for backward compatibility purposes. -4. Increase `min_producer` to the GraphDef version from (2) and remove the - functionality entirely. - -#### Change an op's functionality - -1. Add a new similar op named `SomethingV2` or similar and go through the - process of adding it and switching existing Python wrappers to use it, which - may take three weeks if forward compatibility is desired. -2. Remove the old op (Can only take place with a major version change due to - backward compatibility). -3. Increase `min_consumer` to rule out consumers with the old op, add back the - old op as an alias for `SomethingV2`, and go through the process to switch - existing Python wrappers to use it. -4. Go through the process to remove `SomethingV2`. - -#### Ban a single unsafe consumer version - -1. Bump the `GraphDef` version and add the bad version to `bad_consumers` for - all new GraphDefs. If possible, add to `bad_consumers` only for GraphDefs - which contain a certain op or similar. -2. If existing consumers have the bad version, push them out as soon as - possible. diff --git a/tensorflow/docs_src/tutorials/deep_cnn.md b/tensorflow/docs_src/tutorials/deep_cnn.md index 6a4c9a9b07..44a32d9d1d 100644 --- a/tensorflow/docs_src/tutorials/deep_cnn.md +++ b/tensorflow/docs_src/tutorials/deep_cnn.md @@ -268,7 +268,7 @@ in `cifar10_input.py`. `cifar10_train.py` periodically @{tf.train.Saver$saves} all model parameters in -@{$programmers_guide/saved_model$checkpoint files} +@{$guide/saved_model$checkpoint files} but it does *not* evaluate the model. The checkpoint file will be used by `cifar10_eval.py` to measure the predictive performance (see [Evaluating a Model](#evaluating-a-model) below). diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md index 0f17899dae..212e337637 100644 --- a/tensorflow/docs_src/tutorials/layers.md +++ b/tensorflow/docs_src/tutorials/layers.md @@ -627,7 +627,7 @@ operation earlier when we generated the probabilities in `cnn_model_fn`. > argument, TensorFlow will assign a default name. A couple easy ways to > discover the names applied to operations are to visualize your graph on > @{$graph_viz$TensorBoard}) or to enable the -> @{$programmers_guide/debugger$TensorFlow Debugger (tfdbg)}. +> @{$guide/debugger$TensorFlow Debugger (tfdbg)}. Next, we create the `LoggingTensorHook`, passing `tensors_to_log` to the `tensors` argument. We set `every_n_iter=50`, which specifies that probabilities diff --git a/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py b/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py index 307eede5c0..7402247448 100644 --- a/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py +++ b/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py @@ -17,7 +17,7 @@ This version is like fully_connected_feed.py but uses data converted to a TFRecords file containing tf.train.Example protocol buffers. See: -https://www.tensorflow.org/programmers_guide/reading_data#reading_from_files +https://www.tensorflow.org/guide/reading_data#reading_from_files for context. YOU MUST run convert_to_records before running this (but you only need to diff --git a/tensorflow/java/README.md b/tensorflow/java/README.md index 2f1ce253b2..c7382ff231 100644 --- a/tensorflow/java/README.md +++ b/tensorflow/java/README.md @@ -1,7 +1,7 @@ # TensorFlow for Java > *WARNING*: The TensorFlow Java API is not currently covered by the TensorFlow -> [API stability guarantees](https://www.tensorflow.org/programmers_guide/version_semantics). +> [API stability guarantees](https://www.tensorflow.org/guide/version_semantics). > > For using TensorFlow on Android refer instead to > [contrib/android](https://www.tensorflow.org/code/tensorflow/contrib/android), @@ -23,8 +23,7 @@ native libraries will need to be built from source. 2. Setup the environment to build TensorFlow from source code ([Linux](https://www.tensorflow.org/install/install_sources#PrepareLinux) - or [Mac OS - X](https://www.tensorflow.org/install/install_sources#PrepareMac)). + or [macOS](https://www.tensorflow.org/install/install_sources#PrepareMac)). If you'd like to skip reading those details and do not care about GPU support, try the following: diff --git a/tensorflow/java/src/main/java/org/tensorflow/package-info.java b/tensorflow/java/src/main/java/org/tensorflow/package-info.java index 521c5c610c..f353ee3145 100644 --- a/tensorflow/java/src/main/java/org/tensorflow/package-info.java +++ b/tensorflow/java/src/main/java/org/tensorflow/package-info.java @@ -17,7 +17,7 @@ limitations under the License. * Defines classes to build, save, load and execute TensorFlow models. * *

WARNING: The API is currently experimental and is not covered by TensorFlow API stability + * href="https://www.tensorflow.org/guide/version_semantics">API stability * guarantees. See README.md for installation * instructions. diff --git a/tensorflow/python/data/__init__.py b/tensorflow/python/data/__init__.py index 7efe0948e7..3b9bf2469e 100644 --- a/tensorflow/python/data/__init__.py +++ b/tensorflow/python/data/__init__.py @@ -14,7 +14,7 @@ # ============================================================================== """`tf.data.Dataset` API for input pipelines. -See the @{$datasets$Importing Data} Programmer's Guide for an overview. +See @{$guide/datasets$Importing Data} for an overview. """ from __future__ import absolute_import diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index 6f9b12b123..0e020d86d0 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -212,6 +212,13 @@ class Dataset(object): def from_tensors(tensors): """Creates a `Dataset` with a single element, comprising the given tensors. + Note that if `tensors` contains a NumPy array, and eager execution is not + enabled, the values will be embedded in the graph as one or more + @{tf.constant} operations. For large datasets (> 1 GB), this can waste + memory and run into byte limits of graph serialization. If tensors contains + one or more large NumPy arrays, consider the alternative described in + @{$guide/datasets#consuming_numpy_arrays$this guide}. + Args: tensors: A nested structure of tensors. @@ -224,6 +231,13 @@ class Dataset(object): def from_tensor_slices(tensors): """Creates a `Dataset` whose elements are slices of the given tensors. + Note that if `tensors` contains a NumPy array, and eager execution is not + enabled, the values will be embedded in the graph as one or more + @{tf.constant} operations. For large datasets (> 1 GB), this can waste + memory and run into byte limits of graph serialization. If tensors contains + one or more large NumPy arrays, consider the alternative described in + @{$guide/datasets#consuming_numpy_arrays$this guide}. + Args: tensors: A nested structure of tensors, each having the same size in the 0th dimension. diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD index 09062abd74..2d261f9be7 100644 --- a/tensorflow/python/debug/BUILD +++ b/tensorflow/python/debug/BUILD @@ -5,7 +5,7 @@ # # ":debug_py": Public Python methods and classes of tfdbg. # For API documentation, see https://www.tensorflow.org/api_docs/python/tfdbg -# For a user interface walkthrough, see https://www.tensorflow.org/programmers_guide/debugger +# For a user interface walkthrough, see https://www.tensorflow.org/guide/debugger # ":grpc_debug_server": Server interface for grpc:// debug URLs. package( diff --git a/tensorflow/python/debug/README.md b/tensorflow/python/debug/README.md index 269bbb19bd..9c16af4d79 100644 --- a/tensorflow/python/debug/README.md +++ b/tensorflow/python/debug/README.md @@ -28,7 +28,7 @@ models: * Easy access through session wrappers * Easy integration with common high-level APIs, such as - [TensorFlow Estimators](https://www.tensorflow.org/programmers_guide/estimators) and + [TensorFlow Estimators](https://www.tensorflow.org/guide/estimators) and [Keras](https://keras.io/) * Inspection of runtime tensor values and node connections * Conditional breaking after runs that generate tensors satisfying given @@ -43,7 +43,7 @@ models: ## How to use TFDBG? -* For a walkthrough of TFDBG command-line interface, see https://www.tensorflow.org/programmers_guide/debugger. +* For a walkthrough of TFDBG command-line interface, see https://www.tensorflow.org/guide/debugger. * For information on the web GUI of TFDBG (TensorBoard Debugger Plugin), see [this README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md). * For programmatic use of the API of TFDBG, see https://www.tensorflow.org/api_docs/python/tfdbg. diff --git a/tensorflow/python/debug/examples/README.md b/tensorflow/python/debug/examples/README.md index cb4d484092..3b431e04dc 100644 --- a/tensorflow/python/debug/examples/README.md +++ b/tensorflow/python/debug/examples/README.md @@ -3,7 +3,7 @@ Hi, there! The documentation of **TensorFlow Debugger (tfdbg)** has moved. See the source version at -[this new location](../../../docs_src/programmers_guide/debugger.md). +[this new location](../../../docs_src/guide/debugger.md). See the public website version at -[https://www.tensorflow.org/programmers_guide/debugger](https://www.tensorflow.org/programmers_guide/debugger). +[https://www.tensorflow.org/guide/debugger](https://www.tensorflow.org/guide/debugger). diff --git a/tensorflow/python/estimator/keras.py b/tensorflow/python/estimator/keras.py index 2f439f765e..6856b8b5a9 100644 --- a/tensorflow/python/estimator/keras.py +++ b/tensorflow/python/estimator/keras.py @@ -455,7 +455,7 @@ def model_to_estimator(keras_model=None, """Constructs an `Estimator` instance from given keras model. For usage example, please see - @{$programmers_guide/estimators$creating_estimators_from_keras_models}. + @{$guide/estimators$creating_estimators_from_keras_models}. Args: keras_model: A compiled Keras model object. This argument is mutually diff --git a/tensorflow/python/ops/script_ops.py b/tensorflow/python/ops/script_ops.py index 16c73213d5..f8df9b2c78 100644 --- a/tensorflow/python/ops/script_ops.py +++ b/tensorflow/python/ops/script_ops.py @@ -267,7 +267,7 @@ def eager_py_func(func, inp, Tout, name=None): or print statements as desired, and wrap those functions in `tf.contrib.eager.py_func`. - For more information on eager execution, see @{$programmers_guide/eager}. + For more information on eager execution, see @{$guide/eager}. `tf.contrib.eager.py_func` is similar in spirit to @{tf.py_func}, but unlike the latter, the former lets you use TensorFlow operations in the wrapped diff --git a/tensorflow/python/tools/saved_model_cli.py b/tensorflow/python/tools/saved_model_cli.py index 5b9d25d449..38fed5335e 100644 --- a/tensorflow/python/tools/saved_model_cli.py +++ b/tensorflow/python/tools/saved_model_cli.py @@ -15,7 +15,7 @@ """Command-line interface to inspect and execute a graph in a SavedModel. For detailed usages and examples, please refer to: -https://www.tensorflow.org/programmers_guide/saved_model_cli +https://www.tensorflow.org/guide/saved_model_cli """ @@ -720,7 +720,7 @@ def create_parser(): '\'input4_key=[{"id":[26],"weights":[0.5, 0.5]}]\' \\\n' ' --outdir=/out\n\n' 'For more information about input file format, please see:\n' - 'https://www.tensorflow.org/programmers_guide/saved_model_cli\n') + 'https://www.tensorflow.org/guide/saved_model_cli\n') parser_run = subparsers.add_parser( 'run', description=run_msg, formatter_class=argparse.RawTextHelpFormatter) parser_run.add_argument( diff --git a/third_party/examples/eager/spinn/README.md b/third_party/examples/eager/spinn/README.md index fbb1fde837..e2fd8009a0 100644 --- a/third_party/examples/eager/spinn/README.md +++ b/third_party/examples/eager/spinn/README.md @@ -22,7 +22,7 @@ Other eager execution examples can be found under [tensorflow/contrib/eager/pyth - [`data.py`](../../../../tensorflow/contrib/eager/python/examples/spinn/data.py): Pipeline for loading and preprocessing the [SNLI](https://nlp.stanford.edu/projects/snli/) data and [GloVe](https://nlp.stanford.edu/projects/glove/) word embedding, written - using the [`tf.data`](https://www.tensorflow.org/programmers_guide/datasets) + using the [`tf.data`](https://www.tensorflow.org/guide/datasets) API. - [`spinn.py`](./spinn.py): Model definition and training routines. This example illustrates how one might perform the following actions with -- cgit v1.2.3 From ab60fbc1fcfc600b800ad12c9f76cfccc4fb7087 Mon Sep 17 00:00:00 2001 From: Pete Warden Date: Tue, 26 Jun 2018 09:32:12 -0700 Subject: Fix for RPi OpenBLAS compile issues, by pinning to known good version --- tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh | 8 ++++++++ tensorflow/tools/ci_build/pi/build_raspberry_pi.sh | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh b/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh index 9d8e3df3b5..4afb2f1534 100755 --- a/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh +++ b/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh @@ -27,3 +27,11 @@ curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - apt-get update rm -rf /usr/local/bin/bazel apt-get install -y bazel python3 python3-numpy python3-dev python3-pip + +# We're using Ubuntu 14.04 as our base image because that's needed by the Pi +# cross-compilation chain, but that doesn't have built-in Python 3.5 support, so +# install from a separate repository. +apt-get install -y software-properties-common +add-apt-repository ppa:fkrull/deadsnakes +apt-get update +apt-get install -y python3.5 python3.5-dev diff --git a/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh b/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh index 4d1a30601e..5eff3e415d 100755 --- a/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh +++ b/tensorflow/tools/ci_build/pi/build_raspberry_pi.sh @@ -65,6 +65,10 @@ OPENBLAS_SRC_PATH=/tmp/openblas_src/ sudo rm -rf ${OPENBLAS_SRC_PATH} git clone https://github.com/xianyi/OpenBLAS ${OPENBLAS_SRC_PATH} cd ${OPENBLAS_SRC_PATH} +# The commit after this introduced Fortran compile issues. In theory they should +# be solvable using NOFORTRAN=1 on the make command, but my initial tries didn't +# work, so pinning to the last know good version. +git checkout 5a6a2bed9aff0ba8a18651d5514d029c8cae336a # If this path is changed, you'll also need to update # cxx_builtin_include_directory in third_party/toolchains/cpus/arm/CROSSTOOL.tpl OPENBLAS_INSTALL_PATH=/tmp/openblas_install/ -- cgit v1.2.3 From fe374d31f38ba7fa84284b58d28c55dc0087f2b3 Mon Sep 17 00:00:00 2001 From: Pete Warden Date: Tue, 26 Jun 2018 09:36:22 -0700 Subject: Removed Python 3.5 updates for RPi --- tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh b/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh index 4afb2f1534..9d8e3df3b5 100755 --- a/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh +++ b/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh @@ -27,11 +27,3 @@ curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - apt-get update rm -rf /usr/local/bin/bazel apt-get install -y bazel python3 python3-numpy python3-dev python3-pip - -# We're using Ubuntu 14.04 as our base image because that's needed by the Pi -# cross-compilation chain, but that doesn't have built-in Python 3.5 support, so -# install from a separate repository. -apt-get install -y software-properties-common -add-apt-repository ppa:fkrull/deadsnakes -apt-get update -apt-get install -y python3.5 python3.5-dev -- cgit v1.2.3 From 8025ac34099ed1b38c3cf0c0f84244496b42fedb Mon Sep 17 00:00:00 2001 From: Michael Case Date: Tue, 26 Jun 2018 13:05:25 -0700 Subject: Moving StatusOr from XLA to stream_executor. PiperOrigin-RevId: 202179928 --- tensorflow/compiler/xla/BUILD | 17 +- tensorflow/compiler/xla/service/gpu/BUILD | 1 + .../xla/service/gpu/stream_executor_util.h | 1 + tensorflow/compiler/xla/statusor.cc | 38 -- tensorflow/compiler/xla/statusor.h | 286 +-------- tensorflow/compiler/xla/statusor_internals.h | 245 -------- tensorflow/compiler/xla/statusor_test.cc | 675 -------------------- tensorflow/stream_executor/BUILD | 2 - tensorflow/stream_executor/lib/statusor.cc | 40 ++ tensorflow/stream_executor/lib/statusor.h | 290 ++++++++- .../stream_executor/lib/statusor_internals.h | 248 ++++++++ tensorflow/stream_executor/lib/statusor_test.cc | 676 +++++++++++++++++++++ 12 files changed, 1254 insertions(+), 1265 deletions(-) delete mode 100644 tensorflow/compiler/xla/statusor.cc delete mode 100644 tensorflow/compiler/xla/statusor_internals.h delete mode 100644 tensorflow/compiler/xla/statusor_test.cc create mode 100644 tensorflow/stream_executor/lib/statusor.cc create mode 100644 tensorflow/stream_executor/lib/statusor_internals.h create mode 100644 tensorflow/stream_executor/lib/statusor_test.cc diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index c6deb959a5..afa8ce730b 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -143,30 +143,15 @@ cc_library( cc_library( name = "statusor", - srcs = ["statusor.cc"], hdrs = [ "statusor.h", - "statusor_internals.h", ], visibility = ["//visibility:public"], deps = [ ":status", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", - ], -) - -tf_cc_test( - name = "statusor_test", - size = "small", - srcs = ["statusor_test.cc"], - deps = [ - ":statusor", - ":test", - ":types", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", + "//tensorflow/stream_executor", ], ) diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 68297ad4ae..fe597bfb45 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -727,6 +727,7 @@ cc_library( hdrs = ["stream_executor_util.h"], deps = [ "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/core:stream_executor_no_cuda", ], diff --git a/tensorflow/compiler/xla/service/gpu/stream_executor_util.h b/tensorflow/compiler/xla/service/gpu/stream_executor_util.h index 8218f4fd11..39a6a38d00 100644 --- a/tensorflow/compiler/xla/service/gpu/stream_executor_util.h +++ b/tensorflow/compiler/xla/service/gpu/stream_executor_util.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_STREAM_EXECUTOR_UTIL_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_STREAM_EXECUTOR_UTIL_H_ +#include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/platform/stream_executor_no_cuda.h" diff --git a/tensorflow/compiler/xla/statusor.cc b/tensorflow/compiler/xla/statusor.cc deleted file mode 100644 index 72ab67ff81..0000000000 --- a/tensorflow/compiler/xla/statusor.cc +++ /dev/null @@ -1,38 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/xla/statusor.h" - -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/platform/logging.h" - -namespace xla { -namespace internal_statusor { - -void Helper::HandleInvalidStatusCtorArg(Status* status) { - const char* kMessage = - "An OK status is not a valid constructor argument to StatusOr"; - LOG(ERROR) << kMessage; - // Fall back to tensorflow::error::INTERNAL. - *status = ::tensorflow::errors::Internal(kMessage); -} - -void Helper::Crash(const Status& status) { - LOG(FATAL) << "Attempting to fetch value instead of handling error " - << status; -} - -} // namespace internal_statusor -} // namespace xla diff --git a/tensorflow/compiler/xla/statusor.h b/tensorflow/compiler/xla/statusor.h index 0e1387c939..a32e2ad985 100644 --- a/tensorflow/compiler/xla/statusor.h +++ b/tensorflow/compiler/xla/statusor.h @@ -12,297 +12,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ - -// StatusOr is the union of a Status object and a T object. StatusOr models -// the concept of an object that is either a value, or an error Status -// explaining why such a value is not present. To this end, StatusOr does not -// allow its Status value to be Status::OK. -// -// The primary use-case for StatusOr is as the return value of a -// function which may fail. -// -// Example client usage for a StatusOr, where T is not a pointer: -// -// StatusOr result = DoBigCalculationThatCouldFail(); -// if (result.ok()) { -// float answer = result.ValueOrDie(); -// printf("Big calculation yielded: %f", answer); -// } else { -// LOG(ERROR) << result.status(); -// } -// -// Example client usage for a StatusOr: -// -// StatusOr result = FooFactory::MakeNewFoo(arg); -// if (result.ok()) { -// std::unique_ptr foo(result.ValueOrDie()); -// foo->DoSomethingCool(); -// } else { -// LOG(ERROR) << result.status(); -// } -// -// Example client usage for a StatusOr>: -// -// StatusOr> result = FooFactory::MakeNewFoo(arg); -// if (result.ok()) { -// std::unique_ptr foo = std::move(result.ValueOrDie()); -// foo->DoSomethingCool(); -// } else { -// LOG(ERROR) << result.status(); -// } -// -// Example factory implementation returning StatusOr: -// -// StatusOr FooFactory::MakeNewFoo(int arg) { -// if (arg <= 0) { -// return tensorflow::InvalidArgument("Arg must be positive"); -// } else { -// return new Foo(arg); -// } -// } -// -// Note that the assignment operators require that destroying the currently -// stored value cannot invalidate the argument; in other words, the argument -// cannot be an alias for the current value, or anything owned by the current -// value. #ifndef TENSORFLOW_COMPILER_XLA_STATUSOR_H_ #define TENSORFLOW_COMPILER_XLA_STATUSOR_H_ #include "tensorflow/compiler/xla/status.h" -#include "tensorflow/compiler/xla/statusor_internals.h" -#include "tensorflow/core/platform/macros.h" +#include "tensorflow/stream_executor/lib/statusor.h" namespace xla { -#if defined(__clang__) -// Only clang supports warn_unused_result as a type annotation. -template -class TF_MUST_USE_RESULT StatusOr; -#endif - -template -class StatusOr : private internal_statusor::StatusOrData, - private internal_statusor::TraitsBase< - std::is_copy_constructible::value, - std::is_move_constructible::value> { - template - friend class StatusOr; - - typedef internal_statusor::StatusOrData Base; - - public: - typedef T element_type; - - // Constructs a new StatusOr with Status::UNKNOWN status. This is marked - // 'explicit' to try to catch cases like 'return {};', where people think - // StatusOr> will be initialized with an empty vector, - // instead of a Status::UNKNOWN status. - explicit StatusOr(); - - // StatusOr will be copy constructible/assignable if T is copy - // constructible. - StatusOr(const StatusOr&) = default; - StatusOr& operator=(const StatusOr&) = default; - - // StatusOr will be move constructible/assignable if T is move - // constructible. - StatusOr(StatusOr&&) = default; - StatusOr& operator=(StatusOr&&) = default; - - // Conversion copy/move constructor, T must be convertible from U. - template ::value>::type* = nullptr> - StatusOr(const StatusOr& other); - template ::value>::type* = nullptr> - StatusOr(StatusOr&& other); - - // Conversion copy/move assignment operator, T must be convertible from U. - template ::value>::type* = nullptr> - StatusOr& operator=(const StatusOr& other); - template ::value>::type* = nullptr> - StatusOr& operator=(StatusOr&& other); - - // Constructs a new StatusOr with the given value. After calling this - // constructor, calls to ValueOrDie() will succeed, and calls to status() will - // return OK. - // - // NOTE: Not explicit - we want to use StatusOr as a return type - // so it is convenient and sensible to be able to do 'return T()' - // when the return type is StatusOr. - // - // REQUIRES: T is copy constructible. - StatusOr(const T& value); - - // Constructs a new StatusOr with the given non-ok status. After calling - // this constructor, calls to ValueOrDie() will CHECK-fail. - // - // NOTE: Not explicit - we want to use StatusOr as a return - // value, so it is convenient and sensible to be able to do 'return - // Status()' when the return type is StatusOr. - // - // REQUIRES: !status.ok(). This requirement is DCHECKed. - // In optimized builds, passing Status::OK() here will have the effect - // of passing tensorflow::error::INTERNAL as a fallback. - StatusOr(const Status& status); - StatusOr& operator=(const Status& status); - - // TODO(b/62186997): Add operator=(T) overloads. - - // Similar to the `const T&` overload. - // - // REQUIRES: T is move constructible. - StatusOr(T&& value); - - // RValue versions of the operations declared above. - StatusOr(Status&& status); - StatusOr& operator=(Status&& status); - - // Returns this->status().ok() - bool ok() const { return this->status_.ok(); } - - // Returns a reference to our status. If this contains a T, then - // returns Status::OK(). - const Status& status() const &; - Status status() &&; - - // Returns a reference to our current value, or CHECK-fails if !this->ok(). - // - // Note: for value types that are cheap to copy, prefer simple code: - // - // T value = statusor.ValueOrDie(); - // - // Otherwise, if the value type is expensive to copy, but can be left - // in the StatusOr, simply assign to a reference: - // - // T& value = statusor.ValueOrDie(); // or `const T&` - // - // Otherwise, if the value type supports an efficient move, it can be - // used as follows: - // - // T value = std::move(statusor).ValueOrDie(); - // - // The std::move on statusor instead of on the whole expression enables - // warnings about possible uses of the statusor object after the move. - // C++ style guide waiver for ref-qualified overloads granted in cl/143176389 - // See go/ref-qualifiers for more details on such overloads. - const T& ValueOrDie() const &; - T& ValueOrDie() &; - const T&& ValueOrDie() const &&; - T&& ValueOrDie() &&; - - T ConsumeValueOrDie() { return std::move(ValueOrDie()); } - - // Ignores any errors. This method does nothing except potentially suppress - // complaints from any tools that are checking that errors are not dropped on - // the floor. - void IgnoreError() const; -}; - -//////////////////////////////////////////////////////////////////////////////// -// Implementation details for StatusOr - -template -StatusOr::StatusOr() : Base(Status(tensorflow::error::UNKNOWN, "")) {} - -template -StatusOr::StatusOr(const T& value) : Base(value) {} - -template -StatusOr::StatusOr(const Status& status) : Base(status) {} - -template -StatusOr& StatusOr::operator=(const Status& status) { - this->Assign(status); - return *this; -} - -template -StatusOr::StatusOr(T&& value) : Base(std::move(value)) {} - -template -StatusOr::StatusOr(Status&& status) : Base(std::move(status)) {} - -template -StatusOr& StatusOr::operator=(Status&& status) { - this->Assign(std::move(status)); - return *this; -} - -template -template ::value>::type*> -inline StatusOr::StatusOr(const StatusOr& other) - : Base(static_cast::Base&>(other)) {} - -template -template ::value>::type*> -inline StatusOr& StatusOr::operator=(const StatusOr& other) { - if (other.ok()) - this->Assign(other.ValueOrDie()); - else - this->Assign(other.status()); - return *this; -} - -template -template ::value>::type*> -inline StatusOr::StatusOr(StatusOr&& other) - : Base(static_cast::Base&&>(other)) {} - -template -template ::value>::type*> -inline StatusOr& StatusOr::operator=(StatusOr&& other) { - if (other.ok()) { - this->Assign(std::move(other).ValueOrDie()); - } else { - this->Assign(std::move(other).status()); - } - return *this; -} - -template -const Status& StatusOr::status() const & { - return this->status_; -} -template -Status StatusOr::status() && { - return ok() ? Status::OK() : std::move(this->status_); -} - -template -const T& StatusOr::ValueOrDie() const & { - this->EnsureOk(); - return this->data_; -} - -template -T& StatusOr::ValueOrDie() & { - this->EnsureOk(); - return this->data_; -} - -template -const T&& StatusOr::ValueOrDie() const && { - this->EnsureOk(); - return std::move(this->data_); -} - -template -T&& StatusOr::ValueOrDie() && { - this->EnsureOk(); - return std::move(this->data_); -} - +// Use steam_executor's StatusOr so we don't duplicate code. template -void StatusOr::IgnoreError() const { - // no-op -} +using StatusOr = ::stream_executor::port::StatusOr; } // namespace xla diff --git a/tensorflow/compiler/xla/statusor_internals.h b/tensorflow/compiler/xla/statusor_internals.h deleted file mode 100644 index 14636bd144..0000000000 --- a/tensorflow/compiler/xla/statusor_internals.h +++ /dev/null @@ -1,245 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_XLA_STATUSOR_INTERNALS_H_ -#define TENSORFLOW_COMPILER_XLA_STATUSOR_INTERNALS_H_ - -#include "tensorflow/compiler/xla/status.h" -#include "tensorflow/core/platform/macros.h" - -namespace xla { -namespace internal_statusor { - -class Helper { - public: - // Move type-agnostic error handling to the .cc. - static void HandleInvalidStatusCtorArg(Status*); - TF_ATTRIBUTE_NORETURN static void Crash(const Status& status); -}; - -// Construct an instance of T in `p` through placement new, passing Args... to -// the constructor. -// This abstraction is here mostly for the gcc performance fix. -template -void PlacementNew(void* p, Args&&... args) { -#if defined(__GNUC__) && !defined(__clang__) - // Teach gcc that 'p' cannot be null, fixing code size issues. - if (p == nullptr) __builtin_unreachable(); -#endif - new (p) T(std::forward(args)...); -} - -// Helper base class to hold the data and all operations. -// We move all this to a base class to allow mixing with the appropriate -// TraitsBase specialization. -template -class StatusOrData { - template - friend class StatusOrData; - - public: - StatusOrData() = delete; - - StatusOrData(const StatusOrData& other) { - if (other.ok()) { - MakeValue(other.data_); - MakeStatus(); - } else { - MakeStatus(other.status_); - } - } - - StatusOrData(StatusOrData&& other) noexcept { - if (other.ok()) { - MakeValue(std::move(other.data_)); - MakeStatus(); - } else { - MakeStatus(std::move(other.status_)); - } - } - - template - StatusOrData(const StatusOrData& other) { - if (other.ok()) { - MakeValue(other.data_); - MakeStatus(); - } else { - MakeStatus(other.status_); - } - } - - template - StatusOrData(StatusOrData&& other) { - if (other.ok()) { - MakeValue(std::move(other.data_)); - MakeStatus(); - } else { - MakeStatus(std::move(other.status_)); - } - } - - explicit StatusOrData(const T& value) : data_(value) { MakeStatus(); } - explicit StatusOrData(T&& value) : data_(std::move(value)) { MakeStatus(); } - - explicit StatusOrData(const Status& status) : status_(status) { - EnsureNotOk(); - } - explicit StatusOrData(Status&& status) : status_(std::move(status)) { - EnsureNotOk(); - } - - StatusOrData& operator=(const StatusOrData& other) { - if (this == &other) return *this; - if (other.ok()) - Assign(other.data_); - else - Assign(other.status_); - return *this; - } - - StatusOrData& operator=(StatusOrData&& other) { - if (this == &other) return *this; - if (other.ok()) - Assign(std::move(other.data_)); - else - Assign(std::move(other.status_)); - return *this; - } - - ~StatusOrData() { - if (ok()) { - status_.~Status(); - data_.~T(); - } else { - status_.~Status(); - } - } - - void Assign(const T& value) { - if (ok()) { - data_.~T(); - MakeValue(value); - } else { - MakeValue(value); - status_ = Status::OK(); - } - } - - void Assign(T&& value) { - if (ok()) { - data_.~T(); - MakeValue(std::move(value)); - } else { - MakeValue(std::move(value)); - status_ = Status::OK(); - } - } - - void Assign(const Status& status) { - Clear(); - status_ = status; - EnsureNotOk(); - } - - void Assign(Status&& status) { - Clear(); - status_ = std::move(status); - EnsureNotOk(); - } - - bool ok() const { return status_.ok(); } - - protected: - // status_ will always be active after the constructor. - // We make it a union to be able to initialize exactly how we need without - // waste. - // Eg. in the copy constructor we use the default constructor of Status in - // the ok() path to avoid an extra Ref call. - union { - Status status_; - }; - - // data_ is active iff status_.ok()==true - struct Dummy {}; - union { - // When T is const, we need some non-const object we can cast to void* for - // the placement new. dummy_ is that object. - Dummy dummy_; - T data_; - }; - - void Clear() { - if (ok()) data_.~T(); - } - - void EnsureOk() const { - if (!ok()) Helper::Crash(status_); - } - - void EnsureNotOk() { - if (ok()) Helper::HandleInvalidStatusCtorArg(&status_); - } - - // Construct the value (ie. data_) through placement new with the passed - // argument. - template - void MakeValue(Arg&& arg) { - internal_statusor::PlacementNew(&dummy_, std::forward(arg)); - } - - // Construct the status (ie. status_) through placement new with the passed - // argument. - template - void MakeStatus(Args&&... args) { - internal_statusor::PlacementNew(&status_, - std::forward(args)...); - } -}; - -// Helper base class to allow implicitly deleted constructors and assignment -// operations in StatusOr. -// TraitsBase will explicitly delete what it can't support and StatusOr will -// inherit that behavior implicitly. -template -struct TraitsBase { - TraitsBase() = default; - TraitsBase(const TraitsBase&) = default; - TraitsBase(TraitsBase&&) = default; - TraitsBase& operator=(const TraitsBase&) = default; - TraitsBase& operator=(TraitsBase&&) = default; -}; - -template <> -struct TraitsBase { - TraitsBase() = default; - TraitsBase(const TraitsBase&) = delete; - TraitsBase(TraitsBase&&) = default; - TraitsBase& operator=(const TraitsBase&) = delete; - TraitsBase& operator=(TraitsBase&&) = default; -}; - -template <> -struct TraitsBase { - TraitsBase() = default; - TraitsBase(const TraitsBase&) = delete; - TraitsBase(TraitsBase&&) = delete; - TraitsBase& operator=(const TraitsBase&) = delete; - TraitsBase& operator=(TraitsBase&&) = delete; -}; - -} // namespace internal_statusor -} // namespace xla - -#endif // TENSORFLOW_COMPILER_XLA_STATUSOR_INTERNALS_H_ diff --git a/tensorflow/compiler/xla/statusor_test.cc b/tensorflow/compiler/xla/statusor_test.cc deleted file mode 100644 index 377a618ffb..0000000000 --- a/tensorflow/compiler/xla/statusor_test.cc +++ /dev/null @@ -1,675 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Unit tests for StatusOr - -#include "tensorflow/compiler/xla/statusor.h" - -#include -#include - -#include "tensorflow/compiler/xla/test.h" -#include "tensorflow/compiler/xla/types.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/test_benchmark.h" - -namespace xla { -namespace { - -class Base1 { - public: - virtual ~Base1() {} - int pad_; -}; - -class Base2 { - public: - virtual ~Base2() {} - int yetotherpad_; -}; - -class Derived : public Base1, public Base2 { - public: - ~Derived() override {} - int evenmorepad_; -}; - -class CopyNoAssign { - public: - explicit CopyNoAssign(int value) : foo_(value) {} - CopyNoAssign(const CopyNoAssign& other) : foo_(other.foo_) {} - int foo_; - - private: - const CopyNoAssign& operator=(const CopyNoAssign&); -}; - -class NoDefaultConstructor { - public: - explicit NoDefaultConstructor(int foo); -}; - -static_assert(!std::is_default_constructible(), - "Should not be default-constructible."); - -StatusOr> ReturnUniquePtr() { - // Uses implicit constructor from T&& - return std::unique_ptr(new int(0)); -} - -TEST(StatusOr, ElementType) { - static_assert(std::is_same::element_type, int>(), ""); - static_assert(std::is_same::element_type, char>(), ""); -} - -TEST(StatusOr, NullPointerStatusOr) { - // As a very special case, null-plain-pointer StatusOr used to be an - // error. Test that it no longer is. - StatusOr null_status(nullptr); - EXPECT_TRUE(null_status.ok()); - EXPECT_EQ(null_status.ValueOrDie(), nullptr); -} - -TEST(StatusOr, TestNoDefaultConstructorInitialization) { - // Explicitly initialize it with an error code. - StatusOr statusor(tensorflow::errors::Cancelled("")); - EXPECT_FALSE(statusor.ok()); - EXPECT_EQ(statusor.status().code(), tensorflow::error::CANCELLED); - - // Default construction of StatusOr initializes it with an UNKNOWN error code. - StatusOr statusor2; - EXPECT_FALSE(statusor2.ok()); - EXPECT_EQ(statusor2.status().code(), tensorflow::error::UNKNOWN); -} - -TEST(StatusOr, TestMoveOnlyInitialization) { - StatusOr> thing(ReturnUniquePtr()); - ASSERT_TRUE(thing.ok()); - EXPECT_EQ(0, *thing.ValueOrDie()); - int* previous = thing.ValueOrDie().get(); - - thing = ReturnUniquePtr(); - EXPECT_TRUE(thing.ok()); - EXPECT_EQ(0, *thing.ValueOrDie()); - EXPECT_NE(previous, thing.ValueOrDie().get()); -} - -TEST(StatusOr, TestMoveOnlyStatusCtr) { - StatusOr> thing(tensorflow::errors::Cancelled("")); - ASSERT_FALSE(thing.ok()); -} - -TEST(StatusOr, TestMoveOnlyValueExtraction) { - StatusOr> thing(ReturnUniquePtr()); - ASSERT_TRUE(thing.ok()); - std::unique_ptr ptr = thing.ConsumeValueOrDie(); - EXPECT_EQ(0, *ptr); - - thing = std::move(ptr); - ptr = std::move(thing.ValueOrDie()); - EXPECT_EQ(0, *ptr); -} - -TEST(StatusOr, TestMoveOnlyConversion) { - StatusOr> const_thing(ReturnUniquePtr()); - EXPECT_TRUE(const_thing.ok()); - EXPECT_EQ(0, *const_thing.ValueOrDie()); - - // Test rvalue converting assignment - const int* const_previous = const_thing.ValueOrDie().get(); - const_thing = ReturnUniquePtr(); - EXPECT_TRUE(const_thing.ok()); - EXPECT_EQ(0, *const_thing.ValueOrDie()); - EXPECT_NE(const_previous, const_thing.ValueOrDie().get()); -} - -TEST(StatusOr, TestMoveOnlyVector) { - // Sanity check that StatusOr works in vector. - std::vector>> vec; - vec.push_back(ReturnUniquePtr()); - vec.resize(2); - auto another_vec = std::move(vec); - EXPECT_EQ(0, *another_vec[0].ValueOrDie()); - EXPECT_EQ(tensorflow::error::UNKNOWN, another_vec[1].status().code()); -} - -TEST(StatusOr, TestMoveWithValuesAndErrors) { - StatusOr status_or(string(1000, '0')); - StatusOr value1(string(1000, '1')); - StatusOr value2(string(1000, '2')); - StatusOr error1(Status(tensorflow::error::UNKNOWN, "error1")); - StatusOr error2(Status(tensorflow::error::UNKNOWN, "error2")); - - ASSERT_TRUE(status_or.ok()); - EXPECT_EQ(string(1000, '0'), status_or.ValueOrDie()); - - // Overwrite the value in status_or with another value. - status_or = std::move(value1); - ASSERT_TRUE(status_or.ok()); - EXPECT_EQ(string(1000, '1'), status_or.ValueOrDie()); - - // Overwrite the value in status_or with an error. - status_or = std::move(error1); - ASSERT_FALSE(status_or.ok()); - EXPECT_EQ("error1", status_or.status().error_message()); - - // Overwrite the error in status_or with another error. - status_or = std::move(error2); - ASSERT_FALSE(status_or.ok()); - EXPECT_EQ("error2", status_or.status().error_message()); - - // Overwrite the error with a value. - status_or = std::move(value2); - ASSERT_TRUE(status_or.ok()); - EXPECT_EQ(string(1000, '2'), status_or.ValueOrDie()); -} - -TEST(StatusOr, TestCopyWithValuesAndErrors) { - StatusOr status_or(string(1000, '0')); - StatusOr value1(string(1000, '1')); - StatusOr value2(string(1000, '2')); - StatusOr error1(Status(tensorflow::error::UNKNOWN, "error1")); - StatusOr error2(Status(tensorflow::error::UNKNOWN, "error2")); - - ASSERT_TRUE(status_or.ok()); - EXPECT_EQ(string(1000, '0'), status_or.ValueOrDie()); - - // Overwrite the value in status_or with another value. - status_or = value1; - ASSERT_TRUE(status_or.ok()); - EXPECT_EQ(string(1000, '1'), status_or.ValueOrDie()); - - // Overwrite the value in status_or with an error. - status_or = error1; - ASSERT_FALSE(status_or.ok()); - EXPECT_EQ("error1", status_or.status().error_message()); - - // Overwrite the error in status_or with another error. - status_or = error2; - ASSERT_FALSE(status_or.ok()); - EXPECT_EQ("error2", status_or.status().error_message()); - - // Overwrite the error with a value. - status_or = value2; - ASSERT_TRUE(status_or.ok()); - EXPECT_EQ(string(1000, '2'), status_or.ValueOrDie()); - - // Verify original values unchanged. - EXPECT_EQ(string(1000, '1'), value1.ValueOrDie()); - EXPECT_EQ("error1", error1.status().error_message()); - EXPECT_EQ("error2", error2.status().error_message()); - EXPECT_EQ(string(1000, '2'), value2.ValueOrDie()); -} - -TEST(StatusOr, TestDefaultCtor) { - StatusOr thing; - EXPECT_FALSE(thing.ok()); - EXPECT_EQ(thing.status().code(), tensorflow::error::UNKNOWN); -} - -TEST(StatusOrDeathTest, TestDefaultCtorValue) { - StatusOr thing; - EXPECT_DEATH(thing.ValueOrDie(), ""); - - const StatusOr thing2; - EXPECT_DEATH(thing.ValueOrDie(), ""); -} - -TEST(StatusOr, TestStatusCtor) { - StatusOr thing(Status(tensorflow::error::CANCELLED, "")); - EXPECT_FALSE(thing.ok()); - EXPECT_EQ(thing.status().code(), tensorflow::error::CANCELLED); -} - -TEST(StatusOr, TestValueCtor) { - const int kI = 4; - const StatusOr thing(kI); - EXPECT_TRUE(thing.ok()); - EXPECT_EQ(kI, thing.ValueOrDie()); -} - -TEST(StatusOr, TestCopyCtorStatusOk) { - const int kI = 4; - const StatusOr original(kI); - const StatusOr copy(original); - EXPECT_EQ(copy.status(), original.status()); - EXPECT_EQ(original.ValueOrDie(), copy.ValueOrDie()); -} - -TEST(StatusOr, TestCopyCtorStatusNotOk) { - StatusOr original(Status(tensorflow::error::CANCELLED, "")); - StatusOr copy(original); - EXPECT_EQ(copy.status(), original.status()); -} - -TEST(StatusOr, TestCopyCtorNonAssignable) { - const int kI = 4; - CopyNoAssign value(kI); - StatusOr original(value); - StatusOr copy(original); - EXPECT_EQ(copy.status(), original.status()); - EXPECT_EQ(original.ValueOrDie().foo_, copy.ValueOrDie().foo_); -} - -TEST(StatusOr, TestCopyCtorStatusOKConverting) { - const int kI = 4; - StatusOr original(kI); - StatusOr copy(original); - EXPECT_EQ(copy.status(), original.status()); - EXPECT_DOUBLE_EQ(original.ValueOrDie(), copy.ValueOrDie()); -} - -TEST(StatusOr, TestCopyCtorStatusNotOkConverting) { - StatusOr original(Status(tensorflow::error::CANCELLED, "")); - StatusOr copy(original); - EXPECT_EQ(copy.status(), original.status()); -} - -TEST(StatusOr, TestAssignmentStatusOk) { - const int kI = 4; - StatusOr source(kI); - StatusOr target; - target = source; - EXPECT_EQ(target.status(), source.status()); - EXPECT_EQ(source.ValueOrDie(), target.ValueOrDie()); -} - -TEST(StatusOr, TestAssignmentStatusNotOk) { - StatusOr source(Status(tensorflow::error::CANCELLED, "")); - StatusOr target; - target = source; - EXPECT_EQ(target.status(), source.status()); -} - -TEST(StatusOr, TestStatus) { - StatusOr good(4); - EXPECT_TRUE(good.ok()); - StatusOr bad(Status(tensorflow::error::CANCELLED, "")); - EXPECT_FALSE(bad.ok()); - EXPECT_EQ(bad.status(), Status(tensorflow::error::CANCELLED, "")); -} - -TEST(StatusOr, TestValue) { - const int kI = 4; - StatusOr thing(kI); - EXPECT_EQ(kI, thing.ValueOrDie()); -} - -TEST(StatusOr, TestValueConst) { - const int kI = 4; - const StatusOr thing(kI); - EXPECT_EQ(kI, thing.ValueOrDie()); -} - -TEST(StatusOrDeathTest, TestValueNotOk) { - StatusOr thing(Status(tensorflow::error::CANCELLED, "cancelled")); - EXPECT_DEATH(thing.ValueOrDie(), "cancelled"); -} - -TEST(StatusOrDeathTest, TestValueNotOkConst) { - const StatusOr thing(Status(tensorflow::error::UNKNOWN, "")); - EXPECT_DEATH(thing.ValueOrDie(), ""); -} - -TEST(StatusOr, TestPointerDefaultCtor) { - StatusOr thing; - EXPECT_FALSE(thing.ok()); - EXPECT_EQ(thing.status().code(), tensorflow::error::UNKNOWN); -} - -TEST(StatusOrDeathTest, TestPointerDefaultCtorValue) { - StatusOr thing; - EXPECT_DEATH(thing.ValueOrDie(), ""); -} - -TEST(StatusOr, TestPointerStatusCtor) { - StatusOr thing(Status(tensorflow::error::CANCELLED, "")); - EXPECT_FALSE(thing.ok()); - EXPECT_EQ(thing.status(), Status(tensorflow::error::CANCELLED, "")); -} - -TEST(StatusOr, TestPointerValueCtor) { - const int kI = 4; - StatusOr thing(&kI); - EXPECT_TRUE(thing.ok()); - EXPECT_EQ(&kI, thing.ValueOrDie()); -} - -TEST(StatusOr, TestPointerCopyCtorStatusOk) { - const int kI = 0; - StatusOr original(&kI); - StatusOr copy(original); - EXPECT_EQ(copy.status(), original.status()); - EXPECT_EQ(original.ValueOrDie(), copy.ValueOrDie()); -} - -TEST(StatusOr, TestPointerCopyCtorStatusNotOk) { - StatusOr original(Status(tensorflow::error::CANCELLED, "")); - StatusOr copy(original); - EXPECT_EQ(copy.status(), original.status()); -} - -TEST(StatusOr, TestPointerCopyCtorStatusOKConverting) { - Derived derived; - StatusOr original(&derived); - StatusOr copy(original); - EXPECT_EQ(copy.status(), original.status()); - EXPECT_EQ(static_cast(original.ValueOrDie()), - copy.ValueOrDie()); -} - -TEST(StatusOr, TestPointerCopyCtorStatusNotOkConverting) { - StatusOr original(Status(tensorflow::error::CANCELLED, "")); - StatusOr copy(original); - EXPECT_EQ(copy.status(), original.status()); -} - -TEST(StatusOr, TestPointerAssignmentStatusOk) { - const int kI = 0; - StatusOr source(&kI); - StatusOr target; - target = source; - EXPECT_EQ(target.status(), source.status()); - EXPECT_EQ(source.ValueOrDie(), target.ValueOrDie()); -} - -TEST(StatusOr, TestPointerAssignmentStatusNotOk) { - StatusOr source(Status(tensorflow::error::CANCELLED, "")); - StatusOr target; - target = source; - EXPECT_EQ(target.status(), source.status()); -} - -TEST(StatusOr, TestPointerStatus) { - const int kI = 0; - StatusOr good(&kI); - EXPECT_TRUE(good.ok()); - StatusOr bad(Status(tensorflow::error::CANCELLED, "")); - EXPECT_EQ(bad.status(), Status(tensorflow::error::CANCELLED, "")); -} - -TEST(StatusOr, TestPointerValue) { - const int kI = 0; - StatusOr thing(&kI); - EXPECT_EQ(&kI, thing.ValueOrDie()); -} - -TEST(StatusOr, TestPointerValueConst) { - const int kI = 0; - const StatusOr thing(&kI); - EXPECT_EQ(&kI, thing.ValueOrDie()); -} - -// NOTE(tucker): StatusOr does not support this kind -// of resize op. -// TEST(StatusOr, StatusOrVectorOfUniquePointerCanResize) { -// using EvilType = std::vector>; -// static_assert(std::is_copy_constructible::value, ""); -// std::vector> v(5); -// v.reserve(v.capacity() + 10); -// } - -TEST(StatusOrDeathTest, TestPointerValueNotOk) { - StatusOr thing(Status(tensorflow::error::CANCELLED, "cancelled")); - EXPECT_DEATH(thing.ValueOrDie(), "cancelled"); -} - -TEST(StatusOrDeathTest, TestPointerValueNotOkConst) { - const StatusOr thing(Status(tensorflow::error::CANCELLED, "cancelled")); - EXPECT_DEATH(thing.ValueOrDie(), "cancelled"); -} - -static StatusOr MakeStatus() { return 100; } -// A factory to help us benchmark the various factory styles. All of -// the factory methods are marked as non-inlineable so as to more -// accurately simulate calling a factory for which you do not have -// visibility of implementation. Similarly, the value_ variable is -// marked volatile to prevent the compiler from getting too clever -// about detecting that the same value is used in all loop iterations. -template -class BenchmarkFactory { - public: - // Construct a new factory. Allocate an object which will always - // be the result of the factory methods. - BenchmarkFactory() : value_(new T) {} - - // Destroy this factory, including the result value. - ~BenchmarkFactory() { delete value_; } - - // A trivial factory that just returns the value. There is no status - // object that could be returned to encapsulate an error - T* TrivialFactory() TF_ATTRIBUTE_NOINLINE { return value_; } - - // A more sophisticated factory, which returns a status to indicate - // the result of the operation. The factory result is populated into - // the user provided pointer result. - Status ArgumentFactory(T** result) TF_ATTRIBUTE_NOINLINE { - *result = value_; - return Status::OK(); - } - - Status ArgumentFactoryFail(T** result) TF_ATTRIBUTE_NOINLINE { - *result = nullptr; - return Status(tensorflow::error::CANCELLED, ""); - } - - Status ArgumentFactoryFailShortMsg(T** result) TF_ATTRIBUTE_NOINLINE { - *result = nullptr; - return Status(::tensorflow::error::INTERNAL, ""); - } - - Status ArgumentFactoryFailLongMsg(T** result) TF_ATTRIBUTE_NOINLINE { - *result = nullptr; - return Status(::tensorflow::error::INTERNAL, - "a big string of message junk that will never be read"); - } - - // A factory that returns a StatusOr. If the factory operation - // is OK, then the StatusOr will hold a T*. Otherwise, it will - // hold a status explaining the error. - StatusOr StatusOrFactory() TF_ATTRIBUTE_NOINLINE { - return static_cast(value_); - } - - StatusOr StatusOrFactoryFail() TF_ATTRIBUTE_NOINLINE { - return Status(tensorflow::error::CANCELLED, ""); - } - - StatusOr StatusOrFactoryFailShortMsg() TF_ATTRIBUTE_NOINLINE { - return Status(::tensorflow::error::INTERNAL, ""); - } - - StatusOr StatusOrFactoryFailLongMsg() TF_ATTRIBUTE_NOINLINE { - return Status(::tensorflow::error::INTERNAL, - "a big string of message junk that will never be read"); - } - - private: - T* volatile value_; - TF_DISALLOW_COPY_AND_ASSIGN(BenchmarkFactory); -}; - -// A simple type we use with the factory. -class BenchmarkType { - public: - BenchmarkType() {} - virtual ~BenchmarkType() {} - virtual void DoWork() TF_ATTRIBUTE_NOINLINE {} - - private: - TF_DISALLOW_COPY_AND_ASSIGN(BenchmarkType); -}; - -// Calibrate the amount of time spent just calling DoWork, since each of our -// tests will do this, we can subtract this out of benchmark results. -void BM_CalibrateWorkLoop(int iters) { - tensorflow::testing::StopTiming(); - BenchmarkFactory factory; - BenchmarkType* result = factory.TrivialFactory(); - tensorflow::testing::StartTiming(); - for (int i = 0; i != iters; ++i) { - if (result != nullptr) { - result->DoWork(); - } - } -} -BENCHMARK(BM_CalibrateWorkLoop); - -// Measure the time taken to call into the factory, return the value, -// determine that it is OK, and invoke a trivial function. -void BM_TrivialFactory(int iters) { - tensorflow::testing::StopTiming(); - BenchmarkFactory factory; - tensorflow::testing::StartTiming(); - for (int i = 0; i != iters; ++i) { - BenchmarkType* result = factory.TrivialFactory(); - if (result != nullptr) { - result->DoWork(); - } - } -} -BENCHMARK(BM_TrivialFactory); - -// Measure the time taken to call into the factory, providing an -// out-param for the result, evaluating the status result and the -// result pointer, and invoking the trivial function. -void BM_ArgumentFactory(int iters) { - tensorflow::testing::StopTiming(); - BenchmarkFactory factory; - tensorflow::testing::StartTiming(); - for (int i = 0; i != iters; ++i) { - BenchmarkType* result = nullptr; - Status status = factory.ArgumentFactory(&result); - if (status.ok() && result != nullptr) { - result->DoWork(); - } - } -} -BENCHMARK(BM_ArgumentFactory); - -// Measure the time to use the StatusOr factory, evaluate the result, -// and invoke the trivial function. -void BM_StatusOrFactory(int iters) { - tensorflow::testing::StopTiming(); - BenchmarkFactory factory; - tensorflow::testing::StartTiming(); - for (int i = 0; i != iters; ++i) { - StatusOr result = factory.StatusOrFactory(); - if (result.ok()) { - result.ValueOrDie()->DoWork(); - } - } -} -BENCHMARK(BM_StatusOrFactory); - -// Measure the time taken to call into the factory, providing an -// out-param for the result, evaluating the status result and the -// result pointer, and invoking the trivial function. -void BM_ArgumentFactoryFail(int iters) { - tensorflow::testing::StopTiming(); - BenchmarkFactory factory; - tensorflow::testing::StartTiming(); - for (int i = 0; i != iters; ++i) { - BenchmarkType* result = nullptr; - Status status = factory.ArgumentFactoryFail(&result); - if (status.ok() && result != nullptr) { - result->DoWork(); - } - } -} -BENCHMARK(BM_ArgumentFactoryFail); - -// Measure the time to use the StatusOr factory, evaluate the result, -// and invoke the trivial function. -void BM_StatusOrFactoryFail(int iters) { - tensorflow::testing::StopTiming(); - BenchmarkFactory factory; - tensorflow::testing::StartTiming(); - for (int i = 0; i != iters; ++i) { - StatusOr result = factory.StatusOrFactoryFail(); - if (result.ok()) { - result.ValueOrDie()->DoWork(); - } - } -} -BENCHMARK(BM_StatusOrFactoryFail); - -// Measure the time taken to call into the factory, providing an -// out-param for the result, evaluating the status result and the -// result pointer, and invoking the trivial function. -void BM_ArgumentFactoryFailShortMsg(int iters) { - tensorflow::testing::StopTiming(); - BenchmarkFactory factory; - tensorflow::testing::StartTiming(); - for (int i = 0; i != iters; ++i) { - BenchmarkType* result = nullptr; - Status status = factory.ArgumentFactoryFailShortMsg(&result); - if (status.ok() && result != nullptr) { - result->DoWork(); - } - } -} -BENCHMARK(BM_ArgumentFactoryFailShortMsg); - -// Measure the time to use the StatusOr factory, evaluate the result, -// and invoke the trivial function. -void BM_StatusOrFactoryFailShortMsg(int iters) { - tensorflow::testing::StopTiming(); - BenchmarkFactory factory; - tensorflow::testing::StartTiming(); - for (int i = 0; i != iters; ++i) { - StatusOr result = factory.StatusOrFactoryFailShortMsg(); - if (result.ok()) { - result.ValueOrDie()->DoWork(); - } - } -} -BENCHMARK(BM_StatusOrFactoryFailShortMsg); - -// Measure the time taken to call into the factory, providing an -// out-param for the result, evaluating the status result and the -// result pointer, and invoking the trivial function. -void BM_ArgumentFactoryFailLongMsg(int iters) { - tensorflow::testing::StopTiming(); - BenchmarkFactory factory; - tensorflow::testing::StartTiming(); - for (int i = 0; i != iters; ++i) { - BenchmarkType* result = nullptr; - Status status = factory.ArgumentFactoryFailLongMsg(&result); - if (status.ok() && result != nullptr) { - result->DoWork(); - } - } -} -BENCHMARK(BM_ArgumentFactoryFailLongMsg); - -// Measure the time to use the StatusOr factory, evaluate the result, -// and invoke the trivial function. -void BM_StatusOrFactoryFailLongMsg(int iters) { - tensorflow::testing::StopTiming(); - BenchmarkFactory factory; - tensorflow::testing::StartTiming(); - for (int i = 0; i != iters; ++i) { - StatusOr result = factory.StatusOrFactoryFailLongMsg(); - if (result.ok()) { - result.ValueOrDie()->DoWork(); - } - } -} -BENCHMARK(BM_StatusOrFactoryFailLongMsg); - -} // namespace -} // namespace xla diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD index c68cda0100..21295abed1 100644 --- a/tensorflow/stream_executor/BUILD +++ b/tensorflow/stream_executor/BUILD @@ -33,7 +33,6 @@ cc_library( }), visibility = ["//visibility:public"], deps = [ - "//tensorflow/compiler/xla:statusor", "//tensorflow/core:lib", "//tensorflow/core:ptr_util", "@local_config_cuda//cuda:cuda_headers", @@ -48,7 +47,6 @@ cc_library( deps = [ "//tensorflow/core:lib", "//tensorflow/core:ptr_util", - "//tensorflow/compiler/xla:statusor", "@local_config_cuda//cuda:cuda_headers", ] + if_static([":stream_executor_impl"]), ) diff --git a/tensorflow/stream_executor/lib/statusor.cc b/tensorflow/stream_executor/lib/statusor.cc new file mode 100644 index 0000000000..e0e851f96e --- /dev/null +++ b/tensorflow/stream_executor/lib/statusor.cc @@ -0,0 +1,40 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/stream_executor/lib/statusor.h" + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/logging.h" + +namespace stream_executor { +namespace port { +namespace internal_statusor { + +void Helper::HandleInvalidStatusCtorArg(Status* status) { + const char* kMessage = + "An OK status is not a valid constructor argument to StatusOr"; + LOG(ERROR) << kMessage; + // Fall back to tensorflow::error::INTERNAL. + *status = ::tensorflow::errors::Internal(kMessage); +} + +void Helper::Crash(const Status& status) { + LOG(FATAL) << "Attempting to fetch value instead of handling error " + << status; +} + +} // namespace internal_statusor +} // namespace port +} // namespace stream_executor diff --git a/tensorflow/stream_executor/lib/statusor.h b/tensorflow/stream_executor/lib/statusor.h index dab5909674..3c716acb46 100644 --- a/tensorflow/stream_executor/lib/statusor.h +++ b/tensorflow/stream_executor/lib/statusor.h @@ -1,4 +1,4 @@ -/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,19 +13,297 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// IWYU pragma: private, include "third_party/tensorflow/stream_executor/stream_executor.h" - +// StatusOr is the union of a Status object and a T object. StatusOr models +// the concept of an object that is either a value, or an error Status +// explaining why such a value is not present. To this end, StatusOr does not +// allow its Status value to be Status::OK. +// +// The primary use-case for StatusOr is as the return value of a +// function which may fail. +// +// Example client usage for a StatusOr, where T is not a pointer: +// +// StatusOr result = DoBigCalculationThatCouldFail(); +// if (result.ok()) { +// float answer = result.ValueOrDie(); +// printf("Big calculation yielded: %f", answer); +// } else { +// LOG(ERROR) << result.status(); +// } +// +// Example client usage for a StatusOr: +// +// StatusOr result = FooFactory::MakeNewFoo(arg); +// if (result.ok()) { +// std::unique_ptr foo(result.ValueOrDie()); +// foo->DoSomethingCool(); +// } else { +// LOG(ERROR) << result.status(); +// } +// +// Example client usage for a StatusOr>: +// +// StatusOr> result = FooFactory::MakeNewFoo(arg); +// if (result.ok()) { +// std::unique_ptr foo = std::move(result.ValueOrDie()); +// foo->DoSomethingCool(); +// } else { +// LOG(ERROR) << result.status(); +// } +// +// Example factory implementation returning StatusOr: +// +// StatusOr FooFactory::MakeNewFoo(int arg) { +// if (arg <= 0) { +// return tensorflow::InvalidArgument("Arg must be positive"); +// } else { +// return new Foo(arg); +// } +// } +// +// Note that the assignment operators require that destroying the currently +// stored value cannot invalidate the argument; in other words, the argument +// cannot be an alias for the current value, or anything owned by the current +// value. #ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_H_ #define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_H_ -#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/stream_executor/lib/status.h" +#include "tensorflow/stream_executor/lib/statusor_internals.h" namespace stream_executor { namespace port { -// Use XLA's StatusOr so we don't duplicate code. +#if defined(__clang__) +// Only clang supports warn_unused_result as a type annotation. +template +class TF_MUST_USE_RESULT StatusOr; +#endif + +template +class StatusOr : private internal_statusor::StatusOrData, + private internal_statusor::TraitsBase< + std::is_copy_constructible::value, + std::is_move_constructible::value> { + template + friend class StatusOr; + + typedef internal_statusor::StatusOrData Base; + + public: + typedef T element_type; + + // Constructs a new StatusOr with Status::UNKNOWN status. This is marked + // 'explicit' to try to catch cases like 'return {};', where people think + // StatusOr> will be initialized with an empty vector, + // instead of a Status::UNKNOWN status. + explicit StatusOr(); + + // StatusOr will be copy constructible/assignable if T is copy + // constructible. + StatusOr(const StatusOr&) = default; + StatusOr& operator=(const StatusOr&) = default; + + // StatusOr will be move constructible/assignable if T is move + // constructible. + StatusOr(StatusOr&&) = default; + StatusOr& operator=(StatusOr&&) = default; + + // Conversion copy/move constructor, T must be convertible from U. + template ::value>::type* = nullptr> + StatusOr(const StatusOr& other); + template ::value>::type* = nullptr> + StatusOr(StatusOr&& other); + + // Conversion copy/move assignment operator, T must be convertible from U. + template ::value>::type* = nullptr> + StatusOr& operator=(const StatusOr& other); + template ::value>::type* = nullptr> + StatusOr& operator=(StatusOr&& other); + + // Constructs a new StatusOr with the given value. After calling this + // constructor, calls to ValueOrDie() will succeed, and calls to status() will + // return OK. + // + // NOTE: Not explicit - we want to use StatusOr as a return type + // so it is convenient and sensible to be able to do 'return T()' + // when the return type is StatusOr. + // + // REQUIRES: T is copy constructible. + StatusOr(const T& value); + + // Constructs a new StatusOr with the given non-ok status. After calling + // this constructor, calls to ValueOrDie() will CHECK-fail. + // + // NOTE: Not explicit - we want to use StatusOr as a return + // value, so it is convenient and sensible to be able to do 'return + // Status()' when the return type is StatusOr. + // + // REQUIRES: !status.ok(). This requirement is DCHECKed. + // In optimized builds, passing Status::OK() here will have the effect + // of passing tensorflow::error::INTERNAL as a fallback. + StatusOr(const Status& status); + StatusOr& operator=(const Status& status); + + // TODO(b/62186997): Add operator=(T) overloads. + + // Similar to the `const T&` overload. + // + // REQUIRES: T is move constructible. + StatusOr(T&& value); + + // RValue versions of the operations declared above. + StatusOr(Status&& status); + StatusOr& operator=(Status&& status); + + // Returns this->status().ok() + bool ok() const { return this->status_.ok(); } + + // Returns a reference to our status. If this contains a T, then + // returns Status::OK(). + const Status& status() const &; + Status status() &&; + + // Returns a reference to our current value, or CHECK-fails if !this->ok(). + // + // Note: for value types that are cheap to copy, prefer simple code: + // + // T value = statusor.ValueOrDie(); + // + // Otherwise, if the value type is expensive to copy, but can be left + // in the StatusOr, simply assign to a reference: + // + // T& value = statusor.ValueOrDie(); // or `const T&` + // + // Otherwise, if the value type supports an efficient move, it can be + // used as follows: + // + // T value = std::move(statusor).ValueOrDie(); + // + // The std::move on statusor instead of on the whole expression enables + // warnings about possible uses of the statusor object after the move. + // C++ style guide waiver for ref-qualified overloads granted in cl/143176389 + // See go/ref-qualifiers for more details on such overloads. + const T& ValueOrDie() const &; + T& ValueOrDie() &; + const T&& ValueOrDie() const &&; + T&& ValueOrDie() &&; + + T ConsumeValueOrDie() { return std::move(ValueOrDie()); } + + // Ignores any errors. This method does nothing except potentially suppress + // complaints from any tools that are checking that errors are not dropped on + // the floor. + void IgnoreError() const; +}; + +//////////////////////////////////////////////////////////////////////////////// +// Implementation details for StatusOr + +template +StatusOr::StatusOr() : Base(Status(tensorflow::error::UNKNOWN, "")) {} + +template +StatusOr::StatusOr(const T& value) : Base(value) {} + +template +StatusOr::StatusOr(const Status& status) : Base(status) {} + +template +StatusOr& StatusOr::operator=(const Status& status) { + this->Assign(status); + return *this; +} + +template +StatusOr::StatusOr(T&& value) : Base(std::move(value)) {} + +template +StatusOr::StatusOr(Status&& status) : Base(std::move(status)) {} + +template +StatusOr& StatusOr::operator=(Status&& status) { + this->Assign(std::move(status)); + return *this; +} + +template +template ::value>::type*> +inline StatusOr::StatusOr(const StatusOr& other) + : Base(static_cast::Base&>(other)) {} + +template +template ::value>::type*> +inline StatusOr& StatusOr::operator=(const StatusOr& other) { + if (other.ok()) + this->Assign(other.ValueOrDie()); + else + this->Assign(other.status()); + return *this; +} + +template +template ::value>::type*> +inline StatusOr::StatusOr(StatusOr&& other) + : Base(static_cast::Base&&>(other)) {} + +template +template ::value>::type*> +inline StatusOr& StatusOr::operator=(StatusOr&& other) { + if (other.ok()) { + this->Assign(std::move(other).ValueOrDie()); + } else { + this->Assign(std::move(other).status()); + } + return *this; +} + +template +const Status& StatusOr::status() const & { + return this->status_; +} +template +Status StatusOr::status() && { + return ok() ? Status::OK() : std::move(this->status_); +} + +template +const T& StatusOr::ValueOrDie() const & { + this->EnsureOk(); + return this->data_; +} + +template +T& StatusOr::ValueOrDie() & { + this->EnsureOk(); + return this->data_; +} + +template +const T&& StatusOr::ValueOrDie() const && { + this->EnsureOk(); + return std::move(this->data_); +} + +template +T&& StatusOr::ValueOrDie() && { + this->EnsureOk(); + return std::move(this->data_); +} + template -using StatusOr = ::xla::StatusOr; +void StatusOr::IgnoreError() const { + // no-op +} } // namespace port } // namespace stream_executor diff --git a/tensorflow/stream_executor/lib/statusor_internals.h b/tensorflow/stream_executor/lib/statusor_internals.h new file mode 100644 index 0000000000..09f88f5825 --- /dev/null +++ b/tensorflow/stream_executor/lib/statusor_internals.h @@ -0,0 +1,248 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_INTERNALS_H_ +#define TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_INTERNALS_H_ + + +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/stream_executor/lib/status.h" + +namespace stream_executor { +namespace port { +namespace internal_statusor { + +class Helper { + public: + // Move type-agnostic error handling to the .cc. + static void HandleInvalidStatusCtorArg(Status*); + TF_ATTRIBUTE_NORETURN static void Crash(const Status& status); +}; + +// Construct an instance of T in `p` through placement new, passing Args... to +// the constructor. +// This abstraction is here mostly for the gcc performance fix. +template +void PlacementNew(void* p, Args&&... args) { +#if defined(__GNUC__) && !defined(__clang__) + // Teach gcc that 'p' cannot be null, fixing code size issues. + if (p == nullptr) __builtin_unreachable(); +#endif + new (p) T(std::forward(args)...); +} + +// Helper base class to hold the data and all operations. +// We move all this to a base class to allow mixing with the appropriate +// TraitsBase specialization. +template +class StatusOrData { + template + friend class StatusOrData; + + public: + StatusOrData() = delete; + + StatusOrData(const StatusOrData& other) { + if (other.ok()) { + MakeValue(other.data_); + MakeStatus(); + } else { + MakeStatus(other.status_); + } + } + + StatusOrData(StatusOrData&& other) noexcept { + if (other.ok()) { + MakeValue(std::move(other.data_)); + MakeStatus(); + } else { + MakeStatus(std::move(other.status_)); + } + } + + template + StatusOrData(const StatusOrData& other) { + if (other.ok()) { + MakeValue(other.data_); + MakeStatus(); + } else { + MakeStatus(other.status_); + } + } + + template + StatusOrData(StatusOrData&& other) { + if (other.ok()) { + MakeValue(std::move(other.data_)); + MakeStatus(); + } else { + MakeStatus(std::move(other.status_)); + } + } + + explicit StatusOrData(const T& value) : data_(value) { MakeStatus(); } + explicit StatusOrData(T&& value) : data_(std::move(value)) { MakeStatus(); } + + explicit StatusOrData(const Status& status) : status_(status) { + EnsureNotOk(); + } + explicit StatusOrData(Status&& status) : status_(std::move(status)) { + EnsureNotOk(); + } + + StatusOrData& operator=(const StatusOrData& other) { + if (this == &other) return *this; + if (other.ok()) + Assign(other.data_); + else + Assign(other.status_); + return *this; + } + + StatusOrData& operator=(StatusOrData&& other) { + if (this == &other) return *this; + if (other.ok()) + Assign(std::move(other.data_)); + else + Assign(std::move(other.status_)); + return *this; + } + + ~StatusOrData() { + if (ok()) { + status_.~Status(); + data_.~T(); + } else { + status_.~Status(); + } + } + + void Assign(const T& value) { + if (ok()) { + data_.~T(); + MakeValue(value); + } else { + MakeValue(value); + status_ = Status::OK(); + } + } + + void Assign(T&& value) { + if (ok()) { + data_.~T(); + MakeValue(std::move(value)); + } else { + MakeValue(std::move(value)); + status_ = Status::OK(); + } + } + + void Assign(const Status& status) { + Clear(); + status_ = status; + EnsureNotOk(); + } + + void Assign(Status&& status) { + Clear(); + status_ = std::move(status); + EnsureNotOk(); + } + + bool ok() const { return status_.ok(); } + + protected: + // status_ will always be active after the constructor. + // We make it a union to be able to initialize exactly how we need without + // waste. + // Eg. in the copy constructor we use the default constructor of Status in + // the ok() path to avoid an extra Ref call. + union { + Status status_; + }; + + // data_ is active iff status_.ok()==true + struct Dummy {}; + union { + // When T is const, we need some non-const object we can cast to void* for + // the placement new. dummy_ is that object. + Dummy dummy_; + T data_; + }; + + void Clear() { + if (ok()) data_.~T(); + } + + void EnsureOk() const { + if (!ok()) Helper::Crash(status_); + } + + void EnsureNotOk() { + if (ok()) Helper::HandleInvalidStatusCtorArg(&status_); + } + + // Construct the value (ie. data_) through placement new with the passed + // argument. + template + void MakeValue(Arg&& arg) { + internal_statusor::PlacementNew(&dummy_, std::forward(arg)); + } + + // Construct the status (ie. status_) through placement new with the passed + // argument. + template + void MakeStatus(Args&&... args) { + internal_statusor::PlacementNew(&status_, + std::forward(args)...); + } +}; + +// Helper base class to allow implicitly deleted constructors and assignment +// operations in StatusOr. +// TraitsBase will explicitly delete what it can't support and StatusOr will +// inherit that behavior implicitly. +template +struct TraitsBase { + TraitsBase() = default; + TraitsBase(const TraitsBase&) = default; + TraitsBase(TraitsBase&&) = default; + TraitsBase& operator=(const TraitsBase&) = default; + TraitsBase& operator=(TraitsBase&&) = default; +}; + +template <> +struct TraitsBase { + TraitsBase() = default; + TraitsBase(const TraitsBase&) = delete; + TraitsBase(TraitsBase&&) = default; + TraitsBase& operator=(const TraitsBase&) = delete; + TraitsBase& operator=(TraitsBase&&) = default; +}; + +template <> +struct TraitsBase { + TraitsBase() = default; + TraitsBase(const TraitsBase&) = delete; + TraitsBase(TraitsBase&&) = delete; + TraitsBase& operator=(const TraitsBase&) = delete; + TraitsBase& operator=(TraitsBase&&) = delete; +}; + +} // namespace internal_statusor +} // namespace port +} // namespace stream_executor + +#endif // TENSORFLOW_STREAM_EXECUTOR_LIB_STATUSOR_INTERNALS_H_ diff --git a/tensorflow/stream_executor/lib/statusor_test.cc b/tensorflow/stream_executor/lib/statusor_test.cc new file mode 100644 index 0000000000..56584e1892 --- /dev/null +++ b/tensorflow/stream_executor/lib/statusor_test.cc @@ -0,0 +1,676 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Unit tests for StatusOr + +#include "tensorflow/stream_executor/lib/statusor.h" + +#include +#include + +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/test_benchmark.h" + +namespace stream_executor { +namespace port { +namespace { + +class Base1 { + public: + virtual ~Base1() {} + int pad_; +}; + +class Base2 { + public: + virtual ~Base2() {} + int yetotherpad_; +}; + +class Derived : public Base1, public Base2 { + public: + ~Derived() override {} + int evenmorepad_; +}; + +class CopyNoAssign { + public: + explicit CopyNoAssign(int value) : foo_(value) {} + CopyNoAssign(const CopyNoAssign& other) : foo_(other.foo_) {} + int foo_; + + private: + const CopyNoAssign& operator=(const CopyNoAssign&); +}; + +class NoDefaultConstructor { + public: + explicit NoDefaultConstructor(int foo); +}; + +static_assert(!std::is_default_constructible(), + "Should not be default-constructible."); + +StatusOr> ReturnUniquePtr() { + // Uses implicit constructor from T&& + return std::unique_ptr(new int(0)); +} + +TEST(StatusOr, ElementType) { + static_assert(std::is_same::element_type, int>(), ""); + static_assert(std::is_same::element_type, char>(), ""); +} + +TEST(StatusOr, NullPointerStatusOr) { + // As a very special case, null-plain-pointer StatusOr used to be an + // error. Test that it no longer is. + StatusOr null_status(nullptr); + EXPECT_TRUE(null_status.ok()); + EXPECT_EQ(null_status.ValueOrDie(), nullptr); +} + +TEST(StatusOr, TestNoDefaultConstructorInitialization) { + // Explicitly initialize it with an error code. + StatusOr statusor(tensorflow::errors::Cancelled("")); + EXPECT_FALSE(statusor.ok()); + EXPECT_EQ(statusor.status().code(), tensorflow::error::CANCELLED); + + // Default construction of StatusOr initializes it with an UNKNOWN error code. + StatusOr statusor2; + EXPECT_FALSE(statusor2.ok()); + EXPECT_EQ(statusor2.status().code(), tensorflow::error::UNKNOWN); +} + +TEST(StatusOr, TestMoveOnlyInitialization) { + StatusOr> thing(ReturnUniquePtr()); + ASSERT_TRUE(thing.ok()); + EXPECT_EQ(0, *thing.ValueOrDie()); + int* previous = thing.ValueOrDie().get(); + + thing = ReturnUniquePtr(); + EXPECT_TRUE(thing.ok()); + EXPECT_EQ(0, *thing.ValueOrDie()); + EXPECT_NE(previous, thing.ValueOrDie().get()); +} + +TEST(StatusOr, TestMoveOnlyStatusCtr) { + StatusOr> thing(tensorflow::errors::Cancelled("")); + ASSERT_FALSE(thing.ok()); +} + +TEST(StatusOr, TestMoveOnlyValueExtraction) { + StatusOr> thing(ReturnUniquePtr()); + ASSERT_TRUE(thing.ok()); + std::unique_ptr ptr = thing.ConsumeValueOrDie(); + EXPECT_EQ(0, *ptr); + + thing = std::move(ptr); + ptr = std::move(thing.ValueOrDie()); + EXPECT_EQ(0, *ptr); +} + +TEST(StatusOr, TestMoveOnlyConversion) { + StatusOr> const_thing(ReturnUniquePtr()); + EXPECT_TRUE(const_thing.ok()); + EXPECT_EQ(0, *const_thing.ValueOrDie()); + + // Test rvalue converting assignment + const int* const_previous = const_thing.ValueOrDie().get(); + const_thing = ReturnUniquePtr(); + EXPECT_TRUE(const_thing.ok()); + EXPECT_EQ(0, *const_thing.ValueOrDie()); + EXPECT_NE(const_previous, const_thing.ValueOrDie().get()); +} + +TEST(StatusOr, TestMoveOnlyVector) { + // Sanity check that StatusOr works in vector. + std::vector>> vec; + vec.push_back(ReturnUniquePtr()); + vec.resize(2); + auto another_vec = std::move(vec); + EXPECT_EQ(0, *another_vec[0].ValueOrDie()); + EXPECT_EQ(tensorflow::error::UNKNOWN, another_vec[1].status().code()); +} + +TEST(StatusOr, TestMoveWithValuesAndErrors) { + StatusOr status_or(string(1000, '0')); + StatusOr value1(string(1000, '1')); + StatusOr value2(string(1000, '2')); + StatusOr error1(Status(tensorflow::error::UNKNOWN, "error1")); + StatusOr error2(Status(tensorflow::error::UNKNOWN, "error2")); + + ASSERT_TRUE(status_or.ok()); + EXPECT_EQ(string(1000, '0'), status_or.ValueOrDie()); + + // Overwrite the value in status_or with another value. + status_or = std::move(value1); + ASSERT_TRUE(status_or.ok()); + EXPECT_EQ(string(1000, '1'), status_or.ValueOrDie()); + + // Overwrite the value in status_or with an error. + status_or = std::move(error1); + ASSERT_FALSE(status_or.ok()); + EXPECT_EQ("error1", status_or.status().error_message()); + + // Overwrite the error in status_or with another error. + status_or = std::move(error2); + ASSERT_FALSE(status_or.ok()); + EXPECT_EQ("error2", status_or.status().error_message()); + + // Overwrite the error with a value. + status_or = std::move(value2); + ASSERT_TRUE(status_or.ok()); + EXPECT_EQ(string(1000, '2'), status_or.ValueOrDie()); +} + +TEST(StatusOr, TestCopyWithValuesAndErrors) { + StatusOr status_or(string(1000, '0')); + StatusOr value1(string(1000, '1')); + StatusOr value2(string(1000, '2')); + StatusOr error1(Status(tensorflow::error::UNKNOWN, "error1")); + StatusOr error2(Status(tensorflow::error::UNKNOWN, "error2")); + + ASSERT_TRUE(status_or.ok()); + EXPECT_EQ(string(1000, '0'), status_or.ValueOrDie()); + + // Overwrite the value in status_or with another value. + status_or = value1; + ASSERT_TRUE(status_or.ok()); + EXPECT_EQ(string(1000, '1'), status_or.ValueOrDie()); + + // Overwrite the value in status_or with an error. + status_or = error1; + ASSERT_FALSE(status_or.ok()); + EXPECT_EQ("error1", status_or.status().error_message()); + + // Overwrite the error in status_or with another error. + status_or = error2; + ASSERT_FALSE(status_or.ok()); + EXPECT_EQ("error2", status_or.status().error_message()); + + // Overwrite the error with a value. + status_or = value2; + ASSERT_TRUE(status_or.ok()); + EXPECT_EQ(string(1000, '2'), status_or.ValueOrDie()); + + // Verify original values unchanged. + EXPECT_EQ(string(1000, '1'), value1.ValueOrDie()); + EXPECT_EQ("error1", error1.status().error_message()); + EXPECT_EQ("error2", error2.status().error_message()); + EXPECT_EQ(string(1000, '2'), value2.ValueOrDie()); +} + +TEST(StatusOr, TestDefaultCtor) { + StatusOr thing; + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(thing.status().code(), tensorflow::error::UNKNOWN); +} + +TEST(StatusOrDeathTest, TestDefaultCtorValue) { + StatusOr thing; + EXPECT_DEATH(thing.ValueOrDie(), ""); + + const StatusOr thing2; + EXPECT_DEATH(thing.ValueOrDie(), ""); +} + +TEST(StatusOr, TestStatusCtor) { + StatusOr thing(Status(tensorflow::error::CANCELLED, "")); + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(thing.status().code(), tensorflow::error::CANCELLED); +} + +TEST(StatusOr, TestValueCtor) { + const int kI = 4; + const StatusOr thing(kI); + EXPECT_TRUE(thing.ok()); + EXPECT_EQ(kI, thing.ValueOrDie()); +} + +TEST(StatusOr, TestCopyCtorStatusOk) { + const int kI = 4; + const StatusOr original(kI); + const StatusOr copy(original); + EXPECT_EQ(copy.status(), original.status()); + EXPECT_EQ(original.ValueOrDie(), copy.ValueOrDie()); +} + +TEST(StatusOr, TestCopyCtorStatusNotOk) { + StatusOr original(Status(tensorflow::error::CANCELLED, "")); + StatusOr copy(original); + EXPECT_EQ(copy.status(), original.status()); +} + +TEST(StatusOr, TestCopyCtorNonAssignable) { + const int kI = 4; + CopyNoAssign value(kI); + StatusOr original(value); + StatusOr copy(original); + EXPECT_EQ(copy.status(), original.status()); + EXPECT_EQ(original.ValueOrDie().foo_, copy.ValueOrDie().foo_); +} + +TEST(StatusOr, TestCopyCtorStatusOKConverting) { + const int kI = 4; + StatusOr original(kI); + StatusOr copy(original); + EXPECT_EQ(copy.status(), original.status()); + EXPECT_DOUBLE_EQ(original.ValueOrDie(), copy.ValueOrDie()); +} + +TEST(StatusOr, TestCopyCtorStatusNotOkConverting) { + StatusOr original(Status(tensorflow::error::CANCELLED, "")); + StatusOr copy(original); + EXPECT_EQ(copy.status(), original.status()); +} + +TEST(StatusOr, TestAssignmentStatusOk) { + const int kI = 4; + StatusOr source(kI); + StatusOr target; + target = source; + EXPECT_EQ(target.status(), source.status()); + EXPECT_EQ(source.ValueOrDie(), target.ValueOrDie()); +} + +TEST(StatusOr, TestAssignmentStatusNotOk) { + StatusOr source(Status(tensorflow::error::CANCELLED, "")); + StatusOr target; + target = source; + EXPECT_EQ(target.status(), source.status()); +} + +TEST(StatusOr, TestStatus) { + StatusOr good(4); + EXPECT_TRUE(good.ok()); + StatusOr bad(Status(tensorflow::error::CANCELLED, "")); + EXPECT_FALSE(bad.ok()); + EXPECT_EQ(bad.status(), Status(tensorflow::error::CANCELLED, "")); +} + +TEST(StatusOr, TestValue) { + const int kI = 4; + StatusOr thing(kI); + EXPECT_EQ(kI, thing.ValueOrDie()); +} + +TEST(StatusOr, TestValueConst) { + const int kI = 4; + const StatusOr thing(kI); + EXPECT_EQ(kI, thing.ValueOrDie()); +} + +TEST(StatusOrDeathTest, TestValueNotOk) { + StatusOr thing(Status(tensorflow::error::CANCELLED, "cancelled")); + EXPECT_DEATH(thing.ValueOrDie(), "cancelled"); +} + +TEST(StatusOrDeathTest, TestValueNotOkConst) { + const StatusOr thing(Status(tensorflow::error::UNKNOWN, "")); + EXPECT_DEATH(thing.ValueOrDie(), ""); +} + +TEST(StatusOr, TestPointerDefaultCtor) { + StatusOr thing; + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(thing.status().code(), tensorflow::error::UNKNOWN); +} + +TEST(StatusOrDeathTest, TestPointerDefaultCtorValue) { + StatusOr thing; + EXPECT_DEATH(thing.ValueOrDie(), ""); +} + +TEST(StatusOr, TestPointerStatusCtor) { + StatusOr thing(Status(tensorflow::error::CANCELLED, "")); + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(thing.status(), Status(tensorflow::error::CANCELLED, "")); +} + +TEST(StatusOr, TestPointerValueCtor) { + const int kI = 4; + StatusOr thing(&kI); + EXPECT_TRUE(thing.ok()); + EXPECT_EQ(&kI, thing.ValueOrDie()); +} + +TEST(StatusOr, TestPointerCopyCtorStatusOk) { + const int kI = 0; + StatusOr original(&kI); + StatusOr copy(original); + EXPECT_EQ(copy.status(), original.status()); + EXPECT_EQ(original.ValueOrDie(), copy.ValueOrDie()); +} + +TEST(StatusOr, TestPointerCopyCtorStatusNotOk) { + StatusOr original(Status(tensorflow::error::CANCELLED, "")); + StatusOr copy(original); + EXPECT_EQ(copy.status(), original.status()); +} + +TEST(StatusOr, TestPointerCopyCtorStatusOKConverting) { + Derived derived; + StatusOr original(&derived); + StatusOr copy(original); + EXPECT_EQ(copy.status(), original.status()); + EXPECT_EQ(static_cast(original.ValueOrDie()), + copy.ValueOrDie()); +} + +TEST(StatusOr, TestPointerCopyCtorStatusNotOkConverting) { + StatusOr original(Status(tensorflow::error::CANCELLED, "")); + StatusOr copy(original); + EXPECT_EQ(copy.status(), original.status()); +} + +TEST(StatusOr, TestPointerAssignmentStatusOk) { + const int kI = 0; + StatusOr source(&kI); + StatusOr target; + target = source; + EXPECT_EQ(target.status(), source.status()); + EXPECT_EQ(source.ValueOrDie(), target.ValueOrDie()); +} + +TEST(StatusOr, TestPointerAssignmentStatusNotOk) { + StatusOr source(Status(tensorflow::error::CANCELLED, "")); + StatusOr target; + target = source; + EXPECT_EQ(target.status(), source.status()); +} + +TEST(StatusOr, TestPointerStatus) { + const int kI = 0; + StatusOr good(&kI); + EXPECT_TRUE(good.ok()); + StatusOr bad(Status(tensorflow::error::CANCELLED, "")); + EXPECT_EQ(bad.status(), Status(tensorflow::error::CANCELLED, "")); +} + +TEST(StatusOr, TestPointerValue) { + const int kI = 0; + StatusOr thing(&kI); + EXPECT_EQ(&kI, thing.ValueOrDie()); +} + +TEST(StatusOr, TestPointerValueConst) { + const int kI = 0; + const StatusOr thing(&kI); + EXPECT_EQ(&kI, thing.ValueOrDie()); +} + +// NOTE(tucker): StatusOr does not support this kind +// of resize op. +// TEST(StatusOr, StatusOrVectorOfUniquePointerCanResize) { +// using EvilType = std::vector>; +// static_assert(std::is_copy_constructible::value, ""); +// std::vector> v(5); +// v.reserve(v.capacity() + 10); +// } + +TEST(StatusOrDeathTest, TestPointerValueNotOk) { + StatusOr thing(Status(tensorflow::error::CANCELLED, "cancelled")); + EXPECT_DEATH(thing.ValueOrDie(), "cancelled"); +} + +TEST(StatusOrDeathTest, TestPointerValueNotOkConst) { + const StatusOr thing(Status(tensorflow::error::CANCELLED, "cancelled")); + EXPECT_DEATH(thing.ValueOrDie(), "cancelled"); +} + +static StatusOr MakeStatus() { return 100; } +// A factory to help us benchmark the various factory styles. All of +// the factory methods are marked as non-inlineable so as to more +// accurately simulate calling a factory for which you do not have +// visibility of implementation. Similarly, the value_ variable is +// marked volatile to prevent the compiler from getting too clever +// about detecting that the same value is used in all loop iterations. +template +class BenchmarkFactory { + public: + // Construct a new factory. Allocate an object which will always + // be the result of the factory methods. + BenchmarkFactory() : value_(new T) {} + + // Destroy this factory, including the result value. + ~BenchmarkFactory() { delete value_; } + + // A trivial factory that just returns the value. There is no status + // object that could be returned to encapsulate an error + T* TrivialFactory() TF_ATTRIBUTE_NOINLINE { return value_; } + + // A more sophisticated factory, which returns a status to indicate + // the result of the operation. The factory result is populated into + // the user provided pointer result. + Status ArgumentFactory(T** result) TF_ATTRIBUTE_NOINLINE { + *result = value_; + return Status::OK(); + } + + Status ArgumentFactoryFail(T** result) TF_ATTRIBUTE_NOINLINE { + *result = nullptr; + return Status(tensorflow::error::CANCELLED, ""); + } + + Status ArgumentFactoryFailShortMsg(T** result) TF_ATTRIBUTE_NOINLINE { + *result = nullptr; + return Status(::tensorflow::error::INTERNAL, ""); + } + + Status ArgumentFactoryFailLongMsg(T** result) TF_ATTRIBUTE_NOINLINE { + *result = nullptr; + return Status(::tensorflow::error::INTERNAL, + "a big string of message junk that will never be read"); + } + + // A factory that returns a StatusOr. If the factory operation + // is OK, then the StatusOr will hold a T*. Otherwise, it will + // hold a status explaining the error. + StatusOr StatusOrFactory() TF_ATTRIBUTE_NOINLINE { + return static_cast(value_); + } + + StatusOr StatusOrFactoryFail() TF_ATTRIBUTE_NOINLINE { + return Status(tensorflow::error::CANCELLED, ""); + } + + StatusOr StatusOrFactoryFailShortMsg() TF_ATTRIBUTE_NOINLINE { + return Status(::tensorflow::error::INTERNAL, ""); + } + + StatusOr StatusOrFactoryFailLongMsg() TF_ATTRIBUTE_NOINLINE { + return Status(::tensorflow::error::INTERNAL, + "a big string of message junk that will never be read"); + } + + private: + T* volatile value_; + TF_DISALLOW_COPY_AND_ASSIGN(BenchmarkFactory); +}; + +// A simple type we use with the factory. +class BenchmarkType { + public: + BenchmarkType() {} + virtual ~BenchmarkType() {} + virtual void DoWork() TF_ATTRIBUTE_NOINLINE {} + + private: + TF_DISALLOW_COPY_AND_ASSIGN(BenchmarkType); +}; + +// Calibrate the amount of time spent just calling DoWork, since each of our +// tests will do this, we can subtract this out of benchmark results. +void BM_CalibrateWorkLoop(int iters) { + tensorflow::testing::StopTiming(); + BenchmarkFactory factory; + BenchmarkType* result = factory.TrivialFactory(); + tensorflow::testing::StartTiming(); + for (int i = 0; i != iters; ++i) { + if (result != nullptr) { + result->DoWork(); + } + } +} +BENCHMARK(BM_CalibrateWorkLoop); + +// Measure the time taken to call into the factory, return the value, +// determine that it is OK, and invoke a trivial function. +void BM_TrivialFactory(int iters) { + tensorflow::testing::StopTiming(); + BenchmarkFactory factory; + tensorflow::testing::StartTiming(); + for (int i = 0; i != iters; ++i) { + BenchmarkType* result = factory.TrivialFactory(); + if (result != nullptr) { + result->DoWork(); + } + } +} +BENCHMARK(BM_TrivialFactory); + +// Measure the time taken to call into the factory, providing an +// out-param for the result, evaluating the status result and the +// result pointer, and invoking the trivial function. +void BM_ArgumentFactory(int iters) { + tensorflow::testing::StopTiming(); + BenchmarkFactory factory; + tensorflow::testing::StartTiming(); + for (int i = 0; i != iters; ++i) { + BenchmarkType* result = nullptr; + Status status = factory.ArgumentFactory(&result); + if (status.ok() && result != nullptr) { + result->DoWork(); + } + } +} +BENCHMARK(BM_ArgumentFactory); + +// Measure the time to use the StatusOr factory, evaluate the result, +// and invoke the trivial function. +void BM_StatusOrFactory(int iters) { + tensorflow::testing::StopTiming(); + BenchmarkFactory factory; + tensorflow::testing::StartTiming(); + for (int i = 0; i != iters; ++i) { + StatusOr result = factory.StatusOrFactory(); + if (result.ok()) { + result.ValueOrDie()->DoWork(); + } + } +} +BENCHMARK(BM_StatusOrFactory); + +// Measure the time taken to call into the factory, providing an +// out-param for the result, evaluating the status result and the +// result pointer, and invoking the trivial function. +void BM_ArgumentFactoryFail(int iters) { + tensorflow::testing::StopTiming(); + BenchmarkFactory factory; + tensorflow::testing::StartTiming(); + for (int i = 0; i != iters; ++i) { + BenchmarkType* result = nullptr; + Status status = factory.ArgumentFactoryFail(&result); + if (status.ok() && result != nullptr) { + result->DoWork(); + } + } +} +BENCHMARK(BM_ArgumentFactoryFail); + +// Measure the time to use the StatusOr factory, evaluate the result, +// and invoke the trivial function. +void BM_StatusOrFactoryFail(int iters) { + tensorflow::testing::StopTiming(); + BenchmarkFactory factory; + tensorflow::testing::StartTiming(); + for (int i = 0; i != iters; ++i) { + StatusOr result = factory.StatusOrFactoryFail(); + if (result.ok()) { + result.ValueOrDie()->DoWork(); + } + } +} +BENCHMARK(BM_StatusOrFactoryFail); + +// Measure the time taken to call into the factory, providing an +// out-param for the result, evaluating the status result and the +// result pointer, and invoking the trivial function. +void BM_ArgumentFactoryFailShortMsg(int iters) { + tensorflow::testing::StopTiming(); + BenchmarkFactory factory; + tensorflow::testing::StartTiming(); + for (int i = 0; i != iters; ++i) { + BenchmarkType* result = nullptr; + Status status = factory.ArgumentFactoryFailShortMsg(&result); + if (status.ok() && result != nullptr) { + result->DoWork(); + } + } +} +BENCHMARK(BM_ArgumentFactoryFailShortMsg); + +// Measure the time to use the StatusOr factory, evaluate the result, +// and invoke the trivial function. +void BM_StatusOrFactoryFailShortMsg(int iters) { + tensorflow::testing::StopTiming(); + BenchmarkFactory factory; + tensorflow::testing::StartTiming(); + for (int i = 0; i != iters; ++i) { + StatusOr result = factory.StatusOrFactoryFailShortMsg(); + if (result.ok()) { + result.ValueOrDie()->DoWork(); + } + } +} +BENCHMARK(BM_StatusOrFactoryFailShortMsg); + +// Measure the time taken to call into the factory, providing an +// out-param for the result, evaluating the status result and the +// result pointer, and invoking the trivial function. +void BM_ArgumentFactoryFailLongMsg(int iters) { + tensorflow::testing::StopTiming(); + BenchmarkFactory factory; + tensorflow::testing::StartTiming(); + for (int i = 0; i != iters; ++i) { + BenchmarkType* result = nullptr; + Status status = factory.ArgumentFactoryFailLongMsg(&result); + if (status.ok() && result != nullptr) { + result->DoWork(); + } + } +} +BENCHMARK(BM_ArgumentFactoryFailLongMsg); + +// Measure the time to use the StatusOr factory, evaluate the result, +// and invoke the trivial function. +void BM_StatusOrFactoryFailLongMsg(int iters) { + tensorflow::testing::StopTiming(); + BenchmarkFactory factory; + tensorflow::testing::StartTiming(); + for (int i = 0; i != iters; ++i) { + StatusOr result = factory.StatusOrFactoryFailLongMsg(); + if (result.ok()) { + result.ValueOrDie()->DoWork(); + } + } +} +BENCHMARK(BM_StatusOrFactoryFailLongMsg); + +} // namespace +} // namespace port +} // namespace stream_executor -- cgit v1.2.3 From b8c1732664f41d5af2587e2f093880a3a7d83f43 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Tue, 26 Jun 2018 15:07:23 -0700 Subject: Fix small typo in RELEASE.md --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 879b995a5a..52cd9ef72b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -21,7 +21,7 @@ * The [distributions.Bijector](https://www.tensorflow.org/versions/r1.9/api_docs/python/tf/contrib/distributions/bijectors/Bijector) API supports broadcasting for Bijectors with new API changes. -## Breaking Chances +## Breaking Changes * If you're opening empty variable scopes; replace `variable_scope('', ...)` by `variable_scope(tf.get_variable_scope(), ...)`. -- cgit v1.2.3 From 74ca837950536aaef358abf3e05b31b4d62248f7 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Tue, 26 Jun 2018 15:29:48 -0700 Subject: Update eigen version to a fixed version for ppc64. --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 3c657c4a5b..79274d66ad 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -107,11 +107,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): tf_http_archive( name = "eigen_archive", urls = [ - "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/e5e305a158a0.tar.gz", - "https://bitbucket.org/eigen/eigen/get/e5e305a158a0.tar.gz", + "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/fd6845384b86.tar.gz", + "https://bitbucket.org/eigen/eigen/get/fd6845384b86.tar.gz", ], - sha256 = "8bbe676d69e7f59070c83a949454b8b6344034e0ebbf686b337528e5dc04c7de", - strip_prefix = "eigen-eigen-e5e305a158a0", + sha256 = "d956415d784fa4e42b6a2a45c32556d6aec9d0a3d8ef48baee2522ab762556a9", + strip_prefix = "eigen-eigen-fd6845384b86", build_file = clean_dep("//third_party:eigen.BUILD"), ) -- cgit v1.2.3 From 388a267b1191adf2df4006bf205a19b8a24813db Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Tue, 26 Jun 2018 11:24:37 -0700 Subject: Remove section links that don't go anywhere. --- tensorflow/docs_src/get_started/_index.yaml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tensorflow/docs_src/get_started/_index.yaml b/tensorflow/docs_src/get_started/_index.yaml index 277fc852fb..4060804892 100644 --- a/tensorflow/docs_src/get_started/_index.yaml +++ b/tensorflow/docs_src/get_started/_index.yaml @@ -66,9 +66,7 @@ landing_page: }

- -

Learn and use ML

- +

Learn and use ML

The high-level Keras API provides building blocks to create and @@ -117,9 +115,7 @@ landing_page: - items: - custom_html: >

- -

Research and experimentation

- +

Research and experimentation

Eager execution provides an imperative, define-by-run interface for advanced operations. Write custom layers, forward passes, and training loops with auto‑differentiation. Start with @@ -170,9 +166,7 @@ landing_page:

- custom_html: >
- -

ML at production scale

- +

ML at production scale

Estimators can train large models on multiple machines in a -- cgit v1.2.3 From 9f4fbdb05e35b512dd4a3da5ae80558021a291e5 Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Tue, 26 Jun 2018 14:56:57 -0700 Subject: Fix leftnav for get_started --- tensorflow/docs_src/get_started/leftnav_files | 6 +++--- tensorflow/docs_src/get_started/next_steps.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/docs_src/get_started/leftnav_files b/tensorflow/docs_src/get_started/leftnav_files index 5c400a67f0..99d2b2c3e1 100644 --- a/tensorflow/docs_src/get_started/leftnav_files +++ b/tensorflow/docs_src/get_started/leftnav_files @@ -1,7 +1,7 @@ ### Learn and use ML -basic_classification.md -basic_text_classification.md -basic_regression.md +basic_classification.md: Basic classification +basic_text_classification.md: Text classification +basic_regression.md: Regression overfit_and_underfit.md save_and_restore_models.md next_steps.md diff --git a/tensorflow/docs_src/get_started/next_steps.md b/tensorflow/docs_src/get_started/next_steps.md index 6318a39c6c..01c9f7204a 100644 --- a/tensorflow/docs_src/get_started/next_steps.md +++ b/tensorflow/docs_src/get_started/next_steps.md @@ -1,4 +1,4 @@ -# Next Steps +# Next steps ## Learn more about TensorFlow -- cgit v1.2.3 From 9809978ea09845d5429925b64c20cda461c20a66 Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Tue, 26 Jun 2018 21:34:07 -0700 Subject: Fix checkpoints link in keras guide --- tensorflow/docs_src/guide/keras.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/docs_src/guide/keras.md b/tensorflow/docs_src/guide/keras.md index 83172dab7f..c799e9b12c 100644 --- a/tensorflow/docs_src/guide/keras.md +++ b/tensorflow/docs_src/guide/keras.md @@ -35,7 +35,7 @@ from tensorflow import keras * The `tf.keras` version in the latest TensorFlow release might not be the same as the latest `keras` version from PyPI. Check `tf.keras.__version__`. * When [saving a model's weights](#weights_only), `tf.keras` defaults to the - [checkpoint format](../get_started/checkpoints.md). Pass `save_format='h5'` to + [checkpoint format](./checkpoints.md). Pass `save_format='h5'` to use HDF5. ## Build a simple model @@ -442,7 +442,7 @@ model.load_weights('my_model') ``` By default, this saves the model's weights in the -[TensorFlow checkpoint](../get_started/checkpoints.md) file format. Weights can +[TensorFlow checkpoint](./checkpoints.md) file format. Weights can also be saved to the Keras HDF5 format (the default for the multi-backend implementation of Keras): -- cgit v1.2.3 From 7e99fff9a4e516809abe506dcd85fa96070ce3bb Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Wed, 27 Jun 2018 13:26:46 +0800 Subject: make benchmark_model for tflite build --- tensorflow/contrib/lite/profiling/profile_summarizer.cc | 2 +- tensorflow/contrib/lite/tools/benchmark/benchmark_params.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/lite/profiling/profile_summarizer.cc b/tensorflow/contrib/lite/profiling/profile_summarizer.cc index c37a096588..36e87b666a 100644 --- a/tensorflow/contrib/lite/profiling/profile_summarizer.cc +++ b/tensorflow/contrib/lite/profiling/profile_summarizer.cc @@ -83,7 +83,7 @@ OperatorDetails GetOperatorDetails(const tflite::Interpreter& interpreter, OperatorDetails details; details.name = op_name; if (profiling_string) { - details.name += ":" + string(profiling_string); + details.name += ":" + std::string(profiling_string); } details.inputs = GetTensorNames(interpreter, inputs); details.outputs = GetTensorNames(interpreter, outputs); diff --git a/tensorflow/contrib/lite/tools/benchmark/benchmark_params.h b/tensorflow/contrib/lite/tools/benchmark/benchmark_params.h index 33448dd162..d9471145a9 100644 --- a/tensorflow/contrib/lite/tools/benchmark/benchmark_params.h +++ b/tensorflow/contrib/lite/tools/benchmark/benchmark_params.h @@ -46,11 +46,11 @@ class BenchmarkParam { } virtual ~BenchmarkParam() {} BenchmarkParam(ParamType type) : type_(type) {} + template + static ParamType GetValueType(); private: static void AssertHasSameType(ParamType a, ParamType b); - template - static ParamType GetValueType(); const ParamType type_; }; -- cgit v1.2.3 From 25804da02063df8d3836e8fcf197c9304379e4ae Mon Sep 17 00:00:00 2001 From: Jie Date: Mon, 25 Jun 2018 17:33:41 -0700 Subject: TRT 4.0 update code is compatible with TRT 3.0.4 updated feature support for TRT 4.0 layers added error checking for converter disabled broken shape inference (added TODO) --- .../contrib/tensorrt/convert/convert_graph.cc | 21 + .../contrib/tensorrt/convert/convert_nodes.cc | 975 +++++++++++++++------ .../contrib/tensorrt/kernels/trt_engine_op.cc | 11 + tensorflow/contrib/tensorrt/ops/trt_engine_op.cc | 5 +- 4 files changed, 738 insertions(+), 274 deletions(-) diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc index 13986127ba..9ae569ac78 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc @@ -104,6 +104,27 @@ bool IsTensorRTCandidate(const tensorflow::Node* node) { "DepthwiseConv2dNative", "FusedBatchNorm", "FusedBatchNormV2", + "Div", + "RealDiv", + "Rsqrt", + "Reciprocal", + "Exp", + "Log", + "Sqrt", + "Abs", + "Neg", +#if NV_TENSORRT_MAJOR > 3 + "MatMul", + "BatchMatMul", + "Softmax", + "Minimum", + "Maximum", + "TopKV2", + "Sum", + "Prod", + "Max", + "Min", +#endif // TODO(ben,jie): ... }; // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h) diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index 146b9c7344..22f92d38bf 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -53,6 +53,31 @@ limitations under the License. // would work! #define CHECK_EQ_TYPE(val1, val2) CHECK_EQ((int)val1, (int)val2) +#define TFTRT_RETURN_ERROR_IF_FALSE(ptr, node) \ + do { \ + if (ptr == false) { \ + return tensorflow::errors::Internal(string("TFTRT::") + __FUNCTION__ + \ + "failed to add TRT layer, at: " + \ + node); \ + } \ + } while (0) + +#define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \ + do { \ + if (ptr == nullptr) { \ + return tensorflow::errors::Internal(string("TFTRT::") + __FUNCTION__ + \ + "failed to add TRT layer, at: " + \ + node); \ + } \ + } while (0) + +#define TF_RETURN_IF_OK(status) \ + do { \ + if (status.ok()) { \ + return tensorflow::Status::OK(); \ + } \ + } while (0) + namespace tensorflow { namespace tensorrt { namespace convert { @@ -75,6 +100,11 @@ inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype, case tensorflow::DataType::DT_HALF: *trt_dtype = nvinfer1::DataType::kHALF; break; +#if NV_TENSORRT_MAJOR > 3 + case tensorflow::DataType::DT_INT32: + *trt_dtype = nvinfer1::DataType::kINT32; + break; +#endif default: return tensorflow::errors::InvalidArgument( "Unsupported data type " + tensorflow::DataTypeString(tf_dtype)); @@ -82,6 +112,96 @@ inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype, return tensorflow::Status::OK(); } +// return whether or not the broadcast is feasible; +bool TensorRTGetBroadcastShape(const nvinfer1::Dims& operand_l, + const bool operand_l_is_tensor, + const nvinfer1::Dims& operand_r, + const bool operand_r_is_tensor, + nvinfer1::Dims* operand_l_new_shape, + nvinfer1::Dims* operand_r_new_shape) { + /******************************************************************************* + TensorRT Elementwise op supports broadcast but requires both tensor to be of + Identical rank + + We consider case of: i. Tensor op Const; ii. Tensor op Tensor + note: const op const (constant folding) should fallback to TensorFlow + + broadcast scheme: + T: 1 3 5 (tensor would not have batch dimension) + W: 1 1 3 1 (weight would have all explicit dimensions) + i. fill in explicit dimensions + -> T: -1 1 3 5 (we put a -1 for batch dimension) + -> W: 1 1 3 1 + ii. compare broadcast feasibility + + we cannot support these since TensorRT does not allow manipulation on batch + dimension, we cannot generate output with proper shape + T: 3 5 1 + W: 1 1 1 1 3 5 1 + -> T: 1 1 1 -1 3 5 1 + -> W: 1 1 1 1 3 5 1 + *******************************************************************************/ + static const int max_nb_dims = nvinfer1::Dims::MAX_DIMS + 1; + const size_t element_size = sizeof(operand_l.d[0]); + + // fill in dimensions + int l_s[max_nb_dims]; + std::fill(l_s, l_s + max_nb_dims, 1); + int l_d = operand_l_is_tensor ? operand_l.nbDims + 1 : operand_l.nbDims; + int r_s[max_nb_dims]; + std::fill(r_s, r_s + max_nb_dims, 1); + int r_d = operand_r_is_tensor ? operand_r.nbDims + 1 : operand_r.nbDims; + + int max_d = std::max(l_d, r_d); + std::memcpy(l_s + max_d - operand_l.nbDims, operand_l.d, + operand_l.nbDims * element_size); + std::memcpy(r_s + max_d - operand_r.nbDims, operand_r.d, + operand_r.nbDims * element_size); + + // set -1 for batch dimension, since batch size is not supposed to be + // broadcasted + if (operand_l_is_tensor) { + if (max_d != l_d) { // if broadcast beyond batch dimension, fail + return false; + } + l_s[0] = -1; + } + if (operand_r_is_tensor) { + if (max_d != r_d) { // if broadcast beyond batch dimension, fail + return false; + } + r_s[0] = -1; + } + + // compare broadcast feasibility + for (int i = max_d - 1; i >= 0; i--) { + if ((l_s[i] != r_s[i]) && (l_s[i] != 1) && (r_s[i] != 1)) { + return false; + } + } + + // output new TensorRT Dimension (stripping the batch dimension) + operand_l_new_shape->nbDims = max_d - 1; + std::memcpy(operand_l_new_shape->d, l_s + 1, (max_d - 1) * element_size); + operand_r_new_shape->nbDims = max_d - 1; + std::memcpy(operand_r_new_shape->d, r_s + 1, (max_d - 1) * element_size); + + return true; +} + +inline bool DimsEqual(const nvinfer1::Dims& dim_l, + const nvinfer1::Dims& dim_r) { + if (dim_l.nbDims != dim_r.nbDims) { + return false; + } + for (int i = 0; i < dim_l.nbDims; i++) { + if (dim_l.d[i] != dim_r.d[i]) { + return false; + } + } + return true; +} + inline nvinfer1::Dims GetTensorShape(const tensorflow::Tensor& tensor) { nvinfer1::Dims dims; dims.nbDims = tensor.dims(); @@ -342,7 +462,7 @@ void Reorder2(nvinfer1::DimsHW shape, const T* idata, nvinfer1::DimsHW istrides, for (int h = 0; h < shape.h(); ++h) { for (int w = 0; w < shape.w(); ++w) { odata[h * ostrides.h() + w * ostrides.w()] = - idata[h * ostrides.h() + w * ostrides.w()]; + idata[h * istrides.h() + w * istrides.w()]; } } } @@ -365,11 +485,10 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights, break; } case tensorflow::DataType::DT_HALF: { - Reorder2( - {k, c}, static_cast(iweights.GetValues()), - istrides, - static_cast(const_cast(oweights->GetValues())), - ostrides); + Reorder2({k, c}, static_cast(iweights.GetValues()), + istrides, static_cast( + const_cast(oweights->GetValues())), + ostrides); break; } default: @@ -547,6 +666,9 @@ class Converter { LOG(ERROR) << "Dimension does not match, fail gracefully"; nvinfer1::IShuffleLayer* layer = this->network()->addShuffle(*input_tensor); + if (layer == nullptr) { + return nullptr; + } nvinfer1::Permutation permutation; for (int32_t i = 0; i < dims.nbDims; ++i) { permutation.order[i] = order[i + 1] - 1; @@ -583,7 +705,7 @@ TRT_ShapedWeights ConvertFP32ToFP16(Converter& ctx, // there. //*****************************************************************************/ struct LambdaFactory { - enum class OP_CATEGORY : int { RSQRT = 0, NEG, ADD, MUL, SUB }; + enum class OP_CATEGORY : int { RSQRT = 0, NEG, ADD, MUL, SUB, RECIP }; OP_CATEGORY op; template @@ -595,6 +717,8 @@ struct LambdaFactory { } case OP_CATEGORY::NEG: return [](T t) -> T { return -t; }; + case OP_CATEGORY::RECIP: + return [](T t) -> T { return 1.0 / t; }; default: VLOG(2) << "Not supported op for unary: " << static_cast(op); return nullptr; @@ -790,118 +914,23 @@ tensorflow::Status BinaryCompute(const TRT_ShapedWeights& iweights_l, return tensorflow::Status::OK(); } -tensorflow::Status ConstantFoldUnary( - Converter& ctx, const tensorflow::NodeDef& node_def, - const std::vector& inputs, - std::vector* outputs) { - TRT_ShapedWeights weights_input = inputs.at(0).weights(); - - // Allocate output weights - TRT_ShapedWeights weights_output = ctx.get_temp_weights_like(weights_input); - - // FIXME assume type matches input weights - // Get trt type & shape - // Maybe this part has to be moved into the block of rsqrt later - // Check type consistency - CHECK_EQ(weights_input.type_, - TFAttrs(node_def).get("T")); - - LambdaFactory unary_op; - if (node_def.op() == "Rsqrt") { - // Compute rsqrt - unary_op.op = LambdaFactory::OP_CATEGORY::RSQRT; - auto ret = UnaryCompute(weights_input, &weights_output, unary_op); - // Pass the output - if (ret == tensorflow::Status::OK()) { - outputs->push_back(TRT_TensorOrWeights(weights_output)); - } - return ret; - } else { - return tensorflow::errors::Unimplemented("Binary op not supported: " + - node_def.op()); - } -} - -// TODO(jie,ben) broadcast is needed yet not implemented -// Let's get the simple stuff working first. Maybe we should fall back to TF -// approach for constant folding -tensorflow::Status ConstantFoldBinary( - Converter& ctx, const tensorflow::NodeDef& node_def, - const std::vector& inputs, - std::vector* outputs) { - TRT_ShapedWeights weights_input_l = inputs.at(0).weights(); - TRT_ShapedWeights weights_input_r = inputs.at(1).weights(); - - // Check type consistency - CHECK_EQ(weights_input_l.type_, weights_input_r.type_); - - if (weights_input_l.shape_.nbDims != weights_input_r.shape_.nbDims) - return tensorflow::errors::Unimplemented( - "Binary op implicit broadcast not supported: " + node_def.op()); - - // TODO(jie): constant fold should really fall back to TF. - int num_dims = weights_input_l.shape_.nbDims; - nvinfer1::Dims output_shape; - output_shape.nbDims = num_dims; - VLOG(2) << "nb_dims: " << num_dims - << ", the other: " << weights_input_r.shape_.nbDims; - for (int i = 0; i < num_dims; i++) { - if (weights_input_l.shape_.d[i] == weights_input_r.shape_.d[i]) { - output_shape.d[i] = weights_input_l.shape_.d[i]; - } else if (weights_input_l.shape_.d[i] == 1 || - weights_input_r.shape_.d[i] == 1) { - output_shape.d[i] = - std::max(weights_input_l.shape_.d[i], weights_input_r.shape_.d[i]); - } else { - return tensorflow::errors::Unimplemented( - "Binary op with incompatible shape at, " + node_def.op()); - } - VLOG(2) << "left: " << weights_input_l.shape_.d[i] - << "right: " << weights_input_r.shape_.d[i] - << "output: " << output_shape.d[i]; - } - - // FIXME assume type matches input weights - // Get trt type & shape - TFAttrs attrs(node_def); - // Maybe this part has to be moved into the block of rsqrt later - tensorflow::DataType dtype = attrs.get("T"); - - // Allocate output weights - TRT_ShapedWeights weights_output = ctx.get_temp_weights(dtype, output_shape); - - LambdaFactory binary_op; - if (node_def.op() == "Sub") { - binary_op.op = LambdaFactory::OP_CATEGORY::SUB; - } else if (node_def.op() == "Mul") { - binary_op.op = LambdaFactory::OP_CATEGORY::MUL; - } else if (node_def.op() == "Add") { - binary_op.op = LambdaFactory::OP_CATEGORY::ADD; - } else { - return tensorflow::errors::Unimplemented("Binary op not supported: " + - node_def.op()); - } - auto ret = BinaryCompute(weights_input_l, weights_input_r, &weights_output, - binary_op); - - // Pass the output - if (ret == tensorflow::Status::OK()) { - outputs->push_back(TRT_TensorOrWeights(weights_output)); - } - - return ret; -} - // TODO(jie): broadcast is needed yet not implemented. // Only implemented channel wise for the time being tensorflow::Status BinaryTensorOpWeight( Converter& ctx, const tensorflow::NodeDef& node_def, const nvinfer1::ITensor* tensor, TRT_ShapedWeights weights, - std::vector* outputs) { + std::vector* outputs, bool swapped_inputs) { // FIXME assume type matches input weights // Get trt type & shape // Maybe this part has to be moved into the block of rsqrt later + if (node_def.op() != "Sub" && node_def.op() != "Add" && + node_def.op() != "Mul" && node_def.op() != "Div" && + node_def.op() != "RealDiv") { + return tensorflow::errors::Unimplemented( + "op not supported: " + node_def.op() + ", at: " + node_def.name()); + } + // Check type consistency nvinfer1::DataType ttype; TF_RETURN_IF_ERROR(ConvertDType(weights.type_, &ttype)); @@ -910,6 +939,12 @@ tensorflow::Status BinaryTensorOpWeight( auto dims_w = weights.shape_; auto dims_t = tensor->getDimensions(); + // TODO(jie): addScale checks for input tensor dimension + if (dims_t.nbDims != 3) { + return tensorflow::errors::InvalidArgument( + "addScale requires tensor with rank 3, " + node_def.name()); + } + // default to element-wise auto scale_mode = nvinfer1::ScaleMode::kELEMENTWISE; @@ -980,6 +1015,7 @@ tensorflow::Status BinaryTensorOpWeight( permutation[dims_t.nbDims] = 1; tensor = ctx.TransposeTensor(const_cast(tensor), permutation); + TFTRT_RETURN_ERROR_IF_NULLPTR(tensor, node_def.name()); } else { return tensorflow::errors::InvalidArgument( "Transpose cannot be applied, " + node_def.name()); @@ -997,11 +1033,35 @@ tensorflow::Status BinaryTensorOpWeight( // Maybe I should do a switch if (node_def.op() == "Sub") { - TRT_ShapedWeights neg_weights = ctx.get_temp_weights_like(weights); - LambdaFactory unary_op; - unary_op.op = LambdaFactory::OP_CATEGORY::NEG; - TF_RETURN_IF_ERROR(UnaryCompute(weights, &neg_weights, unary_op)); - shift_weights = neg_weights; + if (swapped_inputs) { + shift_weights = weights; + nvinfer1::IUnaryLayer* layer = + ctx.network()->addUnary(*const_cast(tensor), + nvinfer1::UnaryOperation::kNEG); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + tensor = layer->getOutput(0); + } else { + TRT_ShapedWeights neg_weights = ctx.get_temp_weights_like(weights); + LambdaFactory unary_op; + unary_op.op = LambdaFactory::OP_CATEGORY::NEG; + TF_RETURN_IF_ERROR(UnaryCompute(weights, &neg_weights, unary_op)); + shift_weights = neg_weights; + } + } else if (node_def.op() == "Div" || node_def.op() == "RealDiv") { + if (swapped_inputs) { + scale_weights = weights; + nvinfer1::IUnaryLayer* layer = + ctx.network()->addUnary(*const_cast(tensor), + nvinfer1::UnaryOperation::kRECIP); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + tensor = layer->getOutput(0); + } else { + TRT_ShapedWeights recip_weights = ctx.get_temp_weights_like(weights); + LambdaFactory unary_op; + unary_op.op = LambdaFactory::OP_CATEGORY::RECIP; + TF_RETURN_IF_ERROR(UnaryCompute(weights, &recip_weights, unary_op)); + scale_weights = recip_weights; + } } else if (node_def.op() == "Mul") { scale_weights = weights; } else if (node_def.op() == "Add") { @@ -1014,11 +1074,13 @@ tensorflow::Status BinaryTensorOpWeight( nvinfer1::IScaleLayer* layer = ctx.network()->addScale( *const_cast(tensor), scale_mode, shift_weights, scale_weights, power_weights); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); nvinfer1::ITensor* output_tensor = layer->getOutput(0); // transpose back dimension if (permutation_flag) { output_tensor = ctx.TransposeTensor(output_tensor, permutation); + TFTRT_RETURN_ERROR_IF_NULLPTR(output_tensor, node_def.name()); } // Pass the output @@ -1042,6 +1104,7 @@ tensorflow::Status ConvertConv2DHelper( if (data_format == "NHWC") { tensor = ctx.TransposeTensor(const_cast(tensor), {0, 3, 1, 2}); + TFTRT_RETURN_ERROR_IF_NULLPTR(tensor, node_def.name()); h_index = 1; w_index = 2; // TODO(jie): transpose it @@ -1102,6 +1165,7 @@ tensorflow::Status ConvertConv2DHelper( *const_cast(tensor), nvinfer1::DimsHW(padding[0].first, padding[1].first), nvinfer1::DimsHW(padding[0].second, padding[1].second)); + TFTRT_RETURN_ERROR_IF_NULLPTR(pad_layer, node_def.name()); padding = {{0, 0}, {0, 0}}; tensor = pad_layer->getOutput(0); auto dim_after = tensor->getDimensions(); @@ -1112,6 +1176,7 @@ tensorflow::Status ConvertConv2DHelper( nvinfer1::IConvolutionLayer* layer = ctx.network()->addConvolution(*const_cast(tensor), noutput, kernel_size, weights, biases); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); layer->setStride(stride); layer->setPadding({padding[0].first, padding[1].first}); @@ -1126,6 +1191,7 @@ tensorflow::Status ConvertConv2DHelper( if (data_format == "NHWC") { // TODO(jie): transpose it back! output_tensor = ctx.TransposeTensor(output_tensor, {0, 2, 3, 1}); + TFTRT_RETURN_ERROR_IF_NULLPTR(output_tensor, node_def.name()); } else { VLOG(2) << "NCHW !!!!"; } @@ -1147,18 +1213,71 @@ tensorflow::Status ConvertConv2DHelper( node_def.name()); } +bool PrepareTensorForShape(Converter& ctx, const TRT_TensorOrWeights& input, + const nvinfer1::ITensor** tensor, + const nvinfer1::Dims& dims) { + if (input.is_tensor()) { + if (DimsEqual(input.shape(), dims)) { + *tensor = input.tensor(); + } else { + nvinfer1::IShuffleLayer* layer = ctx.network()->addShuffle( + *const_cast(input.tensor())); + if (layer != nullptr) { + layer->setReshapeDimensions(dims); + *tensor = layer->getOutput(0); + } else { + return false; + } + } + } else { +#if NV_TENSORRT_MAJOR > 3 + nvinfer1::IConstantLayer* layer = + ctx.network()->addConstant(dims, input.weights()); + if (layer != nullptr) { + *tensor = layer->getOutput(0); + } else { + return false; + } +#else + return false; +#endif + } + return true; +} + tensorflow::Status BinaryTensorOpTensor( Converter& ctx, const tensorflow::NodeDef& node_def, - const nvinfer1::ITensor* tensor_l, const nvinfer1::ITensor* tensor_r, + const TRT_TensorOrWeights operand_l, const TRT_TensorOrWeights operand_r, std::vector* outputs) { static const std::unordered_map ops{ {"Add", nvinfer1::ElementWiseOperation::kSUM}, {"Mul", nvinfer1::ElementWiseOperation::kPROD}, {"Sub", nvinfer1::ElementWiseOperation::kSUB}, {"Div", nvinfer1::ElementWiseOperation::kDIV}, + {"RealDiv", nvinfer1::ElementWiseOperation::kDIV}, + {"Minimum", nvinfer1::ElementWiseOperation::kMIN}, + {"Maximum", nvinfer1::ElementWiseOperation::kMAX}, }; - // FIXME assume type matches input weights + const nvinfer1::ITensor* tensor_l; + const nvinfer1::ITensor* tensor_r; + + nvinfer1::Dims dim_l; + nvinfer1::Dims dim_r; + + if (!TensorRTGetBroadcastShape(operand_l.shape(), operand_l.is_tensor(), + operand_r.shape(), operand_r.is_tensor(), + &dim_l, &dim_r)) { + return tensorflow::errors::InvalidArgument( + "Binary op broadcast scheme not supported by TensorRT op: " + + node_def.op() + ", at: " + node_def.name()); + } + + TFTRT_RETURN_ERROR_IF_FALSE( + PrepareTensorForShape(ctx, operand_l, &tensor_l, dim_l), node_def.name()); + TFTRT_RETURN_ERROR_IF_FALSE( + PrepareTensorForShape(ctx, operand_r, &tensor_r, dim_r), node_def.name()); + // get trt type & shape TFAttrs attrs(node_def); // maybe this part has to be moved into the block of rsqrt later @@ -1169,13 +1288,14 @@ tensorflow::Status BinaryTensorOpTensor( CHECK_EQ_TYPE(tensor_r->getType(), dtype); auto op_pair = ops.find(node_def.op()); if (op_pair == ops.end()) - return tensorflow::errors::Unimplemented( - "binary op: " + node_def.op() + - " not supported at: " + node_def.name()); + return tensorflow::errors::Unimplemented("binary op: " + node_def.op() + + " not supported at: " + + node_def.name()); nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise( *const_cast(tensor_l), *const_cast(tensor_r), op_pair->second); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); nvinfer1::ITensor* output_tensor = layer->getOutput(0); @@ -1223,29 +1343,6 @@ tensorflow::Status ConvertPlugin(Converter& ctx, return tensorflow::Status::OK(); } -tensorflow::Status ConvertPlaceholder( - Converter& ctx, const tensorflow::NodeDef& node_def, - const std::vector& inputs, - std::vector* outputs) { - VLOG(2) << "Placeholder should have been replace already"; - return tensorflow::errors::Unimplemented("cannot convert Placeholder op"); - // OK this make sense since we are supposed to replace it with input - TFAttrs attrs(node_def); - nvinfer1::DataType dtype = attrs.get("dtype"); - nvinfer1::Dims dims = attrs.get("shape"); - - dims.nbDims--; - for (int i = 0; i < dims.nbDims; i++) dims.d[i] = dims.d[i + 1]; - - nvinfer1::ITensor* output = - ctx.network()->addInput(node_def.name().c_str(), dtype, dims); - if (!output) { - return tensorflow::errors::InvalidArgument("Failed to create Input layer"); - } - outputs->push_back(TRT_TensorOrWeights(output)); - return tensorflow::Status::OK(); -} - tensorflow::Status ConvertConv2D(Converter& ctx, const tensorflow::NodeDef& node_def, const std::vector& inputs, @@ -1277,11 +1374,10 @@ tensorflow::Status ConvertPool(Converter& ctx, w_index = 2; tensor = ctx.TransposeTensor(const_cast(tensor), {0, 3, 1, 2}); - } else { - VLOG(2) << "NCHW !!!!"; + TFTRT_RETURN_ERROR_IF_NULLPTR(tensor, node_def.name()); } + nvinfer1::PoolingType type; - // TODO(jie): support other pooling type if (node_def.op() == "MaxPool") type = nvinfer1::PoolingType::kMAX; else if (node_def.op() == "AvgPool") @@ -1289,7 +1385,6 @@ tensorflow::Status ConvertPool(Converter& ctx, else return tensorflow::errors::Unimplemented("Only supports Max pool"); - // TODO(jie): NCHW auto tf_stride = attrs.get>("strides"); nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]); @@ -1298,7 +1393,6 @@ tensorflow::Status ConvertPool(Converter& ctx, auto tensor_dim = tensor->getDimensions(); std::vector> padding; - // TODO(jie): padding. if (attrs.get("padding") == "SAME") { // This is NCHW tensor with no batch dimension. // 1 -> h @@ -1307,8 +1401,6 @@ tensorflow::Status ConvertPool(Converter& ctx, stride, ksize, {static_cast(tensor_dim.d[1]), static_cast(tensor_dim.d[2])}); } else if (attrs.get("padding") == "VALID") { - // No padding for valid padding here - VLOG(2) << "No padding added for VALID padding in pool" << node_def.name(); padding = {{0, 0}, {0, 0}}; } else { return tensorflow::errors::Unimplemented( @@ -1317,19 +1409,20 @@ tensorflow::Status ConvertPool(Converter& ctx, if (padding[0].first != padding[0].second || padding[1].first != padding[1].second) { - // TODO(jie): handle asymmetric padding VLOG(2) << "Padding!!!: " << padding[0].first << padding[0].second << padding[1].first << padding[1].second; auto pad_layer = ctx.network()->addPadding( *const_cast(tensor), nvinfer1::DimsHW(padding[0].first, padding[1].first), nvinfer1::DimsHW(padding[0].second, padding[1].second)); + TFTRT_RETURN_ERROR_IF_NULLPTR(pad_layer, node_def.name()); padding = {{0, 0}, {0, 0}}; tensor = pad_layer->getOutput(0); } nvinfer1::IPoolingLayer* layer = ctx.network()->addPooling( *const_cast(tensor), type, ksize); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); layer->setStride(stride); layer->setPadding({padding[0].first, padding[1].first}); @@ -1337,10 +1430,8 @@ tensorflow::Status ConvertPool(Converter& ctx, nvinfer1::ITensor* output_tensor = layer->getOutput(0); if (data_format == "NHWC") { - // TODO(jie): transpose it back! output_tensor = ctx.TransposeTensor(output_tensor, {0, 2, 3, 1}); - } else { - VLOG(2) << "NCHW !!!!"; + TFTRT_RETURN_ERROR_IF_NULLPTR(output_tensor, node_def.name()); } outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); @@ -1353,6 +1444,7 @@ tensorflow::Status ConvertActivation( const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); nvinfer1::IActivationLayer* layer = ctx.network()->addActivation( *const_cast(tensor), nvinfer1::ActivationType::kRELU); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); nvinfer1::ITensor* output_tensor = layer->getOutput(0); outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); @@ -1366,7 +1458,7 @@ tensorflow::Status ConvertScale(Converter& ctx, !inputs.at(1).is_weights()) return tensorflow::errors::Unimplemented( "Only supports tensor op weight for now, at " + node_def.name()); - // Implement tensor binaryOp weight [channel wise] for now; + const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); TRT_ShapedWeights weights = inputs.at(1).weights(); @@ -1378,25 +1470,48 @@ tensorflow::Status ConvertScale(Converter& ctx, TFAttrs attrs(node_def); - // Transpose NHWC auto data_format = attrs.get("data_format"); + int channel_index; + auto dims = tensor->getDimensions(); if (data_format == "NHWC") { - tensor = ctx.TransposeTensor(const_cast(tensor), - {0, 3, 1, 2}); - // TODO(jie): transpose it + // 1). NHWC is really N+C + channel_index = dims.nbDims - 1; // batch dimension is implicit here! } else { - VLOG(2) << "NCHW !!!!"; + // 2). NCHW is really N+CHW + channel_index = dims.nbDims - 3; // batch dimension is implicit here! } - auto dims = tensor->getDimensions(); - VLOG(2) << "tensor dimensions: " << dims.nbDims; - for (int i = 0; i < dims.nbDims; i++) { - VLOG(2) << "i: " << dims.d[i]; + nvinfer1::Permutation permutation; + for (int32_t i = 0; i < dims.nbDims; ++i) { + permutation.order[i] = i; } - dims = weights.shape_; - VLOG(2) << "tensor dimensions: " << dims.nbDims; - for (int i = 0; i < dims.nbDims; i++) { - VLOG(2) << "i: " << dims.d[i]; + + if (channel_index >= 0) { + permutation.order[0] = channel_index; + permutation.order[channel_index] = 0; + } else { + return tensorflow::errors::Unimplemented( + "TFTRT::BiasAdd cannot apply on batch dimension, at " + + node_def.name()); + } + + // TensorRT addScale requires input to be of rank 3, we need to apply + // transpose as well as reshape + if (channel_index != 0 || dims.nbDims != 3) { + nvinfer1::IShuffleLayer* shuffle_layer = + ctx.network()->addShuffle(*const_cast(tensor)); + TFTRT_RETURN_ERROR_IF_NULLPTR(shuffle_layer, node_def.name()); + nvinfer1::Dims reshape_dims; + reshape_dims.nbDims = 3; + reshape_dims.d[0] = 0; // 0 copy from the input + reshape_dims.d[1] = dims.nbDims >= 2 ? 0 : 1; // 0 copy from the input + reshape_dims.d[2] = dims.nbDims >= 3 ? -1 : 1; // -1 infer from the rest + if (channel_index != 0) { // maybe we do not need this check. concerned + // about TRT optimization + shuffle_layer->setFirstTranspose(permutation); + } + shuffle_layer->setReshapeDimensions(reshape_dims); + tensor = shuffle_layer->getOutput(0); } nvinfer1::ScaleMode mode = nvinfer1::ScaleMode::kCHANNEL; @@ -1407,14 +1522,26 @@ tensorflow::Status ConvertScale(Converter& ctx, nvinfer1::IScaleLayer* layer = ctx.network()->addScale(*const_cast(tensor), mode, weights, empty_weights, empty_weights); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); nvinfer1::ITensor* output_tensor = layer->getOutput(0); - if (data_format == "NHWC") { - // TODO(jie): transpose it back! - output_tensor = ctx.TransposeTensor(output_tensor, {0, 2, 3, 1}); - } else { - VLOG(2) << "NCHW !!!!"; + + // restore transpose & reshape + if (channel_index != 0 || dims.nbDims != 3) { + nvinfer1::IShuffleLayer* shuffle_layer = ctx.network()->addShuffle( + *const_cast(output_tensor)); + TFTRT_RETURN_ERROR_IF_NULLPTR(shuffle_layer, node_def.name()); + nvinfer1::Dims reshape_dims = dims; + int tmp = reshape_dims.d[channel_index]; + reshape_dims.d[channel_index] = reshape_dims.d[0]; + reshape_dims.d[0] = tmp; + shuffle_layer->setReshapeDimensions(reshape_dims); + if (channel_index != 0) { + shuffle_layer->setSecondTranspose(permutation); + } + output_tensor = shuffle_layer->getOutput(0); } + outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); } @@ -1567,47 +1694,105 @@ tensorflow::Status ConvertBinary(Converter& ctx, return tensorflow::errors::FailedPrecondition( "Binary ops require two tensor input, at " + node_def.name()); - if (inputs.at(0).is_weights() && inputs.at(1).is_weights()) - return ConstantFoldBinary(ctx, node_def, inputs, outputs); + // Constant folding should have been done by TensorFlow - if (inputs.at(0).is_tensor() && inputs.at(1).is_weights()) - return BinaryTensorOpWeight(ctx, node_def, inputs.at(0).tensor(), - inputs.at(1).weights(), outputs); + if (inputs.at(0).is_weights() && inputs.at(1).is_weights()) + return tensorflow::errors::Unimplemented( + "Constant folding is falled back to TensorFlow, binary op received " + "both input as constant at: " + + node_def.name()); - if (inputs.at(0).is_weights() && inputs.at(1).is_tensor()) - return BinaryTensorOpWeight(ctx, node_def, inputs.at(1).tensor(), - inputs.at(0).weights(), outputs); + // Try to convert into Scale layer first (for better performance) + // Since scale layer supports restricted broadcast policy and op types, we + // allow failure and try to handle it through Elementwise op + // (BinaryTensorOpTensor) + if (inputs.at(0).is_tensor() && inputs.at(1).is_weights()) { + auto status = BinaryTensorOpWeight(ctx, node_def, inputs.at(0).tensor(), + inputs.at(1).weights(), outputs, false); +#if NV_TENSORRT_MAJOR == 3 + TF_RETURN_IF_ERROR(status); +#else + TF_RETURN_IF_OK(status); +#endif + } - if (inputs.at(0).is_tensor() && inputs.at(1).is_tensor()) - return BinaryTensorOpTensor(ctx, node_def, inputs.at(0).tensor(), - inputs.at(1).tensor(), outputs); + if (inputs.at(0).is_weights() && inputs.at(1).is_tensor()) { + auto status = BinaryTensorOpWeight(ctx, node_def, inputs.at(1).tensor(), + inputs.at(0).weights(), outputs, true); +#if NV_TENSORRT_MAJOR == 3 + TF_RETURN_IF_ERROR(status); +#else + TF_RETURN_IF_OK(status); +#endif + } - return tensorflow::errors::Unknown("Binary op input error, at " + - node_def.name()); +#if NV_TENSORRT_MAJOR == 3 + if (inputs.at(0).is_tensor() && inputs.at(1).is_tensor()) { +#endif + return BinaryTensorOpTensor(ctx, node_def, inputs.at(0), inputs.at(1), + outputs); +#if NV_TENSORRT_MAJOR == 3 + } +#endif } tensorflow::Status ConvertUnary(Converter& ctx, const tensorflow::NodeDef& node_def, const std::vector& inputs, std::vector* outputs) { + static const std::unordered_map ops{ + {"Neg", nvinfer1::UnaryOperation::kNEG}, + {"Exp", nvinfer1::UnaryOperation::kEXP}, + {"Log", nvinfer1::UnaryOperation::kLOG}, + {"Sqrt", nvinfer1::UnaryOperation::kSQRT}, + {"Abs", nvinfer1::UnaryOperation::kABS}, + {"Reciprocal", nvinfer1::UnaryOperation::kRECIP}, + }; + if (inputs.size() != 1) return tensorflow::errors::FailedPrecondition( "Unary ops require single tensor input, at " + node_def.name()); +#if NV_TENSORRT_MAJOR == 3 if (inputs.at(0).is_weights()) - return ConstantFoldUnary(ctx, node_def, inputs, outputs); - else if (inputs.at(0).is_tensor()) return tensorflow::errors::Unimplemented( - "Unary op for tensor not supported, at " + node_def.name()); + "Constant folding for unary op is not supported" + node_def.name()); +#endif + + // TODO(jie): check type + const nvinfer1::ITensor* tensor; + TFTRT_RETURN_ERROR_IF_FALSE( + PrepareTensorForShape(ctx, inputs.at(0), &tensor, inputs.at(0).shape()), + node_def.name()); + + nvinfer1::IUnaryLayer* layer; + if (node_def.op() == "Rsqrt") { + layer = ctx.network()->addUnary(*const_cast(tensor), + nvinfer1::UnaryOperation::kSQRT); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + tensor = layer->getOutput(0); + layer = ctx.network()->addUnary(*const_cast(tensor), + nvinfer1::UnaryOperation::kRECIP); + } else if (ops.count(node_def.op()) != 0) { + layer = ctx.network()->addUnary(*const_cast(tensor), + ops.at(node_def.op())); + } else { + return tensorflow::errors::InvalidArgument("Binary op: " + node_def.op() + + " not supported, at " + + node_def.name()); + } - return tensorflow::errors::Unknown("Binary op input error, at " + - node_def.name()); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + nvinfer1::ITensor* output_tensor = layer->getOutput(0); + outputs->push_back(TRT_TensorOrWeights(output_tensor)); + return tensorflow::Status::OK(); } -tensorflow::Status ConvertReduce(Converter& ctx, - const tensorflow::NodeDef& node_def, - const std::vector& inputs, - std::vector* outputs) { +#if NV_TENSORRT_MAJOR == 3 +tensorflow::Status ConvertReducePool( + Converter& ctx, const tensorflow::NodeDef& node_def, + const std::vector& inputs, + std::vector* outputs) { if (inputs.size() != 2 || !inputs.at(0).is_tensor() || !inputs.at(1).is_weights()) return tensorflow::errors::InvalidArgument( @@ -1622,9 +1807,6 @@ tensorflow::Status ConvertReduce(Converter& ctx, TRT_ShapedWeights index_list = inputs.at(1).weights(); TFAttrs attrs(node_def); - // TODO(jie): handle data type. - // Index type here is done through TF type, so I can leverage their - // EnumToDataType for my cast auto index_type = attrs.get("Tidx"); // Only expect to handle INT32 as attributes for now @@ -1633,8 +1815,6 @@ tensorflow::Status ConvertReduce(Converter& ctx, auto index_list_data = static_cast(const_cast(index_list.GetValues())); - // Hack warning: have to fall back to pool layer since reduce is not in public - // TRT yet. if (nb_dims != 4) return tensorflow::errors::InvalidArgument( "TRT only support reduce on 4 dimensional tensors, at" + @@ -1673,6 +1853,7 @@ tensorflow::Status ConvertReduce(Converter& ctx, // Apply permutation before extracting dimension for pool_kernel tensor = ctx.TransposeTensor(const_cast(tensor), permutation_order); + TFTRT_RETURN_ERROR_IF_NULLPTR(tensor, node_def.name()); } // Apply permutation before extracting dimension for pool_kernel @@ -1685,6 +1866,7 @@ tensorflow::Status ConvertReduce(Converter& ctx, nvinfer1::IPoolingLayer* layer = ctx.network()->addPooling(*const_cast(tensor), nvinfer1::PoolingType::kAVERAGE, pool_kernel); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); output_tensor = layer->getOutput(0); } else { return tensorflow::errors::Unimplemented( @@ -1694,10 +1876,82 @@ tensorflow::Status ConvertReduce(Converter& ctx, // Apply permutation before extracting dimension for pool_kernel output_tensor = ctx.TransposeTensor( const_cast(output_tensor), permutation_order); + TFTRT_RETURN_ERROR_IF_NULLPTR(output_tensor, node_def.name()); } outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); } +#endif + +#if NV_TENSORRT_MAJOR > 3 +tensorflow::Status ConvertReduce(Converter& ctx, + const tensorflow::NodeDef& node_def, + const std::vector& inputs, + std::vector* outputs) { + if (inputs.size() != 2 || !inputs.at(0).is_tensor() || + !inputs.at(1).is_weights()) { + return tensorflow::errors::InvalidArgument( + "Input expects tensor and weights, at" + node_def.name()); + } + + const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); + auto dims = tensor->getDimensions(); + int nb_dims = dims.nbDims + 1; + + TRT_ShapedWeights index_list = inputs.at(1).weights(); + + TFAttrs attrs(node_def); + auto index_type = attrs.get("Tidx"); + + // Only expect to handle INT32 as attributes for now + if (index_type != tensorflow::DataType::DT_INT32) + return tensorflow::errors::Unimplemented("Tidx supports only DT_INT32"); + + auto keep_dims = attrs.get("keep_dims"); + + auto index_list_data = + static_cast(const_cast(index_list.GetValues())); + + int axes = 0; + if (index_list.count() == 0) { + return tensorflow::errors::InvalidArgument( + "TRT cannot support reduce on all (batch) dimensions, at" + + node_def.name()); + } else { + for (int i = 0; i < index_list.count(); i++) { + if (index_list_data[i] == 0) { + return tensorflow::errors::InvalidArgument( + "TRT cannot reduce at batch dimension, at" + node_def.name()); + } + axes |= (1 << (index_list_data[i] - 1)); + } + } + + nvinfer1::ReduceOperation reduce_operation; + if (node_def.op() == "Sum") { + reduce_operation = nvinfer1::ReduceOperation::kSUM; + } else if (node_def.op() == "Prod") { + reduce_operation = nvinfer1::ReduceOperation::kPROD; + } else if (node_def.op() == "Max") { + reduce_operation = nvinfer1::ReduceOperation::kMAX; + } else if (node_def.op() == "Min") { + reduce_operation = nvinfer1::ReduceOperation::kMIN; + } else if (node_def.op() == "Mean") { + reduce_operation = nvinfer1::ReduceOperation::kAVG; + } else { + return tensorflow::errors::Unimplemented( + "Op not supported " + node_def.op() + " , at " + node_def.name()); + } + + nvinfer1::ILayer* layer = + ctx.network()->addReduce(*const_cast(tensor), + reduce_operation, axes, keep_dims); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + + outputs->push_back(TRT_TensorOrWeights(layer->getOutput(0))); + return tensorflow::Status::OK(); +} +#endif tensorflow::Status ConvertPad(Converter& ctx, const tensorflow::NodeDef& node_def, @@ -1770,6 +2024,7 @@ tensorflow::Status ConvertPad(Converter& ctx, legit_pad = false; tensor = ctx.TransposeTensor(const_cast(tensor), {0, 3, 2, 1}); + TFTRT_RETURN_ERROR_IF_NULLPTR(tensor, node_def.name()); permuted_pad_index[0] = 3; } @@ -1786,11 +2041,14 @@ tensorflow::Status ConvertPad(Converter& ctx, nvinfer1::IPaddingLayer* layer = ctx.network()->addPadding( *const_cast(tensor), pre_padding, post_padding); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); nvinfer1::ITensor* output_tensor = layer->getOutput(0); - if (!legit_pad) + if (!legit_pad) { output_tensor = ctx.TransposeTensor( const_cast(output_tensor), {0, 3, 2, 1}); + TFTRT_RETURN_ERROR_IF_NULLPTR(output_tensor, node_def.name()); + } outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); @@ -1833,15 +2091,21 @@ tensorflow::Status ConvertConcat(Converter& ctx, return tensorflow::errors::InvalidArgument( "Concatenate on batch dimension not supported, at " + node_def.name()); + if (index < 0) { + index = dim.nbDims + index + 1; + } + +#if NV_TENSORRT_MAJOR == 3 // incase we need permutation; std::vector permutation_order(dim.nbDims + 1); for (int i = 0; i < dim.nbDims + 1; i++) permutation_order[i] = i; if (index != 1) { - permutation_order[1] = index - 1; - permutation_order[index - 1] = 1; + permutation_order[1] = index; + permutation_order[index] = 1; } +#endif std::vector inputs_vec; // Shap chack (all input tensor should have same shape) @@ -1862,11 +2126,14 @@ tensorflow::Status ConvertConcat(Converter& ctx, node_def.name()); } - // TRT does concatenation only on channel! - if (index != 1) +#if NV_TENSORRT_MAJOR == 3 + // TRT3 does concatenation only on channel! + if (index != 1) { tensor_i = ctx.TransposeTensor(const_cast(tensor_i), permutation_order); - + TFTRT_RETURN_ERROR_IF_NULLPTR(tensor_i, node_def.name()); + } +#endif inputs_vec.push_back(tensor_i); } @@ -1874,11 +2141,18 @@ tensorflow::Status ConvertConcat(Converter& ctx, nvinfer1::IConcatenationLayer* layer = ctx.network()->addConcatenation( const_cast(inputs_vec.data()), inputs_vec.size()); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); +#if NV_TENSORRT_MAJOR > 3 + layer->setAxis(index - 1); +#endif nvinfer1::ITensor* output_tensor = layer->getOutput(0); +#if NV_TENSORRT_MAJOR == 3 if (index != 1) { output_tensor = ctx.TransposeTensor(output_tensor, permutation_order); + TFTRT_RETURN_ERROR_IF_NULLPTR(output_tensor, node_def.name()); } +#endif outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); } @@ -1997,112 +2271,249 @@ tensorflow::Status ConvertFusedBatchNorm( combined_offset_weights.GetWeightsForTRT(), combined_scale_weights.GetWeightsForTRT(), dummy_power_weights.GetWeightsForTRT()); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); nvinfer1::ITensor* output_tensor = layer->getOutput(0); outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); } +#if NV_TENSORRT_MAJOR > 3 +tensorflow::Status ConvertMatMulHelper( + Converter& ctx, TRT_TensorOrWeights tensor_input, + TRT_ShapedWeights weights_raw, bool transpose_weight, + std::vector* outputs, string node_name) { + nvinfer1::ITensor* output_tensor; + if (!tensor_input.is_tensor()) { + return tensorflow::errors::InvalidArgument("Input 0 expects tensor"); + } + const nvinfer1::ITensor* tensor = tensor_input.tensor(); + + TRT_ShapedWeights weights(weights_raw.type_); + if (transpose_weight) { + weights = weights_raw; + } else { + TRT_ShapedWeights weights_ck = weights_raw; + weights = ctx.get_temp_weights_like(weights_ck); + ReorderCKtoKC(weights_raw, &weights); + } + TRT_ShapedWeights biases(weights.type_); + + int noutput = weights.shape_.d[0]; + + auto input_dim = tensor->getDimensions(); + while (input_dim.nbDims != 3) { + input_dim.d[input_dim.nbDims++] = 1; + } + TFTRT_RETURN_ERROR_IF_FALSE( + PrepareTensorForShape(ctx, tensor_input, &tensor, input_dim), node_name); + + nvinfer1::IFullyConnectedLayer* layer = ctx.network()->addFullyConnected( + *const_cast(tensor), noutput, weights, biases); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_name); + output_tensor = layer->getOutput(0); + + const nvinfer1::ITensor* temp_tensor; + auto output_dim = output_tensor->getDimensions(); + output_dim.nbDims = 1; + TFTRT_RETURN_ERROR_IF_FALSE( + PrepareTensorForShape(ctx, TRT_TensorOrWeights(output_tensor), + &temp_tensor, output_dim), + node_name); + output_tensor = const_cast(temp_tensor); + outputs->push_back(TRT_TensorOrWeights(output_tensor)); + return tensorflow::Status::OK(); +} + +// inputs are both two dimensional (tensorflow::ops::MatMul) tensorflow::Status ConvertMatMul(Converter& ctx, const tensorflow::NodeDef& node_def, const std::vector& inputs, std::vector* outputs) { + if (!inputs.at(0).is_tensor()) { + return tensorflow::errors::InvalidArgument("Input 0 expects tensor, at" + + node_def.name()); + } + const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); - // TODO(jie): transpose! TFAttrs attrs(node_def); - TRT_ShapedWeights weights_ck = inputs.at(1).weights(); - TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_ck); - ReorderCKtoKC(weights_ck, &weights); - TRT_ShapedWeights biases(weights.type_); + // TODO(jie): INT32 should be converted? + tensorflow::DataType tf_dtype = attrs.get("T"); + if (tf_dtype != tensorflow::DataType::DT_FLOAT && + tf_dtype != tensorflow::DataType::DT_HALF) { + return tensorflow::errors::Unimplemented( + "data type is not supported, for node " + node_def.name() + " got " + + tensorflow::DataTypeString(tf_dtype)); + } - int noutput = weights.shape_.d[0]; + bool transpose_a = attrs.get("transpose_a"); + bool transpose_b = attrs.get("transpose_b"); - nvinfer1::IFullyConnectedLayer* layer = ctx.network()->addFullyConnected( - *const_cast(tensor), noutput, weights, biases); + nvinfer1::ITensor* output_tensor; - nvinfer1::ITensor* output_tensor = layer->getOutput(0); - outputs->push_back(TRT_TensorOrWeights(output_tensor)); - return tensorflow::Status::OK(); + // FullyConnected: + if (transpose_a) { + return tensorflow::errors::Internal( + "Transpose_a is not supported for TensorRT FullyConnected (op: " + + node_def.op() + "), at: " + node_def.name()); + } + if (inputs.at(1).is_tensor()) { + return tensorflow::errors::Internal( + "Operand 1 must be constant for TensorRT FullyConnected (op: " + + node_def.op() + "), at: " + node_def.name()); + } + return ConvertMatMulHelper(ctx, inputs.at(0), inputs.at(1).weights(), + transpose_b, outputs, node_def.name()); } -tensorflow::Status ConvertReshape( +tensorflow::Status ConvertBatchMatMul( Converter& ctx, const tensorflow::NodeDef& node_def, const std::vector& inputs, std::vector* outputs) { - if (inputs.size() != 2 || !inputs.at(0).is_tensor() || - !inputs.at(1).is_weights()) - return tensorflow::errors::InvalidArgument( - "Input expects tensor and weights, at" + node_def.name()); + TFAttrs attrs(node_def); - // implement tensor binaryOp weight [channel wise] for now; - const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); - auto dims = tensor->getDimensions(); - // restore implicit batch dimension + // TODO(jie): INT32 should be converted? + tensorflow::DataType tf_dtype = attrs.get("T"); + if (tf_dtype != tensorflow::DataType::DT_FLOAT && + tf_dtype != tensorflow::DataType::DT_HALF) { + return tensorflow::errors::Unimplemented( + "data type is not supported, for node " + node_def.name() + " got " + + tensorflow::DataTypeString(tf_dtype)); + } - TRT_ShapedWeights shape = inputs.at(1).weights(); + bool transpose_a = attrs.get("adj_x"); + bool transpose_b = attrs.get("adj_y"); - TFAttrs attrs(node_def); + auto dims = inputs.at(0).shape(); + if (dims.nbDims == 1) { // NC * CK is only supported through fully connected + if (transpose_a == false && inputs.at(0).is_tensor() && + inputs.at(1).is_weights()) { + return ConvertMatMulHelper(ctx, inputs.at(0), inputs.at(1).weights(), + transpose_b, outputs, node_def.name()); + } else { + return tensorflow::errors::InvalidArgument( + "Invalid configuration for MatMul, at: " + node_def.name()); + } + } - auto padding_type = attrs.get("Tshape"); + const nvinfer1::ITensor* tensor_l; + const nvinfer1::ITensor* tensor_r; + auto dims_l = inputs.at(0).shape(); + auto dims_r = inputs.at(1).shape(); + if (inputs.at(0).is_weights()) { + if (inputs.at(0).shape().d[0] != 1) { + return tensorflow::errors::InvalidArgument( + "Input 0 as weight assumes broadcast across batch for MatMul, at: " + + node_def.name()); + } else { + for (int i = 0; i < dims_l.nbDims - 1; i++) { + dims_l.d[i] = dims_l.d[i + 1]; + } + dims_l.nbDims--; + } + } + if (inputs.at(1).is_weights()) { + if (inputs.at(1).shape().d[0] != 1) { + return tensorflow::errors::InvalidArgument( + "Input 1 as weight assumes broadcast across batch for MatMul, at: " + + node_def.name()); + } else { + for (int i = 0; i < dims_r.nbDims - 1; i++) { + dims_r.d[i] = dims_r.d[i + 1]; + } + dims_r.nbDims--; + } + } - if (shape.shape_.nbDims != 1) - return tensorflow::errors::InvalidArgument( - "reshape new shape is not 1 dimensional, at " + node_def.name()); + TFTRT_RETURN_ERROR_IF_FALSE( + PrepareTensorForShape(ctx, inputs.at(0), &tensor_l, dims_l), + node_def.name()); + TFTRT_RETURN_ERROR_IF_FALSE( + PrepareTensorForShape(ctx, inputs.at(1), &tensor_r, dims_r), + node_def.name()); - // Only expect to handle INT32 as attributes for now - if (padding_type != tensorflow::DataType::DT_INT32) - return tensorflow::errors::Unimplemented( - "reshape new shape supports only DT_INT32, at " + node_def.name()); + nvinfer1::IMatrixMultiplyLayer* layer = ctx.network()->addMatrixMultiply( + *const_cast(tensor_l), transpose_a, + *const_cast(tensor_r), transpose_b); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + nvinfer1::ITensor* output_tensor = layer->getOutput(0); + outputs->push_back(TRT_TensorOrWeights(output_tensor)); + return tensorflow::Status::OK(); +} +#endif - auto shape_data = static_cast(const_cast(shape.GetValues())); +#if NV_TENSORRT_MAJOR > 3 +tensorflow::Status ConvertSoftmax( + Converter& ctx, const tensorflow::NodeDef& node_def, + const std::vector& inputs, + std::vector* outputs) { + const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); - if (shape_data[0] != -1) + int nbDims = tensor->getDimensions().nbDims; + if (nbDims == 0) { return tensorflow::errors::InvalidArgument( - "reshape new shape first dimension is not -1, at " + node_def.name()); + "TensorRT Softmax cannot apply on batch dimension, at" + + node_def.name()); + } + nvinfer1::ISoftMaxLayer* layer = + ctx.network()->addSoftMax(*const_cast(tensor)); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + // Tensorflow SoftMax assumes applying softmax on the last dimension. + layer->setAxes(1 << (nbDims - 1)); - auto shape_num_dims = shape.shape_.d[0]; - VLOG(2) << "shape dimensions: " << shape_num_dims; - int volume_w = 1; - for (int i = 1; i < shape.shape_.d[0]; i++) volume_w *= shape_data[i]; + nvinfer1::ITensor* output_tensor = layer->getOutput(0); + outputs->push_back(TRT_TensorOrWeights(output_tensor)); + return tensorflow::Status::OK(); +} +#endif - int volume_t = 1; - for (int i = 0; i < dims.nbDims; i++) volume_t *= dims.d[i]; +#if NV_TENSORRT_MAJOR > 3 +tensorflow::Status ConvertTopK(Converter& ctx, + const tensorflow::NodeDef& node_def, + const std::vector& inputs, + std::vector* outputs) { + const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); - VLOG(2) << "volume: " << volume_t << " volume weights: " << volume_w; - if (volume_w != volume_t) + int nbDims = tensor->getDimensions().nbDims; + if (nbDims == 0) { return tensorflow::errors::InvalidArgument( - "volume does not agree between tensor and new shape, at " + - node_def.name()); + "TensorRT TopK cannot apply on batch dimension, at" + node_def.name()); + } - nvinfer1::IShuffleLayer* layer = - ctx.network()->addShuffle(*const_cast(tensor)); + TRT_ShapedWeights k_w = inputs.at(1).weights(); + int k = *(static_cast(const_cast(k_w.GetValues()))); - nvinfer1::Dims reshape_dims; - VLOG(2) << "new dimension: " << shape_num_dims - 1; - reshape_dims.nbDims = shape_num_dims - 1; - for (int32_t i = 0; i < reshape_dims.nbDims; ++i) { - reshape_dims.d[i] = shape_data[i + 1]; + nvinfer1::TopKOperation op; + uint32_t reducedAxes = 0; + if (node_def.op() == "TopKV2") { + op = nvinfer1::TopKOperation::kMAX; + reducedAxes |= 1 << (nbDims - 1); + } else { + return tensorflow::errors::Unimplemented("Operation: " + node_def.op() + + " not implemented, at: " + + node_def.name()); } - layer->setReshapeDimensions(reshape_dims); - VLOG(2) << "new dimension: " << shape_num_dims - 1; - nvinfer1::ITensor* output_tensor = layer->getOutput(0); - auto dims_output = output_tensor->getDimensions(); - VLOG(2) << "output tensor dimension:" << dims_output.nbDims; - outputs->push_back(TRT_TensorOrWeights(output_tensor)); + nvinfer1::ITopKLayer* layer = ctx.network()->addTopK( + *const_cast(tensor), op, k, reducedAxes); + TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); + + nvinfer1::ITensor* output_value_tensor = layer->getOutput(0); + nvinfer1::ITensor* output_indices_tensor = layer->getOutput(1); + outputs->push_back(TRT_TensorOrWeights(output_value_tensor)); + outputs->push_back(TRT_TensorOrWeights(output_indices_tensor)); return tensorflow::Status::OK(); } +#endif void Converter::register_op_converters() { // vgg_16 slim implementation - op_registry_["Placeholder"] = ConvertPlaceholder; op_registry_["Conv2D"] = ConvertConv2D; op_registry_["DepthwiseConv2dNative"] = ConvertConv2DDepthwise; op_registry_["Relu"] = ConvertActivation; op_registry_["MaxPool"] = ConvertPool; op_registry_["AvgPool"] = ConvertPool; - // This could be really handled as ConvertBinary op_registry_["BiasAdd"] = ConvertScale; op_registry_["Const"] = ConvertConst; // TODO(ben,jie): this is a temp hack. @@ -2113,18 +2524,38 @@ void Converter::register_op_converters() { op_registry_["Add"] = ConvertBinary; op_registry_["Mul"] = ConvertBinary; op_registry_["Sub"] = ConvertBinary; - op_registry_["Rsqrt"] = ConvertUnary; - op_registry_["Mean"] = ConvertReduce; op_registry_["Pad"] = ConvertPad; - // TODO(ben,jie): Add more ops op_registry_["ConcatV2"] = ConvertConcat; - op_registry_["MatMul"] = ConvertMatMul; - op_registry_["Reshape"] = ConvertReshape; op_registry_["FusedBatchNorm"] = ConvertFusedBatchNorm; op_registry_["FusedBatchNormV2"] = ConvertFusedBatchNorm; - plugin_converter_ = ConvertPlugin; + op_registry_["Div"] = ConvertBinary; + op_registry_["RealDiv"] = ConvertBinary; + + op_registry_["Rsqrt"] = ConvertUnary; + op_registry_["Reciprocal"] = ConvertUnary; + op_registry_["Exp"] = ConvertUnary; + op_registry_["Log"] = ConvertUnary; + op_registry_["Sqrt"] = ConvertUnary; + op_registry_["Abs"] = ConvertUnary; + op_registry_["Neg"] = ConvertUnary; +#if NV_TENSORRT_MAJOR == 3 + op_registry_["Mean"] = ConvertReducePool; +#endif +#if NV_TENSORRT_MAJOR > 3 + op_registry_["Sum"] = ConvertReduce; + op_registry_["Prod"] = ConvertReduce; + op_registry_["Max"] = ConvertReduce; + op_registry_["Min"] = ConvertReduce; + op_registry_["Mean"] = ConvertReduce; + op_registry_["Maximum"] = ConvertBinary; + op_registry_["Minimum"] = ConvertBinary; + op_registry_["Softmax"] = ConvertSoftmax; + op_registry_["MatMul"] = ConvertMatMul; + op_registry_["BatchMatMul"] = ConvertBatchMatMul; + op_registry_["TopKV2"] = ConvertTopK; +#endif } } // namespace @@ -2215,8 +2646,8 @@ tensorflow::Status ConvertGraphDefToEngine( node_name.c_str(), dtype, input_dim_pseudo_chw); if (!input_tensor) { return tensorflow::errors::InvalidArgument( - "Failed to create Input layer tensor ", node_name, - " rank=", shape.dims() - 1); + "Failed to create Input layer tensor ", node_name, " rank=", + shape.dims() - 1); } VLOG(1) << "Input tensor name :" << node_name; if (!converter.insert_input_tensor(node_name, input_tensor)) { diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc index 75e32559bb..416a55c4ad 100644 --- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc +++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc @@ -316,6 +316,11 @@ void TRTEngineOp::ComputeAsync(tensorflow::OpKernelContext* ctx, ctx->SetStatus(tensorflow::errors::InvalidArgument( "INT8 inputs are not supported!")); return; +#if NV_TENSORRT_MAJOR > 3 + case nvinfer1::DataType::kINT32: + buffers[binding_index] = (void*)(input_tensor.flat().data()); + break; +#endif default: LOG(ERROR) << "Unknown TRT data type: " << int(dtype); ctx->SetStatus(tensorflow::errors::InvalidArgument( @@ -368,6 +373,12 @@ void TRTEngineOp::ComputeAsync(tensorflow::OpKernelContext* ctx, ctx->SetStatus(tensorflow::errors::InvalidArgument( "INT8 outputs are not supported!")); return; +#if NV_TENSORRT_MAJOR > 3 + case nvinfer1::DataType::kINT32: + buffers[binding_index] = + reinterpret_cast(output_tensor->flat().data()); + break; +#endif default: LOG(ERROR) << "Unknown TRT data type: " << static_cast(dtype); ctx->SetStatus(tensorflow::errors::InvalidArgument( diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc index 383635f428..7eaa080586 100644 --- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc +++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc @@ -42,8 +42,9 @@ REGISTER_OP("TRTEngineOp") .Attr("precision_mode: {'FP32', 'FP16', 'INT8', 'INT8CALIB'}") .Attr("calibration_data: string = ''") .Input("in_tensor: InT") - .Output("out_tensor: OutT") - .SetShapeFn(shape_inference::TRTEngineOpShapeInference); + .Output("out_tensor: OutT"); + // TODO(Sami): shape inference not working for concrete input shape + //.SetShapeFn(shape_inference::TRTEngineOpShapeInference); } // namespace tensorflow -- cgit v1.2.3 From 32d4e6fd74fbeb91c8b2fd06c5ab0d4247d1784d Mon Sep 17 00:00:00 2001 From: Michael Case Date: Wed, 27 Jun 2018 09:46:45 -0700 Subject: Update version strings for TF 1.9.0-rc2. --- tensorflow/core/public/version.h | 2 +- tensorflow/docs_src/install/install_c.md | 2 +- tensorflow/docs_src/install/install_go.md | 2 +- tensorflow/docs_src/install/install_java.md | 22 +++++++++++----------- tensorflow/docs_src/install/install_linux.md | 18 +++++++++--------- tensorflow/docs_src/install/install_mac.md | 10 +++++----- tensorflow/docs_src/install/install_sources.md | 4 ++-- tensorflow/tools/pip_package/setup.py | 2 +- 8 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 9e5e747557..0e4a61ac1f 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -24,7 +24,7 @@ limitations under the License. // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") -#define TF_VERSION_SUFFIX "-rc1" +#define TF_VERSION_SUFFIX "-rc2" #define TF_STR_HELPER(x) #x #define TF_STR(x) TF_STR_HELPER(x) diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md index 2f81ae0c40..9aebf2bfa4 100644 --- a/tensorflow/docs_src/install/install_c.md +++ b/tensorflow/docs_src/install/install_c.md @@ -38,7 +38,7 @@ enable TensorFlow for C: OS="linux" # Change to "darwin" for macOS TARGET_DIRECTORY="/usr/local" curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.9.0-rc1.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.9.0-rc2.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index 5451e1b319..1907355341 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go: TF_TYPE="cpu" # Change to "gpu" for GPU support TARGET_DIRECTORY='/usr/local' curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.9.0-rc1.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.9.0-rc2.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index ad3544b595..b9c9912816 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs: org.tensorflow tensorflow - 1.9.0-rc1 + 1.9.0-rc2 ``` @@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow: org.tensorflow tensorflow - 1.9.0-rc1 + 1.9.0-rc2 @@ -124,12 +124,12 @@ instead: org.tensorflow libtensorflow - 1.9.0-rc1 + 1.9.0-rc2 org.tensorflow libtensorflow_jni_gpu - 1.9.0-rc1 + 1.9.0-rc2 ``` @@ -148,7 +148,7 @@ refer to the simpler instructions above instead. Take the following steps to install TensorFlow for Java on Linux or macOS: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc1.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc2.jar), which is the TensorFlow Java Archive (JAR). 2. Decide whether you will run TensorFlow for Java on CPU(s) only or with @@ -167,7 +167,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: OS=$(uname -s | tr '[:upper:]' '[:lower:]') mkdir -p ./jni curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.9.0-rc1.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.9.0-rc2.tar.gz" | tar -xz -C ./jni ### Install on Windows @@ -175,10 +175,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: Take the following steps to install TensorFlow for Java on Windows: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc1.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc2.jar), which is the TensorFlow Java Archive (JAR). 2. Download the following Java Native Interface (JNI) file appropriate for - [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.9.0-rc1.zip). + [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.9.0-rc2.zip). 3. Extract this .zip file. @@ -227,7 +227,7 @@ must be part of your `classpath`. For example, you can include the downloaded `.jar` in your `classpath` by using the `-cp` compilation flag as follows: -

javac -cp libtensorflow-1.9.0-rc1.jar HelloTF.java
+
javac -cp libtensorflow-1.9.0-rc2.jar HelloTF.java
### Running @@ -241,11 +241,11 @@ two files are available to the JVM: For example, the following command line executes the `HelloTF` program on Linux and macOS X: -
java -cp libtensorflow-1.9.0-rc1.jar:. -Djava.library.path=./jni HelloTF
+
java -cp libtensorflow-1.9.0-rc2.jar:. -Djava.library.path=./jni HelloTF
And the following command line executes the `HelloTF` program on Windows: -
java -cp libtensorflow-1.9.0-rc1.jar;. -Djava.library.path=jni HelloTF
+
java -cp libtensorflow-1.9.0-rc2.jar;. -Djava.library.path=jni HelloTF
If the program prints Hello from version, you've successfully installed TensorFlow for Java and are ready to use the API. If the program diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 41619ca230..ae3d50ff39 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -438,7 +438,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: 
      (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc1-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp34-cp34m-linux_x86_64.whl ## Validate your installation @@ -678,14 +678,14 @@ This section documents the relevant values for Linux installations. CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp27-none-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp27-none-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -697,14 +697,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp34-cp34m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp34-cp34m-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -716,14 +716,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp35-cp35m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp35-cp35m-linux_x86_64.whl
@@ -735,14 +735,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp36-cp36m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp36-cp36m-linux_x86_64.whl
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index eeca389617..3de6da1342 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv: TensorFlow in the active Virtualenv is as follows: 
 $ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl If you encounter installation problems, see [Common Installation Problems](#common-installation-problems). @@ -242,7 +242,7 @@ take the following steps: issue the following command: 
 $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl If the preceding command fails, see [installation problems](#common-installation-problems). @@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: TensorFlow for Python 2.7: 
 (targetDirectory)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py2-none-any.whl @@ -518,7 +518,7 @@ The value you specify depends on your Python version. 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py2-none-any.whl
@@ -526,5 +526,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py2-none-a 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc1-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index 7afcd340aa..3520f97c9a 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -328,10 +328,10 @@ Invoke `pip install` to install that pip package. The filename of the `.whl` file depends on your platform. For example, the following command will install the pip package -for TensorFlow 1.9.0rc1 on Linux: +for TensorFlow 1.9.0rc2 on Linux: 
-$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.9.0rc1-py2-none-any.whl
+$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.9.0rc2-py2-none-any.whl
## Validate your installation diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index eb2e359ee5..ed7ce01b6b 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -45,7 +45,7 @@ DOCLINES = __doc__.split('\n') # This version string is semver compatible, but incompatible with pip. # For pip, we will remove all '-' characters from this string, and use the # result for pip. -_VERSION = '1.9.0-rc1' +_VERSION = '1.9.0-rc2' REQUIRED_PACKAGES = [ 'absl-py >= 0.1.6', -- cgit v1.2.3 From 6896a74984efb4b1b77fc36ea274703536ba649d Mon Sep 17 00:00:00 2001 From: Jon Triebenbach Date: Wed, 27 Jun 2018 13:29:53 -0500 Subject: Build OpenBLAS 0.3.0 on ppc64le for TF tests --- configure.py | 7 ++++++ tensorflow/tools/ci_build/Dockerfile.cpu.ppc64le | 2 +- tensorflow/tools/ci_build/Dockerfile.gpu.ppc64le | 2 +- .../ci_build/install/install_openblas_ppc64le.sh | 28 ++++++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100755 tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh diff --git a/configure.py b/configure.py index ad585fa52e..04ad1c9441 100644 --- a/configure.py +++ b/configure.py @@ -1465,6 +1465,13 @@ def main(): environ_cp['TF_NEED_JEMALLOC'] = '0' environ_cp['TF_NEED_TENSORRT'] = '0' + # The numpy package on ppc64le uses OpenBLAS which has multi-threading + # issues that lead to incorrect answers. Set OMP_NUM_THREADS=1 at + # runtime to allow the Tensorflow testcases which compare numpy + # results to Tensorflow results to succeed. + if is_ppc64le(): + write_action_env_to_bazelrc("OMP_NUM_THREADS", 1) + set_build_var(environ_cp, 'TF_NEED_JEMALLOC', 'jemalloc as malloc', 'with_jemalloc', True) set_build_var(environ_cp, 'TF_NEED_GCP', 'Google Cloud Platform', diff --git a/tensorflow/tools/ci_build/Dockerfile.cpu.ppc64le b/tensorflow/tools/ci_build/Dockerfile.cpu.ppc64le index e879c34bbd..ada2c63880 100644 --- a/tensorflow/tools/ci_build/Dockerfile.cpu.ppc64le +++ b/tensorflow/tools/ci_build/Dockerfile.cpu.ppc64le @@ -7,7 +7,7 @@ COPY install/*.sh /install/ RUN /install/install_bootstrap_deb_packages.sh RUN add-apt-repository -y ppa:openjdk-r/ppa RUN /install/install_deb_packages.sh -RUN apt-get update && apt-get install -y libopenblas-dev +RUN /install/install_openblas_ppc64le.sh RUN /install/install_hdf5_ppc64le.sh RUN /install/install_pip_packages.sh RUN /install/install_bazel_from_source.sh diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu.ppc64le b/tensorflow/tools/ci_build/Dockerfile.gpu.ppc64le index 8967138747..a404f129ab 100644 --- a/tensorflow/tools/ci_build/Dockerfile.gpu.ppc64le +++ b/tensorflow/tools/ci_build/Dockerfile.gpu.ppc64le @@ -13,7 +13,7 @@ ARG DEBIAN_FRONTEND=noninteractive RUN /install/install_bootstrap_deb_packages.sh RUN add-apt-repository -y ppa:openjdk-r/ppa RUN /install/install_deb_packages.sh -RUN apt-get update && apt-get install -y libopenblas-dev +RUN /install/install_openblas_ppc64le.sh RUN /install/install_hdf5_ppc64le.sh RUN /install/install_pip_packages.sh RUN /install/install_bazel_from_source.sh diff --git a/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh b/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh new file mode 100755 index 0000000000..9ace25a36f --- /dev/null +++ b/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +OPENBLAS_SRC_PATH=/tmp/openblas_src/ +POWER="POWER8" +USE_OPENMP="USE_OPENMP=1" +OPENBLAS_INSTALL_PATH="/usr" +apt-get install -y gfortran gfortran-5 +sudo rm -rf ${OPENBLAS_SRC_PATH} +git clone -b release-0.3.0 https://github.com/xianyi/OpenBLAS ${OPENBLAS_SRC_PATH} +cd ${OPENBLAS_SRC_PATH} +# Pick up fix for OpenBLAS issue 1571 +git cherry-pick -X theirs 961d25e9c7e4a1758adb1dbeaa15187de69dd052 +make TARGET=${POWER} ${USE_OPENMP} FC=gfortran +make PREFIX=${OPENBLAS_INSTALL_PATH} install -- cgit v1.2.3 From 6336322f193fb300290c04474fad01a8f4d5b8c2 Mon Sep 17 00:00:00 2001 From: Jon Triebenbach Date: Thu, 28 Jun 2018 13:00:15 -0500 Subject: Correct install openblas script --- tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh b/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh index 9ace25a36f..b73962854c 100755 --- a/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh +++ b/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh @@ -18,6 +18,7 @@ OPENBLAS_SRC_PATH=/tmp/openblas_src/ POWER="POWER8" USE_OPENMP="USE_OPENMP=1" OPENBLAS_INSTALL_PATH="/usr" +apt-get update apt-get install -y gfortran gfortran-5 sudo rm -rf ${OPENBLAS_SRC_PATH} git clone -b release-0.3.0 https://github.com/xianyi/OpenBLAS ${OPENBLAS_SRC_PATH} -- cgit v1.2.3 From f93e1b07282216d77e9d7d704f6722a893e9ef73 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Thu, 28 Jun 2018 11:24:37 -0700 Subject: Potential fix for how pip installs headers used for custom ops. These headers were recently moved from site-packages/external into site-packages/tensorflow/include/external. Need to update setup.py to reflect that. --- RELEASE.md | 1 + tensorflow/tools/pip_package/setup.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 52cd9ef72b..21207a7efa 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -24,6 +24,7 @@ ## Breaking Changes * If you're opening empty variable scopes; replace `variable_scope('', ...)` by `variable_scope(tf.get_variable_scope(), ...)`. + * Headers used for building custom ops have been moved from site-packages/external into site-packages/tensorflow/include/external. ## Bug Fixes and Other Changes diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index ed7ce01b6b..8c077580aa 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -170,8 +170,9 @@ class InstallHeaders(Command): # symlink within the directory hierarchy. # NOTE(keveman): Figure out how to customize bdist_wheel package so # we can do the symlink. - if 'external/eigen_archive/' in install_dir: - extra_dir = install_dir.replace('external/eigen_archive', '') + if 'tensorflow/include/external/eigen_archive/' in install_dir: + extra_dir = install_dir.replace( + 'tensorflow/include/external/eigen_archive', '') if not os.path.exists(extra_dir): self.mkpath(extra_dir) self.copy_file(header, extra_dir) @@ -204,13 +205,12 @@ def find_files(pattern, root): yield os.path.join(dirpath, filename) -matches = ['../' + x for x in find_files('*', 'external') if '.py' not in x] - so_lib_paths = [ i for i in os.listdir('.') if os.path.isdir(i) and fnmatch.fnmatch(i, '_solib_*') ] +matches = [] for path in so_lib_paths: matches.extend( ['../' + x for x in find_files('*', path) if '.py' not in x] @@ -225,7 +225,7 @@ headers = (list(find_files('*.h', 'tensorflow/core')) + list(find_files('*.h', 'tensorflow/stream_executor')) + list(find_files('*.h', 'google/protobuf_archive/src')) + list(find_files('*', 'third_party/eigen3')) + - list(find_files('*', 'external/eigen_archive'))) + list(find_files('*', 'tensorflow/include/external/eigen_archive'))) setup( name=project_name, -- cgit v1.2.3 From f09aaf0dd33869253020b095d7c44840d1b430fe Mon Sep 17 00:00:00 2001 From: Michael Case Date: Fri, 29 Jun 2018 10:19:06 -0700 Subject: Exclude test sources from stream executor builds. (#20423) PiperOrigin-RevId: 202423156 --- tensorflow/contrib/cmake/tf_stream_executor.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/cmake/tf_stream_executor.cmake b/tensorflow/contrib/cmake/tf_stream_executor.cmake index 9a37b68119..2f70e59d54 100644 --- a/tensorflow/contrib/cmake/tf_stream_executor.cmake +++ b/tensorflow/contrib/cmake/tf_stream_executor.cmake @@ -76,11 +76,11 @@ if (tensorflow_ENABLE_GPU) list(APPEND tf_stream_executor_srcs ${tf_stream_executor_gpu_srcs}) endif() -#file(GLOB_RECURSE tf_stream_executor_test_srcs -# "${tensorflow_source_dir}/tensorflow/stream_executor/*_test.cc" -# "${tensorflow_source_dir}/tensorflow/stream_executor/*_test.h" -#) -#list(REMOVE_ITEM tf_stream_executor_srcs ${tf_stream_executor_test_srcs}) +file(GLOB_RECURSE tf_stream_executor_test_srcs + "${tensorflow_source_dir}/tensorflow/stream_executor/*test.cc" + "${tensorflow_source_dir}/tensorflow/stream_executor/lib/*test.h" +) +list(REMOVE_ITEM tf_stream_executor_srcs ${tf_stream_executor_test_srcs}) if (NOT WIN32) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lgomp") -- cgit v1.2.3 From 648ef712f2c4fc996551373765aff30a0e48bc4c Mon Sep 17 00:00:00 2001 From: bhack Date: Sat, 30 Jun 2018 14:54:43 +0200 Subject: Advise batch_normalization with model_to_estimator --- tensorflow/docs_src/guide/keras.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/docs_src/guide/keras.md b/tensorflow/docs_src/guide/keras.md index c799e9b12c..d584ebe945 100644 --- a/tensorflow/docs_src/guide/keras.md +++ b/tensorflow/docs_src/guide/keras.md @@ -548,9 +548,11 @@ model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), estimator = keras.estimator.model_to_estimator(model) ``` -Note: Enable [eager execution](./eager.md) for debugging +Note: +* Enable [eager execution](./eager.md) for debugging [Estimator input functions](./premade_estimators.md#create_input_functions) and inspecting data. +* Don't use batch normalization or try to finetune batch normalization models with estimators created from `tf.keras.estimator.model_to_estimator`. More details at [#17950](https://github.com/tensorflow/tensorflow/issues/17950) ### Multiple GPUs -- cgit v1.2.3 From 8fab75ff2e551414d093b8fc3b3e4a78bf91754d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 30 Jun 2018 16:12:27 +0000 Subject: Update .gitignore for cmake generated file While running cmake in Linux: ``` tensorflow/tools/ci_build/ci_build.sh CMAKE tensorflow/tools/ci_build/builds/cmake.sh ``` the following file is generated and left out: ``` ubuntu@ubuntu:~/tensorflow$ git status On branch master Your branch is up-to-date with 'origin/master'. Untracked files: (use "git add ..." to include in what will be committed) estimator_api_init_files_list.txt nothing added to commit but untracked files present (use "git add" to track) ``` This fix add `/estimator_api_init_files_list.txt` in gitignore so that it will not be picked by `git add -A`. Signed-off-by: Yong Tang --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b5306b8b79..5afe375f46 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ Podfile.lock /tensorflow/contrib/lite/examples/ios/simple/data/*.tflite xcuserdata/** /api_init_files_list.txt +/estimator_api_init_files_list.txt # Android .gradle -- cgit v1.2.3 From 9281603eeb149942952e3d8b35c2a121bbaa045e Mon Sep 17 00:00:00 2001 From: Paul Woitaschek Date: Sun, 1 Jul 2018 15:08:07 +0200 Subject: Added minSdkVersion to the manifest Fixes #20453 --- tensorflow/contrib/lite/java/AndroidManifest.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/contrib/lite/java/AndroidManifest.xml b/tensorflow/contrib/lite/java/AndroidManifest.xml index f705feacbe..d9e10900bf 100644 --- a/tensorflow/contrib/lite/java/AndroidManifest.xml +++ b/tensorflow/contrib/lite/java/AndroidManifest.xml @@ -1,6 +1,9 @@ + + + -- cgit v1.2.3 From 1d7fcde539fcff854e261c375c8ec2fbff258c34 Mon Sep 17 00:00:00 2001 From: 张天启 Date: Sun, 1 Jul 2018 22:05:51 +0800 Subject: fix bug in maxout function The line "shape[axis] = -1" will make the shape wrong when dealing with batches with arbitrary sizes. --- tensorflow/contrib/layers/python/layers/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py index b7194ae333..a55d42c151 100644 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ b/tensorflow/contrib/layers/python/layers/layers.py @@ -3117,7 +3117,7 @@ def maxout(inputs, num_units, axis=-1, scope=None): raise ValueError('number of features({}) is not ' 'a multiple of num_units({})'.format( num_channels, num_units)) - shape[axis] = -1 + shape[axis] = num_units shape += [num_channels // num_units] # Dealing with batches with arbitrary sizes -- cgit v1.2.3 From 2629729eef55f27d03a1be661bd827d5176afd51 Mon Sep 17 00:00:00 2001 From: Jie Date: Sat, 30 Jun 2018 23:29:15 -0700 Subject: addressing review comments added Constant broadcast in converter; added integration tests for converter; --- .../contrib/tensorrt/convert/convert_nodes.cc | 131 ++++++++------- .../contrib/tensorrt/kernels/trt_engine_op.cc | 4 +- .../tensorrt/test/unit_tests/BatchMatMulTest.py | 102 ++++++++++++ .../tensorrt/test/unit_tests/BiasaddMatMulTest.py | 140 ++++++++++++++++ .../unit_tests/BinaryTensorWeightBroadcastTest.py | 153 +++++++++++++++++ .../tensorrt/test/unit_tests/ConcatenationTest.py | 100 +++++++++++ .../tensorrt/test/unit_tests/ConstBroadcastTest.py | 82 +++++++++ .../MultiConnectionNeighborEngineTest.py | 100 +++++++++++ .../test/unit_tests/NeighboringEngineTest.py | 84 ++++++++++ .../contrib/tensorrt/test/unit_tests/UnaryTest.py | 130 +++++++++++++++ .../tensorrt/test/unit_tests/VGGBlockNCHWTest.py | 84 ++++++++++ .../tensorrt/test/unit_tests/VGGBlockTest.py | 84 ++++++++++ .../tensorrt/test/unit_tests/base_unit_test.py | 115 +++++++++++++ .../contrib/tensorrt/test/unit_tests/run_test.py | 183 +++++++++++++++++++++ .../contrib/tensorrt/test/unit_tests/unit_tests.py | 65 ++++++++ .../contrib/tensorrt/test/unit_tests/utilities.py | 30 ++++ 16 files changed, 1529 insertions(+), 58 deletions(-) create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/run_test.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/utilities.py diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index 22f92d38bf..9369152d0e 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -53,25 +53,27 @@ limitations under the License. // would work! #define CHECK_EQ_TYPE(val1, val2) CHECK_EQ((int)val1, (int)val2) -#define TFTRT_RETURN_ERROR_IF_FALSE(ptr, node) \ - do { \ - if (ptr == false) { \ - return tensorflow::errors::Internal(string("TFTRT::") + __FUNCTION__ + \ - "failed to add TRT layer, at: " + \ - node); \ - } \ +#define TFTRT_RETURN_ERROR_IF_FALSE(ptr, node) \ + do { \ + if (ptr == false) { \ + return tensorflow::errors::Internal( \ + string("TFTRT::"), __FUNCTION__, \ + "failed to add TRT layer, at: ", \ + node); \ + } \ } while (0) -#define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \ - do { \ - if (ptr == nullptr) { \ - return tensorflow::errors::Internal(string("TFTRT::") + __FUNCTION__ + \ - "failed to add TRT layer, at: " + \ - node); \ - } \ +#define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \ + do { \ + if (ptr == nullptr) { \ + return tensorflow::errors::Internal( \ + string("TFTRT::"), __FUNCTION__, \ + "failed to add TRT layer, at: ", \ + node); \ + } \ } while (0) -#define TF_RETURN_IF_OK(status) \ +#define TFTRT_RETURN_IF_OK(status) \ do { \ if (status.ok()) { \ return tensorflow::Status::OK(); \ @@ -510,6 +512,8 @@ void ReorderRSCKToKCRS(const TRT_ShapedWeights& iweights, VLOG(2) << "c" << iweights.shape_.d[2] << " then " << c; int k = iweights.shape_.d[3] * num_groups; VLOG(2) << "k" << iweights.shape_.d[3] << " then " << k; + VLOG(2) << "r" << iweights.shape_.d[0] << " then " << r; + VLOG(2) << "s" << iweights.shape_.d[1] << " then " << s; oweights->shape_.d[0] = k / num_groups; oweights->shape_.d[1] = c * num_groups; oweights->shape_.d[2] = r; @@ -1119,6 +1123,17 @@ tensorflow::Status ConvertConv2DHelper( VLOG(2) << "groups count: " << num_groups; TRT_ShapedWeights weights_rsck = inputs.at(1).weights(); + + VLOG(2) << "weight shape: " << weights_rsck.shape_.nbDims; + for (int i = 0; i < weights_rsck.shape_.nbDims; i++) { + VLOG(2) << weights_rsck.shape_.d[i]; + } + + if (weights_rsck.shape_.nbDims != 4) { + return tensorflow::errors::Internal( + "Conv2D expects kernel of dimension 4, at: " + node_def.name()); + } + if (ctx.isFP16()) { weights_rsck = ConvertFP32ToFP16(ctx, inputs.at(1).weights()); } @@ -1130,6 +1145,10 @@ tensorflow::Status ConvertConv2DHelper( nvinfer1::DimsHW kernel_size; kernel_size.h() = weights.shape_.d[2]; kernel_size.w() = weights.shape_.d[3]; + VLOG(2) << "RSCK: "; + for (int i = 0; i < 4; i++) { + VLOG(2) << " " << weights.shape_.d[i]; + } VLOG(2) << "kernel size: " << kernel_size.h() << ", " << kernel_size.w(); // TODO(jie): stride. (NHWC/NCHW) @@ -1570,22 +1589,16 @@ tensorflow::Status ConvertConst(Converter& ctx, VLOG(2) << "dimensions: " << tensor.dims(); VLOG(2) << "size: " << weights_tensor.float_val_size(); scalar_shape = GetTensorShape(tensor); + VLOG(2) << "details: "; for (int i = 0; i < scalar_shape.nbDims; i++) VLOG(2) << scalar_shape.d[i]; - if (GetShapeSize(scalar_shape) != weights_tensor.float_val_size()) { - if (weights_tensor.float_val_size() == 1 || - scalar_shape.d[0] == weights_tensor.float_val_size()) { - scalar_shape.nbDims = 1; - // no dimension provided. flatten it - scalar_shape.d[0] = weights_tensor.float_val_size(); - scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL; - } else { - LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and" - << " kUNIFORM, at: " << node_def.name(); - string err_str("Broadcast method is not supported for '"); - StrAppend(&err_str, node_def.name(), "' of type ", node_def.op()); - return tensorflow::errors::InvalidArgument(err_str); - } + if (GetShapeSize(scalar_shape) != weights_tensor.float_val_size() && + weights_tensor.float_val_size() != 1) { + LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and" + << " kUNIFORM, at: " << node_def.name(); + string err_str("Broadcast method is not supported for '"); + StrAppend(&err_str, node_def.name(), "' of type ", node_def.op()); + return tensorflow::errors::InvalidArgument(err_str); } } else { VLOG(2) << "Dimensions: " << tensor.dims(); @@ -1595,18 +1608,25 @@ tensorflow::Status ConvertConst(Converter& ctx, scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL; for (int i = 1; i < nvinfer1::Dims::MAX_DIMS; i++) { scalar_shape.d[i] = 0; - scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL; } } size_t len_data = tensorflow::DataTypeSize(dtype); for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i]; ctx.weight_store()->store_.push_back(std::vector(len_data)); void* dst = static_cast(&(ctx.weight_store()->store_.back()[0])); - std::vector tensor_data( - weights_tensor.float_val().begin(), - weights_tensor.float_val() - .end()); // make a local copy first to flatten - memcpy(dst, tensor_data.data(), len_data); // store into weight store + if (weights_tensor.float_val_size() == 1) { + std::fill_n((float*)dst, GetShapeSize(scalar_shape), + *weights_tensor.float_val().begin()); + } else { + std::vector tensor_data( + weights_tensor.float_val().begin(), + weights_tensor.float_val() + .end()); // make a local copy first to flatten + // doesn't have to be contigous + memcpy(dst, tensor_data.data(), len_data); // store into weight store + } + VLOG(2) << "create shape details: "; + for (int i = 0; i < scalar_shape.nbDims; i++) VLOG(2) << scalar_shape.d[i]; weights = TRT_ShapedWeights(dtype, dst, scalar_shape); } else if (!weights_tensor.int_val().empty()) { VLOG(2) << "int!!!" << node_def.name(); @@ -1614,20 +1634,13 @@ tensorflow::Status ConvertConst(Converter& ctx, if (tensor.dims() > 0) { VLOG(2) << "dimensions: " << tensor.dims(); scalar_shape = GetTensorShape(tensor); - if (GetShapeSize(scalar_shape) != weights_tensor.int_val_size()) { - if (weights_tensor.int_val_size() == 1 || - scalar_shape.d[0] == weights_tensor.int_val_size()) { - scalar_shape.nbDims = 1; - // no dimension provided. flatten it - scalar_shape.d[0] = weights_tensor.int_val_size(); - scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL; - } else { - LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and" - << " kUNIFORM, at: " << node_def.name(); - string err_str("Broadcast method is not supported for '"); - StrAppend(&err_str, node_def.name(), "' of type ", node_def.op()); - return tensorflow::errors::InvalidArgument(err_str); - } + if (GetShapeSize(scalar_shape) != weights_tensor.int_val_size() && + weights_tensor.int_val_size() != 1) { + LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and" + << " kUNIFORM, at: " << node_def.name(); + string err_str("Broadcast method is not supported for '"); + StrAppend(&err_str, node_def.name(), "' of type ", node_def.op()); + return tensorflow::errors::InvalidArgument(err_str); } } else { VLOG(2) << "dimensions: " << tensor.dims(); @@ -1647,11 +1660,17 @@ tensorflow::Status ConvertConst(Converter& ctx, len_data = std::max(len_data, len_tensor); ctx.weight_store()->store_.push_back(std::vector(len_data)); void* dst = static_cast(&(ctx.weight_store()->store_.back()[0])); - std::vector tensor_data( - weights_tensor.int_val().begin(), - weights_tensor.int_val().end()); // make a local copy first to flatten - // doesn't have to be contigous - memcpy(dst, tensor_data.data(), len_tensor); // store into weight store + if (weights_tensor.int_val_size() == 1) { + std::fill_n((int*)dst, GetShapeSize(scalar_shape), + *weights_tensor.int_val().begin()); + } else { + std::vector tensor_data( + weights_tensor.int_val().begin(), + weights_tensor.int_val() + .end()); // make a local copy first to flatten + // doesn't have to be contigous + memcpy(dst, tensor_data.data(), len_tensor); // store into weight store + } weights = TRT_ShapedWeights(dtype, dst, scalar_shape); } else if (!weights_tensor.tensor_content().empty()) { // obsolete method. @@ -1712,7 +1731,7 @@ tensorflow::Status ConvertBinary(Converter& ctx, #if NV_TENSORRT_MAJOR == 3 TF_RETURN_IF_ERROR(status); #else - TF_RETURN_IF_OK(status); + TFTRT_RETURN_IF_OK(status); #endif } @@ -1722,7 +1741,7 @@ tensorflow::Status ConvertBinary(Converter& ctx, #if NV_TENSORRT_MAJOR == 3 TF_RETURN_IF_ERROR(status); #else - TF_RETURN_IF_OK(status); + TFTRT_RETURN_IF_OK(status); #endif } diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc index 416a55c4ad..aad07af628 100644 --- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc +++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc @@ -318,7 +318,7 @@ void TRTEngineOp::ComputeAsync(tensorflow::OpKernelContext* ctx, return; #if NV_TENSORRT_MAJOR > 3 case nvinfer1::DataType::kINT32: - buffers[binding_index] = (void*)(input_tensor.flat().data()); + buffers[binding_index] = (void*)(input_tensor.flat().data()); break; #endif default: @@ -376,7 +376,7 @@ void TRTEngineOp::ComputeAsync(tensorflow::OpKernelContext* ctx, #if NV_TENSORRT_MAJOR > 3 case nvinfer1::DataType::kINT32: buffers[binding_index] = - reinterpret_cast(output_tensor->flat().data()); + reinterpret_cast(output_tensor->flat().data()); break; #endif default: diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py new file mode 100644 index 0000000000..bcd6eb2192 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py @@ -0,0 +1,102 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.layers import core +from tensorflow.python.training import training +from base_unit_test import BaseUnitTest +from utilities import get_all_variables + +class BatchMatMulTest(BaseUnitTest): + """Testing BatchMatMul in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(BatchMatMulTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug=True + self.dynamic_mode_list = {} + self.inp_dims = (12, 5, 8, 12) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.matmul_test + self.expect_nb_nodes = 16 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.ckpt = "./tmp.ckpt" + sess = csess.Session() + + def matmul_test(self): + g = ops.Graph() + gpu_options = cpb2.GPUOptions() + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + b = constant_op.constant( + np.random.randn(12, 5, 12, 7), dtype=dtypes.float32) + x1 = math_ops.matmul(x, b) + b = constant_op.constant( + np.random.randn(5, 1, 1), dtype=dtypes.float32) + x1 = x1 + b + + var = variable_scope.get_variable("test", [12, 5, 12, 7], dtype=dtypes.float32, initializer=init_ops.truncated_normal_initializer) + x2 = math_ops.matmul(x, var) + b = constant_op.constant( + np.random.randn(5, 1, 1), dtype=dtypes.float32) + x2 = x2 * b + + var = variable_scope.get_variable("test2", [12, 84], dtype=dtypes.float32, initializer=init_ops.truncated_normal_initializer) + c = gen_array_ops.reshape(x, [12, 40, 12]) + b = gen_array_ops.reshape(var, [12, 12, 7]) + x3 = math_ops.matmul(c, b) + b = constant_op.constant( + np.random.randn(40, 1), dtype=dtypes.float32) + x3 = x3 + b + x3 = gen_array_ops.reshape(x3, [12, 5, 8, 7]) + + out = x3 + x1 + array_ops.squeeze(out, name="output") + + with csess.Session(config=sessconfig, graph=g) as sess: + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + sess.run(variables.global_variables_initializer()) + saver.save(sess, self.ckpt) + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py new file mode 100644 index 0000000000..ab1e18e0d0 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py @@ -0,0 +1,140 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.layers import core +from tensorflow.python.training import training +from base_unit_test import BaseUnitTest +from utilities import get_all_variables + +class BiasaddMatMulTest(BaseUnitTest): + """Testing BiasAdd MatMul in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(BiasaddMatMulTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug=True + self.dynamic_mode_list = {} + self.inp_dims = (48, 12) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.matmul_test + self.expect_nb_nodes = 53 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def matmul_test(self): + g = ops.Graph() + gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + b = constant_op.constant( + np.random.randn(12, 4), dtype=dtypes.float32) + x1 = math_ops.matmul(x, b) + b = constant_op.constant( + np.random.randn(1, 4), dtype=dtypes.float32) + x1 = x1 + b + + b = constant_op.constant( + np.random.randn(48, 4), dtype=dtypes.float32) + x2 = math_ops.matmul(x, b, transpose_a=True) + x2 = gen_array_ops.reshape(x2, [48, 1]) + + b = constant_op.constant( + np.random.randn(4, 12), dtype=dtypes.float32) + x3 = math_ops.matmul(x, b, transpose_b=True) + + b = constant_op.constant( + np.random.randn(16, 48), dtype=dtypes.float32) + x4 = math_ops.matmul(x, b, transpose_b=True, transpose_a=True) + x4 = gen_array_ops.reshape(x4, [48, 4]) + + x5 = gen_array_ops.reshape(x, [4, 12, 12]) + x5 = core.flatten(x5) + b = constant_op.constant( + np.random.randn(144, 48), dtype=dtypes.float32) + x5 = math_ops.matmul(x5, b) + b = constant_op.constant( + np.random.randn(48), dtype=dtypes.float32) + x5 = nn.bias_add(x5, b) + x5 = gen_array_ops.reshape(x5, [48, 4]) + + x6 = gen_array_ops.reshape(x, [4, 12, 12]) + b = constant_op.constant( + np.random.randn(12), dtype=dtypes.float32) + x6 = nn.bias_add(x6, b, data_format="NHWC") + x6 = gen_array_ops.reshape(x6, [48, -1]) + + x7 = gen_array_ops.reshape(x, [4, 12, 3, 4]) + b = constant_op.constant( + np.random.randn(4), dtype=dtypes.float32) + x7 = nn.bias_add(x7, b, data_format="NHWC") + x7 = gen_array_ops.reshape(x7, [48, -1]) + + x8 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) + b = constant_op.constant( + np.random.randn(2), dtype=dtypes.float32) + x8 = nn.bias_add(x8, b, data_format="NHWC") + x8 = gen_array_ops.reshape(x8, [48, -1]) + + x9 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) + b = constant_op.constant( + np.random.randn(3), dtype=dtypes.float32) + x9 = nn.bias_add(x9, b, data_format="NCHW") + x9 = gen_array_ops.reshape(x9, [48, -1]) + + x10 = gen_array_ops.reshape(x, [4, 12, 3, 4]) + b = constant_op.constant( + np.random.randn(12), dtype=dtypes.float32) + x10 = nn.bias_add(x10, b, data_format="NCHW") + x10 = gen_array_ops.reshape(x10, [48, -1]) + + x11 = gen_array_ops.reshape(x, [4, 12, 12]) + b = constant_op.constant( + np.random.randn(4), dtype=dtypes.float32) + x11 = nn.bias_add(x11, b, data_format="NCHW") + x11 = gen_array_ops.reshape(x11, [48, -1]) + + out = array_ops.concat([x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11], axis=-1) + out = array_ops.squeeze(out, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py new file mode 100644 index 0000000000..2b56ac7a96 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py @@ -0,0 +1,153 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training +from base_unit_test import BaseUnitTest +from utilities import get_all_variables + +class BinaryTensorWeightBroadcastTest(BaseUnitTest): + """unit tests for scale & elementwise layers in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(BinaryTensorWeightBroadcastTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug=True + self.dynamic_mode_list = {} + self.inp_dims = (10, 24, 24, 20) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 35 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.1 + self.allclose_atol = 0.05 + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + # scale + a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(24,1,1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(24,1,1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(24,24,20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(24,24,20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(1,24,1,1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(1,24,1,1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(1,24,24,1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(1,24,24,1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(1,24,24,20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(1,24,24,20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(24,20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(24,20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + gen_array_ops.reshape(x, [5, -1] , name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py new file mode 100644 index 0000000000..b3648fd070 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py @@ -0,0 +1,100 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training +from base_unit_test import BaseUnitTest +from utilities import get_all_variables + +class ConcatenationTest(BaseUnitTest): + """Testing Concatenation in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(ConcatenationTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug=True + self.dynamic_mode_list = {} + self.inp_dims = (2, 3, 3, 1) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 4 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + # scale + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r1 = x / a + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r2 = a / x + a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32) + r3 = a + x + a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32) + r4 = x * a + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r5 = x - a + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r6 = a - x + a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) + r7 = x - a + a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) + r8 = a - x + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r9 = gen_math_ops.maximum(x, a) + a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) + r10 = gen_math_ops.minimum(a, x) + a = constant_op.constant(np.random.randn(3), dtype=dtypes.float32) + r11 = x * a + a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) + r12 = a * x + concat1 = array_ops.concat([r1,r2,r3,r4,r5,r6], axis=-1) + concat2 = array_ops.concat([r7,r8,r9,r10,r11,r12], axis=3) + x = array_ops.concat([concat1, concat2], axis=-1) + + gen_array_ops.reshape(x, [2, -1] , name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py new file mode 100644 index 0000000000..b4fbb57851 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py @@ -0,0 +1,82 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np +import tensorflow as tf + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.layers import core +from tensorflow.python.training import training +from base_unit_test import BaseUnitTest +from utilities import get_all_variables + +class ConstBroadcastTest(BaseUnitTest): + """Testing Constant broadcasting in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(ConstBroadcastTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug=True + self.dynamic_mode_list = {} + self.inp_dims = (5, 12, 12, 2) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.conv_broadcast + self.expect_nb_nodes = 7 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.05 + self.allclose_atol = 0.05 + + def conv_broadcast(self): + g = ops.Graph() + gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + filt1 = tf.constant(1, shape=(3,3,2,1), dtype=tf.float32, name='filt1') + y1 = tf.nn.conv2d(x, filt1, strides=[1,1, 1, 1], padding='SAME', name='y1') + z1 = tf.nn.relu(y1, name='z1') + filt2 = tf.constant(np.random.randn(9), shape=(3,3,1,1), dtype=tf.float32, name='filt2') + y2 = tf.nn.conv2d(z1, filt2, strides=[1,1, 1, 1], padding='SAME', name='y2') + z2 = tf.nn.relu(y2, name='z') + filt3 = tf.constant(np.random.randn(3,3,1,1), shape=(3,3,1,1), dtype=tf.float32, name='filt3') + y3 = tf.nn.conv2d(z2, filt3, strides=[1,1, 1, 1], padding='SAME', name='y3') + z = tf.nn.relu(y3, name='output') + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py new file mode 100644 index 0000000000..5f5f13a102 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py @@ -0,0 +1,100 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training +from base_unit_test import BaseUnitTest +from utilities import get_all_variables + +class MultiConnectionNeighborEngineTest(BaseUnitTest): + """Multi connection neighboring nodes wiring tests in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(MultiConnectionNeighborEngineTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug=True + self.dynamic_mode_list = {} + self.inp_dims = (2, 3, 7, 5) + self.dummy_input = np.random.normal(1.0, 0.5, self.inp_dims) + self.get_network = self.neighboring_tensor_test + self.expect_nb_nodes = 7 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.05 + self.allclose_atol = 0.05 + + def neighboring_tensor_test(self): + g = ops.Graph() + gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + e = constant_op.constant( + np.random.normal(.05, .005, [3,2,3,4]), + name="weights", + dtype=dtypes.float32) + conv = nn.conv2d( + input=x, filter=e, data_format="NCHW",strides=[1, 1, 1, 1], padding="VALID", name="conv") + b = constant_op.constant( + np.random.normal(2.0, 1.0, [1,4,1,1]), name="bias", dtype=dtypes.float32) + t = conv+b + + b = constant_op.constant( + np.random.normal(5.0, 1.0, [1,4,1,1]), name="bias", dtype=dtypes.float32) + q = conv-b + edge = math_ops.sigmoid(q) + + b = constant_op.constant( + np.random.normal(5.0, 1.0, [1,4,1,1]), name="bias", dtype=dtypes.float32) + d = b+conv + edge3 = math_ops.sigmoid(d) + + c = constant_op.constant( + np.random.normal(1.0, 1.0, [1,4,1,1]), name="bias", dtype=dtypes.float32) + edge1 = gen_math_ops.tan(conv) + t = t - edge1 + q = q + edge + t = t + q + t = t + d + t = t - edge3 + array_ops.squeeze(t, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py new file mode 100644 index 0000000000..d7b5eba4fd --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py @@ -0,0 +1,84 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training +from base_unit_test import BaseUnitTest +from utilities import get_all_variables + +class NeighboringEngineTest(BaseUnitTest): + """Neighboring node wiring tests in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(NeighboringEngineTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug=True + self.dynamic_mode_list = {} + self.inp_dims = (2, 3, 7, 5) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.neighboring_tensor_test + self.expect_nb_nodes = 5 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.05 + self.allclose_atol = 0.05 + + def neighboring_tensor_test(self): + g = ops.Graph() + gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + e = constant_op.constant( + np.random.normal(.3, 0.05, [3,2,3,4]), + name="weights", + dtype=dtypes.float32) + conv = nn.conv2d( + input=x, filter=e, data_format="NCHW",strides=[1, 1, 1, 1], padding="VALID", name="conv") + b = constant_op.constant( + np.random.normal(1.0, 1.0, [1,4,1,1]), name="bias", dtype=dtypes.float32) + t = conv*b + + e = gen_math_ops.tan(conv) + t = t - e + array_ops.squeeze(t, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py new file mode 100644 index 0000000000..beb2f2b7a7 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py @@ -0,0 +1,130 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.layers import core +from tensorflow.python.training import training +from base_unit_test import BaseUnitTest +from utilities import get_all_variables + +class UnaryTest(BaseUnitTest): + """Unit tests for unary operations in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(UnaryTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug=True + self.dynamic_mode_list = {} + self.inp_dims = (12, 5, 8, 1, 1, 12) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.unary_test + self.expect_nb_nodes = 17 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.ckpt = "./tmp.ckpt" + + def unary_test(self): + g = ops.Graph() + gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + q = math_ops.abs(x) + q = q + 1.0 + q = gen_math_ops.exp(q) + q = gen_math_ops.log(q) + q = array_ops.squeeze(q, axis=-2) + q = math_ops.abs(q) + q = q + 2.2 + q = gen_math_ops.sqrt(q) + q = gen_math_ops.rsqrt(q) + q = math_ops.negative(q) + q = array_ops.squeeze(q, axis=3) + q = math_ops.abs(q) + q = q + 3.0 + a = gen_math_ops.reciprocal(q) + + x = constant_op.constant( + np.random.randn(5, 8, 12), dtype=dtypes.float32) + q = math_ops.abs(x) + q = q + 2.0 + q = gen_math_ops.exp(q) + q = gen_math_ops.log(q) + q = math_ops.abs(q) + q = q + 2.1 + q = gen_math_ops.sqrt(q) + q = gen_math_ops.rsqrt(q) + q = math_ops.negative(q) + q = math_ops.abs(q) + q = q + 4.0 + b = gen_math_ops.reciprocal(q) + + # TODO(jie): this one will break, broadcasting on batch. + x = variable_scope.get_variable("test", [12, 40, 12], dtype=dtypes.float32, initializer=init_ops.truncated_normal_initializer) + x = gen_array_ops.reshape(x, [12, 5, 8, 1, 12, 1, 1]) + q = math_ops.abs(x) + q = q + 5.0 + q = gen_math_ops.exp(q) + q = array_ops.squeeze(q, axis=[-1, -2, 3]) + q = gen_math_ops.log(q) + q = math_ops.abs(q) + q = q + 5.1 + q = gen_array_ops.reshape(q, [12, 5, 1, 1, 8, 1, 12]) + q = array_ops.squeeze(q, axis=[5, 2, 3]) + q = gen_math_ops.sqrt(q) + q = math_ops.abs(q) + q = q + 5.2 + q = gen_math_ops.rsqrt(q) + q = math_ops.negative(q) + q = math_ops.abs(q) + q = q + 5.3 + c = gen_math_ops.reciprocal(q) + + q = a * b + q = q / c + array_ops.squeeze(q, name="output") + + with csess.Session(config=sessconfig, graph=g) as sess: + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + sess.run(variables.global_variables_initializer()) + saver.save(sess, self.ckpt) + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py new file mode 100644 index 0000000000..f9f5c7d114 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py @@ -0,0 +1,84 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training +from base_unit_test import BaseUnitTest +from utilities import get_all_variables + +class VGGBlockNCHWTest(BaseUnitTest): + """single vgg layer in NCHW unit tests in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(VGGBlockNCHWTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug=True + self.dynamic_mode_list = {} + self.inp_dims = (5, 2, 8, 8) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 3 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + x, mean_x, var_x = nn_impl.fused_batch_norm(x, np.random.randn(2).astype(np.float32), np.random.randn(2).astype(np.float32), mean=np.random.randn(2).astype(np.float32), variance=np.random.randn(2).astype(np.float32), data_format="NCHW", is_training=False) + e = constant_op.constant( + np.random.randn(1,1,2,6), + name="weights", + dtype=dtypes.float32) + conv = nn.conv2d( + input=x, filter=e, data_format="NCHW",strides=[1, 1, 2, 2], padding="SAME", name="conv") + b = constant_op.constant( + np.random.randn(6), name="bias", dtype=dtypes.float32) + t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd") + relu = nn.relu(t, "relu") + idty = array_ops.identity(relu, "ID") + v = nn_ops.max_pool( + idty, [1, 1, 2, 2], [1, 1, 2, 2], "VALID", data_format="NCHW", name="max_pool") + array_ops.squeeze(v, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py new file mode 100644 index 0000000000..fe872ecd2c --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py @@ -0,0 +1,84 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training +from base_unit_test import BaseUnitTest +from utilities import get_all_variables + +class VGGBlockTest(BaseUnitTest): + """single vgg layer test in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(VGGBlockTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug=True + self.dynamic_mode_list = {} + self.inp_dims = (5, 8, 8, 2) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 7 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + x, mean_x, var_x = nn_impl.fused_batch_norm(x, np.random.randn(2).astype(np.float32), np.random.randn(2).astype(np.float32), mean=np.random.randn(2).astype(np.float32), variance=np.random.randn(2).astype(np.float32), is_training=False) + e = constant_op.constant( + np.random.randn(1,1,2,6), + name="weights", + dtype=dtypes.float32) + conv = nn.conv2d( + input=x, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv") + b = constant_op.constant( + np.random.randn(6), name="bias", dtype=dtypes.float32) + t = nn.bias_add(conv, b, name="biasAdd") + relu = nn.relu(t, "relu") + idty = array_ops.identity(relu, "ID") + v = nn_ops.max_pool( + idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool") + array_ops.squeeze(v, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py b/tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py new file mode 100644 index 0000000000..f8997bf304 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py @@ -0,0 +1,115 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Base class to facilitate development of integration tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op as cop +from tensorflow.python.framework import dtypes as dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops as aops +from tensorflow.python.ops import nn as nn +from tensorflow.python.ops import nn_ops as nn_ops + +class BaseUnitTest(object): + """Base class for unit tests in TF-TRT""" + + def __init__(self, log_file='log.txt'): + self.static_mode_list = {} + self.dynamic_mode_list = {} + self.dummy_input = None + self.get_network = None + self.expect_nb_nodes = None + self.test_name = None + self.log_file = log_file + self.ckpt = None + self.allclose_rtol = 0.01 + self.allclose_atol = 0.01 + self.allclose_equal_nan = True + # saves out graphdef + self.debug = False + # require node count check fail leads to test failure + self.check_node_count = False + + def run(self, run_test_context): + run_test_context.run_test(self.get_network, self.static_mode_list, self.dynamic_mode_list, self.dummy_input, self.ckpt) + return self.log_result(run_test_context) + + def log_result(self, run_test_result): + log = open(self.log_file, 'a') + log.write(("================= model: %s\n")%(self.test_name)) + + if self.debug: + open(self.test_name+"_native.pb", 'wb').write(run_test_result.native_network.SerializeToString()) + all_success = True + if len(run_test_result.tftrt_conversion_flag) != 0: + log.write(" -- static_mode\n") + for static_mode in run_test_result.tftrt_conversion_flag: + if self.debug: + open(self.test_name+"_"+static_mode+".pb", 'wb').write(run_test_result.tftrt[static_mode].SerializeToString()) + log.write(" ----\n") + log.write((" mode: [%s]\n")%(static_mode)) + if run_test_result.tftrt_conversion_flag[static_mode]: + if run_test_result.tftrt_nb_nodes[static_mode] != self.expect_nb_nodes: + log.write(("[WARNING]: converted node number does not match (%d,%d,%d)!!!\n")%(run_test_result.tftrt_nb_nodes[static_mode], self.expect_nb_nodes, run_test_result.native_nb_nodes)) + if self.check_node_count: + all_success = False + + if np.array_equal(run_test_result.tftrt_result[static_mode], run_test_result.native_result): + log.write(" output: equal\n") + elif np.allclose(run_test_result.tftrt_result[static_mode], run_test_result.native_result, atol=self.allclose_atol, rtol=self.allclose_rtol, equal_nan=self.allclose_equal_nan): + log.write(" output: allclose\n") + else: + diff = run_test_result.tftrt_result[static_mode]-run_test_result.native_result + log.write("[ERROR]: output does not match!!!\n") + log.write( "max diff: " +str(np.max(diff))) + log.write( "\ntftrt:\n") + log.write(str(run_test_result.tftrt_result[static_mode])) + log.write( "\nnative:\n") + log.write(str(run_test_result.native_result)) + log.write( "\ndiff:\n") + log.write(str(diff)) + all_success = False + else: + log.write("[ERROR]: conversion failed!!!\n") + all_success = False + + if len(run_test_result.tftrt_dynamic_conversion_flag) != 0: + log.write(" -- dynamic_mode\n") + for dynamic_mode in run_test_result.tftrt_dynamic_conversion_flag: + log.write("\n ----\n") + log.write((" mode: [%s]\n")%(dynamic_mode)) + if run_test_result.tftrt_dynamic_conversion_flag[dynamic_mode]: + if np.array_equal(run_test_result.tftrt_dynamic_result[dynamic_mode], run_test_result.native_result): + log.write(" output: equal\n") + elif np.allclose(run_test_result.tftrt_dynamic_result[dynamic_mode], run_test_result.native_result): + log.write(" output: allclose\n") + else: + log.write("[ERROR]: output does not match!!!\n") + all_success = False + else: + log.write("[ERROR]: conversion failed!!!\n") + all_success = False + return all_success diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/run_test.py b/tensorflow/contrib/tensorrt/test/unit_tests/run_test.py new file mode 100644 index 0000000000..afb5cabf33 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/run_test.py @@ -0,0 +1,183 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""script to convert and execute TF-TensorRT graph.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 as cpb2 +from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 +from tensorflow.python.client import session as csess +from tensorflow.python.framework import constant_op as cop +from tensorflow.python.framework import dtypes as dtypes +from tensorflow.python.framework import importer as importer +from tensorflow.python.framework import ops as ops +from tensorflow.python.ops import array_ops as aops +from tensorflow.python.ops import nn as nn +from tensorflow.python.ops import nn_ops as nn_ops +from tensorflow.python.ops import variables +from tensorflow.python.training import training +from utilities import get_all_variables + +OUTPUT_NODE = "output" +INPUT_NODE = "input" +CALIB_COUNT = 5 # calibration iteration + +class RunTest: + """base class to run TR-TRT conversion and execution""" + + def __init__(self): + self.clean() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.clean() + + def clean(self): + self.tftrt = {} + self.tftrt_conversion_flag = {} + self.tftrt_nb_nodes = {} + self.tftrt_result = {} + self.tftrt_dynamic_conversion_flag = {} + self.tftrt_dynamic_result = {} + self.check_file = None + self.native_network = None + + def run_test(self, network, static_mode_list, dynamic_mode_list, dummy_input, file_name=None): + self.native_network = network() + success = True + initialization = False + if file_name!=None: + initialization = True + self.check_file = file_name + self.native_result, self.native_nb_nodes = self.execute_graph(self.native_network, dummy_input, initialization) + for mode in static_mode_list: + try: + self.run_static_convert_network(mode, dummy_input, initialization) + self.tftrt_conversion_flag[mode] = True + except Exception as inst: + self.tftrt_conversion_flag[mode] = False + success = False + for mode in dynamic_mode_list: + try: + self.run_dynamic_convert_network(mode, dummy_input, initialization) + self.tftrt_dynamic_conversion_flag[mode] = True + except Exception as inst: + self.tftrt_dynamic_conversion_flag[mode] = False + success = False + return success + + def run_dynamic_convert_network(self, mode, dummy_input, initialization=True): + inp_dims = dummy_input.shape + if mode == "FP32" or mode == "FP16": + opt_config = rwpb2.RewriterConfig() + opt_config.optimizers.extend(["constfold", "layout"]) + custom_op = opt_config.custom_optimizers.add() + custom_op.name = "TensorRTOptimizer" + custom_op.parameter_map["minimum_segment_size"].i = 3 + custom_op.parameter_map["precision_mode"].s = mode + custom_op.parameter_map["max_batch_size"].i = inp_dims[0] + custom_op.parameter_map["max_workspace_size_bytes"].i = 1 << 25 + print(custom_op) + gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + graph_options = cpb2.GraphOptions(rewrite_options=opt_config) + sessconfig = cpb2.ConfigProto( + gpu_options=gpu_options, graph_options=graph_options) + print(sessconfig) + g = ops.Graph() + ops.reset_default_graph() + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=self.native_network, return_elements=["input", "output"]) + inp = inp.outputs[0] + out = out.outputs[0] + with csess.Session(config=sessconfig, graph=g) as sess: + if (initialization): + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + saver.restore(sess, self.check_file) + self.tftrt_dynamic_result[mode] = sess.run(out, {inp: dummy_input}) + else: + raise Exception("dynamic op mode: " + mode + " not supported") + + def run_static_convert_network(self, mode, dummy_input, initialization=True): + inp_dims = dummy_input.shape + if mode == "FP32" or mode == "FP16" or mode == "INT8": + trt_graph = trt.create_inference_graph( + input_graph_def=self.native_network, + outputs=[OUTPUT_NODE], + max_batch_size=inp_dims[0], + max_workspace_size_bytes=1 << 25, + precision_mode=mode, # TRT Engine precision "FP32","FP16" or "INT8" + minimum_segment_size=2 # minimum number of nodes in an engine + ) + if mode == "INT8": + _ = self.execute_calibration(trt_graph, dummy_input, initialization) + trt_graph = trt.calib_graph_to_infer_graph(trt_graph) + trt_result, nb_nodes = self.execute_graph(trt_graph, dummy_input, initialization) + self.tftrt[mode] = trt_graph + self.tftrt_nb_nodes[mode] = nb_nodes + self.tftrt_result[mode] = trt_result + else: + raise Exception("mode: " + mode + " not supported") + + def execute_graph(self, gdef, dummy_input, initialization=True): + """Run given graphdef once.""" + gpu_options = cpb2.GPUOptions() + sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) + ops.reset_default_graph() + g = ops.Graph() + nb_nodes = 0 + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") + nb_nodes = len(g.get_operations()) + inp = inp.outputs[0] + out = out.outputs[0] + with csess.Session(config=sessconfig, graph=g) as sess: + if (initialization): + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + saver.restore(sess, self.check_file) + val = sess.run(out, {inp: dummy_input}) + return val, nb_nodes + + # Use real data that is representative of the inference dataset + # for calibration. For this test script it is random data. + def execute_calibration(self, gdef, dummy_input, initialization=True): + """Run given calibration graph multiple times.""" + gpu_options = cpb2.GPUOptions() + ops.reset_default_graph() + g = ops.Graph() + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") + inp = inp.outputs[0] + out = out.outputs[0] + with csess.Session( + config=cpb2.ConfigProto(gpu_options=gpu_options), graph=g) as sess: + if (initialization): + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + saver.restore(sess, self.check_file) + for _ in range(CALIB_COUNT): + val = sess.run(out, {inp: dummy_input}) + return val diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py b/tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py new file mode 100644 index 0000000000..31d3625c3c --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py @@ -0,0 +1,65 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to execute and log all integration tests.""" + +from BatchMatMulTest import BatchMatMulTest +from BiasaddMatMulTest import BiasaddMatMulTest +from BinaryTensorWeightBroadcastTest import BinaryTensorWeightBroadcastTest +from ConcatenationTest import ConcatenationTest +from ConvElewiseFusionFailTest import ConvElewiseFusionFailTest +from GatherV2FailTest import GatherV2FailTest +from MultiConnectionNeighborEngineTest import MultiConnectionNeighborEngineTest +from NeighboringEngineTest import NeighboringEngineTest +from UnaryTest import UnaryTest +from VGGBlockNCHWTest import VGGBlockNCHWTest +from VGGBlockTest import VGGBlockTest +from ConstBroadcastTest import ConstBroadcastTest + +from run_test import RunTest + +tests = 0 +passed_test = 0 + +failed_list = [] +test_list = [] + +test_list.append(BatchMatMulTest()) +test_list.append(BiasaddMatMulTest()) +test_list.append(BinaryTensorWeightBroadcastTest()) +test_list.append(ConcatenationTest()) +test_list.append(NeighboringEngineTest()) +test_list.append(UnaryTest()) +test_list.append(VGGBlockNCHWTest()) +test_list.append(VGGBlockTest()) +test_list.append(MultiConnectionNeighborEngineTest()) +test_list.append(ConstBroadcastTest()) + +for test in test_list: + test.debug = True + test.check_node_count = False + with RunTest() as context: + tests+=1 + if test.run(context): + passed_test +=1 + else: + failed_list.append(test.test_name) + print("Failed test: %s\n", test.test_name) + +if passed_test == tests: + print("Passed\n") +else: + print(("%d out of %d passed\n -- failed list:")%(passed_test, tests)) + for test in failed_list: + print(" - " + test) diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/utilities.py b/tensorflow/contrib/tensorrt/test/unit_tests/utilities.py new file mode 100644 index 0000000000..abb999a25f --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests/utilities.py @@ -0,0 +1,30 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities script for TF-TensorRT integration tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.ops import variables +from tensorflow.python.training import training + +def get_all_variables(sess): + var_names = sess.run(variables.report_uninitialized_variables()) + names_var_list = {} + for name in var_names: + names_var_list[name] = sess.graph.get_tensor_by_name(name+":0") + print(var_names) + return names_var_list -- cgit v1.2.3 From 11cd70438e7d7104904bf8f3b24fcaf6fd88eab5 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 2 Jul 2018 13:37:38 -0700 Subject: Fix lint error. --- tensorflow/contrib/model_pruning/python/pruning.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/model_pruning/python/pruning.py b/tensorflow/contrib/model_pruning/python/pruning.py index d843fa26d5..da9d398cbc 100644 --- a/tensorflow/contrib/model_pruning/python/pruning.py +++ b/tensorflow/contrib/model_pruning/python/pruning.py @@ -520,7 +520,8 @@ class Pruning(object): thresholds = get_thresholds() for mask, threshold in zip(masks, thresholds): if not self._exists_in_do_not_prune_list(mask.name): - summary.scalar(mask.op.name + '/sparsity', nn_impl.zero_fraction(mask)) + summary.scalar(mask.op.name + '/sparsity', + nn_impl.zero_fraction(mask)) summary.scalar(threshold.op.name + '/threshold', threshold) def print_hparams(self): -- cgit v1.2.3 From f394207e840c6ea72c153fb2bb4e781f63f5e119 Mon Sep 17 00:00:00 2001 From: Paul Woitaschek Date: Mon, 2 Jul 2018 23:12:26 +0200 Subject: Added target-sdk as well. --- tensorflow/contrib/lite/java/AndroidManifest.xml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/lite/java/AndroidManifest.xml b/tensorflow/contrib/lite/java/AndroidManifest.xml index d9e10900bf..f954bba739 100644 --- a/tensorflow/contrib/lite/java/AndroidManifest.xml +++ b/tensorflow/contrib/lite/java/AndroidManifest.xml @@ -1,10 +1,11 @@ + package="org.tensorflow.lite"> - + - - + + - -- cgit v1.2.3 From 4664191b73959387c190f969d7f1fe3480a585f4 Mon Sep 17 00:00:00 2001 From: Yifei Feng <1192265+yifeif@users.noreply.github.com> Date: Mon, 2 Jul 2018 17:10:20 -0700 Subject: Match for path instead of name --- tensorflow/tools/pip_package/build_pip_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh index 9e41514cfa..b0089d3360 100755 --- a/tensorflow/tools/pip_package/build_pip_package.sh +++ b/tensorflow/tools/pip_package/build_pip_package.sh @@ -27,7 +27,7 @@ function cp_external() { pushd . cd "$src_dir" - for f in `find . ! -type d ! -name '*.py' ! -name '*local_config_cuda*' ! -name '*local_config_tensorrt*' ! -name '*org_tensorflow*'`; do + for f in `find . ! -type d ! -name '*.py' ! -path '*local_config_cuda*' ! -path '*local_config_tensorrt*' ! -path '*org_tensorflow*'`; do mkdir -p "${dest_dir}/$(dirname ${f})" cp "${f}" "${dest_dir}/$(dirname ${f})/" done -- cgit v1.2.3 From 2496c49fab2893c9bfb154c4c2b6ae26d410fc3a Mon Sep 17 00:00:00 2001 From: 张晓飞 <32763586+henry860916@users.noreply.github.com> Date: Tue, 3 Jul 2018 17:04:19 +0800 Subject: Update debugger.md Fix issue of "function object is not subscriptable" in sample code. --- tensorflow/docs_src/guide/debugger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/guide/debugger.md b/tensorflow/docs_src/guide/debugger.md index dc4db58857..dad11e6226 100644 --- a/tensorflow/docs_src/guide/debugger.md +++ b/tensorflow/docs_src/guide/debugger.md @@ -781,7 +781,7 @@ sess.run(b) ``` python import numpy as np -a = tf.Variable(np.ones[10], name="a") +a = tf.Variable(np.ones(10), name="a") b = tf.add(a, a, name="b") sess = tf.Session() sess.run(tf.global_variables_initializer()) -- cgit v1.2.3 From 69e37cef0ca721f76d12a3808521d73299aab7ea Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 3 Jul 2018 12:29:18 +0000 Subject: Update calling of expand_dims with axis This fix updates calling of `expand_dims` with `dim -> axis` as the `dim=` in `tf.expand_dims` has been deprecated and was generating unnecessary warnings. Signed-off-by: Yong Tang --- tensorflow/contrib/learn/python/learn/estimators/head.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py index 339c4e0e36..dee0755204 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/head.py +++ b/tensorflow/contrib/learn/python/learn/estimators/head.py @@ -563,10 +563,10 @@ def _mean_squared_loss(labels, logits, weights=None): labels = ops.convert_to_tensor(labels) # To prevent broadcasting inside "-". if len(labels.get_shape()) == 1: - labels = array_ops.expand_dims(labels, dim=(1,)) + labels = array_ops.expand_dims(labels, axis=(1,)) # TODO(zakaria): make sure it does not recreate the broadcast bug. if len(logits.get_shape()) == 1: - logits = array_ops.expand_dims(logits, dim=(1,)) + logits = array_ops.expand_dims(logits, axis=(1,)) logits.get_shape().assert_is_compatible_with(labels.get_shape()) loss = math_ops.square(logits - math_ops.to_float(labels), name=name) return _compute_weighted_loss(loss, weights) -- cgit v1.2.3 From 00071753077dcd9f1486c1335f05eed80e68efcb Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 3 Jul 2018 12:32:35 +0000 Subject: Additional fix Signed-off-by: Yong Tang --- tensorflow/contrib/learn/python/learn/estimators/head.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py index dee0755204..e9c79f88b0 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/head.py +++ b/tensorflow/contrib/learn/python/learn/estimators/head.py @@ -579,10 +579,10 @@ def _poisson_loss(labels, logits, weights=None): labels = ops.convert_to_tensor(labels) # To prevent broadcasting inside "-". if len(labels.get_shape()) == 1: - labels = array_ops.expand_dims(labels, dim=(1,)) + labels = array_ops.expand_dims(labels, axis=(1,)) # TODO(zakaria): make sure it does not recreate the broadcast bug. if len(logits.get_shape()) == 1: - logits = array_ops.expand_dims(logits, dim=(1,)) + logits = array_ops.expand_dims(logits, axis=(1,)) logits.get_shape().assert_is_compatible_with(labels.get_shape()) loss = nn.log_poisson_loss(labels, logits, compute_full_loss=True, name=name) -- cgit v1.2.3 From a77a9689198675f62ced41eb5c737eec429b8fae Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 3 Jul 2018 12:33:58 +0000 Subject: Fix warning in _log_loss_with_two_classes as well Signed-off-by: Yong Tang --- tensorflow/contrib/learn/python/learn/estimators/head.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py index e9c79f88b0..ded93d4a7f 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/head.py +++ b/tensorflow/contrib/learn/python/learn/estimators/head.py @@ -797,7 +797,7 @@ def _log_loss_with_two_classes(labels, logits, weights=None): # TODO(ptucker): This will break for dynamic shapes. # sigmoid_cross_entropy_with_logits requires [batch_size, 1] labels. if len(labels.get_shape()) == 1: - labels = array_ops.expand_dims(labels, dim=(1,)) + labels = array_ops.expand_dims(labels, axis=(1,)) loss = nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits, name=name) return _compute_weighted_loss(loss, weights) -- cgit v1.2.3 From 486b96a51d6b0b394edf77d182f7283a8ec03e0d Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Tue, 3 Jul 2018 16:17:58 -0700 Subject: Update eager notebooks in 1.9 to match master --- .../nmt_with_attention/nmt_with_attention.ipynb | 909 +++++++++++++++++++++ .../eager/python/examples/notebooks/1_basics.ipynb | 429 ---------- .../python/examples/notebooks/2_gradients.ipynb | 323 -------- .../python/examples/notebooks/3_datasets.ipynb | 209 ----- .../examples/notebooks/3_training_models.ipynb | 485 ----------- .../python/examples/notebooks/4_high_level.ipynb | 551 ------------- .../eager/python/examples/notebooks/README.md | 11 + .../notebooks/automatic_differentiation.ipynb | 364 +++++++++ .../python/examples/notebooks/custom_layers.ipynb | 399 +++++++++ .../examples/notebooks/custom_training.ipynb | 478 +++++++++++ .../python/examples/notebooks/eager_basics.ipynb | 491 +++++++++++ 11 files changed, 2652 insertions(+), 1997 deletions(-) create mode 100644 tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/3_training_models.ipynb delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/4_high_level.ipynb create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/README.md create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/automatic_differentiation.ipynb create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/custom_layers.ipynb create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/eager_basics.ipynb diff --git a/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb new file mode 100644 index 0000000000..34ce5e0cc3 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb @@ -0,0 +1,909 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "nmt_with_attention.ipynb", + "version": "0.3.2", + "views": {}, + "default_view": {}, + "provenance": [ + { + "file_id": "1C4fpM7_7IL8ZzF7Gc5abywqQjeQNS2-U", + "timestamp": 1527858391290 + }, + { + "file_id": "1pExo6aUuw0S6MISFWoinfJv0Ftm9V4qv", + "timestamp": 1527776041613 + } + ], + "private_outputs": true, + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "metadata": { + "id": "AOpGoE2T-YXS", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "##### Copyright 2018 The TensorFlow Authors.\n", + "\n", + "Licensed under the Apache License, Version 2.0 (the \"License\").\n", + "\n", + "# Neural Machine Translation with Attention\n", + "\n", + "
\n", + "\n", + " Run in Google Colab \n", + "\n", + "View source on GitHub
" + ] + }, + { + "metadata": { + "id": "CiwtNgENbx2g", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "This notebook trains a sequence to sequence (seq2seq) model for Spanish to English translation using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). This is an advanced example that assumes some knowledge of sequence to sequence models.\n", + "\n", + "After training the model in this notebook, you will be able to input a Spanish sentence, such as *\"¿todavia estan en casa?\"*, and return the English translation: *\"are you still at home?\"*\n", + "\n", + "The translation quality is reasonable for a toy example, but the generated attention plot is perhaps more interesting. This shows which parts of the input sentence has the model's attention while translating:\n", + "\n", + "\"spanish-english\n", + "\n", + "Note: This example takes approximately 10 mintues to run on a single P100 GPU." + ] + }, + { + "metadata": { + "id": "tnxXKDjq3jEL", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "from __future__ import absolute_import, division, print_function\n", + "\n", + "# Import TensorFlow >= 1.9 and enable eager execution\n", + "import tensorflow as tf\n", + "\n", + "tf.enable_eager_execution()\n", + "\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "import unicodedata\n", + "import re\n", + "import numpy as np\n", + "import os\n", + "import time\n", + "\n", + "print(tf.__version__)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "wfodePkj3jEa", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Download and prepare the dataset\n", + "\n", + "We'll use a language dataset provided by http://www.manythings.org/anki/. This dataset contains language translation pairs in the format:\n", + "\n", + "```\n", + "May I borrow this book?\t¿Puedo tomar prestado este libro?\n", + "```\n", + "\n", + "There are a variety of languages available, but we'll use the English-Spanish dataset. For convenience, we've hosted a copy of this dataset on Google Cloud, but you can also download your own copy. After downloading the dataset, here are the steps we'll take to prepare the data:\n", + "\n", + "1. Add a *start* and *end* token to each sentence.\n", + "2. Clean the sentences by removing special characters.\n", + "3. Create a word index and reverse word index (dictionaries mapping from word → id and id → word).\n", + "4. Pad each sentence to a maximum length." + ] + }, + { + "metadata": { + "id": "kRVATYOgJs1b", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Download the file\n", + "path_to_zip = tf.keras.utils.get_file(\n", + " 'spa-eng.zip', origin='http://download.tensorflow.org/data/spa-eng.zip', \n", + " extract=True)\n", + "\n", + "path_to_file = os.path.dirname(path_to_zip)+\"/spa-eng/spa.txt\"" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "rd0jw-eC3jEh", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Converts the unicode file to ascii\n", + "def unicode_to_ascii(s):\n", + " return ''.join(c for c in unicodedata.normalize('NFD', s)\n", + " if unicodedata.category(c) != 'Mn')\n", + "\n", + "\n", + "def preprocess_sentence(w):\n", + " w = unicode_to_ascii(w.lower().strip())\n", + " \n", + " # creating a space between a word and the punctuation following it\n", + " # eg: \"he is a boy.\" => \"he is a boy .\" \n", + " # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n", + " w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n", + " w = re.sub(r'[\" \"]+', \" \", w)\n", + " \n", + " # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n", + " w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n", + " \n", + " w = w.rstrip().strip()\n", + " \n", + " # adding a start and an end token to the sentence\n", + " # so that the model know when to start and stop predicting.\n", + " w = ' ' + w + ' '\n", + " return w" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "OHn4Dct23jEm", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# 1. Remove the accents\n", + "# 2. Clean the sentences\n", + "# 3. Return word pairs in the format: [ENGLISH, SPANISH]\n", + "def create_dataset(path, num_examples):\n", + " lines = open(path, encoding='UTF-8').read().strip().split('\\n')\n", + " \n", + " word_pairs = [[preprocess_sentence(w) for w in l.split('\\t')] for l in lines[:num_examples]]\n", + " \n", + " return word_pairs" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "9xbqO7Iie9bb", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# This class creates a word -> index mapping (e.g,. \"dad\" -> 5) and vice-versa \n", + "# (e.g., 5 -> \"dad\") for each language,\n", + "class LanguageIndex():\n", + " def __init__(self, lang):\n", + " self.lang = lang\n", + " self.word2idx = {}\n", + " self.idx2word = {}\n", + " self.vocab = set()\n", + " \n", + " self.create_index()\n", + " \n", + " def create_index(self):\n", + " for phrase in self.lang:\n", + " self.vocab.update(phrase.split(' '))\n", + " \n", + " self.vocab = sorted(self.vocab)\n", + " \n", + " self.word2idx[''] = 0\n", + " for index, word in enumerate(self.vocab):\n", + " self.word2idx[word] = index + 1\n", + " \n", + " for word, index in self.word2idx.items():\n", + " self.idx2word[index] = word" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "eAY9k49G3jE_", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def max_length(tensor):\n", + " return max(len(t) for t in tensor)\n", + "\n", + "\n", + "def load_dataset(path, num_examples):\n", + " # creating cleaned input, output pairs\n", + " pairs = create_dataset(path, num_examples)\n", + "\n", + " # index language using the class defined above \n", + " inp_lang = LanguageIndex(sp for en, sp in pairs)\n", + " targ_lang = LanguageIndex(en for en, sp in pairs)\n", + " \n", + " # Vectorize the input and target languages\n", + " \n", + " # Spanish sentences\n", + " input_tensor = [[inp_lang.word2idx[s] for s in sp.split(' ')] for en, sp in pairs]\n", + " \n", + " # English sentences\n", + " target_tensor = [[targ_lang.word2idx[s] for s in en.split(' ')] for en, sp in pairs]\n", + " \n", + " # Calculate max_length of input and output tensor\n", + " # Here, we'll set those to the longest sentence in the dataset\n", + " max_length_inp, max_length_tar = max_length(input_tensor), max_length(target_tensor)\n", + " \n", + " # Padding the input and output tensor to the maximum length\n", + " input_tensor = tf.keras.preprocessing.sequence.pad_sequences(input_tensor, \n", + " maxlen=max_length_inp,\n", + " padding='post')\n", + " \n", + " target_tensor = tf.keras.preprocessing.sequence.pad_sequences(target_tensor, \n", + " maxlen=max_length_tar, \n", + " padding='post')\n", + " \n", + " return input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_tar" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "GOi42V79Ydlr", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Limit the size of the dataset to experiment faster (optional)\n", + "\n", + "Training on the complete dataset of >100,000 sentences will take a long time. To train faster, we can limit the size of the dataset to 30,000 sentences (of course, translation quality degrades with less data):" + ] + }, + { + "metadata": { + "id": "cnxC7q-j3jFD", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Try experimenting with the size of that dataset\n", + "num_examples = 30000\n", + "input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_targ = load_dataset(path_to_file, num_examples)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "4QILQkOs3jFG", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Creating training and validation sets using an 80-20 split\n", + "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n", + "\n", + "# Show length\n", + "len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "rgCLkfv5uO3d", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Create a tf.data dataset" + ] + }, + { + "metadata": { + "id": "TqHsArVZ3jFS", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "BUFFER_SIZE = len(input_tensor_train)\n", + "BATCH_SIZE = 64\n", + "embedding_dim = 256\n", + "units = 1024\n", + "vocab_inp_size = len(inp_lang.word2idx)\n", + "vocab_tar_size = len(targ_lang.word2idx)\n", + "\n", + "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n", + "dataset = dataset.apply(tf.contrib.data.batch_and_drop_remainder(BATCH_SIZE))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "TNfHIF71ulLu", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Write the encoder and decoder model\n", + "\n", + "Here, we'll implement an encoder-decoder model with attention which you can read about in the TensorFlow [Neural Machine Translation (seq2seq) tutorial](https://www.tensorflow.org/tutorials/seq2seq). This example uses a more recent set of APIs. This notebook implements the [attention equations](https://www.tensorflow.org/tutorials/seq2seq#background_on_the_attention_mechanism) from the seq2seq tutorial. The following diagram shows that each input words is assigned a weight by the attention mechanism which is then used by the decoder to predict the next word in the sentence.\n", + "\n", + "\"attention\n", + "\n", + "The input is put through an encoder model which gives us the encoder output of shape *(batch_size, max_length, hidden_size)* and the encoder hidden state of shape *(batch_size, hidden_size)*. \n", + "\n", + "Here are the equations that are implemented:\n", + "\n", + "\"attention\n", + "\"attention\n", + "\n", + "We're using *Bahdanau attention*. Lets decide on notation before writing the simplified form:\n", + "\n", + "* FC = Fully connected (dense) layer\n", + "* EO = Encoder output\n", + "* H = hidden state\n", + "* X = input to the decoder\n", + "\n", + "And the pseudo-code:\n", + "\n", + "* `score = FC(tanh(FC(EO) + FC(H)))`\n", + "* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, hidden_size)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n", + "* `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n", + "* `embedding output` = The input to the decoder X is passed through an embedding layer.\n", + "* `merged vector = concat(embedding output, context vector)`\n", + "* This merged vector is then given to the GRU\n", + " \n", + "The shapes of all the vectors at each step have been specified in the comments in the code:" + ] + }, + { + "metadata": { + "id": "avyJ_4VIUoHb", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def gru(units):\n", + " # If you have a GPU, we recommend using CuDNNGRU(provides a 3x speedup than GRU)\n", + " # the code automatically does that.\n", + " if tf.test.is_gpu_available():\n", + " return tf.keras.layers.CuDNNGRU(units, \n", + " return_sequences=True, \n", + " return_state=True, \n", + " recurrent_initializer='glorot_uniform')\n", + " else:\n", + " return tf.keras.layers.GRU(units, \n", + " return_sequences=True, \n", + " return_state=True, \n", + " recurrent_activation='sigmoid', \n", + " recurrent_initializer='glorot_uniform')" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "nZ2rI24i3jFg", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "class Encoder(tf.keras.Model):\n", + " def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n", + " super(Encoder, self).__init__()\n", + " self.batch_sz = batch_sz\n", + " self.enc_units = enc_units\n", + " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", + " self.gru = gru(self.enc_units)\n", + " \n", + " def call(self, x, hidden):\n", + " x = self.embedding(x)\n", + " output, state = self.gru(x, initial_state = hidden) \n", + " return output, state\n", + " \n", + " def initialize_hidden_state(self):\n", + " return tf.zeros((self.batch_sz, self.enc_units))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "yJ_B3mhW3jFk", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "class Decoder(tf.keras.Model):\n", + " def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n", + " super(Decoder, self).__init__()\n", + " self.batch_sz = batch_sz\n", + " self.dec_units = dec_units\n", + " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", + " self.gru = gru(self.dec_units)\n", + " self.fc = tf.keras.layers.Dense(vocab_size)\n", + " \n", + " # used for attention\n", + " self.W1 = tf.keras.layers.Dense(self.dec_units)\n", + " self.W2 = tf.keras.layers.Dense(self.dec_units)\n", + " self.V = tf.keras.layers.Dense(1)\n", + " \n", + " def call(self, x, hidden, enc_output):\n", + " # enc_output shape == (batch_size, max_length, hidden_size)\n", + " \n", + " # hidden shape == (batch_size, hidden size)\n", + " # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n", + " # we are doing this to perform addition to calculate the score\n", + " hidden_with_time_axis = tf.expand_dims(hidden, 1)\n", + " \n", + " # score shape == (batch_size, max_length, hidden_size)\n", + " score = tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis))\n", + " \n", + " # attention_weights shape == (batch_size, max_length, 1)\n", + " # we get 1 at the last axis because we are applying score to self.V\n", + " attention_weights = tf.nn.softmax(self.V(score), axis=1)\n", + " \n", + " # context_vector shape after sum == (batch_size, hidden_size)\n", + " context_vector = attention_weights * enc_output\n", + " context_vector = tf.reduce_sum(context_vector, axis=1)\n", + " \n", + " # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n", + " x = self.embedding(x)\n", + " \n", + " # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n", + " x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n", + " \n", + " # passing the concatenated vector to the GRU\n", + " output, state = self.gru(x)\n", + " \n", + " # output shape == (batch_size * max_length, hidden_size)\n", + " output = tf.reshape(output, (-1, output.shape[2]))\n", + " \n", + " # output shape == (batch_size * max_length, vocab)\n", + " x = self.fc(output)\n", + " \n", + " return x, state, attention_weights\n", + " \n", + " def initialize_hidden_state(self):\n", + " return tf.zeros((self.batch_sz, self.dec_units))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "P5UY8wko3jFp", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n", + "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "_ch_71VbIRfK", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Define the optimizer and the loss function" + ] + }, + { + "metadata": { + "id": "WmTHr5iV3jFr", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "optimizer = tf.train.AdamOptimizer()\n", + "\n", + "\n", + "def loss_function(real, pred):\n", + " mask = 1 - np.equal(real, 0)\n", + " loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n", + " return tf.reduce_mean(loss_)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "hpObfY22IddU", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Training\n", + "\n", + "1. Pass the *input* through the *encoder* which return *encoder output* and the *encoder hidden state*.\n", + "2. The encoder output, encoder hidden state and the decoder input (which is the *start token*) is passed to the decoder.\n", + "3. The decoder returns the *predictions* and the *decoder hidden state*.\n", + "4. The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n", + "5. Use *teacher forcing* to decide the next input to the decoder.\n", + "6. *Teacher forcing* is the technique where the *target word* is passed as the *next input* to the decoder.\n", + "7. The final step is to calculate the gradients and apply it to the optimizer and backpropagate." + ] + }, + { + "metadata": { + "id": "ddefjBMa3jF0", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "EPOCHS = 10\n", + "\n", + "for epoch in range(EPOCHS):\n", + " start = time.time()\n", + " \n", + " hidden = encoder.initialize_hidden_state()\n", + " total_loss = 0\n", + " \n", + " for (batch, (inp, targ)) in enumerate(dataset):\n", + " loss = 0\n", + " \n", + " with tf.GradientTape() as tape:\n", + " enc_output, enc_hidden = encoder(inp, hidden)\n", + " \n", + " dec_hidden = enc_hidden\n", + " \n", + " dec_input = tf.expand_dims([targ_lang.word2idx['']] * BATCH_SIZE, 1) \n", + " \n", + " # Teacher forcing - feeding the target as the next input\n", + " for t in range(1, targ.shape[1]):\n", + " # passing enc_output to the decoder\n", + " predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n", + " \n", + " loss += loss_function(targ[:, t], predictions)\n", + " \n", + " # using teacher forcing\n", + " dec_input = tf.expand_dims(targ[:, t], 1)\n", + " \n", + " total_loss += (loss / int(targ.shape[1]))\n", + " \n", + " variables = encoder.variables + decoder.variables\n", + " \n", + " gradients = tape.gradient(loss, variables)\n", + " \n", + " optimizer.apply_gradients(zip(gradients, variables), tf.train.get_or_create_global_step())\n", + "\n", + " if batch % 100 == 0:\n", + " print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n", + " batch,\n", + " loss.numpy() / int(targ.shape[1])))\n", + " \n", + " print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n", + " total_loss/len(input_tensor)))\n", + " print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "mU3Ce8M6I3rz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Translate\n", + "\n", + "* The evaluate function is similar to the training loop, except we don't use *teacher forcing* here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n", + "* Stop predicting when the model predicts the *end token*.\n", + "* And store the *attention weights for every time step*.\n", + "\n", + "Note: The encoder output is calculated only once for one input." + ] + }, + { + "metadata": { + "id": "EbQpyYs13jF_", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n", + " attention_plot = np.zeros((max_length_targ, max_length_inp))\n", + " \n", + " sentence = preprocess_sentence(sentence)\n", + "\n", + " inputs = [inp_lang.word2idx[i] for i in sentence.split(' ')]\n", + " inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n", + " inputs = tf.convert_to_tensor(inputs)\n", + " \n", + " result = ''\n", + "\n", + " hidden = [tf.zeros((1, units))]\n", + " enc_out, enc_hidden = encoder(inputs, hidden)\n", + "\n", + " dec_hidden = enc_hidden\n", + " dec_input = tf.expand_dims([targ_lang.word2idx['']], 0)\n", + "\n", + " for t in range(max_length_targ):\n", + " predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n", + " \n", + " # storing the attention weigths to plot later on\n", + " attention_weights = tf.reshape(attention_weights, (-1, ))\n", + " attention_plot[t] = attention_weights.numpy()\n", + "\n", + " predicted_id = tf.multinomial(tf.exp(predictions), num_samples=1)[0][0].numpy()\n", + "\n", + " result += targ_lang.idx2word[predicted_id] + ' '\n", + "\n", + " if targ_lang.idx2word[predicted_id] == '':\n", + " return result, sentence, attention_plot\n", + " \n", + " # the predicted ID is fed back into the model\n", + " dec_input = tf.expand_dims([predicted_id], 0)\n", + "\n", + " return result, sentence, attention_plot" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "s5hQWlbN3jGF", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# function for plotting the attention weights\n", + "def plot_attention(attention, sentence, predicted_sentence):\n", + " fig = plt.figure(figsize=(10,10))\n", + " ax = fig.add_subplot(1, 1, 1)\n", + " ax.matshow(attention, cmap='viridis')\n", + " \n", + " fontdict = {'fontsize': 14}\n", + " \n", + " ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n", + " ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n", + "\n", + " plt.show()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "sl9zUHzg3jGI", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def translate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n", + " result, sentence, attention_plot = evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)\n", + " \n", + " print('Input: {}'.format(sentence))\n", + " print('Predicted translation: {}'.format(result))\n", + " \n", + " attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n", + " plot_attention(attention_plot, sentence.split(' '), result.split(' '))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "WrAM0FDomq3E", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "translate('hace mucho frio aqui.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "zSx2iM36EZQZ", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "translate('esta es mi vida.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "A3LLCx3ZE0Ls", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "translate('¿todavia estan en casa?', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "DUQVLVqUE1YW", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# wrong translation\n", + "translate('trata de averiguarlo.', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "RTe5P5ioMJwN", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Next steps\n", + "\n", + "* [Download a different dataset](http://www.manythings.org/anki/) to experiment with translations, for example, English to German, or English to French.\n", + "* Experiment with training on a larger dataset, or using more epochs\n" + ] + } + ] +} \ No newline at end of file diff --git a/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb deleted file mode 100644 index 51d10a7784..0000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb +++ /dev/null @@ -1,429 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U9i2Dsh-ziXr" - }, - "source": [ - "# An introduction to TensorFlow\n", - "\n", - "This is an introductory tutorial for using TensorFlow. It will cover:\n", - "\n", - "* Importing required packages\n", - "* Creating and using Tensors\n", - "* Using GPU acceleration\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1JcS5iBXMRO" - }, - "source": [ - "## Import TensorFlow\n", - "\n", - "To get started, import the `tensorflow` module and enable eager execution.\n", - "Eager execution enables a more interactive frontend to TensorFlow, the details of which we will discuss much later." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "RlIWhyeLoYnG" - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H9UySOPLXdaw" - }, - "source": [ - "## Tensors\n", - "\n", - "A Tensor is a multi-dimensional array. Similar to NumPy `ndarray` objects, `Tensor` objects have a data type and a shape. Additionally, Tensors can reside in accelerator (like GPU) memory. TensorFlow offers a rich library of operations ([tf.add](https://www.tensorflow.org/api_docs/python/tf/add), [tf.matmul](https://www.tensorflow.org/api_docs/python/tf/matmul), [tf.linalg.inv](https://www.tensorflow.org/api_docs/python/tf/linalg/inv) etc.) that consume and produce Tensors. These operations automatically convert native Python types. For example:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 125 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 320, - "status": "ok", - "timestamp": 1526420535530, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "ngUe237Wt48W", - "outputId": "b1a1cd60-4eb3-443d-cd6b-68406390784e" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tf.Tensor(3, shape=(), dtype=int32)\n", - "tf.Tensor([4 6], shape=(2,), dtype=int32)\n", - "tf.Tensor(25, shape=(), dtype=int32)\n", - "tf.Tensor(6, shape=(), dtype=int32)\n", - "tf.Tensor(aGVsbG8gd29ybGQ, shape=(), dtype=string)\n", - "tf.Tensor(13, shape=(), dtype=int32)\n" - ] - } - ], - "source": [ - "print(tf.add(1, 2))\n", - "print(tf.add([1, 2], [3, 4]))\n", - "print(tf.square(5))\n", - "print(tf.reduce_sum([1, 2, 3]))\n", - "print(tf.encode_base64(\"hello world\"))\n", - "\n", - "# Operator overloading is also supported\n", - "print(tf.square(2) + tf.square(3))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IDY4WsYRhP81" - }, - "source": [ - "Each Tensor has a shape and a datatype" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 53 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 215, - "status": "ok", - "timestamp": 1526420538162, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "srYWH1MdJNG7", - "outputId": "5e4ac41c-5115-4e50-eba0-42e249c16561" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(1, 2)\n", - "\u003cdtype: 'int32'\u003e\n" - ] - } - ], - "source": [ - "x = tf.matmul([[1]], [[2, 3]])\n", - "print(x.shape)\n", - "print(x.dtype)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eBPw8e8vrsom" - }, - "source": [ - "The most obvious differences between NumPy arrays and TensorFlow Tensors are:\n", - "\n", - "1. Tensors can be backed by accelerator memory (like GPU, TPU).\n", - "2. Tensors are immutable." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dwi1tdW3JBw6" - }, - "source": [ - "### NumPy Compatibility\n", - "\n", - "Conversion between TensorFlow Tensors and NumPy ndarrays is quite simple as:\n", - "* TensorFlow operations automatically convert NumPy ndarrays to Tensors.\n", - "* NumPy operations automatically convert Tensors to NumPy ndarrays.\n", - "\n", - "Tensors can be explicitly converted to NumPy ndarrays by invoking the `.numpy()` method on them.\n", - "These conversions are typically cheap as the array and Tensor share the underlying memory representation if possible. However, sharing the underlying representation isn't always possible since the Tensor may be hosted in GPU memory while NumPy arrays are always backed by host memory, and the conversion will thus involve a copy from GPU to host memory." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 251 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 238, - "status": "ok", - "timestamp": 1526420540562, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "lCUWzso6mbqR", - "outputId": "fd0a22bc-8249-49dd-fcbd-63161cc47e46" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "TensorFlow operations convert numpy arrays to Tensors automatically\n", - "tf.Tensor(\n", - "[[ 42. 42. 42.]\n", - " [ 42. 42. 42.]\n", - " [ 42. 42. 42.]], shape=(3, 3), dtype=float64)\n", - "And NumPy operations convert Tensors to numpy arrays automatically\n", - "[[ 43. 43. 43.]\n", - " [ 43. 43. 43.]\n", - " [ 43. 43. 43.]]\n", - "The .numpy() method explicitly converts a Tensor to a numpy array\n", - "[[ 42. 42. 42.]\n", - " [ 42. 42. 42.]\n", - " [ 42. 42. 42.]]\n" - ] - } - ], - "source": [ - "import numpy as np\n", - "\n", - "ndarray = np.ones([3, 3])\n", - "\n", - "print(\"TensorFlow operations convert numpy arrays to Tensors automatically\")\n", - "tensor = tf.multiply(ndarray, 42)\n", - "print(tensor)\n", - "\n", - "\n", - "print(\"And NumPy operations convert Tensors to numpy arrays automatically\")\n", - "print(np.add(tensor, 1))\n", - "\n", - "print(\"The .numpy() method explicitly converts a Tensor to a numpy array\")\n", - "print(tensor.numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PBNP8yTRfu_X" - }, - "source": [ - "## GPU acceleration\n", - "\n", - "Many TensorFlow operations can be accelerated by using the GPU for computation. Without any annotations, TensorFlow automatically decides whether to use the GPU or CPU for an operation (and copies the tensor between CPU and GPU memory if necessary). Tensors produced by an operation are typically backed by the memory of the device on which the operation executed. For example:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 53 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 340, - "status": "ok", - "timestamp": 1526420543562, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "3Twf_Rw-gQFM", - "outputId": "2239ae2b-adf3-4895-b1f3-464cf5361d1b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Is there a GPU available: False\n", - "Is the Tensor on GPU #0: False\n" - ] - } - ], - "source": [ - "x = tf.random_uniform([3, 3])\n", - "\n", - "print(\"Is there a GPU available: \"),\n", - "print(tf.test.is_gpu_available())\n", - "\n", - "print(\"Is the Tensor on GPU #0: \"),\n", - "print(x.device.endswith('GPU:0'))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vpgYzgVXW2Ud" - }, - "source": [ - "### Device Names\n", - "\n", - "The `Tensor.device` property provides a fully qualified string name of the device hosting the contents of the Tensor. This name encodes a bunch of details, such as an identifier of the network address of the host on which this program is executing and the device within that host. This is required for distributed execution of TensorFlow programs, but we'll skip that for now. The string will end with `GPU:\u003cN\u003e` if the tensor is placed on the `N`-th tensor on the host." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZWZQCimzuqyP" - }, - "source": [ - "\n", - "\n", - "### Explicit Device Placement\n", - "\n", - "The term \"placement\" in TensorFlow refers to how individual operations are assigned (placed on) a device for execution. As mentioned above, when there is no explicit guidance provided, TensorFlow automatically decides which device to execute an operation, and copies Tensors to that device if needed. However, TensorFlow operations can be explicitly placed on specific devices using the `tf.device` context manager. For example:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 53 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 1762, - "status": "ok", - "timestamp": 1526420547562, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "RjkNZTuauy-Q", - "outputId": "2e613293-ccac-4db2-b793-8ceb5b5adcfd" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "On CPU:\n", - "10 loops, best of 3: 35.8 ms per loop\n" - ] - } - ], - "source": [ - "def time_matmul(x):\n", - " %timeit tf.matmul(x, x)\n", - "\n", - "# Force execution on CPU\n", - "print(\"On CPU:\")\n", - "with tf.device(\"CPU:0\"):\n", - " x = tf.random_uniform([1000, 1000])\n", - " assert x.device.endswith(\"CPU:0\")\n", - " time_matmul(x)\n", - "\n", - "# Force execution on GPU #0 if available\n", - "if tf.test.is_gpu_available():\n", - " with tf.device(\"GPU:0\"): # Or GPU:1 for the 2nd GPU, GPU:2 for the 3rd etc.\n", - " x = tf.random_uniform([1000, 1000])\n", - " assert x.device.endswith(\"GPU:0\")\n", - " time_matmul(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YEOJTNiOvnpQ" - }, - "source": [ - "## Next Steps\n", - "\n", - "In this tutorial we covered the most fundamental concepts in TensorFlow - `Tensor`s, operations, and devices.\n", - "In [the next tutorial](https://github.com/tensorflow/models/tree/master/official/contrib/eager/python/examples/notebooks/2_gradients.ipynb) we will cover automatic differentiation - a building block required for training many machine learning models like neural networks." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "default_view": {}, - "name": "TensorFlow: An introduction", - "provenance": [], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb deleted file mode 100644 index 9c1af9c208..0000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb +++ /dev/null @@ -1,323 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vDJ4XzMqodTy" - }, - "source": [ - "# Automatic Differentiation\n", - "\n", - "In the previous tutorial we introduced `Tensor`s and operations on them. In this tutorial we will cover [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation), a key technique for optimizing machine learning models." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GQJysDM__Qb0" - }, - "source": [ - "## Setup\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "OiMPZStlibBv" - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "tf.enable_eager_execution()\n", - "\n", - "tfe = tf.contrib.eager # Shorthand for some symbols" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1CLWJl0QliB0" - }, - "source": [ - "## Derivatives of a function\n", - "\n", - "TensorFlow provides APIs for automatic differentiation - computing the derivative of a function. The way that more closely mimics the math is to encapsulate the computation in a Python function, say `f`, and use `tfe.gradients_function` to create a function that computes the derivatives of `f` with respect to its arguments. If you're familiar with [autograd](https://github.com/HIPS/autograd) for differentiating numpy functions, this will be familiar. For example: " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "9FViq92UX7P8" - }, - "outputs": [], - "source": [ - "from math import pi\n", - "\n", - "def f(x):\n", - " return tf.square(tf.sin(x))\n", - "\n", - "assert f(pi/2).numpy() == 1.0\n", - "\n", - "\n", - "# grad_f will return a list of derivatives of f\n", - "# with respect to its arguments. Since f() has a single argument,\n", - "# grad_f will return a list with a single element.\n", - "grad_f = tfe.gradients_function(f)\n", - "assert tf.abs(grad_f(pi/2)[0]).numpy() \u003c 1e-7" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "v9fPs8RyopCf" - }, - "source": [ - "### Higher-order gradients\n", - "\n", - "The same API can be used to differentiate as many times as you like:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 276 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 730, - "status": "ok", - "timestamp": 1527005655565, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "3D0ZvnGYo0rW", - "outputId": "e23f8cc6-6813-4944-f20f-825b8a03c2ff" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEDCAYAAAAhsS8XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXd0HNX5sJ/ZXrTq3ZLV3IvcDdgGGwOm2WCbHhJa6C2B\nUBISQioBfoQPkjhACA4QCIQSDITQbGMbsHHvVbZ6s7q0vc18f4xmJVltJa0q+5zDOXhn9s7dqzvv\nfe/briBJkkSYMGHChBkxqAa7A2HChAkTJrSEBXuYMGHCjDDCgj1MmDBhRhhhwR4mTJgwI4ywYA8T\nJkyYEUZYsIcJEybMCCNkgl0URVasWMHtt98eqibDhAkTJkwvCJlgf+2118jJyQlVc2HChAkTppeE\nRLBXVlayceNGrrjiilA0FyZMmDBh+kBIBPvjjz/OQw89hCAIoWguTJgwYcL0gT4L9g0bNhAfH8/E\niRMJVycIEyZMmMFH6GutmGeeeYYPP/wQtVqN2+3Gbrdz3nnn8dRTT3X6HUmSwtp9CKittvH8UxsQ\nxZY/4aXXTGfa7PRB7NXAU1dj5y9PrIfmYUgeFcnya2aQmBI5uB0bYE5WNPHS/9uE6JcHYukVucw8\nPWOQezXw7NhcyCfvH0Bqfi+uumkO4ycnD3KvBpY+C/bWbNu2jdWrV/PCCy90e291tTVUj+03EhIs\nQ7qfWzfls2tzMTNPH01UrJEv/3eU5LRIVnx/5mB3rUP6azw3fnaMQ7vLOX1RNrVVNvIOVZGeFcPS\nq6YNmT6GmlP7KYoi/3ltF9WVNhacO4btXxfi9fi5+Mpc0jJjhkw/+5t9O0r5Zu1xDEYtpy/KZuOn\nR4mOM3HlTbNRqTo3UAynv3swhOPYhymSJJF3sAqtTs35l05mQm4K6VkxVJY2UVdtH+zuDRgOu4ej\n+yqIjDYwbW4a514yiYTkCMqKGnC7vIPdvQFjz9YSqittjJuSxNTZaVywcgoAX3xwCL9PHOTeDRyH\ndpej0ai47PqZTJyWwoTcFOprHBzdf3KwuzaghFSwz507NyhtPUzfOVnehLXRRdbYeLQ6DQATp6UC\ncGhv+WB2bUA5sLMMv19i2pz0gEaWNS4BUZQoOlE3yL0bGDxuHzu+LsRk1jH/nDEApI6OZtL0VFxO\nLyfLmwa5hwNDU4OT+loHozJiiIw2AjB7QSYajYrtXxXg9foHuYcDR1hjH6bkHawCYOzkxMBnmWPj\nMJq1HDtwEt93YBJ7PT4O7CrDYNQwPrfFhpo1Lh6AgmPVg9W1AaWyrBG/X2JCbjIGozbweXqWbIIp\nLawfrK4NKMX58kI+Oic28FmERc/UOWnYbR7yDn53tPawYB+GiKLI8SNVGEzaNvZTtVrFhKkpuF0+\n8o+OfKGWd7gKt8vHlJmj0GrVgc9j4kxExRopzq/7Tixw5cWNAKSkR7f5PHV0NIIApUXfEcHevEMb\nnR3b5vPxU5IAqChpHPA+DRZhwT4MKS2sx+XwMmZCYjuH0MRpKQAc3lsxGF0bUJQXNWdiYpvPBUEg\ne1w8Pq9IyXdAW60oaUAQ5Gig1uj0GhJTIqkqb8Lj9g1S7wYGn89PWXE90XGmgBlGITrWhN6gobIs\nLNjDDGE6MsMoRMUYSUiOoLKsacQ7zaoqrGh1amLiTO2uZY1LAKDgWM1Ad2tA8Xr9VFVYSUi2oNNr\n2l1Py4xBkqC8pGEQejdwVJQ04vOKZJyirYO80CeNiqSpwYXD5h6E3g08YcE+zJAkiZKCOswWHUmp\nHcdpJyRbEEWJupqRGx3jdvloqHWQmGLpMCciMcWCOUJH0fEaRHHkLnBV5U2IokRKelSH10dlyOaZ\nkW5nD5hhcuI6vJ48Sh6fyrLvhiM5LNiHGQ67B6fDS2JyZKdJXgnJcqxrdeXQj8vtLcpv6ywJSRAE\nMsfF43L6RvTLXF4sa+Kn2tcVkkdFodGoKCvqu8b+zjtv8f3vX8Fvf/ton9sKNUX5tWi0KlLSOl7g\nFDPVSJ4LrWm/dwszpKk5aQMgLimi03u+C4JdCeFLTOk8YSMlLYqDu8qpOWkjtRPBN9wpb/YzpHai\nsas1KlLSoygpqMdhc2OK0Pf6WWvWvMsf//hnkpNTet1Gf9BY76Sxzknm2DjUmo51VXlnBye/I3b2\nsMY+zKitkgV7fGLngj02wYxKLVBdaRuobg04VRWyYO/MHAUQlyCPkTJmIw2/X+RkeRNxCWb0Bm2n\n943KaA577IPW/vTTf6C8vIyHH76ft99+s9ft9AeKUzQto/MMW61OQ1xiBFWV1hHve4Kwxj7sUDT2\n+C40drVaRVyCmdpqG36/iFo9stZvSZKoKrditugwWzrXQKNijahUwojLxH17/XF25VXj8fhx+Hzo\nGh1s/+vmTu8X/SJ2RA59egTDxhMd3jNnQiJXLh7TaRsPPPAztm79lj//+UUiI4dWDZ76Zl9SXBfK\nDshmqZqTNqpPWgM295HKyHrjvwPUVNnQGzRERHa9pU5ItiD6pREn1ADsVjcOu6fbIl9qtYqYeBN1\nNfYRWXlU0Ty7W7hVzddFf181VYlApbUhhDLHYxPMXd6XnCbPl5PfATt7WGMfRng9PhrrnM2JJ11X\nx5Tt7BVUn7QGbO4jhZPliuO0+98VlxBBbZWdpgYnUTHtwyKHI1cuHsNdV83g1b9upuhELdf/cG63\ntvN/v7ydpgYnN99xxoirrFpX48Bk1rXJuu2IlsiYRqYxsiughjX2YURts2bSlRlGocWBOvLsy4p9\nPZiyvLGJshZXWzXydi71tXYMJm1QDtHYeBM+r4itaWTFcXs9PqyNLmLiu1+0IyL1mCN0VJY2jcgd\nXGvCgn0YEbCvd2NLBIiNN6NSCdSMwMiYqoqeaeww8hyoPq9fFmixwe1CouPkBa6+ti8L3NDT9Otq\nHED3ZhiQQ2ATUyNx2D3YbZ7+7tqgEhbsw4hgHKcKao2K2AQztVWyA3WkIIoS1ZVWYuJNHWZankpc\n8wtfO8J8DbLfAKI7yLrtiNhmjba+WRD2hnfe+YDIyKHldAzY1+O7F+xAIEu5sa734zAcCAv2YURt\nlQ2VWgj6ZU5ItuD3S4GogZFAU4MTr8dPQlJwfgNThA6DUTPinMi11fIi31E5hY5Q5kx97cgSaMrc\nDkZjB7luDEBDnbPf+jQUCAv2YYIoitRW24mNNwcdvjgS7eyN9fILGR1r7OZOGUEQiE2IoLFeXhBG\nCjXNpqXoYE0xMSYEoa+mmKGHUjYjJi44wR7VPG/CGnuYIUFDnRO/TwzKDKOg3DuS7MuNzZpWVJAC\nDVrMMSOpdk5AsAepsas1KiJjjNTXOEaU47Cuxo7ZokdvCC7AL6yxhxlS9MRxqqBotU0NI2cSN9bL\nmlZUTHAaO7Qkrijmi5GAYpazRBmC/k5snBm3y4fTMTKODHS7vNitnqDNMAAGoxaDUUNDWGMPMxRQ\nbMTdZde1Rm/QojdoaGxw9Ve3BhzFFNMTwa68+HUjJORRkiRqquxExciZtcESHXCgjoxxCETEBBHq\n2JqoWBNNDc4RFVRwKn0W7B6PhyuuuILly5ezbNky/vKXv4SiX2FOQdG6eyLQlPubGpyI4sjYfjfW\nOzGatEFFxCgoERMjJTLGYfPgcfuCdpwqxI4wB2rAcRpkRIxCdIwRSQJr48hReE6lz4Jdp9Px2muv\nsWbNGtasWcOmTZvYt29fKPoWphVNDU7UGhWmCF2PvhcZbUT0S9itwz8xxe8XsTa6Ag6wYNHq1ETF\nGKkbIaYYRTAHa19XiGkWgL0NeWxdtvebb77ijTdeDfq7lZUVfPHFp0Hd+/jjv2bjxvXd3te6lMCa\nNe/x2Wf/C6r9qICdXR6HTz75L9XVLUdJPvnk7ykqKgyqraFKSEoKGI3yi+bxePD5RvYRXINFU4OL\nyChDj9PBFQ2/sd7ZI3vsUMTa6EKS6FVpgMgYIyX5TjxuX4+0/aGIIpCCTU5SUByHvY2MObVs7/z5\nZ7a7x+/3o1ar231eXl7GF198xnnnXdCrZ3eE4gyPjDawfPllQX9PGQfFEf+//33EzJlTSUrKAODh\nh38esj4OFiGZ4aIosnLlSoqLi7n22mvJzc0NRbNhmnG7vLhdvnZnWgZDZLQszGVTTudlTYcDgYiY\nHpqjACKjlHFw9SiyaCjS0EuNXatTY4nU98oU07ps78UXX4LFYuHIkUPcd99DPP74r7FYIsnLO8r4\n8ROZP/9MnnvuaQRBQKvV8OyzL/Dii6soKirkppuu5YILlnLllde0af+ZZ55k9+6dpKSktonaOXr0\nCH/+8zO4XC6ioqL5+c8fIzY2jnvuuQ3RFUt1XSGxH5Zht9sxmUycccYCfve7x3jpJXk3UVlZwcMP\n38+rr77JK6/8nW+++QqHw4lWSmTS9HvYsGEdR44c5sEHH0Sj0fL886t54IF7ufvu+zh8+ADl5eXc\neee9gKzZHz16hB//+AE+//wT3nnnLfx+H5MmTeEnP/npkKrBExLBrlKpWLNmDTabjTvvvJPjx48z\nZkznJUDD9IymZufnqYf0BoMiBEdCZExDLyJiFJQFztroHPaC/Vv315ROK+RP+ZsRCnomTJxjPfh8\nInnfbGgjiGYkTmXlmKWdfu/Usr2ffPLfNt8vLS3mT396AYCHH76Pn/zkp0yZkktEhIamJg+33343\nb731Ok8++f/atb1x45eUlpbwz3++TU1NDd///hUsXXopPp+PZ599iieeeIaoqGjWrfuCF19cxc9+\n9kskScLpsHPdlT9l6VXTWL36bwBkZGTi9/uoqCgnJSWVdes+55xzzgPgssuu4oYbbsbn9XPj9+9k\n566t3P/Idbz33tv88pe/ICGhbWGwRYvO5fbbbwwI9nXrPuf6639IUVEh69Z9zgsvrEatVvPHPz7J\n559/wvnnX9Sjv0V/EtI9aUREBHPnzuWrr77qVrAnJAyPioNDoZ/VzdUMU9OiO+1PZ58b9HLFO5fD\nNyR+S1/64HHKCUaZ2fE9bidttLxb8fukbr87FMapK9xuH6oIAU0v6uyrNSp8PhEkUKtbBLPJqOv2\nd6tUEBdnJjragsViwNj8HYNBy8KFSwPfP/30uTz//HMsW7aMJUuWkJSURHS0CZ1O0+Ezjh07wIoV\nl5KQYCEhwcK8eWcQGWnEZquhoCCfBx+8F0mSEEWRxMREEhIsCAhkpE4nMTmShAQLZrMes9lAQoKF\npUsvZuvWTdxyyy1s2rSeZ599loQEC7t2bebll1/G6XRSVV9FWXkaCQkWtFo1ktQyL7RaNTExJsaO\nTSczM4OKigJGjx5NeXkpixcv4I033uD48WPccceNSJKE2+0mLS15SM2bPgv2uro6tFotFosFl8vF\nli1buPXWW7v9XnX10C9OlZBgGRL9LC2WDyJWaYQO+9NVPyVJQqNVUV1pHfTf0tfxPFkhn5QjIva4\nHalZhlWUNnb53aHyN+8Mr9dPbN5YZo45gwvPn9Lj7x/aU87GT49x9sUTmDA1uc217n63KErU1trw\netVYrS6cTg/V1VZcLi8+X8vcXLHiGqZNm8uWLV9z5ZVX8swzq2hocODx+Dp8htPpwWZzB6653V6a\nmpzU1dnIysrm+edXt+uny+VFY9Gh0amorrZit7uRJDXV1VZOO+0sHn30p8yaNQ+/X8JojKGsrJZf\n/erXrF79OvHxCTx8/2+wNjgpL6vH6/W3+f1er5/6egfV1VYWLDibd99dQ0ZGJvPnL6S62orV6mTJ\nkou47ba7ejR+oSDYxaPPUTHV1dVcd911XHrppVxxxRUsWLCAhQsX9rXZMK1QzCi9McUIgkBktJHG\nBuewzzhsqHNiMut65fxsbYoZziip8PGJPQvxU1Ac6LZ+DPUrKyslOzuHa6+9nilTplBcXIjJZMZu\n79hpO23aTNau/RxRFKmpqWHXrp0AjB6dSX19AwcO7AfA5/NRUJAPtBwy0tE7MWpUGmq1ilde+TuL\nF8tmGI/HgyBAZGQUDoeD44W7geY5ZTJhs3UcMbVw4WK++mpDG5POrFlz2bBhHfX1ssLV1NREZWVl\nr8aqv+izxj5+/Hjef//9UPQlTCcoSTmW6N5FtcihfnacDi8mc8/CJYcKfr+IrcnV6yPN9AY59r1p\nmMcuK6nwSjninqIIdmtTb8YhOHv+O++8ya5dO1Cr1YwfP47TT58PgFqt4cYbv8eFFy5r4zxduPBs\ndu3azvXXX016egYzZswCQKPR8LvfPcmzz/4fNpsNUfRz5ZXXkJWVjd8vtfk9p7J48RKef/5P3HLL\nnYBsJl62bAXXXXcVKSmpZGeNw14vv1sXXbSMxx57DK1Wx/PPr27jO7BYLGRmZlNcXMiECZMAyMzM\n4pZb7uT+++9CFCW0Wi333/8QycnJHfZlMBCkQVLjhvJ2V2GobMtff/5b/H6R6++e1+H17vq5ef0J\n9m4rYcX3Z5CcNnhlV/synvW1dt56aTsTpiZz9sUTetXGO//YQUOtg5t/cmanEQxD5W/eGbu/Lebb\nDflcddOcwCEiPcHn8/PS018xKiOaS66Z3g89bEt/jeen/zlAwbEarr9nXq+UleL8Wj5+ez9zFmQy\ne0HmkP+7KwyYKSZM/6JoqpG91NahVSz7MI6MCZQS6GFyUmssUQZ8PhGnffgesqBkS0b38pg/jUaN\nyawb9lmX1kYXGo0Ko6nr4/A6I1AMrH5kZOGeSliwD3FsTW4kCSKjei/QomLkRUERjsORvsSwKyj2\n2OFsjrE1m1D6Mg4RUfrmeTV8fS7WRheWXiTsKUREGlCpBJrqh+9c6IqwYB/iBBynoRBoI0Fj78OB\n1C3JWsP3ZbY2udHp1d0e3NwVkVEGRFEatsfDuV0+3C5fnzKpVSoBc4QOm3X4zoWuCAv2IU5LclLv\nJ7GinQxrjT0g2Hs/DgHH4TBd4CRJwtroIiKyb6UhlO/3Z2RMf6LsWvpaIiMi0oDd6hmRVR7Dgn2I\n05dQRwWVSsASbRjW205rkwuDSYtW1/tAroDGPkwFmsftw+vxY4nU96kdRSAO13FQ+t3bKDGFiCh5\nHEdCgbxTCQv2IU6LYO/bJI6KNuJyyjVnhhuSJGFvchNhCZFAG6amGGujLIAi+qipBmLZexXyOPhY\nlV1sCDR2kP1YI42wYB/iNDXI3v++xp8PZzu72+XD5xOJ6KOmqtGoMUcM34gQJfbc0kdTjPL94TYO\nu3fv5KGH7gv0uzNTzD333MbRo0e6bU/Z+diaXPzpT39i587tverX22+/idvdsjg89NCPsdsHt0R0\nWLAPYSRJoqnBiSW6995/BWXbaRuG207lRY6w9L3ssCXagK3JNSztqoqG3dcFztI8F4abYAcQBLoV\n7MGiaOyNDU7uvfdeZs2a06t23nnnTdzulrF86qlnMZsHt9Dc8C5MPcJxu3x43H5S0ntvX1dQzBjD\n0Z6oLEZ9FWggh41WljZht7r75LcYDBRTTF8FmlanwWDU9Eiwu1wufvnLn1JdXYUoilx//c0sXnxu\np2V1y8pK+b//exybrQlJEvjtb58gNXUUq1Y9x9atmxEEFddddxPnnHMeu3fvZPXqvxEVFU1BwQkm\nTJjIo4/+FoBvv93Mn//8DNHRMYwdOx6ApkYnGq0qEBnkdrt5/PFfU1RUSEZGBh5PS7TP9u3f8vLL\nf8Pr9TJqVBqPPPIYBoOBK664hLMXXcAXm7/Er1vGV9veZNas09HrDfzvfx/xm9/8AZB3Cf/+9xs8\n8cQzPP30Exw9egi3282iRedw00238u67b1FTU80999xOdHQ0zz33PFdccQkvv/xP3njjNZKTU1ix\n4nIAVq/+G2azmauuupZ//euffPnlF3i9Ps46axE33dR9fa2eEBbsQxjlxeurLRFaBPtwtCfam0In\n2C2tQh6Hm2BXNHb/hv+yY9WePu065tg8iH6J/IffBsAyew4JV1zd6f1bt24mPj6Bp556FgCHw95l\nWd1f//oXXHfdjaxYsZTy8jpEUWTjxvWcOJHHa6/9m/r6Om6++TpmzJgJQF7eMV5//R3i4uK4444f\nsn//XsaPn8hTT/2eP//5RUaNSuOXv/wZ0D6Gfc2adzEajbzyyr84ceI4N910LQCNjQ28+upqnnvu\nr+j1Bt5441Xeeut1brjhZvk3R5pZMu8uRqfHcrSkVB6XOafx9NN/wO12odcbWLfuCxYvXgLAbbfd\nhcViQRRFfvSjO8jPP87ll1/Nv//9ZqCcsYzcr3PPXcJzz/0xINjXr1/LM8/8me3bv6W0tJiXXnoN\nSZJ4+OH72bt3D9OmhS4TOCzYhzC2EAo087DW2BUTRN8XuMCBG43D7+ARa5MLlUpAq1XTVxe4oBKQ\n/CKSJJs3uiM7ewyrVj3HCy/8hTPOWMC0adPJzz9Bfv4J7rvvruayuhLx8Qk4HA5qaqpZsEAuBqjV\nypr1vn17OPfc8wGIiYllxoxZHD58CJPJxKRJk4mPjwdgzJhxVFRUYDAYSU0dxahRaQAsWXIhH3zw\nH3kXm9ayKO/Zs5srmhelnJwxjBkzDoCDBw9QWJjPHXf8EEmS8Pl8TJkyLfC9JUvO57//ymuj7KjV\nak477Qy+/vorFi1azJYtX3PXXT8CYN26z/jwwzX4/X7q6mopKCggO3sMIDX/pyD//9ix42loaKC2\ntob6+noiIyNJTEzinXfeYvv2bdx007VyXXmni9LS4rBg/66gCGFzH6NBWrcxHCMhrMoCF4JxaHEi\nD79xsDW6MVv0JF55NQl33dKn2ibfrDvOvu2lrLxuJkmp3Z/MlZ4+mpdffp0tW77hxRf/wty5p3PW\nWYvIzs5pV1bX4ei4iuOpma6t/60IfwC1WoXf3/HS5fPKu5RTzVGtfVBKu5IkMWfO6Tz22O86bMto\nNBIRaWj3TixefB7/+c/bREZamDhxMkajkYqKct566w1efvmfmM0RPP74r/F4uleSzj77HL78ci21\ntbWcc86SQL9+8IMbuOSSFd1+v7eEnadDmIBtOQQCTa2WD8Iejs5TW5MbQQCzpe+VKZXdj32YmaT8\nPhGH3ROyc2t7GvJYU1ODXq9nyZILuOaa73Ps2NFOy+qaTGYSE5P46qsNAHi9XtxuF9OmzWTdui8Q\nRZH6+nr27dvDpEmTO31mRkYmlZUVlJeXAbB27Wf4mmuntx6H6dNn8PnnnwCQn3+cEyfyAJg8eSr7\n9++lrEw2s7jdLkpKits8IyJSj8ftlw8faWbGjFkcO3aUDz9cEyjVa7fbMRqNmExm6upq+fbbzYH7\nuypJvHjxeaxb9zkbN67n7LPPAeC0007n448/xOl0No9tdaAEcKgIa+xDmFBq7CAvEDVVNiRJGlLn\nM3aHvcmFKUKHStV3PSSwcxlmC5xijuprcpKCEvIYbJJSfv5xVq16DpVKQKPR8sADP+uyrO4vfvFr\n/u//HueVV15CENT89rdPsHDh2Rw8uI8bbrgGQVBx5533EhMTS2FhQZtnKXNTp9Px4IOP8OCDPyI6\nOobc3OmcrKiT+99KsC9ffjmPP/5rbrjhe4wdO45Jk+QDSKKjo3nkkcf41a8ewePxIggCt9xyB+np\no1Hs4Ip5T1kwQD7qc968BXzyycf84he/BmDMmLGMHTueH/zgKlJTR5Gb22LSueSS5TzwwL3Exyfw\n3HPP07q8cVZWNg6Hg4SEJGJj4wCYM+d0iooKuf32GwEwmUw8+uhviYkJnWkwXLa3Cwa7lOcH/9pD\neXEDtz54FuoujkELtp99LXXaV3oznqIo8dLTm0hIsbDyBzND0o9X/vQNOr2G7912Wkj6OBCUFtbz\n0Vt7mTUvg7lnZfW5nzUnrbzzj51MmZnKmUvGhbCnbQn1eH79RR77d5Zx+Q2zSEju+1F0u7YUsXVj\nAVf/cC4xCb2vQzRQhMv2jgDsVjdGs7ZLod4ThmPIo8PuQRSlkJijFMwWPXbr8KpuGKr6KAqBujnD\nLJY9lKGvcjtKlNTwS9zrirBgH6JIkoTN2vc0+tZERA6/kMdQJeW0JsKix+cTh1V5hUCSVojGQT5R\nSh1wTA8X7FY3KrXQp+qWrVHGczifVdARYcE+RHG7fPh9Ysjs69CqNsYwKlVqDziQQ6OpwvAM/VQW\n41Bp7CDb2a2NrmG1c7Fb3Zgj9CHzEQV8DcO48mlH9FmwV1ZWct1113HRRRexbNkyXnvttVD06zuP\nLYQhfgrDWaCFUmMfjg5UpU5MSOdDpB6vx4/X4+/+5iGAKMqRQaGIjlIwRegQhJGnsfc5KkatVvOz\nn/2MiRMnYrfbWblyJfPnzycnJycU/fvOEuqIGBie2afWfjDFBBY42zAah0YXRpMWjVYdsjbNES0L\nvU4/9APkHHYvktTS71AghwHrh/VZBR3RZ409ISGBiRMnAmA2m8nJyaGqqqrPHfuuE8oYdgVThKzp\nDCfB3qKxh84EEXAiD5NxCPhbQjgGMPwWuP5QdkAOIW1qdCGKw8ck1R0htbGXlpZy5MgRcnNzQ9ls\nv2I/sA9nfv5gd6Md/TGJ1WpV83Fg7V9kT2UFjsOHQvasUKE4y3p7aHFHKFv51uMgSRKi14vo9SL5\nhpZT1enwIvqlkO5aAMzNC73d2lI0S3S7se3Zjd/WtuyszWbj/fffDfxbKaHbEU8++XuKigq7fX5X\nbbRGKcMbeCeC0NhffvnFoMvwRkQakEQJR/MC9/bbb+JyOrHu2IansmJIlOHtKSHbf9ntdu69914e\neeQRzGZzt/cHG4/Z3xT+8xU8dfWkLL2IjB9ci1rfdtIMVj+V1OnRmXHExoduPKNiTVSWNRIfFwGS\nSPlHH1P15QYchUUApF9zFaOvvrL3HQ9RPxUcNg9R0UYSE7tPew+WSItcVsDr8ZOQYEH0ejn8uz/Q\nsGcvxwFUKjK+/z3SLuu/lO+eUOlpBCA+IaLN+PV1bqamRcv/I8lt+Z1ODj3zJE2HDiOo1URNyyV1\n6UXEzJqJ293IRx/9h1tvlZNqoqNN6PWaDvvw9NNPtPm3co8oim2SzLpqozVarZqYGBP2etlhmjIq\nqsvviKLIT3/6QPcD0ExisoXjh6vQqNUkJFh49+03mJF/HOn4CZIvWMI//vFy0G0NFUIi2H0+H/fe\ney+XXnoGQVm+AAAgAElEQVQp5557blDfGSpJIEm33U3l6r9R8dHH1Gzbwagf/wRdQiIwuMkqNVXy\nc90eb7d96Ek/DUYNol+iuKgW19frqHn3bVCrMU+bjqesjJI3/43D4SFu2aV9/g196SfIafQ2q5vU\n9KiQ/x10ejX1tQ6qq61UvfUGDXv2ohuVhikhDmt+AUX/fANfXDLmyVNC+tzeUFoip5urNEJgHEIx\nN33N1SGrKps4WVpD2XPP4Dx2FOOEiYgOBw27dtOwZy8Zj/2Gx//2V4qLi1m27BJmzz6NM86YT0ND\nE7fddme7Urv33HMbd999H+PHT2DJkrO46qpr2bbtW+6++8fY7fY2ZXg9Hl+733FqGV673Ul9vYP6\nCh8V1cf42aOrEVRSuzK8F198Cdu3b2XlyivZunUz8+efGVQZ3sYGG3GWCZQUTeTdF/8fVSdP8ugX\nnxEdHcOq85exaNHZg16GVyHYxTwkgv2RRx5hzJgxXH/99aFobkAxZmeT8cvfUPOfd2hY+wU1775N\n6h13D3a3sFvdGIyhdZZBS9hgQ1k19o8+QB1hIePXv0MTFYW3tpbS/3uC2g/eR9DpiD3/wpA+u6co\ntt9Q25ahJUnJumsnDWu/QJeSyuhHHiUpLZ6SbXspfuL3VP79RTIe+y2a6OiQP78nKONgajZBbF5/\ngsK8GsQ+HhaimJSP7KvEsXsHWXlHiZg9h5RbbkdQq7Ht3kX5qj9R9cY/uf32uykszGf16jcAWUB2\nVGp36tRpbZ7hdDrJyRnDD394Gx6Ph6uvXtGuDO+pdFaGt7qqhgN5a3nxpb+RkBTdrgyvTqdn1aqX\nALnMMARXhvfEkZM89PC9HDt8mNOLi3lPq+WPj/6G1IVnN4dVDn4Z3p7SZxv7zp07+eijj/j2229Z\nvnw5K1asYNOmTaHo24Ch0ulIuOp76DOzsO3cgfuUQkEDTX8kJykoNvvyz9Yjud3EX34lmqgoALRx\ncaQ9+DDqqGhqP3i/nZ11oOmPUEeFCIset8tH+auvIOh0pNx+F6pmM5whK5uEK67Cb7VS8dILSOLg\nnrak2MAVm3ioUELBRb+Ir64Oc+40Um6+DUEtKxMRM2ZinjET57Gj2Hbvavd9pdSuIAiBUrunotFo\nWLhwMQBFRYXtyvB2xJ49uwPXWpfhPX7iCI22kzz007u48cbv8emnH3Py5MnA95SCXa1pXYbX7/ez\nZcvXnHmmXE543brPuOmm7/PL395Do/Ukx3ftQLTZEIwmLDNntYqVb1+G9/jxvEAZ3m3btgbK8N50\n07UUFxdRWjq4MqTPGvusWbM4fPhwKPoyqAiCQPzyFZQ9+wy1H35A6l33DFpfPG4fPm9ok5MUApl2\nReUk5owhct78Nte1cfHELDmfmnf+TeNXm4i98KKQ9yFY+iPrVEFxwDk9kPW9a9GPGtXmevQ55+E4\nchj7nt3Y9+4hYkZo6tT0BsWpp8yHeYtzuPSq6SExT73+/Ld4GxsZW7uDhB8/jqBpKxISr7qGwoMH\nqPv4w3YLXDCldnU6Xa+SiToqw+tyekhLnsA//vFih98xGjs+OKW7MrySX8Odt/4Ya1UtKrMZVSft\nwOCV4e0p4czTVpgmT8WQnYNt905cxUWD1g8lWsPcLwJN1vpcmggSr/0BQgcVE6POPAtBr6dh/dpB\njRCx2xRNNfTjYDLJWqkvbhSR889sd10QBOIvXQlA49eDuwPtL40dwKgRcUtajJNz0aWktruujU8g\n9qKlaB0ObLW1PW6/dVZrR2V4O6KzMrwW4yiq6gq6LMPbEd2V4XW6rZRXH8EraIg9/0LM5oghV4a3\np4QFeysEQSDuUnnVrf1wzaD1w94PMewK2qZqAPyJ6RhGZ3R4j9pkJmr+Anz1dR1uwQeK/opbBlDX\nyMJFGJ/b4eIGoE9PR5+ZhX3/PnwNDSHvQ7DYbW40GlW/JBFprDVIggrDWZ0HPcScfwGRlkhydDqu\nu+5q/vrXP7W7p7WG3dn/63Q6Hnro5zz44I+4665bSOlgIQG5DK/D4eCGG77Hm2++zqRJU/B6fKgF\nI8vOv5lf/eoRrr/+Gm677SaKAwpY57sCpQzv1q1bmDdPXsRbl+F96onfkBSdgU+tJ3rxOYEyvD/6\n0R3t2u6sDO95553P7bffyPXXX82jjz6M0+notD8DQbhs7ylIkkTJE7/HdeI4s/72PFbVwJ+LeWhv\nORs/OcbZF09gwtTkbu/vSYTEybfe5D8FSSTGaLns9vaaqoLnZCWFP/8phpwxjP7ZL4Lue6j6CfDZ\n+wfJP1rNdXefEXKtffsfVrFDmMycuUnMXjyx0z42bFhP1euvEb/ycmIvWhrSPgTLK3/+Bp2ubZnh\nkETFNNTz6ZNvURI1kcuun0liSuchpZWvrKbp602kPfAwpgkTO73vVEIVWVZfY+etv29n4rQUFl04\nvs/ttWl7/Vo+3tSI0xTLzQ8uGtJnFYTL9vYSQRCInLcAgLqt2walD/Z+qBMDIIki9p3b0IsunGLX\n2p8uKRlz7jRcJ47jzD8R0n4Ei8Mun5xkNIXWBOEuL0dVIv8mp6/rqCPL3NMRtFoav/lqUIpl+f0i\nTrs3kDUcShq+XI/eKzvIFbNXZ0SedjoA1m1bQ96PYLDb+m/3Ztu1E53fgU8Uhk3dnO4IC/YOiJg+\nAwSB2i3fDsrzbf1kgnDmHcNXX49Rr8Jh93QrqKIXy9tza6tjwAYSu9WDyaxDpQqtBtX09Sb0Pnvg\nGV2hNpmImDUb78mTOPOOhbQfweC094+fQZIkmrZuwaCSfSjdFYYzjp+AOioK687tg+J3USKkQlkA\nDMBvteI8dpSICNkRPFzKK3RHWLB3gCYqCuOYsTQdPoKvqWnAn99iYw/tJLZukxeqiPhI/H4Jj7vr\nF9Q0YSIqgwH7/n0Drq1KkoTD7gm5pir5fDRt+Qa9SYtaLQRV4TFqwVkANH61MaR9CYYWB3Jox8FT\nUY6vpoao0SmAnOHbFYJKhWX2XES7HfuhgyHtSzD0lyPdtncPiCIxaXJSYncL/XAhLNg7IWLGTJAk\n7Ht2D/izbVY3Or0arS50zjLJ58O6cwfqqCgik2SnT3eTWNBoME2egre6Gm9l+xjl/sTjluvRm0L8\nIjvzjuG3Womae7qcpBSEhmYcPwFNfDz23bsGXFvtLweyfd9eAGInjmnznK6wzJVt/IqCMJD0V0CB\nbfdOABInZAItoaXDnbBg74SIGbOAlj/8QOKweUKumdgPHUS02bDMnoup+eVw2LufxObmTEJbsyAY\nKPorxM9+8IDcbm4uZoseh82Dv5sMTkEQME/JRXS5cBUMbME4RZMO9c7Fvm8vCALxM+SSCcEscIbs\nHLTxCdh270Z0D6wA7I8FTnS5cBw8gG5UGjHpSfJzutm5DBfCgr0TtAkJmLMycRw+hN85cLWa/c1H\ntoX6Rbbtkhcoy9zTWqr6BTGJzVOnyvfu3xfS/nSHsuiEWmN3HDyAoNFgHDs+ICQUO3ZXmCdPBhhw\nM0TAaRjCcfA77DiP52HIysIQG41OrwlqLgiCgGXuaUhu14BXArXb3KjVAnpD6Hax9gP7kXw+ImbM\nDJykFLaxfweIPf00JJ8P+/6B01Zb6oKEVrA7jxxGZTJhyMoOtN2dXRVAExWNPjNLNmEM4ALXHxq7\nr6kJd0kxxrHjUOn1LQePBGGGMI6fCCoVjmaNf6DoD03VcfAgiGJgN2a26II+VcvUXBTNcWSABbvV\ng9kSuiPxoGU3HjFzVuDIwWDeieFAWLB3QdzpcwGwD2CSjqNZezSZQ/cie2uq8dZUYxw3HkGlajk5\nJ0jtxDw1F/x+HIcGTqgFxiGEgt1xWNa2TZNk4WQyB7/AqU0mDNk5uAry8XeSldgf2PvBFKPY1825\nzYI9Qq6b4/N2H+pnyM5B0GpxHDkSsv50h9/ffCReCHctkt+Pfd9eNHFx6NNHN5+jGjbFfCcwZWSg\njo7GcfTIgEWFOPohCkJ5CZXEkp4INICIZgFg3zdw5pieHKoQLIq2bWo2q/Rk5wLIJXwlaUC1VbtN\nPrZOG6Iqn5IoYj+wD3VUNPrmzOOWk5S6HweVVotxzFg8pSX4rAMTMRYI+QzhrsVdXITodGKeMhVB\nEFCpBExmXdh5+l1AEARM48bjb2rC26qKXH/SH84yx1G5SJsi2I1mbY+0E31GJmpLJPb9ewes0mGo\nw/wkScJ+8CBqiwV9Wnpz280CLQgnMoBpkrwgOAbQzu6whfbwZldhAX6rFfPU3IBZo+UkpeDGwdg8\nj5xHj4asX13RktcRwnfimNx347gJgc9MEXrstu7zO4YDYcHeDcaxcvqy89jATGJFyChadV+RJAnn\nkcOoLRZ0qXIFQ5VKhdEUvHYiqFSYp0zF39SEp6wsJP3qDiXr1BCirFNPeRn+xgZMkyYHasP0VGM3\nZGahMhqxHzwwIC+/z+vH7fKFdtfSvCgpTnHo+dmnioLgODIwVV0d/RDDrrzPxrHjAp+ZI3T4fWK3\n+R3DgbBg7wbjOFmwO/IGRrAHJnGItp3eqpNytun4CW2KXZkidEFlnyooL4DzeF5I+tUdoc46DZhh\nJrWciNRTk5SgVmOaOAlfTQ3eATiwvT+Sk5S/n6KwyO03C/Ygk3MMGZkIegPOgRLsIfa3SKKIM+8Y\n2oQEtLGxgc+VMOCRkKQUFuzdoEtJQRURMWAae8AUEyKNXdGqTi3cZI7Q4fOKeNzB1cYwjh0LDIxg\n74+sUyVMUQlbBNDpNWi0qh5FQgSiQgbAkRyIkArRIi+JIq4Tx9EmJaGJbCn4pZg4gtXY5XDRcXgq\nK/A19H952lC/E56yMkSHo83iBq1MUiPAzh4W7N0gqFQYx47DV1uLt7am35/nsHnQaENXotXZiWBX\n4sODSVIC0CY3L3An+l+whzrrVBJFXMfz0CYno4mOaXPNZNYFbWMHMI1vti/n9f84hNqR7ikvQ3Q6\nMeaMbfO5orH3xHFomiDbph1H+z865tSjAfuKsvtWduMKph7kdwx1woI9CEwBO3v/F4Gy290hsyVK\nkoTjyBHU0dFok9qW/+2xGUIQMOaMwVdT0+9aWqhj2D3lZYguF8bsMe2umSL0uBxeRDE4k5Q2KUle\n4PKPh6RvXRHqyKCAGWZMW8FuNMsFsHq0c5kwSf7OAJye5rSHVmMP2NfHnaqx93yBG6qEBXsQKBPA\n2c92dlFsLtEaqi1nRTl+axOm8RPbJXa0bL+Df5kVgdDf5phQZ50qZYcNOe0FuzlChySB09GDBS47\nR17gGvv38I1Ql6pV/m6GUwS77EzXYg8iA1dBP3o0KpMJ59H+F+x2m6f5oJG+h3xKkoTz2FFZ2UlI\naHOtJToorLED8MgjjzBv3jyWLVsWiuaGHPr0dFQGQyBEqr9w2r1A6JxErhOyVqnYx1ujJED1RDsZ\nKMEeao3ddUIW7MacnHbXerpzATlJB8DVz3XqQ+08dR0/jspsRpfc/vAWU4QuqNIKCoJKhTFnDN7q\n6n6vgKr4W0KRdeo9eRJ/UxOmcePbtWfqYeLeUCYkgn3lypW8/PLLoWhqSCKo1RjGjMVbWYmvsbHf\nnhNq779SsEoRRK3paagfgD4zE0GjwXm8f80QIR+H/BOoDIZAuGdrejMOxmbN33mifwW70idjCHZw\nvoYGOfs4Z0yHRwGazDo8bj/eILJPFQxZ2QD9WhhNFCWcdk/ozTCnOE4BjCYtKpUwIsoKhESwz549\nm8jIzo/VGgmYAuaY/rOzh7rgkzM/H0GnQz8qrd21nhQCU1BpdegzMuWsvX6s7hdK27LfbsdTUY4h\nK7tjgdbDJCUAQ1YWCEJgR9RfOOweDEYtanXfX9PO7OsKyjj0RGs3ZCuCvf8WOJfTiySFbpFX3l/j\nuHHtrgmCIIcBjwCNPfSn445QAtvvgnwss+cAYPPa2V65G5WgIjMyndSIFLSq4IZUkiRKqmwcKqzH\n6/Oj16pxVcs1SEKhnYhuN56yUoxjxiKo29smjQETRM8msXHMGFwnjuMqyA9E2tS7GjhQewS7186M\nxFySTAndtNKCy+OjqNJKQYUVm9NLXJSB6pPyGZmhMEEoQqejXUvrZ/RES1MZjOhSR+EqKkTy+RA0\nGlw+F2W2SmpddVg9NqbGTySxB+NQ3eCk+KSVmkYXjXYPybEmbFY3lsj+ta8rKHPObvMQGR3cOb+G\nTEWwFwQ+a/JYOVhzBLVKjU6tY5pxLALB/4ayGjvHShpweXx4vCKG5jyLUNVOchacQGU0ouvkIG1T\nhI6aShuSJA3ps0+7Y9AEe7CHsg42Sj995qmUCgL+smI0ESJrDn/G+vxvcPtbBIJereOHs65mUdYZ\nnbbncvt4d30ea7cXU9voanMtFRiFig0HK7GkRzNtbPCC4dTxbDxYDJJEzKTxnY61KUKH2+Xr0d9C\nNWsa9Z99iqqiGPuUFJ7f/k8K6ksC1z/K/4zxcdlcPuVipiVP6rSftY1O3vz8KGu3FeM/JSJlAgIR\nCHyxr4JLzxpDQkzvDxR3VpYCkDRzKrEd/E7RKz9b8kuBvgUzHo1TJnLys1JM9joKLV6e3foyTW5b\n4PoHJ/7HOTkLuHzyxUQbOt7NSpLEoYI63t9wnG2HKmmdKyYAs1FRWu9k8+EqLpqXiVbTdoHuyd+t\nvCgfQaMhbfZU1Pr2QjIxSW5Lq1YF326ChbKUZNyFBcTGGvmycAtv7H0fu7elCqjmoIZrc5dz4biz\nUQkd7zx8fpEvthbxxbZi8kraOqQjgfGo2Ha8moRJSSyYntprgeuz2zlWWUlU7lQSk6La/5wECzGx\nZqrKrUSY9CEvGT2QDJpgD8XJ5f3NqSes65JTsB4/zs8+e4I6dwMx+miWZi3BrDVT2FTCjpO7+eu2\n1yisKueirPPaTEBJktiTV8O/1h6jtsmN2aDh9MlJ5GbHYTHpcHv9HPy2GFu5lX2FdWx9YTNn5qZw\n9TljMXYT097RSfB1u+WEHCk5vdOxNpq0NDW4evS38CXIduqSndt5Ub0Zl8/FxNhxTImbiElrZGvF\nTo7WHucPm1Zx69TrmBrfItwTEixUVDby4TcFfLatBK9PJCnWxPQxcWSlRBJl1lHX5GbfF3l4PH4+\n2JTPf78uYMVZ2Vxw2mhUvXiha/fLBbs8cakd/k63V3ZY19bYqa62djiWHZI6GoAvv3iP12MLUQkq\nFqbNJ9mUiFqlYm3RRj4/vomvCrfx4xm3k2ZpqyHaXV5Wf3yY3XlybkRWSiRzJiQSH2Ug0qyjsKSB\nE5sK8UgSf//gAO9/mcfV54xj1viEwFgG+3cT3W5s+QUYMjKpa/IA7XcnIvKqUlHeSHxK8AuGdnQW\nrq1b+MM7v2efUIlBrWdZ9gVEaE04vE6+LPuKV/e8y9aivVw/+WoidW3bLqux8/f/HqKo0oogQG5O\nHLPGJWAx6dBpVRzeW0HV4WqqrW6een0H//06hmvPG0dKnDnoPiooNeRVqe3fCWU8NVp58SkuriMu\nIaLHz+hvgl10QybYR0LhnO7QjE7HU1GOVFXDBdPO56LMc1GrZC3qtJRZLEybx1/3ruZ/hWupdzdy\n7YTLEQQBUZR4Y+0xvtxVhlolcPEZGSw9IxO9rq0GdnJfJTbgnqum8eaXJ/hqXwWHCuu5fflkclLb\naxhdoURsKHbQjjBF6KmtsuP1+II+hk9jiUSKjcZZkI97ZgLXTbqKuckzA9fnJs8krz6fv+59mb/v\n/ye35t7A5DjZP9Fk9/D/3t7L4aJ6Yix6li/IYt7UZNStbN+SJLH/02MkJ0bww9mjeG/jCd7dcIJj\nJQ3cvHQSEUZt0GMgiSKu/BNok5JQR3T8khqMvXOYGZtNO1VH9hC5KI2bp/6A7KjMwPXTk2ezsWwz\n7+V9xPP7/sFDs+8hSi9r7gUVTTy/5gA1jS7GpUez8qxsxqZFtVEEIlUCJyhk/oxR5Khh3c4yVr2/\nn0vmZ3LJgqwe9dVdUgx+f9dzQTHN9cDGDqDPysK6dQuewgJy58zmqvHLida3zNWLpy7iua//wcHa\nI/xt32vcN/P2wDuzbmcp/15/HJ9fZP6UZFYuzCHmlNBOZ7mVqsPVfO+C8aw7Ws3+/FoeW72dW5dN\nYvaExB711VUom4wMWZ2PnzIOTrsHgt8wDzlC4jz9yU9+wtVXX01BQQGLFi3ivffeC0WzQwqv38s2\nXSUAC8VMlmYtCUxQhWRzIg/MvovRllFsqdjOloodeLx+Vr2/ny93lZGWEMGvb5rLZQtz2gl1kF8q\ntVpgfGYsj14/m6XzMqizunj6zT0cKqzrUX9dBfmoLZFoYuM6vcds7rkDtdZZzwmLG6Nb5Oa0S9oI\ndYWxMdncnnsjgiDw0v5XKWgspqzGzk+e28jhonpmjI3ndzefxpnTUtsIdWjJOjVb9MyfmsKvbpzL\n5KxY9p2o5TevbKemIfjDPjyVFXKmZQeJSQqCIGDsRbnWfK0Nl05gVK3Iw3N+3EaoA6hVahann8ml\n2RfS4G7kxX2v4vF72Hm0isf/uZPaRheXzM/koWtmMC49up15QVlooqMMXLV4LI/dMJuEaAMfflPI\nX98/gKsHhapcRYUAGDK6EGi98DUA7NLLO46JNjO3TP1BG6EOEG2I5I7cG5mdNJ2CpiI+yP8ESZJ4\nb+MJ3vjiGEa9mrtXTuWHSye1E+rQ4sxNSbLw4ytyuXP5FNRqgefXHGDtjpJ293dFQLBndqXs9G4c\nhhohEex//OMf+frrrzlw4AAbNmzgsssuC0WzQ4qPC77ggFGO1811xXRq54vUWbhl6nUY1HrezfuQ\nJ975ht15NUzMiOGn184kNb7zLaTdJod1CYKARq1i5Vk53L1iKn5R5Nl39rEnL7iSBr6GBnx1dRiy\ns7u0R5osPZvEkiTx5tH3qIyRp022tXPn5vjYMdw85Qd4RR+vHnybJ/+1g8paB8vmZXLXyqmdmpdO\nTaOPNOu478ppLJ2XSU2ji6fe3E1NY3DCPbBr6SB+vTXmCB32HhREs3nsvHbk31TGa7FYvZjdnX/v\nvIxFnJ48myJrCX/Z9i9e+OAgGo2K+66axvIzszstcnZqyOeohAgevX4OE0ZHs+tYNb//xza8vuBC\nE92FhYBcfrkzeqOx76s+yIeu3fhVkNOk79SGLggC14xfSaIpnnXFm1i1di0fbykiMcbIo9fPZua4\nzlXj1rH8giAwe0IiP/3eTCLNOv61No/3NgYfkeMqKGhWdmI7vUcJKuhJstZQJJx5GgQn7VWsL/kK\nX1IcqFS4iwq6vD/WEMOKMctw+92UGzczd1Ii9105DVMX5zVKUnO87ikOmxnjEvjRFdNQqWDV+/vZ\nd6K22/4G4tezOtdMAMzmniVkbKvcxeG6YwGNx11U1OX9U+InMit+FtWuKlxRedxxWS4rzsru0lau\nCJbWsdsqQWDlWdmsODNLFu7/Ck64K9Ea3Y2DyaxD9Eu4Xd1rwZIk8caRd2n0WIkeI0cFKZpgRwiC\nwDUTVhKvTeaE8xCaqHruv3IaU7I630lBx4WvIoxa7r9qOtPHxLMnr5oXPjiIr5uDuEHW2AW9ocPE\nJIWeFkRz+py8ceRdVFodmrQ0vKWliN7Ov2vQGPjh5O+jktQckjaQkqziZ9fOJD6qa8d4R+WbM5It\n/PwHs0iKMfLxliI+3VrcbX99TU346moxZGV1qewoCoUzrLGPbCRJ4p28D/FLflZOvBR9Wjru4mIk\nX+dCQJQkDuw04a9PQB1Vx4QZTWi6iUV2OeV6JR3F607OjOX+K6ejUgk8/8EBik927TQLVrD3ZNvZ\n5LHybt6H6NU6zjvjGvk5XQg0gEa7h6Nbk5G8OvTp+czO7d7x4+iiLsiy+Vksbxbu/+/tvTi6EcTu\n4iJQqzuM429NT8Zhb81B9tUcZGx0NhNzF7Y8pwsKym1U7pPNIElTCsgZ1X3OR2dJWhq1ijuWT2ba\n2Hh259Xwj/8d7nKnIbrdchz/6NEdxvG3xmTWBa2xf160AZvXzgWZ5xA1Zjz4/biLuxawhw77cBWN\nQ9B4mTCnhqggok4cNg9GU/vyzfHRRh64egbRETre/vI4Ww5UdtmOMle72rVA730NQ42wYO+GvTUH\nOVx3jImx45iWMAVDZhaSz4e7vPMDJ9798gTbDlUxyn0GBrWeTwq/aBMW2RHdnZw0Lj2aW5ZOwuPx\n8+w7e6lrcnV4H7QW7F072QICLYhJvOb4/3D4nFyacxEJcaloE5PkOO5OhIrXJ7Lq/f1U1/qZrJ+P\niI+Xd/272+d0V6L1kvlZLJmTTkWtg+c/OIC/kxOdJJ8Pd0kx+lFpCJquHcPBVroUJZGP8j9DQDYt\nKDZrxYbdETUNTv7yn/34bdGMi5hMtfsk31bs6PI50PU4aDVqfn7jaeSkRrLl4En+u7nz57uL5bBX\nfWb3DldThB6n3dNtQbQ6Vz1flnxFtD6KxekLMGQpOR6dL/Q7j1bx7/XHMTtyiNPHsa1qBycd1V0+\np7vyzXFRBu6/ajomvYbV/zvMwS78UO4gHKcARlNYsI94vH4v7+V9hFpQc8XYSxAEAUPzC9LZJN5y\nsJJPtxWTEmfivhWncXb6mdi8djaVbu7yWQFbYhfJSbMnJHLl4jE02Dw89+4+3B2kf0uShKuwAG1S\nMmpT1yFhwdZJqXLUsK1yF6nmZM4cdToAhsxMRLsdX017u78kSbzxxVGOlzYyd2Iid5x1PuNixrC7\n4gDHG7rW8oMpJ3Dl2WOYlhPHwYI63lrbcfanp6ICyefDkJnZ5fOgbXJOV+w4uYdK+0lOS5lFkjkR\nTXQ06sjITk1STreP597bh9Xh5drzxnL9tOXoVFo+PPEpTl/nCzO0ONI7K99s1Gu457Jc4iL1vP9V\nAbuPdSwkXUWKwzCzy+eBPA6SJO8eu+Kj/M/wij4uyb4AnVrXUlqgsOPSAkWVVv720SF0WjX3XT6D\n5WMvDCySXeH1+PF5xS7nQlpCBPdenosgwAtrDlDdiXM9GMcpgFqjQm/QhJ2nI5mvirZT56rnrLQz\nSN51YmQAACAASURBVDLLoVXKit/RJC6qtPLqJ0cw6tXcc1kuEUYti9PPxKgxsLZ4Iy5f5xqhI8ia\n00vmpLNoeiolVTZe+7T9IdvemmpEpxNDN1tOCH7b+VnReiQkLsg8J+AgU7a0rg78Det3lbFpbwUZ\nSRZuvGgiKpWKZdnny20Vru/yWcEcqqBSCdx6yWRGJZhZt6uUTXvL292jaNHKgc1dEczOxS/6+Tj/\nc9SCmosyzwVk+7l+dCa+ulr81rbmMUmS+Pt/D1FWbeecWWmcPTONaH0USzIWY/XaWF+8qcs+OZr9\nLV3ZgyPNOu65LBedVsXf/nuI0mpbu3sCAi2I+dCShdv5PC2xlrG9cjdpEanMSZ4BgDYxEUFv6NAU\nY3V4WPX+frw+kdsumUxGsoUZCVPJsKSzu2ofRU2dR7Z0ZZZrzbj0aK49bxx2l49V/9nfTuGRJAlX\nQQGa2Lg2B4x0hnK62HAmLNg7QZREPjwiv8jnjl4Y+FyXOgpBpwts7RRsTi+r3t+Pxydy89JJJMea\nADBpjUFp7cEWvhIEgWvOHUd28zb8y91tTUKK9qgfPbrb36jRqtHp1V1O4hpnHdsqd5FkSmRGYss5\nmYqgcDVHXCjklTbw5to8Ik1a7rlsKnqtHNaZHZXB5MRxHKo7SnFTaafPC/ZlNuo1/OiyXMwGDa9/\nfoyiyraC1V0s90s/OrPLdiC4sgJbKrZT46pjfuppxBlboioMGfLC4TrFzv7p1uJANNTV57SEWy4e\nfSZmjYmNZZvxdGKeCzjSgygtMTrJwg8vnoTb42fVf/bjPCUM0l1UhMpgQJuY1G1bxiAW+k8L5UV+\n+ZiLAou8oFJhGD0aT0V5mxpCoiTx9Bs7qWkO7Zw+Nl6+XxC4NOdCAD488Wmnz+rJwe4Lp49i4fRU\niqtsvHqKwuOrq8NvberWDKNgMssZ2X7fwBzc3h+EBXsn7Ks5RLn1JHOTZ7aJzRXUavTpo3GXlSF6\n5IknShIvfXQoMIFnnFIKYHH6AowaY7PW3vEWvCfHf2k1Ku5cPoUIo5Y31+ZxpJVtUXHkBaOhKc/r\n6kX+vOhLREnkgszFbcLZFE3Y3cq+3OTw8MIHB5GQuGP5FGIjDW3aWjHxAgA+K/qy0+c57B50ejUa\nbfe1t+Ojjdy8dBI+v8hf1+zH4WoxIbiKikClQp/eteMUujdJ+UU/nxauR6vSckHm4jbXlJ1L63E4\nWlzPuxtPEB2h47ZLJreJ1derdZyZdgZ2r6NTW3tXjvSOmDMhkQtPG83Jeif/+KRFqIkuJ57KCvSj\nM7p1nEL3C1yNs5a91QcYbRnFhJi2NWf0ozNAknCXtSzaH35dwK4jVUzJjm2XVDU+dgzjonM4Up9H\nqbX9jgtaFcULsk7M984dR05qJN8ePMmGVgpPixkmSMHeA9/TUCUs2DtAkiQ+L/oSAaGNtq5gyMgA\nUcRdKk/iT74tYn9+LZOz2k9gAKPGyDnpZ2L3Odhcsb3DZ/a0VG1spIHbL52MKEk8+c8d2Jrtoorm\nqE/vXmMHWai5HF78HYTN1bsa+LZiB4nGeGYlTmtzTW0yoU1KDjhQlcWt3upm5VnZjB8d0669qUkT\nyLCks7f6AJX2kx32x9HDEq3TxsSzdF4G1Q0u/v5fOUJEEkXcJcXoUkeh0nbfVncF0fbWHKTe3cAZ\nKbMD2aMKp2rsjTY3z39wEAGB2y+dQmQHv2Vh2jw0Kg3rSr5ClNqPe7C7ltasaM5e3XGkivW7ypr7\nJDtOgxVoxm58DV+WfI2ExOL0s9qZiJQdorJjPFhQx0ffFJIYY+TWZZM7DHFdPPpMud3Srzt8Xkeh\nr12h1ai4Q1F41uUFdnGKshOMWQ5GRmRMWLB3QF7DCYqaSpgzahrJ5vZpywFttaSIYyUNvL+pgBiL\nnluWTeo0RvvMUWegUWnYVLq5y5fZaAo+ZX5SZizLF2RR0+Dk7/89hF8UcRcVoYmL6zSF/lSUhcTl\naO8w21S2Bb/k57yMRe2ybEHeFYgOB97qaj7eXMjBgjpyc+K48PSOXyBBEDg/82wkJD4v2tDuut8v\n4nL0/ASp5QuymZgRw57jNXy2rQRPZQWSxxP0rkWtVmHo4gShDSXfALAwbX67a5rYOFQREbiLChFF\niRc/PEiT3cPli3IYlx7dYXuROgunJc9s1oAPtrvem8ObNWoVt186BYtJy1vr8sgvbwoqMak1gRju\nDsbB4ZWVkmh9FDMTc9tdNzSbvNwlRdRb3fzto4OoVAIPXzen0zIQk+MmkGiMZ0flbqye9v6B3pz5\nGhtpaN7FSc27OF/LLjZowa4cQhMW7COKdc2OrUsnLunwuiLYrScKeOED+bT62y6ZTKSp8wkYoTMz\nO3E61c5aDte1P4HIYfc0F/rv2Z/k4jMymT4ugX0nalm34SB+a1PQmgl0blf1ij42l2/DrDExO2lG\nh99VIi3yt+9nzdcFxEbquXlp54sbwNT4SSQa49lZtReb197mmtOhnCDVs6p6ijM1yqzjvY0nKN4j\nH9emzwh+HMzmjk8QKrGWcaKxgImx4zpc5AVBwDA6A+//Z++9oyS560PfT3WOk3ty3JyjNiqsJAQS\nCiRjHgbDRRhjHDg8Xb/jc1+wr6/TxX6PCxiuMRgso4vBZIQQKGu1knalzTnvTs6xezqHqvdHdfX0\nzHRPV3XXzG6P+nMO54jpqq7f/vpX39/3942jo/zqlYtc7pli++oaHtzdsuDz3tVyDwICL/W8Ns8B\nnm+jkUq3lc8+thFRlPjGL87jV8JeVUTEwMLRQW8OHCWaiHJv850ZN3lLQ4Ncvri7i2/98gLTwRgf\nuX8VazKc3BQMgoF7W+4iLiV4vf/IvM+12NjT2bKymkf2yae4J399iXBvD6bKKoxudQW0SqaYZch4\naIIL41foKGtldXXmI6y1sQmMRgbOX2HKH+VDB1Zk1c7SOdCyH4BDfW/O+yzfLjEGg8Cffmwn5S4L\nxw+eBtRrJpDdvnxq5Cz+WIC9jXdgMWbWuJQN5PQbZzAIAn/4/k05i3QZBAN3Ne0lLsbn2ZgLaVpc\n7rTw2ffJpqmzb+QxD65kB6HobOfjweRvdW8GbV1BmYdTh85QU27j04/M7zE7lzpnLZtq1tPl66Fr\nTmRIPqYYhY0dVTx2ZzvjvjCjF6/JjlOPumJZNocFQZgv0BJigoN9b2I1WrizcU/GewWTCUtzC6He\nPq71TLBzjYcHdub2b+yp34ndZONQ/xFi4uy5L2QePnB3B2tbKrh0sYfE1JSqYAIFRw7TXDFQEuxz\nODxwFAmJu5Lx2pkQTCZC5R5c02NsX1HFQ3vULZpWdzMdZW1cGL/CaHCmNEAsliAaSeTdJabCbeVz\n79tIXVj+zkRt5iYCmZhJzpn9Mh/qO4KAwN2N2WvLm5plrbQiMMZv37eKlU3qKlDubbgDs8HE6/1v\nzTJL5auhKaxvq+T9d3VQ4RtBQsDctLDWnI5ycvGnNTKejvo5PnyaWnsNG6rnt1JTiNfKpYwbouP8\n4Qc24bSpM6cdaJI3+jcH3p7190Ln4X13drCp2YUjMEmgok6V4xRkJcHumF8Q7ezYRaYiXvY27MJh\nzl4CIFBei0FMsMYa5vGH16mqm24zWdnfuJvpqJ+Tw2dmfabFkT4Xo8HAH7x/Ix2CbGcPVOSOClIo\n2diXGQkxwZuDR7Gb7OyY4yxM5/zNca7HnJilBJ/YVampTviB5v1ISBzqnwl9DGl0EmVibWslO8rk\nF/IHF0JZMzLnkmkR90730+nrZn31GjyO7DVNfnFsiCmTi6b4FA/snN9PNBtOs4OdtdsYC41zZWIm\nwUirsywTj+xppSE2yZiljGeOZY62yIQjJdhnopYODxwlLsY50Hxn1gJXsbjIDy7K9+yqiNHRoL5F\n5NqqVVTbqjgxfJpQfCaxphBNFWQB/cntZRiQuBiyc6l7UvW9mWK4lY3nrizaOsDIZJBDI7IA/vB6\nKw6VmxvIG5yAwBsDb836e9BfWK/TCpeVRzrkMT3fk8AXVCeoS6aYZcaZsQtMR/3srd+Z1fwwPBHk\nn5++wIhdFniG4eylBTKxvXYzZRa3XNI3IduU83GWZaJ8eoSIxcGZ4Rg/Oaiu6l0mU8yhPtneqWiU\nmXjr4hDPvd2D112DNRpE1Nip/u5m+USUblstVKABJMZGMSViTLk8PHO4S3VFzJR9OdlvVZREDg8c\nxWIws6dhfmlihe+/dJXzkxA3WamYHtE0VoNg4M7G3UTFGMeGTqX+rkcTa9PYIAAjtiq59rvKcscO\np4V4TCSajIcfD01weeIaK8rbaHRlLiIWiSX4p5+fp9con9hck5kjnrJRba9iXdVqbnq7GUxGSyUS\nIuFQrOAuRmU+OSP3pljGN35+XlXRNKvNJNfoLwn25cGb/UnNpCmzZhIMx/jqT84SjMTZcfc2gJyF\nj+ZiMpjY23AHoXiI06Pn5O/N01mWTsLvJz4+TvmqFdRVO3n+aC+vn82tsc7VTkLxMMeHT1Ftq8xq\nfugemubffn0Zm8XI6js2AvMTdHLR5m6hxd3E2bGLTIbldmip7NsCBFqkV/491u/ZjNlk4F9+dYHB\n8UCOu2bmwZ8U7NcmbzIWnmB77Rbspszmh4On+3nt9ACtdW6cHe3ERoY1N/ne27ALg2DgjYG3U05U\nPZpYK+ty54Ht+EMxvp4hIzMTc9fD4cFjSEhZbeuiJPHtZy7SM+Jn3a4NcvXTXm3vBMD+xt3y8waO\nAoX5W9KJ9HZjcDhZtbGdK71TfO+FqznLM880tS4J9qJnJDjG5clrrKrooN453x4nihL//MsLDE0E\neXB3C3fcK0eKaBVoAPsa5GbYR5LOQz00VeVlcrS384UPyxmZTz13Jecx3GY3z3KYnRw5Q1SMsa9h\nd0bzw4QvzNd/dpZoXOSzj22kZu2qWc9XiyAI3N20FwmJI8nYfj02OGUc9RtW86n3riMUSfA/fniG\nqRyOsJQpxidfd3hQFjCKwJnL2RtjfO/5q7jsZrm+fFurnKDTp635Q7nVzZaajfT7B1NO1KA/e+Er\ntUR65cqW++/blsrI/PavLuYs8JV+gkuICY4MHMNusmUMcQT46cEbnLg6yrrWCn7noY1Y6hsI9/Qg\nqTQFKmyp2YDL7OTtoRPExLgu74QYDhEbGcHa2spnHt1Ia62LQ2cGePlE9sxnBSVxr1g7w5UEexJF\nuNzVON9pKkoS//aby5y/OcHmFdX89r2rMNrtmGvr5BK+Gn/8WkcNqyo6uDp5nbHQuC6mmHBaEkZ9\nlYM/+ZCc/v8/f3aOgbHsGqviMFM0pCMDxxEQ2Nuwc961/lCM//GjM4z7IvzWgRVsW12TSoTKVbo2\nEztrt2IxmHlr8ASiJBIMROXa2xra380lPUFr38Z6Pnh3B+O+MF/58Zl56fbppNvYg7Egp0fPU+fw\nsHJOZySQW9v90y/OYzQKfOHDW/BU2LG2JHMbNJ7gYMZ2/cbAW8RjCaKReEFrQUomz1kbGzGYzXz8\n3WtY21LBiSujPPX8lQXXa7rGfmH8Mt6oj11127EY54/ntdP9/ObtHuqqHPzRBzdjMhqwtrUhRcLE\nRrSZY0wGE3sadhKIBTk7ekGnTb5PTtBqacVqkes3lTkt/ODlaxy9tPD4lBr9UQ2dqm4nSoId2Z76\n9uAJ7CYbWz2bZn0mSRLfe+Eqb5wbpL3ezR+8b2OqNrS1pQUxGCA+kbv5xVz2N8ia4JHB4/os4jkZ\np2tbK/nUe9cRjMT5hx+con8B4a5oJ0OBYTp93ayrWk2lbXb4Zjga58s/OsPAWID37Grh4WQSkqmq\nCoPTSaRXm6YKcvOFHbVbGQ9PcH3qplx72zm/9rYWIr09mKpmErQe3d/OPVsb6Rn28z9/fo5INLM5\nIt0Uc3T4FHExzr6GXfMiO/pH/Xz1x2eIxUU+976NqUggm5J5mYcZQnaiVnJy5CyTPjlRpxDBHh0a\nQopGU2vBZDTw+d/aQmudrLH+7FDmKozpzw0GoryZNItkMsO8fnaAp567gtNm4n//7S2pMFdbARuc\n8k4cHjiqi8Ye7p1dN6m63MYXPrwFq9nIvzxzMWtFTCj+FnklwQ5cmriKN+pjZ922WU5TUZT4wUvX\nOHiqn5Zal1z7Oa0LUioDNQ9tdXvtZmxGK28NHk/VAS/UFCPHLM/UqblzcwMff/cafIEo//D9k/SN\nzM/uA7C7LMSiCd7slU1DiqlIwReM8qUfnqZz0Medm+r5yP2rUgJPEASsLa3ERoZJhNT3I1XY23AH\nMLPBFTIHce8UCa93VsyyIAh84sE1bFtVw8WuSf6/H55KlV9Ix2I1YTAK+H0RDg8cxSAY2DPn1HKj\n38sX//0kvmCM333PWrantXSzNDSC0ZiXYDcIBvbU7ySaiHKm/zKgjzkqPVHNYTPxnz+yLdV16Iev\nXEPMoLkr8z/pnebixBVa3U00u2eHzx483c+Tv76Mw2bi//joduoqHanPlLkP5zEP9c5aVpZ3cHny\nGmNTXnk8eig7afPQ0VDGEx/Zislo4J9+cT6rcz1XeYXbHV0E+6FDh3jooYd48MEH+da3vqXHVy4p\niq17X1LIAATCMf76X9/mpRN9NNY4+dOPbpuXfKMkwITz0E4sRgs767YxFfEy4Z1esPZ2LhKRCNHB\nQawt87vkvGtnM598cC3TwRh///2TnL0xfyErNeBP9VzAaXKwxbMx9dnAWIC/+e5xbvT72Luxjk89\nvG5eeKcyD1GN9mWAVRUdeOzVnB68KNfeLmhzk58/t06O0WDgjz64ib0b67jR7+Pv//0k497ZxdgE\nQcDhtOD1Buj3D7K5ej1llplMxbM3xvh//+MUoUiC33tkPfdtnx3eKZhMWBubiPT1IiXU9SJNZ09y\n7V3ol7OSC5qHLPWCypwW/vSj22iodvD80V65xO2cE4wiSPvGhxElkb1pm3xCFPnF6zd56rkruB1m\n/uxjO2irn53NaU3mNuSzwQHsa5Sf1z0qO/4Lm4ceBLMZS33DrL+vbq7gCx/egtEg8LWfnuVXh7vm\n+R6cRR7yWLBgF0WRv/7rv+Y73/kOv/rVr3j22We5cUN9g9lbTSAW5NzoBeqddbS55UV5Y8DLX/3b\nMY5fGmZjRxX/5eM7MpYLSBU+ykNjB9ifXMTT06FUE+t8CPb0yl1yWjIn5Ny7vYnfe2Q9kViCr/z4\nLN9/8eqsRsj25CKOhBLsqt+O2WAiIYocPNXP3/6vmbKrv//ohlmVChUUAZKPI1kQBFlrj8jfq4um\nmqEAmslo4DOPbuCBO5rpHwvwF//6NgdP9c/SWh1Oi6yhSTMCJhiO893nLvOVH59FkuBPPrSZOzc3\nzPt+5blSLEZ0WJt9GaDGXsWaipWMTckRQos1DzXldv6vT+xkXWsFp66N8bf/6wRXe6dSnyuCdGzK\ni0kwckfdtuT/D/H3/36KX77ZRXWZjT/72A5aaufXIzK6XJiqqvMW7Ns9m7EYLYwm5yHfkE8pHic6\n0I+lqRnBOD/BaV1bJX/2sR1UuK387NBN/vt3j+JNc7Ar85CpzEQxkJ+KmMbZs2dpa2ujqUnWYB55\n5BFefvllVuboDH+7cGz4FHEpwd76ndwY8PGrw12phtEfeWAN79nRlNXmayqvwFhenpd9GeSQv3pH\nHVLEgLUy/58ikOzmtFBFxzs3N9BS6+Kbv7zASyf6OHVtjPt2NHHXlobUIjbFrGyr3s6JKyM8/UYn\nfaMBrBYjv//oBvZtyt4I2VqAfRnktPJXzsjOaz00VVuWeTAIAr/zrtU0e1z88JXrPPX8FY5cGOL+\nHc1sXVWN3WkGUaDcUEGdqY3nj/bw/NEepvxRmj1OHn94/YIJSNbWVjgsR6RYG9Vn/yrsbbiD586f\nBPKfB0mSiPT2YK7xYHQ4Ml7jtMlNsb//4lUOnh7gi/9+kl3rannPrhbaG9wYTQKJMGz2bGRiUuSn\np65w5PwQkViC3etr+eSDaxdMQLK2thI4fYq4dwo86uqzKNhMVnZ4tjB8TijIkR4dHJA7aC1QVmJF\nYxn/9VO7+Oenz/PW+SFOXh7hwLYmHtzdoqo2/e1MwYJ9eHiYhoYZDaauro5z584V+rVLxuHXbuK2\n1fLTX0SIhk4AsKa5nA/cvYK772hldHThxtHWllaC58+R8PtVV1RUEASBXVU76ZQgYgzm/W8I3OyS\nx5KjNkprnZu/+NQufn7oJgdP9/OTgzf4+aGbNDsEagFLqIIvfvs6kgQCcNeWBj50zwoqciSJWOrl\nAlD5OMwAKm0VtFrlscfN+dfnCPf2YLDbMdXUZL1GEATu2drI5hXVfO+FK5y6Nsa1Pi9mk4GV9ghu\nrIhDTfyXf5ZzGkxGgQ/e3cF797blbEg+43PpgT3ZSzFkY1vtZl6NyzZ2m4Yqn+nEp6ZITE9jX71m\nwetMRgOffGgd+zc38IOXrnHs8gjHLo9gtRhZb4hgilk5fzzB4SHZgVpdZuV337OG/Zvqc54srS2y\nYI/09sIq9WUdFPY27OTZ2GWwJPJ2pCvm0VzlqxXz1MkbE/zwxSu8eLyXF4/3Umkzsgq41jfIPopD\nSU2nYMGeb5ynR+NOvliU9zVisVThrK6mbUMZ79ndxuZVM4Ih1zgDa1cRPH8O2/QYFR2Zj+gLsT+4\nk05OMMl43nMyeLMTwWikactaDJbcmt7nP7qDx9+/mVeO9/DayT68kWvgr0OYqmJ9exWbV9Zw59ZG\nOhrV1X4BGGxvI9DVTXWFDYM5u1DK9m9cX76Wq/gYZgCPJ3Ps+EIkwmGuDg9TtnEDtbW50/o9Hjd/\n9bkauod8vHlmgDfODBCMD+CmkcRoLVtX13Dn1ib2b26gXGX2Y9yxnj5AGh7I+7f0mGqJAn7HOOs8\nC6+nTM+Y6L4KQNW61arG4PG42bOliWMXhzhxeYSzN4eJ+idwBMqxhl3csb6Sh/a2cceGeowqhaxh\n01omngHT+FDWcS5Edc0WXox3EbIFcFeYsZltuW+aw/SY/Oy6LesoU/H8h+vKeffuVl4+1suxi8Pc\nnOwkOi4QIXbbyCotFCzY6+vrGRiYyXAcHh6mtjZ3NblcmvBS4XY5KBMcfOKTM45TZWwejzvnOMUa\n+eUbOXeZWEO75uf7RuQ42QlxnDOd17KmbWdDEkUC3d2Y6xsY90YA9RrvvnW17F3r4YsHj8BwHfva\nV/LgYzPt77T8RoaGJqTrNxg4dzWrlrTQfNojZYCPs5PnGRrOXP99IUI3roMkYahv1DRuh1Hg3Tua\n2L3RzZd+fhTGG/n9d+1gzUY5SS0aijIaUn8cN9d4mL5xk5ERX14+E1vcSVgI8XLnm7Q5s5/Ass3l\n+DlZ449X1WmahxV1Lvl/66Z58RdhhEAl//UTu1MmoYnxzBFVmYiVy9FCE5ev0Yz2dz0WjSMkjMRM\nYV64eDjl79DC1JVrIAiEnFVEVDzf43EzNRlk56pqdq6q5t8vXeTwwDH+eNunbxtZBeo3yYKdp5s3\nb6anp4f+/n6i0SjPPvss73rXuwr92iXD6ZKTc/I9eaQch3nalxUbXswS4a2hzK3SFiI2MoIYDmsq\nS5pOr7+fgbhc7yYRzj/LrpAIIYBIQN7gvMIklyauar9/AYehGo4NnyJqliNlCnGYWVtaSUxPk/BO\n5b44A2JIQLLEOTt2nmBMe/io1m5BczkyeCxlDss3httUXYPBbi/4nYibI6mINS2k/Ax1dRhs2rX9\nSCLKyZEzVNrKWVe1OvcNtyEFC3aj0cif//mf8+lPf5pHH32URx55pGgcpyB73RMFZJjJHdqteduX\nlZfHZIWjQydJiNpC5RSBls1hmIu3Bk8gGuIYjIU5itK7SuVD+sv81tAJzfcXItglSeLI4HEkc2zW\nWPIhFcedx3qQJEmO5XdZiIlxToyc1vwdkd4ejC43psrsDS6yMRme4vLENdxuuTZOvvOQym0YHiYR\nztzjdyGUd6LM7eCGt5ORYPZEokzEx8cQQ6G834nTI+cIJyLsadiZtarn7Y4uo77nnnt4/vnneeGF\nF/jsZz+rx1cuGWo61C+EYDBgbW6RO7THtH+H8vKsbmhjOurn4sQVTfcXItBiYpzjQ6dwW1w4XbaC\nsuysTc0gCPlvcIEoJrOB2rIazo1eIBDT5kyO9PSA0Sg3QdFIl6+HocAwq+rlzamgeSigxEIkHEcU\nJWoqKhAQNGuriWSbQmtLa15moLcGTyAhsaJWbpBR8AYnSQS7ta8H5bntHvm31DoPhZ7elAYwe+vv\nyHHl7Utxbkc6okdYk7W1FUSRaL/6+t8KyrH/jhbZtq2kcatFrfc/E+fGLhKIB9lVvx2nq7CiRwab\nDXNdHZFe7bVzYKb29r6GO4hLCY4Pq9dWpUSCSF8v1sYmBJN2t5FSUXBfm1yeVw+NPZ/QT+W55W4H\nG6rX0u3rZcA/pPr+mYxT7WtBlETeGjyGxWBmbf0KoHCTFID/Zqfme5WNdVVdG3aTnbcHj2s6yabe\niTzMUWOhCa5O3ZAT5xboRXC7844X7Hp0S0lpaXmYIZTnrqxrodXdxIXxy0xFvKrvj/T2YPXUaA61\nhJkyxXc27sbutCBJEM6Qbq8WW2sbYihEbEzb0VkUJULBKA6XlV11OzAIBt5MK2Obi+jQEFIslteL\nHI6HOT5yhipbJRtqV2O1mQpaC6bKKowud14ae3oxOKWsw9z2gQtRiGC/PtWZKlNcWe6aNZ58UHwu\ngc4uzfcq819WZmdX3Ta80WlNJ9lCNPa3FW29QbvD9naiJNiz9PzUQiGOQ7n9lwmTycj+xt2pgmRq\niHu9JLxTODsy92ZdiLHQOJcnr7GyvJ16Z50uRY9mzBDa5iEciiFJ8m9RbnWzuWYD/f5BeqZzl1eV\nn9clP19D82qFkyNniSai7G24A4NgwOW2FiTYBUHA2tpKbHSURDB3Hfh00ovBba5Zj9Ps4O2hE6q1\n1ZlSAtrnQaluuq9hly6nWKV2TiAfjT2tAJgSEXNk4Jjq+yM9PRjLyzGVqw/XBbmD2uHBY1iNw/az\nyAAAIABJREFUFrZ7Nue+4TamJNh1qAlhaWzKu8FAeu3tO+q2YTaYOTxwdFYv0GwoJwRnR7vm5x5O\nvihK5T5dTi55OlDnNthQxvRG/1tZ70lH2VBteQi0wwPHEBBSdYJcZTbCwRgJFZ12sjErUUkDMxq7\nFZPBxO76HfhjAc6MXVB1f7inB8FiwVKvLWQ2FA9xauQcHns1qyo6sCeTowra4JK1c4Ld3Zpr56QL\n9hZXE02uBs6NX8IXzR12mPD7iU+M56WtX5y4wlTEy676HdhMhXVuutWUBLsOGrvBYsFS30Ckt1dT\ng4FU+6/kGOReq1sYC09wbTJ7aVWFcHdSsK9coWm8CTHBW4PHsJvsbE82ULiVGvvcssXrq1ZTZavk\n+PBpQvHcURWRnm4QhKy1crIxmFamuMomR5G43PILHQ7mb5KaqSFU2DwovQFe7zuS9R4FMRYjOjiA\ntblZdfNqhbcGTxATY+xv2I0gCBiNBmx287ym1lqxtrUhRqNEhwY13Rf0y450s8WIIAjsb1B/kk1F\nieVhlns9qUjcnaEnQ7FREuw6VXGztrbKDQZG1duXQ0nh4XDOZGoq2qrSwWchlKO3a4U2wX5+/BLe\n6DS767enyhTrobGbysowVlRoPrnMbTSS3gv0+PCphW6diVmu1R6zrPgY0rskKYK9kHlImeY0nlzm\ntoOrd9aypmIlV6duMBRYuLBYdKAfEgnNZhhJkni9/wgmwTgrEShTU2utKPMQ6dY+D+lF8XbXb8ds\nMPN6/5GcJ9l87eujgXEujl+ho6x1XpniYuQdL9hNJiMWa2EOM8jPgTpjgpg59q0ob6PeUcupkXN4\nIwsfPSM93Rhdbiw12rz3mRooOJNp84U2FrC1thGfnCQ+rb65daZGI/uUXqD9CztR42NjiMFgqtGF\nWsLxMEcGj1NucbOlZkPq704dBLu5tg7BastbY7enbfR3N8s1Z17PYZbK13F6ZfI6w8FRttduxW2Z\nccA7nBaikQRxFX1Ss2FtawcgnPSBqCEVy59WBM1hdrC7fjvj4UkujF9e8P5wlpLFuXj55htyb9em\n4tfWoSTYAXRpXGtTFnFXl+p7UppqmkATBIEDzXeSkBK80Z/9CJ4IBOSY5bY2TTHLI8HRlGbS5Jqp\nRTLjMCvw+J2HGSJTa8ByaxmbazbQ5x9I9QLNhCI0tEbEvDV0gnAizN1N+zAZZkIkXW7brDHlg2Aw\nYG1J5jZE1X9PwB9JOdIVttZspMzi5u2hE0QS2b8rX8fpoeQaO9A8u2iZLj6X5hbZ96RBY1cc6XPL\n9R5ovhOAg71vLnh/pLtbDr1VUdZEISEmePnmYewmOzuz9HYtNkqCHXkRh0M6Ocw0LOJsLfH2NOzE\nbrJzqP8IsURmW2+mLjlqeLVX1kzua7l71t9TDrMCN7h8en9mm4d7mmRh82rv61nvjeQRsyxKIq/1\nvYlJMHLXHA3NVVa4xg7JVnnJ3qNqCQWiqYQ5BaNBjpYKxcOcWCC2P9zTI/sZmptVP28yPMXZ0Qu0\nuBppL5ut4ephojRYrdibGjU1t86k7AA0uRpYVSF3VxoKjGS8VwyHiQ4NYm1t0+RnOD16Dm/Yx976\nnRl7uxYjJcGOPkX1jQ4H5to6wt1dquOvszWxthot3NW4B38skDVRJ9zdBYBNQ4ifPxbgyOBxqmyV\nbJvT29VoNGBzmAno4GsArSappAliTqnatZWraHY1cnLkLGOhiYz3ztRGUX/0vjRxjZHgGDvrts0y\nP0CaYC/UcagxQkh2pMczNpa4q3EPAgIH+97MuLYkUSTS24uloUFVdU+FN/rfQkLinub98059ejWa\ncK1ckWxunVkYz2WhXqeK1n4oy0k20tsjN5xJnp7VIEkSL/a8hoDAPc3aSy3frpQEO/o5UG1tbXJz\n67HMfRTnEsiiqQIcaN6PQTDwat8bGV/mGYHWrnp8b/S/TUyMcV/znRmrJzqdloJfZHONB4PDkYrY\nUUMwEMXuNGOYo2UJgsC7Wu9BQuKVLFp7uKcHU2UVJnfuUr0KB/veAODepKBIJ2WKKXiD09YPN7TA\nWqi0VbCzbiv9/kHOj1+a93lsZBgpEtZkhgnHw7ze/xYOkz3VJSkdvRpNOJOOfbV29oUE+9aajZRb\nynh78HjGaKl8lJ0rk9fpne5nT/N2ah2e3DcUCSXBjj72REhzFiUXWC5CSU3VmaHed6Wtgu2ezfT7\nB7k2Nb/VYKS7G4PdPqt59ULExDiv9b2JzWhjX2PmeucOl+wwixXgMBMEAVtbO7HhIRJBdfVeFmpi\nvbN2K5XWCo4MHMUfm53wIzevntKkrQ/4h7g4foUV5e20ls03WzidFgRBB5NUY5Pc3FqlSWohgQbw\nnrb7AHi+65V5G324S04CsmlIVDvUf4RAPMj9LXdnND/oEQYMssYO6k2UC82D0WDkQPN+wolIRlv7\njGBvVz2+F7pfBeD969+j+p5ioCTY0U+w2zQK9kAggsEgYLVlrm9yX8tdAPxmzssshsNEh4dkW6JK\nx+nx4dP4otNy+QBT5rBAvY7fyganRluNRePEogkcWZpZGA1G7mu5i6gY4/W+2ZEh+djXn+18AYD3\ntN2b8XPBIMz0Pi0AwWTC2tSsurl1LsHe5Gpgc80GOn098zZ6xWFva1Mn2COJKC/3HMJmtKXMG3PR\n6xSrJM+p3uCy2NgVDjTvx2ly8HLvoXlljSPd3QhWG+Y6dQla3b5erkxeZ23lKlZW5Vfm+HalJNjR\nJzkH0h2oXaquV7JOswnnjvI2NlSt5erkdS5PXEv9PdIrN69Wm4QRS8T4deeLmAQj97ZkfpFhZh4K\nFWqK5hjuzJ1OHgwosfzZbcPKZnSw7w3CaUfwlIamUmPv8fVxevQ87WWtbKpen/U6R4EF0RSsrW1y\nc+vB3MXhsvlb0nmw7X4Anu96ddbfw12dsuNU5Ty80f8W/liA+1ruxGG2Z7xGL43d5HTKvqcedb6n\nXPNgM9l4oPUAoXiIV5MmNQAxEiE6OICttVW14/TF7oPAzGloOVES7OinsRudTswejyoHaqZ43Uy8\nb+V7AXj6xq9TyRlhjbVRXus/zER4kgPNd6YyLDOhxNMXHPrZnhTs3SoE+5xyAhm/z2Tj/pa78ccC\nPN89I9RmTBDqErSeufk8AI+teHDBk47DaSURF4lG8jdJyePqmDXOhcgWGZROR3kraytXcXnyGlfH\n5MxkKZEg0tONpbEJgzV3Gnw0EePFnoNYjZZ5kVHpWG0mDEZBl2bO1tY2xECA+MR4zmuV9ZDJiaxw\nT/N+XGYnr/a+ntLatTpOu329nB49T6u7ibWVq1TdU0yUBDv6aewgmyHEQID4+MIO1Eg4jpiQcgr2\nFncjd9Rto9c/wMmRs/K93eodp/5YgOe6XsZhsvNQ+/0LXjtz/C4sIsRUVS1XOFQR069GoAE80HqA\nSmsFr/S+zlhoAkmSCHfexFhRgakid1OJ61OdXJy4wpqKlTm74ug1D6kNrjN3eYhcphiFhzveDcC/\nnvwhoiQSHRpEikZTz8rFq72vMx31c6D5TpxmR9brBEE2Sekh2BVnphqHeiAQxeYwY1ygcbjNZE1q\n7WFe6T2U/O6u5LPacz5DlER+dPVpJCQ+uOqRvGrX3+6UBDtgs5sRhMJty6Dezp7LlpjOYysexCgY\neebm88TFOOGebtXFnp7rfJlQPMx729+FY4EXGfQ7uQiCgLW9ndjYKAn/wr0y1ZggACxGCx9Y9TBx\nMc7Prz9LfHKShNerSlsXJZFfXP81AI+tfDDn9XqZIaxNzQgmkzqTlMr1sKqig931O7g52cOhviMz\np5b29pzPGA6O8uuul3CbXTzQeiDn9Ypg18MkBRBRc3LxR3HmWAsga+1ui4sXe15jKDCcMn+q0djf\nGjxOl6+HnbVbWbMMtXUoCXZgRjsp1LYMaY7DHNrJjKaa+/hcY6/m7qa9jIXGefbys0T7+7C1tee0\nJfb7BznUf4QaWxV3N+/P+Rw9Ty6KoMm5wanUVEGOkFlR3s7p0XN0npdjme0qBPvzXa/S6etmR+0W\nVpS357xeL1+DYDJhbWsn0tebMwM1FIgiCLKSkYsPrXoUp8XBMzefw3dD7g9rzeE4FSWRf7/0E+Ji\nnI+s/cCC2rqCw2VBTEhEwvm1jVRIKTs5NrhYNJF0pOdeC1ajhY+u+SBxMc5TF39EuKsLwWrNqewE\nY0GevvEbLEYLH1z1iOp/Q7FREuxJHAU2tVZIFYDKqbHnti2n89iKB6m113DhzKuy4zRH4a9ALMi3\nzn6XhJTgw2veh9mQu7OQXho7gK09Gb+cQ0tTa4oBeQP+8OrHEBA4d+ol+Tk5BHunt5tfd71IhbWc\nj679kJqhF9wuMR1be4ecgZqjMJrib1FjFnBbXHx8ywcJJyIMXz0jtwTMUdnyzYG3ueHtZKtnk+pa\n44rSESgwWcvocmGuqyfcdXPBDFTF9KVG2QHYVruZXXU76J/sITI4ILcEXEDZkSSJH1/7Jf5YgIfb\nH6DSVqHtH1JEFCTYn3vuOR599FHWr1/PhQvqakbfrjicFuJxkVi0MIeZ0eXCVFOT04G6UHJSJmwm\nG7+36XdpnJDHF2/OrpmIksi/XfgBY+EJHmq7n81pRa4WwmwxYjIb9NXYcwl2laYYhbayFt634iEq\nRmQTj9CcvRJfOB7m3y78AEmS+E8bPqpKS4UZwVKojR3SI4Sy29klSSLojy7oMJzL/Sv2s8rVimPE\nR6imbMGWgDemuvj59Wexm2z8b2s+oNqmrOcGZ1+xEjEUWrCEb0CDeVLhI2veR7vfgiBJRBqqFrz2\nmZvPc3ToJK3uplQo8XKlIMG+Zs0avv71r7NrV3G3kQL9Mu1A1lZFv3/BEr4zyUnqF3Gzu5GdIbmS\n43+E3s7YvT0hJvjZtV9xceIKG6rX8sgK9YkXejrMTBWVGMsrcjpQlSbWFqv6XqUPtNxDw6TERJmR\n73U9k7HD0Hhokq+c+iZj4Qne3XYvaypXqv5+XU8uyRPFQoI9GkkQj4ua1oJBMPCJqvswiXDdHeLZ\nzhczXnd54hpfP/0vxMQ4v7vutym3qs/Q1cskBaROmOGb2edB2UDU2NgVHGYHDxnWAfBi4irnxi5m\nvO5g75s83/0KHns1f7T192YVfluOFCTYV6xYQXt7e8Hmi9sBPe3L9lWyQyZ841rWawIabMsKkiTh\nGJwk6rJxnXH++7Gv8ubA28QSMSRJotPbzd8f/0de7XsDj72axzf8DgZB20+smKREsfDf1NbeTnxy\ngrh3Kus1akI+5xIfGcYUjROsr+T06Dn+7uiXOTt6QY6UiYc5N3aRvz/+VXqn+9nXsItHO7RlFerl\nPAW5hK/B4Vjw5JIyy6k0QSiYBuSNPVBXwW+6XuKpiz+k0ys3E58IT/JKzyG+ceZfEZH47OZPsq1W\nW7s3p1OfujkAthXyxhq+OT+LWkFLQEE65UNyiejBWgvfPPtdXuh+lfHQJJIk0Tc9wJMXvs9Prv0S\nt8XFn2z7zLz6QMuR5b1taSC1iHXQ0uwrZcEeun6dsn2ZE4JSha80CLX4xAQJr5eqHTt5fOPd/MeV\nn/H9yz/l+5d/ikEwpOLc72zczftXPpwzCiYTDqc11dRaq8Cdi629g8CZ04S7unBtnV+PRBQlQoEo\ndU3qtUiYMe9s2v4uhhoDHB44xjfPfReb0Uo4IQsho2Dkd9Z+iDsb92gOZzOaDHJTax0EuyAI2No7\nCF68QMLvz9h0PJDH6Q1InYYevPPjdE78hreHTvD20AncZhfTMdlUZTGY+YMtn8oZ4pkJXcOAm5oR\nzGbCndkFeyCPDU6SJELXr2EsL+f37v5j/vncv/H0jd/w9I3f4DQ5CMTlshaNznr+04aPUmPX1rug\nWMkp2B9//HHGMhS1euKJJ7j//oXjohfC43Hnfe9iUN8oCxdBmj22fMYpVmygz2Ih1n0z6/3RcByH\n00J9vfqGu2NXzwFQvXkDWzfdza6Ojfzowq+YCE4RSUSxGM18eOPDrPdof4kVqmuc3LwyitVsKvg3\nMm3byPjTP8cw1IvnATkZJv07/dMRJAkqq5yanjU9JJfCbb1jO19Ys5rf8j3ED889Q59vkFpnNR5H\nNfd27GNVdXte4/Z43JRV2Jn2hnVZp8GN6whevIB1apjKjoZ5nw/2eAGoayjT9LxY900MFgsb9uzm\nHw17OTt8iYOdR7gwcpXtDRvZ0bCZXU1bqXLk5yS0W+UInXhMLGgelHuHV6/Cd/kKVS4TRvv8jFcx\nLp8SW1orqax2qvruyOgoiakpqvbuYf2qjaxs/L95vfsoNya6uTnZTXtVM4+tfTfbGzbm3OBvN5lU\nCDkF+5NPPrkoDx4dzd2YdimJJ731I8PTqbF5PO68x2ltayd4/RpDPSMZF7HPG8JVZtP0/aOnzgOQ\nqGtO3mfmtzs+OG+chcytYJQXf3/fJEZLYUFTiepGEATGz5zH8eD0vHGODcv/bTQZNI158uIVMBoJ\nuqoJj05jxcUn1/zO7IvE/OZBGaPVZmJ0KMbgwBQm8/xKmFoQa5sAGD59gXjzfFv/0IAs2EVJUj3m\nSrtAsKcX+9p1jE/K2ZdNplY+vroV0vb1RABGA/mtB1GUEASYnAjkvabSf3NjcxtcvETf8XM41s0v\n6TAxLhd5C0diqp83ffQMAIaW9uQ9RvbX7GN/zewSvGNjC+dTFPKuLyVqNx/dwh2L3c7u1Cm0S8G2\nchUksyPnEo8liEYSmk0doZs3wGDQVL1OK3ral40OB9bmFsKdNxFj8xuGaAl1VJDicSK9PVhbWjGY\nc8d858tSOlDzsS37Ll8BScK+Kv/TWS4MSkG06cLnANLs7FnmIdVBSsNGGrpxHZgxf5aQKUiwv/TS\nSxw4cIAzZ87wuc99js985jN6jWvJ0VOgAakXLpxceOnkLdB6urE2NauqCZIvelX1U7CvXo0Ui2Us\njKY11BFk+7oUj2PX2MBbK3rOg6miAlNVNaEb1zPGcSvKRKbyzdnwXZCjP+yr1xQ8voXQqyAazETG\nhLI4UIP++R2kchG6cT2ZCLa8qjMWSkHO0wceeIAHHnhAr7HcUowmAza7WZfQLgDbSlk7CV2fHxmT\nj0CL9PUixWIprWex0H2DW72WqVdeJnTtKuzbMeuzlNPQrX4eglfkZsb2Net0GV829HQcAtjXrmX6\nyGGiA/1yL9A0gn456zS9iXUufJcugyBgX7nI68FlZXTITzQSx2or7IRkqqzCWFFB+OYNJEmaZfNO\nxEUi4Tg1deojVsRIhEhPN7aOFRjMy6OlnV6UMk/TcLosuoR2AZjcZZjr6uRFPEdLyycRQ9FycmWc\nFopTx9hlkDV2QBbsc8hHUw1dvSJ/75q1OowuO8qY9BLsjrXyRhRMjj+dgD+C3WGZ10EqG2Isiv/a\ndaytbRhsmcvu6oWe60EQBOwdK0l4vfMqPeZzig13dYIolswwGSgJ9jQcbqvcQShaWG0MBfuKVXK2\n3eDsbDslo1GTQFM01UW0qQLYHMkOQjpkXYKcqGT2eAhdn2+GCE5re5mleJzQtatYGpswlWkLkdSK\ncnIJ6DQP9qRgV35HBSXrVJNA60yao1Yv7lqAxTjBJTf6K7M3uFSoo1P9O6GYOW2LfGopRkqCPQ29\ntVVbMlEpNCdRSUvhK5CbFQcvX8JUVY25tk6XsWXDYBCwOy26vcgA9lVrEIMBgr19s/4e8EcwGAVV\nha9Arr8jRaPY1y6utg76m2LMNR5MlVWErlyZZa/OJ+s0nDTvLbZ9HcDp1i9JCcCxXi5vEbw0O0M0\nmEcsf8lxmp2SYE9D7+O3suDC1+YIdo2mmEhPD2IggGPDhiWpHe10yZUu9Yp0UgSQ7+Lslzngj+J0\nWVX/mxRtVzFrLCZ6a6qCIGBfu5aEf5rowExHpXyyToNXZbOWfdXiC/bUyUWnebA0NWN0uwlcujBr\nfWk1xUiiSOjGdUzV1arq8b/TKAn2NGZqY+ijnVgam+RFfPH87EWs0XmqaDeKtrPYOF3WlDNLD5Tj\nt+/ijBlCFCWC/ogmDW2pHKdAMuzOoFt0EMxsSKErl1J/05p1Koki4RvXsDU2YCpXn9yWL3qfXASD\nAce69SSmpoilFQTT+k5EeroR/f4leyeKjZJgTyNlitEpblcwGHBs3ETC6yXa15v6e9AvF74yW9TF\n6wYvyZUzHeuWSLAnj9+BaX02OHN9A0aXG9/FGYEWDkaRJPWaqhSPE7p+DUtD46Lb1xWcLqu+Jqk1\n8x2oWjX2aH8fYihE2frsPVv1xKljpUsFx/qNAATSzDFaywkEzstZ2M5N2urfvFMoCfY0UuVaddLY\nYWbhKQsRwO+PqDZBiLGoLNCampdEQwP9fQ2CIGBfs4bo2BjRoaFZ36021DHc3YUUiaSckEuBw2kh\nFNSnIBqAubYWU2XlLDu7Vo1dOb2VbVwawa6EYOql7EBmO7tWG3vg/DkQhNQmUWI2JcGeht4CDcCx\ncRMIQkqwJ+Ii4WAspRXnInzjBlI0uqRHTr01dgDnlq0A+M+ckr9bY6ijEua4FPZ1BYfLgiRBKKjn\nBreOxLQvFSml1d/iP30KBIHKnTtyX6wDBoMBu9Osq0nK7PHIkVKXLyEl5JLLWk6xiUCA8I3r2Fas\nxOhUV1PmnUZJsKeRqsmuo8ZucpdhbWsndP0aYjg0I9BUaqpLbV8H/TrnpOPcvFXe4M6clr97WqOm\nelk24yx2/Ho6etuXIS2e/bL8u2rZ4BJ+P6Hr17CtWImlYum6/zhdVgL+iK5lQxzrNyCGQqkG14FA\nRHUHqeClCyBJODdv0W08y42SYE/DaJS1Ez01dkiaYxIJgpcupR291WmqwUsXwGDAsQQhfgrKpqPn\nPJjKy3GvWU3o+jUSfr8mm2oiECB4+RLW1rYlM0dBWv0gHU8ujqRpzn/yhPzdGrJOA+fPgihmLIG8\nmDhcFuKxwruLzfrOpAkleOkCoigSCsRK9nUdKQn2OSyGdpJuZ1eEhBpTTCIYINzZKadML3KGYTqu\nRTDFAFTt3gWiSODc2Rmbqop58J8+CYkE7juWtlNXyiSl48nFXFWFbeUqQlcuE/f5CGrIOvWflk87\nzq3bdRuPGmYK5OnoSF6XPLlcukgoEEs+J/fpTZIkAufPYXS5sbaW6sNkoyTY5+BcBO3E1rECg8NB\n4MI5/Elh6VIj0E6evCVHTovVhNFk0NUkBVC1+w5AtrPPmCByv8z+48cAcO1cWsGu/EZ+nTc4985d\nIElMnzyhOutUiscJnj+L2ePB0pi9z+ti4FgkE6WtYwWhq1fwDcvlBdTMQ7S/j8TUFI6NmxZsXP1O\npzQzc1gM+7JgNOLYsJH42BjTQxOAOk3Vd+RNAMr27Mtxpb4IgiAnKekYCQFgb2nB7PEQPH+OgC+C\n2WLM2es0EQwQuHgBa0srlrrFzbqdy4wTWd95cN0hb3BTx0+qzjoNXrmMGA7j3Lp9SZLU0tGz92k6\n7n37QRQZPymH86oxTwbOlcwwaigJ9jnoHcuu4Eoen729Q7Oek43Y+BihK5exr1mL2ePRdSxqcLqt\nBANREon5ZWbzRRAEnFu3I4bDBLxBVRqa/9QpSCRwLbEZBtJ8DTpr7OaqamwrVjJ1U85tUGNbDiSj\niVzbltYMA/pnZCuU7doDRiMTV+X67K6yhedBkiR8bx0GoxHHpk26jmW5URLsc9C7NoaCa+cdGBxO\npsenEYTcx07fW0cAKNu7X9dxqEV5mUM6hrmBLJhEDISjkioNzX9CNsMstX0dwGQyYrObdDfFgLwe\nIkbZb5Jrk5dEEf/p0xgcjkUvApeJmdr0+s6D0e3GuXkLfp86v1P4+jWi/X24tu/E5F6aJLVipSTY\n57BYx06DxULZnXcRESzYzCzoLJMkCd+RNxFMpluiqcLiRMaAXJ0yXlkLgMO+cMxyIhggcOE81pYW\nLHX1uo5DLU63VXeNHeSNShHsuTT2wNkzxCfGcW3fiWBa+v7zi3WKBSjbt5+ISW66nsvvNHXwFQAq\n7r1P93EsN0qCfQ56t8hLp/yee4kYnViiC/dfjHR1EhsawrV9B0aHQ/dxqGExQv0ABJMJy54DAJgm\nBhe8dvrYMdkMs8RO03ScbiuxaIJoRJ+6OQrm6hrE2mYArGSfY0mSmPj1rwCofM9Duo5BLQ6XFUEA\n/3RY9+92btlGxCr38XQ4sod8xqd9+E8cx9LQuKTZx8VKSbDPYTGSUhSkihpEgxGzf4LIQH/W6xSn\nqXvfrTHDwOKE+ikIa2THl3TzEmI08zyLkQgTv3oawWymbP9duo9BLYsV+gkgJRtbx46+kfWa0LWr\nhG/ewLltO9amJt3HoAaDQcDhshLw6T8HBrOZmKMKczxE5NrlrNf53ngdKR6n/MB9S+48LkZKgn0O\n9mSjCb1NEEDKlmiNB/AefDXjNdGhQbyvH8JYXoFzw61zEC3m8Tuc/EqzfwLf4cxCbfKlF4hPTlL5\n7gcxV1XpPga1KCeXxbCzx9zVAMRPHSHS25PxmolfPwtA1Xsf0f35WnCVWQn49auboyBJEiEs2OIB\nxn72E6T4/JORJIp4XzuIYLFQtv/WKTvFREGC/R/+4R9473vfy/vf/34+//nP4/cvbGIoBpTO7Ho7\nT2FG+7WbJbxvHCLS2zvrc0kUGXryO0ixGLUf+/gtsacqLKbGrmyaNqJMPv+bVL0QhbjPx+RvnsXo\nclP50MO6P18Li1E3R8E/HUEQwBIPMfqTH837PNLbQ/D8Wexr1t7yZhIutxVRlHR3pkfCcRIJCWeZ\njUh3FxPP/XreNVMvv0hsbBT37r0YHaXaMGooSLDfddddPPvsszz99NO0tbXxzW9+U69x3VIcLquu\njSYUFOHg2b0NKRql/2tfJj41lfp88sXnCd+4jnvXbjmJ5RaSciIvgkBTNovqbZuIjY4y+sMfzGqb\nN/7M04jhMFXve/8t8zEoLKZgD/giuMpsODdsIHjh/KwKoHHvFEPffRK49do6LF6yljJpmH0fAAAa\nGklEQVSvVWtXYKyoYPyZp4mklbj2nznN6I/+A2N5OdXve7+uz17OFCTY9+/fn4ru2LZtG0PJkqzF\njtNlIREXCQVjun6vsohrNq+j5kMfJj4xQf/XvkLg/Dkmnv8N47/4GUa3m9qPfULX5+aDEuq3GCYp\nZR4aH3svloZGpl55iYGvfYXg1Sv0f+0reF99GXNdHRX33Kv7s7WSEmg6z0MiIRLwR3G5rdR8+CMg\nCAz809cY/fF/ELx0kZ6//SsiXZ2U7b8zVV/mVuJMxpj7dbazKxuFu8pJ3Sc/BYkEg//yTbyHXmP6\nxDEGv/UNBLOZpj/5Auaqal2fvZzR7az/k5/8hEceufWahR64ymwA+KZCGC36uSHSC4BVvPcRosPD\n+N58nf6vfEm+QBCo/cSnMLrduj2zEBwuK36f/pEQQX8Um92EraaKlv/z/2Hwm/9E4NxZAufOAnIr\nvdqPfeKWmqIUUmGfOgs0xTnvKrNia22j/vd+n7Gf/pjJ559j8vnnAKj+4G9R9fCjt4WzcLGcyOm1\nk1ybtlF+zwG8h15j+KknU9c0/OGfYOtYoetzlzs535zHH3+csbGxeX9/4oknuP/++wH4xje+gdls\n5rHHHlP9YI/n9hBemahvLOP8yX68UyHWbtQvfjoWkW3JbR3VWG1map74Y3qb5DR5Z1srrlUrsdXn\n97zFmM/KagcTowHKy+w5U//V4vG4CQailFfak2N2U/fXf0H3975P4GYnTR98P+Vbt9xSYZY+l5Ik\nYbYYiYRius5xKOmU9tSV4fG48Tz2IB0P3sfwiy8x+tobNH3wfVTv26t6nItNJCg7NRNxUfNzF7pe\nTMjmzqaWSjweNzX/+fP4H32IYE8Pwd4+XCtX4jlwd/4D12mcxUbOt/XJJ59c8POf//znvPbaazz1\n1FOaHjw6Oq3p+qVEMMpCxTcZ0nWck+MBzBYjvukwJGOCHe95FAAJmAam83iex+NelPlUmh50dY5T\nWV24rdvjcTPQP0UkHMdqM80as/PhD+AEYsDY2K1zwmeaS4fLwtSUvmuht2cSAKNZmPW9pt1307D7\nbkQWfkcW6zfPRizp4B4dntb03FzjHB2SP4snEjPXVTVgqGrAtW2PfM0S/DuXej7zRe3mU5Cd4dCh\nQ3z729/mG9/4BhaL+qbEtzvKsdM7FdL1ewMamzffapyL0CpwOmnaUcxdxYDLbSUcjJGI61c3J6Ch\nyuftgMNpwWAQdDfF+DWUsS6hnoLO13/zN39DLBbj05/+NABbt27lL//yL/UY1y1FKUbkndRPsMfj\nCcKhONW1Lt2+c7FZjIgQxWbvLi8ewZ6ejVxWoU9dfH9qgysOgSYnKVkWJSrGZjdhNqtr7F5CHQUJ\n9hdeeEGvcdxWKCnUemrsWhpL3C4ojkM9X+Zpb1JTLRKBBmkRIdN6CnZlHopng3O5rQwP+BBFCYNB\nHx+IPKfFMwfFQinzNAMGg4DLbcWno8ZejEdOd1Lo6Bnipphi3MUk0Bahbo5/OoLJbMBqu/WRP2px\nlVmRJHRrbB2NxIlFE0VjjiomSoI9C64yG9O+sG71yFM2VZV9HW8HFG1y2qtfyGNRmmIWySTlcltv\ni1BGtSjzoFcIbDEqO8VCSbBnwVWe1E50SkyZidctHuep1WbCYjWltGw9mPZGVNWjv53Q2yQVi8n+\nlmIywwC43PJ49drgis2BXEyUBHsWUtqqTkJN0XqLSVMFKCu3Me0N61Zewe8L43RbMRqLZ+nNJOfo\nu8kXm0Bz6Zx9qrbBRgntFM/btcS4dV7ExSrYXeVW4jGRcKjw8gpiQiQwHSk6TdWuc6hfSqAVkQMZ\n0kwxemvsRTYPxUBJsGfBlXIc6qOx+7xhLFYjVlv2ZgK3I8pGpMcG5/OGkaSZTbNYUJp769Vowl+E\nDmSYEcC6bXAlG/uiURLsWdDz2ClJEtPeMGXl+oTKLSXuVN2cwoWaEj7qKrJTC8gbXGA6qkuS0kyo\nY3EJNLtDPrnodYpN+Z2KKKCgWCgJ9iwojiI9NPZwKEY8JhadGQbSNXYdBHsyfLTYNHYgFb+uh8/F\nX6Q2doNB55PLdASL1ahbHaISM5QEexasNhNWm4lpHbSTYrWvw8yY9Qh59E4GgeJKylFwV+h3cim2\nrNN0nGU2gv4ooljYyUWSJDnkswjXQjFQEuwLUF5h10VTTQn2Isyw01ewh2Z9ZzFRVj5TyrlQ/NMR\nrDYTZkvxaaoutz5hwJFwnGgkUco6XSRKgn0ByirtRCMJIuHCOtQrWl4xCjRZABl1MUEUsynGrZhi\nCtzgZE01UnRmGAW9fE/KWtCrREOJ2ZQE+wKUJxddoTZFRRiUFaFgFwQBV5lVN429WDXVMp1MMak0\n+iLc3CDNmV7gelBOPiWNfXEoCfYFKK9MCvYCtZNitrGDvCEVenKRJAnvVKho58DhtGA0GZj2FmaK\nmYlhL855KEu+E4XWUVI2yJLGvjiUBPsCpDT2As0QPm84lZ5fjLh0iIwJh2JFrakKgoC73Fawxp4K\ndSxSU0x5pbwWCq18OqOxlwT7YlAS7AugaCeFRMYoMezFqqmCPsdvRaAVW1JOOmXlNiLheEEnF8W2\nrJwGiw1XmQ1B0FFjL+L34namJNgXQA+NPZTsvFPMtsRULHsBgl0xRxVzeJvyGxZijil2wW40GnCX\n23TR2F1lVoymkghaDEqzugDu8qR2UsDxu9jt66BPyGOqDnt5cZogANzJzOFC1oMSy1+sgh3ksYcC\nMaKR/E4uibiI3xcpaeuLSEmwL4DRaKCswo53In/tRLElLgvBXsDJxZ/snFTM86BHZIx3MoTdaS5a\nfwvM2MXznQdlHZXs64tHSbDnoKLKTjgUy7u64XLQ2O0OczIiJH9fQzE2sZ7LzMklv40+kRCZ9oYp\nr3ToOawlRzlt5NsTuBTquPgUpDZ89atf5eWXX8ZgMFBdXc0Xv/hFPB6PXmO7LSivcsCNCbyTIWx2\n7ZUZZ2LYi1c7EQQBd4Gx7FMTQSxWE3ZHcVW3TCelqeY5D9PJ6pbFbIaBdI09T8E+mXwninwebmcK\n0tg/85nP8Mtf/pJf/OIX3HvvvXz961/Xa1y3DRVV8uKbGg/mdf+Mxl68tmWQtVUlZFEroijhnQxR\nU+sqqlZwc0nVD8rTBKGY9IpesCshjwVr7MU9D7czBQl2p9OZ+u9QKITBsPwsOxVV8rF5ajI/we7z\nhrHZzUWZbZmOklKfz8s87Q0jJiSqa525L77NcZfbknXltXeUUtaQoiwUKwVr7KnkpJIpZrEoWNp8\n+ctf5umnn8btdvPUU0/pMabbivKkYM/HgSpJEn5vmOpal97DWnIqq+V5mBwPUFOn7d+jnHZqlsE8\nlFXYGBv2EwxENdcR9y2T+ihmsxGny5J3LLtvKoTZYszLtFlCHTkF++OPP87Y2Ni8vz/xxBPcf//9\nPPHEEzzxxBN861vf4nvf+x6f//znVT3Y43FrH+0toL2jGrPFiN8X0TxmnzdEIiFRU+ta9H/vYn9/\n+4oa3uQ60VBC87OuXxgBWJJ50IOFxljXUM7NK2MYMWj+twT9sgN+5eparLbCT3C3ci6ra130dE5Q\nWenAZDIueG36OCVJwucNU1XjpLa2bLGHqYliWJtqybm6nnzySVVf9Oijj/IHf/AHqgX76Oi0qutu\nJR6Pm7ExP+UVdsZH/YyM+DTZiHs7JwCwuyyL+u/1eNyLPp8Gs/zv7uuZ1Pysvp5JAKprF3+chZJr\nLk0W2dzY0zWOzaVN4xwdnsbhtOCbDkGB07AUv/lCOJwWkODm9bHUaS4Tc8cZDESJRRM4Fvmd0Mqt\nnk+1qN18CjKKd3d3p/775ZdfZsWKFYV83W1LeZWdeEzU3OtxYjQAQLWn+G3LTpcFs8XI5HhA871T\n40EEAapqijvMD/KPZU8kRPy+cNE7ThXyLQZWcpwuDQWdB7/0pS/R2dmJwWCgsbGR//bf/pte47qt\nSDlQJ0Ka4rAnxmQhWFlT/IJdEAQqaxyMDfkRRVGTo3xyIoi73JbzyF4MKGtB6wbnm1oeoY4KqVh2\njQ7UkuN0aShIsP/jP/6jXuO4rSlXQh4ngjS3V6q+b2IsgMEgLJuXubLaycjANN7J8ILH73TCoRjh\nYIy6huVhv3SX2zBbjIyPaBPsqVICRR4Ro1Be0thva5ZffOIiUJFHZIwkSUyOBamodmA0Lo9pTkXG\njKkXalMTyRA/lRvB7Y4gCFTXOpmaCBKPq4/pXy4x7AqKxq1VY1fWTrGHfN7uLA+Js8ikkpQ0xLL7\nfRFi0QRVy8AMo1BZo5gh1M+DEuqobI7LgSqPC0mCyTH186AIwOUi2K02M1abSXNew9iwH4vVVNQl\nNoqBkmBXgdVmxuYwa9LYFcfpcnAYKlRWy5uUFvuysgksF40dZpzhym+shuWmsYN8gvNNhojH1J1c\nYtEEUxMhauqKOwO5GCgJdpVUVNnxTYVIJERV1yuO06plEBGj4C63YTQZNGmqisau1iZfDCgJZ+Oj\nftX3eCdDOFyWos9ATsdT70aSYFzlBqfM13JIVLvdKQl2lVRUOpAk9WFuyykiRsFgEKiosjM1HlSd\nUj81EcRqMy2rLEPFvKbWgRoJx5n2qnc4Fws19bJDfHRIXfz32LAs2Ks1Zi6X0E5JsKskPTJGDROj\nAYwmw7Lz/lfWOInHRVWVHhMJEd9UmIpqx7I6elttJtxlVtWmGEXw1TbcXpmWheKplwW0WsE+PlLS\n2JeKkmBXiWJSGVOxiEVRYmo8SGW1A4Nh+Qg0SK8Zk3uD802FEEWJymXkOFWoqnURDEQJBaM5rx0Z\n9AFQu0xCPhUqqx2YTAZNGrvBIKSc8CUWj5JgV0ldo6xtDfX7cl477Q0Rj4vLKiJGIeVAVWFnV65Z\nTo5TBcWBqsYcMzwgrxllDS0XDAYD1XUuJsdyh36Kosj4aICqGueyCf+9nSnNsErsDgsVVXaGB3yI\n4sL25YlRWaAtJ8epwkzIo3qBphzZlxNqHaiSJDEyMI3TbcHpLu6a/Jnw1LkRRSnnBuedCJGIiyX7\n+hJREuwaqG8qJxZN5EzQmXGcLj9NtbzSjsEgpOylCzHQO4XBIFDXWL4EI1taUiGPOQRaYDpCMBBd\ndvZ1BbV29jHFvl4S7EtCSbBroK5ZMcd4F7wuFeq4DE0xRqOB2sYyxob9RMLZ+8DGonHGhvx46t2Y\nLcVfI2Yu5VV2jEYhZ6jfyKDiOF1e9nUFj8rIGCUipuQ4XRpKgl0D9U2y5jnUl93OLkkSw33eZZ1d\n19xWgSTBQM9U1muG+mWTVUPL8tPWQbYvV9Y4mRgLLGiaW672dYXKGtmBOja08AkuFepYEuxLQkmw\na6Cy2oHFalpQY58cDzLti9C6onJZhfiloxRC6+uazHrNYK88R40tFUsypltBtcdJIi4u6G9QNHZF\ns11uGAwGqmtdTIwFsjpQJUlibMSPu9ymS4ORErkpCXYNCIJAfVMZvqkwwUDmMLeeG+MAtKyoXsqh\nLSm1jWWYzAb6urNr7AO98mf1zctTYwdobJM3uO7r4xk/F0WJ0aFpKmtkhWC54ql3IYpS1rj+oD9K\nOBgr2deXkJJg10h9k3ykHs4S9thzU+6a1LqiasnGtNQYjQYaWyuYGg/iz9B8JB5PMDLgo6bOtaw1\ntPZV1QgCdF6d3zoS5HIKsWhi2TpOFXLZ2ZV3Yrmao25HSoJdI3WKnT2DOSYaiTPY68VT75Jbhy1j\nmpPaan8Gc8zIwDSJxPK1ryvY7GYaWysYGZzOuMEp9vXl6jhV8CT/fdlMc9cuDgOwan3tko3pnU5J\nsGukrtGNIGROVOrrmkQUJVqXsRlGoSkp2Pu657/Mg0kzzHK2ryt0rKkBoCuD1t6f7PW63DXVqhon\nVR4nXdfG55kofd4Q/d1T1DeXL9tggtuRkmDXiNliorrWxeigj3BodrhfygyzcvmaYRSqa53YHGb6\nuybnFQQbSDpOl7vGDtCxWhbsN6+Ozvq7fzrCjUujVFTZl71tWRAENmxrQBQlrpwbmvXZhdMDAKze\nUNLWlxJdBPt3vvMd1q1bx9RUdmfacmLNxjoSCYmTR2aaeUuSRM/NcWx207K3qYL8Mje3VRDwR2cV\nRgsGogz1e6msdmB3LG9zFICrzEZtg5uBnqlZG/25432IosTWPS3LNjoqnTUb6zCaDFw6Mzhroz9/\nsh+DQWDlOs8tHN07j4IF+9DQEIcPH6axsVGP8RQFm3Y04S6zcu5Ef6rK4cRogMB0lJaOqmVX+Csb\nTcmwx7PH+1N/e/2Fa8RjIhu3v3PWQ8eaGiRpJjomEo5z8fQADqeFNRvrbvHolgarzcyqdR68k7Lp\nBeTQ38E+Ly0dle+ITf52omDB/nd/93f82Z/9mR5jKRqMJgO77ulATEgcfb0T72SIF56+CEB78mj+\nTmD1+lqqPE4unhrg3Ik+blwe4eaVUeqby9m0s+lWD2/J6Fgja6PnT8kb/cUzA0QjCTbf0YTJtPyy\nbrOxYZu8mV86M5A0ywwCsGrDO2Nzu50oKBbtlVdeoaGhgbVr1+o1nqJhzca6/7+9u4tpMkvjAP6v\ntIDDOKaK06DD6CwOG4gFRhPdgURtbeSjVlFRboymDUZvrCB+hKJGA8aAqJekxAjRZDTK2myI0Wym\nWiEIIsYFN6Q6bHAcjAVRMhSj9OvZC9dO2NJqzOgp5fndnSYn+acfT09P3/c56Or4DY/+PYBfe19g\n7I0H6Uu/mVI/OWXRUuQVKPH3c/fQ+nMvZNFSREmnQZX31ymx/fCOfPYXSPxOjt/6hvGT+Q6ipNMg\ni46aUr9aAEAx7yvI47/Af+zP8fiXFng8Psiio/Dd95F/MUG4eW9h1+v1GBoK/Me/uLgYZrMZZ8+e\n9T/2oafqRAKJRIK/rfwLrl56ALfLixU5yf4Vy1QyY2Yscjcq8Y+f/gXXmAc/qpIi6uDqD5W3KQ29\nPQPobP0Vvw+/RvrSRMTERs6pUR9CIpFg8Y/z0fLPX/DVzFjMmhOHH5Z+G1HHAU4WEvrIavzo0SPo\n9XrExsa+7Y8yMACFQoHLly9j9mz+hmaMMVE+urD/P7VaDYvFgpkzI/8SN8YYC2d/2nXsEolkSm3F\nMMZYuPrTVuyMMcbCA995yhhjEYYLO2OMRRgu7IwxFmGEFXa73Y7CwkLk5+ejoKAADx48EBXlvc6f\nP4+cnBzodDrU1NSIjhNUuPfsqa6uRm5uLtatW4ddu3ZhdPT9B2J/Ts3NzcjJyUF2djbq6upEx5mQ\nw+HA1q1bkZeXB51Oh3PnzomOFJTP58P69euxc+dO0VGCcjqdMBqNyM3NhVarRVdXl+hIE2poaMCa\nNWug0+lQWloKl2vig378SBCDwUAtLS1ERGSz2WjLli2iooTU3t5Oer2e3G43ERG9ePFCcKKJPXv2\njAwGA6lUKhoeHhYdZ0Ktra3k9XqJiOjEiRNUU1MjONEfvF4vaTQa6u/vJ5fLRWvXrqXe3l7RsQIM\nDg5ST08PERGNjo7S6tWrwzInEVF9fT2VlpbSjh07REcJ6sCBA9TY2EhERG63m5xOp+BEgRwOB6nV\nahobGyMiot27d5PFYgk5R9iKXSKRwOl8e+KK0+mEQhGe/SQuXLiA7du3Qyp9e/fcrFnh2ZJ3MvTs\nyczMxLRpb99yGRkZcDgc75nx+XR3d2P+/PmYN28eZDIZtFotrFar6FgB5syZg5SUFABAXFwckpKS\nMDg4KDhVIIfDgVu3bmHTpk2iowQ1OjqKzs5ObNy4EQAglUrx5Zfh2WLZ5/Ph9evX8Hg8ePPmDb7+\nOnQbZGH3+paVlaGoqAhVVVUgIly8eFFUlJAeP36Mzs5OnD59GjExMdi/fz+USqXoWONMxp49jY2N\n0Gq1omP4DQwMICEhwT9WKBRhvT0IAP39/bDb7UhLSxMdJcC7hca7xVs46u/vh1wuR1lZGex2OxYt\nWoTy8nLExobXgSAKhQJ6vR4rV67E9OnTkZWVhczMzJBzPmlhD9ZnpqSkBLdv30Z5eTk0Gg2uX78O\nk8mE+vr6TxknqFD9cLxeL0ZGRnDp0iV0d3ejuLhYyEpusvTsCfWaq9VqAEBtbS1kMhl0Ot3njheU\nyOfsY7x69QpGoxEmkwlxcXGi44xjs9kQHx+PlJQU3LlzR3ScoDweD3p6enD48GEolUocO3YMdXV1\nMBqNoqONMzIyAqvVips3b2LGjBkwGo1oamoK/fn55BtEQSxZsmTcePHixYKShFZUVEQdHR3+sUaj\noZcvXwpMNN7Dhw8pMzOT1Go1qVQqSk1NJZVKRUNDQ6KjTejKlStUWFjo3y8MF/fv3yeDweAfm81m\nMpvNAhMF53a7yWAwUENDg+goEzp58iStWLGC1Go1ZWVlUUZGBu3bt090rADPnz8ntVrtH9+9ezcs\n/w+4du0alZeX+8cWi4WOHj0aco6wPXaFQoGOjg4AQFtbGxYsWCAqSkgajQZtbW0AgL6+Png8Hsjl\ncsGp/pCcnIzW1lZYrVbcuHEDCoUCFoslLBuxNTc348yZM6itrUV0dHgdvKBUKvHkyRM8ffoULpcL\nV69exapVq0THmpDJZMLChQuxbds20VEmtGfPHthsNlitVpw6dQrLli1DdXW16FgB4uPjkZCQgL6+\nPgBAe3s7kpKSBKcKNHfuXHR1dWFsbAxE9EE5he2xV1RUoLKyEj6fDzExMaioqBAVJaQNGzbAZDJB\np9NBJpOhqqpKdKSQwrlnT2VlJdxuNwwGAwAgPT0dR44cERvqf6KionDo0CEYDAYQEQoKCsLyQ37v\n3j00NTUhOTkZ+fn5kEgkKCkpwfLly0VHm5QOHjyIvXv3wuPxIDExEcePHxcdKUBaWhqys7ORn58P\nqVSK1NRUbN68OeQc7hXDGGMRhu88ZYyxCMOFnTHGIgwXdsYYizBc2BljLMJwYWeMsQjDhZ0xxiIM\nF3bGGIswXNgZYyzC/Be68EGj7hfMcwAAAABJRU5ErkJggg==\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f385e198650\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "def f(x):\n", - " return tf.square(tf.sin(x))\n", - "\n", - "def grad(f):\n", - " return lambda x: tfe.gradients_function(f)(x)[0]\n", - "\n", - "x = tf.lin_space(-2*pi, 2*pi, 100) # 100 points between -2π and +2π\n", - "\n", - "import matplotlib.pyplot as plt\n", - "\n", - "plt.plot(x, f(x), label=\"f\")\n", - "plt.plot(x, grad(f)(x), label=\"first derivative\")\n", - "plt.plot(x, grad(grad(f))(x), label=\"second derivative\")\n", - "plt.plot(x, grad(grad(grad(f)))(x), label=\"third derivative\")\n", - "plt.legend()\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-39gouo7mtgu" - }, - "source": [ - "## Gradient tapes\n", - "\n", - "Every differentiable TensorFlow operation has an associated gradient function. For example, the gradient function of `tf.square(x)` would be a function that returns `2.0 * x`. To compute the gradient of a user-defined function (like `f(x)` in the example above), TensorFlow first \"records\" all the operations applied to compute the output of the function. We call this record a \"tape\". It then uses that tape and the gradients functions associated with each primitive operation to compute the gradients of the user-defined function using [reverse mode differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation).\n", - "\n", - "Since operations are recorded as they are executed, Python control flow (using `if`s and `while`s for example) is naturally handled:\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "MH0UfjympWf7" - }, - "outputs": [], - "source": [ - "def f(x, y):\n", - " output = 1\n", - " for i in range(y):\n", - " output = tf.multiply(output, x)\n", - " return output\n", - "\n", - "def g(x, y):\n", - " # Return the gradient of `f` with respect to it's first parameter\n", - " return tfe.gradients_function(f)(x, y)[0]\n", - "\n", - "assert f(3.0, 2).numpy() == 9.0 # f(x, 2) is essentially x * x\n", - "assert g(3.0, 2).numpy() == 6.0 # And its gradient will be 2 * x\n", - "assert f(4.0, 3).numpy() == 64.0 # f(x, 3) is essentially x * x * x\n", - "assert g(4.0, 3).numpy() == 48.0 # And its gradient will be 3 * x * x" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aNmR5-jhpX2t" - }, - "source": [ - "At times it may be inconvenient to encapsulate computation of interest into a function. For example, if you want the gradient of the output with respect to intermediate values computed in the function. In such cases, the slightly more verbose but explicit [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) context is useful. All computation inside the context of a `tf.GradientTape` is \"recorded\".\n", - "\n", - "For example:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "bAFeIE8EuVIq" - }, - "outputs": [], - "source": [ - "x = tf.ones((2, 2))\n", - " \n", - "# TODO(b/78880779): Remove the 'persistent=True' argument and use\n", - "# a single t.gradient() call when the bug is resolved.\n", - "with tf.GradientTape(persistent=True) as t:\n", - " # TODO(ashankar): Explain with \"watch\" argument better?\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# Use the same tape to compute the derivative of z with respect to the\n", - "# intermediate value y.\n", - "dz_dy = t.gradient(z, y)\n", - "assert dz_dy.numpy() == 8.0\n", - "\n", - "# Derivative of z with respect to the original input tensor x\n", - "dz_dx = t.gradient(z, x)\n", - "for i in [0, 1]:\n", - " for j in [0, 1]:\n", - " assert dz_dx[i][j].numpy() == 8.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DK05KXrAAld3" - }, - "source": [ - "### Higher-order gradients\n", - "\n", - "Operations inside of the `GradientTape` context manager are recorded for automatic differentiation. If gradients are computed in that context, then the gradient computation is recorded as well. As a result, the exact same API works for higher-order gradients as well. For example:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "cPQgthZ7ugRJ" - }, - "outputs": [], - "source": [ - "# TODO(ashankar): Should we use the persistent tape here instead? Follow up on Tom and Alex's discussion\n", - "\n", - "x = tf.constant(1.0) # Convert the Python 1.0 to a Tensor object\n", - "\n", - "with tf.GradientTape() as t:\n", - " with tf.GradientTape() as t2:\n", - " t2.watch(x)\n", - " y = x * x * x\n", - " # Compute the gradient inside the 't' context manager\n", - " # which means the gradient computation is differentiable as well.\n", - " dy_dx = t2.gradient(y, x)\n", - "d2y_dx2 = t.gradient(dy_dx, x)\n", - "\n", - "assert dy_dx.numpy() == 3.0\n", - "assert d2y_dx2.numpy() == 6.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4U1KKzUpNl58" - }, - "source": [ - "## Next Steps\n", - "\n", - "In this tutorial we covered gradient computation in TensorFlow. With that we have enough of the primitives required to build an train neural networks, which we will cover in the [next tutorial](https://github.com/tensorflow/models/tree/master/official/contrib/eager/python/examples/notebooks/3_neural_networks.ipynb)." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "default_view": {}, - "name": "Automatic Differentiation", - "provenance": [], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb deleted file mode 100644 index d268cbcd91..0000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb +++ /dev/null @@ -1,209 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U9i2Dsh-ziXr" - }, - "source": [ - "# Eager Execution Tutorial: Importing Data\n", - "\n", - "This notebook demonstrates the use of the [`tf.data.Dataset` API](https://www.tensorflow.org/guide/datasets) to build pipelines to feed data to your program. It covers:\n", - "\n", - "* Creating a `Dataset`.\n", - "* Iteration over a `Dataset` with eager execution enabled.\n", - "\n", - "We recommend using the `Dataset`s API for building performant, complex input pipelines from simple, re-usable pieces that will feed your model's training or evaluation loops.\n", - "\n", - "If you're familiar with TensorFlow graphs, the API for constructing the `Dataset` object remains exactly the same when eager execution is enabled, but the process of iterating over elements of the dataset is slightly simpler.\n", - "You can use Python iteration over the `tf.data.Dataset` object and do not need to explicitly create an `tf.data.Iterator` object.\n", - "As a result, the discussion on iterators in the [TensorFlow Guide](https://www.tensorflow.org/guide/datasets) is not relevant when eager execution is enabled." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1JcS5iBXMRO" - }, - "source": [ - "# Setup: Enable eager execution\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "RlIWhyeLoYnG" - }, - "outputs": [], - "source": [ - "# Import TensorFlow.\n", - "import tensorflow as tf\n", - "\n", - "# Enable eager execution\n", - "tf.enable_eager_execution()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H9UySOPLXdaw" - }, - "source": [ - "# Step 1: Create a source `Dataset`\n", - "\n", - "Create a _source_ dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensor_slices) or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset). See the [TensorFlow Guide](https://www.tensorflow.org/guide/datasets#reading_input_data) for more information." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "WPTUfGq6kJ5w" - }, - "outputs": [], - "source": [ - "ds_tensors = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n", - "\n", - "# Create a CSV file\n", - "import tempfile\n", - "_, filename = tempfile.mkstemp()\n", - "with open(filename, 'w') as f:\n", - " f.write(\"\"\"Line 1\n", - "Line 2\n", - "Line 3\n", - " \"\"\")\n", - "ds_file = tf.data.TextLineDataset(filename)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "twBfWd5xyu_d" - }, - "source": [ - "# Step 2: Apply transformations\n", - "\n", - "Use the transformations functions like [`map`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#map), [`batch`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch), [`shuffle`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shuffle) etc. to apply transformations to the records of the dataset. See the [API documentation for `tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) for details." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "ngUe237Wt48W" - }, - "outputs": [], - "source": [ - "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", - "ds_file = ds_file.batch(2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IDY4WsYRhP81" - }, - "source": [ - "# Step 3: Iterate\n", - "\n", - "When eager execution is enabled `Dataset` objects support iteration.\n", - "If you're familiar with the use of `Dataset`s in TensorFlow graphs, note that there is no need for calls to `Dataset.make_one_shot_iterator()` or `get_next()` calls." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "base_uri": "https://localhost:8080/", - "height": 153 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 388, - "status": "ok", - "timestamp": 1525154629129, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "lCUWzso6mbqR", - "outputId": "8e4b0298-d27d-4ac7-e26a-ef94af0594ec" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Elements of ds_tensors:\n", - "tf.Tensor([1 9], shape=(2,), dtype=int32)\n", - "tf.Tensor([16 25], shape=(2,), dtype=int32)\n", - "tf.Tensor([ 4 36], shape=(2,), dtype=int32)\n", - "\n", - "Elements in ds_file:\n", - "tf.Tensor(['Line 1' 'Line 2'], shape=(2,), dtype=string)\n", - "tf.Tensor(['Line 3' ' '], shape=(2,), dtype=string)\n" - ] - } - ], - "source": [ - "print('Elements of ds_tensors:')\n", - "for x in ds_tensors:\n", - " print(x)\n", - "\n", - "print('\\nElements in ds_file:')\n", - "for x in ds_file:\n", - " print(x)" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "default_view": {}, - "name": "Eager Execution Tutorial: Importing Data", - "provenance": [], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/3_training_models.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/3_training_models.ipynb deleted file mode 100644 index 84f1d031d4..0000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/3_training_models.ipynb +++ /dev/null @@ -1,485 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k2o3TTG4TFpt" - }, - "source": [ - "# Training Models\n", - "\n", - "In the previous tutorial we covered the TensorFlow APIs for automatic differentiation, a basic building block for machine learning.\n", - "In this tutorial we will use the TensorFlow primitives introduced in the prior tutorials to do some simple machine learning.\n", - "\n", - "TensorFlow also includes a higher-level neural networks API (`tf.keras`) which provides useful abstractions to reduce boilerplate. We strongly recommend those higher level APIs for people working with neural networks. However, in this short tutorial we cover neural network training from first principles to establish a strong foundation." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3LXMVuV0VhDr" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "PJ64L90aVir3" - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "tf.enable_eager_execution()\n", - "tfe = tf.contrib.eager # Shorthand for some symbols" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eMAWbDJFVmMk" - }, - "source": [ - "## Variables\n", - "\n", - "Tensors in TensorFlow are immutable stateless objects. Machine learning models, however, need to have changing state: as your model trains, the same code to compute predictions should behave differently over time (hopefully with a lower loss!). To represent this state which needs to change over the course of your computation, you can choose to rely on the fact that Python is a stateful programming language:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "VkJwtLS_Jbn8" - }, - "outputs": [], - "source": [ - "# Using python state\n", - "x = tf.zeros([10, 10])\n", - "x += 2 # This is equivalent to x = x + 2, which does not mutate the original\n", - " # value of x\n", - "print(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wfneTXy7JcUz" - }, - "source": [ - "TensorFlow, however, has stateful operations built in, and these are often more pleasant to use than low-level Python representations of your state. To represent weights in a model, for example, it's often convenient and efficient to use TensorFlow variables.\n", - "\n", - "A Variable is an object which stores a value and, when used in a TensorFlow computation, will implicitly read from this stored value. There are operations (`tf.assign_sub`, `tf.scatter_update`, etc) which manipulate the value stored in a TensorFlow variable." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "itxmrMil6DQi" - }, - "outputs": [], - "source": [ - "v = tfe.Variable(1.0)\n", - "assert v.numpy() == 1.0\n", - "\n", - "# Re-assign the value\n", - "v.assign(3.0)\n", - "assert v.numpy() == 3.0\n", - "\n", - "# Use `v` in a TensorFlow operation like tf.square() and reassign\n", - "v.assign(tf.square(v))\n", - "assert v.numpy() == 9.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-paSaeq1JzwC" - }, - "source": [ - "Computations using Variables are automatically traced when computing gradients. For Variables representing embeddings TensorFlow will do sparse updates by default, which are more computation and memory efficient.\n", - "\n", - "Using Variables is also a way to quickly let a reader of your code know that this piece of state is mutable." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BMiFcDzE7Qu3" - }, - "source": [ - "## Example: Fitting a linear model\n", - "\n", - "Let's now put the few concepts we have so far ---`Tensor`, `GradientTape`, `Variable` --- to build and train a simple model. This typically involves a few steps:\n", - "\n", - "1. Define the model.\n", - "2. Define a loss function.\n", - "3. Obtain training data.\n", - "4. Run through the training data and use an \"optimizer\" to adjust the variables to fit the data.\n", - "\n", - "In this tutorial, we'll walk through a trivial example of a simple linear model: `f(x) = x * W + b`, which has two variables - `W` and `b`. Furthermore, we'll synthesize data such that a well trained model would have `W = 3.0` and `b = 2.0`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFzH64Jn9PIm" - }, - "source": [ - "### Define the model\n", - "\n", - "Let's define a simple class to encapsulate the variables and the computation." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "_WRu7Pze7wk8" - }, - "outputs": [], - "source": [ - "class Model(object):\n", - " def __init__(self):\n", - " # Initialize variable to (5.0, 0.0)\n", - " # In practice, these should be initialized to random values.\n", - " self.W = tfe.Variable(5.0)\n", - " self.b = tfe.Variable(0.0)\n", - " \n", - " def __call__(self, x):\n", - " return self.W * x + self.b\n", - " \n", - "model = Model()\n", - "\n", - "assert model(3.0).numpy() == 15.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xa6j_yXa-j79" - }, - "source": [ - "### Define a loss function\n", - "\n", - "A loss function measures how well the output of a model for a given input matches the desired output. Let's use the standard L2 loss." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "Y0ysUFGY924U" - }, - "outputs": [], - "source": [ - "def loss(predicted_y, desired_y):\n", - " return tf.reduce_mean(tf.square(predicted_y - desired_y))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qutT_fkl_CBc" - }, - "source": [ - "### Obtain training data\n", - "\n", - "Let's synthesize the training data with some noise." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "gxPTb-kt_N5m" - }, - "outputs": [], - "source": [ - "TRUE_W = 3.0\n", - "TRUE_b = 2.0\n", - "NUM_EXAMPLES = 1000\n", - "\n", - "inputs = tf.random_normal(shape=[NUM_EXAMPLES])\n", - "noise = tf.random_normal(shape=[NUM_EXAMPLES])\n", - "outputs = inputs * TRUE_W + TRUE_b + noise" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-50nq-wPBsAW" - }, - "source": [ - "Before we train the model let's visualize where the model stands right now. We'll plot the model's predictions in red and the training data in blue." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 293 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 1210, - "status": "ok", - "timestamp": 1527005898290, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "_eb83LtrB4nt", - "outputId": "3873f508-72fb-41e7-a7f5-3f513deefe38" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEDCAYAAAA2k7/eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXlgU1X2xz/pAhRautCWUsCwWVlcUHHGBUFQcSg7uM8P\nFLUICo4VpygObihI3UdmUHBB0IGZQbEgFNGCqKgMolV2pKylCy1pukDp+n5/3LxmaUsDTUjSns8/\nbZKXd09C+b7zvvfccw2apmkIgiAITR4/TwcgCIIgnB9E8AVBEJoJIviCIAjNBBF8QRCEZoIIviAI\nQjNBBF8QBKGZENDYE+Tk5JCUlER+fj7+/v7cdtttTJgwgcLCQhITEzl27BidOnXijTfeICQkxBUx\nC4IgCOeAobF1+Hl5eeTn59OrVy9OnjzJ2LFj+ec//8mnn35KWFgYCQkJLFy4kKKiIh5//HFXxS0I\ngiCcJY22dKKioujVqxcAbdq0oXv37uTm5pKWlsaYMWMAGDNmDF999VVjhxIEQRAagUs9/MzMTPbs\n2cNll13GiRMniIyMBNRFoaCgwJVDCYIgCGeJywT/5MmTPPLII8ycOZM2bdpgMBhcdWpBEATBBbhE\n8CsrK3nkkUcYNWoUN910EwDt2rUjPz8fUD5/REREg+eRtj6CIAjuo9FVOgAzZ86kR48e3HPPPTXP\nDR48mE8//ZRJkyaxcuVKbrzxxgbPYzAYyMsrdkVIbiUqKkTidCESp2vxhTh9IUbwrTidodGCv23b\nNlavXk1cXByjR4/GYDCQmJhIQkICjz76KJ988gmxsbG8+eabjR1KEARBaASNFvwrr7yS3bt31/na\n4sWLG3t6QRAEwUXISltBEIRmggi+IAhCM0EEXxAEoZkggi8IgtBMEMEXBEFoJojgC4IgNBNE8AVB\nEJoJIviCIAjNBBF8QRCEZoIIviAIQjNBBF8QBKGZIIIvCILQTBDBFwRBaCaI4AuCIDQTRPAFQRCa\nCSL4giAIzQQRfEEQhLOk0GTi84R7+XbIDXyecA+FBSZPh+QULtnTVhAEoTnx7YzHuDflUwyAlv4z\nizEwfNFiT4fVIJLhC4IgnCWhhw9hsPxusDz2BVwi+DNnzuTaa69lxIgRNc/Nnz+fAQMGMGbMGMaM\nGcM333zjiqEEQRA8TqHRiGb5XQMKjV08GI3zuMTSGTt2LOPHjycpKcnu+YkTJzJx4kRXDCEIguA1\nXJ/8OosxEHr4EIXGLlyf/JqnQ3IKlwh+v379OHbsWK3nNU2r42hBEATfJjQ8wic8e0fc6uF//PHH\njBo1iqeeeori4mJ3DiUIgiA0gNsE/+677+arr74iJSWFyMhI5s6d666hBEEQXMLRjAwW9u3FWmN7\nFvbtxeGMDE+H5FLcVpYZERFR8/vtt9/O5MmTnXpfVFSIu0JyKRKna5E4XYsvxOmNMb53xQhmZh1T\n5Zalx5h3ww08cfSop8NyGS4TfEe/Pi8vj6ioKAC+/PJL4uLinDpPXp73Wz9RUSESpwuROF2LL8Tp\nTTEWmkx8O+MxQg8fIjory67cMtZk8po4z4SzF0+XCP706dPZsmULZrOZG264gWnTprFlyxZ2796N\nn58fHTt25Pnnn3fFUIIgCC7FdhHVXFSZpcHyM8vGqWgKuETwX3311VrPjRs3zhWnFgRBcCu2i6ju\nBp4JDKR7QACZ4RH839dfezAy1yMrbQVBaNbYLqK6AOgaP4L4w7lMSt+NsXt3T4bmcqSXjiAIzRpf\nXUR1LojgC4LQrPHVRVTnglg6giA0WXy1jbG7kAxfEIQmi6+2MXYXIviCIDQZbGvqC41Ggg9k+GQb\nY3chgi8IQpPBMaOfE9vRrq7eV9oYuwsRfEEQfB49s/dbn2qX0V8QEcHiq/7odAWOyWRmxoyNHD7c\nFqOxkPffHwX4uzv884YIviAIPk2hycS/B1/HJVnH2In9StnK7heelWc/Y8ZGUlLGAwbS0zWmTFnO\n/PnD3RK3JxDBFwTBJzmakUHquOFE5mTTpbqaAcD1wDygQ1AQ1UOGnnVN/eHDbcHmHuHgwWDXBu1h\nRPAFQfBJUscNt3a2BJYDdwG9gRNDhp5TNY7RWEh6uvUeoWvXEhdG7HlE8AVB8BkKTSY2PDqVgB+/\nI8ZstvPrg1HCvz22I3ec42rZ5OTBwFKLh1/EggUjqapyTezegAi+IAhejz4pq23aQBuzmWHAAuz9\n+t8CA8mPH8Edya8RGn5uXS7tu7w3vS1aRfAFQfBq9ElZR/vmbuBZwGgwkN0hlqEr19C5a7dGjdXU\nJ22ltYIgCF7NtzMe4xKL2IPVvrkAuAgwjBzDpPTdjRZ7aPqTtiL4giB4NaGHD1GC1WDR7ZvktqFs\nbH85r2eMJiHhUwoKzI0ey2gstBtJJm0FQRDciF5u2anARGZ4BC169eIBlI3TBsuk7MbNPJ60Sdkv\nuQa279CApSxaNOasx7NdbNWhw0mGDn2P7OxImbQVBEFwF/rEbNba1cysqKjZSPyF6mo+GzWW0MOH\nOGHsUjMp62i/HD7cttZK2eTkwYSHh51xXEffftSopaxffyMAERHes/euKxDBFwTBK9D74HwO9u0R\nCs3E11FT71gzbzQW1RJvZ7L+ui4cTRWXCP7MmTP5+uuvadeuHatXrwagsLCQxMREjh07RqdOnXjj\njTcICXFuZ3VBEJo+u7Zt48vRQ+ladpqDBgNRrVtjAIqxL7fMrKfE0rFmPjl5EHfcsY2zFe+6LhxN\nFZcI/tixYxk/fjxJSUk1zy1cuJBrrrmGhIQEFi5cyDvvvMPjjz/uiuEEQWgCfDkmntllp5XMahpP\nnzyJBsQDy4BC/MhoFcawxcvqfH94eFit7P1cxLuuC0dTxSWC369fP44dO2b3XFpaGh999BEAY8aM\nYfz48SL4giCwa9s20sbG0+10qZ110x14JSwMf8LZZO7HKt6G0+Hs/8dSFi3q69S5z0W867pwNMS5\nzBV4A27z8E0mE5GRkQBERUVRUFDgrqEEQfAQZyN8+qTs6VUruUjTOIS9dZMNxAwczN8Pjyc9fXTN\n+87GUz8X8T4XzmWuwBvwuknbqCjf8PklTtcicbqW8xXn1Kmf2wlfy5bL+fe/76p1nPnECVbc1J/e\nmZmUAEOBpajOltHAAYOBsBtvZMz7i1g3ZZ2dLRMXV4qfXxUPPZTKwYPBdO1azIIF8UREnJ+Muq7v\nMisrHNu5gqyscJ/423Cb4Ldr1478/HwiIyPJy8sjIsK53ha+UAIVFeUbpVoSp2uROGuzb18QtsK3\nb18QeXnFtTL/+PIVzMjMtGuN0BUYDsxqFcRfjuQCUFEFs2dfT1mZ1ZaZPXsQ99+/qubCsnWrRlnZ\nUubNG+R2W6W+7zI21oTt/UlsbIFH/zacvdi4TPA1+65DDB48mE8//ZRJkyaxcuVKbrzxRlcNJQiC\nl1DfJKlueRgwcUH6FPBbb+fXtwF+A7a0CuLmVevszlmXLVNX6aQnbRVfneh1ieBPnz6dLVu2YDab\nueGGG5g2bRqTJk3iL3/5C5988gmxsbG8+eabrhhKEAQvoi7hKzSZCN34D17hUfIoYS4VLKu29+t3\nderEHWnfOd3Vsq4Liyfr58/XXIGrcYngv/rqq3U+v3jxYlecXhAEL8VW+ApNJj57YAKl337NDSgp\n7mD5GY+yccotO1FNfn8RuXknSUhY6ZQlU9eFJSlpQ7Opn3cVXjdpKwiCb/LtjMeI/fZr7sKayb9k\n+RkG3AkstuxEFRYRwr33fe60JaNfWPS5gTvu2Far742v2CqeRARfEASnqasMs9BUwOJxCVyanU4+\nUIgSeAOqffFLQPuwMAwDB3N98muYTGamTv2c9evB1pLJyPBvMOOvr++NyWQmKcn36uLPNyL4giA4\nja3g/pqeT+DqP3BJ9UHmY83ql6E2J9GAn4Hc9pezLCqRblRzHX4251iGrbNvMh1mx44nOVPGX59v\n76t18ecbEXxBEJxGF1wDJ5jM5fyjOrNWs7NC4G3gd/zZenUS3/74ol0LY6toK2c/KKiCIUPgwIE4\nsrLOPAnrOHl7/PguDhzowaZNucDnqE488U26AVpjEMEXBKEW9a2g7djhGMb0eG5mHa3R6mx2to6r\nWcUPwCrC9uRbXjEDqaxfD+HhO4GBNe8oKytl69Z8evUKtjuTPglr36++nPbtnyY39yrgJFlZUxg7\ndgFm85PY3mMYjZVn/BzNFRF8QRBqoSySEcA60tPD2bp1CY/cW0nv1GfpDeQBuWDX7Kwc2A2s4l+W\nV04C+ZbfU4E7KS01UFqqERT0DKWlLYGZVFcbyMrSqK5+gVGjate2O9o1YWGvACNrYi0o6ITtPUZY\n2GmSk2+u873N3eoRwReEZsDZZrrKElkH3Ik/WxmSNYaCOdX0B0qAB4BVwNNAJ9QFIB9Y1vZeKNoO\n/Aj8iWuuWU6LFktZvx5KS62iXFraDyXS1ucKC411irGjbw/tsL0TCA8/Smmp9fHAgQE1n6059bp3\nBhF8QWgG1JXpnqk1gdFYyK/pp4nnEv7ATiKBUGCA5edyIALoDOwliNfIJCzsM7744g/MmfOz5Zyr\nSU4eTnh4GAkJn5KSYmv8nLT8tBXuzDpjd/Ttr7mmmhYtrHcCM2eOYs6cule9Nqde984ggi8ITRDH\njP7AgTY01JrgxImX+emnYsrKuhKifcxf2EAw0BdqGp6lAnehWiMUA7tpxRtsB8Ixm1vx7LPf0aJF\na8s41nYrtgunjh/fRVbWFEs8y/DzKyYm5gQrV1ptGltqL7q6pdbdyaJFRiff27xr9UXwBaEJ4ijm\nsbFzcJwQdbQ7Nm8uAm0ioxjFzeykEHgCa06+HNCnVf8H/MKlrGUssMvyTDybNy+kqOhBHD1z2xW5\nBQVXMmvWOvbtC8JorCQ5Of6M9lJj2hj4agsEdyGCLwhNEEcxj4jowlVXnbk1gb92kkR6MM/yzCrs\nnfM2wK/Atxh4mXuB14A1qJ6X6hynToXSkGceHh7Gv/99l090Hm1qiOALQhNEedcFqInXlvz++06O\nHGmFn18nOnSoAuztjuh2+7k07Q36Y5XrEuzLLb8DXmYDMAhQ1TLV1aUUFS1BOfoltG5toqiocZ65\nlFK6DxF8QfBRMjIOM27cKgoKOhEefpSVK0fRtavyspOTB7N16wKyslR9elnZGMrKlgHxpKauZfPm\nLwgOziW8bTsuP/4SxvTddMZe5IcCs4COwC78mc8e4D+Wo4rp1CmW7t0rSUmZgC7w/fq9xZ49cy0x\nZTJzZm1fXm+toCyd2oIupZTuQwRfEFzI+cxOx41bVSPopaUaI0c+w9VX9yArK5zYWBMREUa7lasQ\nhFoDO4PiIhN9i/7ENVk/0Qb4G/AKcDvKq28DbAPKgHXczKqaupyLgRGAxv79s3jvvTuxnRQtLw+0\ni2nOnKW1JlQbEnQppXQfIviC4EKczU6dvTDUd5zJZCYnR8O2nUBeXsuasdUuTHOxN2X2AH3wYz/j\nuYgYNHoDpZYjwlBVOCGWM5qBv/Moyqu3LacEMHD6tCrBtP18Q4akoZorpALBbNqUQ0GB2e6z2Qt6\nIZs25TJkSFrN55NSSvchgi8ILsTZ7NTZC4PjcWVl7wGQlpZLdfVMbNsJaFqE3dgFBbG0ajWL06fb\no0Q4GH/SmcooWgNXo8wZ/Qy3AWuBTGB/y1DSus7B//dDVFUtAyqBY8Bky/mV+Dt+PiXWa8HSJNls\nHk5S0lK71saHDlUCHwPDgLWYzY+Tnm79HqSU0n2I4AuCC3E2O63vwmCb0cfE5PH99/arUX/80Q+z\neSLUallWTnCwieJi69iqdcFsYmJe4FTO10xhA3HAPuBFrEK/BJiLMmwOGAL5vPsLxPVpz6fJg3n0\n0dWkpgLkoMR+Hcrw2QU8SEzMJ3YtjWfOvJJNm/6H2Xzmjpb6pC+0q3WslFK6DxF8QXAhzmanHTpk\nk57+L5SBUkSHDrZ7waoeNtAeVd9uFfGiohzL744ty/IpKTlOy5azKC/vhqYFoaZdDbSqzOMeNjCX\nusstw1E9cF7yv4viqo9hv4Hd+zVSU5+ksjIEg8FAy5a5lJXNQ9PiwLLQKiZmPgZDJCkp92N7pzJw\noL/dqlr9oud4kevS5UKMxsI6jxXcgwi+ILgQ57PTQLDbG+o9TCazpc1viuX1AcD1wDwgBmhBdXWU\n5fjrUHl5NKqTzd1o2mbKyu5CtTK7k0BWkMjtdM2HFtRfbvk9MI+HoeoKm6MKKS9vA1wClHD69KXA\nBJt3LeP06WNkZ3fA8U7l3/++krouenXd/Yh9c35xu+APHjyY4OBg/Pz8CAgIYMWKFe4eUhC8nuzs\nSGyFMjs7khkzNmI2P255vgBVUdMH8ENJdjzqYvAekAHMwX4dbIjl8XX48xBTeJvLLM9uwb7c8kng\nAiCdIBaRALyB/YYka1G1O/r5P8T+viAEaFeniNd30bMV97i4UmbPHiT2zXnG7YJvMBhYunQpoaGh\n7h5KEHwG+4VRbTh+fCdVVRdhFdV1wAzL4+G0aJFEefkBVG/KAqAH9gIcDBThzxb+j6uJRTnt+j1E\nf+ApoCfKfd9LIPN4CUjEKub6VuOlqEla2/PnYX9fUMw111Rb2hA7l6HbintUVIistPUAbhd8TdOo\nrq529zCC4HHOpgbfcWFUVtYITKZZwDisjQysgtuyZSTl5UlYBVffHlx/vJcW/EoiH2EE2qLuC/Qz\nhANGlNjPYxgwGnXXsAw4hf1W48ss77I9fy7qziIAP78sbrklnDfeGC4Zuo9xXjL8+++/H4PBwB13\n3MHtt9/u7iEFwSMkJq4hNbUt4E96egDl5Z/z4Yf/V+ex4eFhREf3tlsYdfp0F5RfHw38jvLvwwGN\n4uJg7DPuzsALQCcCWMOdfEJHqJmYreuSsA94jf0og8d2/mA2yh5S7REgwTKOyvZbtjxAWdlUoAug\nMWKErHz1Vdwu+MuXLycqKgqTycTEiRPp1q0b/fr1q/f4qKgQd4fkEiRO1+LNcZ44Yeahh1I5eDCY\nrl2LWbAgnoiIsFrHfPVVNqA6RYLGjz++WvO5Tpww88ADKWzapAF5DBgQRocO2PnfaguRGTaP56E8\n/BIgG3v5Pgp0pxWf8hc+IRi4FPtLQk9Urm4GdhDCAn4BuqPyfNsjL0c1QHseqEB1vDcAd9Kp0zx+\n/fVxpkxJZd++X8jP38vhw0amTl1d5/dwNnjzv7ktvhKnM7hd8KOiogCIiIjg5ptvZvv27WcUfF/w\n9XzFf5Q4XUNCwqqaUsmtW4P57rt/sHHjBMLDw2r62eTktKO6ugu2QlpcHMK+fUctG4Cssus5k5Ky\nBPgNeBWIRAl4H+yFuDd6GwNYgLVBcQkBHGIqM7kQ5ei3pnb1zS6gCEjmJ9Qq226Wcxc5HKkvv7rC\n8rt1nLCwzlRV+TN//nASElaSnj6DzEx9Edi5Z/re/m+u40txOoNbBb+0tJTq6mratGnDqVOn+O67\n75g6dao7hxSEs8IZ3z0jwx94B1XXspOsrF4MGrSEjRsn2PWzUatHrUJaWdmKQYOWEh3d27K61FbM\nNVT3Guvyp8DAX6ioGIO9ZBuAdJSFcyd+7Gc0/ehOUU0bYw1l7tyLtQ/O98AxgviI7aisvrvlqF7A\nXtQU7oVAS9RkrS781cDdNWfu3n1pzfcgPW58H7cKfn5+PlOnTsVgMFBVVcWIESPo37+/O4cUhLPC\nmRYHJtNhVCHjcvQtQbKyNJKSllJQYFuHPgyVscehWo/dR1bWf8nK8kdNehage/JwANs+OFBJUFB7\n2rV7kZycSNS0613AZtQdwAECuZ1EVnARqqmZ7eWjI+oeIBz4BQMvk4Ta+1XP6kNRtf0VwHPAEWAp\nEAXMx8+vkN69+9K5cxHwHtnZkbJdYBPErYLfuXNnUlJS3DmEIDQKZ7LWdu3iLJOr9hOnq1ZVomm7\nsWb1eulxgeXnRqADavJ1OJCEqoTRd4PNRU3QLgBKKSp63tJL/hmU4P8XmI4BE4MZSD920hvV0cYP\ne1PGhDJqnuIOVOb+PKp/zjKgHFWRU2zzGb5HZfnqDDExc9mwwdrKWL/zueOObTV3PrJIyveRlbZC\ns8aZrLVbt5Ns365qz21lVrUviEOJahCqQUEgyjL5K9a+MwuAKajFSvYNz2AkyqefZxnNgLJdDgNt\n8eN1pjKDi6gkFHUPEYqa2l2GtbNlJvAmM1C1OXrzhDCUPbPaMsYCwsJ2YzYPx/Hi1a5dnN1nru/O\nR6pzfBsRfKFZo2etGRmtMZn2kZFhJCHhUzsv33qMH/v3P83p00bUQqRhwHpU24NiVLZ+P8qqWYeq\naTegxHYJyj5xXK2q/x6MkvA2qJLMSIJYwiNs4VpqbyLeFdiBWob1Bd1YxR2oOwioPX2rHgcGmtiy\nZQJJSUvZtCnHIvzqmG7dTtl9L+LXN01E8IVmjb5w6J57PmbHji5kZYWwY0cOP/zwHuXlFwD5XHNN\nMG+8MYJZs75jx47nsbY+eB3ohxL7oSj/Xq+jz0ZZKmGWn0eAVjiuVlVoKKPmYcBAC/L4Cw/QAlUh\nb9s8Qd9E/DCQg4G5bEU1MzuFaocQgrJwnkDdfRxH1c8vY8CAkJrPW1BgJimpfntG/PqmiQi+0OQ4\n212nTCYzX32VhbWG/l8cP/4Mutilpi6jRYuNZGWFY93cIwNV6a7L8XxUhYttHf0ylKWi96XRPfVi\nlOtegrJgwlGZfSEB/I9HeYCXUFOqtvcDbVDSvhmYxzxURq8Bn6CqbabYjP0CMBbdVoqN3cE//zm+\n5jM3tEJW/PqmiQi+0OQ42z1RZ8zYSEVFP6zyGoKj9ZKSchz4BiW5T6KyedvVqi+gJktt31cIvI+q\njLH11FehBD8Y/QLhzxbGE04HrF1yjlG7q2UJLfgHPwArLec5iTJ42tuN3bZtB6677hNLtY2Z5OTx\ntS56Z7owSsuEpokIvtDkcPSfMzL87TbpePLJK5k792ebTUayUTX2+i5MjguTTKiKmj4o774QVSrp\n2Opgj8P7TqKmWG1ragpQ9fVRwDEMHORmRtGXHXRB1ebonW3uRuX/eqOFDYRyqtfrxBauo23bzhQV\n7aBduzgOHy6nqGgnaq5AjT1oUIsGBVs2C29+iOALTQ6r/1wIrGXv3kPs2KGqY9LTNdasmUVl5eya\n15V4ZwMXofZvLUJZNsrDV4+fw75LDdiLeybKdHkS1aYsFHjA8vM9VK+aXqhFVOpcLXmTKXSnDfZe\n/RKsPStPAj8Bb/MhsbGZpG+6tdbn7dv3LYqKpqAvuwoK+onk5IRaxzkiE7PNDz9PByAIrsRkMlNe\nXkFY2AcEBLwCDKWiwr7LTGVlZ8vjVNRkaxFqknMsSozboiY6W6BE+yrss/k+KL9cL4FcjppwDUDZ\nQS1R+XmY5fho1H+1/6F8/+W04s88yqNcBfzB4ewRqPqeA0ApLXibn4AJhIZ2r/Mzq5LKcJTFNJKe\nPS8/45yFjtFYiLrEgEzMNg8kwxe8nrOZhH300S9Yt05tuWetbdFw3A5Q/QxGTWr2xl5y+6Hq4/X3\nV1PbqgkDLkbZKDp6q4IdDsf/BDwGrCKA9fyFJfRE4xDKvsHh6L3Ad0AyM7Dtf1lYmFHnZ7auE1DH\nXXjh6TN9nTXIxGzzQwRf8HrOxmv+8UfbLvB6bcsAVHVMIcq6Kbc8PoaycRzr1k/avL8Y5bnvRmX9\nB1CLqqC21/+75Zi7gVmoydTjwP34cZw7uI8LqKpZLfsAyux5DGsPnP8BvxDLWraj7gqWoyZ9Aykp\naUtBgbnWxc5RuBcsGElVVcPfq0zMNj9E8AWv5+y8Zj1710V4p817v0cJcg9U1h2E6lLZBiW90ZZj\nJlse630oo4CHULZJAaqRWm/LuZdg7SMfClwLfInK9A1AJa14iGmspSWq4YFt82Mj8E/LmX/AwFv8\nRHT0ajiul4BqqN2n/CkqaklS0sZaIu0o3BERvtHhUTj/iOALXo+zi4BMJjMtWxZjbTlcSfv2p+jQ\noYqYmFOsWxeNmjgNQVXbPIG1cuYN1H+HauADVNOx6Vjl+VUgFtXorA/KytmL/cbeT6IuAN2Bv+HP\nVhKYRDBVhKHW49ree8SiMv1iYBVhFF74MqN676CkJJy0tGVAlkMMS2RiVWgUIviC1+Ho2c+ceSWO\nXrPtMR06ZAOB/PCDH2ZzT/SOM4GBc7jiigt4440refTRNahM3LZ2XpffdcCzNs/PcXjdgLqABKP6\n4kRZXg/HWk+Ti6rq6QQYaM10pvI6ccAh1KXAcQeqXagcftfVf+XzVbNqPv+QIWmoLQhXO8QQjtFo\nbvwXLDRbRPAFr8Pesy9g69YFREf3tpuwTUhYaXPMv7AX8uXAXVRUXEpqan9++WW+peVwFUpiQdkx\noGrsq7AX1hhqL3tqgbXR2WzUHMCtKBtHb5u8kAC+ZzjzuAhqvPo4y1nuxtp4YR/wHZEcaD+Jrz+c\nbPf5rXc09s3aYmN3kJw8HkE4V0TwBa/D3rNfR1bWk2RlWSds580bxKZNucBnqLr2AOBDy/GjUR0r\n56JWn75ETs5L2Fe5t8Bq59S1+2suKovXNwjMBfoC/0JZOlGoOv3lqHmAUYCBEN5iCjsJwbbxMDxt\n+WlEratNAl7hCoYOfYCvLRuB22Jt1uaPyTSXdu3i6NbtVJ2rZQXhbBDBF7wG3aZRu0Ppq17bYJt9\n792rcdllb1NW9kfURGknVL2LrdeeidqnNQJrWwMsP09TO6PvgrU12V7L43hUnX4R9nbPMlRWfxKV\nvz+PH/uJJ5o+VDAX1SvT9uzdUReAjqgan9dYQlBQFR9+OK7O70GqZwR3IYIveA22Vg5otG37EuXl\nJzl9Wm8ZUMC+fXuprn4Re4G3ldeLUNOhQ1HevGPVTjFqktb2Ob2RgV4FvxtlEd2Ftbe8fv5y1F3E\nt0B/gunNFPbQBWtdTrHD2fcAJ4C5PI1a2KURGvqi6744QXASEXzBa3AsvywpiaK6uhxYCORjMBRS\nXd0fewFuR+3e7yFY+9G/i/1WIUUoF/0pVO69H2XLrEb5+WGoCdqXUP89CrHtUaNkPRQD2VxPF66h\niDhUBX6X0UVuAAAgAElEQVRLyxHxWHtiHgR+IJhv+NYS0xLgd/r0EWtGOP9IawXB45w4YSYhYSUH\nDuzFdql/dfUhVOVLS+AhNK071kVSYF3s9CyqNn4Z1lYJuhV0G9bSS/189wDXAPcB/ijhH2455/2o\n7cCfQOXl01F2zyrURSKHAJ5kEg9yDUX0Rjn8D6LuJf6Gala8C7WI6oer/8qivbsIC/sSVc4ZCEzn\nxIm62yQIgjtxe4b/zTffMGfOHDRNY9y4cUyaNMndQwpegG3ZZExMHgZDJdnZHepsjfDQQ6kWK8ex\nX/x0rJt+L0fVzt+OypL1TUNOowTeH+XNL0CJ/SFUZh6GyvR160evrNmCEnRQUv0qtdsi2/aoAX/2\ncieP0Qkl246LqK6wRP078D3h7G8/hc/evIvw8DAGDowmJcW6w5T0rRE8gVsFv7q6mtmzZ7N48WKi\no6O59dZbufHGG+neXbKbpo6jH6+EfDTp6Rrl5e/QokXrmjr7I0d0K0fvF78ENXG6DjWRql8AilCZ\nfABqQrYUJdIXUbsscwLwIsryse1cqS+gMqIyeX3B1KWoUk1be2h/zWMD+dxOEp1Qa2mzqb2Iapcl\nor/zHPA05GrMmbOURYuM0rdG8ArcKvi//fYbRqORjh07AjBs2DDS0tJE8JsBGRn+WCtfirGVx82b\n8ygq6g74k54eQIcOv6AmQnWhPWY51rZ0ch5KmF8GrkZZO9NRG4E4ZubBKHHvbvndtsGZXrnTEmuZ\nZXeUXD+OtavNVmASBhYxhme4kZyada/hqBoix0VUR4BlrETdbahY9JWxUnkjeANuFfzc3Fw6dOhQ\n87h9+/Zs377dnUMKHka3cvbu3U99PeSLiqqwzchPnnyRsLBXMJs7oCpkOlN7pWs0ag9Z2wqd5aiL\nQ0vs5fc3lGUzHXVn8aHl+TxUM7OHgR9Qwr4AlZf/EVv7BvJoxTKmMZN5DiPehSoYnYOq9P8dSOav\nQDLWuxn1WcW6EbwJtwq+pmkNH+RAVFSIGyJxPRJn3Uyd+rnFyvkMW8E2GNqiaUtQAh2DdW/YYIqK\nqhk6tA2pqX6orQIN1M6hg1Btix07YZajBFvvn3MUlbFnoCyhLOy3F1mG6pXzrOW5EahGadZiSj/2\n8Sce4BKUfeM4Iqj7h2LgZ+C+las5tKyYgwdX07GjCU2rICtrNV27lrBgwUgiIs7/34ov/H36Qozg\nO3E6g1sFPyYmhqysrJrHubm5REdHn/E9vtDlLyrKN7oReiLOffuCUNKod3pUQqtpLVFTnX1Q2fda\nrFn+cNLSnqBt21YUFenyOgwl4hEosR9qeY/tRWAr1lYJlUCZ5fl01DKnO1HzAbaSHYK6IDjePagW\nyi1ZwZ9ZSRRK7B0bJ+9EXbIOA/P4KzCPqsX1t2uuqjr/f9O+8PfpCzGCb8XpDG4V/EsuuYQjR45w\n7NgxoqKiWLNmDa+99po7hxRsUOWOq5zaOMRVqD4wBajVrh+ibJTTqHLIO1HSeT3wH2xFt7z8QgwG\n6ySpyqFjUcuWdGtoKMoaCkNV1kSiBL81+mbg1sVYRcBbqAzfceGVfZ8cP78f8av+HxN5kWiUQXQZ\nSuyHYu/qlwDbCWAZe1AXDqSDpeAzuFXw/f39mTVrFvfddx+apnHrrbfKhO15xFrueP42qU5OHszW\nrQvIyrLtJvMSyh/XbZxAVAXMIpS9UwSUU1b2V1Stew+sE7ftUNXtF6JEvhR1Z6B78Ccs4ziutu1v\nGddoOacR5d/HoCqBVJ+cmBgThTmnmcrrNVO8rbGK/TqsG5OUAIb7JnHqxLWQ0s0ynvj0gu/g9jr8\nAQMGMGDAAHcPI9TBwYPBOL9xyJlxdpvB8PAwoqN7k5VlK8DtgV9RkqnbOONQojsCdVF4EXVR6Im6\nIPwN6wVjBipTvxh157Ac1YuyBEgE3qb2att1KMG3nW69HVXW+RtgwJ9D9Mx5mb5Qk9lXAL9YzqqL\n/fdArrErr//8ExVVgRQUmJESS8EXkdYKTZiuXYvZurXhjUOcwXGbQb2WPiOjNSbTXiIiutC9eyXJ\nyYOJicnDXoBboeri11HbT9d/jwIWo5oRnLa8pxRVNtkVtSh8JKoLpm255nLUBWW25Ryhlvd84zBW\nBaq0cwYQTkve5EFeJghVgW9bxT8Pa9u0X4G+H3zMjcNGEGbZSUpKLAVfRQS/CbNgQTxlZWfORPXM\nvS7hts3gHfvc/PBDMWbzg+gymZX1Pjt2BLFmzVpURfoLqJbCe1GLnlRFTm0/HcvvIVgbmC1Bib6+\n4YgJlYNrqDsAx7qZwyg//3eU7/8ZyhKy9sDx89tD9+4XcOD3KQzj31yE2tMqHzUlbHvGCNQ9QC6w\ns/f/MX2YbR2/IPguIvhNmIiIhjNRxxWxWVnL2bFjZK1NRxy3GSwpsb0AFKJE/lkqKx27wFdYfgaj\nJmuXo7L3LagFSgtQ3vpfLOcyoKpt9K0D9bJJtayp9sYkO1Fi74eaatVQ62BNGAwLMBgKCAgopLz8\nSTJ/f5XH+DctsC/UdOyGvxdY3fmv9L7iYj4Su0ZoQojgN3McM3clzKvsNh3ZsuUFIiO70bLlLMrK\nugIFVFaWohqSrUMJdBeH8/RC9YzvgzJJ/FENyu5CyeoW1MpWfd1qqOW9GsqnL0RV46iWxMHBwbRu\nncHJkyGcPDkLuBJ1FzAFa0Y/E1sZ17TOaFoorQK2MLG8KxEUcrHlXbaRhqIWUYUDu6KiefS7//FE\neIQLvl1B8C5E8Jsp1s1Gcqg94Wm/yjUnpx05OX7AH1AZ9RSsbvdc6l4odQRrqeQIm2MvRpVq9gJS\nULtP9QdmWc5/EjVluhblxa8FWtO2bQGXXRZJaupk9L48+lixsVmUlMTY1PAb0Dcab8F7TDr1Fn1Q\nS7JKLKPbRhqGarV22YrV3DZgoAu+XUHwTkTwmxG2lTbHj+8kK+shlOwtIyTkFOXlBygr80ctWtJ3\nnApFudlTsIq33mCgN9YLg75QKhrlpV+OfR6t7yk7EiXYek2+vvq1o+U1nWLgH+hZe1aWRnb2U8BS\nlGe/gKCgIMLDs4mIMFJdfYCiIpvaevZzEy25iHIuR80QBAKnLL+/iJrizQUyW7Zk8jdb6Ny1G4LQ\nlBHBb0bY+/WjgPctrxRQXNwG5YPbNv3VO0teQG3bR29yZrtQqhglpzEoJ9w2j24DVGP1823PV4i6\nSNger0u09ThNuxp1UVCWTXi4ucZ6ggJiY+cSHd2bnN8/5s8nV9ADlc3bVuC8iqrSH44ylHq/9TZT\n7ri7MV+rIPgMIvhNGMeVthkZAdgLbQFK0O+zPLbfzi8oKJLQ0AxycsB+Zep2AgK+oby8M0pC26EE\nXpU8qvLKu7GuUf0NdRFohVoEFYSSXF2GQ1H5ti7HJSg7Zz72F4GdqBYIYfj5taekRP8cAOFEtA1j\n4P4JtDhZXNPw7ANq32f8AmwA/mgptxSE5oIIfhPmgQdSSElR1S7p6RrR0S9gK6ABAcFUVtq2Frbv\nHBMenkV09CXk5NyAEu8yoAXV1X+mvPxfqIlafU3qfJTYg8r030bZNDstj5+ynONFVEbvKO6foTJ6\n2wtBEQbDU5bM/iQwGVXeeSctWhykqKhnTbxBPMWf9syhB8qmsZ3yrVXT87fnmPlIoku+Y0HwJUTw\nmzCbNtlPvppMkSi/3AAcoqoqFNiBVWSHoiZXewO7aN26DXv2bANyUPZNN5QL/jFK7HeiRPtt7Gvs\ny7BfHDUHqxVkQElxLPbibsDffzdVVXrXSwNt215ARUUwpaW23n4psbFzadu2M3v2DMOfF3iAp4lE\nTfmWoNbTrkXdY4xCFYgagX1Az7feZqRYOEIzRQTfx6ivxUFdzzvWo1RXF6AmX5cBT6BpytYJCHgG\n6EBl5WGUtbINuICMjN/RtBmo0kt9kdW/UBuRLMde1Gdh3SzcfkMSgyHC0iq72CaeocTEvMjp07EY\nDCauvroNYCQ19YGacw4atJStW49SWmr9DLGxOaSnTyMh4VP279nCgzxNR2qvvT2NMpbyLaMa3nqb\nv4rQC80cEXwfo74WB5s2VWI2twRuID09FFjK1Ve3JDX1JZS1coyIiALy8x0nTcMJDu6C2TwRtcI1\nEHgMNUmql17G2hyvi7njxGtfVG/6LNRCKqtI33ijgV275pKV1cVyvjhiY/ewceM9hIeH1bSgLSgw\n06KF/cpgs7mQMWPmUlDQifDwTFauHMnRjAx6bnyEKyiqKcB0XHt7CDUzkN82lAlfbpIKHEFABN/n\naKjFgV4yefhwW7p00YBpNa/17fsOO3a8YKmpt9opRUU5qOy8LepPwlY+9SZluoAXYW2LYOuOH0Jd\nWKpQ3S5fom3bKAYNakFy8jAAkpI2cvhwT4uYj6/VfK2uHjXh4WGkp0+refzh669wdO7zRKNaIDiu\nHNCAHwEzEP3W20yXrF4QahDB9zEcWxyoCpnaJZNGYxHHjkXYvZafH0OfPhXk5LRCTZqGAK2orn4I\nlQ8/i6qksfXWT6ImVZfTqlUZoaEZ5OYuQS2YeslyjmKUp1+NukNQq2mvu+49OwFvTMOxXdu28Wn8\nYDprGiGWT51nGd22Z/1m4GhQax7/+nvJ6gXBARF8H0H36A8caENs7BxLk7MqysurSE21XgDCwvYw\ncGABTz55Bbfeuhpb8TYai9i06TQwFaugv4+qfAFlyVyCtY/8LtS+sGHAnUREzKWg4EJUnxtFQMBz\nVFY+jaqLWYtqoaA2B8/OjnTJZ1/98VL2JD7MGyhhn24T/VxUfVBHVGfLuLfe5nHJ6gWhTkTwfQTH\nJmeXXvoe0IKjR1sTGzuXdu3i6NbtFMnJdxIeHkZCwkoyMyej574xMb9SXh5JUVE7lH0TjxLyAlQd\n/nKs1TS6NdQH+CcBAZFERBwnK2sq6uKgoQt8dXVny/kqsDY8U6tnjcZKwPle+nXxzovPU/LmK/S0\njOLY2bIT6rK0u20od4lXLwhnRATfwzgrho7e/Y8/+mE2Wy8AV11l3c3KZDKzaVMlqi7+LgCOH99h\n6UNjK+h3Uv8kbAVwjKFDI/jww7sZMiSN48f15z9E7Vg1nerqcMv5PrR7f1jYaZKTbwZqTzQ7s/NW\nWspnbEmYQBuse8sOpfZWJzuB61es5o/SA0cQGkQE38M4K4a1vXt9az8A+92sZszYiNlchbJWQoAi\nqqvD7I4PCqrA3382JSX+qBW2O7H37gNRxY7v2Yy/FvssXu+pY8DP7xjV1db4Bg4MqLlwOV6sGtp5\n6z8LF3D0bzO4Cvu2CMtRl6VZqLqhA35+jFi3kd59Lz/j+QRBUIjgexhnxVDV1VtLFsvL29h590Zj\nUc3dwvr1oNab2u4rOwfb3HjIEPjii3KsneGvB55BbczdApVPG2p8+OTkwWza9CVms2MBJIDGLbdE\n1Cqp1HG8WNW389aWDRv45s7RdEXtcWVfza9G247aB6vlW28zQ7x6QTgr3Cb48+fP5z//+Q/t2rUD\nIDExUfa2rQNnxdCxZLGumvWkJFuf374vjiqvXEZY2GkGDgwgOXkQ69dX2RwTDlyFqrixdrLU4wkP\nD2PgQH9SUmwXQe0gOrqamJh8gHptKceLVV07b+kWzlUood9B7f2xvgcKWrfmwY1SgSMI54JbM/yJ\nEycyceJEdw7h8zgjhnWhXwD0rP6OO7ZZetvrXWRKsG5Q0gb4BX//Mq65xkhy8gjCw8MID8+yW8Wq\nO+V610nHeGrHOr5mgjgl5X7qs6XOtAfsrm3bWDlyCK0qKojCauH0x9pBPwzVmu1SaYsgCI3CrYKv\nVmoKZ+JMYujMhO6jj37BunVKbK37wd4DDMVgeBlNe9Hy2giqql4lNXUKLVooQV65cpRlFWsHNO0A\nXbv2IC5udZ2LovRY580bVBNTUtIGkpMHn7VHD8q++e6uMcRpGq1QbdG+xv5+oyewBwiZ+wp/u39S\ng+cUBOHMuFXwP/74Y1JSUrj44ot54oknCAkJcedwTQZd6Otql+B4cfjxRz9sxTYg4DQXX/yZpea+\nu4PnrpqS6YLctavRbhWrM9Q1yWw0ak7ZUjqrP17KvsSHa/bK0uvpY7G3cHYAwdNncKeIvSC4hEYJ\n/sSJE8nPz6/1fGJiInfffTcPP/wwBoOB119/nblz5zJnzpwGzxkV5RsXBXfFeeKEmZtu+pjMTH17\nQGs1TFZWeK1xDYYT2MpkSEgxv/zyIACjRy+289z1n3FxpWcV/4kTZh56KJWDB4PZv78a2wtMVlY4\n69Zdz5Qpyzl4MJiuXUtYsGAkERG1z3/49995+7rr0PLyuBb7GYYY1KaFy1BtEQ4FBjLhhx+49Mor\nnY7zfNDc/z5diS/ECL4TpzM0SvA/+OADp467/fbbmTx5slPH5uUVNyak84Le7MsdJCSsIjPTdutA\na7uE2NiCWuNefXUbUlP1LpXFXHGFH6NHL7HYQBUMHfoemZlhnDixj4gII927L2X27EHk5RU7vQYg\nIWGVzWSw/d61sbEFVFX5M3/+8Jrjq6pq/ztu2bCBNXeOph2qyfJOVF2QXsV/AHVZy42MYsSaL7nN\nMinrTX8P7vx3dyW+EKcvxAi+FaczuM3SycvLIyoqCoAvv/ySuLg4dw3VpFB2i307ML1dgu0Eqi7W\nmZnRxMbutWm10NbOchk1ailpabcAt9Qay9k1APYe/TDCwl6hS5cLnZpkLjSZ+PD2MZz67Rcisd9A\nUe+8vxXVxrjrW2/zkEzKCoLbcJvgv/zyy+zevRs/Pz86duzI888/766hmhQxMXnArVhbIvzGpk33\n1Mq8HVst6CtthwxJw9kJVGcnW+1LR0MZOLA9ixbd2OBnKTSZePe6fgSeyKcDEIf9fUsUkI5aQnaD\nbDcoCG7HbYKfnJzsrlM3aQyGSlS/GmXRXH55O6daLehi7Wxd/9kce7alo4UmEx/eOoLAHdvpgloC\n1prabYx/B4au38TAmwf4xG2zIPg6stLWy8jO7oCavtQff1bncfWJta04x8WVMnt2/eLsrJCfqXTU\nkS0bNrDqztG0R7VAsF3nexfWNsbfA/1XrJa2CIJwHhHB9zIam3XbinNDE05nI+QNUWgy8cn/3U7Z\nT/8jGrVm19a+CQcWoBZRHbj8Sh5Y/gmh4REuGVsQBOcQwT+POFMV8+STV7J1q76l31FmzhxV57lc\nKdaNpdBk4tUr+hB56iRdgQyUjWNr32QBtGzFdau/kKxeEDyECP55xJmqmLlzfyYr60nAQGmpxpw5\nS1m0yOiJcJ1CX0TVDvsKnGewt286zn1FFlAJgodp8oJfV1ataZzzhhyNwZmqmHNpU+AJ0lI+Iz1h\nAj1QmyN2wN7CuQDV1fJXoK9U4AiCV9DkBb+urBo46w05XIEz/vzZVNl4gkKTifWJD3MkdY1da4SZ\n2Fs4+1FCP12EXhC8hiYv+PVnzOc/i3amKuZcu2eeD45mZLDwun6EVVfVqqmPRO2EG4US+/C/PSdZ\nvSB4GU1e8OvOmM+u2ZercGai1ZsmY3UKTSa+eHgSpWnrCUNtObgT1XxZb41QBJQBmZf2JfG/n0kF\njiB4IU1e8OvPmL0zi/Y2jmZk8J8Bf2RuRTnLgenozZZVa4RoVEYfcNUfeOCj/4jQC4IX0+QFv76M\n2duyaG9k17ZtpA4dRFfq3ua8N/CjwY/79hwQoRcEH6DJC75w9hzNyODzUX/C73guc4FXULZNMbW3\nHBz6xUYRe0HwEUTwBTv0rL43qtfNEdTq2GUooX8JaAvkRbfn9tVfyN6yguBDiOALgEXoRw7hgooK\nLgGGoerrXwKmAGtRk7SFLVpy7efrZbWsIPggIvjNHL0C50Taeru6erXHFrRHZfeHUZ0tbxOhFwSf\nRQTfDTi7k5SnSUv5jG8SJhAFGKlrjy3YB1RFRnHXmi/FvhEEH0cEvwHqEu+GthNzdicpT1JoMvFz\nwgQ6A0+gsnjbCdm9wGZUVi/2jSA0DUTwG6Au8f7sswlnfI8398PRWyPkfLWei4BAVKTxKBvnJJCN\n2nLwZulXLwhNCj9PB+DtnIt4G42FqDwZvKkfztGMDN699CJOpK7huYoKgoBjqEjDgDtRi6hCbhzC\ntL2H+OOAgZ4MVxAEFyMZfgOcSzMzb+yHU2gy8emga7m2vAwT1qx+CfA00BXYHxjI7d9tFa9eEJoo\njRL8devWMX/+fDIyMlixYgV9+vSpee2dd97hk08+wd/fn6eeeor+/fs3OlhPcC7i7S39cMwnTvDf\ne+6hcPO3VBYXM1vTMAAfY83qpwFzAgOpvOkW7ntjviyiEoQmTKMEPy4ujvnz5/P000/bPZ+RkUFq\naipr164lJyeHiRMnsn79egwGQz1n8l68RbzPlqMZGSy89goiNI0eqEVU24FLUTX2r6I6XO5v2Yp7\nf9sjQi8IzYBGCX63burWX9M0u+fT0tKIj48nICCATp06YTQa+e2337jssssaM5zgJLp9E6VpdrtQ\nPY0S/FDADJyIiua2z9eL2AtCM8EtHn5ubi59+/atedy+fXtyc3PdMZTgQKHJxL8HX0e306VUYF9b\n3xVYDByL7ci9GzeL0AtCM6NBwZ84cSL5+fm1nk9MTGTw4MF1vscx4wectnMaqnH3FrwtTvOJE6Q8\n8ACZa9Yws6LCzqvXM/x9QI9Ro3j4/fcJi/Ausfe277M+JE7X4Qsxgu/E6QwNCv4HH3xw1ieNiYkh\nOzu75nFOTg7R0dFOvTcvr/isxzvfREWFeE2cu7Zt48sx8XQ9Xcpx7FfMDgNeADqixF7fW7aiyru+\nZ2/6Ps+ExOk6fCFG8K04ncFldfi2Wf3gwYNZu3Yt5eXlHD16lCNHjnDppZe6aijBhi/HxDP7dCn3\no1bM7sW6AiAU8IvtyIC9h5h+vEi2HBSEZk6jPPyvvvqK2bNnU1BQwOTJk+nZsyfvvvsuPXr0YOjQ\noQwbNoyAgACeeeYZn6zQ8WaOZmSQOm443U6X2vn03VFtEsqBnE6duCPtO/HqBUEAwKDVZbh7EF+5\nffJUnIUmE9/OeIyDa1fzXEUFy1BdLXWffhYQGhZG62uu488fLaGiKtAjcZ4NvnTbLHG6Bl+IEXwr\nTmeQlbY+gi702qYNtDSb6YZ9D5xS4ECrIG5eta6m/01YhG/8sQqCcH4QwfcRvp3xGPemfGqXydv2\nwJkT25G/pO/2ZIiCIHg5IvhejJ7Vhx4+hHbogJ1X3xO1G1V7Pz+yYzowdOUazwUqCIJPIILvxdhm\n9Y419dlhYcQMHMz1ya/JpKwgCE4hgu9l7Nq2jS9G/wljWRm5wALgblRN/SthYXTv0o1CYxfGiNAL\ngnCWiOB7GV+OiefFsrKaTH4ZkIry6SMHDub6RYs9GZ4gCD6MCL6X0a3stJ1XHwKYgoJYPGQo1ye/\n5sHIBEHwdUTwPYztxGyh0cjuwBZo5dYMvxioHjKU4ZLZC4LQSETwPYxduWX6z/x94CCe+vF7jGVl\nHDcYaHX9QMZIZi8IggsQwfcwoYcP2Vk4nQsLuftonidDEgShiSKbmJ9HCk0mPk+4l2+H3MDnCfdQ\nWGCi0Gi02e4cCo1dPBihIAhNGcnwzyOO9s1iDFyf/DqLMVg8/C4yMSsIgtsQwT+PONo3oYcPERoe\nIROygiCcF8TSOY+IfSMIgieRDN8NOJZaXp/8OqHhEWLfCILgUUTw3UBdXv3wRYvFvhEEwaOIpeMG\n6vLqBUEQPI0IvhsQr14QBG9ELB03IF69IAjeSKMEf926dcyfP5+MjAxWrFhBnz59ADh27Bjx8fF0\n69YNgMsuu4xnn3220cH6CuLVC4LgjTRK8OPi4pg/fz5PP/10rdcuuOACVq5c2ZjTC4IgCC6kUYKv\nZ/CapjVwpCAIguBp3DZpm5mZydixYxk/fjw//fSTu4YRBEEQnKTBDH/ixInk5+fXej4xMZHBgwfX\n+Z7o6Gi+/vprQkND2blzJw8//DBr1qyhTZs2DQYUFRXiRNjnD/OJE6Q+9BDBBw9S3LUr8QsWAN4X\nZ31InK5F4nQdvhAj+E6cztCg4H/wwQdnfdLAwEBCQ0MB6NOnD507d+bQoUM1k7pnIi+v+KzHcyef\nJ0yyLqLaupXFZZVM/OwTr4uzLqKiQiROFyJxug5fiBF8K05ncJmlY+vjm0wmqqurATh69ChHjhyh\nc+fOrhrqvCKLqARBaCo0atL2q6++Yvbs2RQUFDB58mR69uzJu+++y08//cTf//53AgIC8PPz4/nn\nn6dt27auivm8Umg0oqX/XLPloCyiEgTBV2mU4N90003cdNNNtZ4fMmQIQ4YMacypvQZZRCUIQlNB\nVto2gCyiEgShqSC9dARBEJoJzVLw69pbVhAEoanTLC2d+vrVC4IgNGWaZYYvpZaCIDRHmqXgS796\nQRCaI03e0qlrf1kptRQEoTnS5AW/Pr9ePHtBEJobTd7SEb9eEARB0eQFX/x6QRAERZO3dMSvFwRB\nUDR5wZfWCIIgCIomb+kIgiAIChF8QRCEZoIIviAIQjNBBF8QBKGZIIIvCILQTBDBFwRBaCY0SvCT\nk5MZOnQoo0aNYtq0aZSUlNS89s477zBkyBCGDh3Kd9991+hABUEQhMbRKMHv378/a9asISUlBaPR\nyDvvvAPA/v37SU1NZe3atSxatIjnnnsOTdMaOJsgCILgThol+Ndeey1+fuoUffv2JScnB4ANGzYQ\nHx9PQEAAnTp1wmg08ttvvzU+WkEQBOGccZmHv2LFCgYOHAhAbm4uHTp0qHmtffv25ObmumooQRAE\n4RxosLXCxIkTyc/Pr/V8YmIigwcPBmDBggUEBgYyfPhwgDrtG4PBUOs5QRAE4fzRoOB/8MEHZ3x9\n5cqVbNq0iSVLltQ8FxMTQ3Z2ds3jnJwcoqOjnQooKirEqeM8jcTpWiRO1+ILcfpCjOA7cTpDoyyd\nb775hnfffZcFCxbQokWLmucHDx7M2rVrKS8v5+jRoxw5coRLL7200cEKgiAI545Ba0T5zJAhQ6io\nqIMzjrUAAATvSURBVCAsLAyAyy67jGeffRZQZZkrVqwgICCAp556iv79+7skYEEQBOHcaJTgC4Ig\nCL6DrLQVBEFoJojgC4IgNBNE8AVBEJoJXiv47733Hj179sRsNns6lDp58803GTlyJKNHj+b+++8n\nLy/P0yHVyZn6HXkT69atY/jw4fTq1YudO3d6Ohw7vvnmG/70pz9xyy23sHDhQk+HUy8zZ87k2muv\nZcSIEZ4OpV5ycnKYMGEC8fHxjBgxwq6c25soLy/ntttuY/To0YwYMYL58+d7OqR6qa6uZsyYMUye\nPLnhgzUvJDs7W7vvvvu0QYMGaQUFBZ4Op05KSkpqfl+yZIn29NNPezCa+tm8ebNWVVWlaZqmvfzy\ny9orr7zi4YjqJiMjQzt48KA2fvx4bceOHZ4Op4aqqirtpptu0jIzM7Xy8nJt5MiR2v79+z0dVp1s\n3bpV27VrlzZ8+HBPh1Ivx48f13bt2qVpmvo/NGTIEK/9Pk+dOqVpmqZVVlZqt912m/brr796OKK6\n+eCDD7Tp06drDz74YIPHemWGP2fOHJKSkjwdxhlp06ZNze+lpaU1PYW8jfr6HXkb3bp1o0uXLl7X\nZO+3337DaDTSsWNHAgMDGTZsGGlpaZ4Oq0769etH27ZtPR3GGYmKiqJXr16A+j/UvXt3jh8/7uGo\n6iYoKAhQ2X5lZaWHo6mbnJwcNm3axG233ebU8Q2utD3fbNiwgQ4dOnDRRRd5OpQGef3110lJSSEk\nJMRrb01tWbFiBcOGDfN0GD5FXX2htm/f7sGImg6ZmZns2bPHaxdlVldXM3bsWI4cOcKf//xnr4xT\nT46Li4udOt4jgl9ff55HH32Ud955h/fff7/mOU9mfA31EUpMTCQxMZGFCxfy0UcfMW3aNA9EeXb9\njjzp7zoTp7fhbXccTYWTJ0/yyCOPMHPmTLu7ZW/Cz8+Pzz77jJKSEh566CH2799Pjx49PB1WDV9/\n/TWRkZH06tWLLVu2OPUejwh+ff159u3bx7Fjxxg1ahSappGbm8u4ceP473//S7t27c5zlA33EdIZ\nPnw4Dz74oMcE/1z6HXkCZ79PbyImJoasrKyax7m5uU73hRLqprKykkceeYRRo0Zx0003eTqcBgkO\nDuYPf/gD3377rVcJ/s8//8yGDRvYtGkTZWVlnDx5kqSkJJKTk+t9j1cZz3FxcWzevJm0tDQ2bNhA\n+/btWblypUfEviEOHz5c83taWhrdunXzYDT1U1+/I2/Gm7LqSy65hCNHjnDs2DHKy8tZs2YNN954\no6fDqhdv+u7qY+bMmfTo0YN77rnH06HUi8lkqrFJTp8+zQ8//OB1/8cfe+wxvv76a9LS0njttdf4\n4x//eEaxBy/08G0xGAxe+wf86quvcvDgQfz8/IiNjeW5557zdEh18sILL1BRUcF9990H2Pc78ia+\n+uorZs+eTUFBAZMnT6Znz568++67ng4Lf39/Zs2axX333Yemadx66610797d02HVyfTp09myZQtm\ns5kbbriBadOmMW7cOE+HZce2bdtYvXo1cXFxjB49GoPBQGJiIgMGDPB0aHbk5eXxxBNPUF1dTXV1\nNfHx8TX7ffgy0ktHEAShmeBVlo4gCILgPkTwBUEQmgki+IIgCM0EEXxBEIRmggi+IAhCM0EEXxAE\noZkggi8IgtBMEMEXBEFoJvw//5K32R/vBHAAAAAASUVORK5CYII=\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f5be3c99f50\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Current loss: 9.48636\n" - ] - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.scatter(inputs, outputs, c='b')\n", - "plt.scatter(inputs, model(inputs), c='r')\n", - "plt.show()\n", - "\n", - "print('Current loss: '),\n", - "print(loss(model(inputs), outputs).numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sSDP-yeq_4jE" - }, - "source": [ - "### Define a training loop\n", - "\n", - "We now have our network and our training data. Let's train it, i.e., use the training data to update the model's variables (`W` and `b`) so that the loss goes down using [gradient descent](https://en.wikipedia.org/wiki/Gradient_descent). There are many variants of the gradient descent scheme that are captured in `tf.train.Optimizer` implementations. We'd highly recommend using those implementations, but in the spirit of building from first principles, in this particular example we will implement the basic math ourselves." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "MBIACgdnA55X" - }, - "outputs": [], - "source": [ - "def train(model, inputs, outputs, learning_rate):\n", - " with tf.GradientTape() as t:\n", - " current_loss = loss(model(inputs), outputs)\n", - " dW, db = t.gradient(current_loss, [model.W, model.b])\n", - " model.W.assign_sub(learning_rate * dW)\n", - " model.b.assign_sub(learning_rate * db)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RwWPaJryD2aN" - }, - "source": [ - "Finally, let's repeatedly run through the training data and see how `W` and `b` evolve." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 446 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 569, - "status": "ok", - "timestamp": 1527005915434, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "XdfkR223D9dW", - "outputId": "c43591ae-d5ac-4f2b-a8e7-bfce607e0919" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 0: W=5.00 b=0.00, loss=9.48636\n", - "Epoch 1: W=4.58 b=0.42, loss=6.28101\n", - "Epoch 2: W=4.24 b=0.76, loss=4.29357\n", - "Epoch 3: W=3.98 b=1.02, loss=3.06128\n", - "Epoch 4: W=3.78 b=1.23, loss=2.29721\n", - "Epoch 5: W=3.61 b=1.39, loss=1.82345\n", - "Epoch 6: W=3.49 b=1.52, loss=1.52970\n", - "Epoch 7: W=3.38 b=1.62, loss=1.34756\n", - "Epoch 8: W=3.30 b=1.70, loss=1.23463\n", - "Epoch 9: W=3.24 b=1.76, loss=1.16460\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAEDCAYAAAD+/1UIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VOXdPvD7zJZ9XwmELQkQIAELsiTsi6xiEBGXAiIW\nbV8WBY2K0tLa4lbsr283qxURtIoioAi8SpFNg6whi0FJKAoJBgLZt5k5c87vj5OZLIRkgEnOGXJ/\nritXJsmZyT0sN1+enPOMIMuyDCIicgs6tQMQEZHzWNpERG6EpU1E5EZY2kREboSlTUTkRljaRERu\nxODMQePGjYOvry90Oh0MBgM2b97c1rmIiKgZTpW2IAjYuHEjAgIC2joPERG1wKnlEVmWIUlSW2ch\nIqJWCM5cETl+/HgEBARAEATMmTMH9957b3tkIyKiJpxaHvnggw8QFhaG4uJiLFiwAD179sTgwYPb\nOhsRETXh1PJIWFgYACA4OBgTJ05EVlZWi8fL3t6AIADdugFvvglYrTeflIiIWl8eqampgSRJ8PHx\nQXV1NR5++GEsXrwYI0aMuPadCgtRvfoFeL2zDkJtLWxdu6NqRSrMs+8DDE4N9y4XFuaHoqIKVb73\ntTCTc7SYCdBmLmZyjlYzOaPVSfvy5ct44IEHkJKSgjlz5mDcuHEtFzYAREai6oWXUHwkA9WPPApd\n4QX4L/sVgpMGwWPTvwFRdCocERE15tQPIm9Ew3/FdBcK4P3ntfB89x0IVivEmFhUP/kMzCmzAL2+\nLb79VbT6LysztU6LmQBt5mIm52g1kzPa5YpIKaozKl9+DcWHT6Jm7gLof/wB/r98BEGjh8Fj28cA\nTyckInJKu17GLnWJRuXaP6P40AnUPDgP+jN58F+0AEFjhsO0fRvLm4ioFarsPSJ1647KP/0VxWnH\nUTvnAehPf4+AhfMQNG4ETDs/A/hiOkREzVJ1wyipR09U/OV1lHx9FLX3zIH+uxwEPPQAAieMgunz\nXSxvIqImNLHLny0mDhV/fxMlB4+g9u57YMjORMDcOQicNAamPV+wvImI6miitO1scb1Q8fo6lOz/\nBrUzZsJ4Mh0B99+DwKnjYdy7h+VNRNftL395DR999IHj4+XLl2DVqlWOj//61/+HDz/8txrRboim\nStvO1iceFf96B8V702CeNgPG48cQOGcmAu+cBOOBfSxvInJa//6JyM7OAKBsfldWVorc3FzH17Oz\nM5GQMECteNdNk6VtZ+vXH+Vvv4uSPQdhnjwVxiPfIPCeGQhImQpj2ldqxyMiN5CQMBBZWZkAgLNn\nz6Bnzxj4+PigsrISVqsVP/74A+Liequc0nnqXFN+ncSEASjf8AEMJ0/A+9UX4bH7c5hSpsIycjSq\nnloJcdhwtSMSkRN8Vj8Pj+3bXPqY5jtTULX699f8emhoKPR6Ay5duoisrEz075+I6uoyZGdnwsfH\nBzExsTCotL3GjdD0pN2UOPBnKH/vI5Ts2gPL2PEwHdyPoBmTEDD7LhiOHlY7HhFpVGJiIrKyMpCd\nrZT2gAEDkJWVgaws91oaAdxk0m5KHHQ7yjZtheHIYfi8sgam/Xth2r8X5vETUZ26EuJtg9SOSETN\nqFr9+xan4rbSr18isrIy8d//KssjHh4y/vnPf8HX1wfTpt3V7nluhltN2k2JQ4aibPMnKP1kFyzJ\nI+GxZzeCJo2F/8/vhSHzpNrxiEgjEhIGIC3tIPz9/SEIAgICAlBZWYHs7Cz075+gdrzr4talbWcd\nnoyyrTtQuuUzWIYlweOL/0PQhFHwn3c/9HU/gCCijismJhbl5WXo3z+x0ef8/Pzg7+9er33bLrv8\ntStZhvHAPvi8/AcYjx0BAJin3wWPXz+Hom69lRdn0Ait7jTGTM7RYi5mco5WMznjlpi0GxEEWEeP\nRemO3Sj9YAusPxsEj88+AYYMQeCEUfBc/xaEinK1UxIR3ZBbr7TtBAHWcRNQuutLlG7aCsycCUNO\nNvxSn0BIQm/4Ll8CQ/pxXqhDRG7l1i1tO0GAdex4YMsWFJ88hapnV0EKCYHXu+8gaNJYBI4fCc+3\n/wWhvEztpERErbr1S7sBKSIS1U88heKjmSj9YAvM02bAcOpb+D29HCGJveH7xGIYThzj9E1EmtWh\nSttBp4N13ASUv/2uMn2v/DWk0DB4vbcBQZPHIWjcCHiue5PTNxFpTscs7QakiEhUP/4kio9koHTT\nVpinzYD++1Pwe2aFMn0//j8wHD/K6ZuINKHDl7aDTgfr2PHK9J2eg8rnfgMpNBxe/96IoCnjETQ2\nmdM3kZsqLPwJ8+bNUTuGS7C0myFFRKJm2QoUHzmpTN/T74L+9HfK9J3QC77LfgXDsSOcvonciKCh\nazRuBku7Jfbpe91GXEk/hcrnV0MKj4DX++8iaOoEZfp+6w0IZaVqJyWiVoiiiD/8YTXmz78fy5Yt\ng9lsVjvSDbn1roi8BpddASVJMB7YB6+N62Ha9RkEUYTs5QXzXXejZu5DEAcPcfqqS61elcVMztFi\nLq1nWr3aA9u3u3afujvvFLF6dcsFXFj4E2bPnoF//GMd+vdPwJ/+9CI6dYrGfff93KVZbkbHvSKy\nrel0sI4Zh/K3NuDKye9Q+fxvIYVHwPOD9xA0bSKCxiTB861/cvom0piIiEjH5lAzZsxAZmaGyolu\njFtuzaoVcng4apY+gZrFy2A8uB+eG9fDY+d2+D37FHx/92uYZ8xEzbwF1zV9E93KVq82tzoVt5Wm\na9ru+leSk7Yr6HSwjh6Lin+9o0zfq34HKSISnpv+XTd9D4fnv16HUFqidlKiDquw8Cd8+202AGDH\njh1ITByocqIbw9J2MTk8HDVLHkfxN+ko3fwpau+6G/q8XPitTEVIYm/4LXkMhiOHeeYJUTvr3r0H\ndu36DPPn34+ysjKkpNyjdqQbwh9EtgOhqAiem/4Nz41vw3D2vwAAsU88DAsfxpVREyH16KlKruZo\n/QdZWqLFXMzkHK1mcgYn7XYgh4WhZvEylBw6gdKPt6M25W7oz+QBTz2FkKEDETR6OLxf/gMMWRmc\nwImoRfxBZHvS6WAdORrWkaNReeUKQr/eA/Omj2A6sA8+a1+Gz9qXYYvuCvOUabBMvRPWIcMAN3qV\naCJqe2wElcghIcDChSifcS+EygoYv/wPPHZ+BtPuz+H9xj/g/cY/IAUHwzxpKixTpsMyeizg5aV2\nbCJSGUtbA2RfP1hmzIRlxkzAYoHx64NKgf/fDni9/y683n8Xsrc3LGMnwDx1OiwTJ0EODFI7NhGp\ngKWtNSYTrGPHKy/c8PJaGE4cg8euHTDt3A6PHZ/CY8enkA0GWJNGKgU+ZRqkTlFqpyaidsLS1jKd\nDuLgIRAHD0HV86uhP/09PHZ9BtPO7TAd2AvTgb3AMytg/dkgmKdMh2XqnbDF9VI7NRG1IZ494i4E\nAbbefVD9+JMo/WI/rqTnoOLFV2EZOQaGjJPw/cNvEZw8GEFJg+Dz+9XKHuCSpHZqItVVVlZi69bN\nbfb406dPQGVlJQDgypXLGDnydmRlZTT4+kSUl7vuxcSdLm1JkjBz5kw89thjLvvmdOOkzl1Qu/BR\nlH38Ka7knEH5X/8J89Q7oS/Ih/f/voagKeMRPDAevqlPwLjvS8BiUTsykSoqKsqxdetHzX5NcsFg\n07dvArKzMwEA2dmZ6NWrD7KylI/PnfsRgYFB8Pf3v+nvY+d0aW/YsAExMTEu+8bkOnJQMMz33o/y\n9e/h8qmzKHvnfdTOeQCCuRZe699C4L0pCOkbA79fPgLT9m1A3VRA1BG8/vpfceFCAR5++EH8/e//\ni/T045g3bx5++9vnMX/+fVe9QML777+Lt99+EwBQUJCPFSuW4pFH5mHx4kU4d+7Hqx4/ISHRUdpZ\nWZmYM+dBfPttfYknJCS69Pk4taZdWFiI/fv347HHHsPbb7/t0gDkYt7esEyZBsuUaYAowvhNGky7\nPoPHzs/g+fGH8Pz4Q8geHrCMGQfLlOkw3zEFcmio2qmpAwke1L/Zzxcfz3bJ8U398pdL8MMP/8W6\nde8BANLTjyMrKwsbNnyIyMhIFBb+dM0XSHjllTVITV2Jzp27ICcnG2vXvoQ///kfjY7p3z8R69e/\nBQA4depbPPLIY/joo38DUEo8IWGAUzmd5VRpr1mzBqmpqaio0NZln9QKgwHWEaNgHTEKVb9/GYas\nDOUslF074PH5Lnh8vgu+Oh2sQ4fDMnU6zFOmA2HN/wUhupUkJiYiMjKyxWNqamqQnZ2BVauehn23\nD1EUrzqub99+yM39HrW1tbDZbPD09ERUVGcUFOQjOzsD99/v2j27Wy3tffv2ITQ0FPHx8Th8+LDT\nD+zsdfTtqcNnGj9SeVv7CpCbC3zyCYStW2E6lAbToa/hu+pZICEBYWPGAGPGAKNGARqZwrX4ewdo\nM5fmMzWzxAAAYde68/Ue34TFUg69XufIEBjoDS8vL8fHklQNQajPaDQCOp0JwcHeCAgIwPbtn7by\nHfzQvXs37N//OQYMSEBYmB+GDBmMrKxjKC8vw6Br/E/hRrVa2idOnMCXX36J/fv3w2w2o6qqCqmp\nqXjllVdavJ8WN2NhpgYCI4H5jwLzH4Vw8SI8Pt8Jj53bYUr7CsjKAv7yFwCAGN8X1uHJsCSPhHVY\nMuQwZ/+quI4Wf+8AbeZipqvV1sqoqKh0ZCgtrQZQ31GSZMLly1dw5kwBPD09sXv3HgwbloSaGhkR\nEZ3w4YdbMXbsBABAXl4uYmPjrvoeffr0w7p1b2PhwkdRVFSBbt164YUXViE+vp/Tz93Zf2xbLe3l\ny5dj+fLlAIAjR45g3bp1rRY2uRc5IgK18xagdt4ChPmbUPrFPhi/Pghj2tcwHjsMw6kceK1TfjAj\n9u4D6/BkWJNHwjJ8BOTwcJXTE7XM3z8ACQkDMH/+fRg6NAnDhyc3+rrBYMCCBY9g0aL5iIrqjG7d\nuju+9utfv4A//vElvPPOOthsIsaPv6PZ0k5IGIDNmzehXz/llXF69+6DoqIizJgx0+XP57q2ZrWX\n9uuvv97qsfzXvnVukcligSH9BEyHvlKK/OhhCNXVji+Lcb1gHT4C1uQRsCaNgBTR8jqhSzJphBZz\nMZNztJrJGdxPW0VumclqheHkCRgPfQ3T1wdhOHIYuqr6UwjFmFhYk0Y43lxxib0Wf50AbeZiJudo\nNZMzeBk7XR+jEeLtQyHePhQ1S5crJZ55UllKSTsI4+Fv4LVxPbw2rgcA2Lr3UNbD65ZUpM5d1M1P\n5OZY2nRzjEaIg26HOOh21Cx5HBBFGLIylBI/9BWMh9Lg9d4GeL23AQBg69odluQR9SUe3VXlJ0Dk\nXlja5FoGA8TbBkG8bRBq/mcpYLPB8G0WjF9/VV/iddvNAoAtuiusSSNgsS+ndO3mvi+TTdQOWNrU\ntvR6iIkDISYORM0vFwM2G/Q538KUdtAxjXtu+jc8NylXkNk6d3Gsh1uSRkDq3kPlJ0CkLSxtal96\nPWwJiahJSETNo/8DSBL0p3Ial/hHH8Dzow8AALZOUcCY0fDqkwAxcQDEhETI/gEqPwki9bC0SV06\nHWz9+qOmX3/U/OKXSol//x2MaV/BlKYsqeD99+GL9x13EXv0VKb3hAF1RT5Aefk2og6ApU3aotPB\nFt8Xtvi+qF24CJBlhJUWonx/GgyZGcpb1kl4frIF+GSL4262LtH1JZ44AGLiwDY5Z5zcT2VlJXbv\n/j/MnHlPm32PNWt+i+TkkRg9elybfQ87ljZpmyAAvXrBHNQJ5pRZyudkGbr8844CN2RmwJhxEh67\nPoPHrs8cd7WFR9SXeMJAiIkDIHWJ5g86Oxj7ftpNS1uSJOh07vc6MCxtcj+CACm6KyzRXWGZdqfj\n07qLhTBknmwwkWfA4z9fwOM/XziOkYKCHAVuf7N17wm44V9edzVokE+znz9+vMolxzfVcD9tvV4P\nLy9vREVF4ttvc/Dqq39Gaurj2LBhEwBlL+3a2hosWPALFBTk47XXXkFZWSk8PT2Rmvocunbtds3v\nc/ToYXz44fsoKSnG4sVPIClphFP5rhdLm24ZUkQkLBMnwzJxsuNzwpUrMGTVl7gh82T962va7+fr\nBzEh0bE+LiYOhC02DjDwr8etoOF+2unpx5Ga+gTWrn0VRqPfTe+l3VBh4U/429/eRH7+eSxd+hg2\nbdoGo9Ho8ufDP5V0S5NDQmAdMw7WMfVrjUJ5GQzZWfVTeVYGjIcPwXTo6/r7eXlB7NvfsT4uJg6A\n2DseMJnUeBq3FGcn5Bs9vjV9+/ZDVFRUi5exO7uXdkPjxk0EAHTpEo2oqM748ccfmt1c6maxtKnD\nkf0DHOeCO1RVwZCT3WAiz4AhIx3G40fr72c0QozvpxR4/0Rg2CAIIZ2VnQ65Tu42PD09Hbf1ej1s\ntvrXibRYzAAAWZbg5+fveLUbZzSd2K81wd8sljYRAPj4OPZUcTCbYfgup9FZK4Zvs2HMPOk4JBSA\n5B8AW2wsbDFxsMX1glj33tajJ+Dh0f7PhRrx9vZGdd3OlE33xwsKCkZpaQnKy8vh6emJtLSvMGxY\nEry9fdCpUxT27v1Pq3tp2+3d+x9MnjwNFy4U4MKFghbXv28GS5voWjw8IA64DeKA2+o/Z7VCn3sa\nhqwM+F/4EeaMbOjP5MKQlQnjieON7i7rdJC6doMYG+codFtsHMTYXsqLSXA6bxcN99M2mTwQHBzs\n+Jor9tK2i47uhsWLF6GkpBhPPbWyTdazAW7Nqipmco4WMwFNcokidOd+hOFMLvS5udCfyVXKPS8X\nustFV93XMZ3H1he5LTbupqdzLf5aMZNzuDUrUXsyGCD1jIGlZwzQ4OwVABBKS6DPy4U+LxeGuvf6\nvNOtT+f2Iq9bcuF0TgBLm6jNyYFBEAcPgTh4CMwNvyCK0J/7oa7E86DPO11X7KeVc8sbnF8O1E3n\nccpSixjXS1lyccF0Ts7bsGEd9u79DwRBgCzLEAQBY8dOwNy5C9otA5dHVMRMztFiJqBtc101neee\nVpZczv4XgtXa6FjHdB7XCx59eqEyJBK26GhIXaJh69IVcmioqhO6Fn//tJrJGZy0iTTIqem8bu3c\nUFfoHrs/B3Z/Dt+mj+XlBVvnLkqJR3eFFN0VtrpCl6KjIUV2AvT6dnx2dDNY2kTuxGCArWcsbD1j\ngTumNPqSUFKM0MorKMv8Dvr8c9Dln4f+/Pm69z/CkJfb7EPKBgOkqM6wdbFP59GOYpeio2HrHM3l\nFw1haRPdIuSgYKBXN1iir3FaWmUl9PnnlUI/fx76/PPQ5Z9zFLvx0NcQrrFaaguPUAo8uiukLg0K\nvW5al32d+6893TyWNlFH4esLW5942PrEN/91sxm6CwV1ZX4e+vPn6m+fOwdDxkkYjx9r9q5SYKBS\n4F2i69bT64sdiX0A2YNLMC7C0iYihYcHpB49IfXo2fzXbTboLhbWTen1yy/224b/5kHIzmz2rqF6\nPaTQMEjhEZAiIhq/D49s9DG8vdvwSbo/ljYROUevhxTVGVJUZ4hDh139dVmGUFzcYPlFKXPv4iKI\n5wuUrXPP5ELIymjx20h+/pDCwyFFRCrvHcVu/1wEpIhIyMHBHXJLXZY2EbmGIEAOCYEYEgI0uPTf\nO8wPpQ1OrxMqK6C7dBG6ixfr3hdCd+lS3fv6z+v/e+aaa+xA3Q9Qw8KbTO0RjlJvWPJosEmUu2Np\nE1G7kn39YPP1U86AaYnVCt2Vy1eVeePCvwjD96cgZKS3+FBSQGD91B4RAXTtAm9PX0jBIZCCgyEH\nBUMKCoYcEgIpKFjTJc/SJiJtMhohRXZSziNviSxDqChvMq03md7r3gy5px13a/71cOoe0ttbKfSg\nukIPaVDswcH1X6u7LQcHQ/bxbZeLmFjaROTeBAGyfwBs/gHKKw61xGKB7nIRQsQqlJ45D11JMYSS\nYuiuXGl0Wygpga6kGIYzeRCqnXsRBtlobDSty0H1hS4FBSsTfXDj4pcDAq97XZ6lTUQdh8kEKaoz\nEOYHa9dezt3HbFYKvbgYuuIrSrHbbxcX15e9/eOfLsBwKseph5Z1OsiBgZCCQ4AG/wtoCUubiKgl\nHh7KEk1kJ9icvY8oQigtVQq9boq/ZvHX3XYWS5uIyNUMBsihobCFhgJOvkxkmJMP3fFOciQicmMs\nbSIiN8LSJiJyIyxtIiI30uoPIi0WCx588EFYrVbYbDZMmjQJixcvbo9sRETURKulbTKZsGHDBnh5\necFms+H+++/HqFGjkJiY2B75iIioAaeWR7y8vAAoU7coim0aiIiIrs2p0pYkCSkpKUhOTkZycjKn\nbCIilTh1cY1Op8O2bdtQWVmJX/3qV8jLy0NsbAs7dHXvjmDp6i0Vi49nN3t48KD+zX7epcfrhKsy\nqZoHuCqT6nmaZNJEngaZNJPH7tyPmsrD42+N41tzXVdE+vr6YsiQITh48GDLpQ1Ar7t6t6trvkR8\nM8e2xfFNM6mdp2kmLeRpmEkreeyZtJSnxfuolMd+/FX3UznPVffVQJ5GH2skj7MEWW5hl3EAxcXF\nMBqN8PPzQ21tLRYuXIhFixZh9OjRLT5wUYNNz7UgLMyPmZzATM7TYi5mco5WMzmj1Um7qKgIzzzz\nDCRJgiRJmDp1aquFTUREbaPV0u7duze2bt3aHlmIiKgVvCKSiMiNsLSJiNwIS5uIyI2wtImI3AhL\nm4jIjbC0iYjcCEubiMiNsLSJiNwIS5uIyI2wtImI3AhLm4jIjbC0iYjcCEubiMiNsLSJiNwIS5uI\nyI2wtImI3AhLm4jIjbC0iYjcCEubiMiNsLSJiNwIS5uIyI2wtImI3AhLm4jIjbC0iYjcCEubiMiN\nsLSJiNwIS5uIyI2wtImI3AhLm4jIjbC0iYjcCEubiMiNsLSJiNwIS5uIyI2wtImI3AhLm4jIjbC0\niYjciKG1AwoLC5GamorLly9Dr9dj9uzZmDdvXntkIyKiJlotbb1ej2effRbx8fGoqqrC3XffjeTk\nZMTExLRHPiIiaqDV5ZGwsDDEx8cDAHx8fBATE4NLly61eTAiIrrada1p5+fn47vvvkNiYmJb5SEi\noha0ujxiV1VVhaVLl2LlypXw8fFp8dju3QFJuvqY48ermj1+0KDmH8+Vx+t0V2dSMw+AqzKpnadp\nJi3kaZhJK3nszp1r9tOq5eHxt8bxrXGqtEVRxNKlS3HXXXdhwoQJTj2wTnf1EB8W5neNY5t/DFcf\n3zST2nmaZtJCnoaZtJLHnklLeVq6j1p57Mc3vZ/aeZre1kKehh9rJY+zBFmW5dYOSk1NRVBQEJ59\n9lmnH7ioqOKGArWVsDA/ZnICMzlPi7mYyTlazeSMVte0jx8/ju3bt+Obb75BSkoKZs6ciQMHDtx0\nQCIiun6tLo8MGjQIp06dao8sRETUCl4RSUTkRljaRERuhKVNRORGWNpERG6EpU1E5EacviKSiIiu\nnyQBZWVASYmA4uL6t5ISwfG5khIBn37q3OOxtImInGSxoFHRNizgpkWsfAyUlgqQJMFlGVjaRNTh\nyDJQWYmrCvdaU7D981VVzpWvXi8jKEhGaKiMuDgJQUEygoOVt6Ag1L2XHe+DgmQAvk49NkubiG4Z\nNTVAUZGAixcFXLqkw6VLyu2iIuVj5fMCLl8GLBbnLhv39lZKtUcPqVHR1pdw4/INCZHh5wcIrhuu\nG2FpE5GmSZIyEV+6JDhK2F7IDd8uXtShvLzlpvTwkBERIWPgQMDfX2yxfO23vbza6Yk6iaVNRKqo\nqUGjwm1cwvVTcVGRAFFsuYxDQiR07izhtttkhIfLiIiQEB5uvy3X3Zbg769MwMqGUTXt9Exdi6VN\nRC4likBhoYD8fB3OnxdQUQGcPevRYEpWStn5qVhCeLjUqIAblnJYmAyjsZ2enAawtInoutTUABcu\nCDh/Xof8fB3y8+23laK+cEGAzda0kE2OW9eaiusnYuVzbbku7M5Y2kTUSFkZGpVw49sCLl9u/po8\nQZARGSnjZz+T0KWL/U1GfLwnPD2rEBGhnE3RkabitsDSJupAZFlZR25Ywsq0XH+7oqL58dZolNG5\ns4z4eBFdusjo0kVCdLTkuB0VJcNkuvp+YWGeKCqS2viZdRwsbaJbiNUKnDvXtJDrlzIKCgSYzc2X\nso+P3KiEu3SxfywhOlpZtmjppdeofbC0idyMLAM//SQgL0+H3FwdzpzRIS9PeV9QAEhS8xdphIZK\niI9X1pPrC7m+mAMDuYbsDljaRBpVXQ2cOaOUccNyzsvTobr66naNiJCQlARERFgbTczR0TI6d5bg\n7a3CkyCXY2kTqcg+Nefm1heyfWrOz796LcLTU0bPnhJiYxu/xcQoZ1so5x/XqvBMqL2wtInagX1q\nbljK9um5uak5MlLCyJEiYmIal3OXLlxX7uhY2kQuIsvK+csNJ2b7W0HBtafmuDjJUc72277O7R1E\nHRBLm+g6WSzA99/rcOkScOKEqdH03NzU3KmTMjU3XMqIi5PQuTOnZrp+LG2iFtTUADk5OmRm6pGV\npbw/dUoHq9Vezh4AAC+va681c2omV2JpE9WprASys/XIzKwv6dOndY0uyfbwkJGQIKF/fxsGDzYh\nIqIasbGcmqn9sLSpQyopAbKylIJW3utx5kzj1vX2ljF4sA2JiRISEpT3cXGS4zLssDATiopsKqSn\njoylTbe8S5cEx9KGvaTPnWtc0AEBMkaOFJGQICEx0YbERBt69uT0TNrD0qZbhv3sjYblnJmpQ2Fh\n4+YNDZUwbpyIxESbo6S7dpV5NSC5BZY2uSVZBn74QXAUs30N+sqVxgUdFSVh8mRrgwlaQmQkC5rc\nF0ubNM9mA06f1jUq56ws/VWb6HfrJiEpyepYg05IkBAWJquUmqhtsLRJc8xmID1dj6+/1iMtTY/j\nx4Hqah/H1wVBeYXrCRPqp+f+/W0IDFQxNFE7YWmT6mprgRMnlIJOS9Pj2DE9amvrp+h+/YCEBKtj\nDbpfPxvPfaYOi6VN7a6mBjh+vL6kjx/XO/Z4FgQZfftKSE62YfhwG4YPF9G7NzdBIrJjaVObq64G\njh2rL+lQSYIFAAANpklEQVQTJ/SwWOpLun9/CUlJNiQl2TBsmIigIJUDE2kYS5tcrqrq6pK2X/at\n09WXdHKyiKFDuRZNdD1Y2nTTKiuBo0ftJW1AeroOolhf0omJ9klaKemAAJUDE7kxljZdt8pK4MgR\n+9kdBmRk1Je0Xi9jwAAJw4crk/SQITb4+6scmOgW0mppr1y5Evv27UNISAi2b9/eHplIYyoqgMOH\n6yfpjIz6TZT0ehkDB0pIShKRnGzDkCE8s4OoLbVa2nfffTfmzp2L1NTU9shDGlBeDnzzjVLQaWnK\nFYeSpJS0wSDjttskJCeLGD6cJU3U3lot7cGDB6OgoKA9spBKZBk4eVKHXbsMOHgQSE/3dZS00ajs\ndGc/u+P2223w8WnlAYmozXBNu4OyWoFDh/TYtcuAXbsMuHBB2bPDaARuv92G5GSlpAcPtvFVvIk0\npM1KOyzMr60e+oZ19EzV1cAXXwBbtwLbtyt7SgNAYCAwdy6QkgJMmgT4+BigtX/Ptfh7B2gzFzM5\nR4uZnNFmfzOLiira6qFvSFiYX4fMVFICfPGFATt3GrBvnwE1NcqyR2SkhAULREydKiIpyebY2N/H\np2P+Ot0ILeZiJudoNZMznCptWeZOae7kwgUBu3YpRZ2Wpnec6REba8PUqUpRDxwocYN/IjfUammv\nWLEChw8fRmlpKcaMGYMlS5Zg1qxZ7ZGNrkNurg47dypFnZ6ud3z+ttuUop4yRUSvXpKKCYnIFVot\n7bVr17ZHDrpOkqSc8WEv6rw8paj1euVls+xFHRXF/yUR3Uq09dMmapHVCqSl1Z/x8dNPyvqGl5eM\nKVOsmDpVxB13cMMlolsZS1vjqquBvXuVaXr3bgNKS5X16cBAGffeqxT1mDEiT8sj6iBY2hpUUgJ8\n/rlS1Pv315/xERUlYdYspaiHDas/44OIOg6WtkYUFAiOZY+GZ3z06lX/g8SBAyW+IC1RB8fSVtGp\nU8C775qwc6cBJ0/Wn/Hxs5/ZT82zIjaWP0gkonos7XZWWgp89JER775rxKlTAOABg0HGqFH1Z3x0\n6sSiJqLmsbTbgSwDR4/qsGGDCZ9+akBtrQCjUcbMmcCECTWYOFHkq7cQkVNY2m2orEyZqjduNOLU\nKWX5o0cPCXPnmjFnjoi+fX1RVCSqnJKI3AlL28VkGTh2TIeNG0345BPlzA+jUcZdd1kxd64VI0bY\nePk4Ed0wlraLlJUBmzcbsWFD/VTdrZuEuXMtuP9+K8LCuE5NRDePpX0TZBk4cUJZq962TZmqDQYZ\nM2YoU/XIkZyqici1WNo3oLy8fqrOyWk8Vd93nxXh4ZyqiahtsLSdJMtAeroOGzYYsW2bEdXVylQ9\nfboV8+ZZMWoUp2oianss7VZUVChT9caNRmRnK1N11671U3VEBKdqImo/LO1m2F/oduNGI7ZsUaZq\nvV7GtGnKVD16NKdqIlIHS7uBysr6qTorq36q/vnPlTNAOFUTkdpY2gAyMpS16o8/rp+qp05Vpuox\nYzhVE5F2dNjSrqwEtmxRzgDJzFSm6i5dJCxdasEDD1gRGcmpmoi0p8OVdmamDu+8o6xVV1UpU/Xk\nyVbMn69M1Xp9649BRKSWDlHalZXAtm3A3//u7dgCtXNnCYsXK1M1d9UjIndxS5d2eTnwxhsmvP66\nCeXlgE6nw+TJylr12LGcqonI/dySpV1ZCbz1lgl/+5sJpaUCQkIkrF4tICWliq9OTkRu7ZYq7epq\nYN06I/72NxOuXNEhMFDGc8+ZsXChBT16+KGoiIVNRO7tlijt2lpgwwYj/vxnE4qKdPD3l5Gaasai\nRRb4+6udjojIddy6tM1m4N13lbIuLNTBx0fG8uVmPPaYha8EQ0S3JLcsbasVeP99I/70JxMKCnTw\n9paxZIkZv/qVFSEhXAIholuXW5W2KAIffWTA2rUeOHdOB09PGY89ZsGSJRa+yAARdQhuUdo2G7Bl\niwF//KMHzp7VwWSS8cgjFixbZuF+IETUoWi6tCUJ+PRTA1591YTcXD2MRhkPPWTB449beOoeEXVI\nmixtSQJ27lTK+tQpPfR6GT//uVLWXbuyrImo49JUacsy8MUXerz8sgeys/XQ6WTMmWPF8uVm9OjB\nsiYi0kRpyzKwd69S1unpegiCjLvvtuLJJ82IjWVZExHZqVrasgwcPKiU9dGjykYgM2ZY8eSTFvTp\nI6kZjYhIk1Qr7UOH9HjpJRMOHVIiTJlixVNPWdC/P8uaiOha2r20jx7V4aWXPHDwoPKtJ04UkZpq\nxoABLGsiotY49UJaBw4cwOTJkzFp0iS88cYbN/SNTpzQ4b77vDBtmg8OHjRgzBgRu3ZV4b33aljY\nREROanXSliQJL7zwAtavX4/w8HDcc889GD9+PGJiYpz6BllZOrzyigc+/1z5ViNGiEhNtWDYMNvN\nJSci6oBaLe3MzEx069YNnTt3BgBMmzYNe/bsabW0c3J0ePVVE3bsMAIAhg4V8fTTFowYwbImIrpR\nrZb2xYsX0alTJ8fHERERyMrKavE+990HfPihN2RZwKBBNjz9tBmjR9sgCDcfmIioI2u1tGX5+s+T\n3rQJGDBAwtNPmzF+PMuaiMhVWi3tyMhIXLhwwfHxxYsXER4e3uJ9lJ7XA/C+yXiuFRbmp3aEqzCT\nc7SYCdBmLmZyjhYzOaPVs0cSEhJw7tw5FBQUwGKxYMeOHRg/fnx7ZCMioiZanbT1ej1WrVqFhx9+\nGLIs45577nH6zBEiInItQb6RRWsiIlKFUxfXEBGRNrC0iYjcCEubiMiNuHTDqAMHDmDNmjWQZRmz\nZs3CokWLXPnwN2TlypXYt28fQkJCsH37drXjAAAKCwuRmpqKy5cvQ6/XY/bs2Zg3b56qmSwWCx58\n8EFYrVbYbDZMmjQJixcvVjWTnSRJmDVrFiIiIvD666+rHQfjxo2Dr68vdDodDAYDNm/erHYkVFRU\n4LnnnkNubi50Oh3WrFmDAQMGqJrp7NmzeOKJJyAIAmRZxvnz57Fs2TLV/6yvX78emzdvhiAI6NWr\nF1588UWYTCZVM73zzjuOP0et9oHsIjabTZ4wYYKcn58vWywWecaMGXJeXp6rHv6GHT16VM7JyZGn\nT5+udhSHS5cuyTk5ObIsy3JlZaV8xx13aOLXqrq6WpZlWRZFUZ49e7ackZGhciLF22+/La9YsUJ+\n9NFH1Y4iy7Isjxs3Ti4tLVU7RiNPP/20vHnzZlmWZdlqtcoVFRUqJ2rMZrPJycnJ8oULF1TNUVhY\nKI8bN042m82yLMvysmXL5K1bt6qa6fTp0/L06dNls9ksi6IoP/TQQ/KPP/54zeNdtjzScI8So9Ho\n2KNEbYMHD4a/v7/aMRoJCwtDfHw8AMDHxwcxMTG4dOmSyqkALy8vAMrULYqiymkUhYWF2L9/P2bP\nnq12FAdZliFJ2tmZsrKyEseOHcOsWbMAAAaDAb6+viqnaiwtLQ1du3ZttCWGWiRJQk1NDURRRG1t\nbasXC7a1M2fOYODAgTCZTNDr9bj99tuxe/fuax7vstJubo8SLRSR1uXn5+O7775DYmKi2lEgSRJS\nUlKQnJyM5ORkTWRas2YNUlNTIWhoLwRBELBw4ULMmjULH374odpxkJ+fj6CgIDz77LOYOXMmVq1a\nhdraWrVjNbJz505MmzZN7RiIiIjAggULMGbMGIwaNQp+fn5ISkpSNVNcXByOHj2KsrIy1NTU4MCB\nA/jpp5+uebzLSlvm6d7XraqqCkuXLsXKlSvh4+OjdhzodDps27YNBw4cQEZGBvLy8lTNs2/fPoSG\nhiI+Pl5Tf74++OADbNmyBW+++Sbee+89HDt2TNU8oigiJycHDzzwALZu3QpPT88b3ve+LVitVnz5\n5ZeYMmWK2lFQXl6OPXv2YO/evTh48CCqq6tV/1lXTEwMfvGLX2DBggVYtGgR+vTpA4Ph2j9udFlp\n38geJR2ZKIpYunQp7rrrLkyYMEHtOI34+vpiyJAhOHjwoKo5Tpw4gS+//BLjx4/HihUrcPjwYaSm\npqqaCVCWtwAgODgYEydObHXXy7YWGRmJyMhIJCQkAAAmTZqEnJwcVTM1dODAAfTr1w/BwcFqR0Fa\nWhqio6MRGBgIvV6PiRMnIj09Xe1YmDVrFrZs2YKNGzciICAA3bp1u+axLittLe9RoqUpzW7lypWI\njY3F/Pnz1Y4CACguLkZFRQUAoLa2FocOHULPnj1VzbR8+XLs27cPe/bswWuvvYahQ4filVdeUTVT\nTU0NqqqqAADV1dX46quvEBcXp2qm0NBQdOrUCWfPngUAfPPNN5raamLHjh2YPn262jEAAFFRUcjI\nyIDZbIYsy5r5tSouLgYAXLhwAbt3727x18tlp/xpdY8S+4RWWlqKMWPGYMmSJY4f2Kjl+PHj2L59\nO3r16oWUlBQIgoAnnngCo0aNUi1TUVERnnnmGUiSBEmSMHXqVIwePVq1PFp1+fJlLF68GIIgwGaz\n4c4778SIESPUjoXnn38eTz75JERRRHR0NF588UW1IwFQBoC0tDT87ne/UzsKACAxMRGTJk1CSkoK\nDAYD+vbti3vvvVftWFiyZAnKyspgMBjwm9/8Bn5+196BkHuPEBG5EV4RSUTkRljaRERuhKVNRORG\nWNpERG6EpU1E5EZY2kREboSlTUTkRljaRERu5P8D+7Wym3BFpegAAAAASUVORK5CYII=\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f5be4b8ec50\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "model = Model()\n", - "\n", - "# Collect the history of W-values and b-values to plot later\n", - "Ws, bs = [], []\n", - "epochs = range(10)\n", - "for epoch in epochs:\n", - " Ws.append(model.W.numpy())\n", - " bs.append(model.b.numpy())\n", - " current_loss = loss(model(inputs), outputs)\n", - "\n", - " train(model, inputs, outputs, learning_rate=0.1)\n", - " print('Epoch %2d: W=%1.2f b=%1.2f, loss=%2.5f' %\n", - " (epoch, Ws[-1], bs[-1], current_loss))\n", - "\n", - "# Let's plot it all\n", - "plt.plot(epochs, Ws, 'r',\n", - " epochs, bs, 'b')\n", - "plt.plot([TRUE_W] * len(epochs), 'r--',\n", - " [TRUE_b] * len(epochs), 'b--')\n", - "plt.legend(['W', 'b', 'true W', 'true_b'])\n", - "plt.show()\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vPnIVuaSJwWz" - }, - "source": [ - "## Next Steps\n", - "\n", - "In this tutorial we covered `Variable`s and built and trained a simple linear model using the TensorFlow primitives discussed so far.\n", - "\n", - "In theory, this is pretty much all you need to use TensorFlow for your machine learning research.\n", - "In practice, particularly for neural networks, the higher level APIs like `tf.keras` will be much more convenient since it provides higher level building blocks (called \"layers\"), utilities to save and restore state, a suite of loss functions, a suite of optimization strategies etc. \n", - "\n", - "The [next tutorial](TODO) will cover these higher level APIs." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "default_view": {}, - "name": "Training Models", - "provenance": [], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/4_high_level.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/4_high_level.ipynb deleted file mode 100644 index 4fe3a0e3f3..0000000000 --- a/tensorflow/contrib/eager/python/examples/notebooks/4_high_level.ipynb +++ /dev/null @@ -1,551 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "pwX7Fii1rwsJ" - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "tf.enable_eager_execution()\n", - "tfe = tf.contrib.eager\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UEu3q4jmpKVT" - }, - "source": [ - "# High level API\n", - "\n", - "We recommend using `tf.keras` as a high-level API for building neural networks. That said, most TensorFlow APIs are usable with eager execution.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zSFfVVjkrrsI" - }, - "source": [ - "## Layers: common sets of useful operations\n", - "\n", - "Most of the time when writing code for machine learning models you want to operate at a higher level of abstraction than individual operations and manipulation of individual variables.\n", - "\n", - "Many machine learning models are expressible as the composition and stacking of relatively simple layers, and TensorFlow provides both a set of many common layers as a well as easy ways for you to write your own application-specific layers either from scratch or as the composition of existing layers.\n", - "\n", - "TensorFlow includes the full [Keras](https://keras.io) API in the tf.keras package, and the Keras layers are very useful when building your own models.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "8PyXlPl-4TzQ" - }, - "outputs": [], - "source": [ - "# In the tf.keras.layers package, layers are objects. To construct a layer,\n", - "# simply construct the object. Most layers take as a first argument the number\n", - "# of output dimensions / channels.\n", - "layer = tf.keras.layers.Dense(100)\n", - "# The number of input dimensionss is often unnecessary, as it can be inferred\n", - "# the first time the layer is used, but it can be provided if you want to \n", - "# specify it manually, which is useful in some complex models.\n", - "layer = tf.keras.layers.Dense(10, input_shape=(None, 5))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fn69xxPO5Psr" - }, - "source": [ - "The full list of pre-existing layers can be seen in [the documentation](https://www.tensorflow.org/api_docs/python/tf/keras/layers). It includes Dense (a fully-connected layer),\n", - "Conv2D, LSTM, BatchNormalization, Dropout, and many others." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 204 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 244, - "status": "ok", - "timestamp": 1527783641557, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "E3XKNknP5Mhb", - "outputId": "c5d52434-d980-4488-efa7-5660819d0207" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "\u003ctf.Tensor: id=30, shape=(10, 10), dtype=float32, numpy=\n", - "array([[ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", - " [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]], dtype=float32)\u003e" - ] - }, - "execution_count": 3, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "# To use a layer, simply call it.\n", - "layer(tf.zeros([10, 5]))" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 221 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 320, - "status": "ok", - "timestamp": 1527783642457, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "Wt_Nsv-L5t2s", - "outputId": "f0d96dce-0128-4080-bfe2-0ee6fbc0ad90" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[\u003ctf.Variable 'dense_1/kernel:0' shape=(5, 10) dtype=float32, numpy=\n", - " array([[ 0.43788117, -0.62099844, -0.30525017, -0.59352523, 0.1783089 ,\n", - " 0.47078604, -0.23620895, -0.30482283, 0.01366901, -0.1288507 ],\n", - " [ 0.18407935, -0.56550485, 0.54180616, -0.42254075, 0.3702994 ,\n", - " 0.36705834, -0.29678228, 0.36660975, 0.36717761, 0.46269661],\n", - " [ 0.1709305 , -0.11529458, 0.32710236, 0.46300393, -0.62802851,\n", - " 0.51641601, 0.39624029, 0.26918125, -0.25196898, 0.21353298],\n", - " [ 0.35752094, 0.44161648, 0.61500639, -0.12653333, 0.41629118,\n", - " 0.36193585, 0.066082 , -0.59253877, 0.47318751, 0.17115968],\n", - " [-0.22554061, -0.17727301, 0.5525015 , 0.3678053 , -0.00454676,\n", - " 0.24066836, -0.53640735, 0.13792562, -0.10727292, 0.59708995]], dtype=float32)\u003e,\n", - " \u003ctf.Variable 'dense_1/bias:0' shape=(10,) dtype=float32, numpy=array([ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32)\u003e]" - ] - }, - "execution_count": 4, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "# Layers have many useful methods. For example, you can inspect all variables\n", - "# in a layer by calling layer.variables. In this case a fully-connected layer\n", - "# will have variables for weights and biases.\n", - "layer.variables" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 221 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 226, - "status": "ok", - "timestamp": 1527783643252, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "6ilvKjz8_4MQ", - "outputId": "f647fced-c2d7-41a3-c237-242036784665" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(\u003ctf.Variable 'dense_1/kernel:0' shape=(5, 10) dtype=float32, numpy=\n", - " array([[ 0.43788117, -0.62099844, -0.30525017, -0.59352523, 0.1783089 ,\n", - " 0.47078604, -0.23620895, -0.30482283, 0.01366901, -0.1288507 ],\n", - " [ 0.18407935, -0.56550485, 0.54180616, -0.42254075, 0.3702994 ,\n", - " 0.36705834, -0.29678228, 0.36660975, 0.36717761, 0.46269661],\n", - " [ 0.1709305 , -0.11529458, 0.32710236, 0.46300393, -0.62802851,\n", - " 0.51641601, 0.39624029, 0.26918125, -0.25196898, 0.21353298],\n", - " [ 0.35752094, 0.44161648, 0.61500639, -0.12653333, 0.41629118,\n", - " 0.36193585, 0.066082 , -0.59253877, 0.47318751, 0.17115968],\n", - " [-0.22554061, -0.17727301, 0.5525015 , 0.3678053 , -0.00454676,\n", - " 0.24066836, -0.53640735, 0.13792562, -0.10727292, 0.59708995]], dtype=float32)\u003e,\n", - " \u003ctf.Variable 'dense_1/bias:0' shape=(10,) dtype=float32, numpy=array([ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32)\u003e)" - ] - }, - "execution_count": 5, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "# The variables are also accessible through nice accessors\n", - "layer.kernel, layer.bias" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O0kDbE54-5VS" - }, - "source": [ - "## Implementing custom layers\n", - "The best way to implement your own layer is extending the tf.keras.Layer class and implementing:\n", - " * `__init__` , where you can do all input-independent initialization\n", - " * `build`, where you know the shapes of the input tensors and can do the rest of the initialization\n", - " * `call`, where you do the forward computation\n", - "\n", - "Note that you don't have to wait until `build` is called to create your variables, you can also create them in `__init__`. However, the advantage of creating them in `build` is that it enables late variable creation based on the shape of the inputs the layer will operate on. On the other hand, creating variables in `__init__` would mean that shapes requires to create the variables will need to be explicitly specified." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 391 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 251, - "status": "ok", - "timestamp": 1527783661512, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "5Byl3n1k5kIy", - "outputId": "6e7f9285-649a-4132-82ce-73ea92f15862" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tf.Tensor(\n", - "[[ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n", - " [ 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]], shape=(10, 10), dtype=float32)\n", - "[\u003ctf.Variable 'my_dense_layer_1/kernel:0' shape=(5, 10) dtype=float32, numpy=\n", - "array([[-0.4011991 , 0.22458655, -0.33237562, -0.25117266, 0.33528614,\n", - " -0.01392961, 0.58580834, -0.16346583, 0.28465688, -0.47191954],\n", - " [-0.52922136, 0.22416979, -0.58209574, -0.60914612, 0.05226624,\n", - " -0.18325993, 0.5591442 , -0.24718609, 0.37148207, 0.40475875],\n", - " [ 0.16912812, -0.47618777, -0.38989353, 0.30105609, -0.08085585,\n", - " 0.44758242, 0.545829 , 0.51421839, 0.11063248, 0.20159996],\n", - " [ 0.34073615, -0.59835428, 0.06498981, -0.44489855, -0.34302285,\n", - " 0.20969599, 0.35527444, -0.03173476, -0.22227573, 0.09303057],\n", - " [ 0.41764337, -0.06435019, -0.52509922, -0.39957345, 0.56811184,\n", - " 0.23481232, -0.61666459, 0.31144124, -0.11532354, -0.42421889]], dtype=float32)\u003e]\n" - ] - } - ], - "source": [ - "class MyDenseLayer(tf.keras.layers.Layer):\n", - " def __init__(self, num_outputs):\n", - " super(MyDenseLayer, self).__init__()\n", - " self.num_outputs = num_outputs\n", - " \n", - " def build(self, input_shape):\n", - " self.kernel = self.add_variable(\"kernel\", \n", - " shape=[input_shape[-1].value, \n", - " self.num_outputs])\n", - " \n", - " def call(self, input):\n", - " return tf.matmul(input, self.kernel)\n", - " \n", - "layer = MyDenseLayer(10)\n", - "print(layer(tf.zeros([10, 5])))\n", - "print(layer.variables)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tk8E2vY0-z4Z" - }, - "source": [ - "Note that you don't have to wait until `build` is called to create your variables, you can also create them in `__init__`.\n", - "\n", - "Overall code is easier to read and maintain if it uses standard layers whenever possible, as other readers will be familiar with the behavior of standard layers. If you want to use a layer which is not present in tf.keras.layers or tf.contrib.layers, consider filing a [github issue](http://github.com/tensorflow/tensorflow/issues/new) or, even better, sending us a pull request!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Qhg4KlbKrs3G" - }, - "source": [ - "## Models: composing layers\n", - "\n", - "Many interesting layer-like things in machine learning models are implemented by composing existing layers. For example, each residual block in a resnet is a composition of convolutions, batch normalizations, and a shortcut.\n", - "\n", - "The main class used when creating a layer-like thing which contains other layers is tf.keras.Model. Implementing one is done by inheriting from tf.keras.Model." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 190 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 420, - "status": "ok", - "timestamp": 1527783698512, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 420 - }, - "id": "N30DTXiRASlb", - "outputId": "a8b23a8e-5cf9-4bbf-f93b-6c763d74e2b3" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tf.Tensor(\n", - "[[[[ 0. 0. 0.]\n", - " [ 0. 0. 0.]\n", - " [ 0. 0. 0.]]\n", - "\n", - " [[ 0. 0. 0.]\n", - " [ 0. 0. 0.]\n", - " [ 0. 0. 0.]]]], shape=(1, 2, 3, 3), dtype=float32)\n", - "['resnet_identity_block_1/conv2d_3/kernel:0', 'resnet_identity_block_1/conv2d_3/bias:0', 'resnet_identity_block_1/batch_normalization_3/gamma:0', 'resnet_identity_block_1/batch_normalization_3/beta:0', 'resnet_identity_block_1/conv2d_4/kernel:0', 'resnet_identity_block_1/conv2d_4/bias:0', 'resnet_identity_block_1/batch_normalization_4/gamma:0', 'resnet_identity_block_1/batch_normalization_4/beta:0', 'resnet_identity_block_1/conv2d_5/kernel:0', 'resnet_identity_block_1/conv2d_5/bias:0', 'resnet_identity_block_1/batch_normalization_5/gamma:0', 'resnet_identity_block_1/batch_normalization_5/beta:0', 'resnet_identity_block_1/batch_normalization_3/moving_mean:0', 'resnet_identity_block_1/batch_normalization_3/moving_variance:0', 'resnet_identity_block_1/batch_normalization_4/moving_mean:0', 'resnet_identity_block_1/batch_normalization_4/moving_variance:0', 'resnet_identity_block_1/batch_normalization_5/moving_mean:0', 'resnet_identity_block_1/batch_normalization_5/moving_variance:0']\n" - ] - } - ], - "source": [ - "class ResnetIdentityBlock(tf.keras.Model):\n", - " def __init__(self, kernel_size, filters):\n", - " super(ResnetIdentityBlock, self).__init__(name='')\n", - " filters1, filters2, filters3 = filters\n", - "\n", - " self.conv2a = tf.keras.layers.Conv2D(filters1, (1, 1))\n", - " self.bn2a = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2b = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same')\n", - " self.bn2b = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2c = tf.keras.layers.Conv2D(filters3, (1, 1))\n", - " self.bn2c = tf.keras.layers.BatchNormalization()\n", - "\n", - " def call(self, input_tensor, training=False):\n", - " x = self.conv2a(input_tensor)\n", - " x = self.bn2a(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2b(x)\n", - " x = self.bn2b(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2c(x)\n", - " x = self.bn2c(x, training=training)\n", - "\n", - " x += input_tensor\n", - " return tf.nn.relu(x)\n", - "\n", - " \n", - "block = ResnetIdentityBlock(1, [1, 2, 3])\n", - "print(block(tf.zeros([1, 2, 3, 3])))\n", - "print([x.name for x in block.variables])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wYfucVw65PMj" - }, - "source": [ - "Much of the time, however, models which compose many layers simply call one layer after the other. This can be done in very little code using tf.keras.Sequential" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "base_uri": "https://localhost:8080/", - "height": 153 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 361, - "status": "ok", - "timestamp": 1526674830777, - "user": { - "displayName": "Alexandre Passos", - "photoUrl": "//lh4.googleusercontent.com/-kmTTWXEgAPw/AAAAAAAAAAI/AAAAAAAAAC0/q_DoOzKGwds/s50-c-k-no/photo.jpg", - "userId": "108023195365833072773" - }, - "user_tz": 420 - }, - "id": "L9frk7Ur4uvJ", - "outputId": "882e9076-b6d9-4380-bb1e-7c6b57d54c39" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "\u003ctf.Tensor: id=1423, shape=(1, 2, 3, 3), dtype=float32, numpy=\n", - "array([[[[0., 0., 0.],\n", - " [0., 0., 0.],\n", - " [0., 0., 0.]],\n", - "\n", - " [[0., 0., 0.],\n", - " [0., 0., 0.],\n", - " [0., 0., 0.]]]], dtype=float32)\u003e" - ] - }, - "execution_count": 26, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - " my_seq = tf.keras.Sequential([tf.keras.layers.Conv2D(1, (1, 1)),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(2, 1, \n", - " padding='same'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(3, (1, 1)),\n", - " tf.keras.layers.BatchNormalization()])\n", - "my_seq(tf.zeros([1, 2, 3, 3]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c5YwYcnuK-wc" - }, - "source": [ - "# Next steps\n", - "\n", - "Now you can go back to the previous notebook and adapt the linear regression example to use layers and models to be better structured." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "default_view": {}, - "name": "4 - High level API - TensorFlow Eager.ipynb", - "provenance": [], - "version": "0.3.2", - "views": {} - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/README.md b/tensorflow/contrib/eager/python/examples/notebooks/README.md new file mode 100644 index 0000000000..0d5ed84894 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/README.md @@ -0,0 +1,11 @@ +## Research and experimentation + +Eager execution provides an imperative, define-by-run interface for advanced +operations. Write custom layers, forward passes, and training loops with auto +differentiation. Start with these notebooks, then read the +[eager execution guide](https://www.tensorflow.org/guide/eager). + +1. [Eager execution basics](./eager_basics.ipynb) +2. [Automatic differentiation and gradient tapes](./automatic_differentiation.ipynb) +3. [Custom training: basics](./custom_training.ipynb) +4. [Custom layers](./custom_layers.ipynb) diff --git a/tensorflow/contrib/eager/python/examples/notebooks/automatic_differentiation.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/automatic_differentiation.ipynb new file mode 100644 index 0000000000..a18882fafa --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/automatic_differentiation.ipynb @@ -0,0 +1,364 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "automatic_differentiation.ipynb", + "version": "0.3.2", + "views": {}, + "default_view": {}, + "provenance": [], + "private_outputs": true, + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "metadata": { + "id": "t09eeeR5prIJ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "##### Copyright 2018 The TensorFlow Authors." + ] + }, + { + "metadata": { + "id": "GCCk8_dHpuNf", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "form" + }, + "cell_type": "code", + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "xh8WkEwWpnm7", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Automatic differentiation and gradient tape" + ] + }, + { + "metadata": { + "id": "idv0bPeCp325", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "
\n", + "\n", + " Run in Google Colab\n", + "\n", + "View source on GitHub
" + ] + }, + { + "metadata": { + "id": "vDJ4XzMqodTy", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "In the previous tutorial we introduced `Tensor`s and operations on them. In this tutorial we will cover [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation), a key technique for optimizing machine learning models." + ] + }, + { + "metadata": { + "id": "GQJysDM__Qb0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup\n" + ] + }, + { + "metadata": { + "id": "OiMPZStlibBv", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "import tensorflow as tf\n", + "tf.enable_eager_execution()\n", + "\n", + "tfe = tf.contrib.eager # Shorthand for some symbols" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "1CLWJl0QliB0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Derivatives of a function\n", + "\n", + "TensorFlow provides APIs for automatic differentiation - computing the derivative of a function. The way that more closely mimics the math is to encapsulate the computation in a Python function, say `f`, and use `tfe.gradients_function` to create a function that computes the derivatives of `f` with respect to its arguments. If you're familiar with [autograd](https://github.com/HIPS/autograd) for differentiating numpy functions, this will be familiar. For example: " + ] + }, + { + "metadata": { + "id": "9FViq92UX7P8", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "from math import pi\n", + "\n", + "def f(x):\n", + " return tf.square(tf.sin(x))\n", + "\n", + "assert f(pi/2).numpy() == 1.0\n", + "\n", + "\n", + "# grad_f will return a list of derivatives of f\n", + "# with respect to its arguments. Since f() has a single argument,\n", + "# grad_f will return a list with a single element.\n", + "grad_f = tfe.gradients_function(f)\n", + "assert tf.abs(grad_f(pi/2)[0]).numpy() < 1e-7" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "v9fPs8RyopCf", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Higher-order gradients\n", + "\n", + "The same API can be used to differentiate as many times as you like:\n" + ] + }, + { + "metadata": { + "id": "3D0ZvnGYo0rW", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def f(x):\n", + " return tf.square(tf.sin(x))\n", + "\n", + "def grad(f):\n", + " return lambda x: tfe.gradients_function(f)(x)[0]\n", + "\n", + "x = tf.lin_space(-2*pi, 2*pi, 100) # 100 points between -2π and +2π\n", + "\n", + "import matplotlib.pyplot as plt\n", + "\n", + "plt.plot(x, f(x), label=\"f\")\n", + "plt.plot(x, grad(f)(x), label=\"first derivative\")\n", + "plt.plot(x, grad(grad(f))(x), label=\"second derivative\")\n", + "plt.plot(x, grad(grad(grad(f)))(x), label=\"third derivative\")\n", + "plt.legend()\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "-39gouo7mtgu", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Gradient tapes\n", + "\n", + "Every differentiable TensorFlow operation has an associated gradient function. For example, the gradient function of `tf.square(x)` would be a function that returns `2.0 * x`. To compute the gradient of a user-defined function (like `f(x)` in the example above), TensorFlow first \"records\" all the operations applied to compute the output of the function. We call this record a \"tape\". It then uses that tape and the gradients functions associated with each primitive operation to compute the gradients of the user-defined function using [reverse mode differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation).\n", + "\n", + "Since operations are recorded as they are executed, Python control flow (using `if`s and `while`s for example) is naturally handled:\n", + "\n" + ] + }, + { + "metadata": { + "id": "MH0UfjympWf7", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def f(x, y):\n", + " output = 1\n", + " for i in range(y):\n", + " output = tf.multiply(output, x)\n", + " return output\n", + "\n", + "def g(x, y):\n", + " # Return the gradient of `f` with respect to it's first parameter\n", + " return tfe.gradients_function(f)(x, y)[0]\n", + "\n", + "assert f(3.0, 2).numpy() == 9.0 # f(x, 2) is essentially x * x\n", + "assert g(3.0, 2).numpy() == 6.0 # And its gradient will be 2 * x\n", + "assert f(4.0, 3).numpy() == 64.0 # f(x, 3) is essentially x * x * x\n", + "assert g(4.0, 3).numpy() == 48.0 # And its gradient will be 3 * x * x" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "aNmR5-jhpX2t", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "At times it may be inconvenient to encapsulate computation of interest into a function. For example, if you want the gradient of the output with respect to intermediate values computed in the function. In such cases, the slightly more verbose but explicit [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) context is useful. All computation inside the context of a `tf.GradientTape` is \"recorded\".\n", + "\n", + "For example:" + ] + }, + { + "metadata": { + "id": "bAFeIE8EuVIq", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "x = tf.ones((2, 2))\n", + " \n", + "# TODO(b/78880779): Remove the 'persistent=True' argument and use\n", + "# a single t.gradient() call when the bug is resolved.\n", + "with tf.GradientTape(persistent=True) as t:\n", + " # TODO(ashankar): Explain with \"watch\" argument better?\n", + " t.watch(x)\n", + " y = tf.reduce_sum(x)\n", + " z = tf.multiply(y, y)\n", + "\n", + "# Use the same tape to compute the derivative of z with respect to the\n", + "# intermediate value y.\n", + "dz_dy = t.gradient(z, y)\n", + "assert dz_dy.numpy() == 8.0\n", + "\n", + "# Derivative of z with respect to the original input tensor x\n", + "dz_dx = t.gradient(z, x)\n", + "for i in [0, 1]:\n", + " for j in [0, 1]:\n", + " assert dz_dx[i][j].numpy() == 8.0" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "DK05KXrAAld3", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Higher-order gradients\n", + "\n", + "Operations inside of the `GradientTape` context manager are recorded for automatic differentiation. If gradients are computed in that context, then the gradient computation is recorded as well. As a result, the exact same API works for higher-order gradients as well. For example:" + ] + }, + { + "metadata": { + "id": "cPQgthZ7ugRJ", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# TODO(ashankar): Should we use the persistent tape here instead? Follow up on Tom and Alex's discussion\n", + "\n", + "x = tf.constant(1.0) # Convert the Python 1.0 to a Tensor object\n", + "\n", + "with tf.GradientTape() as t:\n", + " with tf.GradientTape() as t2:\n", + " t2.watch(x)\n", + " y = x * x * x\n", + " # Compute the gradient inside the 't' context manager\n", + " # which means the gradient computation is differentiable as well.\n", + " dy_dx = t2.gradient(y, x)\n", + "d2y_dx2 = t.gradient(dy_dx, x)\n", + "\n", + "assert dy_dx.numpy() == 3.0\n", + "assert d2y_dx2.numpy() == 6.0" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "4U1KKzUpNl58", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Next Steps\n", + "\n", + "In this tutorial we covered gradient computation in TensorFlow. With that we have enough of the primitives required to build an train neural networks, which we will cover in the [next tutorial](https://github.com/tensorflow/models/tree/master/official/contrib/eager/python/examples/notebooks/3_neural_networks.ipynb)." + ] + } + ] +} \ No newline at end of file diff --git a/tensorflow/contrib/eager/python/examples/notebooks/custom_layers.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/custom_layers.ipynb new file mode 100644 index 0000000000..54fbf2a7e1 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/custom_layers.ipynb @@ -0,0 +1,399 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "custom_layers.ipynb", + "version": "0.3.2", + "views": {}, + "default_view": {}, + "provenance": [], + "private_outputs": true, + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "cells": [ + { + "metadata": { + "id": "tDnwEv8FtJm7", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "##### Copyright 2018 The TensorFlow Authors." + ] + }, + { + "metadata": { + "id": "JlknJBWQtKkI", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "form" + }, + "cell_type": "code", + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "60RdWsg1tETW", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Custom layers" + ] + }, + { + "metadata": { + "id": "BcJg7Enms86w", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "
\n", + "\n", + " Run in Google Colab\n", + "\n", + "View source on GitHub
" + ] + }, + { + "metadata": { + "id": "UEu3q4jmpKVT", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "We recommend using `tf.keras` as a high-level API for building neural networks. That said, most TensorFlow APIs are usable with eager execution.\n" + ] + }, + { + "metadata": { + "id": "pwX7Fii1rwsJ", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "import tensorflow as tf\n", + "tfe = tf.contrib.eager\n", + "\n", + "tf.enable_eager_execution()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "zSFfVVjkrrsI", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Layers: common sets of useful operations\n", + "\n", + "Most of the time when writing code for machine learning models you want to operate at a higher level of abstraction than individual operations and manipulation of individual variables.\n", + "\n", + "Many machine learning models are expressible as the composition and stacking of relatively simple layers, and TensorFlow provides both a set of many common layers as a well as easy ways for you to write your own application-specific layers either from scratch or as the composition of existing layers.\n", + "\n", + "TensorFlow includes the full [Keras](https://keras.io) API in the tf.keras package, and the Keras layers are very useful when building your own models.\n" + ] + }, + { + "metadata": { + "id": "8PyXlPl-4TzQ", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# In the tf.keras.layers package, layers are objects. To construct a layer,\n", + "# simply construct the object. Most layers take as a first argument the number\n", + "# of output dimensions / channels.\n", + "layer = tf.keras.layers.Dense(100)\n", + "# The number of input dimensions is often unnecessary, as it can be inferred\n", + "# the first time the layer is used, but it can be provided if you want to \n", + "# specify it manually, which is useful in some complex models.\n", + "layer = tf.keras.layers.Dense(10, input_shape=(None, 5))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "Fn69xxPO5Psr", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The full list of pre-existing layers can be seen in [the documentation](https://www.tensorflow.org/api_docs/python/tf/keras/layers). It includes Dense (a fully-connected layer),\n", + "Conv2D, LSTM, BatchNormalization, Dropout, and many others." + ] + }, + { + "metadata": { + "id": "E3XKNknP5Mhb", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# To use a layer, simply call it.\n", + "layer(tf.zeros([10, 5]))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "Wt_Nsv-L5t2s", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Layers have many useful methods. For example, you can inspect all variables\n", + "# in a layer by calling layer.variables. In this case a fully-connected layer\n", + "# will have variables for weights and biases.\n", + "layer.variables" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "6ilvKjz8_4MQ", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# The variables are also accessible through nice accessors\n", + "layer.kernel, layer.bias" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "O0kDbE54-5VS", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Implementing custom layers\n", + "The best way to implement your own layer is extending the tf.keras.Layer class and implementing:\n", + " * `__init__` , where you can do all input-independent initialization\n", + " * `build`, where you know the shapes of the input tensors and can do the rest of the initialization\n", + " * `call`, where you do the forward computation\n", + "\n", + "Note that you don't have to wait until `build` is called to create your variables, you can also create them in `__init__`. However, the advantage of creating them in `build` is that it enables late variable creation based on the shape of the inputs the layer will operate on. On the other hand, creating variables in `__init__` would mean that shapes required to create the variables will need to be explicitly specified." + ] + }, + { + "metadata": { + "id": "5Byl3n1k5kIy", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "class MyDenseLayer(tf.keras.layers.Layer):\n", + " def __init__(self, num_outputs):\n", + " super(MyDenseLayer, self).__init__()\n", + " self.num_outputs = num_outputs\n", + " \n", + " def build(self, input_shape):\n", + " self.kernel = self.add_variable(\"kernel\", \n", + " shape=[input_shape[-1].value, \n", + " self.num_outputs])\n", + " \n", + " def call(self, input):\n", + " return tf.matmul(input, self.kernel)\n", + " \n", + "layer = MyDenseLayer(10)\n", + "print(layer(tf.zeros([10, 5])))\n", + "print(layer.variables)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "tk8E2vY0-z4Z", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Note that you don't have to wait until `build` is called to create your variables, you can also create them in `__init__`.\n", + "\n", + "Overall code is easier to read and maintain if it uses standard layers whenever possible, as other readers will be familiar with the behavior of standard layers. If you want to use a layer which is not present in tf.keras.layers or tf.contrib.layers, consider filing a [github issue](http://github.com/tensorflow/tensorflow/issues/new) or, even better, sending us a pull request!" + ] + }, + { + "metadata": { + "id": "Qhg4KlbKrs3G", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Models: composing layers\n", + "\n", + "Many interesting layer-like things in machine learning models are implemented by composing existing layers. For example, each residual block in a resnet is a composition of convolutions, batch normalizations, and a shortcut.\n", + "\n", + "The main class used when creating a layer-like thing which contains other layers is tf.keras.Model. Implementing one is done by inheriting from tf.keras.Model." + ] + }, + { + "metadata": { + "id": "N30DTXiRASlb", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "class ResnetIdentityBlock(tf.keras.Model):\n", + " def __init__(self, kernel_size, filters):\n", + " super(ResnetIdentityBlock, self).__init__(name='')\n", + " filters1, filters2, filters3 = filters\n", + "\n", + " self.conv2a = tf.keras.layers.Conv2D(filters1, (1, 1))\n", + " self.bn2a = tf.keras.layers.BatchNormalization()\n", + "\n", + " self.conv2b = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same')\n", + " self.bn2b = tf.keras.layers.BatchNormalization()\n", + "\n", + " self.conv2c = tf.keras.layers.Conv2D(filters3, (1, 1))\n", + " self.bn2c = tf.keras.layers.BatchNormalization()\n", + "\n", + " def call(self, input_tensor, training=False):\n", + " x = self.conv2a(input_tensor)\n", + " x = self.bn2a(x, training=training)\n", + " x = tf.nn.relu(x)\n", + "\n", + " x = self.conv2b(x)\n", + " x = self.bn2b(x, training=training)\n", + " x = tf.nn.relu(x)\n", + "\n", + " x = self.conv2c(x)\n", + " x = self.bn2c(x, training=training)\n", + "\n", + " x += input_tensor\n", + " return tf.nn.relu(x)\n", + "\n", + " \n", + "block = ResnetIdentityBlock(1, [1, 2, 3])\n", + "print(block(tf.zeros([1, 2, 3, 3])))\n", + "print([x.name for x in block.variables])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "wYfucVw65PMj", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Much of the time, however, models which compose many layers simply call one layer after the other. This can be done in very little code using tf.keras.Sequential" + ] + }, + { + "metadata": { + "id": "L9frk7Ur4uvJ", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + " my_seq = tf.keras.Sequential([tf.keras.layers.Conv2D(1, (1, 1)),\n", + " tf.keras.layers.BatchNormalization(),\n", + " tf.keras.layers.Conv2D(2, 1, \n", + " padding='same'),\n", + " tf.keras.layers.BatchNormalization(),\n", + " tf.keras.layers.Conv2D(3, (1, 1)),\n", + " tf.keras.layers.BatchNormalization()])\n", + "my_seq(tf.zeros([1, 2, 3, 3]))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "c5YwYcnuK-wc", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Next steps\n", + "\n", + "Now you can go back to the previous notebook and adapt the linear regression example to use layers and models to be better structured." + ] + } + ] +} \ No newline at end of file diff --git a/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb new file mode 100644 index 0000000000..0a781d2153 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb @@ -0,0 +1,478 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Custom training: basics", + "version": "0.3.2", + "views": {}, + "default_view": {}, + "provenance": [], + "private_outputs": true, + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "metadata": { + "id": "5rmpybwysXGV", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "##### Copyright 2018 The TensorFlow Authors." + ] + }, + { + "metadata": { + "id": "m8y3rGtQsYP2", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "form" + }, + "cell_type": "code", + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "hrXv0rU9sIma", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Custom training: basics" + ] + }, + { + "metadata": { + "id": "7S0BwJ_8sLu7", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "
\n", + "\n", + " Run in Google Colab\n", + "\n", + "View source on GitHub
" + ] + }, + { + "metadata": { + "id": "k2o3TTG4TFpt", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "In the previous tutorial we covered the TensorFlow APIs for automatic differentiation, a basic building block for machine learning.\n", + "In this tutorial we will use the TensorFlow primitives introduced in the prior tutorials to do some simple machine learning.\n", + "\n", + "TensorFlow also includes a higher-level neural networks API (`tf.keras`) which provides useful abstractions to reduce boilerplate. We strongly recommend those higher level APIs for people working with neural networks. However, in this short tutorial we cover neural network training from first principles to establish a strong foundation." + ] + }, + { + "metadata": { + "id": "3LXMVuV0VhDr", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup" + ] + }, + { + "metadata": { + "id": "PJ64L90aVir3", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "import tensorflow as tf\n", + "tfe = tf.contrib.eager # Shorthand for some symbols\n", + "\n", + "tf.enable_eager_execution()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "eMAWbDJFVmMk", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Variables\n", + "\n", + "Tensors in TensorFlow are immutable stateless objects. Machine learning models, however, need to have changing state: as your model trains, the same code to compute predictions should behave differently over time (hopefully with a lower loss!). To represent this state which needs to change over the course of your computation, you can choose to rely on the fact that Python is a stateful programming language:\n" + ] + }, + { + "metadata": { + "id": "VkJwtLS_Jbn8", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "# Using python state\n", + "x = tf.zeros([10, 10])\n", + "x += 2 # This is equivalent to x = x + 2, which does not mutate the original\n", + " # value of x\n", + "print(x)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "wfneTXy7JcUz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "TensorFlow, however, has stateful operations built in, and these are often more pleasant to use than low-level Python representations of your state. To represent weights in a model, for example, it's often convenient and efficient to use TensorFlow variables.\n", + "\n", + "A Variable is an object which stores a value and, when used in a TensorFlow computation, will implicitly read from this stored value. There are operations (`tf.assign_sub`, `tf.scatter_update`, etc) which manipulate the value stored in a TensorFlow variable." + ] + }, + { + "metadata": { + "id": "itxmrMil6DQi", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "v = tfe.Variable(1.0)\n", + "assert v.numpy() == 1.0\n", + "\n", + "# Re-assign the value\n", + "v.assign(3.0)\n", + "assert v.numpy() == 3.0\n", + "\n", + "# Use `v` in a TensorFlow operation like tf.square() and reassign\n", + "v.assign(tf.square(v))\n", + "assert v.numpy() == 9.0" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "-paSaeq1JzwC", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Computations using Variables are automatically traced when computing gradients. For Variables representing embeddings TensorFlow will do sparse updates by default, which are more computation and memory efficient.\n", + "\n", + "Using Variables is also a way to quickly let a reader of your code know that this piece of state is mutable." + ] + }, + { + "metadata": { + "id": "BMiFcDzE7Qu3", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Example: Fitting a linear model\n", + "\n", + "Let's now put the few concepts we have so far ---`Tensor`, `GradientTape`, `Variable` --- to build and train a simple model. This typically involves a few steps:\n", + "\n", + "1. Define the model.\n", + "2. Define a loss function.\n", + "3. Obtain training data.\n", + "4. Run through the training data and use an \"optimizer\" to adjust the variables to fit the data.\n", + "\n", + "In this tutorial, we'll walk through a trivial example of a simple linear model: `f(x) = x * W + b`, which has two variables - `W` and `b`. Furthermore, we'll synthesize data such that a well trained model would have `W = 3.0` and `b = 2.0`." + ] + }, + { + "metadata": { + "id": "gFzH64Jn9PIm", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Define the model\n", + "\n", + "Let's define a simple class to encapsulate the variables and the computation." + ] + }, + { + "metadata": { + "id": "_WRu7Pze7wk8", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "class Model(object):\n", + " def __init__(self):\n", + " # Initialize variable to (5.0, 0.0)\n", + " # In practice, these should be initialized to random values.\n", + " self.W = tfe.Variable(5.0)\n", + " self.b = tfe.Variable(0.0)\n", + " \n", + " def __call__(self, x):\n", + " return self.W * x + self.b\n", + " \n", + "model = Model()\n", + "\n", + "assert model(3.0).numpy() == 15.0" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "xa6j_yXa-j79", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Define a loss function\n", + "\n", + "A loss function measures how well the output of a model for a given input matches the desired output. Let's use the standard L2 loss." + ] + }, + { + "metadata": { + "id": "Y0ysUFGY924U", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def loss(predicted_y, desired_y):\n", + " return tf.reduce_mean(tf.square(predicted_y - desired_y))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "qutT_fkl_CBc", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Obtain training data\n", + "\n", + "Let's synthesize the training data with some noise." + ] + }, + { + "metadata": { + "id": "gxPTb-kt_N5m", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "TRUE_W = 3.0\n", + "TRUE_b = 2.0\n", + "NUM_EXAMPLES = 1000\n", + "\n", + "inputs = tf.random_normal(shape=[NUM_EXAMPLES])\n", + "noise = tf.random_normal(shape=[NUM_EXAMPLES])\n", + "outputs = inputs * TRUE_W + TRUE_b + noise" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "-50nq-wPBsAW", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Before we train the model let's visualize where the model stands right now. We'll plot the model's predictions in red and the training data in blue." + ] + }, + { + "metadata": { + "id": "_eb83LtrB4nt", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.scatter(inputs, outputs, c='b')\n", + "plt.scatter(inputs, model(inputs), c='r')\n", + "plt.show()\n", + "\n", + "print('Current loss: '),\n", + "print(loss(model(inputs), outputs).numpy())" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "sSDP-yeq_4jE", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Define a training loop\n", + "\n", + "We now have our network and our training data. Let's train it, i.e., use the training data to update the model's variables (`W` and `b`) so that the loss goes down using [gradient descent](https://en.wikipedia.org/wiki/Gradient_descent). There are many variants of the gradient descent scheme that are captured in `tf.train.Optimizer` implementations. We'd highly recommend using those implementations, but in the spirit of building from first principles, in this particular example we will implement the basic math ourselves." + ] + }, + { + "metadata": { + "id": "MBIACgdnA55X", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def train(model, inputs, outputs, learning_rate):\n", + " with tf.GradientTape() as t:\n", + " current_loss = loss(model(inputs), outputs)\n", + " dW, db = t.gradient(current_loss, [model.W, model.b])\n", + " model.W.assign_sub(learning_rate * dW)\n", + " model.b.assign_sub(learning_rate * db)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "RwWPaJryD2aN", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Finally, let's repeatedly run through the training data and see how `W` and `b` evolve." + ] + }, + { + "metadata": { + "id": "XdfkR223D9dW", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "model = Model()\n", + "\n", + "# Collect the history of W-values and b-values to plot later\n", + "Ws, bs = [], []\n", + "epochs = range(10)\n", + "for epoch in epochs:\n", + " Ws.append(model.W.numpy())\n", + " bs.append(model.b.numpy())\n", + " current_loss = loss(model(inputs), outputs)\n", + "\n", + " train(model, inputs, outputs, learning_rate=0.1)\n", + " print('Epoch %2d: W=%1.2f b=%1.2f, loss=%2.5f' %\n", + " (epoch, Ws[-1], bs[-1], current_loss))\n", + "\n", + "# Let's plot it all\n", + "plt.plot(epochs, Ws, 'r',\n", + " epochs, bs, 'b')\n", + "plt.plot([TRUE_W] * len(epochs), 'r--',\n", + " [TRUE_b] * len(epochs), 'b--')\n", + "plt.legend(['W', 'b', 'true W', 'true_b'])\n", + "plt.show()\n", + " " + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vPnIVuaSJwWz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Next Steps\n", + "\n", + "In this tutorial we covered `Variable`s and built and trained a simple linear model using the TensorFlow primitives discussed so far.\n", + "\n", + "In theory, this is pretty much all you need to use TensorFlow for your machine learning research.\n", + "In practice, particularly for neural networks, the higher level APIs like `tf.keras` will be much more convenient since it provides higher level building blocks (called \"layers\"), utilities to save and restore state, a suite of loss functions, a suite of optimization strategies etc. \n", + "\n", + "The [next tutorial](TODO) will cover these higher level APIs." + ] + } + ] +} \ No newline at end of file diff --git a/tensorflow/contrib/eager/python/examples/notebooks/eager_basics.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/eager_basics.ipynb new file mode 100644 index 0000000000..b37a18c9a6 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/eager_basics.ipynb @@ -0,0 +1,491 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "eager_basics.ipynb", + "version": "0.3.2", + "views": {}, + "default_view": {}, + "provenance": [], + "private_outputs": true, + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "metadata": { + "id": "iPpI7RaYoZuE", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "##### Copyright 2018 The TensorFlow Authors." + ] + }, + { + "metadata": { + "id": "hro2InpHobKk", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "form" + }, + "cell_type": "code", + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "U9i2Dsh-ziXr", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Eager execution basics" + ] + }, + { + "metadata": { + "id": "Hndw-YcxoOJK", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "
\n", + "\n", + " Run in Google Colab\n", + "\n", + "View source on GitHub
" + ] + }, + { + "metadata": { + "id": "6sILUVbHoSgH", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "This is an introductory tutorial for using TensorFlow. It will cover:\n", + "\n", + "* Importing required packages\n", + "* Creating and using Tensors\n", + "* Using GPU acceleration\n", + "* Datasets" + ] + }, + { + "metadata": { + "id": "z1JcS5iBXMRO", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Import TensorFlow\n", + "\n", + "To get started, import the `tensorflow` module and enable eager execution.\n", + "Eager execution enables a more interactive frontend to TensorFlow, the details of which we will discuss much later." + ] + }, + { + "metadata": { + "id": "RlIWhyeLoYnG", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "code" + }, + "cell_type": "code", + "source": [ + "import tensorflow as tf\n", + "\n", + "tf.enable_eager_execution()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "H9UySOPLXdaw", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Tensors\n", + "\n", + "A Tensor is a multi-dimensional array. Similar to NumPy `ndarray` objects, `Tensor` objects have a data type and a shape. Additionally, Tensors can reside in accelerator (like GPU) memory. TensorFlow offers a rich library of operations ([tf.add](https://www.tensorflow.org/api_docs/python/tf/add), [tf.matmul](https://www.tensorflow.org/api_docs/python/tf/matmul), [tf.linalg.inv](https://www.tensorflow.org/api_docs/python/tf/linalg/inv) etc.) that consume and produce Tensors. These operations automatically convert native Python types. For example:\n" + ] + }, + { + "metadata": { + "id": "ngUe237Wt48W", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "code" + }, + "cell_type": "code", + "source": [ + "print(tf.add(1, 2))\n", + "print(tf.add([1, 2], [3, 4]))\n", + "print(tf.square(5))\n", + "print(tf.reduce_sum([1, 2, 3]))\n", + "print(tf.encode_base64(\"hello world\"))\n", + "\n", + "# Operator overloading is also supported\n", + "print(tf.square(2) + tf.square(3))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "IDY4WsYRhP81", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Each Tensor has a shape and a datatype" + ] + }, + { + "metadata": { + "id": "srYWH1MdJNG7", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "x = tf.matmul([[1]], [[2, 3]])\n", + "print(x.shape)\n", + "print(x.dtype)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "eBPw8e8vrsom", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The most obvious differences between NumPy arrays and TensorFlow Tensors are:\n", + "\n", + "1. Tensors can be backed by accelerator memory (like GPU, TPU).\n", + "2. Tensors are immutable." + ] + }, + { + "metadata": { + "id": "Dwi1tdW3JBw6", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### NumPy Compatibility\n", + "\n", + "Conversion between TensorFlow Tensors and NumPy ndarrays is quite simple as:\n", + "* TensorFlow operations automatically convert NumPy ndarrays to Tensors.\n", + "* NumPy operations automatically convert Tensors to NumPy ndarrays.\n", + "\n", + "Tensors can be explicitly converted to NumPy ndarrays by invoking the `.numpy()` method on them.\n", + "These conversions are typically cheap as the array and Tensor share the underlying memory representation if possible. However, sharing the underlying representation isn't always possible since the Tensor may be hosted in GPU memory while NumPy arrays are always backed by host memory, and the conversion will thus involve a copy from GPU to host memory." + ] + }, + { + "metadata": { + "id": "lCUWzso6mbqR", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "import numpy as np\n", + "\n", + "ndarray = np.ones([3, 3])\n", + "\n", + "print(\"TensorFlow operations convert numpy arrays to Tensors automatically\")\n", + "tensor = tf.multiply(ndarray, 42)\n", + "print(tensor)\n", + "\n", + "\n", + "print(\"And NumPy operations convert Tensors to numpy arrays automatically\")\n", + "print(np.add(tensor, 1))\n", + "\n", + "print(\"The .numpy() method explicitly converts a Tensor to a numpy array\")\n", + "print(tensor.numpy())" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "PBNP8yTRfu_X", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## GPU acceleration\n", + "\n", + "Many TensorFlow operations can be accelerated by using the GPU for computation. Without any annotations, TensorFlow automatically decides whether to use the GPU or CPU for an operation (and copies the tensor between CPU and GPU memory if necessary). Tensors produced by an operation are typically backed by the memory of the device on which the operation executed. For example:" + ] + }, + { + "metadata": { + "id": "3Twf_Rw-gQFM", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "cellView": "code" + }, + "cell_type": "code", + "source": [ + "x = tf.random_uniform([3, 3])\n", + "\n", + "print(\"Is there a GPU available: \"),\n", + "print(tf.test.is_gpu_available())\n", + "\n", + "print(\"Is the Tensor on GPU #0: \"),\n", + "print(x.device.endswith('GPU:0'))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vpgYzgVXW2Ud", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Device Names\n", + "\n", + "The `Tensor.device` property provides a fully qualified string name of the device hosting the contents of the Tensor. This name encodes a bunch of details, such as an identifier of the network address of the host on which this program is executing and the device within that host. This is required for distributed execution of TensorFlow programs, but we'll skip that for now. The string will end with `GPU:` if the tensor is placed on the `N`-th tensor on the host." + ] + }, + { + "metadata": { + "id": "ZWZQCimzuqyP", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + "\n", + "### Explicit Device Placement\n", + "\n", + "The term \"placement\" in TensorFlow refers to how individual operations are assigned (placed on) a device for execution. As mentioned above, when there is no explicit guidance provided, TensorFlow automatically decides which device to execute an operation, and copies Tensors to that device if needed. However, TensorFlow operations can be explicitly placed on specific devices using the `tf.device` context manager. For example:" + ] + }, + { + "metadata": { + "id": "RjkNZTuauy-Q", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "def time_matmul(x):\n", + " %timeit tf.matmul(x, x)\n", + "\n", + "# Force execution on CPU\n", + "print(\"On CPU:\")\n", + "with tf.device(\"CPU:0\"):\n", + " x = tf.random_uniform([1000, 1000])\n", + " assert x.device.endswith(\"CPU:0\")\n", + " time_matmul(x)\n", + "\n", + "# Force execution on GPU #0 if available\n", + "if tf.test.is_gpu_available():\n", + " with tf.device(\"GPU:0\"): # Or GPU:1 for the 2nd GPU, GPU:2 for the 3rd etc.\n", + " x = tf.random_uniform([1000, 1000])\n", + " assert x.device.endswith(\"GPU:0\")\n", + " time_matmul(x)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "o1K4dlhhHtQj", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Datasets\n", + "\n", + "This section demonstrates the use of the [`tf.data.Dataset` API](https://www.tensorflow.org/guide/datasets) to build pipelines to feed data to your model. It covers:\n", + "\n", + "* Creating a `Dataset`.\n", + "* Iteration over a `Dataset` with eager execution enabled.\n", + "\n", + "We recommend using the `Dataset`s API for building performant, complex input pipelines from simple, re-usable pieces that will feed your model's training or evaluation loops.\n", + "\n", + "If you're familiar with TensorFlow graphs, the API for constructing the `Dataset` object remains exactly the same when eager execution is enabled, but the process of iterating over elements of the dataset is slightly simpler.\n", + "You can use Python iteration over the `tf.data.Dataset` object and do not need to explicitly create an `tf.data.Iterator` object.\n", + "As a result, the discussion on iterators in the [TensorFlow Guide](https://www.tensorflow.org/guide/datasets) is not relevant when eager execution is enabled." + ] + }, + { + "metadata": { + "id": "zI0fmOynH-Ne", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Create a source `Dataset`\n", + "\n", + "Create a _source_ dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensor_slices) or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset). See the [TensorFlow Guide](https://www.tensorflow.org/guide/datasets#reading_input_data) for more information." + ] + }, + { + "metadata": { + "id": "F04fVOHQIBiG", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "ds_tensors = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n", + "\n", + "# Create a CSV file\n", + "import tempfile\n", + "_, filename = tempfile.mkstemp()\n", + "\n", + "with open(filename, 'w') as f:\n", + " f.write(\"\"\"Line 1\n", + "Line 2\n", + "Line 3\n", + " \"\"\")\n", + "\n", + "ds_file = tf.data.TextLineDataset(filename)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vbxIhC-5IPdf", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Apply transformations\n", + "\n", + "Use the transformations functions like [`map`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#map), [`batch`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch), [`shuffle`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shuffle) etc. to apply transformations to the records of the dataset. See the [API documentation for `tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) for details." + ] + }, + { + "metadata": { + "id": "uXSDZWE-ISsd", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", + "\n", + "ds_file = ds_file.batch(2)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "A8X1GNfoIZKJ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Iterate\n", + "\n", + "When eager execution is enabled `Dataset` objects support iteration.\n", + "If you're familiar with the use of `Dataset`s in TensorFlow graphs, note that there is no need for calls to `Dataset.make_one_shot_iterator()` or `get_next()` calls." + ] + }, + { + "metadata": { + "id": "ws-WKRk5Ic6-", + "colab_type": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + } + }, + "cell_type": "code", + "source": [ + "print('Elements of ds_tensors:')\n", + "for x in ds_tensors:\n", + " print(x)\n", + "\n", + "print('\\nElements in ds_file:')\n", + "for x in ds_file:\n", + " print(x)" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file -- cgit v1.2.3 From 2b13b7ac7253e6f0d7d96855b1b3e7fee49277a7 Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Tue, 3 Jul 2018 16:40:14 -0700 Subject: Update docs_src in 1.9 to match master --- tensorflow/docs_src/community/leftnav_files | 1 - tensorflow/docs_src/community/swift.md | 60 -- tensorflow/docs_src/get_started/_index.yaml | 249 ------- .../docs_src/get_started/basic_classification.md | 3 - .../docs_src/get_started/basic_regression.md | 3 - .../get_started/basic_text_classification.md | 3 - tensorflow/docs_src/get_started/eager.md | 3 - tensorflow/docs_src/get_started/leftnav_files | 10 - tensorflow/docs_src/get_started/next_steps.md | 36 - .../docs_src/get_started/overfit_and_underfit.md | 3 - .../get_started/save_and_restore_models.md | 3 - tensorflow/docs_src/guide/custom_estimators.md | 8 +- .../docs_src/guide/datasets_for_estimators.md | 6 +- tensorflow/docs_src/guide/debugger.md | 30 +- tensorflow/docs_src/guide/eager.md | 12 +- tensorflow/docs_src/guide/graphs.md | 2 +- tensorflow/docs_src/guide/keras.md | 24 +- tensorflow/docs_src/guide/saved_model.md | 9 +- .../docs_src/guide/tensorboard_histograms.md | 4 +- tensorflow/docs_src/install/install_c.md | 2 +- tensorflow/docs_src/install/install_go.md | 2 +- tensorflow/docs_src/install/install_java.md | 24 +- tensorflow/docs_src/install/install_linux.md | 24 +- tensorflow/docs_src/install/install_mac.md | 13 +- tensorflow/docs_src/install/install_raspbian.md | 2 +- tensorflow/docs_src/install/install_sources.md | 36 +- tensorflow/docs_src/install/install_windows.md | 2 +- tensorflow/docs_src/mobile/leftnav_files | 1 + tensorflow/docs_src/mobile/linking_libs.md | 2 +- tensorflow/docs_src/mobile/mobile_intro.md | 3 +- tensorflow/docs_src/mobile/prepare_models.md | 4 +- tensorflow/docs_src/mobile/tflite/demo_android.md | 24 +- tensorflow/docs_src/mobile/tflite/devguide.md | 9 +- tensorflow/docs_src/mobile/tflite/index.md | 17 +- tensorflow/docs_src/mobile/tflite/performance.md | 174 +++++ tensorflow/docs_src/performance/quantization.md | 2 +- .../performance/xla/operation_semantics.md | 39 +- tensorflow/docs_src/tutorials/_index.yaml | 251 +++++++ tensorflow/docs_src/tutorials/_toc.yaml | 93 +++ tensorflow/docs_src/tutorials/audio_recognition.md | 631 ------------------ tensorflow/docs_src/tutorials/deep_cnn.md | 452 ------------- .../tutorials/eager/custom_training_walkthrough.md | 3 + tensorflow/docs_src/tutorials/eager/index.md | 13 + tensorflow/docs_src/tutorials/image_recognition.md | 456 ------------- tensorflow/docs_src/tutorials/image_retraining.md | 4 - tensorflow/docs_src/tutorials/images/deep_cnn.md | 446 +++++++++++++ .../docs_src/tutorials/images/image_recognition.md | 455 +++++++++++++ tensorflow/docs_src/tutorials/images/layers.md | 694 ++++++++++++++++++++ tensorflow/docs_src/tutorials/index.md | 59 -- .../tutorials/keras/basic_classification.md | 3 + .../docs_src/tutorials/keras/basic_regression.md | 3 + .../tutorials/keras/basic_text_classification.md | 3 + tensorflow/docs_src/tutorials/keras/index.md | 22 + .../tutorials/keras/overfit_and_underfit.md | 3 + .../tutorials/keras/save_and_restore_models.md | 3 + tensorflow/docs_src/tutorials/kernel_methods.md | 304 --------- tensorflow/docs_src/tutorials/layers.md | 727 --------------------- tensorflow/docs_src/tutorials/leftnav_files | 23 - tensorflow/docs_src/tutorials/linear.md | 237 ------- tensorflow/docs_src/tutorials/mandelbrot.md | 116 ---- tensorflow/docs_src/tutorials/next_steps.md | 36 + tensorflow/docs_src/tutorials/non-ml/mandelbrot.md | 116 ++++ tensorflow/docs_src/tutorials/non-ml/pdes.md | 140 ++++ tensorflow/docs_src/tutorials/pdes.md | 141 ---- tensorflow/docs_src/tutorials/recurrent.md | 232 ------- .../docs_src/tutorials/recurrent_quickdraw.md | 411 ------------ .../tutorials/representation/kernel_methods.md | 304 +++++++++ .../docs_src/tutorials/representation/linear.md | 237 +++++++ .../docs_src/tutorials/representation/wide.md | 461 +++++++++++++ .../tutorials/representation/wide_and_deep.md | 243 +++++++ .../docs_src/tutorials/representation/word2vec.md | 405 ++++++++++++ tensorflow/docs_src/tutorials/seq2seq.md | 5 - .../tutorials/sequences/audio_recognition.md | 631 ++++++++++++++++++ .../docs_src/tutorials/sequences/recurrent.md | 232 +++++++ .../tutorials/sequences/recurrent_quickdraw.md | 411 ++++++++++++ tensorflow/docs_src/tutorials/wide.md | 461 ------------- tensorflow/docs_src/tutorials/wide_and_deep.md | 243 ------- tensorflow/docs_src/tutorials/word2vec.md | 405 ------------ 78 files changed, 5561 insertions(+), 5403 deletions(-) delete mode 100644 tensorflow/docs_src/community/swift.md delete mode 100644 tensorflow/docs_src/get_started/_index.yaml delete mode 100644 tensorflow/docs_src/get_started/basic_classification.md delete mode 100644 tensorflow/docs_src/get_started/basic_regression.md delete mode 100644 tensorflow/docs_src/get_started/basic_text_classification.md delete mode 100644 tensorflow/docs_src/get_started/eager.md delete mode 100644 tensorflow/docs_src/get_started/leftnav_files delete mode 100644 tensorflow/docs_src/get_started/next_steps.md delete mode 100644 tensorflow/docs_src/get_started/overfit_and_underfit.md delete mode 100644 tensorflow/docs_src/get_started/save_and_restore_models.md create mode 100644 tensorflow/docs_src/mobile/tflite/performance.md create mode 100644 tensorflow/docs_src/tutorials/_index.yaml create mode 100644 tensorflow/docs_src/tutorials/_toc.yaml delete mode 100644 tensorflow/docs_src/tutorials/audio_recognition.md delete mode 100644 tensorflow/docs_src/tutorials/deep_cnn.md create mode 100644 tensorflow/docs_src/tutorials/eager/custom_training_walkthrough.md create mode 100644 tensorflow/docs_src/tutorials/eager/index.md delete mode 100644 tensorflow/docs_src/tutorials/image_recognition.md delete mode 100644 tensorflow/docs_src/tutorials/image_retraining.md create mode 100644 tensorflow/docs_src/tutorials/images/deep_cnn.md create mode 100644 tensorflow/docs_src/tutorials/images/image_recognition.md create mode 100644 tensorflow/docs_src/tutorials/images/layers.md delete mode 100644 tensorflow/docs_src/tutorials/index.md create mode 100644 tensorflow/docs_src/tutorials/keras/basic_classification.md create mode 100644 tensorflow/docs_src/tutorials/keras/basic_regression.md create mode 100644 tensorflow/docs_src/tutorials/keras/basic_text_classification.md create mode 100644 tensorflow/docs_src/tutorials/keras/index.md create mode 100644 tensorflow/docs_src/tutorials/keras/overfit_and_underfit.md create mode 100644 tensorflow/docs_src/tutorials/keras/save_and_restore_models.md delete mode 100644 tensorflow/docs_src/tutorials/kernel_methods.md delete mode 100644 tensorflow/docs_src/tutorials/layers.md delete mode 100644 tensorflow/docs_src/tutorials/leftnav_files delete mode 100644 tensorflow/docs_src/tutorials/linear.md delete mode 100755 tensorflow/docs_src/tutorials/mandelbrot.md create mode 100644 tensorflow/docs_src/tutorials/next_steps.md create mode 100644 tensorflow/docs_src/tutorials/non-ml/mandelbrot.md create mode 100644 tensorflow/docs_src/tutorials/non-ml/pdes.md delete mode 100755 tensorflow/docs_src/tutorials/pdes.md delete mode 100644 tensorflow/docs_src/tutorials/recurrent.md delete mode 100644 tensorflow/docs_src/tutorials/recurrent_quickdraw.md create mode 100644 tensorflow/docs_src/tutorials/representation/kernel_methods.md create mode 100644 tensorflow/docs_src/tutorials/representation/linear.md create mode 100644 tensorflow/docs_src/tutorials/representation/wide.md create mode 100644 tensorflow/docs_src/tutorials/representation/wide_and_deep.md create mode 100644 tensorflow/docs_src/tutorials/representation/word2vec.md delete mode 100644 tensorflow/docs_src/tutorials/seq2seq.md create mode 100644 tensorflow/docs_src/tutorials/sequences/audio_recognition.md create mode 100644 tensorflow/docs_src/tutorials/sequences/recurrent.md create mode 100644 tensorflow/docs_src/tutorials/sequences/recurrent_quickdraw.md delete mode 100644 tensorflow/docs_src/tutorials/wide.md delete mode 100644 tensorflow/docs_src/tutorials/wide_and_deep.md delete mode 100644 tensorflow/docs_src/tutorials/word2vec.md diff --git a/tensorflow/docs_src/community/leftnav_files b/tensorflow/docs_src/community/leftnav_files index 2bae60d9dd..0bd1f14de9 100644 --- a/tensorflow/docs_src/community/leftnav_files +++ b/tensorflow/docs_src/community/leftnav_files @@ -6,4 +6,3 @@ groups.md documentation.md style_guide.md benchmarks.md -swift.md diff --git a/tensorflow/docs_src/community/swift.md b/tensorflow/docs_src/community/swift.md deleted file mode 100644 index d1625d3b93..0000000000 --- a/tensorflow/docs_src/community/swift.md +++ /dev/null @@ -1,60 +0,0 @@ -

- -

- -# Swift for TensorFlow - -Welcome to the Swift for TensorFlow development community! - -Swift for TensorFlow is a new way to develop machine learning models. It -gives you the power of -[TensorFlow](https://www.tensorflow.org) directly -integrated into the [Swift programming language](https://swift.org/about). -With Swift, you can write the following imperative code, and Swift -automatically turns it into **a single TensorFlow Graph** and runs it -with the full performance of TensorFlow Sessions on CPU, GPU and -[TPU](https://cloud.google.com/tpu/docs/tpus). - -```swift -import TensorFlow - -var x = Tensor([[1, 2], [3, 4]]) - -for i in 1...5 { - x += x ⊗ x -} - -print(x) -``` - -Swift combines the flexibility of -[Eager Execution](https://www.tensorflow.org/programmers_guide/eager) with the -high performance of [Graphs and Sessions](https://www.tensorflow.org/programmers_guide/graphs). -Behind the scenes, Swift analyzes your Tensor code and automatically builds -graphs for you. Swift also catches type errors and shape mismatches before -running your code, and has [Automatic Differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) -built right in. We believe that machine learning tools are so important that -they deserve **a first-class language and a compiler**. - -Note: Swift for TensorFlow is an early stage research project. It has been -released to enable open source development and is not yet ready for general use -by machine learning developers. - -## Open Source - -We have released Swift for TensorFlow as an open-source project on GitHub! - -Our [documentation repository](https://github.com/tensorflow/swift) contains a -[project overview](https://github.com/tensorflow/swift/blob/master/docs/DesignOverview.md) -and [technical papers](https://github.com/tensorflow/swift/tree/master/docs) -explaining specific areas in depth. There are also instructions for [installing -pre-built packages](https://github.com/tensorflow/swift/blob/master/Installation.md) -(for macOS and Ubuntu) as well as a simple -[usage tutorial](https://github.com/tensorflow/swift/blob/master/Usage.md). - -Moving forward, we will use an open design model and all discussions will be -public. - -[Sign up here to join the community Google -group](https://groups.google.com/a/tensorflow.org/d/forum/swift), which we will -use for announcements and general discussion. diff --git a/tensorflow/docs_src/get_started/_index.yaml b/tensorflow/docs_src/get_started/_index.yaml deleted file mode 100644 index 4060804892..0000000000 --- a/tensorflow/docs_src/get_started/_index.yaml +++ /dev/null @@ -1,249 +0,0 @@ -project_path: /_project.yaml -book_path: /_book.yaml -description: -landing_page: - show_side_navs: True - rows: - - description: > -

Get Started with TensorFlow

-

- TensorFlow is an open-source machine learning library for research and - production. TensorFlow offers APIs for beginners and experts to develop - for desktop, mobile, web, and cloud. See the sections below to get - started. -

- items: - - custom_html: > - -
-

Learn and use ML

-
-

- The high-level Keras API provides building blocks to create and - train deep learning models. Start with these beginner-friendly - notebook examples, then read the - TensorFlow Keras guide. -

-
    -
  1. Basic classification
    2. -
  2. Text classification
    3. -
  3. Regression
    4. -
  4. Overfitting and underfitting
    5. -
  5. Save and load
    6. -
-
-
- Read the Keras guide -
-
- - classname: tfo-landing-row-item-code-block - code_block: | - 
-        import tensorflow as tf
-        mnist = tf.keras.datasets.mnist
-
-        (x_train, y_train),(x_test, y_test) = mnist.load_data()
-        x_train, x_test = x_train / 255.0, x_test / 255.0
-
-        model = tf.keras.models.Sequential([
-          tf.keras.layers.Flatten(),
-          tf.keras.layers.Dense(512, activation=tf.nn.relu),
-          tf.keras.layers.Dropout(0.2),
-          tf.keras.layers.Dense(10, activation=tf.nn.softmax)
-        ])
-        model.compile(optimizer='adam',
-                      loss='sparse_categorical_crossentropy',
-                      metrics=['accuracy'])
-
-        model.fit(x_train, y_train, epochs=5)
-        model.evaluate(x_test, y_test)
-
- {% dynamic if request.tld != 'cn' %} - Run in a Notebook - {% dynamic endif %} - - - items: - - custom_html: > -
-

Research and experimentation

-
-

- Eager execution provides an imperative, define-by-run interface for advanced operations. Write custom layers, forward passes, and training loops with auto‑differentiation. Start with - these notebooks, then read the eager execution guide. -

-
    -
  1. - {% dynamic if request.tld == 'cn' %} - Eager execution basics - {% dynamic else %} - Eager execution basics - {% dynamic endif %} -
    2. -
  2. - {% dynamic if request.tld == 'cn' %} - Automatic differentiation and gradient tapes - {% dynamic else %} - Automatic differentiation and gradient tapes - {% dynamic endif %} -
    3. -
  3. - {% dynamic if request.tld == 'cn' %} - Variables, models, and training - {% dynamic else %} - Variables, models, and training - {% dynamic endif %} -
    4. -
  4. - {% dynamic if request.tld == 'cn' %} - Custom layers - {% dynamic else %} - Custom layers - {% dynamic endif %} -
    5. -
  5. Custom training walkthrough
    6. -
  6. - {% dynamic if request.tld == 'cn' %} - Example: Neural machine translation w/ attention - {% dynamic else %} - Example: Neural machine translation w/ attention - {% dynamic endif %} -
    7. -
-
-
- Read the eager execution guide -
-
- - custom_html: > -
-

ML at production scale

-
-

- Estimators can train large models on multiple machines in a - production environment. Try the examples below and read the - Estimators guide. -

-
    -
  1. How to build a simple text classifier with TF-Hub
    2. -
  2. Classifying Higgs boson processes
    3. -
  3. Wide and deep learning using estimators
    4. -
-
-
- Read the Estimators guide -
-
- - - description: > -

Google Colab: An easy way to learn and use TensorFlow

-

- Colaboratory - is a Google research project created to help disseminate machine learning - education and research. It's a Jupyter notebook environment that requires - no setup to use and runs entirely in the cloud. - Read the blog post. -

- - - description: > -

Build your first ML app

-

Create and deploy TensorFlow models on web and mobile.

- background: grey - items: - - custom_html: > -
- -

Web developers

- -
- TensorFlow.js is a WebGL accelerated, JavaScript library to train and - deploy ML models in the browser and for Node.js. -
-
- - custom_html: > -
- -

Mobile developers

- -
- TensorFlow Lite is lightweight solution for mobile and embedded devices. -
-
- - - description: > -

Videos and updates

-

- Subscribe to the TensorFlow - YouTube channel - and blog for - the latest videos and updates. -

- items: - - description: > -

Get started with TensorFlow's High-Level APIs

- youtube_id: tjsHSIG8I08 - buttons: - - label: Watch the video - path: https://www.youtube.com/watch?v=tjsHSIG8I08 - - description: > -

Eager execution

- youtube_id: T8AW0fKP0Hs - background: grey - buttons: - - label: Watch the video - path: https://www.youtube.com/watch?v=T8AW0fKP0Hs - - description: > -

tf.data: Fast, flexible, and easy-to-use input pipelines

- youtube_id: uIcqeP7MFH0 - buttons: - - label: Watch the video - path: https://www.youtube.com/watch?v=uIcqeP7MFH0 diff --git a/tensorflow/docs_src/get_started/basic_classification.md b/tensorflow/docs_src/get_started/basic_classification.md deleted file mode 100644 index 91bbd85b24..0000000000 --- a/tensorflow/docs_src/get_started/basic_classification.md +++ /dev/null @@ -1,3 +0,0 @@ -# Basic Classification - -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_classification.ipynb) diff --git a/tensorflow/docs_src/get_started/basic_regression.md b/tensorflow/docs_src/get_started/basic_regression.md deleted file mode 100644 index a535f22f5a..0000000000 --- a/tensorflow/docs_src/get_started/basic_regression.md +++ /dev/null @@ -1,3 +0,0 @@ -# Basic Regression - -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_regression.ipynb) diff --git a/tensorflow/docs_src/get_started/basic_text_classification.md b/tensorflow/docs_src/get_started/basic_text_classification.md deleted file mode 100644 index 7c5d4f7896..0000000000 --- a/tensorflow/docs_src/get_started/basic_text_classification.md +++ /dev/null @@ -1,3 +0,0 @@ -# Basic Text Classification - -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_text_classification.ipynb) diff --git a/tensorflow/docs_src/get_started/eager.md b/tensorflow/docs_src/get_started/eager.md deleted file mode 100644 index ddf239485a..0000000000 --- a/tensorflow/docs_src/get_started/eager.md +++ /dev/null @@ -1,3 +0,0 @@ -# Custom Training Walkthrough - -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/r1.9.0/samples/core/get_started/eager.ipynb) diff --git a/tensorflow/docs_src/get_started/leftnav_files b/tensorflow/docs_src/get_started/leftnav_files deleted file mode 100644 index 99d2b2c3e1..0000000000 --- a/tensorflow/docs_src/get_started/leftnav_files +++ /dev/null @@ -1,10 +0,0 @@ -### Learn and use ML -basic_classification.md: Basic classification -basic_text_classification.md: Text classification -basic_regression.md: Regression -overfit_and_underfit.md -save_and_restore_models.md -next_steps.md - -### Research and experimentation -eager.md diff --git a/tensorflow/docs_src/get_started/next_steps.md b/tensorflow/docs_src/get_started/next_steps.md deleted file mode 100644 index 01c9f7204a..0000000000 --- a/tensorflow/docs_src/get_started/next_steps.md +++ /dev/null @@ -1,36 +0,0 @@ -# Next steps - -## Learn more about TensorFlow - -* The [TensorFlow Guide](/guide) includes usage guides for the - high-level APIs, as well as advanced TensorFlow operations. -* [Premade Estimators](/guide/premade_estimators) are designed to - get results out of the box. Use TensorFlow without building your own models. -* [TensorFlow.js](https://js.tensorflow.org/) allows web developers to train and - deploy ML models in the browser and using Node.js. -* [TFLite](/mobile/tflite) allows mobile developers to do inference efficiently - on mobile devices. -* [TensorFlow Serving](/serving) is an open-source project that can put - TensorFlow models in production quickly. -* The [ecosystem](/ecosystem) contains more projects, including - [Magenta](https://magenta.tensorflow.org/), [TFX](/tfx), - [Swift for TensorFlow](https://github.com/tensorflow/swift), and more. - -## Learn more about machine learning - -Recommended resources include: - -* [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/), - a course from Google that introduces machine learning concepts. -* [CS 20: Tensorflow for Deep Learning Research](http://web.stanford.edu/class/cs20si/), - notes from an intro course from Stanford. -* [CS231n: Convolutional Neural Networks for Visual Recognition](http://cs231n.stanford.edu/), - a course that teaches how convolutional networks work. -* [Machine Learning Recipes](https://www.youtube.com/watch?v=cKxRvEZd3Mw&list=PLOU2XLYxmsIIuiBfYad6rFYQU_jL2ryal), - a video series that introduces basic machine learning concepts with few prerequisites. -* [Deep Learning with Python](https://www.manning.com/books/deep-learning-with-python), - a book by Francois Chollet about the Keras API, as well as an excellent hands on intro to Deep Learning. -* [Hands-on Machine Learning with Scikit-Learn and TensorFlow](https://github.com/ageron/handson-ml), - a book by Aurélien Geron's that is a clear getting-started guide to data science and deep learning. -* [Deep Learning](https://www.deeplearningbook.org/), a book by Ian Goodfellow et al. - that provides a technical dive into learning machine learning. diff --git a/tensorflow/docs_src/get_started/overfit_and_underfit.md b/tensorflow/docs_src/get_started/overfit_and_underfit.md deleted file mode 100644 index e5b5ae7b5a..0000000000 --- a/tensorflow/docs_src/get_started/overfit_and_underfit.md +++ /dev/null @@ -1,3 +0,0 @@ -# Overfitting and Underfitting - -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/overfit_and_underfit.ipynb) diff --git a/tensorflow/docs_src/get_started/save_and_restore_models.md b/tensorflow/docs_src/get_started/save_and_restore_models.md deleted file mode 100644 index 44b3772945..0000000000 --- a/tensorflow/docs_src/get_started/save_and_restore_models.md +++ /dev/null @@ -1,3 +0,0 @@ -# Save and restore Models - -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/save_and_restore_models.ipynb) diff --git a/tensorflow/docs_src/guide/custom_estimators.md b/tensorflow/docs_src/guide/custom_estimators.md index fb20b35c12..a63e2bafb3 100644 --- a/tensorflow/docs_src/guide/custom_estimators.md +++ b/tensorflow/docs_src/guide/custom_estimators.md @@ -362,10 +362,10 @@ model's loss. This is the that will be optimized. We can calculate the loss by calling @{tf.losses.sparse_softmax_cross_entropy}. -The value returned by this function will be lowest, approximately 0, -probability of the correct class (at index `label`) is near 1.0. The loss value -returned is progressively larger as the probability of the correct class -decreases. +The value returned by this function will be approximately 0 at lowest, +when the probability of the correct class (at index `label`) is near 1.0. +The loss value returned is progressively larger as the probability of the +correct class decreases. This function returns the average over the whole batch. diff --git a/tensorflow/docs_src/guide/datasets_for_estimators.md b/tensorflow/docs_src/guide/datasets_for_estimators.md index b04af78cd8..b55a5731a4 100644 --- a/tensorflow/docs_src/guide/datasets_for_estimators.md +++ b/tensorflow/docs_src/guide/datasets_for_estimators.md @@ -76,9 +76,9 @@ Let's walk through the `train_input_fn()`. The function starts by using the @{tf.data.Dataset.from_tensor_slices} function to create a @{tf.data.Dataset} representing slices of the array. The array is sliced across the first dimension. For example, an array containing the -@{$tutorials/layers$mnist training data} has a shape of `(60000, 28, 28)`. -Passing this to `from_tensor_slices` returns a `Dataset` object containing -60000 slices, each one a 28x28 image. +MNIST training data has a shape of `(60000, 28, 28)`. Passing this to +`from_tensor_slices` returns a `Dataset` object containing 60000 slices, each one +a 28x28 image. The code that returns this `Dataset` is as follows: diff --git a/tensorflow/docs_src/guide/debugger.md b/tensorflow/docs_src/guide/debugger.md index 6bd941886d..8d78fe6fbd 100644 --- a/tensorflow/docs_src/guide/debugger.md +++ b/tensorflow/docs_src/guide/debugger.md @@ -17,7 +17,7 @@ how to use the graphical user interface (GUI) of tfdbg, i.e., the Note: The TensorFlow debugger uses a [curses](https://en.wikipedia.org/wiki/Curses_\(programming_library\))-based text user interface. On Mac OS X, the `ncurses` library is required and can be -installed with `brew install homebrew/dupes/ncurses`. On Windows, curses isn't as +installed with `brew install ncurses`. On Windows, curses isn't as well supported, so a [readline](https://en.wikipedia.org/wiki/GNU_Readline)-based interface can be used with tfdbg by installing `pyreadline` with `pip`. If you use Anaconda3, you can install it with a command such as @@ -33,8 +33,9 @@ and [`inf`s](https://en.wikipedia.org/wiki/Infinity), a frequently-encountered type of bug in TensorFlow model development. The following example is for users who use the low-level [`Session`](https://www.tensorflow.org/api_docs/python/tf/Session) API of -TensorFlow. A later section of this document describes how to use **tfdbg** -with a higher-level API, namely `Estimator`s. +TensorFlow. Later sections of this document describe how to use **tfdbg** +with higher-level APIs of TensorFlow, including `tf.estimator`, +`tf.keras` / `keras` and `tf.contrib.slim`. To *observe* such an issue, run the following command without the debugger (the source code can be found [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/debug/examples/debug_mnist.py)): @@ -209,6 +210,7 @@ Try the following commands at the `tfdbg>` prompt (referencing the code at | **`config`** | | **Set or show persistent TFDBG UI configuration.** | | | | `set` | Set the value of a config item: {`graph_recursion_depth`, `mouse_mode`}. | `config set graph_recursion_depth 3` | | | `show` | Show current persistent UI configuration. | `config show` | +| **`version`** | | **Print the version of TensorFlow and its key dependencies.** | `version` | | **`help`** | | **Print general help information** | `help` | | | `help ` | Print help for given command. | `help lt` | @@ -461,7 +463,6 @@ predict_results = classifier.predict(predict_input_fn, hooks=hooks) ``` [debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_tflearn_iris.py), -based on [tf-learn's iris tutorial](https://www.tensorflow.org/versions/r1.8/get_started/tflearn), contains a full example of how to use the tfdbg with `Estimator`s. To run this example, do: @@ -477,20 +478,31 @@ for more details. ## Debugging Keras Models with TFDBG -To use TFDBG with [Keras](https://keras.io/), let the Keras backend use -a TFDBG-wrapped Session object. For example, to use the CLI wrapper: +To use TFDBG with +[tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras), +let the Keras backend use a TFDBG-wrapped Session object. For example, to use +the CLI wrapper: ``` python import tensorflow as tf -from keras import backend as keras_backend from tensorflow.python import debug as tf_debug -keras_backend.set_session(tf_debug.LocalCLIDebugWrapperSession(tf.Session())) +tf.keras.backend.set_session(tf_debug.LocalCLIDebugWrapperSession(tf.Session())) # Define your keras model, called "model". -model.fit(...) # This will break into the TFDBG CLI. + +# Calls to `fit()`, 'evaluate()` and `predict()` methods will break into the +# TFDBG CLI. +model.fit(...) +model.evaluate(...) +model.predict(...) ``` +With minor modification, the preceding code example also works for the +[non-TensorFlow version of Keras](https://keras.io/) running against a +TensorFlow backend. You just need to replace `tf.keras.backend` with +`keras.backend`. + ## Debugging tf-slim with TFDBG TFDBG supports debugging of training and evaluation with diff --git a/tensorflow/docs_src/guide/eager.md b/tensorflow/docs_src/guide/eager.md index 00d02b4455..003ca265fe 100644 --- a/tensorflow/docs_src/guide/eager.md +++ b/tensorflow/docs_src/guide/eager.md @@ -149,16 +149,17 @@ it to implement your own layer: ```py class MySimpleLayer(tf.keras.layers.Layer): def __init__(self, output_units): + super(MySimpleLayer, self).__init__() self.output_units = output_units - def build(self, input): + def build(self, input_shape): # The build method gets called the first time your layer is used. # Creating variables on build() allows you to make their shape depend - # on the input shape and hence remove the need for the user to specify + # on the input shape and hence removes the need for the user to specify # full shapes. It is possible to create variables during __init__() if # you already know their full shapes. self.kernel = self.add_variable( - "kernel", [input.shape[-1], self.output_units]) + "kernel", [input_shape[-1], self.output_units]) def call(self, input): # Override call() instead of __call__ so we can perform some bookkeeping. @@ -315,9 +316,8 @@ for (batch, (images, labels)) in enumerate(dataset): The following example creates a multi-layer model that classifies the standard -[MNIST handwritten digits](https://www.tensorflow.org/tutorials/layers). It -demonstrates the optimizer and layer APIs to build trainable graphs in an eager -execution environment. +MNIST handwritten digits. It demonstrates the optimizer and layer APIs to build +trainable graphs in an eager execution environment. ### Train a model diff --git a/tensorflow/docs_src/guide/graphs.md b/tensorflow/docs_src/guide/graphs.md index e6246ef148..492f97c191 100644 --- a/tensorflow/docs_src/guide/graphs.md +++ b/tensorflow/docs_src/guide/graphs.md @@ -486,7 +486,7 @@ subgraph inside. ![](../images/mnist_deep.png) For more information about visualizing your TensorFlow application with -TensorBoard, see the [TensorBoard tutorial](../get_started/summaries_and_tensorboard.md). +TensorBoard, see the [TensorBoard guide](./summaries_and_tensorboard.md). ## Programming with multiple graphs diff --git a/tensorflow/docs_src/guide/keras.md b/tensorflow/docs_src/guide/keras.md index d584ebe945..1d846df104 100644 --- a/tensorflow/docs_src/guide/keras.md +++ b/tensorflow/docs_src/guide/keras.md @@ -221,7 +221,7 @@ To *evaluate* the inference-mode loss and metrics for the data provided: ```python model.evaluate(x, y, batch_size=32) -model.evaluate(dataset, steps=30 +model.evaluate(dataset, steps=30) ``` And to *predict* the output of the last layer in inference for the data provided, @@ -548,11 +548,9 @@ model.compile(optimizer=tf.train.RMSPropOptimizer(0.001), estimator = keras.estimator.model_to_estimator(model) ``` -Note: -* Enable [eager execution](./eager.md) for debugging +Note: Enable [eager execution](./eager.md) for debugging [Estimator input functions](./premade_estimators.md#create_input_functions) and inspecting data. -* Don't use batch normalization or try to finetune batch normalization models with estimators created from `tf.keras.estimator.model_to_estimator`. More details at [#17950](https://github.com/tensorflow/tensorflow/issues/17950) ### Multiple GPUs @@ -583,15 +581,6 @@ model.compile(loss='binary_crossentropy', optimizer=optimizer) model.summary() ``` -Convert the Keras model to a `tf.estimator.Estimator` instance: - -```python -keras_estimator = keras.estimator.model_to_estimator( - keras_model=model, - config=config, - model_dir='/tmp/model_dir') -``` - Define an *input pipeline*. The `input_fn` returns a `tf.data.Dataset` object used to distribute the data across multiple devices—with each device processing a slice of the input batch. @@ -617,6 +606,15 @@ strategy = tf.contrib.distribute.MirroredStrategy() config = tf.estimator.RunConfig(train_distribute=strategy) ``` +Convert the Keras model to a `tf.estimator.Estimator` instance: + +```python +keras_estimator = keras.estimator.model_to_estimator( + keras_model=model, + config=config, + model_dir='/tmp/model_dir') +``` + Finally, train the `Estimator` instance by providing the `input_fn` and `steps` arguments: diff --git a/tensorflow/docs_src/guide/saved_model.md b/tensorflow/docs_src/guide/saved_model.md index 27ef7bb0da..acc3d3ca0b 100644 --- a/tensorflow/docs_src/guide/saved_model.md +++ b/tensorflow/docs_src/guide/saved_model.md @@ -794,11 +794,12 @@ Here's the syntax: ``` usage: saved_model_cli run [-h] --dir DIR --tag_set TAG_SET --signature_def SIGNATURE_DEF_KEY [--inputs INPUTS] - [--input_exprs INPUT_EXPRS] [--outdir OUTDIR] + [--input_exprs INPUT_EXPRS] + [--input_examples INPUT_EXAMPLES] [--outdir OUTDIR] [--overwrite] [--tf_debug] ``` -The `run` command provides the following two ways to pass inputs to the model: +The `run` command provides the following three ways to pass inputs to the model: * `--inputs` option enables you to pass numpy ndarray in files. * `--input_exprs` option enables you to pass Python expressions. @@ -847,7 +848,7 @@ dictionary is stored in the pickle file and the value corresponding to the *variable_name* will be used. -#### `--inputs_exprs` +#### `--input_exprs` To pass inputs through Python expressions, specify the `--input_exprs` option. This can be useful for when you don't have data @@ -869,7 +870,7 @@ example: (Note that the `numpy` module is already available to you as `np`.) -#### `--inputs_examples` +#### `--input_examples` To pass `tf.train.Example` as inputs, specify the `--input_examples` option. For each input key, it takes a list of dictionary, where each dictionary is an diff --git a/tensorflow/docs_src/guide/tensorboard_histograms.md b/tensorflow/docs_src/guide/tensorboard_histograms.md index 918deda190..af8f2cadd1 100644 --- a/tensorflow/docs_src/guide/tensorboard_histograms.md +++ b/tensorflow/docs_src/guide/tensorboard_histograms.md @@ -13,8 +13,8 @@ TensorFlow has an op which is perfect for this purpose. As is usually the case with TensorBoard, we will ingest data using a summary op; in this case, ['tf.summary.histogram'](https://www.tensorflow.org/api_docs/python/tf/summary/histogram). -For a primer on how summaries work, please see the general -[TensorBoard tutorial](https://www.tensorflow.org/get_started/summaries_and_tensorboard). +For a primer on how summaries work, please see the +[TensorBoard guide](./summaries_and_tensorboard.md). Here is a code snippet that will generate some histogram summaries containing normally distributed data, where the mean of the distribution increases over diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md index 9aebf2bfa4..2901848745 100644 --- a/tensorflow/docs_src/install/install_c.md +++ b/tensorflow/docs_src/install/install_c.md @@ -38,7 +38,7 @@ enable TensorFlow for C: OS="linux" # Change to "darwin" for macOS TARGET_DIRECTORY="/usr/local" curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.9.0-rc2.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.9.0-rc0.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index 1907355341..2c126df5aa 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go: TF_TYPE="cpu" # Change to "gpu" for GPU support TARGET_DIRECTORY='/usr/local' curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.9.0-rc2.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.9.0-rc0.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index b9c9912816..692dfc9cef 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs: org.tensorflow tensorflow - 1.9.0-rc2 + 1.9.0-rc0 ``` @@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow: org.tensorflow tensorflow - 1.9.0-rc2 + 1.9.0-rc0 @@ -124,12 +124,12 @@ instead: org.tensorflow libtensorflow - 1.9.0-rc2 + 1.9.0-rc0 org.tensorflow libtensorflow_jni_gpu - 1.9.0-rc2 + 1.9.0-rc0 ``` @@ -148,7 +148,7 @@ refer to the simpler instructions above instead. Take the following steps to install TensorFlow for Java on Linux or macOS: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc2.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc0.jar), which is the TensorFlow Java Archive (JAR). 2. Decide whether you will run TensorFlow for Java on CPU(s) only or with @@ -167,7 +167,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: OS=$(uname -s | tr '[:upper:]' '[:lower:]') mkdir -p ./jni curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.9.0-rc2.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.9.0-rc0.tar.gz" | tar -xz -C ./jni ### Install on Windows @@ -175,13 +175,13 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: Take the following steps to install TensorFlow for Java on Windows: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc2.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc0.jar), which is the TensorFlow Java Archive (JAR). 2. Download the following Java Native Interface (JNI) file appropriate for - [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.9.0-rc2.zip). + [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.9.0-rc0.zip). 3. Extract this .zip file. - +__Note__: The native library (`tensorflow_jni.dll`) requires `msvcp140.dll` at runtime, which is included in the [Visual C++ 2015 Redistributable](https://www.microsoft.com/en-us/download/details.aspx?id=48145) package. ### Validate the installation @@ -227,7 +227,7 @@ must be part of your `classpath`. For example, you can include the downloaded `.jar` in your `classpath` by using the `-cp` compilation flag as follows: -
javac -cp libtensorflow-1.9.0-rc2.jar HelloTF.java
+
javac -cp libtensorflow-1.9.0-rc0.jar HelloTF.java
### Running @@ -241,11 +241,11 @@ two files are available to the JVM: For example, the following command line executes the `HelloTF` program on Linux and macOS X: -
java -cp libtensorflow-1.9.0-rc2.jar:. -Djava.library.path=./jni HelloTF
+
java -cp libtensorflow-1.9.0-rc0.jar:. -Djava.library.path=./jni HelloTF
And the following command line executes the `HelloTF` program on Windows: -
java -cp libtensorflow-1.9.0-rc2.jar;. -Djava.library.path=jni HelloTF
+
java -cp libtensorflow-1.9.0-rc0.jar;. -Djava.library.path=jni HelloTF
If the program prints Hello from version, you've successfully installed TensorFlow for Java and are ready to use the API. If the program diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index ae3d50ff39..f21c073a1b 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -339,9 +339,7 @@ Docker will download the TensorFlow binary image the first time you launch it. #### GPU support -Prior to installing TensorFlow with GPU support, ensure that your system meets all -[NVIDIA software requirements](#NVIDIARequirements). To launch a Docker container -with NVidia GPU support, enter a command of the following format: +To launch a Docker container with NVidia GPU support, enter a command of the following format (this [does not require any local CUDA installation](https://github.com/nvidia/nvidia-docker/wiki/CUDA#requirements)): 
 $ nvidia-docker run -it -p hostPort:containerPort TensorFlowGPUImage
@@ -438,7 +436,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      
      (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp34-cp34m-linux_x86_64.whl

+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
## Validate your installation @@ -491,7 +489,7 @@ TensorFlow programs: If the system outputs an error message instead of a greeting, see [Common installation problems](#common_installation_problems). -To learn more, see [Get Started with TensorFlow](https://www.tensorflow.org/get_started). +To learn more, see the [TensorFlow tutorials](../tutorials/). ## TensorFlow GPU support @@ -678,14 +676,14 @@ This section documents the relevant values for Linux installations. CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp27-none-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp27-none-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -697,14 +695,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -716,14 +714,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp35-cp35m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp35-cp35m-linux_x86_64.whl
@@ -735,14 +733,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp36-cp36m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp36-cp36m-linux_x86_64.whl
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index 3de6da1342..c6f0c17924 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv: TensorFlow in the active Virtualenv is as follows: 
 $ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py3-none-any.whl If you encounter installation problems, see [Common Installation Problems](#common-installation-problems). @@ -242,7 +242,7 @@ take the following steps: issue the following command: 
 $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py3-none-any.whl If the preceding command fails, see [installation problems](#common-installation-problems). @@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: TensorFlow for Python 2.7: 
 (targetDirectory)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py2-none-any.whl @@ -403,8 +403,7 @@ writing TensorFlow programs: If the system outputs an error message instead of a greeting, see [Common installation problems](#common_installation_problems). -To learn more, see [Get Started with TensorFlow](https://www.tensorflow.org/get_started). - +To learn more, see the [TensorFlow tutorials](../tutorials/). ## Common installation problems @@ -518,7 +517,7 @@ The value you specify depends on your Python version. 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py2-none-any.whl
@@ -526,5 +525,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py2-none-a 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py3-none-any.whl
diff --git a/tensorflow/docs_src/install/install_raspbian.md b/tensorflow/docs_src/install/install_raspbian.md index 0caab6d335..46c4944ca7 100644 --- a/tensorflow/docs_src/install/install_raspbian.md +++ b/tensorflow/docs_src/install/install_raspbian.md @@ -230,7 +230,7 @@ problems, despite the log message. If the system outputs an error message instead of a greeting, see [Common installation problems](#common_installation_problems). -To learn more, see [Get Started with TensorFlow](https://www.tensorflow.org/get_started). +To learn more, see the [TensorFlow tutorials](../tutorials/). ## Common installation problems diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index 3520f97c9a..fc1f6d05bd 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -81,7 +81,7 @@ or [macOS](#PrepareMac) - + ## Prepare environment for Linux Before building TensorFlow on Linux, install the following build @@ -289,17 +289,27 @@ Note: If you're only interested in building the libraries for the TensorFlow C or Java APIs, see [Build the C or Java libraries](#BuildCorJava), you do not need to build the pip package in that case. -To build a pip package for TensorFlow with CPU-only support, -you would typically invoke the following command: +### CPU-only support + +To build a pip package for TensorFlow with CPU-only support: + +
+$ bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package
+
+ +To build a pip package for TensorFlow with CPU-only support for the Intel® MKL-DNN: 
-$ bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package
+$ bazel build --config=mkl --config=opt //tensorflow/tools/pip_package:build_pip_package
-To build a pip package for TensorFlow with GPU support, -invoke the following command: +### GPU support + +To build a pip package for TensorFlow with GPU support: -
$ bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package
+
+$ bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package
+
**NOTE on gcc 5 or later:** the binary pip packages available on the TensorFlow website are built with gcc 4, which uses the older ABI. To @@ -328,10 +338,10 @@ Invoke `pip install` to install that pip package. The filename of the `.whl` file depends on your platform. For example, the following command will install the pip package -for TensorFlow 1.9.0rc2 on Linux: +for TensorFlow 1.9.0rc0 on Linux: 
-$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.9.0rc2-py2-none-any.whl
+$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.9.0rc0-py2-none-any.whl
## Validate your installation @@ -362,7 +372,7 @@ TensorFlow programs: 
Hello, TensorFlow!
-To learn more, see [Get Started with TensorFlow](https://www.tensorflow.org/get_started). +To learn more, see the [TensorFlow tutorials](../tutorials/). If the system outputs an error message instead of a greeting, see [Common installation problems](#common_installation_problems). @@ -373,9 +383,9 @@ The build and installation problems you encounter typically depend on the operating system. See the "Common installation problems" section of one of the following guides: - * @{$install_linux#CommonInstallationProblems$Installing TensorFlow on Linux} - * @{$install_mac#CommonInstallationProblems$Installing TensorFlow on Mac OS} - * @{$install_windows#CommonInstallationProblems$Installing TensorFlow on Windows} + * @{$install_linux#common_installation_problems$Installing TensorFlow on Linux} + * @{$install_mac#common_installation_problems$Installing TensorFlow on Mac OS} + * @{$install_windows#common_installation_problems$Installing TensorFlow on Windows} Beyond the errors documented in those two guides, the following table notes additional errors specific to building TensorFlow. Note that we diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md index 7fe94f0bc3..7b7b17ce81 100644 --- a/tensorflow/docs_src/install/install_windows.md +++ b/tensorflow/docs_src/install/install_windows.md @@ -157,7 +157,7 @@ TensorFlow programs: If the system outputs an error message instead of a greeting, see [Common installation problems](#common_installation_problems). -To learn more, see [Get Started with TensorFlow](https://www.tensorflow.org/get_started). +To learn more, see the [TensorFlow tutorials](../tutorials/). ## Common installation problems diff --git a/tensorflow/docs_src/mobile/leftnav_files b/tensorflow/docs_src/mobile/leftnav_files index 585470d5f0..97340ef7e1 100644 --- a/tensorflow/docs_src/mobile/leftnav_files +++ b/tensorflow/docs_src/mobile/leftnav_files @@ -4,6 +4,7 @@ tflite/index.md tflite/devguide.md tflite/demo_android.md tflite/demo_ios.md +tflite/performance.md >>> ### TensorFlow Mobile mobile_intro.md diff --git a/tensorflow/docs_src/mobile/linking_libs.md b/tensorflow/docs_src/mobile/linking_libs.md index cf0db59021..efef5dd0da 100644 --- a/tensorflow/docs_src/mobile/linking_libs.md +++ b/tensorflow/docs_src/mobile/linking_libs.md @@ -27,7 +27,7 @@ called `libandroid_tensorflow_inference_java.jar`. There are three ways to include this functionality in your program: 1. Include the jcenter AAR which contains it, as in this - [example app](https://github.com/googlecodelabs/tensorflow-for-poets-2/blob/master/android/build.gradle#L59-L65) + [example app](https://github.com/googlecodelabs/tensorflow-for-poets-2/blob/master/android/tfmobile/build.gradle#L59-L65) 2. Download the nightly precompiled version from [ci.tensorflow.org](http://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/). diff --git a/tensorflow/docs_src/mobile/mobile_intro.md b/tensorflow/docs_src/mobile/mobile_intro.md index 241f01d460..baad443308 100644 --- a/tensorflow/docs_src/mobile/mobile_intro.md +++ b/tensorflow/docs_src/mobile/mobile_intro.md @@ -38,7 +38,8 @@ speech-driven interface, and many of these require on-device processing. Most of the time a user isn’t giving commands, and so streaming audio continuously to a remote server would be a waste of bandwidth, since it would mostly be silence or background noises. To solve this problem it’s common to have a small neural -network running on-device @{$tutorials/audio_recognition$listening out for a particular keyword}. +network running on-device +[listening out for a particular keyword](../tutorials/sequences/audio_recognition). Once that keyword has been spotted, the rest of the conversation can be transmitted over to the server for further processing if more computing power is needed. diff --git a/tensorflow/docs_src/mobile/prepare_models.md b/tensorflow/docs_src/mobile/prepare_models.md index 8b22c04d87..2b84dbb973 100644 --- a/tensorflow/docs_src/mobile/prepare_models.md +++ b/tensorflow/docs_src/mobile/prepare_models.md @@ -105,8 +105,8 @@ inline constants so everything’s in one file. To handle the conversion, you need the `freeze_graph.py` script, that’s held in [`tensorflow/python/tools/freeze_graph.py`](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py). You’ll run it like this: - bazel build tensorflow/tools:freeze_graph - bazel-bin/tensorflow/tools/freeze_graph \ + bazel build tensorflow/python/tools:freeze_graph + bazel-bin/tensorflow/python/tools/freeze_graph \ --input_graph=/tmp/model/my_graph.pb \ --input_checkpoint=/tmp/model/model.ckpt-1000 \ --output_graph=/tmp/frozen_graph.pb \ diff --git a/tensorflow/docs_src/mobile/tflite/demo_android.md b/tensorflow/docs_src/mobile/tflite/demo_android.md index 7f2f8882a2..fdf0bcf3c1 100644 --- a/tensorflow/docs_src/mobile/tflite/demo_android.md +++ b/tensorflow/docs_src/mobile/tflite/demo_android.md @@ -1,7 +1,7 @@ # Android Demo App An example Android application using TensorFLow Lite is available -[on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app). +[on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo). The demo is a sample camera app that classifies images continuously using either a quantized Mobilenet model or a floating point Inception-v3 model. To run the demo, a device running Android 5.0 ( API 21) or higher is required. @@ -44,20 +44,22 @@ app: Android Studio project. * Install all the Gradle extensions it requests. -To get a model, either: +Now you can build and run the demo app. -* Download the quantized [Mobilenet TensorFlow Lite model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip) - and unzip and copy `mobilenet_quant_v1_224.tflite` to the assets directory: - `tensorflow/contrib/lite/java/demo/app/src/main/assets/`. -* Or, download the floating point [Inception-v3 model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip) - and unzip and copy `inceptionv3_non_slim_2015.tflite` to the assets - directory. Change the chosen classifier in - [Camera2BasicFragment.java](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java)
+The build process downloads the quantized [Mobilenet TensorFlow Lite model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip), and unzips it into the assets directory: `tensorflow/contrib/lite/java/demo/app/src/main/assets/`. + +Some additional details are available on the +[TF Lite Android App page](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md). + +### Using other models + +To use a different model: +* Download the floating point [Inception-v3 model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip). +* Unzip and copy `inceptionv3_non_slim_2015.tflite` to the assets directory. +* Change the chosen classifier in [Camera2BasicFragment.java](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java)
from: `classifier = new ImageClassifierQuantizedMobileNet(getActivity());`
to: `classifier = new ImageClassifierFloatInception(getActivity());`. -Now you can build and run the demo app. - ## Build TensorFlow Lite and the demo app from source diff --git a/tensorflow/docs_src/mobile/tflite/devguide.md b/tensorflow/docs_src/mobile/tflite/devguide.md index 4133bc172a..b168d6c183 100644 --- a/tensorflow/docs_src/mobile/tflite/devguide.md +++ b/tensorflow/docs_src/mobile/tflite/devguide.md @@ -54,10 +54,11 @@ both floating point and quantized inference. ### Train a custom model A developer may choose to train a custom model using Tensorflow (see the -@{$tutorials} for examples of building and training models). If you have already -written a model, the first step is to export this to a @{tf.GraphDef} file. This -is required because some formats do not store the model structure outside the -code, and we must communicate with other parts of the framework. See +[TensorFlow tutorials](../../tutorials/) for examples of building and training +models). If you have already written a model, the first step is to export this +to a @{tf.GraphDef} file. This is required because some formats do not store the +model structure outside the code, and we must communicate with other parts of the +framework. See [Exporting the Inference Graph](https://github.com/tensorflow/models/blob/master/research/slim/README.md) to create .pb file for the custom model. diff --git a/tensorflow/docs_src/mobile/tflite/index.md b/tensorflow/docs_src/mobile/tflite/index.md index 5622034827..3d1733024e 100644 --- a/tensorflow/docs_src/mobile/tflite/index.md +++ b/tensorflow/docs_src/mobile/tflite/index.md @@ -37,8 +37,9 @@ a custom (less-dynamic) memory allocator to ensure minimal load, initialization, and execution latency. TensorFlow Lite provides an interface to leverage hardware acceleration, if -available on the device. It does so via the Android Neural Networks library, -released as part of Android O-MR1. +available on the device. It does so via the +[Android Neural Networks API](https://developer.android.com/ndk/guides/neuralnetworks/index.html), +available on Android 8.1 (API level 27) and higher. ## Why do we need a new mobile-specific library? @@ -116,6 +117,10 @@ following: Wear](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html) to all first-party and third-party apps. + Also see the complete list of + [TensorFlow Lite's supported models](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/models.md), + including the model sizes, performance numbers, and downloadable model files. + - Quantized versions of the MobileNet model, which runs faster than the non-quantized (float) version on CPU. @@ -131,10 +136,10 @@ compatibility with this release. ## Getting Started We recommend you try out TensorFlow Lite with the pre-tested models indicated -above. If you have an existing mode, you will need to test whether your model is -compatible with both the converter and the supported operator set. To test your -model, see the [documentation on -GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite). +above. If you have an existing model, you will need to test whether your model +is compatible with both the converter and the supported operator set. To test +your model, see the +[documentation on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite). ### Retrain Inception-V3 or MobileNet for a custom data set diff --git a/tensorflow/docs_src/mobile/tflite/performance.md b/tensorflow/docs_src/mobile/tflite/performance.md new file mode 100644 index 0000000000..79bacaaa1b --- /dev/null +++ b/tensorflow/docs_src/mobile/tflite/performance.md @@ -0,0 +1,174 @@ +# Performance + +This document lists TensorFlow Lite performance benchmarks when running well +known models on some Android and iOS devices. + +These performance benchmark numbers were generated with the +[Android TFLite benchmark binary](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark) +and the [iOS benchmark app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark/ios). + +# Android performance benchmarks + +For Android benchmarks, the CPU affinity is set to use big cores on the device to +reduce variance (see [details](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark#reducing-variance-between-runs-on-android)). + +It assumes that models were download and unzipped to the +`/data/local/tmp/tflite_models` directory. The benchmark binary is built +using [these instructions](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark#on-android) +and assumed in the `/data/local/tmp` directory. + +To run the benchmark: + +``` +adb shell taskset ${CPU_MASK} /data/local/tmp/benchmark_model \ + --num_threads=1 \ + --graph=/data/local/tmp/tflite_models/${GRAPH} \ + --warmup_runs=1 \ + --num_runs=50 \ + --use_nnapi=false +``` + +Here, `${GRAPH}` is the name of model and `${CPU_MASK}` is the CPU affinity +chosen according to the following table: + +Device | CPU_MASK | +-------| ---------- +Pixel 2 | f0 | +Pixel xl | 0c | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model NameDevice Mean inference time (std dev)
+ Mobilenet_1.0_224(float) + Pixel 2 166.5 ms (2.6 ms)
Pixel xl 122.9 ms (1.8 ms)
+ Mobilenet_1.0_224 (quant) + Pixel 2 69.5 ms (0.9 ms)
Pixel xl 78.9 ms (2.2 ms)
+ NASNet mobile + Pixel 2 273.8 ms (3.5 ms)
Pixel xl 210.8 ms (4.2 ms)
+ SqueezeNet + Pixel 2 234.0 ms (2.1 ms)
Pixel xl 158.0 ms (2.1 ms)
+ Inception_ResNet_V2 + Pixel 2 2846.0 ms (15.0 ms)
Pixel xl 1973.0 ms (15.0 ms)
+ Inception_V4 + Pixel 2 3180.0 ms (11.7 ms)
Pixel xl 2262.0 ms (21.0 ms)
+ +# iOS benchmarks + +To run iOS benchmarks, the [benchmark +app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark/ios) +was modified to include the appropriate model and `benchmark_params.json` was +modified to set `num_threads` to 1. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model NameDevice Mean inference time (std dev)
+ Mobilenet_1.0_224(float) + iPhone 8 32.2 ms (0.8 ms)
+ Mobilenet_1.0_224 (quant) + iPhone 8 24.4 ms (0.8 ms)
+ NASNet mobile + iPhone 8 60.3 ms (0.6 ms)
+ SqueezeNet + iPhone 8 44.3 (0.7 ms)
+ Inception_ResNet_V2 + iPhone 8562.4 ms (18.2 ms)
+ Inception_V4 + iPhone 8 661.0 ms (29.2 ms)
diff --git a/tensorflow/docs_src/performance/quantization.md b/tensorflow/docs_src/performance/quantization.md index 2fea02d861..c97f74139c 100644 --- a/tensorflow/docs_src/performance/quantization.md +++ b/tensorflow/docs_src/performance/quantization.md @@ -227,8 +227,8 @@ of 30.0f, and an 8-bit array, the quantized values represent the following: - +
QuantizedFloat
0-10.0
25530.0
12810.0
25530.0
Table 2: Example quantized value range diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md index 5887c3d88b..4c4f3f3934 100644 --- a/tensorflow/docs_src/performance/xla/operation_semantics.md +++ b/tensorflow/docs_src/performance/xla/operation_semantics.md @@ -581,12 +581,21 @@ Computes a sum across replicas. Arguments | Type | Semantics --------- | ------- | ----------------------------- `operand` | `XlaOp` | Array to sum across replicas. +| `replica_group_ids` | `int64` vector | Group ID for each replica. | The output shape is the same as the input shape. For example, if there are two replicas and the operand has the value `(1.0, 2.5)` and `(3.0, 5.25)` respectively on the two replicas, then the output value from this op will be `(4.0, 7.75)` on both replicas. +`replica_group_ids` identifies the group ID of each replica. The group ID must +either be empty (all replicas belong to a single group), or contain the same +number of elements as the number of replicas. For example, if +`replica_group_ids` = {0, 1, 2, 3, 0, 1, 2, 3} has eight replicas, there are +four subgroups of replica IDs: {0, 4}, {1, 5}, {2, 6}, and {3, 7}. The size of +each subgroup *must* be identical, so, for example, using: +`replica_group_ids` = {0, 1, 2, 0} for four replicas is invalid. + Computing the result of CrossReplicaSum requires having one input from each replica, so if one replica executes a CrossReplicaSum node more times than another, then the former replica will wait forever. Since the replicas are all @@ -1299,12 +1308,10 @@ See also : : : parameters of type T and M of : : : : arbitrary type : | `dimensions` | `int64` array | array of map dimensions | -| `static_operands` | sequence of M `XlaOp`s | M arrays of arbitrary type | Applies a scalar function over the given `operands` arrays, producing an array of the same dimensions where each element is the result of the mapped function -applied to the corresponding elements in the input arrays with `static_operands` -given as additional input to `computation`. +applied to the corresponding elements in the input arrays. The mapped function is an arbitrary computation with the restriction that it has N inputs of scalar type `T` and a single output with type `S`. The output has @@ -2003,13 +2010,35 @@ Slice(b, {2, 1}, {4, 3}) produces: See also [`XlaBuilder::Sort`](https://www.tensorflow.org/code/tensorflow/compiler/xla/client/xla_client/xla_builder.h). -Sorts the elements in the operand. +There are two versions of the Sort instruction: a single-operand and a +two-operand version. `Sort(operand)` +Arguments | Type | Semantics +--------- | ------- | -------------------- +`operand` | `XlaOp` | The operand to sort. + +Sorts the elements in the operand in ascending order. The operand must be rank-1. +If the operand's elements have floating point type, and the operand contains +NaN elements, the order of elements in the output is implementation-defined. + +`Sort(key, value)` + +Sorts both the key and the value operands. The keys are sorted as in the +single-operand version. The values are sorted according to the order of their +corresponding keys. For example, if the inputs are `keys = [3, 1]` and +`values = [42, 50]`, then the output of the sort is the tuple `{[1, 3], [50, 42]}`. +The sort is not guaranteed to be stable, that is, if the keys array contains +duplicates, the order of their corresponding values may not be preserved. + Arguments | Type | Semantics --------- | ------- | ------------------- -`operand` | `XlaOp` | The operand to sort +`keys` | `XlaOp` | The sort keys. +`values` | `XlaOp` | The values to sort. + +The `keys` and `values` operand must both be rank-1, and must have the same +dimensions, but may have different element types. ## Transpose diff --git a/tensorflow/docs_src/tutorials/_index.yaml b/tensorflow/docs_src/tutorials/_index.yaml new file mode 100644 index 0000000000..6fc8155669 --- /dev/null +++ b/tensorflow/docs_src/tutorials/_index.yaml @@ -0,0 +1,251 @@ +project_path: /_project.yaml +book_path: /_book.yaml +description: +landing_page: + show_side_navs: True + rows: + - description: > +

Get Started with TensorFlow

+

+ TensorFlow is an open-source machine learning library for research and + production. TensorFlow offers APIs for beginners and experts to develop + for desktop, mobile, web, and cloud. See the sections below to get + started. +

+ items: + - custom_html: > + +
+

Learn and use ML

+
+

+ The high-level Keras API provides building blocks to create and + train deep learning models. Start with these beginner-friendly + notebook examples, then read the + TensorFlow Keras guide. +

+
    +
  1. Basic classification
    2. +
  2. Text classification
    3. +
  3. Regression
    4. +
  4. Overfitting and underfitting
    5. +
  5. Save and load
    6. +
+
+
+ Read the Keras guide +
+
+ - classname: tfo-landing-row-item-code-block + code_block: | + 
+        import tensorflow as tf
+        mnist = tf.keras.datasets.mnist
+
+        (x_train, y_train),(x_test, y_test) = mnist.load_data()
+        x_train, x_test = x_train / 255.0, x_test / 255.0
+
+        model = tf.keras.models.Sequential([
+          tf.keras.layers.Flatten(),
+          tf.keras.layers.Dense(512, activation=tf.nn.relu),
+          tf.keras.layers.Dropout(0.2),
+          tf.keras.layers.Dense(10, activation=tf.nn.softmax)
+        ])
+        model.compile(optimizer='adam',
+                      loss='sparse_categorical_crossentropy',
+                      metrics=['accuracy'])
+
+        model.fit(x_train, y_train, epochs=5)
+        model.evaluate(x_test, y_test)
+
+ {% dynamic if request.tld != 'cn' %} + Run in a Notebook + {% dynamic endif %} + + - items: + - custom_html: > +
+

Research and experimentation

+
+

+ Eager execution provides an imperative, define-by-run interface for advanced operations. Write custom layers, forward passes, and training loops with auto‑differentiation. Start with + these notebooks, then read the eager execution guide. +

+
    +
  1. + {% dynamic if request.tld == 'cn' %} + Eager execution basics + {% dynamic else %} + Eager execution basics + {% dynamic endif %} +
    2. +
  2. + {% dynamic if request.tld == 'cn' %} + Automatic differentiation and gradient tape + {% dynamic else %} + Automatic differentiation and gradient tape + {% dynamic endif %} +
    3. +
  3. + {% dynamic if request.tld == 'cn' %} + Custom training: basics + {% dynamic else %} + Custom training: basics + {% dynamic endif %} +
    4. +
  4. + {% dynamic if request.tld == 'cn' %} + Custom layers + {% dynamic else %} + Custom layers + {% dynamic endif %} +
    5. +
  5. Custom training: walkthrough
    6. +
  6. + {% dynamic if request.tld == 'cn' %} + Example: Neural machine translation w/ attention + {% dynamic else %} + Example: Neural machine translation w/ attention + {% dynamic endif %} +
    7. +
+
+
+ Read the eager execution guide +
+
+ - custom_html: > +
+

ML at production scale

+
+

+ Estimators can train large models on multiple machines in a + production environment. Read the + Estimators guide for details. +

+
    +
  1. Build a Convolutional Neural Network using Estimators
    2. +
  2. How to build a simple text classifier with TF-Hub
    3. +
  3. Classifying Higgs boson processes
    4. +
  4. Wide and deep learning using Estimators
    5. +
  5. Large-scale linear models
    6. +
+
+
+ Read the Estimators guide +
+
+ + - description: > +

Google Colab: An easy way to learn and use TensorFlow

+

+ Colaboratory + is a Google research project created to help disseminate machine learning + education and research. It's a Jupyter notebook environment that requires + no setup to use and runs entirely in the cloud. + Read the blog post. +

+ + - description: > +

Build your first ML app

+

Create and deploy TensorFlow models on web and mobile.

+ background: grey + items: + - custom_html: > +
+ +

Web developers

+ +
+ TensorFlow.js is a WebGL accelerated, JavaScript library to train and + deploy ML models in the browser and for Node.js. +
+
+ - custom_html: > +
+ +

Mobile developers

+ +
+ TensorFlow Lite is lightweight solution for mobile and embedded devices. +
+
+ + - description: > +

Videos and updates

+

+ Subscribe to the TensorFlow + YouTube channel + and blog for + the latest videos and updates. +

+ items: + - description: > +

Get started with TensorFlow's High-Level APIs

+ youtube_id: tjsHSIG8I08 + buttons: + - label: Watch the video + path: https://www.youtube.com/watch?v=tjsHSIG8I08 + - description: > +

Eager execution

+ youtube_id: T8AW0fKP0Hs + background: grey + buttons: + - label: Watch the video + path: https://www.youtube.com/watch?v=T8AW0fKP0Hs + - description: > +

tf.data: Fast, flexible, and easy-to-use input pipelines

+ youtube_id: uIcqeP7MFH0 + buttons: + - label: Watch the video + path: https://www.youtube.com/watch?v=uIcqeP7MFH0 diff --git a/tensorflow/docs_src/tutorials/_toc.yaml b/tensorflow/docs_src/tutorials/_toc.yaml new file mode 100644 index 0000000000..d46d570a93 --- /dev/null +++ b/tensorflow/docs_src/tutorials/_toc.yaml @@ -0,0 +1,93 @@ +toc: +- title: Get started with TensorFlow + path: /tutorials/ + +- title: Learn and use ML + style: accordion + section: + - title: Overview + path: /tutorials/keras/ + - title: Basic classification + path: /tutorials/keras/basic_classification + - title: Text classification + path: /tutorials/keras/basic_text_classification + - title: Regression + path: /tutorials/keras/basic_regression + - title: Overfitting and underfitting + path: /tutorials/keras/overfit_and_underfit + - title: Save and restore models + path: /tutorials/keras/save_and_restore_models + +- title: Research and experimentation + style: accordion + section: + - title: Overview + path: /tutorials/eager/ + - title: Eager execution + path: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/eager_intro.ipynb + status: external + - title: Automatic differentiation + path: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/automatic_differentiation.ipynb + status: external + - title: "Custom training: basics" + path: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb + status: external + - title: Custom layers + path: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/custom_layers.ipynb + status: external + - title: "Custom training: walkthrough" + path: /tutorials/eager/custom_training_walkthrough + - title: Neural machine translation + path: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb + status: external + +- title: Images + style: accordion + section: + - title: Build a CNN using Estimators + path: /tutorials/images/layers + - title: Image recognition + path: /tutorials/images/image_recognition + - title: Image retraining + path: /hub/tutorials/image_retraining + - title: Advanced CNN + path: /tutorials/images/deep_cnn + +- title: Sequences + style: accordion + section: + - title: Recurrent neural network + path: /tutorials/sequences/recurrent + - title: Drawing classification + path: /tutorials/sequences/recurrent_quickdraw + - title: Simple audio recognition + path: /tutorials/sequences/audio_recognition + - title: Neural machine translation + path: https://github.com/tensorflow/nmt + status: external + +- title: Data representation + style: accordion + section: + - title: Linear models + path: /tutorials/representation/wide + - title: Wide and deep learning + path: /tutorials/representation/wide_and_deep + - title: Vector representations of words + path: /tutorials/representation/word2vec + - title: Kernel methods + path: /tutorials/representation/kernel_methods + - title: Large-scale linear models + path: /tutorials/representation/linear + +- title: Non-ML + style: accordion + section: + - title: Mandelbrot set + path: /tutorials/non-ml/mandelbrot + - title: Partial differential equations + path: /tutorials/non-ml/pdes + +- break: True +- title: Next steps + path: /tutorials/next_steps diff --git a/tensorflow/docs_src/tutorials/audio_recognition.md b/tensorflow/docs_src/tutorials/audio_recognition.md deleted file mode 100644 index d7a8da6f96..0000000000 --- a/tensorflow/docs_src/tutorials/audio_recognition.md +++ /dev/null @@ -1,631 +0,0 @@ -# Simple Audio Recognition - -This tutorial will show you how to build a basic speech recognition network that -recognizes ten different words. It's important to know that real speech and -audio recognition systems are much more complex, but like MNIST for images, it -should give you a basic understanding of the techniques involved. Once you've -completed this tutorial, you'll have a model that tries to classify a one second -audio clip as either silence, an unknown word, "yes", "no", "up", "down", -"left", "right", "on", "off", "stop", or "go". You'll also be able to take this -model and run it in an Android application. - -## Preparation - -You should make sure you have TensorFlow installed, and since the script -downloads over 1GB of training data, you'll need a good internet connection and -enough free space on your machine. The training process itself can take several -hours, so make sure you have a machine available for that long. - -## Training - -To begin the training process, go to the TensorFlow source tree and run: - -```bash -python tensorflow/examples/speech_commands/train.py -``` - -The script will start off by downloading the [Speech Commands -dataset](https://storage.cloud.google.com/download.tensorflow.org/data/speech_commands_v0.02.tar.gz), -which consists of over 105,000 WAVE audio files of people saying thirty -different words. This data was collected by Google and released under a CC BY -license, and you can help improve it by [contributing five minutes of your own -voice](https://aiyprojects.withgoogle.com/open_speech_recording). The archive is -over 2GB, so this part may take a while, but you should see progress logs, and -once it's been downloaded once you won't need to do this step again. You can -find more information about this dataset in this -[Speech Commands paper](https://arxiv.org/abs/1804.03209). - -Once the downloading has completed, you'll see logging information that looks -like this: - -``` -I0730 16:53:44.766740 55030 train.py:176] Training from step: 1 -I0730 16:53:47.289078 55030 train.py:217] Step #1: rate 0.001000, accuracy 7.0%, cross entropy 2.611571 -``` - -This shows that the initialization process is done and the training loop has -begun. You'll see that it outputs information for every training step. Here's a -break down of what it means: - -`Step #1` shows that we're on the first step of the training loop. In this case -there are going to be 18,000 steps in total, so you can look at the step number -to get an idea of how close it is to finishing. - -`rate 0.001000` is the learning rate that's controlling the speed of the -network's weight updates. Early on this is a comparatively high number (0.001), -but for later training cycles it will be reduced 10x, to 0.0001. - -`accuracy 7.0%` is the how many classes were correctly predicted on this -training step. This value will often fluctuate a lot, but should increase on -average as training progresses. The model outputs an array of numbers, one for -each label, and each number is the predicted likelihood of the input being that -class. The predicted label is picked by choosing the entry with the highest -score. The scores are always between zero and one, with higher values -representing more confidence in the result. - -`cross entropy 2.611571` is the result of the loss function that we're using to -guide the training process. This is a score that's obtained by comparing the -vector of scores from the current training run to the correct labels, and this -should trend downwards during training. - -After a hundred steps, you should see a line like this: - -`I0730 16:54:41.813438 55030 train.py:252] Saving to -"/tmp/speech_commands_train/conv.ckpt-100"` - -This is saving out the current trained weights to a checkpoint file. If your -training script gets interrupted, you can look for the last saved checkpoint and -then restart the script with -`--start_checkpoint=/tmp/speech_commands_train/conv.ckpt-100` as a command line -argument to start from that point. - -## Confusion Matrix - -After four hundred steps, this information will be logged: - -``` -I0730 16:57:38.073667 55030 train.py:243] Confusion Matrix: - [[258 0 0 0 0 0 0 0 0 0 0 0] - [ 7 6 26 94 7 49 1 15 40 2 0 11] - [ 10 1 107 80 13 22 0 13 10 1 0 4] - [ 1 3 16 163 6 48 0 5 10 1 0 17] - [ 15 1 17 114 55 13 0 9 22 5 0 9] - [ 1 1 6 97 3 87 1 12 46 0 0 10] - [ 8 6 86 84 13 24 1 9 9 1 0 6] - [ 9 3 32 112 9 26 1 36 19 0 0 9] - [ 8 2 12 94 9 52 0 6 72 0 0 2] - [ 16 1 39 74 29 42 0 6 37 9 0 3] - [ 15 6 17 71 50 37 0 6 32 2 1 9] - [ 11 1 6 151 5 42 0 8 16 0 0 20]] -``` - -The first section is a [confusion -matrix](https://www.tensorflow.org/api_docs/python/tf/confusion_matrix). To -understand what it means, you first need to know the labels being used, which in -this case are "_silence_", "_unknown_", "yes", "no", "up", "down", "left", -"right", "on", "off", "stop", and "go". Each column represents a set of samples -that were predicted to be each label, so the first column represents all the -clips that were predicted to be silence, the second all those that were -predicted to be unknown words, the third "yes", and so on. - -Each row represents clips by their correct, ground truth labels. The first row -is all the clips that were silence, the second clips that were unknown words, -the third "yes", etc. - -This matrix can be more useful than just a single accuracy score because it -gives a good summary of what mistakes the network is making. In this example you -can see that all of the entries in the first row are zero, apart from the -initial one. Because the first row is all the clips that are actually silence, -this means that none of them were mistakenly labeled as words, so we have no -false negatives for silence. This shows the network is already getting pretty -good at distinguishing silence from words. - -If we look down the first column though, we see a lot of non-zero values. The -column represents all the clips that were predicted to be silence, so positive -numbers outside of the first cell are errors. This means that some clips of real -spoken words are actually being predicted to be silence, so we do have quite a -few false positives. - -A perfect model would produce a confusion matrix where all of the entries were -zero apart from a diagonal line through the center. Spotting deviations from -that pattern can help you figure out how the model is most easily confused, and -once you've identified the problems you can address them by adding more data or -cleaning up categories. - -## Validation - -After the confusion matrix, you should see a line like this: - -`I0730 16:57:38.073777 55030 train.py:245] Step 400: Validation accuracy = 26.3% -(N=3093)` - -It's good practice to separate your data set into three categories. The largest -(in this case roughly 80% of the data) is used for training the network, a -smaller set (10% here, known as "validation") is reserved for evaluation of the -accuracy during training, and another set (the last 10%, "testing") is used to -evaluate the accuracy once after the training is complete. - -The reason for this split is that there's always a danger that networks will -start memorizing their inputs during training. By keeping the validation set -separate, you can ensure that the model works with data it's never seen before. -The testing set is an additional safeguard to make sure that you haven't just -been tweaking your model in a way that happens to work for both the training and -validation sets, but not a broader range of inputs. - -The training script automatically separates the data set into these three -categories, and the logging line above shows the accuracy of model when run on -the validation set. Ideally, this should stick fairly close to the training -accuracy. If the training accuracy increases but the validation doesn't, that's -a sign that overfitting is occurring, and your model is only learning things -about the training clips, not broader patterns that generalize. - -## Tensorboard - -A good way to visualize how the training is progressing is using Tensorboard. By -default, the script saves out events to /tmp/retrain_logs, and you can load -these by running: - -`tensorboard --logdir /tmp/retrain_logs` - -Then navigate to [http://localhost:6006](http://localhost:6006) in your browser, -and you'll see charts and graphs showing your models progress. - -
- -
- -## Training Finished - -After a few hours of training (depending on your machine's speed), the script -should have completed all 18,000 steps. It will print out a final confusion -matrix, along with an accuracy score, all run on the testing set. With the -default settings, you should see an accuracy of between 85% and 90%. - -Because audio recognition is particularly useful on mobile devices, next we'll -export it to a compact format that's easy to work with on those platforms. To do -that, run this command line: - -``` -python tensorflow/examples/speech_commands/freeze.py \ ---start_checkpoint=/tmp/speech_commands_train/conv.ckpt-18000 \ ---output_file=/tmp/my_frozen_graph.pb -``` - -Once the frozen model has been created, you can test it with the `label_wav.py` -script, like this: - -``` -python tensorflow/examples/speech_commands/label_wav.py \ ---graph=/tmp/my_frozen_graph.pb \ ---labels=/tmp/speech_commands_train/conv_labels.txt \ ---wav=/tmp/speech_dataset/left/a5d485dc_nohash_0.wav -``` - -This should print out three labels: - -``` -left (score = 0.81477) -right (score = 0.14139) -_unknown_ (score = 0.03808) -``` - -Hopefully "left" is the top score since that's the correct label, but since the -training is random it may not for the first file you try. Experiment with some -of the other .wav files in that same folder to see how well it does. - -The scores are between zero and one, and higher values mean the model is more -confident in its prediction. - -## Running the Model in an Android App - -The easiest way to see how this model works in a real application is to download -[the prebuilt Android demo -applications](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#prebuilt-components) -and install them on your phone. You'll see 'TF Speech' appear in your app list, -and opening it will show you the same list of action words we've just trained -our model on, starting with "Yes" and "No". Once you've given the app permission -to use the microphone, you should be able to try saying those words and see them -highlighted in the UI when the model recognizes one of them. - -You can also build this application yourself, since it's open source and -[available as part of the TensorFlow repository on -github](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#building-in-android-studio-using-the-tensorflow-aar-from-jcenter). -By default it downloads [a pretrained model from -tensorflow.org](http://download.tensorflow.org/models/speech_commands_v0.02.zip), -but you can easily [replace it with a model you've trained -yourself](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#install-model-files-optional). -If you do this, you'll need to make sure that the constants in [the main -SpeechActivity Java source -file](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android/src/org/tensorflow/demo/SpeechActivity.java) -like `SAMPLE_RATE` and `SAMPLE_DURATION` match any changes you've made to the -defaults while training. You'll also see that there's a [Java version of the -RecognizeCommands -module](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android/src/org/tensorflow/demo/RecognizeCommands.java) -that's very similar to the C++ version in this tutorial. If you've tweaked -parameters for that, you can also update them in SpeechActivity to get the same -results as in your server testing. - -The demo app updates its UI list of results automatically based on the labels -text file you copy into assets alongside your frozen graph, which means you can -easily try out different models without needing to make any code changes. You -will need to update `LABEL_FILENAME` and `MODEL_FILENAME` to point to the files -you've added if you change the paths though. - -## How does this Model Work? - -The architecture used in this tutorial is based on some described in the paper -[Convolutional Neural Networks for Small-footprint Keyword -Spotting](http://www.isca-speech.org/archive/interspeech_2015/papers/i15_1478.pdf). -It was chosen because it's comparatively simple, quick to train, and easy to -understand, rather than being state of the art. There are lots of different -approaches to building neural network models to work with audio, including -[recurrent networks](https://svds.com/tensorflow-rnn-tutorial/) or [dilated -(atrous) -convolutions](https://deepmind.com/blog/wavenet-generative-model-raw-audio/). -This tutorial is based on the kind of convolutional network that will feel very -familiar to anyone who's worked with image recognition. That may seem surprising -at first though, since audio is inherently a one-dimensional continuous signal -across time, not a 2D spatial problem. - -We solve that issue by defining a window of time we believe our spoken words -should fit into, and converting the audio signal in that window into an image. -This is done by grouping the incoming audio samples into short segments, just a -few milliseconds long, and calculating the strength of the frequencies across a -set of bands. Each set of frequency strengths from a segment is treated as a -vector of numbers, and those vectors are arranged in time order to form a -two-dimensional array. This array of values can then be treated like a -single-channel image, and is known as a -[spectrogram](https://en.wikipedia.org/wiki/Spectrogram). If you want to view -what kind of image an audio sample produces, you can run the `wav_to_spectrogram -tool: - -``` -bazel run tensorflow/examples/wav_to_spectrogram:wav_to_spectrogram -- \ ---input_wav=/tmp/speech_dataset/happy/ab00c4b2_nohash_0.wav \ ---output_image=/tmp/spectrogram.png -``` - -If you open up `/tmp/spectrogram.png` you should see something like this: - -
- -
- -Because of TensorFlow's memory order, time in this image is increasing from top -to bottom, with frequencies going from left to right, unlike the usual -convention for spectrograms where time is left to right. You should be able to -see a couple of distinct parts, with the first syllable "Ha" distinct from -"ppy". - -Because the human ear is more sensitive to some frequencies than others, it's -been traditional in speech recognition to do further processing to this -representation to turn it into a set of [Mel-Frequency Cepstral -Coefficients](https://en.wikipedia.org/wiki/Mel-frequency_cepstrum), or MFCCs -for short. This is also a two-dimensional, one-channel representation so it can -be treated like an image too. If you're targeting general sounds rather than -speech you may find you can skip this step and operate directly on the -spectrograms. - -The image that's produced by these processing steps is then fed into a -multi-layer convolutional neural network, with a fully-connected layer followed -by a softmax at the end. You can see the definition of this portion in -[tensorflow/examples/speech_commands/models.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/models.py). - -## Streaming Accuracy - -Most audio recognition applications need to run on a continuous stream of audio, -rather than on individual clips. A typical way to use a model in this -environment is to apply it repeatedly at different offsets in time and average -the results over a short window to produce a smoothed prediction. If you think -of the input as an image, it's continuously scrolling along the time axis. The -words we want to recognize can start at any time, so we need to take a series of -snapshots to have a chance of having an alignment that captures most of the -utterance in the time window we feed into the model. If we sample at a high -enough rate, then we have a good chance of capturing the word in multiple -windows, so averaging the results improves the overall confidence of the -prediction. - -For an example of how you can use your model on streaming data, you can look at -[test_streaming_accuracy.cc](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/). -This uses the -[RecognizeCommands](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/recognize_commands.h) -class to run through a long-form input audio, try to spot words, and compare -those predictions against a ground truth list of labels and times. This makes it -a good example of applying a model to a stream of audio signals over time. - -You'll need a long audio file to test it against, along with labels showing -where each word was spoken. If you don't want to record one yourself, you can -generate some synthetic test data using the `generate_streaming_test_wav` -utility. By default this will create a ten minute .wav file with words roughly -every three seconds, and a text file containing the ground truth of when each -word was spoken. These words are pulled from the test portion of your current -dataset, mixed in with background noise. To run it, use: - -``` -bazel run tensorflow/examples/speech_commands:generate_streaming_test_wav -``` - -This will save a .wav file to `/tmp/speech_commands_train/streaming_test.wav`, -and a text file listing the labels to -`/tmp/speech_commands_train/streaming_test_labels.txt`. You can then run -accuracy testing with: - -``` -bazel run tensorflow/examples/speech_commands:test_streaming_accuracy -- \ ---graph=/tmp/my_frozen_graph.pb \ ---labels=/tmp/speech_commands_train/conv_labels.txt \ ---wav=/tmp/speech_commands_train/streaming_test.wav \ ---ground_truth=/tmp/speech_commands_train/streaming_test_labels.txt \ ---verbose -``` - -This will output information about the number of words correctly matched, how -many were given the wrong labels, and how many times the model triggered when -there was no real word spoken. There are various parameters that control how the -signal averaging works, including `--average_window_ms` which sets the length of -time to average results over, `--clip_stride_ms` which is the time between -applications of the model, `--suppression_ms` which stops subsequent word -detections from triggering for a certain time after an initial one is found, and -`--detection_threshold`, which controls how high the average score must be -before it's considered a solid result. - -You'll see that the streaming accuracy outputs three numbers, rather than just -the one metric used in training. This is because different applications have -varying requirements, with some being able to tolerate frequent incorrect -results as long as real words are found (high recall), while others very focused -on ensuring the predicted labels are highly likely to be correct even if some -aren't detected (high precision). The numbers from the tool give you an idea of -how your model will perform in an application, and you can try tweaking the -signal averaging parameters to tune it to give the kind of performance you want. -To understand what the right parameters are for your application, you can look -at generating an [ROC -curve](https://en.wikipedia.org/wiki/Receiver_operating_characteristic) to help -you understand the tradeoffs. - -## RecognizeCommands - -The streaming accuracy tool uses a simple decoder contained in a small C++ class -called -[RecognizeCommands](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/recognize_commands.h). -This class is fed the output of running the TensorFlow model over time, it -averages the signals, and returns information about a label when it has enough -evidence to think that a recognized word has been found. The implementation is -fairly small, just keeping track of the last few predictions and averaging them, -so it's easy to port to other platforms and languages as needed. For example, -it's convenient to do something similar at the Java level on Android, or Python -on the Raspberry Pi. As long as these implementations share the same logic, you -can tune the parameters that control the averaging using the streaming test -tool, and then transfer them over to your application to get similar results. - -## Advanced Training - -The defaults for the training script are designed to produce good end to end -results in a comparatively small file, but there are a lot of options you can -change to customize the results for your own requirements. - -### Custom Training Data - -By default the script will download the [Speech Commands -dataset](https://download.tensorflow.org/data/speech_commands_v0.01.tgz), but -you can also supply your own training data. To train on your own data, you -should make sure that you have at least several hundred recordings of each sound -you would like to recognize, and arrange them into folders by class. For -example, if you were trying to recognize dog barks from cat miaows, you would -create a root folder called `animal_sounds`, and then within that two -sub-folders called `bark` and `miaow`. You would then organize your audio files -into the appropriate folders. - -To point the script to your new audio files, you'll need to set `--data_url=` to -disable downloading of the Speech Commands dataset, and -`--data_dir=/your/data/folder/` to find the files you've just created. - -The files themselves should be 16-bit little-endian PCM-encoded WAVE format. The -sample rate defaults to 16,000, but as long as all your audio is consistently -the same rate (the script doesn't support resampling) you can change this with -the `--sample_rate` argument. The clips should also all be roughly the same -duration. The default expected duration is one second, but you can set this with -the `--clip_duration_ms` flag. If you have clips with variable amounts of -silence at the start, you can look at word alignment tools to standardize them -([here's a quick and dirty approach you can use -too](https://petewarden.com/2017/07/17/a-quick-hack-to-align-single-word-audio-recordings/)). - -One issue to watch out for is that you may have very similar repetitions of the -same sounds in your dataset, and these can give misleading metrics if they're -spread across your training, validation, and test sets. For example, the Speech -Commands set has people repeating the same word multiple times. Each one of -those repetitions is likely to be pretty close to the others, so if training was -overfitting and memorizing one, it could perform unrealistically well when it -saw a very similar copy in the test set. To avoid this danger, Speech Commands -trys to ensure that all clips featuring the same word spoken by a single person -are put into the same partition. Clips are assigned to training, test, or -validation sets based on a hash of their filename, to ensure that the -assignments remain steady even as new clips are added and avoid any training -samples migrating into the other sets. To make sure that all a given speaker's -words are in the same bucket, [the hashing -function](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/input_data.py) -ignores anything in a filename after '_nohash_' when calculating the -assignments. This means that if you have file names like `pete_nohash_0.wav` and -`pete_nohash_1.wav`, they're guaranteed to be in the same set. - -### Unknown Class - -It's likely that your application will hear sounds that aren't in your training -set, and you'll want the model to indicate that it doesn't recognize the noise -in those cases. To help the network learn what sounds to ignore, you need to -provide some clips of audio that are neither of your classes. To do this, you'd -create `quack`, `oink`, and `moo` subfolders and populate them with noises from -other animals your users might encounter. The `--wanted_words` argument to the -script defines which classes you care about, all the others mentioned in -subfolder names will be used to populate an `_unknown_` class during training. -The Speech Commands dataset has twenty words in its unknown classes, including -the digits zero through nine and random names like "Sheila". - -By default 10% of the training examples are picked from the unknown classes, but -you can control this with the `--unknown_percentage` flag. Increasing this will -make the model less likely to mistake unknown words for wanted ones, but making -it too large can backfire as the model might decide it's safest to categorize -all words as unknown! - -### Background Noise - -Real applications have to recognize audio even when there are other irrelevant -sounds happening in the environment. To build a model that's robust to this kind -of interference, we need to train against recorded audio with similar -properties. The files in the Speech Commands dataset were captured on a variety -of devices by users in many different environments, not in a studio, so that -helps add some realism to the training. To add even more, you can mix in random -segments of environmental audio to the training inputs. In the Speech Commands -set there's a special folder called `_background_noise_` which contains -minute-long WAVE files with white noise and recordings of machinery and everyday -household activity. - -Small snippets of these files are chosen at random and mixed at a low volume -into clips during training. The loudness is also chosen randomly, and controlled -by the `--background_volume` argument as a proportion where 0 is silence, and 1 -is full volume. Not all clips have background added, so the -`--background_frequency` flag controls what proportion have them mixed in. - -Your own application might operate in its own environment with different -background noise patterns than these defaults, so you can supply your own audio -clips in the `_background_noise_` folder. These should be the same sample rate -as your main dataset, but much longer in duration so that a good set of random -segments can be selected from them. - -### Silence - -In most cases the sounds you care about will be intermittent and so it's -important to know when there's no matching audio. To support this, there's a -special `_silence_` label that indicates when the model detects nothing -interesting. Because there's never complete silence in real environments, we -actually have to supply examples with quiet and irrelevant audio. For this, we -reuse the `_background_noise_` folder that's also mixed in to real clips, -pulling short sections of the audio data and feeding those in with the ground -truth class of `_silence_`. By default 10% of the training data is supplied like -this, but the `--silence_percentage` can be used to control the proportion. As -with unknown words, setting this higher can weight the model results in favor of -true positives for silence, at the expense of false negatives for words, but too -large a proportion can cause it to fall into the trap of always guessing -silence. - -### Time Shifting - -Adding in background noise is one way of distorting the training data in a -realistic way to effectively increase the size of the dataset, and so increase -overall accuracy, and time shifting is another. This involves a random offset in -time of the training sample data, so that a small part of the start or end is -cut off and the opposite section is padded with zeroes. This mimics the natural -variations in starting time in the training data, and is controlled with the -`--time_shift_ms` flag, which defaults to 100ms. Increasing this value will -provide more variation, but at the risk of cutting off important parts of the -audio. A related way of augmenting the data with realistic distortions is by -using [time stretching and pitch -scaling](https://en.wikipedia.org/wiki/Audio_time_stretching_and_pitch_scaling), -but that's outside the scope of this tutorial. - -## Customizing the Model - -The default model used for this script is pretty large, taking over 800 million -FLOPs for each inference and using 940,000 weight parameters. This runs at -usable speeds on desktop machines or modern phones, but it involves too many -calculations to run at interactive speeds on devices with more limited -resources. To support these use cases, there's a couple of alternatives -available: - - -**low_latency_conv** -Based on the 'cnn-one-fstride4' topology described in the [Convolutional -Neural Networks for Small-footprint Keyword Spotting -paper](http://www.isca-speech.org/archive/interspeech_2015/papers/i15_1478.pdf). -The accuracy is slightly lower than 'conv' but the number of weight parameters -is about the same, and it only needs 11 million FLOPs to run one prediction, -making it much faster. - -To use this model, you specify `--model_architecture=low_latency_conv` on -the command line. You'll also need to update the training rates and the number -of steps, so the full command will look like: - -``` -python tensorflow/examples/speech_commands/train \ ---model_architecture=low_latency_conv \ ---how_many_training_steps=20000,6000 \ ---learning_rate=0.01,0.001 -``` - -This asks the script to train with a learning rate of 0.01 for 20,000 steps, and -then do a fine-tuning pass of 6,000 steps with a 10x smaller rate. - -**low_latency_svdf** -Based on the topology presented in the [Compressing Deep Neural Networks using a -Rank-Constrained Topology paper](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43813.pdf). -The accuracy is also lower than 'conv' but it only uses about 750 thousand -parameters, and most significantly, it allows for an optimized execution at -test time (i.e. when you will actually use it in your application), resulting -in 750 thousand FLOPs. - -To use this model, you specify `--model_architecture=low_latency_svdf` on -the command line, and update the training rates and the number -of steps, so the full command will look like: - -``` -python tensorflow/examples/speech_commands/train \ ---model_architecture=low_latency_svdf \ ---how_many_training_steps=100000,35000 \ ---learning_rate=0.01,0.005 -``` - -Note that despite requiring a larger number of steps than the previous two -topologies, the reduced number of computations means that training should take -about the same time, and at the end reach an accuracy of around 85%. -You can also further tune the topology fairly easily for computation and -accuracy by changing these parameters in the SVDF layer: - -* rank - The rank of the approximation (higher typically better, but results in - more computation). -* num_units - Similar to other layer types, specifies the number of nodes in - the layer (more nodes better quality, and more computation). - -Regarding runtime, since the layer allows optimizations by caching some of the -internal neural network activations, you need to make sure to use a consistent -stride (e.g. 'clip_stride_ms' flag) both when you freeze the graph, and when -executing the model in streaming mode (e.g. test_streaming_accuracy.cc). - -**Other parameters to customize** -If you want to experiment with customizing models, a good place to start is by -tweaking the spectrogram creation parameters. This has the effect of altering -the size of the input image to the model, and the creation code in -[models.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/models.py) -will adjust the number of computations and weights automatically to fit with -different dimensions. If you make the input smaller, the model will need fewer -computations to process it, so it can be a great way to trade off some accuracy -for improved latency. The `--window_stride_ms` controls how far apart each -frequency analysis sample is from the previous. If you increase this value, then -fewer samples will be taken for a given duration, and the time axis of the input -will shrink. The `--dct_coefficient_count` flag controls how many buckets are -used for the frequency counting, so reducing this will shrink the input in the -other dimension. The `--window_size_ms` argument doesn't affect the size, but -does control how wide the area used to calculate the frequencies is for each -sample. Reducing the duration of the training samples, controlled by -`--clip_duration_ms`, can also help if the sounds you're looking for are short, -since that also reduces the time dimension of the input. You'll need to make -sure that all your training data contains the right audio in the initial portion -of the clip though. - -If you have an entirely different model in mind for your problem, you may find -that you can plug it into -[models.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/models.py) -and have the rest of the script handle all of the preprocessing and training -mechanics. You would add a new clause to `create_model`, looking for the name of -your architecture and then calling a model creation function. This function is -given the size of the spectrogram input, along with other model information, and -is expected to create TensorFlow ops to read that in and produce an output -prediction vector, and a placeholder to control the dropout rate. The rest of -the script will handle integrating this model into a larger graph doing the -input calculations and applying softmax and a loss function to train it. - -One common problem when you're adjusting models and training hyper-parameters is -that not-a-number values can creep in, thanks to numerical precision issues. In -general you can solve these by reducing the magnitude of things like learning -rates and weight initialization functions, but if they're persistent you can -enable the `--check_nans` flag to track down the source of the errors. This will -insert check ops between most regular operations in TensorFlow, and abort the -training process with a useful error message when they're encountered. diff --git a/tensorflow/docs_src/tutorials/deep_cnn.md b/tensorflow/docs_src/tutorials/deep_cnn.md deleted file mode 100644 index 44a32d9d1d..0000000000 --- a/tensorflow/docs_src/tutorials/deep_cnn.md +++ /dev/null @@ -1,452 +0,0 @@ -# Convolutional Neural Networks - -> **NOTE:** This tutorial is intended for *advanced* users of TensorFlow -and assumes expertise and experience in machine learning. - -## Overview - -CIFAR-10 classification is a common benchmark problem in machine learning. The -problem is to classify RGB 32x32 pixel images across 10 categories: -``` -airplane, automobile, bird, cat, deer, dog, frog, horse, ship, and truck. -``` - -For more details refer to the [CIFAR-10 page](https://www.cs.toronto.edu/~kriz/cifar.html) -and a [Tech Report](https://www.cs.toronto.edu/~kriz/learning-features-2009-TR.pdf) -by Alex Krizhevsky. - -### Goals - -The goal of this tutorial is to build a relatively small [convolutional neural -network](https://en.wikipedia.org/wiki/Convolutional_neural_network) (CNN) for -recognizing images. In the process, this tutorial: - -1. Highlights a canonical organization for network architecture, -training and evaluation. -2. Provides a template for constructing larger and more sophisticated models. - -The reason CIFAR-10 was selected was that it is complex enough to exercise -much of TensorFlow's ability to scale to large models. At the same time, -the model is small enough to train fast, which is ideal for trying out -new ideas and experimenting with new techniques. - -### Highlights of the Tutorial -The CIFAR-10 tutorial demonstrates several important constructs for -designing larger and more sophisticated models in TensorFlow: - -* Core mathematical components including @{tf.nn.conv2d$convolution} -([wiki](https://en.wikipedia.org/wiki/Convolution)), -@{tf.nn.relu$rectified linear activations} -([wiki](https://en.wikipedia.org/wiki/Rectifier_(neural_networks))), -@{tf.nn.max_pool$max pooling} -([wiki](https://en.wikipedia.org/wiki/Convolutional_neural_network#Pooling_layer)) -and @{tf.nn.local_response_normalization$local response normalization} -(Chapter 3.3 in -[AlexNet paper](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf)). -* @{$summaries_and_tensorboard$Visualization} -of network activities during training, including input images, -losses and distributions of activations and gradients. -* Routines for calculating the -@{tf.train.ExponentialMovingAverage$moving average} -of learned parameters and using these averages -during evaluation to boost predictive performance. -* Implementation of a -@{tf.train.exponential_decay$learning rate schedule} -that systematically decrements over time. -* Prefetching @{tf.train.shuffle_batch$queues} -for input -data to isolate the model from disk latency and expensive image pre-processing. - -We also provide a [multi-GPU version](#training-a-model-using-multiple-gpu-cards) -of the model which demonstrates: - -* Configuring a model to train across multiple GPU cards in parallel. -* Sharing and updating variables among multiple GPUs. - -We hope that this tutorial provides a launch point for building larger CNNs for -vision tasks on TensorFlow. - -### Model Architecture - -The model in this CIFAR-10 tutorial is a multi-layer architecture consisting of -alternating convolutions and nonlinearities. These layers are followed by fully -connected layers leading into a softmax classifier. The model follows the -architecture described by -[Alex Krizhevsky](https://code.google.com/p/cuda-convnet/), with a few -differences in the top few layers. - -This model achieves a peak performance of about 86% accuracy within a few hours -of training time on a GPU. Please see [below](#evaluating-a-model) and the code -for details. It consists of 1,068,298 learnable parameters and requires about -19.5M multiply-add operations to compute inference on a single image. - -## Code Organization - -The code for this tutorial resides in -[`models/tutorials/image/cifar10/`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/). - -File | Purpose ---- | --- -[`cifar10_input.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10_input.py) | Reads the native CIFAR-10 binary file format. -[`cifar10.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10.py) | Builds the CIFAR-10 model. -[`cifar10_train.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10_train.py) | Trains a CIFAR-10 model on a CPU or GPU. -[`cifar10_multi_gpu_train.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10_multi_gpu_train.py) | Trains a CIFAR-10 model on multiple GPUs. -[`cifar10_eval.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10_eval.py) | Evaluates the predictive performance of a CIFAR-10 model. - - -## CIFAR-10 Model - -The CIFAR-10 network is largely contained in -[`cifar10.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10.py). -The complete training -graph contains roughly 765 operations. We find that we can make the code most -reusable by constructing the graph with the following modules: - -1. [**Model inputs:**](#model-inputs) `inputs()` and `distorted_inputs()` add -operations that read and preprocess CIFAR images for evaluation and training, -respectively. -1. [**Model prediction:**](#model-prediction) `inference()` -adds operations that perform inference, i.e. classification, on supplied images. -1. [**Model training:**](#model-training) `loss()` and `train()` -add operations that compute the loss, -gradients, variable updates and visualization summaries. - -### Model Inputs - -The input part of the model is built by the functions `inputs()` and -`distorted_inputs()` which read images from the CIFAR-10 binary data files. -These files contain fixed byte length records, so we use -@{tf.FixedLengthRecordReader}. -See @{$reading_data#reading-from-files$Reading Data} to -learn more about how the `Reader` class works. - -The images are processed as follows: - -* They are cropped to 24 x 24 pixels, centrally for evaluation or - @{tf.random_crop$randomly} for training. -* They are @{tf.image.per_image_standardization$approximately whitened} - to make the model insensitive to dynamic range. - -For training, we additionally apply a series of random distortions to -artificially increase the data set size: - -* @{tf.image.random_flip_left_right$Randomly flip} the image from left to right. -* Randomly distort the @{tf.image.random_brightness$image brightness}. -* Randomly distort the @{tf.image.random_contrast$image contrast}. - -Please see the @{$python/image$Images} page for the list of -available distortions. We also attach an -@{tf.summary.image} to the images -so that we may visualize them in @{$summaries_and_tensorboard$TensorBoard}. -This is a good practice to verify that inputs are built correctly. - -
- -
- -Reading images from disk and distorting them can use a non-trivial amount of -processing time. To prevent these operations from slowing down training, we run -them inside 16 separate threads which continuously fill a TensorFlow -@{tf.train.shuffle_batch$queue}. - -### Model Prediction - -The prediction part of the model is constructed by the `inference()` function -which adds operations to compute the *logits* of the predictions. That part of -the model is organized as follows: - -Layer Name | Description ---- | --- -`conv1` | @{tf.nn.conv2d$convolution} and @{tf.nn.relu$rectified linear} activation. -`pool1` | @{tf.nn.max_pool$max pooling}. -`norm1` | @{tf.nn.local_response_normalization$local response normalization}. -`conv2` | @{tf.nn.conv2d$convolution} and @{tf.nn.relu$rectified linear} activation. -`norm2` | @{tf.nn.local_response_normalization$local response normalization}. -`pool2` | @{tf.nn.max_pool$max pooling}. -`local3` | @{$python/nn$fully connected layer with rectified linear activation}. -`local4` | @{$python/nn$fully connected layer with rectified linear activation}. -`softmax_linear` | linear transformation to produce logits. - -Here is a graph generated from TensorBoard describing the inference operation: - -
- -
- -> **EXERCISE**: The output of `inference` are un-normalized logits. Try editing -the network architecture to return normalized predictions using -@{tf.nn.softmax}. - -The `inputs()` and `inference()` functions provide all the components -necessary to perform an evaluation of a model. We now shift our focus towards -building operations for training a model. - -> **EXERCISE:** The model architecture in `inference()` differs slightly from -the CIFAR-10 model specified in -[cuda-convnet](https://code.google.com/p/cuda-convnet/). In particular, the top -layers of Alex's original model are locally connected and not fully connected. -Try editing the architecture to exactly reproduce the locally connected -architecture in the top layer. - -### Model Training - -The usual method for training a network to perform N-way classification is -[multinomial logistic regression](https://en.wikipedia.org/wiki/Multinomial_logistic_regression), -aka. *softmax regression*. Softmax regression applies a -@{tf.nn.softmax$softmax} nonlinearity to the -output of the network and calculates the -@{tf.nn.sparse_softmax_cross_entropy_with_logits$cross-entropy} -between the normalized predictions and the label index. -For regularization, we also apply the usual -@{tf.nn.l2_loss$weight decay} losses to all learned -variables. The objective function for the model is the sum of the cross entropy -loss and all these weight decay terms, as returned by the `loss()` function. - -We visualize it in TensorBoard with a @{tf.summary.scalar}: - -![CIFAR-10 Loss](https://www.tensorflow.org/images/cifar_loss.png "CIFAR-10 Total Loss") - -We train the model using standard -[gradient descent](https://en.wikipedia.org/wiki/Gradient_descent) -algorithm (see @{$python/train$Training} for other methods) -with a learning rate that -@{tf.train.exponential_decay$exponentially decays} -over time. - -![CIFAR-10 Learning Rate Decay](https://www.tensorflow.org/images/cifar_lr_decay.png "CIFAR-10 Learning Rate Decay") - -The `train()` function adds the operations needed to minimize the objective by -calculating the gradient and updating the learned variables (see -@{tf.train.GradientDescentOptimizer} -for details). It returns an operation that executes all the calculations -needed to train and update the model for one batch of images. - -## Launching and Training the Model - -We have built the model, let's now launch it and run the training operation with -the script `cifar10_train.py`. - -```shell -python cifar10_train.py -``` - -> **NOTE:** The first time you run any target in the CIFAR-10 tutorial, -the CIFAR-10 dataset is automatically downloaded. The data set is ~160MB -so you may want to grab a quick cup of coffee for your first run. - -You should see the output: - -```shell -Filling queue with 20000 CIFAR images before starting to train. This will take a few minutes. -2015-11-04 11:45:45.927302: step 0, loss = 4.68 (2.0 examples/sec; 64.221 sec/batch) -2015-11-04 11:45:49.133065: step 10, loss = 4.66 (533.8 examples/sec; 0.240 sec/batch) -2015-11-04 11:45:51.397710: step 20, loss = 4.64 (597.4 examples/sec; 0.214 sec/batch) -2015-11-04 11:45:54.446850: step 30, loss = 4.62 (391.0 examples/sec; 0.327 sec/batch) -2015-11-04 11:45:57.152676: step 40, loss = 4.61 (430.2 examples/sec; 0.298 sec/batch) -2015-11-04 11:46:00.437717: step 50, loss = 4.59 (406.4 examples/sec; 0.315 sec/batch) -... -``` - -The script reports the total loss every 10 steps as well as the speed at which -the last batch of data was processed. A few comments: - -* The first batch of data can be inordinately slow (e.g. several minutes) as the -preprocessing threads fill up the shuffling queue with 20,000 processed CIFAR -images. - -* The reported loss is the average loss of the most recent batch. Remember that -this loss is the sum of the cross entropy and all weight decay terms. - -* Keep an eye on the processing speed of a batch. The numbers shown above were -obtained on a Tesla K40c. If you are running on a CPU, expect slower performance. - - -> **EXERCISE:** When experimenting, it is sometimes annoying that the first -training step can take so long. Try decreasing the number of images that -initially fill up the queue. Search for `min_fraction_of_examples_in_queue` -in `cifar10_input.py`. - -`cifar10_train.py` periodically @{tf.train.Saver$saves} -all model parameters in -@{$guide/saved_model$checkpoint files} -but it does *not* evaluate the model. The checkpoint file -will be used by `cifar10_eval.py` to measure the predictive -performance (see [Evaluating a Model](#evaluating-a-model) below). - - -If you followed the previous steps, then you have now started training -a CIFAR-10 model. [Congratulations!](https://www.youtube.com/watch?v=9bZkp7q19f0) - -The terminal text returned from `cifar10_train.py` provides minimal insight into -how the model is training. We want more insight into the model during training: - -* Is the loss *really* decreasing or is that just noise? -* Is the model being provided appropriate images? -* Are the gradients, activations and weights reasonable? -* What is the learning rate currently at? - -@{$summaries_and_tensorboard$TensorBoard} provides this -functionality, displaying data exported periodically from `cifar10_train.py` via -a -@{tf.summary.FileWriter}. - -For instance, we can watch how the distribution of activations and degree of -sparsity in `local3` features evolve during training: - -
- - -
- -Individual loss functions, as well as the total loss, are particularly -interesting to track over time. However, the loss exhibits a considerable amount -of noise due to the small batch size employed by training. In practice we find -it extremely useful to visualize their moving averages in addition to their raw -values. See how the scripts use -@{tf.train.ExponentialMovingAverage} -for this purpose. - -## Evaluating a Model - -Let us now evaluate how well the trained model performs on a hold-out data set. -The model is evaluated by the script `cifar10_eval.py`. It constructs the model -with the `inference()` function and uses all 10,000 images in the evaluation set -of CIFAR-10. It calculates the *precision at 1:* how often the top prediction -matches the true label of the image. - -To monitor how the model improves during training, the evaluation script runs -periodically on the latest checkpoint files created by the `cifar10_train.py`. - -```shell -python cifar10_eval.py -``` - -> Be careful not to run the evaluation and training binary on the same GPU or -else you might run out of memory. Consider running the evaluation on -a separate GPU if available or suspending the training binary while running -the evaluation on the same GPU. - -You should see the output: - -```shell -2015-11-06 08:30:44.391206: precision @ 1 = 0.860 -... -``` - -The script merely returns the precision @ 1 periodically -- in this case -it returned 86% accuracy. `cifar10_eval.py` also -exports summaries that may be visualized in TensorBoard. These summaries -provide additional insight into the model during evaluation. - -The training script calculates the -@{tf.train.ExponentialMovingAverage$moving average} -version of all learned variables. The evaluation script substitutes -all learned model parameters with the moving average version. This -substitution boosts model performance at evaluation time. - -> **EXERCISE:** Employing averaged parameters may boost predictive performance -by about 3% as measured by precision @ 1. Edit `cifar10_eval.py` to not employ -the averaged parameters for the model and verify that the predictive performance -drops. - - -## Training a Model Using Multiple GPU Cards - -Modern workstations may contain multiple GPUs for scientific computation. -TensorFlow can leverage this environment to run the training operation -concurrently across multiple cards. - -Training a model in a parallel, distributed fashion requires -coordinating training processes. For what follows we term *model replica* -to be one copy of a model training on a subset of data. - -Naively employing asynchronous updates of model parameters -leads to sub-optimal training performance -because an individual model replica might be trained on a stale -copy of the model parameters. Conversely, employing fully synchronous -updates will be as slow as the slowest model replica. - -In a workstation with multiple GPU cards, each GPU will have similar speed -and contain enough memory to run an entire CIFAR-10 model. Thus, we opt to -design our training system in the following manner: - -* Place an individual model replica on each GPU. -* Update model parameters synchronously by waiting for all GPUs to finish -processing a batch of data. - -Here is a diagram of this model: - -
- -
- -Note that each GPU computes inference as well as the gradients for a unique -batch of data. This setup effectively permits dividing up a larger batch -of data across the GPUs. - -This setup requires that all GPUs share the model parameters. A well-known -fact is that transferring data to and from GPUs is quite slow. For this -reason, we decide to store and update all model parameters on the CPU (see -green box). A fresh set of model parameters is transferred to the GPU -when a new batch of data is processed by all GPUs. - -The GPUs are synchronized in operation. All gradients are accumulated from -the GPUs and averaged (see green box). The model parameters are updated with -the gradients averaged across all model replicas. - -### Placing Variables and Operations on Devices - -Placing operations and variables on devices requires some special -abstractions. - -The first abstraction we require is a function for computing inference and -gradients for a single model replica. In the code we term this abstraction -a "tower". We must set two attributes for each tower: - -* A unique name for all operations within a tower. -@{tf.name_scope} provides -this unique name by prepending a scope. For instance, all operations in -the first tower are prepended with `tower_0`, e.g. `tower_0/conv1/Conv2D`. - -* A preferred hardware device to run the operation within a tower. -@{tf.device} specifies this. For -instance, all operations in the first tower reside within `device('/device:GPU:0')` -scope indicating that they should be run on the first GPU. - -All variables are pinned to the CPU and accessed via -@{tf.get_variable} -in order to share them in a multi-GPU version. -See how-to on @{$variables$Sharing Variables}. - -### Launching and Training the Model on Multiple GPU cards - -If you have several GPU cards installed on your machine you can use them to -train the model faster with the `cifar10_multi_gpu_train.py` script. This -version of the training script parallelizes the model across multiple GPU cards. - -```shell -python cifar10_multi_gpu_train.py --num_gpus=2 -``` - -Note that the number of GPU cards used defaults to 1. Additionally, if only 1 -GPU is available on your machine, all computations will be placed on it, even if -you ask for more. - -> **EXERCISE:** The default settings for `cifar10_train.py` is to -run on a batch size of 128. Try running `cifar10_multi_gpu_train.py` on 2 GPUs -with a batch size of 64 and compare the training speed. - -## Next Steps - -[Congratulations!](https://www.youtube.com/watch?v=9bZkp7q19f0) You have -completed the CIFAR-10 tutorial. - -If you are now interested in developing and training your own image -classification system, we recommend forking this tutorial and replacing -components to address your image classification problem. - - -> **EXERCISE:** Download the -[Street View House Numbers (SVHN)](http://ufldl.stanford.edu/housenumbers/) data set. -Fork the CIFAR-10 tutorial and swap in the SVHN as the input data. Try adapting -the network architecture to improve predictive performance. diff --git a/tensorflow/docs_src/tutorials/eager/custom_training_walkthrough.md b/tensorflow/docs_src/tutorials/eager/custom_training_walkthrough.md new file mode 100644 index 0000000000..b45fbefac0 --- /dev/null +++ b/tensorflow/docs_src/tutorials/eager/custom_training_walkthrough.md @@ -0,0 +1,3 @@ +# Custom training: walkthrough + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/eager.ipynb) diff --git a/tensorflow/docs_src/tutorials/eager/index.md b/tensorflow/docs_src/tutorials/eager/index.md new file mode 100644 index 0000000000..5445e0c343 --- /dev/null +++ b/tensorflow/docs_src/tutorials/eager/index.md @@ -0,0 +1,13 @@ +# Research and experimentation + +Eager execution provides an imperative, define-by-run interface for advanced +operations. Write custom layers, forward passes, and training loops with +auto differentiation. Start with these notebooks, then read the +[eager execution guide](../../guide/eager). + +1. [Eager execution](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/eager_intro.ipynb){:.external} +2. [Automatic differentiation and gradient tape](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/automatic_differentiation.ipynb){:.external} +3. [Custom training: basics](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb){:.external} +4. [Custom layers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/custom_layers.ipynb){:.external} +5. [Custom training: walkthrough](/tutorials/eager/custom_training_walkthrough) +6. [Advanced example: Neural machine translation with attention](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/nmt_with_attention/nmt_with_attention.ipynb){:.external} diff --git a/tensorflow/docs_src/tutorials/image_recognition.md b/tensorflow/docs_src/tutorials/image_recognition.md deleted file mode 100644 index 332bcf54f0..0000000000 --- a/tensorflow/docs_src/tutorials/image_recognition.md +++ /dev/null @@ -1,456 +0,0 @@ -# Image Recognition - -Our brains make vision seem easy. It doesn't take any effort for humans to -tell apart a lion and a jaguar, read a sign, or recognize a human's face. -But these are actually hard problems to solve with a computer: they only -seem easy because our brains are incredibly good at understanding images. - -In the last few years, the field of machine learning has made tremendous -progress on addressing these difficult problems. In particular, we've -found that a kind of model called a deep -[convolutional neural network](https://colah.github.io/posts/2014-07-Conv-Nets-Modular/) -can achieve reasonable performance on hard visual recognition tasks -- -matching or exceeding human performance in some domains. - -Researchers have demonstrated steady progress -in computer vision by validating their work against -[ImageNet](http://www.image-net.org) -- an academic benchmark for computer vision. -Successive models continue to show improvements, each time achieving -a new state-of-the-art result: -[QuocNet], [AlexNet], [Inception (GoogLeNet)], [BN-Inception-v2]. -Researchers both internal and external to Google have published papers describing all -these models but the results are still hard to reproduce. -We're now taking the next step by releasing code for running image recognition -on our latest model, [Inception-v3]. - -[QuocNet]: https://static.googleusercontent.com/media/research.google.com/en//archive/unsupervised_icml2012.pdf -[AlexNet]: https://www.cs.toronto.edu/~fritz/absps/imagenet.pdf -[Inception (GoogLeNet)]: https://arxiv.org/abs/1409.4842 -[BN-Inception-v2]: https://arxiv.org/abs/1502.03167 -[Inception-v3]: https://arxiv.org/abs/1512.00567 - -Inception-v3 is trained for the [ImageNet] Large Visual Recognition Challenge -using the data from 2012. This is a standard task in computer vision, -where models try to classify entire -images into [1000 classes], like "Zebra", "Dalmatian", and "Dishwasher". -For example, here are the results from [AlexNet] classifying some images: - -
- -
- -To compare models, we examine how often the model fails to predict the -correct answer as one of their top 5 guesses -- termed "top-5 error rate". -[AlexNet] achieved by setting a top-5 error rate of 15.3% on the 2012 -validation data set; [Inception (GoogLeNet)] achieved 6.67%; -[BN-Inception-v2] achieved 4.9%; [Inception-v3] reaches 3.46%. - -> How well do humans do on ImageNet Challenge? There's a [blog post] by -Andrej Karpathy who attempted to measure his own performance. He reached -5.1% top-5 error rate. - -[ImageNet]: http://image-net.org/ -[1000 classes]: http://image-net.org/challenges/LSVRC/2014/browse-synsets -[blog post]: https://karpathy.github.io/2014/09/02/what-i-learned-from-competing-against-a-convnet-on-imagenet/ - -This tutorial will teach you how to use [Inception-v3]. You'll learn how to -classify images into [1000 classes] in Python or C++. We'll also discuss how to -extract higher level features from this model which may be reused for other -vision tasks. - -We're excited to see what the community will do with this model. - - -##Usage with Python API - -`classify_image.py` downloads the trained model from `tensorflow.org` -when the program is run for the first time. You'll need about 200M of free space -available on your hard disk. - -Start by cloning the [TensorFlow models repo](https://github.com/tensorflow/models) from GitHub. Run the following commands: - - cd models/tutorials/image/imagenet - python classify_image.py - -The above command will classify a supplied image of a panda bear. - -
- -
- -If the model runs correctly, the script will produce the following output: - - giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca (score = 0.88493) - indri, indris, Indri indri, Indri brevicaudatus (score = 0.00878) - lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens (score = 0.00317) - custard apple (score = 0.00149) - earthstar (score = 0.00127) - -If you wish to supply other JPEG images, you may do so by editing -the `--image_file` argument. - -> If you download the model data to a different directory, you -will need to point `--model_dir` to the directory used. - -## Usage with the C++ API - -You can run the same [Inception-v3] model in C++ for use in production -environments. You can download the archive containing the GraphDef that defines -the model like this (running from the root directory of the TensorFlow -repository): - -```bash -curl -L "https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz" | - tar -C tensorflow/examples/label_image/data -xz -``` - -Next, we need to compile the C++ binary that includes the code to load and run the graph. -If you've followed -@{$install_sources$the instructions to download the source installation of TensorFlow} -for your platform, you should be able to build the example by -running this command from your shell terminal: - -```bash -bazel build tensorflow/examples/label_image/... -``` - -That should create a binary executable that you can then run like this: - -```bash -bazel-bin/tensorflow/examples/label_image/label_image -``` - -This uses the default example image that ships with the framework, and should -output something similar to this: - -``` -I tensorflow/examples/label_image/main.cc:206] military uniform (653): 0.834306 -I tensorflow/examples/label_image/main.cc:206] mortarboard (668): 0.0218692 -I tensorflow/examples/label_image/main.cc:206] academic gown (401): 0.0103579 -I tensorflow/examples/label_image/main.cc:206] pickelhaube (716): 0.00800814 -I tensorflow/examples/label_image/main.cc:206] bulletproof vest (466): 0.00535088 -``` -In this case, we're using the default image of -[Admiral Grace Hopper](https://en.wikipedia.org/wiki/Grace_Hopper), and you can -see the network correctly identifies she's wearing a military uniform, with a high -score of 0.8. - - -
- -
- -Next, try it out on your own images by supplying the --image= argument, e.g. - -```bash -bazel-bin/tensorflow/examples/label_image/label_image --image=my_image.png -``` - -If you look inside the [`tensorflow/examples/label_image/main.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc) -file, you can find out -how it works. We hope this code will help you integrate TensorFlow into -your own applications, so we will walk step by step through the main functions: - -The command line flags control where the files are loaded from, and properties of the input images. -The model expects to get square 299x299 RGB images, so those are the `input_width` -and `input_height` flags. We also need to scale the pixel values from integers that -are between 0 and 255 to the floating point values that the graph operates on. -We control the scaling with the `input_mean` and `input_std` flags: we first subtract -`input_mean` from each pixel value, then divide it by `input_std`. - -These values probably look somewhat magical, but they are just defined by the -original model author based on what he/she wanted to use as input images for -training. If you have a graph that you've trained yourself, you'll just need -to adjust the values to match whatever you used during your training process. - -You can see how they're applied to an image in the -[`ReadTensorFromImageFile()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc#L88) -function. - -```C++ -// Given an image file name, read in the data, try to decode it as an image, -// resize it to the requested size, and then scale the values as desired. -Status ReadTensorFromImageFile(string file_name, const int input_height, - const int input_width, const float input_mean, - const float input_std, - std::vector* out_tensors) { - tensorflow::GraphDefBuilder b; -``` -We start by creating a `GraphDefBuilder`, which is an object we can use to -specify a model to run or load. - -```C++ - string input_name = "file_reader"; - string output_name = "normalized"; - tensorflow::Node* file_reader = - tensorflow::ops::ReadFile(tensorflow::ops::Const(file_name, b.opts()), - b.opts().WithName(input_name)); -``` -We then start creating nodes for the small model we want to run -to load, resize, and scale the pixel values to get the result the main model -expects as its input. The first node we create is just a `Const` op that holds a -tensor with the file name of the image we want to load. That's then passed as the -first input to the `ReadFile` op. You might notice we're passing `b.opts()` as the last -argument to all the op creation functions. The argument ensures that the node is added to -the model definition held in the `GraphDefBuilder`. We also name the `ReadFile` -operator by making the `WithName()` call to `b.opts()`. This gives a name to the node, -which isn't strictly necessary since an automatic name will be assigned if you don't -do this, but it does make debugging a bit easier. - -```C++ - // Now try to figure out what kind of file it is and decode it. - const int wanted_channels = 3; - tensorflow::Node* image_reader; - if (tensorflow::StringPiece(file_name).ends_with(".png")) { - image_reader = tensorflow::ops::DecodePng( - file_reader, - b.opts().WithAttr("channels", wanted_channels).WithName("png_reader")); - } else { - // Assume if it's not a PNG then it must be a JPEG. - image_reader = tensorflow::ops::DecodeJpeg( - file_reader, - b.opts().WithAttr("channels", wanted_channels).WithName("jpeg_reader")); - } - // Now cast the image data to float so we can do normal math on it. - tensorflow::Node* float_caster = tensorflow::ops::Cast( - image_reader, tensorflow::DT_FLOAT, b.opts().WithName("float_caster")); - // The convention for image ops in TensorFlow is that all images are expected - // to be in batches, so that they're four-dimensional arrays with indices of - // [batch, height, width, channel]. Because we only have a single image, we - // have to add a batch dimension of 1 to the start with ExpandDims(). - tensorflow::Node* dims_expander = tensorflow::ops::ExpandDims( - float_caster, tensorflow::ops::Const(0, b.opts()), b.opts()); - // Bilinearly resize the image to fit the required dimensions. - tensorflow::Node* resized = tensorflow::ops::ResizeBilinear( - dims_expander, tensorflow::ops::Const({input_height, input_width}, - b.opts().WithName("size")), - b.opts()); - // Subtract the mean and divide by the scale. - tensorflow::ops::Div( - tensorflow::ops::Sub( - resized, tensorflow::ops::Const({input_mean}, b.opts()), b.opts()), - tensorflow::ops::Const({input_std}, b.opts()), - b.opts().WithName(output_name)); -``` -We then keep adding more nodes, to decode the file data as an image, to cast the -integers into floating point values, to resize it, and then finally to run the -subtraction and division operations on the pixel values. - -```C++ - // This runs the GraphDef network definition that we've just constructed, and - // returns the results in the output tensor. - tensorflow::GraphDef graph; - TF_RETURN_IF_ERROR(b.ToGraphDef(&graph)); -``` -At the end of this we have -a model definition stored in the b variable, which we turn into a full graph -definition with the `ToGraphDef()` function. - -```C++ - std::unique_ptr session( - tensorflow::NewSession(tensorflow::SessionOptions())); - TF_RETURN_IF_ERROR(session->Create(graph)); - TF_RETURN_IF_ERROR(session->Run({}, {output_name}, {}, out_tensors)); - return Status::OK(); -``` -Then we create a @{tf.Session} -object, which is the interface to actually running the graph, and run it, -specifying which node we want to get the output from, and where to put the -output data. - -This gives us a vector of `Tensor` objects, which in this case we know will only be a -single object long. You can think of a `Tensor` as a multi-dimensional array in this -context, and it holds a 299 pixel high, 299 pixel wide, 3 channel image as float -values. If you have your own image-processing framework in your product already, you -should be able to use that instead, as long as you apply the same transformations -before you feed images into the main graph. - -This is a simple example of creating a small TensorFlow graph dynamically in C++, -but for the pre-trained Inception model we want to load a much larger definition from -a file. You can see how we do that in the `LoadGraph()` function. - -```C++ -// Reads a model graph definition from disk, and creates a session object you -// can use to run it. -Status LoadGraph(string graph_file_name, - std::unique_ptr* session) { - tensorflow::GraphDef graph_def; - Status load_graph_status = - ReadBinaryProto(tensorflow::Env::Default(), graph_file_name, &graph_def); - if (!load_graph_status.ok()) { - return tensorflow::errors::NotFound("Failed to load compute graph at '", - graph_file_name, "'"); - } -``` -If you've looked through the image loading code, a lot of the terms should seem familiar. Rather than -using a `GraphDefBuilder` to produce a `GraphDef` object, we load a protobuf file that -directly contains the `GraphDef`. - -```C++ - session->reset(tensorflow::NewSession(tensorflow::SessionOptions())); - Status session_create_status = (*session)->Create(graph_def); - if (!session_create_status.ok()) { - return session_create_status; - } - return Status::OK(); -} -``` -Then we create a Session object from that `GraphDef` and -pass it back to the caller so that they can run it at a later time. - -The `GetTopLabels()` function is a lot like the image loading, except that in this case -we want to take the results of running the main graph, and turn it into a sorted list -of the highest-scoring labels. Just like the image loader, it creates a -`GraphDefBuilder`, adds a couple of nodes to it, and then runs the short graph to get a -pair of output tensors. In this case they represent the sorted scores and index -positions of the highest results. - -```C++ -// Analyzes the output of the Inception graph to retrieve the highest scores and -// their positions in the tensor, which correspond to categories. -Status GetTopLabels(const std::vector& outputs, int how_many_labels, - Tensor* indices, Tensor* scores) { - tensorflow::GraphDefBuilder b; - string output_name = "top_k"; - tensorflow::ops::TopK(tensorflow::ops::Const(outputs[0], b.opts()), - how_many_labels, b.opts().WithName(output_name)); - // This runs the GraphDef network definition that we've just constructed, and - // returns the results in the output tensors. - tensorflow::GraphDef graph; - TF_RETURN_IF_ERROR(b.ToGraphDef(&graph)); - std::unique_ptr session( - tensorflow::NewSession(tensorflow::SessionOptions())); - TF_RETURN_IF_ERROR(session->Create(graph)); - // The TopK node returns two outputs, the scores and their original indices, - // so we have to append :0 and :1 to specify them both. - std::vector out_tensors; - TF_RETURN_IF_ERROR(session->Run({}, {output_name + ":0", output_name + ":1"}, - {}, &out_tensors)); - *scores = out_tensors[0]; - *indices = out_tensors[1]; - return Status::OK(); -``` -The `PrintTopLabels()` function takes those sorted results, and prints them out in a -friendly way. The `CheckTopLabel()` function is very similar, but just makes sure that -the top label is the one we expect, for debugging purposes. - -At the end, [`main()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc#L252) -ties together all of these calls. - -```C++ -int main(int argc, char* argv[]) { - // We need to call this to set up global state for TensorFlow. - tensorflow::port::InitMain(argv[0], &argc, &argv); - Status s = tensorflow::ParseCommandLineFlags(&argc, argv); - if (!s.ok()) { - LOG(ERROR) << "Error parsing command line flags: " << s.ToString(); - return -1; - } - - // First we load and initialize the model. - std::unique_ptr session; - string graph_path = tensorflow::io::JoinPath(FLAGS_root_dir, FLAGS_graph); - Status load_graph_status = LoadGraph(graph_path, &session); - if (!load_graph_status.ok()) { - LOG(ERROR) << load_graph_status; - return -1; - } -``` -We load the main graph. - -```C++ - // Get the image from disk as a float array of numbers, resized and normalized - // to the specifications the main graph expects. - std::vector resized_tensors; - string image_path = tensorflow::io::JoinPath(FLAGS_root_dir, FLAGS_image); - Status read_tensor_status = ReadTensorFromImageFile( - image_path, FLAGS_input_height, FLAGS_input_width, FLAGS_input_mean, - FLAGS_input_std, &resized_tensors); - if (!read_tensor_status.ok()) { - LOG(ERROR) << read_tensor_status; - return -1; - } - const Tensor& resized_tensor = resized_tensors[0]; -``` -Load, resize, and process the input image. - -```C++ - // Actually run the image through the model. - std::vector outputs; - Status run_status = session->Run({{FLAGS_input_layer, resized_tensor}}, - {FLAGS_output_layer}, {}, &outputs); - if (!run_status.ok()) { - LOG(ERROR) << "Running model failed: " << run_status; - return -1; - } -``` -Here we run the loaded graph with the image as an input. - -```C++ - // This is for automated testing to make sure we get the expected result with - // the default settings. We know that label 866 (military uniform) should be - // the top label for the Admiral Hopper image. - if (FLAGS_self_test) { - bool expected_matches; - Status check_status = CheckTopLabel(outputs, 866, &expected_matches); - if (!check_status.ok()) { - LOG(ERROR) << "Running check failed: " << check_status; - return -1; - } - if (!expected_matches) { - LOG(ERROR) << "Self-test failed!"; - return -1; - } - } -``` -For testing purposes we can check to make sure we get the output we expect here. - -```C++ - // Do something interesting with the results we've generated. - Status print_status = PrintTopLabels(outputs, FLAGS_labels); -``` -Finally we print the labels we found. - -```C++ - if (!print_status.ok()) { - LOG(ERROR) << "Running print failed: " << print_status; - return -1; - } -``` - -The error handling here is using TensorFlow's `Status` -object, which is very convenient because it lets you know whether any error has -occurred with the `ok()` checker, and then can be printed out to give a readable error -message. - -In this case we are demonstrating object recognition, but you should be able to -use very similar code on other models you've found or trained yourself, across -all -sorts of domains. We hope this small example gives you some ideas on how to use -TensorFlow within your own products. - -> **EXERCISE**: Transfer learning is the idea that, if you know how to solve a task well, you -should be able to transfer some of that understanding to solving related -problems. One way to perform transfer learning is to remove the final -classification layer of the network and extract -the [next-to-last layer of the CNN](https://arxiv.org/abs/1310.1531), in this case a 2048 dimensional vector. -There's a guide to doing this @{$image_retraining$in the how-to section}. - - -## Resources for Learning More - -To learn about neural networks in general, Michael Nielsen's -[free online book](http://neuralnetworksanddeeplearning.com/chap1.html) -is an excellent resource. For convolutional neural networks in particular, -Chris Olah has some -[nice blog posts](https://colah.github.io/posts/2014-07-Conv-Nets-Modular/), -and Michael Nielsen's book has a -[great chapter](http://neuralnetworksanddeeplearning.com/chap6.html) -covering them. - -To find out more about implementing convolutional neural networks, you can jump -to the TensorFlow @{$deep_cnn$deep convolutional networks tutorial}, -or start a bit more gently with our @{$layers$MNIST starter tutorial}. -Finally, if you want to get up to speed on research in this area, you can -read the recent work of all the papers referenced in this tutorial. - diff --git a/tensorflow/docs_src/tutorials/image_retraining.md b/tensorflow/docs_src/tutorials/image_retraining.md deleted file mode 100644 index 27784eef9c..0000000000 --- a/tensorflow/docs_src/tutorials/image_retraining.md +++ /dev/null @@ -1,4 +0,0 @@ -# How to Retrain Inception's Final Layer for New Categories - -**NOTE: This tutorial has moved to** -https://github.com/tensorflow/hub/tree/master/docs/tutorials/image_retraining.md diff --git a/tensorflow/docs_src/tutorials/images/deep_cnn.md b/tensorflow/docs_src/tutorials/images/deep_cnn.md new file mode 100644 index 0000000000..1590f15eb9 --- /dev/null +++ b/tensorflow/docs_src/tutorials/images/deep_cnn.md @@ -0,0 +1,446 @@ +# Advanced Convolutional Neural Networks + +## Overview + +CIFAR-10 classification is a common benchmark problem in machine learning. The +problem is to classify RGB 32x32 pixel images across 10 categories: +``` +airplane, automobile, bird, cat, deer, dog, frog, horse, ship, and truck. +``` + +For more details refer to the [CIFAR-10 page](https://www.cs.toronto.edu/~kriz/cifar.html) +and a [Tech Report](https://www.cs.toronto.edu/~kriz/learning-features-2009-TR.pdf) +by Alex Krizhevsky. + +### Goals + +The goal of this tutorial is to build a relatively small [convolutional neural +network](https://en.wikipedia.org/wiki/Convolutional_neural_network) (CNN) for +recognizing images. In the process, this tutorial: + +1. Highlights a canonical organization for network architecture, +training and evaluation. +2. Provides a template for constructing larger and more sophisticated models. + +The reason CIFAR-10 was selected was that it is complex enough to exercise +much of TensorFlow's ability to scale to large models. At the same time, +the model is small enough to train fast, which is ideal for trying out +new ideas and experimenting with new techniques. + +### Highlights of the Tutorial +The CIFAR-10 tutorial demonstrates several important constructs for +designing larger and more sophisticated models in TensorFlow: + +* Core mathematical components including @{tf.nn.conv2d$convolution} +([wiki](https://en.wikipedia.org/wiki/Convolution)), +@{tf.nn.relu$rectified linear activations} +([wiki](https://en.wikipedia.org/wiki/Rectifier_(neural_networks))), +@{tf.nn.max_pool$max pooling} +([wiki](https://en.wikipedia.org/wiki/Convolutional_neural_network#Pooling_layer)) +and @{tf.nn.local_response_normalization$local response normalization} +(Chapter 3.3 in +[AlexNet paper](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf)). +* @{$summaries_and_tensorboard$Visualization} +of network activities during training, including input images, +losses and distributions of activations and gradients. +* Routines for calculating the +@{tf.train.ExponentialMovingAverage$moving average} +of learned parameters and using these averages +during evaluation to boost predictive performance. +* Implementation of a +@{tf.train.exponential_decay$learning rate schedule} +that systematically decrements over time. +* Prefetching @{tf.train.shuffle_batch$queues} +for input +data to isolate the model from disk latency and expensive image pre-processing. + +We also provide a [multi-GPU version](#training-a-model-using-multiple-gpu-cards) +of the model which demonstrates: + +* Configuring a model to train across multiple GPU cards in parallel. +* Sharing and updating variables among multiple GPUs. + +We hope that this tutorial provides a launch point for building larger CNNs for +vision tasks on TensorFlow. + +### Model Architecture + +The model in this CIFAR-10 tutorial is a multi-layer architecture consisting of +alternating convolutions and nonlinearities. These layers are followed by fully +connected layers leading into a softmax classifier. The model follows the +architecture described by +[Alex Krizhevsky](https://code.google.com/p/cuda-convnet/), with a few +differences in the top few layers. + +This model achieves a peak performance of about 86% accuracy within a few hours +of training time on a GPU. Please see [below](#evaluating-a-model) and the code +for details. It consists of 1,068,298 learnable parameters and requires about +19.5M multiply-add operations to compute inference on a single image. + +## Code Organization + +The code for this tutorial resides in +[`models/tutorials/image/cifar10/`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/). + +File | Purpose +--- | --- +[`cifar10_input.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10_input.py) | Reads the native CIFAR-10 binary file format. +[`cifar10.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10.py) | Builds the CIFAR-10 model. +[`cifar10_train.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10_train.py) | Trains a CIFAR-10 model on a CPU or GPU. +[`cifar10_multi_gpu_train.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10_multi_gpu_train.py) | Trains a CIFAR-10 model on multiple GPUs. +[`cifar10_eval.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10_eval.py) | Evaluates the predictive performance of a CIFAR-10 model. + + +## CIFAR-10 Model + +The CIFAR-10 network is largely contained in +[`cifar10.py`](https://www.tensorflow.org/code/tensorflow_models/tutorials/image/cifar10/cifar10.py). +The complete training +graph contains roughly 765 operations. We find that we can make the code most +reusable by constructing the graph with the following modules: + +1. [**Model inputs:**](#model-inputs) `inputs()` and `distorted_inputs()` add +operations that read and preprocess CIFAR images for evaluation and training, +respectively. +1. [**Model prediction:**](#model-prediction) `inference()` +adds operations that perform inference, i.e. classification, on supplied images. +1. [**Model training:**](#model-training) `loss()` and `train()` +add operations that compute the loss, +gradients, variable updates and visualization summaries. + +### Model Inputs + +The input part of the model is built by the functions `inputs()` and +`distorted_inputs()` which read images from the CIFAR-10 binary data files. +These files contain fixed byte length records, so we use +@{tf.FixedLengthRecordReader}. +See @{$reading_data#reading-from-files$Reading Data} to +learn more about how the `Reader` class works. + +The images are processed as follows: + +* They are cropped to 24 x 24 pixels, centrally for evaluation or + @{tf.random_crop$randomly} for training. +* They are @{tf.image.per_image_standardization$approximately whitened} + to make the model insensitive to dynamic range. + +For training, we additionally apply a series of random distortions to +artificially increase the data set size: + +* @{tf.image.random_flip_left_right$Randomly flip} the image from left to right. +* Randomly distort the @{tf.image.random_brightness$image brightness}. +* Randomly distort the @{tf.image.random_contrast$image contrast}. + +Please see the @{$python/image$Images} page for the list of +available distortions. We also attach an +@{tf.summary.image} to the images +so that we may visualize them in @{$summaries_and_tensorboard$TensorBoard}. +This is a good practice to verify that inputs are built correctly. + +
+ +
+ +Reading images from disk and distorting them can use a non-trivial amount of +processing time. To prevent these operations from slowing down training, we run +them inside 16 separate threads which continuously fill a TensorFlow +@{tf.train.shuffle_batch$queue}. + +### Model Prediction + +The prediction part of the model is constructed by the `inference()` function +which adds operations to compute the *logits* of the predictions. That part of +the model is organized as follows: + +Layer Name | Description +--- | --- +`conv1` | @{tf.nn.conv2d$convolution} and @{tf.nn.relu$rectified linear} activation. +`pool1` | @{tf.nn.max_pool$max pooling}. +`norm1` | @{tf.nn.local_response_normalization$local response normalization}. +`conv2` | @{tf.nn.conv2d$convolution} and @{tf.nn.relu$rectified linear} activation. +`norm2` | @{tf.nn.local_response_normalization$local response normalization}. +`pool2` | @{tf.nn.max_pool$max pooling}. +`local3` | @{$python/nn$fully connected layer with rectified linear activation}. +`local4` | @{$python/nn$fully connected layer with rectified linear activation}. +`softmax_linear` | linear transformation to produce logits. + +Here is a graph generated from TensorBoard describing the inference operation: + +
+ +
+ +> **EXERCISE**: The output of `inference` are un-normalized logits. Try editing +the network architecture to return normalized predictions using +@{tf.nn.softmax}. + +The `inputs()` and `inference()` functions provide all the components +necessary to perform an evaluation of a model. We now shift our focus towards +building operations for training a model. + +> **EXERCISE:** The model architecture in `inference()` differs slightly from +the CIFAR-10 model specified in +[cuda-convnet](https://code.google.com/p/cuda-convnet/). In particular, the top +layers of Alex's original model are locally connected and not fully connected. +Try editing the architecture to exactly reproduce the locally connected +architecture in the top layer. + +### Model Training + +The usual method for training a network to perform N-way classification is +[multinomial logistic regression](https://en.wikipedia.org/wiki/Multinomial_logistic_regression), +aka. *softmax regression*. Softmax regression applies a +@{tf.nn.softmax$softmax} nonlinearity to the +output of the network and calculates the +@{tf.nn.sparse_softmax_cross_entropy_with_logits$cross-entropy} +between the normalized predictions and the label index. +For regularization, we also apply the usual +@{tf.nn.l2_loss$weight decay} losses to all learned +variables. The objective function for the model is the sum of the cross entropy +loss and all these weight decay terms, as returned by the `loss()` function. + +We visualize it in TensorBoard with a @{tf.summary.scalar}: + +![CIFAR-10 Loss](https://www.tensorflow.org/images/cifar_loss.png "CIFAR-10 Total Loss") + +We train the model using standard +[gradient descent](https://en.wikipedia.org/wiki/Gradient_descent) +algorithm (see @{$python/train$Training} for other methods) +with a learning rate that +@{tf.train.exponential_decay$exponentially decays} +over time. + +![CIFAR-10 Learning Rate Decay](https://www.tensorflow.org/images/cifar_lr_decay.png "CIFAR-10 Learning Rate Decay") + +The `train()` function adds the operations needed to minimize the objective by +calculating the gradient and updating the learned variables (see +@{tf.train.GradientDescentOptimizer} +for details). It returns an operation that executes all the calculations +needed to train and update the model for one batch of images. + +## Launching and Training the Model + +We have built the model, let's now launch it and run the training operation with +the script `cifar10_train.py`. + +```shell +python cifar10_train.py +``` + +> **NOTE:** The first time you run any target in the CIFAR-10 tutorial, +the CIFAR-10 dataset is automatically downloaded. The data set is ~160MB +so you may want to grab a quick cup of coffee for your first run. + +You should see the output: + +```shell +Filling queue with 20000 CIFAR images before starting to train. This will take a few minutes. +2015-11-04 11:45:45.927302: step 0, loss = 4.68 (2.0 examples/sec; 64.221 sec/batch) +2015-11-04 11:45:49.133065: step 10, loss = 4.66 (533.8 examples/sec; 0.240 sec/batch) +2015-11-04 11:45:51.397710: step 20, loss = 4.64 (597.4 examples/sec; 0.214 sec/batch) +2015-11-04 11:45:54.446850: step 30, loss = 4.62 (391.0 examples/sec; 0.327 sec/batch) +2015-11-04 11:45:57.152676: step 40, loss = 4.61 (430.2 examples/sec; 0.298 sec/batch) +2015-11-04 11:46:00.437717: step 50, loss = 4.59 (406.4 examples/sec; 0.315 sec/batch) +... +``` + +The script reports the total loss every 10 steps as well as the speed at which +the last batch of data was processed. A few comments: + +* The first batch of data can be inordinately slow (e.g. several minutes) as the +preprocessing threads fill up the shuffling queue with 20,000 processed CIFAR +images. + +* The reported loss is the average loss of the most recent batch. Remember that +this loss is the sum of the cross entropy and all weight decay terms. + +* Keep an eye on the processing speed of a batch. The numbers shown above were +obtained on a Tesla K40c. If you are running on a CPU, expect slower performance. + + +> **EXERCISE:** When experimenting, it is sometimes annoying that the first +training step can take so long. Try decreasing the number of images that +initially fill up the queue. Search for `min_fraction_of_examples_in_queue` +in `cifar10_input.py`. + +`cifar10_train.py` periodically @{tf.train.Saver$saves} +all model parameters in +@{$guide/saved_model$checkpoint files} +but it does *not* evaluate the model. The checkpoint file +will be used by `cifar10_eval.py` to measure the predictive +performance (see [Evaluating a Model](#evaluating-a-model) below). + + +If you followed the previous steps, then you have now started training +a CIFAR-10 model. [Congratulations!](https://www.youtube.com/watch?v=9bZkp7q19f0) + +The terminal text returned from `cifar10_train.py` provides minimal insight into +how the model is training. We want more insight into the model during training: + +* Is the loss *really* decreasing or is that just noise? +* Is the model being provided appropriate images? +* Are the gradients, activations and weights reasonable? +* What is the learning rate currently at? + +@{$summaries_and_tensorboard$TensorBoard} provides this +functionality, displaying data exported periodically from `cifar10_train.py` via +a +@{tf.summary.FileWriter}. + +For instance, we can watch how the distribution of activations and degree of +sparsity in `local3` features evolve during training: + +
+ + +
+ +Individual loss functions, as well as the total loss, are particularly +interesting to track over time. However, the loss exhibits a considerable amount +of noise due to the small batch size employed by training. In practice we find +it extremely useful to visualize their moving averages in addition to their raw +values. See how the scripts use +@{tf.train.ExponentialMovingAverage} +for this purpose. + +## Evaluating a Model + +Let us now evaluate how well the trained model performs on a hold-out data set. +The model is evaluated by the script `cifar10_eval.py`. It constructs the model +with the `inference()` function and uses all 10,000 images in the evaluation set +of CIFAR-10. It calculates the *precision at 1:* how often the top prediction +matches the true label of the image. + +To monitor how the model improves during training, the evaluation script runs +periodically on the latest checkpoint files created by the `cifar10_train.py`. + +```shell +python cifar10_eval.py +``` + +> Be careful not to run the evaluation and training binary on the same GPU or +else you might run out of memory. Consider running the evaluation on +a separate GPU if available or suspending the training binary while running +the evaluation on the same GPU. + +You should see the output: + +```shell +2015-11-06 08:30:44.391206: precision @ 1 = 0.860 +... +``` + +The script merely returns the precision @ 1 periodically -- in this case +it returned 86% accuracy. `cifar10_eval.py` also +exports summaries that may be visualized in TensorBoard. These summaries +provide additional insight into the model during evaluation. + +The training script calculates the +@{tf.train.ExponentialMovingAverage$moving average} +version of all learned variables. The evaluation script substitutes +all learned model parameters with the moving average version. This +substitution boosts model performance at evaluation time. + +> **EXERCISE:** Employing averaged parameters may boost predictive performance +by about 3% as measured by precision @ 1. Edit `cifar10_eval.py` to not employ +the averaged parameters for the model and verify that the predictive performance +drops. + + +## Training a Model Using Multiple GPU Cards + +Modern workstations may contain multiple GPUs for scientific computation. +TensorFlow can leverage this environment to run the training operation +concurrently across multiple cards. + +Training a model in a parallel, distributed fashion requires +coordinating training processes. For what follows we term *model replica* +to be one copy of a model training on a subset of data. + +Naively employing asynchronous updates of model parameters +leads to sub-optimal training performance +because an individual model replica might be trained on a stale +copy of the model parameters. Conversely, employing fully synchronous +updates will be as slow as the slowest model replica. + +In a workstation with multiple GPU cards, each GPU will have similar speed +and contain enough memory to run an entire CIFAR-10 model. Thus, we opt to +design our training system in the following manner: + +* Place an individual model replica on each GPU. +* Update model parameters synchronously by waiting for all GPUs to finish +processing a batch of data. + +Here is a diagram of this model: + +
+ +
+ +Note that each GPU computes inference as well as the gradients for a unique +batch of data. This setup effectively permits dividing up a larger batch +of data across the GPUs. + +This setup requires that all GPUs share the model parameters. A well-known +fact is that transferring data to and from GPUs is quite slow. For this +reason, we decide to store and update all model parameters on the CPU (see +green box). A fresh set of model parameters is transferred to the GPU +when a new batch of data is processed by all GPUs. + +The GPUs are synchronized in operation. All gradients are accumulated from +the GPUs and averaged (see green box). The model parameters are updated with +the gradients averaged across all model replicas. + +### Placing Variables and Operations on Devices + +Placing operations and variables on devices requires some special +abstractions. + +The first abstraction we require is a function for computing inference and +gradients for a single model replica. In the code we term this abstraction +a "tower". We must set two attributes for each tower: + +* A unique name for all operations within a tower. +@{tf.name_scope} provides +this unique name by prepending a scope. For instance, all operations in +the first tower are prepended with `tower_0`, e.g. `tower_0/conv1/Conv2D`. + +* A preferred hardware device to run the operation within a tower. +@{tf.device} specifies this. For +instance, all operations in the first tower reside within `device('/device:GPU:0')` +scope indicating that they should be run on the first GPU. + +All variables are pinned to the CPU and accessed via +@{tf.get_variable} +in order to share them in a multi-GPU version. +See how-to on @{$variables$Sharing Variables}. + +### Launching and Training the Model on Multiple GPU cards + +If you have several GPU cards installed on your machine you can use them to +train the model faster with the `cifar10_multi_gpu_train.py` script. This +version of the training script parallelizes the model across multiple GPU cards. + +```shell +python cifar10_multi_gpu_train.py --num_gpus=2 +``` + +Note that the number of GPU cards used defaults to 1. Additionally, if only 1 +GPU is available on your machine, all computations will be placed on it, even if +you ask for more. + +> **EXERCISE:** The default settings for `cifar10_train.py` is to +run on a batch size of 128. Try running `cifar10_multi_gpu_train.py` on 2 GPUs +with a batch size of 64 and compare the training speed. + +## Next Steps + +If you are now interested in developing and training your own image +classification system, we recommend forking this tutorial and replacing +components to address your image classification problem. + + +> **EXERCISE:** Download the +[Street View House Numbers (SVHN)](http://ufldl.stanford.edu/housenumbers/) data set. +Fork the CIFAR-10 tutorial and swap in the SVHN as the input data. Try adapting +the network architecture to improve predictive performance. diff --git a/tensorflow/docs_src/tutorials/images/image_recognition.md b/tensorflow/docs_src/tutorials/images/image_recognition.md new file mode 100644 index 0000000000..432d470d0c --- /dev/null +++ b/tensorflow/docs_src/tutorials/images/image_recognition.md @@ -0,0 +1,455 @@ +# Image Recognition + +Our brains make vision seem easy. It doesn't take any effort for humans to +tell apart a lion and a jaguar, read a sign, or recognize a human's face. +But these are actually hard problems to solve with a computer: they only +seem easy because our brains are incredibly good at understanding images. + +In the last few years, the field of machine learning has made tremendous +progress on addressing these difficult problems. In particular, we've +found that a kind of model called a deep +[convolutional neural network](https://colah.github.io/posts/2014-07-Conv-Nets-Modular/) +can achieve reasonable performance on hard visual recognition tasks -- +matching or exceeding human performance in some domains. + +Researchers have demonstrated steady progress +in computer vision by validating their work against +[ImageNet](http://www.image-net.org) -- an academic benchmark for computer vision. +Successive models continue to show improvements, each time achieving +a new state-of-the-art result: +[QuocNet], [AlexNet], [Inception (GoogLeNet)], [BN-Inception-v2]. +Researchers both internal and external to Google have published papers describing all +these models but the results are still hard to reproduce. +We're now taking the next step by releasing code for running image recognition +on our latest model, [Inception-v3]. + +[QuocNet]: https://static.googleusercontent.com/media/research.google.com/en//archive/unsupervised_icml2012.pdf +[AlexNet]: https://www.cs.toronto.edu/~fritz/absps/imagenet.pdf +[Inception (GoogLeNet)]: https://arxiv.org/abs/1409.4842 +[BN-Inception-v2]: https://arxiv.org/abs/1502.03167 +[Inception-v3]: https://arxiv.org/abs/1512.00567 + +Inception-v3 is trained for the [ImageNet] Large Visual Recognition Challenge +using the data from 2012. This is a standard task in computer vision, +where models try to classify entire +images into [1000 classes], like "Zebra", "Dalmatian", and "Dishwasher". +For example, here are the results from [AlexNet] classifying some images: + +
+ +
+ +To compare models, we examine how often the model fails to predict the +correct answer as one of their top 5 guesses -- termed "top-5 error rate". +[AlexNet] achieved by setting a top-5 error rate of 15.3% on the 2012 +validation data set; [Inception (GoogLeNet)] achieved 6.67%; +[BN-Inception-v2] achieved 4.9%; [Inception-v3] reaches 3.46%. + +> How well do humans do on ImageNet Challenge? There's a [blog post] by +Andrej Karpathy who attempted to measure his own performance. He reached +5.1% top-5 error rate. + +[ImageNet]: http://image-net.org/ +[1000 classes]: http://image-net.org/challenges/LSVRC/2014/browse-synsets +[blog post]: https://karpathy.github.io/2014/09/02/what-i-learned-from-competing-against-a-convnet-on-imagenet/ + +This tutorial will teach you how to use [Inception-v3]. You'll learn how to +classify images into [1000 classes] in Python or C++. We'll also discuss how to +extract higher level features from this model which may be reused for other +vision tasks. + +We're excited to see what the community will do with this model. + + +##Usage with Python API + +`classify_image.py` downloads the trained model from `tensorflow.org` +when the program is run for the first time. You'll need about 200M of free space +available on your hard disk. + +Start by cloning the [TensorFlow models repo](https://github.com/tensorflow/models) from GitHub. Run the following commands: + + cd models/tutorials/image/imagenet + python classify_image.py + +The above command will classify a supplied image of a panda bear. + +
+ +
+ +If the model runs correctly, the script will produce the following output: + + giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca (score = 0.88493) + indri, indris, Indri indri, Indri brevicaudatus (score = 0.00878) + lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens (score = 0.00317) + custard apple (score = 0.00149) + earthstar (score = 0.00127) + +If you wish to supply other JPEG images, you may do so by editing +the `--image_file` argument. + +> If you download the model data to a different directory, you +will need to point `--model_dir` to the directory used. + +## Usage with the C++ API + +You can run the same [Inception-v3] model in C++ for use in production +environments. You can download the archive containing the GraphDef that defines +the model like this (running from the root directory of the TensorFlow +repository): + +```bash +curl -L "https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz" | + tar -C tensorflow/examples/label_image/data -xz +``` + +Next, we need to compile the C++ binary that includes the code to load and run the graph. +If you've followed +@{$install_sources$the instructions to download the source installation of TensorFlow} +for your platform, you should be able to build the example by +running this command from your shell terminal: + +```bash +bazel build tensorflow/examples/label_image/... +``` + +That should create a binary executable that you can then run like this: + +```bash +bazel-bin/tensorflow/examples/label_image/label_image +``` + +This uses the default example image that ships with the framework, and should +output something similar to this: + +``` +I tensorflow/examples/label_image/main.cc:206] military uniform (653): 0.834306 +I tensorflow/examples/label_image/main.cc:206] mortarboard (668): 0.0218692 +I tensorflow/examples/label_image/main.cc:206] academic gown (401): 0.0103579 +I tensorflow/examples/label_image/main.cc:206] pickelhaube (716): 0.00800814 +I tensorflow/examples/label_image/main.cc:206] bulletproof vest (466): 0.00535088 +``` +In this case, we're using the default image of +[Admiral Grace Hopper](https://en.wikipedia.org/wiki/Grace_Hopper), and you can +see the network correctly identifies she's wearing a military uniform, with a high +score of 0.8. + + +
+ +
+ +Next, try it out on your own images by supplying the --image= argument, e.g. + +```bash +bazel-bin/tensorflow/examples/label_image/label_image --image=my_image.png +``` + +If you look inside the [`tensorflow/examples/label_image/main.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc) +file, you can find out +how it works. We hope this code will help you integrate TensorFlow into +your own applications, so we will walk step by step through the main functions: + +The command line flags control where the files are loaded from, and properties of the input images. +The model expects to get square 299x299 RGB images, so those are the `input_width` +and `input_height` flags. We also need to scale the pixel values from integers that +are between 0 and 255 to the floating point values that the graph operates on. +We control the scaling with the `input_mean` and `input_std` flags: we first subtract +`input_mean` from each pixel value, then divide it by `input_std`. + +These values probably look somewhat magical, but they are just defined by the +original model author based on what he/she wanted to use as input images for +training. If you have a graph that you've trained yourself, you'll just need +to adjust the values to match whatever you used during your training process. + +You can see how they're applied to an image in the +[`ReadTensorFromImageFile()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc#L88) +function. + +```C++ +// Given an image file name, read in the data, try to decode it as an image, +// resize it to the requested size, and then scale the values as desired. +Status ReadTensorFromImageFile(string file_name, const int input_height, + const int input_width, const float input_mean, + const float input_std, + std::vector* out_tensors) { + tensorflow::GraphDefBuilder b; +``` +We start by creating a `GraphDefBuilder`, which is an object we can use to +specify a model to run or load. + +```C++ + string input_name = "file_reader"; + string output_name = "normalized"; + tensorflow::Node* file_reader = + tensorflow::ops::ReadFile(tensorflow::ops::Const(file_name, b.opts()), + b.opts().WithName(input_name)); +``` +We then start creating nodes for the small model we want to run +to load, resize, and scale the pixel values to get the result the main model +expects as its input. The first node we create is just a `Const` op that holds a +tensor with the file name of the image we want to load. That's then passed as the +first input to the `ReadFile` op. You might notice we're passing `b.opts()` as the last +argument to all the op creation functions. The argument ensures that the node is added to +the model definition held in the `GraphDefBuilder`. We also name the `ReadFile` +operator by making the `WithName()` call to `b.opts()`. This gives a name to the node, +which isn't strictly necessary since an automatic name will be assigned if you don't +do this, but it does make debugging a bit easier. + +```C++ + // Now try to figure out what kind of file it is and decode it. + const int wanted_channels = 3; + tensorflow::Node* image_reader; + if (tensorflow::StringPiece(file_name).ends_with(".png")) { + image_reader = tensorflow::ops::DecodePng( + file_reader, + b.opts().WithAttr("channels", wanted_channels).WithName("png_reader")); + } else { + // Assume if it's not a PNG then it must be a JPEG. + image_reader = tensorflow::ops::DecodeJpeg( + file_reader, + b.opts().WithAttr("channels", wanted_channels).WithName("jpeg_reader")); + } + // Now cast the image data to float so we can do normal math on it. + tensorflow::Node* float_caster = tensorflow::ops::Cast( + image_reader, tensorflow::DT_FLOAT, b.opts().WithName("float_caster")); + // The convention for image ops in TensorFlow is that all images are expected + // to be in batches, so that they're four-dimensional arrays with indices of + // [batch, height, width, channel]. Because we only have a single image, we + // have to add a batch dimension of 1 to the start with ExpandDims(). + tensorflow::Node* dims_expander = tensorflow::ops::ExpandDims( + float_caster, tensorflow::ops::Const(0, b.opts()), b.opts()); + // Bilinearly resize the image to fit the required dimensions. + tensorflow::Node* resized = tensorflow::ops::ResizeBilinear( + dims_expander, tensorflow::ops::Const({input_height, input_width}, + b.opts().WithName("size")), + b.opts()); + // Subtract the mean and divide by the scale. + tensorflow::ops::Div( + tensorflow::ops::Sub( + resized, tensorflow::ops::Const({input_mean}, b.opts()), b.opts()), + tensorflow::ops::Const({input_std}, b.opts()), + b.opts().WithName(output_name)); +``` +We then keep adding more nodes, to decode the file data as an image, to cast the +integers into floating point values, to resize it, and then finally to run the +subtraction and division operations on the pixel values. + +```C++ + // This runs the GraphDef network definition that we've just constructed, and + // returns the results in the output tensor. + tensorflow::GraphDef graph; + TF_RETURN_IF_ERROR(b.ToGraphDef(&graph)); +``` +At the end of this we have +a model definition stored in the b variable, which we turn into a full graph +definition with the `ToGraphDef()` function. + +```C++ + std::unique_ptr session( + tensorflow::NewSession(tensorflow::SessionOptions())); + TF_RETURN_IF_ERROR(session->Create(graph)); + TF_RETURN_IF_ERROR(session->Run({}, {output_name}, {}, out_tensors)); + return Status::OK(); +``` +Then we create a @{tf.Session} +object, which is the interface to actually running the graph, and run it, +specifying which node we want to get the output from, and where to put the +output data. + +This gives us a vector of `Tensor` objects, which in this case we know will only be a +single object long. You can think of a `Tensor` as a multi-dimensional array in this +context, and it holds a 299 pixel high, 299 pixel wide, 3 channel image as float +values. If you have your own image-processing framework in your product already, you +should be able to use that instead, as long as you apply the same transformations +before you feed images into the main graph. + +This is a simple example of creating a small TensorFlow graph dynamically in C++, +but for the pre-trained Inception model we want to load a much larger definition from +a file. You can see how we do that in the `LoadGraph()` function. + +```C++ +// Reads a model graph definition from disk, and creates a session object you +// can use to run it. +Status LoadGraph(string graph_file_name, + std::unique_ptr* session) { + tensorflow::GraphDef graph_def; + Status load_graph_status = + ReadBinaryProto(tensorflow::Env::Default(), graph_file_name, &graph_def); + if (!load_graph_status.ok()) { + return tensorflow::errors::NotFound("Failed to load compute graph at '", + graph_file_name, "'"); + } +``` +If you've looked through the image loading code, a lot of the terms should seem familiar. Rather than +using a `GraphDefBuilder` to produce a `GraphDef` object, we load a protobuf file that +directly contains the `GraphDef`. + +```C++ + session->reset(tensorflow::NewSession(tensorflow::SessionOptions())); + Status session_create_status = (*session)->Create(graph_def); + if (!session_create_status.ok()) { + return session_create_status; + } + return Status::OK(); +} +``` +Then we create a Session object from that `GraphDef` and +pass it back to the caller so that they can run it at a later time. + +The `GetTopLabels()` function is a lot like the image loading, except that in this case +we want to take the results of running the main graph, and turn it into a sorted list +of the highest-scoring labels. Just like the image loader, it creates a +`GraphDefBuilder`, adds a couple of nodes to it, and then runs the short graph to get a +pair of output tensors. In this case they represent the sorted scores and index +positions of the highest results. + +```C++ +// Analyzes the output of the Inception graph to retrieve the highest scores and +// their positions in the tensor, which correspond to categories. +Status GetTopLabels(const std::vector& outputs, int how_many_labels, + Tensor* indices, Tensor* scores) { + tensorflow::GraphDefBuilder b; + string output_name = "top_k"; + tensorflow::ops::TopK(tensorflow::ops::Const(outputs[0], b.opts()), + how_many_labels, b.opts().WithName(output_name)); + // This runs the GraphDef network definition that we've just constructed, and + // returns the results in the output tensors. + tensorflow::GraphDef graph; + TF_RETURN_IF_ERROR(b.ToGraphDef(&graph)); + std::unique_ptr session( + tensorflow::NewSession(tensorflow::SessionOptions())); + TF_RETURN_IF_ERROR(session->Create(graph)); + // The TopK node returns two outputs, the scores and their original indices, + // so we have to append :0 and :1 to specify them both. + std::vector out_tensors; + TF_RETURN_IF_ERROR(session->Run({}, {output_name + ":0", output_name + ":1"}, + {}, &out_tensors)); + *scores = out_tensors[0]; + *indices = out_tensors[1]; + return Status::OK(); +``` +The `PrintTopLabels()` function takes those sorted results, and prints them out in a +friendly way. The `CheckTopLabel()` function is very similar, but just makes sure that +the top label is the one we expect, for debugging purposes. + +At the end, [`main()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc#L252) +ties together all of these calls. + +```C++ +int main(int argc, char* argv[]) { + // We need to call this to set up global state for TensorFlow. + tensorflow::port::InitMain(argv[0], &argc, &argv); + Status s = tensorflow::ParseCommandLineFlags(&argc, argv); + if (!s.ok()) { + LOG(ERROR) << "Error parsing command line flags: " << s.ToString(); + return -1; + } + + // First we load and initialize the model. + std::unique_ptr session; + string graph_path = tensorflow::io::JoinPath(FLAGS_root_dir, FLAGS_graph); + Status load_graph_status = LoadGraph(graph_path, &session); + if (!load_graph_status.ok()) { + LOG(ERROR) << load_graph_status; + return -1; + } +``` +We load the main graph. + +```C++ + // Get the image from disk as a float array of numbers, resized and normalized + // to the specifications the main graph expects. + std::vector resized_tensors; + string image_path = tensorflow::io::JoinPath(FLAGS_root_dir, FLAGS_image); + Status read_tensor_status = ReadTensorFromImageFile( + image_path, FLAGS_input_height, FLAGS_input_width, FLAGS_input_mean, + FLAGS_input_std, &resized_tensors); + if (!read_tensor_status.ok()) { + LOG(ERROR) << read_tensor_status; + return -1; + } + const Tensor& resized_tensor = resized_tensors[0]; +``` +Load, resize, and process the input image. + +```C++ + // Actually run the image through the model. + std::vector outputs; + Status run_status = session->Run({{FLAGS_input_layer, resized_tensor}}, + {FLAGS_output_layer}, {}, &outputs); + if (!run_status.ok()) { + LOG(ERROR) << "Running model failed: " << run_status; + return -1; + } +``` +Here we run the loaded graph with the image as an input. + +```C++ + // This is for automated testing to make sure we get the expected result with + // the default settings. We know that label 866 (military uniform) should be + // the top label for the Admiral Hopper image. + if (FLAGS_self_test) { + bool expected_matches; + Status check_status = CheckTopLabel(outputs, 866, &expected_matches); + if (!check_status.ok()) { + LOG(ERROR) << "Running check failed: " << check_status; + return -1; + } + if (!expected_matches) { + LOG(ERROR) << "Self-test failed!"; + return -1; + } + } +``` +For testing purposes we can check to make sure we get the output we expect here. + +```C++ + // Do something interesting with the results we've generated. + Status print_status = PrintTopLabels(outputs, FLAGS_labels); +``` +Finally we print the labels we found. + +```C++ + if (!print_status.ok()) { + LOG(ERROR) << "Running print failed: " << print_status; + return -1; + } +``` + +The error handling here is using TensorFlow's `Status` +object, which is very convenient because it lets you know whether any error has +occurred with the `ok()` checker, and then can be printed out to give a readable error +message. + +In this case we are demonstrating object recognition, but you should be able to +use very similar code on other models you've found or trained yourself, across +all +sorts of domains. We hope this small example gives you some ideas on how to use +TensorFlow within your own products. + +> **EXERCISE**: Transfer learning is the idea that, if you know how to solve a task well, you +should be able to transfer some of that understanding to solving related +problems. One way to perform transfer learning is to remove the final +classification layer of the network and extract +the [next-to-last layer of the CNN](https://arxiv.org/abs/1310.1531), in this case a 2048 dimensional vector. + + +## Resources for Learning More + +To learn about neural networks in general, Michael Nielsen's +[free online book](http://neuralnetworksanddeeplearning.com/chap1.html) +is an excellent resource. For convolutional neural networks in particular, +Chris Olah has some +[nice blog posts](https://colah.github.io/posts/2014-07-Conv-Nets-Modular/), +and Michael Nielsen's book has a +[great chapter](http://neuralnetworksanddeeplearning.com/chap6.html) +covering them. + +To find out more about implementing convolutional neural networks, you can jump +to the TensorFlow @{$deep_cnn$deep convolutional networks tutorial}, +or start a bit more gently with our @{$layers$MNIST starter tutorial}. +Finally, if you want to get up to speed on research in this area, you can +read the recent work of all the papers referenced in this tutorial. + diff --git a/tensorflow/docs_src/tutorials/images/layers.md b/tensorflow/docs_src/tutorials/images/layers.md new file mode 100644 index 0000000000..12a215b50c --- /dev/null +++ b/tensorflow/docs_src/tutorials/images/layers.md @@ -0,0 +1,694 @@ +# Build a Convolutional Neural Network using Estimators + +The TensorFlow @{tf.layers$`layers` module} provides a high-level API that makes +it easy to construct a neural network. It provides methods that facilitate the +creation of dense (fully connected) layers and convolutional layers, adding +activation functions, and applying dropout regularization. In this tutorial, +you'll learn how to use `layers` to build a convolutional neural network model +to recognize the handwritten digits in the MNIST data set. + +![handwritten digits 0–9 from the MNIST data set](https://www.tensorflow.org/images/mnist_0-9.png) + +**The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) comprises 60,000 +training examples and 10,000 test examples of the handwritten digits 0–9, +formatted as 28x28-pixel monochrome images.** + +## Getting Started + +Let's set up the skeleton for our TensorFlow program. Create a file called +`cnn_mnist.py`, and add the following code: + +```python +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# Imports +import numpy as np +import tensorflow as tf + +tf.logging.set_verbosity(tf.logging.INFO) + +# Our application logic will be added here + +if __name__ == "__main__": + tf.app.run() +``` + +As you work through the tutorial, you'll add code to construct, train, and +evaluate the convolutional neural network. The complete, final code can be +[found here](https://www.tensorflow.org/code/tensorflow/examples/tutorials/layers/cnn_mnist.py). + +## Intro to Convolutional Neural Networks + +Convolutional neural networks (CNNs) are the current state-of-the-art model +architecture for image classification tasks. CNNs apply a series of filters to +the raw pixel data of an image to extract and learn higher-level features, which +the model can then use for classification. CNNs contains three components: + +* **Convolutional layers**, which apply a specified number of convolution + filters to the image. For each subregion, the layer performs a set of + mathematical operations to produce a single value in the output feature map. + Convolutional layers then typically apply a + [ReLU activation function](https://en.wikipedia.org/wiki/Rectifier_\(neural_networks\)) to + the output to introduce nonlinearities into the model. + +* **Pooling layers**, which + [downsample the image data](https://en.wikipedia.org/wiki/Convolutional_neural_network#Pooling_layer) + extracted by the convolutional layers to reduce the dimensionality of the + feature map in order to decrease processing time. A commonly used pooling + algorithm is max pooling, which extracts subregions of the feature map + (e.g., 2x2-pixel tiles), keeps their maximum value, and discards all other + values. + +* **Dense (fully connected) layers**, which perform classification on the + features extracted by the convolutional layers and downsampled by the + pooling layers. In a dense layer, every node in the layer is connected to + every node in the preceding layer. + +Typically, a CNN is composed of a stack of convolutional modules that perform +feature extraction. Each module consists of a convolutional layer followed by a +pooling layer. The last convolutional module is followed by one or more dense +layers that perform classification. The final dense layer in a CNN contains a +single node for each target class in the model (all the possible classes the +model may predict), with a +[softmax](https://en.wikipedia.org/wiki/Softmax_function) activation function to +generate a value between 0–1 for each node (the sum of all these softmax values +is equal to 1). We can interpret the softmax values for a given image as +relative measurements of how likely it is that the image falls into each target +class. + +> Note: For a more comprehensive walkthrough of CNN architecture, see Stanford +> University's +> Convolutional Neural Networks for Visual Recognition course materials.

+ +## Building the CNN MNIST Classifier {#building_the_cnn_mnist_classifier} + +Let's build a model to classify the images in the MNIST dataset using the +following CNN architecture: + +1. **Convolutional Layer #1**: Applies 32 5x5 filters (extracting 5x5-pixel + subregions), with ReLU activation function +2. **Pooling Layer #1**: Performs max pooling with a 2x2 filter and stride of 2 + (which specifies that pooled regions do not overlap) +3. **Convolutional Layer #2**: Applies 64 5x5 filters, with ReLU activation + function +4. **Pooling Layer #2**: Again, performs max pooling with a 2x2 filter and + stride of 2 +5. **Dense Layer #1**: 1,024 neurons, with dropout regularization rate of 0.4 + (probability of 0.4 that any given element will be dropped during training) +6. **Dense Layer #2 (Logits Layer)**: 10 neurons, one for each digit target + class (0–9). + +The `tf.layers` module contains methods to create each of the three layer types +above: + +* `conv2d()`. Constructs a two-dimensional convolutional layer. Takes number + of filters, filter kernel size, padding, and activation function as + arguments. +* `max_pooling2d()`. Constructs a two-dimensional pooling layer using the + max-pooling algorithm. Takes pooling filter size and stride as arguments. +* `dense()`. Constructs a dense layer. Takes number of neurons and activation + function as arguments. + +Each of these methods accepts a tensor as input and returns a transformed tensor +as output. This makes it easy to connect one layer to another: just take the +output from one layer-creation method and supply it as input to another. + +Open `cnn_mnist.py` and add the following `cnn_model_fn` function, which +conforms to the interface expected by TensorFlow's Estimator API (more on this +later in [Create the Estimator](#create-the-estimator)). `cnn_mnist.py` takes +MNIST feature data, labels, and +@{tf.estimator.ModeKeys$model mode} (`TRAIN`, `EVAL`, `PREDICT`) as arguments; +configures the CNN; and returns predictions, loss, and a training operation: + +```python +def cnn_model_fn(features, labels, mode): + """Model function for CNN.""" + # Input Layer + input_layer = tf.reshape(features["x"], [-1, 28, 28, 1]) + + # Convolutional Layer #1 + conv1 = tf.layers.conv2d( + inputs=input_layer, + filters=32, + kernel_size=[5, 5], + padding="same", + activation=tf.nn.relu) + + # Pooling Layer #1 + pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2) + + # Convolutional Layer #2 and Pooling Layer #2 + conv2 = tf.layers.conv2d( + inputs=pool1, + filters=64, + kernel_size=[5, 5], + padding="same", + activation=tf.nn.relu) + pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2) + + # Dense Layer + pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64]) + dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu) + dropout = tf.layers.dropout( + inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) + + # Logits Layer + logits = tf.layers.dense(inputs=dropout, units=10) + + predictions = { + # Generate predictions (for PREDICT and EVAL mode) + "classes": tf.argmax(input=logits, axis=1), + # Add `softmax_tensor` to the graph. It is used for PREDICT and by the + # `logging_hook`. + "probabilities": tf.nn.softmax(logits, name="softmax_tensor") + } + + if mode == tf.estimator.ModeKeys.PREDICT: + return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) + + # Calculate Loss (for both TRAIN and EVAL modes) + loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) + + # Configure the Training Op (for TRAIN mode) + if mode == tf.estimator.ModeKeys.TRAIN: + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) + train_op = optimizer.minimize( + loss=loss, + global_step=tf.train.get_global_step()) + return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) + + # Add evaluation metrics (for EVAL mode) + eval_metric_ops = { + "accuracy": tf.metrics.accuracy( + labels=labels, predictions=predictions["classes"])} + return tf.estimator.EstimatorSpec( + mode=mode, loss=loss, eval_metric_ops=eval_metric_ops) +``` + +The following sections (with headings corresponding to each code block above) +dive deeper into the `tf.layers` code used to create each layer, as well as how +to calculate loss, configure the training op, and generate predictions. If +you're already experienced with CNNs and @{$custom_estimators$TensorFlow `Estimator`s}, +and find the above code intuitive, you may want to skim these sections or just +skip ahead to ["Training and Evaluating the CNN MNIST Classifier"](#train_eval_mnist). + +### Input Layer + +The methods in the `layers` module for creating convolutional and pooling layers +for two-dimensional image data expect input tensors to have a shape of +[batch_size, image_height, image_width, +channels] by default. This behavior can be changed using the data_format parameter; defined as follows: + + +* _`batch_size`_. Size of the subset of examples to use when performing + gradient descent during training. +* _`image_height`_. Height of the example images. +* _`image_width`_. Width of the example images. +* _`channels`_. Number of color channels in the example images. For color + images, the number of channels is 3 (red, green, blue). For monochrome + images, there is just 1 channel (black). +* _`data_format`_. A string, one of `channels_last` (default) or `channels_first`. + `channels_last` corresponds to inputs with shape + `(batch, ..., channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, ...)`. + +Here, our MNIST dataset is composed of monochrome 28x28 pixel images, so the +desired shape for our input layer is [batch_size, 28, 28, +1]. + +To convert our input feature map (`features`) to this shape, we can perform the +following `reshape` operation: + +```python +input_layer = tf.reshape(features["x"], [-1, 28, 28, 1]) +``` + +Note that we've indicated `-1` for batch size, which specifies that this +dimension should be dynamically computed based on the number of input values in +`features["x"]`, holding the size of all other dimensions constant. This allows +us to treat `batch_size` as a hyperparameter that we can tune. For example, if +we feed examples into our model in batches of 5, `features["x"]` will contain +3,920 values (one value for each pixel in each image), and `input_layer` will +have a shape of `[5, 28, 28, 1]`. Similarly, if we feed examples in batches of +100, `features["x"]` will contain 78,400 values, and `input_layer` will have a +shape of `[100, 28, 28, 1]`. + +### Convolutional Layer #1 + +In our first convolutional layer, we want to apply 32 5x5 filters to the input +layer, with a ReLU activation function. We can use the `conv2d()` method in the +`layers` module to create this layer as follows: + +```python +conv1 = tf.layers.conv2d( + inputs=input_layer, + filters=32, + kernel_size=[5, 5], + padding="same", + activation=tf.nn.relu) +``` + +The `inputs` argument specifies our input tensor, which must have the shape +[batch_size, image_height, image_width, +channels]. Here, we're connecting our first convolutional layer +to `input_layer`, which has the shape [batch_size, 28, 28, +1]. + +> Note: conv2d() will instead accept a shape of +> [batch_size, channels, image_height, image_width] when passed the argument +> data_format=channels_first. + +The `filters` argument specifies the number of filters to apply (here, 32), and +`kernel_size` specifies the dimensions of the filters as [height, +width] (here, [5, 5]). + +

TIP: If filter height and width have the same value, you can instead specify a +single integer for kernel_size—e.g., kernel_size=5.

+ +The `padding` argument specifies one of two enumerated values +(case-insensitive): `valid` (default value) or `same`. To specify that the +output tensor should have the same height and width values as the input tensor, +we set `padding=same` here, which instructs TensorFlow to add 0 values to the +edges of the input tensor to preserve height and width of 28. (Without padding, +a 5x5 convolution over a 28x28 tensor will produce a 24x24 tensor, as there are +24x24 locations to extract a 5x5 tile from a 28x28 grid.) + +The `activation` argument specifies the activation function to apply to the +output of the convolution. Here, we specify ReLU activation with +@{tf.nn.relu}. + +Our output tensor produced by `conv2d()` has a shape of +[batch_size, 28, 28, 32]: the same height and width +dimensions as the input, but now with 32 channels holding the output from each +of the filters. + +### Pooling Layer #1 + +Next, we connect our first pooling layer to the convolutional layer we just +created. We can use the `max_pooling2d()` method in `layers` to construct a +layer that performs max pooling with a 2x2 filter and stride of 2: + +```python +pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2) +``` + +Again, `inputs` specifies the input tensor, with a shape of +[batch_size, image_height, image_width, +channels]. Here, our input tensor is `conv1`, the output from +the first convolutional layer, which has a shape of [batch_size, +28, 28, 32]. + +> Note: As with conv2d(), max_pooling2d() will instead +> accept a shape of [batch_size, channels, +> image_height, image_width] when passed the argument +> data_format=channels_first. + +The `pool_size` argument specifies the size of the max pooling filter as +[height, width] (here, `[2, 2]`). If both +dimensions have the same value, you can instead specify a single integer (e.g., +`pool_size=2`). + +The `strides` argument specifies the size of the stride. Here, we set a stride +of 2, which indicates that the subregions extracted by the filter should be +separated by 2 pixels in both the height and width dimensions (for a 2x2 filter, +this means that none of the regions extracted will overlap). If you want to set +different stride values for height and width, you can instead specify a tuple or +list (e.g., `stride=[3, 6]`). + +Our output tensor produced by `max_pooling2d()` (`pool1`) has a shape of +[batch_size, 14, 14, 32]: the 2x2 filter reduces height and width by 50% each. + +### Convolutional Layer #2 and Pooling Layer #2 + +We can connect a second convolutional and pooling layer to our CNN using +`conv2d()` and `max_pooling2d()` as before. For convolutional layer #2, we +configure 64 5x5 filters with ReLU activation, and for pooling layer #2, we use +the same specs as pooling layer #1 (a 2x2 max pooling filter with stride of 2): + +```python +conv2 = tf.layers.conv2d( + inputs=pool1, + filters=64, + kernel_size=[5, 5], + padding="same", + activation=tf.nn.relu) + +pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2) +``` + +Note that convolutional layer #2 takes the output tensor of our first pooling +layer (`pool1`) as input, and produces the tensor `conv2` as output. `conv2` +has a shape of [batch_size, 14, 14, 64], the same height and width as `pool1` (due to `padding="same"`), and 64 channels for the 64 +filters applied. + +Pooling layer #2 takes `conv2` as input, producing `pool2` as output. `pool2` +has shape [batch_size, 7, 7, 64] (50% reduction of height and width from `conv2`). + +### Dense Layer + +Next, we want to add a dense layer (with 1,024 neurons and ReLU activation) to +our CNN to perform classification on the features extracted by the +convolution/pooling layers. Before we connect the layer, however, we'll flatten +our feature map (`pool2`) to shape [batch_size, +features], so that our tensor has only two dimensions: + +```python +pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64]) +``` + +In the `reshape()` operation above, the `-1` signifies that the *`batch_size`* +dimension will be dynamically calculated based on the number of examples in our +input data. Each example has 7 (`pool2` height) * 7 (`pool2` width) * 64 +(`pool2` channels) features, so we want the `features` dimension to have a value +of 7 * 7 * 64 (3136 in total). The output tensor, `pool2_flat`, has shape +[batch_size, 3136]. + +Now, we can use the `dense()` method in `layers` to connect our dense layer as +follows: + +```python +dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu) +``` + +The `inputs` argument specifies the input tensor: our flattened feature map, +`pool2_flat`. The `units` argument specifies the number of neurons in the dense +layer (1,024). The `activation` argument takes the activation function; again, +we'll use `tf.nn.relu` to add ReLU activation. + +To help improve the results of our model, we also apply dropout regularization +to our dense layer, using the `dropout` method in `layers`: + +```python +dropout = tf.layers.dropout( + inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) +``` + +Again, `inputs` specifies the input tensor, which is the output tensor from our +dense layer (`dense`). + +The `rate` argument specifies the dropout rate; here, we use `0.4`, which means +40% of the elements will be randomly dropped out during training. + +The `training` argument takes a boolean specifying whether or not the model is +currently being run in training mode; dropout will only be performed if +`training` is `True`. Here, we check if the `mode` passed to our model function +`cnn_model_fn` is `TRAIN` mode. + +Our output tensor `dropout` has shape [batch_size, 1024]. + +### Logits Layer + +The final layer in our neural network is the logits layer, which will return the +raw values for our predictions. We create a dense layer with 10 neurons (one for +each target class 0–9), with linear activation (the default): + +```python +logits = tf.layers.dense(inputs=dropout, units=10) +``` + +Our final output tensor of the CNN, `logits`, has shape +[batch_size, 10]. + +### Generate Predictions {#generate_predictions} + +The logits layer of our model returns our predictions as raw values in a +[batch_size, 10]-dimensional tensor. Let's convert these +raw values into two different formats that our model function can return: + +* The **predicted class** for each example: a digit from 0–9. +* The **probabilities** for each possible target class for each example: the + probability that the example is a 0, is a 1, is a 2, etc. + +For a given example, our predicted class is the element in the corresponding row +of the logits tensor with the highest raw value. We can find the index of this +element using the @{tf.argmax} +function: + +```python +tf.argmax(input=logits, axis=1) +``` + +The `input` argument specifies the tensor from which to extract maximum +values—here `logits`. The `axis` argument specifies the axis of the `input` +tensor along which to find the greatest value. Here, we want to find the largest +value along the dimension with index of 1, which corresponds to our predictions +(recall that our logits tensor has shape [batch_size, +10]). + +We can derive probabilities from our logits layer by applying softmax activation +using @{tf.nn.softmax}: + +```python +tf.nn.softmax(logits, name="softmax_tensor") +``` + +> Note: We use the `name` argument to explicitly name this operation +> `softmax_tensor`, so we can reference it later. (We'll set up logging for the +> softmax values in ["Set Up a Logging Hook"](#set-up-a-logging-hook)). + +We compile our predictions in a dict, and return an `EstimatorSpec` object: + +```python +predictions = { + "classes": tf.argmax(input=logits, axis=1), + "probabilities": tf.nn.softmax(logits, name="softmax_tensor") +} +if mode == tf.estimator.ModeKeys.PREDICT: + return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) +``` + +### Calculate Loss {#calculating-loss} + +For both training and evaluation, we need to define a +[loss function](https://en.wikipedia.org/wiki/Loss_function) +that measures how closely the model's predictions match the target classes. For +multiclass classification problems like MNIST, +[cross entropy](https://en.wikipedia.org/wiki/Cross_entropy) is typically used +as the loss metric. The following code calculates cross entropy when the model +runs in either `TRAIN` or `EVAL` mode: + +```python +loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) +``` + +Let's take a closer look at what's happening above. + +Our `labels` tensor contains a list of prediction indices for our examples, e.g. `[1, +9, ...]`. `logits` contains the linear outputs of our last layer. + +`tf.losses.sparse_softmax_cross_entropy`, calculates the softmax crossentropy +(aka: categorical crossentropy, negative log-likelihood) from these two inputs +in an efficient, numerically stable way. + + +### Configure the Training Op + +In the previous section, we defined loss for our CNN as the softmax +cross-entropy of the logits layer and our labels. Let's configure our model to +optimize this loss value during training. We'll use a learning rate of 0.001 and +[stochastic gradient descent](https://en.wikipedia.org/wiki/Stochastic_gradient_descent) +as the optimization algorithm: + +```python +if mode == tf.estimator.ModeKeys.TRAIN: + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) + train_op = optimizer.minimize( + loss=loss, + global_step=tf.train.get_global_step()) + return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) +``` + +> Note: For a more in-depth look at configuring training ops for Estimator model +> functions, see @{$custom_estimators#defining-the-training-op-for-the-model$"Defining the training op for the model"} +> in the @{$custom_estimators$"Creating Estimations in tf.estimator"} tutorial. + + +### Add evaluation metrics + +To add accuracy metric in our model, we define `eval_metric_ops` dict in EVAL +mode as follows: + +```python +eval_metric_ops = { + "accuracy": tf.metrics.accuracy( + labels=labels, predictions=predictions["classes"])} +return tf.estimator.EstimatorSpec( + mode=mode, loss=loss, eval_metric_ops=eval_metric_ops) +``` + + +## Training and Evaluating the CNN MNIST Classifier + +We've coded our MNIST CNN model function; now we're ready to train and evaluate +it. + +### Load Training and Test Data + +First, let's load our training and test data. Add a `main()` function to +`cnn_mnist.py` with the following code: + +```python +def main(unused_argv): + # Load training and eval data + mnist = tf.contrib.learn.datasets.load_dataset("mnist") + train_data = mnist.train.images # Returns np.array + train_labels = np.asarray(mnist.train.labels, dtype=np.int32) + eval_data = mnist.test.images # Returns np.array + eval_labels = np.asarray(mnist.test.labels, dtype=np.int32) +``` + +We store the training feature data (the raw pixel values for 55,000 images of +hand-drawn digits) and training labels (the corresponding value from 0–9 for +each image) as [numpy +arrays](https://docs.scipy.org/doc/numpy/reference/generated/numpy.array.html) +in `train_data` and `train_labels`, respectively. Similarly, we store the +evaluation feature data (10,000 images) and evaluation labels in `eval_data` +and `eval_labels`, respectively. + +### Create the Estimator {#create-the-estimator} + +Next, let's create an `Estimator` (a TensorFlow class for performing high-level +model training, evaluation, and inference) for our model. Add the following code +to `main()`: + +```python +# Create the Estimator +mnist_classifier = tf.estimator.Estimator( + model_fn=cnn_model_fn, model_dir="/tmp/mnist_convnet_model") +``` + +The `model_fn` argument specifies the model function to use for training, +evaluation, and prediction; we pass it the `cnn_model_fn` we created in +["Building the CNN MNIST Classifier."](#building-the-cnn-mnist-classifier) The +`model_dir` argument specifies the directory where model data (checkpoints) will +be saved (here, we specify the temp directory `/tmp/mnist_convnet_model`, but +feel free to change to another directory of your choice). + +> Note: For an in-depth walkthrough of the TensorFlow `Estimator` API, see the +> tutorial @{$custom_estimators$"Creating Estimators in tf.estimator."} + +### Set Up a Logging Hook {#set_up_a_logging_hook} + +Since CNNs can take a while to train, let's set up some logging so we can track +progress during training. We can use TensorFlow's @{tf.train.SessionRunHook} to create a +@{tf.train.LoggingTensorHook} +that will log the probability values from the softmax layer of our CNN. Add the +following to `main()`: + +```python +# Set up logging for predictions +tensors_to_log = {"probabilities": "softmax_tensor"} +logging_hook = tf.train.LoggingTensorHook( + tensors=tensors_to_log, every_n_iter=50) +``` + +We store a dict of the tensors we want to log in `tensors_to_log`. Each key is a +label of our choice that will be printed in the log output, and the +corresponding label is the name of a `Tensor` in the TensorFlow graph. Here, our +`probabilities` can be found in `softmax_tensor`, the name we gave our softmax +operation earlier when we generated the probabilities in `cnn_model_fn`. + +> Note: If you don't explicitly assign a name to an operation via the `name` +> argument, TensorFlow will assign a default name. A couple easy ways to +> discover the names applied to operations are to visualize your graph on +> @{$graph_viz$TensorBoard}) or to enable the +> @{$guide/debugger$TensorFlow Debugger (tfdbg)}. + +Next, we create the `LoggingTensorHook`, passing `tensors_to_log` to the +`tensors` argument. We set `every_n_iter=50`, which specifies that probabilities +should be logged after every 50 steps of training. + +### Train the Model + +Now we're ready to train our model, which we can do by creating `train_input_fn` +and calling `train()` on `mnist_classifier`. Add the following to `main()`: + +```python +# Train the model +train_input_fn = tf.estimator.inputs.numpy_input_fn( + x={"x": train_data}, + y=train_labels, + batch_size=100, + num_epochs=None, + shuffle=True) +mnist_classifier.train( + input_fn=train_input_fn, + steps=20000, + hooks=[logging_hook]) +``` + +In the `numpy_input_fn` call, we pass the training feature data and labels to +`x` (as a dict) and `y`, respectively. We set a `batch_size` of `100` (which +means that the model will train on minibatches of 100 examples at each step). +`num_epochs=None` means that the model will train until the specified number of +steps is reached. We also set `shuffle=True` to shuffle the training data. +In the `train` call, we set `steps=20000` +(which means the model will train for 20,000 steps total). We pass our +`logging_hook` to the `hooks` argument, so that it will be triggered during +training. + +### Evaluate the Model + +Once training is complete, we want to evaluate our model to determine its +accuracy on the MNIST test set. We call the `evaluate` method, which evaluates +the metrics we specified in `eval_metric_ops` argument in the `model_fn`. +Add the following to `main()`: + +```python +# Evaluate the model and print results +eval_input_fn = tf.estimator.inputs.numpy_input_fn( + x={"x": eval_data}, + y=eval_labels, + num_epochs=1, + shuffle=False) +eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn) +print(eval_results) +``` + +To create `eval_input_fn`, we set `num_epochs=1`, so that the model evaluates +the metrics over one epoch of data and returns the result. We also set +`shuffle=False` to iterate through the data sequentially. + +### Run the Model + +We've coded the CNN model function, `Estimator`, and the training/evaluation +logic; now let's see the results. Run `cnn_mnist.py`. + +> Note: Training CNNs is quite computationally intensive. Estimated completion +> time of `cnn_mnist.py` will vary depending on your processor, but will likely +> be upwards of 1 hour on CPU. To train more quickly, you can decrease the +> number of `steps` passed to `train()`, but note that this will affect accuracy. + +As the model trains, you'll see log output like the following: + +```python +INFO:tensorflow:loss = 2.36026, step = 1 +INFO:tensorflow:probabilities = [[ 0.07722801 0.08618255 0.09256398, ...]] +... +INFO:tensorflow:loss = 2.13119, step = 101 +INFO:tensorflow:global_step/sec: 5.44132 +... +INFO:tensorflow:Loss for final step: 0.553216. + +INFO:tensorflow:Restored model from /tmp/mnist_convnet_model +INFO:tensorflow:Eval steps [0,inf) for training step 20000. +INFO:tensorflow:Input iterator is exhausted. +INFO:tensorflow:Saving evaluation summary for step 20000: accuracy = 0.9733, loss = 0.0902271 +{'loss': 0.090227105, 'global_step': 20000, 'accuracy': 0.97329998} +``` + +Here, we've achieved an accuracy of 97.3% on our test data set. + +## Additional Resources + +To learn more about TensorFlow Estimators and CNNs in TensorFlow, see the +following resources: + +* @{$custom_estimators$Creating Estimators in tf.estimator} + provides an introduction to the TensorFlow Estimator API. It walks through + configuring an Estimator, writing a model function, calculating loss, and + defining a training op. +* @{$deep_cnn} walks through how to build a MNIST CNN classification model + *without estimators* using lower-level TensorFlow operations. diff --git a/tensorflow/docs_src/tutorials/index.md b/tensorflow/docs_src/tutorials/index.md deleted file mode 100644 index 6bd3a3a897..0000000000 --- a/tensorflow/docs_src/tutorials/index.md +++ /dev/null @@ -1,59 +0,0 @@ -# Tutorials - - -This section contains tutorials demonstrating how to do specific tasks -in TensorFlow. If you are new to TensorFlow, we recommend reading -[Get Started with TensorFlow](/get_started/). - -## Images - -These tutorials cover different aspects of image recognition: - - * @{$layers$MNIST}, which introduces convolutional neural networks (CNNs) and - demonstrates how to build a CNN in TensorFlow. - * @{$image_recognition}, which introduces the field of image recognition and - uses a pre-trained model (Inception) for recognizing images. - * @{$image_retraining}, which has a wonderfully self-explanatory title. - * @{$deep_cnn}, which demonstrates how to build a small CNN for recognizing - images. This tutorial is aimed at advanced TensorFlow users. - - -## Sequences - -These tutorials focus on machine learning problems dealing with sequence data. - - * @{$recurrent}, which demonstrates how to use a - recurrent neural network to predict the next word in a sentence. - * @{$seq2seq}, which demonstrates how to use a - sequence-to-sequence model to translate text from English to French. - * @{$recurrent_quickdraw} - builds a classification model for drawings, directly from the sequence of - pen strokes. - * @{$audio_recognition}, which shows how to - build a basic speech recognition network. - -## Data representation - -These tutorials demonstrate various data representations that can be used in -TensorFlow. - - * @{$wide}, uses - @{tf.feature_column$feature columns} to feed a variety of data types - to linear model, to solve a classification problem. - * @{$wide_and_deep}, builds on the - above linear model tutorial, adding a deep feed-forward neural network - component and a DNN-compatible data representation. - * @{$word2vec}, which demonstrates how to - create an embedding for words. - * @{$kernel_methods}, - which shows how to improve the quality of a linear model by using explicit - kernel mappings. - -## Non Machine Learning - -Although TensorFlow specializes in machine learning, the core of TensorFlow is -a powerful numeric computation system which you can also use to solve other -kinds of math problems. For example: - - * @{$mandelbrot} - * @{$pdes} diff --git a/tensorflow/docs_src/tutorials/keras/basic_classification.md b/tensorflow/docs_src/tutorials/keras/basic_classification.md new file mode 100644 index 0000000000..91bbd85b24 --- /dev/null +++ b/tensorflow/docs_src/tutorials/keras/basic_classification.md @@ -0,0 +1,3 @@ +# Basic Classification + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_classification.ipynb) diff --git a/tensorflow/docs_src/tutorials/keras/basic_regression.md b/tensorflow/docs_src/tutorials/keras/basic_regression.md new file mode 100644 index 0000000000..a535f22f5a --- /dev/null +++ b/tensorflow/docs_src/tutorials/keras/basic_regression.md @@ -0,0 +1,3 @@ +# Basic Regression + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_regression.ipynb) diff --git a/tensorflow/docs_src/tutorials/keras/basic_text_classification.md b/tensorflow/docs_src/tutorials/keras/basic_text_classification.md new file mode 100644 index 0000000000..7c5d4f7896 --- /dev/null +++ b/tensorflow/docs_src/tutorials/keras/basic_text_classification.md @@ -0,0 +1,3 @@ +# Basic Text Classification + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_text_classification.ipynb) diff --git a/tensorflow/docs_src/tutorials/keras/index.md b/tensorflow/docs_src/tutorials/keras/index.md new file mode 100644 index 0000000000..9d42281c8f --- /dev/null +++ b/tensorflow/docs_src/tutorials/keras/index.md @@ -0,0 +1,22 @@ +# Learn and use machine learning + +This notebook collection is inspired by the book +*[Deep Learning with Python](https://books.google.com/books?id=Yo3CAQAACAAJ)*. +These tutorials use `tf.keras`, TensorFlow's high-level Python API for building +and training deep learning models. To learn more about using Keras with +TensorFlow, see the [TensorFlow Keras Guide](../../guide/keras). + +Publisher's note: *Deep Learning with Python* introduces the field of deep +learning using the Python language and the powerful Keras library. Written by +Keras creator and Google AI researcher François Chollet, this book builds your +understanding through intuitive explanations and practical examples. + +To learn about machine learning fundamentals and concepts, consider taking the +[Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/). +Additional TensorFlow and machine learning resources are listed in [next steps](../next_steps). + +1. [Basic classification](./basic_classification) +2. [Text classification](./basic_text_classification) +3. [Regression](./basic_regression) +4. [Overfitting and underfitting](./overfit_and_underfit) +5. [Save and restore models](./save_and_restore_models) diff --git a/tensorflow/docs_src/tutorials/keras/overfit_and_underfit.md b/tensorflow/docs_src/tutorials/keras/overfit_and_underfit.md new file mode 100644 index 0000000000..e5b5ae7b5a --- /dev/null +++ b/tensorflow/docs_src/tutorials/keras/overfit_and_underfit.md @@ -0,0 +1,3 @@ +# Overfitting and Underfitting + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/overfit_and_underfit.ipynb) diff --git a/tensorflow/docs_src/tutorials/keras/save_and_restore_models.md b/tensorflow/docs_src/tutorials/keras/save_and_restore_models.md new file mode 100644 index 0000000000..44b3772945 --- /dev/null +++ b/tensorflow/docs_src/tutorials/keras/save_and_restore_models.md @@ -0,0 +1,3 @@ +# Save and restore Models + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/save_and_restore_models.ipynb) diff --git a/tensorflow/docs_src/tutorials/kernel_methods.md b/tensorflow/docs_src/tutorials/kernel_methods.md deleted file mode 100644 index 205e2a2d2c..0000000000 --- a/tensorflow/docs_src/tutorials/kernel_methods.md +++ /dev/null @@ -1,304 +0,0 @@ -# Improving Linear Models Using Explicit Kernel Methods - -Note: This document uses a deprecated version of @{tf.estimator}, -which has a @{tf.contrib.learn.Estimator$different interface}. -It also uses other `contrib` methods whose -@{$version_compat#not_covered$API may not be stable}. - -In this tutorial, we demonstrate how combining (explicit) kernel methods with -linear models can drastically increase the latters' quality of predictions -without significantly increasing training and inference times. Unlike dual -kernel methods, explicit (primal) kernel methods scale well with the size of the -training dataset both in terms of training/inference times and in terms of -memory requirements. - -**Intended audience:** Even though we provide a high-level overview of concepts -related to explicit kernel methods, this tutorial primarily targets readers who -already have at least basic knowledge of kernel methods and Support Vector -Machines (SVMs). If you are new to kernel methods, refer to either of the -following sources for an introduction: - -* If you have a strong mathematical background: -[Kernel Methods in Machine Learning](https://arxiv.org/pdf/math/0701907.pdf) -* [Kernel method wikipedia page](https://en.wikipedia.org/wiki/Kernel_method) - -Currently, TensorFlow supports explicit kernel mappings for dense features only; -TensorFlow will provide support for sparse features at a later release. - -This tutorial uses [tf.contrib.learn](https://www.tensorflow.org/code/tensorflow/contrib/learn/python/learn) -(TensorFlow's high-level Machine Learning API) Estimators for our ML models. -If you are not familiar with this API, [tf.estimator Quickstart](https://www.tensorflow.org/get_started/estimator) -is a good place to start. We will use the MNIST dataset. The tutorial consists -of the following steps: - -* Load and prepare MNIST data for classification. -* Construct a simple linear model, train it, and evaluate it on the eval data. -* Replace the linear model with a kernelized linear model, re-train, and -re-evaluate. - -## Load and prepare MNIST data for classification -Run the following utility command to load the MNIST dataset: - -```python -data = tf.contrib.learn.datasets.mnist.load_mnist() -``` -The preceding method loads the entire MNIST dataset (containing 70K samples) and -splits it into train, validation, and test data with 55K, 5K, and 10K samples -respectively. Each split contains one numpy array for images (with shape -[sample_size, 784]) and one for labels (with shape [sample_size, 1]). In this -tutorial, we only use the train and validation splits to train and evaluate our -models respectively. - -In order to feed data to a `tf.contrib.learn Estimator`, it is helpful to convert -it to Tensors. For this, we will use an `input function` which adds Ops to the -TensorFlow graph that, when executed, create mini-batches of Tensors to be used -downstream. For more background on input functions, check -@{$premade_estimators#create_input_functions$this section on input functions}. -In this example, we will use the `tf.train.shuffle_batch` Op which, besides -converting numpy arrays to Tensors, allows us to specify the batch_size and -whether to randomize the input every time the input_fn Ops are executed -(randomization typically expedites convergence during training). The full code -for loading and preparing the data is shown in the snippet below. In this -example, we use mini-batches of size 256 for training and the entire sample -(5K entries) for evaluation. Feel free to experiment with different batch sizes. - -```python -import numpy as np -import tensorflow as tf - -def get_input_fn(dataset_split, batch_size, capacity=10000, min_after_dequeue=3000): - - def _input_fn(): - images_batch, labels_batch = tf.train.shuffle_batch( - tensors=[dataset_split.images, dataset_split.labels.astype(np.int32)], - batch_size=batch_size, - capacity=capacity, - min_after_dequeue=min_after_dequeue, - enqueue_many=True, - num_threads=4) - features_map = {'images': images_batch} - return features_map, labels_batch - - return _input_fn - -data = tf.contrib.learn.datasets.mnist.load_mnist() - -train_input_fn = get_input_fn(data.train, batch_size=256) -eval_input_fn = get_input_fn(data.validation, batch_size=5000) - -``` - -## Training a simple linear model -We can now train a linear model over the MNIST dataset. We will use the -@{tf.contrib.learn.LinearClassifier} estimator with 10 classes representing the -10 digits. The input features form a 784-dimensional dense vector which can -be specified as follows: - -```python -image_column = tf.contrib.layers.real_valued_column('images', dimension=784) -``` - -The full code for constructing, training and evaluating a LinearClassifier -estimator is as follows: - -```python -import time - -# Specify the feature(s) to be used by the estimator. -image_column = tf.contrib.layers.real_valued_column('images', dimension=784) -estimator = tf.contrib.learn.LinearClassifier(feature_columns=[image_column], n_classes=10) - -# Train. -start = time.time() -estimator.fit(input_fn=train_input_fn, steps=2000) -end = time.time() -print('Elapsed time: {} seconds'.format(end - start)) - -# Evaluate and report metrics. -eval_metrics = estimator.evaluate(input_fn=eval_input_fn, steps=1) -print(eval_metrics) -``` -The following table summarizes the results on the eval data. - -metric | value -:------------ | :------------ -loss | 0.25 to 0.30 -accuracy | 92.5% -training time | ~25 seconds on my machine - -Note: Metrics will vary depending on various factors. - -In addition to experimenting with the (training) batch size and the number of -training steps, there are a couple other parameters that can be tuned as well. -For instance, you can change the optimization method used to minimize the loss -by explicitly selecting another optimizer from the collection of -[available optimizers](https://www.tensorflow.org/code/tensorflow/python/training). -As an example, the following code constructs a LinearClassifier estimator that -uses the Follow-The-Regularized-Leader (FTRL) optimization strategy with a -specific learning rate and L2-regularization. - - -```python -optimizer = tf.train.FtrlOptimizer(learning_rate=5.0, l2_regularization_strength=1.0) -estimator = tf.contrib.learn.LinearClassifier( - feature_columns=[image_column], n_classes=10, optimizer=optimizer) -``` - -Regardless of the values of the parameters, the maximum accuracy a linear model -can achieve on this dataset caps at around **93%**. - -## Using explicit kernel mappings with the linear model. -The relatively high error (~7%) of the linear model over MNIST indicates that -the input data is not linearly separable. We will use explicit kernel mappings -to reduce the classification error. - -**Intuition:** The high-level idea is to use a non-linear map to transform the -input space to another feature space (of possibly higher dimension) where the -(transformed) features are (almost) linearly separable and then apply a linear -model on the mapped features. This is shown in the following figure: - -
- -
- - -### Technical details -In this example we will use **Random Fourier Features**, introduced in the -["Random Features for Large-Scale Kernel Machines"](https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf) -paper by Rahimi and Recht, to map the input data. Random Fourier Features map a -vector \\(\mathbf{x} \in \mathbb{R}^d\\) to \\(\mathbf{x'} \in \mathbb{R}^D\\) -via the following mapping: - -$$ -RFFM(\cdot): \mathbb{R}^d \to \mathbb{R}^D, \quad -RFFM(\mathbf{x}) = \cos(\mathbf{\Omega} \cdot \mathbf{x}+ \mathbf{b}) -$$ - -where \\(\mathbf{\Omega} \in \mathbb{R}^{D \times d}\\), -\\(\mathbf{x} \in \mathbb{R}^d,\\) \\(\mathbf{b} \in \mathbb{R}^D\\) and the -cosine is applied element-wise. - -In this example, the entries of \\(\mathbf{\Omega}\\) and \\(\mathbf{b}\\) are -sampled from distributions such that the mapping satisfies the following -property: - -$$ -RFFM(\mathbf{x})^T \cdot RFFM(\mathbf{y}) \approx -e^{-\frac{\|\mathbf{x} - \mathbf{y}\|^2}{2 \sigma^2}} -$$ - -The right-hand-side quantity of the expression above is known as the RBF (or -Gaussian) kernel function. This function is one of the most-widely used kernel -functions in Machine Learning and implicitly measures similarity in a different, -much higher dimensional space than the original one. See -[Radial basis function kernel](https://en.wikipedia.org/wiki/Radial_basis_function_kernel) -for more details. - -### Kernel classifier -@{tf.contrib.kernel_methods.KernelLinearClassifier} is a pre-packaged -`tf.contrib.learn` estimator that combines the power of explicit kernel mappings -with linear models. Its constructor is almost identical to that of the -LinearClassifier estimator with the additional option to specify a list of -explicit kernel mappings to be applied to each feature the classifier uses. The -following code snippet demonstrates how to replace LinearClassifier with -KernelLinearClassifier. - - -```python -# Specify the feature(s) to be used by the estimator. This is identical to the -# code used for the LinearClassifier. -image_column = tf.contrib.layers.real_valued_column('images', dimension=784) -optimizer = tf.train.FtrlOptimizer( - learning_rate=50.0, l2_regularization_strength=0.001) - - -kernel_mapper = tf.contrib.kernel_methods.RandomFourierFeatureMapper( - input_dim=784, output_dim=2000, stddev=5.0, name='rffm') -kernel_mappers = {image_column: [kernel_mapper]} -estimator = tf.contrib.kernel_methods.KernelLinearClassifier( - n_classes=10, optimizer=optimizer, kernel_mappers=kernel_mappers) - -# Train. -start = time.time() -estimator.fit(input_fn=train_input_fn, steps=2000) -end = time.time() -print('Elapsed time: {} seconds'.format(end - start)) - -# Evaluate and report metrics. -eval_metrics = estimator.evaluate(input_fn=eval_input_fn, steps=1) -print(eval_metrics) -``` -The only additional parameter passed to `KernelLinearClassifier` is a dictionary -from feature_columns to a list of kernel mappings to be applied to the -corresponding feature column. The following lines instruct the classifier to -first map the initial 784-dimensional images to 2000-dimensional vectors using -random Fourier features and then learn a linear model on the transformed -vectors: - -```python -kernel_mapper = tf.contrib.kernel_methods.RandomFourierFeatureMapper( - input_dim=784, output_dim=2000, stddev=5.0, name='rffm') -kernel_mappers = {image_column: [kernel_mapper]} -estimator = tf.contrib.kernel_methods.KernelLinearClassifier( - n_classes=10, optimizer=optimizer, kernel_mappers=kernel_mappers) -``` -Notice the `stddev` parameter. This is the standard deviation (\\(\sigma\\)) of -the approximated RBF kernel and controls the similarity measure used in -classification. `stddev` is typically determined via hyperparameter tuning. - -The results of running the preceding code are summarized in the following table. -We can further increase the accuracy by increasing the output dimension of the -mapping and tuning the standard deviation. - -metric | value -:------------ | :------------ -loss | 0.10 -accuracy | 97% -training time | ~35 seconds on my machine - - -### stddev -The classification quality is very sensitive to the value of stddev. The -following table shows the accuracy of the classifier on the eval data for -different values of stddev. The optimal value is stddev=5.0. Notice how too -small or too high stddev values can dramatically decrease the accuracy of the -classification. - -stddev | eval accuracy -:----- | :------------ -1.0 | 0.1362 -2.0 | 0.4764 -4.0 | 0.9654 -5.0 | 0.9766 -8.0 | 0.9714 -16.0 | 0.8878 - -### Output dimension -Intuitively, the larger the output dimension of the mapping, the closer the -inner product of two mapped vectors approximates the kernel, which typically -translates to better classification accuracy. Another way to think about this is -that the output dimension equals the number of weights of the linear model; the -larger this dimension, the larger the "degrees of freedom" of the model. -However, after a certain threshold, higher output dimensions increase the -accuracy by very little, while making training take more time. This is shown in -the following two Figures which depict the eval accuracy as a function of the -output dimension and the training time, respectively. - -![image](https://www.tensorflow.org/versions/master/images/acc_vs_outdim.png) -![image](https://www.tensorflow.org/versions/master/images/acc-vs-trn_time.png) - - -## Summary -Explicit kernel mappings combine the predictive power of nonlinear models with -the scalability of linear models. Unlike traditional dual kernel methods, -explicit kernel methods can scale to millions or hundreds of millions of -samples. When using explicit kernel mappings, consider the following tips: - -* Random Fourier Features can be particularly effective for datasets with dense -features. -* The parameters of the kernel mapping are often data-dependent. Model quality -can be very sensitive to these parameters. Use hyperparameter tuning to find the -optimal values. -* If you have multiple numerical features, concatenate them into a single -multi-dimensional feature and apply the kernel mapping to the concatenated -vector. diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md deleted file mode 100644 index 212e337637..0000000000 --- a/tensorflow/docs_src/tutorials/layers.md +++ /dev/null @@ -1,727 +0,0 @@ -# A Guide to TF Layers: Building a Convolutional Neural Network - -The TensorFlow @{tf.layers$`layers` module} provides a high-level API that makes -it easy to construct a neural network. It provides methods that facilitate the -creation of dense (fully connected) layers and convolutional layers, adding -activation functions, and applying dropout regularization. In this tutorial, -you'll learn how to use `layers` to build a convolutional neural network model -to recognize the handwritten digits in the MNIST data set. - -![handwritten digits 0–9 from the MNIST data set](https://www.tensorflow.org/images/mnist_0-9.png) - -**The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) comprises 60,000 -training examples and 10,000 test examples of the handwritten digits 0–9, -formatted as 28x28-pixel monochrome images.** - -## Getting Started - -Let's set up the skeleton for our TensorFlow program. Create a file called -`cnn_mnist.py`, and add the following code: - -```python -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# Imports -import numpy as np -import tensorflow as tf - -tf.logging.set_verbosity(tf.logging.INFO) - -# Our application logic will be added here - -if __name__ == "__main__": - tf.app.run() -``` - -As you work through the tutorial, you'll add code to construct, train, and -evaluate the convolutional neural network. The complete, final code can be -[found here](https://www.tensorflow.org/code/tensorflow/examples/tutorials/layers/cnn_mnist.py). - -## Intro to Convolutional Neural Networks - -Convolutional neural networks (CNNs) are the current state-of-the-art model -architecture for image classification tasks. CNNs apply a series of filters to -the raw pixel data of an image to extract and learn higher-level features, which -the model can then use for classification. CNNs contains three components: - -* **Convolutional layers**, which apply a specified number of convolution - filters to the image. For each subregion, the layer performs a set of - mathematical operations to produce a single value in the output feature map. - Convolutional layers then typically apply a - [ReLU activation function](https://en.wikipedia.org/wiki/Rectifier_\(neural_networks\)) to - the output to introduce nonlinearities into the model. - -* **Pooling layers**, which - [downsample the image data](https://en.wikipedia.org/wiki/Convolutional_neural_network#Pooling_layer) - extracted by the convolutional layers to reduce the dimensionality of the - feature map in order to decrease processing time. A commonly used pooling - algorithm is max pooling, which extracts subregions of the feature map - (e.g., 2x2-pixel tiles), keeps their maximum value, and discards all other - values. - -* **Dense (fully connected) layers**, which perform classification on the - features extracted by the convolutional layers and downsampled by the - pooling layers. In a dense layer, every node in the layer is connected to - every node in the preceding layer. - -Typically, a CNN is composed of a stack of convolutional modules that perform -feature extraction. Each module consists of a convolutional layer followed by a -pooling layer. The last convolutional module is followed by one or more dense -layers that perform classification. The final dense layer in a CNN contains a -single node for each target class in the model (all the possible classes the -model may predict), with a -[softmax](https://en.wikipedia.org/wiki/Softmax_function) activation function to -generate a value between 0–1 for each node (the sum of all these softmax values -is equal to 1). We can interpret the softmax values for a given image as -relative measurements of how likely it is that the image falls into each target -class. - -> Note: For a more comprehensive walkthrough of CNN architecture, see Stanford -> University's -> Convolutional Neural Networks for Visual Recognition course materials.

- -## Building the CNN MNIST Classifier {#building_the_cnn_mnist_classifier} - -Let's build a model to classify the images in the MNIST dataset using the -following CNN architecture: - -1. **Convolutional Layer #1**: Applies 32 5x5 filters (extracting 5x5-pixel - subregions), with ReLU activation function -2. **Pooling Layer #1**: Performs max pooling with a 2x2 filter and stride of 2 - (which specifies that pooled regions do not overlap) -3. **Convolutional Layer #2**: Applies 64 5x5 filters, with ReLU activation - function -4. **Pooling Layer #2**: Again, performs max pooling with a 2x2 filter and - stride of 2 -5. **Dense Layer #1**: 1,024 neurons, with dropout regularization rate of 0.4 - (probability of 0.4 that any given element will be dropped during training) -6. **Dense Layer #2 (Logits Layer)**: 10 neurons, one for each digit target - class (0–9). - -The `tf.layers` module contains methods to create each of the three layer types -above: - -* `conv2d()`. Constructs a two-dimensional convolutional layer. Takes number - of filters, filter kernel size, padding, and activation function as - arguments. -* `max_pooling2d()`. Constructs a two-dimensional pooling layer using the - max-pooling algorithm. Takes pooling filter size and stride as arguments. -* `dense()`. Constructs a dense layer. Takes number of neurons and activation - function as arguments. - -Each of these methods accepts a tensor as input and returns a transformed tensor -as output. This makes it easy to connect one layer to another: just take the -output from one layer-creation method and supply it as input to another. - -Open `cnn_mnist.py` and add the following `cnn_model_fn` function, which -conforms to the interface expected by TensorFlow's Estimator API (more on this -later in [Create the Estimator](#create-the-estimator)). `cnn_mnist.py` takes -MNIST feature data, labels, and -@{tf.estimator.ModeKeys$model mode} (`TRAIN`, `EVAL`, `PREDICT`) as arguments; -configures the CNN; and returns predictions, loss, and a training operation: - -```python -def cnn_model_fn(features, labels, mode): - """Model function for CNN.""" - # Input Layer - input_layer = tf.reshape(features["x"], [-1, 28, 28, 1]) - - # Convolutional Layer #1 - conv1 = tf.layers.conv2d( - inputs=input_layer, - filters=32, - kernel_size=[5, 5], - padding="same", - activation=tf.nn.relu) - - # Pooling Layer #1 - pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2) - - # Convolutional Layer #2 and Pooling Layer #2 - conv2 = tf.layers.conv2d( - inputs=pool1, - filters=64, - kernel_size=[5, 5], - padding="same", - activation=tf.nn.relu) - pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2) - - # Dense Layer - pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64]) - dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu) - dropout = tf.layers.dropout( - inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) - - # Logits Layer - logits = tf.layers.dense(inputs=dropout, units=10) - - predictions = { - # Generate predictions (for PREDICT and EVAL mode) - "classes": tf.argmax(input=logits, axis=1), - # Add `softmax_tensor` to the graph. It is used for PREDICT and by the - # `logging_hook`. - "probabilities": tf.nn.softmax(logits, name="softmax_tensor") - } - - if mode == tf.estimator.ModeKeys.PREDICT: - return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) - - # Calculate Loss (for both TRAIN and EVAL modes) - loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) - - # Configure the Training Op (for TRAIN mode) - if mode == tf.estimator.ModeKeys.TRAIN: - optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) - train_op = optimizer.minimize( - loss=loss, - global_step=tf.train.get_global_step()) - return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) - - # Add evaluation metrics (for EVAL mode) - eval_metric_ops = { - "accuracy": tf.metrics.accuracy( - labels=labels, predictions=predictions["classes"])} - return tf.estimator.EstimatorSpec( - mode=mode, loss=loss, eval_metric_ops=eval_metric_ops) -``` - -The following sections (with headings corresponding to each code block above) -dive deeper into the `tf.layers` code used to create each layer, as well as how -to calculate loss, configure the training op, and generate predictions. If -you're already experienced with CNNs and @{$custom_estimators$TensorFlow `Estimator`s}, -and find the above code intuitive, you may want to skim these sections or just -skip ahead to ["Training and Evaluating the CNN MNIST Classifier"](#train_eval_mnist). - -### Input Layer - -The methods in the `layers` module for creating convolutional and pooling layers -for two-dimensional image data expect input tensors to have a shape of -[batch_size, image_height, image_width, -channels] by default. This behavior can be changed using the data_format parameter; defined as follows: - - -* _`batch_size`_. Size of the subset of examples to use when performing - gradient descent during training. -* _`image_height`_. Height of the example images. -* _`image_width`_. Width of the example images. -* _`channels`_. Number of color channels in the example images. For color - images, the number of channels is 3 (red, green, blue). For monochrome - images, there is just 1 channel (black). -* _`data_format`_. A string, one of `channels_last` (default) or `channels_first`. - `channels_last` corresponds to inputs with shape - `(batch, ..., channels)` while `channels_first` corresponds to - inputs with shape `(batch, channels, ...)`. - -Here, our MNIST dataset is composed of monochrome 28x28 pixel images, so the -desired shape for our input layer is [batch_size, 28, 28, -1]. - -To convert our input feature map (`features`) to this shape, we can perform the -following `reshape` operation: - -```python -input_layer = tf.reshape(features["x"], [-1, 28, 28, 1]) -``` - -Note that we've indicated `-1` for batch size, which specifies that this -dimension should be dynamically computed based on the number of input values in -`features["x"]`, holding the size of all other dimensions constant. This allows -us to treat `batch_size` as a hyperparameter that we can tune. For example, if -we feed examples into our model in batches of 5, `features["x"]` will contain -3,920 values (one value for each pixel in each image), and `input_layer` will -have a shape of `[5, 28, 28, 1]`. Similarly, if we feed examples in batches of -100, `features["x"]` will contain 78,400 values, and `input_layer` will have a -shape of `[100, 28, 28, 1]`. - -### Convolutional Layer #1 - -In our first convolutional layer, we want to apply 32 5x5 filters to the input -layer, with a ReLU activation function. We can use the `conv2d()` method in the -`layers` module to create this layer as follows: - -```python -conv1 = tf.layers.conv2d( - inputs=input_layer, - filters=32, - kernel_size=[5, 5], - padding="same", - activation=tf.nn.relu) -``` - -The `inputs` argument specifies our input tensor, which must have the shape -[batch_size, image_height, image_width, -channels]. Here, we're connecting our first convolutional layer -to `input_layer`, which has the shape [batch_size, 28, 28, -1]. - -> Note: conv2d() will instead accept a shape of -> [batch_size, channels, image_height, image_width] when passed the argument -> data_format=channels_first. - -The `filters` argument specifies the number of filters to apply (here, 32), and -`kernel_size` specifies the dimensions of the filters as [height, -width] (here, [5, 5]). - -

TIP: If filter height and width have the same value, you can instead specify a -single integer for kernel_size—e.g., kernel_size=5.

- -The `padding` argument specifies one of two enumerated values -(case-insensitive): `valid` (default value) or `same`. To specify that the -output tensor should have the same height and width values as the input tensor, -we set `padding=same` here, which instructs TensorFlow to add 0 values to the -edges of the input tensor to preserve height and width of 28. (Without padding, -a 5x5 convolution over a 28x28 tensor will produce a 24x24 tensor, as there are -24x24 locations to extract a 5x5 tile from a 28x28 grid.) - -The `activation` argument specifies the activation function to apply to the -output of the convolution. Here, we specify ReLU activation with -@{tf.nn.relu}. - -Our output tensor produced by `conv2d()` has a shape of -[batch_size, 28, 28, 32]: the same height and width -dimensions as the input, but now with 32 channels holding the output from each -of the filters. - -### Pooling Layer #1 - -Next, we connect our first pooling layer to the convolutional layer we just -created. We can use the `max_pooling2d()` method in `layers` to construct a -layer that performs max pooling with a 2x2 filter and stride of 2: - -```python -pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2) -``` - -Again, `inputs` specifies the input tensor, with a shape of -[batch_size, image_height, image_width, -channels]. Here, our input tensor is `conv1`, the output from -the first convolutional layer, which has a shape of [batch_size, -28, 28, 32]. - -> Note: As with conv2d(), max_pooling2d() will instead -> accept a shape of [batch_size, channels, -> image_height, image_width] when passed the argument -> data_format=channels_first. - -The `pool_size` argument specifies the size of the max pooling filter as -[height, width] (here, `[2, 2]`). If both -dimensions have the same value, you can instead specify a single integer (e.g., -`pool_size=2`). - -The `strides` argument specifies the size of the stride. Here, we set a stride -of 2, which indicates that the subregions extracted by the filter should be -separated by 2 pixels in both the height and width dimensions (for a 2x2 filter, -this means that none of the regions extracted will overlap). If you want to set -different stride values for height and width, you can instead specify a tuple or -list (e.g., `stride=[3, 6]`). - -Our output tensor produced by `max_pooling2d()` (`pool1`) has a shape of -[batch_size, 14, 14, 32]: the 2x2 filter reduces height and width by 50% each. - -### Convolutional Layer #2 and Pooling Layer #2 - -We can connect a second convolutional and pooling layer to our CNN using -`conv2d()` and `max_pooling2d()` as before. For convolutional layer #2, we -configure 64 5x5 filters with ReLU activation, and for pooling layer #2, we use -the same specs as pooling layer #1 (a 2x2 max pooling filter with stride of 2): - -```python -conv2 = tf.layers.conv2d( - inputs=pool1, - filters=64, - kernel_size=[5, 5], - padding="same", - activation=tf.nn.relu) - -pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2) -``` - -Note that convolutional layer #2 takes the output tensor of our first pooling -layer (`pool1`) as input, and produces the tensor `conv2` as output. `conv2` -has a shape of [batch_size, 14, 14, 64], the same height and width as `pool1` (due to `padding="same"`), and 64 channels for the 64 -filters applied. - -Pooling layer #2 takes `conv2` as input, producing `pool2` as output. `pool2` -has shape [batch_size, 7, 7, 64] (50% reduction of height and width from `conv2`). - -### Dense Layer - -Next, we want to add a dense layer (with 1,024 neurons and ReLU activation) to -our CNN to perform classification on the features extracted by the -convolution/pooling layers. Before we connect the layer, however, we'll flatten -our feature map (`pool2`) to shape [batch_size, -features], so that our tensor has only two dimensions: - -```python -pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64]) -``` - -In the `reshape()` operation above, the `-1` signifies that the *`batch_size`* -dimension will be dynamically calculated based on the number of examples in our -input data. Each example has 7 (`pool2` height) * 7 (`pool2` width) * 64 -(`pool2` channels) features, so we want the `features` dimension to have a value -of 7 * 7 * 64 (3136 in total). The output tensor, `pool2_flat`, has shape -[batch_size, 3136]. - -Now, we can use the `dense()` method in `layers` to connect our dense layer as -follows: - -```python -dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu) -``` - -The `inputs` argument specifies the input tensor: our flattened feature map, -`pool2_flat`. The `units` argument specifies the number of neurons in the dense -layer (1,024). The `activation` argument takes the activation function; again, -we'll use `tf.nn.relu` to add ReLU activation. - -To help improve the results of our model, we also apply dropout regularization -to our dense layer, using the `dropout` method in `layers`: - -```python -dropout = tf.layers.dropout( - inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) -``` - -Again, `inputs` specifies the input tensor, which is the output tensor from our -dense layer (`dense`). - -The `rate` argument specifies the dropout rate; here, we use `0.4`, which means -40% of the elements will be randomly dropped out during training. - -The `training` argument takes a boolean specifying whether or not the model is -currently being run in training mode; dropout will only be performed if -`training` is `True`. Here, we check if the `mode` passed to our model function -`cnn_model_fn` is `TRAIN` mode. - -Our output tensor `dropout` has shape [batch_size, 1024]. - -### Logits Layer - -The final layer in our neural network is the logits layer, which will return the -raw values for our predictions. We create a dense layer with 10 neurons (one for -each target class 0–9), with linear activation (the default): - -```python -logits = tf.layers.dense(inputs=dropout, units=10) -``` - -Our final output tensor of the CNN, `logits`, has shape -[batch_size, 10]. - -### Generate Predictions {#generate_predictions} - -The logits layer of our model returns our predictions as raw values in a -[batch_size, 10]-dimensional tensor. Let's convert these -raw values into two different formats that our model function can return: - -* The **predicted class** for each example: a digit from 0–9. -* The **probabilities** for each possible target class for each example: the - probability that the example is a 0, is a 1, is a 2, etc. - -For a given example, our predicted class is the element in the corresponding row -of the logits tensor with the highest raw value. We can find the index of this -element using the @{tf.argmax} -function: - -```python -tf.argmax(input=logits, axis=1) -``` - -The `input` argument specifies the tensor from which to extract maximum -values—here `logits`. The `axis` argument specifies the axis of the `input` -tensor along which to find the greatest value. Here, we want to find the largest -value along the dimension with index of 1, which corresponds to our predictions -(recall that our logits tensor has shape [batch_size, -10]). - -We can derive probabilities from our logits layer by applying softmax activation -using @{tf.nn.softmax}: - -```python -tf.nn.softmax(logits, name="softmax_tensor") -``` - -> Note: We use the `name` argument to explicitly name this operation -> `softmax_tensor`, so we can reference it later. (We'll set up logging for the -> softmax values in ["Set Up a Logging Hook"](#set-up-a-logging-hook)). - -We compile our predictions in a dict, and return an `EstimatorSpec` object: - -```python -predictions = { - "classes": tf.argmax(input=logits, axis=1), - "probabilities": tf.nn.softmax(logits, name="softmax_tensor") -} -if mode == tf.estimator.ModeKeys.PREDICT: - return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) -``` - -### Calculate Loss {#calculating-loss} - -For both training and evaluation, we need to define a -[loss function](https://en.wikipedia.org/wiki/Loss_function) -that measures how closely the model's predictions match the target classes. For -multiclass classification problems like MNIST, -[cross entropy](https://en.wikipedia.org/wiki/Cross_entropy) is typically used -as the loss metric. The following code calculates cross entropy when the model -runs in either `TRAIN` or `EVAL` mode: - -```python -onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10) -loss = tf.losses.softmax_cross_entropy( - onehot_labels=onehot_labels, logits=logits) -``` - -Let's take a closer look at what's happening above. - -Our `labels` tensor contains a list of predictions for our examples, e.g. `[1, -9, ...]`. In order to calculate cross-entropy, first we need to convert `labels` -to the corresponding -[one-hot encoding](https://www.quora.com/What-is-one-hot-encoding-and-when-is-it-used-in-data-science): - -```none -[[0, 1, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 1], - ...] -``` - -We use the @{tf.one_hot} function -to perform this conversion. `tf.one_hot()` has two required arguments: - -* `indices`. The locations in the one-hot tensor that will have "on - values"—i.e., the locations of `1` values in the tensor shown above. -* `depth`. The depth of the one-hot tensor—i.e., the number of target classes. - Here, the depth is `10`. - -The following code creates the one-hot tensor for our labels, `onehot_labels`: - -```python -onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10) -``` - -Because `labels` contains a series of values from 0–9, `indices` is just our -`labels` tensor, with values cast to integers. The `depth` is `10` because we -have 10 possible target classes, one for each digit. - -Next, we compute cross-entropy of `onehot_labels` and the softmax of the -predictions from our logits layer. `tf.losses.softmax_cross_entropy()` takes -`onehot_labels` and `logits` as arguments, performs softmax activation on -`logits`, calculates cross-entropy, and returns our `loss` as a scalar `Tensor`: - -```python -loss = tf.losses.softmax_cross_entropy( - onehot_labels=onehot_labels, logits=logits) -``` - -### Configure the Training Op - -In the previous section, we defined loss for our CNN as the softmax -cross-entropy of the logits layer and our labels. Let's configure our model to -optimize this loss value during training. We'll use a learning rate of 0.001 and -[stochastic gradient descent](https://en.wikipedia.org/wiki/Stochastic_gradient_descent) -as the optimization algorithm: - -```python -if mode == tf.estimator.ModeKeys.TRAIN: - optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) - train_op = optimizer.minimize( - loss=loss, - global_step=tf.train.get_global_step()) - return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) -``` - -> Note: For a more in-depth look at configuring training ops for Estimator model -> functions, see @{$custom_estimators#defining-the-training-op-for-the-model$"Defining the training op for the model"} -> in the @{$custom_estimators$"Creating Estimations in tf.estimator"} tutorial. - - -### Add evaluation metrics - -To add accuracy metric in our model, we define `eval_metric_ops` dict in EVAL -mode as follows: - -```python -eval_metric_ops = { - "accuracy": tf.metrics.accuracy( - labels=labels, predictions=predictions["classes"])} -return tf.estimator.EstimatorSpec( - mode=mode, loss=loss, eval_metric_ops=eval_metric_ops) -``` - - -## Training and Evaluating the CNN MNIST Classifier - -We've coded our MNIST CNN model function; now we're ready to train and evaluate -it. - -### Load Training and Test Data - -First, let's load our training and test data. Add a `main()` function to -`cnn_mnist.py` with the following code: - -```python -def main(unused_argv): - # Load training and eval data - mnist = tf.contrib.learn.datasets.load_dataset("mnist") - train_data = mnist.train.images # Returns np.array - train_labels = np.asarray(mnist.train.labels, dtype=np.int32) - eval_data = mnist.test.images # Returns np.array - eval_labels = np.asarray(mnist.test.labels, dtype=np.int32) -``` - -We store the training feature data (the raw pixel values for 55,000 images of -hand-drawn digits) and training labels (the corresponding value from 0–9 for -each image) as [numpy -arrays](https://docs.scipy.org/doc/numpy/reference/generated/numpy.array.html) -in `train_data` and `train_labels`, respectively. Similarly, we store the -evaluation feature data (10,000 images) and evaluation labels in `eval_data` -and `eval_labels`, respectively. - -### Create the Estimator {#create-the-estimator} - -Next, let's create an `Estimator` (a TensorFlow class for performing high-level -model training, evaluation, and inference) for our model. Add the following code -to `main()`: - -```python -# Create the Estimator -mnist_classifier = tf.estimator.Estimator( - model_fn=cnn_model_fn, model_dir="/tmp/mnist_convnet_model") -``` - -The `model_fn` argument specifies the model function to use for training, -evaluation, and prediction; we pass it the `cnn_model_fn` we created in -["Building the CNN MNIST Classifier."](#building-the-cnn-mnist-classifier) The -`model_dir` argument specifies the directory where model data (checkpoints) will -be saved (here, we specify the temp directory `/tmp/mnist_convnet_model`, but -feel free to change to another directory of your choice). - -> Note: For an in-depth walkthrough of the TensorFlow `Estimator` API, see the -> tutorial @{$custom_estimators$"Creating Estimators in tf.estimator."} - -### Set Up a Logging Hook {#set_up_a_logging_hook} - -Since CNNs can take a while to train, let's set up some logging so we can track -progress during training. We can use TensorFlow's @{tf.train.SessionRunHook} to create a -@{tf.train.LoggingTensorHook} -that will log the probability values from the softmax layer of our CNN. Add the -following to `main()`: - -```python -# Set up logging for predictions -tensors_to_log = {"probabilities": "softmax_tensor"} -logging_hook = tf.train.LoggingTensorHook( - tensors=tensors_to_log, every_n_iter=50) -``` - -We store a dict of the tensors we want to log in `tensors_to_log`. Each key is a -label of our choice that will be printed in the log output, and the -corresponding label is the name of a `Tensor` in the TensorFlow graph. Here, our -`probabilities` can be found in `softmax_tensor`, the name we gave our softmax -operation earlier when we generated the probabilities in `cnn_model_fn`. - -> Note: If you don't explicitly assign a name to an operation via the `name` -> argument, TensorFlow will assign a default name. A couple easy ways to -> discover the names applied to operations are to visualize your graph on -> @{$graph_viz$TensorBoard}) or to enable the -> @{$guide/debugger$TensorFlow Debugger (tfdbg)}. - -Next, we create the `LoggingTensorHook`, passing `tensors_to_log` to the -`tensors` argument. We set `every_n_iter=50`, which specifies that probabilities -should be logged after every 50 steps of training. - -### Train the Model - -Now we're ready to train our model, which we can do by creating `train_input_fn` -and calling `train()` on `mnist_classifier`. Add the following to `main()`: - -```python -# Train the model -train_input_fn = tf.estimator.inputs.numpy_input_fn( - x={"x": train_data}, - y=train_labels, - batch_size=100, - num_epochs=None, - shuffle=True) -mnist_classifier.train( - input_fn=train_input_fn, - steps=20000, - hooks=[logging_hook]) -``` - -In the `numpy_input_fn` call, we pass the training feature data and labels to -`x` (as a dict) and `y`, respectively. We set a `batch_size` of `100` (which -means that the model will train on minibatches of 100 examples at each step). -`num_epochs=None` means that the model will train until the specified number of -steps is reached. We also set `shuffle=True` to shuffle the training data. -In the `train` call, we set `steps=20000` -(which means the model will train for 20,000 steps total). We pass our -`logging_hook` to the `hooks` argument, so that it will be triggered during -training. - -### Evaluate the Model - -Once training is complete, we want to evaluate our model to determine its -accuracy on the MNIST test set. We call the `evaluate` method, which evaluates -the metrics we specified in `eval_metric_ops` argument in the `model_fn`. -Add the following to `main()`: - -```python -# Evaluate the model and print results -eval_input_fn = tf.estimator.inputs.numpy_input_fn( - x={"x": eval_data}, - y=eval_labels, - num_epochs=1, - shuffle=False) -eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn) -print(eval_results) -``` - -To create `eval_input_fn`, we set `num_epochs=1`, so that the model evaluates -the metrics over one epoch of data and returns the result. We also set -`shuffle=False` to iterate through the data sequentially. - -### Run the Model - -We've coded the CNN model function, `Estimator`, and the training/evaluation -logic; now let's see the results. Run `cnn_mnist.py`. - -> Note: Training CNNs is quite computationally intensive. Estimated completion -> time of `cnn_mnist.py` will vary depending on your processor, but will likely -> be upwards of 1 hour on CPU. To train more quickly, you can decrease the -> number of `steps` passed to `train()`, but note that this will affect accuracy. - -As the model trains, you'll see log output like the following: - -```python -INFO:tensorflow:loss = 2.36026, step = 1 -INFO:tensorflow:probabilities = [[ 0.07722801 0.08618255 0.09256398, ...]] -... -INFO:tensorflow:loss = 2.13119, step = 101 -INFO:tensorflow:global_step/sec: 5.44132 -... -INFO:tensorflow:Loss for final step: 0.553216. - -INFO:tensorflow:Restored model from /tmp/mnist_convnet_model -INFO:tensorflow:Eval steps [0,inf) for training step 20000. -INFO:tensorflow:Input iterator is exhausted. -INFO:tensorflow:Saving evaluation summary for step 20000: accuracy = 0.9733, loss = 0.0902271 -{'loss': 0.090227105, 'global_step': 20000, 'accuracy': 0.97329998} -``` - -Here, we've achieved an accuracy of 97.3% on our test data set. - -## Additional Resources - -To learn more about TensorFlow Estimators and CNNs in TensorFlow, see the -following resources: - -* @{$custom_estimators$Creating Estimators in tf.estimator} - provides an introduction to the TensorFlow Estimator API. It walks through - configuring an Estimator, writing a model function, calculating loss, and - defining a training op. -* @{$deep_cnn} walks through how to build a MNIST CNN classification model - *without estimators* using lower-level TensorFlow operations. diff --git a/tensorflow/docs_src/tutorials/leftnav_files b/tensorflow/docs_src/tutorials/leftnav_files deleted file mode 100644 index 888052428f..0000000000 --- a/tensorflow/docs_src/tutorials/leftnav_files +++ /dev/null @@ -1,23 +0,0 @@ -index.md - -### Images -layers.md: MNIST -image_recognition.md: Image Recognition -image_retraining.md: Image Retraining -deep_cnn.md - -### Sequences -recurrent.md -seq2seq.md: Neural Machine Translation -recurrent_quickdraw.md: Drawing Classification -audio_recognition.md - -### Data Representation -wide.md: Linear Models -wide_and_deep.md: Wide & Deep Learning -word2vec.md -kernel_methods.md: Kernel Methods - -### Non-ML -mandelbrot.md -pdes.md diff --git a/tensorflow/docs_src/tutorials/linear.md b/tensorflow/docs_src/tutorials/linear.md deleted file mode 100644 index 3f247ade26..0000000000 --- a/tensorflow/docs_src/tutorials/linear.md +++ /dev/null @@ -1,237 +0,0 @@ -# Large-scale Linear Models with TensorFlow - -@{tf.estimator$Estimators} provides (among other things) a rich set of tools for -working with linear models in TensorFlow. This document provides an overview of -those tools. It explains: - - * What a linear model is. - * Why you might want to use a linear model. - * How Estimators make it easy to build linear models in TensorFlow. - * How you can use Estimators to combine linear models with. - deep learning to get the advantages of both. - -Read this overview to decide whether the Estimator's linear model tools might -be useful to you. Then do the @{$wide$Linear Models tutorial} to -give it a try. This overview uses code samples from the tutorial, but the -tutorial walks through the code in greater detail. - -To understand this overview it will help to have some familiarity -with basic machine learning concepts, and also with -@{$premade_estimators$Estimators}. - -[TOC] - -## What is a linear model? - -A **linear model** uses a single weighted sum of features to make a prediction. -For example, if you have [data](https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.names) -on age, years of education, and weekly hours of -work for a population, a model can learn weights for each of those numbers so that -their weighted sum estimates a person's salary. You can also use linear models -for classification. - -Some linear models transform the weighted sum into a more convenient form. For -example, [**logistic regression**](https://developers.google.com/machine-learning/glossary/#logistic_regression) plugs the weighted sum into the logistic -function to turn the output into a value between 0 and 1. But you still just -have one weight for each input feature. - -## Why would you want to use a linear model? - -Why would you want to use so simple a model when recent research has -demonstrated the power of more complex neural networks with many layers? - -Linear models: - - * train quickly, compared to deep neural nets. - * can work well on very large feature sets. - * can be trained with algorithms that don't require a lot of fiddling - with learning rates, etc. - * can be interpreted and debugged more easily than neural nets. - You can examine the weights assigned to each feature to figure out what's - having the biggest impact on a prediction. - * provide an excellent starting point for learning about machine learning. - * are widely used in industry. - -## How do Estimators help you build linear models? - -You can build a linear model from scratch in TensorFlow without the help of a -special API. But Estimators provides some tools that make it easier to build -effective large-scale linear models. - -### Feature columns and transformations - -Much of the work of designing a linear model consists of transforming raw data -into suitable input features. Tensorflow uses the `FeatureColumn` abstraction to -enable these transformations. - -A `FeatureColumn` represents a single feature in your data. A `FeatureColumn` -may represent a quantity like 'height', or it may represent a category like -'eye_color' where the value is drawn from a set of discrete possibilities like -{'blue', 'brown', 'green'}. - -In the case of both *continuous features* like 'height' and *categorical -features* like 'eye_color', a single value in the data might get transformed -into a sequence of numbers before it is input into the model. The -`FeatureColumn` abstraction lets you manipulate the feature as a single -semantic unit in spite of this fact. You can specify transformations and -select features to include without dealing with specific indices in the -tensors you feed into the model. - -#### Sparse columns - -Categorical features in linear models are typically translated into a sparse -vector in which each possible value has a corresponding index or id. For -example, if there are only three possible eye colors you can represent -'eye_color' as a length 3 vector: 'brown' would become [1, 0, 0], 'blue' would -become [0, 1, 0] and 'green' would become [0, 0, 1]. These vectors are called -"sparse" because they may be very long, with many zeros, when the set of -possible values is very large (such as all English words). - -While you don't need to use categorical columns to use the linear model tools -provided by Estimators, one of the strengths of linear models is their ability -to deal with large sparse vectors. Sparse features are a primary use case for -the linear model tools provided by Estimators. - -##### Encoding sparse columns - -`FeatureColumn` handles the conversion of categorical values into vectors -automatically, with code like this: - -```python -eye_color = tf.feature_column.categorical_column_with_vocabulary_list( - "eye_color", vocabulary_list=["blue", "brown", "green"]) -``` - -where `eye_color` is the name of a column in your source data. - -You can also generate `FeatureColumn`s for categorical features for which you -don't know all possible values. For this case you would use -`categorical_column_with_hash_bucket()`, which uses a hash function to assign -indices to feature values. - -```python -education = tf.feature_column.categorical_column_with_hash_bucket( - "education", hash_bucket_size=1000) -``` - -##### Feature Crosses - -Because linear models assign independent weights to separate features, they -can't learn the relative importance of specific combinations of feature -values. If you have a feature 'favorite_sport' and a feature 'home_city' and -you're trying to predict whether a person likes to wear red, your linear model -won't be able to learn that baseball fans from St. Louis especially like to -wear red. - -You can get around this limitation by creating a new feature -'favorite_sport_x_home_city'. The value of this feature for a given person is -just the concatenation of the values of the two source features: -'baseball_x_stlouis', for example. This sort of combination feature is called -a *feature cross*. - -The `crossed_column()` method makes it easy to set up feature crosses: - -```python -sport_x_city = tf.feature_column.crossed_column( - ["sport", "city"], hash_bucket_size=int(1e4)) -``` - -#### Continuous columns - -You can specify a continuous feature like so: - -```python -age = tf.feature_column.numeric_column("age") -``` - -Although, as a single real number, a continuous feature can often be input -directly into the model, Tensorflow offers useful transformations for this sort -of column as well. - -##### Bucketization - -*Bucketization* turns a continuous column into a categorical column. This -transformation lets you use continuous features in feature crosses, or learn -cases where specific value ranges have particular importance. - -Bucketization divides the range of possible values into subranges called -buckets: - -```python -age_buckets = tf.feature_column.bucketized_column( - age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) -``` - -The bucket into which a value falls becomes the categorical label for -that value. - -#### Input function - -`FeatureColumn`s provide a specification for the input data for your model, -indicating how to represent and transform the data. But they do not provide -the data itself. You provide the data through an input function. - -The input function must return a dictionary of tensors. Each key corresponds to -the name of a `FeatureColumn`. Each key's value is a tensor containing the -values of that feature for all data instances. See -@{$premade_estimators#input_fn} for a -more comprehensive look at input functions, and `input_fn` in the -[linear models tutorial code](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py) -for an example implementation of an input function. - -The input function is passed to the `train()` and `evaluate()` calls that -initiate training and testing, as described in the next section. - -### Linear estimators - -Tensorflow estimator classes provide a unified training and evaluation harness -for regression and classification models. They take care of the details of the -training and evaluation loops and allow the user to focus on model inputs and -architecture. - -To build a linear estimator, you can use either the -`tf.estimator.LinearClassifier` estimator or the -`tf.estimator.LinearRegressor` estimator, for classification and -regression respectively. - -As with all tensorflow estimators, to run the estimator you just: - - 1. Instantiate the estimator class. For the two linear estimator classes, - you pass a list of `FeatureColumn`s to the constructor. - 2. Call the estimator's `train()` method to train it. - 3. Call the estimator's `evaluate()` method to see how it does. - -For example: - -```python -e = tf.estimator.LinearClassifier( - feature_columns=[ - native_country, education, occupation, workclass, marital_status, - race, age_buckets, education_x_occupation, - age_buckets_x_race_x_occupation], - model_dir=YOUR_MODEL_DIRECTORY) -e.train(input_fn=input_fn_train, steps=200) -# Evaluate for one step (one pass through the test data). -results = e.evaluate(input_fn=input_fn_test) - -# Print the stats for the evaluation. -for key in sorted(results): - print("%s: %s" % (key, results[key])) -``` - -### Wide and deep learning - -The `tf.estimator` module also provides an estimator class that lets you jointly -train a linear model and a deep neural network. This novel approach combines the -ability of linear models to "memorize" key features with the generalization -ability of neural nets. Use `tf.estimator.DNNLinearCombinedClassifier` to -create this sort of "wide and deep" model: - -```python -e = tf.estimator.DNNLinearCombinedClassifier( - model_dir=YOUR_MODEL_DIR, - linear_feature_columns=wide_columns, - dnn_feature_columns=deep_columns, - dnn_hidden_units=[100, 50]) -``` -For more information, see the @{$wide_and_deep$Wide and Deep Learning tutorial}. diff --git a/tensorflow/docs_src/tutorials/mandelbrot.md b/tensorflow/docs_src/tutorials/mandelbrot.md deleted file mode 100755 index 1c0a548129..0000000000 --- a/tensorflow/docs_src/tutorials/mandelbrot.md +++ /dev/null @@ -1,116 +0,0 @@ -# Mandelbrot Set - -Visualizing the [Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set) -doesn't have anything to do with machine learning, but it makes for a fun -example of how one can use TensorFlow for general mathematics. This is -actually a pretty naive implementation of the visualization, but it makes the -point. (We may end up providing a more elaborate implementation down the line -to produce more truly beautiful images.) - - -## Basic Setup - -We'll need a few imports to get started. - -```python -# Import libraries for simulation -import tensorflow as tf -import numpy as np - -# Imports for visualization -import PIL.Image -from io import BytesIO -from IPython.display import Image, display -``` - -Now we'll define a function to actually display the image once we have -iteration counts. - -```python -def DisplayFractal(a, fmt='jpeg'): - """Display an array of iteration counts as a - colorful picture of a fractal.""" - a_cyclic = (6.28*a/20.0).reshape(list(a.shape)+[1]) - img = np.concatenate([10+20*np.cos(a_cyclic), - 30+50*np.sin(a_cyclic), - 155-80*np.cos(a_cyclic)], 2) - img[a==a.max()] = 0 - a = img - a = np.uint8(np.clip(a, 0, 255)) - f = BytesIO() - PIL.Image.fromarray(a).save(f, fmt) - display(Image(data=f.getvalue())) -``` - -## Session and Variable Initialization - -For playing around like this, we often use an interactive session, but a regular -session would work as well. - -```python -sess = tf.InteractiveSession() -``` - -It's handy that we can freely mix NumPy and TensorFlow. - -```python -# Use NumPy to create a 2D array of complex numbers - -Y, X = np.mgrid[-1.3:1.3:0.005, -2:1:0.005] -Z = X+1j*Y -``` - -Now we define and initialize TensorFlow tensors. - -```python -xs = tf.constant(Z.astype(np.complex64)) -zs = tf.Variable(xs) -ns = tf.Variable(tf.zeros_like(xs, tf.float32)) -``` - -TensorFlow requires that you explicitly initialize variables before using them. - -```python -tf.global_variables_initializer().run() -``` - -## Defining and Running the Computation - -Now we specify more of the computation... - -```python -# Compute the new values of z: z^2 + x -zs_ = zs*zs + xs - -# Have we diverged with this new value? -not_diverged = tf.abs(zs_) < 4 - -# Operation to update the zs and the iteration count. -# -# Note: We keep computing zs after they diverge! This -# is very wasteful! There are better, if a little -# less simple, ways to do this. -# -step = tf.group( - zs.assign(zs_), - ns.assign_add(tf.cast(not_diverged, tf.float32)) - ) -``` - -... and run it for a couple hundred steps - -```python -for i in range(200): step.run() -``` - -Let's see what we've got. - -```python -DisplayFractal(ns.eval()) -``` - -![jpeg](https://www.tensorflow.org/images/mandelbrot_output.jpg) - -Not bad! - - diff --git a/tensorflow/docs_src/tutorials/next_steps.md b/tensorflow/docs_src/tutorials/next_steps.md new file mode 100644 index 0000000000..01c9f7204a --- /dev/null +++ b/tensorflow/docs_src/tutorials/next_steps.md @@ -0,0 +1,36 @@ +# Next steps + +## Learn more about TensorFlow + +* The [TensorFlow Guide](/guide) includes usage guides for the + high-level APIs, as well as advanced TensorFlow operations. +* [Premade Estimators](/guide/premade_estimators) are designed to + get results out of the box. Use TensorFlow without building your own models. +* [TensorFlow.js](https://js.tensorflow.org/) allows web developers to train and + deploy ML models in the browser and using Node.js. +* [TFLite](/mobile/tflite) allows mobile developers to do inference efficiently + on mobile devices. +* [TensorFlow Serving](/serving) is an open-source project that can put + TensorFlow models in production quickly. +* The [ecosystem](/ecosystem) contains more projects, including + [Magenta](https://magenta.tensorflow.org/), [TFX](/tfx), + [Swift for TensorFlow](https://github.com/tensorflow/swift), and more. + +## Learn more about machine learning + +Recommended resources include: + +* [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/), + a course from Google that introduces machine learning concepts. +* [CS 20: Tensorflow for Deep Learning Research](http://web.stanford.edu/class/cs20si/), + notes from an intro course from Stanford. +* [CS231n: Convolutional Neural Networks for Visual Recognition](http://cs231n.stanford.edu/), + a course that teaches how convolutional networks work. +* [Machine Learning Recipes](https://www.youtube.com/watch?v=cKxRvEZd3Mw&list=PLOU2XLYxmsIIuiBfYad6rFYQU_jL2ryal), + a video series that introduces basic machine learning concepts with few prerequisites. +* [Deep Learning with Python](https://www.manning.com/books/deep-learning-with-python), + a book by Francois Chollet about the Keras API, as well as an excellent hands on intro to Deep Learning. +* [Hands-on Machine Learning with Scikit-Learn and TensorFlow](https://github.com/ageron/handson-ml), + a book by Aurélien Geron's that is a clear getting-started guide to data science and deep learning. +* [Deep Learning](https://www.deeplearningbook.org/), a book by Ian Goodfellow et al. + that provides a technical dive into learning machine learning. diff --git a/tensorflow/docs_src/tutorials/non-ml/mandelbrot.md b/tensorflow/docs_src/tutorials/non-ml/mandelbrot.md new file mode 100644 index 0000000000..1c0a548129 --- /dev/null +++ b/tensorflow/docs_src/tutorials/non-ml/mandelbrot.md @@ -0,0 +1,116 @@ +# Mandelbrot Set + +Visualizing the [Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set) +doesn't have anything to do with machine learning, but it makes for a fun +example of how one can use TensorFlow for general mathematics. This is +actually a pretty naive implementation of the visualization, but it makes the +point. (We may end up providing a more elaborate implementation down the line +to produce more truly beautiful images.) + + +## Basic Setup + +We'll need a few imports to get started. + +```python +# Import libraries for simulation +import tensorflow as tf +import numpy as np + +# Imports for visualization +import PIL.Image +from io import BytesIO +from IPython.display import Image, display +``` + +Now we'll define a function to actually display the image once we have +iteration counts. + +```python +def DisplayFractal(a, fmt='jpeg'): + """Display an array of iteration counts as a + colorful picture of a fractal.""" + a_cyclic = (6.28*a/20.0).reshape(list(a.shape)+[1]) + img = np.concatenate([10+20*np.cos(a_cyclic), + 30+50*np.sin(a_cyclic), + 155-80*np.cos(a_cyclic)], 2) + img[a==a.max()] = 0 + a = img + a = np.uint8(np.clip(a, 0, 255)) + f = BytesIO() + PIL.Image.fromarray(a).save(f, fmt) + display(Image(data=f.getvalue())) +``` + +## Session and Variable Initialization + +For playing around like this, we often use an interactive session, but a regular +session would work as well. + +```python +sess = tf.InteractiveSession() +``` + +It's handy that we can freely mix NumPy and TensorFlow. + +```python +# Use NumPy to create a 2D array of complex numbers + +Y, X = np.mgrid[-1.3:1.3:0.005, -2:1:0.005] +Z = X+1j*Y +``` + +Now we define and initialize TensorFlow tensors. + +```python +xs = tf.constant(Z.astype(np.complex64)) +zs = tf.Variable(xs) +ns = tf.Variable(tf.zeros_like(xs, tf.float32)) +``` + +TensorFlow requires that you explicitly initialize variables before using them. + +```python +tf.global_variables_initializer().run() +``` + +## Defining and Running the Computation + +Now we specify more of the computation... + +```python +# Compute the new values of z: z^2 + x +zs_ = zs*zs + xs + +# Have we diverged with this new value? +not_diverged = tf.abs(zs_) < 4 + +# Operation to update the zs and the iteration count. +# +# Note: We keep computing zs after they diverge! This +# is very wasteful! There are better, if a little +# less simple, ways to do this. +# +step = tf.group( + zs.assign(zs_), + ns.assign_add(tf.cast(not_diverged, tf.float32)) + ) +``` + +... and run it for a couple hundred steps + +```python +for i in range(200): step.run() +``` + +Let's see what we've got. + +```python +DisplayFractal(ns.eval()) +``` + +![jpeg](https://www.tensorflow.org/images/mandelbrot_output.jpg) + +Not bad! + + diff --git a/tensorflow/docs_src/tutorials/non-ml/pdes.md b/tensorflow/docs_src/tutorials/non-ml/pdes.md new file mode 100644 index 0000000000..b5a0fa834a --- /dev/null +++ b/tensorflow/docs_src/tutorials/non-ml/pdes.md @@ -0,0 +1,140 @@ +# Partial Differential Equations + +TensorFlow isn't just for machine learning. Here we give a (somewhat +pedestrian) example of using TensorFlow for simulating the behavior of a +[partial differential equation]( +https://en.wikipedia.org/wiki/Partial_differential_equation). +We'll simulate the surface of square pond as a few raindrops land on it. + + +## Basic Setup + +A few imports we'll need. + +```python +#Import libraries for simulation +import tensorflow as tf +import numpy as np + +#Imports for visualization +import PIL.Image +from io import BytesIO +from IPython.display import clear_output, Image, display +``` + +A function for displaying the state of the pond's surface as an image. + +```python +def DisplayArray(a, fmt='jpeg', rng=[0,1]): + """Display an array as a picture.""" + a = (a - rng[0])/float(rng[1] - rng[0])*255 + a = np.uint8(np.clip(a, 0, 255)) + f = BytesIO() + PIL.Image.fromarray(a).save(f, fmt) + clear_output(wait = True) + display(Image(data=f.getvalue())) +``` + +Here we start an interactive TensorFlow session for convenience in playing +around. A regular session would work as well if we were doing this in an +executable .py file. + +```python +sess = tf.InteractiveSession() +``` + +## Computational Convenience Functions + + +```python +def make_kernel(a): + """Transform a 2D array into a convolution kernel""" + a = np.asarray(a) + a = a.reshape(list(a.shape) + [1,1]) + return tf.constant(a, dtype=1) + +def simple_conv(x, k): + """A simplified 2D convolution operation""" + x = tf.expand_dims(tf.expand_dims(x, 0), -1) + y = tf.nn.depthwise_conv2d(x, k, [1, 1, 1, 1], padding='SAME') + return y[0, :, :, 0] + +def laplace(x): + """Compute the 2D laplacian of an array""" + laplace_k = make_kernel([[0.5, 1.0, 0.5], + [1.0, -6., 1.0], + [0.5, 1.0, 0.5]]) + return simple_conv(x, laplace_k) +``` + +## Define the PDE + +Our pond is a perfect 500 x 500 square, as is the case for most ponds found in +nature. + +```python +N = 500 +``` + +Here we create our pond and hit it with some rain drops. + +```python +# Initial Conditions -- some rain drops hit a pond + +# Set everything to zero +u_init = np.zeros([N, N], dtype=np.float32) +ut_init = np.zeros([N, N], dtype=np.float32) + +# Some rain drops hit a pond at random points +for n in range(40): + a,b = np.random.randint(0, N, 2) + u_init[a,b] = np.random.uniform() + +DisplayArray(u_init, rng=[-0.1, 0.1]) +``` + +![jpeg](https://www.tensorflow.org/images/pde_output_1.jpg) + + +Now let's specify the details of the differential equation. + + +```python +# Parameters: +# eps -- time resolution +# damping -- wave damping +eps = tf.placeholder(tf.float32, shape=()) +damping = tf.placeholder(tf.float32, shape=()) + +# Create variables for simulation state +U = tf.Variable(u_init) +Ut = tf.Variable(ut_init) + +# Discretized PDE update rules +U_ = U + eps * Ut +Ut_ = Ut + eps * (laplace(U) - damping * Ut) + +# Operation to update the state +step = tf.group( + U.assign(U_), + Ut.assign(Ut_)) +``` + +## Run The Simulation + +This is where it gets fun -- running time forward with a simple for loop. + +```python +# Initialize state to initial conditions +tf.global_variables_initializer().run() + +# Run 1000 steps of PDE +for i in range(1000): + # Step simulation + step.run({eps: 0.03, damping: 0.04}) + DisplayArray(U.eval(), rng=[-0.1, 0.1]) +``` + +![jpeg](../../images/pde_output_2.jpg) + +Look! Ripples! diff --git a/tensorflow/docs_src/tutorials/pdes.md b/tensorflow/docs_src/tutorials/pdes.md deleted file mode 100755 index 425e8d7084..0000000000 --- a/tensorflow/docs_src/tutorials/pdes.md +++ /dev/null @@ -1,141 +0,0 @@ -# Partial Differential Equations - -TensorFlow isn't just for machine learning. Here we give a (somewhat -pedestrian) example of using TensorFlow for simulating the behavior of a -[partial differential equation]( -https://en.wikipedia.org/wiki/Partial_differential_equation). -We'll simulate the surface of square pond as a few raindrops land on it. - - -## Basic Setup - -A few imports we'll need. - -```python -#Import libraries for simulation -import tensorflow as tf -import numpy as np - -#Imports for visualization -import PIL.Image -from io import BytesIO -from IPython.display import clear_output, Image, display -``` - -A function for displaying the state of the pond's surface as an image. - -```python -def DisplayArray(a, fmt='jpeg', rng=[0,1]): - """Display an array as a picture.""" - a = (a - rng[0])/float(rng[1] - rng[0])*255 - a = np.uint8(np.clip(a, 0, 255)) - f = BytesIO() - PIL.Image.fromarray(a).save(f, fmt) - clear_output(wait = True) - display(Image(data=f.getvalue())) -``` - -Here we start an interactive TensorFlow session for convenience in playing -around. A regular session would work as well if we were doing this in an -executable .py file. - -```python -sess = tf.InteractiveSession() -``` - -## Computational Convenience Functions - - -```python -def make_kernel(a): - """Transform a 2D array into a convolution kernel""" - a = np.asarray(a) - a = a.reshape(list(a.shape) + [1,1]) - return tf.constant(a, dtype=1) - -def simple_conv(x, k): - """A simplified 2D convolution operation""" - x = tf.expand_dims(tf.expand_dims(x, 0), -1) - y = tf.nn.depthwise_conv2d(x, k, [1, 1, 1, 1], padding='SAME') - return y[0, :, :, 0] - -def laplace(x): - """Compute the 2D laplacian of an array""" - laplace_k = make_kernel([[0.5, 1.0, 0.5], - [1.0, -6., 1.0], - [0.5, 1.0, 0.5]]) - return simple_conv(x, laplace_k) -``` - -## Define the PDE - -Our pond is a perfect 500 x 500 square, as is the case for most ponds found in -nature. - -```python -N = 500 -``` - -Here we create our pond and hit it with some rain drops. - -```python -# Initial Conditions -- some rain drops hit a pond - -# Set everything to zero -u_init = np.zeros([N, N], dtype=np.float32) -ut_init = np.zeros([N, N], dtype=np.float32) - -# Some rain drops hit a pond at random points -for n in range(40): - a,b = np.random.randint(0, N, 2) - u_init[a,b] = np.random.uniform() - -DisplayArray(u_init, rng=[-0.1, 0.1]) -``` - -![jpeg](https://www.tensorflow.org/images/pde_output_1.jpg) - - -Now let's specify the details of the differential equation. - - -```python -# Parameters: -# eps -- time resolution -# damping -- wave damping -eps = tf.placeholder(tf.float32, shape=()) -damping = tf.placeholder(tf.float32, shape=()) - -# Create variables for simulation state -U = tf.Variable(u_init) -Ut = tf.Variable(ut_init) - -# Discretized PDE update rules -U_ = U + eps * Ut -Ut_ = Ut + eps * (laplace(U) - damping * Ut) - -# Operation to update the state -step = tf.group( - U.assign(U_), - Ut.assign(Ut_)) -``` - -## Run The Simulation - -This is where it gets fun -- running time forward with a simple for loop. - -```python -# Initialize state to initial conditions -tf.global_variables_initializer().run() - -# Run 1000 steps of PDE -for i in range(1000): - # Step simulation - step.run({eps: 0.03, damping: 0.04}) - DisplayArray(U.eval(), rng=[-0.1, 0.1]) -``` - -![jpeg](../images/pde_output_2.jpg) - -Look! Ripples! - diff --git a/tensorflow/docs_src/tutorials/recurrent.md b/tensorflow/docs_src/tutorials/recurrent.md deleted file mode 100644 index 14da2c8785..0000000000 --- a/tensorflow/docs_src/tutorials/recurrent.md +++ /dev/null @@ -1,232 +0,0 @@ -# Recurrent Neural Networks - -## Introduction - -Take a look at [this great article](https://colah.github.io/posts/2015-08-Understanding-LSTMs/) -for an introduction to recurrent neural networks and LSTMs in particular. - -## Language Modeling - -In this tutorial we will show how to train a recurrent neural network on -a challenging task of language modeling. The goal of the problem is to fit a -probabilistic model which assigns probabilities to sentences. It does so by -predicting next words in a text given a history of previous words. For this -purpose we will use the [Penn Tree Bank](https://catalog.ldc.upenn.edu/ldc99t42) -(PTB) dataset, which is a popular benchmark for measuring the quality of these -models, whilst being small and relatively fast to train. - -Language modeling is key to many interesting problems such as speech -recognition, machine translation, or image captioning. It is also fun -- -take a look [here](https://karpathy.github.io/2015/05/21/rnn-effectiveness/). - -For the purpose of this tutorial, we will reproduce the results from -[Zaremba et al., 2014](https://arxiv.org/abs/1409.2329) -([pdf](https://arxiv.org/pdf/1409.2329.pdf)), which achieves very good quality -on the PTB dataset. - -## Tutorial Files - -This tutorial references the following files from `models/tutorials/rnn/ptb` in the [TensorFlow models repo](https://github.com/tensorflow/models): - -File | Purpose ---- | --- -`ptb_word_lm.py` | The code to train a language model on the PTB dataset. -`reader.py` | The code to read the dataset. - -## Download and Prepare the Data - -The data required for this tutorial is in the `data/` directory of the -[PTB dataset from Tomas Mikolov's webpage](http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz). - -The dataset is already preprocessed and contains overall 10000 different words, -including the end-of-sentence marker and a special symbol (\) for rare -words. In `reader.py`, we convert each word to a unique integer identifier, -in order to make it easy for the neural network to process the data. - -## The Model - -### LSTM - -The core of the model consists of an LSTM cell that processes one word at a -time and computes probabilities of the possible values for the next word in the -sentence. The memory state of the network is initialized with a vector of zeros -and gets updated after reading each word. For computational reasons, we will -process data in mini-batches of size `batch_size`. In this example, it is -important to note that `current_batch_of_words` does not correspond to a -"sentence" of words. Every word in a batch should correspond to a time t. -TensorFlow will automatically sum the gradients of each batch for you. - -For example: - -``` - t=0 t=1 t=2 t=3 t=4 -[The, brown, fox, is, quick] -[The, red, fox, jumped, high] - -words_in_dataset[0] = [The, The] -words_in_dataset[1] = [brown, red] -words_in_dataset[2] = [fox, fox] -words_in_dataset[3] = [is, jumped] -words_in_dataset[4] = [quick, high] -batch_size = 2, time_steps = 5 -``` - -The basic pseudocode is as follows: - -```python -words_in_dataset = tf.placeholder(tf.float32, [time_steps, batch_size, num_features]) -lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size) -# Initial state of the LSTM memory. -hidden_state = tf.zeros([batch_size, lstm.state_size]) -current_state = tf.zeros([batch_size, lstm.state_size]) -state = hidden_state, current_state -probabilities = [] -loss = 0.0 -for current_batch_of_words in words_in_dataset: - # The value of state is updated after processing each batch of words. - output, state = lstm(current_batch_of_words, state) - - # The LSTM output can be used to make next word predictions - logits = tf.matmul(output, softmax_w) + softmax_b - probabilities.append(tf.nn.softmax(logits)) - loss += loss_function(probabilities, target_words) -``` - -### Truncated Backpropagation - -By design, the output of a recurrent neural network (RNN) depends on arbitrarily -distant inputs. Unfortunately, this makes backpropagation computation difficult. -In order to make the learning process tractable, it is common practice to create -an "unrolled" version of the network, which contains a fixed number -(`num_steps`) of LSTM inputs and outputs. The model is then trained on this -finite approximation of the RNN. This can be implemented by feeding inputs of -length `num_steps` at a time and performing a backward pass after each -such input block. - -Here is a simplified block of code for creating a graph which performs -truncated backpropagation: - -```python -# Placeholder for the inputs in a given iteration. -words = tf.placeholder(tf.int32, [batch_size, num_steps]) - -lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size) -# Initial state of the LSTM memory. -initial_state = state = tf.zeros([batch_size, lstm.state_size]) - -for i in range(num_steps): - # The value of state is updated after processing each batch of words. - output, state = lstm(words[:, i], state) - - # The rest of the code. - # ... - -final_state = state -``` - -And this is how to implement an iteration over the whole dataset: - -```python -# A numpy array holding the state of LSTM after each batch of words. -numpy_state = initial_state.eval() -total_loss = 0.0 -for current_batch_of_words in words_in_dataset: - numpy_state, current_loss = session.run([final_state, loss], - # Initialize the LSTM state from the previous iteration. - feed_dict={initial_state: numpy_state, words: current_batch_of_words}) - total_loss += current_loss -``` - -### Inputs - -The word IDs will be embedded into a dense representation (see the -@{$word2vec$Vector Representations Tutorial}) before feeding to -the LSTM. This allows the model to efficiently represent the knowledge about -particular words. It is also easy to write: - -```python -# embedding_matrix is a tensor of shape [vocabulary_size, embedding size] -word_embeddings = tf.nn.embedding_lookup(embedding_matrix, word_ids) -``` - -The embedding matrix will be initialized randomly and the model will learn to -differentiate the meaning of words just by looking at the data. - -### Loss Function - -We want to minimize the average negative log probability of the target words: - -$$ \text{loss} = -\frac{1}{N}\sum_{i=1}^{N} \ln p_{\text{target}_i} $$ - -It is not very difficult to implement but the function -`sequence_loss_by_example` is already available, so we can just use it here. - -The typical measure reported in the papers is average per-word perplexity (often -just called perplexity), which is equal to - -$$e^{-\frac{1}{N}\sum_{i=1}^{N} \ln p_{\text{target}_i}} = e^{\text{loss}} $$ - -and we will monitor its value throughout the training process. - -### Stacking multiple LSTMs - -To give the model more expressive power, we can add multiple layers of LSTMs -to process the data. The output of the first layer will become the input of -the second and so on. - -We have a class called `MultiRNNCell` that makes the implementation seamless: - -```python -def lstm_cell(): - return tf.contrib.rnn.BasicLSTMCell(lstm_size) -stacked_lstm = tf.contrib.rnn.MultiRNNCell( - [lstm_cell() for _ in range(number_of_layers)]) - -initial_state = state = stacked_lstm.zero_state(batch_size, tf.float32) -for i in range(num_steps): - # The value of state is updated after processing each batch of words. - output, state = stacked_lstm(words[:, i], state) - - # The rest of the code. - # ... - -final_state = state -``` - -## Run the Code - -Before running the code, download the PTB dataset, as discussed at the beginning -of this tutorial. Then, extract the PTB dataset underneath your home directory -as follows: - -```bsh -tar xvfz simple-examples.tgz -C $HOME -``` -_(Note: On Windows, you may need to use -[other tools](https://wiki.haskell.org/How_to_unpack_a_tar_file_in_Windows).)_ - -Now, clone the [TensorFlow models repo](https://github.com/tensorflow/models) -from GitHub. Run the following commands: - -```bsh -cd models/tutorials/rnn/ptb -python ptb_word_lm.py --data_path=$HOME/simple-examples/data/ --model=small -``` - -There are 3 supported model configurations in the tutorial code: "small", -"medium" and "large". The difference between them is in size of the LSTMs and -the set of hyperparameters used for training. - -The larger the model, the better results it should get. The `small` model should -be able to reach perplexity below 120 on the test set and the `large` one below -80, though it might take several hours to train. - -## What Next? - -There are several tricks that we haven't mentioned that make the model better, -including: - -* decreasing learning rate schedule, -* dropout between the LSTM layers. - -Study the code and modify it to improve the model even further. diff --git a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md deleted file mode 100644 index 1afd861738..0000000000 --- a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md +++ /dev/null @@ -1,411 +0,0 @@ -# Recurrent Neural Networks for Drawing Classification - -[Quick, Draw!]: http://quickdraw.withgoogle.com - -[Quick, Draw!] is a game where a player is challenged to draw a number of -objects and see if a computer can recognize the drawing. - -The recognition in [Quick, Draw!] is performed by a classifier that takes the -user input, given as a sequence of strokes of points in x and y, and recognizes -the object category that the user tried to draw. - -In this tutorial we'll show how to build an RNN-based recognizer for this -problem. The model will use a combination of convolutional layers, LSTM layers, -and a softmax output layer to classify the drawings: - -
![RNN model structure](../images/quickdraw_model.png)
- -The figure above shows the structure of the model that we will build in this -tutorial. The input is a drawing that is encoded as a sequence of strokes of -points in x, y, and n, where n indicates whether a the point is the first point -in a new stroke. - -Then, a series of 1-dimensional convolutions is applied. Then LSTM layers are -applied and the sum of the outputs of all LSTM steps is fed into a softmax layer -to make a classification decision among the classes of drawings that we know. - -This tutorial uses the data from actual [Quick, Draw!] games [that is publicly -available](https://quickdraw.withgoogle.com/data). This dataset contains of 50M -drawings in 345 categories. - -## Run the tutorial code - -To try the code for this tutorial: - -1. @{$install$Install TensorFlow} if you haven't already. -1. Download the [tutorial code] -(https://github.com/tensorflow/models/tree/master/tutorials/rnn/quickdraw/train_model.py). -1. [Download the data](#download-the-data) in `TFRecord` format from - [here](http://download.tensorflow.org/data/quickdraw_tutorial_dataset_v1.tar.gz) and unzip it. More details about [how to - obtain the original Quick, Draw! - data](#optional_download_the_full_quick_draw_data) and [how to convert that - to `TFRecord` files](#optional_converting_the_data) is available below. - -1. Execute the tutorial code with the following command to train the RNN-based - model described in this tutorial. Make sure to adjust the paths to point to - the unzipped data from the download in step 3. - -```shell - python train_model.py \ - --training_data=rnn_tutorial_data/training.tfrecord-?????-of-????? \ - --eval_data=rnn_tutorial_data/eval.tfrecord-?????-of-????? \ - --classes_file=rnn_tutorial_data/training.tfrecord.classes -``` - -## Tutorial details - -### Download the data - -We make the data that we use in this tutorial available as `TFRecord` files -containing `TFExamples`. You can download the data from here: - -http://download.tensorflow.org/data/quickdraw_tutorial_dataset_v1.tar.gz - -Alternatively you can download the original data in `ndjson` format from the -Google cloud and convert it to the `TFRecord` files containing `TFExamples` -yourself as described in the next section. - -### Optional: Download the full Quick Draw Data - -The full [Quick, Draw!](https://quickdraw.withgoogle.com) -[dataset](https://quickdraw.withgoogle.com/data) is available on Google Cloud -Storage as [ndjson](http://ndjson.org/) files separated by category. You can -[browse the list of files in Cloud -Console](https://console.cloud.google.com/storage/quickdraw_dataset). - -To download the data we recommend using -[gsutil](https://cloud.google.com/storage/docs/gsutil_install#install) to -download the entire dataset. Note that the original .ndjson files require -downloading ~22GB. - -Then use the following command to check that your gsutil installation works and -that you can access the data bucket: - -```shell -gsutil ls -r "gs://quickdraw_dataset/full/simplified/*" -``` - -which will output a long list of files like the following: - -```shell -gs://quickdraw_dataset/full/simplified/The Eiffel Tower.ndjson -gs://quickdraw_dataset/full/simplified/The Great Wall of China.ndjson -gs://quickdraw_dataset/full/simplified/The Mona Lisa.ndjson -gs://quickdraw_dataset/full/simplified/aircraft carrier.ndjson -... -``` - -Then create a folder and download the dataset there. - -```shell -mkdir rnn_tutorial_data -cd rnn_tutorial_data -gsutil -m cp "gs://quickdraw_dataset/full/simplified/*" . -``` - -This download will take a while and download a bit more than 23GB of data. - -### Optional: Converting the data - -To convert the `ndjson` files to -@{$python/python_io#TFRecords_Format_Details$TFRecord} files containing -[`tf.train.Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto) -protos run the following command. - -```shell - python create_dataset.py --ndjson_path rnn_tutorial_data \ - --output_path rnn_tutorial_data -``` - -This will store the data in 10 shards of -@{$python/python_io#TFRecords_Format_Details$TFRecord} files with 10000 items -per class for the training data and 1000 items per class as eval data. - -This conversion process is described in more detail in the following. - -The original QuickDraw data is formatted as `ndjson` files where each line -contains a JSON object like the following: - -```json -{"word":"cat", - "countrycode":"VE", - "timestamp":"2017-03-02 23:25:10.07453 UTC", - "recognized":true, - "key_id":"5201136883597312", - "drawing":[ - [ - [130,113,99,109,76,64,55,48,48,51,59,86,133,154,170,203,214,217,215,208,186,176,162,157,132], - [72,40,27,79,82,88,100,120,134,152,165,184,189,186,179,152,131,114,100,89,76,0,31,65,70] - ],[ - [76,28,7], - [136,128,128] - ],[ - [76,23,0], - [160,164,175] - ],[ - [87,52,37], - [175,191,204] - ],[ - [174,220,246,251], - [134,132,136,139] - ],[ - [175,255], - [147,168] - ],[ - [171,208,215], - [164,198,210] - ],[ - [130,110,108,111,130,139,139,119], - [129,134,137,144,148,144,136,130] - ],[ - [107,106], - [96,113] - ] - ] -} -``` - -For our purpose of building a classifier we only care about the fields "`word`" -and "`drawing`". While parsing the ndjson files, we process them line by line -using a function that converts the strokes from the `drawing` field into a -tensor of size `[number of points, 3]` containing the differences of consecutive -points. This function also returns the class name as a string. - -```python -def parse_line(ndjson_line): - """Parse an ndjson line and return ink (as np array) and classname.""" - sample = json.loads(ndjson_line) - class_name = sample["word"] - inkarray = sample["drawing"] - stroke_lengths = [len(stroke[0]) for stroke in inkarray] - total_points = sum(stroke_lengths) - np_ink = np.zeros((total_points, 3), dtype=np.float32) - current_t = 0 - for stroke in inkarray: - for i in [0, 1]: - np_ink[current_t:(current_t + len(stroke[0])), i] = stroke[i] - current_t += len(stroke[0]) - np_ink[current_t - 1, 2] = 1 # stroke_end - # Preprocessing. - # 1. Size normalization. - lower = np.min(np_ink[:, 0:2], axis=0) - upper = np.max(np_ink[:, 0:2], axis=0) - scale = upper - lower - scale[scale == 0] = 1 - np_ink[:, 0:2] = (np_ink[:, 0:2] - lower) / scale - # 2. Compute deltas. - np_ink = np_ink[1:, 0:2] - np_ink[0:-1, 0:2] - return np_ink, class_name -``` - -Since we want the data to be shuffled for writing we read from each of the -category files in random order and write to a random shard. - -For the training data we read the first 10000 items for each class and for the -eval data we read the next 1000 items for each class. - -This data is then reformatted into a tensor of shape `[num_training_samples, -max_length, 3]`. Then we determine the bounding box of the original drawing in -screen coordinates and normalize the size such that the drawing has unit height. - -
![Size normalization](../images/quickdraw_sizenormalization.png)
- -Finally, we compute the differences between consecutive points and store these -as a `VarLenFeature` in a -[tensorflow.Example](https://www.tensorflow.org/code/tensorflow/core/example/example.proto) -under the key `ink`. In addition we store the `class_index` as a single entry -`FixedLengthFeature` and the `shape` of the `ink` as a `FixedLengthFeature` of -length 2. - -### Defining the model - -To define the model we create a new `Estimator`. If you want to read more about -estimators, we recommend @{$custom_estimators$this tutorial}. - -To build the model, we: - -1. reshape the input back into the original shape - where the mini batch is - padded to the maximal length of its contents. In addition to the ink data we - also have the lengths for each example and the target class. This happens in - the function [`_get_input_tensors`](#-get-input-tensors). - -1. pass the input through to a series of convolution layers in - [`_add_conv_layers`](#-add-conv-layers). - -1. pass the output of the convolutions into a series of bidirectional LSTM - layers in [`_add_rnn_layers`](#-add-rnn-layers). At the end of that, the - outputs for each time step are summed up to have a compact, fixed length - embedding of the input. - -1. classify this embedding using a softmax layer in - [`_add_fc_layers`](#-add-fc-layers). - -In code this looks like: - -```python -inks, lengths, targets = _get_input_tensors(features, targets) -convolved = _add_conv_layers(inks) -final_state = _add_rnn_layers(convolved, lengths) -logits =_add_fc_layers(final_state) -``` - -### _get_input_tensors - -To obtain the input features we first obtain the shape from the features dict -and then create a 1D tensor of size `[batch_size]` containing the lengths of the -input sequences. The ink is stored as a SparseTensor in the features dict which -we convert into a dense tensor and then reshape to be `[batch_size, ?, 3]`. And -finally, if targets were passed in we make sure they are stored as a 1D tensor -of size `[batch_size]` - -In code this looks like this: - -```python -shapes = features["shape"] -lengths = tf.squeeze( - tf.slice(shapes, begin=[0, 0], size=[params["batch_size"], 1])) -inks = tf.reshape( - tf.sparse_tensor_to_dense(features["ink"]), - [params["batch_size"], -1, 3]) -if targets is not None: - targets = tf.squeeze(targets) -``` - -### _add_conv_layers - -The desired number of convolution layers and the lengths of the filters is -configured through the parameters `num_conv` and `conv_len` in the `params` -dict. - -The input is a sequence where each point has dimensionality 3. We are going to -use 1D convolutions where we treat the 3 input features as channels. That means -that the input is a `[batch_size, length, 3]` tensor and the output will be a -`[batch_size, length, number_of_filters]` tensor. - -```python -convolved = inks -for i in range(len(params.num_conv)): - convolved_input = convolved - if params.batch_norm: - convolved_input = tf.layers.batch_normalization( - convolved_input, - training=(mode == tf.estimator.ModeKeys.TRAIN)) - # Add dropout layer if enabled and not first convolution layer. - if i > 0 and params.dropout: - convolved_input = tf.layers.dropout( - convolved_input, - rate=params.dropout, - training=(mode == tf.estimator.ModeKeys.TRAIN)) - convolved = tf.layers.conv1d( - convolved_input, - filters=params.num_conv[i], - kernel_size=params.conv_len[i], - activation=None, - strides=1, - padding="same", - name="conv1d_%d" % i) -return convolved, lengths -``` - -### _add_rnn_layers - -We pass the output from the convolutions into bidirectional LSTM layers for -which we use a helper function from contrib. - -```python -outputs, _, _ = contrib_rnn.stack_bidirectional_dynamic_rnn( - cells_fw=[cell(params.num_nodes) for _ in range(params.num_layers)], - cells_bw=[cell(params.num_nodes) for _ in range(params.num_layers)], - inputs=convolved, - sequence_length=lengths, - dtype=tf.float32, - scope="rnn_classification") -``` - -see the code for more details and how to use `CUDA` accelerated implementations. - -To create a compact, fixed-length embedding, we sum up the output of the LSTMs. -We first zero out the regions of the batch where the sequences have no data. - -```python -mask = tf.tile( - tf.expand_dims(tf.sequence_mask(lengths, tf.shape(outputs)[1]), 2), - [1, 1, tf.shape(outputs)[2]]) -zero_outside = tf.where(mask, outputs, tf.zeros_like(outputs)) -outputs = tf.reduce_sum(zero_outside, axis=1) -``` - -### _add_fc_layers - -The embedding of the input is passed into a fully connected layer which we then -use as a softmax layer. - -```python -tf.layers.dense(final_state, params.num_classes) -``` - -### Loss, predictions, and optimizer - -Finally, we need to add a loss, a training op, and predictions to create the -`ModelFn`: - -```python -cross_entropy = tf.reduce_mean( - tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=targets, logits=logits)) -# Add the optimizer. -train_op = tf.contrib.layers.optimize_loss( - loss=cross_entropy, - global_step=tf.train.get_global_step(), - learning_rate=params.learning_rate, - optimizer="Adam", - # some gradient clipping stabilizes training in the beginning. - clip_gradients=params.gradient_clipping_norm, - summaries=["learning_rate", "loss", "gradients", "gradient_norm"]) -predictions = tf.argmax(logits, axis=1) -return model_fn_lib.ModelFnOps( - mode=mode, - predictions={"logits": logits, - "predictions": predictions}, - loss=cross_entropy, - train_op=train_op, - eval_metric_ops={"accuracy": tf.metrics.accuracy(targets, predictions)}) -``` - -### Training and evaluating the model - -To train and evaluate the model we can rely on the functionalities of the -`Estimator` APIs and easily run training and evaluation with the `Experiment` -APIs: - -```python - estimator = tf.estimator.Estimator( - model_fn=model_fn, - model_dir=output_dir, - config=config, - params=model_params) - # Train the model. - tf.contrib.learn.Experiment( - estimator=estimator, - train_input_fn=get_input_fn( - mode=tf.contrib.learn.ModeKeys.TRAIN, - tfrecord_pattern=FLAGS.training_data, - batch_size=FLAGS.batch_size), - train_steps=FLAGS.steps, - eval_input_fn=get_input_fn( - mode=tf.contrib.learn.ModeKeys.EVAL, - tfrecord_pattern=FLAGS.eval_data, - batch_size=FLAGS.batch_size), - min_eval_frequency=1000) -``` - -Note that this tutorial is just a quick example on a relatively small dataset to -get you familiar with the APIs of recurrent neural networks and estimators. Such -models can be even more powerful if you try them on a large dataset. - -When training the model for 1M steps you can expect to get an accuracy of -approximately of approximately 70% on the top-1 candidate. Note that this -accuracy is sufficient to build the quickdraw game because of the game dynamics -the user will be able to adjust their drawing until it is ready. Also, the game -does not use the top-1 candidate only but accepts a drawing as correct if the -target category shows up with a score better than a fixed threshold. diff --git a/tensorflow/docs_src/tutorials/representation/kernel_methods.md b/tensorflow/docs_src/tutorials/representation/kernel_methods.md new file mode 100644 index 0000000000..f3c232c511 --- /dev/null +++ b/tensorflow/docs_src/tutorials/representation/kernel_methods.md @@ -0,0 +1,304 @@ +# Improving Linear Models Using Explicit Kernel Methods + +Note: This document uses a deprecated version of @{tf.estimator}, +which has a @{tf.contrib.learn.Estimator$different interface}. +It also uses other `contrib` methods whose +@{$version_compat#not_covered$API may not be stable}. + +In this tutorial, we demonstrate how combining (explicit) kernel methods with +linear models can drastically increase the latters' quality of predictions +without significantly increasing training and inference times. Unlike dual +kernel methods, explicit (primal) kernel methods scale well with the size of the +training dataset both in terms of training/inference times and in terms of +memory requirements. + +**Intended audience:** Even though we provide a high-level overview of concepts +related to explicit kernel methods, this tutorial primarily targets readers who +already have at least basic knowledge of kernel methods and Support Vector +Machines (SVMs). If you are new to kernel methods, refer to either of the +following sources for an introduction: + +* If you have a strong mathematical background: +[Kernel Methods in Machine Learning](https://arxiv.org/pdf/math/0701907.pdf) +* [Kernel method wikipedia page](https://en.wikipedia.org/wiki/Kernel_method) + +Currently, TensorFlow supports explicit kernel mappings for dense features only; +TensorFlow will provide support for sparse features at a later release. + +This tutorial uses [tf.contrib.learn](https://www.tensorflow.org/code/tensorflow/contrib/learn/python/learn) +(TensorFlow's high-level Machine Learning API) Estimators for our ML models. +If you are not familiar with this API, The [Estimator guide](../../guide/estimators.md) +is a good place to start. We will use the MNIST dataset. The tutorial consists +of the following steps: + +* Load and prepare MNIST data for classification. +* Construct a simple linear model, train it, and evaluate it on the eval data. +* Replace the linear model with a kernelized linear model, re-train, and +re-evaluate. + +## Load and prepare MNIST data for classification +Run the following utility command to load the MNIST dataset: + +```python +data = tf.contrib.learn.datasets.mnist.load_mnist() +``` +The preceding method loads the entire MNIST dataset (containing 70K samples) and +splits it into train, validation, and test data with 55K, 5K, and 10K samples +respectively. Each split contains one numpy array for images (with shape +[sample_size, 784]) and one for labels (with shape [sample_size, 1]). In this +tutorial, we only use the train and validation splits to train and evaluate our +models respectively. + +In order to feed data to a `tf.contrib.learn Estimator`, it is helpful to convert +it to Tensors. For this, we will use an `input function` which adds Ops to the +TensorFlow graph that, when executed, create mini-batches of Tensors to be used +downstream. For more background on input functions, check +@{$premade_estimators#create_input_functions$this section on input functions}. +In this example, we will use the `tf.train.shuffle_batch` Op which, besides +converting numpy arrays to Tensors, allows us to specify the batch_size and +whether to randomize the input every time the input_fn Ops are executed +(randomization typically expedites convergence during training). The full code +for loading and preparing the data is shown in the snippet below. In this +example, we use mini-batches of size 256 for training and the entire sample +(5K entries) for evaluation. Feel free to experiment with different batch sizes. + +```python +import numpy as np +import tensorflow as tf + +def get_input_fn(dataset_split, batch_size, capacity=10000, min_after_dequeue=3000): + + def _input_fn(): + images_batch, labels_batch = tf.train.shuffle_batch( + tensors=[dataset_split.images, dataset_split.labels.astype(np.int32)], + batch_size=batch_size, + capacity=capacity, + min_after_dequeue=min_after_dequeue, + enqueue_many=True, + num_threads=4) + features_map = {'images': images_batch} + return features_map, labels_batch + + return _input_fn + +data = tf.contrib.learn.datasets.mnist.load_mnist() + +train_input_fn = get_input_fn(data.train, batch_size=256) +eval_input_fn = get_input_fn(data.validation, batch_size=5000) + +``` + +## Training a simple linear model +We can now train a linear model over the MNIST dataset. We will use the +@{tf.contrib.learn.LinearClassifier} estimator with 10 classes representing the +10 digits. The input features form a 784-dimensional dense vector which can +be specified as follows: + +```python +image_column = tf.contrib.layers.real_valued_column('images', dimension=784) +``` + +The full code for constructing, training and evaluating a LinearClassifier +estimator is as follows: + +```python +import time + +# Specify the feature(s) to be used by the estimator. +image_column = tf.contrib.layers.real_valued_column('images', dimension=784) +estimator = tf.contrib.learn.LinearClassifier(feature_columns=[image_column], n_classes=10) + +# Train. +start = time.time() +estimator.fit(input_fn=train_input_fn, steps=2000) +end = time.time() +print('Elapsed time: {} seconds'.format(end - start)) + +# Evaluate and report metrics. +eval_metrics = estimator.evaluate(input_fn=eval_input_fn, steps=1) +print(eval_metrics) +``` +The following table summarizes the results on the eval data. + +metric | value +:------------ | :------------ +loss | 0.25 to 0.30 +accuracy | 92.5% +training time | ~25 seconds on my machine + +Note: Metrics will vary depending on various factors. + +In addition to experimenting with the (training) batch size and the number of +training steps, there are a couple other parameters that can be tuned as well. +For instance, you can change the optimization method used to minimize the loss +by explicitly selecting another optimizer from the collection of +[available optimizers](https://www.tensorflow.org/code/tensorflow/python/training). +As an example, the following code constructs a LinearClassifier estimator that +uses the Follow-The-Regularized-Leader (FTRL) optimization strategy with a +specific learning rate and L2-regularization. + + +```python +optimizer = tf.train.FtrlOptimizer(learning_rate=5.0, l2_regularization_strength=1.0) +estimator = tf.contrib.learn.LinearClassifier( + feature_columns=[image_column], n_classes=10, optimizer=optimizer) +``` + +Regardless of the values of the parameters, the maximum accuracy a linear model +can achieve on this dataset caps at around **93%**. + +## Using explicit kernel mappings with the linear model. +The relatively high error (~7%) of the linear model over MNIST indicates that +the input data is not linearly separable. We will use explicit kernel mappings +to reduce the classification error. + +**Intuition:** The high-level idea is to use a non-linear map to transform the +input space to another feature space (of possibly higher dimension) where the +(transformed) features are (almost) linearly separable and then apply a linear +model on the mapped features. This is shown in the following figure: + +
+ +
+ + +### Technical details +In this example we will use **Random Fourier Features**, introduced in the +["Random Features for Large-Scale Kernel Machines"](https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf) +paper by Rahimi and Recht, to map the input data. Random Fourier Features map a +vector \\(\mathbf{x} \in \mathbb{R}^d\\) to \\(\mathbf{x'} \in \mathbb{R}^D\\) +via the following mapping: + +$$ +RFFM(\cdot): \mathbb{R}^d \to \mathbb{R}^D, \quad +RFFM(\mathbf{x}) = \cos(\mathbf{\Omega} \cdot \mathbf{x}+ \mathbf{b}) +$$ + +where \\(\mathbf{\Omega} \in \mathbb{R}^{D \times d}\\), +\\(\mathbf{x} \in \mathbb{R}^d,\\) \\(\mathbf{b} \in \mathbb{R}^D\\) and the +cosine is applied element-wise. + +In this example, the entries of \\(\mathbf{\Omega}\\) and \\(\mathbf{b}\\) are +sampled from distributions such that the mapping satisfies the following +property: + +$$ +RFFM(\mathbf{x})^T \cdot RFFM(\mathbf{y}) \approx +e^{-\frac{\|\mathbf{x} - \mathbf{y}\|^2}{2 \sigma^2}} +$$ + +The right-hand-side quantity of the expression above is known as the RBF (or +Gaussian) kernel function. This function is one of the most-widely used kernel +functions in Machine Learning and implicitly measures similarity in a different, +much higher dimensional space than the original one. See +[Radial basis function kernel](https://en.wikipedia.org/wiki/Radial_basis_function_kernel) +for more details. + +### Kernel classifier +@{tf.contrib.kernel_methods.KernelLinearClassifier} is a pre-packaged +`tf.contrib.learn` estimator that combines the power of explicit kernel mappings +with linear models. Its constructor is almost identical to that of the +LinearClassifier estimator with the additional option to specify a list of +explicit kernel mappings to be applied to each feature the classifier uses. The +following code snippet demonstrates how to replace LinearClassifier with +KernelLinearClassifier. + + +```python +# Specify the feature(s) to be used by the estimator. This is identical to the +# code used for the LinearClassifier. +image_column = tf.contrib.layers.real_valued_column('images', dimension=784) +optimizer = tf.train.FtrlOptimizer( + learning_rate=50.0, l2_regularization_strength=0.001) + + +kernel_mapper = tf.contrib.kernel_methods.RandomFourierFeatureMapper( + input_dim=784, output_dim=2000, stddev=5.0, name='rffm') +kernel_mappers = {image_column: [kernel_mapper]} +estimator = tf.contrib.kernel_methods.KernelLinearClassifier( + n_classes=10, optimizer=optimizer, kernel_mappers=kernel_mappers) + +# Train. +start = time.time() +estimator.fit(input_fn=train_input_fn, steps=2000) +end = time.time() +print('Elapsed time: {} seconds'.format(end - start)) + +# Evaluate and report metrics. +eval_metrics = estimator.evaluate(input_fn=eval_input_fn, steps=1) +print(eval_metrics) +``` +The only additional parameter passed to `KernelLinearClassifier` is a dictionary +from feature_columns to a list of kernel mappings to be applied to the +corresponding feature column. The following lines instruct the classifier to +first map the initial 784-dimensional images to 2000-dimensional vectors using +random Fourier features and then learn a linear model on the transformed +vectors: + +```python +kernel_mapper = tf.contrib.kernel_methods.RandomFourierFeatureMapper( + input_dim=784, output_dim=2000, stddev=5.0, name='rffm') +kernel_mappers = {image_column: [kernel_mapper]} +estimator = tf.contrib.kernel_methods.KernelLinearClassifier( + n_classes=10, optimizer=optimizer, kernel_mappers=kernel_mappers) +``` +Notice the `stddev` parameter. This is the standard deviation (\\(\sigma\\)) of +the approximated RBF kernel and controls the similarity measure used in +classification. `stddev` is typically determined via hyperparameter tuning. + +The results of running the preceding code are summarized in the following table. +We can further increase the accuracy by increasing the output dimension of the +mapping and tuning the standard deviation. + +metric | value +:------------ | :------------ +loss | 0.10 +accuracy | 97% +training time | ~35 seconds on my machine + + +### stddev +The classification quality is very sensitive to the value of stddev. The +following table shows the accuracy of the classifier on the eval data for +different values of stddev. The optimal value is stddev=5.0. Notice how too +small or too high stddev values can dramatically decrease the accuracy of the +classification. + +stddev | eval accuracy +:----- | :------------ +1.0 | 0.1362 +2.0 | 0.4764 +4.0 | 0.9654 +5.0 | 0.9766 +8.0 | 0.9714 +16.0 | 0.8878 + +### Output dimension +Intuitively, the larger the output dimension of the mapping, the closer the +inner product of two mapped vectors approximates the kernel, which typically +translates to better classification accuracy. Another way to think about this is +that the output dimension equals the number of weights of the linear model; the +larger this dimension, the larger the "degrees of freedom" of the model. +However, after a certain threshold, higher output dimensions increase the +accuracy by very little, while making training take more time. This is shown in +the following two Figures which depict the eval accuracy as a function of the +output dimension and the training time, respectively. + +![image](https://www.tensorflow.org/versions/master/images/acc_vs_outdim.png) +![image](https://www.tensorflow.org/versions/master/images/acc-vs-trn_time.png) + + +## Summary +Explicit kernel mappings combine the predictive power of nonlinear models with +the scalability of linear models. Unlike traditional dual kernel methods, +explicit kernel methods can scale to millions or hundreds of millions of +samples. When using explicit kernel mappings, consider the following tips: + +* Random Fourier Features can be particularly effective for datasets with dense +features. +* The parameters of the kernel mapping are often data-dependent. Model quality +can be very sensitive to these parameters. Use hyperparameter tuning to find the +optimal values. +* If you have multiple numerical features, concatenate them into a single +multi-dimensional feature and apply the kernel mapping to the concatenated +vector. diff --git a/tensorflow/docs_src/tutorials/representation/linear.md b/tensorflow/docs_src/tutorials/representation/linear.md new file mode 100644 index 0000000000..3f247ade26 --- /dev/null +++ b/tensorflow/docs_src/tutorials/representation/linear.md @@ -0,0 +1,237 @@ +# Large-scale Linear Models with TensorFlow + +@{tf.estimator$Estimators} provides (among other things) a rich set of tools for +working with linear models in TensorFlow. This document provides an overview of +those tools. It explains: + + * What a linear model is. + * Why you might want to use a linear model. + * How Estimators make it easy to build linear models in TensorFlow. + * How you can use Estimators to combine linear models with. + deep learning to get the advantages of both. + +Read this overview to decide whether the Estimator's linear model tools might +be useful to you. Then do the @{$wide$Linear Models tutorial} to +give it a try. This overview uses code samples from the tutorial, but the +tutorial walks through the code in greater detail. + +To understand this overview it will help to have some familiarity +with basic machine learning concepts, and also with +@{$premade_estimators$Estimators}. + +[TOC] + +## What is a linear model? + +A **linear model** uses a single weighted sum of features to make a prediction. +For example, if you have [data](https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.names) +on age, years of education, and weekly hours of +work for a population, a model can learn weights for each of those numbers so that +their weighted sum estimates a person's salary. You can also use linear models +for classification. + +Some linear models transform the weighted sum into a more convenient form. For +example, [**logistic regression**](https://developers.google.com/machine-learning/glossary/#logistic_regression) plugs the weighted sum into the logistic +function to turn the output into a value between 0 and 1. But you still just +have one weight for each input feature. + +## Why would you want to use a linear model? + +Why would you want to use so simple a model when recent research has +demonstrated the power of more complex neural networks with many layers? + +Linear models: + + * train quickly, compared to deep neural nets. + * can work well on very large feature sets. + * can be trained with algorithms that don't require a lot of fiddling + with learning rates, etc. + * can be interpreted and debugged more easily than neural nets. + You can examine the weights assigned to each feature to figure out what's + having the biggest impact on a prediction. + * provide an excellent starting point for learning about machine learning. + * are widely used in industry. + +## How do Estimators help you build linear models? + +You can build a linear model from scratch in TensorFlow without the help of a +special API. But Estimators provides some tools that make it easier to build +effective large-scale linear models. + +### Feature columns and transformations + +Much of the work of designing a linear model consists of transforming raw data +into suitable input features. Tensorflow uses the `FeatureColumn` abstraction to +enable these transformations. + +A `FeatureColumn` represents a single feature in your data. A `FeatureColumn` +may represent a quantity like 'height', or it may represent a category like +'eye_color' where the value is drawn from a set of discrete possibilities like +{'blue', 'brown', 'green'}. + +In the case of both *continuous features* like 'height' and *categorical +features* like 'eye_color', a single value in the data might get transformed +into a sequence of numbers before it is input into the model. The +`FeatureColumn` abstraction lets you manipulate the feature as a single +semantic unit in spite of this fact. You can specify transformations and +select features to include without dealing with specific indices in the +tensors you feed into the model. + +#### Sparse columns + +Categorical features in linear models are typically translated into a sparse +vector in which each possible value has a corresponding index or id. For +example, if there are only three possible eye colors you can represent +'eye_color' as a length 3 vector: 'brown' would become [1, 0, 0], 'blue' would +become [0, 1, 0] and 'green' would become [0, 0, 1]. These vectors are called +"sparse" because they may be very long, with many zeros, when the set of +possible values is very large (such as all English words). + +While you don't need to use categorical columns to use the linear model tools +provided by Estimators, one of the strengths of linear models is their ability +to deal with large sparse vectors. Sparse features are a primary use case for +the linear model tools provided by Estimators. + +##### Encoding sparse columns + +`FeatureColumn` handles the conversion of categorical values into vectors +automatically, with code like this: + +```python +eye_color = tf.feature_column.categorical_column_with_vocabulary_list( + "eye_color", vocabulary_list=["blue", "brown", "green"]) +``` + +where `eye_color` is the name of a column in your source data. + +You can also generate `FeatureColumn`s for categorical features for which you +don't know all possible values. For this case you would use +`categorical_column_with_hash_bucket()`, which uses a hash function to assign +indices to feature values. + +```python +education = tf.feature_column.categorical_column_with_hash_bucket( + "education", hash_bucket_size=1000) +``` + +##### Feature Crosses + +Because linear models assign independent weights to separate features, they +can't learn the relative importance of specific combinations of feature +values. If you have a feature 'favorite_sport' and a feature 'home_city' and +you're trying to predict whether a person likes to wear red, your linear model +won't be able to learn that baseball fans from St. Louis especially like to +wear red. + +You can get around this limitation by creating a new feature +'favorite_sport_x_home_city'. The value of this feature for a given person is +just the concatenation of the values of the two source features: +'baseball_x_stlouis', for example. This sort of combination feature is called +a *feature cross*. + +The `crossed_column()` method makes it easy to set up feature crosses: + +```python +sport_x_city = tf.feature_column.crossed_column( + ["sport", "city"], hash_bucket_size=int(1e4)) +``` + +#### Continuous columns + +You can specify a continuous feature like so: + +```python +age = tf.feature_column.numeric_column("age") +``` + +Although, as a single real number, a continuous feature can often be input +directly into the model, Tensorflow offers useful transformations for this sort +of column as well. + +##### Bucketization + +*Bucketization* turns a continuous column into a categorical column. This +transformation lets you use continuous features in feature crosses, or learn +cases where specific value ranges have particular importance. + +Bucketization divides the range of possible values into subranges called +buckets: + +```python +age_buckets = tf.feature_column.bucketized_column( + age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) +``` + +The bucket into which a value falls becomes the categorical label for +that value. + +#### Input function + +`FeatureColumn`s provide a specification for the input data for your model, +indicating how to represent and transform the data. But they do not provide +the data itself. You provide the data through an input function. + +The input function must return a dictionary of tensors. Each key corresponds to +the name of a `FeatureColumn`. Each key's value is a tensor containing the +values of that feature for all data instances. See +@{$premade_estimators#input_fn} for a +more comprehensive look at input functions, and `input_fn` in the +[linear models tutorial code](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py) +for an example implementation of an input function. + +The input function is passed to the `train()` and `evaluate()` calls that +initiate training and testing, as described in the next section. + +### Linear estimators + +Tensorflow estimator classes provide a unified training and evaluation harness +for regression and classification models. They take care of the details of the +training and evaluation loops and allow the user to focus on model inputs and +architecture. + +To build a linear estimator, you can use either the +`tf.estimator.LinearClassifier` estimator or the +`tf.estimator.LinearRegressor` estimator, for classification and +regression respectively. + +As with all tensorflow estimators, to run the estimator you just: + + 1. Instantiate the estimator class. For the two linear estimator classes, + you pass a list of `FeatureColumn`s to the constructor. + 2. Call the estimator's `train()` method to train it. + 3. Call the estimator's `evaluate()` method to see how it does. + +For example: + +```python +e = tf.estimator.LinearClassifier( + feature_columns=[ + native_country, education, occupation, workclass, marital_status, + race, age_buckets, education_x_occupation, + age_buckets_x_race_x_occupation], + model_dir=YOUR_MODEL_DIRECTORY) +e.train(input_fn=input_fn_train, steps=200) +# Evaluate for one step (one pass through the test data). +results = e.evaluate(input_fn=input_fn_test) + +# Print the stats for the evaluation. +for key in sorted(results): + print("%s: %s" % (key, results[key])) +``` + +### Wide and deep learning + +The `tf.estimator` module also provides an estimator class that lets you jointly +train a linear model and a deep neural network. This novel approach combines the +ability of linear models to "memorize" key features with the generalization +ability of neural nets. Use `tf.estimator.DNNLinearCombinedClassifier` to +create this sort of "wide and deep" model: + +```python +e = tf.estimator.DNNLinearCombinedClassifier( + model_dir=YOUR_MODEL_DIR, + linear_feature_columns=wide_columns, + dnn_feature_columns=deep_columns, + dnn_hidden_units=[100, 50]) +``` +For more information, see the @{$wide_and_deep$Wide and Deep Learning tutorial}. diff --git a/tensorflow/docs_src/tutorials/representation/wide.md b/tensorflow/docs_src/tutorials/representation/wide.md new file mode 100644 index 0000000000..27ce75a30d --- /dev/null +++ b/tensorflow/docs_src/tutorials/representation/wide.md @@ -0,0 +1,461 @@ +# TensorFlow Linear Model Tutorial + +In this tutorial, we will use the tf.estimator API in TensorFlow to solve a +binary classification problem: Given census data about a person such as age, +education, marital status, and occupation (the features), we will try to predict +whether or not the person earns more than 50,000 dollars a year (the target +label). We will train a **logistic regression** model, and given an individual's +information our model will output a number between 0 and 1, which can be +interpreted as the probability that the individual has an annual income of over +50,000 dollars. + +## Setup + +To try the code for this tutorial: + +1. @{$install$Install TensorFlow} if you haven't already. + +2. Download [the tutorial code](https://github.com/tensorflow/models/tree/master/official/wide_deep/). + +3. Execute the data download script we provide to you: + + $ python data_download.py + +4. Execute the tutorial code with the following command to train the linear +model described in this tutorial: + + $ python wide_deep.py --model_type=wide + +Read on to find out how this code builds its linear model. + +## Reading The Census Data + +The dataset we'll be using is the +[Census Income Dataset](https://archive.ics.uci.edu/ml/datasets/Census+Income). +We have provided +[data_download.py](https://github.com/tensorflow/models/tree/master/official/wide_deep/data_download.py) +which downloads the code and performs some additional cleanup. + +Since the task is a binary classification problem, we'll construct a label +column named "label" whose value is 1 if the income is over 50K, and 0 +otherwise. For reference, see `input_fn` in +[wide_deep.py](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py). + +Next, let's take a look at the dataframe and see which columns we can use to +predict the target label. The columns can be grouped into two types—categorical +and continuous columns: + +* A column is called **categorical** if its value can only be one of the + categories in a finite set. For example, the relationship status of a person + (wife, husband, unmarried, etc.) or the education level (high school, + college, etc.) are categorical columns. +* A column is called **continuous** if its value can be any numerical value in + a continuous range. For example, the capital gain of a person (e.g. $14,084) + is a continuous column. + +Here's a list of columns available in the Census Income dataset: + +| Column Name | Type | Description | +| -------------- | ----------- | --------------------------------- | +| age | Continuous | The age of the individual | +| workclass | Categorical | The type of employer the | +: : : individual has (government, : +: : : military, private, etc.). : +| fnlwgt | Continuous | The number of people the census | +: : : takers believe that observation : +: : : represents (sample weight). Final : +: : : weight will not be used. : +| education | Categorical | The highest level of education | +: : : achieved for that individual. : +| education_num | Continuous | The highest level of education in | +: : : numerical form. : +| marital_status | Categorical | Marital status of the individual. | +| occupation | Categorical | The occupation of the individual. | +| relationship | Categorical | Wife, Own-child, Husband, | +: : : Not-in-family, Other-relative, : +: : : Unmarried. : +| race | Categorical | Amer-Indian-Eskimo, Asian-Pac- | +: : : Islander, Black, White, Other. : +| gender | Categorical | Female, Male. | +| capital_gain | Continuous | Capital gains recorded. | +| capital_loss | Continuous | Capital Losses recorded. | +| hours_per_week | Continuous | Hours worked per week. | +| native_country | Categorical | Country of origin of the | +: : : individual. : +| income_bracket | Categorical | ">50K" or "<=50K", meaning | +: : : whether the person makes more : +: : : than $50,000 annually. : + +## Converting Data into Tensors + +When building a tf.estimator model, the input data is specified by means of an +Input Builder function. This builder function will not be called until it is +later passed to tf.estimator.Estimator methods such as `train` and `evaluate`. +The purpose of this function is to construct the input data, which is +represented in the form of @{tf.Tensor}s or @{tf.SparseTensor}s. +In more detail, the input builder function returns the following as a pair: + +1. `features`: A dict from feature column names to `Tensors` or + `SparseTensors`. +2. `labels`: A `Tensor` containing the label column. + +The keys of the `features` will be used to construct columns in the next +section. Because we want to call the `train` and `evaluate` methods with +different data, we define a method that returns an input function based on the +given data. Note that the returned input function will be called while +constructing the TensorFlow graph, not while running the graph. What it is +returning is a representation of the input data as the fundamental unit of +TensorFlow computations, a `Tensor` (or `SparseTensor`). + +Each continuous column in the train or test data will be converted into a +`Tensor`, which in general is a good format to represent dense data. For +categorical data, we must represent the data as a `SparseTensor`. This data +format is good for representing sparse data. Our `input_fn` uses the `tf.data` +API, which makes it easy to apply transformations to our dataset: + +```python +def input_fn(data_file, num_epochs, shuffle, batch_size): + """Generate an input function for the Estimator.""" + assert tf.gfile.Exists(data_file), ( + '%s not found. Please make sure you have either run data_download.py or ' + 'set both arguments --train_data and --test_data.' % data_file) + + def parse_csv(value): + print('Parsing', data_file) + columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS) + features = dict(zip(_CSV_COLUMNS, columns)) + labels = features.pop('income_bracket') + return features, tf.equal(labels, '>50K') + + # Extract lines from input files using the Dataset API. + dataset = tf.data.TextLineDataset(data_file) + + if shuffle: + dataset = dataset.shuffle(buffer_size=_SHUFFLE_BUFFER) + + dataset = dataset.map(parse_csv, num_parallel_calls=5) + + # We call repeat after shuffling, rather than before, to prevent separate + # epochs from blending together. + dataset = dataset.repeat(num_epochs) + dataset = dataset.batch(batch_size) + + iterator = dataset.make_one_shot_iterator() + features, labels = iterator.get_next() + return features, labels +``` + +## Selecting and Engineering Features for the Model + +Selecting and crafting the right set of feature columns is key to learning an +effective model. A **feature column** can be either one of the raw columns in +the original dataframe (let's call them **base feature columns**), or any new +columns created based on some transformations defined over one or multiple base +columns (let's call them **derived feature columns**). Basically, "feature +column" is an abstract concept of any raw or derived variable that can be used +to predict the target label. + +### Base Categorical Feature Columns + +To define a feature column for a categorical feature, we can create a +`CategoricalColumn` using the tf.feature_column API. If you know the set of all +possible feature values of a column and there are only a few of them, you can +use `categorical_column_with_vocabulary_list`. Each key in the list will get +assigned an auto-incremental ID starting from 0. For example, for the +`relationship` column we can assign the feature string "Husband" to an integer +ID of 0 and "Not-in-family" to 1, etc., by doing: + +```python +relationship = tf.feature_column.categorical_column_with_vocabulary_list( + 'relationship', [ + 'Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried', + 'Other-relative']) +``` + +What if we don't know the set of possible values in advance? Not a problem. We +can use `categorical_column_with_hash_bucket` instead: + +```python +occupation = tf.feature_column.categorical_column_with_hash_bucket( + 'occupation', hash_bucket_size=1000) +``` + +What will happen is that each possible value in the feature column `occupation` +will be hashed to an integer ID as we encounter them in training. See an example +illustration below: + +ID | Feature +--- | ------------- +... | +9 | `"Machine-op-inspct"` +... | +103 | `"Farming-fishing"` +... | +375 | `"Protective-serv"` +... | + +No matter which way we choose to define a `SparseColumn`, each feature string +will be mapped into an integer ID by looking up a fixed mapping or by hashing. +Note that hashing collisions are possible, but may not significantly impact the +model quality. Under the hood, the `LinearModel` class is responsible for +managing the mapping and creating `tf.Variable` to store the model parameters +(also known as model weights) for each feature ID. The model parameters will be +learned through the model training process we'll go through later. + +We'll do the similar trick to define the other categorical features: + +```python +education = tf.feature_column.categorical_column_with_vocabulary_list( + 'education', [ + 'Bachelors', 'HS-grad', '11th', 'Masters', '9th', 'Some-college', + 'Assoc-acdm', 'Assoc-voc', '7th-8th', 'Doctorate', 'Prof-school', + '5th-6th', '10th', '1st-4th', 'Preschool', '12th']) + +marital_status = tf.feature_column.categorical_column_with_vocabulary_list( + 'marital_status', [ + 'Married-civ-spouse', 'Divorced', 'Married-spouse-absent', + 'Never-married', 'Separated', 'Married-AF-spouse', 'Widowed']) + +relationship = tf.feature_column.categorical_column_with_vocabulary_list( + 'relationship', [ + 'Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried', + 'Other-relative']) + +workclass = tf.feature_column.categorical_column_with_vocabulary_list( + 'workclass', [ + 'Self-emp-not-inc', 'Private', 'State-gov', 'Federal-gov', + 'Local-gov', '?', 'Self-emp-inc', 'Without-pay', 'Never-worked']) + +# To show an example of hashing: +occupation = tf.feature_column.categorical_column_with_hash_bucket( + 'occupation', hash_bucket_size=1000) +``` + +### Base Continuous Feature Columns + +Similarly, we can define a `NumericColumn` for each continuous feature column +that we want to use in the model: + +```python +age = tf.feature_column.numeric_column('age') +education_num = tf.feature_column.numeric_column('education_num') +capital_gain = tf.feature_column.numeric_column('capital_gain') +capital_loss = tf.feature_column.numeric_column('capital_loss') +hours_per_week = tf.feature_column.numeric_column('hours_per_week') +``` + +### Making Continuous Features Categorical through Bucketization + +Sometimes the relationship between a continuous feature and the label is not +linear. As a hypothetical example, a person's income may grow with age in the +early stage of one's career, then the growth may slow at some point, and finally +the income decreases after retirement. In this scenario, using the raw `age` as +a real-valued feature column might not be a good choice because the model can +only learn one of the three cases: + +1. Income always increases at some rate as age grows (positive correlation), +1. Income always decreases at some rate as age grows (negative correlation), or +1. Income stays the same no matter at what age (no correlation) + +If we want to learn the fine-grained correlation between income and each age +group separately, we can leverage **bucketization**. Bucketization is a process +of dividing the entire range of a continuous feature into a set of consecutive +bins/buckets, and then converting the original numerical feature into a bucket +ID (as a categorical feature) depending on which bucket that value falls into. +So, we can define a `bucketized_column` over `age` as: + +```python +age_buckets = tf.feature_column.bucketized_column( + age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) +``` + +where the `boundaries` is a list of bucket boundaries. In this case, there are +10 boundaries, resulting in 11 age group buckets (from age 17 and below, 18-24, +25-29, ..., to 65 and over). + +### Intersecting Multiple Columns with CrossedColumn + +Using each base feature column separately may not be enough to explain the data. +For example, the correlation between education and the label (earning > 50,000 +dollars) may be different for different occupations. Therefore, if we only learn +a single model weight for `education="Bachelors"` and `education="Masters"`, we +won't be able to capture every single education-occupation combination (e.g. +distinguishing between `education="Bachelors" AND occupation="Exec-managerial"` +and `education="Bachelors" AND occupation="Craft-repair"`). To learn the +differences between different feature combinations, we can add **crossed feature +columns** to the model. + +```python +education_x_occupation = tf.feature_column.crossed_column( + ['education', 'occupation'], hash_bucket_size=1000) +``` + +We can also create a `CrossedColumn` over more than two columns. Each +constituent column can be either a base feature column that is categorical +(`SparseColumn`), a bucketized real-valued feature column (`BucketizedColumn`), +or even another `CrossColumn`. Here's an example: + +```python +age_buckets_x_education_x_occupation = tf.feature_column.crossed_column( + [age_buckets, 'education', 'occupation'], hash_bucket_size=1000) +``` + +## Defining The Logistic Regression Model + +After processing the input data and defining all the feature columns, we're now +ready to put them all together and build a Logistic Regression model. In the +previous section we've seen several types of base and derived feature columns, +including: + +* `CategoricalColumn` +* `NumericColumn` +* `BucketizedColumn` +* `CrossedColumn` + +All of these are subclasses of the abstract `FeatureColumn` class, and can be +added to the `feature_columns` field of a model: + +```python +base_columns = [ + education, marital_status, relationship, workclass, occupation, + age_buckets, +] +crossed_columns = [ + tf.feature_column.crossed_column( + ['education', 'occupation'], hash_bucket_size=1000), + tf.feature_column.crossed_column( + [age_buckets, 'education', 'occupation'], hash_bucket_size=1000), +] + +model_dir = tempfile.mkdtemp() +model = tf.estimator.LinearClassifier( + model_dir=model_dir, feature_columns=base_columns + crossed_columns) +``` + +The model also automatically learns a bias term, which controls the prediction +one would make without observing any features (see the section "How Logistic +Regression Works" for more explanations). The learned model files will be stored +in `model_dir`. + +## Training and Evaluating Our Model + +After adding all the features to the model, now let's look at how to actually +train the model. Training a model is just a single command using the +tf.estimator API: + +```python +model.train(input_fn=lambda: input_fn(train_data, num_epochs, True, batch_size)) +``` + +After the model is trained, we can evaluate how good our model is at predicting +the labels of the holdout data: + +```python +results = model.evaluate(input_fn=lambda: input_fn( + test_data, 1, False, batch_size)) +for key in sorted(results): + print('%s: %s' % (key, results[key])) +``` + +The first line of the final output should be something like +`accuracy: 0.83557522`, which means the accuracy is 83.6%. Feel free to try more +features and transformations and see if you can do even better! + +After the model is evaluated, we can use the model to predict whether an individual has an annual income of over +50,000 dollars given an individual's information input. +```python + pred_iter = model.predict(input_fn=lambda: input_fn(FLAGS.test_data, 1, False, 1)) + for pred in pred_iter: + print(pred['classes']) +``` + +The model prediction output would be like `[b'1']` or `[b'0']` which means whether corresponding individual has an annual income of over 50,000 dollars or not. + +If you'd like to see a working end-to-end example, you can download our +[example code](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py) +and set the `model_type` flag to `wide`. + +## Adding Regularization to Prevent Overfitting + +Regularization is a technique used to avoid **overfitting**. Overfitting happens +when your model does well on the data it is trained on, but worse on test data +that the model has not seen before, such as live traffic. Overfitting generally +occurs when a model is excessively complex, such as having too many parameters +relative to the number of observed training data. Regularization allows for you +to control your model's complexity and makes the model more generalizable to +unseen data. + +In the Linear Model library, you can add L1 and L2 regularizations to the model +as: + +``` +model = tf.estimator.LinearClassifier( + model_dir=model_dir, feature_columns=base_columns + crossed_columns, + optimizer=tf.train.FtrlOptimizer( + learning_rate=0.1, + l1_regularization_strength=1.0, + l2_regularization_strength=1.0)) +``` + +One important difference between L1 and L2 regularization is that L1 +regularization tends to make model weights stay at zero, creating sparser +models, whereas L2 regularization also tries to make the model weights closer to +zero but not necessarily zero. Therefore, if you increase the strength of L1 +regularization, you will have a smaller model size because many of the model +weights will be zero. This is often desirable when the feature space is very +large but sparse, and when there are resource constraints that prevent you from +serving a model that is too large. + +In practice, you should try various combinations of L1, L2 regularization +strengths and find the best parameters that best control overfitting and give +you a desirable model size. + +## How Logistic Regression Works + +Finally, let's take a minute to talk about what the Logistic Regression model +actually looks like in case you're not already familiar with it. We'll denote +the label as \\(Y\\), and the set of observed features as a feature vector +\\(\mathbf{x}=[x_1, x_2, ..., x_d]\\). We define \\(Y=1\\) if an individual +earned > 50,000 dollars and \\(Y=0\\) otherwise. In Logistic Regression, the +probability of the label being positive (\\(Y=1\\)) given the features +\\(\mathbf{x}\\) is given as: + +$$ P(Y=1|\mathbf{x}) = \frac{1}{1+\exp(-(\mathbf{w}^T\mathbf{x}+b))}$$ + +where \\(\mathbf{w}=[w_1, w_2, ..., w_d]\\) are the model weights for the +features \\(\mathbf{x}=[x_1, x_2, ..., x_d]\\). \\(b\\) is a constant that is +often called the **bias** of the model. The equation consists of two parts—A +linear model and a logistic function: + +* **Linear Model**: First, we can see that \\(\mathbf{w}^T\mathbf{x}+b = b + + w_1x_1 + ... +w_dx_d\\) is a linear model where the output is a linear + function of the input features \\(\mathbf{x}\\). The bias \\(b\\) is the + prediction one would make without observing any features. The model weight + \\(w_i\\) reflects how the feature \\(x_i\\) is correlated with the positive + label. If \\(x_i\\) is positively correlated with the positive label, the + weight \\(w_i\\) increases, and the probability \\(P(Y=1|\mathbf{x})\\) will + be closer to 1. On the other hand, if \\(x_i\\) is negatively correlated + with the positive label, then the weight \\(w_i\\) decreases and the + probability \\(P(Y=1|\mathbf{x})\\) will be closer to 0. + +* **Logistic Function**: Second, we can see that there's a logistic function + (also known as the sigmoid function) \\(S(t) = 1/(1+\exp(-t))\\) being + applied to the linear model. The logistic function is used to convert the + output of the linear model \\(\mathbf{w}^T\mathbf{x}+b\\) from any real + number into the range of \\([0, 1]\\), which can be interpreted as a + probability. + +Model training is an optimization problem: The goal is to find a set of model +weights (i.e. model parameters) to minimize a **loss function** defined over the +training data, such as logistic loss for Logistic Regression models. The loss +function measures the discrepancy between the ground-truth label and the model's +prediction. If the prediction is very close to the ground-truth label, the loss +value will be low; if the prediction is very far from the label, then the loss +value would be high. + +## Learn Deeper + +If you're interested in learning more, check out our +@{$wide_and_deep$Wide & Deep Learning Tutorial} where we'll show you how to +combine the strengths of linear models and deep neural networks by jointly +training them using the tf.estimator API. diff --git a/tensorflow/docs_src/tutorials/representation/wide_and_deep.md b/tensorflow/docs_src/tutorials/representation/wide_and_deep.md new file mode 100644 index 0000000000..44677a810b --- /dev/null +++ b/tensorflow/docs_src/tutorials/representation/wide_and_deep.md @@ -0,0 +1,243 @@ +# TensorFlow Wide & Deep Learning Tutorial + +In the previous @{$wide$TensorFlow Linear Model Tutorial}, we trained a logistic +regression model to predict the probability that the individual has an annual +income of over 50,000 dollars using the +[Census Income Dataset](https://archive.ics.uci.edu/ml/datasets/Census+Income). +TensorFlow is great for training deep neural networks too, and you might be +thinking which one you should choose—well, why not both? Would it be possible to +combine the strengths of both in one model? + +In this tutorial, we'll introduce how to use the tf.estimator API to jointly +train a wide linear model and a deep feed-forward neural network. This approach +combines the strengths of memorization and generalization. It's useful for +generic large-scale regression and classification problems with sparse input +features (e.g., categorical features with a large number of possible feature +values). If you're interested in learning more about how Wide & Deep Learning +works, please check out our [research paper](https://arxiv.org/abs/1606.07792). + +![Wide & Deep Spectrum of Models](https://www.tensorflow.org/images/wide_n_deep.svg "Wide & Deep") + +The figure above shows a comparison of a wide model (logistic regression with +sparse features and transformations), a deep model (feed-forward neural network +with an embedding layer and several hidden layers), and a Wide & Deep model +(joint training of both). At a high level, there are only 3 steps to configure a +wide, deep, or Wide & Deep model using the tf.estimator API: + +1. Select features for the wide part: Choose the sparse base columns and + crossed columns you want to use. +1. Select features for the deep part: Choose the continuous columns, the + embedding dimension for each categorical column, and the hidden layer sizes. +1. Put them all together in a Wide & Deep model + (`DNNLinearCombinedClassifier`). + +And that's it! Let's go through a simple example. + +## Setup + +To try the code for this tutorial: + +1. @{$install$Install TensorFlow} if you haven't already. + +2. Download [the tutorial code](https://github.com/tensorflow/models/tree/master/official/wide_deep/). + +3. Execute the data download script we provide to you: + + $ python data_download.py + +4. Execute the tutorial code with the following command to train the wide and +deep model described in this tutorial: + + $ python wide_deep.py + +Read on to find out how this code builds its model. + + +## Define Base Feature Columns + +First, let's define the base categorical and continuous feature columns that +we'll use. These base columns will be the building blocks used by both the wide +part and the deep part of the model. + +```python +import tensorflow as tf + +# Continuous columns +age = tf.feature_column.numeric_column('age') +education_num = tf.feature_column.numeric_column('education_num') +capital_gain = tf.feature_column.numeric_column('capital_gain') +capital_loss = tf.feature_column.numeric_column('capital_loss') +hours_per_week = tf.feature_column.numeric_column('hours_per_week') + +education = tf.feature_column.categorical_column_with_vocabulary_list( + 'education', [ + 'Bachelors', 'HS-grad', '11th', 'Masters', '9th', 'Some-college', + 'Assoc-acdm', 'Assoc-voc', '7th-8th', 'Doctorate', 'Prof-school', + '5th-6th', '10th', '1st-4th', 'Preschool', '12th']) + +marital_status = tf.feature_column.categorical_column_with_vocabulary_list( + 'marital_status', [ + 'Married-civ-spouse', 'Divorced', 'Married-spouse-absent', + 'Never-married', 'Separated', 'Married-AF-spouse', 'Widowed']) + +relationship = tf.feature_column.categorical_column_with_vocabulary_list( + 'relationship', [ + 'Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried', + 'Other-relative']) + +workclass = tf.feature_column.categorical_column_with_vocabulary_list( + 'workclass', [ + 'Self-emp-not-inc', 'Private', 'State-gov', 'Federal-gov', + 'Local-gov', '?', 'Self-emp-inc', 'Without-pay', 'Never-worked']) + +# To show an example of hashing: +occupation = tf.feature_column.categorical_column_with_hash_bucket( + 'occupation', hash_bucket_size=1000) + +# Transformations. +age_buckets = tf.feature_column.bucketized_column( + age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) +``` + +## The Wide Model: Linear Model with Crossed Feature Columns + +The wide model is a linear model with a wide set of sparse and crossed feature +columns: + +```python +base_columns = [ + education, marital_status, relationship, workclass, occupation, + age_buckets, +] + +crossed_columns = [ + tf.feature_column.crossed_column( + ['education', 'occupation'], hash_bucket_size=1000), + tf.feature_column.crossed_column( + [age_buckets, 'education', 'occupation'], hash_bucket_size=1000), +] +``` + +You can also see the @{$wide$TensorFlow Linear Model Tutorial} for more details. + +Wide models with crossed feature columns can memorize sparse interactions +between features effectively. That being said, one limitation of crossed feature +columns is that they do not generalize to feature combinations that have not +appeared in the training data. Let's add a deep model with embeddings to fix +that. + +## The Deep Model: Neural Network with Embeddings + +The deep model is a feed-forward neural network, as shown in the previous +figure. Each of the sparse, high-dimensional categorical features are first +converted into a low-dimensional and dense real-valued vector, often referred to +as an embedding vector. These low-dimensional dense embedding vectors are +concatenated with the continuous features, and then fed into the hidden layers +of a neural network in the forward pass. The embedding values are initialized +randomly, and are trained along with all other model parameters to minimize the +training loss. If you're interested in learning more about embeddings, check out +the TensorFlow tutorial on @{$word2vec$Vector Representations of Words} or +[Word embedding](https://en.wikipedia.org/wiki/Word_embedding) on Wikipedia. + +Another way to represent categorical columns to feed into a neural network is +via a one-hot or multi-hot representation. This is often appropriate for +categorical columns with only a few possible values. As an example of a one-hot +representation, for the relationship column, `"Husband"` can be represented as +[1, 0, 0, 0, 0, 0], and `"Not-in-family"` as [0, 1, 0, 0, 0, 0], etc. This is a +fixed representation, whereas embeddings are more flexible and calculated at +training time. + +We'll configure the embeddings for the categorical columns using +`embedding_column`, and concatenate them with the continuous columns. +We also use `indicator_column` to create multi-hot representations of some +categorical columns. + +```python +deep_columns = [ + age, + education_num, + capital_gain, + capital_loss, + hours_per_week, + tf.feature_column.indicator_column(workclass), + tf.feature_column.indicator_column(education), + tf.feature_column.indicator_column(marital_status), + tf.feature_column.indicator_column(relationship), + # To show an example of embedding + tf.feature_column.embedding_column(occupation, dimension=8), +] +``` + +The higher the `dimension` of the embedding is, the more degrees of freedom the +model will have to learn the representations of the features. For simplicity, we +set the dimension to 8 for all feature columns here. Empirically, a more +informed decision for the number of dimensions is to start with a value on the +order of \\(\log_2(n)\\) or \\(k\sqrt[4]n\\), where \\(n\\) is the number of +unique features in a feature column and \\(k\\) is a small constant (usually +smaller than 10). + +Through dense embeddings, deep models can generalize better and make predictions +on feature pairs that were previously unseen in the training data. However, it +is difficult to learn effective low-dimensional representations for feature +columns when the underlying interaction matrix between two feature columns is +sparse and high-rank. In such cases, the interaction between most feature pairs +should be zero except a few, but dense embeddings will lead to nonzero +predictions for all feature pairs, and thus can over-generalize. On the other +hand, linear models with crossed features can memorize these “exception rules” +effectively with fewer model parameters. + +Now, let's see how to jointly train wide and deep models and allow them to +complement each other’s strengths and weaknesses. + +## Combining Wide and Deep Models into One + +The wide models and deep models are combined by summing up their final output +log odds as the prediction, then feeding the prediction to a logistic loss +function. All the graph definition and variable allocations have already been +handled for you under the hood, so you simply need to create a +`DNNLinearCombinedClassifier`: + +```python +model = tf.estimator.DNNLinearCombinedClassifier( + model_dir='/tmp/census_model', + linear_feature_columns=base_columns + crossed_columns, + dnn_feature_columns=deep_columns, + dnn_hidden_units=[100, 50]) +``` + +## Training and Evaluating The Model + +Before we train the model, let's read in the Census dataset as we did in the +@{$wide$TensorFlow Linear Model tutorial}. See `data_download.py` as well as +`input_fn` within +[`wide_deep.py`](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py). + +After reading in the data, you can train and evaluate the model: + +```python +# Train and evaluate the model every `FLAGS.epochs_per_eval` epochs. +for n in range(FLAGS.train_epochs // FLAGS.epochs_per_eval): + model.train(input_fn=lambda: input_fn( + FLAGS.train_data, FLAGS.epochs_per_eval, True, FLAGS.batch_size)) + + results = model.evaluate(input_fn=lambda: input_fn( + FLAGS.test_data, 1, False, FLAGS.batch_size)) + + # Display evaluation metrics + print('Results at epoch', (n + 1) * FLAGS.epochs_per_eval) + print('-' * 30) + + for key in sorted(results): + print('%s: %s' % (key, results[key])) +``` + +The final output accuracy should be somewhere around 85.5%. If you'd like to +see a working end-to-end example, you can download our +[example code](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py). + +Note that this tutorial is just a quick example on a small dataset to get you +familiar with the API. Wide & Deep Learning will be even more powerful if you +try it on a large dataset with many sparse feature columns that have a large +number of possible feature values. Again, feel free to take a look at our +[research paper](https://arxiv.org/abs/1606.07792) for more ideas about how to +apply Wide & Deep Learning in real-world large-scale machine learning problems. diff --git a/tensorflow/docs_src/tutorials/representation/word2vec.md b/tensorflow/docs_src/tutorials/representation/word2vec.md new file mode 100644 index 0000000000..3fe7352bd2 --- /dev/null +++ b/tensorflow/docs_src/tutorials/representation/word2vec.md @@ -0,0 +1,405 @@ +# Vector Representations of Words + +In this tutorial we look at the word2vec model by +[Mikolov et al.](https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf) +This model is used for learning vector representations of words, called "word +embeddings". + +## Highlights + +This tutorial is meant to highlight the interesting, substantive parts of +building a word2vec model in TensorFlow. + +* We start by giving the motivation for why we would want to +represent words as vectors. +* We look at the intuition behind the model and how it is trained +(with a splash of math for good measure). +* We also show a simple implementation of the model in TensorFlow. +* Finally, we look at ways to make the naive version scale better. + +We walk through the code later during the tutorial, but if you'd prefer to dive +straight in, feel free to look at the minimalistic implementation in +[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py) +This basic example contains the code needed to download some data, train on it a +bit and visualize the result. Once you get comfortable with reading and running +the basic version, you can graduate to +[models/tutorials/embedding/word2vec.py](https://www.tensorflow.org/code/tensorflow_models/tutorials/embedding/word2vec.py) +which is a more serious implementation that showcases some more advanced +TensorFlow principles about how to efficiently use threads to move data into a +text model, how to checkpoint during training, etc. + +But first, let's look at why we would want to learn word embeddings in the first +place. Feel free to skip this section if you're an Embedding Pro and you'd just +like to get your hands dirty with the details. + +## Motivation: Why Learn Word Embeddings? + +Image and audio processing systems work with rich, high-dimensional datasets +encoded as vectors of the individual raw pixel-intensities for image data, or +e.g. power spectral density coefficients for audio data. For tasks like object +or speech recognition we know that all the information required to successfully +perform the task is encoded in the data (because humans can perform these tasks +from the raw data). However, natural language processing systems traditionally +treat words as discrete atomic symbols, and therefore 'cat' may be represented +as `Id537` and 'dog' as `Id143`. These encodings are arbitrary, and provide +no useful information to the system regarding the relationships that may exist +between the individual symbols. This means that the model can leverage +very little of what it has learned about 'cats' when it is processing data about +'dogs' (such that they are both animals, four-legged, pets, etc.). Representing +words as unique, discrete ids furthermore leads to data sparsity, and usually +means that we may need more data in order to successfully train statistical +models. Using vector representations can overcome some of these obstacles. + +
+ +
+ +[Vector space models](https://en.wikipedia.org/wiki/Vector_space_model) (VSMs) +represent (embed) words in a continuous vector space where semantically +similar words are mapped to nearby points ('are embedded nearby each other'). +VSMs have a long, rich history in NLP, but all methods depend in some way or +another on the +[Distributional Hypothesis](https://en.wikipedia.org/wiki/Distributional_semantics#Distributional_Hypothesis), +which states that words that appear in the same contexts share +semantic meaning. The different approaches that leverage this principle can be +divided into two categories: *count-based methods* (e.g. +[Latent Semantic Analysis](https://en.wikipedia.org/wiki/Latent_semantic_analysis)), +and *predictive methods* (e.g. +[neural probabilistic language models](http://www.scholarpedia.org/article/Neural_net_language_models)). + +This distinction is elaborated in much more detail by +[Baroni et al.](http://clic.cimec.unitn.it/marco/publications/acl2014/baroni-etal-countpredict-acl2014.pdf), +but in a nutshell: Count-based methods compute the statistics of +how often some word co-occurs with its neighbor words in a large text corpus, +and then map these count-statistics down to a small, dense vector for each word. +Predictive models directly try to predict a word from its neighbors in terms of +learned small, dense *embedding vectors* (considered parameters of the +model). + +Word2vec is a particularly computationally-efficient predictive model for +learning word embeddings from raw text. It comes in two flavors, the Continuous +Bag-of-Words model (CBOW) and the Skip-Gram model (Section 3.1 and 3.2 in [Mikolov et al.](https://arxiv.org/pdf/1301.3781.pdf)). Algorithmically, these +models are similar, except that CBOW predicts target words (e.g. 'mat') from +source context words ('the cat sits on the'), while the skip-gram does the +inverse and predicts source context-words from the target words. This inversion +might seem like an arbitrary choice, but statistically it has the effect that +CBOW smoothes over a lot of the distributional information (by treating an +entire context as one observation). For the most part, this turns out to be a +useful thing for smaller datasets. However, skip-gram treats each context-target +pair as a new observation, and this tends to do better when we have larger +datasets. We will focus on the skip-gram model in the rest of this tutorial. + + +## Scaling up with Noise-Contrastive Training + +Neural probabilistic language models are traditionally trained using the +[maximum likelihood](https://en.wikipedia.org/wiki/Maximum_likelihood) (ML) +principle to maximize the probability of the next word \\(w_t\\) (for "target") +given the previous words \\(h\\) (for "history") in terms of a +[*softmax* function](https://en.wikipedia.org/wiki/Softmax_function), + +$$ +\begin{align} +P(w_t | h) &= \text{softmax}(\text{score}(w_t, h)) \\ + &= \frac{\exp \{ \text{score}(w_t, h) \} } + {\sum_\text{Word w' in Vocab} \exp \{ \text{score}(w', h) \} } +\end{align} +$$ + +where \\(\text{score}(w_t, h)\\) computes the compatibility of word \\(w_t\\) +with the context \\(h\\) (a dot product is commonly used). We train this model +by maximizing its [log-likelihood](https://en.wikipedia.org/wiki/Likelihood_function) +on the training set, i.e. by maximizing + +$$ +\begin{align} + J_\text{ML} &= \log P(w_t | h) \\ + &= \text{score}(w_t, h) - + \log \left( \sum_\text{Word w' in Vocab} \exp \{ \text{score}(w', h) \} \right). +\end{align} +$$ + +This yields a properly normalized probabilistic model for language modeling. +However this is very expensive, because we need to compute and normalize each +probability using the score for all other \\(V\\) words \\(w'\\) in the current +context \\(h\\), *at every training step*. + +
+ +
+ +On the other hand, for feature learning in word2vec we do not need a full +probabilistic model. The CBOW and skip-gram models are instead trained using a +binary classification objective ([logistic regression](https://en.wikipedia.org/wiki/Logistic_regression)) +to discriminate the real target words \\(w_t\\) from \\(k\\) imaginary (noise) words \\(\tilde w\\), in the +same context. We illustrate this below for a CBOW model. For skip-gram the +direction is simply inverted. + +
+ +
+ +Mathematically, the objective (for each example) is to maximize + +$$J_\text{NEG} = \log Q_\theta(D=1 |w_t, h) + + k \mathop{\mathbb{E}}_{\tilde w \sim P_\text{noise}} + \left[ \log Q_\theta(D = 0 |\tilde w, h) \right]$$ + +where \\(Q_\theta(D=1 | w, h)\\) is the binary logistic regression probability +under the model of seeing the word \\(w\\) in the context \\(h\\) in the dataset +\\(D\\), calculated in terms of the learned embedding vectors \\(\theta\\). In +practice we approximate the expectation by drawing \\(k\\) contrastive words +from the noise distribution (i.e. we compute a +[Monte Carlo average](https://en.wikipedia.org/wiki/Monte_Carlo_integration)). + +This objective is maximized when the model assigns high probabilities +to the real words, and low probabilities to noise words. Technically, this is +called +[Negative Sampling](https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf), +and there is good mathematical motivation for using this loss function: +The updates it proposes approximate the updates of the softmax function in the +limit. But computationally it is especially appealing because computing the +loss function now scales only with the number of *noise words* that we +select (\\(k\\)), and not *all words* in the vocabulary (\\(V\\)). This makes it +much faster to train. We will actually make use of the very similar +[noise-contrastive estimation (NCE)](https://papers.nips.cc/paper/5165-learning-word-embeddings-efficiently-with-noise-contrastive-estimation.pdf) +loss, for which TensorFlow has a handy helper function `tf.nn.nce_loss()`. + +Let's get an intuitive feel for how this would work in practice! + +## The Skip-gram Model + +As an example, let's consider the dataset + +`the quick brown fox jumped over the lazy dog` + +We first form a dataset of words and the contexts in which they appear. We +could define 'context' in any way that makes sense, and in fact people have +looked at syntactic contexts (i.e. the syntactic dependents of the current +target word, see e.g. +[Levy et al.](https://levyomer.files.wordpress.com/2014/04/dependency-based-word-embeddings-acl-2014.pdf)), +words-to-the-left of the target, words-to-the-right of the target, etc. For now, +let's stick to the vanilla definition and define 'context' as the window +of words to the left and to the right of a target word. Using a window +size of 1, we then have the dataset + +`([the, brown], quick), ([quick, fox], brown), ([brown, jumped], fox), ...` + +of `(context, target)` pairs. Recall that skip-gram inverts contexts and +targets, and tries to predict each context word from its target word, so the +task becomes to predict 'the' and 'brown' from 'quick', 'quick' and 'fox' from +'brown', etc. Therefore our dataset becomes + +`(quick, the), (quick, brown), (brown, quick), (brown, fox), ...` + +of `(input, output)` pairs. The objective function is defined over the entire +dataset, but we typically optimize this with +[stochastic gradient descent](https://en.wikipedia.org/wiki/Stochastic_gradient_descent) +(SGD) using one example at a time (or a 'minibatch' of `batch_size` examples, +where typically `16 <= batch_size <= 512`). So let's look at one step of +this process. + +Let's imagine at training step \\(t\\) we observe the first training case above, +where the goal is to predict `the` from `quick`. We select `num_noise` number +of noisy (contrastive) examples by drawing from some noise distribution, +typically the unigram distribution, \\(P(w)\\). For simplicity let's say +`num_noise=1` and we select `sheep` as a noisy example. Next we compute the +loss for this pair of observed and noisy examples, i.e. the objective at time +step \\(t\\) becomes + +$$J^{(t)}_\text{NEG} = \log Q_\theta(D=1 | \text{the, quick}) + + \log(Q_\theta(D=0 | \text{sheep, quick}))$$ + +The goal is to make an update to the embedding parameters \\(\theta\\) to improve +(in this case, maximize) this objective function. We do this by deriving the +gradient of the loss with respect to the embedding parameters \\(\theta\\), i.e. +\\(\frac{\partial}{\partial \theta} J_\text{NEG}\\) (luckily TensorFlow provides +easy helper functions for doing this!). We then perform an update to the +embeddings by taking a small step in the direction of the gradient. When this +process is repeated over the entire training set, this has the effect of +'moving' the embedding vectors around for each word until the model is +successful at discriminating real words from noise words. + +We can visualize the learned vectors by projecting them down to 2 dimensions +using for instance something like the +[t-SNE dimensionality reduction technique](https://lvdmaaten.github.io/tsne/). +When we inspect these visualizations it becomes apparent that the vectors +capture some general, and in fact quite useful, semantic information about +words and their relationships to one another. It was very interesting when we +first discovered that certain directions in the induced vector space specialize +towards certain semantic relationships, e.g. *male-female*, *verb tense* and +even *country-capital* relationships between words, as illustrated in the figure +below (see also for example +[Mikolov et al., 2013](https://www.aclweb.org/anthology/N13-1090)). + +
+ +
+ +This explains why these vectors are also useful as features for many canonical +NLP prediction tasks, such as part-of-speech tagging or named entity recognition +(see for example the original work by +[Collobert et al., 2011](https://arxiv.org/abs/1103.0398) +([pdf](https://arxiv.org/pdf/1103.0398.pdf)), or follow-up work by +[Turian et al., 2010](https://www.aclweb.org/anthology/P10-1040)). + +But for now, let's just use them to draw pretty pictures! + +## Building the Graph + +This is all about embeddings, so let's define our embedding matrix. +This is just a big random matrix to start. We'll initialize the values to be +uniform in the unit cube. + +```python +embeddings = tf.Variable( + tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) +``` + +The noise-contrastive estimation loss is defined in terms of a logistic regression +model. For this, we need to define the weights and biases for each word in the +vocabulary (also called the `output weights` as opposed to the `input +embeddings`). So let's define that. + +```python +nce_weights = tf.Variable( + tf.truncated_normal([vocabulary_size, embedding_size], + stddev=1.0 / math.sqrt(embedding_size))) +nce_biases = tf.Variable(tf.zeros([vocabulary_size])) +``` + +Now that we have the parameters in place, we can define our skip-gram model +graph. For simplicity, let's suppose we've already integerized our text corpus +with a vocabulary so that each word is represented as an integer (see +[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py) +for the details). The skip-gram model takes two inputs. One is a batch full of +integers representing the source context words, the other is for the target +words. Let's create placeholder nodes for these inputs, so that we can feed in +data later. + +```python +# Placeholders for inputs +train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) +train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) +``` + +Now what we need to do is look up the vector for each of the source words in +the batch. TensorFlow has handy helpers that make this easy. + +```python +embed = tf.nn.embedding_lookup(embeddings, train_inputs) +``` + +Ok, now that we have the embeddings for each word, we'd like to try to predict +the target word using the noise-contrastive training objective. + +```python +# Compute the NCE loss, using a sample of the negative labels each time. +loss = tf.reduce_mean( + tf.nn.nce_loss(weights=nce_weights, + biases=nce_biases, + labels=train_labels, + inputs=embed, + num_sampled=num_sampled, + num_classes=vocabulary_size)) +``` + +Now that we have a loss node, we need to add the nodes required to compute +gradients and update the parameters, etc. For this we will use stochastic +gradient descent, and TensorFlow has handy helpers to make this easy as well. + +```python +# We use the SGD optimizer. +optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0).minimize(loss) +``` + +## Training the Model + +Training the model is then as simple as using a `feed_dict` to push data into +the placeholders and calling +@{tf.Session.run} with this new data +in a loop. + +```python +for inputs, labels in generate_batch(...): + feed_dict = {train_inputs: inputs, train_labels: labels} + _, cur_loss = session.run([optimizer, loss], feed_dict=feed_dict) +``` + +See the full example code in +[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py). + +## Visualizing the Learned Embeddings + +After training has finished we can visualize the learned embeddings using +t-SNE. + +
+ +
+ +Et voila! As expected, words that are similar end up clustering nearby each +other. For a more heavyweight implementation of word2vec that showcases more of +the advanced features of TensorFlow, see the implementation in +[models/tutorials/embedding/word2vec.py](https://www.tensorflow.org/code/tensorflow_models/tutorials/embedding/word2vec.py). + +## Evaluating Embeddings: Analogical Reasoning + +Embeddings are useful for a wide variety of prediction tasks in NLP. Short of +training a full-blown part-of-speech model or named-entity model, one simple way +to evaluate embeddings is to directly use them to predict syntactic and semantic +relationships like `king is to queen as father is to ?`. This is called +*analogical reasoning* and the task was introduced by +[Mikolov and colleagues +](https://www.aclweb.org/anthology/N13-1090). +Download the dataset for this task from +[download.tensorflow.org](http://download.tensorflow.org/data/questions-words.txt). + +To see how we do this evaluation, have a look at the `build_eval_graph()` and +`eval()` functions in +[models/tutorials/embedding/word2vec.py](https://www.tensorflow.org/code/tensorflow_models/tutorials/embedding/word2vec.py). + +The choice of hyperparameters can strongly influence the accuracy on this task. +To achieve state-of-the-art performance on this task requires training over a +very large dataset, carefully tuning the hyperparameters and making use of +tricks like subsampling the data, which is out of the scope of this tutorial. + + +## Optimizing the Implementation + +Our vanilla implementation showcases the flexibility of TensorFlow. For +example, changing the training objective is as simple as swapping out the call +to `tf.nn.nce_loss()` for an off-the-shelf alternative such as +`tf.nn.sampled_softmax_loss()`. If you have a new idea for a loss function, you +can manually write an expression for the new objective in TensorFlow and let +the optimizer compute its derivatives. This flexibility is invaluable in the +exploratory phase of machine learning model development, where we are trying +out several different ideas and iterating quickly. + +Once you have a model structure you're satisfied with, it may be worth +optimizing your implementation to run more efficiently (and cover more data in +less time). For example, the naive code we used in this tutorial would suffer +compromised speed because we use Python for reading and feeding data items -- +each of which require very little work on the TensorFlow back-end. If you find +your model is seriously bottlenecked on input data, you may want to implement a +custom data reader for your problem, as described in +@{$new_data_formats$New Data Formats}. For the case of Skip-Gram +modeling, we've actually already done this for you as an example in +[models/tutorials/embedding/word2vec.py](https://www.tensorflow.org/code/tensorflow_models/tutorials/embedding/word2vec.py). + +If your model is no longer I/O bound but you want still more performance, you +can take things further by writing your own TensorFlow Ops, as described in +@{$adding_an_op$Adding a New Op}. Again we've provided an +example of this for the Skip-Gram case +[models/tutorials/embedding/word2vec_optimized.py](https://www.tensorflow.org/code/tensorflow_models/tutorials/embedding/word2vec_optimized.py). +Feel free to benchmark these against each other to measure performance +improvements at each stage. + +## Conclusion + +In this tutorial we covered the word2vec model, a computationally efficient +model for learning word embeddings. We motivated why embeddings are useful, +discussed efficient training techniques and showed how to implement all of this +in TensorFlow. Overall, we hope that this has show-cased how TensorFlow affords +you the flexibility you need for early experimentation, and the control you +later need for bespoke optimized implementation. diff --git a/tensorflow/docs_src/tutorials/seq2seq.md b/tensorflow/docs_src/tutorials/seq2seq.md deleted file mode 100644 index 8928ba4f7d..0000000000 --- a/tensorflow/docs_src/tutorials/seq2seq.md +++ /dev/null @@ -1,5 +0,0 @@ -# Sequence-to-Sequence Models - -Please check out the -[tensorflow neural machine translation tutorial](https://github.com/tensorflow/nmt) -for building sequence-to-sequence models with the latest Tensorflow API. diff --git a/tensorflow/docs_src/tutorials/sequences/audio_recognition.md b/tensorflow/docs_src/tutorials/sequences/audio_recognition.md new file mode 100644 index 0000000000..d7a8da6f96 --- /dev/null +++ b/tensorflow/docs_src/tutorials/sequences/audio_recognition.md @@ -0,0 +1,631 @@ +# Simple Audio Recognition + +This tutorial will show you how to build a basic speech recognition network that +recognizes ten different words. It's important to know that real speech and +audio recognition systems are much more complex, but like MNIST for images, it +should give you a basic understanding of the techniques involved. Once you've +completed this tutorial, you'll have a model that tries to classify a one second +audio clip as either silence, an unknown word, "yes", "no", "up", "down", +"left", "right", "on", "off", "stop", or "go". You'll also be able to take this +model and run it in an Android application. + +## Preparation + +You should make sure you have TensorFlow installed, and since the script +downloads over 1GB of training data, you'll need a good internet connection and +enough free space on your machine. The training process itself can take several +hours, so make sure you have a machine available for that long. + +## Training + +To begin the training process, go to the TensorFlow source tree and run: + +```bash +python tensorflow/examples/speech_commands/train.py +``` + +The script will start off by downloading the [Speech Commands +dataset](https://storage.cloud.google.com/download.tensorflow.org/data/speech_commands_v0.02.tar.gz), +which consists of over 105,000 WAVE audio files of people saying thirty +different words. This data was collected by Google and released under a CC BY +license, and you can help improve it by [contributing five minutes of your own +voice](https://aiyprojects.withgoogle.com/open_speech_recording). The archive is +over 2GB, so this part may take a while, but you should see progress logs, and +once it's been downloaded once you won't need to do this step again. You can +find more information about this dataset in this +[Speech Commands paper](https://arxiv.org/abs/1804.03209). + +Once the downloading has completed, you'll see logging information that looks +like this: + +``` +I0730 16:53:44.766740 55030 train.py:176] Training from step: 1 +I0730 16:53:47.289078 55030 train.py:217] Step #1: rate 0.001000, accuracy 7.0%, cross entropy 2.611571 +``` + +This shows that the initialization process is done and the training loop has +begun. You'll see that it outputs information for every training step. Here's a +break down of what it means: + +`Step #1` shows that we're on the first step of the training loop. In this case +there are going to be 18,000 steps in total, so you can look at the step number +to get an idea of how close it is to finishing. + +`rate 0.001000` is the learning rate that's controlling the speed of the +network's weight updates. Early on this is a comparatively high number (0.001), +but for later training cycles it will be reduced 10x, to 0.0001. + +`accuracy 7.0%` is the how many classes were correctly predicted on this +training step. This value will often fluctuate a lot, but should increase on +average as training progresses. The model outputs an array of numbers, one for +each label, and each number is the predicted likelihood of the input being that +class. The predicted label is picked by choosing the entry with the highest +score. The scores are always between zero and one, with higher values +representing more confidence in the result. + +`cross entropy 2.611571` is the result of the loss function that we're using to +guide the training process. This is a score that's obtained by comparing the +vector of scores from the current training run to the correct labels, and this +should trend downwards during training. + +After a hundred steps, you should see a line like this: + +`I0730 16:54:41.813438 55030 train.py:252] Saving to +"/tmp/speech_commands_train/conv.ckpt-100"` + +This is saving out the current trained weights to a checkpoint file. If your +training script gets interrupted, you can look for the last saved checkpoint and +then restart the script with +`--start_checkpoint=/tmp/speech_commands_train/conv.ckpt-100` as a command line +argument to start from that point. + +## Confusion Matrix + +After four hundred steps, this information will be logged: + +``` +I0730 16:57:38.073667 55030 train.py:243] Confusion Matrix: + [[258 0 0 0 0 0 0 0 0 0 0 0] + [ 7 6 26 94 7 49 1 15 40 2 0 11] + [ 10 1 107 80 13 22 0 13 10 1 0 4] + [ 1 3 16 163 6 48 0 5 10 1 0 17] + [ 15 1 17 114 55 13 0 9 22 5 0 9] + [ 1 1 6 97 3 87 1 12 46 0 0 10] + [ 8 6 86 84 13 24 1 9 9 1 0 6] + [ 9 3 32 112 9 26 1 36 19 0 0 9] + [ 8 2 12 94 9 52 0 6 72 0 0 2] + [ 16 1 39 74 29 42 0 6 37 9 0 3] + [ 15 6 17 71 50 37 0 6 32 2 1 9] + [ 11 1 6 151 5 42 0 8 16 0 0 20]] +``` + +The first section is a [confusion +matrix](https://www.tensorflow.org/api_docs/python/tf/confusion_matrix). To +understand what it means, you first need to know the labels being used, which in +this case are "_silence_", "_unknown_", "yes", "no", "up", "down", "left", +"right", "on", "off", "stop", and "go". Each column represents a set of samples +that were predicted to be each label, so the first column represents all the +clips that were predicted to be silence, the second all those that were +predicted to be unknown words, the third "yes", and so on. + +Each row represents clips by their correct, ground truth labels. The first row +is all the clips that were silence, the second clips that were unknown words, +the third "yes", etc. + +This matrix can be more useful than just a single accuracy score because it +gives a good summary of what mistakes the network is making. In this example you +can see that all of the entries in the first row are zero, apart from the +initial one. Because the first row is all the clips that are actually silence, +this means that none of them were mistakenly labeled as words, so we have no +false negatives for silence. This shows the network is already getting pretty +good at distinguishing silence from words. + +If we look down the first column though, we see a lot of non-zero values. The +column represents all the clips that were predicted to be silence, so positive +numbers outside of the first cell are errors. This means that some clips of real +spoken words are actually being predicted to be silence, so we do have quite a +few false positives. + +A perfect model would produce a confusion matrix where all of the entries were +zero apart from a diagonal line through the center. Spotting deviations from +that pattern can help you figure out how the model is most easily confused, and +once you've identified the problems you can address them by adding more data or +cleaning up categories. + +## Validation + +After the confusion matrix, you should see a line like this: + +`I0730 16:57:38.073777 55030 train.py:245] Step 400: Validation accuracy = 26.3% +(N=3093)` + +It's good practice to separate your data set into three categories. The largest +(in this case roughly 80% of the data) is used for training the network, a +smaller set (10% here, known as "validation") is reserved for evaluation of the +accuracy during training, and another set (the last 10%, "testing") is used to +evaluate the accuracy once after the training is complete. + +The reason for this split is that there's always a danger that networks will +start memorizing their inputs during training. By keeping the validation set +separate, you can ensure that the model works with data it's never seen before. +The testing set is an additional safeguard to make sure that you haven't just +been tweaking your model in a way that happens to work for both the training and +validation sets, but not a broader range of inputs. + +The training script automatically separates the data set into these three +categories, and the logging line above shows the accuracy of model when run on +the validation set. Ideally, this should stick fairly close to the training +accuracy. If the training accuracy increases but the validation doesn't, that's +a sign that overfitting is occurring, and your model is only learning things +about the training clips, not broader patterns that generalize. + +## Tensorboard + +A good way to visualize how the training is progressing is using Tensorboard. By +default, the script saves out events to /tmp/retrain_logs, and you can load +these by running: + +`tensorboard --logdir /tmp/retrain_logs` + +Then navigate to [http://localhost:6006](http://localhost:6006) in your browser, +and you'll see charts and graphs showing your models progress. + +
+ +
+ +## Training Finished + +After a few hours of training (depending on your machine's speed), the script +should have completed all 18,000 steps. It will print out a final confusion +matrix, along with an accuracy score, all run on the testing set. With the +default settings, you should see an accuracy of between 85% and 90%. + +Because audio recognition is particularly useful on mobile devices, next we'll +export it to a compact format that's easy to work with on those platforms. To do +that, run this command line: + +``` +python tensorflow/examples/speech_commands/freeze.py \ +--start_checkpoint=/tmp/speech_commands_train/conv.ckpt-18000 \ +--output_file=/tmp/my_frozen_graph.pb +``` + +Once the frozen model has been created, you can test it with the `label_wav.py` +script, like this: + +``` +python tensorflow/examples/speech_commands/label_wav.py \ +--graph=/tmp/my_frozen_graph.pb \ +--labels=/tmp/speech_commands_train/conv_labels.txt \ +--wav=/tmp/speech_dataset/left/a5d485dc_nohash_0.wav +``` + +This should print out three labels: + +``` +left (score = 0.81477) +right (score = 0.14139) +_unknown_ (score = 0.03808) +``` + +Hopefully "left" is the top score since that's the correct label, but since the +training is random it may not for the first file you try. Experiment with some +of the other .wav files in that same folder to see how well it does. + +The scores are between zero and one, and higher values mean the model is more +confident in its prediction. + +## Running the Model in an Android App + +The easiest way to see how this model works in a real application is to download +[the prebuilt Android demo +applications](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#prebuilt-components) +and install them on your phone. You'll see 'TF Speech' appear in your app list, +and opening it will show you the same list of action words we've just trained +our model on, starting with "Yes" and "No". Once you've given the app permission +to use the microphone, you should be able to try saying those words and see them +highlighted in the UI when the model recognizes one of them. + +You can also build this application yourself, since it's open source and +[available as part of the TensorFlow repository on +github](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#building-in-android-studio-using-the-tensorflow-aar-from-jcenter). +By default it downloads [a pretrained model from +tensorflow.org](http://download.tensorflow.org/models/speech_commands_v0.02.zip), +but you can easily [replace it with a model you've trained +yourself](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#install-model-files-optional). +If you do this, you'll need to make sure that the constants in [the main +SpeechActivity Java source +file](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android/src/org/tensorflow/demo/SpeechActivity.java) +like `SAMPLE_RATE` and `SAMPLE_DURATION` match any changes you've made to the +defaults while training. You'll also see that there's a [Java version of the +RecognizeCommands +module](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android/src/org/tensorflow/demo/RecognizeCommands.java) +that's very similar to the C++ version in this tutorial. If you've tweaked +parameters for that, you can also update them in SpeechActivity to get the same +results as in your server testing. + +The demo app updates its UI list of results automatically based on the labels +text file you copy into assets alongside your frozen graph, which means you can +easily try out different models without needing to make any code changes. You +will need to update `LABEL_FILENAME` and `MODEL_FILENAME` to point to the files +you've added if you change the paths though. + +## How does this Model Work? + +The architecture used in this tutorial is based on some described in the paper +[Convolutional Neural Networks for Small-footprint Keyword +Spotting](http://www.isca-speech.org/archive/interspeech_2015/papers/i15_1478.pdf). +It was chosen because it's comparatively simple, quick to train, and easy to +understand, rather than being state of the art. There are lots of different +approaches to building neural network models to work with audio, including +[recurrent networks](https://svds.com/tensorflow-rnn-tutorial/) or [dilated +(atrous) +convolutions](https://deepmind.com/blog/wavenet-generative-model-raw-audio/). +This tutorial is based on the kind of convolutional network that will feel very +familiar to anyone who's worked with image recognition. That may seem surprising +at first though, since audio is inherently a one-dimensional continuous signal +across time, not a 2D spatial problem. + +We solve that issue by defining a window of time we believe our spoken words +should fit into, and converting the audio signal in that window into an image. +This is done by grouping the incoming audio samples into short segments, just a +few milliseconds long, and calculating the strength of the frequencies across a +set of bands. Each set of frequency strengths from a segment is treated as a +vector of numbers, and those vectors are arranged in time order to form a +two-dimensional array. This array of values can then be treated like a +single-channel image, and is known as a +[spectrogram](https://en.wikipedia.org/wiki/Spectrogram). If you want to view +what kind of image an audio sample produces, you can run the `wav_to_spectrogram +tool: + +``` +bazel run tensorflow/examples/wav_to_spectrogram:wav_to_spectrogram -- \ +--input_wav=/tmp/speech_dataset/happy/ab00c4b2_nohash_0.wav \ +--output_image=/tmp/spectrogram.png +``` + +If you open up `/tmp/spectrogram.png` you should see something like this: + +
+ +
+ +Because of TensorFlow's memory order, time in this image is increasing from top +to bottom, with frequencies going from left to right, unlike the usual +convention for spectrograms where time is left to right. You should be able to +see a couple of distinct parts, with the first syllable "Ha" distinct from +"ppy". + +Because the human ear is more sensitive to some frequencies than others, it's +been traditional in speech recognition to do further processing to this +representation to turn it into a set of [Mel-Frequency Cepstral +Coefficients](https://en.wikipedia.org/wiki/Mel-frequency_cepstrum), or MFCCs +for short. This is also a two-dimensional, one-channel representation so it can +be treated like an image too. If you're targeting general sounds rather than +speech you may find you can skip this step and operate directly on the +spectrograms. + +The image that's produced by these processing steps is then fed into a +multi-layer convolutional neural network, with a fully-connected layer followed +by a softmax at the end. You can see the definition of this portion in +[tensorflow/examples/speech_commands/models.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/models.py). + +## Streaming Accuracy + +Most audio recognition applications need to run on a continuous stream of audio, +rather than on individual clips. A typical way to use a model in this +environment is to apply it repeatedly at different offsets in time and average +the results over a short window to produce a smoothed prediction. If you think +of the input as an image, it's continuously scrolling along the time axis. The +words we want to recognize can start at any time, so we need to take a series of +snapshots to have a chance of having an alignment that captures most of the +utterance in the time window we feed into the model. If we sample at a high +enough rate, then we have a good chance of capturing the word in multiple +windows, so averaging the results improves the overall confidence of the +prediction. + +For an example of how you can use your model on streaming data, you can look at +[test_streaming_accuracy.cc](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/). +This uses the +[RecognizeCommands](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/recognize_commands.h) +class to run through a long-form input audio, try to spot words, and compare +those predictions against a ground truth list of labels and times. This makes it +a good example of applying a model to a stream of audio signals over time. + +You'll need a long audio file to test it against, along with labels showing +where each word was spoken. If you don't want to record one yourself, you can +generate some synthetic test data using the `generate_streaming_test_wav` +utility. By default this will create a ten minute .wav file with words roughly +every three seconds, and a text file containing the ground truth of when each +word was spoken. These words are pulled from the test portion of your current +dataset, mixed in with background noise. To run it, use: + +``` +bazel run tensorflow/examples/speech_commands:generate_streaming_test_wav +``` + +This will save a .wav file to `/tmp/speech_commands_train/streaming_test.wav`, +and a text file listing the labels to +`/tmp/speech_commands_train/streaming_test_labels.txt`. You can then run +accuracy testing with: + +``` +bazel run tensorflow/examples/speech_commands:test_streaming_accuracy -- \ +--graph=/tmp/my_frozen_graph.pb \ +--labels=/tmp/speech_commands_train/conv_labels.txt \ +--wav=/tmp/speech_commands_train/streaming_test.wav \ +--ground_truth=/tmp/speech_commands_train/streaming_test_labels.txt \ +--verbose +``` + +This will output information about the number of words correctly matched, how +many were given the wrong labels, and how many times the model triggered when +there was no real word spoken. There are various parameters that control how the +signal averaging works, including `--average_window_ms` which sets the length of +time to average results over, `--clip_stride_ms` which is the time between +applications of the model, `--suppression_ms` which stops subsequent word +detections from triggering for a certain time after an initial one is found, and +`--detection_threshold`, which controls how high the average score must be +before it's considered a solid result. + +You'll see that the streaming accuracy outputs three numbers, rather than just +the one metric used in training. This is because different applications have +varying requirements, with some being able to tolerate frequent incorrect +results as long as real words are found (high recall), while others very focused +on ensuring the predicted labels are highly likely to be correct even if some +aren't detected (high precision). The numbers from the tool give you an idea of +how your model will perform in an application, and you can try tweaking the +signal averaging parameters to tune it to give the kind of performance you want. +To understand what the right parameters are for your application, you can look +at generating an [ROC +curve](https://en.wikipedia.org/wiki/Receiver_operating_characteristic) to help +you understand the tradeoffs. + +## RecognizeCommands + +The streaming accuracy tool uses a simple decoder contained in a small C++ class +called +[RecognizeCommands](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/recognize_commands.h). +This class is fed the output of running the TensorFlow model over time, it +averages the signals, and returns information about a label when it has enough +evidence to think that a recognized word has been found. The implementation is +fairly small, just keeping track of the last few predictions and averaging them, +so it's easy to port to other platforms and languages as needed. For example, +it's convenient to do something similar at the Java level on Android, or Python +on the Raspberry Pi. As long as these implementations share the same logic, you +can tune the parameters that control the averaging using the streaming test +tool, and then transfer them over to your application to get similar results. + +## Advanced Training + +The defaults for the training script are designed to produce good end to end +results in a comparatively small file, but there are a lot of options you can +change to customize the results for your own requirements. + +### Custom Training Data + +By default the script will download the [Speech Commands +dataset](https://download.tensorflow.org/data/speech_commands_v0.01.tgz), but +you can also supply your own training data. To train on your own data, you +should make sure that you have at least several hundred recordings of each sound +you would like to recognize, and arrange them into folders by class. For +example, if you were trying to recognize dog barks from cat miaows, you would +create a root folder called `animal_sounds`, and then within that two +sub-folders called `bark` and `miaow`. You would then organize your audio files +into the appropriate folders. + +To point the script to your new audio files, you'll need to set `--data_url=` to +disable downloading of the Speech Commands dataset, and +`--data_dir=/your/data/folder/` to find the files you've just created. + +The files themselves should be 16-bit little-endian PCM-encoded WAVE format. The +sample rate defaults to 16,000, but as long as all your audio is consistently +the same rate (the script doesn't support resampling) you can change this with +the `--sample_rate` argument. The clips should also all be roughly the same +duration. The default expected duration is one second, but you can set this with +the `--clip_duration_ms` flag. If you have clips with variable amounts of +silence at the start, you can look at word alignment tools to standardize them +([here's a quick and dirty approach you can use +too](https://petewarden.com/2017/07/17/a-quick-hack-to-align-single-word-audio-recordings/)). + +One issue to watch out for is that you may have very similar repetitions of the +same sounds in your dataset, and these can give misleading metrics if they're +spread across your training, validation, and test sets. For example, the Speech +Commands set has people repeating the same word multiple times. Each one of +those repetitions is likely to be pretty close to the others, so if training was +overfitting and memorizing one, it could perform unrealistically well when it +saw a very similar copy in the test set. To avoid this danger, Speech Commands +trys to ensure that all clips featuring the same word spoken by a single person +are put into the same partition. Clips are assigned to training, test, or +validation sets based on a hash of their filename, to ensure that the +assignments remain steady even as new clips are added and avoid any training +samples migrating into the other sets. To make sure that all a given speaker's +words are in the same bucket, [the hashing +function](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/input_data.py) +ignores anything in a filename after '_nohash_' when calculating the +assignments. This means that if you have file names like `pete_nohash_0.wav` and +`pete_nohash_1.wav`, they're guaranteed to be in the same set. + +### Unknown Class + +It's likely that your application will hear sounds that aren't in your training +set, and you'll want the model to indicate that it doesn't recognize the noise +in those cases. To help the network learn what sounds to ignore, you need to +provide some clips of audio that are neither of your classes. To do this, you'd +create `quack`, `oink`, and `moo` subfolders and populate them with noises from +other animals your users might encounter. The `--wanted_words` argument to the +script defines which classes you care about, all the others mentioned in +subfolder names will be used to populate an `_unknown_` class during training. +The Speech Commands dataset has twenty words in its unknown classes, including +the digits zero through nine and random names like "Sheila". + +By default 10% of the training examples are picked from the unknown classes, but +you can control this with the `--unknown_percentage` flag. Increasing this will +make the model less likely to mistake unknown words for wanted ones, but making +it too large can backfire as the model might decide it's safest to categorize +all words as unknown! + +### Background Noise + +Real applications have to recognize audio even when there are other irrelevant +sounds happening in the environment. To build a model that's robust to this kind +of interference, we need to train against recorded audio with similar +properties. The files in the Speech Commands dataset were captured on a variety +of devices by users in many different environments, not in a studio, so that +helps add some realism to the training. To add even more, you can mix in random +segments of environmental audio to the training inputs. In the Speech Commands +set there's a special folder called `_background_noise_` which contains +minute-long WAVE files with white noise and recordings of machinery and everyday +household activity. + +Small snippets of these files are chosen at random and mixed at a low volume +into clips during training. The loudness is also chosen randomly, and controlled +by the `--background_volume` argument as a proportion where 0 is silence, and 1 +is full volume. Not all clips have background added, so the +`--background_frequency` flag controls what proportion have them mixed in. + +Your own application might operate in its own environment with different +background noise patterns than these defaults, so you can supply your own audio +clips in the `_background_noise_` folder. These should be the same sample rate +as your main dataset, but much longer in duration so that a good set of random +segments can be selected from them. + +### Silence + +In most cases the sounds you care about will be intermittent and so it's +important to know when there's no matching audio. To support this, there's a +special `_silence_` label that indicates when the model detects nothing +interesting. Because there's never complete silence in real environments, we +actually have to supply examples with quiet and irrelevant audio. For this, we +reuse the `_background_noise_` folder that's also mixed in to real clips, +pulling short sections of the audio data and feeding those in with the ground +truth class of `_silence_`. By default 10% of the training data is supplied like +this, but the `--silence_percentage` can be used to control the proportion. As +with unknown words, setting this higher can weight the model results in favor of +true positives for silence, at the expense of false negatives for words, but too +large a proportion can cause it to fall into the trap of always guessing +silence. + +### Time Shifting + +Adding in background noise is one way of distorting the training data in a +realistic way to effectively increase the size of the dataset, and so increase +overall accuracy, and time shifting is another. This involves a random offset in +time of the training sample data, so that a small part of the start or end is +cut off and the opposite section is padded with zeroes. This mimics the natural +variations in starting time in the training data, and is controlled with the +`--time_shift_ms` flag, which defaults to 100ms. Increasing this value will +provide more variation, but at the risk of cutting off important parts of the +audio. A related way of augmenting the data with realistic distortions is by +using [time stretching and pitch +scaling](https://en.wikipedia.org/wiki/Audio_time_stretching_and_pitch_scaling), +but that's outside the scope of this tutorial. + +## Customizing the Model + +The default model used for this script is pretty large, taking over 800 million +FLOPs for each inference and using 940,000 weight parameters. This runs at +usable speeds on desktop machines or modern phones, but it involves too many +calculations to run at interactive speeds on devices with more limited +resources. To support these use cases, there's a couple of alternatives +available: + + +**low_latency_conv** +Based on the 'cnn-one-fstride4' topology described in the [Convolutional +Neural Networks for Small-footprint Keyword Spotting +paper](http://www.isca-speech.org/archive/interspeech_2015/papers/i15_1478.pdf). +The accuracy is slightly lower than 'conv' but the number of weight parameters +is about the same, and it only needs 11 million FLOPs to run one prediction, +making it much faster. + +To use this model, you specify `--model_architecture=low_latency_conv` on +the command line. You'll also need to update the training rates and the number +of steps, so the full command will look like: + +``` +python tensorflow/examples/speech_commands/train \ +--model_architecture=low_latency_conv \ +--how_many_training_steps=20000,6000 \ +--learning_rate=0.01,0.001 +``` + +This asks the script to train with a learning rate of 0.01 for 20,000 steps, and +then do a fine-tuning pass of 6,000 steps with a 10x smaller rate. + +**low_latency_svdf** +Based on the topology presented in the [Compressing Deep Neural Networks using a +Rank-Constrained Topology paper](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43813.pdf). +The accuracy is also lower than 'conv' but it only uses about 750 thousand +parameters, and most significantly, it allows for an optimized execution at +test time (i.e. when you will actually use it in your application), resulting +in 750 thousand FLOPs. + +To use this model, you specify `--model_architecture=low_latency_svdf` on +the command line, and update the training rates and the number +of steps, so the full command will look like: + +``` +python tensorflow/examples/speech_commands/train \ +--model_architecture=low_latency_svdf \ +--how_many_training_steps=100000,35000 \ +--learning_rate=0.01,0.005 +``` + +Note that despite requiring a larger number of steps than the previous two +topologies, the reduced number of computations means that training should take +about the same time, and at the end reach an accuracy of around 85%. +You can also further tune the topology fairly easily for computation and +accuracy by changing these parameters in the SVDF layer: + +* rank - The rank of the approximation (higher typically better, but results in + more computation). +* num_units - Similar to other layer types, specifies the number of nodes in + the layer (more nodes better quality, and more computation). + +Regarding runtime, since the layer allows optimizations by caching some of the +internal neural network activations, you need to make sure to use a consistent +stride (e.g. 'clip_stride_ms' flag) both when you freeze the graph, and when +executing the model in streaming mode (e.g. test_streaming_accuracy.cc). + +**Other parameters to customize** +If you want to experiment with customizing models, a good place to start is by +tweaking the spectrogram creation parameters. This has the effect of altering +the size of the input image to the model, and the creation code in +[models.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/models.py) +will adjust the number of computations and weights automatically to fit with +different dimensions. If you make the input smaller, the model will need fewer +computations to process it, so it can be a great way to trade off some accuracy +for improved latency. The `--window_stride_ms` controls how far apart each +frequency analysis sample is from the previous. If you increase this value, then +fewer samples will be taken for a given duration, and the time axis of the input +will shrink. The `--dct_coefficient_count` flag controls how many buckets are +used for the frequency counting, so reducing this will shrink the input in the +other dimension. The `--window_size_ms` argument doesn't affect the size, but +does control how wide the area used to calculate the frequencies is for each +sample. Reducing the duration of the training samples, controlled by +`--clip_duration_ms`, can also help if the sounds you're looking for are short, +since that also reduces the time dimension of the input. You'll need to make +sure that all your training data contains the right audio in the initial portion +of the clip though. + +If you have an entirely different model in mind for your problem, you may find +that you can plug it into +[models.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/speech_commands/models.py) +and have the rest of the script handle all of the preprocessing and training +mechanics. You would add a new clause to `create_model`, looking for the name of +your architecture and then calling a model creation function. This function is +given the size of the spectrogram input, along with other model information, and +is expected to create TensorFlow ops to read that in and produce an output +prediction vector, and a placeholder to control the dropout rate. The rest of +the script will handle integrating this model into a larger graph doing the +input calculations and applying softmax and a loss function to train it. + +One common problem when you're adjusting models and training hyper-parameters is +that not-a-number values can creep in, thanks to numerical precision issues. In +general you can solve these by reducing the magnitude of things like learning +rates and weight initialization functions, but if they're persistent you can +enable the `--check_nans` flag to track down the source of the errors. This will +insert check ops between most regular operations in TensorFlow, and abort the +training process with a useful error message when they're encountered. diff --git a/tensorflow/docs_src/tutorials/sequences/recurrent.md b/tensorflow/docs_src/tutorials/sequences/recurrent.md new file mode 100644 index 0000000000..715cc7856a --- /dev/null +++ b/tensorflow/docs_src/tutorials/sequences/recurrent.md @@ -0,0 +1,232 @@ +# Recurrent Neural Networks + +## Introduction + +See [Understanding LSTM Networks](https://colah.github.io/posts/2015-08-Understanding-LSTMs/){:.external} +for an introduction to recurrent neural networks and LSTMs. + +## Language Modeling + +In this tutorial we will show how to train a recurrent neural network on +a challenging task of language modeling. The goal of the problem is to fit a +probabilistic model which assigns probabilities to sentences. It does so by +predicting next words in a text given a history of previous words. For this +purpose we will use the [Penn Tree Bank](https://catalog.ldc.upenn.edu/ldc99t42) +(PTB) dataset, which is a popular benchmark for measuring the quality of these +models, whilst being small and relatively fast to train. + +Language modeling is key to many interesting problems such as speech +recognition, machine translation, or image captioning. It is also fun -- +take a look [here](https://karpathy.github.io/2015/05/21/rnn-effectiveness/). + +For the purpose of this tutorial, we will reproduce the results from +[Zaremba et al., 2014](https://arxiv.org/abs/1409.2329) +([pdf](https://arxiv.org/pdf/1409.2329.pdf)), which achieves very good quality +on the PTB dataset. + +## Tutorial Files + +This tutorial references the following files from `models/tutorials/rnn/ptb` in the [TensorFlow models repo](https://github.com/tensorflow/models): + +File | Purpose +--- | --- +`ptb_word_lm.py` | The code to train a language model on the PTB dataset. +`reader.py` | The code to read the dataset. + +## Download and Prepare the Data + +The data required for this tutorial is in the `data/` directory of the +[PTB dataset from Tomas Mikolov's webpage](http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz). + +The dataset is already preprocessed and contains overall 10000 different words, +including the end-of-sentence marker and a special symbol (\) for rare +words. In `reader.py`, we convert each word to a unique integer identifier, +in order to make it easy for the neural network to process the data. + +## The Model + +### LSTM + +The core of the model consists of an LSTM cell that processes one word at a +time and computes probabilities of the possible values for the next word in the +sentence. The memory state of the network is initialized with a vector of zeros +and gets updated after reading each word. For computational reasons, we will +process data in mini-batches of size `batch_size`. In this example, it is +important to note that `current_batch_of_words` does not correspond to a +"sentence" of words. Every word in a batch should correspond to a time t. +TensorFlow will automatically sum the gradients of each batch for you. + +For example: + +``` + t=0 t=1 t=2 t=3 t=4 +[The, brown, fox, is, quick] +[The, red, fox, jumped, high] + +words_in_dataset[0] = [The, The] +words_in_dataset[1] = [brown, red] +words_in_dataset[2] = [fox, fox] +words_in_dataset[3] = [is, jumped] +words_in_dataset[4] = [quick, high] +batch_size = 2, time_steps = 5 +``` + +The basic pseudocode is as follows: + +```python +words_in_dataset = tf.placeholder(tf.float32, [time_steps, batch_size, num_features]) +lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size) +# Initial state of the LSTM memory. +hidden_state = tf.zeros([batch_size, lstm.state_size]) +current_state = tf.zeros([batch_size, lstm.state_size]) +state = hidden_state, current_state +probabilities = [] +loss = 0.0 +for current_batch_of_words in words_in_dataset: + # The value of state is updated after processing each batch of words. + output, state = lstm(current_batch_of_words, state) + + # The LSTM output can be used to make next word predictions + logits = tf.matmul(output, softmax_w) + softmax_b + probabilities.append(tf.nn.softmax(logits)) + loss += loss_function(probabilities, target_words) +``` + +### Truncated Backpropagation + +By design, the output of a recurrent neural network (RNN) depends on arbitrarily +distant inputs. Unfortunately, this makes backpropagation computation difficult. +In order to make the learning process tractable, it is common practice to create +an "unrolled" version of the network, which contains a fixed number +(`num_steps`) of LSTM inputs and outputs. The model is then trained on this +finite approximation of the RNN. This can be implemented by feeding inputs of +length `num_steps` at a time and performing a backward pass after each +such input block. + +Here is a simplified block of code for creating a graph which performs +truncated backpropagation: + +```python +# Placeholder for the inputs in a given iteration. +words = tf.placeholder(tf.int32, [batch_size, num_steps]) + +lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size) +# Initial state of the LSTM memory. +initial_state = state = tf.zeros([batch_size, lstm.state_size]) + +for i in range(num_steps): + # The value of state is updated after processing each batch of words. + output, state = lstm(words[:, i], state) + + # The rest of the code. + # ... + +final_state = state +``` + +And this is how to implement an iteration over the whole dataset: + +```python +# A numpy array holding the state of LSTM after each batch of words. +numpy_state = initial_state.eval() +total_loss = 0.0 +for current_batch_of_words in words_in_dataset: + numpy_state, current_loss = session.run([final_state, loss], + # Initialize the LSTM state from the previous iteration. + feed_dict={initial_state: numpy_state, words: current_batch_of_words}) + total_loss += current_loss +``` + +### Inputs + +The word IDs will be embedded into a dense representation (see the +@{$word2vec$Vector Representations Tutorial}) before feeding to +the LSTM. This allows the model to efficiently represent the knowledge about +particular words. It is also easy to write: + +```python +# embedding_matrix is a tensor of shape [vocabulary_size, embedding size] +word_embeddings = tf.nn.embedding_lookup(embedding_matrix, word_ids) +``` + +The embedding matrix will be initialized randomly and the model will learn to +differentiate the meaning of words just by looking at the data. + +### Loss Function + +We want to minimize the average negative log probability of the target words: + +$$ \text{loss} = -\frac{1}{N}\sum_{i=1}^{N} \ln p_{\text{target}_i} $$ + +It is not very difficult to implement but the function +`sequence_loss_by_example` is already available, so we can just use it here. + +The typical measure reported in the papers is average per-word perplexity (often +just called perplexity), which is equal to + +$$e^{-\frac{1}{N}\sum_{i=1}^{N} \ln p_{\text{target}_i}} = e^{\text{loss}} $$ + +and we will monitor its value throughout the training process. + +### Stacking multiple LSTMs + +To give the model more expressive power, we can add multiple layers of LSTMs +to process the data. The output of the first layer will become the input of +the second and so on. + +We have a class called `MultiRNNCell` that makes the implementation seamless: + +```python +def lstm_cell(): + return tf.contrib.rnn.BasicLSTMCell(lstm_size) +stacked_lstm = tf.contrib.rnn.MultiRNNCell( + [lstm_cell() for _ in range(number_of_layers)]) + +initial_state = state = stacked_lstm.zero_state(batch_size, tf.float32) +for i in range(num_steps): + # The value of state is updated after processing each batch of words. + output, state = stacked_lstm(words[:, i], state) + + # The rest of the code. + # ... + +final_state = state +``` + +## Run the Code + +Before running the code, download the PTB dataset, as discussed at the beginning +of this tutorial. Then, extract the PTB dataset underneath your home directory +as follows: + +```bsh +tar xvfz simple-examples.tgz -C $HOME +``` +_(Note: On Windows, you may need to use +[other tools](https://wiki.haskell.org/How_to_unpack_a_tar_file_in_Windows).)_ + +Now, clone the [TensorFlow models repo](https://github.com/tensorflow/models) +from GitHub. Run the following commands: + +```bsh +cd models/tutorials/rnn/ptb +python ptb_word_lm.py --data_path=$HOME/simple-examples/data/ --model=small +``` + +There are 3 supported model configurations in the tutorial code: "small", +"medium" and "large". The difference between them is in size of the LSTMs and +the set of hyperparameters used for training. + +The larger the model, the better results it should get. The `small` model should +be able to reach perplexity below 120 on the test set and the `large` one below +80, though it might take several hours to train. + +## What Next? + +There are several tricks that we haven't mentioned that make the model better, +including: + +* decreasing learning rate schedule, +* dropout between the LSTM layers. + +Study the code and modify it to improve the model even further. diff --git a/tensorflow/docs_src/tutorials/sequences/recurrent_quickdraw.md b/tensorflow/docs_src/tutorials/sequences/recurrent_quickdraw.md new file mode 100644 index 0000000000..37bce5b76d --- /dev/null +++ b/tensorflow/docs_src/tutorials/sequences/recurrent_quickdraw.md @@ -0,0 +1,411 @@ +# Recurrent Neural Networks for Drawing Classification + +[Quick, Draw!]: http://quickdraw.withgoogle.com + +[Quick, Draw!] is a game where a player is challenged to draw a number of +objects and see if a computer can recognize the drawing. + +The recognition in [Quick, Draw!] is performed by a classifier that takes the +user input, given as a sequence of strokes of points in x and y, and recognizes +the object category that the user tried to draw. + +In this tutorial we'll show how to build an RNN-based recognizer for this +problem. The model will use a combination of convolutional layers, LSTM layers, +and a softmax output layer to classify the drawings: + +
![RNN model structure](../../images/quickdraw_model.png)
+ +The figure above shows the structure of the model that we will build in this +tutorial. The input is a drawing that is encoded as a sequence of strokes of +points in x, y, and n, where n indicates whether a the point is the first point +in a new stroke. + +Then, a series of 1-dimensional convolutions is applied. Then LSTM layers are +applied and the sum of the outputs of all LSTM steps is fed into a softmax layer +to make a classification decision among the classes of drawings that we know. + +This tutorial uses the data from actual [Quick, Draw!] games [that is publicly +available](https://quickdraw.withgoogle.com/data). This dataset contains of 50M +drawings in 345 categories. + +## Run the tutorial code + +To try the code for this tutorial: + +1. @{$install$Install TensorFlow} if you haven't already. +1. Download the [tutorial code] +(https://github.com/tensorflow/models/tree/master/tutorials/rnn/quickdraw/train_model.py). +1. [Download the data](#download-the-data) in `TFRecord` format from + [here](http://download.tensorflow.org/data/quickdraw_tutorial_dataset_v1.tar.gz) and unzip it. More details about [how to + obtain the original Quick, Draw! + data](#optional_download_the_full_quick_draw_data) and [how to convert that + to `TFRecord` files](#optional_converting_the_data) is available below. + +1. Execute the tutorial code with the following command to train the RNN-based + model described in this tutorial. Make sure to adjust the paths to point to + the unzipped data from the download in step 3. + +```shell + python train_model.py \ + --training_data=rnn_tutorial_data/training.tfrecord-?????-of-????? \ + --eval_data=rnn_tutorial_data/eval.tfrecord-?????-of-????? \ + --classes_file=rnn_tutorial_data/training.tfrecord.classes +``` + +## Tutorial details + +### Download the data + +We make the data that we use in this tutorial available as `TFRecord` files +containing `TFExamples`. You can download the data from here: + +http://download.tensorflow.org/data/quickdraw_tutorial_dataset_v1.tar.gz + +Alternatively you can download the original data in `ndjson` format from the +Google cloud and convert it to the `TFRecord` files containing `TFExamples` +yourself as described in the next section. + +### Optional: Download the full Quick Draw Data + +The full [Quick, Draw!](https://quickdraw.withgoogle.com) +[dataset](https://quickdraw.withgoogle.com/data) is available on Google Cloud +Storage as [ndjson](http://ndjson.org/) files separated by category. You can +[browse the list of files in Cloud +Console](https://console.cloud.google.com/storage/quickdraw_dataset). + +To download the data we recommend using +[gsutil](https://cloud.google.com/storage/docs/gsutil_install#install) to +download the entire dataset. Note that the original .ndjson files require +downloading ~22GB. + +Then use the following command to check that your gsutil installation works and +that you can access the data bucket: + +```shell +gsutil ls -r "gs://quickdraw_dataset/full/simplified/*" +``` + +which will output a long list of files like the following: + +```shell +gs://quickdraw_dataset/full/simplified/The Eiffel Tower.ndjson +gs://quickdraw_dataset/full/simplified/The Great Wall of China.ndjson +gs://quickdraw_dataset/full/simplified/The Mona Lisa.ndjson +gs://quickdraw_dataset/full/simplified/aircraft carrier.ndjson +... +``` + +Then create a folder and download the dataset there. + +```shell +mkdir rnn_tutorial_data +cd rnn_tutorial_data +gsutil -m cp "gs://quickdraw_dataset/full/simplified/*" . +``` + +This download will take a while and download a bit more than 23GB of data. + +### Optional: Converting the data + +To convert the `ndjson` files to +@{$python/python_io#TFRecords_Format_Details$TFRecord} files containing +[`tf.train.Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto) +protos run the following command. + +```shell + python create_dataset.py --ndjson_path rnn_tutorial_data \ + --output_path rnn_tutorial_data +``` + +This will store the data in 10 shards of +@{$python/python_io#TFRecords_Format_Details$TFRecord} files with 10000 items +per class for the training data and 1000 items per class as eval data. + +This conversion process is described in more detail in the following. + +The original QuickDraw data is formatted as `ndjson` files where each line +contains a JSON object like the following: + +```json +{"word":"cat", + "countrycode":"VE", + "timestamp":"2017-03-02 23:25:10.07453 UTC", + "recognized":true, + "key_id":"5201136883597312", + "drawing":[ + [ + [130,113,99,109,76,64,55,48,48,51,59,86,133,154,170,203,214,217,215,208,186,176,162,157,132], + [72,40,27,79,82,88,100,120,134,152,165,184,189,186,179,152,131,114,100,89,76,0,31,65,70] + ],[ + [76,28,7], + [136,128,128] + ],[ + [76,23,0], + [160,164,175] + ],[ + [87,52,37], + [175,191,204] + ],[ + [174,220,246,251], + [134,132,136,139] + ],[ + [175,255], + [147,168] + ],[ + [171,208,215], + [164,198,210] + ],[ + [130,110,108,111,130,139,139,119], + [129,134,137,144,148,144,136,130] + ],[ + [107,106], + [96,113] + ] + ] +} +``` + +For our purpose of building a classifier we only care about the fields "`word`" +and "`drawing`". While parsing the ndjson files, we process them line by line +using a function that converts the strokes from the `drawing` field into a +tensor of size `[number of points, 3]` containing the differences of consecutive +points. This function also returns the class name as a string. + +```python +def parse_line(ndjson_line): + """Parse an ndjson line and return ink (as np array) and classname.""" + sample = json.loads(ndjson_line) + class_name = sample["word"] + inkarray = sample["drawing"] + stroke_lengths = [len(stroke[0]) for stroke in inkarray] + total_points = sum(stroke_lengths) + np_ink = np.zeros((total_points, 3), dtype=np.float32) + current_t = 0 + for stroke in inkarray: + for i in [0, 1]: + np_ink[current_t:(current_t + len(stroke[0])), i] = stroke[i] + current_t += len(stroke[0]) + np_ink[current_t - 1, 2] = 1 # stroke_end + # Preprocessing. + # 1. Size normalization. + lower = np.min(np_ink[:, 0:2], axis=0) + upper = np.max(np_ink[:, 0:2], axis=0) + scale = upper - lower + scale[scale == 0] = 1 + np_ink[:, 0:2] = (np_ink[:, 0:2] - lower) / scale + # 2. Compute deltas. + np_ink = np_ink[1:, 0:2] - np_ink[0:-1, 0:2] + return np_ink, class_name +``` + +Since we want the data to be shuffled for writing we read from each of the +category files in random order and write to a random shard. + +For the training data we read the first 10000 items for each class and for the +eval data we read the next 1000 items for each class. + +This data is then reformatted into a tensor of shape `[num_training_samples, +max_length, 3]`. Then we determine the bounding box of the original drawing in +screen coordinates and normalize the size such that the drawing has unit height. + +
![Size normalization](../../images/quickdraw_sizenormalization.png)
+ +Finally, we compute the differences between consecutive points and store these +as a `VarLenFeature` in a +[tensorflow.Example](https://www.tensorflow.org/code/tensorflow/core/example/example.proto) +under the key `ink`. In addition we store the `class_index` as a single entry +`FixedLengthFeature` and the `shape` of the `ink` as a `FixedLengthFeature` of +length 2. + +### Defining the model + +To define the model we create a new `Estimator`. If you want to read more about +estimators, we recommend @{$custom_estimators$this tutorial}. + +To build the model, we: + +1. reshape the input back into the original shape - where the mini batch is + padded to the maximal length of its contents. In addition to the ink data we + also have the lengths for each example and the target class. This happens in + the function [`_get_input_tensors`](#-get-input-tensors). + +1. pass the input through to a series of convolution layers in + [`_add_conv_layers`](#-add-conv-layers). + +1. pass the output of the convolutions into a series of bidirectional LSTM + layers in [`_add_rnn_layers`](#-add-rnn-layers). At the end of that, the + outputs for each time step are summed up to have a compact, fixed length + embedding of the input. + +1. classify this embedding using a softmax layer in + [`_add_fc_layers`](#-add-fc-layers). + +In code this looks like: + +```python +inks, lengths, targets = _get_input_tensors(features, targets) +convolved = _add_conv_layers(inks) +final_state = _add_rnn_layers(convolved, lengths) +logits =_add_fc_layers(final_state) +``` + +### _get_input_tensors + +To obtain the input features we first obtain the shape from the features dict +and then create a 1D tensor of size `[batch_size]` containing the lengths of the +input sequences. The ink is stored as a SparseTensor in the features dict which +we convert into a dense tensor and then reshape to be `[batch_size, ?, 3]`. And +finally, if targets were passed in we make sure they are stored as a 1D tensor +of size `[batch_size]` + +In code this looks like this: + +```python +shapes = features["shape"] +lengths = tf.squeeze( + tf.slice(shapes, begin=[0, 0], size=[params["batch_size"], 1])) +inks = tf.reshape( + tf.sparse_tensor_to_dense(features["ink"]), + [params["batch_size"], -1, 3]) +if targets is not None: + targets = tf.squeeze(targets) +``` + +### _add_conv_layers + +The desired number of convolution layers and the lengths of the filters is +configured through the parameters `num_conv` and `conv_len` in the `params` +dict. + +The input is a sequence where each point has dimensionality 3. We are going to +use 1D convolutions where we treat the 3 input features as channels. That means +that the input is a `[batch_size, length, 3]` tensor and the output will be a +`[batch_size, length, number_of_filters]` tensor. + +```python +convolved = inks +for i in range(len(params.num_conv)): + convolved_input = convolved + if params.batch_norm: + convolved_input = tf.layers.batch_normalization( + convolved_input, + training=(mode == tf.estimator.ModeKeys.TRAIN)) + # Add dropout layer if enabled and not first convolution layer. + if i > 0 and params.dropout: + convolved_input = tf.layers.dropout( + convolved_input, + rate=params.dropout, + training=(mode == tf.estimator.ModeKeys.TRAIN)) + convolved = tf.layers.conv1d( + convolved_input, + filters=params.num_conv[i], + kernel_size=params.conv_len[i], + activation=None, + strides=1, + padding="same", + name="conv1d_%d" % i) +return convolved, lengths +``` + +### _add_rnn_layers + +We pass the output from the convolutions into bidirectional LSTM layers for +which we use a helper function from contrib. + +```python +outputs, _, _ = contrib_rnn.stack_bidirectional_dynamic_rnn( + cells_fw=[cell(params.num_nodes) for _ in range(params.num_layers)], + cells_bw=[cell(params.num_nodes) for _ in range(params.num_layers)], + inputs=convolved, + sequence_length=lengths, + dtype=tf.float32, + scope="rnn_classification") +``` + +see the code for more details and how to use `CUDA` accelerated implementations. + +To create a compact, fixed-length embedding, we sum up the output of the LSTMs. +We first zero out the regions of the batch where the sequences have no data. + +```python +mask = tf.tile( + tf.expand_dims(tf.sequence_mask(lengths, tf.shape(outputs)[1]), 2), + [1, 1, tf.shape(outputs)[2]]) +zero_outside = tf.where(mask, outputs, tf.zeros_like(outputs)) +outputs = tf.reduce_sum(zero_outside, axis=1) +``` + +### _add_fc_layers + +The embedding of the input is passed into a fully connected layer which we then +use as a softmax layer. + +```python +tf.layers.dense(final_state, params.num_classes) +``` + +### Loss, predictions, and optimizer + +Finally, we need to add a loss, a training op, and predictions to create the +`ModelFn`: + +```python +cross_entropy = tf.reduce_mean( + tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=targets, logits=logits)) +# Add the optimizer. +train_op = tf.contrib.layers.optimize_loss( + loss=cross_entropy, + global_step=tf.train.get_global_step(), + learning_rate=params.learning_rate, + optimizer="Adam", + # some gradient clipping stabilizes training in the beginning. + clip_gradients=params.gradient_clipping_norm, + summaries=["learning_rate", "loss", "gradients", "gradient_norm"]) +predictions = tf.argmax(logits, axis=1) +return model_fn_lib.ModelFnOps( + mode=mode, + predictions={"logits": logits, + "predictions": predictions}, + loss=cross_entropy, + train_op=train_op, + eval_metric_ops={"accuracy": tf.metrics.accuracy(targets, predictions)}) +``` + +### Training and evaluating the model + +To train and evaluate the model we can rely on the functionalities of the +`Estimator` APIs and easily run training and evaluation with the `Experiment` +APIs: + +```python + estimator = tf.estimator.Estimator( + model_fn=model_fn, + model_dir=output_dir, + config=config, + params=model_params) + # Train the model. + tf.contrib.learn.Experiment( + estimator=estimator, + train_input_fn=get_input_fn( + mode=tf.contrib.learn.ModeKeys.TRAIN, + tfrecord_pattern=FLAGS.training_data, + batch_size=FLAGS.batch_size), + train_steps=FLAGS.steps, + eval_input_fn=get_input_fn( + mode=tf.contrib.learn.ModeKeys.EVAL, + tfrecord_pattern=FLAGS.eval_data, + batch_size=FLAGS.batch_size), + min_eval_frequency=1000) +``` + +Note that this tutorial is just a quick example on a relatively small dataset to +get you familiar with the APIs of recurrent neural networks and estimators. Such +models can be even more powerful if you try them on a large dataset. + +When training the model for 1M steps you can expect to get an accuracy of +approximately of approximately 70% on the top-1 candidate. Note that this +accuracy is sufficient to build the quickdraw game because of the game dynamics +the user will be able to adjust their drawing until it is ready. Also, the game +does not use the top-1 candidate only but accepts a drawing as correct if the +target category shows up with a score better than a fixed threshold. diff --git a/tensorflow/docs_src/tutorials/wide.md b/tensorflow/docs_src/tutorials/wide.md deleted file mode 100644 index 27ce75a30d..0000000000 --- a/tensorflow/docs_src/tutorials/wide.md +++ /dev/null @@ -1,461 +0,0 @@ -# TensorFlow Linear Model Tutorial - -In this tutorial, we will use the tf.estimator API in TensorFlow to solve a -binary classification problem: Given census data about a person such as age, -education, marital status, and occupation (the features), we will try to predict -whether or not the person earns more than 50,000 dollars a year (the target -label). We will train a **logistic regression** model, and given an individual's -information our model will output a number between 0 and 1, which can be -interpreted as the probability that the individual has an annual income of over -50,000 dollars. - -## Setup - -To try the code for this tutorial: - -1. @{$install$Install TensorFlow} if you haven't already. - -2. Download [the tutorial code](https://github.com/tensorflow/models/tree/master/official/wide_deep/). - -3. Execute the data download script we provide to you: - - $ python data_download.py - -4. Execute the tutorial code with the following command to train the linear -model described in this tutorial: - - $ python wide_deep.py --model_type=wide - -Read on to find out how this code builds its linear model. - -## Reading The Census Data - -The dataset we'll be using is the -[Census Income Dataset](https://archive.ics.uci.edu/ml/datasets/Census+Income). -We have provided -[data_download.py](https://github.com/tensorflow/models/tree/master/official/wide_deep/data_download.py) -which downloads the code and performs some additional cleanup. - -Since the task is a binary classification problem, we'll construct a label -column named "label" whose value is 1 if the income is over 50K, and 0 -otherwise. For reference, see `input_fn` in -[wide_deep.py](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py). - -Next, let's take a look at the dataframe and see which columns we can use to -predict the target label. The columns can be grouped into two types—categorical -and continuous columns: - -* A column is called **categorical** if its value can only be one of the - categories in a finite set. For example, the relationship status of a person - (wife, husband, unmarried, etc.) or the education level (high school, - college, etc.) are categorical columns. -* A column is called **continuous** if its value can be any numerical value in - a continuous range. For example, the capital gain of a person (e.g. $14,084) - is a continuous column. - -Here's a list of columns available in the Census Income dataset: - -| Column Name | Type | Description | -| -------------- | ----------- | --------------------------------- | -| age | Continuous | The age of the individual | -| workclass | Categorical | The type of employer the | -: : : individual has (government, : -: : : military, private, etc.). : -| fnlwgt | Continuous | The number of people the census | -: : : takers believe that observation : -: : : represents (sample weight). Final : -: : : weight will not be used. : -| education | Categorical | The highest level of education | -: : : achieved for that individual. : -| education_num | Continuous | The highest level of education in | -: : : numerical form. : -| marital_status | Categorical | Marital status of the individual. | -| occupation | Categorical | The occupation of the individual. | -| relationship | Categorical | Wife, Own-child, Husband, | -: : : Not-in-family, Other-relative, : -: : : Unmarried. : -| race | Categorical | Amer-Indian-Eskimo, Asian-Pac- | -: : : Islander, Black, White, Other. : -| gender | Categorical | Female, Male. | -| capital_gain | Continuous | Capital gains recorded. | -| capital_loss | Continuous | Capital Losses recorded. | -| hours_per_week | Continuous | Hours worked per week. | -| native_country | Categorical | Country of origin of the | -: : : individual. : -| income_bracket | Categorical | ">50K" or "<=50K", meaning | -: : : whether the person makes more : -: : : than $50,000 annually. : - -## Converting Data into Tensors - -When building a tf.estimator model, the input data is specified by means of an -Input Builder function. This builder function will not be called until it is -later passed to tf.estimator.Estimator methods such as `train` and `evaluate`. -The purpose of this function is to construct the input data, which is -represented in the form of @{tf.Tensor}s or @{tf.SparseTensor}s. -In more detail, the input builder function returns the following as a pair: - -1. `features`: A dict from feature column names to `Tensors` or - `SparseTensors`. -2. `labels`: A `Tensor` containing the label column. - -The keys of the `features` will be used to construct columns in the next -section. Because we want to call the `train` and `evaluate` methods with -different data, we define a method that returns an input function based on the -given data. Note that the returned input function will be called while -constructing the TensorFlow graph, not while running the graph. What it is -returning is a representation of the input data as the fundamental unit of -TensorFlow computations, a `Tensor` (or `SparseTensor`). - -Each continuous column in the train or test data will be converted into a -`Tensor`, which in general is a good format to represent dense data. For -categorical data, we must represent the data as a `SparseTensor`. This data -format is good for representing sparse data. Our `input_fn` uses the `tf.data` -API, which makes it easy to apply transformations to our dataset: - -```python -def input_fn(data_file, num_epochs, shuffle, batch_size): - """Generate an input function for the Estimator.""" - assert tf.gfile.Exists(data_file), ( - '%s not found. Please make sure you have either run data_download.py or ' - 'set both arguments --train_data and --test_data.' % data_file) - - def parse_csv(value): - print('Parsing', data_file) - columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS) - features = dict(zip(_CSV_COLUMNS, columns)) - labels = features.pop('income_bracket') - return features, tf.equal(labels, '>50K') - - # Extract lines from input files using the Dataset API. - dataset = tf.data.TextLineDataset(data_file) - - if shuffle: - dataset = dataset.shuffle(buffer_size=_SHUFFLE_BUFFER) - - dataset = dataset.map(parse_csv, num_parallel_calls=5) - - # We call repeat after shuffling, rather than before, to prevent separate - # epochs from blending together. - dataset = dataset.repeat(num_epochs) - dataset = dataset.batch(batch_size) - - iterator = dataset.make_one_shot_iterator() - features, labels = iterator.get_next() - return features, labels -``` - -## Selecting and Engineering Features for the Model - -Selecting and crafting the right set of feature columns is key to learning an -effective model. A **feature column** can be either one of the raw columns in -the original dataframe (let's call them **base feature columns**), or any new -columns created based on some transformations defined over one or multiple base -columns (let's call them **derived feature columns**). Basically, "feature -column" is an abstract concept of any raw or derived variable that can be used -to predict the target label. - -### Base Categorical Feature Columns - -To define a feature column for a categorical feature, we can create a -`CategoricalColumn` using the tf.feature_column API. If you know the set of all -possible feature values of a column and there are only a few of them, you can -use `categorical_column_with_vocabulary_list`. Each key in the list will get -assigned an auto-incremental ID starting from 0. For example, for the -`relationship` column we can assign the feature string "Husband" to an integer -ID of 0 and "Not-in-family" to 1, etc., by doing: - -```python -relationship = tf.feature_column.categorical_column_with_vocabulary_list( - 'relationship', [ - 'Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried', - 'Other-relative']) -``` - -What if we don't know the set of possible values in advance? Not a problem. We -can use `categorical_column_with_hash_bucket` instead: - -```python -occupation = tf.feature_column.categorical_column_with_hash_bucket( - 'occupation', hash_bucket_size=1000) -``` - -What will happen is that each possible value in the feature column `occupation` -will be hashed to an integer ID as we encounter them in training. See an example -illustration below: - -ID | Feature ---- | ------------- -... | -9 | `"Machine-op-inspct"` -... | -103 | `"Farming-fishing"` -... | -375 | `"Protective-serv"` -... | - -No matter which way we choose to define a `SparseColumn`, each feature string -will be mapped into an integer ID by looking up a fixed mapping or by hashing. -Note that hashing collisions are possible, but may not significantly impact the -model quality. Under the hood, the `LinearModel` class is responsible for -managing the mapping and creating `tf.Variable` to store the model parameters -(also known as model weights) for each feature ID. The model parameters will be -learned through the model training process we'll go through later. - -We'll do the similar trick to define the other categorical features: - -```python -education = tf.feature_column.categorical_column_with_vocabulary_list( - 'education', [ - 'Bachelors', 'HS-grad', '11th', 'Masters', '9th', 'Some-college', - 'Assoc-acdm', 'Assoc-voc', '7th-8th', 'Doctorate', 'Prof-school', - '5th-6th', '10th', '1st-4th', 'Preschool', '12th']) - -marital_status = tf.feature_column.categorical_column_with_vocabulary_list( - 'marital_status', [ - 'Married-civ-spouse', 'Divorced', 'Married-spouse-absent', - 'Never-married', 'Separated', 'Married-AF-spouse', 'Widowed']) - -relationship = tf.feature_column.categorical_column_with_vocabulary_list( - 'relationship', [ - 'Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried', - 'Other-relative']) - -workclass = tf.feature_column.categorical_column_with_vocabulary_list( - 'workclass', [ - 'Self-emp-not-inc', 'Private', 'State-gov', 'Federal-gov', - 'Local-gov', '?', 'Self-emp-inc', 'Without-pay', 'Never-worked']) - -# To show an example of hashing: -occupation = tf.feature_column.categorical_column_with_hash_bucket( - 'occupation', hash_bucket_size=1000) -``` - -### Base Continuous Feature Columns - -Similarly, we can define a `NumericColumn` for each continuous feature column -that we want to use in the model: - -```python -age = tf.feature_column.numeric_column('age') -education_num = tf.feature_column.numeric_column('education_num') -capital_gain = tf.feature_column.numeric_column('capital_gain') -capital_loss = tf.feature_column.numeric_column('capital_loss') -hours_per_week = tf.feature_column.numeric_column('hours_per_week') -``` - -### Making Continuous Features Categorical through Bucketization - -Sometimes the relationship between a continuous feature and the label is not -linear. As a hypothetical example, a person's income may grow with age in the -early stage of one's career, then the growth may slow at some point, and finally -the income decreases after retirement. In this scenario, using the raw `age` as -a real-valued feature column might not be a good choice because the model can -only learn one of the three cases: - -1. Income always increases at some rate as age grows (positive correlation), -1. Income always decreases at some rate as age grows (negative correlation), or -1. Income stays the same no matter at what age (no correlation) - -If we want to learn the fine-grained correlation between income and each age -group separately, we can leverage **bucketization**. Bucketization is a process -of dividing the entire range of a continuous feature into a set of consecutive -bins/buckets, and then converting the original numerical feature into a bucket -ID (as a categorical feature) depending on which bucket that value falls into. -So, we can define a `bucketized_column` over `age` as: - -```python -age_buckets = tf.feature_column.bucketized_column( - age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) -``` - -where the `boundaries` is a list of bucket boundaries. In this case, there are -10 boundaries, resulting in 11 age group buckets (from age 17 and below, 18-24, -25-29, ..., to 65 and over). - -### Intersecting Multiple Columns with CrossedColumn - -Using each base feature column separately may not be enough to explain the data. -For example, the correlation between education and the label (earning > 50,000 -dollars) may be different for different occupations. Therefore, if we only learn -a single model weight for `education="Bachelors"` and `education="Masters"`, we -won't be able to capture every single education-occupation combination (e.g. -distinguishing between `education="Bachelors" AND occupation="Exec-managerial"` -and `education="Bachelors" AND occupation="Craft-repair"`). To learn the -differences between different feature combinations, we can add **crossed feature -columns** to the model. - -```python -education_x_occupation = tf.feature_column.crossed_column( - ['education', 'occupation'], hash_bucket_size=1000) -``` - -We can also create a `CrossedColumn` over more than two columns. Each -constituent column can be either a base feature column that is categorical -(`SparseColumn`), a bucketized real-valued feature column (`BucketizedColumn`), -or even another `CrossColumn`. Here's an example: - -```python -age_buckets_x_education_x_occupation = tf.feature_column.crossed_column( - [age_buckets, 'education', 'occupation'], hash_bucket_size=1000) -``` - -## Defining The Logistic Regression Model - -After processing the input data and defining all the feature columns, we're now -ready to put them all together and build a Logistic Regression model. In the -previous section we've seen several types of base and derived feature columns, -including: - -* `CategoricalColumn` -* `NumericColumn` -* `BucketizedColumn` -* `CrossedColumn` - -All of these are subclasses of the abstract `FeatureColumn` class, and can be -added to the `feature_columns` field of a model: - -```python -base_columns = [ - education, marital_status, relationship, workclass, occupation, - age_buckets, -] -crossed_columns = [ - tf.feature_column.crossed_column( - ['education', 'occupation'], hash_bucket_size=1000), - tf.feature_column.crossed_column( - [age_buckets, 'education', 'occupation'], hash_bucket_size=1000), -] - -model_dir = tempfile.mkdtemp() -model = tf.estimator.LinearClassifier( - model_dir=model_dir, feature_columns=base_columns + crossed_columns) -``` - -The model also automatically learns a bias term, which controls the prediction -one would make without observing any features (see the section "How Logistic -Regression Works" for more explanations). The learned model files will be stored -in `model_dir`. - -## Training and Evaluating Our Model - -After adding all the features to the model, now let's look at how to actually -train the model. Training a model is just a single command using the -tf.estimator API: - -```python -model.train(input_fn=lambda: input_fn(train_data, num_epochs, True, batch_size)) -``` - -After the model is trained, we can evaluate how good our model is at predicting -the labels of the holdout data: - -```python -results = model.evaluate(input_fn=lambda: input_fn( - test_data, 1, False, batch_size)) -for key in sorted(results): - print('%s: %s' % (key, results[key])) -``` - -The first line of the final output should be something like -`accuracy: 0.83557522`, which means the accuracy is 83.6%. Feel free to try more -features and transformations and see if you can do even better! - -After the model is evaluated, we can use the model to predict whether an individual has an annual income of over -50,000 dollars given an individual's information input. -```python - pred_iter = model.predict(input_fn=lambda: input_fn(FLAGS.test_data, 1, False, 1)) - for pred in pred_iter: - print(pred['classes']) -``` - -The model prediction output would be like `[b'1']` or `[b'0']` which means whether corresponding individual has an annual income of over 50,000 dollars or not. - -If you'd like to see a working end-to-end example, you can download our -[example code](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py) -and set the `model_type` flag to `wide`. - -## Adding Regularization to Prevent Overfitting - -Regularization is a technique used to avoid **overfitting**. Overfitting happens -when your model does well on the data it is trained on, but worse on test data -that the model has not seen before, such as live traffic. Overfitting generally -occurs when a model is excessively complex, such as having too many parameters -relative to the number of observed training data. Regularization allows for you -to control your model's complexity and makes the model more generalizable to -unseen data. - -In the Linear Model library, you can add L1 and L2 regularizations to the model -as: - -``` -model = tf.estimator.LinearClassifier( - model_dir=model_dir, feature_columns=base_columns + crossed_columns, - optimizer=tf.train.FtrlOptimizer( - learning_rate=0.1, - l1_regularization_strength=1.0, - l2_regularization_strength=1.0)) -``` - -One important difference between L1 and L2 regularization is that L1 -regularization tends to make model weights stay at zero, creating sparser -models, whereas L2 regularization also tries to make the model weights closer to -zero but not necessarily zero. Therefore, if you increase the strength of L1 -regularization, you will have a smaller model size because many of the model -weights will be zero. This is often desirable when the feature space is very -large but sparse, and when there are resource constraints that prevent you from -serving a model that is too large. - -In practice, you should try various combinations of L1, L2 regularization -strengths and find the best parameters that best control overfitting and give -you a desirable model size. - -## How Logistic Regression Works - -Finally, let's take a minute to talk about what the Logistic Regression model -actually looks like in case you're not already familiar with it. We'll denote -the label as \\(Y\\), and the set of observed features as a feature vector -\\(\mathbf{x}=[x_1, x_2, ..., x_d]\\). We define \\(Y=1\\) if an individual -earned > 50,000 dollars and \\(Y=0\\) otherwise. In Logistic Regression, the -probability of the label being positive (\\(Y=1\\)) given the features -\\(\mathbf{x}\\) is given as: - -$$ P(Y=1|\mathbf{x}) = \frac{1}{1+\exp(-(\mathbf{w}^T\mathbf{x}+b))}$$ - -where \\(\mathbf{w}=[w_1, w_2, ..., w_d]\\) are the model weights for the -features \\(\mathbf{x}=[x_1, x_2, ..., x_d]\\). \\(b\\) is a constant that is -often called the **bias** of the model. The equation consists of two parts—A -linear model and a logistic function: - -* **Linear Model**: First, we can see that \\(\mathbf{w}^T\mathbf{x}+b = b + - w_1x_1 + ... +w_dx_d\\) is a linear model where the output is a linear - function of the input features \\(\mathbf{x}\\). The bias \\(b\\) is the - prediction one would make without observing any features. The model weight - \\(w_i\\) reflects how the feature \\(x_i\\) is correlated with the positive - label. If \\(x_i\\) is positively correlated with the positive label, the - weight \\(w_i\\) increases, and the probability \\(P(Y=1|\mathbf{x})\\) will - be closer to 1. On the other hand, if \\(x_i\\) is negatively correlated - with the positive label, then the weight \\(w_i\\) decreases and the - probability \\(P(Y=1|\mathbf{x})\\) will be closer to 0. - -* **Logistic Function**: Second, we can see that there's a logistic function - (also known as the sigmoid function) \\(S(t) = 1/(1+\exp(-t))\\) being - applied to the linear model. The logistic function is used to convert the - output of the linear model \\(\mathbf{w}^T\mathbf{x}+b\\) from any real - number into the range of \\([0, 1]\\), which can be interpreted as a - probability. - -Model training is an optimization problem: The goal is to find a set of model -weights (i.e. model parameters) to minimize a **loss function** defined over the -training data, such as logistic loss for Logistic Regression models. The loss -function measures the discrepancy between the ground-truth label and the model's -prediction. If the prediction is very close to the ground-truth label, the loss -value will be low; if the prediction is very far from the label, then the loss -value would be high. - -## Learn Deeper - -If you're interested in learning more, check out our -@{$wide_and_deep$Wide & Deep Learning Tutorial} where we'll show you how to -combine the strengths of linear models and deep neural networks by jointly -training them using the tf.estimator API. diff --git a/tensorflow/docs_src/tutorials/wide_and_deep.md b/tensorflow/docs_src/tutorials/wide_and_deep.md deleted file mode 100644 index 44677a810b..0000000000 --- a/tensorflow/docs_src/tutorials/wide_and_deep.md +++ /dev/null @@ -1,243 +0,0 @@ -# TensorFlow Wide & Deep Learning Tutorial - -In the previous @{$wide$TensorFlow Linear Model Tutorial}, we trained a logistic -regression model to predict the probability that the individual has an annual -income of over 50,000 dollars using the -[Census Income Dataset](https://archive.ics.uci.edu/ml/datasets/Census+Income). -TensorFlow is great for training deep neural networks too, and you might be -thinking which one you should choose—well, why not both? Would it be possible to -combine the strengths of both in one model? - -In this tutorial, we'll introduce how to use the tf.estimator API to jointly -train a wide linear model and a deep feed-forward neural network. This approach -combines the strengths of memorization and generalization. It's useful for -generic large-scale regression and classification problems with sparse input -features (e.g., categorical features with a large number of possible feature -values). If you're interested in learning more about how Wide & Deep Learning -works, please check out our [research paper](https://arxiv.org/abs/1606.07792). - -![Wide & Deep Spectrum of Models](https://www.tensorflow.org/images/wide_n_deep.svg "Wide & Deep") - -The figure above shows a comparison of a wide model (logistic regression with -sparse features and transformations), a deep model (feed-forward neural network -with an embedding layer and several hidden layers), and a Wide & Deep model -(joint training of both). At a high level, there are only 3 steps to configure a -wide, deep, or Wide & Deep model using the tf.estimator API: - -1. Select features for the wide part: Choose the sparse base columns and - crossed columns you want to use. -1. Select features for the deep part: Choose the continuous columns, the - embedding dimension for each categorical column, and the hidden layer sizes. -1. Put them all together in a Wide & Deep model - (`DNNLinearCombinedClassifier`). - -And that's it! Let's go through a simple example. - -## Setup - -To try the code for this tutorial: - -1. @{$install$Install TensorFlow} if you haven't already. - -2. Download [the tutorial code](https://github.com/tensorflow/models/tree/master/official/wide_deep/). - -3. Execute the data download script we provide to you: - - $ python data_download.py - -4. Execute the tutorial code with the following command to train the wide and -deep model described in this tutorial: - - $ python wide_deep.py - -Read on to find out how this code builds its model. - - -## Define Base Feature Columns - -First, let's define the base categorical and continuous feature columns that -we'll use. These base columns will be the building blocks used by both the wide -part and the deep part of the model. - -```python -import tensorflow as tf - -# Continuous columns -age = tf.feature_column.numeric_column('age') -education_num = tf.feature_column.numeric_column('education_num') -capital_gain = tf.feature_column.numeric_column('capital_gain') -capital_loss = tf.feature_column.numeric_column('capital_loss') -hours_per_week = tf.feature_column.numeric_column('hours_per_week') - -education = tf.feature_column.categorical_column_with_vocabulary_list( - 'education', [ - 'Bachelors', 'HS-grad', '11th', 'Masters', '9th', 'Some-college', - 'Assoc-acdm', 'Assoc-voc', '7th-8th', 'Doctorate', 'Prof-school', - '5th-6th', '10th', '1st-4th', 'Preschool', '12th']) - -marital_status = tf.feature_column.categorical_column_with_vocabulary_list( - 'marital_status', [ - 'Married-civ-spouse', 'Divorced', 'Married-spouse-absent', - 'Never-married', 'Separated', 'Married-AF-spouse', 'Widowed']) - -relationship = tf.feature_column.categorical_column_with_vocabulary_list( - 'relationship', [ - 'Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried', - 'Other-relative']) - -workclass = tf.feature_column.categorical_column_with_vocabulary_list( - 'workclass', [ - 'Self-emp-not-inc', 'Private', 'State-gov', 'Federal-gov', - 'Local-gov', '?', 'Self-emp-inc', 'Without-pay', 'Never-worked']) - -# To show an example of hashing: -occupation = tf.feature_column.categorical_column_with_hash_bucket( - 'occupation', hash_bucket_size=1000) - -# Transformations. -age_buckets = tf.feature_column.bucketized_column( - age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65]) -``` - -## The Wide Model: Linear Model with Crossed Feature Columns - -The wide model is a linear model with a wide set of sparse and crossed feature -columns: - -```python -base_columns = [ - education, marital_status, relationship, workclass, occupation, - age_buckets, -] - -crossed_columns = [ - tf.feature_column.crossed_column( - ['education', 'occupation'], hash_bucket_size=1000), - tf.feature_column.crossed_column( - [age_buckets, 'education', 'occupation'], hash_bucket_size=1000), -] -``` - -You can also see the @{$wide$TensorFlow Linear Model Tutorial} for more details. - -Wide models with crossed feature columns can memorize sparse interactions -between features effectively. That being said, one limitation of crossed feature -columns is that they do not generalize to feature combinations that have not -appeared in the training data. Let's add a deep model with embeddings to fix -that. - -## The Deep Model: Neural Network with Embeddings - -The deep model is a feed-forward neural network, as shown in the previous -figure. Each of the sparse, high-dimensional categorical features are first -converted into a low-dimensional and dense real-valued vector, often referred to -as an embedding vector. These low-dimensional dense embedding vectors are -concatenated with the continuous features, and then fed into the hidden layers -of a neural network in the forward pass. The embedding values are initialized -randomly, and are trained along with all other model parameters to minimize the -training loss. If you're interested in learning more about embeddings, check out -the TensorFlow tutorial on @{$word2vec$Vector Representations of Words} or -[Word embedding](https://en.wikipedia.org/wiki/Word_embedding) on Wikipedia. - -Another way to represent categorical columns to feed into a neural network is -via a one-hot or multi-hot representation. This is often appropriate for -categorical columns with only a few possible values. As an example of a one-hot -representation, for the relationship column, `"Husband"` can be represented as -[1, 0, 0, 0, 0, 0], and `"Not-in-family"` as [0, 1, 0, 0, 0, 0], etc. This is a -fixed representation, whereas embeddings are more flexible and calculated at -training time. - -We'll configure the embeddings for the categorical columns using -`embedding_column`, and concatenate them with the continuous columns. -We also use `indicator_column` to create multi-hot representations of some -categorical columns. - -```python -deep_columns = [ - age, - education_num, - capital_gain, - capital_loss, - hours_per_week, - tf.feature_column.indicator_column(workclass), - tf.feature_column.indicator_column(education), - tf.feature_column.indicator_column(marital_status), - tf.feature_column.indicator_column(relationship), - # To show an example of embedding - tf.feature_column.embedding_column(occupation, dimension=8), -] -``` - -The higher the `dimension` of the embedding is, the more degrees of freedom the -model will have to learn the representations of the features. For simplicity, we -set the dimension to 8 for all feature columns here. Empirically, a more -informed decision for the number of dimensions is to start with a value on the -order of \\(\log_2(n)\\) or \\(k\sqrt[4]n\\), where \\(n\\) is the number of -unique features in a feature column and \\(k\\) is a small constant (usually -smaller than 10). - -Through dense embeddings, deep models can generalize better and make predictions -on feature pairs that were previously unseen in the training data. However, it -is difficult to learn effective low-dimensional representations for feature -columns when the underlying interaction matrix between two feature columns is -sparse and high-rank. In such cases, the interaction between most feature pairs -should be zero except a few, but dense embeddings will lead to nonzero -predictions for all feature pairs, and thus can over-generalize. On the other -hand, linear models with crossed features can memorize these “exception rules” -effectively with fewer model parameters. - -Now, let's see how to jointly train wide and deep models and allow them to -complement each other’s strengths and weaknesses. - -## Combining Wide and Deep Models into One - -The wide models and deep models are combined by summing up their final output -log odds as the prediction, then feeding the prediction to a logistic loss -function. All the graph definition and variable allocations have already been -handled for you under the hood, so you simply need to create a -`DNNLinearCombinedClassifier`: - -```python -model = tf.estimator.DNNLinearCombinedClassifier( - model_dir='/tmp/census_model', - linear_feature_columns=base_columns + crossed_columns, - dnn_feature_columns=deep_columns, - dnn_hidden_units=[100, 50]) -``` - -## Training and Evaluating The Model - -Before we train the model, let's read in the Census dataset as we did in the -@{$wide$TensorFlow Linear Model tutorial}. See `data_download.py` as well as -`input_fn` within -[`wide_deep.py`](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py). - -After reading in the data, you can train and evaluate the model: - -```python -# Train and evaluate the model every `FLAGS.epochs_per_eval` epochs. -for n in range(FLAGS.train_epochs // FLAGS.epochs_per_eval): - model.train(input_fn=lambda: input_fn( - FLAGS.train_data, FLAGS.epochs_per_eval, True, FLAGS.batch_size)) - - results = model.evaluate(input_fn=lambda: input_fn( - FLAGS.test_data, 1, False, FLAGS.batch_size)) - - # Display evaluation metrics - print('Results at epoch', (n + 1) * FLAGS.epochs_per_eval) - print('-' * 30) - - for key in sorted(results): - print('%s: %s' % (key, results[key])) -``` - -The final output accuracy should be somewhere around 85.5%. If you'd like to -see a working end-to-end example, you can download our -[example code](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py). - -Note that this tutorial is just a quick example on a small dataset to get you -familiar with the API. Wide & Deep Learning will be even more powerful if you -try it on a large dataset with many sparse feature columns that have a large -number of possible feature values. Again, feel free to take a look at our -[research paper](https://arxiv.org/abs/1606.07792) for more ideas about how to -apply Wide & Deep Learning in real-world large-scale machine learning problems. diff --git a/tensorflow/docs_src/tutorials/word2vec.md b/tensorflow/docs_src/tutorials/word2vec.md deleted file mode 100644 index 3fe7352bd2..0000000000 --- a/tensorflow/docs_src/tutorials/word2vec.md +++ /dev/null @@ -1,405 +0,0 @@ -# Vector Representations of Words - -In this tutorial we look at the word2vec model by -[Mikolov et al.](https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf) -This model is used for learning vector representations of words, called "word -embeddings". - -## Highlights - -This tutorial is meant to highlight the interesting, substantive parts of -building a word2vec model in TensorFlow. - -* We start by giving the motivation for why we would want to -represent words as vectors. -* We look at the intuition behind the model and how it is trained -(with a splash of math for good measure). -* We also show a simple implementation of the model in TensorFlow. -* Finally, we look at ways to make the naive version scale better. - -We walk through the code later during the tutorial, but if you'd prefer to dive -straight in, feel free to look at the minimalistic implementation in -[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py) -This basic example contains the code needed to download some data, train on it a -bit and visualize the result. Once you get comfortable with reading and running -the basic version, you can graduate to -[models/tutorials/embedding/word2vec.py](https://www.tensorflow.org/code/tensorflow_models/tutorials/embedding/word2vec.py) -which is a more serious implementation that showcases some more advanced -TensorFlow principles about how to efficiently use threads to move data into a -text model, how to checkpoint during training, etc. - -But first, let's look at why we would want to learn word embeddings in the first -place. Feel free to skip this section if you're an Embedding Pro and you'd just -like to get your hands dirty with the details. - -## Motivation: Why Learn Word Embeddings? - -Image and audio processing systems work with rich, high-dimensional datasets -encoded as vectors of the individual raw pixel-intensities for image data, or -e.g. power spectral density coefficients for audio data. For tasks like object -or speech recognition we know that all the information required to successfully -perform the task is encoded in the data (because humans can perform these tasks -from the raw data). However, natural language processing systems traditionally -treat words as discrete atomic symbols, and therefore 'cat' may be represented -as `Id537` and 'dog' as `Id143`. These encodings are arbitrary, and provide -no useful information to the system regarding the relationships that may exist -between the individual symbols. This means that the model can leverage -very little of what it has learned about 'cats' when it is processing data about -'dogs' (such that they are both animals, four-legged, pets, etc.). Representing -words as unique, discrete ids furthermore leads to data sparsity, and usually -means that we may need more data in order to successfully train statistical -models. Using vector representations can overcome some of these obstacles. - -
- -
- -[Vector space models](https://en.wikipedia.org/wiki/Vector_space_model) (VSMs) -represent (embed) words in a continuous vector space where semantically -similar words are mapped to nearby points ('are embedded nearby each other'). -VSMs have a long, rich history in NLP, but all methods depend in some way or -another on the -[Distributional Hypothesis](https://en.wikipedia.org/wiki/Distributional_semantics#Distributional_Hypothesis), -which states that words that appear in the same contexts share -semantic meaning. The different approaches that leverage this principle can be -divided into two categories: *count-based methods* (e.g. -[Latent Semantic Analysis](https://en.wikipedia.org/wiki/Latent_semantic_analysis)), -and *predictive methods* (e.g. -[neural probabilistic language models](http://www.scholarpedia.org/article/Neural_net_language_models)). - -This distinction is elaborated in much more detail by -[Baroni et al.](http://clic.cimec.unitn.it/marco/publications/acl2014/baroni-etal-countpredict-acl2014.pdf), -but in a nutshell: Count-based methods compute the statistics of -how often some word co-occurs with its neighbor words in a large text corpus, -and then map these count-statistics down to a small, dense vector for each word. -Predictive models directly try to predict a word from its neighbors in terms of -learned small, dense *embedding vectors* (considered parameters of the -model). - -Word2vec is a particularly computationally-efficient predictive model for -learning word embeddings from raw text. It comes in two flavors, the Continuous -Bag-of-Words model (CBOW) and the Skip-Gram model (Section 3.1 and 3.2 in [Mikolov et al.](https://arxiv.org/pdf/1301.3781.pdf)). Algorithmically, these -models are similar, except that CBOW predicts target words (e.g. 'mat') from -source context words ('the cat sits on the'), while the skip-gram does the -inverse and predicts source context-words from the target words. This inversion -might seem like an arbitrary choice, but statistically it has the effect that -CBOW smoothes over a lot of the distributional information (by treating an -entire context as one observation). For the most part, this turns out to be a -useful thing for smaller datasets. However, skip-gram treats each context-target -pair as a new observation, and this tends to do better when we have larger -datasets. We will focus on the skip-gram model in the rest of this tutorial. - - -## Scaling up with Noise-Contrastive Training - -Neural probabilistic language models are traditionally trained using the -[maximum likelihood](https://en.wikipedia.org/wiki/Maximum_likelihood) (ML) -principle to maximize the probability of the next word \\(w_t\\) (for "target") -given the previous words \\(h\\) (for "history") in terms of a -[*softmax* function](https://en.wikipedia.org/wiki/Softmax_function), - -$$ -\begin{align} -P(w_t | h) &= \text{softmax}(\text{score}(w_t, h)) \\ - &= \frac{\exp \{ \text{score}(w_t, h) \} } - {\sum_\text{Word w' in Vocab} \exp \{ \text{score}(w', h) \} } -\end{align} -$$ - -where \\(\text{score}(w_t, h)\\) computes the compatibility of word \\(w_t\\) -with the context \\(h\\) (a dot product is commonly used). We train this model -by maximizing its [log-likelihood](https://en.wikipedia.org/wiki/Likelihood_function) -on the training set, i.e. by maximizing - -$$ -\begin{align} - J_\text{ML} &= \log P(w_t | h) \\ - &= \text{score}(w_t, h) - - \log \left( \sum_\text{Word w' in Vocab} \exp \{ \text{score}(w', h) \} \right). -\end{align} -$$ - -This yields a properly normalized probabilistic model for language modeling. -However this is very expensive, because we need to compute and normalize each -probability using the score for all other \\(V\\) words \\(w'\\) in the current -context \\(h\\), *at every training step*. - -
- -
- -On the other hand, for feature learning in word2vec we do not need a full -probabilistic model. The CBOW and skip-gram models are instead trained using a -binary classification objective ([logistic regression](https://en.wikipedia.org/wiki/Logistic_regression)) -to discriminate the real target words \\(w_t\\) from \\(k\\) imaginary (noise) words \\(\tilde w\\), in the -same context. We illustrate this below for a CBOW model. For skip-gram the -direction is simply inverted. - -
- -
- -Mathematically, the objective (for each example) is to maximize - -$$J_\text{NEG} = \log Q_\theta(D=1 |w_t, h) + - k \mathop{\mathbb{E}}_{\tilde w \sim P_\text{noise}} - \left[ \log Q_\theta(D = 0 |\tilde w, h) \right]$$ - -where \\(Q_\theta(D=1 | w, h)\\) is the binary logistic regression probability -under the model of seeing the word \\(w\\) in the context \\(h\\) in the dataset -\\(D\\), calculated in terms of the learned embedding vectors \\(\theta\\). In -practice we approximate the expectation by drawing \\(k\\) contrastive words -from the noise distribution (i.e. we compute a -[Monte Carlo average](https://en.wikipedia.org/wiki/Monte_Carlo_integration)). - -This objective is maximized when the model assigns high probabilities -to the real words, and low probabilities to noise words. Technically, this is -called -[Negative Sampling](https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf), -and there is good mathematical motivation for using this loss function: -The updates it proposes approximate the updates of the softmax function in the -limit. But computationally it is especially appealing because computing the -loss function now scales only with the number of *noise words* that we -select (\\(k\\)), and not *all words* in the vocabulary (\\(V\\)). This makes it -much faster to train. We will actually make use of the very similar -[noise-contrastive estimation (NCE)](https://papers.nips.cc/paper/5165-learning-word-embeddings-efficiently-with-noise-contrastive-estimation.pdf) -loss, for which TensorFlow has a handy helper function `tf.nn.nce_loss()`. - -Let's get an intuitive feel for how this would work in practice! - -## The Skip-gram Model - -As an example, let's consider the dataset - -`the quick brown fox jumped over the lazy dog` - -We first form a dataset of words and the contexts in which they appear. We -could define 'context' in any way that makes sense, and in fact people have -looked at syntactic contexts (i.e. the syntactic dependents of the current -target word, see e.g. -[Levy et al.](https://levyomer.files.wordpress.com/2014/04/dependency-based-word-embeddings-acl-2014.pdf)), -words-to-the-left of the target, words-to-the-right of the target, etc. For now, -let's stick to the vanilla definition and define 'context' as the window -of words to the left and to the right of a target word. Using a window -size of 1, we then have the dataset - -`([the, brown], quick), ([quick, fox], brown), ([brown, jumped], fox), ...` - -of `(context, target)` pairs. Recall that skip-gram inverts contexts and -targets, and tries to predict each context word from its target word, so the -task becomes to predict 'the' and 'brown' from 'quick', 'quick' and 'fox' from -'brown', etc. Therefore our dataset becomes - -`(quick, the), (quick, brown), (brown, quick), (brown, fox), ...` - -of `(input, output)` pairs. The objective function is defined over the entire -dataset, but we typically optimize this with -[stochastic gradient descent](https://en.wikipedia.org/wiki/Stochastic_gradient_descent) -(SGD) using one example at a time (or a 'minibatch' of `batch_size` examples, -where typically `16 <= batch_size <= 512`). So let's look at one step of -this process. - -Let's imagine at training step \\(t\\) we observe the first training case above, -where the goal is to predict `the` from `quick`. We select `num_noise` number -of noisy (contrastive) examples by drawing from some noise distribution, -typically the unigram distribution, \\(P(w)\\). For simplicity let's say -`num_noise=1` and we select `sheep` as a noisy example. Next we compute the -loss for this pair of observed and noisy examples, i.e. the objective at time -step \\(t\\) becomes - -$$J^{(t)}_\text{NEG} = \log Q_\theta(D=1 | \text{the, quick}) + - \log(Q_\theta(D=0 | \text{sheep, quick}))$$ - -The goal is to make an update to the embedding parameters \\(\theta\\) to improve -(in this case, maximize) this objective function. We do this by deriving the -gradient of the loss with respect to the embedding parameters \\(\theta\\), i.e. -\\(\frac{\partial}{\partial \theta} J_\text{NEG}\\) (luckily TensorFlow provides -easy helper functions for doing this!). We then perform an update to the -embeddings by taking a small step in the direction of the gradient. When this -process is repeated over the entire training set, this has the effect of -'moving' the embedding vectors around for each word until the model is -successful at discriminating real words from noise words. - -We can visualize the learned vectors by projecting them down to 2 dimensions -using for instance something like the -[t-SNE dimensionality reduction technique](https://lvdmaaten.github.io/tsne/). -When we inspect these visualizations it becomes apparent that the vectors -capture some general, and in fact quite useful, semantic information about -words and their relationships to one another. It was very interesting when we -first discovered that certain directions in the induced vector space specialize -towards certain semantic relationships, e.g. *male-female*, *verb tense* and -even *country-capital* relationships between words, as illustrated in the figure -below (see also for example -[Mikolov et al., 2013](https://www.aclweb.org/anthology/N13-1090)). - -
- -
- -This explains why these vectors are also useful as features for many canonical -NLP prediction tasks, such as part-of-speech tagging or named entity recognition -(see for example the original work by -[Collobert et al., 2011](https://arxiv.org/abs/1103.0398) -([pdf](https://arxiv.org/pdf/1103.0398.pdf)), or follow-up work by -[Turian et al., 2010](https://www.aclweb.org/anthology/P10-1040)). - -But for now, let's just use them to draw pretty pictures! - -## Building the Graph - -This is all about embeddings, so let's define our embedding matrix. -This is just a big random matrix to start. We'll initialize the values to be -uniform in the unit cube. - -```python -embeddings = tf.Variable( - tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) -``` - -The noise-contrastive estimation loss is defined in terms of a logistic regression -model. For this, we need to define the weights and biases for each word in the -vocabulary (also called the `output weights` as opposed to the `input -embeddings`). So let's define that. - -```python -nce_weights = tf.Variable( - tf.truncated_normal([vocabulary_size, embedding_size], - stddev=1.0 / math.sqrt(embedding_size))) -nce_biases = tf.Variable(tf.zeros([vocabulary_size])) -``` - -Now that we have the parameters in place, we can define our skip-gram model -graph. For simplicity, let's suppose we've already integerized our text corpus -with a vocabulary so that each word is represented as an integer (see -[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py) -for the details). The skip-gram model takes two inputs. One is a batch full of -integers representing the source context words, the other is for the target -words. Let's create placeholder nodes for these inputs, so that we can feed in -data later. - -```python -# Placeholders for inputs -train_inputs = tf.placeholder(tf.int32, shape=[batch_size]) -train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) -``` - -Now what we need to do is look up the vector for each of the source words in -the batch. TensorFlow has handy helpers that make this easy. - -```python -embed = tf.nn.embedding_lookup(embeddings, train_inputs) -``` - -Ok, now that we have the embeddings for each word, we'd like to try to predict -the target word using the noise-contrastive training objective. - -```python -# Compute the NCE loss, using a sample of the negative labels each time. -loss = tf.reduce_mean( - tf.nn.nce_loss(weights=nce_weights, - biases=nce_biases, - labels=train_labels, - inputs=embed, - num_sampled=num_sampled, - num_classes=vocabulary_size)) -``` - -Now that we have a loss node, we need to add the nodes required to compute -gradients and update the parameters, etc. For this we will use stochastic -gradient descent, and TensorFlow has handy helpers to make this easy as well. - -```python -# We use the SGD optimizer. -optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0).minimize(loss) -``` - -## Training the Model - -Training the model is then as simple as using a `feed_dict` to push data into -the placeholders and calling -@{tf.Session.run} with this new data -in a loop. - -```python -for inputs, labels in generate_batch(...): - feed_dict = {train_inputs: inputs, train_labels: labels} - _, cur_loss = session.run([optimizer, loss], feed_dict=feed_dict) -``` - -See the full example code in -[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py). - -## Visualizing the Learned Embeddings - -After training has finished we can visualize the learned embeddings using -t-SNE. - -
- -
- -Et voila! As expected, words that are similar end up clustering nearby each -other. For a more heavyweight implementation of word2vec that showcases more of -the advanced features of TensorFlow, see the implementation in -[models/tutorials/embedding/word2vec.py](https://www.tensorflow.org/code/tensorflow_models/tutorials/embedding/word2vec.py). - -## Evaluating Embeddings: Analogical Reasoning - -Embeddings are useful for a wide variety of prediction tasks in NLP. Short of -training a full-blown part-of-speech model or named-entity model, one simple way -to evaluate embeddings is to directly use them to predict syntactic and semantic -relationships like `king is to queen as father is to ?`. This is called -*analogical reasoning* and the task was introduced by -[Mikolov and colleagues -](https://www.aclweb.org/anthology/N13-1090). -Download the dataset for this task from -[download.tensorflow.org](http://download.tensorflow.org/data/questions-words.txt). - -To see how we do this evaluation, have a look at the `build_eval_graph()` and -`eval()` functions in -[models/tutorials/embedding/word2vec.py](https://www.tensorflow.org/code/tensorflow_models/tutorials/embedding/word2vec.py). - -The choice of hyperparameters can strongly influence the accuracy on this task. -To achieve state-of-the-art performance on this task requires training over a -very large dataset, carefully tuning the hyperparameters and making use of -tricks like subsampling the data, which is out of the scope of this tutorial. - - -## Optimizing the Implementation - -Our vanilla implementation showcases the flexibility of TensorFlow. For -example, changing the training objective is as simple as swapping out the call -to `tf.nn.nce_loss()` for an off-the-shelf alternative such as -`tf.nn.sampled_softmax_loss()`. If you have a new idea for a loss function, you -can manually write an expression for the new objective in TensorFlow and let -the optimizer compute its derivatives. This flexibility is invaluable in the -exploratory phase of machine learning model development, where we are trying -out several different ideas and iterating quickly. - -Once you have a model structure you're satisfied with, it may be worth -optimizing your implementation to run more efficiently (and cover more data in -less time). For example, the naive code we used in this tutorial would suffer -compromised speed because we use Python for reading and feeding data items -- -each of which require very little work on the TensorFlow back-end. If you find -your model is seriously bottlenecked on input data, you may want to implement a -custom data reader for your problem, as described in -@{$new_data_formats$New Data Formats}. For the case of Skip-Gram -modeling, we've actually already done this for you as an example in -[models/tutorials/embedding/word2vec.py](https://www.tensorflow.org/code/tensorflow_models/tutorials/embedding/word2vec.py). - -If your model is no longer I/O bound but you want still more performance, you -can take things further by writing your own TensorFlow Ops, as described in -@{$adding_an_op$Adding a New Op}. Again we've provided an -example of this for the Skip-Gram case -[models/tutorials/embedding/word2vec_optimized.py](https://www.tensorflow.org/code/tensorflow_models/tutorials/embedding/word2vec_optimized.py). -Feel free to benchmark these against each other to measure performance -improvements at each stage. - -## Conclusion - -In this tutorial we covered the word2vec model, a computationally efficient -model for learning word embeddings. We motivated why embeddings are useful, -discussed efficient training techniques and showed how to implement all of this -in TensorFlow. Overall, we hope that this has show-cased how TensorFlow affords -you the flexibility you need for early experimentation, and the control you -later need for bespoke optimized implementation. -- cgit v1.2.3 From b46fde9a42f97d66535a2dde60642ce22473f80c Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Tue, 3 Jul 2018 16:56:01 -0700 Subject: fix rc2 --- tensorflow/docs_src/install/install_c.md | 2 +- tensorflow/docs_src/install/install_go.md | 2 +- tensorflow/docs_src/install/install_java.md | 22 +++++++++++----------- tensorflow/docs_src/install/install_linux.md | 18 +++++++++--------- tensorflow/docs_src/install/install_mac.md | 10 +++++----- tensorflow/docs_src/install/install_sources.md | 4 ++-- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md index 2901848745..9aebf2bfa4 100644 --- a/tensorflow/docs_src/install/install_c.md +++ b/tensorflow/docs_src/install/install_c.md @@ -38,7 +38,7 @@ enable TensorFlow for C: OS="linux" # Change to "darwin" for macOS TARGET_DIRECTORY="/usr/local" curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.9.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.9.0-rc2.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index 2c126df5aa..1907355341 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go: TF_TYPE="cpu" # Change to "gpu" for GPU support TARGET_DIRECTORY='/usr/local' curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.9.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.9.0-rc2.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index 692dfc9cef..1fbdcc2b47 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs: org.tensorflow tensorflow - 1.9.0-rc0 + 1.9.0-rc2 ``` @@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow: org.tensorflow tensorflow - 1.9.0-rc0 + 1.9.0-rc2 @@ -124,12 +124,12 @@ instead: org.tensorflow libtensorflow - 1.9.0-rc0 + 1.9.0-rc2 org.tensorflow libtensorflow_jni_gpu - 1.9.0-rc0 + 1.9.0-rc2 ``` @@ -148,7 +148,7 @@ refer to the simpler instructions above instead. Take the following steps to install TensorFlow for Java on Linux or macOS: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc0.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc2.jar), which is the TensorFlow Java Archive (JAR). 2. Decide whether you will run TensorFlow for Java on CPU(s) only or with @@ -167,7 +167,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: OS=$(uname -s | tr '[:upper:]' '[:lower:]') mkdir -p ./jni curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.9.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.9.0-rc2.tar.gz" | tar -xz -C ./jni ### Install on Windows @@ -175,10 +175,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: Take the following steps to install TensorFlow for Java on Windows: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc0.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc2.jar), which is the TensorFlow Java Archive (JAR). 2. Download the following Java Native Interface (JNI) file appropriate for - [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.9.0-rc0.zip). + [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.9.0-rc2.zip). 3. Extract this .zip file. __Note__: The native library (`tensorflow_jni.dll`) requires `msvcp140.dll` at runtime, which is included in the [Visual C++ 2015 Redistributable](https://www.microsoft.com/en-us/download/details.aspx?id=48145) package. @@ -227,7 +227,7 @@ must be part of your `classpath`. For example, you can include the downloaded `.jar` in your `classpath` by using the `-cp` compilation flag as follows: -
javac -cp libtensorflow-1.9.0-rc0.jar HelloTF.java
+
javac -cp libtensorflow-1.9.0-rc2.jar HelloTF.java
### Running @@ -241,11 +241,11 @@ two files are available to the JVM: For example, the following command line executes the `HelloTF` program on Linux and macOS X: -
java -cp libtensorflow-1.9.0-rc0.jar:. -Djava.library.path=./jni HelloTF
+
java -cp libtensorflow-1.9.0-rc2.jar:. -Djava.library.path=./jni HelloTF
And the following command line executes the `HelloTF` program on Windows: -
java -cp libtensorflow-1.9.0-rc0.jar;. -Djava.library.path=jni HelloTF
+
java -cp libtensorflow-1.9.0-rc2.jar;. -Djava.library.path=jni HelloTF
If the program prints Hello from version, you've successfully installed TensorFlow for Java and are ready to use the API. If the program diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index f21c073a1b..8efa166073 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -436,7 +436,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: 
      (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp34-cp34m-linux_x86_64.whl ## Validate your installation @@ -676,14 +676,14 @@ This section documents the relevant values for Linux installations. CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp27-none-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp27-none-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -695,14 +695,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp34-cp34m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp34-cp34m-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -714,14 +714,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp35-cp35m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp35-cp35m-linux_x86_64.whl
@@ -733,14 +733,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp36-cp36m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp36-cp36m-linux_x86_64.whl
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index c6f0c17924..5b593d1ca9 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv: TensorFlow in the active Virtualenv is as follows: 
 $ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl If you encounter installation problems, see [Common Installation Problems](#common-installation-problems). @@ -242,7 +242,7 @@ take the following steps: issue the following command: 
 $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl If the preceding command fails, see [installation problems](#common-installation-problems). @@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: TensorFlow for Python 2.7: 
 (targetDirectory)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py2-none-any.whl @@ -517,7 +517,7 @@ The value you specify depends on your Python version. 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py2-none-any.whl
@@ -525,5 +525,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py2-none-a 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc0-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index fc1f6d05bd..3801fc0f83 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -338,10 +338,10 @@ Invoke `pip install` to install that pip package. The filename of the `.whl` file depends on your platform. For example, the following command will install the pip package -for TensorFlow 1.9.0rc0 on Linux: +for TensorFlow 1.9.0rc2 on Linux: 
-$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.9.0rc0-py2-none-any.whl
+$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.9.0rc2-py2-none-any.whl
## Validate your installation -- cgit v1.2.3 From 3db3f1f83a562587cc207106258751d089c48ae4 Mon Sep 17 00:00:00 2001 From: Loo Rong Jie Date: Wed, 4 Jul 2018 13:18:02 +0800 Subject: [MSVC] Disable C++ exceptions --- tensorflow/contrib/cmake/CMakeLists.txt | 29 +++++++++++++++++++++-------- tensorflow/tensorflow.bzl | 9 ++++++--- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt index a0a5b0e00c..693393c1e7 100644 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ b/tensorflow/contrib/cmake/CMakeLists.txt @@ -145,26 +145,39 @@ if(WIN32) # temporary fix for #18241 add_definitions(-DEIGEN_DEFAULT_DENSE_INDEX_TYPE=std::int64_t) endif() - add_definitions(-DNOMINMAX -D_WIN32_WINNT=0x0A00 -DLANG_CXX11) - add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN32_LEAN_AND_MEAN -DNOGDI -DPLATFORM_WINDOWS) + add_definitions(-DNOMINMAX -D_WIN32_WINNT=0x0A00) + add_definitions(-DWIN32_LEAN_AND_MEAN -DNOGDI -DPLATFORM_WINDOWS) add_definitions(-DTENSORFLOW_USE_EIGEN_THREADPOOL -DEIGEN_HAS_C99_MATH) add_definitions(-DTF_COMPILE_LIBRARY) - add_definitions(/bigobj /nologo /EHsc /GF /MP /Gm-) + add_compile_options(/bigobj /GF /MP /Gm-) # Suppress warnings to reduce build log size. - add_definitions(/wd4267 /wd4244 /wd4800 /wd4503 /wd4554 /wd4996 /wd4348 /wd4018) - add_definitions(/wd4099 /wd4146 /wd4267 /wd4305 /wd4307) - add_definitions(/wd4715 /wd4722 /wd4723 /wd4838 /wd4309 /wd4334) - add_definitions(/wd4003 /wd4244 /wd4267 /wd4503 /wd4506 /wd4800 /wd4996) + add_compile_options(/wd4267 /wd4244 /wd4800 /wd4503 /wd4554 /wd4996 /wd4348 /wd4018) + add_compile_options(/wd4099 /wd4146 /wd4267 /wd4305 /wd4307) + add_compile_options(/wd4715 /wd4722 /wd4723 /wd4838 /wd4309 /wd4334) + add_compile_options(/wd4003 /wd4244 /wd4267 /wd4503 /wd4506 /wd4800 /wd4996) # Suppress linker warnings. set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /ignore:4049 /ignore:4197 /ignore:4217 /ignore:4221") set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} /ignore:4049 /ignore:4197 /ignore:4217 /ignore:4221") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /ignore:4049 /ignore:4197 /ignore:4217 /ignore:4221") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP") set(CMAKE_CXX_FLAGS_DEBUG "/D_DEBUG /MDd /Ob2") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /D_ITERATOR_DEBUG_LEVEL=0") set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} /D_ITERATOR_DEBUG_LEVEL=0") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /D_ITERATOR_DEBUG_LEVEL=0") + set(compiler_flags + CMAKE_CXX_FLAGS + CMAKE_CXX_FLAGS_DEBUG + CMAKE_CXX_FLAGS_RELEASE + CMAKE_C_FLAGS + CMAKE_C_FLAGS_DEBUG + CMAKE_C_FLAGS_RELEASE + ) + # No exception + foreach(flag ${compiler_flags}) + string(REPLACE "/EHsc" "/EHs-c-" ${flag} "${${flag}}") + endforeach() + add_definitions(/D_HAS_EXCEPTIONS=0) + # Try to avoid flaky failures due to failed generation of generate.stamp files. set(CMAKE_SUPPRESS_REGENERATION ON) endif() diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index e4632c4811..c3bd854940 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -180,9 +180,12 @@ def get_win_copts(is_external=False): "/DEIGEN_AVOID_STL_ARRAY", "/Iexternal/gemmlowp", "/wd4018", # -Wno-sign-compare - "/U_HAS_EXCEPTIONS", - "/D_HAS_EXCEPTIONS=1", - "/EHsc", # -fno-exceptions + # Bazel's CROSSTOOL currently pass /EHsc to enable exception by + # default. We can't pass /EHs-c- to disable exception, otherwise + # we will get a waterfall of flag conflict warnings. Wait for + # Bazel to fix this. + # "/D_HAS_EXCEPTIONS=0", + # "/EHs-c-", "/DNOGDI", ] if is_external: -- cgit v1.2.3 From 86ad3363e18f7df3f86f30150f7b13d389c459e5 Mon Sep 17 00:00:00 2001 From: Loo Rong Jie Date: Wed, 4 Jul 2018 19:32:20 +0800 Subject: Suppress /wd4577 'noexcept with no exception handling mode' warning --- tensorflow/contrib/cmake/CMakeLists.txt | 2 ++ tensorflow/tensorflow.bzl | 1 + 2 files changed, 3 insertions(+) diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt index 693393c1e7..708618dcb0 100644 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ b/tensorflow/contrib/cmake/CMakeLists.txt @@ -177,6 +177,8 @@ if(WIN32) string(REPLACE "/EHsc" "/EHs-c-" ${flag} "${${flag}}") endforeach() add_definitions(/D_HAS_EXCEPTIONS=0) + # Suppress 'noexcept used with no exception handling mode specified' warning + add_compile_options(/wd4577) # Try to avoid flaky failures due to failed generation of generate.stamp files. set(CMAKE_SUPPRESS_REGENERATION ON) diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index c3bd854940..46d554a19d 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -186,6 +186,7 @@ def get_win_copts(is_external=False): # Bazel to fix this. # "/D_HAS_EXCEPTIONS=0", # "/EHs-c-", + "/wd4577", "/DNOGDI", ] if is_external: -- cgit v1.2.3 From 64117da0c36f0697467ce6d56a7be6837da24d2f Mon Sep 17 00:00:00 2001 From: Mahmoud Abuzaina Date: Thu, 5 Jul 2018 13:56:22 -0700 Subject: Fixing AVX performance issue --- tensorflow/tensorflow.bzl | 5 +++++ third_party/mkl_dnn/BUILD | 8 ++++++++ third_party/mkl_dnn/mkldnn.BUILD | 29 ++++++++++++++++++++++++++++- tools/bazel.rc | 4 ++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index e4632c4811..3e3fbeb8f8 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -24,6 +24,10 @@ load( "if_mkl", "if_mkl_lnx_x64" ) +load( + "//third_party/mkl_dnn:build_defs.bzl", + "if_mkl_open_source_only", +) def register_extension_info(**kwargs): pass @@ -214,6 +218,7 @@ def tf_copts(android_optimization_level_override="-O2", is_external=False): + if_cuda(["-DGOOGLE_CUDA=1"]) + if_tensorrt(["-DGOOGLE_TENSORRT=1"]) + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML"]) + + if_mkl_open_source_only(["-DDO_NOT_USE_ML"]) + if_mkl_lnx_x64(["-fopenmp"]) + if_android_arm(["-mfpu=neon"]) + if_linux_x86_64(["-msse3"]) diff --git a/third_party/mkl_dnn/BUILD b/third_party/mkl_dnn/BUILD index 5b01f6e3e4..17a0074abe 100644 --- a/third_party/mkl_dnn/BUILD +++ b/third_party/mkl_dnn/BUILD @@ -1 +1,9 @@ licenses(["notice"]) + +config_setting( + name = "using_mkl_dnn_only", + values = { + "define": "using_mkl_dnn_only=true", + }, + visibility = ["//visibility:public"], +) diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD index 68f24aabae..57d2e1292b 100644 --- a/third_party/mkl_dnn/mkldnn.BUILD +++ b/third_party/mkl_dnn/mkldnn.BUILD @@ -1,5 +1,10 @@ exports_files(["LICENSE"]) +load( + "@org_tensorflow//third_party/mkl_dnn:build_defs.bzl", + "if_mkl_open_source_only", +) + config_setting( name = "clang_linux_x86_64", values = { @@ -15,7 +20,14 @@ cc_library( "src/cpu/*.cpp", ]), hdrs = glob(["include/*"]), - copts = ["-fexceptions"] + select({ + copts = [ + "-fexceptions", + "-DUSE_MKL", + "-DUSE_CBLAS", + ] + if_mkl_open_source_only([ + "-UUSE_MKL", + "-UUSE_CBLAS", + ]) + select({ "@org_tensorflow//tensorflow:linux_x86_64": [ "-fopenmp", # only works with gcc ], @@ -33,4 +45,19 @@ cc_library( ], nocopts = "-fno-exceptions", visibility = ["//visibility:public"], + deps = select({ + "@org_tensorflow//tensorflow:linux_x86_64": [ + "@mkl_linux//:mkl_headers", + "@mkl_linux//:mkl_libs_linux", + ], + "@org_tensorflow//tensorflow:darwin": [ + "@mkl_darwin//:mkl_headers", + "@mkl_darwin//:mkl_libs_darwin", + ], + "@org_tensorflow//tensorflow:windows": [ + "@mkl_windows//:mkl_headers", + "@mkl_windows//:mkl_libs_windows", + ], + "//conditions:default": [], + }), ) diff --git a/tools/bazel.rc b/tools/bazel.rc index 1c1e6afb65..b3a9e6f0ef 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -27,6 +27,10 @@ build --define framework_shared_object=true build:mkl --define=using_mkl=true build:mkl -c opt +# This config option is used to enable MKL-DNN open source library only, +# without depending on MKL binary version. +build:mkl_open_source_only --define=using_mkl_dnn_only=true + build:download_clang --crosstool_top=@local_config_download_clang//:toolchain build:download_clang --define=using_clang=true -- cgit v1.2.3 From 9f3bd2cf1eccdc76ed1934ade96c6cd4464bb8b2 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Fri, 6 Jul 2018 05:46:42 -0700 Subject: lint fix --- tensorflow/examples/tutorials/mnist/mnist_deep.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/examples/tutorials/mnist/mnist_deep.py b/tensorflow/examples/tutorials/mnist/mnist_deep.py index 47d2777813..5d8d8d84fe 100644 --- a/tensorflow/examples/tutorials/mnist/mnist_deep.py +++ b/tensorflow/examples/tutorials/mnist/mnist_deep.py @@ -170,7 +170,9 @@ def main(_): accuracy_l = [] for _ in range(20): batch = mnist.test.next_batch(500, shuffle=False) - accuracy_l.append(accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0})) + accuracy_l.append(accuracy.eval(feed_dict={x: batch[0], + y_: batch[1], + keep_prob: 1.0})) print('test accuracy %g' % numpy.mean(accuracy_l)) -- cgit v1.2.3 From d33bc55210478d58b858704bfa92316860b777fa Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Fri, 6 Jul 2018 09:27:31 -0700 Subject: Updating the version to 1.9.0 official. --- tensorflow/core/public/version.h | 2 +- tensorflow/docs_src/install/install_c.md | 2 +- tensorflow/docs_src/install/install_go.md | 2 +- tensorflow/docs_src/install/install_java.md | 22 +++++++++++----------- tensorflow/docs_src/install/install_linux.md | 18 +++++++++--------- tensorflow/docs_src/install/install_mac.md | 10 +++++----- tensorflow/docs_src/install/install_sources.md | 4 ++-- tensorflow/tools/pip_package/setup.py | 2 +- 8 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 0e4a61ac1f..cea5e8ffb0 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -24,7 +24,7 @@ limitations under the License. // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") -#define TF_VERSION_SUFFIX "-rc2" +#define TF_VERSION_SUFFIX "" #define TF_STR_HELPER(x) #x #define TF_STR(x) TF_STR_HELPER(x) diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md index 9aebf2bfa4..362a03cd56 100644 --- a/tensorflow/docs_src/install/install_c.md +++ b/tensorflow/docs_src/install/install_c.md @@ -38,7 +38,7 @@ enable TensorFlow for C: OS="linux" # Change to "darwin" for macOS TARGET_DIRECTORY="/usr/local" curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.9.0-rc2.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.9.0.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index 1907355341..a4f2e5733b 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go: TF_TYPE="cpu" # Change to "gpu" for GPU support TARGET_DIRECTORY='/usr/local' curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.9.0-rc2.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.9.0.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index b9c9912816..643c3b715f 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs: org.tensorflow tensorflow - 1.9.0-rc2 + 1.9.0 ``` @@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow: org.tensorflow tensorflow - 1.9.0-rc2 + 1.9.0 @@ -124,12 +124,12 @@ instead: org.tensorflow libtensorflow - 1.9.0-rc2 + 1.9.0 org.tensorflow libtensorflow_jni_gpu - 1.9.0-rc2 + 1.9.0 ``` @@ -148,7 +148,7 @@ refer to the simpler instructions above instead. Take the following steps to install TensorFlow for Java on Linux or macOS: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc2.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0.jar), which is the TensorFlow Java Archive (JAR). 2. Decide whether you will run TensorFlow for Java on CPU(s) only or with @@ -167,7 +167,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: OS=$(uname -s | tr '[:upper:]' '[:lower:]') mkdir -p ./jni curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.9.0-rc2.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.9.0.tar.gz" | tar -xz -C ./jni ### Install on Windows @@ -175,10 +175,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS: Take the following steps to install TensorFlow for Java on Windows: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0-rc2.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.9.0.jar), which is the TensorFlow Java Archive (JAR). 2. Download the following Java Native Interface (JNI) file appropriate for - [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.9.0-rc2.zip). + [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.9.0.zip). 3. Extract this .zip file. @@ -227,7 +227,7 @@ must be part of your `classpath`. For example, you can include the downloaded `.jar` in your `classpath` by using the `-cp` compilation flag as follows: -
javac -cp libtensorflow-1.9.0-rc2.jar HelloTF.java
+
javac -cp libtensorflow-1.9.0.jar HelloTF.java
### Running @@ -241,11 +241,11 @@ two files are available to the JVM: For example, the following command line executes the `HelloTF` program on Linux and macOS X: -
java -cp libtensorflow-1.9.0-rc2.jar:. -Djava.library.path=./jni HelloTF
+
java -cp libtensorflow-1.9.0.jar:. -Djava.library.path=./jni HelloTF
And the following command line executes the `HelloTF` program on Windows: -
java -cp libtensorflow-1.9.0-rc2.jar;. -Djava.library.path=jni HelloTF
+
java -cp libtensorflow-1.9.0.jar;. -Djava.library.path=jni HelloTF
If the program prints Hello from version, you've successfully installed TensorFlow for Java and are ready to use the API. If the program diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index ae3d50ff39..abec8ca072 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -438,7 +438,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: 
      (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0-cp34-cp34m-linux_x86_64.whl ## Validate your installation @@ -678,14 +678,14 @@ This section documents the relevant values for Linux installations. CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0-cp27-none-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0-cp27-none-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -697,14 +697,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0-cp34-cp34m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0-cp34-cp34m-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -716,14 +716,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0-cp35-cp35m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0-cp35-cp35m-linux_x86_64.whl
@@ -735,14 +735,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc2-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0-cp36-cp36m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc2-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0-cp36-cp36m-linux_x86_64.whl
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index 3de6da1342..167d17adb4 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv: TensorFlow in the active Virtualenv is as follows: 
 $ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0-py3-none-any.whl If you encounter installation problems, see [Common Installation Problems](#common-installation-problems). @@ -242,7 +242,7 @@ take the following steps: issue the following command: 
 $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0-py3-none-any.whl If the preceding command fails, see [installation problems](#common-installation-problems). @@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: TensorFlow for Python 2.7: 
 (targetDirectory)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0-py2-none-any.whl @@ -518,7 +518,7 @@ The value you specify depends on your Python version. 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0-py2-none-any.whl
@@ -526,5 +526,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py2-none-a 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0rc2-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.9.0-py3-none-any.whl
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index 3520f97c9a..79da209928 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -328,10 +328,10 @@ Invoke `pip install` to install that pip package. The filename of the `.whl` file depends on your platform. For example, the following command will install the pip package -for TensorFlow 1.9.0rc2 on Linux: +for TensorFlow 1.9.0 on Linux: 
-$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.9.0rc2-py2-none-any.whl
+$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.9.0-py2-none-any.whl
## Validate your installation diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 8c077580aa..dc9d059bab 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -45,7 +45,7 @@ DOCLINES = __doc__.split('\n') # This version string is semver compatible, but incompatible with pip. # For pip, we will remove all '-' characters from this string, and use the # result for pip. -_VERSION = '1.9.0-rc2' +_VERSION = '1.9.0' REQUIRED_PACKAGES = [ 'absl-py >= 0.1.6', -- cgit v1.2.3 From a522d458dacd3a34c4ff2e6b76556f623fe7dbd6 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Fri, 29 Jun 2018 22:43:22 -0700 Subject: Remove unused gcp and hdfs config flags, as these are on by default now. PiperOrigin-RevId: 202753310 --- tensorflow/tools/ci_build/ci_parameterized_build.sh | 2 +- tensorflow/tools/ci_build/ci_sanity.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh index e621f85652..6aaeb14aee 100755 --- a/tensorflow/tools/ci_build/ci_parameterized_build.sh +++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh @@ -132,7 +132,7 @@ BAZEL_CMD="bazel test" BAZEL_BUILD_ONLY_CMD="bazel build" BAZEL_CLEAN_CMD="bazel clean" -DEFAULT_BAZEL_CONFIGS="--config=gcp --config=hdfs" +DEFAULT_BAZEL_CONFIGS="" PIP_CMD="${CI_BUILD_DIR}/builds/pip.sh" PIP_TEST_TUTORIALS_FLAG="--test_tutorials" diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index 05676f9551..0dd32ad1a8 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -543,7 +543,7 @@ SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_check_futures_test" "d SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "Check that python files have certain __future__ imports" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Test entries in /tensorflow/contrib/cmake/python_{modules|protos|protos_cc}.txt for validity and consistency" "Check file names for cases") INCREMENTAL_FLAG="" -DEFAULT_BAZEL_CONFIGS="--config=hdfs --config=gcp" +DEFAULT_BAZEL_CONFIGS="" # Parse command-line arguments BAZEL_FLAGS=${DEFAULT_BAZEL_CONFIGS} -- cgit v1.2.3 From 1107fb018307dfdc35fbe1c2d2f2a378c45aeb18 Mon Sep 17 00:00:00 2001 From: Tristan Rice Date: Wed, 20 Jun 2018 13:19:50 -0700 Subject: Cast: support casting to and from quantized types --- tensorflow/core/framework/tensor.h | 1 + tensorflow/core/kernels/cast_op.cc | 53 ++++++++++++++++++---- tensorflow/core/kernels/cast_op.h | 2 + tensorflow/core/kernels/cast_op_test.cc | 12 ++++- .../python/kernel_tests/distributions/util_test.py | 4 +- 5 files changed, 61 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h index d2f2609d3b..1b19ab5da3 100644 --- a/tensorflow/core/framework/tensor.h +++ b/tensorflow/core/framework/tensor.h @@ -482,6 +482,7 @@ class Tensor { friend class VariableOp; // For access to set_shape friend class AutoReloadVariableOp; // For access to set_shape friend class TensorTestHelper; // For access to set_shape + friend class CastOpBase; // For access to set_dtype; friend class OpKernelContext; // For access to RefCountIsOne(). friend class ScopedAllocator; // For access to buf_. friend class XlaTensor; // For access to RefCountIsOne(). diff --git a/tensorflow/core/kernels/cast_op.cc b/tensorflow/core/kernels/cast_op.cc index 626db9131a..85b8a5ea6b 100644 --- a/tensorflow/core/kernels/cast_op.cc +++ b/tensorflow/core/kernels/cast_op.cc @@ -53,8 +53,39 @@ typedef Eigen::SyclDevice SYCLDevice; FN(arg0, std::complex) CastOpBase::CastOpBase(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("SrcT", &src_dtype_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("DstT", &dst_dtype_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("SrcT", &external_src_dtype_)); + + OP_REQUIRES_OK(ctx, ctx->GetAttr("DstT", &external_dst_dtype_)); + + // Quantized data types use the same underlying format as their non quantized + // version so we use the non quantized implementation for casting. + if (external_dst_dtype_ == DT_QUINT8) { + dst_dtype_ = DT_UINT8; + } else if (external_dst_dtype_ == DT_QINT8) { + dst_dtype_ = DT_INT8; + } else if (external_dst_dtype_ == DT_QINT32) { + dst_dtype_ = DT_INT32; + } else if (external_dst_dtype_ == DT_QINT16) { + dst_dtype_ = DT_INT16; + } else if (external_dst_dtype_ == DT_QUINT16) { + dst_dtype_ = DT_UINT16; + } else { + dst_dtype_ = external_dst_dtype_; + } + + if (external_src_dtype_ == DT_QUINT8) { + src_dtype_ = DT_UINT8; + } else if (external_src_dtype_ == DT_QINT8) { + src_dtype_ = DT_INT8; + } else if (external_src_dtype_ == DT_QINT32) { + src_dtype_ = DT_INT32; + } else if (external_src_dtype_ == DT_QINT16) { + src_dtype_ = DT_INT16; + } else if (external_src_dtype_ == DT_QUINT16) { + src_dtype_ = DT_UINT16; + } else { + src_dtype_ = external_src_dtype_; + } } void CastOpBase::Compute(OpKernelContext* ctx) { @@ -62,15 +93,19 @@ void CastOpBase::Compute(OpKernelContext* ctx) { if (work_ == nullptr) { ctx->set_output(0, inp); } else { + Tensor in; + in.UnsafeCopyFromInternal(inp, src_dtype_, inp.shape()); Tensor* out = nullptr; - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, inp.shape(), &out)); - work_(ctx, inp, out); + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, in.shape(), &out)); + out->set_dtype(dst_dtype_); + work_(ctx, in, out); + out->set_dtype(external_dst_dtype_); } } Status CastOpBase::Unimplemented() { - return errors::Unimplemented("Cast ", DataTypeString(src_dtype_), " to ", - DataTypeString(dst_dtype_), " is not supported"); + return errors::Unimplemented("Cast ", DataTypeString(external_src_dtype_), " to ", + DataTypeString(external_dst_dtype_), " is not supported"); } CpuCastOp::CpuCastOp(OpKernelConstruction* ctx) : CastOpBase(ctx) { @@ -78,7 +113,7 @@ CpuCastOp::CpuCastOp(OpKernelConstruction* ctx) : CastOpBase(ctx) { } Status CpuCastOp::Prepare() { - if (src_dtype_ == dst_dtype_) { + if (external_src_dtype_ == external_dst_dtype_) { work_ = nullptr; // Identity return Status::OK(); } @@ -127,7 +162,7 @@ class GpuCastOp : public CastOpBase { private: Status Prepare() { - if (src_dtype_ == dst_dtype_) { + if (external_src_dtype_ == external_dst_dtype_) { work_ = nullptr; // Identity return Status::OK(); } @@ -203,7 +238,7 @@ class SyclCastOp : public CastOpBase { private: Status Prepare() { - if (src_dtype_ == dst_dtype_) { + if (external_src_dtype_ == external_dst_dtype_) { work_ = nullptr; // Identity return Status::OK(); } diff --git a/tensorflow/core/kernels/cast_op.h b/tensorflow/core/kernels/cast_op.h index 16d2e0e0a5..aae1e7ff19 100644 --- a/tensorflow/core/kernels/cast_op.h +++ b/tensorflow/core/kernels/cast_op.h @@ -36,6 +36,8 @@ class CastOpBase : public OpKernel { protected: DataType src_dtype_; DataType dst_dtype_; + DataType external_src_dtype_; + DataType external_dst_dtype_; std::function work_ = nullptr; Status Unimplemented(); diff --git a/tensorflow/core/kernels/cast_op_test.cc b/tensorflow/core/kernels/cast_op_test.cc index 7da9d28a3d..b74dc25837 100644 --- a/tensorflow/core/kernels/cast_op_test.cc +++ b/tensorflow/core/kernels/cast_op_test.cc @@ -76,7 +76,12 @@ class CastOpTest : public OpsTestBase { TEST_CAST(in, half); \ TEST_CAST(in, float); \ TEST_CAST(in, double); \ - TEST_CAST(in, bfloat16); + TEST_CAST(in, bfloat16); \ + TEST_CAST(in, quint8); \ + TEST_CAST(in, qint8); \ + TEST_CAST(in, qint32); \ + TEST_CAST(in, qint16); \ + TEST_CAST(in, quint16); TEST_ALL_CASTS_FROM(uint8) TEST_ALL_CASTS_FROM(uint16) @@ -87,6 +92,11 @@ TEST_ALL_CASTS_FROM(half) TEST_ALL_CASTS_FROM(float) TEST_ALL_CASTS_FROM(double) TEST_ALL_CASTS_FROM(bfloat16) +TEST_ALL_CASTS_FROM(quint8) +TEST_ALL_CASTS_FROM(qint8) +TEST_ALL_CASTS_FROM(qint32) +TEST_ALL_CASTS_FROM(qint16) +TEST_ALL_CASTS_FROM(quint16) #undef TEST_ALL_CASTS_FROM #undef TEST_CAST diff --git a/tensorflow/python/kernel_tests/distributions/util_test.py b/tensorflow/python/kernel_tests/distributions/util_test.py index 9d38ffcb4a..53f143abd6 100644 --- a/tensorflow/python/kernel_tests/distributions/util_test.py +++ b/tensorflow/python/kernel_tests/distributions/util_test.py @@ -311,8 +311,10 @@ class EmbedCheckCategoricalEventShapeTest(test.TestCase): @test_util.run_in_graph_and_eager_modes def testUnsupportedDtype(self): with self.test_session(): + param = ops.convert_to_tensor( + np.ones([2**11+1]).astype(dtypes.qint16.as_numpy_dtype), + dtype=dtypes.qint16) with self.assertRaises(TypeError): - param = array_ops.ones([int(2**11+1)], dtype=dtypes.qint16) du.embed_check_categorical_event_shape(param) -- cgit v1.2.3 From 184aac7d1502f39c293b848820dd9149a0e3a766 Mon Sep 17 00:00:00 2001 From: Andrew Ginns Date: Fri, 6 Jul 2018 21:23:27 +0100 Subject: Typo in taskset for CPU affinity --- tensorflow/contrib/lite/tools/benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/tools/benchmark/README.md b/tensorflow/contrib/lite/tools/benchmark/README.md index 93769305bd..f1e257ad10 100644 --- a/tensorflow/contrib/lite/tools/benchmark/README.md +++ b/tensorflow/contrib/lite/tools/benchmark/README.md @@ -115,7 +115,7 @@ E.g. for running the benchmark on big cores on Pixel 2 with a single thread one can use the following command: ``` -adb shell tasket f0 /data/local/tmp/benchmark_model \ +adb shell taskset f0 /data/local/tmp/benchmark_model \ --graph=/data/local/tmp/mobilenet_quant_v1_224.tflite \ --input_layer="input" \ --input_layer_shape="1,224,224,3" \ -- cgit v1.2.3 From 2203a79228ba40888515b0d71c99719b4429563a Mon Sep 17 00:00:00 2001 From: gracehoney <31743510+aaroey@users.noreply.github.com> Date: Fri, 6 Jul 2018 14:51:58 -0700 Subject: Fix the order of adding const nodes to keep the topological order, and prevent adding them multiple times --- tensorflow/contrib/tensorrt/convert/convert_graph.cc | 13 ++++++++++--- tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py | 7 ++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc index 189944f29b..911585bcd5 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc @@ -248,6 +248,7 @@ tensorflow::Status GetEngineInfo( const std::vector& reverse_topo_order, EngineInfo* info) { std::vector subgraph_node_ids; + std::set added_const_node_ids; // Used to prevent double insertion. std::set segment_devices; int input_port = 0; int output_port = 0; @@ -257,6 +258,7 @@ tensorflow::Status GetEngineInfo( // edge, thus there must not be any duplicates since source nodes of // input/output edges must be in different split of the graph. // TODO(aaroey): consider using node id and port instead. + // TODO(aaroey): using topo order instead of reverting reverse topo order. std::unordered_map created_edges; for (auto it = reverse_topo_order.rbegin(); it != reverse_topo_order.rend(); ++it) { @@ -275,8 +277,7 @@ tensorflow::Status GetEngineInfo( << " neither have requested device nor assigned device"; } } - int node_id = node->id(); - subgraph_node_ids.push_back(node_id); + const int node_id = node->id(); for (const auto edge : node->in_edges()) { auto input_node = edge->src(); if (segment_nodes.count(input_node->name()) == 0) { @@ -286,7 +287,10 @@ tensorflow::Status GetEngineInfo( // won't be removed from the graph. If it doesn't have any edges, TF // will prune it out. if (input_node->type_string() == "Const") { - subgraph_node_ids.push_back(input_node->id()); + if (added_const_node_ids.count(input_node->id()) == 0) { + added_const_node_ids.insert(input_node->id()); + subgraph_node_ids.push_back(input_node->id()); + } } else if (!edge->IsControlEdge() && !input_node->IsSource()) { string s(input_node->name()); StrAppend(&s, ":", edge->src_output()); @@ -304,6 +308,9 @@ tensorflow::Status GetEngineInfo( } } } + // We need to add possible const input nodes before adding this node in + // order to keep the topological order. + subgraph_node_ids.push_back(node_id); for (const auto edge : node->out_edges()) { auto output_node = edge->dst(); if (segment_nodes.count(output_node->name()) == 0 && diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py index d9c41f90d0..3c68c6e4e9 100644 --- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py +++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py @@ -290,7 +290,12 @@ class TfTrtIntegrationTest(test_util.TensorFlowTestCase): def testIdempotence(self): # Test that applying tensorrt optimizer or offline conversion tools multiple # times to the same graph will result in same graph. - # TODO(aaroey): implement this. + # + # TODO(aaroey): currently the conversion is not deterministic, this is + # mainly because during tensorflow::ConvertGraphDefToGraph(), the graph uses + # EdgeSet which use a map keyed by Edge*, so the order of input/output edges + # of a node is nondeterministic, thus the order for segmenter to contract + # edges is nondeterministic. Need to evaluate whether we should fix this. pass -- cgit v1.2.3 From a979fb29de16dff6495b51cb1363eea752b43513 Mon Sep 17 00:00:00 2001 From: AG Ramesh Date: Sun, 8 Jul 2018 20:38:26 -0700 Subject: Add reorder primitive reuse --- tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc | 10 +++------- tensorflow/core/kernels/mkl_conv_grad_input_ops.cc | 7 ++----- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc index 4e80f5acce..87849e48b8 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc @@ -872,12 +872,11 @@ class MklConv2DCustomBackpropFilterOp } // check if src and diff_dst need reorder - std::vector net; T *src_data = nullptr; if (fwd_src_md.data.format != conv2d_bwd_filter->GetSrcMemoryFormat()) { src.SetUsrMem(fwd_src_md, &src_tensor); src.CheckReorderToOpMem( - bwd_filter_pd->src_primitive_desc(), &net); + bwd_filter_pd->src_primitive_desc()); src_data = static_cast(src.GetOpMem().get_data_handle()); } else { src_data = static_cast(const_cast( @@ -889,14 +888,13 @@ class MklConv2DCustomBackpropFilterOp conv2d_bwd_filter->GetDiffDstMemoryFormat()) { diff_dst.SetUsrMem(diff_dst_md, &diff_dst_tensor); diff_dst.CheckReorderToOpMem( - bwd_filter_pd->diff_dst_primitive_desc(), &net); + bwd_filter_pd->diff_dst_primitive_desc()); diff_dst_data = static_cast( diff_dst.GetOpMem().get_data_handle()); } else { diff_dst_data = static_cast(const_cast( diff_dst_tensor.flat().data())); } - stream(stream::kind::eager).submit(net).wait(); // For backward filter, convert diff_filter back to Tensorflow layout // Here we prepare to reorder op memory back to user memory @@ -929,9 +927,7 @@ class MklConv2DCustomBackpropFilterOp // Reorder diff_filter back to Tensorflow layout if necessary if (diff_filter_reorder_required) { - std::vector net; - diff_filter.InsertReorderToUserMem(&net); - stream(stream::kind::eager).submit(net).wait(); + diff_filter.InsertReorderToUserMem(); } } catch (mkldnn::error& e) { string error_msg = "Status: " + std::to_string(e.status) + diff --git a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc index 0af4568b47..60a048779f 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc @@ -722,14 +722,12 @@ class MklConv2DCustomBackpropInputOp diff_src_tensor->flat().data())); // check if filter and diff_dst need reorder - std::vector net; T* filter_data = nullptr; if (fwd_filter_md.data.format != conv2d_bwd_input->GetFilterMemoryFormat()) { filter.SetUsrMem(fwd_filter_md, &filter_tensor); filter.CheckReorderToOpMem( - bwd_input_pd->weights_primitive_desc(), - &net); + bwd_input_pd->weights_primitive_desc()); filter_data = static_cast(filter.GetOpMem().get_data_handle()); } else { filter_data = static_cast(const_cast( @@ -741,14 +739,13 @@ class MklConv2DCustomBackpropInputOp conv2d_bwd_input->GetDiffDstMemoryFormat()) { diff_dst.SetUsrMem(diff_dst_md, &diff_dst_tensor); diff_dst.CheckReorderToOpMem( - bwd_input_pd->diff_dst_primitive_desc(), &net); + bwd_input_pd->diff_dst_primitive_desc()); diff_dst_data = static_cast( diff_dst.GetOpMem().get_data_handle()); } else { diff_dst_data = static_cast(const_cast( diff_dst_tensor.flat().data())); } - stream(stream::kind::eager).submit(net).wait(); // execute convolution input bwd conv2d_bwd_input->Execute(diff_src_data, filter_data, diff_dst_data); -- cgit v1.2.3 From a3aa2e1243e01b91b88f2d5061bed14ff8935ebe Mon Sep 17 00:00:00 2001 From: Jie Date: Sun, 8 Jul 2018 23:14:36 -0700 Subject: [addressing review comments] i. correcting code style 2. adding comments on functions --- .../contrib/tensorrt/convert/convert_nodes.cc | 77 +++++++++++----------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index 9369152d0e..497da88cae 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -53,24 +53,24 @@ limitations under the License. // would work! #define CHECK_EQ_TYPE(val1, val2) CHECK_EQ((int)val1, (int)val2) -#define TFTRT_RETURN_ERROR_IF_FALSE(ptr, node) \ - do { \ - if (ptr == false) { \ - return tensorflow::errors::Internal( \ - string("TFTRT::"), __FUNCTION__, \ - "failed to add TRT layer, at: ", \ - node); \ - } \ +#define TFTRT_RETURN_ERROR_IF_FALSE(ptr, node) \ + do { \ + if (ptr == false) { \ + return tensorflow::errors::Internal( \ + string("TFTRT::"), __FUNCTION__, \ + "failed to add TRT layer, at: ", \ + node); \ + } \ } while (0) -#define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \ - do { \ - if (ptr == nullptr) { \ - return tensorflow::errors::Internal( \ - string("TFTRT::"), __FUNCTION__, \ - "failed to add TRT layer, at: ", \ - node); \ - } \ +#define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \ + do { \ + if (ptr == nullptr) { \ + return tensorflow::errors::Internal( \ + string("TFTRT::"), __FUNCTION__, \ + "failed to add TRT layer, at: ", \ + node); \ + } \ } while (0) #define TFTRT_RETURN_IF_OK(status) \ @@ -125,7 +125,9 @@ bool TensorRTGetBroadcastShape(const nvinfer1::Dims& operand_l, TensorRT Elementwise op supports broadcast but requires both tensor to be of Identical rank - We consider case of: i. Tensor op Const; ii. Tensor op Tensor + We consider case of: + 1. operand_l to be a Tensor & operand_r to be a Const; + 2. operand_l to be a Tensor & operand_r to be a Tensor; note: const op const (constant folding) should fallback to TensorFlow broadcast scheme: @@ -143,7 +145,7 @@ bool TensorRTGetBroadcastShape(const nvinfer1::Dims& operand_l, -> T: 1 1 1 -1 3 5 1 -> W: 1 1 1 1 3 5 1 *******************************************************************************/ - static const int max_nb_dims = nvinfer1::Dims::MAX_DIMS + 1; + const int max_nb_dims = nvinfer1::Dims::MAX_DIMS + 1; const size_t element_size = sizeof(operand_l.d[0]); // fill in dimensions @@ -923,11 +925,9 @@ tensorflow::Status BinaryCompute(const TRT_ShapedWeights& iweights_l, tensorflow::Status BinaryTensorOpWeight( Converter& ctx, const tensorflow::NodeDef& node_def, const nvinfer1::ITensor* tensor, TRT_ShapedWeights weights, - std::vector* outputs, bool swapped_inputs) { - // FIXME assume type matches input weights - // Get trt type & shape - // Maybe this part has to be moved into the block of rsqrt later - + bool swapped_inputs, std::vector* outputs) { + // tensor is the left operand while weights is the right operand; + // when swapped_inputs set to true, those two are swapped. if (node_def.op() != "Sub" && node_def.op() != "Add" && node_def.op() != "Mul" && node_def.op() != "Div" && node_def.op() != "RealDiv") { @@ -1232,9 +1232,10 @@ tensorflow::Status ConvertConv2DHelper( node_def.name()); } +// Helper function converts input into tensor with shape specified by dims. bool PrepareTensorForShape(Converter& ctx, const TRT_TensorOrWeights& input, - const nvinfer1::ITensor** tensor, - const nvinfer1::Dims& dims) { + const nvinfer1::Dims& dims, + const nvinfer1::ITensor** tensor) { if (input.is_tensor()) { if (DimsEqual(input.shape(), dims)) { *tensor = input.tensor(); @@ -1266,7 +1267,7 @@ bool PrepareTensorForShape(Converter& ctx, const TRT_TensorOrWeights& input, tensorflow::Status BinaryTensorOpTensor( Converter& ctx, const tensorflow::NodeDef& node_def, - const TRT_TensorOrWeights operand_l, const TRT_TensorOrWeights operand_r, + const TRT_TensorOrWeights& operand_l, const TRT_TensorOrWeights& operand_r, std::vector* outputs) { static const std::unordered_map ops{ {"Add", nvinfer1::ElementWiseOperation::kSUM}, @@ -1293,9 +1294,9 @@ tensorflow::Status BinaryTensorOpTensor( } TFTRT_RETURN_ERROR_IF_FALSE( - PrepareTensorForShape(ctx, operand_l, &tensor_l, dim_l), node_def.name()); + PrepareTensorForShape(ctx, operand_l, dim_l, &tensor_l), node_def.name()); TFTRT_RETURN_ERROR_IF_FALSE( - PrepareTensorForShape(ctx, operand_r, &tensor_r, dim_r), node_def.name()); + PrepareTensorForShape(ctx, operand_r, dim_r, &tensor_r), node_def.name()); // get trt type & shape TFAttrs attrs(node_def); @@ -1727,7 +1728,7 @@ tensorflow::Status ConvertBinary(Converter& ctx, // (BinaryTensorOpTensor) if (inputs.at(0).is_tensor() && inputs.at(1).is_weights()) { auto status = BinaryTensorOpWeight(ctx, node_def, inputs.at(0).tensor(), - inputs.at(1).weights(), outputs, false); + inputs.at(1).weights(), false, outputs); #if NV_TENSORRT_MAJOR == 3 TF_RETURN_IF_ERROR(status); #else @@ -1737,7 +1738,7 @@ tensorflow::Status ConvertBinary(Converter& ctx, if (inputs.at(0).is_weights() && inputs.at(1).is_tensor()) { auto status = BinaryTensorOpWeight(ctx, node_def, inputs.at(1).tensor(), - inputs.at(0).weights(), outputs, true); + inputs.at(0).weights(), true, outputs); #if NV_TENSORRT_MAJOR == 3 TF_RETURN_IF_ERROR(status); #else @@ -1781,7 +1782,7 @@ tensorflow::Status ConvertUnary(Converter& ctx, // TODO(jie): check type const nvinfer1::ITensor* tensor; TFTRT_RETURN_ERROR_IF_FALSE( - PrepareTensorForShape(ctx, inputs.at(0), &tensor, inputs.at(0).shape()), + PrepareTensorForShape(ctx, inputs.at(0), inputs.at(0).shape(), &tensor), node_def.name()); nvinfer1::IUnaryLayer* layer; @@ -2300,7 +2301,7 @@ tensorflow::Status ConvertFusedBatchNorm( tensorflow::Status ConvertMatMulHelper( Converter& ctx, TRT_TensorOrWeights tensor_input, TRT_ShapedWeights weights_raw, bool transpose_weight, - std::vector* outputs, string node_name) { + string node_name, std::vector* outputs) { nvinfer1::ITensor* output_tensor; if (!tensor_input.is_tensor()) { return tensorflow::errors::InvalidArgument("Input 0 expects tensor"); @@ -2324,7 +2325,7 @@ tensorflow::Status ConvertMatMulHelper( input_dim.d[input_dim.nbDims++] = 1; } TFTRT_RETURN_ERROR_IF_FALSE( - PrepareTensorForShape(ctx, tensor_input, &tensor, input_dim), node_name); + PrepareTensorForShape(ctx, tensor_input, input_dim, &tensor), node_name); nvinfer1::IFullyConnectedLayer* layer = ctx.network()->addFullyConnected( *const_cast(tensor), noutput, weights, biases); @@ -2336,7 +2337,7 @@ tensorflow::Status ConvertMatMulHelper( output_dim.nbDims = 1; TFTRT_RETURN_ERROR_IF_FALSE( PrepareTensorForShape(ctx, TRT_TensorOrWeights(output_tensor), - &temp_tensor, output_dim), + output_dim, &temp_tensor), node_name); output_tensor = const_cast(temp_tensor); outputs->push_back(TRT_TensorOrWeights(output_tensor)); @@ -2383,7 +2384,7 @@ tensorflow::Status ConvertMatMul(Converter& ctx, node_def.op() + "), at: " + node_def.name()); } return ConvertMatMulHelper(ctx, inputs.at(0), inputs.at(1).weights(), - transpose_b, outputs, node_def.name()); + transpose_b, node_def.name(), outputs); } tensorflow::Status ConvertBatchMatMul( @@ -2409,7 +2410,7 @@ tensorflow::Status ConvertBatchMatMul( if (transpose_a == false && inputs.at(0).is_tensor() && inputs.at(1).is_weights()) { return ConvertMatMulHelper(ctx, inputs.at(0), inputs.at(1).weights(), - transpose_b, outputs, node_def.name()); + transpose_b, node_def.name(), outputs); } else { return tensorflow::errors::InvalidArgument( "Invalid configuration for MatMul, at: " + node_def.name()); @@ -2446,10 +2447,10 @@ tensorflow::Status ConvertBatchMatMul( } TFTRT_RETURN_ERROR_IF_FALSE( - PrepareTensorForShape(ctx, inputs.at(0), &tensor_l, dims_l), + PrepareTensorForShape(ctx, inputs.at(0), dims_l, &tensor_l), node_def.name()); TFTRT_RETURN_ERROR_IF_FALSE( - PrepareTensorForShape(ctx, inputs.at(1), &tensor_r, dims_r), + PrepareTensorForShape(ctx, inputs.at(1), dims_r, &tensor_r), node_def.name()); nvinfer1::IMatrixMultiplyLayer* layer = ctx.network()->addMatrixMultiply( -- cgit v1.2.3 From 4902f909d206fe6ca669b8f8a060f18b32f184e0 Mon Sep 17 00:00:00 2001 From: Jie Date: Mon, 9 Jul 2018 01:48:48 -0700 Subject: [addressing review comments] 1. addressing comments on coding style 2. addressing comments on macro logics 3. using absolute path for python tests 4. added building targets for unit tests 5. formating python scripts (using yapf with google style + 2 spaces indent) --- tensorflow/contrib/tensorrt/BUILD | 33 ++++++++++++++ .../contrib/tensorrt/convert/convert_nodes.cc | 14 +++--- .../tensorrt/test/unit_tests/BatchMatMulTest.py | 28 +++++++----- .../tensorrt/test/unit_tests/BiasaddMatMulTest.py | 51 +++++++++------------- .../unit_tests/BinaryTensorWeightBroadcastTest.py | 47 ++++++++++++-------- .../tensorrt/test/unit_tests/ConcatenationTest.py | 17 ++++---- .../tensorrt/test/unit_tests/ConstBroadcastTest.py | 37 ++++++++++------ .../MultiConnectionNeighborEngineTest.py | 40 +++++++++++------ .../test/unit_tests/NeighboringEngineTest.py | 24 ++++++---- .../contrib/tensorrt/test/unit_tests/UnaryTest.py | 17 +++++--- .../tensorrt/test/unit_tests/VGGBlockNCHWTest.py | 36 ++++++++++----- .../tensorrt/test/unit_tests/VGGBlockTest.py | 23 ++++++---- .../tensorrt/test/unit_tests/base_unit_test.py | 48 +++++++++++++------- .../contrib/tensorrt/test/unit_tests/run_test.py | 34 +++++++++------ .../contrib/tensorrt/test/unit_tests/unit_tests.py | 40 +++++++++-------- .../contrib/tensorrt/test/unit_tests/utilities.py | 3 +- 16 files changed, 303 insertions(+), 189 deletions(-) diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index adda0b758b..4de0cf9d18 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -341,6 +341,39 @@ py_test( ], ) +py_test( + name = "converter_unit_tests", + srcs = [ + "test/unit_tests/base_unit_test.py", + "test/unit_tests/BatchMatMulTest.py", + "test/unit_tests/BiasaddMatMulTest.py", + "test/unit_tests/BinaryTensorWeightBroadcastTest.py", + "test/unit_tests/ConcatenationTest.py", + "test/unit_tests/ConstBroadcastTest.py", + "test/unit_tests/MultiConnectionNeighborEngineTest.py", + "test/unit_tests/NeighboringEngineTest.py", + "test/unit_tests/run_test.py", + "test/unit_tests/UnaryTest.py", + "test/unit_tests/unit_tests.py", + "test/unit_tests/utilities.py", + "test/unit_tests/VGGBlockNCHWTest.py", + "test/unit_tests/VGGBlockTest.py", + ], + main = "test/unit_tests/unit_tests.py", + srcs_version = "PY2AND3", + tags = [ + "manual", + "notap", + ], + deps = [ + ":init_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:layers", + "//tensorflow/python:training", + ], +) + cc_library( name = "utils", hdrs = ["convert/utils.h"], diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index 497da88cae..e19adfacad 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -1619,11 +1619,10 @@ tensorflow::Status ConvertConst(Converter& ctx, std::fill_n((float*)dst, GetShapeSize(scalar_shape), *weights_tensor.float_val().begin()); } else { + // make a local copy first to flatten doesn't have to be contigous std::vector tensor_data( weights_tensor.float_val().begin(), - weights_tensor.float_val() - .end()); // make a local copy first to flatten - // doesn't have to be contigous + weights_tensor.float_val().end()); memcpy(dst, tensor_data.data(), len_data); // store into weight store } VLOG(2) << "create shape details: "; @@ -1665,11 +1664,10 @@ tensorflow::Status ConvertConst(Converter& ctx, std::fill_n((int*)dst, GetShapeSize(scalar_shape), *weights_tensor.int_val().begin()); } else { + // make a local copy first to flatten doesn't have to be contigous std::vector tensor_data( weights_tensor.int_val().begin(), - weights_tensor.int_val() - .end()); // make a local copy first to flatten - // doesn't have to be contigous + weights_tensor.int_val().end()); memcpy(dst, tensor_data.data(), len_tensor); // store into weight store } weights = TRT_ShapedWeights(dtype, dst, scalar_shape); @@ -1901,9 +1899,7 @@ tensorflow::Status ConvertReducePool( outputs->push_back(TRT_TensorOrWeights(output_tensor)); return tensorflow::Status::OK(); } -#endif - -#if NV_TENSORRT_MAJOR > 3 +#elif NV_TENSORRT_MAJOR > 3 tensorflow::Status ConvertReduce(Converter& ctx, const tensorflow::NodeDef& node_def, const std::vector& inputs, diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py index bcd6eb2192..d26be35458 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py @@ -41,8 +41,9 @@ from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.layers import core from tensorflow.python.training import training -from base_unit_test import BaseUnitTest -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables + class BatchMatMulTest(BaseUnitTest): """Testing BatchMatMul in TF-TRT conversion""" @@ -50,14 +51,14 @@ class BatchMatMulTest(BaseUnitTest): def __init__(self, log_file='log.txt'): super(BatchMatMulTest, self).__init__() self.static_mode_list = {"FP32", "FP16"} - self.debug=True + self.debug = True self.dynamic_mode_list = {} self.inp_dims = (12, 5, 8, 12) self.dummy_input = np.random.random_sample(self.inp_dims) self.get_network = self.matmul_test self.expect_nb_nodes = 16 self.log_file = log_file - self.test_name = self.__class__.__name__ + self.test_name = self.__class__.__name__ self.ckpt = "./tmp.ckpt" sess = csess.Session() @@ -72,22 +73,25 @@ class BatchMatMulTest(BaseUnitTest): b = constant_op.constant( np.random.randn(12, 5, 12, 7), dtype=dtypes.float32) x1 = math_ops.matmul(x, b) - b = constant_op.constant( - np.random.randn(5, 1, 1), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32) x1 = x1 + b - var = variable_scope.get_variable("test", [12, 5, 12, 7], dtype=dtypes.float32, initializer=init_ops.truncated_normal_initializer) + var = variable_scope.get_variable( + "test", [12, 5, 12, 7], + dtype=dtypes.float32, + initializer=init_ops.truncated_normal_initializer) x2 = math_ops.matmul(x, var) - b = constant_op.constant( - np.random.randn(5, 1, 1), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32) x2 = x2 * b - var = variable_scope.get_variable("test2", [12, 84], dtype=dtypes.float32, initializer=init_ops.truncated_normal_initializer) + var = variable_scope.get_variable( + "test2", [12, 84], + dtype=dtypes.float32, + initializer=init_ops.truncated_normal_initializer) c = gen_array_ops.reshape(x, [12, 40, 12]) b = gen_array_ops.reshape(var, [12, 12, 7]) x3 = math_ops.matmul(c, b) - b = constant_op.constant( - np.random.randn(40, 1), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(40, 1), dtype=dtypes.float32) x3 = x3 + b x3 = gen_array_ops.reshape(x3, [12, 5, 8, 7]) diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py index ab1e18e0d0..81b43422fd 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py @@ -41,8 +41,9 @@ from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.layers import core from tensorflow.python.training import training -from base_unit_test import BaseUnitTest -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables + class BiasaddMatMulTest(BaseUnitTest): """Testing BiasAdd MatMul in TF-TRT conversion""" @@ -50,14 +51,14 @@ class BiasaddMatMulTest(BaseUnitTest): def __init__(self, log_file='log.txt'): super(BiasaddMatMulTest, self).__init__() self.static_mode_list = {"FP32", "FP16"} - self.debug=True + self.debug = True self.dynamic_mode_list = {} self.inp_dims = (48, 12) self.dummy_input = np.random.random_sample(self.inp_dims) self.get_network = self.matmul_test self.expect_nb_nodes = 53 self.log_file = log_file - self.test_name = self.__class__.__name__ + self.test_name = self.__class__.__name__ def matmul_test(self): g = ops.Graph() @@ -67,74 +68,62 @@ class BiasaddMatMulTest(BaseUnitTest): x = array_ops.placeholder( dtype=dtypes.float32, shape=self.inp_dims, name="input") - b = constant_op.constant( - np.random.randn(12, 4), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(12, 4), dtype=dtypes.float32) x1 = math_ops.matmul(x, b) - b = constant_op.constant( - np.random.randn(1, 4), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(1, 4), dtype=dtypes.float32) x1 = x1 + b - b = constant_op.constant( - np.random.randn(48, 4), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(48, 4), dtype=dtypes.float32) x2 = math_ops.matmul(x, b, transpose_a=True) x2 = gen_array_ops.reshape(x2, [48, 1]) - b = constant_op.constant( - np.random.randn(4, 12), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(4, 12), dtype=dtypes.float32) x3 = math_ops.matmul(x, b, transpose_b=True) - b = constant_op.constant( - np.random.randn(16, 48), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(16, 48), dtype=dtypes.float32) x4 = math_ops.matmul(x, b, transpose_b=True, transpose_a=True) x4 = gen_array_ops.reshape(x4, [48, 4]) x5 = gen_array_ops.reshape(x, [4, 12, 12]) x5 = core.flatten(x5) - b = constant_op.constant( - np.random.randn(144, 48), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(144, 48), dtype=dtypes.float32) x5 = math_ops.matmul(x5, b) - b = constant_op.constant( - np.random.randn(48), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(48), dtype=dtypes.float32) x5 = nn.bias_add(x5, b) x5 = gen_array_ops.reshape(x5, [48, 4]) x6 = gen_array_ops.reshape(x, [4, 12, 12]) - b = constant_op.constant( - np.random.randn(12), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32) x6 = nn.bias_add(x6, b, data_format="NHWC") x6 = gen_array_ops.reshape(x6, [48, -1]) x7 = gen_array_ops.reshape(x, [4, 12, 3, 4]) - b = constant_op.constant( - np.random.randn(4), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32) x7 = nn.bias_add(x7, b, data_format="NHWC") x7 = gen_array_ops.reshape(x7, [48, -1]) x8 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) - b = constant_op.constant( - np.random.randn(2), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(2), dtype=dtypes.float32) x8 = nn.bias_add(x8, b, data_format="NHWC") x8 = gen_array_ops.reshape(x8, [48, -1]) x9 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) - b = constant_op.constant( - np.random.randn(3), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(3), dtype=dtypes.float32) x9 = nn.bias_add(x9, b, data_format="NCHW") x9 = gen_array_ops.reshape(x9, [48, -1]) x10 = gen_array_ops.reshape(x, [4, 12, 3, 4]) - b = constant_op.constant( - np.random.randn(12), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32) x10 = nn.bias_add(x10, b, data_format="NCHW") x10 = gen_array_ops.reshape(x10, [48, -1]) x11 = gen_array_ops.reshape(x, [4, 12, 12]) - b = constant_op.constant( - np.random.randn(4), dtype=dtypes.float32) + b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32) x11 = nn.bias_add(x11, b, data_format="NCHW") x11 = gen_array_ops.reshape(x11, [48, -1]) - out = array_ops.concat([x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11], axis=-1) + out = array_ops.concat( + [x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11], axis=-1) out = array_ops.squeeze(out, name="output") return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py index 2b56ac7a96..46c8814405 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py @@ -40,8 +40,9 @@ from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.training import training -from base_unit_test import BaseUnitTest -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables + class BinaryTensorWeightBroadcastTest(BaseUnitTest): """unit tests for scale & elementwise layers in TF-TRT""" @@ -49,14 +50,14 @@ class BinaryTensorWeightBroadcastTest(BaseUnitTest): def __init__(self, log_file='log.txt'): super(BinaryTensorWeightBroadcastTest, self).__init__() self.static_mode_list = {"FP32", "FP16"} - self.debug=True + self.debug = True self.dynamic_mode_list = {} self.inp_dims = (10, 24, 24, 20) self.dummy_input = np.random.random_sample(self.inp_dims) self.get_network = self.get_simple_graph_def self.expect_nb_nodes = 35 - self.log_file = log_file - self.test_name = self.__class__.__name__ + self.log_file = log_file + self.test_name = self.__class__.__name__ self.allclose_rtol = 0.1 self.allclose_atol = 0.05 @@ -79,22 +80,24 @@ class BinaryTensorWeightBroadcastTest(BaseUnitTest): x = math_ops.sigmoid(f) # scale - a = constant_op.constant(np.random.randn(24,1,1), dtype=dtypes.float32) + a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32) f = x + a x = math_ops.sigmoid(f) # scale - a = constant_op.constant(np.random.randn(24,1,1), dtype=dtypes.float32) + a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32) f = a + x x = math_ops.sigmoid(f) # scale - a = constant_op.constant(np.random.randn(24,24,20), dtype=dtypes.float32) + a = constant_op.constant( + np.random.randn(24, 24, 20), dtype=dtypes.float32) f = a + x x = math_ops.sigmoid(f) # scale - a = constant_op.constant(np.random.randn(24,24,20), dtype=dtypes.float32) + a = constant_op.constant( + np.random.randn(24, 24, 20), dtype=dtypes.float32) f = x + a x = math_ops.sigmoid(f) @@ -103,51 +106,57 @@ class BinaryTensorWeightBroadcastTest(BaseUnitTest): f = x + a x = math_ops.sigmoid(f) - # elementwise + # elementwise a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) f = a + x x = math_ops.sigmoid(f) # elementwise - a = constant_op.constant(np.random.randn(1,24,1,1), dtype=dtypes.float32) + a = constant_op.constant( + np.random.randn(1, 24, 1, 1), dtype=dtypes.float32) f = a + x x = math_ops.sigmoid(f) # elementwise - a = constant_op.constant(np.random.randn(1,24,1,1), dtype=dtypes.float32) + a = constant_op.constant( + np.random.randn(1, 24, 1, 1), dtype=dtypes.float32) f = x + a x = math_ops.sigmoid(f) # elementwise - a = constant_op.constant(np.random.randn(1,24,24,1), dtype=dtypes.float32) + a = constant_op.constant( + np.random.randn(1, 24, 24, 1), dtype=dtypes.float32) f = a + x x = math_ops.sigmoid(f) # elementwise - a = constant_op.constant(np.random.randn(1,24,24,1), dtype=dtypes.float32) + a = constant_op.constant( + np.random.randn(1, 24, 24, 1), dtype=dtypes.float32) f = x + a x = math_ops.sigmoid(f) # elementwise - a = constant_op.constant(np.random.randn(1,24,24,20), dtype=dtypes.float32) + a = constant_op.constant( + np.random.randn(1, 24, 24, 20), dtype=dtypes.float32) f = a + x x = math_ops.sigmoid(f) # elementwise - a = constant_op.constant(np.random.randn(1,24,24,20), dtype=dtypes.float32) + a = constant_op.constant( + np.random.randn(1, 24, 24, 20), dtype=dtypes.float32) f = x + a x = math_ops.sigmoid(f) # elementwise - a = constant_op.constant(np.random.randn(24,20), dtype=dtypes.float32) + a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32) f = a + x x = math_ops.sigmoid(f) # elementwise - a = constant_op.constant(np.random.randn(24,20), dtype=dtypes.float32) + a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32) f = x + a x = math_ops.sigmoid(f) - gen_array_ops.reshape(x, [5, -1] , name="output") + gen_array_ops.reshape(x, [5, -1], name="output") return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py index b3648fd070..3a3098e4ed 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py @@ -40,8 +40,9 @@ from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.training import training -from base_unit_test import BaseUnitTest -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables + class ConcatenationTest(BaseUnitTest): """Testing Concatenation in TF-TRT conversion""" @@ -49,14 +50,14 @@ class ConcatenationTest(BaseUnitTest): def __init__(self, log_file='log.txt'): super(ConcatenationTest, self).__init__() self.static_mode_list = {"FP32", "FP16"} - self.debug=True + self.debug = True self.dynamic_mode_list = {} self.inp_dims = (2, 3, 3, 1) self.dummy_input = np.random.random_sample(self.inp_dims) self.get_network = self.get_simple_graph_def self.expect_nb_nodes = 4 - self.log_file = log_file - self.test_name = self.__class__.__name__ + self.log_file = log_file + self.test_name = self.__class__.__name__ def get_simple_graph_def(self): g = ops.Graph() @@ -91,10 +92,10 @@ class ConcatenationTest(BaseUnitTest): r11 = x * a a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) r12 = a * x - concat1 = array_ops.concat([r1,r2,r3,r4,r5,r6], axis=-1) - concat2 = array_ops.concat([r7,r8,r9,r10,r11,r12], axis=3) + concat1 = array_ops.concat([r1, r2, r3, r4, r5, r6], axis=-1) + concat2 = array_ops.concat([r7, r8, r9, r10, r11, r12], axis=3) x = array_ops.concat([concat1, concat2], axis=-1) - gen_array_ops.reshape(x, [2, -1] , name="output") + gen_array_ops.reshape(x, [2, -1], name="output") return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py index b4fbb57851..7fb7d6f611 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py @@ -20,7 +20,6 @@ from __future__ import print_function import argparse import numpy as np -import tensorflow as tf from tensorflow.contrib import tensorrt as trt from tensorflow.core.protobuf import config_pb2 as cpb2 @@ -42,8 +41,9 @@ from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.layers import core from tensorflow.python.training import training -from base_unit_test import BaseUnitTest -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables + class ConstBroadcastTest(BaseUnitTest): """Testing Constant broadcasting in TF-TRT""" @@ -51,14 +51,14 @@ class ConstBroadcastTest(BaseUnitTest): def __init__(self, log_file='log.txt'): super(ConstBroadcastTest, self).__init__() self.static_mode_list = {"FP32", "FP16"} - self.debug=True + self.debug = True self.dynamic_mode_list = {} self.inp_dims = (5, 12, 12, 2) self.dummy_input = np.random.random_sample(self.inp_dims) self.get_network = self.conv_broadcast self.expect_nb_nodes = 7 self.log_file = log_file - self.test_name = self.__class__.__name__ + self.test_name = self.__class__.__name__ self.allclose_rtol = 0.05 self.allclose_atol = 0.05 @@ -69,14 +69,23 @@ class ConstBroadcastTest(BaseUnitTest): with g.as_default(): x = array_ops.placeholder( dtype=dtypes.float32, shape=self.inp_dims, name="input") - filt1 = tf.constant(1, shape=(3,3,2,1), dtype=tf.float32, name='filt1') - y1 = tf.nn.conv2d(x, filt1, strides=[1,1, 1, 1], padding='SAME', name='y1') - z1 = tf.nn.relu(y1, name='z1') - filt2 = tf.constant(np.random.randn(9), shape=(3,3,1,1), dtype=tf.float32, name='filt2') - y2 = tf.nn.conv2d(z1, filt2, strides=[1,1, 1, 1], padding='SAME', name='y2') - z2 = tf.nn.relu(y2, name='z') - filt3 = tf.constant(np.random.randn(3,3,1,1), shape=(3,3,1,1), dtype=tf.float32, name='filt3') - y3 = tf.nn.conv2d(z2, filt3, strides=[1,1, 1, 1], padding='SAME', name='y3') - z = tf.nn.relu(y3, name='output') + filt1 = constant_op.constant( + 1, shape=(3, 3, 2, 1), dtype=dtypes.float32, name='filt1') + y1 = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='SAME', name='y1') + z1 = nn.relu(y1, name='z1') + filt2 = constant_op.constant( + np.random.randn(9), + shape=(3, 3, 1, 1), + dtype=dtypes.float32, + name='filt2') + y2 = nn.conv2d(z1, filt2, strides=[1, 1, 1, 1], padding='SAME', name='y2') + z2 = nn.relu(y2, name='z') + filt3 = constant_op.constant( + np.random.randn(3, 3, 1, 1), + shape=(3, 3, 1, 1), + dtype=dtypes.float32, + name='filt3') + y3 = nn.conv2d(z2, filt3, strides=[1, 1, 1, 1], padding='SAME', name='y3') + z = nn.relu(y3, name='output') return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py index 5f5f13a102..1bbfd0078d 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py @@ -40,8 +40,9 @@ from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.training import training -from base_unit_test import BaseUnitTest -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables + class MultiConnectionNeighborEngineTest(BaseUnitTest): """Multi connection neighboring nodes wiring tests in TF-TRT""" @@ -49,14 +50,14 @@ class MultiConnectionNeighborEngineTest(BaseUnitTest): def __init__(self, log_file='log.txt'): super(MultiConnectionNeighborEngineTest, self).__init__() self.static_mode_list = {"FP32", "FP16"} - self.debug=True + self.debug = True self.dynamic_mode_list = {} self.inp_dims = (2, 3, 7, 5) self.dummy_input = np.random.normal(1.0, 0.5, self.inp_dims) self.get_network = self.neighboring_tensor_test self.expect_nb_nodes = 7 self.log_file = log_file - self.test_name = self.__class__.__name__ + self.test_name = self.__class__.__name__ self.allclose_rtol = 0.05 self.allclose_atol = 0.05 @@ -68,27 +69,40 @@ class MultiConnectionNeighborEngineTest(BaseUnitTest): x = array_ops.placeholder( dtype=dtypes.float32, shape=self.inp_dims, name="input") e = constant_op.constant( - np.random.normal(.05, .005, [3,2,3,4]), + np.random.normal(.05, .005, [3, 2, 3, 4]), name="weights", dtype=dtypes.float32) conv = nn.conv2d( - input=x, filter=e, data_format="NCHW",strides=[1, 1, 1, 1], padding="VALID", name="conv") + input=x, + filter=e, + data_format="NCHW", + strides=[1, 1, 1, 1], + padding="VALID", + name="conv") b = constant_op.constant( - np.random.normal(2.0, 1.0, [1,4,1,1]), name="bias", dtype=dtypes.float32) - t = conv+b + np.random.normal(2.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + t = conv + b b = constant_op.constant( - np.random.normal(5.0, 1.0, [1,4,1,1]), name="bias", dtype=dtypes.float32) - q = conv-b + np.random.normal(5.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + q = conv - b edge = math_ops.sigmoid(q) b = constant_op.constant( - np.random.normal(5.0, 1.0, [1,4,1,1]), name="bias", dtype=dtypes.float32) - d = b+conv + np.random.normal(5.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + d = b + conv edge3 = math_ops.sigmoid(d) c = constant_op.constant( - np.random.normal(1.0, 1.0, [1,4,1,1]), name="bias", dtype=dtypes.float32) + np.random.normal(1.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) edge1 = gen_math_ops.tan(conv) t = t - edge1 q = q + edge diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py index d7b5eba4fd..8341e89519 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py @@ -40,8 +40,9 @@ from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.training import training -from base_unit_test import BaseUnitTest -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables + class NeighboringEngineTest(BaseUnitTest): """Neighboring node wiring tests in TF-TRT conversion""" @@ -49,14 +50,14 @@ class NeighboringEngineTest(BaseUnitTest): def __init__(self, log_file='log.txt'): super(NeighboringEngineTest, self).__init__() self.static_mode_list = {"FP32", "FP16"} - self.debug=True + self.debug = True self.dynamic_mode_list = {} self.inp_dims = (2, 3, 7, 5) self.dummy_input = np.random.random_sample(self.inp_dims) self.get_network = self.neighboring_tensor_test self.expect_nb_nodes = 5 self.log_file = log_file - self.test_name = self.__class__.__name__ + self.test_name = self.__class__.__name__ self.allclose_rtol = 0.05 self.allclose_atol = 0.05 @@ -68,14 +69,21 @@ class NeighboringEngineTest(BaseUnitTest): x = array_ops.placeholder( dtype=dtypes.float32, shape=self.inp_dims, name="input") e = constant_op.constant( - np.random.normal(.3, 0.05, [3,2,3,4]), + np.random.normal(.3, 0.05, [3, 2, 3, 4]), name="weights", dtype=dtypes.float32) conv = nn.conv2d( - input=x, filter=e, data_format="NCHW",strides=[1, 1, 1, 1], padding="VALID", name="conv") + input=x, + filter=e, + data_format="NCHW", + strides=[1, 1, 1, 1], + padding="VALID", + name="conv") b = constant_op.constant( - np.random.normal(1.0, 1.0, [1,4,1,1]), name="bias", dtype=dtypes.float32) - t = conv*b + np.random.normal(1.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + t = conv * b e = gen_math_ops.tan(conv) t = t - e diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py index beb2f2b7a7..8ac4c2a308 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py @@ -41,8 +41,9 @@ from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.layers import core from tensorflow.python.training import training -from base_unit_test import BaseUnitTest -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables + class UnaryTest(BaseUnitTest): """Unit tests for unary operations in TF-TRT""" @@ -50,14 +51,14 @@ class UnaryTest(BaseUnitTest): def __init__(self, log_file='log.txt'): super(UnaryTest, self).__init__() self.static_mode_list = {"FP32", "FP16"} - self.debug=True + self.debug = True self.dynamic_mode_list = {} self.inp_dims = (12, 5, 8, 1, 1, 12) self.dummy_input = np.random.random_sample(self.inp_dims) self.get_network = self.unary_test self.expect_nb_nodes = 17 self.log_file = log_file - self.test_name = self.__class__.__name__ + self.test_name = self.__class__.__name__ self.ckpt = "./tmp.ckpt" def unary_test(self): @@ -82,8 +83,7 @@ class UnaryTest(BaseUnitTest): q = q + 3.0 a = gen_math_ops.reciprocal(q) - x = constant_op.constant( - np.random.randn(5, 8, 12), dtype=dtypes.float32) + x = constant_op.constant(np.random.randn(5, 8, 12), dtype=dtypes.float32) q = math_ops.abs(x) q = q + 2.0 q = gen_math_ops.exp(q) @@ -98,7 +98,10 @@ class UnaryTest(BaseUnitTest): b = gen_math_ops.reciprocal(q) # TODO(jie): this one will break, broadcasting on batch. - x = variable_scope.get_variable("test", [12, 40, 12], dtype=dtypes.float32, initializer=init_ops.truncated_normal_initializer) + x = variable_scope.get_variable( + "test", [12, 40, 12], + dtype=dtypes.float32, + initializer=init_ops.truncated_normal_initializer) x = gen_array_ops.reshape(x, [12, 5, 8, 1, 12, 1, 1]) q = math_ops.abs(x) q = q + 5.0 diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py index f9f5c7d114..a714cc8d9d 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py @@ -40,8 +40,9 @@ from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.training import training -from base_unit_test import BaseUnitTest -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables + class VGGBlockNCHWTest(BaseUnitTest): """single vgg layer in NCHW unit tests in TF-TRT""" @@ -49,14 +50,14 @@ class VGGBlockNCHWTest(BaseUnitTest): def __init__(self, log_file='log.txt'): super(VGGBlockNCHWTest, self).__init__() self.static_mode_list = {"FP32", "FP16"} - self.debug=True + self.debug = True self.dynamic_mode_list = {} self.inp_dims = (5, 2, 8, 8) self.dummy_input = np.random.random_sample(self.inp_dims) self.get_network = self.get_simple_graph_def self.expect_nb_nodes = 3 - self.log_file = log_file - self.test_name = self.__class__.__name__ + self.log_file = log_file + self.test_name = self.__class__.__name__ def get_simple_graph_def(self): g = ops.Graph() @@ -65,20 +66,33 @@ class VGGBlockNCHWTest(BaseUnitTest): with g.as_default(): x = array_ops.placeholder( dtype=dtypes.float32, shape=self.inp_dims, name="input") - x, mean_x, var_x = nn_impl.fused_batch_norm(x, np.random.randn(2).astype(np.float32), np.random.randn(2).astype(np.float32), mean=np.random.randn(2).astype(np.float32), variance=np.random.randn(2).astype(np.float32), data_format="NCHW", is_training=False) + x, mean_x, var_x = nn_impl.fused_batch_norm( + x, + np.random.randn(2).astype(np.float32), + np.random.randn(2).astype(np.float32), + mean=np.random.randn(2).astype(np.float32), + variance=np.random.randn(2).astype(np.float32), + data_format="NCHW", + is_training=False) e = constant_op.constant( - np.random.randn(1,1,2,6), - name="weights", - dtype=dtypes.float32) + np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32) conv = nn.conv2d( - input=x, filter=e, data_format="NCHW",strides=[1, 1, 2, 2], padding="SAME", name="conv") + input=x, + filter=e, + data_format="NCHW", + strides=[1, 1, 2, 2], + padding="SAME", + name="conv") b = constant_op.constant( np.random.randn(6), name="bias", dtype=dtypes.float32) t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd") relu = nn.relu(t, "relu") idty = array_ops.identity(relu, "ID") v = nn_ops.max_pool( - idty, [1, 1, 2, 2], [1, 1, 2, 2], "VALID", data_format="NCHW", name="max_pool") + idty, [1, 1, 2, 2], [1, 1, 2, 2], + "VALID", + data_format="NCHW", + name="max_pool") array_ops.squeeze(v, name="output") return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py index fe872ecd2c..77601a23e7 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py @@ -40,8 +40,9 @@ from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops from tensorflow.python.training import training -from base_unit_test import BaseUnitTest -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables + class VGGBlockTest(BaseUnitTest): """single vgg layer test in TF-TRT conversion""" @@ -49,14 +50,14 @@ class VGGBlockTest(BaseUnitTest): def __init__(self, log_file='log.txt'): super(VGGBlockTest, self).__init__() self.static_mode_list = {"FP32", "FP16"} - self.debug=True + self.debug = True self.dynamic_mode_list = {} self.inp_dims = (5, 8, 8, 2) self.dummy_input = np.random.random_sample(self.inp_dims) self.get_network = self.get_simple_graph_def self.expect_nb_nodes = 7 - self.log_file = log_file - self.test_name = self.__class__.__name__ + self.log_file = log_file + self.test_name = self.__class__.__name__ def get_simple_graph_def(self): g = ops.Graph() @@ -65,11 +66,15 @@ class VGGBlockTest(BaseUnitTest): with g.as_default(): x = array_ops.placeholder( dtype=dtypes.float32, shape=self.inp_dims, name="input") - x, mean_x, var_x = nn_impl.fused_batch_norm(x, np.random.randn(2).astype(np.float32), np.random.randn(2).astype(np.float32), mean=np.random.randn(2).astype(np.float32), variance=np.random.randn(2).astype(np.float32), is_training=False) + x, mean_x, var_x = nn_impl.fused_batch_norm( + x, + np.random.randn(2).astype(np.float32), + np.random.randn(2).astype(np.float32), + mean=np.random.randn(2).astype(np.float32), + variance=np.random.randn(2).astype(np.float32), + is_training=False) e = constant_op.constant( - np.random.randn(1,1,2,6), - name="weights", - dtype=dtypes.float32) + np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32) conv = nn.conv2d( input=x, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv") b = constant_op.constant( diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py b/tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py index f8997bf304..ec8e6e3d1d 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py @@ -33,6 +33,7 @@ from tensorflow.python.ops import array_ops as aops from tensorflow.python.ops import nn as nn from tensorflow.python.ops import nn_ops as nn_ops + class BaseUnitTest(object): """Base class for unit tests in TF-TRT""" @@ -54,42 +55,55 @@ class BaseUnitTest(object): self.check_node_count = False def run(self, run_test_context): - run_test_context.run_test(self.get_network, self.static_mode_list, self.dynamic_mode_list, self.dummy_input, self.ckpt) + run_test_context.run_test(self.get_network, self.static_mode_list, + self.dynamic_mode_list, self.dummy_input, + self.ckpt) return self.log_result(run_test_context) def log_result(self, run_test_result): log = open(self.log_file, 'a') - log.write(("================= model: %s\n")%(self.test_name)) + log.write(("================= model: %s\n") % (self.test_name)) if self.debug: - open(self.test_name+"_native.pb", 'wb').write(run_test_result.native_network.SerializeToString()) + open(self.test_name + "_native.pb", + 'wb').write(run_test_result.native_network.SerializeToString()) all_success = True if len(run_test_result.tftrt_conversion_flag) != 0: log.write(" -- static_mode\n") for static_mode in run_test_result.tftrt_conversion_flag: if self.debug: - open(self.test_name+"_"+static_mode+".pb", 'wb').write(run_test_result.tftrt[static_mode].SerializeToString()) + open(self.test_name + "_" + static_mode + ".pb", + 'wb').write(run_test_result.tftrt[static_mode].SerializeToString()) log.write(" ----\n") - log.write((" mode: [%s]\n")%(static_mode)) + log.write((" mode: [%s]\n") % (static_mode)) if run_test_result.tftrt_conversion_flag[static_mode]: if run_test_result.tftrt_nb_nodes[static_mode] != self.expect_nb_nodes: - log.write(("[WARNING]: converted node number does not match (%d,%d,%d)!!!\n")%(run_test_result.tftrt_nb_nodes[static_mode], self.expect_nb_nodes, run_test_result.native_nb_nodes)) + log.write( + ("[WARNING]: converted node number does not match (%d,%d,%d)!!!\n" + ) % (run_test_result.tftrt_nb_nodes[static_mode], + self.expect_nb_nodes, run_test_result.native_nb_nodes)) if self.check_node_count: all_success = False - if np.array_equal(run_test_result.tftrt_result[static_mode], run_test_result.native_result): + if np.array_equal(run_test_result.tftrt_result[static_mode], + run_test_result.native_result): log.write(" output: equal\n") - elif np.allclose(run_test_result.tftrt_result[static_mode], run_test_result.native_result, atol=self.allclose_atol, rtol=self.allclose_rtol, equal_nan=self.allclose_equal_nan): + elif np.allclose( + run_test_result.tftrt_result[static_mode], + run_test_result.native_result, + atol=self.allclose_atol, + rtol=self.allclose_rtol, + equal_nan=self.allclose_equal_nan): log.write(" output: allclose\n") else: - diff = run_test_result.tftrt_result[static_mode]-run_test_result.native_result + diff = run_test_result.tftrt_result[static_mode] - run_test_result.native_result log.write("[ERROR]: output does not match!!!\n") - log.write( "max diff: " +str(np.max(diff))) - log.write( "\ntftrt:\n") + log.write("max diff: " + str(np.max(diff))) + log.write("\ntftrt:\n") log.write(str(run_test_result.tftrt_result[static_mode])) - log.write( "\nnative:\n") + log.write("\nnative:\n") log.write(str(run_test_result.native_result)) - log.write( "\ndiff:\n") + log.write("\ndiff:\n") log.write(str(diff)) all_success = False else: @@ -100,11 +114,13 @@ class BaseUnitTest(object): log.write(" -- dynamic_mode\n") for dynamic_mode in run_test_result.tftrt_dynamic_conversion_flag: log.write("\n ----\n") - log.write((" mode: [%s]\n")%(dynamic_mode)) + log.write((" mode: [%s]\n") % (dynamic_mode)) if run_test_result.tftrt_dynamic_conversion_flag[dynamic_mode]: - if np.array_equal(run_test_result.tftrt_dynamic_result[dynamic_mode], run_test_result.native_result): + if np.array_equal(run_test_result.tftrt_dynamic_result[dynamic_mode], + run_test_result.native_result): log.write(" output: equal\n") - elif np.allclose(run_test_result.tftrt_dynamic_result[dynamic_mode], run_test_result.native_result): + elif np.allclose(run_test_result.tftrt_dynamic_result[dynamic_mode], + run_test_result.native_result): log.write(" output: allclose\n") else: log.write("[ERROR]: output does not match!!!\n") diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/run_test.py b/tensorflow/contrib/tensorrt/test/unit_tests/run_test.py index afb5cabf33..b322ae63a2 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/run_test.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/run_test.py @@ -33,11 +33,12 @@ from tensorflow.python.ops import nn as nn from tensorflow.python.ops import nn_ops as nn_ops from tensorflow.python.ops import variables from tensorflow.python.training import training -from utilities import get_all_variables +from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables OUTPUT_NODE = "output" INPUT_NODE = "input" -CALIB_COUNT = 5 # calibration iteration +CALIB_COUNT = 5 # calibration iteration + class RunTest: """base class to run TR-TRT conversion and execution""" @@ -61,14 +62,20 @@ class RunTest: self.check_file = None self.native_network = None - def run_test(self, network, static_mode_list, dynamic_mode_list, dummy_input, file_name=None): + def run_test(self, + network, + static_mode_list, + dynamic_mode_list, + dummy_input, + file_name=None): self.native_network = network() success = True initialization = False - if file_name!=None: + if file_name != None: initialization = True self.check_file = file_name - self.native_result, self.native_nb_nodes = self.execute_graph(self.native_network, dummy_input, initialization) + self.native_result, self.native_nb_nodes = self.execute_graph( + self.native_network, dummy_input, initialization) for mode in static_mode_list: try: self.run_static_convert_network(mode, dummy_input, initialization) @@ -79,7 +86,7 @@ class RunTest: for mode in dynamic_mode_list: try: self.run_dynamic_convert_network(mode, dummy_input, initialization) - self.tftrt_dynamic_conversion_flag[mode] = True + self.tftrt_dynamic_conversion_flag[mode] = True except Exception as inst: self.tftrt_dynamic_conversion_flag[mode] = False success = False @@ -122,17 +129,18 @@ class RunTest: inp_dims = dummy_input.shape if mode == "FP32" or mode == "FP16" or mode == "INT8": trt_graph = trt.create_inference_graph( - input_graph_def=self.native_network, - outputs=[OUTPUT_NODE], - max_batch_size=inp_dims[0], - max_workspace_size_bytes=1 << 25, - precision_mode=mode, # TRT Engine precision "FP32","FP16" or "INT8" - minimum_segment_size=2 # minimum number of nodes in an engine + input_graph_def=self.native_network, + outputs=[OUTPUT_NODE], + max_batch_size=inp_dims[0], + max_workspace_size_bytes=1 << 25, + precision_mode=mode, # TRT Engine precision "FP32","FP16" or "INT8" + minimum_segment_size=2 # minimum number of nodes in an engine ) if mode == "INT8": _ = self.execute_calibration(trt_graph, dummy_input, initialization) trt_graph = trt.calib_graph_to_infer_graph(trt_graph) - trt_result, nb_nodes = self.execute_graph(trt_graph, dummy_input, initialization) + trt_result, nb_nodes = self.execute_graph(trt_graph, dummy_input, + initialization) self.tftrt[mode] = trt_graph self.tftrt_nb_nodes[mode] = nb_nodes self.tftrt_result[mode] = trt_result diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py b/tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py index 31d3625c3c..2828249878 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py @@ -14,20 +14,24 @@ # ============================================================================== """Script to execute and log all integration tests.""" -from BatchMatMulTest import BatchMatMulTest -from BiasaddMatMulTest import BiasaddMatMulTest -from BinaryTensorWeightBroadcastTest import BinaryTensorWeightBroadcastTest -from ConcatenationTest import ConcatenationTest -from ConvElewiseFusionFailTest import ConvElewiseFusionFailTest -from GatherV2FailTest import GatherV2FailTest -from MultiConnectionNeighborEngineTest import MultiConnectionNeighborEngineTest -from NeighboringEngineTest import NeighboringEngineTest -from UnaryTest import UnaryTest -from VGGBlockNCHWTest import VGGBlockNCHWTest -from VGGBlockTest import VGGBlockTest -from ConstBroadcastTest import ConstBroadcastTest +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function -from run_test import RunTest +import tensorflow.contrib.tensorrt.test + +from tensorflow.contrib.tensorrt.test.unit_tests.BatchMatMulTest import BatchMatMulTest +from tensorflow.contrib.tensorrt.test.unit_tests.BiasaddMatMulTest import BiasaddMatMulTest +from tensorflow.contrib.tensorrt.test.unit_tests.BinaryTensorWeightBroadcastTest import BinaryTensorWeightBroadcastTest +from tensorflow.contrib.tensorrt.test.unit_tests.ConcatenationTest import ConcatenationTest +from tensorflow.contrib.tensorrt.test.unit_tests.MultiConnectionNeighborEngineTest import MultiConnectionNeighborEngineTest +from tensorflow.contrib.tensorrt.test.unit_tests.NeighboringEngineTest import NeighboringEngineTest +from tensorflow.contrib.tensorrt.test.unit_tests.UnaryTest import UnaryTest +from tensorflow.contrib.tensorrt.test.unit_tests.VGGBlockNCHWTest import VGGBlockNCHWTest +from tensorflow.contrib.tensorrt.test.unit_tests.VGGBlockTest import VGGBlockTest +from tensorflow.contrib.tensorrt.test.unit_tests.ConstBroadcastTest import ConstBroadcastTest + +from tensorflow.contrib.tensorrt.test.unit_tests.run_test import RunTest tests = 0 passed_test = 0 @@ -47,12 +51,12 @@ test_list.append(MultiConnectionNeighborEngineTest()) test_list.append(ConstBroadcastTest()) for test in test_list: - test.debug = True - test.check_node_count = False + test.debug = True + test.check_node_count = False with RunTest() as context: - tests+=1 + tests += 1 if test.run(context): - passed_test +=1 + passed_test += 1 else: failed_list.append(test.test_name) print("Failed test: %s\n", test.test_name) @@ -60,6 +64,6 @@ for test in test_list: if passed_test == tests: print("Passed\n") else: - print(("%d out of %d passed\n -- failed list:")%(passed_test, tests)) + print(("%d out of %d passed\n -- failed list:") % (passed_test, tests)) for test in failed_list: print(" - " + test) diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/utilities.py b/tensorflow/contrib/tensorrt/test/unit_tests/utilities.py index abb999a25f..47a03ece9f 100644 --- a/tensorflow/contrib/tensorrt/test/unit_tests/utilities.py +++ b/tensorflow/contrib/tensorrt/test/unit_tests/utilities.py @@ -21,10 +21,11 @@ from __future__ import print_function from tensorflow.python.ops import variables from tensorflow.python.training import training + def get_all_variables(sess): var_names = sess.run(variables.report_uninitialized_variables()) names_var_list = {} for name in var_names: - names_var_list[name] = sess.graph.get_tensor_by_name(name+":0") + names_var_list[name] = sess.graph.get_tensor_by_name(name + ":0") print(var_names) return names_var_list -- cgit v1.2.3 From 6c4ffe1bee9e7f8e1b181680251898bc842260b1 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Mon, 9 Jul 2018 23:39:52 +0800 Subject: move GetValueType() to protected: --- tensorflow/contrib/lite/tools/benchmark/benchmark_params.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/lite/tools/benchmark/benchmark_params.h b/tensorflow/contrib/lite/tools/benchmark/benchmark_params.h index d9471145a9..c98f47bb0d 100644 --- a/tensorflow/contrib/lite/tools/benchmark/benchmark_params.h +++ b/tensorflow/contrib/lite/tools/benchmark/benchmark_params.h @@ -31,6 +31,8 @@ class TypedBenchmarkParam; class BenchmarkParam { protected: enum class ParamType { TYPE_INT32, TYPE_FLOAT, TYPE_BOOL, TYPE_STRING }; + template + static ParamType GetValueType(); public: template @@ -46,8 +48,6 @@ class BenchmarkParam { } virtual ~BenchmarkParam() {} BenchmarkParam(ParamType type) : type_(type) {} - template - static ParamType GetValueType(); private: static void AssertHasSameType(ParamType a, ParamType b); -- cgit v1.2.3 From 5ed41f6da78bf79ad1ec75918d5e7b315ad4bec8 Mon Sep 17 00:00:00 2001 From: Guillaume Klein Date: Tue, 10 Jul 2018 11:41:13 +0200 Subject: Fix masking of beam ids in gather_tree_from_array The `sequence_length` argument that is passed to the function is the lengths of the **reordered** predictions and was incorrectly used to mask beam ids *before* reordering. Instead, we can reorder beam ids without caring about out of range steps and only select the reodered ids that are in bounds. The added test covers a beam trajectory that previously produced an out of range error because `gather_tree` returned `end_token` (here `beam_width + 1`) for some steps. --- .../kernel_tests/beam_search_decoder_test.py | 42 ++++++++++++++++++++++ .../seq2seq/python/ops/beam_search_decoder.py | 14 +++----- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py index 178328619f..4073b390fc 100644 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py +++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py @@ -132,6 +132,48 @@ class TestGatherTree(test.TestCase): def test_gather_tree_from_array_2d(self): self._test_gather_tree_from_array(depth_ndims=2) + def test_gather_tree_from_array_complex_trajectory(self): + # Max. time = 7, batch = 1, beam = 5. + array = np.expand_dims(np.array( + [[[25, 12, 114, 89, 97]], + [[9, 91, 64, 11, 162]], + [[34, 34, 34, 34, 34]], + [[2, 4, 2, 2, 4]], + [[2, 3, 6, 2, 2]], + [[2, 2, 2, 3, 2]], + [[2, 2, 2, 2, 2]]]), -1) + parent_ids = np.array( + [[[0, 0, 0, 0, 0]], + [[0, 0, 0, 0, 0]], + [[0, 1, 2, 3, 4]], + [[0, 0, 1, 2, 1]], + [[0, 1, 1, 2, 3]], + [[0, 1, 3, 1, 2]], + [[0, 1, 2, 3, 4]]]) + expected_array = np.expand_dims(np.array( + [[[25, 25, 25, 25, 25]], + [[9, 9, 91, 9, 9]], + [[34, 34, 34, 34, 34]], + [[2, 4, 2, 4, 4]], + [[2, 3, 6, 3, 6]], + [[2, 2, 2, 3, 2]], + [[2, 2, 2, 2, 2]]]), -1) + sequence_length = [[4, 6, 4, 7, 6]] + + array = ops.convert_to_tensor( + array, dtype=dtypes.float32) + parent_ids = ops.convert_to_tensor( + parent_ids, dtype=dtypes.int32) + expected_array = ops.convert_to_tensor( + expected_array, dtype=dtypes.float32) + + sorted_array = beam_search_decoder.gather_tree_from_array( + array, parent_ids, sequence_length) + + with self.test_session() as sess: + sorted_array, expected_array = sess.run([sorted_array, expected_array]) + self.assertAllEqual(expected_array, sorted_array) + class TestArrayShapeChecks(test.TestCase): diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py index c7fbeea310..f17dbb0fe3 100644 --- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py +++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py @@ -145,24 +145,20 @@ def gather_tree_from_array(t, parent_ids, sequence_length): array_ops.expand_dims(math_ops.range(beam_width), 0), 0) beam_ids = array_ops.tile(beam_ids, [max_time, batch_size, 1]) - mask = array_ops.sequence_mask( - sequence_length, maxlen=max_time, dtype=dtypes.int32) - mask = array_ops.transpose(mask, perm=[2, 0, 1]) - - # Use beam_width + 1 to mark the end of beam. - masked_beam_ids = (beam_ids * mask) + (1 - mask) * (beam_width + 1) - max_sequence_lengths = math_ops.to_int32( math_ops.reduce_max(sequence_length, axis=1)) sorted_beam_ids = beam_search_ops.gather_tree( - step_ids=masked_beam_ids, + step_ids=beam_ids, parent_ids=parent_ids, max_sequence_lengths=max_sequence_lengths, end_token=beam_width + 1) # For out of range steps, simply copy the same beam. + in_bound_steps = array_ops.transpose( + array_ops.sequence_mask(sequence_length, maxlen=max_time), + perm=[2, 0, 1]) sorted_beam_ids = array_ops.where( - math_ops.cast(mask, dtypes.bool), x=sorted_beam_ids, y=beam_ids) + in_bound_steps, x=sorted_beam_ids, y=beam_ids) # Generate indices for gather_nd. time_ind = array_ops.tile(array_ops.reshape( -- cgit v1.2.3 From 08b54912c07b993b661d30a8a5c2d7b558ca9b81 Mon Sep 17 00:00:00 2001 From: mktozk Date: Tue, 10 Jul 2018 19:31:00 +0900 Subject: Replace custom relu6 with tf.keras.layers.ReLU() --- tensorflow/python/keras/applications/mobilenet.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tensorflow/python/keras/applications/mobilenet.py b/tensorflow/python/keras/applications/mobilenet.py index e56c695a28..08c3f69e5c 100644 --- a/tensorflow/python/keras/applications/mobilenet.py +++ b/tensorflow/python/keras/applications/mobilenet.py @@ -87,6 +87,7 @@ from tensorflow.python.keras.layers import Dropout from tensorflow.python.keras.layers import GlobalAveragePooling2D from tensorflow.python.keras.layers import GlobalMaxPooling2D from tensorflow.python.keras.layers import Input +from tensorflow.python.keras.layers import ReLU from tensorflow.python.keras.layers import Reshape from tensorflow.python.keras.layers import ZeroPadding2D from tensorflow.python.keras.models import Model @@ -100,10 +101,6 @@ from tensorflow.python.util.tf_export import tf_export BASE_WEIGHT_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.6/' -def relu6(x): - return K.relu(x, max_value=6) - - @tf_export('keras.applications.mobilenet.preprocess_input') def preprocess_input(x): """Preprocesses a numpy array encoding a batch of images. @@ -130,12 +127,6 @@ def MobileNet(input_shape=None, classes=1000): """Instantiates the MobileNet architecture. - To load a MobileNet model via `load_model`, import the custom - objects `relu6` and pass them to the `custom_objects` parameter. - E.g. - model = load_model('mobilenet.h5', custom_objects={ - 'relu6': mobilenet.relu6}) - Arguments: input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape @@ -412,7 +403,7 @@ def _conv_block(inputs, filters, alpha, kernel=(3, 3), strides=(1, 1)): strides=strides, name='conv1')(x) x = BatchNormalization(axis=channel_axis, name='conv1_bn')(x) - return Activation(relu6, name='conv1_relu')(x) + return ReLU(6, name='conv1_relu')(x) def _depthwise_conv_block(inputs, @@ -479,7 +470,7 @@ def _depthwise_conv_block(inputs, use_bias=False, name='conv_dw_%d' % block_id)(x) x = BatchNormalization(axis=channel_axis, name='conv_dw_%d_bn' % block_id)(x) - x = Activation(relu6, name='conv_dw_%d_relu' % block_id)(x) + x = ReLU(6, name='conv_dw_%d_relu' % block_id)(x) x = Conv2D( pointwise_conv_filters, (1, 1), @@ -489,4 +480,4 @@ def _depthwise_conv_block(inputs, name='conv_pw_%d' % block_id)( x) x = BatchNormalization(axis=channel_axis, name='conv_pw_%d_bn' % block_id)(x) - return Activation(relu6, name='conv_pw_%d_relu' % block_id)(x) + return ReLU(6, name='conv_pw_%d_relu' % block_id)(x) -- cgit v1.2.3 From c9cb73604bbe12c79badb03c7f3e1ab817e2e25e Mon Sep 17 00:00:00 2001 From: mktozk Date: Tue, 10 Jul 2018 19:33:23 +0900 Subject: Remove unused imports --- tensorflow/python/keras/applications/mobilenet.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tensorflow/python/keras/applications/mobilenet.py b/tensorflow/python/keras/applications/mobilenet.py index 08c3f69e5c..7285e03963 100644 --- a/tensorflow/python/keras/applications/mobilenet.py +++ b/tensorflow/python/keras/applications/mobilenet.py @@ -72,13 +72,9 @@ from __future__ import print_function import os from tensorflow.python.keras import backend as K -from tensorflow.python.keras import constraints -from tensorflow.python.keras import initializers -from tensorflow.python.keras import regularizers from tensorflow.python.keras.applications import imagenet_utils from tensorflow.python.keras.applications.imagenet_utils import _obtain_input_shape from tensorflow.python.keras.applications.imagenet_utils import decode_predictions -from tensorflow.python.keras.engine.base_layer import InputSpec from tensorflow.python.keras.layers import Activation from tensorflow.python.keras.layers import BatchNormalization from tensorflow.python.keras.layers import Conv2D @@ -91,7 +87,6 @@ from tensorflow.python.keras.layers import ReLU from tensorflow.python.keras.layers import Reshape from tensorflow.python.keras.layers import ZeroPadding2D from tensorflow.python.keras.models import Model -from tensorflow.python.keras.utils import conv_utils from tensorflow.python.keras.utils import layer_utils from tensorflow.python.keras.utils.data_utils import get_file from tensorflow.python.platform import tf_logging as logging -- cgit v1.2.3 From f01ef8253068e7ccae92146ece6add3b57b1b916 Mon Sep 17 00:00:00 2001 From: TShapinsky Date: Tue, 10 Jul 2018 13:16:38 -0400 Subject: Fix incorrect documentation --- .../src/org/tensorflow/demo/TensorFlowObjectDetectionAPIModel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/TensorFlowObjectDetectionAPIModel.java b/tensorflow/examples/android/src/org/tensorflow/demo/TensorFlowObjectDetectionAPIModel.java index 614d3c7dd7..9739e58018 100644 --- a/tensorflow/examples/android/src/org/tensorflow/demo/TensorFlowObjectDetectionAPIModel.java +++ b/tensorflow/examples/android/src/org/tensorflow/demo/TensorFlowObjectDetectionAPIModel.java @@ -137,7 +137,7 @@ public class TensorFlowObjectDetectionAPIModel implements Classifier { Trace.beginSection("recognizeImage"); Trace.beginSection("preprocessBitmap"); - // Preprocess the image data from 0-255 int to normalized float based + // Preprocess the image data to extract R, G and B bytes from int of form 0x00RRGGBB // on the provided parameters. bitmap.getPixels(intValues, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight()); -- cgit v1.2.3 From a6ffc43178f3da4f658be38274e647521e430ee2 Mon Sep 17 00:00:00 2001 From: mktozk Date: Wed, 11 Jul 2018 05:10:19 +0900 Subject: Fix test_custom_objects --- tensorflow/python/estimator/keras_test.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tensorflow/python/estimator/keras_test.py b/tensorflow/python/estimator/keras_test.py index 7a4457f5a4..788219378d 100644 --- a/tensorflow/python/estimator/keras_test.py +++ b/tensorflow/python/estimator/keras_test.py @@ -477,14 +477,8 @@ class TestKerasEstimator(test_util.TensorFlowTestCase): keras_mobile = mobilenet.MobileNet(weights=None) keras_mobile.compile(loss='categorical_crossentropy', optimizer='adam') custom_objects = { - 'relu6': mobilenet.relu6, 'DepthwiseConv2D': mobilenet.DepthwiseConv2D } - with self.assertRaisesRegexp(ValueError, 'relu6'): - with self.test_session(): - keras_lib.model_to_estimator( - keras_model=keras_mobile, - model_dir=tempfile.mkdtemp(dir=self._base_dir)) with self.test_session(): keras_lib.model_to_estimator( -- cgit v1.2.3 From 4e09f05994ea3b9cb3291880b3fa8cecc60ec153 Mon Sep 17 00:00:00 2001 From: gracehoney <31743510+aaroey@users.noreply.github.com> Date: Tue, 10 Jul 2018 14:37:41 -0700 Subject: Add TODOs and fix some formatting issues --- .../contrib/tensorrt/convert/convert_nodes.cc | 406 +++++++++++---------- 1 file changed, 223 insertions(+), 183 deletions(-) diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index e19adfacad..b1f08ef0d0 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -49,9 +49,9 @@ limitations under the License. #if GOOGLE_TENSORRT #include "tensorrt/include/NvInfer.h" -// Check if the types are equal. Cast to int first so that failure log message -// would work! -#define CHECK_EQ_TYPE(val1, val2) CHECK_EQ((int)val1, (int)val2) +// Check if the types are equal. Cast to int first so that failure log message +// would work! +#define TFTRT_CHECK_EQ_TYPE(val1, val2) CHECK_EQ((int)val1, (int)val2) #define TFTRT_RETURN_ERROR_IF_FALSE(ptr, node) \ do { \ @@ -109,42 +109,42 @@ inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype, #endif default: return tensorflow::errors::InvalidArgument( - "Unsupported data type " + tensorflow::DataTypeString(tf_dtype)); + "Unsupported data type ", tensorflow::DataTypeString(tf_dtype)); } return tensorflow::Status::OK(); } -// return whether or not the broadcast is feasible; +// Return whether or not the broadcast is feasible; bool TensorRTGetBroadcastShape(const nvinfer1::Dims& operand_l, const bool operand_l_is_tensor, const nvinfer1::Dims& operand_r, const bool operand_r_is_tensor, nvinfer1::Dims* operand_l_new_shape, nvinfer1::Dims* operand_r_new_shape) { - /******************************************************************************* - TensorRT Elementwise op supports broadcast but requires both tensor to be of - Identical rank - - We consider case of: - 1. operand_l to be a Tensor & operand_r to be a Const; - 2. operand_l to be a Tensor & operand_r to be a Tensor; - note: const op const (constant folding) should fallback to TensorFlow - - broadcast scheme: - T: 1 3 5 (tensor would not have batch dimension) - W: 1 1 3 1 (weight would have all explicit dimensions) - i. fill in explicit dimensions - -> T: -1 1 3 5 (we put a -1 for batch dimension) - -> W: 1 1 3 1 - ii. compare broadcast feasibility - - we cannot support these since TensorRT does not allow manipulation on batch - dimension, we cannot generate output with proper shape - T: 3 5 1 - W: 1 1 1 1 3 5 1 - -> T: 1 1 1 -1 3 5 1 - -> W: 1 1 1 1 3 5 1 - *******************************************************************************/ + // *************************************************************************** + // TensorRT Elementwise op supports broadcast but requires both tensor to be + // of Identical rank + // + // We consider case of: + // 1. operand_l to be a Tensor & operand_r to be a Const; + // 2. operand_l to be a Tensor & operand_r to be a Tensor; + // note: const op const (constant folding) should fallback to TensorFlow + // + // broadcast scheme: + // T: 1 3 5 (tensor would not have batch dimension) + // W: 1 1 3 1 (weight would have all explicit dimensions) + // i. fill in explicit dimensions + // -> T: -1 1 3 5 (we put a -1 for batch dimension) + // -> W: 1 1 3 1 + // ii. compare broadcast feasibility + // + // We cannot support the following since TensorRT does not allow manipulation + // on batch dimension, we cannot generate output with proper shape + // T: 3 5 1 + // W: 1 1 1 1 3 5 1 + // -> T: 1 1 1 -1 3 5 1 + // -> W: 1 1 1 1 3 5 1 + // *************************************************************************** const int max_nb_dims = nvinfer1::Dims::MAX_DIMS + 1; const size_t element_size = sizeof(operand_l.d[0]); @@ -215,7 +215,7 @@ inline nvinfer1::Dims GetTensorShape(const tensorflow::Tensor& tensor) { return dims; } -inline int64_t GetShapeSize(nvinfer1::Dims shape) { +inline int64_t GetShapeSize(const nvinfer1::Dims& shape) { // Returns total number of elements in shape int64_t count = 1; for (int d = 0; d < shape.nbDims; ++d) { @@ -228,7 +228,7 @@ static std::vector> CreateSamePadding( const nvinfer1::DimsHW& stride, const nvinfer1::DimsHW& kernel, const std::vector& input_dims) { std::vector> padding(input_dims.size()); - CHECK_EQ((size_t)stride.nbDims, input_dims.size()); // TODO(jie): N+C? NC+? + CHECK_EQ(stride.nbDims, input_dims.size()); // TODO(jie): N+C? NC+? for (size_t i = 0; i < input_dims.size(); ++i) { // Formula to calculate the padding @@ -258,6 +258,7 @@ string GetCommonNameScope(const string& op_name_a, const string& op_name_b) { return op_name_a.substr(0, last_scope_separator); } +// Class to convert TF weight to TRT weight. class TRT_ShapedWeights { public: TRT_ShapedWeights(tensorflow::DataType type, const void* values, @@ -269,12 +270,14 @@ class TRT_ShapedWeights { explicit TRT_ShapedWeights(tensorflow::DataType type) : shape_(), type_(type), values_(nullptr), empty_weight_flag_(true) {} + // TODO(aaroey): use rvalue reference. TRT_ShapedWeights(const TRT_ShapedWeights& rhs) : shape_(rhs.shape_), type_(rhs.type_), values_(rhs.values_), empty_weight_flag_(rhs.empty_weight_flag_) {} + // TODO(aaroey): use GetShapeSize() instead. int64_t count() const { int64_t c = 1; for (int i = 0; i < shape_.nbDims; i++) c *= shape_.d[i]; @@ -292,6 +295,7 @@ class TRT_ShapedWeights { const void* GetValues() const { return values_; } + // TODO(aaroey): get rid of this method. void SetValues(const void* values) { values_ = values; } size_t size_bytes() const { @@ -302,10 +306,12 @@ class TRT_ShapedWeights { // Default converter operator nvinfer1::Weights() const { return GetWeightsForTRT(); } + // TODO(aaroey): make these private. nvinfer1::Dims shape_; tensorflow::DataType type_; private: + // TODO(aaroey): this should not be const as it's always from TRTWeightStore. const void* values_; bool empty_weight_flag_; }; @@ -316,6 +322,7 @@ class TRT_TensorOrWeights { : tensor_(tensor), weights_(DT_FLOAT), variant_(TRT_NODE_TENSOR) {} explicit TRT_TensorOrWeights(const TRT_ShapedWeights& weights) : tensor_(nullptr), weights_(weights), variant_(TRT_NODE_WEIGHTS) {} + // TODO(aaroey): use rvalue reference. TRT_TensorOrWeights(const TRT_TensorOrWeights& rhs) : tensor_(rhs.tensor_), weights_(rhs.weights_), variant_(rhs.variant_) {} ~TRT_TensorOrWeights() {} @@ -324,19 +331,19 @@ class TRT_TensorOrWeights { bool is_weights() const { return variant_ == TRT_NODE_WEIGHTS; } nvinfer1::ITensor* tensor() { - CHECK_EQ(is_tensor(), true); + CHECK(is_tensor()); return tensor_; } const nvinfer1::ITensor* tensor() const { - CHECK_EQ(is_tensor(), true); + CHECK(is_tensor()); return tensor_; } TRT_ShapedWeights& weights() { - CHECK_EQ(is_weights(), true); + CHECK(is_weights()); return weights_; } const TRT_ShapedWeights& weights() const { - CHECK_EQ(is_weights(), true); + CHECK(is_weights()); return weights_; } nvinfer1::Dims shape() const { @@ -360,21 +367,25 @@ class TFAttrs { attrs_.insert({attr.first, &attr.second}); } } - bool count(string key) const { return attrs_.count(key); } - tensorflow::AttrValue const* at(string key) const { + + bool count(const string& key) const { return attrs_.count(key); } + + tensorflow::AttrValue const* at(const string& key) const { if (!attrs_.count(key)) { LOG(FATAL) << "Attribute not found: " << key; } return attrs_.at(key); } + template T get(const string& key) const; + template T get(const string& key, const T& default_value) const { return attrs_.count(key) ? this->get(key) : default_value; } - std::vector GetAllAttrKey() { + std::vector GetAllAttrKeys() const { std::vector attr_list; for (const auto& attr_item : attrs_) { attr_list.emplace_back(attr_item.first); @@ -409,15 +420,6 @@ std::vector TFAttrs::get>(const string& key) const { auto attr = this->at(key)->list().s(); return std::vector(attr.begin(), attr.end()); } -template <> -nvinfer1::Dims TFAttrs::get(const string& key) const { - auto values = this->get>(key); - nvinfer1::Dims dims; - dims.nbDims = values.size(); - std::copy(values.begin(), values.end(), dims.d); - // Note: No dimension type information is included - return dims; -} template <> nvinfer1::DataType TFAttrs::get(const string& key) const { @@ -443,10 +445,11 @@ bool TFAttrs::get(const string& key) const { } // TODO(jie): reorder4 & reorder2 should be merged? +// TODO(aaroey): fix the order of parameters. template -void Reorder4(nvinfer1::DimsNCHW shape, const T* idata, - nvinfer1::DimsNCHW istrides, T* odata, - nvinfer1::DimsNCHW ostrides) { +void Reorder4(const nvinfer1::DimsNCHW& shape, const T* idata, + const nvinfer1::DimsNCHW& istrides, T* odata, + const nvinfer1::DimsNCHW& ostrides) { for (int n = 0; n < shape.n(); ++n) { for (int c = 0; c < shape.c(); ++c) { for (int h = 0; h < shape.h(); ++h) { @@ -461,8 +464,9 @@ void Reorder4(nvinfer1::DimsNCHW shape, const T* idata, } template -void Reorder2(nvinfer1::DimsHW shape, const T* idata, nvinfer1::DimsHW istrides, - T* odata, nvinfer1::DimsHW ostrides) { +void Reorder2(const nvinfer1::DimsHW& shape, const T* idata, + const nvinfer1::DimsHW& istrides, T* odata, + const nvinfer1::DimsHW& ostrides) { for (int h = 0; h < shape.h(); ++h) { for (int w = 0; w < shape.w(); ++w) { odata[h * ostrides.h() + w * ostrides.w()] = @@ -474,16 +478,17 @@ void Reorder2(nvinfer1::DimsHW shape, const T* idata, nvinfer1::DimsHW istrides, // TODO(jie): fallback to tensorflow!! void ReorderCKtoKC(const TRT_ShapedWeights& iweights, TRT_ShapedWeights* oweights) { - int c = iweights.shape_.d[0]; - int k = iweights.shape_.d[1]; + const int c = iweights.shape_.d[0]; + const int k = iweights.shape_.d[1]; oweights->shape_.d[0] = k; oweights->shape_.d[1] = c; - nvinfer1::DimsHW istrides = {1, k}; - nvinfer1::DimsHW ostrides = {c, 1}; + const nvinfer1::DimsHW istrides = {1, k}; + const nvinfer1::DimsHW ostrides = {c, 1}; switch (iweights.type_) { case tensorflow::DataType::DT_FLOAT: { Reorder2({k, c}, static_cast(iweights.GetValues()), istrides, + // TODO(aaroey): get rid of all the const_cast like this. static_cast(const_cast(oweights->GetValues())), ostrides); break; @@ -505,14 +510,15 @@ void ReorderRSCKToKCRS(const TRT_ShapedWeights& iweights, TRT_ShapedWeights* oweights, int num_groups) { CHECK_EQ(iweights.type_, oweights->type_); CHECK_EQ(iweights.size_bytes(), oweights->size_bytes()); - int r = iweights.shape_.d[0]; - int s = iweights.shape_.d[1]; - // TRT requires GKcRS, while TF depthwise has RSCK - // where c=1, C=G + // K indexes over output channels, C over input channels, and R and S over the + // height and width of the convolution + const int r = iweights.shape_.d[0]; + const int s = iweights.shape_.d[1]; + // TRT requires GKcRS, while TF depthwise has RSCK where c=1, C=G VLOG(2) << "num_groups: " << num_groups; - int c = iweights.shape_.d[2] / num_groups; + const int c = iweights.shape_.d[2] / num_groups; VLOG(2) << "c" << iweights.shape_.d[2] << " then " << c; - int k = iweights.shape_.d[3] * num_groups; + const int k = iweights.shape_.d[3] * num_groups; VLOG(2) << "k" << iweights.shape_.d[3] << " then " << k; VLOG(2) << "r" << iweights.shape_.d[0] << " then " << r; VLOG(2) << "s" << iweights.shape_.d[1] << " then " << s; @@ -520,8 +526,8 @@ void ReorderRSCKToKCRS(const TRT_ShapedWeights& iweights, oweights->shape_.d[1] = c * num_groups; oweights->shape_.d[2] = r; oweights->shape_.d[3] = s; - nvinfer1::DimsNCHW istrides = {1, k, s * k * c, c * k}; - nvinfer1::DimsNCHW ostrides = {c * r * s, r * s, s, 1}; + const nvinfer1::DimsNCHW istrides = {1, k, s * k * c, c * k}; + const nvinfer1::DimsNCHW ostrides = {c * r * s, r * s, s, 1}; switch (iweights.type_) { case tensorflow::DataType::DT_FLOAT: { Reorder4({k, c, r, s}, static_cast(iweights.GetValues()), @@ -553,11 +559,14 @@ using OpConverter = std::vector*)>; class Converter { + // TODO(aaroey): fix the order of members. std::unordered_map trt_tensors_; std::unordered_map op_registry_; OpConverter plugin_converter_; nvinfer1::INetworkDefinition* trt_network_; std::list> temp_bufs_; + // TODO(aaroey): inline the definition of TRTWeightStore here, and add APIs to + // operate the stored weights instead of operating it directly. TRTWeightStore* weight_store_; bool fp16_; void register_op_converters(); @@ -565,7 +574,7 @@ class Converter { std::vector* inputs) { for (auto const& input_name : node_def.input()) { /************************************************************************* - * TODO(jie) handle case 1) here + * TODO(jie): handle case 1) here. * Normalizes the inputs and extracts associated metadata: * 1) Inputs can contain a colon followed by a suffix of characters. * That suffix may be a single number (e.g. inputName:1) or several @@ -579,6 +588,7 @@ class Converter { if (input_name[0] == '^') continue; string name = input_name; auto first = name.find_first_of(':'); + // TODO(aaroey): why removing the colon but not the zero? A bug? if (first != string::npos && first + 2 == name.size() && name[first + 1] == '0') name.erase(first); @@ -587,12 +597,13 @@ class Converter { if (trt_tensors_.count(name)) { inputs->push_back(trt_tensors_.at(name)); } else { - string str("Node "); - StrAppend(&str, node_def.name(), " should have an input named '", name, + // TODO(aaroey): this should not happen, make it a CHECK. + // TODO(aaroey): use StrCat for pattern like this. + string msg("Node "); + StrAppend(&msg, node_def.name(), " should have an input named '", name, "' but it is not available"); - LOG(WARNING) << "input: " << name << " not available for node at " - << node_def.name(); - return tensorflow::errors::InvalidArgument(str); + LOG(ERROR) << msg; + return tensorflow::errors::InvalidArgument(msg); } } return tensorflow::Status::OK(); @@ -613,6 +624,7 @@ class Converter { weights.SetValues(weight_store_->store_.back().data()); return weights; } + // TODO(aaroey): fix all the namings. bool isFP16() { return fp16_; } TRT_ShapedWeights get_temp_weights_like(const TRT_ShapedWeights& weights) { return this->get_temp_weights(weights.type_, weights.shape_); @@ -621,7 +633,7 @@ class Converter { tensorflow::Status convert_node(const tensorflow::NodeDef& node_def) { std::vector inputs; TF_RETURN_IF_ERROR(this->get_inputs(node_def, &inputs)); - string op = node_def.op(); + const string& op = node_def.op(); std::vector outputs; if (PluginFactoryTensorRT::GetInstance()->IsPlugin(op)) { TF_RETURN_IF_ERROR(plugin_converter_(*this, node_def, inputs, &outputs)); @@ -634,7 +646,7 @@ class Converter { TF_RETURN_IF_ERROR(op_converter(*this, node_def, inputs, &outputs)); } for (size_t i = 0; i < outputs.size(); ++i) { - TRT_TensorOrWeights output = outputs.at(i); + TRT_TensorOrWeights& output = outputs[i]; // TODO(jie): tf protobuf seems to be omitting the :0 suffix string output_name = node_def.name(); if (i != 0) output_name = StrCat(output_name, ":", i); @@ -652,20 +664,20 @@ class Converter { nvinfer1::INetworkDefinition* network() { return trt_network_; } - TRT_TensorOrWeights get_tensor(string name) { + TRT_TensorOrWeights get_tensor(const string& name) { if (!trt_tensors_.count(name)) { return TRT_TensorOrWeights(nullptr); } return trt_tensors_.at(name); } - bool insert_input_tensor(string name, nvinfer1::ITensor* tensor) { + bool insert_input_tensor(const string& name, nvinfer1::ITensor* tensor) { return trt_tensors_.insert({name, TRT_TensorOrWeights(tensor)}).second; } nvinfer1::ITensor* TransposeTensor(nvinfer1::ITensor* input_tensor, - std::vector order) { - auto dims = input_tensor->getDimensions(); + const std::vector& order) { + const auto dims = input_tensor->getDimensions(); // TODO(jie): change the return to status and properly exit if (order.size() - 1 != size_t(dims.nbDims)) @@ -705,11 +717,12 @@ TRT_ShapedWeights ConvertFP32ToFP16(Converter& ctx, } return weights; } + // **************************************************************************** // Constant folding functions // TODO(jie): once optimizer kicks in, we should have done constant folding // there. -//*****************************************************************************/ +// ***************************************************************************** struct LambdaFactory { enum class OP_CATEGORY : int { RSQRT = 0, NEG, ADD, MUL, SUB, RECIP }; OP_CATEGORY op; @@ -758,7 +771,6 @@ struct LambdaFactory { VLOG(2) << "LAMBDA VAL : " << val; return l + val; }; - // Return [val](T l)-> T {return l+val;}; case OP_CATEGORY::SUB: return [val](T l) -> T { VLOG(2) << "LAMBDA VAL : " << val; @@ -818,11 +830,13 @@ std::function LambdaFactory::unary() { } case OP_CATEGORY::NEG: return [](Eigen::half t) -> Eigen::half { return -t; }; + // TODO(aaroey): can we support RECIP? default: VLOG(2) << "Not supported op for unary: " << static_cast(op); return nullptr; } } + tensorflow::Status UnaryCompute(const TRT_ShapedWeights& iweights, TRT_ShapedWeights* oweights, LambdaFactory unary_op) { @@ -868,6 +882,7 @@ tensorflow::Status BinaryCompute(const TRT_ShapedWeights& iweights_l, if (iweights_l.count() != iweights_r.count()) { // We only supports broadcast of RankZero if (iweights_l.count() == 1) { + // TODO(aaroey): Remove loggings like this. VLOG(2) << "I bet it is not working!" << (*inp_l); std::transform(inp_r, inp_r + iweights_r.count(), oup, binary_op.broadcast_l(*inp_l)); @@ -928,6 +943,7 @@ tensorflow::Status BinaryTensorOpWeight( bool swapped_inputs, std::vector* outputs) { // tensor is the left operand while weights is the right operand; // when swapped_inputs set to true, those two are swapped. + // TODO(aaroey): use a set. if (node_def.op() != "Sub" && node_def.op() != "Add" && node_def.op() != "Mul" && node_def.op() != "Div" && node_def.op() != "RealDiv") { @@ -1115,11 +1131,10 @@ tensorflow::Status ConvertConv2DHelper( } // tensor after transpose (NCHW) - auto tensor_dim = tensor->getDimensions(); + const auto tensor_dim = tensor->getDimensions(); int num_groups = group; - if (num_groups == 0) // depthwise convolution - num_groups = tensor_dim.d[0]; + if (num_groups == 0) num_groups = tensor_dim.d[0]; // depthwise convolution VLOG(2) << "groups count: " << num_groups; TRT_ShapedWeights weights_rsck = inputs.at(1).weights(); @@ -1141,7 +1156,7 @@ tensorflow::Status ConvertConv2DHelper( TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck); ReorderRSCKToKCRS(weights_rsck, &weights, num_groups); TRT_ShapedWeights biases(weights.type_); - int noutput = weights.shape_.d[0] * num_groups; + const int noutput = weights.shape_.d[0] * num_groups; nvinfer1::DimsHW kernel_size; kernel_size.h() = weights.shape_.d[2]; kernel_size.w() = weights.shape_.d[3]; @@ -1152,11 +1167,11 @@ tensorflow::Status ConvertConv2DHelper( VLOG(2) << "kernel size: " << kernel_size.h() << ", " << kernel_size.w(); // TODO(jie): stride. (NHWC/NCHW) - auto tf_stride = attrs.get>("strides"); + const auto tf_stride = attrs.get>("strides"); VLOG(2) << "h_INDEX" << h_index << ", w_index " << w_index; VLOG(2) << "stride!!!: " << tf_stride[0] << tf_stride[1] << tf_stride[2] << tf_stride[3]; - nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]); + const nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]); std::vector> padding; // TODO(jie): padding. @@ -1304,15 +1319,16 @@ tensorflow::Status BinaryTensorOpTensor( nvinfer1::DataType dtype = attrs.get("T"); // check type consistency - CHECK_EQ_TYPE(tensor_l->getType(), dtype); - CHECK_EQ_TYPE(tensor_r->getType(), dtype); + TFTRT_CHECK_EQ_TYPE(tensor_l->getType(), dtype); + TFTRT_CHECK_EQ_TYPE(tensor_r->getType(), dtype); auto op_pair = ops.find(node_def.op()); - if (op_pair == ops.end()) - return tensorflow::errors::Unimplemented("binary op: " + node_def.op() + - " not supported at: " + - node_def.name()); + if (op_pair == ops.end()) { + return tensorflow::errors::Unimplemented( + "binary op: ", node_def.op(), " not supported at: ", node_def.name()); + } nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise( + // TODO(aaroey): will tensor_l/tensor_r get modified? *const_cast(tensor_l), *const_cast(tensor_r), op_pair->second); TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); @@ -1342,7 +1358,7 @@ tensorflow::Status ConvertPlugin(Converter& ctx, // passing attributes // TODO(jie): support more general attribute TFAttrs attrs(node_def); - auto attr_key_vector = attrs.GetAllAttrKey(); + auto attr_key_vector = attrs.GetAllAttrKeys(); for (auto attr_key : attr_key_vector) { // TODO(jie): support only list of float for toy example here. auto data = attrs.get>(attr_key); @@ -1388,7 +1404,7 @@ tensorflow::Status ConvertPool(Converter& ctx, int h_index = 2; int w_index = 3; - auto data_format = attrs.get("data_format"); + const auto data_format = attrs.get("data_format"); if (data_format == "NHWC") { h_index = 1; w_index = 2; @@ -1398,33 +1414,36 @@ tensorflow::Status ConvertPool(Converter& ctx, } nvinfer1::PoolingType type; - if (node_def.op() == "MaxPool") + if (node_def.op() == "MaxPool") { type = nvinfer1::PoolingType::kMAX; - else if (node_def.op() == "AvgPool") + } else if (node_def.op() == "AvgPool") { type = nvinfer1::PoolingType::kAVERAGE; - else - return tensorflow::errors::Unimplemented("Only supports Max pool"); + } else { + return tensorflow::errors::Unimplemented( + "Unsupported pool type: ", node_def.op()); + } - auto tf_stride = attrs.get>("strides"); - nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]); + const auto tf_stride = attrs.get>("strides"); + const nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]); - auto tf_kernel = attrs.get>("ksize"); - nvinfer1::DimsHW ksize(tf_kernel[h_index], tf_kernel[w_index]); + const auto tf_kernel = attrs.get>("ksize"); + const nvinfer1::DimsHW ksize(tf_kernel[h_index], tf_kernel[w_index]); auto tensor_dim = tensor->getDimensions(); std::vector> padding; - if (attrs.get("padding") == "SAME") { + const string padding_type = attrs.get("padding"); + if (padding_type == "SAME") { // This is NCHW tensor with no batch dimension. // 1 -> h // 2 -> w padding = CreateSamePadding( stride, ksize, {static_cast(tensor_dim.d[1]), static_cast(tensor_dim.d[2])}); - } else if (attrs.get("padding") == "VALID") { + } else if (padding_type == "VALID") { padding = {{0, 0}, {0, 0}}; } else { return tensorflow::errors::Unimplemented( - "Current MaxPool cannot support padding other than SAME"); + "Unsupported padding type: ", padding_type); } if (padding[0].first != padding[0].second || @@ -1475,24 +1494,23 @@ tensorflow::Status ConvertScale(Converter& ctx, const std::vector& inputs, std::vector* outputs) { if (inputs.size() != 2 || !inputs.at(0).is_tensor() || - !inputs.at(1).is_weights()) + !inputs.at(1).is_weights()) { return tensorflow::errors::Unimplemented( - "Only supports tensor op weight for now, at " + node_def.name()); + "ConvertScale only supports tensorweight: ", node_def.name()); + } const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); - TRT_ShapedWeights weights = inputs.at(1).weights(); if (ctx.isFP16()) { weights = ConvertFP32ToFP16(ctx, inputs.at(1).weights()); } TRT_ShapedWeights empty_weights(weights.type_); - TFAttrs attrs(node_def); - auto data_format = attrs.get("data_format"); + const auto data_format = attrs.get("data_format"); int channel_index; - auto dims = tensor->getDimensions(); + const auto dims = tensor->getDimensions(); if (data_format == "NHWC") { // 1). NHWC is really N+C channel_index = dims.nbDims - 1; // batch dimension is implicit here! @@ -1511,7 +1529,7 @@ tensorflow::Status ConvertScale(Converter& ctx, permutation.order[channel_index] = 0; } else { return tensorflow::errors::Unimplemented( - "TFTRT::BiasAdd cannot apply on batch dimension, at " + + "TFTRT::BiasAdd cannot apply on batch dimension, at ", node_def.name()); } @@ -1526,8 +1544,8 @@ tensorflow::Status ConvertScale(Converter& ctx, reshape_dims.d[0] = 0; // 0 copy from the input reshape_dims.d[1] = dims.nbDims >= 2 ? 0 : 1; // 0 copy from the input reshape_dims.d[2] = dims.nbDims >= 3 ? -1 : 1; // -1 infer from the rest - if (channel_index != 0) { // maybe we do not need this check. concerned - // about TRT optimization + if (channel_index != 0) { + // maybe we do not need this check. concerned about TRT optimization shuffle_layer->setFirstTranspose(permutation); } shuffle_layer->setReshapeDimensions(reshape_dims); @@ -1578,11 +1596,13 @@ tensorflow::Status ConvertConst(Converter& ctx, // Create shaped weights as output tensorflow::Tensor tensor; - if (!tensor.FromProto(weights_tensor)) - return tensorflow::errors::Internal("Cannot parse weight tensor proto: " + + if (!tensor.FromProto(weights_tensor)) { + return tensorflow::errors::Internal("Cannot parse weight tensor proto: ", node_def.name()); + } TRT_ShapedWeights weights(dtype); + // TODO(aaroey): we should choose the array using dtype and shape. if (!weights_tensor.float_val().empty()) { VLOG(2) << "SCALAR!!!" << node_def.name(); nvinfer1::Dims scalar_shape; @@ -1595,8 +1615,8 @@ tensorflow::Status ConvertConst(Converter& ctx, VLOG(2) << scalar_shape.d[i]; if (GetShapeSize(scalar_shape) != weights_tensor.float_val_size() && weights_tensor.float_val_size() != 1) { - LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and" - << " kUNIFORM, at: " << node_def.name(); + LOG(ERROR) << "Broadcast on weights only supports kCHANNEL and" + << " kUNIFORM, at: " << node_def.name(); string err_str("Broadcast method is not supported for '"); StrAppend(&err_str, node_def.name(), "' of type ", node_def.op()); return tensorflow::errors::InvalidArgument(err_str); @@ -1611,6 +1631,7 @@ tensorflow::Status ConvertConst(Converter& ctx, scalar_shape.d[i] = 0; } } + // TODO(aaroey): use GetShapeSize(). size_t len_data = tensorflow::DataTypeSize(dtype); for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i]; ctx.weight_store()->store_.push_back(std::vector(len_data)); @@ -1619,6 +1640,7 @@ tensorflow::Status ConvertConst(Converter& ctx, std::fill_n((float*)dst, GetShapeSize(scalar_shape), *weights_tensor.float_val().begin()); } else { + // TODO(aaroey): get rid of this copy as RepeatedField is always contigous // make a local copy first to flatten doesn't have to be contigous std::vector tensor_data( weights_tensor.float_val().begin(), @@ -1629,6 +1651,8 @@ tensorflow::Status ConvertConst(Converter& ctx, for (int i = 0; i < scalar_shape.nbDims; i++) VLOG(2) << scalar_shape.d[i]; weights = TRT_ShapedWeights(dtype, dst, scalar_shape); } else if (!weights_tensor.int_val().empty()) { + // TODO(aaroey): this is very similar to the above code for float, merge + // them. VLOG(2) << "int!!!" << node_def.name(); nvinfer1::Dims scalar_shape; if (tensor.dims() > 0) { @@ -1653,7 +1677,7 @@ tensorflow::Status ConvertConst(Converter& ctx, scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL; } } - // we should not have converted //if (ctx.isFP16()) { + // we should not have converted size_t len_data = tensorflow::DataTypeSize(dtype); for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i]; size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32); @@ -1664,7 +1688,8 @@ tensorflow::Status ConvertConst(Converter& ctx, std::fill_n((int*)dst, GetShapeSize(scalar_shape), *weights_tensor.int_val().begin()); } else { - // make a local copy first to flatten doesn't have to be contigous + // TODO(aaroey): get rid of this copy as RepeatedField is always contigous + // make a local copy first to flatten doesn't have to be contigous std::vector tensor_data( weights_tensor.int_val().begin(), weights_tensor.int_val().end()); @@ -1672,9 +1697,10 @@ tensorflow::Status ConvertConst(Converter& ctx, } weights = TRT_ShapedWeights(dtype, dst, scalar_shape); } else if (!weights_tensor.tensor_content().empty()) { - // obsolete method. - // After optimization path, we do not see weights in this format. - // fp16 conversion technically should be needed here. + // obsolete method. + // After optimization path, we do not see weights in this format. + // TODO(aaroey): why? + // fp16 conversion technically should be needed here. VLOG(2) << "TENSOR!!!" << node_def.name(); const auto& content = weights_tensor.tensor_content(); @@ -1689,7 +1715,7 @@ tensorflow::Status ConvertConst(Converter& ctx, } } else { return tensorflow::errors::Unimplemented( - "Not supported constant type, at " + node_def.name()); + "Not supported constant type, at ", node_def.name()); } // Pass the output outputs->push_back(TRT_TensorOrWeights(weights)); @@ -1708,17 +1734,19 @@ tensorflow::Status ConvertBinary(Converter& ctx, const tensorflow::NodeDef& node_def, const std::vector& inputs, std::vector* outputs) { - if (inputs.size() != 2) + if (inputs.size() != 2) { return tensorflow::errors::FailedPrecondition( - "Binary ops require two tensor input, at " + node_def.name()); + "Binary ops require two tensor input, at ", node_def.name()); + } // Constant folding should have been done by TensorFlow - if (inputs.at(0).is_weights() && inputs.at(1).is_weights()) + if (inputs.at(0).is_weights() && inputs.at(1).is_weights()) { return tensorflow::errors::Unimplemented( "Constant folding is falled back to TensorFlow, binary op received " - "both input as constant at: " + + "both input as constant at: ", node_def.name()); + } // Try to convert into Scale layer first (for better performance) // Since scale layer supports restricted broadcast policy and op types, we @@ -1752,6 +1780,7 @@ tensorflow::Status ConvertBinary(Converter& ctx, #if NV_TENSORRT_MAJOR == 3 } #endif + return tensorflow::Status::OK(); } tensorflow::Status ConvertUnary(Converter& ctx, @@ -1767,14 +1796,16 @@ tensorflow::Status ConvertUnary(Converter& ctx, {"Reciprocal", nvinfer1::UnaryOperation::kRECIP}, }; - if (inputs.size() != 1) + if (inputs.size() != 1) { return tensorflow::errors::FailedPrecondition( - "Unary ops require single tensor input, at " + node_def.name()); + "Unary ops require single tensor input, at ", node_def.name()); + } #if NV_TENSORRT_MAJOR == 3 - if (inputs.at(0).is_weights()) + if (inputs.at(0).is_weights()) { return tensorflow::errors::Unimplemented( - "Constant folding for unary op is not supported" + node_def.name()); + "Constant folding for unary op is not supported", node_def.name()); + } #endif // TODO(jie): check type @@ -1795,9 +1826,8 @@ tensorflow::Status ConvertUnary(Converter& ctx, layer = ctx.network()->addUnary(*const_cast(tensor), ops.at(node_def.op())); } else { - return tensorflow::errors::InvalidArgument("Binary op: " + node_def.op() + - " not supported, at " + - node_def.name()); + return tensorflow::errors::InvalidArgument( + "Binary op: ", node_def.op(), " not supported, at ", node_def.name()); } TFTRT_RETURN_ERROR_IF_NULLPTR(layer, node_def.name()); @@ -1812,45 +1842,48 @@ tensorflow::Status ConvertReducePool( const std::vector& inputs, std::vector* outputs) { if (inputs.size() != 2 || !inputs.at(0).is_tensor() || - !inputs.at(1).is_weights()) + !inputs.at(1).is_weights()) { return tensorflow::errors::InvalidArgument( - "Input expects tensor and weights, at" + node_def.name()); + "Input expects tensor and weights, at", node_def.name()); + } // Implement tensor binaryOp weight [channel wise] for now; const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); - auto dims = tensor->getDimensions(); + const auto dims = tensor->getDimensions(); // Restore implicit batch dimension - int nb_dims = dims.nbDims + 1; + const int nb_dims = dims.nbDims + 1; TRT_ShapedWeights index_list = inputs.at(1).weights(); - TFAttrs attrs(node_def); auto index_type = attrs.get("Tidx"); // Only expect to handle INT32 as attributes for now - if (index_type != tensorflow::DataType::DT_INT32) + if (index_type != tensorflow::DataType::DT_INT32) { return tensorflow::errors::Unimplemented("Tidx supports only DT_INT32"); - auto index_list_data = + } + const auto index_list_data = static_cast(const_cast(index_list.GetValues())); - if (nb_dims != 4) + if (nb_dims != 4) { return tensorflow::errors::InvalidArgument( - "TRT only support reduce on 4 dimensional tensors, at" + + "TRT only support reduce on 4 dimensional tensors, at", node_def.name()); - if (index_list.count() > 2) + } + if (index_list.count() > 2) { return tensorflow::errors::InvalidArgument( - "TRT cannot support reduce on more than 2 dimensions, at" + + "TRT cannot support reduce on more than 2 dimensions, at", node_def.name()); + } std::set idx_set; // We cannot operate on Channel. permutation flag used to transpose tensor int permuted_index = -1; for (int i = 0; i < index_list.count(); i++) { - if (index_list_data[i] == 0) - return tensorflow::errors::InvalidArgument("TRT cannot reduce at 0, at" + + if (index_list_data[i] == 0) { + return tensorflow::errors::InvalidArgument("TRT cannot reduce at 0, at", node_def.name()); + } if (index_list_data[i] == 1) permuted_index = 1; - idx_set.emplace(index_list_data[i]); } @@ -1888,7 +1921,7 @@ tensorflow::Status ConvertReducePool( output_tensor = layer->getOutput(0); } else { return tensorflow::errors::Unimplemented( - "Op not supported " + node_def.op() + " , at " + node_def.name()); + "Op not supported ", node_def.op(), " , at ", node_def.name()); } if (permuted_index != -1) { // Apply permutation before extracting dimension for pool_kernel @@ -1907,37 +1940,34 @@ tensorflow::Status ConvertReduce(Converter& ctx, if (inputs.size() != 2 || !inputs.at(0).is_tensor() || !inputs.at(1).is_weights()) { return tensorflow::errors::InvalidArgument( - "Input expects tensor and weights, at" + node_def.name()); + "Input expects tensor and weights, at", node_def.name()); } const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); - auto dims = tensor->getDimensions(); - int nb_dims = dims.nbDims + 1; - TRT_ShapedWeights index_list = inputs.at(1).weights(); TFAttrs attrs(node_def); auto index_type = attrs.get("Tidx"); // Only expect to handle INT32 as attributes for now - if (index_type != tensorflow::DataType::DT_INT32) + if (index_type != tensorflow::DataType::DT_INT32) { return tensorflow::errors::Unimplemented("Tidx supports only DT_INT32"); + } - auto keep_dims = attrs.get("keep_dims"); - + const auto keep_dims = attrs.get("keep_dims"); auto index_list_data = static_cast(const_cast(index_list.GetValues())); int axes = 0; if (index_list.count() == 0) { return tensorflow::errors::InvalidArgument( - "TRT cannot support reduce on all (batch) dimensions, at" + + "TRT cannot support reduce on all (batch) dimensions, at", node_def.name()); } else { for (int i = 0; i < index_list.count(); i++) { if (index_list_data[i] == 0) { return tensorflow::errors::InvalidArgument( - "TRT cannot reduce at batch dimension, at" + node_def.name()); + "TRT cannot reduce at batch dimension, at", node_def.name()); } axes |= (1 << (index_list_data[i] - 1)); } @@ -1956,7 +1986,7 @@ tensorflow::Status ConvertReduce(Converter& ctx, reduce_operation = nvinfer1::ReduceOperation::kAVG; } else { return tensorflow::errors::Unimplemented( - "Op not supported " + node_def.op() + " , at " + node_def.name()); + "Op not supported ", node_def.op(), " , at ", node_def.name()); } nvinfer1::ILayer* layer = @@ -1973,16 +2003,18 @@ tensorflow::Status ConvertPad(Converter& ctx, const tensorflow::NodeDef& node_def, const std::vector& inputs, std::vector* outputs) { + // TODO(aaroey): make a routine for this check and reuse it. if (inputs.size() != 2 || !inputs.at(0).is_tensor() || - !inputs.at(1).is_weights()) + !inputs.at(1).is_weights()) { return tensorflow::errors::InvalidArgument( - "Input expects tensor and weights, at" + node_def.name()); + "Input expects tensor and weights, at", node_def.name()); + } // Implement tensor binaryOp weight [channel wise] for now; const nvinfer1::ITensor* tensor = inputs.at(0).tensor(); - auto dims = tensor->getDimensions(); + const auto dims = tensor->getDimensions(); // Restore implicit batch dimension - int nb_dims = dims.nbDims + 1; + const int nb_dims = dims.nbDims + 1; TRT_ShapedWeights pads = inputs.at(1).weights(); @@ -1992,21 +2024,24 @@ tensorflow::Status ConvertPad(Converter& ctx, auto padding_type = attrs.get("Tpaddings"); // TODO(jie): handle data type conversion for TRT? - if (pads.shape_.d[0] != nb_dims || pads.shape_.d[1] != 2) + if (pads.shape_.d[0] != nb_dims || pads.shape_.d[1] != 2) { return tensorflow::errors::InvalidArgument( - "Pad only supports explicit padding on 4 dimensional tensor, at " + + "Pad only supports explicit padding on 4 dimensional tensor, at ", node_def.name()); + } // Only expect to handle INT32 as attributes for now - if (padding_type != tensorflow::DataType::DT_INT32) + if (padding_type != tensorflow::DataType::DT_INT32) { return tensorflow::errors::Unimplemented( "Tpaddings supports only DT_INT32"); + } auto pad_data = static_cast(const_cast(pads.GetValues())); std::vector pad_index; for (int i = 0; i < nb_dims; i++) { - if (pad_data[2 * i] != 0 || pad_data[2 * i + 1] != 0) + if (pad_data[2 * i] != 0 || pad_data[2 * i + 1] != 0) { pad_index.push_back(i); + } } // No padding at all, we should exit @@ -2016,20 +2051,23 @@ tensorflow::Status ConvertPad(Converter& ctx, } // Only supports padding on less than 2 axis GIE-2579 - if (pad_index.size() > 2) + if (pad_index.size() > 2) { return tensorflow::errors::InvalidArgument( "Padding layer does not support padding on > 2"); + } // Padding on batch dimension is not supported - if (pad_index[0] == 0) + if (pad_index[0] == 0) { return tensorflow::errors::InvalidArgument( "Padding layer does not support padding on batch dimension"); + } // Not doing the legit thing here. ignoring padding on dim 1 and 3; // TODO(jie): implement pad as uff parser - if (pad_index.size() == 2 && pad_index[0] == 0 && pad_index[1] == 3) + if (pad_index.size() == 2 && pad_index[0] == 0 && pad_index[1] == 3) { return tensorflow::errors::Unimplemented( "Padding layer does not support padding on dimension 1 and 3 yet"); + } bool legit_pad = true; nvinfer1::DimsHW pre_padding(0, 0); @@ -2077,9 +2115,10 @@ tensorflow::Status ConvertConcat(Converter& ctx, // not including the last input (axis) here int input_size = static_cast(inputs.size()) - 1; - if (!inputs.at(0).is_tensor()) + if (!inputs.at(0).is_tensor()) { return tensorflow::errors::InvalidArgument( - "Concat in TRT support only Tensor input, at " + node_def.name()); + "Concat in TRT support only Tensor input, at ", node_def.name()); + } // We are retrieving the axis TRT_ShapedWeights axis = inputs.at(input_size).weights(); @@ -2091,7 +2130,7 @@ tensorflow::Status ConvertConcat(Converter& ctx, // Only expect to handle INT32 as index attributes for now if (index_type != tensorflow::DataType::DT_INT32) return tensorflow::errors::Unimplemented( - "Tidx supports only DT_INT32, at " + node_def.name()); + "Tidx supports only DT_INT32, at ", node_def.name()); int index = *(static_cast(const_cast(axis.GetValues()))); @@ -2099,14 +2138,14 @@ tensorflow::Status ConvertConcat(Converter& ctx, auto dim = inputs.at(0).tensor()->getDimensions(); // dimension check - if (index > dim.nbDims + 1) + if (index > dim.nbDims + 1) { return tensorflow::errors::InvalidArgument( - "Concatenate on axis out of dimension range, at " + node_def.name()); - - if (index == 0) + "Concatenate on axis out of dimension range, at ", node_def.name()); + } + if (index == 0) { return tensorflow::errors::InvalidArgument( - "Concatenate on batch dimension not supported, at " + node_def.name()); - + "Concatenate on batch dimension not supported, at ", node_def.name()); + } if (index < 0) { index = dim.nbDims + index + 1; } @@ -2129,17 +2168,18 @@ tensorflow::Status ConvertConcat(Converter& ctx, for (int i = 0; i < input_size; i++) { auto tensor_i = inputs.at(i).tensor(); auto dim_i = tensor_i->getDimensions(); - if (dim_i.nbDims != dim.nbDims) + if (dim_i.nbDims != dim.nbDims) { return tensorflow::errors::InvalidArgument( - "Concatenate receives inputs with inconsistent dimensions, at " + + "Concatenate receives inputs with inconsistent dimensions, at ", node_def.name()); - + } for (int j = 0; j < dim.nbDims; j++) { // check dimension consistency on non-concatenate axis - if (j != index - 1 && dim_i.d[j] != dim.d[j]) + if (j != index - 1 && dim_i.d[j] != dim.d[j]) { return tensorflow::errors::InvalidArgument( - "Concatenate receives inputs with inconsistent shape, at" + + "Concatenate receives inputs with inconsistent shape, at", node_def.name()); + } } #if NV_TENSORRT_MAJOR == 3 -- cgit v1.2.3 From 123033191cff00721ab59ff5a052463edb3b97bb Mon Sep 17 00:00:00 2001 From: mktozk Date: Wed, 11 Jul 2018 07:35:57 +0900 Subject: Fix test_custom_objects --- tensorflow/python/estimator/keras_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/python/estimator/keras_test.py b/tensorflow/python/estimator/keras_test.py index 788219378d..6774355561 100644 --- a/tensorflow/python/estimator/keras_test.py +++ b/tensorflow/python/estimator/keras_test.py @@ -480,6 +480,12 @@ class TestKerasEstimator(test_util.TensorFlowTestCase): 'DepthwiseConv2D': mobilenet.DepthwiseConv2D } + with self.assertRaisesRegexp(ValueError, 'DepthwiseConv2D'): + with self.test_session(): + keras_lib.model_to_estimator( + keras_model=keras_mobile, + model_dir=tempfile.mkdtemp(dir=self._base_dir)) + with self.test_session(): keras_lib.model_to_estimator( keras_model=keras_mobile, -- cgit v1.2.3 From d3b4945e6de5343bcf7c37507e9a1d1ff8d9e496 Mon Sep 17 00:00:00 2001 From: Mahmoud Abuzaina Date: Tue, 10 Jul 2018 17:35:27 -0700 Subject: Added missing file --- third_party/mkl_dnn/build_defs.bzl | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 third_party/mkl_dnn/build_defs.bzl diff --git a/third_party/mkl_dnn/build_defs.bzl b/third_party/mkl_dnn/build_defs.bzl new file mode 100644 index 0000000000..108d82e683 --- /dev/null +++ b/third_party/mkl_dnn/build_defs.bzl @@ -0,0 +1,13 @@ +def if_mkl_open_source_only(if_true, if_false = []): + """Shorthand for select()'ing on whether we're building with + MKL-DNN open source lib only, without depending on MKL binary form. + + Returns a select statement which evaluates to if_true if we're building + with MKL-DNN open source lib only. Otherwise, + the select statement evaluates to if_false. + + """ + return select({ + str(Label("//third_party/mkl_dnn:using_mkl_dnn_only")): if_true, + "//conditions:default": if_false + }) \ No newline at end of file -- cgit v1.2.3 From 998e6aa7741dba78b3240a140de18674f61c669b Mon Sep 17 00:00:00 2001 From: SneakyFish5 <32284796+SneakyFish5@users.noreply.github.com> Date: Tue, 10 Jul 2018 21:25:49 -0500 Subject: Fix a couple typos --- tensorflow/contrib/copy_graph/python/util/copy_elements.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/copy_graph/python/util/copy_elements.py b/tensorflow/contrib/copy_graph/python/util/copy_elements.py index a0dd3881a8..5931c8a279 100644 --- a/tensorflow/contrib/copy_graph/python/util/copy_elements.py +++ b/tensorflow/contrib/copy_graph/python/util/copy_elements.py @@ -18,7 +18,7 @@ These functions allow for recursive copying of elements (ops and variables) from one graph to another. The copied elements are initialized inside a user-specified scope in the other graph. There are separate functions to copy ops and variables. -There is also a function to retrive the copied version of an op from the +There is also a function to retrieve the copied version of an op from the first graph inside a scope in the second graph. @@copy_op_to_graph @@ -77,7 +77,7 @@ def copy_variable_to_graph(org_instance, to_graph, scope=''): else: collections.append(scope + '/' + name) - #See if its trainable. + #See if it's trainable. trainable = ( org_instance in org_instance.graph.get_collection( ops.GraphKeys.TRAINABLE_VARIABLES)) @@ -162,7 +162,7 @@ def copy_op_to_graph(org_instance, to_graph, variables, scope=''): if isinstance(org_instance, ops.Tensor): - #If its a Tensor, it is one of the outputs of the underlying + #If it's a Tensor, it is one of the outputs of the underlying #op. Therefore, copy the op itself and return the appropriate #output. op = org_instance.op -- cgit v1.2.3 From c668ad5c222f8699bd298a04f4eb612ce8a7f8d0 Mon Sep 17 00:00:00 2001 From: mktozk Date: Wed, 11 Jul 2018 12:44:14 +0900 Subject: Update test_custom_objects --- tensorflow/python/estimator/keras_test.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tensorflow/python/estimator/keras_test.py b/tensorflow/python/estimator/keras_test.py index 6774355561..7a3c5a9bf1 100644 --- a/tensorflow/python/estimator/keras_test.py +++ b/tensorflow/python/estimator/keras_test.py @@ -32,7 +32,6 @@ from tensorflow.python.estimator.inputs import numpy_io from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras import testing_utils -from tensorflow.python.keras.applications import mobilenet from tensorflow.python.keras.optimizers import SGD from tensorflow.python.ops.parsing_ops import gen_parsing_ops from tensorflow.python.platform import gfile @@ -60,9 +59,9 @@ def simple_sequential_model(): return model -def simple_functional_model(): +def simple_functional_model(activation='relu'): a = keras.layers.Input(shape=_INPUT_SIZE) - b = keras.layers.Dense(16, activation='relu')(a) + b = keras.layers.Dense(16, activation=activation)(a) b = keras.layers.Dropout(0.1)(b) b = keras.layers.Dense(_NUM_CLASS, activation='softmax')(b) model = keras.models.Model(inputs=[a], outputs=[b]) @@ -474,21 +473,25 @@ class TestKerasEstimator(test_util.TensorFlowTestCase): est_keras.train(input_fn=invald_output_name_input_fn, steps=100) def test_custom_objects(self): - keras_mobile = mobilenet.MobileNet(weights=None) - keras_mobile.compile(loss='categorical_crossentropy', optimizer='adam') + + def relu6(x): + return keras.backend.relu(x, max_value=6) + + keras_model = simple_functional_model(activation=relu6) + keras_model.compile(loss='categorical_crossentropy', optimizer='adam') custom_objects = { - 'DepthwiseConv2D': mobilenet.DepthwiseConv2D + 'relu6': relu6 } - with self.assertRaisesRegexp(ValueError, 'DepthwiseConv2D'): + with self.assertRaisesRegexp(ValueError, 'relu6'): with self.test_session(): keras_lib.model_to_estimator( - keras_model=keras_mobile, + keras_model=keras_model, model_dir=tempfile.mkdtemp(dir=self._base_dir)) with self.test_session(): keras_lib.model_to_estimator( - keras_model=keras_mobile, + keras_model=keras_model, model_dir=tempfile.mkdtemp(dir=self._base_dir), custom_objects=custom_objects) -- cgit v1.2.3 From 5351dbef668f0bd542fb0b1f3e2e9288972db69a Mon Sep 17 00:00:00 2001 From: Jie Date: Wed, 11 Jul 2018 01:40:27 -0700 Subject: addressing review comments --- tensorflow/contrib/tensorrt/BUILD | 30 ++-- .../contrib/tensorrt/convert/convert_nodes.cc | 67 +++----- tensorflow/contrib/tensorrt/test/base_unit_test.py | 118 +++++++++++++ .../contrib/tensorrt/test/batch_matmul_test.py | 97 +++++++++++ .../contrib/tensorrt/test/biasadd_matmul_test.py | 116 +++++++++++++ .../test/binary_tensor_weight_broadcast_test.py | 148 ++++++++++++++++ .../contrib/tensorrt/test/concatenation_test.py | 87 ++++++++++ .../contrib/tensorrt/test/const_broadcast_test.py | 75 ++++++++ .../test/multi_connection_neighbor_engine_test.py | 101 +++++++++++ .../tensorrt/test/neighboring_engine_test.py | 78 +++++++++ tensorflow/contrib/tensorrt/test/run_test.py | 184 ++++++++++++++++++++ .../tensorrt/test/tf_trt_integration_test.py | 16 +- tensorflow/contrib/tensorrt/test/unary_test.py | 125 ++++++++++++++ tensorflow/contrib/tensorrt/test/unit_tests.py | 67 ++++++++ .../tensorrt/test/unit_tests/BatchMatMulTest.py | 106 ------------ .../tensorrt/test/unit_tests/BiasaddMatMulTest.py | 129 -------------- .../unit_tests/BinaryTensorWeightBroadcastTest.py | 162 ----------------- .../tensorrt/test/unit_tests/ConcatenationTest.py | 101 ----------- .../tensorrt/test/unit_tests/ConstBroadcastTest.py | 91 ---------- .../MultiConnectionNeighborEngineTest.py | 114 ------------ .../test/unit_tests/NeighboringEngineTest.py | 92 ---------- .../contrib/tensorrt/test/unit_tests/UnaryTest.py | 133 -------------- .../tensorrt/test/unit_tests/VGGBlockNCHWTest.py | 98 ----------- .../tensorrt/test/unit_tests/VGGBlockTest.py | 89 ---------- .../tensorrt/test/unit_tests/base_unit_test.py | 131 -------------- .../contrib/tensorrt/test/unit_tests/run_test.py | 191 --------------------- .../contrib/tensorrt/test/unit_tests/unit_tests.py | 69 -------- .../contrib/tensorrt/test/unit_tests/utilities.py | 31 ---- tensorflow/contrib/tensorrt/test/utilities.py | 30 ++++ .../contrib/tensorrt/test/vgg_block_nchw_test.py | 85 +++++++++ tensorflow/contrib/tensorrt/test/vgg_block_test.py | 76 ++++++++ 31 files changed, 1435 insertions(+), 1602 deletions(-) create mode 100644 tensorflow/contrib/tensorrt/test/base_unit_test.py create mode 100644 tensorflow/contrib/tensorrt/test/batch_matmul_test.py create mode 100644 tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py create mode 100644 tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py create mode 100644 tensorflow/contrib/tensorrt/test/concatenation_test.py create mode 100644 tensorflow/contrib/tensorrt/test/const_broadcast_test.py create mode 100644 tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py create mode 100644 tensorflow/contrib/tensorrt/test/neighboring_engine_test.py create mode 100644 tensorflow/contrib/tensorrt/test/run_test.py create mode 100644 tensorflow/contrib/tensorrt/test/unary_test.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/run_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests/utilities.py create mode 100644 tensorflow/contrib/tensorrt/test/utilities.py create mode 100644 tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py create mode 100644 tensorflow/contrib/tensorrt/test/vgg_block_test.py diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index 4de0cf9d18..d957ca0861 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -344,22 +344,22 @@ py_test( py_test( name = "converter_unit_tests", srcs = [ - "test/unit_tests/base_unit_test.py", - "test/unit_tests/BatchMatMulTest.py", - "test/unit_tests/BiasaddMatMulTest.py", - "test/unit_tests/BinaryTensorWeightBroadcastTest.py", - "test/unit_tests/ConcatenationTest.py", - "test/unit_tests/ConstBroadcastTest.py", - "test/unit_tests/MultiConnectionNeighborEngineTest.py", - "test/unit_tests/NeighboringEngineTest.py", - "test/unit_tests/run_test.py", - "test/unit_tests/UnaryTest.py", - "test/unit_tests/unit_tests.py", - "test/unit_tests/utilities.py", - "test/unit_tests/VGGBlockNCHWTest.py", - "test/unit_tests/VGGBlockTest.py", + "test/base_unit_test.py", + "test/batch_matmul_test.py", + "test/biasadd_matmul_test.py", + "test/binary_tensor_weight_broadcast_test.py", + "test/concatenation_test.py", + "test/const_broadcast_test.py", + "test/multi_connection_neighbor_engine_test.py", + "test/neighboring_engine_test.py", + "test/run_test.py", + "test/unary_test.py", + "test/unit_tests.py", + "test/utilities.py", + "test/vgg_block_nchw_test.py", + "test/vgg_block_test.py", ], - main = "test/unit_tests/unit_tests.py", + main = "test/unit_tests.py", srcs_version = "PY2AND3", tags = [ "manual", diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index e19adfacad..1df4dccc69 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -53,33 +53,26 @@ limitations under the License. // would work! #define CHECK_EQ_TYPE(val1, val2) CHECK_EQ((int)val1, (int)val2) -#define TFTRT_RETURN_ERROR_IF_FALSE(ptr, node) \ - do { \ - if (ptr == false) { \ - return tensorflow::errors::Internal( \ - string("TFTRT::"), __FUNCTION__, \ - "failed to add TRT layer, at: ", \ - node); \ - } \ +#define TFTRT_INTERNAL_ERROR_AT_NODE(node) \ + do { \ + return tensorflow::errors::Internal( \ + "TFTRT::", __FUNCTION__, "failed to add TRT layer, at: ", node); \ + } while (0) + +#define TFTRT_RETURN_ERROR_IF_FALSE(status, node) \ + do { \ + if (status == false) { \ + TFTRT_INTERNAL_ERROR_AT_NODE(node); \ + } \ } while (0) #define TFTRT_RETURN_ERROR_IF_NULLPTR(ptr, node) \ do { \ if (ptr == nullptr) { \ - return tensorflow::errors::Internal( \ - string("TFTRT::"), __FUNCTION__, \ - "failed to add TRT layer, at: ", \ - node); \ + TFTRT_INTERNAL_ERROR_AT_NODE(node); \ } \ } while (0) -#define TFTRT_RETURN_IF_OK(status) \ - do { \ - if (status.ok()) { \ - return tensorflow::Status::OK(); \ - } \ - } while (0) - namespace tensorflow { namespace tensorrt { namespace convert { @@ -1714,44 +1707,34 @@ tensorflow::Status ConvertBinary(Converter& ctx, // Constant folding should have been done by TensorFlow - if (inputs.at(0).is_weights() && inputs.at(1).is_weights()) + if (inputs.at(0).is_weights() && inputs.at(1).is_weights()) { return tensorflow::errors::Unimplemented( "Constant folding is falled back to TensorFlow, binary op received " "both input as constant at: " + node_def.name()); + } // Try to convert into Scale layer first (for better performance) // Since scale layer supports restricted broadcast policy and op types, we // allow failure and try to handle it through Elementwise op // (BinaryTensorOpTensor) + Status status = tensorflow::Status::OK(); if (inputs.at(0).is_tensor() && inputs.at(1).is_weights()) { - auto status = BinaryTensorOpWeight(ctx, node_def, inputs.at(0).tensor(), - inputs.at(1).weights(), false, outputs); -#if NV_TENSORRT_MAJOR == 3 - TF_RETURN_IF_ERROR(status); -#else - TFTRT_RETURN_IF_OK(status); -#endif - } - - if (inputs.at(0).is_weights() && inputs.at(1).is_tensor()) { - auto status = BinaryTensorOpWeight(ctx, node_def, inputs.at(1).tensor(), - inputs.at(0).weights(), true, outputs); + status = BinaryTensorOpWeight(ctx, node_def, inputs.at(0).tensor(), + inputs.at(1).weights(), false, outputs); + } else if (inputs.at(0).is_weights() && inputs.at(1).is_tensor()) { + status = BinaryTensorOpWeight(ctx, node_def, inputs.at(1).tensor(), + inputs.at(0).weights(), true, outputs); #if NV_TENSORRT_MAJOR == 3 - TF_RETURN_IF_ERROR(status); + } else { #else - TFTRT_RETURN_IF_OK(status); -#endif } - -#if NV_TENSORRT_MAJOR == 3 - if (inputs.at(0).is_tensor() && inputs.at(1).is_tensor()) { + if (inputs.at(0).is_tensor() && inputs.at(1).is_tensor() || !status.ok()) { #endif - return BinaryTensorOpTensor(ctx, node_def, inputs.at(0), inputs.at(1), - outputs); -#if NV_TENSORRT_MAJOR == 3 + status = BinaryTensorOpTensor(ctx, node_def, inputs.at(0), inputs.at(1), + outputs); } -#endif + return status; } tensorflow::Status ConvertUnary(Converter& ctx, diff --git a/tensorflow/contrib/tensorrt/test/base_unit_test.py b/tensorflow/contrib/tensorrt/test/base_unit_test.py new file mode 100644 index 0000000000..8a6c648ab6 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/base_unit_test.py @@ -0,0 +1,118 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Base class to facilitate development of integration tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + + +class BaseUnitTest(object): + """Base class for unit tests in TF-TRT""" + + def __init__(self, log_file='log.txt'): + self.static_mode_list = {} + self.dynamic_mode_list = {} + self.dummy_input = None + self.get_network = None + self.expect_nb_nodes = None + self.test_name = None + self.log_file = log_file + self.ckpt = None + self.allclose_rtol = 0.01 + self.allclose_atol = 0.01 + self.allclose_equal_nan = True + # saves out graphdef + self.debug = False + # require node count check fail leads to test failure + self.check_node_count = False + + def run(self, run_test_context): + run_test_context.run_test(self.get_network, self.static_mode_list, + self.dynamic_mode_list, self.dummy_input, + self.ckpt) + return self.log_result(run_test_context) + + def log_result(self, run_test_result): + log = open(self.log_file, 'a') + log.write(("================= model: %s\n") % (self.test_name)) + + if self.debug: + open(self.test_name + "_native.pb", + 'wb').write(run_test_result.native_network.SerializeToString()) + all_success = True + if len(run_test_result.tftrt_conversion_flag) != 0: + log.write(" -- static_mode\n") + for static_mode in run_test_result.tftrt_conversion_flag: + if self.debug: + open(self.test_name + "_" + static_mode + ".pb", + 'wb').write(run_test_result.tftrt[static_mode].SerializeToString()) + log.write(" ----\n") + log.write((" mode: [%s]\n") % (static_mode)) + if run_test_result.tftrt_conversion_flag[static_mode]: + if run_test_result.tftrt_nb_nodes[static_mode] != self.expect_nb_nodes: + log.write( + ("[WARNING]: converted node number does not match (%d,%d,%d)!!!\n" + ) % (run_test_result.tftrt_nb_nodes[static_mode], + self.expect_nb_nodes, run_test_result.native_nb_nodes)) + if self.check_node_count: + all_success = False + + if np.array_equal(run_test_result.tftrt_result[static_mode], + run_test_result.native_result): + log.write(" output: equal\n") + elif np.allclose( + run_test_result.tftrt_result[static_mode], + run_test_result.native_result, + atol=self.allclose_atol, + rtol=self.allclose_rtol, + equal_nan=self.allclose_equal_nan): + log.write(" output: allclose\n") + else: + diff = run_test_result.tftrt_result[static_mode] - run_test_result.native_result + log.write("[ERROR]: output does not match!!!\n") + log.write("max diff: " + str(np.max(diff))) + log.write("\ntftrt:\n") + log.write(str(run_test_result.tftrt_result[static_mode])) + log.write("\nnative:\n") + log.write(str(run_test_result.native_result)) + log.write("\ndiff:\n") + log.write(str(diff)) + all_success = False + else: + log.write("[ERROR]: conversion failed!!!\n") + all_success = False + + if len(run_test_result.tftrt_dynamic_conversion_flag) != 0: + log.write(" -- dynamic_mode\n") + for dynamic_mode in run_test_result.tftrt_dynamic_conversion_flag: + log.write("\n ----\n") + log.write((" mode: [%s]\n") % (dynamic_mode)) + if run_test_result.tftrt_dynamic_conversion_flag[dynamic_mode]: + if np.array_equal(run_test_result.tftrt_dynamic_result[dynamic_mode], + run_test_result.native_result): + log.write(" output: equal\n") + elif np.allclose(run_test_result.tftrt_dynamic_result[dynamic_mode], + run_test_result.native_result): + log.write(" output: allclose\n") + else: + log.write("[ERROR]: output does not match!!!\n") + all_success = False + else: + log.write("[ERROR]: conversion failed!!!\n") + all_success = False + return all_success diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py new file mode 100644 index 0000000000..3c83a3a562 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py @@ -0,0 +1,97 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.client import session +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.utilities import get_all_variables + + +class BatchMatMulTest(BaseUnitTest): + """Testing BatchMatMul in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(BatchMatMulTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (12, 5, 8, 12) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.matmul_test + self.expect_nb_nodes = 16 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.ckpt = "./tmp.ckpt" + sess = session.Session() + + def matmul_test(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions() + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + b = constant_op.constant( + np.random.randn(12, 5, 12, 7), dtype=dtypes.float32) + x1 = math_ops.matmul(x, b) + b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32) + x1 = x1 + b + + var = variable_scope.get_variable( + "test", [12, 5, 12, 7], + dtype=dtypes.float32, + initializer=init_ops.truncated_normal_initializer) + x2 = math_ops.matmul(x, var) + b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32) + x2 = x2 * b + + var = variable_scope.get_variable( + "test2", [12, 84], + dtype=dtypes.float32, + initializer=init_ops.truncated_normal_initializer) + c = gen_array_ops.reshape(x, [12, 40, 12]) + b = gen_array_ops.reshape(var, [12, 12, 7]) + x3 = math_ops.matmul(c, b) + b = constant_op.constant(np.random.randn(40, 1), dtype=dtypes.float32) + x3 = x3 + b + x3 = gen_array_ops.reshape(x3, [12, 5, 8, 7]) + + out = x3 + x1 + array_ops.squeeze(out, name="output") + + with session.Session(config=sessconfig, graph=g) as sess: + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + sess.run(variables.global_variables_initializer()) + saver.save(sess, self.ckpt) + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py new file mode 100644 index 0000000000..1ac6f5cb6a --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py @@ -0,0 +1,116 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.layers import core +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class BiasaddMatMulTest(BaseUnitTest): + """Testing BiasAdd MatMul in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(BiasaddMatMulTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (48, 12) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.matmul_test + self.expect_nb_nodes = 53 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def matmul_test(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + b = constant_op.constant(np.random.randn(12, 4), dtype=dtypes.float32) + x1 = math_ops.matmul(x, b) + b = constant_op.constant(np.random.randn(1, 4), dtype=dtypes.float32) + x1 = x1 + b + + b = constant_op.constant(np.random.randn(48, 4), dtype=dtypes.float32) + x2 = math_ops.matmul(x, b, transpose_a=True) + x2 = gen_array_ops.reshape(x2, [48, 1]) + + b = constant_op.constant(np.random.randn(4, 12), dtype=dtypes.float32) + x3 = math_ops.matmul(x, b, transpose_b=True) + + b = constant_op.constant(np.random.randn(16, 48), dtype=dtypes.float32) + x4 = math_ops.matmul(x, b, transpose_b=True, transpose_a=True) + x4 = gen_array_ops.reshape(x4, [48, 4]) + + x5 = gen_array_ops.reshape(x, [4, 12, 12]) + x5 = core.flatten(x5) + b = constant_op.constant(np.random.randn(144, 48), dtype=dtypes.float32) + x5 = math_ops.matmul(x5, b) + b = constant_op.constant(np.random.randn(48), dtype=dtypes.float32) + x5 = nn.bias_add(x5, b) + x5 = gen_array_ops.reshape(x5, [48, 4]) + + x6 = gen_array_ops.reshape(x, [4, 12, 12]) + b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32) + x6 = nn.bias_add(x6, b, data_format="NHWC") + x6 = gen_array_ops.reshape(x6, [48, -1]) + + x7 = gen_array_ops.reshape(x, [4, 12, 3, 4]) + b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32) + x7 = nn.bias_add(x7, b, data_format="NHWC") + x7 = gen_array_ops.reshape(x7, [48, -1]) + + x8 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) + b = constant_op.constant(np.random.randn(2), dtype=dtypes.float32) + x8 = nn.bias_add(x8, b, data_format="NHWC") + x8 = gen_array_ops.reshape(x8, [48, -1]) + + x9 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) + b = constant_op.constant(np.random.randn(3), dtype=dtypes.float32) + x9 = nn.bias_add(x9, b, data_format="NCHW") + x9 = gen_array_ops.reshape(x9, [48, -1]) + + x10 = gen_array_ops.reshape(x, [4, 12, 3, 4]) + b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32) + x10 = nn.bias_add(x10, b, data_format="NCHW") + x10 = gen_array_ops.reshape(x10, [48, -1]) + + x11 = gen_array_ops.reshape(x, [4, 12, 12]) + b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32) + x11 = nn.bias_add(x11, b, data_format="NCHW") + x11 = gen_array_ops.reshape(x11, [48, -1]) + + out = array_ops.concat( + [x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11], axis=-1) + out = array_ops.squeeze(out, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py new file mode 100644 index 0000000000..5233a493d0 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py @@ -0,0 +1,148 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import math_ops +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class BinaryTensorWeightBroadcastTest(BaseUnitTest): + """unit tests for scale & elementwise layers in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(BinaryTensorWeightBroadcastTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (10, 24, 24, 20) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 35 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.1 + self.allclose_atol = 0.05 + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + # scale + a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant( + np.random.randn(24, 24, 20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant( + np.random.randn(24, 24, 20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 1, 1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 1, 1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 24, 1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 24, 1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 24, 20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 24, 20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + gen_array_ops.reshape(x, [5, -1], name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py new file mode 100644 index 0000000000..de0817d2e8 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/concatenation_test.py @@ -0,0 +1,87 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class ConcatenationTest(BaseUnitTest): + """Testing Concatenation in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(ConcatenationTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (2, 3, 3, 1) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 4 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + # scale + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r1 = x / a + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r2 = a / x + a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32) + r3 = a + x + a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32) + r4 = x * a + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r5 = x - a + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r6 = a - x + a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) + r7 = x - a + a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) + r8 = a - x + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r9 = gen_math_ops.maximum(x, a) + a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) + r10 = gen_math_ops.minimum(a, x) + a = constant_op.constant(np.random.randn(3), dtype=dtypes.float32) + r11 = x * a + a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) + r12 = a * x + concat1 = array_ops.concat([r1, r2, r3, r4, r5, r6], axis=-1) + concat2 = array_ops.concat([r7, r8, r9, r10, r11, r12], axis=3) + x = array_ops.concat([concat1, concat2], axis=-1) + + gen_array_ops.reshape(x, [2, -1], name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py new file mode 100644 index 0000000000..74d39d9015 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py @@ -0,0 +1,75 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class ConstBroadcastTest(BaseUnitTest): + """Testing Constant broadcasting in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(ConstBroadcastTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (5, 12, 12, 2) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.conv_broadcast + self.expect_nb_nodes = 7 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.05 + self.allclose_atol = 0.05 + + def conv_broadcast(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + filt1 = constant_op.constant( + 1, shape=(3, 3, 2, 1), dtype=dtypes.float32, name='filt1') + y1 = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='SAME', name='y1') + z1 = nn.relu(y1, name='z1') + filt2 = constant_op.constant( + np.random.randn(9), + shape=(3, 3, 1, 1), + dtype=dtypes.float32, + name='filt2') + y2 = nn.conv2d(z1, filt2, strides=[1, 1, 1, 1], padding='SAME', name='y2') + z2 = nn.relu(y2, name='z') + filt3 = constant_op.constant( + np.random.randn(3, 3, 1, 1), + shape=(3, 3, 1, 1), + dtype=dtypes.float32, + name='filt3') + y3 = nn.conv2d(z2, filt3, strides=[1, 1, 1, 1], padding='SAME', name='y3') + z = nn.relu(y3, name='output') + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py new file mode 100644 index 0000000000..291b4d16c1 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py @@ -0,0 +1,101 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class MultiConnectionNeighborEngineTest(BaseUnitTest): + """Multi connection neighboring nodes wiring tests in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(MultiConnectionNeighborEngineTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (2, 3, 7, 5) + self.dummy_input = np.random.normal(1.0, 0.5, self.inp_dims) + self.get_network = self.neighboring_tensor_test + self.expect_nb_nodes = 7 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.05 + self.allclose_atol = 0.05 + + def neighboring_tensor_test(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + e = constant_op.constant( + np.random.normal(.05, .005, [3, 2, 3, 4]), + name="weights", + dtype=dtypes.float32) + conv = nn.conv2d( + input=x, + filter=e, + data_format="NCHW", + strides=[1, 1, 1, 1], + padding="VALID", + name="conv") + b = constant_op.constant( + np.random.normal(2.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + t = conv + b + + b = constant_op.constant( + np.random.normal(5.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + q = conv - b + edge = math_ops.sigmoid(q) + + b = constant_op.constant( + np.random.normal(5.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + d = b + conv + edge3 = math_ops.sigmoid(d) + + c = constant_op.constant( + np.random.normal(1.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + edge1 = gen_math_ops.tan(conv) + t = t - edge1 + q = q + edge + t = t + q + t = t + d + t = t - edge3 + array_ops.squeeze(t, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py new file mode 100644 index 0000000000..f916db3504 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py @@ -0,0 +1,78 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import gen_math_ops +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class NeighboringEngineTest(BaseUnitTest): + """Neighboring node wiring tests in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(NeighboringEngineTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (2, 3, 7, 5) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.neighboring_tensor_test + self.expect_nb_nodes = 5 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.05 + self.allclose_atol = 0.05 + + def neighboring_tensor_test(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + e = constant_op.constant( + np.random.normal(.3, 0.05, [3, 2, 3, 4]), + name="weights", + dtype=dtypes.float32) + conv = nn.conv2d( + input=x, + filter=e, + data_format="NCHW", + strides=[1, 1, 1, 1], + padding="VALID", + name="conv") + b = constant_op.constant( + np.random.normal(1.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + t = conv * b + + e = gen_math_ops.tan(conv) + t = t - e + array_ops.squeeze(t, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/run_test.py b/tensorflow/contrib/tensorrt/test/run_test.py new file mode 100644 index 0000000000..4d109cc378 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/run_test.py @@ -0,0 +1,184 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""script to convert and execute TF-TensorRT graph.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 +from tensorflow.core.protobuf import rewriter_config_pb2 +from tensorflow.python.client import session +from tensorflow.python.framework import importer +from tensorflow.python.framework import ops +from tensorflow.python.training import training +from tensorflow.contrib.tensorrt.test.utilities import get_all_variables + +OUTPUT_NODE = "output" +INPUT_NODE = "input" +CALIB_COUNT = 5 # calibration iteration + + +class RunTest: + """base class to run TR-TRT conversion and execution""" + + def __init__(self): + self.clean() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.clean() + + def clean(self): + self.tftrt = {} + self.tftrt_conversion_flag = {} + self.tftrt_nb_nodes = {} + self.tftrt_result = {} + self.tftrt_dynamic_conversion_flag = {} + self.tftrt_dynamic_result = {} + self.check_file = None + self.native_network = None + + def run_test(self, + network, + static_mode_list, + dynamic_mode_list, + dummy_input, + file_name=None): + self.native_network = network() + success = True + initialization = False + if file_name != None: + initialization = True + self.check_file = file_name + self.native_result, self.native_nb_nodes = self.execute_graph( + self.native_network, dummy_input, initialization) + for mode in static_mode_list: + try: + self.run_static_convert_network(mode, dummy_input, initialization) + self.tftrt_conversion_flag[mode] = True + except Exception as inst: + self.tftrt_conversion_flag[mode] = False + success = False + for mode in dynamic_mode_list: + try: + self.run_dynamic_convert_network(mode, dummy_input, initialization) + self.tftrt_dynamic_conversion_flag[mode] = True + except Exception as inst: + self.tftrt_dynamic_conversion_flag[mode] = False + success = False + return success + + def run_dynamic_convert_network(self, mode, dummy_input, initialization=True): + inp_dims = dummy_input.shape + if mode == "FP32" or mode == "FP16": + opt_config = rewriter_config_pb2.RewriterConfig() + opt_config.optimizers.extend(["constfold", "layout"]) + custom_op = opt_config.custom_optimizers.add() + custom_op.name = "TensorRTOptimizer" + custom_op.parameter_map["minimum_segment_size"].i = 3 + custom_op.parameter_map["precision_mode"].s = mode + custom_op.parameter_map["max_batch_size"].i = inp_dims[0] + custom_op.parameter_map["max_workspace_size_bytes"].i = 1 << 25 + print(custom_op) + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + graph_options = config_pb2.GraphOptions(rewrite_options=opt_config) + sessconfig = config_pb2.ConfigProto( + gpu_options=gpu_options, graph_options=graph_options) + print(sessconfig) + g = ops.Graph() + ops.reset_default_graph() + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=self.native_network, return_elements=["input", "output"]) + inp = inp.outputs[0] + out = out.outputs[0] + with session.Session(config=sessconfig, graph=g) as sess: + if (initialization): + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + saver.restore(sess, self.check_file) + self.tftrt_dynamic_result[mode] = sess.run(out, {inp: dummy_input}) + else: + raise Exception("dynamic op mode: " + mode + " not supported") + + def run_static_convert_network(self, mode, dummy_input, initialization=True): + inp_dims = dummy_input.shape + if mode == "FP32" or mode == "FP16" or mode == "INT8": + trt_graph = trt.create_inference_graph( + input_graph_def=self.native_network, + outputs=[OUTPUT_NODE], + max_batch_size=inp_dims[0], + max_workspace_size_bytes=1 << 25, + precision_mode=mode, # TRT Engine precision "FP32","FP16" or "INT8" + minimum_segment_size=2 # minimum number of nodes in an engine + ) + if mode == "INT8": + _ = self.execute_calibration(trt_graph, dummy_input, initialization) + trt_graph = trt.calib_graph_to_infer_graph(trt_graph) + trt_result, nb_nodes = self.execute_graph(trt_graph, dummy_input, + initialization) + self.tftrt[mode] = trt_graph + self.tftrt_nb_nodes[mode] = nb_nodes + self.tftrt_result[mode] = trt_result + else: + raise Exception("mode: " + mode + " not supported") + + def execute_graph(self, gdef, dummy_input, initialization=True): + """Run given graphdef once.""" + gpu_options = config_pb2.GPUOptions() + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + ops.reset_default_graph() + g = ops.Graph() + nb_nodes = 0 + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") + nb_nodes = len(g.get_operations()) + inp = inp.outputs[0] + out = out.outputs[0] + with session.Session(config=sessconfig, graph=g) as sess: + if (initialization): + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + saver.restore(sess, self.check_file) + val = sess.run(out, {inp: dummy_input}) + return val, nb_nodes + + # Use real data that is representative of the inference dataset + # for calibration. For this test script it is random data. + def execute_calibration(self, gdef, dummy_input, initialization=True): + """Run given calibration graph multiple times.""" + gpu_options = config_pb2.GPUOptions() + ops.reset_default_graph() + g = ops.Graph() + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") + inp = inp.outputs[0] + out = out.outputs[0] + with session.Session( + config=config_pb2.ConfigProto(gpu_options=gpu_options), + graph=g) as sess: + if (initialization): + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + saver.restore(sess, self.check_file) + for _ in range(CALIB_COUNT): + val = sess.run(out, {inp: dummy_input}) + return val diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py index d9c41f90d0..854a1ae168 100644 --- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py +++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py @@ -124,15 +124,15 @@ TestGraph = namedtuple("TestGraph", TEST_GRAPHS = { "SingleEngineGraph": - TestGraph( - gdef=GetSingleEngineGraphDef(), - num_expected_engines=1, - expected_output_dims=(100, 6, 6, 6)), + TestGraph( + gdef=GetSingleEngineGraphDef(), + num_expected_engines=1, + expected_output_dims=(100, 6, 6, 6)), "MultiEngineGraph": - TestGraph( - gdef=GetMultiEngineGraphDef(), - num_expected_engines=2, - expected_output_dims=(100, 12, 12, 6)), + TestGraph( + gdef=GetMultiEngineGraphDef(), + num_expected_engines=2, + expected_output_dims=(100, 12, 12, 6)), # TODO(aaroey): add a large complex graph to test. } diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py new file mode 100644 index 0000000000..a054939ce2 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unary_test.py @@ -0,0 +1,125 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.client import session +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.utilities import get_all_variables + + +class UnaryTest(BaseUnitTest): + """Unit tests for unary operations in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(UnaryTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (12, 5, 8, 1, 1, 12) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.unary_test + self.expect_nb_nodes = 17 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.ckpt = "./tmp.ckpt" + + def unary_test(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + q = math_ops.abs(x) + q = q + 1.0 + q = gen_math_ops.exp(q) + q = gen_math_ops.log(q) + q = array_ops.squeeze(q, axis=-2) + q = math_ops.abs(q) + q = q + 2.2 + q = gen_math_ops.sqrt(q) + q = gen_math_ops.rsqrt(q) + q = math_ops.negative(q) + q = array_ops.squeeze(q, axis=3) + q = math_ops.abs(q) + q = q + 3.0 + a = gen_math_ops.reciprocal(q) + + x = constant_op.constant(np.random.randn(5, 8, 12), dtype=dtypes.float32) + q = math_ops.abs(x) + q = q + 2.0 + q = gen_math_ops.exp(q) + q = gen_math_ops.log(q) + q = math_ops.abs(q) + q = q + 2.1 + q = gen_math_ops.sqrt(q) + q = gen_math_ops.rsqrt(q) + q = math_ops.negative(q) + q = math_ops.abs(q) + q = q + 4.0 + b = gen_math_ops.reciprocal(q) + + # TODO(jie): this one will break, broadcasting on batch. + x = variable_scope.get_variable( + "test", [12, 40, 12], + dtype=dtypes.float32, + initializer=init_ops.truncated_normal_initializer) + x = gen_array_ops.reshape(x, [12, 5, 8, 1, 12, 1, 1]) + q = math_ops.abs(x) + q = q + 5.0 + q = gen_math_ops.exp(q) + q = array_ops.squeeze(q, axis=[-1, -2, 3]) + q = gen_math_ops.log(q) + q = math_ops.abs(q) + q = q + 5.1 + q = gen_array_ops.reshape(q, [12, 5, 1, 1, 8, 1, 12]) + q = array_ops.squeeze(q, axis=[5, 2, 3]) + q = gen_math_ops.sqrt(q) + q = math_ops.abs(q) + q = q + 5.2 + q = gen_math_ops.rsqrt(q) + q = math_ops.negative(q) + q = math_ops.abs(q) + q = q + 5.3 + c = gen_math_ops.reciprocal(q) + + q = a * b + q = q / c + array_ops.squeeze(q, name="output") + + with session.Session(config=sessconfig, graph=g) as sess: + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + sess.run(variables.global_variables_initializer()) + saver.save(sess, self.ckpt) + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests.py b/tensorflow/contrib/tensorrt/test/unit_tests.py new file mode 100644 index 0000000000..ac6e3b13ee --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests.py @@ -0,0 +1,67 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to execute and log all integration tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.tensorrt.test.batch_matmul_test import BatchMatMulTest +from tensorflow.contrib.tensorrt.test.biasadd_matmul_test import BiasaddMatMulTest +from tensorflow.contrib.tensorrt.test.binary_tensor_weight_broadcast_test import BinaryTensorWeightBroadcastTest +from tensorflow.contrib.tensorrt.test.concatenation_test import ConcatenationTest +from tensorflow.contrib.tensorrt.test.multi_connection_neighbor_engine_test import MultiConnectionNeighborEngineTest +from tensorflow.contrib.tensorrt.test.neighboring_engine_test import NeighboringEngineTest +from tensorflow.contrib.tensorrt.test.unary_test import UnaryTest +from tensorflow.contrib.tensorrt.test.vgg_block_nchw_test import VGGBlockNCHWTest +from tensorflow.contrib.tensorrt.test.vgg_block_test import VGGBlockTest +from tensorflow.contrib.tensorrt.test.const_broadcast_test import ConstBroadcastTest + +from tensorflow.contrib.tensorrt.test.run_test import RunTest + +tests = 0 +passed_test = 0 + +failed_list = [] +test_list = [] + +test_list.append(BatchMatMulTest()) +test_list.append(BiasaddMatMulTest()) +test_list.append(BinaryTensorWeightBroadcastTest()) +test_list.append(ConcatenationTest()) +test_list.append(NeighboringEngineTest()) +test_list.append(UnaryTest()) +test_list.append(VGGBlockNCHWTest()) +test_list.append(VGGBlockTest()) +test_list.append(MultiConnectionNeighborEngineTest()) +test_list.append(ConstBroadcastTest()) + +for test in test_list: + test.debug = True + test.check_node_count = False + with RunTest() as context: + tests += 1 + if test.run(context): + passed_test += 1 + else: + failed_list.append(test.test_name) + print("Failed test: %s\n", test.test_name) + +if passed_test == tests: + print("Passed\n") +else: + print(("%d out of %d passed\n -- failed list:") % (passed_test, tests)) + for test in failed_list: + print(" - " + test) diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py deleted file mode 100644 index d26be35458..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/BatchMatMulTest.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.layers import core -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - - -class BatchMatMulTest(BaseUnitTest): - """Testing BatchMatMul in TF-TRT conversion""" - - def __init__(self, log_file='log.txt'): - super(BatchMatMulTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (12, 5, 8, 12) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.matmul_test - self.expect_nb_nodes = 16 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.ckpt = "./tmp.ckpt" - sess = csess.Session() - - def matmul_test(self): - g = ops.Graph() - gpu_options = cpb2.GPUOptions() - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - - b = constant_op.constant( - np.random.randn(12, 5, 12, 7), dtype=dtypes.float32) - x1 = math_ops.matmul(x, b) - b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32) - x1 = x1 + b - - var = variable_scope.get_variable( - "test", [12, 5, 12, 7], - dtype=dtypes.float32, - initializer=init_ops.truncated_normal_initializer) - x2 = math_ops.matmul(x, var) - b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32) - x2 = x2 * b - - var = variable_scope.get_variable( - "test2", [12, 84], - dtype=dtypes.float32, - initializer=init_ops.truncated_normal_initializer) - c = gen_array_ops.reshape(x, [12, 40, 12]) - b = gen_array_ops.reshape(var, [12, 12, 7]) - x3 = math_ops.matmul(c, b) - b = constant_op.constant(np.random.randn(40, 1), dtype=dtypes.float32) - x3 = x3 + b - x3 = gen_array_ops.reshape(x3, [12, 5, 8, 7]) - - out = x3 + x1 - array_ops.squeeze(out, name="output") - - with csess.Session(config=sessconfig, graph=g) as sess: - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - sess.run(variables.global_variables_initializer()) - saver.save(sess, self.ckpt) - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py deleted file mode 100644 index 81b43422fd..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/BiasaddMatMulTest.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.layers import core -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - - -class BiasaddMatMulTest(BaseUnitTest): - """Testing BiasAdd MatMul in TF-TRT conversion""" - - def __init__(self, log_file='log.txt'): - super(BiasaddMatMulTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (48, 12) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.matmul_test - self.expect_nb_nodes = 53 - self.log_file = log_file - self.test_name = self.__class__.__name__ - - def matmul_test(self): - g = ops.Graph() - gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - - b = constant_op.constant(np.random.randn(12, 4), dtype=dtypes.float32) - x1 = math_ops.matmul(x, b) - b = constant_op.constant(np.random.randn(1, 4), dtype=dtypes.float32) - x1 = x1 + b - - b = constant_op.constant(np.random.randn(48, 4), dtype=dtypes.float32) - x2 = math_ops.matmul(x, b, transpose_a=True) - x2 = gen_array_ops.reshape(x2, [48, 1]) - - b = constant_op.constant(np.random.randn(4, 12), dtype=dtypes.float32) - x3 = math_ops.matmul(x, b, transpose_b=True) - - b = constant_op.constant(np.random.randn(16, 48), dtype=dtypes.float32) - x4 = math_ops.matmul(x, b, transpose_b=True, transpose_a=True) - x4 = gen_array_ops.reshape(x4, [48, 4]) - - x5 = gen_array_ops.reshape(x, [4, 12, 12]) - x5 = core.flatten(x5) - b = constant_op.constant(np.random.randn(144, 48), dtype=dtypes.float32) - x5 = math_ops.matmul(x5, b) - b = constant_op.constant(np.random.randn(48), dtype=dtypes.float32) - x5 = nn.bias_add(x5, b) - x5 = gen_array_ops.reshape(x5, [48, 4]) - - x6 = gen_array_ops.reshape(x, [4, 12, 12]) - b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32) - x6 = nn.bias_add(x6, b, data_format="NHWC") - x6 = gen_array_ops.reshape(x6, [48, -1]) - - x7 = gen_array_ops.reshape(x, [4, 12, 3, 4]) - b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32) - x7 = nn.bias_add(x7, b, data_format="NHWC") - x7 = gen_array_ops.reshape(x7, [48, -1]) - - x8 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) - b = constant_op.constant(np.random.randn(2), dtype=dtypes.float32) - x8 = nn.bias_add(x8, b, data_format="NHWC") - x8 = gen_array_ops.reshape(x8, [48, -1]) - - x9 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) - b = constant_op.constant(np.random.randn(3), dtype=dtypes.float32) - x9 = nn.bias_add(x9, b, data_format="NCHW") - x9 = gen_array_ops.reshape(x9, [48, -1]) - - x10 = gen_array_ops.reshape(x, [4, 12, 3, 4]) - b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32) - x10 = nn.bias_add(x10, b, data_format="NCHW") - x10 = gen_array_ops.reshape(x10, [48, -1]) - - x11 = gen_array_ops.reshape(x, [4, 12, 12]) - b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32) - x11 = nn.bias_add(x11, b, data_format="NCHW") - x11 = gen_array_ops.reshape(x11, [48, -1]) - - out = array_ops.concat( - [x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11], axis=-1) - out = array_ops.squeeze(out, name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py deleted file mode 100644 index 46c8814405..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/BinaryTensorWeightBroadcastTest.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - - -class BinaryTensorWeightBroadcastTest(BaseUnitTest): - """unit tests for scale & elementwise layers in TF-TRT""" - - def __init__(self, log_file='log.txt'): - super(BinaryTensorWeightBroadcastTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (10, 24, 24, 20) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.get_simple_graph_def - self.expect_nb_nodes = 35 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.allclose_rtol = 0.1 - self.allclose_atol = 0.05 - - def get_simple_graph_def(self): - g = ops.Graph() - gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - - # scale - a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # scale - a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # scale - a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # scale - a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # scale - a = constant_op.constant( - np.random.randn(24, 24, 20), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # scale - a = constant_op.constant( - np.random.randn(24, 24, 20), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 1, 1), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 1, 1), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 24, 1), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 24, 1), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 24, 20), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 24, 20), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - gen_array_ops.reshape(x, [5, -1], name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py deleted file mode 100644 index 3a3098e4ed..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/ConcatenationTest.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - - -class ConcatenationTest(BaseUnitTest): - """Testing Concatenation in TF-TRT conversion""" - - def __init__(self, log_file='log.txt'): - super(ConcatenationTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (2, 3, 3, 1) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.get_simple_graph_def - self.expect_nb_nodes = 4 - self.log_file = log_file - self.test_name = self.__class__.__name__ - - def get_simple_graph_def(self): - g = ops.Graph() - gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - - # scale - a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) - r1 = x / a - a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) - r2 = a / x - a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32) - r3 = a + x - a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32) - r4 = x * a - a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) - r5 = x - a - a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) - r6 = a - x - a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) - r7 = x - a - a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) - r8 = a - x - a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) - r9 = gen_math_ops.maximum(x, a) - a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) - r10 = gen_math_ops.minimum(a, x) - a = constant_op.constant(np.random.randn(3), dtype=dtypes.float32) - r11 = x * a - a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) - r12 = a * x - concat1 = array_ops.concat([r1, r2, r3, r4, r5, r6], axis=-1) - concat2 = array_ops.concat([r7, r8, r9, r10, r11, r12], axis=3) - x = array_ops.concat([concat1, concat2], axis=-1) - - gen_array_ops.reshape(x, [2, -1], name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py deleted file mode 100644 index 7fb7d6f611..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/ConstBroadcastTest.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.layers import core -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - - -class ConstBroadcastTest(BaseUnitTest): - """Testing Constant broadcasting in TF-TRT""" - - def __init__(self, log_file='log.txt'): - super(ConstBroadcastTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (5, 12, 12, 2) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.conv_broadcast - self.expect_nb_nodes = 7 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.allclose_rtol = 0.05 - self.allclose_atol = 0.05 - - def conv_broadcast(self): - g = ops.Graph() - gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - filt1 = constant_op.constant( - 1, shape=(3, 3, 2, 1), dtype=dtypes.float32, name='filt1') - y1 = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='SAME', name='y1') - z1 = nn.relu(y1, name='z1') - filt2 = constant_op.constant( - np.random.randn(9), - shape=(3, 3, 1, 1), - dtype=dtypes.float32, - name='filt2') - y2 = nn.conv2d(z1, filt2, strides=[1, 1, 1, 1], padding='SAME', name='y2') - z2 = nn.relu(y2, name='z') - filt3 = constant_op.constant( - np.random.randn(3, 3, 1, 1), - shape=(3, 3, 1, 1), - dtype=dtypes.float32, - name='filt3') - y3 = nn.conv2d(z2, filt3, strides=[1, 1, 1, 1], padding='SAME', name='y3') - z = nn.relu(y3, name='output') - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py deleted file mode 100644 index 1bbfd0078d..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/MultiConnectionNeighborEngineTest.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - - -class MultiConnectionNeighborEngineTest(BaseUnitTest): - """Multi connection neighboring nodes wiring tests in TF-TRT""" - - def __init__(self, log_file='log.txt'): - super(MultiConnectionNeighborEngineTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (2, 3, 7, 5) - self.dummy_input = np.random.normal(1.0, 0.5, self.inp_dims) - self.get_network = self.neighboring_tensor_test - self.expect_nb_nodes = 7 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.allclose_rtol = 0.05 - self.allclose_atol = 0.05 - - def neighboring_tensor_test(self): - g = ops.Graph() - gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - e = constant_op.constant( - np.random.normal(.05, .005, [3, 2, 3, 4]), - name="weights", - dtype=dtypes.float32) - conv = nn.conv2d( - input=x, - filter=e, - data_format="NCHW", - strides=[1, 1, 1, 1], - padding="VALID", - name="conv") - b = constant_op.constant( - np.random.normal(2.0, 1.0, [1, 4, 1, 1]), - name="bias", - dtype=dtypes.float32) - t = conv + b - - b = constant_op.constant( - np.random.normal(5.0, 1.0, [1, 4, 1, 1]), - name="bias", - dtype=dtypes.float32) - q = conv - b - edge = math_ops.sigmoid(q) - - b = constant_op.constant( - np.random.normal(5.0, 1.0, [1, 4, 1, 1]), - name="bias", - dtype=dtypes.float32) - d = b + conv - edge3 = math_ops.sigmoid(d) - - c = constant_op.constant( - np.random.normal(1.0, 1.0, [1, 4, 1, 1]), - name="bias", - dtype=dtypes.float32) - edge1 = gen_math_ops.tan(conv) - t = t - edge1 - q = q + edge - t = t + q - t = t + d - t = t - edge3 - array_ops.squeeze(t, name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py deleted file mode 100644 index 8341e89519..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/NeighboringEngineTest.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - - -class NeighboringEngineTest(BaseUnitTest): - """Neighboring node wiring tests in TF-TRT conversion""" - - def __init__(self, log_file='log.txt'): - super(NeighboringEngineTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (2, 3, 7, 5) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.neighboring_tensor_test - self.expect_nb_nodes = 5 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.allclose_rtol = 0.05 - self.allclose_atol = 0.05 - - def neighboring_tensor_test(self): - g = ops.Graph() - gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - e = constant_op.constant( - np.random.normal(.3, 0.05, [3, 2, 3, 4]), - name="weights", - dtype=dtypes.float32) - conv = nn.conv2d( - input=x, - filter=e, - data_format="NCHW", - strides=[1, 1, 1, 1], - padding="VALID", - name="conv") - b = constant_op.constant( - np.random.normal(1.0, 1.0, [1, 4, 1, 1]), - name="bias", - dtype=dtypes.float32) - t = conv * b - - e = gen_math_ops.tan(conv) - t = t - e - array_ops.squeeze(t, name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py deleted file mode 100644 index 8ac4c2a308..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/UnaryTest.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.layers import core -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - - -class UnaryTest(BaseUnitTest): - """Unit tests for unary operations in TF-TRT""" - - def __init__(self, log_file='log.txt'): - super(UnaryTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (12, 5, 8, 1, 1, 12) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.unary_test - self.expect_nb_nodes = 17 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.ckpt = "./tmp.ckpt" - - def unary_test(self): - g = ops.Graph() - gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - q = math_ops.abs(x) - q = q + 1.0 - q = gen_math_ops.exp(q) - q = gen_math_ops.log(q) - q = array_ops.squeeze(q, axis=-2) - q = math_ops.abs(q) - q = q + 2.2 - q = gen_math_ops.sqrt(q) - q = gen_math_ops.rsqrt(q) - q = math_ops.negative(q) - q = array_ops.squeeze(q, axis=3) - q = math_ops.abs(q) - q = q + 3.0 - a = gen_math_ops.reciprocal(q) - - x = constant_op.constant(np.random.randn(5, 8, 12), dtype=dtypes.float32) - q = math_ops.abs(x) - q = q + 2.0 - q = gen_math_ops.exp(q) - q = gen_math_ops.log(q) - q = math_ops.abs(q) - q = q + 2.1 - q = gen_math_ops.sqrt(q) - q = gen_math_ops.rsqrt(q) - q = math_ops.negative(q) - q = math_ops.abs(q) - q = q + 4.0 - b = gen_math_ops.reciprocal(q) - - # TODO(jie): this one will break, broadcasting on batch. - x = variable_scope.get_variable( - "test", [12, 40, 12], - dtype=dtypes.float32, - initializer=init_ops.truncated_normal_initializer) - x = gen_array_ops.reshape(x, [12, 5, 8, 1, 12, 1, 1]) - q = math_ops.abs(x) - q = q + 5.0 - q = gen_math_ops.exp(q) - q = array_ops.squeeze(q, axis=[-1, -2, 3]) - q = gen_math_ops.log(q) - q = math_ops.abs(q) - q = q + 5.1 - q = gen_array_ops.reshape(q, [12, 5, 1, 1, 8, 1, 12]) - q = array_ops.squeeze(q, axis=[5, 2, 3]) - q = gen_math_ops.sqrt(q) - q = math_ops.abs(q) - q = q + 5.2 - q = gen_math_ops.rsqrt(q) - q = math_ops.negative(q) - q = math_ops.abs(q) - q = q + 5.3 - c = gen_math_ops.reciprocal(q) - - q = a * b - q = q / c - array_ops.squeeze(q, name="output") - - with csess.Session(config=sessconfig, graph=g) as sess: - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - sess.run(variables.global_variables_initializer()) - saver.save(sess, self.ckpt) - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py deleted file mode 100644 index a714cc8d9d..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockNCHWTest.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - - -class VGGBlockNCHWTest(BaseUnitTest): - """single vgg layer in NCHW unit tests in TF-TRT""" - - def __init__(self, log_file='log.txt'): - super(VGGBlockNCHWTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (5, 2, 8, 8) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.get_simple_graph_def - self.expect_nb_nodes = 3 - self.log_file = log_file - self.test_name = self.__class__.__name__ - - def get_simple_graph_def(self): - g = ops.Graph() - gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - x, mean_x, var_x = nn_impl.fused_batch_norm( - x, - np.random.randn(2).astype(np.float32), - np.random.randn(2).astype(np.float32), - mean=np.random.randn(2).astype(np.float32), - variance=np.random.randn(2).astype(np.float32), - data_format="NCHW", - is_training=False) - e = constant_op.constant( - np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32) - conv = nn.conv2d( - input=x, - filter=e, - data_format="NCHW", - strides=[1, 1, 2, 2], - padding="SAME", - name="conv") - b = constant_op.constant( - np.random.randn(6), name="bias", dtype=dtypes.float32) - t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd") - relu = nn.relu(t, "relu") - idty = array_ops.identity(relu, "ID") - v = nn_ops.max_pool( - idty, [1, 1, 2, 2], [1, 1, 2, 2], - "VALID", - data_format="NCHW", - name="max_pool") - array_ops.squeeze(v, name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py b/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py deleted file mode 100644 index 77601a23e7..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/VGGBlockTest.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - - -class VGGBlockTest(BaseUnitTest): - """single vgg layer test in TF-TRT conversion""" - - def __init__(self, log_file='log.txt'): - super(VGGBlockTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (5, 8, 8, 2) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.get_simple_graph_def - self.expect_nb_nodes = 7 - self.log_file = log_file - self.test_name = self.__class__.__name__ - - def get_simple_graph_def(self): - g = ops.Graph() - gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - x, mean_x, var_x = nn_impl.fused_batch_norm( - x, - np.random.randn(2).astype(np.float32), - np.random.randn(2).astype(np.float32), - mean=np.random.randn(2).astype(np.float32), - variance=np.random.randn(2).astype(np.float32), - is_training=False) - e = constant_op.constant( - np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32) - conv = nn.conv2d( - input=x, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv") - b = constant_op.constant( - np.random.randn(6), name="bias", dtype=dtypes.float32) - t = nn.bias_add(conv, b, name="biasAdd") - relu = nn.relu(t, "relu") - idty = array_ops.identity(relu, "ID") - v = nn_ops.max_pool( - idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool") - array_ops.squeeze(v, name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py b/tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py deleted file mode 100644 index ec8e6e3d1d..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/base_unit_test.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Base class to facilitate development of integration tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op as cop -from tensorflow.python.framework import dtypes as dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops as aops -from tensorflow.python.ops import nn as nn -from tensorflow.python.ops import nn_ops as nn_ops - - -class BaseUnitTest(object): - """Base class for unit tests in TF-TRT""" - - def __init__(self, log_file='log.txt'): - self.static_mode_list = {} - self.dynamic_mode_list = {} - self.dummy_input = None - self.get_network = None - self.expect_nb_nodes = None - self.test_name = None - self.log_file = log_file - self.ckpt = None - self.allclose_rtol = 0.01 - self.allclose_atol = 0.01 - self.allclose_equal_nan = True - # saves out graphdef - self.debug = False - # require node count check fail leads to test failure - self.check_node_count = False - - def run(self, run_test_context): - run_test_context.run_test(self.get_network, self.static_mode_list, - self.dynamic_mode_list, self.dummy_input, - self.ckpt) - return self.log_result(run_test_context) - - def log_result(self, run_test_result): - log = open(self.log_file, 'a') - log.write(("================= model: %s\n") % (self.test_name)) - - if self.debug: - open(self.test_name + "_native.pb", - 'wb').write(run_test_result.native_network.SerializeToString()) - all_success = True - if len(run_test_result.tftrt_conversion_flag) != 0: - log.write(" -- static_mode\n") - for static_mode in run_test_result.tftrt_conversion_flag: - if self.debug: - open(self.test_name + "_" + static_mode + ".pb", - 'wb').write(run_test_result.tftrt[static_mode].SerializeToString()) - log.write(" ----\n") - log.write((" mode: [%s]\n") % (static_mode)) - if run_test_result.tftrt_conversion_flag[static_mode]: - if run_test_result.tftrt_nb_nodes[static_mode] != self.expect_nb_nodes: - log.write( - ("[WARNING]: converted node number does not match (%d,%d,%d)!!!\n" - ) % (run_test_result.tftrt_nb_nodes[static_mode], - self.expect_nb_nodes, run_test_result.native_nb_nodes)) - if self.check_node_count: - all_success = False - - if np.array_equal(run_test_result.tftrt_result[static_mode], - run_test_result.native_result): - log.write(" output: equal\n") - elif np.allclose( - run_test_result.tftrt_result[static_mode], - run_test_result.native_result, - atol=self.allclose_atol, - rtol=self.allclose_rtol, - equal_nan=self.allclose_equal_nan): - log.write(" output: allclose\n") - else: - diff = run_test_result.tftrt_result[static_mode] - run_test_result.native_result - log.write("[ERROR]: output does not match!!!\n") - log.write("max diff: " + str(np.max(diff))) - log.write("\ntftrt:\n") - log.write(str(run_test_result.tftrt_result[static_mode])) - log.write("\nnative:\n") - log.write(str(run_test_result.native_result)) - log.write("\ndiff:\n") - log.write(str(diff)) - all_success = False - else: - log.write("[ERROR]: conversion failed!!!\n") - all_success = False - - if len(run_test_result.tftrt_dynamic_conversion_flag) != 0: - log.write(" -- dynamic_mode\n") - for dynamic_mode in run_test_result.tftrt_dynamic_conversion_flag: - log.write("\n ----\n") - log.write((" mode: [%s]\n") % (dynamic_mode)) - if run_test_result.tftrt_dynamic_conversion_flag[dynamic_mode]: - if np.array_equal(run_test_result.tftrt_dynamic_result[dynamic_mode], - run_test_result.native_result): - log.write(" output: equal\n") - elif np.allclose(run_test_result.tftrt_dynamic_result[dynamic_mode], - run_test_result.native_result): - log.write(" output: allclose\n") - else: - log.write("[ERROR]: output does not match!!!\n") - all_success = False - else: - log.write("[ERROR]: conversion failed!!!\n") - all_success = False - return all_success diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/run_test.py b/tensorflow/contrib/tensorrt/test/unit_tests/run_test.py deleted file mode 100644 index b322ae63a2..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/run_test.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""script to convert and execute TF-TensorRT graph.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 as cpb2 -from tensorflow.core.protobuf import rewriter_config_pb2 as rwpb2 -from tensorflow.python.client import session as csess -from tensorflow.python.framework import constant_op as cop -from tensorflow.python.framework import dtypes as dtypes -from tensorflow.python.framework import importer as importer -from tensorflow.python.framework import ops as ops -from tensorflow.python.ops import array_ops as aops -from tensorflow.python.ops import nn as nn -from tensorflow.python.ops import nn_ops as nn_ops -from tensorflow.python.ops import variables -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.unit_tests.utilities import get_all_variables - -OUTPUT_NODE = "output" -INPUT_NODE = "input" -CALIB_COUNT = 5 # calibration iteration - - -class RunTest: - """base class to run TR-TRT conversion and execution""" - - def __init__(self): - self.clean() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.clean() - - def clean(self): - self.tftrt = {} - self.tftrt_conversion_flag = {} - self.tftrt_nb_nodes = {} - self.tftrt_result = {} - self.tftrt_dynamic_conversion_flag = {} - self.tftrt_dynamic_result = {} - self.check_file = None - self.native_network = None - - def run_test(self, - network, - static_mode_list, - dynamic_mode_list, - dummy_input, - file_name=None): - self.native_network = network() - success = True - initialization = False - if file_name != None: - initialization = True - self.check_file = file_name - self.native_result, self.native_nb_nodes = self.execute_graph( - self.native_network, dummy_input, initialization) - for mode in static_mode_list: - try: - self.run_static_convert_network(mode, dummy_input, initialization) - self.tftrt_conversion_flag[mode] = True - except Exception as inst: - self.tftrt_conversion_flag[mode] = False - success = False - for mode in dynamic_mode_list: - try: - self.run_dynamic_convert_network(mode, dummy_input, initialization) - self.tftrt_dynamic_conversion_flag[mode] = True - except Exception as inst: - self.tftrt_dynamic_conversion_flag[mode] = False - success = False - return success - - def run_dynamic_convert_network(self, mode, dummy_input, initialization=True): - inp_dims = dummy_input.shape - if mode == "FP32" or mode == "FP16": - opt_config = rwpb2.RewriterConfig() - opt_config.optimizers.extend(["constfold", "layout"]) - custom_op = opt_config.custom_optimizers.add() - custom_op.name = "TensorRTOptimizer" - custom_op.parameter_map["minimum_segment_size"].i = 3 - custom_op.parameter_map["precision_mode"].s = mode - custom_op.parameter_map["max_batch_size"].i = inp_dims[0] - custom_op.parameter_map["max_workspace_size_bytes"].i = 1 << 25 - print(custom_op) - gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - graph_options = cpb2.GraphOptions(rewrite_options=opt_config) - sessconfig = cpb2.ConfigProto( - gpu_options=gpu_options, graph_options=graph_options) - print(sessconfig) - g = ops.Graph() - ops.reset_default_graph() - with g.as_default(): - inp, out = importer.import_graph_def( - graph_def=self.native_network, return_elements=["input", "output"]) - inp = inp.outputs[0] - out = out.outputs[0] - with csess.Session(config=sessconfig, graph=g) as sess: - if (initialization): - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - saver.restore(sess, self.check_file) - self.tftrt_dynamic_result[mode] = sess.run(out, {inp: dummy_input}) - else: - raise Exception("dynamic op mode: " + mode + " not supported") - - def run_static_convert_network(self, mode, dummy_input, initialization=True): - inp_dims = dummy_input.shape - if mode == "FP32" or mode == "FP16" or mode == "INT8": - trt_graph = trt.create_inference_graph( - input_graph_def=self.native_network, - outputs=[OUTPUT_NODE], - max_batch_size=inp_dims[0], - max_workspace_size_bytes=1 << 25, - precision_mode=mode, # TRT Engine precision "FP32","FP16" or "INT8" - minimum_segment_size=2 # minimum number of nodes in an engine - ) - if mode == "INT8": - _ = self.execute_calibration(trt_graph, dummy_input, initialization) - trt_graph = trt.calib_graph_to_infer_graph(trt_graph) - trt_result, nb_nodes = self.execute_graph(trt_graph, dummy_input, - initialization) - self.tftrt[mode] = trt_graph - self.tftrt_nb_nodes[mode] = nb_nodes - self.tftrt_result[mode] = trt_result - else: - raise Exception("mode: " + mode + " not supported") - - def execute_graph(self, gdef, dummy_input, initialization=True): - """Run given graphdef once.""" - gpu_options = cpb2.GPUOptions() - sessconfig = cpb2.ConfigProto(gpu_options=gpu_options) - ops.reset_default_graph() - g = ops.Graph() - nb_nodes = 0 - with g.as_default(): - inp, out = importer.import_graph_def( - graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") - nb_nodes = len(g.get_operations()) - inp = inp.outputs[0] - out = out.outputs[0] - with csess.Session(config=sessconfig, graph=g) as sess: - if (initialization): - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - saver.restore(sess, self.check_file) - val = sess.run(out, {inp: dummy_input}) - return val, nb_nodes - - # Use real data that is representative of the inference dataset - # for calibration. For this test script it is random data. - def execute_calibration(self, gdef, dummy_input, initialization=True): - """Run given calibration graph multiple times.""" - gpu_options = cpb2.GPUOptions() - ops.reset_default_graph() - g = ops.Graph() - with g.as_default(): - inp, out = importer.import_graph_def( - graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") - inp = inp.outputs[0] - out = out.outputs[0] - with csess.Session( - config=cpb2.ConfigProto(gpu_options=gpu_options), graph=g) as sess: - if (initialization): - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - saver.restore(sess, self.check_file) - for _ in range(CALIB_COUNT): - val = sess.run(out, {inp: dummy_input}) - return val diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py b/tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py deleted file mode 100644 index 2828249878..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/unit_tests.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Script to execute and log all integration tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow.contrib.tensorrt.test - -from tensorflow.contrib.tensorrt.test.unit_tests.BatchMatMulTest import BatchMatMulTest -from tensorflow.contrib.tensorrt.test.unit_tests.BiasaddMatMulTest import BiasaddMatMulTest -from tensorflow.contrib.tensorrt.test.unit_tests.BinaryTensorWeightBroadcastTest import BinaryTensorWeightBroadcastTest -from tensorflow.contrib.tensorrt.test.unit_tests.ConcatenationTest import ConcatenationTest -from tensorflow.contrib.tensorrt.test.unit_tests.MultiConnectionNeighborEngineTest import MultiConnectionNeighborEngineTest -from tensorflow.contrib.tensorrt.test.unit_tests.NeighboringEngineTest import NeighboringEngineTest -from tensorflow.contrib.tensorrt.test.unit_tests.UnaryTest import UnaryTest -from tensorflow.contrib.tensorrt.test.unit_tests.VGGBlockNCHWTest import VGGBlockNCHWTest -from tensorflow.contrib.tensorrt.test.unit_tests.VGGBlockTest import VGGBlockTest -from tensorflow.contrib.tensorrt.test.unit_tests.ConstBroadcastTest import ConstBroadcastTest - -from tensorflow.contrib.tensorrt.test.unit_tests.run_test import RunTest - -tests = 0 -passed_test = 0 - -failed_list = [] -test_list = [] - -test_list.append(BatchMatMulTest()) -test_list.append(BiasaddMatMulTest()) -test_list.append(BinaryTensorWeightBroadcastTest()) -test_list.append(ConcatenationTest()) -test_list.append(NeighboringEngineTest()) -test_list.append(UnaryTest()) -test_list.append(VGGBlockNCHWTest()) -test_list.append(VGGBlockTest()) -test_list.append(MultiConnectionNeighborEngineTest()) -test_list.append(ConstBroadcastTest()) - -for test in test_list: - test.debug = True - test.check_node_count = False - with RunTest() as context: - tests += 1 - if test.run(context): - passed_test += 1 - else: - failed_list.append(test.test_name) - print("Failed test: %s\n", test.test_name) - -if passed_test == tests: - print("Passed\n") -else: - print(("%d out of %d passed\n -- failed list:") % (passed_test, tests)) - for test in failed_list: - print(" - " + test) diff --git a/tensorflow/contrib/tensorrt/test/unit_tests/utilities.py b/tensorflow/contrib/tensorrt/test/unit_tests/utilities.py deleted file mode 100644 index 47a03ece9f..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests/utilities.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities script for TF-TensorRT integration tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import variables -from tensorflow.python.training import training - - -def get_all_variables(sess): - var_names = sess.run(variables.report_uninitialized_variables()) - names_var_list = {} - for name in var_names: - names_var_list[name] = sess.graph.get_tensor_by_name(name + ":0") - print(var_names) - return names_var_list diff --git a/tensorflow/contrib/tensorrt/test/utilities.py b/tensorflow/contrib/tensorrt/test/utilities.py new file mode 100644 index 0000000000..0ea5f5b883 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/utilities.py @@ -0,0 +1,30 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities script for TF-TensorRT integration tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.ops import variables + + +def get_all_variables(sess): + var_names = sess.run(variables.report_uninitialized_variables()) + names_var_list = {} + for name in var_names: + names_var_list[name] = sess.graph.get_tensor_by_name(name + ":0") + print(var_names) + return names_var_list diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py new file mode 100644 index 0000000000..9a759eb994 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py @@ -0,0 +1,85 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class VGGBlockNCHWTest(BaseUnitTest): + """single vgg layer in NCHW unit tests in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(VGGBlockNCHWTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (5, 2, 8, 8) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 3 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + x, mean_x, var_x = nn_impl.fused_batch_norm( + x, + np.random.randn(2).astype(np.float32), + np.random.randn(2).astype(np.float32), + mean=np.random.randn(2).astype(np.float32), + variance=np.random.randn(2).astype(np.float32), + data_format="NCHW", + is_training=False) + e = constant_op.constant( + np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32) + conv = nn.conv2d( + input=x, + filter=e, + data_format="NCHW", + strides=[1, 1, 2, 2], + padding="SAME", + name="conv") + b = constant_op.constant( + np.random.randn(6), name="bias", dtype=dtypes.float32) + t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd") + relu = nn.relu(t, "relu") + idty = array_ops.identity(relu, "ID") + v = nn_ops.max_pool( + idty, [1, 1, 2, 2], [1, 1, 2, 2], + "VALID", + data_format="NCHW", + name="max_pool") + array_ops.squeeze(v, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py new file mode 100644 index 0000000000..04176d58ca --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/vgg_block_test.py @@ -0,0 +1,76 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class VGGBlockTest(BaseUnitTest): + """single vgg layer test in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(VGGBlockTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (5, 8, 8, 2) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 7 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + x, mean_x, var_x = nn_impl.fused_batch_norm( + x, + np.random.randn(2).astype(np.float32), + np.random.randn(2).astype(np.float32), + mean=np.random.randn(2).astype(np.float32), + variance=np.random.randn(2).astype(np.float32), + is_training=False) + e = constant_op.constant( + np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32) + conv = nn.conv2d( + input=x, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv") + b = constant_op.constant( + np.random.randn(6), name="bias", dtype=dtypes.float32) + t = nn.bias_add(conv, b, name="biasAdd") + relu = nn.relu(t, "relu") + idty = array_ops.identity(relu, "ID") + v = nn_ops.max_pool( + idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool") + array_ops.squeeze(v, name="output") + + return g.as_graph_def() -- cgit v1.2.3 From 6af33fb0436e8c9abad792f33a17e8f662e7311d Mon Sep 17 00:00:00 2001 From: gracehoney <31743510+aaroey@users.noreply.github.com> Date: Mon, 9 Jul 2018 14:50:17 -0700 Subject: Fix shape fn --- tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc | 62 +++++++++++++++--------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc index 227ac120dd..f30dba59ad 100644 --- a/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc +++ b/tensorflow/contrib/tensorrt/shape_fn/trt_shfn.cc @@ -28,36 +28,50 @@ limitations under the License. namespace tensorflow { namespace shape_inference { -tensorflow::Status TRTEngineOpShapeInference(InferenceContext* context) { - std::vector shapes; - for (int i = 0; i < context->num_outputs(); ++i) { - context->set_output(i, context->UnknownShape()); +tensorflow::Status TRTEngineOpShapeInference(InferenceContext* c) { + for (int i = 0; i < c->num_outputs(); ++i) { + c->set_output(i, c->UnknownShape()); } - auto status = context->GetAttr("input_shapes", &shapes); - // it is ok to not to have shapes - if (!status.ok()) return Status::OK(); - if ((int)shapes.size() != context->num_inputs()) return Status::OK(); - bool different_input = false; - for (int i = 0; i < context->num_inputs(); ++i) { - if (shapes.at(i) != context->input_tensor(i)->shape()) - different_input = true; + + // Check the sanity of the input shapes. + std::vector input_shapes; + TF_RETURN_IF_ERROR(c->GetAttr("input_shapes", &input_shapes)); + if (input_shapes.size() != c->num_inputs()) { + return tensorflow::errors::InvalidArgument( + "The actual number of inputs doesn't match the number of input " + "shapes set in the attr: ", + c->num_inputs(), " vs ", input_shapes.size()); + } + bool input_match = true; + for (int i = 0; i < c->num_inputs(); ++i) { + ShapeHandle handle; + TF_RETURN_IF_ERROR( + c->MakeShapeFromTensorShape(input_shapes.at(i), &handle)); + ShapeHandle merged; + if (!c->Merge(c->input(i), handle, &merged).ok()) { + // Input shape doesn't match what was set in attr, fine. + input_match = false; + } } - if (different_input) return Status::OK(); - shapes.resize(0); - status = context->GetAttr("output_shapes", &shapes); - if (!status.ok()) return Status::OK(); - if ((int)shapes.size() != context->num_outputs()) return Status::OK(); - std::vector shape_handles(shapes.size()); - for (size_t i = 0; i < shapes.size(); ++i) { - status = - context->MakeShapeFromTensorShape(shapes.at(i), &shape_handles.at(i)); - if (!status.ok()) return Status::OK(); + + // Check the sanity of the output shapes. + std::vector output_shapes; + TF_RETURN_IF_ERROR(c->GetAttr("output_shapes", &output_shapes)); + if (output_shapes.size() != c->num_outputs()) { + return tensorflow::errors::InvalidArgument( + "The actual number of outputs doesn't match the number of output " + "shapes set in the attr: ", + c->num_outputs(), " vs ", output_shapes.size()); } - for (int i = 0; i < context->num_outputs(); ++i) { - context->set_output(i, shape_handles.at(i)); + for (size_t i = 0; i < output_shapes.size(); ++i) { + ShapeHandle handle; + TF_RETURN_IF_ERROR( + c->MakeShapeFromTensorShape(output_shapes.at(i), &handle)); + if (input_match) c->set_output(i, handle); } return Status::OK(); } + } // namespace shape_inference } // namespace tensorflow -- cgit v1.2.3 From 21c4e58510b7126b39d50220716f92ce1716ed2a Mon Sep 17 00:00:00 2001 From: Jie Date: Wed, 11 Jul 2018 01:49:16 -0700 Subject: enabling shape inference function after cherry picking the fix --- tensorflow/contrib/tensorrt/ops/trt_engine_op.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc index 7eaa080586..383635f428 100644 --- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc +++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc @@ -42,9 +42,8 @@ REGISTER_OP("TRTEngineOp") .Attr("precision_mode: {'FP32', 'FP16', 'INT8', 'INT8CALIB'}") .Attr("calibration_data: string = ''") .Input("in_tensor: InT") - .Output("out_tensor: OutT"); - // TODO(Sami): shape inference not working for concrete input shape - //.SetShapeFn(shape_inference::TRTEngineOpShapeInference); + .Output("out_tensor: OutT") + .SetShapeFn(shape_inference::TRTEngineOpShapeInference); } // namespace tensorflow -- cgit v1.2.3 From 54addf8f9215bd1142d9fa073d6c1c029a341ef6 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Wed, 11 Jul 2018 09:19:58 -0700 Subject: Remove unavailable sudo from docker installer script. --- tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh b/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh index b73962854c..107cc61ff5 100755 --- a/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh +++ b/tensorflow/tools/ci_build/install/install_openblas_ppc64le.sh @@ -20,7 +20,7 @@ USE_OPENMP="USE_OPENMP=1" OPENBLAS_INSTALL_PATH="/usr" apt-get update apt-get install -y gfortran gfortran-5 -sudo rm -rf ${OPENBLAS_SRC_PATH} +rm -rf ${OPENBLAS_SRC_PATH} git clone -b release-0.3.0 https://github.com/xianyi/OpenBLAS ${OPENBLAS_SRC_PATH} cd ${OPENBLAS_SRC_PATH} # Pick up fix for OpenBLAS issue 1571 -- cgit v1.2.3 From 2de343c329ff252ed0cb419f29c3ce3765b3da84 Mon Sep 17 00:00:00 2001 From: Jie Date: Wed, 11 Jul 2018 11:01:48 -0700 Subject: removing unit tests (for follow up PR) disabling shape inference function (Because of the fully defined shape requirement from TF) --- tensorflow/contrib/tensorrt/BUILD | 33 ---- tensorflow/contrib/tensorrt/ops/trt_engine_op.cc | 9 +- tensorflow/contrib/tensorrt/test/base_unit_test.py | 118 ------------- .../contrib/tensorrt/test/batch_matmul_test.py | 97 ----------- .../contrib/tensorrt/test/biasadd_matmul_test.py | 116 ------------- .../test/binary_tensor_weight_broadcast_test.py | 148 ----------------- .../contrib/tensorrt/test/concatenation_test.py | 87 ---------- .../contrib/tensorrt/test/const_broadcast_test.py | 75 --------- .../test/multi_connection_neighbor_engine_test.py | 101 ----------- .../tensorrt/test/neighboring_engine_test.py | 78 --------- tensorflow/contrib/tensorrt/test/run_test.py | 184 --------------------- .../tensorrt/test/tf_trt_integration_test.py | 16 +- tensorflow/contrib/tensorrt/test/unary_test.py | 125 -------------- tensorflow/contrib/tensorrt/test/unit_tests.py | 67 -------- tensorflow/contrib/tensorrt/test/utilities.py | 30 ---- .../contrib/tensorrt/test/vgg_block_nchw_test.py | 85 ---------- tensorflow/contrib/tensorrt/test/vgg_block_test.py | 76 --------- 17 files changed, 15 insertions(+), 1430 deletions(-) delete mode 100644 tensorflow/contrib/tensorrt/test/base_unit_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/batch_matmul_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/concatenation_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/const_broadcast_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/neighboring_engine_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/run_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/unary_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests.py delete mode 100644 tensorflow/contrib/tensorrt/test/utilities.py delete mode 100644 tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/vgg_block_test.py diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index d957ca0861..adda0b758b 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -341,39 +341,6 @@ py_test( ], ) -py_test( - name = "converter_unit_tests", - srcs = [ - "test/base_unit_test.py", - "test/batch_matmul_test.py", - "test/biasadd_matmul_test.py", - "test/binary_tensor_weight_broadcast_test.py", - "test/concatenation_test.py", - "test/const_broadcast_test.py", - "test/multi_connection_neighbor_engine_test.py", - "test/neighboring_engine_test.py", - "test/run_test.py", - "test/unary_test.py", - "test/unit_tests.py", - "test/utilities.py", - "test/vgg_block_nchw_test.py", - "test/vgg_block_test.py", - ], - main = "test/unit_tests.py", - srcs_version = "PY2AND3", - tags = [ - "manual", - "notap", - ], - deps = [ - ":init_py", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:layers", - "//tensorflow/python:training", - ], -) - cc_library( name = "utils", hdrs = ["convert/utils.h"], diff --git a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc index 383635f428..836238a0df 100644 --- a/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc +++ b/tensorflow/contrib/tensorrt/ops/trt_engine_op.cc @@ -42,8 +42,13 @@ REGISTER_OP("TRTEngineOp") .Attr("precision_mode: {'FP32', 'FP16', 'INT8', 'INT8CALIB'}") .Attr("calibration_data: string = ''") .Input("in_tensor: InT") - .Output("out_tensor: OutT") - .SetShapeFn(shape_inference::TRTEngineOpShapeInference); + .Output("out_tensor: OutT"); +// TODO(jie): TF requires concrete output shape for concrete input shapes. This +// is tricky for batch dimension, since we cannot ensure which input would carry +// the correct batch dimension (for the current stage of the implementation, we +// do require all input tensor to carry the same batch size, but this could +// change in the future). Hence we disable shape inference function as a WAR. +//.SetShapeFn(shape_inference::TRTEngineOpShapeInference); } // namespace tensorflow diff --git a/tensorflow/contrib/tensorrt/test/base_unit_test.py b/tensorflow/contrib/tensorrt/test/base_unit_test.py deleted file mode 100644 index 8a6c648ab6..0000000000 --- a/tensorflow/contrib/tensorrt/test/base_unit_test.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Base class to facilitate development of integration tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - - -class BaseUnitTest(object): - """Base class for unit tests in TF-TRT""" - - def __init__(self, log_file='log.txt'): - self.static_mode_list = {} - self.dynamic_mode_list = {} - self.dummy_input = None - self.get_network = None - self.expect_nb_nodes = None - self.test_name = None - self.log_file = log_file - self.ckpt = None - self.allclose_rtol = 0.01 - self.allclose_atol = 0.01 - self.allclose_equal_nan = True - # saves out graphdef - self.debug = False - # require node count check fail leads to test failure - self.check_node_count = False - - def run(self, run_test_context): - run_test_context.run_test(self.get_network, self.static_mode_list, - self.dynamic_mode_list, self.dummy_input, - self.ckpt) - return self.log_result(run_test_context) - - def log_result(self, run_test_result): - log = open(self.log_file, 'a') - log.write(("================= model: %s\n") % (self.test_name)) - - if self.debug: - open(self.test_name + "_native.pb", - 'wb').write(run_test_result.native_network.SerializeToString()) - all_success = True - if len(run_test_result.tftrt_conversion_flag) != 0: - log.write(" -- static_mode\n") - for static_mode in run_test_result.tftrt_conversion_flag: - if self.debug: - open(self.test_name + "_" + static_mode + ".pb", - 'wb').write(run_test_result.tftrt[static_mode].SerializeToString()) - log.write(" ----\n") - log.write((" mode: [%s]\n") % (static_mode)) - if run_test_result.tftrt_conversion_flag[static_mode]: - if run_test_result.tftrt_nb_nodes[static_mode] != self.expect_nb_nodes: - log.write( - ("[WARNING]: converted node number does not match (%d,%d,%d)!!!\n" - ) % (run_test_result.tftrt_nb_nodes[static_mode], - self.expect_nb_nodes, run_test_result.native_nb_nodes)) - if self.check_node_count: - all_success = False - - if np.array_equal(run_test_result.tftrt_result[static_mode], - run_test_result.native_result): - log.write(" output: equal\n") - elif np.allclose( - run_test_result.tftrt_result[static_mode], - run_test_result.native_result, - atol=self.allclose_atol, - rtol=self.allclose_rtol, - equal_nan=self.allclose_equal_nan): - log.write(" output: allclose\n") - else: - diff = run_test_result.tftrt_result[static_mode] - run_test_result.native_result - log.write("[ERROR]: output does not match!!!\n") - log.write("max diff: " + str(np.max(diff))) - log.write("\ntftrt:\n") - log.write(str(run_test_result.tftrt_result[static_mode])) - log.write("\nnative:\n") - log.write(str(run_test_result.native_result)) - log.write("\ndiff:\n") - log.write(str(diff)) - all_success = False - else: - log.write("[ERROR]: conversion failed!!!\n") - all_success = False - - if len(run_test_result.tftrt_dynamic_conversion_flag) != 0: - log.write(" -- dynamic_mode\n") - for dynamic_mode in run_test_result.tftrt_dynamic_conversion_flag: - log.write("\n ----\n") - log.write((" mode: [%s]\n") % (dynamic_mode)) - if run_test_result.tftrt_dynamic_conversion_flag[dynamic_mode]: - if np.array_equal(run_test_result.tftrt_dynamic_result[dynamic_mode], - run_test_result.native_result): - log.write(" output: equal\n") - elif np.allclose(run_test_result.tftrt_dynamic_result[dynamic_mode], - run_test_result.native_result): - log.write(" output: allclose\n") - else: - log.write("[ERROR]: output does not match!!!\n") - all_success = False - else: - log.write("[ERROR]: conversion failed!!!\n") - all_success = False - return all_success diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py deleted file mode 100644 index 3c83a3a562..0000000000 --- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.utilities import get_all_variables - - -class BatchMatMulTest(BaseUnitTest): - """Testing BatchMatMul in TF-TRT conversion""" - - def __init__(self, log_file='log.txt'): - super(BatchMatMulTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (12, 5, 8, 12) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.matmul_test - self.expect_nb_nodes = 16 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.ckpt = "./tmp.ckpt" - sess = session.Session() - - def matmul_test(self): - g = ops.Graph() - gpu_options = config_pb2.GPUOptions() - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - - b = constant_op.constant( - np.random.randn(12, 5, 12, 7), dtype=dtypes.float32) - x1 = math_ops.matmul(x, b) - b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32) - x1 = x1 + b - - var = variable_scope.get_variable( - "test", [12, 5, 12, 7], - dtype=dtypes.float32, - initializer=init_ops.truncated_normal_initializer) - x2 = math_ops.matmul(x, var) - b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32) - x2 = x2 * b - - var = variable_scope.get_variable( - "test2", [12, 84], - dtype=dtypes.float32, - initializer=init_ops.truncated_normal_initializer) - c = gen_array_ops.reshape(x, [12, 40, 12]) - b = gen_array_ops.reshape(var, [12, 12, 7]) - x3 = math_ops.matmul(c, b) - b = constant_op.constant(np.random.randn(40, 1), dtype=dtypes.float32) - x3 = x3 + b - x3 = gen_array_ops.reshape(x3, [12, 5, 8, 7]) - - out = x3 + x1 - array_ops.squeeze(out, name="output") - - with session.Session(config=sessconfig, graph=g) as sess: - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - sess.run(variables.global_variables_initializer()) - saver.save(sess, self.ckpt) - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py deleted file mode 100644 index 1ac6f5cb6a..0000000000 --- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.layers import core -from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest - - -class BiasaddMatMulTest(BaseUnitTest): - """Testing BiasAdd MatMul in TF-TRT conversion""" - - def __init__(self, log_file='log.txt'): - super(BiasaddMatMulTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (48, 12) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.matmul_test - self.expect_nb_nodes = 53 - self.log_file = log_file - self.test_name = self.__class__.__name__ - - def matmul_test(self): - g = ops.Graph() - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - - b = constant_op.constant(np.random.randn(12, 4), dtype=dtypes.float32) - x1 = math_ops.matmul(x, b) - b = constant_op.constant(np.random.randn(1, 4), dtype=dtypes.float32) - x1 = x1 + b - - b = constant_op.constant(np.random.randn(48, 4), dtype=dtypes.float32) - x2 = math_ops.matmul(x, b, transpose_a=True) - x2 = gen_array_ops.reshape(x2, [48, 1]) - - b = constant_op.constant(np.random.randn(4, 12), dtype=dtypes.float32) - x3 = math_ops.matmul(x, b, transpose_b=True) - - b = constant_op.constant(np.random.randn(16, 48), dtype=dtypes.float32) - x4 = math_ops.matmul(x, b, transpose_b=True, transpose_a=True) - x4 = gen_array_ops.reshape(x4, [48, 4]) - - x5 = gen_array_ops.reshape(x, [4, 12, 12]) - x5 = core.flatten(x5) - b = constant_op.constant(np.random.randn(144, 48), dtype=dtypes.float32) - x5 = math_ops.matmul(x5, b) - b = constant_op.constant(np.random.randn(48), dtype=dtypes.float32) - x5 = nn.bias_add(x5, b) - x5 = gen_array_ops.reshape(x5, [48, 4]) - - x6 = gen_array_ops.reshape(x, [4, 12, 12]) - b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32) - x6 = nn.bias_add(x6, b, data_format="NHWC") - x6 = gen_array_ops.reshape(x6, [48, -1]) - - x7 = gen_array_ops.reshape(x, [4, 12, 3, 4]) - b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32) - x7 = nn.bias_add(x7, b, data_format="NHWC") - x7 = gen_array_ops.reshape(x7, [48, -1]) - - x8 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) - b = constant_op.constant(np.random.randn(2), dtype=dtypes.float32) - x8 = nn.bias_add(x8, b, data_format="NHWC") - x8 = gen_array_ops.reshape(x8, [48, -1]) - - x9 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) - b = constant_op.constant(np.random.randn(3), dtype=dtypes.float32) - x9 = nn.bias_add(x9, b, data_format="NCHW") - x9 = gen_array_ops.reshape(x9, [48, -1]) - - x10 = gen_array_ops.reshape(x, [4, 12, 3, 4]) - b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32) - x10 = nn.bias_add(x10, b, data_format="NCHW") - x10 = gen_array_ops.reshape(x10, [48, -1]) - - x11 = gen_array_ops.reshape(x, [4, 12, 12]) - b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32) - x11 = nn.bias_add(x11, b, data_format="NCHW") - x11 = gen_array_ops.reshape(x11, [48, -1]) - - out = array_ops.concat( - [x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11], axis=-1) - out = array_ops.squeeze(out, name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py deleted file mode 100644 index 5233a493d0..0000000000 --- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import math_ops -from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest - - -class BinaryTensorWeightBroadcastTest(BaseUnitTest): - """unit tests for scale & elementwise layers in TF-TRT""" - - def __init__(self, log_file='log.txt'): - super(BinaryTensorWeightBroadcastTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (10, 24, 24, 20) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.get_simple_graph_def - self.expect_nb_nodes = 35 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.allclose_rtol = 0.1 - self.allclose_atol = 0.05 - - def get_simple_graph_def(self): - g = ops.Graph() - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - - # scale - a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # scale - a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # scale - a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # scale - a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # scale - a = constant_op.constant( - np.random.randn(24, 24, 20), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # scale - a = constant_op.constant( - np.random.randn(24, 24, 20), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 1, 1), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 1, 1), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 24, 1), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 24, 1), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 24, 20), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant( - np.random.randn(1, 24, 24, 20), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32) - f = a + x - x = math_ops.sigmoid(f) - - # elementwise - a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32) - f = x + a - x = math_ops.sigmoid(f) - - gen_array_ops.reshape(x, [5, -1], name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py deleted file mode 100644 index de0817d2e8..0000000000 --- a/tensorflow/contrib/tensorrt/test/concatenation_test.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest - - -class ConcatenationTest(BaseUnitTest): - """Testing Concatenation in TF-TRT conversion""" - - def __init__(self, log_file='log.txt'): - super(ConcatenationTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (2, 3, 3, 1) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.get_simple_graph_def - self.expect_nb_nodes = 4 - self.log_file = log_file - self.test_name = self.__class__.__name__ - - def get_simple_graph_def(self): - g = ops.Graph() - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - - # scale - a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) - r1 = x / a - a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) - r2 = a / x - a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32) - r3 = a + x - a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32) - r4 = x * a - a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) - r5 = x - a - a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) - r6 = a - x - a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) - r7 = x - a - a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) - r8 = a - x - a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) - r9 = gen_math_ops.maximum(x, a) - a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) - r10 = gen_math_ops.minimum(a, x) - a = constant_op.constant(np.random.randn(3), dtype=dtypes.float32) - r11 = x * a - a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) - r12 = a * x - concat1 = array_ops.concat([r1, r2, r3, r4, r5, r6], axis=-1) - concat2 = array_ops.concat([r7, r8, r9, r10, r11, r12], axis=3) - x = array_ops.concat([concat1, concat2], axis=-1) - - gen_array_ops.reshape(x, [2, -1], name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py deleted file mode 100644 index 74d39d9015..0000000000 --- a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest - - -class ConstBroadcastTest(BaseUnitTest): - """Testing Constant broadcasting in TF-TRT""" - - def __init__(self, log_file='log.txt'): - super(ConstBroadcastTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (5, 12, 12, 2) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.conv_broadcast - self.expect_nb_nodes = 7 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.allclose_rtol = 0.05 - self.allclose_atol = 0.05 - - def conv_broadcast(self): - g = ops.Graph() - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - filt1 = constant_op.constant( - 1, shape=(3, 3, 2, 1), dtype=dtypes.float32, name='filt1') - y1 = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='SAME', name='y1') - z1 = nn.relu(y1, name='z1') - filt2 = constant_op.constant( - np.random.randn(9), - shape=(3, 3, 1, 1), - dtype=dtypes.float32, - name='filt2') - y2 = nn.conv2d(z1, filt2, strides=[1, 1, 1, 1], padding='SAME', name='y2') - z2 = nn.relu(y2, name='z') - filt3 = constant_op.constant( - np.random.randn(3, 3, 1, 1), - shape=(3, 3, 1, 1), - dtype=dtypes.float32, - name='filt3') - y3 = nn.conv2d(z2, filt3, strides=[1, 1, 1, 1], padding='SAME', name='y3') - z = nn.relu(y3, name='output') - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py deleted file mode 100644 index 291b4d16c1..0000000000 --- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest - - -class MultiConnectionNeighborEngineTest(BaseUnitTest): - """Multi connection neighboring nodes wiring tests in TF-TRT""" - - def __init__(self, log_file='log.txt'): - super(MultiConnectionNeighborEngineTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (2, 3, 7, 5) - self.dummy_input = np.random.normal(1.0, 0.5, self.inp_dims) - self.get_network = self.neighboring_tensor_test - self.expect_nb_nodes = 7 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.allclose_rtol = 0.05 - self.allclose_atol = 0.05 - - def neighboring_tensor_test(self): - g = ops.Graph() - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - e = constant_op.constant( - np.random.normal(.05, .005, [3, 2, 3, 4]), - name="weights", - dtype=dtypes.float32) - conv = nn.conv2d( - input=x, - filter=e, - data_format="NCHW", - strides=[1, 1, 1, 1], - padding="VALID", - name="conv") - b = constant_op.constant( - np.random.normal(2.0, 1.0, [1, 4, 1, 1]), - name="bias", - dtype=dtypes.float32) - t = conv + b - - b = constant_op.constant( - np.random.normal(5.0, 1.0, [1, 4, 1, 1]), - name="bias", - dtype=dtypes.float32) - q = conv - b - edge = math_ops.sigmoid(q) - - b = constant_op.constant( - np.random.normal(5.0, 1.0, [1, 4, 1, 1]), - name="bias", - dtype=dtypes.float32) - d = b + conv - edge3 = math_ops.sigmoid(d) - - c = constant_op.constant( - np.random.normal(1.0, 1.0, [1, 4, 1, 1]), - name="bias", - dtype=dtypes.float32) - edge1 = gen_math_ops.tan(conv) - t = t - edge1 - q = q + edge - t = t + q - t = t + d - t = t - edge3 - array_ops.squeeze(t, name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py deleted file mode 100644 index f916db3504..0000000000 --- a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import gen_math_ops -from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest - - -class NeighboringEngineTest(BaseUnitTest): - """Neighboring node wiring tests in TF-TRT conversion""" - - def __init__(self, log_file='log.txt'): - super(NeighboringEngineTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (2, 3, 7, 5) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.neighboring_tensor_test - self.expect_nb_nodes = 5 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.allclose_rtol = 0.05 - self.allclose_atol = 0.05 - - def neighboring_tensor_test(self): - g = ops.Graph() - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - e = constant_op.constant( - np.random.normal(.3, 0.05, [3, 2, 3, 4]), - name="weights", - dtype=dtypes.float32) - conv = nn.conv2d( - input=x, - filter=e, - data_format="NCHW", - strides=[1, 1, 1, 1], - padding="VALID", - name="conv") - b = constant_op.constant( - np.random.normal(1.0, 1.0, [1, 4, 1, 1]), - name="bias", - dtype=dtypes.float32) - t = conv * b - - e = gen_math_ops.tan(conv) - t = t - e - array_ops.squeeze(t, name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/run_test.py b/tensorflow/contrib/tensorrt/test/run_test.py deleted file mode 100644 index 4d109cc378..0000000000 --- a/tensorflow/contrib/tensorrt/test/run_test.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""script to convert and execute TF-TensorRT graph.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 -from tensorflow.core.protobuf import rewriter_config_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import importer -from tensorflow.python.framework import ops -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.utilities import get_all_variables - -OUTPUT_NODE = "output" -INPUT_NODE = "input" -CALIB_COUNT = 5 # calibration iteration - - -class RunTest: - """base class to run TR-TRT conversion and execution""" - - def __init__(self): - self.clean() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.clean() - - def clean(self): - self.tftrt = {} - self.tftrt_conversion_flag = {} - self.tftrt_nb_nodes = {} - self.tftrt_result = {} - self.tftrt_dynamic_conversion_flag = {} - self.tftrt_dynamic_result = {} - self.check_file = None - self.native_network = None - - def run_test(self, - network, - static_mode_list, - dynamic_mode_list, - dummy_input, - file_name=None): - self.native_network = network() - success = True - initialization = False - if file_name != None: - initialization = True - self.check_file = file_name - self.native_result, self.native_nb_nodes = self.execute_graph( - self.native_network, dummy_input, initialization) - for mode in static_mode_list: - try: - self.run_static_convert_network(mode, dummy_input, initialization) - self.tftrt_conversion_flag[mode] = True - except Exception as inst: - self.tftrt_conversion_flag[mode] = False - success = False - for mode in dynamic_mode_list: - try: - self.run_dynamic_convert_network(mode, dummy_input, initialization) - self.tftrt_dynamic_conversion_flag[mode] = True - except Exception as inst: - self.tftrt_dynamic_conversion_flag[mode] = False - success = False - return success - - def run_dynamic_convert_network(self, mode, dummy_input, initialization=True): - inp_dims = dummy_input.shape - if mode == "FP32" or mode == "FP16": - opt_config = rewriter_config_pb2.RewriterConfig() - opt_config.optimizers.extend(["constfold", "layout"]) - custom_op = opt_config.custom_optimizers.add() - custom_op.name = "TensorRTOptimizer" - custom_op.parameter_map["minimum_segment_size"].i = 3 - custom_op.parameter_map["precision_mode"].s = mode - custom_op.parameter_map["max_batch_size"].i = inp_dims[0] - custom_op.parameter_map["max_workspace_size_bytes"].i = 1 << 25 - print(custom_op) - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - graph_options = config_pb2.GraphOptions(rewrite_options=opt_config) - sessconfig = config_pb2.ConfigProto( - gpu_options=gpu_options, graph_options=graph_options) - print(sessconfig) - g = ops.Graph() - ops.reset_default_graph() - with g.as_default(): - inp, out = importer.import_graph_def( - graph_def=self.native_network, return_elements=["input", "output"]) - inp = inp.outputs[0] - out = out.outputs[0] - with session.Session(config=sessconfig, graph=g) as sess: - if (initialization): - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - saver.restore(sess, self.check_file) - self.tftrt_dynamic_result[mode] = sess.run(out, {inp: dummy_input}) - else: - raise Exception("dynamic op mode: " + mode + " not supported") - - def run_static_convert_network(self, mode, dummy_input, initialization=True): - inp_dims = dummy_input.shape - if mode == "FP32" or mode == "FP16" or mode == "INT8": - trt_graph = trt.create_inference_graph( - input_graph_def=self.native_network, - outputs=[OUTPUT_NODE], - max_batch_size=inp_dims[0], - max_workspace_size_bytes=1 << 25, - precision_mode=mode, # TRT Engine precision "FP32","FP16" or "INT8" - minimum_segment_size=2 # minimum number of nodes in an engine - ) - if mode == "INT8": - _ = self.execute_calibration(trt_graph, dummy_input, initialization) - trt_graph = trt.calib_graph_to_infer_graph(trt_graph) - trt_result, nb_nodes = self.execute_graph(trt_graph, dummy_input, - initialization) - self.tftrt[mode] = trt_graph - self.tftrt_nb_nodes[mode] = nb_nodes - self.tftrt_result[mode] = trt_result - else: - raise Exception("mode: " + mode + " not supported") - - def execute_graph(self, gdef, dummy_input, initialization=True): - """Run given graphdef once.""" - gpu_options = config_pb2.GPUOptions() - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - ops.reset_default_graph() - g = ops.Graph() - nb_nodes = 0 - with g.as_default(): - inp, out = importer.import_graph_def( - graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") - nb_nodes = len(g.get_operations()) - inp = inp.outputs[0] - out = out.outputs[0] - with session.Session(config=sessconfig, graph=g) as sess: - if (initialization): - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - saver.restore(sess, self.check_file) - val = sess.run(out, {inp: dummy_input}) - return val, nb_nodes - - # Use real data that is representative of the inference dataset - # for calibration. For this test script it is random data. - def execute_calibration(self, gdef, dummy_input, initialization=True): - """Run given calibration graph multiple times.""" - gpu_options = config_pb2.GPUOptions() - ops.reset_default_graph() - g = ops.Graph() - with g.as_default(): - inp, out = importer.import_graph_def( - graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") - inp = inp.outputs[0] - out = out.outputs[0] - with session.Session( - config=config_pb2.ConfigProto(gpu_options=gpu_options), - graph=g) as sess: - if (initialization): - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - saver.restore(sess, self.check_file) - for _ in range(CALIB_COUNT): - val = sess.run(out, {inp: dummy_input}) - return val diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py index 854a1ae168..d9c41f90d0 100644 --- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py +++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py @@ -124,15 +124,15 @@ TestGraph = namedtuple("TestGraph", TEST_GRAPHS = { "SingleEngineGraph": - TestGraph( - gdef=GetSingleEngineGraphDef(), - num_expected_engines=1, - expected_output_dims=(100, 6, 6, 6)), + TestGraph( + gdef=GetSingleEngineGraphDef(), + num_expected_engines=1, + expected_output_dims=(100, 6, 6, 6)), "MultiEngineGraph": - TestGraph( - gdef=GetMultiEngineGraphDef(), - num_expected_engines=2, - expected_output_dims=(100, 12, 12, 6)), + TestGraph( + gdef=GetMultiEngineGraphDef(), + num_expected_engines=2, + expected_output_dims=(100, 12, 12, 6)), # TODO(aaroey): add a large complex graph to test. } diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py deleted file mode 100644 index a054939ce2..0000000000 --- a/tensorflow/contrib/tensorrt/test/unary_test.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables -from tensorflow.python.ops import init_ops -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest -from tensorflow.contrib.tensorrt.test.utilities import get_all_variables - - -class UnaryTest(BaseUnitTest): - """Unit tests for unary operations in TF-TRT""" - - def __init__(self, log_file='log.txt'): - super(UnaryTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (12, 5, 8, 1, 1, 12) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.unary_test - self.expect_nb_nodes = 17 - self.log_file = log_file - self.test_name = self.__class__.__name__ - self.ckpt = "./tmp.ckpt" - - def unary_test(self): - g = ops.Graph() - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - q = math_ops.abs(x) - q = q + 1.0 - q = gen_math_ops.exp(q) - q = gen_math_ops.log(q) - q = array_ops.squeeze(q, axis=-2) - q = math_ops.abs(q) - q = q + 2.2 - q = gen_math_ops.sqrt(q) - q = gen_math_ops.rsqrt(q) - q = math_ops.negative(q) - q = array_ops.squeeze(q, axis=3) - q = math_ops.abs(q) - q = q + 3.0 - a = gen_math_ops.reciprocal(q) - - x = constant_op.constant(np.random.randn(5, 8, 12), dtype=dtypes.float32) - q = math_ops.abs(x) - q = q + 2.0 - q = gen_math_ops.exp(q) - q = gen_math_ops.log(q) - q = math_ops.abs(q) - q = q + 2.1 - q = gen_math_ops.sqrt(q) - q = gen_math_ops.rsqrt(q) - q = math_ops.negative(q) - q = math_ops.abs(q) - q = q + 4.0 - b = gen_math_ops.reciprocal(q) - - # TODO(jie): this one will break, broadcasting on batch. - x = variable_scope.get_variable( - "test", [12, 40, 12], - dtype=dtypes.float32, - initializer=init_ops.truncated_normal_initializer) - x = gen_array_ops.reshape(x, [12, 5, 8, 1, 12, 1, 1]) - q = math_ops.abs(x) - q = q + 5.0 - q = gen_math_ops.exp(q) - q = array_ops.squeeze(q, axis=[-1, -2, 3]) - q = gen_math_ops.log(q) - q = math_ops.abs(q) - q = q + 5.1 - q = gen_array_ops.reshape(q, [12, 5, 1, 1, 8, 1, 12]) - q = array_ops.squeeze(q, axis=[5, 2, 3]) - q = gen_math_ops.sqrt(q) - q = math_ops.abs(q) - q = q + 5.2 - q = gen_math_ops.rsqrt(q) - q = math_ops.negative(q) - q = math_ops.abs(q) - q = q + 5.3 - c = gen_math_ops.reciprocal(q) - - q = a * b - q = q / c - array_ops.squeeze(q, name="output") - - with session.Session(config=sessconfig, graph=g) as sess: - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - sess.run(variables.global_variables_initializer()) - saver.save(sess, self.ckpt) - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests.py b/tensorflow/contrib/tensorrt/test/unit_tests.py deleted file mode 100644 index ac6e3b13ee..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Script to execute and log all integration tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensorrt.test.batch_matmul_test import BatchMatMulTest -from tensorflow.contrib.tensorrt.test.biasadd_matmul_test import BiasaddMatMulTest -from tensorflow.contrib.tensorrt.test.binary_tensor_weight_broadcast_test import BinaryTensorWeightBroadcastTest -from tensorflow.contrib.tensorrt.test.concatenation_test import ConcatenationTest -from tensorflow.contrib.tensorrt.test.multi_connection_neighbor_engine_test import MultiConnectionNeighborEngineTest -from tensorflow.contrib.tensorrt.test.neighboring_engine_test import NeighboringEngineTest -from tensorflow.contrib.tensorrt.test.unary_test import UnaryTest -from tensorflow.contrib.tensorrt.test.vgg_block_nchw_test import VGGBlockNCHWTest -from tensorflow.contrib.tensorrt.test.vgg_block_test import VGGBlockTest -from tensorflow.contrib.tensorrt.test.const_broadcast_test import ConstBroadcastTest - -from tensorflow.contrib.tensorrt.test.run_test import RunTest - -tests = 0 -passed_test = 0 - -failed_list = [] -test_list = [] - -test_list.append(BatchMatMulTest()) -test_list.append(BiasaddMatMulTest()) -test_list.append(BinaryTensorWeightBroadcastTest()) -test_list.append(ConcatenationTest()) -test_list.append(NeighboringEngineTest()) -test_list.append(UnaryTest()) -test_list.append(VGGBlockNCHWTest()) -test_list.append(VGGBlockTest()) -test_list.append(MultiConnectionNeighborEngineTest()) -test_list.append(ConstBroadcastTest()) - -for test in test_list: - test.debug = True - test.check_node_count = False - with RunTest() as context: - tests += 1 - if test.run(context): - passed_test += 1 - else: - failed_list.append(test.test_name) - print("Failed test: %s\n", test.test_name) - -if passed_test == tests: - print("Passed\n") -else: - print(("%d out of %d passed\n -- failed list:") % (passed_test, tests)) - for test in failed_list: - print(" - " + test) diff --git a/tensorflow/contrib/tensorrt/test/utilities.py b/tensorflow/contrib/tensorrt/test/utilities.py deleted file mode 100644 index 0ea5f5b883..0000000000 --- a/tensorflow/contrib/tensorrt/test/utilities.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities script for TF-TensorRT integration tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import variables - - -def get_all_variables(sess): - var_names = sess.run(variables.report_uninitialized_variables()) - names_var_list = {} - for name in var_names: - names_var_list[name] = sess.graph.get_tensor_by_name(name + ":0") - print(var_names) - return names_var_list diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py deleted file mode 100644 index 9a759eb994..0000000000 --- a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest - - -class VGGBlockNCHWTest(BaseUnitTest): - """single vgg layer in NCHW unit tests in TF-TRT""" - - def __init__(self, log_file='log.txt'): - super(VGGBlockNCHWTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (5, 2, 8, 8) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.get_simple_graph_def - self.expect_nb_nodes = 3 - self.log_file = log_file - self.test_name = self.__class__.__name__ - - def get_simple_graph_def(self): - g = ops.Graph() - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - x, mean_x, var_x = nn_impl.fused_batch_norm( - x, - np.random.randn(2).astype(np.float32), - np.random.randn(2).astype(np.float32), - mean=np.random.randn(2).astype(np.float32), - variance=np.random.randn(2).astype(np.float32), - data_format="NCHW", - is_training=False) - e = constant_op.constant( - np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32) - conv = nn.conv2d( - input=x, - filter=e, - data_format="NCHW", - strides=[1, 1, 2, 2], - padding="SAME", - name="conv") - b = constant_op.constant( - np.random.randn(6), name="bias", dtype=dtypes.float32) - t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd") - relu = nn.relu(t, "relu") - idty = array_ops.identity(relu, "ID") - v = nn_ops.max_pool( - idty, [1, 1, 2, 2], [1, 1, 2, 2], - "VALID", - data_format="NCHW", - name="max_pool") - array_ops.squeeze(v, name="output") - - return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py deleted file mode 100644 index 04176d58ca..0000000000 --- a/tensorflow/contrib/tensorrt/test/vgg_block_test.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.ops import nn_impl -from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest - - -class VGGBlockTest(BaseUnitTest): - """single vgg layer test in TF-TRT conversion""" - - def __init__(self, log_file='log.txt'): - super(VGGBlockTest, self).__init__() - self.static_mode_list = {"FP32", "FP16"} - self.debug = True - self.dynamic_mode_list = {} - self.inp_dims = (5, 8, 8, 2) - self.dummy_input = np.random.random_sample(self.inp_dims) - self.get_network = self.get_simple_graph_def - self.expect_nb_nodes = 7 - self.log_file = log_file - self.test_name = self.__class__.__name__ - - def get_simple_graph_def(self): - g = ops.Graph() - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - with g.as_default(): - x = array_ops.placeholder( - dtype=dtypes.float32, shape=self.inp_dims, name="input") - x, mean_x, var_x = nn_impl.fused_batch_norm( - x, - np.random.randn(2).astype(np.float32), - np.random.randn(2).astype(np.float32), - mean=np.random.randn(2).astype(np.float32), - variance=np.random.randn(2).astype(np.float32), - is_training=False) - e = constant_op.constant( - np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32) - conv = nn.conv2d( - input=x, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv") - b = constant_op.constant( - np.random.randn(6), name="bias", dtype=dtypes.float32) - t = nn.bias_add(conv, b, name="biasAdd") - relu = nn.relu(t, "relu") - idty = array_ops.identity(relu, "ID") - v = nn_ops.max_pool( - idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool") - array_ops.squeeze(v, name="output") - - return g.as_graph_def() -- cgit v1.2.3 From c82cac172297865c9bf106dfcc11195934b95478 Mon Sep 17 00:00:00 2001 From: Mahmoud Abuzaina Date: Wed, 11 Jul 2018 12:13:38 -0700 Subject: Exporting build_defs.bzl in BUILD file --- third_party/mkl_dnn/BUILD | 2 ++ third_party/mkl_dnn/mkldnn.BUILD | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/third_party/mkl_dnn/BUILD b/third_party/mkl_dnn/BUILD index 17a0074abe..6a36b50e70 100644 --- a/third_party/mkl_dnn/BUILD +++ b/third_party/mkl_dnn/BUILD @@ -1,5 +1,7 @@ licenses(["notice"]) +exports_files(["build_defs.bzl"]) + config_setting( name = "using_mkl_dnn_only", values = { diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD index 57d2e1292b..b9c12d4535 100644 --- a/third_party/mkl_dnn/mkldnn.BUILD +++ b/third_party/mkl_dnn/mkldnn.BUILD @@ -1,4 +1,7 @@ -exports_files(["LICENSE"]) +exports_files([ + "LICENSE", + "build_defs.bzl", +]) load( "@org_tensorflow//third_party/mkl_dnn:build_defs.bzl", -- cgit v1.2.3 From 47a2ac52147e9bbb41637170bf7e468367f82976 Mon Sep 17 00:00:00 2001 From: Mahmoud Abuzaina Date: Wed, 11 Jul 2018 13:21:10 -0700 Subject: Adding MKL_DNN license to BUILD files --- tensorflow/tools/lib_package/BUILD | 2 ++ tensorflow/tools/pip_package/BUILD | 1 + third_party/mkl_dnn/BUILD | 2 -- third_party/mkl_dnn/mkldnn.BUILD | 5 +---- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD index 173f418dc8..ef3eb16121 100644 --- a/tensorflow/tools/lib_package/BUILD +++ b/tensorflow/tools/lib_package/BUILD @@ -143,6 +143,7 @@ genrule( "@zlib_archive//:zlib.h", ] + if_mkl([ "//third_party/mkl:LICENSE", + "@mkl_dnn//:LICENSE", ]), outs = ["include/tensorflow/c/LICENSE"], cmd = "$(location :concat_licenses.sh) $(SRCS) >$@", @@ -182,6 +183,7 @@ genrule( "@zlib_archive//:zlib.h", ] + if_mkl([ "//third_party/mkl:LICENSE", + "@mkl_dnn//:LICENSE", ]), outs = ["include/tensorflow/jni/LICENSE"], cmd = "$(location :concat_licenses.sh) $(SRCS) >$@", diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index c9d53f46c3..cfafbcbfa1 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -168,6 +168,7 @@ filegroup( "@org_python_pypi_backports_weakref//:LICENSE", ] + if_mkl([ "//third_party/mkl:LICENSE", + "@mkl_dnn//:LICENSE", ]) + tf_additional_license_deps(), ) diff --git a/third_party/mkl_dnn/BUILD b/third_party/mkl_dnn/BUILD index 6a36b50e70..17a0074abe 100644 --- a/third_party/mkl_dnn/BUILD +++ b/third_party/mkl_dnn/BUILD @@ -1,7 +1,5 @@ licenses(["notice"]) -exports_files(["build_defs.bzl"]) - config_setting( name = "using_mkl_dnn_only", values = { diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD index b9c12d4535..57d2e1292b 100644 --- a/third_party/mkl_dnn/mkldnn.BUILD +++ b/third_party/mkl_dnn/mkldnn.BUILD @@ -1,7 +1,4 @@ -exports_files([ - "LICENSE", - "build_defs.bzl", -]) +exports_files(["LICENSE"]) load( "@org_tensorflow//third_party/mkl_dnn:build_defs.bzl", -- cgit v1.2.3 From 456aaa2fdbf821296a31f5493955f4653ae119dd Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Wed, 11 Jul 2018 18:35:43 +0000 Subject: [ROCm] Interface changes for pooling APIs in StreamExecutor Due to the design of MIOpen, the DNN library on ROCm platform, an instance of ScratchAllocator has to be passed into pooling routines. This commit address such interface changes and the implementation in CUDA StreamExecutor. --- tensorflow/stream_executor/cuda/cuda_dnn.cc | 21 ++++++---- tensorflow/stream_executor/cuda/cuda_dnn.h | 21 ++++++---- tensorflow/stream_executor/dnn.h | 21 ++++++---- tensorflow/stream_executor/stream.cc | 59 +++++++++++++++++++---------- tensorflow/stream_executor/stream.h | 21 ++++++---- 5 files changed, 94 insertions(+), 49 deletions(-) diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc index 84916385a8..c12eb1c61f 100644 --- a/tensorflow/stream_executor/cuda/cuda_dnn.cc +++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc @@ -3603,7 +3603,8 @@ bool CudnnSupport::DoPoolForward( const dnn::BatchDescriptor& input_dimensions, const DeviceMemory& input_data, const dnn::BatchDescriptor& output_dimensions, - DeviceMemory* output_data) { + DeviceMemory* output_data, + ScratchAllocator* workspace_allocator) { // Alpha is the scaling factor for input. double alpha = 1.0; // Beta is the scaling factor for output. @@ -3628,7 +3629,8 @@ bool CudnnSupport::DoPoolForward( const dnn::BatchDescriptor& input_dimensions, const DeviceMemory& input_data, const dnn::BatchDescriptor& output_dimensions, - DeviceMemory* output_data) { + DeviceMemory* output_data, + ScratchAllocator* workspace_allocator) { // Alpha is the scaling factor for input. float alpha = 1.0; // Beta is the scaling factor for output. @@ -3653,7 +3655,8 @@ bool CudnnSupport::DoPoolForward( const dnn::BatchDescriptor& input_dimensions, const DeviceMemory& input_data, const dnn::BatchDescriptor& output_dimensions, - DeviceMemory* output_data) { + DeviceMemory* output_data, + ScratchAllocator* workspace_allocator) { // Alpha is the scaling factor for input. float alpha = 1.0; // Beta is the scaling factor for output. @@ -3679,7 +3682,8 @@ bool CudnnSupport::DoPoolBackward( const dnn::BatchDescriptor& output_dimensions, const DeviceMemory& output_data, const DeviceMemory& input_diff_data, - DeviceMemory* output_diff_data) { + DeviceMemory* output_diff_data, + ScratchAllocator* workspace_allocator) { // Alpha is the scaling factor for input. double alpha = 1.0; // Beta is the scaling factor for output. @@ -3708,7 +3712,8 @@ bool CudnnSupport::DoPoolBackward( const dnn::BatchDescriptor& output_dimensions, const DeviceMemory& output_data, const DeviceMemory& input_diff_data, - DeviceMemory* output_diff_data) { + DeviceMemory* output_diff_data, + ScratchAllocator* workspace_allocator) { // Alpha is the scaling factor for input. float alpha = 1.0; // Beta is the scaling factor for output. @@ -3737,7 +3742,8 @@ bool CudnnSupport::DoPoolBackward( const dnn::BatchDescriptor& output_dimensions, const DeviceMemory& output_data, const DeviceMemory& input_diff_data, - DeviceMemory* output_diff_data) { + DeviceMemory* output_diff_data, + ScratchAllocator* workspace_allocator) { // Alpha is the scaling factor for input. float alpha = 1.0; // Beta is the scaling factor for output. @@ -3806,7 +3812,8 @@ bool CudnnSupport::DoNormalizeBackwardWithDimensions( const dnn::BatchDescriptor& dimensions, const DeviceMemory& raw_data, const DeviceMemory& normalized_data, const DeviceMemory& normalized_variable_gradient, - DeviceMemory* raw_variable_gradient) { + DeviceMemory* raw_variable_gradient, + ScratchAllocator* workspace_allocator) { // Check for unsupported modes. if (normalize_descriptor.wrap_around()) { LOG(ERROR) << "CUDA LRN does not support cudnn-around mode"; diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h index c924d41cb5..77f9223710 100644 --- a/tensorflow/stream_executor/cuda/cuda_dnn.h +++ b/tensorflow/stream_executor/cuda/cuda_dnn.h @@ -515,21 +515,24 @@ class CudnnSupport : public dnn::DnnSupport { const dnn::BatchDescriptor& input_dimensions, const DeviceMemory& input_data, const dnn::BatchDescriptor& output_dimensions, - DeviceMemory* output_data) override; + DeviceMemory* output_data, + ScratchAllocator* workspace_allocator = nullptr) override; bool DoPoolForward(Stream* stream, const dnn::PoolingDescriptor& pooling_dimensions, const dnn::BatchDescriptor& input_dimensions, const DeviceMemory& input_data, const dnn::BatchDescriptor& output_dimensions, - DeviceMemory* output_data) override; + DeviceMemory* output_data, + ScratchAllocator* workspace_allocator = nullptr) override; bool DoPoolForward(Stream* stream, const dnn::PoolingDescriptor& pooling_dimensions, const dnn::BatchDescriptor& input_dimensions, const DeviceMemory& input_data, const dnn::BatchDescriptor& output_dimensions, - DeviceMemory* output_data) override; + DeviceMemory* output_data, + ScratchAllocator* workspace_allocator = nullptr) override; bool DoPoolBackward(Stream* stream, const dnn::PoolingDescriptor& pooling_dimensions, @@ -538,7 +541,8 @@ class CudnnSupport : public dnn::DnnSupport { const dnn::BatchDescriptor& output_dimensions, const DeviceMemory& output_data, const DeviceMemory& input_diff_data, - DeviceMemory* output_diff_data) override; + DeviceMemory* output_diff_data, + ScratchAllocator* workspace_allocator = nullptr) override; bool DoPoolBackward(Stream* stream, const dnn::PoolingDescriptor& pooling_dimensions, @@ -547,7 +551,8 @@ class CudnnSupport : public dnn::DnnSupport { const dnn::BatchDescriptor& output_dimensions, const DeviceMemory& output_data, const DeviceMemory& input_diff_data, - DeviceMemory* output_diff_data) override; + DeviceMemory* output_diff_data, + ScratchAllocator* workspace_allocator = nullptr) override; bool DoPoolBackward(Stream* stream, const dnn::PoolingDescriptor& pooling_dimensions, @@ -556,7 +561,8 @@ class CudnnSupport : public dnn::DnnSupport { const dnn::BatchDescriptor& output_dimensions, const DeviceMemory& output_data, const DeviceMemory& input_diff_data, - DeviceMemory* output_diff_data) override; + DeviceMemory* output_diff_data, + ScratchAllocator* workspace_allocator = nullptr) override; bool DoNormalize(Stream* stream, const dnn::NormalizeDescriptor& normalize_descriptor, @@ -575,7 +581,8 @@ class CudnnSupport : public dnn::DnnSupport { const DeviceMemory& raw_data, const DeviceMemory& normalized_data, const DeviceMemory& normalized_variable_gradient, - DeviceMemory* raw_variable_gradient) override; + DeviceMemory* raw_variable_gradient, + ScratchAllocator* workspace_allocator = nullptr) override; bool DoDepthConcatenate( Stream* stream, port::ArraySlice input_dimensions, diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h index 9eca5abe1a..75705e2b49 100644 --- a/tensorflow/stream_executor/dnn.h +++ b/tensorflow/stream_executor/dnn.h @@ -1552,14 +1552,16 @@ class DnnSupport { const dnn::BatchDescriptor& input_dimensions, const DeviceMemory& input_data, const dnn::BatchDescriptor& output_dimensions, - DeviceMemory* output_data) = 0; + DeviceMemory* output_data, + ScratchAllocator* workspace_allocator = nullptr) = 0; virtual bool DoPoolForward(Stream* stream, const dnn::PoolingDescriptor& pooling_dimensions, const dnn::BatchDescriptor& input_dimensions, const DeviceMemory& input_data, const dnn::BatchDescriptor& output_dimensions, - DeviceMemory* output_data) { + DeviceMemory* output_data, + ScratchAllocator* workspace_allocator = nullptr) { LOG(FATAL) << "DoPoolForward not implemented for double."; return false; } @@ -1569,7 +1571,8 @@ class DnnSupport { const dnn::BatchDescriptor& input_dimensions, const DeviceMemory& input_data, const dnn::BatchDescriptor& output_dimensions, - DeviceMemory* output_data) { + DeviceMemory* output_data, + ScratchAllocator* workspace_allocator = nullptr) { LOG(FATAL) << "DoPoolForward not implemented for float16."; return false; } @@ -1582,7 +1585,8 @@ class DnnSupport { const dnn::BatchDescriptor& output_dimensions, const DeviceMemory& output_data, const DeviceMemory& input_diff_data, - DeviceMemory* output_diff_data) { + DeviceMemory* output_diff_data, + ScratchAllocator* workspace_allocator = nullptr) { LOG(FATAL) << "DoPoolBackward not implemented."; return false; } @@ -1594,7 +1598,8 @@ class DnnSupport { const dnn::BatchDescriptor& output_dimensions, const DeviceMemory& output_data, const DeviceMemory& input_diff_data, - DeviceMemory* output_diff_data) { + DeviceMemory* output_diff_data, + ScratchAllocator* workspace_allocator = nullptr) { LOG(FATAL) << "DoPoolBackward not implemented."; return false; } @@ -1606,7 +1611,8 @@ class DnnSupport { const dnn::BatchDescriptor& output_dimensions, const DeviceMemory& output_data, const DeviceMemory& input_diff_data, - DeviceMemory* output_diff_data) { + DeviceMemory* output_diff_data, + ScratchAllocator* workspace_allocator = nullptr) { LOG(FATAL) << "DoPoolBackward not implemented."; return false; } @@ -1653,7 +1659,8 @@ class DnnSupport { const DeviceMemory& raw_data, const DeviceMemory& normalized_data, const DeviceMemory& normalized_variable_gradient, - DeviceMemory* raw_variable_gradient) { + DeviceMemory* raw_variable_gradient, + ScratchAllocator* workspace_allocator = nullptr) { return false; } diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc index 9369183133..3c285a9416 100644 --- a/tensorflow/stream_executor/stream.cc +++ b/tensorflow/stream_executor/stream.cc @@ -1377,15 +1377,17 @@ Stream &Stream::ThenPoolForward( const dnn::BatchDescriptor &input_dimensions, const DeviceMemory &input_data, const dnn::BatchDescriptor &output_dimensions, - DeviceMemory *output_data) { + DeviceMemory *output_data, + ScratchAllocator *workspace_allocator) { VLOG_CALL(PARAM(pooling_dimensions), PARAM(input_dimensions), - PARAM(input_data), PARAM(output_dimensions), PARAM(output_data)); + PARAM(input_data), PARAM(output_dimensions), PARAM(output_data), + PARAM(workspace_allocator)); if (ok()) { if (dnn::DnnSupport *dnn = parent_->AsDnn()) { CheckError(dnn->DoPoolForward(this, pooling_dimensions, input_dimensions, input_data, output_dimensions, - output_data)); + output_data, workspace_allocator)); } else { SetError(); LOG(WARNING) @@ -1401,15 +1403,17 @@ Stream &Stream::ThenPoolForward( const dnn::BatchDescriptor &input_dimensions, const DeviceMemory &input_data, const dnn::BatchDescriptor &output_dimensions, - DeviceMemory *output_data) { + DeviceMemory *output_data, + ScratchAllocator *workspace_allocator) { VLOG_CALL(PARAM(pooling_dimensions), PARAM(input_dimensions), - PARAM(input_data), PARAM(output_dimensions), PARAM(output_data)); + PARAM(input_data), PARAM(output_dimensions), PARAM(output_data), + PARAM(workspace_allocator)); if (ok()) { if (dnn::DnnSupport *dnn = parent_->AsDnn()) { CheckError(dnn->DoPoolForward(this, pooling_dimensions, input_dimensions, input_data, output_dimensions, - output_data)); + output_data, workspace_allocator)); } else { SetErrorAndLogNoDnnSupport(); } @@ -1422,15 +1426,17 @@ Stream &Stream::ThenPoolForward( const dnn::BatchDescriptor &input_dimensions, const DeviceMemory &input_data, const dnn::BatchDescriptor &output_dimensions, - DeviceMemory *output_data) { + DeviceMemory *output_data, + ScratchAllocator *workspace_allocator) { VLOG_CALL(PARAM(pooling_dimensions), PARAM(input_dimensions), - PARAM(input_data), PARAM(output_dimensions), PARAM(output_data)); + PARAM(input_data), PARAM(output_dimensions), PARAM(output_data), + PARAM(workspace_allocator)); if (ok()) { if (dnn::DnnSupport *dnn = parent_->AsDnn()) { CheckError(dnn->DoPoolForward(this, pooling_dimensions, input_dimensions, input_data, output_dimensions, - output_data)); + output_data, workspace_allocator)); } else { SetErrorAndLogNoDnnSupport(); } @@ -1445,16 +1451,19 @@ Stream &Stream::ThenPoolBackward( const dnn::BatchDescriptor &output_dimensions, const DeviceMemory &output_data, const DeviceMemory &input_diff_data, - DeviceMemory *output_diff_data) { + DeviceMemory *output_diff_data, + ScratchAllocator *workspace_allocator) { VLOG_CALL(PARAM(pooling_dimensions), PARAM(input_dimensions), PARAM(input_data), PARAM(output_dimensions), PARAM(output_data), - PARAM(input_diff_data), PARAM(output_diff_data)); + PARAM(input_diff_data), PARAM(output_diff_data), + PARAM(workspace_allocator)); if (ok()) { if (dnn::DnnSupport *dnn = parent_->AsDnn()) { CheckError(dnn->DoPoolBackward(this, pooling_dimensions, input_dimensions, input_data, output_dimensions, output_data, - input_diff_data, output_diff_data)); + input_diff_data, output_diff_data, + workspace_allocator)); } else { SetError(); LOG(WARNING) @@ -1472,16 +1481,19 @@ Stream &Stream::ThenPoolBackward( const dnn::BatchDescriptor &output_dimensions, const DeviceMemory &output_data, const DeviceMemory &input_diff_data, - DeviceMemory *output_diff_data) { + DeviceMemory *output_diff_data, + ScratchAllocator *workspace_allocator) { VLOG_CALL(PARAM(pooling_dimensions), PARAM(input_dimensions), PARAM(input_data), PARAM(output_dimensions), PARAM(output_data), - PARAM(input_diff_data), PARAM(output_diff_data)); + PARAM(input_diff_data), PARAM(output_diff_data), + PARAM(workspace_allocator)); if (ok()) { if (dnn::DnnSupport *dnn = parent_->AsDnn()) { CheckError(dnn->DoPoolBackward(this, pooling_dimensions, input_dimensions, input_data, output_dimensions, output_data, - input_diff_data, output_diff_data)); + input_diff_data, output_diff_data, + workspace_allocator)); } else { SetErrorAndLogNoDnnSupport(); } @@ -1496,16 +1508,19 @@ Stream &Stream::ThenPoolBackward( const dnn::BatchDescriptor &output_dimensions, const DeviceMemory &output_data, const DeviceMemory &input_diff_data, - DeviceMemory *output_diff_data) { + DeviceMemory *output_diff_data, + ScratchAllocator *workspace_allocator) { VLOG_CALL(PARAM(pooling_dimensions), PARAM(input_dimensions), PARAM(input_data), PARAM(output_dimensions), PARAM(output_data), - PARAM(input_diff_data), PARAM(output_diff_data)); + PARAM(input_diff_data), PARAM(output_diff_data), + PARAM(workspace_allocator)); if (ok()) { if (dnn::DnnSupport *dnn = parent_->AsDnn()) { CheckError(dnn->DoPoolBackward(this, pooling_dimensions, input_dimensions, input_data, output_dimensions, output_data, - input_diff_data, output_diff_data)); + input_diff_data, output_diff_data, + workspace_allocator)); } else { SetErrorAndLogNoDnnSupport(); } @@ -1552,16 +1567,18 @@ Stream &Stream::ThenNormalizeBackwardWithDimensions( const dnn::BatchDescriptor &dimensions, const DeviceMemory &raw_data, const DeviceMemory &normalized_data, const DeviceMemory &normalized_variable_gradient, - DeviceMemory *raw_variable_gradient) { + DeviceMemory *raw_variable_gradient, + ScratchAllocator *workspace_allocator) { VLOG_CALL(PARAM(normalize_descriptor), PARAM(dimensions), PARAM(raw_data), PARAM(normalized_data), PARAM(normalized_variable_gradient), - PARAM(raw_variable_gradient)); + PARAM(raw_variable_gradient), PARAM(workspace_allocator)); if (ok()) { if (dnn::DnnSupport *dnn = parent_->AsDnn()) { CheckError(dnn->DoNormalizeBackwardWithDimensions( this, normalize_descriptor, dimensions, raw_data, normalized_data, - normalized_variable_gradient, raw_variable_gradient)); + normalized_variable_gradient, raw_variable_gradient, + workspace_allocator)); } else { SetErrorAndLogNoDnnSupport(); } diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h index e8885e1eb6..63d64947c8 100644 --- a/tensorflow/stream_executor/stream.h +++ b/tensorflow/stream_executor/stream.h @@ -629,19 +629,22 @@ class Stream { const dnn::BatchDescriptor &input_dimensions, const DeviceMemory &input_data, const dnn::BatchDescriptor &output_dimensions, - DeviceMemory *output_data); + DeviceMemory *output_data, + ScratchAllocator *workspace_allocator = nullptr); Stream &ThenPoolForward(const dnn::PoolingDescriptor &pooling_dimensions, const dnn::BatchDescriptor &input_dimensions, const DeviceMemory &input_data, const dnn::BatchDescriptor &output_dimensions, - DeviceMemory *output_data); + DeviceMemory *output_data, + ScratchAllocator *workspace_allocator = nullptr); Stream &ThenPoolForward(const dnn::PoolingDescriptor &pooling_dimensions, const dnn::BatchDescriptor &input_dimensions, const DeviceMemory &input_data, const dnn::BatchDescriptor &output_dimensions, - DeviceMemory *output_data); + DeviceMemory *output_data, + ScratchAllocator *workspace_allocator = nullptr); Stream &ThenPoolBackward(const dnn::PoolingDescriptor &pooling_dimensions, const dnn::BatchDescriptor &input_dimensions, @@ -649,7 +652,8 @@ class Stream { const dnn::BatchDescriptor &output_dimensions, const DeviceMemory &output_data, const DeviceMemory &input_diff_data, - DeviceMemory *output_diff_data); + DeviceMemory *output_diff_data, + ScratchAllocator *workspace_allocator = nullptr); Stream &ThenPoolBackward(const dnn::PoolingDescriptor &pooling_dimensions, const dnn::BatchDescriptor &input_dimensions, @@ -657,7 +661,8 @@ class Stream { const dnn::BatchDescriptor &output_dimensions, const DeviceMemory &output_data, const DeviceMemory &input_diff_data, - DeviceMemory *output_diff_data); + DeviceMemory *output_diff_data, + ScratchAllocator *workspace_allocator = nullptr); Stream &ThenPoolBackward(const dnn::PoolingDescriptor &pooling_dimensions, const dnn::BatchDescriptor &input_dimensions, @@ -665,7 +670,8 @@ class Stream { const dnn::BatchDescriptor &output_dimensions, const DeviceMemory &output_data, const DeviceMemory &input_diff_data, - DeviceMemory *output_diff_data); + DeviceMemory *output_diff_data, + ScratchAllocator *workspace_allocator = nullptr); Stream &ThenNormalize(const dnn::NormalizeDescriptor &normalize_descriptor, const DeviceMemory &input_data, @@ -684,7 +690,8 @@ class Stream { const DeviceMemory &raw_data, const DeviceMemory &normalized_data, const DeviceMemory &normalized_variable_gradient, - DeviceMemory *raw_variable_gradient); + DeviceMemory *raw_variable_gradient, + ScratchAllocator *workspace_allocator = nullptr); Stream &ThenActivate(dnn::ActivationMode activation_mode, const dnn::BatchDescriptor &dimensions, -- cgit v1.2.3 From 03ab64c3a68f3b990bf690ede06e3066ad4e35a0 Mon Sep 17 00:00:00 2001 From: Mahmoud Abuzaina Date: Wed, 11 Jul 2018 16:10:24 -0700 Subject: Adding LICENSE file to mkl_dnn --- tensorflow/tools/lib_package/BUILD | 4 ++-- tensorflow/tools/pip_package/BUILD | 2 +- third_party/mkl_dnn/BUILD | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD index ef3eb16121..44d8a37a8f 100644 --- a/tensorflow/tools/lib_package/BUILD +++ b/tensorflow/tools/lib_package/BUILD @@ -143,7 +143,7 @@ genrule( "@zlib_archive//:zlib.h", ] + if_mkl([ "//third_party/mkl:LICENSE", - "@mkl_dnn//:LICENSE", + "//third_party/mkl_dnn:LICENSE", ]), outs = ["include/tensorflow/c/LICENSE"], cmd = "$(location :concat_licenses.sh) $(SRCS) >$@", @@ -183,7 +183,7 @@ genrule( "@zlib_archive//:zlib.h", ] + if_mkl([ "//third_party/mkl:LICENSE", - "@mkl_dnn//:LICENSE", + "//third_party/mkl_dnn:LICENSE", ]), outs = ["include/tensorflow/jni/LICENSE"], cmd = "$(location :concat_licenses.sh) $(SRCS) >$@", diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index cfafbcbfa1..bff3042990 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -168,7 +168,7 @@ filegroup( "@org_python_pypi_backports_weakref//:LICENSE", ] + if_mkl([ "//third_party/mkl:LICENSE", - "@mkl_dnn//:LICENSE", + "//third_party/mkl_dnn:LICENSE", ]) + tf_additional_license_deps(), ) diff --git a/third_party/mkl_dnn/BUILD b/third_party/mkl_dnn/BUILD index 17a0074abe..d075809ee9 100644 --- a/third_party/mkl_dnn/BUILD +++ b/third_party/mkl_dnn/BUILD @@ -1,5 +1,7 @@ licenses(["notice"]) +exports_files(["LICENSE"]) + config_setting( name = "using_mkl_dnn_only", values = { -- cgit v1.2.3 From 954c5a95783740cf4b65f6f806dd1fb2e07d3f62 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 12 Jul 2018 03:29:08 +0000 Subject: Fix incorrect link in security advisory TFSA-2018-001 This fix fixes the issue raised in 20722 where the commit link in security advisory TFSA-2018-001 was incorrect. This fix fixes 20722. Signed-off-by: Yong Tang --- tensorflow/security/advisory/tfsa-2018-001.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/security/advisory/tfsa-2018-001.md b/tensorflow/security/advisory/tfsa-2018-001.md index bb97543a21..1966789c84 100644 --- a/tensorflow/security/advisory/tfsa-2018-001.md +++ b/tensorflow/security/advisory/tfsa-2018-001.md @@ -22,7 +22,7 @@ TensorFlow 1.3.0, 1.3.1, 1.4.0, 1.4.1, 1.5.0, 1.5.1, 1.6.0 ### Mitigation We have patched the vulnerability in GitHub commit -[49f73c55](https://github.com/tensorflow/tensorflow/commit/49f73c55d56edffebde4bca4a407ad69c1cae4333c55). +[49f73c55](https://github.com/tensorflow/tensorflow/commit/49f73c55d56edffebde4bca4a407ad69c1cae433). If users are running TensorFlow in production or on untrusted data, they are encouraged to apply this patch. -- cgit v1.2.3 From b87e1f58ead4c6bc906b29092fd486e89a578d2e Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 12 Jul 2018 06:08:42 +0000 Subject: Fix invalid link in security advisories page The link in security advisories page was invalid, should be `[SECURITY.md](https://...)` instead of `(https://...)[SECURITY.md]`. This fix correct the link issue. Signed-off-by: Yong Tang --- tensorflow/security/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/security/index.md b/tensorflow/security/index.md index ea39e17ab2..0f176151c2 100644 --- a/tensorflow/security/index.md +++ b/tensorflow/security/index.md @@ -4,7 +4,7 @@ We regularly publish security advisories about using TensorFlow. *Note*: In conjunction with these security advisories, we strongly encourage TensorFlow users to read and understand TensorFlow's security model as outlined -in (https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md)[SECURITY.md]. +in [SECURITY.md](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md). | Advisory Number | Type | Versions affected | Reported by | Additional Information | |-----------------|--------------------|:-----------------:|-----------------------|-----------------------------| -- cgit v1.2.3 From 571d3dc5747e04fe0a80be185e64532cf74e1fb0 Mon Sep 17 00:00:00 2001 From: gracehoney <31743510+aaroey@users.noreply.github.com> Date: Wed, 11 Jul 2018 23:54:26 -0700 Subject: Let segmenter able to remove ineligible input/output nodes. --- .../contrib/tensorrt/convert/convert_graph.cc | 9 +- .../contrib/tensorrt/convert/convert_nodes.cc | 8 + .../contrib/tensorrt/convert/convert_nodes.h | 4 + tensorflow/contrib/tensorrt/segment/segment.cc | 186 +++++++-- tensorflow/contrib/tensorrt/segment/segment.h | 20 +- .../contrib/tensorrt/segment/segment_test.cc | 460 ++++++++------------- 6 files changed, 338 insertions(+), 349 deletions(-) diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc index 1c4fd4a0ce..359fac36f5 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc @@ -107,8 +107,10 @@ bool IsTensorRTCandidate(const tensorflow::Node* node) { // TODO(ben,jie): ... }; // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h) - return (candidate_ops.count(node->type_string()) || - PluginFactoryTensorRT::GetInstance()->IsPlugin(node->type_string())); + if (!candidate_ops.count(node->type_string()) && + !PluginFactoryTensorRT::GetInstance()->IsPlugin(node->type_string())) { + return false; + } } tensorflow::Status BuildNodeMap( @@ -720,7 +722,8 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) { segment_options.minimum_segment_size = params.minimum_segment_size; tensorflow::tensorrt::segment::SegmentNodesVector initial_segments; TF_RETURN_IF_ERROR(tensorrt::segment::SegmentGraph( - &graph, IsTensorRTCandidate, segment_options, &initial_segments)); + &graph, IsTensorRTCandidate, IsTensorRTInputCandidate, + IsTensorRTOutputCandidate, segment_options, &initial_segments)); if (initial_segments.size() > 1) { VLOG(0) << "MULTIPLE tensorrt candidate conversion: " << initial_segments.size(); diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index 146b9c7344..8f6656e4ad 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -2391,6 +2391,14 @@ tensorflow::Status ConvertSegmentToGraphDef( return tensorflow::Status::OK(); } +bool IsTensorRTInputCandidate(const tensorflow::Node* node) { + return true; +} + +bool IsTensorRTOutputCandidate(const tensorflow::Node* node) { + return true; +} + } // namespace convert } // namespace tensorrt } // namespace tensorflow diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h index 7684d8d4a2..872ba6a080 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h @@ -128,6 +128,10 @@ tensorflow::Status ConvertGraphDefToEngine( TrtUniquePtrType* engine, bool* convert_successfully); +bool IsTensorRTInputCandidate(const tensorflow::Node* node); + +bool IsTensorRTOutputCandidate(const tensorflow::Node* node); + } // namespace convert } // namespace tensorrt } // namespace tensorflow diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc index cc42913eca..5c0898b29a 100644 --- a/tensorflow/contrib/tensorrt/segment/segment.cc +++ b/tensorflow/contrib/tensorrt/segment/segment.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/contrib/tensorrt/segment/segment.h" +#include #include #include #include @@ -32,6 +33,7 @@ namespace tensorflow { namespace tensorrt { namespace segment { using ::tensorflow::strings::StrAppend; + // A simple graph representation to mirror tensorflow::Graph. This structure // helps saving memory since segmenter modifies the graph in place, preventing // the need to create a copy of the graph. It is composed of edges and nodes. @@ -215,7 +217,7 @@ namespace { bool CheckCycles(const std::unique_ptr& g, const SimpleNode* src, const std::vector& start) { - // copied from TF ReverseDFS. + // Copied from TF ReverseDFS, which only works for tensorflow::Graph. struct Work { SimpleNode* node; bool leave; // Are we entering or leaving n? @@ -269,6 +271,24 @@ bool CanContractEdge(const SimpleEdge* edge, // 1. Get all nodes incoming to 'dst', excluding 'src' // 2. Reverse DFS from those nodes // 3. If reverse DFS reaches 'src' then we have a cycle + // + // TODO(aaroey): there are several problems with the current approach: + // 1. src->dst->src, this is not detected but it should be; + // 2. src->dst->...(any node sequence that doesn't contain src)...->dst, this + // is detected but it should not be. + // + // Note that it's fine that dst connects back to src indirectly (i.e. through + // a path with length > 1 that consists of intermedia nodes other than src). + // While loops is one example. + // + // The goal is to make sure that the trt subgraph: + // 1. has no loops (i.e. is a DAG), and + // 2. if there is a path in the subgraph from X to Y (X and Y are both nodes + // in the subgraph), then all paths from X to Y are in the subgraph. + // + // To achieve this goal, the correct way seems to be: + // 1. remove any direct edge from src->dst; + // 2. detect if src can reach dst, if so they cannot be merged. std::vector dfs_start_nodes; for (SimpleNode* node : dst->in_nodes()) { if (node != src) { @@ -276,8 +296,8 @@ bool CanContractEdge(const SimpleEdge* edge, } } - bool is_cycle = CheckCycles(graph, src, dfs_start_nodes); - return !is_cycle; + const bool has_cycle = CheckCycles(graph, src, dfs_start_nodes); + return !has_cycle; } } // namespace @@ -342,22 +362,20 @@ void ContractEdge(SimpleEdge* edge, SimpleGraph* graph, } tensorflow::Status SegmentGraph( - const tensorflow::GraphDef& gdef, - const std::function& candidate_fn, - const SegmentOptions& options, SegmentNodesVector* segments) { - // Create a Graph representation of the GraphDef. - tensorflow::FunctionLibraryDefinition flib(tensorflow::OpRegistry::Global(), - gdef.library()); - tensorflow::Graph graph(flib); - TF_RETURN_IF_ERROR(tensorflow::ConvertGraphDefToGraph( - tensorflow::GraphConstructorOptions(), gdef, &graph)); - return SegmentGraph(&graph, candidate_fn, options, segments); -} - -tensorflow::Status SegmentGraph( - tensorflow::Graph* tf_graph, + const tensorflow::Graph* tf_graph, const std::function& candidate_fn, + const std::function& input_candidate_fn, + const std::function& output_candidate_fn, const SegmentOptions& options, SegmentNodesVector* segments) { + // Steps: + // 1. run the segmentation algorithm to find all the segments, which uses + // candidate_fn to determine the candidates segment nodes; + // 2. for each segments, remove the nodes that are inputs/outputs of the + // segment but are not eligible, using input/output_candidate_fn to + // determine the eligibilities; + // 3. convert the segment into expected return format and return the result. + + // --------------------------------- Step 1 --------------------------------- auto graph = std::unique_ptr(new SimpleGraph(tf_graph)); // Use a union-find to collect the nodes that belong to the same // segment. A node value of nullptr indicates that the node is not a candidate @@ -372,14 +390,19 @@ tensorflow::Status SegmentGraph( node_segments.emplace_back(node); } - // The segmentation algorithm below visits nodes in reverse - // topological order and attempts to merge nodes along output - // edges. That means that subgraphs grow from the output-side of the - // network towards the inputs. In general this is not guaranteed to - // produce a globally optimal segmentation. In the future if we have - // a measure of how beneficial it is to include a given node in a - // TRT subgraph then we can revisit this algorithm to take advantage - // of that information. + // The segmentation algorithm below visits nodes in reverse topological order + // and attempts to merge nodes along output edges. That means that subgraphs + // grow from the output-side of the network towards the inputs. + // + // In general this is not guaranteed to produce a globally optimal + // segmentation. For exaample, consider graph with node {A, B, C, D} and edges + // {A->B, A->C, B->D, C->D), where A, B, D are trt compatible but C is not, so + // in theory we can choose to contract either A, B or B, D but not both, but + // here it always choose to contract B, D. + // + // In the future if we have a measure of how beneficial it is to include a + // given node in a TRT subgraph then we can revisit this algorithm to take + // advantage of that information. std::vector tforder; tensorflow::GetPostOrder(*tf_graph, &tforder); // use postorder implementation from tensorflow and construct mirror in @@ -392,13 +415,11 @@ tensorflow::Status SegmentGraph( for (const SimpleNode* node : order) { // All output nodes of 'node' have been visited... VLOG(2) << "Trying node " << node->name() << " id=" << node->id(); - // 'node' must be a TRT candidate... if (node_segments[node->id()].Value() == nullptr) { VLOG(2) << "... not a TRT candidate"; continue; } - // Contract output edges to combine 'node' with output // nodes. Iterate since combining two nodes may unblock other // combining. @@ -416,7 +437,6 @@ tensorflow::Status SegmentGraph( VLOG(2) << "... ... not a TRT candidate"; continue; } - if (CanContractEdge(out_edge, graph)) { VLOG(2) << "... ... can contract"; contract_edges.insert(out_edge); @@ -424,11 +444,9 @@ tensorflow::Status SegmentGraph( VLOG(2) << "... ... cannot contract, would form cycle"; } } - if (contract_edges.empty()) { break; } - // Contract edges and collect the adjacent nodes into the same // segment/subgraph. while (!contract_edges.empty()) { @@ -457,11 +475,22 @@ tensorflow::Status SegmentGraph( // Collect the segments/subgraphs. Each subgraph is represented by a // set of the names of the nodes in that subgraph. - std::unordered_map> sg_map; + + // A map from the segment identifier (currently the name of the root node of + // the segment tree) to the segment nodes set. + std::unordered_map> sg_map; + + // A map from the segment identifier (currently the name of the root node of + // the segment tree) to the device names that the nodes in the segment are + // assigned to. + // + // TODO(aaroey): nodes assigned to different devices should not be merged, + // fix this. std::unordered_map> device_maps; + for (auto& u : node_segments) { if ((u.Value() != nullptr) && (u.ParentValue() != nullptr)) { - sg_map[u.ParentValue()->name()].insert(u.Value()->name()); + sg_map[u.ParentValue()->name()].insert(u.Value()->tf_node()); auto tf_node = u.Value()->tf_node(); // has_assigned_device_name() is expected to return true // when called from optimization pass. However, since graph @@ -482,25 +511,104 @@ tensorflow::Status SegmentGraph( } } + // --------------------------------- Step 2 --------------------------------- + // Remove ineligible input/output nodes. + for (auto& itr : sg_map) { + std::set& segment_nodes = itr.second; + VLOG(1) << "Segment original size: " << segment_nodes.size(); + while (true) { + std::deque in_nodes_que, out_nodes_que; + // Find an input node that is not eligible and add it to the queue. + // Nodes that has no incoming edges should not be treated as "input", + // as there are really no inputs to them. Similar for output nodes. + for (auto node : segment_nodes) { + bool added = false; + for (const tensorflow::Edge* edge : node->in_edges()) { + if (!edge->IsControlEdge() && !edge->src()->IsSource() && + !segment_nodes.count(edge->src())) { // 'node' is an input node. + if (!input_candidate_fn(node)) { + in_nodes_que.push_back(node); + added = true; + break; + } + } + } + if (added) continue; // Only adding the node once to either queue. + for (const tensorflow::Edge* edge : node->out_edges()) { + if (!edge->dst()->IsSink() && !edge->IsControlEdge() && + !segment_nodes.count(edge->dst())) { // 'node' is an output node. + if (!output_candidate_fn(node)) { + out_nodes_que.push_back(node); + break; + } + } + } + } + if (in_nodes_que.empty() && out_nodes_que.empty()) { + // No more ineligible input/output nodes. + break; + } + // Now for each ineligible node, remove all of its inputs or outputs from + // the subgraph. + // + // It can be proven that, if the original subgraph: + // 1. is a DAG, and + // 2. all paths between two nodes in the subgraph are all inside the + // subgraph + // then after doing this operation the resulting subgraph will keep the + // same properties 1 and 2. + // + // For simplicity we use heuristics: for input nodes remove all its + // input, for output nodes remove all its output. In this way, for common + // cases the number of removed nodes should be minimum. + auto remove_nodes = [&segment_nodes]( + bool is_input_nodes, + std::deque* que) { + // Run a BFS on the queue to find all the input/output nodes. + std::set visited; + while (!que->empty()) { + auto node = que->front(); + que->pop_front(); + if (!visited.insert(node).second) continue; + segment_nodes.erase(node); + for (auto in : is_input_nodes ? node->in_nodes() : node->out_nodes()) { + if (segment_nodes.count(in)) { + que->push_back(in); + VLOG(2) << "Need to remove node " << in->name() + << " because one of its " + << (is_input_nodes ? "output" : "input") + << " nodes in the graph was removed: " << node->name(); + } + } + } + }; + remove_nodes(true, &in_nodes_que); + remove_nodes(false, &out_nodes_que); + } + VLOG(1) << "Segment new size: " << segment_nodes.size(); + } + + // --------------------------------- Step 3 --------------------------------- // Convert the segments into the expected return format for (const auto& itr : sg_map) { - const auto& segment_node_names = itr.second; + const std::set& segment_nodes = itr.second; if (VLOG_IS_ON(1)) { string s; - for (const auto& name : segment_node_names) { - s += " " + name; - } - VLOG(1) << "Segment " << segments->size() << ":" << s; + for (auto node : segment_nodes) s += " " + node->name(); + VLOG(1) << "Segment " << segments->size() << ": " << s; } // Don't use small segments. - if (static_cast(segment_node_names.size()) < + if (static_cast(segment_nodes.size()) < options.minimum_segment_size) { VLOG(1) << "Segment " << segments->size() << " has only " - << segment_node_names.size() << " nodes, dropping"; + << segment_nodes.size() << " nodes, dropping"; continue; } + // TODO(sami): Make segmenter placement aware once trtscopes are in place + std::set segment_node_names; + for (auto node : itr.second) segment_node_names.insert(node->name()); const auto& dev_itr = device_maps.find(itr.first); if (dev_itr == device_maps.end() || dev_itr->second.empty()) { VLOG(1) << "No device assigned to segment " << segments->size(); diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h index 81b4bfe49f..ab75135054 100644 --- a/tensorflow/contrib/tensorrt/segment/segment.h +++ b/tensorflow/contrib/tensorrt/segment/segment.h @@ -40,22 +40,6 @@ struct SegmentOptions { std::set exclude_node_list; }; -// Get the subgraphs of a graph that can be handled by TensorRT. -// -// @param gdef The GraphDef describing the network -// @param candidate_fn A function that returns true for a NodeDef if -// that node can be handled by TensorRT. -// @param segments Returns the TensorRT segments/subgraphs. Each entry -// in the vector describes a subgraph by giving a set of the names of -// all the NodeDefs in that subgraph. -// @return the status. -// -// TODO(aaroey): remove this method. -tensorflow::Status SegmentGraph( - const tensorflow::GraphDef& gdef, - const std::function& candidate_fn, - const SegmentOptions& options, SegmentNodesVector* segments); - // Get the subgraphs of a graph that can be handled by TensorRT. // // @param graph tensorflow::Graph of the network @@ -66,8 +50,10 @@ tensorflow::Status SegmentGraph( // all the NodeDefs in that subgraph. // @return the status. tensorflow::Status SegmentGraph( - tensorflow::Graph* tf_graph, + const tensorflow::Graph* tf_graph, const std::function& candidate_fn, + const std::function& input_candidate_fn, + const std::function& output_candidate_fn, const SegmentOptions& options, SegmentNodesVector* segments); } // namespace segment diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/contrib/tensorrt/segment/segment_test.cc index f5b2d258d7..a43cf4f416 100644 --- a/tensorflow/contrib/tensorrt/segment/segment_test.cc +++ b/tensorflow/contrib/tensorrt/segment/segment_test.cc @@ -14,350 +14,230 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/contrib/tensorrt/segment/segment.h" -#include "tensorflow/c/c_api.h" -#include "tensorflow/core/framework/graph.pb.h" + +#include "tensorflow/cc/framework/scope.h" +#include "tensorflow/cc/ops/standard_ops.h" +#include "tensorflow/core/graph/testlib.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/public/session.h" namespace tensorflow { namespace tensorrt { namespace segment { namespace test { +namespace ops = ::tensorflow::ops; class SegmentTest : public ::testing::Test { - public: - bool GetGraphDef(TF_Graph* graph, tensorflow::GraphDef* graph_def); - - TF_Operation* Placeholder(TF_Graph* graph, TF_Status* s, const char* name); - TF_Operation* Add(TF_Operation* l, TF_Operation* r, TF_Graph* graph, - TF_Status* s, const char* name); - + protected: std::function MakeCandidateFn( - const std::set& node_names); + const std::set& node_names) { + return [node_names](const tensorflow::Node* node) -> bool { + return node_names.find(node->name()) != node_names.end(); + }; + } - protected: - void PlaceholderHelper(TF_Graph* graph, TF_Status* s, const char* name, - TF_Operation** op); - void AddHelper(TF_Operation* l, TF_Operation* r, TF_Graph* graph, - TF_Status* s, const char* name, TF_Operation** op, bool check); + void RunTest(const tensorflow::Graph* graph, + const std::set& candidates, + const std::set& input_candidates, + const std::set& output_candidates, + const std::vector>& expected_segments) { + SegmentNodesVector segments; + TF_EXPECT_OK(SegmentGraph( + graph, MakeCandidateFn(candidates), MakeCandidateFn(input_candidates), + MakeCandidateFn(output_candidates), default_options_, &segments)); + ValidateSegment(segments, expected_segments); + } + + void ValidateSegment(const SegmentNodesVector& segments, + const std::vector>& expected_segments) { + EXPECT_EQ(expected_segments.size(), segments.size()); + for (int i = 0; i < segments.size(); ++i) { + const auto& segment_node_names = segments[i].first; + const auto& expected = expected_segments[i]; + for (const auto& name : expected) { + EXPECT_TRUE(segment_node_names.count(name)) + << "Segment " << i << " is missing expected node: " << name; + } + if (segment_node_names.size() == expected.size()) continue; + for (const auto& name : segment_node_names) { + EXPECT_TRUE(expected.count(name)) + << "Unexpected node found in segment " << i << ": " << name; + } + } + } SegmentOptions default_options_; }; -bool SegmentTest::GetGraphDef(TF_Graph* graph, - tensorflow::GraphDef* graph_def) { - TF_Status* s = TF_NewStatus(); - TF_Buffer* buffer = TF_NewBuffer(); - TF_GraphToGraphDef(graph, buffer, s); - bool ret = TF_GetCode(s) == TF_OK; - EXPECT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - if (ret) ret = graph_def->ParseFromArray(buffer->data, buffer->length); - TF_DeleteBuffer(buffer); - TF_DeleteStatus(s); - return ret; -} - -std::function SegmentTest::MakeCandidateFn( - const std::set& node_names) { - return [node_names](const tensorflow::Node* node) -> bool { - return node_names.find(node->name()) != node_names.end(); - }; -} - -void SegmentTest::PlaceholderHelper(TF_Graph* graph, TF_Status* s, - const char* name, TF_Operation** op) { - TF_OperationDescription* desc = TF_NewOperation(graph, "Placeholder", name); - TF_SetAttrType(desc, "dtype", TF_INT32); - *op = TF_FinishOperation(desc, s); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - ASSERT_NE(*op, nullptr); -} - -TF_Operation* SegmentTest::Placeholder(TF_Graph* graph, TF_Status* s, - const char* name) { - TF_Operation* op; - PlaceholderHelper(graph, s, name, &op); - return op; -} - -void SegmentTest::AddHelper(TF_Operation* l, TF_Operation* r, TF_Graph* graph, - TF_Status* s, const char* name, TF_Operation** op, - bool check) { - TF_OperationDescription* desc = TF_NewOperation(graph, "AddN", name); - TF_Output add_inputs[2] = {{l, 0}, {r, 0}}; - TF_AddInputList(desc, add_inputs, 2); - *op = TF_FinishOperation(desc, s); - if (check) { - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - ASSERT_NE(*op, nullptr); - } -} - -TF_Operation* SegmentTest::Add(TF_Operation* l, TF_Operation* r, - TF_Graph* graph, TF_Status* s, - const char* name) { - TF_Operation* op; - AddHelper(l, r, graph, s, name, &op, true); - return op; +std::set operator-(const std::set& lhs, const string& rhs) { + std::set result = lhs; + CHECK(result.erase(rhs)); + return result; } TEST_F(SegmentTest, Empty) { - TF_Graph* graph = TF_NewGraph(); - - GraphDef graph_def; - ASSERT_TRUE(GetGraphDef(graph, &graph_def)); - - SegmentNodesVector segments; - ASSERT_EQ( - SegmentGraph(graph_def, MakeCandidateFn({}), default_options_, &segments), - tensorflow::Status::OK()); - + Scope s = Scope::NewRootScope(); + tensorflow::Graph g(OpRegistry::Global()); + TF_EXPECT_OK(s.ToGraph(&g)); // Expect no segments/subgraphs. - EXPECT_TRUE(segments.empty()); - TF_DeleteGraph(graph); + RunTest(&g, {}, {}, {}, {}); } TEST_F(SegmentTest, Simple) { - TF_Status* s = TF_NewStatus(); - TF_Graph* graph = TF_NewGraph(); - // feed - // // || + // // \\ // add0 add1 - // | | / + // | \ / // | add2 - // | / || + // | / \\ // add3 add4 - // | / + // \ / // - // - TF_Operation* feed = Placeholder(graph, s, "feed"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("feed"), string(TF_OperationName(feed))); - - TF_Operation* add0 = Add(feed, feed, graph, s, "add0"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add1 = Add(feed, feed, graph, s, "add1"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add2 = Add(add0, add1, graph, s, "add2"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add3 = Add(add0, add2, graph, s, "add3"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("add3"), string(TF_OperationName(add3))); - TF_Operation* add4 = Add(add2, add2, graph, s, "add4"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("add4"), string(TF_OperationName(add4))); - - GraphDef graph_def; - ASSERT_TRUE(GetGraphDef(graph, &graph_def)); - - SegmentNodesVector segments; - ASSERT_EQ( - SegmentGraph(graph_def, - MakeCandidateFn({"add0", "add1", "add2", "add3", "add4"}), - default_options_, &segments), - tensorflow::Status::OK()); - - // Expect all Add operations to be collapsed into a single segment - ASSERT_EQ(segments.size(), 1); - std::vector expected{"add0", "add1", "add2", "add3", "add4"}; - for (const auto& ex : expected) { - EXPECT_TRUE(segments[0].first.find(ex) != segments[0].first.end()) - << "Missing expected node " << ex; - } - TF_DeleteGraph(graph); - TF_DeleteStatus(s); + Scope s = Scope::NewRootScope(); + auto feed = ops::Placeholder(s.WithOpName("feed"), DT_FLOAT); + auto add0 = ops::Add(s.WithOpName("add0"), feed, feed); + auto add1 = ops::Add(s.WithOpName("add1"), feed, feed); + auto add2 = ops::Add(s.WithOpName("add2"), add0, add1); + auto add3 = ops::Add(s.WithOpName("add3"), add0, add2); + auto add4 = ops::Add(s.WithOpName("add4"), add2, add2); + tensorflow::Graph g(OpRegistry::Global()); + TF_EXPECT_OK(s.ToGraph(&g)); + + // All Add operations are candidates, and we expect all of them to be + // collapsed into a single segment + const std::set all_adds = {"add0", "add1", "add2", "add3", "add4"}; + RunTest(&g, all_adds, all_adds, all_adds, {all_adds}); + + // Make add1 not a candidate, and we expect all other Add operations to be + // collapsed into a single segment + auto without_add1 = all_adds - "add1"; + RunTest(&g, without_add1, without_add1, without_add1, {without_add1}); + + // Make add1 not a candidate and add2 not an input candidate, and we expect + // add0 and add2 are removed from the segment. + auto without_add2 = all_adds - "add2"; + RunTest(&g, without_add1, without_add2, without_add1, {{"add3", "add4"}}); + + // Making add2 not an input candidate itself won't affect anything. + RunTest(&g, all_adds, without_add2, all_adds, {all_adds}); + + // Making add1 not an input candidate. + RunTest(&g, all_adds, without_add1, all_adds, {without_add1}); + + // Making add3 not an output candidate doesn't affect anything, since it's + // output is sink. + auto without_add3 = all_adds - "add3"; + RunTest(&g, all_adds, all_adds, without_add3, {all_adds}); } TEST_F(SegmentTest, AvoidCycle) { - TF_Status* s = TF_NewStatus(); - TF_Graph* graph = TF_NewGraph(); - - // add2 is not a TRT candidate so add0/add3 cannot be formed as a - // subgraph - // // feed - // // || + // // \\ // add0 add1 - // | | / + // | \ / // | add2 - // | / || + // | / \\ // add3 add4 - // | / + // \ / // - // - TF_Operation* feed = Placeholder(graph, s, "feed"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("feed"), string(TF_OperationName(feed))); - - TF_Operation* add0 = Add(feed, feed, graph, s, "add0"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add1 = Add(feed, feed, graph, s, "add1"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add2 = Add(add0, add1, graph, s, "add2"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add3 = Add(add0, add2, graph, s, "add3"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("add3"), string(TF_OperationName(add3))); - TF_Operation* add4 = Add(add2, add2, graph, s, "add4"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("add4"), string(TF_OperationName(add4))); - - GraphDef graph_def; - ASSERT_TRUE(GetGraphDef(graph, &graph_def)); - - SegmentNodesVector segments; - ASSERT_EQ( - SegmentGraph(graph_def, MakeCandidateFn({"add0", "add1", "add3", "add4"}), - default_options_, &segments), - tensorflow::Status::OK()); - - // Expect no subgraphs - EXPECT_EQ(segments.size(), 0); - TF_DeleteGraph(graph); - TF_DeleteStatus(s); + Scope s = Scope::NewRootScope(); + auto feed = ops::Placeholder(s.WithOpName("feed"), DT_FLOAT); + auto add0 = ops::Add(s.WithOpName("add0"), feed, feed); + auto add1 = ops::Add(s.WithOpName("add1"), feed, feed); + auto add2 = ops::Add(s.WithOpName("add2"), add0, add1); + auto add3 = ops::Add(s.WithOpName("add3"), add0, add2); + auto add4 = ops::Add(s.WithOpName("add4"), add2, add2); + tensorflow::Graph g(OpRegistry::Global()); + TF_EXPECT_OK(s.ToGraph(&g)); + + // add2 is not a TRT candidate so there should be no segments generated. + const std::set without_add2 = {"add0", "add1", "add3", "add4"}; + RunTest(&g, without_add2, without_add2, without_add2, {}); } TEST_F(SegmentTest, Multiple) { - TF_Status* s = TF_NewStatus(); - TF_Graph* graph = TF_NewGraph(); - - // add5 is not a TRT candidate so two subgraphs should be formed - // - // feed - // // || || - // add0 add1 add7 - // | | / / || - // | add2-----add5 add8 - // | / | | | | - // add3 add4 add6 - // | | / - // - // - TF_Operation* feed = Placeholder(graph, s, "feed"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("feed"), string(TF_OperationName(feed))); - - TF_Operation* add0 = Add(feed, feed, graph, s, "add0"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add1 = Add(feed, feed, graph, s, "add1"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add7 = Add(feed, feed, graph, s, "add7"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add2 = Add(add0, add1, graph, s, "add2"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add5 = Add(add2, add7, graph, s, "add5"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add8 = Add(add7, add7, graph, s, "add8"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add3 = Add(add0, add2, graph, s, "add3"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("add3"), string(TF_OperationName(add3))); - TF_Operation* add4 = Add(add2, add5, graph, s, "add4"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("add4"), string(TF_OperationName(add4))); - TF_Operation* add6 = Add(add5, add8, graph, s, "add6"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("add6"), string(TF_OperationName(add6))); - - GraphDef graph_def; - ASSERT_TRUE(GetGraphDef(graph, &graph_def)); - - SegmentNodesVector segments; - ASSERT_EQ(SegmentGraph(graph_def, - MakeCandidateFn({"add0", "add1", "add2", "add3", - "add4", "add6", "add7", "add8"}), - default_options_, &segments), - tensorflow::Status::OK()); - - // Expect two subgraphs - EXPECT_EQ(segments.size(), 2); - - std::vector expected0{"add6", "add8"}; - for (const auto& ex : expected0) { - EXPECT_TRUE(segments[0].first.find(ex) != segments[0].first.end()) - << "Missing expected node " << ex; - } - - std::vector expected1{"add0", "add1", "add2", "add3"}; - for (const auto& ex : expected1) { - EXPECT_TRUE(segments[1].first.find(ex) != segments[1].first.end()) - << "Missing expected node " << ex; - } - TF_DeleteGraph(graph); - TF_DeleteStatus(s); + // feed + // // || \\ + // add0 add1 add7 + // | \ / / \\ + // | add2 / \\ + // | || \ | || + // | || add5 add8 + // | / \ / \ / + // add3 add4 add6 + // \ | / + // + Scope s = Scope::NewRootScope(); + auto feed = ops::Placeholder(s.WithOpName("feed"), DT_FLOAT); + auto add0 = ops::Add(s.WithOpName("add0"), feed, feed); + auto add1 = ops::Add(s.WithOpName("add1"), feed, feed); + auto add7 = ops::Add(s.WithOpName("add7"), feed, feed); + auto add2 = ops::Add(s.WithOpName("add2"), add0, add1); + auto add5 = ops::Add(s.WithOpName("add5"), add2, add7); + auto add8 = ops::Add(s.WithOpName("add8"), add7, add7); + auto add3 = ops::Add(s.WithOpName("add3"), add0, add2); + auto add4 = ops::Add(s.WithOpName("add4"), add2, add5); + auto add6 = ops::Add(s.WithOpName("add6"), add5, add8); + tensorflow::Graph g(OpRegistry::Global()); + TF_EXPECT_OK(s.ToGraph(&g)); + + const std::set all_adds = {"add0", "add1", "add2", "add3", "add4", + "add5", "add6", "add7", "add8"}; + // Make add5 not a TRT candidate, and we expect two segments. + auto without_add5 = all_adds - "add5"; + RunTest(&g, without_add5, without_add5, without_add5, + {{"add6", "add8"}, {"add0", "add1", "add2", "add3"}}); + + // Make add8 not a candidate and add6 not an input candidate, then all direct + // and indirect inputs of add6 will be removed from the segment. + auto without_add8 = all_adds - "add8"; + auto without_add6 = all_adds - "add6"; + RunTest(&g, without_add8, without_add6, all_adds, {{"add3", "add4"}}); + + // Make add3 not a candidate and add0 not an output candidate, then all + // direct and indirect outputs of add0 will be removed from the segment. + auto without_add3 = all_adds - "add3"; + auto without_add0 = all_adds - "add0"; + RunTest(&g, without_add3, all_adds, without_add0, {{"add1", "add7", "add8"}}); } TEST_F(SegmentTest, BigIfElse) { - TF_Status* s = TF_NewStatus(); - TF_Graph* graph = TF_NewGraph(); - - // add2 is not a TRT candidate - // // feed // || // add0 - // // || + // // \\ // add1 add4 // || || // add2 add5 // || || // add3 add6 - // || // + // \\ // // add7 // || // - // - TF_Operation* feed = Placeholder(graph, s, "feed"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("feed"), string(TF_OperationName(feed))); - - TF_Operation* add0 = Add(feed, feed, graph, s, "add0"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add1 = Add(add0, add0, graph, s, "add1"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add2 = Add(add1, add1, graph, s, "add2"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add3 = Add(add2, add2, graph, s, "add3"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add4 = Add(add0, add0, graph, s, "add4"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add5 = Add(add4, add4, graph, s, "add5"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add6 = Add(add5, add5, graph, s, "add6"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - TF_Operation* add7 = Add(add3, add6, graph, s, "add7"); - ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - EXPECT_EQ(string("add7"), string(TF_OperationName(add7))); - - GraphDef graph_def; - ASSERT_TRUE(GetGraphDef(graph, &graph_def)); - - SegmentNodesVector segments; - ASSERT_EQ(SegmentGraph(graph_def, - MakeCandidateFn({"add0", "add1", "add3", "add4", - "add5", "add6", "add7"}), - default_options_, &segments), - tensorflow::Status::OK()); - - // Expect 2 subgraphs - EXPECT_EQ(segments.size(), 2); - - std::vector expected0{"add3", "add4", "add5", "add6", "add7"}; - for (const auto& ex : expected0) { - EXPECT_TRUE(segments[0].first.find(ex) != segments[0].first.end()) - << "Missing expected node " << ex; - } - - std::vector expected1{"add0", "add1"}; - for (const auto& ex : expected1) { - EXPECT_TRUE(segments[1].first.find(ex) != segments[1].first.end()) - << "Missing expected node " << ex; - } - TF_DeleteGraph(graph); - TF_DeleteStatus(s); + Scope s = Scope::NewRootScope(); + auto feed = ops::Placeholder(s.WithOpName("feed"), DT_FLOAT); + auto add0 = ops::Add(s.WithOpName("add0"), feed, feed); + auto add1 = ops::Add(s.WithOpName("add1"), add0, add0); + auto add2 = ops::Add(s.WithOpName("add2"), add1, add1); + auto add3 = ops::Add(s.WithOpName("add3"), add2, add2); + auto add4 = ops::Add(s.WithOpName("add4"), add0, add0); + auto add5 = ops::Add(s.WithOpName("add5"), add4, add4); + auto add6 = ops::Add(s.WithOpName("add6"), add5, add5); + auto add7 = ops::Add(s.WithOpName("add7"), add3, add6); + tensorflow::Graph g(OpRegistry::Global()); + TF_EXPECT_OK(s.ToGraph(&g)); + + // Make add2 not a TRT candidate, and we expect 2 segments. + const std::set all_adds = {"add0", "add1", "add2", "add3", + "add4", "add5", "add6", "add7"}; + RunTest(&g, all_adds - "add2", all_adds, all_adds, + {{"add3", "add4", "add5", "add6", "add7"}, {"add0", "add1"}}); } } // namespace test -- cgit v1.2.3 From 622a111e88dfbac430556ec8213230345189f466 Mon Sep 17 00:00:00 2001 From: Thomas Joerg Date: Thu, 12 Jul 2018 00:59:10 -0700 Subject: Fix typos. PiperOrigin-RevId: 204256888 --- tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h index a1cc38401c..59547c16d7 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h @@ -118,7 +118,7 @@ class IrEmitterUnnested : public IrEmitter { // Emits code that reduces a matrix of shape [height x width] to a vector of // [width]. Other parameters have the same meaning as those of // `EmitReductionToVector`. Note that input shape might not be - // [height x width], but can be bitcast to [height x weight] with "height" + // [height x width], but can be bitcast to [height x width] with "height" // being the major dimension. Status EmitColumnReduction( int64 height, int64 width, HloInstruction* reduce, @@ -134,7 +134,7 @@ class IrEmitterUnnested : public IrEmitter { // Emits code that reduces a 3D tensor of shape [depth x height x width] to a // vector of shape [height]. Other parameters have the same meaning as those // of `EmitReductionToVector`. Note that input shape might not be - // [depth x height x width], but can be bitcast to [depth x height x weight] + // [depth x height x width], but can be bitcast to [depth x height x width] // with "depth" being the most major dimension. Status EmitRowReduction( int64 depth, int64 height, int64 width, HloInstruction* reduce, -- cgit v1.2.3 From 2b57d8d521fff37529d375e702fb2cd9ac2b36eb Mon Sep 17 00:00:00 2001 From: Adrian Kuegel Date: Thu, 12 Jul 2018 01:35:47 -0700 Subject: Support unsigned indices in DynamicSlice and DynamicUpdateSlice. For unsigned indices, we need to use unsigned comparisons when clamping the start_indices. This CL also adds regression tests for DynamicSlice and DynamicUpdateSlice. PiperOrigin-RevId: 204260644 --- .../compiler/xla/service/elemental_ir_emitter.cc | 36 ++++++++++------------ tensorflow/compiler/xla/tests/dynamic_ops_test.cc | 23 +++++++++++++- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc index bd68685153..004a80d19d 100644 --- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc @@ -1568,16 +1568,15 @@ StatusOr ElementalIrEmitter::EmitElementalDynamicSlice( // to officially document different behavior. start_index_value = ir_builder_->CreateSExtOrTrunc(start_index_value, index_type); - llvm::Value* operand_dim_size = - index_typed_const(input_hlo->shape().dimensions(i)); - llvm::Value* output_dim_size = - index_typed_const(hlo->shape().dimensions(i)); + int64 largest_valid_start_index = + input_hlo->shape().dimensions(i) - hlo->shape().dimensions(i); + CHECK_GE(largest_valid_start_index, 0); + bool is_signed = ShapeUtil::ElementIsSigned(hlo->operand(1)->shape()); start_index_value = EmitIntegralMin( - ir_builder_->CreateSub(operand_dim_size, output_dim_size), - EmitIntegralMax(index_typed_const(0), start_index_value, - /*is_signed=*/true), - /*is_signed=*/true); + index_typed_const(largest_valid_start_index), + EmitIntegralMax(index_typed_const(0), start_index_value, is_signed), + is_signed); start_index_value->setName( AsStringRef(IrName(hlo, StrCat("start_idx", i)))); @@ -1663,14 +1662,12 @@ StatusOr ElementalIrEmitter::EmitElementalGather( // clamp(gather_dim_component_extended, 0, largest_valid_start_index); // TODO(b/111078873): This is implementation defined behavior. - bool is_signed = ShapeUtil::ElementIsSigned(indices_shape); auto gather_dim_component_extended_inbound = EmitIntegralMin( index.GetConstantWithIndexType(largest_valid_start_index), EmitIntegralMax(index.GetConstantWithIndexType(0), - gather_dim_component_extended, - /*is_signed=*/is_signed), - /*is_signed=*/is_signed); + gather_dim_component_extended, is_signed), + is_signed); operand_index[operand_dim] = ir_builder_->CreateAdd( operand_index[operand_dim], gather_dim_component_extended_inbound); @@ -1726,16 +1723,17 @@ StatusOr ElementalIrEmitter::EmitElementalDynamicUpdateSlice( // to officially document different behavior. start_index_value = ir_builder_->CreateSExtOrTrunc(start_index_value, index_type); - llvm::Value* input_dim_size = - index_typed_const(input_hlo->shape().dimensions(i)); llvm::Value* update_dim_size = index_typed_const(update_hlo->shape().dimensions(i)); + int64 largest_valid_start_index = + input_hlo->shape().dimensions(i) - update_hlo->shape().dimensions(i); + CHECK_GE(largest_valid_start_index, 0); - start_index_value = - EmitIntegralMin(ir_builder_->CreateSub(input_dim_size, update_dim_size), - EmitIntegralMax(index_typed_const(0), start_index_value, - /*is_signed=*/true), - /*is_signed=*/true); + bool is_signed = ShapeUtil::ElementIsSigned(start_hlo->shape()); + start_index_value = EmitIntegralMin( + index_typed_const(largest_valid_start_index), + EmitIntegralMax(index_typed_const(0), start_index_value, is_signed), + is_signed); start_index_value->setName( AsStringRef(IrName(hlo, StrCat("start_idx", i)))); diff --git a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc index b063b6bdef..88ac96d6b0 100644 --- a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc +++ b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc @@ -202,18 +202,28 @@ XLA_TEST_F(DynamicSliceTest, Int32R1) { TestR1(); } XLA_TEST_F(DynamicSliceTest, Int32R1OOB) { TestR1OOB(); } XLA_TEST_F(DynamicSliceTest, Int64R1) { TestR1(); } XLA_TEST_F(DynamicSliceTest, UInt64R1) { TestR1(); } +XLA_TEST_F(DynamicSliceTest, UInt32R1OOB) { + RunR1({0, 1, 2, 3, 4}, {2147483648u}, {2}, {3, 4}); +} XLA_TEST_F(DynamicSliceTest, Int32R2BF16) { TestR2(); } XLA_TEST_F(DynamicSliceTest, Int32R2) { TestR2(); } XLA_TEST_F(DynamicSliceTest, Int32R2OOB) { TestR2OOB(); } XLA_TEST_F(DynamicSliceTest, Int64R2) { TestR2(); } XLA_TEST_F(DynamicSliceTest, UInt64R2) { TestR2(); } +XLA_TEST_F(DynamicSliceTest, UInt32R2OOB) { + RunR2({{0, 1}, {2, 3}}, {2147483648u, 0}, {1, 1}, {{2}}); +} XLA_TEST_F(DynamicSliceTest, Int32R3BF16) { TestR3(); } XLA_TEST_F(DynamicSliceTest, Int32R3) { TestR3(); } XLA_TEST_F(DynamicSliceTest, Int32R3OOB) { TestR3OOB(); } XLA_TEST_F(DynamicSliceTest, Int64R3) { TestR3(); } XLA_TEST_F(DynamicSliceTest, UInt64R3) { TestR3(); } +XLA_TEST_F(DynamicSliceTest, UInt32R3OOB) { + RunR3({{{0, 1}, {2, 3}}, {{4, 5}, {6, 7}}}, + {2147483648u, 0, 2147483648u}, {1, 1, 1}, {{{5}}}); +} XLA_TEST_F(DynamicSliceTest, Int32R1Pred) { // Slice at dimension start. @@ -530,21 +540,32 @@ XLA_TEST_F(DynamicUpdateSliceTest, Int32R0) { TestR0(); } XLA_TEST_F(DynamicUpdateSliceTest, Int64R0) { TestR0(); } XLA_TEST_F(DynamicUpdateSliceTest, UInt64R0) { TestR0(); } -// TODO(b/71820067): The CPU parallel backend failed for this on 2018-01-10. XLA_TEST_F(DynamicUpdateSliceTest, Int32R1BF16) { TestR1(); } XLA_TEST_F(DynamicUpdateSliceTest, Int32R1) { TestR1(); } XLA_TEST_F(DynamicUpdateSliceTest, Int64R1) { TestR1(); } XLA_TEST_F(DynamicUpdateSliceTest, UInt64R1) { TestR1(); } +XLA_TEST_F(DynamicUpdateSliceTest, UInt32R1OOB) { + RunR1({0, 1, 2, 3, 4}, {5, 6}, {2147483648u}, {0, 1, 2, 5, 6}); +} XLA_TEST_F(DynamicUpdateSliceTest, Int32R2BF16) { TestR2(); } XLA_TEST_F(DynamicUpdateSliceTest, Int32R2) { TestR2(); } XLA_TEST_F(DynamicUpdateSliceTest, Int64R2) { TestR2(); } XLA_TEST_F(DynamicUpdateSliceTest, UInt64R2) { TestR2(); } +XLA_TEST_F(DynamicUpdateSliceTest, UInt32R2OOB) { + RunR2({{0, 1}, {2, 3}}, {{4}}, {2147483648u, 0}, + {{0, 1}, {4, 3}}); +} XLA_TEST_F(DynamicUpdateSliceTest, Int32R3BF16) { TestR3(); } XLA_TEST_F(DynamicUpdateSliceTest, Int32R3) { TestR3(); } XLA_TEST_F(DynamicUpdateSliceTest, Int64R3) { TestR3(); } XLA_TEST_F(DynamicUpdateSliceTest, UInt64R3) { TestR3(); } +XLA_TEST_F(DynamicUpdateSliceTest, UInt32R3OOB) { + RunR3({{{0, 1}, {2, 3}}, {{4, 5}, {6, 7}}}, {{{8}}}, + {2147483648u, 0, 2147483648u}, + {{{0, 1}, {2, 3}}, {{4, 8}, {6, 7}}}); +} XLA_TEST_F(DynamicUpdateSliceTest, Int32OOBBF16) { TestOOB(); } XLA_TEST_F(DynamicUpdateSliceTest, Int32OOB) { TestOOB(); } -- cgit v1.2.3 From 6cc63839221b34d47534987d9de84812924ade7e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 03:11:32 -0700 Subject: Improve performance of layout assignment Previously creating the layout constraints was proportional to the number of buffers multiplied by the number of computations what is bad when both of them is big. The new implementation is proportional with the total number of instructions by calculating the effected buffers from the tuple point sets. PiperOrigin-RevId: 204270924 --- .../compiler/xla/service/layout_assignment.cc | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc index fedc83c8f8..46a6d57353 100644 --- a/tensorflow/compiler/xla/service/layout_assignment.cc +++ b/tensorflow/compiler/xla/service/layout_assignment.cc @@ -59,7 +59,6 @@ namespace xla { // anonymous namespace, instead of three or four spread all over this file. namespace { - } // namespace std::ostream& operator<<(std::ostream& out, @@ -113,14 +112,18 @@ LayoutConstraints::LayoutConstraints( HloComputation* computation) : points_to_analysis_(points_to_analysis), computation_(computation) { // Gather all array-shaped logical buffers into unconstrained_buffer_ids. - for (LogicalBuffer::Id id = 0; id < points_to_analysis_.num_logical_buffers(); - id++) { - auto& buffer = points_to_analysis_.logical_buffer(id); - // The points to analysis is computed per module, restrict constraints to - // array buffers in this computation. - if (buffer.IsArray() && buffer.instruction()->parent() == computation) { - unconstrained_buffer_ids_.insert(buffer.id()); - } + for (HloInstruction* inst : computation_->instructions()) { + points_to_analysis_.GetPointsToSet(inst).ForEachElement( + [&](const ShapeIndex&, const PointsToSet::BufferList& buffers) { + for (const LogicalBuffer* buffer : buffers) { + // The points to analysis is computed per module, restrict + // constraints to array buffers in this computation. + if (buffer->IsArray() && + buffer->instruction()->parent() == computation) { + unconstrained_buffer_ids_.insert(buffer->id()); + } + } + }); } } -- cgit v1.2.3 From eccc1d4c102b1b3b03b98dbc362f799cb540a1da Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Thu, 12 Jul 2018 03:33:08 -0700 Subject: [XLA:GPU] Unify infeed and outfeed queue implementations This makes the infeed queue behave like the outfeed queue and merges the two implementations. It shouldn't change functionality. There was also quite a bit of unused code in infeed_manager that's gone now. PiperOrigin-RevId: 204273197 --- tensorflow/compiler/xla/service/gpu/BUILD | 11 ++- .../xla/service/gpu/gpu_transfer_manager.cc | 46 +++++------ .../xla/service/gpu/gpu_transfer_manager.h | 7 +- .../compiler/xla/service/gpu/infeed_manager.cc | 69 +---------------- .../compiler/xla/service/gpu/infeed_manager.h | 82 ++++---------------- .../compiler/xla/service/gpu/infeed_thunk.cc | 16 ++-- .../compiler/xla/service/gpu/outfeed_manager.cc | 19 ----- .../compiler/xla/service/gpu/outfeed_manager.h | 27 +------ .../compiler/xla/service/gpu/outfeed_thunk.cc | 2 +- tensorflow/compiler/xla/service/gpu/xfeed_queue.h | 89 ++++++++++++++++++++++ 10 files changed, 146 insertions(+), 222 deletions(-) create mode 100644 tensorflow/compiler/xla/service/gpu/xfeed_queue.h diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 9fca3a51c8..59172e53d3 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -542,6 +542,7 @@ cc_library( ":outfeed_manager", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:literal_util", + "//tensorflow/compiler/xla:shape_tree", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", @@ -639,14 +640,21 @@ cc_library( ], ) +cc_library( + name = "xfeed_queue", + hdrs = ["xfeed_queue.h"], + deps = ["//tensorflow/core:lib"], +) + cc_library( name = "infeed_manager", srcs = ["infeed_manager.cc"], hdrs = ["infeed_manager.h"], deps = [ + ":xfeed_queue", + "//tensorflow/compiler/xla:shape_tree", "//tensorflow/compiler/xla:types", "//tensorflow/compiler/xla:util", - "//tensorflow/core:lib", "//tensorflow/core:stream_executor_no_cuda", ], ) @@ -656,6 +664,7 @@ cc_library( srcs = ["outfeed_manager.cc"], hdrs = ["outfeed_manager.h"], deps = [ + ":xfeed_queue", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_tree", "//tensorflow/compiler/xla:shape_util", diff --git a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc index 3c8018a030..63466539fa 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc @@ -36,6 +36,7 @@ limitations under the License. #include "tensorflow/core/platform/stream_executor_no_cuda.h" namespace xla { +namespace gpu { // TODO(b/30467474) Once GPU infeed implementation settles, consider // folding back the cpu and gpu infeed implementations into a generic @@ -52,48 +53,37 @@ Status GpuTransferManager::TransferLiteralToInfeed( VLOG(2) << "Transferring literal to infeed with shape: " << ShapeUtil::HumanString(shape); - if (!ShapeUtil::IsTuple(shape)) { - int64 size = GetByteSizeRequirement(shape); - return TransferBufferToInfeed(executor, size, literal.untyped_data()); - } - // For a tuple, we transfer each of its elements to the device and // enqueue the resulting destination device addresses with the // infeed manager. - std::vector buffers; - auto cleanup = tensorflow::gtl::MakeCleanup([buffers]() { - for (gpu::InfeedBuffer* b : buffers) { - b->Done(); - } - }); + ShapeTree buffer_tree(shape); TF_RETURN_IF_ERROR(ShapeUtil::ForEachSubshapeWithStatus( shape, [&](const Shape& literal_subshape, const ShapeIndex& index) { if (ShapeUtil::IsArray(literal_subshape)) { int64 tuple_element_size = GetByteSizeRequirement(literal_subshape); TF_ASSIGN_OR_RETURN( - gpu::InfeedBuffer * buffer, + *buffer_tree.mutable_element(index), TransferBufferToInfeedInternal(executor, tuple_element_size, literal.untyped_data(index))); - buffers.push_back(buffer); } return Status::OK(); })); - cleanup.release(); - return EnqueueBuffersToInfeed(executor, buffers); + return EnqueueBuffersToInfeed(executor, std::move(buffer_tree)); } Status GpuTransferManager::TransferBufferToInfeed(se::StreamExecutor* executor, int64 size, const void* source) { - TF_ASSIGN_OR_RETURN(gpu::InfeedBuffer * buffer, - TransferBufferToInfeedInternal(executor, size, source)); - return EnqueueBuffersToInfeed(executor, {buffer}); + return InternalError( + "Attempted to transfer data to infeed on a GPU device using " + "TransferBufferToInfeed. This should be done using " + "TransferLiteralToInfeed instead."); } Status GpuTransferManager::EnqueueBuffersToInfeed( - se::StreamExecutor* executor, std::vector buffers) { + se::StreamExecutor* executor, ShapeTree buffers) { gpu::InfeedManager* infeed_manager = gpu::GetOrCreateInfeedManager(); se::Stream* stream = infeed_manager->GetStream(executor); @@ -103,21 +93,18 @@ Status GpuTransferManager::EnqueueBuffersToInfeed( // possible. Status block_status = stream->BlockHostUntilDone(); if (!block_status.ok()) { - for (gpu::InfeedBuffer* b : buffers) { - b->Done(); - } return InternalError("Failed to complete data transfer on stream %p: %s", stream, block_status.error_message().c_str()); } - infeed_manager->EnqueueBuffers(buffers); + infeed_manager->EnqueueDestination(std::move(buffers)); VLOG(2) << "Infeed data transferred"; return Status::OK(); } -StatusOr GpuTransferManager::TransferBufferToInfeedInternal( +StatusOr GpuTransferManager::TransferBufferToInfeedInternal( se::StreamExecutor* executor, int64 size, const void* source) { if (size > std::numeric_limits::max()) { return InvalidArgument("Infeed shape is too large: needs %lld bytes", size); @@ -133,12 +120,12 @@ StatusOr GpuTransferManager::TransferBufferToInfeedInternal( return InternalError("Failed to obtain a stream"); } - gpu::InfeedBuffer* buffer = new gpu::InfeedBuffer(executor, size); - stream->ThenMemcpy(buffer->device_memory(), source, size); + InfeedBuffer buffer(executor, size); + stream->ThenMemcpy(buffer.device_memory(), source, size); VLOG(2) << "Queued infeed data on stream " << stream; - return buffer; + return std::move(buffer); } static std::unique_ptr ShapeTreeToLiteral( @@ -191,17 +178,18 @@ Status GpuTransferManager::TransferLiteralFromOutfeed( // Give the tree of buffers to the outfeed mananger. The device will fill it // while we're waiting for it below. gpu::OutfeedManager* outfeed_manager = gpu::GetOrCreateOutfeedManager(); - outfeed_manager->EnqueueOutfeedDestination(&outfeed_buffers); + outfeed_manager->EnqueueDestination(&outfeed_buffers); // Now turn the tree of buffers back into a literal. *literal = std::move(*ShapeTreeToLiteral(&outfeed_buffers)); return Status::OK(); } +} // namespace gpu } // namespace xla static std::unique_ptr CreateGpuTransferManager() { - return xla::MakeUnique(); + return xla::MakeUnique(); } static bool InitModule() { diff --git a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h index 9dff1e5a50..7a5fe6979f 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h +++ b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/generic_transfer_manager.h" #include "tensorflow/compiler/xla/service/gpu/infeed_manager.h" #include "tensorflow/compiler/xla/service/transfer_manager.h" +#include "tensorflow/compiler/xla/shape_tree.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/platform/macros.h" @@ -28,6 +29,7 @@ limitations under the License. #include "tensorflow/core/platform/types.h" namespace xla { +namespace gpu { // An implementation of the XLA GenericTransferManager that // handles GPU-specific infeed. @@ -47,17 +49,18 @@ class GpuTransferManager : public GenericTransferManager { private: // Initiates the infeed data transfers. InfeedBuffer->Done() must be // called to clean up the memory allocated for InfeedBuffer. - StatusOr TransferBufferToInfeedInternal( + StatusOr TransferBufferToInfeedInternal( se::StreamExecutor* executor, int64 size, const void* source); // Enqueues infeed data buffers with the infeed manager after their // transfer completes. Status EnqueueBuffersToInfeed(se::StreamExecutor* executor, - std::vector buffers); + ShapeTree buffers); TF_DISALLOW_COPY_AND_ASSIGN(GpuTransferManager); }; +} // namespace gpu } // namespace xla #endif // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_TRANSFER_MANAGER_H_ diff --git a/tensorflow/compiler/xla/service/gpu/infeed_manager.cc b/tensorflow/compiler/xla/service/gpu/infeed_manager.cc index ae310beefa..c5f0cdf6cd 100644 --- a/tensorflow/compiler/xla/service/gpu/infeed_manager.cc +++ b/tensorflow/compiler/xla/service/gpu/infeed_manager.cc @@ -15,76 +15,13 @@ limitations under the License. #include "tensorflow/compiler/xla/service/gpu/infeed_manager.h" -#include "tensorflow/compiler/xla/map_util.h" #include "tensorflow/compiler/xla/ptr_util.h" -#include "tensorflow/core/platform/logging.h" namespace xla { namespace gpu { -InfeedManager::InfeedManager() : host_to_device_executor_(nullptr) {} - -void InfeedManager::Reset() { - tensorflow::mutex_lock l(mu_); - CHECK(dequeued_buffer_.empty()); - for (auto buffer : enqueued_buffer_) { - buffer->Done(); - } - enqueued_buffer_.clear(); -} - -void InfeedManager::EnqueueBuffers(const std::vector& buffers) { - tensorflow::mutex_lock l(mu_); - bool was_empty = enqueued_buffer_.empty(); - for (gpu::InfeedBuffer* b : buffers) { - enqueued_buffer_.push_back(b); - } - if (was_empty) { - // This has the potential to suffer from the notified thread - // immediately trying and failing to acquire mu_, but seems - // preferable to the alternative of notifying outside the lock - // on every enqueue. - cv_.notify_one(); - } -} - -InfeedBuffer* InfeedManager::BlockingDequeueBuffer() { - bool became_empty = false; - InfeedBuffer* current_buffer; - { - tensorflow::mutex_lock l(mu_); - while (enqueued_buffer_.empty()) { - cv_.wait(l); - } - current_buffer = enqueued_buffer_.front(); - enqueued_buffer_.pop_front(); - dequeued_buffer_.insert(current_buffer); - if (enqueued_buffer_.empty()) { - became_empty = true; - } - } - if (became_empty) { - for (const auto& callback : on_empty_callbacks_) { - callback(); - } - } - return current_buffer; -} - -void InfeedManager::ReleaseBuffers(const std::vector& buffers) { - { - tensorflow::mutex_lock l(mu_); - for (gpu::InfeedBuffer* b : buffers) { - CHECK(ContainsKey(dequeued_buffer_, b)); - dequeued_buffer_.erase(b); - } - } - for (gpu::InfeedBuffer* b : buffers) { - b->Done(); - } -} - se::Stream* InfeedManager::GetStream(se::StreamExecutor* executor) { + tensorflow::mutex_lock l(host_to_device_stream_mu_); if (host_to_device_executor_ == nullptr) { host_to_device_executor_ = executor; host_to_device_stream_ = MakeUnique(executor); @@ -100,10 +37,6 @@ se::Stream* InfeedManager::GetStream(se::StreamExecutor* executor) { return host_to_device_stream_.get(); } -void InfeedManager::RegisterOnEmptyCallback(std::function callback) { - on_empty_callbacks_.push_back(std::move(callback)); -} - InfeedManager* GetOrCreateInfeedManager() { static InfeedManager* manager = new InfeedManager; return manager; diff --git a/tensorflow/compiler/xla/service/gpu/infeed_manager.h b/tensorflow/compiler/xla/service/gpu/infeed_manager.h index a3fc15cfe3..7e418882e0 100644 --- a/tensorflow/compiler/xla/service/gpu/infeed_manager.h +++ b/tensorflow/compiler/xla/service/gpu/infeed_manager.h @@ -20,12 +20,9 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_INFEED_MANAGER_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_INFEED_MANAGER_H_ -#include -#include - +#include "tensorflow/compiler/xla/service/gpu/xfeed_queue.h" +#include "tensorflow/compiler/xla/shape_tree.h" #include "tensorflow/compiler/xla/types.h" -#include "tensorflow/core/lib/gtl/flatset.h" -#include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/stream_executor_no_cuda.h" namespace xla { @@ -47,90 +44,41 @@ namespace gpu { // the client. The client manages the memory of the buffer. class InfeedBuffer { public: + InfeedBuffer() = default; InfeedBuffer(se::StreamExecutor* executor, int64 length) - : executor_(executor), length_(length) { - device_memory_ = executor_->AllocateArray(length); - CHECK(!device_memory_.is_null()); + : device_memory_(executor, executor->AllocateArray(length)), + length_(length) { + CHECK(!device_memory_->is_null()); } - ~InfeedBuffer() { executor_->Deallocate(&device_memory_); } - int64 length() const { return length_; } - // Callback to signal that this buffer is consumed. This helps the - // client to manage memory for the infeed buffers. - void Done() { delete this; } - - se::DeviceMemoryBase* device_memory() { return &device_memory_; } + se::DeviceMemoryBase* device_memory() { return device_memory_.ptr(); } private: - se::StreamExecutor* executor_; // Not owned. - const int64 length_; - se::DeviceMemoryBase device_memory_; + se::ScopedDeviceMemory device_memory_; + int64 length_; }; // Client-side class used to enqueue infeed buffers. -class InfeedManager { +class InfeedManager : public XfeedQueue> { public: - InfeedManager(); - - // Calls the completion callback for any enqueued buffers that have - // not been dequeued by the runtime, and empties the infeed - // queue. Reset may not be called while a runtime computation is - // processing a dequeued buffer. The only safe way to ensure this - // condition is to call Reset when no computation is taking place. - void Reset(); - - // Adds a set of buffers to the infeed queue atomically. buffer->Done - // will be called when the buffer will no longer be accessed by the - // InfeedManager, either as a result of a call to Reset or because the - // runtime has dequeued and used the buffer. - void EnqueueBuffers(const std::vector& buffers); - - // Blocks until the infeed queue is non-empty, then returns the - // buffer at the head of the queue. Adds the current buffer to the - // to-be released set. - InfeedBuffer* BlockingDequeueBuffer(); - - // Releases a set of buffers from the to-be released set. - void ReleaseBuffers(const std::vector& buffers); - // Returns a cached stream associated with an executor. Allocates a // new stream on the first invocation. On subsequent invocations, if // the cached executor is not the same as the requested executor, // returns null. se::Stream* GetStream(se::StreamExecutor* executor); - // Registers a callback that will be called when 'enqueued_buffer_' becomes - // empty. - void RegisterOnEmptyCallback(std::function callback); - private: - // TODO(b/30467474): Revisit if this mutex becomes a point of - // contention. - tensorflow::mutex mu_; - - // Condition variable that is signaled every time a buffer is - // enqueued to an empty queue. - tensorflow::condition_variable cv_; - - // InfeedBuffer* queue contents are not owned, but buffer->Done must - // be called when the buffer is no longer needed by the runtime. - std::deque enqueued_buffer_; - - // Buffers that are dequeued and currently being processed by the - // runtime. Not owned. - tensorflow::gtl::FlatSet dequeued_buffer_; + // Mutex for serializing the creation of host_to_device_stream_. + tensorflow::mutex host_to_device_stream_mu_; // Cached host to device stream for queuing infeed data. - std::unique_ptr host_to_device_stream_; + std::unique_ptr host_to_device_stream_ + GUARDED_BY(host_to_device_stream_mu_); // Executor that the host_to_device_stream belongs to. Not owned. - se::StreamExecutor* host_to_device_executor_; - - // List of callbacks which will be called when 'enqueued_buffer_' becomes - // empty. - std::vector> on_empty_callbacks_; + se::StreamExecutor* host_to_device_executor_ = nullptr; }; // Singleton creator-or-accessor: Returns the GPU infeed manager. diff --git a/tensorflow/compiler/xla/service/gpu/infeed_thunk.cc b/tensorflow/compiler/xla/service/gpu/infeed_thunk.cc index 62915febb1..964efd3657 100644 --- a/tensorflow/compiler/xla/service/gpu/infeed_thunk.cc +++ b/tensorflow/compiler/xla/service/gpu/infeed_thunk.cc @@ -38,9 +38,10 @@ Status InfeedThunk::ExecuteOnStream(const BufferAllocations& buffer_allocations, se::DeviceMemoryBase data_address = buffer_allocations.GetDeviceAddress(infeed_slices_.element({0})); InfeedManager* infeed_manager = GetOrCreateInfeedManager(); - std::vector infeed_buffers; const Shape& data_shape = ShapeUtil::GetTupleElementShape(hlo_instruction()->shape(), 0); + ShapeTree infeed_buffers = + infeed_manager->BlockingGetNextDestination(); if (ShapeUtil::IsTuple(data_shape)) { CHECK(!ShapeUtil::IsNestedTuple(data_shape)); // Transfer the tuple elements first. @@ -51,8 +52,7 @@ Status InfeedThunk::ExecuteOnStream(const BufferAllocations& buffer_allocations, se::DeviceMemoryBase tuple_element_address = buffer_allocations.GetDeviceAddress(tuple_element_buffer); - InfeedBuffer* buffer = infeed_manager->BlockingDequeueBuffer(); - infeed_buffers.push_back(buffer); + InfeedBuffer* buffer = infeed_buffers.mutable_element({i}); stream->ThenMemcpy(&tuple_element_address, *(buffer->device_memory()), buffer->length()); tuple_element_addresses.push_back(tuple_element_address.opaque()); @@ -62,19 +62,17 @@ Status InfeedThunk::ExecuteOnStream(const BufferAllocations& buffer_allocations, stream->ThenMemcpy(&data_address, tuple_element_addresses.data(), host_size); } else { - InfeedBuffer* buffer = infeed_manager->BlockingDequeueBuffer(); - infeed_buffers.push_back(buffer); + InfeedBuffer* buffer = infeed_buffers.mutable_element({}); stream->ThenMemcpy(&data_address, *(buffer->device_memory()), buffer->length()); } // Construct top-level tuple of infeed containing the data and the token. Use // a nullptr for the token, it should never be dereferenced. - std::vector infeed_addresses = {data_address.opaque(), nullptr}; + void* infeed_addresses[] = {data_address.opaque(), nullptr}; se::DeviceMemoryBase top_level_address = buffer_allocations.GetDeviceAddress(infeed_slices_.element({})); - stream->ThenMemcpy(&top_level_address, infeed_addresses.data(), - 2 * sizeof(void*)); + stream->ThenMemcpy(&top_level_address, infeed_addresses, 2 * sizeof(void*)); Status block_status = stream->BlockHostUntilDone(); if (!block_status.ok()) { @@ -82,8 +80,6 @@ Status InfeedThunk::ExecuteOnStream(const BufferAllocations& buffer_allocations, stream, block_status.error_message().c_str()); } - infeed_manager->ReleaseBuffers(infeed_buffers); - VLOG(2) << "Infeeding to GPU complete"; return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/gpu/outfeed_manager.cc b/tensorflow/compiler/xla/service/gpu/outfeed_manager.cc index 47744548b9..4aaf0c9e14 100644 --- a/tensorflow/compiler/xla/service/gpu/outfeed_manager.cc +++ b/tensorflow/compiler/xla/service/gpu/outfeed_manager.cc @@ -23,25 +23,6 @@ limitations under the License. namespace xla { namespace gpu { -void OutfeedManager::EnqueueOutfeedDestination( - ShapeTree>* buffers) { - tensorflow::mutex_lock l(mu_); - enqueued_buffers_.push_back(buffers); - cv_.notify_one(); -} - -ShapeTree>* -OutfeedManager::BlockingGetNextOutfeedDestination() { - tensorflow::mutex_lock l(mu_); - while (enqueued_buffers_.empty()) { - cv_.wait(l); - } - ShapeTree>* current_buffer = - enqueued_buffers_.front(); - enqueued_buffers_.pop_front(); - return current_buffer; -} - OutfeedManager* GetOrCreateOutfeedManager() { static auto* manager = new OutfeedManager; return manager; diff --git a/tensorflow/compiler/xla/service/gpu/outfeed_manager.h b/tensorflow/compiler/xla/service/gpu/outfeed_manager.h index f580c24e17..a752eb7011 100644 --- a/tensorflow/compiler/xla/service/gpu/outfeed_manager.h +++ b/tensorflow/compiler/xla/service/gpu/outfeed_manager.h @@ -16,10 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_OUTFEED_MANAGER_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_OUTFEED_MANAGER_H_ -#include -#include - #include "tensorflow/compiler/xla/literal.h" +#include "tensorflow/compiler/xla/service/gpu/xfeed_queue.h" #include "tensorflow/compiler/xla/shape_tree.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/notification.h" @@ -60,28 +58,7 @@ class OutfeedBuffer { // Manages a thread-safe queue of buffers. The buffers are supposed to be // produced by the transfer manager and consumed by the device. -class OutfeedManager { - public: - // Adds a tree of buffers to the queue. The individual buffers correspond to - // the elements of a tuple and may be nullptr if the buffer is a tuple index - // buffer. - void EnqueueOutfeedDestination( - ShapeTree>* buffers); - - // Blocks until the queue is non-empty, then returns the buffer at the head of - // the queue. - ShapeTree>* - BlockingGetNextOutfeedDestination(); - - private: - tensorflow::mutex mu_; - - // Condition variable that is signaled every time a buffer is enqueued. - tensorflow::condition_variable cv_; - - // The queue of trees of buffers. OutfeedBuffer* queue contents are not owned. - std::deque>*> enqueued_buffers_; -}; +using OutfeedManager = XfeedQueue>*>; // Singleton creator-or-accessor: Returns the GPU outfeed manager. OutfeedManager* GetOrCreateOutfeedManager(); diff --git a/tensorflow/compiler/xla/service/gpu/outfeed_thunk.cc b/tensorflow/compiler/xla/service/gpu/outfeed_thunk.cc index 4c0f1421e9..7986e63f43 100644 --- a/tensorflow/compiler/xla/service/gpu/outfeed_thunk.cc +++ b/tensorflow/compiler/xla/service/gpu/outfeed_thunk.cc @@ -36,7 +36,7 @@ Status OutfeedThunk::ExecuteOnStream( auto op_profiler = profiler->MakeScopedInstructionProfiler(hlo_instruction()); OutfeedManager* outfeed_manager = GetOrCreateOutfeedManager(); ShapeTree>* outfeed_buffers = - outfeed_manager->BlockingGetNextOutfeedDestination(); + outfeed_manager->BlockingGetNextDestination(); // Nothing to be done for empty tuples. if (ShapeUtil::IsEmptyTuple(hlo_instruction()->operand(0)->shape())) { diff --git a/tensorflow/compiler/xla/service/gpu/xfeed_queue.h b/tensorflow/compiler/xla/service/gpu/xfeed_queue.h new file mode 100644 index 0000000000..737c7eb025 --- /dev/null +++ b/tensorflow/compiler/xla/service/gpu/xfeed_queue.h @@ -0,0 +1,89 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_XFEED_QUEUE_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_XFEED_QUEUE_H_ + +#include +#include + +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/notification.h" +#include "tensorflow/core/platform/thread_annotations.h" + +namespace xla { +namespace gpu { + +// TODO(b/30467474) Once GPU outfeed implementation settles, consider +// folding back the cpu and gpu outfeed implementations into a generic +// one if possible. + +// Manages a thread-safe queue of buffers. +template +class XfeedQueue { + public: + // Adds a tree of buffers to the queue. The individual buffers correspond to + // the elements of a tuple and may be nullptr if the buffer is a tuple index + // buffer. + void EnqueueDestination(BufferType buffers) { + tensorflow::mutex_lock l(mu_); + enqueued_buffers_.push_back(std::move(buffers)); + cv_.notify_one(); + } + + // Blocks until the queue is non-empty, then returns the buffer at the head of + // the queue. + BufferType BlockingGetNextDestination() { + bool became_empty; + BufferType current_buffer; + { + tensorflow::mutex_lock l(mu_); + while (enqueued_buffers_.empty()) { + cv_.wait(l); + } + current_buffer = std::move(enqueued_buffers_.front()); + enqueued_buffers_.pop_front(); + became_empty = enqueued_buffers_.empty(); + } + if (became_empty) { + for (const auto& callback : on_empty_callbacks_) { + callback(); + } + } + return current_buffer; + } + + void RegisterOnEmptyCallback(std::function callback) { + on_empty_callbacks_.push_back(std::move(callback)); + } + + private: + tensorflow::mutex mu_; + + // Condition variable that is signaled every time a buffer is enqueued. + tensorflow::condition_variable cv_; + + // The queue of trees of buffers. Buffer* queue contents are not owned. + std::deque enqueued_buffers_ GUARDED_BY(mu_); + + // List of callbacks which will be called when 'enqueued_buffers_' becomes + // empty. + std::vector> on_empty_callbacks_; +}; + +} // namespace gpu +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_XFEED_QUEUE_H_ -- cgit v1.2.3 From b8ff1c197ddb8642ff50b876cfecdb6741777fab Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Thu, 12 Jul 2018 05:05:45 -0700 Subject: [XLA:GPU] Enable outfeed test on GPU Outfeed is supported now. PiperOrigin-RevId: 204281960 --- tensorflow/compiler/xla/tests/local_client_execute_test.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tensorflow/compiler/xla/tests/local_client_execute_test.cc b/tensorflow/compiler/xla/tests/local_client_execute_test.cc index 7c003fb81f..2f4d197ae6 100644 --- a/tensorflow/compiler/xla/tests/local_client_execute_test.cc +++ b/tensorflow/compiler/xla/tests/local_client_execute_test.cc @@ -866,9 +866,7 @@ XLA_TEST_F(LocalClientExecuteTest, InfeedTest) { LiteralTestUtil::ExpectR1Equal({-4.0, 125.0, 45.0}, *result); } -// TODO(b/34359662): Support infeed/outfeed on GPU and CPU parallel. -// 2017-10-18. -XLA_TEST_F(LocalClientExecuteTest, DISABLED_ON_GPU(InfeedOutfeedTest)) { +XLA_TEST_F(LocalClientExecuteTest, InfeedOutfeedTest) { XlaBuilder builder(TestName()); const Shape shape = ShapeUtil::MakeShape(F32, {3}); auto in = Infeed(&builder, shape); -- cgit v1.2.3 From 8e3001f5e7ffa71b565c4c67a5e2c800fa1ab7af Mon Sep 17 00:00:00 2001 From: Dmitry Klimenkov Date: Thu, 12 Jul 2018 15:56:12 +0300 Subject: Putting batch_axis,seq_axis instead batch_dim, seq_dim --- tensorflow/python/ops/rnn.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py index deba133fb9..aea01a8081 100644 --- a/tensorflow/python/ops/rnn.py +++ b/tensorflow/python/ops/rnn.py @@ -427,14 +427,14 @@ def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, if seq_lengths is not None: return array_ops.reverse_sequence( input=input_, seq_lengths=seq_lengths, - seq_dim=seq_dim, batch_dim=batch_dim) + seq_axis=seq_dim, batch_axis=batch_dim) else: return array_ops.reverse(input_, axis=[seq_dim]) with vs.variable_scope("bw") as bw_scope: inputs_reverse = _reverse( inputs, seq_lengths=sequence_length, - seq_dim=time_dim, batch_dim=batch_dim) + seq_axis=time_dim, batch_axis=batch_dim) tmp, output_state_bw = dynamic_rnn( cell=cell_bw, inputs=inputs_reverse, sequence_length=sequence_length, initial_state=initial_state_bw, dtype=dtype, @@ -443,7 +443,7 @@ def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, output_bw = _reverse( tmp, seq_lengths=sequence_length, - seq_dim=time_dim, batch_dim=batch_dim) + seq_axis=time_dim, batch_axis=batch_dim) outputs = (output_fw, output_bw) output_states = (output_state_fw, output_state_bw) -- cgit v1.2.3 From 4665633c5f3fcd98de8c4cab909c8ac64c28ed0f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 06:12:43 -0700 Subject: Add TraceableStack object and use it for the colocation stack to improve (future) error messages. The TraceableStack directly manages TraceableObjects. The TraceableStack object acts as a regular stack for arbitrary objects. Its object-encapsulation class, TraceableObject, includes metadata fields including an arbitrary name, and members and methods to track the filenames and line numbers of callers of the stack's call push_obj(). The contents of a TraceableStack can easily be summarized by a dict mapping the TraceableObject.name to new TraceableObjects that contain only the name, file, and line number metadata. This is useful for constructing public return values. PiperOrigin-RevId: 204288464 --- tensorflow/python/BUILD | 37 ++++++ tensorflow/python/framework/ops.py | 99 +++------------ tensorflow/python/framework/traceable_stack.py | 135 +++++++++++++++++++++ .../python/framework/traceable_stack_test.py | 131 ++++++++++++++++++++ tensorflow/python/util/tf_stack.py | 97 +++++++++++++++ 5 files changed, 416 insertions(+), 83 deletions(-) create mode 100644 tensorflow/python/framework/traceable_stack.py create mode 100644 tensorflow/python/framework/traceable_stack_test.py create mode 100644 tensorflow/python/util/tf_stack.py diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index d60d37df50..d00debe1a1 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -822,6 +822,7 @@ py_library( ":platform", ":registry", ":tensor_shape", + ":traceable_stack", ":util", ":versions", "//tensorflow/core:protos_all_py", @@ -887,6 +888,17 @@ py_library( ], ) +# This target is maintained separately from :util to provide separate visibility +# for legacy users who were granted visibility when the functions were private +# members of ops.Graph. +py_library( + name = "tf_stack", + srcs = ["util/tf_stack.py"], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], + deps = [], +) + py_library( name = "tensor_shape", srcs = ["framework/tensor_shape.py"], @@ -921,6 +933,16 @@ py_library( ], ) +py_library( + name = "traceable_stack", + srcs = ["framework/traceable_stack.py"], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], + deps = [ + ":util", + ], +) + py_library( name = "versions", srcs = ["framework/versions.py"], @@ -1207,6 +1229,21 @@ py_test( ], ) +py_test( + name = "framework_traceable_stack_test", + size = "small", + srcs = ["framework/traceable_stack_test.py"], + main = "framework/traceable_stack_test.py", + srcs_version = "PY2AND3", + deps = [ + ":framework_test_lib", + ":platform_test", + ":test_ops", + ":traceable_stack", + ":util", + ], +) + tf_gen_op_wrapper_py( name = "test_ops", out = "framework/test_ops.py", diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index b07c57d265..c4f58f0847 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -20,7 +20,6 @@ from __future__ import print_function import collections import copy -import linecache import os import re import sys @@ -48,7 +47,9 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import op_def_registry from tensorflow.python.framework import registry +from tensorflow.python.util import tf_stack from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import traceable_stack from tensorflow.python.framework import versions from tensorflow.python.ops import control_flow_util from tensorflow.python.platform import app @@ -1713,7 +1714,7 @@ class Operation(object): self._id_value = self._graph._next_id() # pylint: disable=protected-access self._original_op = original_op - self._traceback = self._graph._extract_stack() # pylint: disable=protected-access + self._traceback = tf_stack.extract_stack() self._control_flow_context = self.graph._get_control_flow_context() # pylint: disable=protected-access # Initialize self._c_op. @@ -2154,7 +2155,7 @@ class Operation(object): @property def traceback(self): """Returns the call stack from when this operation was constructed.""" - return self._graph._convert_stack(self._traceback) # pylint: disable=protected-access + return tf_stack.convert_stack(self._traceback) @property def traceback_with_start_lines(self): @@ -2163,9 +2164,8 @@ class Operation(object): Returns: A list of 5-tuples (filename, lineno, name, code, func_start_lineno). """ - return self._graph._convert_stack( # pylint: disable=protected-access - self._traceback, - include_func_start_lineno=True) + return tf_stack.convert_stack(self._traceback, + include_func_start_lineno=True) def _set_attr(self, attr_name, attr_value): """Private method used to set an attribute in the node_def.""" @@ -2617,7 +2617,6 @@ def _name_from_scope_name(name): _MUTATION_LOCK_GROUP = 0 _SESSION_RUN_LOCK_GROUP = 1 - @tf_export("Graph") class Graph(object): """A TensorFlow computation, represented as a dataflow graph. @@ -2726,7 +2725,7 @@ class Graph(object): self._building_function = False # Stack of colocate_with ops. After switch_to_thread_local(), # self._thread_local._colocation_stack is used instead. - self._graph_colocation_stack = [] + self._graph_colocation_stack = traceable_stack.TraceableStack() # Set of tensors that are dangerous to feed! self._unfeedable_tensors = set() # Set of operations that are dangerous to fetch! @@ -2766,36 +2765,6 @@ class Graph(object): """Temporary hack; can be overridden to force C API usage.""" return _USE_C_API - def _convert_stack(self, stack, include_func_start_lineno=False): - """Converts a stack extracted using _extract_stack() to a traceback stack. - - Args: - stack: A list of n 5-tuples, - (filename, lineno, name, frame_globals, func_start_lineno). - include_func_start_lineno: True if function start line number should be - included as the 5th entry in return tuples. - - Returns: - A list of n 4-tuples or 5-tuples - (filename, lineno, name, code, [optional: func_start_lineno]), where the - code tuple element is calculated from the corresponding elements of the - input tuple. - """ - ret = [] - for (filename, lineno, name, frame_globals, func_start_lineno, - unused_frame_info) in stack: - linecache.checkcache(filename) - line = linecache.getline(filename, lineno, frame_globals) - if line: - line = line.strip() - else: - line = None - if include_func_start_lineno: - ret.append((filename, lineno, name, line, func_start_lineno)) - else: - ret.append((filename, lineno, name, line)) - return ret - # Note: this method is private because the API of tf.Graph() is public and # frozen, and this functionality is still not ready for public visibility. @tf_contextlib.contextmanager @@ -2821,46 +2790,6 @@ class Graph(object): def _variable_creator_stack(self, variable_creator_stack): self._thread_local._variable_creator_stack = variable_creator_stack - def _extract_stack(self): - """A lightweight, extensible re-implementation of traceback.extract_stack. - - NOTE(mrry): traceback.extract_stack eagerly retrieves the line of code for - each stack frame using linecache, which results in an abundance of stat() - calls. This implementation does not retrieve the code, and any consumer - should apply _convert_stack to the result to obtain a traceback that can - be formatted etc. using traceback methods. - - Derived classes can implement _extract_frame_info() to add extra information - to the traceback. - - Returns: - A list of 6-tuples - (filename, lineno, name, frame_globals, func_start_lineno, custom_info) - corresponding to the call stack of the current thread. - """ - try: - raise ZeroDivisionError - except ZeroDivisionError: - f = sys.exc_info()[2].tb_frame.f_back - ret = [] - while f is not None: - lineno = f.f_lineno - co = f.f_code - filename = co.co_filename - name = co.co_name - frame_globals = f.f_globals - func_start_lineno = co.co_firstlineno - frame_info = self._extract_frame_info(f) - ret.append((filename, lineno, name, frame_globals, func_start_lineno, - frame_info)) - f = f.f_back - ret.reverse() - return ret - - def _extract_frame_info(self, frame): # pylint: disable=unused-argument - """Extracts custom information from a frame in an op traceback.""" - return None - def _check_not_finalized(self): """Check if the graph is finalized. @@ -3301,7 +3230,7 @@ class Graph(object): if self._colocation_stack: all_colocation_groups = [] - for colocation_op in self._colocation_stack: + for colocation_op in self._colocation_stack.peek_objs(): all_colocation_groups.extend(colocation_op.colocation_groups()) if colocation_op.device: # Make this device match the device of the colocated op, to provide @@ -4074,10 +4003,10 @@ class Graph(object): if ignore_existing: current_stack = self._colocation_stack - self._colocation_stack = [] + self._colocation_stack = traceable_stack.TraceableStack() if op is not None: - self._colocation_stack.append(op) + self._colocation_stack.push_obj(op, name=op.name, offset=1) try: yield @@ -4085,7 +4014,7 @@ class Graph(object): # Restore device function stack self._device_function_stack = device_fn_tmp if op is not None: - self._colocation_stack.pop() + self._colocation_stack.pop_obj() # Reset the colocation stack if requested. if ignore_existing: @@ -4712,11 +4641,15 @@ class Graph(object): @property def _colocation_stack(self): + """Return thread-local copy of colocation stack.""" if self._stack_state_is_thread_local: # This may be called from a thread where colocation_stack doesn't yet # exist. if not hasattr(self._thread_local, "_colocation_stack"): - self._thread_local._colocation_stack = self._graph_colocation_stack[:] + stack_copy_for_this_thread = self._graph_colocation_stack.copy() + # pylint: disable=protected-access + self._thread_local._colocation_stack = stack_copy_for_this_thread + # pylint: enable=protected-access return self._thread_local._colocation_stack else: return self._graph_colocation_stack diff --git a/tensorflow/python/framework/traceable_stack.py b/tensorflow/python/framework/traceable_stack.py new file mode 100644 index 0000000000..1b7c6bd7c5 --- /dev/null +++ b/tensorflow/python/framework/traceable_stack.py @@ -0,0 +1,135 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A simple stack that associates filename and line numbers with each object.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.util import tf_stack + + +class TraceableObject(object): + """Wrap an object together with its the code definition location.""" + + # Return codes for the set_filename_and_line_from_caller() method. + SUCCESS, HEURISTIC_USED, FAILURE = (0, 1, 2) + + def __init__(self, obj, name=None, filename=None, lineno=None): + self.obj = obj + self.name = name + self.filename = filename + self.lineno = lineno + + def set_filename_and_line_from_caller(self, offset=0): + """Set filename and line using the caller's stack frame. + + If the requested stack information is not available, a heuristic may + be applied and self.HEURISTIC USED will be returned. If the heuristic + fails then no change will be made to the filename and lineno members + (None by default) and self.FAILURE will be returned. + + Args: + offset: Integer. If 0, the caller's stack frame is used. If 1, + the caller's caller's stack frame is used. Larger values are + permissible but if out-of-range (larger than the number of stack + frames available) the outermost stack frame will be used. + + Returns: + TraceableObject.SUCCESS if appropriate stack information was found, + TraceableObject.HEURISTIC_USED if the offset was larger than the stack, + and TraceableObject.FAILURE if the stack was empty. + """ + # Offset is defined in "Args" as relative to the caller. We are one frame + # beyond the caller. + local_offset = offset + 1 + + frame_records = tf_stack.extract_stack() + if not frame_records: + return self.FAILURE + if len(frame_records) >= local_offset: + # Negative indexing is one-indexed instead of zero-indexed. + negative_offset = -(local_offset + 1) + self.filename, self.lineno = frame_records[negative_offset][:2] + return self.SUCCESS + else: + # If the offset is too large then we use the largest offset possible, + # meaning we use the outermost stack frame at index 0. + self.filename, self.lineno = frame_records[0][:2] + return self.HEURISTIC_USED + + def copy_metadata(self): + """Return a TraceableObject like this one, but without the object.""" + return self.__class__(None, name=self.name, filename=self.filename, + lineno=self.lineno) + + +class TraceableStack(object): + """A stack of TraceableObjects.""" + + def __init__(self, existing_stack=None): + """Constructor. + + Args: + existing_stack: [TraceableObject, ...] If provided, this object will + set its new stack to a SHALLOW COPY of existing_stack. + """ + self._stack = existing_stack[:] if existing_stack else [] + + def push_obj(self, obj, name=None, offset=0): + """Add object to the stack and record its filename and line information. + + Args: + obj: An object to store on the stack. + name: A name for the object, used for dict keys in get_item_metadata_dict. + offset: Integer. If 0, the caller's stack frame is used. If 1, + the caller's caller's stack frame is used. + + Returns: + TraceableObject.SUCCESS if appropriate stack information was found, + TraceableObject.HEURISTIC_USED if the stack was smaller than expected, + and TraceableObject.FAILURE if the stack was empty. + """ + traceable_obj = TraceableObject(obj, name=name) + self._stack.append(traceable_obj) + # Offset is defined in "Args" as relative to the caller. We are 1 frame + # beyond the caller and need to compensate. + return traceable_obj.set_filename_and_line_from_caller(offset + 1) + + def pop_obj(self): + """Remove last-inserted object and return it, without filename/line info.""" + return self._stack.pop().obj + + def peek_objs(self): + """Return list of stored objects ordered newest to oldest.""" + return [t_obj.obj for t_obj in reversed(self._stack)] + + def peek_traceable_objs(self): + """Return list of stored TraceableObjects ordered newest to oldest.""" + return list(reversed(self._stack)) + + def __len__(self): + """Return number of items on the stack, and used for truth-value testing.""" + return len(self._stack) + + def copy(self): + """Return a copy of self referencing the same objects but in a new list. + + This method is implemented to support thread-local stacks. + + Returns: + TraceableStack with a new list that holds existing objects. + """ + return TraceableStack(self._stack) diff --git a/tensorflow/python/framework/traceable_stack_test.py b/tensorflow/python/framework/traceable_stack_test.py new file mode 100644 index 0000000000..168f9083b0 --- /dev/null +++ b/tensorflow/python/framework/traceable_stack_test.py @@ -0,0 +1,131 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tensorflow.python.framework.traceable_stack.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import test_util +from tensorflow.python.framework import traceable_stack +from tensorflow.python.platform import googletest +from tensorflow.python.util import tf_inspect as inspect + +_LOCAL_OBJECT = lambda x: x +_THIS_FILENAME = inspect.getsourcefile(_LOCAL_OBJECT) + + +class TraceableObjectTest(test_util.TensorFlowTestCase): + + def testSetFilenameAndLineFromCallerUsesCallersStack(self): + t_obj = traceable_stack.TraceableObject(17) + + # Do not separate placeholder from the set_filename_and_line_from_caller() + # call one line below it as it is used to calculate the latter's line + # number. + placeholder = lambda x: x + result = t_obj.set_filename_and_line_from_caller() + + expected_lineno = inspect.getsourcelines(placeholder)[1] + 1 + self.assertEqual(expected_lineno, t_obj.lineno) + self.assertEqual(_THIS_FILENAME, t_obj.filename) + self.assertEqual(t_obj.SUCCESS, result) + + def testSetFilenameAndLineFromCallerRespectsOffset(self): + + def call_set_filename_and_line_from_caller(t_obj): + # We expect to retrieve the line number from _our_ caller. + return t_obj.set_filename_and_line_from_caller(offset=1) + + t_obj = traceable_stack.TraceableObject(None) + # Do not separate placeholder from the + # call_set_filename_and_line_from_caller() call one line below it as it is + # used to calculate the latter's line number. + placeholder = lambda x: x + result = call_set_filename_and_line_from_caller(t_obj) + + expected_lineno = inspect.getsourcelines(placeholder)[1] + 1 + self.assertEqual(expected_lineno, t_obj.lineno) + self.assertEqual(t_obj.SUCCESS, result) + + def testSetFilenameAndLineFromCallerHandlesRidiculousOffset(self): + t_obj = traceable_stack.TraceableObject('The quick brown fox.') + # This line shouldn't die. + result = t_obj.set_filename_and_line_from_caller(offset=300) + + # We expect a heuristic to be used because we are not currently 300 frames + # down on the stack. The filename should be some wacky thing from the + # outermost stack frame -- definitely not equal to this filename. + self.assertEqual(t_obj.HEURISTIC_USED, result) + self.assertNotEqual(_THIS_FILENAME, t_obj.filename) + + +class TraceableStackTest(test_util.TensorFlowTestCase): + + def testPushPeekPopObj(self): + t_stack = traceable_stack.TraceableStack() + t_stack.push_obj(42.0) + t_stack.push_obj('hope') + + expected_lifo_peek = ['hope', 42.0] + self.assertEqual(expected_lifo_peek, t_stack.peek_objs()) + + self.assertEqual('hope', t_stack.pop_obj()) + self.assertEqual(42.0, t_stack.pop_obj()) + + def testPushPopPreserveLifoOrdering(self): + t_stack = traceable_stack.TraceableStack() + t_stack.push_obj(0) + t_stack.push_obj(1) + t_stack.push_obj(2) + t_stack.push_obj(3) + + obj_3 = t_stack.pop_obj() + obj_2 = t_stack.pop_obj() + obj_1 = t_stack.pop_obj() + obj_0 = t_stack.pop_obj() + + self.assertEqual(3, obj_3) + self.assertEqual(2, obj_2) + self.assertEqual(1, obj_1) + self.assertEqual(0, obj_0) + + def testPushObjSetsFilenameAndLineInfoForCaller(self): + t_stack = traceable_stack.TraceableStack() + + # We expect that the line number recorded for the 1-object will come from + # the call to t_stack.push_obj(1). Do not separate the next two lines! + placeholder_1 = lambda x: x + t_stack.push_obj(1) + + # We expect that the line number recorded for the 2-object will come from + # the call to call_push_obj() and _not_ the call to t_stack.push_obj(). + def call_push_obj(obj): + t_stack.push_obj(obj, offset=1) + + # Do not separate the next two lines! + placeholder_2 = lambda x: x + call_push_obj(2) + + expected_lineno_1 = inspect.getsourcelines(placeholder_1)[1] + 1 + expected_lineno_2 = inspect.getsourcelines(placeholder_2)[1] + 1 + + t_obj_2, t_obj_1 = t_stack.peek_traceable_objs() + self.assertEqual(expected_lineno_2, t_obj_2.lineno) + self.assertEqual(expected_lineno_1, t_obj_1.lineno) + + +if __name__ == '__main__': + googletest.main() diff --git a/tensorflow/python/util/tf_stack.py b/tensorflow/python/util/tf_stack.py new file mode 100644 index 0000000000..dacc1ce83e --- /dev/null +++ b/tensorflow/python/util/tf_stack.py @@ -0,0 +1,97 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions used to extract and analyze stacks. Faster than Python libs.""" +# pylint: disable=g-bad-name +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import linecache +import sys + + +def extract_stack(extract_frame_info_fn=None): + """A lightweight, extensible re-implementation of traceback.extract_stack. + + NOTE(mrry): traceback.extract_stack eagerly retrieves the line of code for + each stack frame using linecache, which results in an abundance of stat() + calls. This implementation does not retrieve the code, and any consumer + should apply _convert_stack to the result to obtain a traceback that can + be formatted etc. using traceback methods. + + Args: + extract_frame_info_fn: Optional callable fn(stack_frame) applied to each + stack frame. This callable's return value is stored as the sixth (last) + element of the returned tuples. If not provided, the returned tuples + will have None as their sixth value. + + Returns: + A list of 6-tuples + (filename, lineno, name, frame_globals, func_start_lineno, custom_info) + corresponding to the call stack of the current thread. The returned tuples + have the innermost stack frame at the end, unlike the Python inspect + module's stack() function. + """ + default_fn = lambda f: None + extract_frame_info_fn = extract_frame_info_fn or default_fn + try: + raise ZeroDivisionError + except ZeroDivisionError: + f = sys.exc_info()[2].tb_frame.f_back + ret = [] + while f is not None: + lineno = f.f_lineno + co = f.f_code + filename = co.co_filename + name = co.co_name + frame_globals = f.f_globals + func_start_lineno = co.co_firstlineno + frame_info = extract_frame_info_fn(f) + ret.append((filename, lineno, name, frame_globals, func_start_lineno, + frame_info)) + f = f.f_back + ret.reverse() + return ret + + +def convert_stack(stack, include_func_start_lineno=False): + """Converts a stack extracted using extract_stack() to a traceback stack. + + Args: + stack: A list of n 5-tuples, + (filename, lineno, name, frame_globals, func_start_lineno). + include_func_start_lineno: True if function start line number should be + included as the 5th entry in return tuples. + + Returns: + A list of n 4-tuples or 5-tuples + (filename, lineno, name, code, [optional: func_start_lineno]), where the + code tuple element is calculated from the corresponding elements of the + input tuple. + """ + ret = [] + for (filename, lineno, name, frame_globals, func_start_lineno, + unused_frame_info) in stack: + linecache.checkcache(filename) + line = linecache.getline(filename, lineno, frame_globals) + if line: + line = line.strip() + else: + line = None + if include_func_start_lineno: + ret.append((filename, lineno, name, line, func_start_lineno)) + else: + ret.append((filename, lineno, name, line)) + return ret -- cgit v1.2.3 From afc16bed74798fb47bfbb8fceda19b92dbbf67bc Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Thu, 12 Jul 2018 16:47:27 +0200 Subject: [tf.keras] Fix deprecated use of normal distribution in initializers Usage of `distribution='normal'` in `VarianceScaling` initializer was deprecated in #20197 and replaced with `truncated_normal`. --- tensorflow/python/keras/initializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/keras/initializers.py b/tensorflow/python/keras/initializers.py index 28beb6760d..ea104ea65d 100644 --- a/tensorflow/python/keras/initializers.py +++ b/tensorflow/python/keras/initializers.py @@ -58,7 +58,7 @@ def lecun_normal(seed=None): Backprop](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf) """ return VarianceScaling( - scale=1., mode='fan_in', distribution='normal', seed=seed) + scale=1., mode='fan_in', distribution='truncated_normal', seed=seed) @tf_export('keras.initializers.lecun_uniform') @@ -101,7 +101,7 @@ def he_normal(seed=None): He et al., http://arxiv.org/abs/1502.01852 """ return VarianceScaling( - scale=2., mode='fan_in', distribution='normal', seed=seed) + scale=2., mode='fan_in', distribution='truncated_normal', seed=seed) @tf_export('keras.initializers.he_uniform') -- cgit v1.2.3 From 34a1b6780b55764802cd490e50481d4d2ed8355c Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Thu, 12 Jul 2018 08:22:15 -0700 Subject: Add an owner set property to QN objects. This assists extracting symbols that are affected when objects or containers are mutated in place. Also included is a utility method that simplifies constructing a QN from string. PiperOrigin-RevId: 204303017 --- tensorflow/contrib/autograph/pyct/qual_names.py | 28 ++++++++++++++++++++-- .../contrib/autograph/pyct/qual_names_test.py | 9 +++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/autograph/pyct/qual_names.py b/tensorflow/contrib/autograph/pyct/qual_names.py index da07013cf4..fb81404edc 100644 --- a/tensorflow/contrib/autograph/pyct/qual_names.py +++ b/tensorflow/contrib/autograph/pyct/qual_names.py @@ -30,6 +30,7 @@ import collections import gast from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import parser class Symbol(collections.namedtuple('Symbol', ['name'])): @@ -89,7 +90,8 @@ class QN(object): if not isinstance(base, (str, StringLiteral, NumberLiteral)): # TODO(mdan): Require Symbol instead of string. raise ValueError( - 'For simple QNs, base must be a string or a Literal object.') + 'for simple QNs, base must be a string or a Literal object;' + ' got instead "%s"' % type(base)) assert '.' not in base and '[' not in base and ']' not in base self._parent = None self.qn = (base,) @@ -112,6 +114,22 @@ class QN(object): raise ValueError('Cannot get parent of simple name "%s".' % self.qn[0]) return self._parent + @property + def owner_set(self): + """Returns all the symbols (simple or composite) that own this QN. + + In other words, if this symbol was modified, the symbols in the owner set + may also be affected. + + Examples: + 'a.b[c.d]' has two owners, 'a' and 'a.b' + """ + owners = set() + if self.has_attr() or self.has_subscript(): + owners.add(self.parent) + owners.update(self.parent.owner_set) + return owners + @property def support_set(self): """Returns the set of simple symbols that this QN relies on. @@ -122,7 +140,7 @@ class QN(object): Examples: 'a.b' has only one support symbol, 'a' - 'a[i]' has two roots, 'a' and 'i' + 'a[i]' has two support symbols, 'a' and 'i' """ # TODO(mdan): This might be the set of Name nodes in the AST. Track those? roots = set() @@ -231,3 +249,9 @@ class QnResolver(gast.NodeTransformer): def resolve(node): return QnResolver().visit(node) + + +def from_str(qn_str): + node = parser.parse_expression(qn_str) + node = resolve(node) + return anno.getanno(node, anno.Basic.QN) diff --git a/tensorflow/contrib/autograph/pyct/qual_names_test.py b/tensorflow/contrib/autograph/pyct/qual_names_test.py index 264afd508c..c793c2bb39 100644 --- a/tensorflow/contrib/autograph/pyct/qual_names_test.py +++ b/tensorflow/contrib/autograph/pyct/qual_names_test.py @@ -30,6 +30,15 @@ from tensorflow.python.platform import test class QNTest(test.TestCase): + def test_from_str(self): + a = QN('a') + b = QN('b') + a_dot_b = QN(a, attr='b') + a_sub_b = QN(a, subscript=b) + self.assertEqual(qual_names.from_str('a.b'), a_dot_b) + self.assertEqual(qual_names.from_str('a'), a) + self.assertEqual(qual_names.from_str('a[b]'), a_sub_b) + def test_basic(self): a = QN('a') self.assertEqual(a.qn, ('a',)) -- cgit v1.2.3 From 0ca8c47bfe47da178d976c7fd8c8ac8df1b2ba19 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 12 Jul 2018 08:54:19 -0700 Subject: Cleaning up test proto for `tensorflow/contrib/rpc`. PiperOrigin-RevId: 204307008 --- tensorflow/contrib/rpc/python/kernel_tests/BUILD | 3 - .../rpc/python/kernel_tests/rpc_op_test_base.py | 52 ++++---- .../python/kernel_tests/rpc_op_test_servicer.py | 8 +- .../rpc/python/kernel_tests/test_example.proto | 147 +-------------------- 4 files changed, 34 insertions(+), 176 deletions(-) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/BUILD b/tensorflow/contrib/rpc/python/kernel_tests/BUILD index 2311c15a68..cb0b89ae55 100644 --- a/tensorflow/contrib/rpc/python/kernel_tests/BUILD +++ b/tensorflow/contrib/rpc/python/kernel_tests/BUILD @@ -1,5 +1,3 @@ -# TODO(b/76425722): Port everything in here to OS (currently excluded). - package(default_visibility = ["//visibility:public"]) licenses(["notice"]) # Apache 2.0 @@ -17,7 +15,6 @@ tf_proto_library( srcs = ["test_example.proto"], has_services = 1, cc_api_version = 2, - protodeps = ["//tensorflow/core:protos_all"], ) py_library( diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py index 27273d16b1..1c23c28860 100644 --- a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py +++ b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_base.py @@ -51,23 +51,23 @@ class RpcOpTestBase(object): def testScalarHostPortRpc(self): with self.test_session() as sess: request_tensors = ( - test_example_pb2.TestCase(shape=[1, 2, 3]).SerializeToString()) + test_example_pb2.TestCase(values=[1, 2, 3]).SerializeToString()) response_tensors = self.rpc( - method=self.get_method_name('IncrementTestShapes'), + method=self.get_method_name('Increment'), address=self._address, request=request_tensors) self.assertEqual(response_tensors.shape, ()) response_values = sess.run(response_tensors) response_message = test_example_pb2.TestCase() self.assertTrue(response_message.ParseFromString(response_values)) - self.assertAllEqual([2, 3, 4], response_message.shape) + self.assertAllEqual([2, 3, 4], response_message.values) def testScalarHostPortTryRpc(self): with self.test_session() as sess: request_tensors = ( - test_example_pb2.TestCase(shape=[1, 2, 3]).SerializeToString()) + test_example_pb2.TestCase(values=[1, 2, 3]).SerializeToString()) response_tensors, status_code, status_message = self.try_rpc( - method=self.get_method_name('IncrementTestShapes'), + method=self.get_method_name('Increment'), address=self._address, request=request_tensors) self.assertEqual(status_code.shape, ()) @@ -77,7 +77,7 @@ class RpcOpTestBase(object): sess.run((response_tensors, status_code, status_message))) response_message = test_example_pb2.TestCase() self.assertTrue(response_message.ParseFromString(response_values)) - self.assertAllEqual([2, 3, 4], response_message.shape) + self.assertAllEqual([2, 3, 4], response_message.values) # For the base Rpc op, don't expect to get error status back. self.assertEqual(errors.OK, status_code_values) self.assertEqual(b'', status_message_values) @@ -86,7 +86,7 @@ class RpcOpTestBase(object): with self.test_session() as sess: request_tensors = [] response_tensors = self.rpc( - method=self.get_method_name('IncrementTestShapes'), + method=self.get_method_name('Increment'), address=self._address, request=request_tensors) self.assertAllEqual(response_tensors.shape, [0]) @@ -95,7 +95,7 @@ class RpcOpTestBase(object): def testInvalidMethod(self): for method in [ - '/InvalidService.IncrementTestShapes', + '/InvalidService.Increment', self.get_method_name('InvalidMethodName') ]: with self.test_session() as sess: @@ -115,12 +115,12 @@ class RpcOpTestBase(object): with self.assertRaises(errors.UnavailableError): sess.run( self.rpc( - method=self.get_method_name('IncrementTestShapes'), + method=self.get_method_name('Increment'), address=address, request='')) _, status_code_value, status_message_value = sess.run( self.try_rpc( - method=self.get_method_name('IncrementTestShapes'), + method=self.get_method_name('Increment'), address=address, request='')) self.assertEqual(errors.UNAVAILABLE, status_code_value) @@ -182,10 +182,10 @@ class RpcOpTestBase(object): with self.test_session() as sess: request_tensors = [ test_example_pb2.TestCase( - shape=[i, i + 1, i + 2]).SerializeToString() for i in range(20) + values=[i, i + 1, i + 2]).SerializeToString() for i in range(20) ] response_tensors = self.rpc( - method=self.get_method_name('IncrementTestShapes'), + method=self.get_method_name('Increment'), address=self._address, request=request_tensors) self.assertEqual(response_tensors.shape, (20,)) @@ -194,17 +194,17 @@ class RpcOpTestBase(object): for i in range(20): response_message = test_example_pb2.TestCase() self.assertTrue(response_message.ParseFromString(response_values[i])) - self.assertAllEqual([i + 1, i + 2, i + 3], response_message.shape) + self.assertAllEqual([i + 1, i + 2, i + 3], response_message.values) def testVecHostPortManyParallelRpcs(self): with self.test_session() as sess: request_tensors = [ test_example_pb2.TestCase( - shape=[i, i + 1, i + 2]).SerializeToString() for i in range(20) + values=[i, i + 1, i + 2]).SerializeToString() for i in range(20) ] many_response_tensors = [ self.rpc( - method=self.get_method_name('IncrementTestShapes'), + method=self.get_method_name('Increment'), address=self._address, request=request_tensors) for _ in range(10) ] @@ -216,25 +216,25 @@ class RpcOpTestBase(object): for i in range(20): response_message = test_example_pb2.TestCase() self.assertTrue(response_message.ParseFromString(response_values[i])) - self.assertAllEqual([i + 1, i + 2, i + 3], response_message.shape) + self.assertAllEqual([i + 1, i + 2, i + 3], response_message.values) def testVecHostPortRpcUsingEncodeAndDecodeProto(self): with self.test_session() as sess: request_tensors = encode_proto_op.encode_proto( message_type='tensorflow.contrib.rpc.TestCase', - field_names=['shape'], + field_names=['values'], sizes=[[3]] * 20, values=[ [[i, i + 1, i + 2] for i in range(20)], ]) response_tensor_strings = self.rpc( - method=self.get_method_name('IncrementTestShapes'), + method=self.get_method_name('Increment'), address=self._address, request=request_tensors) _, (response_shape,) = decode_proto_op.decode_proto( bytes=response_tensor_strings, message_type='tensorflow.contrib.rpc.TestCase', - field_names=['shape'], + field_names=['values'], output_types=[dtypes.int32]) response_shape_values = sess.run(response_shape) self.assertAllEqual([[i + 1, i + 2, i + 3] @@ -285,9 +285,9 @@ class RpcOpTestBase(object): addresses = flatten([[ self._address, 'unix:/tmp/this_unix_socket_doesnt_exist_97820348!!@' ] for _ in range(10)]) - request = test_example_pb2.TestCase(shape=[0, 1, 2]).SerializeToString() + request = test_example_pb2.TestCase(values=[0, 1, 2]).SerializeToString() response_tensors, status_code, _ = self.try_rpc( - method=self.get_method_name('IncrementTestShapes'), + method=self.get_method_name('Increment'), address=addresses, request=request) response_tensors_values, status_code_values = sess.run((response_tensors, @@ -303,9 +303,9 @@ class RpcOpTestBase(object): flatten = lambda x: list(itertools.chain.from_iterable(x)) with self.test_session() as sess: methods = flatten( - [[self.get_method_name('IncrementTestShapes'), 'InvalidMethodName'] + [[self.get_method_name('Increment'), 'InvalidMethodName'] for _ in range(10)]) - request = test_example_pb2.TestCase(shape=[0, 1, 2]).SerializeToString() + request = test_example_pb2.TestCase(values=[0, 1, 2]).SerializeToString() response_tensors, status_code, _ = self.try_rpc( method=methods, address=self._address, request=request) response_tensors_values, status_code_values = sess.run((response_tensors, @@ -325,10 +325,10 @@ class RpcOpTestBase(object): ] for _ in range(10)]) requests = [ test_example_pb2.TestCase( - shape=[i, i + 1, i + 2]).SerializeToString() for i in range(20) + values=[i, i + 1, i + 2]).SerializeToString() for i in range(20) ] response_tensors, status_code, _ = self.try_rpc( - method=self.get_method_name('IncrementTestShapes'), + method=self.get_method_name('Increment'), address=addresses, request=requests) response_tensors_values, status_code_values = sess.run((response_tensors, @@ -343,4 +343,4 @@ class RpcOpTestBase(object): response_message = test_example_pb2.TestCase() self.assertTrue( response_message.ParseFromString(response_tensors_values[i])) - self.assertAllEqual([i + 1, i + 2, i + 3], response_message.shape) + self.assertAllEqual([i + 1, i + 2, i + 3], response_message.values) diff --git a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py index 7cbd636cb1..265254aa51 100644 --- a/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py +++ b/tensorflow/contrib/rpc/python/kernel_tests/rpc_op_test_servicer.py @@ -30,8 +30,8 @@ from tensorflow.contrib.rpc.python.kernel_tests import test_example_pb2_grpc class RpcOpTestServicer(test_example_pb2_grpc.TestCaseServiceServicer): """Test servicer for RpcOp tests.""" - def IncrementTestShapes(self, request, context): - """Increment the entries in the shape attribute of request. + def Increment(self, request, context): + """Increment the entries in the `values` attribute of request. Args: request: input TestCase. @@ -40,8 +40,8 @@ class RpcOpTestServicer(test_example_pb2_grpc.TestCaseServiceServicer): Returns: output TestCase. """ - for i in range(len(request.shape)): - request.shape[i] += 1 + for i in range(len(request.values)): + request.values[i] += 1 return request def AlwaysFailWithInvalidArgument(self, request, context): diff --git a/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto b/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto index 96f4550f62..8141466349 100644 --- a/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto +++ b/tensorflow/contrib/rpc/python/kernel_tests/test_example.proto @@ -1,29 +1,17 @@ // Test description and protos to work with it. -// -// Many of the protos in this file are for unit tests that haven't been written yet. syntax = "proto2"; -import "tensorflow/core/framework/types.proto"; - package tensorflow.contrib.rpc; -// A TestCase holds a proto and a bunch of assertions -// about how it should decode. +// A TestCase holds a sequence of values. message TestCase { - // A batch of primitives to be serialized and decoded. - repeated RepeatedPrimitiveValue primitive = 1; - // The shape of the batch. - repeated int32 shape = 2; - // Expected sizes for each field. - repeated int32 sizes = 3; - // Expected values for each field. - repeated FieldSpec field = 4; + repeated int32 values = 1; }; service TestCaseService { - // Copy input, and increment each entry in 'shape' by 1. - rpc IncrementTestShapes(TestCase) returns (TestCase) { + // Copy input, and increment each entry in 'values' by 1. + rpc Increment(TestCase) returns (TestCase) { } // Sleep forever. @@ -42,130 +30,3 @@ service TestCaseService { rpc SometimesFailWithInvalidArgument(TestCase) returns (TestCase) { } }; - -// FieldSpec describes the expected output for a single field. -message FieldSpec { - optional string name = 1; - optional tensorflow.DataType dtype = 2; - optional RepeatedPrimitiveValue expected = 3; -}; - -message TestValue { - optional PrimitiveValue primitive_value = 1; - optional EnumValue enum_value = 2; - optional MessageValue message_value = 3; - optional RepeatedMessageValue repeated_message_value = 4; - optional RepeatedPrimitiveValue repeated_primitive_value = 6; -} - -message PrimitiveValue { - optional double double_value = 1; - optional float float_value = 2; - optional int64 int64_value = 3; - optional uint64 uint64_value = 4; - optional int32 int32_value = 5; - optional fixed64 fixed64_value = 6; - optional fixed32 fixed32_value = 7; - optional bool bool_value = 8; - optional string string_value = 9; - optional bytes bytes_value = 12; - optional uint32 uint32_value = 13; - optional sfixed32 sfixed32_value = 15; - optional sfixed64 sfixed64_value = 16; - optional sint32 sint32_value = 17; - optional sint64 sint64_value = 18; -} - -// NOTE: This definition must be kept in sync with PackedPrimitiveValue. -message RepeatedPrimitiveValue { - repeated double double_value = 1; - repeated float float_value = 2; - repeated int64 int64_value = 3; - repeated uint64 uint64_value = 4; - repeated int32 int32_value = 5; - repeated fixed64 fixed64_value = 6; - repeated fixed32 fixed32_value = 7; - repeated bool bool_value = 8; - repeated string string_value = 9; - repeated bytes bytes_value = 12; - repeated uint32 uint32_value = 13; - repeated sfixed32 sfixed32_value = 15; - repeated sfixed64 sfixed64_value = 16; - repeated sint32 sint32_value = 17; - repeated sint64 sint64_value = 18; - repeated PrimitiveValue message_value = 19; -} - -// A PackedPrimitiveValue looks exactly the same as a RepeatedPrimitiveValue -// in the text format, but the binary serializion is different. -// We test the packed representations by loading the same test cases -// using this definition instead of RepeatedPrimitiveValue. -// NOTE: This definition must be kept in sync with RepeatedPrimitiveValue -// in every way except the packed=true declaration. -message PackedPrimitiveValue { - repeated double double_value = 1 [packed = true]; - repeated float float_value = 2 [packed = true]; - repeated int64 int64_value = 3 [packed = true]; - repeated uint64 uint64_value = 4 [packed = true]; - repeated int32 int32_value = 5 [packed = true]; - repeated fixed64 fixed64_value = 6 [packed = true]; - repeated fixed32 fixed32_value = 7 [packed = true]; - repeated bool bool_value = 8 [packed = true]; - repeated string string_value = 9; - repeated bytes bytes_value = 12; - repeated uint32 uint32_value = 13 [packed = true]; - repeated sfixed32 sfixed32_value = 15 [packed = true]; - repeated sfixed64 sfixed64_value = 16 [packed = true]; - repeated sint32 sint32_value = 17 [packed = true]; - repeated sint64 sint64_value = 18 [packed = true]; - repeated PrimitiveValue message_value = 19; -} - -message EnumValue { - enum Color { - RED = 0; - ORANGE = 1; - YELLOW = 2; - GREEN = 3; - BLUE = 4; - INDIGO = 5; - VIOLET = 6; - }; - optional Color enum_value = 14; - repeated Color repeated_enum_value = 15; -} - - -message InnerMessageValue { - optional float float_value = 2; - repeated bytes bytes_values = 8; -} - -message MiddleMessageValue { - repeated int32 int32_values = 5; - optional InnerMessageValue message_value = 11; - optional uint32 uint32_value = 13; -} - -message MessageValue { - optional double double_value = 1; - optional MiddleMessageValue message_value = 11; -} - -message RepeatedMessageValue { - message NestedMessageValue { - optional float float_value = 2; - repeated bytes bytes_values = 8; - } - - repeated NestedMessageValue message_values = 11; -} - -// Message containing fields with field numbers higher than any field above. An -// instance of this message is prepended to each binary message in the test to -// exercise the code path that handles fields encoded out of order of field -// number. -message ExtraFields { - optional string string_value = 1776; - optional bool bool_value = 1777; -} -- cgit v1.2.3 From f50416cc05aa860ee88a55c62d440e77bfc6913e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 09:14:30 -0700 Subject: Automated rollback of commit a4e79e23bace78e3d89d8273828f9d82ad6f1b95 PiperOrigin-RevId: 204310030 --- tensorflow/contrib/lite/kernels/BUILD | 1 - tensorflow/contrib/lite/kernels/transpose_conv.cc | 111 +++---------------- .../contrib/lite/kernels/transpose_conv_test.cc | 121 +++------------------ 3 files changed, 30 insertions(+), 203 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD index edce73989c..33594c138b 100644 --- a/tensorflow/contrib/lite/kernels/BUILD +++ b/tensorflow/contrib/lite/kernels/BUILD @@ -984,7 +984,6 @@ tf_cc_test( ":builtin_ops", "//tensorflow/contrib/lite:framework", "//tensorflow/contrib/lite/kernels:test_util", - "@com_google_absl//absl/memory", "@com_google_googletest//:gtest", ], ) diff --git a/tensorflow/contrib/lite/kernels/transpose_conv.cc b/tensorflow/contrib/lite/kernels/transpose_conv.cc index 7182374a6f..8b9deeed20 100644 --- a/tensorflow/contrib/lite/kernels/transpose_conv.cc +++ b/tensorflow/contrib/lite/kernels/transpose_conv.cc @@ -22,7 +22,6 @@ limitations under the License. #include "tensorflow/contrib/lite/builtin_op_data.h" #include "tensorflow/contrib/lite/context.h" -#include "tensorflow/contrib/lite/kernels/eigen_support.h" #include "tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h" #include "tensorflow/contrib/lite/kernels/internal/tensor.h" #include "tensorflow/contrib/lite/kernels/kernel_util.h" @@ -39,35 +38,9 @@ constexpr int kWeightsTensor = 1; constexpr int kDataInputTensor = 2; constexpr int kOutputTensor = 0; -const int kTensorNotAllocated = -1; - -struct OpData { - // IDs are the arbitrary identifiers used by TF Lite to identify and access - // memory buffers. - int im2col_id = kTensorNotAllocated; - - // im2col is the only temporary currently tracked, therefore always index 0. - // If more temporaries are added, they should be properly tracked. - int32_t im2col_index = 0; -}; - -void* Init(TfLiteContext* context, const char* buffer, size_t length) { - // This is a builtin op, so we don't use the contents in 'buffer', if any. - // Instead, we allocate a new object to use as scratch space for im2col, and - // to carry information from Prepare() to Eval(). - auto* data = new OpData; - eigen_support::IncrementUsageCounter(context); - return data; -} - -void Free(TfLiteContext* context, void* buffer) { - eigen_support::DecrementUsageCounter(context); - delete reinterpret_cast(buffer); -} - -TfLiteStatus ResizeOutputTensor(TfLiteContext* context, - const TfLiteTensor* output_shape, - TfLiteTensor* output) { +TfLiteStatus ResizeOutputShape(TfLiteContext* context, + const TfLiteTensor* output_shape, + TfLiteTensor* output) { // Currently only support int32 for output shape. if (output_shape->type != kTfLiteInt32) { context->ReportError(context, "Output shape is %d, not int32.", @@ -83,60 +56,15 @@ TfLiteStatus ResizeOutputTensor(TfLiteContext* context, return context->ResizeTensor(context, output, output_shape_array); } -// Allocate temporary im2col tensor. -static TfLiteStatus AllocateIm2colTensor(TfLiteContext* context, - TfLiteNode* node) { - OpData* data = reinterpret_cast(node->user_data); - if (data->im2col_id == kTensorNotAllocated) { - context->AddTensors(context, 1, &data->im2col_id); - } - - TfLiteIntArrayFree(node->temporaries); - node->temporaries = TfLiteIntArrayCreate(1); - node->temporaries->data[data->im2col_index] = data->im2col_id; - - return kTfLiteOk; -} - -TfLiteStatus ResizeIm2ColTensor(TfLiteContext* context, - const TfLiteTensor* output_shape, - const TfLiteTensor* weights, - const TfLiteTensor* input, - TfLiteTensor* im2col) { - if (output_shape->type != kTfLiteInt32) { - context->ReportError(context, "im2col shape is %d, not int32.", - output_shape->type); - return kTfLiteError; - } - TF_LITE_ENSURE_EQ(context, NumElements(output_shape), 4); - TfLiteIntArray* im2col_shape_array = TfLiteIntArrayCreate(4); - im2col_shape_array->data[0] = output_shape->data.i32[0]; - im2col_shape_array->data[1] = output_shape->data.i32[1]; - im2col_shape_array->data[2] = output_shape->data.i32[2]; - const int input_depth = SizeOfDimension(input, 3); - const int filter_width = SizeOfDimension(weights, 1); - const int filter_height = SizeOfDimension(weights, 2); - im2col_shape_array->data[3] = input_depth * filter_height * filter_width; - - im2col->type = input->type; - im2col->allocation_type = kTfLiteArenaRw; - return context->ResizeTensor(context, im2col, im2col_shape_array); -} - TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, NumInputs(node), 3); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); - TF_LITE_ENSURE_STATUS(AllocateIm2colTensor(context, node)); - const TfLiteTensor* output_shape = GetInput(context, node, kOutputShapeTensor); const TfLiteTensor* weights = GetInput(context, node, kWeightsTensor); const TfLiteTensor* input = GetInput(context, node, kDataInputTensor); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - OpData* user_data = reinterpret_cast(node->user_data); - TfLiteTensor* im2col = - &context->tensors[node->temporaries->data[user_data->im2col_index]]; TF_LITE_ENSURE_EQ(context, NumDimensions(output_shape), 1); TF_LITE_ENSURE_EQ(context, NumDimensions(input), 4); @@ -153,15 +81,11 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, SizeOfDimension(input, 3), SizeOfDimension(weights, 3)); - if (IsConstantTensor(output_shape)) { - TF_LITE_ENSURE_STATUS(ResizeOutputTensor(context, output_shape, output)); - TF_LITE_ENSURE_STATUS( - ResizeIm2ColTensor(context, output_shape, weights, input, im2col)); - } else { - // Defer resizing until Eval(). + if (!IsConstantTensor(output_shape)) { SetTensorToDynamic(output); + return kTfLiteOk; } - return kTfLiteOk; + return ResizeOutputShape(context, output_shape, output); } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { @@ -170,19 +94,13 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* weights = GetInput(context, node, kWeightsTensor); const TfLiteTensor* input = GetInput(context, node, kDataInputTensor); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - OpData* user_data = reinterpret_cast(node->user_data); - TfLiteTensor* im2col = - &context->tensors[node->temporaries->data[user_data->im2col_index]]; + const auto* params = reinterpret_cast(node->builtin_data); if (IsDynamicTensor(output)) { TF_LITE_ENSURE_OK(context, - ResizeOutputTensor(context, output_shape, output)); - } - if (IsDynamicTensor(im2col)) { - TF_LITE_ENSURE_OK(context, ResizeIm2ColTensor(context, output_shape, - weights, input, im2col)); + ResizeOutputShape(context, output_shape, output)); } // Get height and width of the output image. @@ -201,12 +119,17 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { // Currently only support float32. switch (input->type) { case kTfLiteFloat32: - optimized_ops::TransposeConv( + reference_ops::TransposeConv( GetTensorData(input), GetTensorDims(input), GetTensorData(weights), GetTensorDims(weights), stride_width, stride_height, padding_size.width, padding_size.height, GetTensorData(output), GetTensorDims(output), - GetTensorData(im2col), GetTensorDims(im2col)); + // Last two args specify im2col which reference_ops ignores. + // (Note this does not lead to a performance regression, as the + // previous optimized version was just a copy of the reference code.) + // TODO(b/110208176): Allocate im2col tensors and switch to + // optimized_ops. + GetTensorData(output), GetTensorDims(output)); break; default: context->ReportError(context, "Type %d, not currently supported.", @@ -219,8 +142,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace transpose_conv TfLiteRegistration* Register_TRANSPOSE_CONV() { - static TfLiteRegistration r = {transpose_conv::Init, transpose_conv::Free, - transpose_conv::Prepare, transpose_conv::Eval}; + static TfLiteRegistration r = {nullptr, nullptr, transpose_conv::Prepare, + transpose_conv::Eval}; return &r; } diff --git a/tensorflow/contrib/lite/kernels/transpose_conv_test.cc b/tensorflow/contrib/lite/kernels/transpose_conv_test.cc index c741df19de..55df897180 100644 --- a/tensorflow/contrib/lite/kernels/transpose_conv_test.cc +++ b/tensorflow/contrib/lite/kernels/transpose_conv_test.cc @@ -14,7 +14,6 @@ limitations under the License. ==============================================================================*/ #include #include -#include "absl/memory/memory.h" #include "tensorflow/contrib/lite/interpreter.h" #include "tensorflow/contrib/lite/kernels/register.h" #include "tensorflow/contrib/lite/kernels/test_util.h" @@ -25,49 +24,9 @@ namespace { using ::testing::ElementsAreArray; -class ConstTransposeConvOpModel : public SingleOpModel { - // Just to be extra confusing, transpose_conv has an _input_ named - // "output_shape". This input sets the shape of the output tensor of the op. - // In this version of the test class, "output_shape" is a constant that must - // be specified in the constructor. - public: - ConstTransposeConvOpModel(TfLiteRegistration* registration, - std::initializer_list input_shape, - std::initializer_list filter_shape, - std::initializer_list output_shape_data, - Padding padding, int stride_w, int stride_h) { - output_shape_ = AddConstInput(TensorType_INT32, output_shape_data, - {static_cast(output_shape_data.size())}); - filter_ = AddInput(TensorType_FLOAT32); - input_ = AddInput(TensorType_FLOAT32); - output_ = AddOutput(TensorType_FLOAT32); - SetBuiltinOp( - BuiltinOperator_TRANSPOSE_CONV, BuiltinOptions_TransposeConvOptions, - CreateTransposeConvOptions(builder_, padding, stride_w, stride_h) - .Union()); - resolver_ = absl::make_unique( - BuiltinOperator_TRANSPOSE_CONV, registration); - BuildInterpreter({{4}, filter_shape, input_shape}); - } - - int output_shape() { return output_shape_; } - int filter() { return filter_; } - int input() { return input_; } - - std::vector GetOutput() { return ExtractVector(output_); } - std::vector GetOutputShape() { return GetTensorShape(output_); } - - private: - int output_shape_; - int filter_; - int input_; - int output_; -}; - class TransposeConvOpModel : public SingleOpModel { public: - TransposeConvOpModel(TfLiteRegistration* registration, - std::initializer_list input_shape, + TransposeConvOpModel(std::initializer_list input_shape, std::initializer_list filter_shape, Padding padding, int stride_w, int stride_h) { output_shape_ = AddInput(TensorType_INT32); @@ -78,8 +37,6 @@ class TransposeConvOpModel : public SingleOpModel { BuiltinOperator_TRANSPOSE_CONV, BuiltinOptions_TransposeConvOptions, CreateTransposeConvOptions(builder_, padding, stride_w, stride_h) .Union()); - resolver_ = absl::make_unique( - BuiltinOperator_TRANSPOSE_CONV, registration); BuildInterpreter({{4}, filter_shape, input_shape}); } @@ -97,15 +54,6 @@ class TransposeConvOpModel : public SingleOpModel { int output_; }; -const auto kKernelMap = new std::map({}); - -class TransposeConvOpTest : public SingleOpTest { - protected: - const std::map& GetKernelMap() override { - return *kKernelMap; - } -}; - // Test case: // output = tf.nn.conv2d_backprop_input( // tf.constant([ 1, 4, 4, 1 ]), @@ -113,9 +61,8 @@ class TransposeConvOpTest : public SingleOpTest { // tf.constant(np.arange(1, 17), shape=[ 1, 4, 4, 1 ], dtype=tf.float32), // [1, 1, 1, 1 ], // "SAME") -TEST_P(TransposeConvOpTest, SimpleTest) { - TransposeConvOpModel m(GetRegistration(), {1, 4, 4, 1}, {1, 3, 3, 1}, - Padding_SAME, 1, 1); +TEST(TransposeConvOpModelTest, SimpleTest) { + TransposeConvOpModel m({1, 4, 4, 1}, {1, 3, 3, 1}, Padding_SAME, 1, 1); m.PopulateTensor(m.output_shape(), {1, 4, 4, 1}); m.PopulateTensor(m.filter(), {1, 2, 3, 4, 5, 6, 7, 8, 9}); m.PopulateTensor( @@ -128,21 +75,6 @@ TEST_P(TransposeConvOpTest, SimpleTest) { EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 4, 1})); } -// Test case: Same as above, but with a const "output_shape" -TEST_P(TransposeConvOpTest, ConstSimpleTest) { - ConstTransposeConvOpModel m(GetRegistration(), {1, 4, 4, 1}, {1, 4, 4, 1}, - {1, 3, 3, 1}, Padding_SAME, 1, 1); - m.PopulateTensor(m.filter(), {1, 2, 3, 4, 5, 6, 7, 8, 9}); - m.PopulateTensor( - m.input(), {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}); - m.Invoke(); - - EXPECT_THAT(m.GetOutput(), - ElementsAreArray({29, 62, 83, 75, 99, 192, 237, 198, 207, 372, - 417, 330, 263, 446, 485, 365})); - EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 4, 4, 1})); -} - // Test case: // filter = tf.constant(np.arange(1, 19), // shape=[ 3, 3, 1, 2 ], @@ -155,9 +87,8 @@ TEST_P(TransposeConvOpTest, ConstSimpleTest) { // "SAME") // And filter value is derived by: // filter = tf.reshape(tf.transpose(filter, perm=[3, 0, 1, 2]), shape=[18, 1]) -TEST_P(TransposeConvOpTest, TwoFiltersTest) { - TransposeConvOpModel m(GetRegistration(), {1, 4, 4, 2}, {1, 3, 3, 2}, - Padding_SAME, 1, 1); +TEST(TransposeConvOpModelTest, TwoFiltersTest) { + TransposeConvOpModel m({1, 4, 4, 2}, {1, 3, 3, 2}, Padding_SAME, 1, 1); m.PopulateTensor(m.output_shape(), {1, 4, 4, 1}); m.PopulateTensor(m.filter(), {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}); @@ -185,9 +116,8 @@ TEST_P(TransposeConvOpTest, TwoFiltersTest) { // "VALID") // And filter value is derived by: // filter = tf.reshape(tf.transpose(filter, perm=[3, 0, 1, 2]), shape=[1, 18]) -TEST_P(TransposeConvOpTest, PaddingValidTest) { - TransposeConvOpModel m(GetRegistration(), {1, 4, 4, 2}, {1, 3, 3, 2}, - Padding_VALID, 1, 1); +TEST(TransposeConvOpModelTest, PaddingValidTest) { + TransposeConvOpModel m({1, 4, 4, 2}, {1, 3, 3, 2}, Padding_VALID, 1, 1); m.PopulateTensor(m.output_shape(), {1, 6, 6, 1}); m.PopulateTensor(m.filter(), {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}); @@ -216,9 +146,8 @@ TEST_P(TransposeConvOpTest, PaddingValidTest) { // tf.constant(np.arange(1, 5), shape=[ 1, 2, 2, 1 ], dtype=tf.float32), // [1, 2, 2, 1 ], // "VALID") -TEST_P(TransposeConvOpTest, StrideValidTest) { - TransposeConvOpModel m(GetRegistration(), {1, 2, 2, 1}, {1, 3, 3, 1}, - Padding_VALID, 2, 2); +TEST(TransposeConvOpModelTest, StrideValidTest) { + TransposeConvOpModel m({1, 2, 2, 1}, {1, 3, 3, 1}, Padding_VALID, 2, 2); m.PopulateTensor(m.output_shape(), {1, 5, 5, 1}); m.PopulateTensor(m.filter(), {1, 2, 3, 4, 5, 6, 7, 8, 9}); m.PopulateTensor(m.input(), {1, 2, 3, 4}); @@ -241,9 +170,8 @@ TEST_P(TransposeConvOpTest, StrideValidTest) { // tf.constant(np.arange(1, 5), shape=[ 1, 2, 2, 1 ], dtype=tf.float32), // [1, 2, 2, 1 ], // "VALID") -TEST_P(TransposeConvOpTest, MultiChannelTest) { - TransposeConvOpModel m(GetRegistration(), {1, 2, 2, 1}, {2, 3, 3, 1}, - Padding_VALID, 2, 2); +TEST(TransposeConvOpModelTest, MultiChannelTest) { + TransposeConvOpModel m({1, 2, 2, 1}, {2, 3, 3, 1}, Padding_VALID, 2, 2); m.PopulateTensor(m.output_shape(), {1, 5, 5, 2}); m.PopulateTensor(m.filter(), {1, 3, 5, 7, 9, 11, 13, 15, 17, 2, 4, 6, 8, 10, 12, 14, 16, 18}); @@ -259,24 +187,6 @@ TEST_P(TransposeConvOpTest, MultiChannelTest) { EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 5, 5, 2})); } -// Test case: Same as above, but with a const "output_shape" -TEST_P(TransposeConvOpTest, ConstMultiChannelTest) { - ConstTransposeConvOpModel m(GetRegistration(), {1, 2, 2, 1}, {2, 3, 3, 1}, - {1, 5, 5, 2}, Padding_VALID, 2, 2); - m.PopulateTensor(m.filter(), {1, 3, 5, 7, 9, 11, 13, 15, 17, 2, 4, 6, - 8, 10, 12, 14, 16, 18}); - m.PopulateTensor(m.input(), {1, 2, 3, 4}); - m.Invoke(); - - EXPECT_THAT( - m.GetOutput(), - ElementsAreArray({1, 2, 3, 4, 7, 10, 6, 8, 10, 12, 7, 8, 9, - 10, 25, 28, 18, 20, 22, 24, 16, 20, 24, 28, 62, 72, - 42, 48, 54, 60, 21, 24, 27, 30, 61, 68, 36, 40, 44, - 48, 39, 42, 45, 48, 103, 110, 60, 64, 68, 72})); - EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 5, 5, 2})); -} - // Test case: // filter = tf.constant(np.random.randint(1, 10, size=9), // shape=[ 3, 3, 1, 1 ], @@ -289,9 +199,8 @@ TEST_P(TransposeConvOpTest, ConstMultiChannelTest) { // "SAME") // And filter value is derived by: // filter = tf.reshape(tf.transpose(filter, perm=[3, 0, 1, 2]), shape=[-1]) -TEST_P(TransposeConvOpTest, AccuracyTest) { - TransposeConvOpModel m(GetRegistration(), {1, 1, 2, 1}, {1, 3, 3, 1}, - Padding_SAME, 3, 3); +TEST(TransposeConvOpModelTest, AccuracyTest) { + TransposeConvOpModel m({1, 1, 2, 1}, {1, 3, 3, 1}, Padding_SAME, 3, 3); m.PopulateTensor(m.output_shape(), {1, 3, 4, 1}); m.PopulateTensor(m.filter(), {9, 5, 6, 9, 8, 5, 3, 1, 4}); m.PopulateTensor(m.input(), {323, 521}); @@ -303,10 +212,6 @@ TEST_P(TransposeConvOpTest, AccuracyTest) { EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 3, 4, 1})); } -INSTANTIATE_TEST_CASE_P( - TransposeConvOpTest, TransposeConvOpTest, - ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap))); - } // namespace } // namespace tflite -- cgit v1.2.3 From 81752cd3ef069eccdf20c36b5404ea600caf7b63 Mon Sep 17 00:00:00 2001 From: Thor Johnsen Date: Thu, 5 Jul 2018 13:29:04 -0500 Subject: merged crop_and_resize with resize_bilinear_op internals --- tensorflow/core/kernels/BUILD | 10 +- tensorflow/core/kernels/crop_and_resize_op.cc | 55 ++- .../core/kernels/crop_resize_bilinear_core.h | 436 +++++++++++++++++++++ tensorflow/core/kernels/resize_bilinear_op.cc | 151 +------ 4 files changed, 481 insertions(+), 171 deletions(-) create mode 100644 tensorflow/core/kernels/crop_resize_bilinear_core.h diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 7599cf7db2..b9f51f49ce 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -557,6 +557,12 @@ cc_header_only_library( deps = [":image_resizer_state"], ) +cc_library( + name = "crop_resize_bilinear_core", + hdrs = ["crop_resize_bilinear_core.h"], + visibility = ["//visibility:private"], +) + # OpKernel libraries ---------------------------------------------------------- ARRAY_DEPS = [ @@ -2129,7 +2135,7 @@ tf_kernel_library( tf_kernel_library( name = "crop_and_resize_op", prefix = "crop_and_resize_op", - deps = IMAGE_DEPS, + deps = IMAGE_DEPS + [":crop_resize_bilinear_core"], ) tf_kernel_library( @@ -2195,7 +2201,7 @@ tf_kernel_library( tf_kernel_library( name = "resize_bilinear_op", prefix = "resize_bilinear_op", - deps = IMAGE_DEPS, + deps = IMAGE_DEPS + [":crop_resize_bilinear_core"], ) tf_kernel_library( diff --git a/tensorflow/core/kernels/crop_and_resize_op.cc b/tensorflow/core/kernels/crop_and_resize_op.cc index 99d01b4db6..de8274db89 100644 --- a/tensorflow/core/kernels/crop_and_resize_op.cc +++ b/tensorflow/core/kernels/crop_and_resize_op.cc @@ -33,6 +33,7 @@ limitations under the License. #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/work_sharder.h" +#include "tensorflow/core/kernels/crop_resize_bilinear_core.h" #if GOOGLE_CUDA #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h" @@ -249,39 +250,27 @@ struct CropAndResize { continue; } if (method_name == "bilinear") { - const int top_y_index = floorf(in_y); - const int bottom_y_index = ceilf(in_y); - const float y_lerp = in_y - top_y_index; - - for (int x = 0; x < crop_width; ++x) { - const float in_x = (crop_width > 1) - ? x1 * (image_width - 1) + x * width_scale - : 0.5 * (x1 + x2) * (image_width - 1); - if (in_x < 0 || in_x > image_width - 1) { - for (int d = 0; d < depth; ++d) { - crops(b, y, x, d) = extrapolation_value; - } - continue; - } - const int left_x_index = floorf(in_x); - const int right_x_index = ceilf(in_x); - const float x_lerp = in_x - left_x_index; - - for (int d = 0; d < depth; ++d) { - const float top_left(static_cast( - image(b_in, top_y_index, left_x_index, d))); - const float top_right(static_cast( - image(b_in, top_y_index, right_x_index, d))); - const float bottom_left(static_cast( - image(b_in, bottom_y_index, left_x_index, d))); - const float bottom_right(static_cast( - image(b_in, bottom_y_index, right_x_index, d))); - const float top = top_left + (top_right - top_left) * x_lerp; - const float bottom = - bottom_left + (bottom_right - bottom_left) * x_lerp; - crops(b, y, x, d) = top + (bottom - top) * y_lerp; - } - } + CachedInterpolation *interp_x=0l, *interp_y=0l; + int min_ix, max_ix, min_iy, max_iy; + compute_interpolation_weights(crop_width,image_width,x1,x2,min_ix,max_ix,interp_x); + compute_interpolation_weights(crop_height,image_height,y1,y2,min_iy,max_iy,interp_y); + + // multiply by depth to avoid multiplication in resize_single_image. + for (int i = min_ix; i <= max_ix; ++i) { + interp_x[i-min_ix].lower *= depth; + interp_x[i-min_ix].upper *= depth; + } + + crop_resize_single_image( + image.data() + (int64)b_in * (int64)image_height * (int64)image_width * (int64)depth, + image_height,image_width,crop_height,crop_width,depth, + min_ix,max_ix,interp_x, + min_iy,max_iy,interp_y, + extrapolation_value,false,false, + crops.data() + (int64)b * (int64)crop_height * (int64)crop_width * (int64)depth); + + delete [] interp_y; + delete [] interp_x; } else { // method == "nearest" for (int x = 0; x < crop_width; ++x) { const float in_x = (crop_width > 1) diff --git a/tensorflow/core/kernels/crop_resize_bilinear_core.h b/tensorflow/core/kernels/crop_resize_bilinear_core.h new file mode 100644 index 0000000000..5f707c6296 --- /dev/null +++ b/tensorflow/core/kernels/crop_resize_bilinear_core.h @@ -0,0 +1,436 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_KERNELS_CROP_RESIZE_BILINEAR_CORE_H_ +#define TENSORFLOW_CORE_KERNELS_CROP_RESIZE_BILINEAR_CORE_H_ + +namespace tensorflow { +namespace { +// Compute the interpolation indices only once. +struct CachedInterpolation { + int lower; // Lower source index used in the interpolation + int upper; // Upper source index used in the interpolation + // 1-D linear iterpolation scale (see: + // https://en.wikipedia.org/wiki/Bilinear_interpolation) + float lerp; +}; + +inline bool compute_single_interpolation_weight( + const int in_size, + const float out2in_scale, + const float out2in_start, + const bool clip, + const int i, + int& lower, + int& upper, + float& lerp) { + const float in = i * out2in_scale + out2in_start; + lower = (int)floor(in); + upper = (int)ceil(in); + lerp = (float)(in - (float)lower); + if (clip) { + if (lower < 0) lower = 0; + else if (lower >= in_size) lower = in_size - 1; + if (upper < 0) upper = 0; + else if (upper >= in_size) upper = in_size - 1; + return true; + } else { + return (lower >= 0 && upper < in_size) ? true : false; + } +} +/** + * Compute interpolation values for output indexes in range [out_start,out_start+out_size-1]. + * Returns true if all output indexes have lower and upper (input) indexes within range [0,in_size-1]. + */ +inline bool compute_interpolation_weights( + const int min_i, + const int max_i, + const int in_size, + const float out2in_scale, + const float out2in_start, + const bool clip, + CachedInterpolation* interpolation) { + bool rval = true; + int num_i = max_i - min_i + 1; + for (int i = 0; i < num_i; ++i) { + if (!compute_single_interpolation_weight( + in_size,out2in_scale,out2in_start,clip, + i+min_i, + interpolation[i].lower,interpolation[i].upper,interpolation[i].lerp)) { + rval = false; + } + } + return rval; +} +/** + * Compatibility method for resize_bilinear_op.cc + */ +inline void compute_interpolation_weights( + const int out_size, + const int in_size, + const float out2in_scale, + CachedInterpolation* interpolation) { + interpolation[out_size].lower = 0; + interpolation[out_size].upper = 0; + const bool clip = true; + if (!compute_interpolation_weights(0,out_size-1,in_size,out2in_scale,0.0f,clip,interpolation)) { + // Should never happen, check for it anyway + printf("Warning! Interpolation values have lower,upper indexes outside of range [0,in_size-1]\n"); + } +} +/** + * Compute minimum and maximum (output) i where both lower and upper (input) is in range [0,in_size-1] + * If no values of i satisfy condition, min_i = in_size, max_i = -1 and method returns false. + * Returns true if min_i >= max_i. + */ +inline bool compute_minmax_indexes( + const int out_size, + const int in_size, + const float out2in_scale, + const float out2in_start, + int& min_i, + int& max_i) { + min_i = out_size; + max_i = -1; + int lower, upper; + float lerp; + for (int i = 0; i < out_size; ++i) { + if (compute_single_interpolation_weight(in_size,out2in_scale,out2in_start,false,i,lower,upper,lerp)) { + if (i < min_i) min_i = i; + if (i > max_i) max_i = i; + } + } + return (min_i <= max_i) ? true : false; +} +/** + * Compute interpolation weights for crop_and_resize_op.cc + * Also computes extrapolation areas. + * Returns true if at least one point requires interpolation, false otherwise. + */ +inline bool compute_interpolation_weights( + const int out_size, + const int in_size, + const float x1, // lower bounding box, crop region starts at in_size*x1 + const float x2, // upper bounding box, crop region ends at in_size*x2 + int& min_i, + int& max_i, + CachedInterpolation*& interpolation) { + float out2in_start = out_size > 1 ? (float)(in_size-1) * (float)x1 : (float)(in_size-1) * (float)(x1 + x2) / 2.0f; + float out2in_scale = out_size > 1 ? (float)(x2-x1) * (float)(in_size-1) / (float)(out_size-1) : 0.0f; + if (compute_minmax_indexes(out_size,in_size,out2in_scale,out2in_start,min_i,max_i)) { + interpolation = new CachedInterpolation[max_i-min_i+1]; + bool all_inputs_ok = compute_interpolation_weights(min_i,max_i,in_size,out2in_scale,out2in_start,false,interpolation); + if (!all_inputs_ok) { + // should never happen, purpose of compute_minmax_indexes is to ensure that all inputs are ok. + printf("Error! compute_interpolation_weights returned input indexes outside valid range - SEGV will likely ensue.\n"); + } + return true; + } else { + interpolation = 0l; + return false; + } +} + +template U cast_to(float v, float min_val, float max_val, U u_min_val, U u_max_val); +template +inline U cast_to(float v, float min_val, float max_val, U u_min_val, U u_max_val) { + if (v < min_val) + return u_min_val; + else if (v > max_val) + return u_max_val; + else + return static_cast(v); +} +template<> +inline float cast_to(float v, float min_val, float max_val, float u_min_val, float u_max_val) { + return v; +} + +inline float compute_lerp(const float top_left, const float top_right, + const float bottom_left, const float bottom_right, + const float x_lerp, const float y_lerp) { + const float top = top_left + (top_right - top_left) * x_lerp; + const float bottom = bottom_left + (bottom_right - bottom_left) * x_lerp; + return top + (bottom - top) * y_lerp; +} + +/** + * Computes the bilinear interpolation from the appropriate 4 float points + * and the linear interpolation weights. + * Accepts input tensors of type T and produces output tensors of type U. + * Optionally flips horizontal and/or vertical axis. + */ +template +void crop_resize_single_image( + const T* image, + const int64 in_height, const int64 in_width, + const int64 out_height, const int64 out_width, + const int channels, + const int min_ix, const int max_ix, + const CachedInterpolation* xs, + const int min_iy, const int max_iy, + const CachedInterpolation* ys, + const float extrapolated_value, + const bool flip_x, + const bool flip_y, + U* output) TF_ATTRIBUTE_NOINLINE; +template +void crop_resize_single_image( + const T* image, + const int64 in_height, const int64 in_width, + const int64 out_height, const int64 out_width, + const int channels, + const int min_ix, const int max_ix, + const CachedInterpolation* xs, + const int min_iy, const int max_iy, + const CachedInterpolation* ys, + const float extrapolated_value, + const bool flip_x, + const bool flip_y, + U* output) { + const int64 in_row_size = in_width * channels; + const int64 out_row_size = out_width * channels; + U u_min_val = std::numeric_limits::min(); + U u_max_val = std::numeric_limits::max(); + float min_val = static_cast(u_min_val); + float max_val = static_cast(u_max_val); + U uEx = cast_to(extrapolated_value,min_val,max_val,u_min_val,u_max_val); + // low y extrapolation zone + if (min_iy > 0) { + U* p = flip_y ? output + out_row_size * (out_height - min_iy) : output; + int64 nn = out_row_size * (int64)min_iy; + for (int64 i = 0; i < nn; ++i) p[i] = uEx; + } + // high y extrapolation zone + if (max_iy < out_height-1) { + U* p = flip_y ? output : output + out_row_size * (max_iy + 1); + int64 nn = out_row_size * (int64)(out_height - 1 - max_iy); + for (int64 i = 0; i < nn; ++i) p[i] = uEx; + } + // low x extrapolation zone + if (min_ix > 0) { + for (int iy = min_iy; iy <= max_iy; ++iy) { + int xx0 = flip_x ? (out_width-min_ix)*channels : 0; + int nxx = min_ix*channels; + U* p = output + xx0 + out_row_size * (int64)(flip_y ? out_height - 1 - iy : iy); + for (int ix = 0; ix < nxx; ++ix) { + p[ix] = uEx; + } + } + } + // high x extrapolation zone + if (max_ix < out_width-1) { + for (int iy = min_iy; iy <= max_iy; ++iy) { + int xx0 = flip_x ? 0 : (max_ix+1)*channels; + int nxx = (out_width-1-max_ix)*channels; + U* p = output + xx0 + out_row_size * (int64)(flip_y ? out_height - 1 - iy : iy); + for (int ix = 0; ix < nxx; ++ix) { + p[ix] = uEx; + } + } + } + U* output_y_ptr = output + out_row_size * (int64)(flip_y ? out_height - 1 - min_iy : min_iy); + // interpolation zone + if (channels == 1) { + for (int y = min_iy; y <= max_iy; ++y) { + const int iy = y - min_iy; + const T* ys_input_lower_ptr = image + ys[iy].lower * in_row_size; + const T* ys_input_upper_ptr = image + ys[iy].upper * in_row_size; + const float ys_lerp = ys[iy].lerp; + const int x0 = flip_x ? out_width - 1 - max_ix : min_ix; + const int x1 = flip_x ? out_width - 1 - min_ix : max_ix; + for (int x = x0; x <= x1; ++x) { + const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix; + const int64 xs_lower = xs[ix].lower; + const int64 xs_upper = xs[ix].upper; + const float xs_lerp = xs[ix].lerp; + + // Read channel 0. + const float top_left0(ys_input_lower_ptr[xs_lower]); + const float top_right0(ys_input_lower_ptr[xs_upper]); + const float bottom_left0(ys_input_upper_ptr[xs_lower]); + const float bottom_right0(ys_input_upper_ptr[xs_upper]); + + // Compute output. + float result0 = compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0,xs_lerp, ys_lerp); + output_y_ptr[x] = cast_to(result0,min_val,max_val,u_min_val,u_max_val); + } + output_y_ptr = flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; + } + } else if (channels == 2) { + for (int y = min_iy; y <= max_iy; ++y) { + const int iy = y - min_iy; + const T* ys_input_lower_ptr = image + ys[iy].lower * in_row_size; + const T* ys_input_upper_ptr = image + ys[iy].upper * in_row_size; + const float ys_lerp = ys[iy].lerp; + const int x0 = flip_x ? out_width - 1 - max_ix : min_ix; + const int x1 = flip_x ? out_width - 1 - min_ix : max_ix; + for (int x = x0; x <= x1; ++x) { + const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix; + const int64 xs_lower = xs[ix].lower; + const int64 xs_upper = xs[ix].upper; + const float xs_lerp = xs[ix].lerp; + + // Read channel 0. + const float top_left0(ys_input_lower_ptr[xs_lower + 0]); + const float top_right0(ys_input_lower_ptr[xs_upper + 0]); + const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]); + const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]); + + // Read channel 1. + const float top_left1(ys_input_lower_ptr[xs_lower + 1]); + const float top_right1(ys_input_lower_ptr[xs_upper + 1]); + const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]); + const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]); + + // Compute output. + float result0 = compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0, + xs_lerp, ys_lerp); + float result1 = compute_lerp(top_left1, top_right1, bottom_left1, bottom_right1, + xs_lerp, ys_lerp); + output_y_ptr[x*2 + 0] = cast_to(result0,min_val,max_val,u_min_val,u_max_val); + output_y_ptr[x*2 + 1] = cast_to(result1,min_val,max_val,u_min_val,u_max_val); + } + output_y_ptr = flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; + } + } else if (channels == 3) { + for (int y = min_iy; y <= max_iy; ++y) { + const int iy = y - min_iy; + const T* ys_input_lower_ptr = image + ys[iy].lower * in_row_size; + const T* ys_input_upper_ptr = image + ys[iy].upper * in_row_size; + const float ys_lerp = ys[iy].lerp; + const int x0 = flip_x ? out_width - 1 - max_ix : min_ix; + const int x1 = flip_x ? out_width - 1 - min_ix : max_ix; + for (int x = x0; x <= x1; ++x) { + const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix; + const int64 xs_lower = xs[ix].lower; + const int64 xs_upper = xs[ix].upper; + const float xs_lerp = xs[ix].lerp; + + // Read channel 0. + const float top_left0(ys_input_lower_ptr[xs_lower + 0]); + const float top_right0(ys_input_lower_ptr[xs_upper + 0]); + const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]); + const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]); + + // Read channel 1. + const float top_left1(ys_input_lower_ptr[xs_lower + 1]); + const float top_right1(ys_input_lower_ptr[xs_upper + 1]); + const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]); + const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]); + + // Read channel 2. + const float top_left2(ys_input_lower_ptr[xs_lower + 2]); + const float top_right2(ys_input_lower_ptr[xs_upper + 2]); + const float bottom_left2(ys_input_upper_ptr[xs_lower + 2]); + const float bottom_right2(ys_input_upper_ptr[xs_upper + 2]); + + // Compute output. + float result0 = compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0, + xs_lerp, ys_lerp); + float result1 = compute_lerp(top_left1, top_right1, bottom_left1, bottom_right1, + xs_lerp, ys_lerp); + float result2 = compute_lerp(top_left2, top_right2, bottom_left2, bottom_right2, + xs_lerp, ys_lerp); + output_y_ptr[x*3 + 0] = cast_to(result0,min_val,max_val,u_min_val,u_max_val); + output_y_ptr[x*3 + 1] = cast_to(result1,min_val,max_val,u_min_val,u_max_val); + output_y_ptr[x*3 + 2] = cast_to(result2,min_val,max_val,u_min_val,u_max_val); + } + output_y_ptr = flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; + } + } else if (channels == 4) { + for (int y = min_iy; y <= max_iy; ++y) { + const int iy = y - min_iy; + const T* ys_input_lower_ptr = image + ys[iy].lower * in_row_size; + const T* ys_input_upper_ptr = image + ys[iy].upper * in_row_size; + const float ys_lerp = ys[iy].lerp; + const int x0 = flip_x ? out_width - 1 - max_ix : min_ix; + const int x1 = flip_x ? out_width - 1 - min_ix : max_ix; + for (int x = x0; x <= x1; ++x) { + const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix; + const int64 xs_lower = xs[ix].lower; + const int64 xs_upper = xs[ix].upper; + const float xs_lerp = xs[ix].lerp; + + // Read channel 0. + const float top_left0(ys_input_lower_ptr[xs_lower + 0]); + const float top_right0(ys_input_lower_ptr[xs_upper + 0]); + const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]); + const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]); + + // Read channel 1. + const float top_left1(ys_input_lower_ptr[xs_lower + 1]); + const float top_right1(ys_input_lower_ptr[xs_upper + 1]); + const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]); + const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]); + + // Read channel 2. + const float top_left2(ys_input_lower_ptr[xs_lower + 2]); + const float top_right2(ys_input_lower_ptr[xs_upper + 2]); + const float bottom_left2(ys_input_upper_ptr[xs_lower + 2]); + const float bottom_right2(ys_input_upper_ptr[xs_upper + 2]); + + // Read channel 3. + const float top_left3(ys_input_lower_ptr[xs_lower + 3]); + const float top_right3(ys_input_lower_ptr[xs_upper + 3]); + const float bottom_left3(ys_input_upper_ptr[xs_lower + 3]); + const float bottom_right3(ys_input_upper_ptr[xs_upper + 3]); + + // Compute output. + float result0 = compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0, + xs_lerp, ys_lerp); + float result1 = compute_lerp(top_left1, top_right1, bottom_left1, bottom_right1, + xs_lerp, ys_lerp); + float result2 = compute_lerp(top_left2, top_right2, bottom_left2, bottom_right2, + xs_lerp, ys_lerp); + float result3 = compute_lerp(top_left3, top_right3, bottom_left3, bottom_right3, + xs_lerp, ys_lerp); + output_y_ptr[x*4 + 0] = cast_to(result0,min_val,max_val,u_min_val,u_max_val); + output_y_ptr[x*4 + 1] = cast_to(result1,min_val,max_val,u_min_val,u_max_val); + output_y_ptr[x*4 + 2] = cast_to(result2,min_val,max_val,u_min_val,u_max_val); + output_y_ptr[x*4 + 3] = cast_to(result3,min_val,max_val,u_min_val,u_max_val); + } + output_y_ptr = flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; + } + } else { + for (int y = min_iy; y <= max_iy; ++y) { + const int iy = y - min_iy; + const T* ys_input_lower_ptr = image + ys[iy].lower * in_row_size; + const T* ys_input_upper_ptr = image + ys[iy].upper * in_row_size; + const float ys_lerp = ys[iy].lerp; + const int x0 = flip_x ? out_width - 1 - max_ix : min_ix; + const int x1 = flip_x ? out_width - 1 - min_ix : max_ix; + for (int x = x0; x <= x1; ++x) { + const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix; + const int64 xs_lower = xs[ix].lower; + const int64 xs_upper = xs[ix].upper; + const float xs_lerp = xs[ix].lerp; + for (int ichan = 0; ichan < channels; ++ichan) { + const float top_left0(ys_input_lower_ptr[xs_lower + ichan]); + const float top_right0(ys_input_lower_ptr[xs_upper + ichan]); + const float bottom_left0(ys_input_upper_ptr[xs_lower + ichan]); + const float bottom_right0(ys_input_upper_ptr[xs_upper + ichan]); + float result0 = compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0, + xs_lerp, ys_lerp); + output_y_ptr[x*channels + ichan] = cast_to(result0,min_val,max_val,u_min_val,u_max_val); + } + } + output_y_ptr = flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; + } + } +} +} // namespace +} // namespace tensorflow +#endif diff --git a/tensorflow/core/kernels/resize_bilinear_op.cc b/tensorflow/core/kernels/resize_bilinear_op.cc index dde59e8e74..3f1589dcee 100644 --- a/tensorflow/core/kernels/resize_bilinear_op.cc +++ b/tensorflow/core/kernels/resize_bilinear_op.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/kernels/image_resizer_state.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/kernels/crop_resize_bilinear_core.h" namespace tensorflow { @@ -63,140 +64,6 @@ class ResizeBilinearOp : public OpKernel { bool align_corners_; }; -namespace { -// Compute the interpolation indices only once. -struct CachedInterpolation { - int64 lower; // Lower source index used in the interpolation - int64 upper; // Upper source index used in the interpolation - // 1-D linear iterpolation scale (see: - // https://en.wikipedia.org/wiki/Bilinear_interpolation) - float lerp; -}; - -inline void compute_interpolation_weights(const int64 out_size, - const int64 in_size, - const float scale, - CachedInterpolation* interpolation) { - interpolation[out_size].lower = 0; - interpolation[out_size].upper = 0; - for (int64 i = out_size - 1; i >= 0; --i) { - const float in = i * scale; - interpolation[i].lower = static_cast(in); - interpolation[i].upper = std::min(interpolation[i].lower + 1, in_size - 1); - interpolation[i].lerp = in - interpolation[i].lower; - } -} - -/** - * Computes the bilinear interpolation from the appropriate 4 float points - * and the linear interpolation weights. - */ -inline float compute_lerp(const float top_left, const float top_right, - const float bottom_left, const float bottom_right, - const float x_lerp, const float y_lerp) { - const float top = top_left + (top_right - top_left) * x_lerp; - const float bottom = bottom_left + (bottom_right - bottom_left) * x_lerp; - return top + (bottom - top) * y_lerp; -} - -template -void resize_image( - typename TTypes::ConstTensor images, const int batch_size, - const int64 in_height, const int64 in_width, const int64 out_height, - const int64 out_width, const int channels, - const std::vector& xs, - const std::vector& ys, - typename TTypes::Tensor output) TF_ATTRIBUTE_NOINLINE; -template -void resize_image(typename TTypes::ConstTensor images, - const int batch_size, const int64 in_height, - const int64 in_width, const int64 out_height, - const int64 out_width, const int channels, - const std::vector& xs_vec, - const std::vector& ys, - typename TTypes::Tensor output) { - const int64 in_row_size = in_width * channels; - const int64 in_batch_num_values = in_height * in_row_size; - const int64 out_row_size = out_width * channels; - - const T* input_b_ptr = images.data(); - const CachedInterpolation* xs = xs_vec.data(); - - if (channels == 3) { - float* output_y_ptr = output.data(); - for (int b = 0; b < batch_size; ++b) { - for (int64 y = 0; y < out_height; ++y) { - const T* ys_input_lower_ptr = input_b_ptr + ys[y].lower * in_row_size; - const T* ys_input_upper_ptr = input_b_ptr + ys[y].upper * in_row_size; - const float ys_lerp = ys[y].lerp; - for (int64 x = 0; x < out_width; ++x) { - const int64 xs_lower = xs[x].lower; - const int64 xs_upper = xs[x].upper; - const float xs_lerp = xs[x].lerp; - - // Read channel 0. - const float top_left0(ys_input_lower_ptr[xs_lower + 0]); - const float top_right0(ys_input_lower_ptr[xs_upper + 0]); - const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]); - const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]); - - // Read channel 1. - const float top_left1(ys_input_lower_ptr[xs_lower + 1]); - const float top_right1(ys_input_lower_ptr[xs_upper + 1]); - const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]); - const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]); - - // Read channel 2. - const float top_left2(ys_input_lower_ptr[xs_lower + 2]); - const float top_right2(ys_input_lower_ptr[xs_upper + 2]); - const float bottom_left2(ys_input_upper_ptr[xs_lower + 2]); - const float bottom_right2(ys_input_upper_ptr[xs_upper + 2]); - - // Compute output. - output_y_ptr[x * channels + 0] = - compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0, - xs_lerp, ys_lerp); - output_y_ptr[x * channels + 1] = - compute_lerp(top_left1, top_right1, bottom_left1, bottom_right1, - xs_lerp, ys_lerp); - output_y_ptr[x * channels + 2] = - compute_lerp(top_left2, top_right2, bottom_left2, bottom_right2, - xs_lerp, ys_lerp); - } - output_y_ptr += out_row_size; - } - input_b_ptr += in_batch_num_values; - } - } else { - float* output_y_ptr = output.data(); - for (int b = 0; b < batch_size; ++b) { - for (int64 y = 0; y < out_height; ++y) { - const T* ys_input_lower_ptr = input_b_ptr + ys[y].lower * in_row_size; - const T* ys_input_upper_ptr = input_b_ptr + ys[y].upper * in_row_size; - const float ys_lerp = ys[y].lerp; - for (int64 x = 0; x < out_width; ++x) { - auto xs_lower = xs[x].lower; - auto xs_upper = xs[x].upper; - auto xs_lerp = xs[x].lerp; - for (int c = 0; c < channels; ++c) { - const float top_left(ys_input_lower_ptr[xs_lower + c]); - const float top_right(ys_input_lower_ptr[xs_upper + c]); - const float bottom_left(ys_input_upper_ptr[xs_lower + c]); - const float bottom_right(ys_input_upper_ptr[xs_upper + c]); - output_y_ptr[x * channels + c] = - compute_lerp(top_left, top_right, bottom_left, bottom_right, - xs_lerp, ys_lerp); - } - } - output_y_ptr += out_row_size; - } - input_b_ptr += in_batch_num_values; - } - } -} - -} // namespace - // Partial specialization of ResizeBilinear functor for a CPUDevice. namespace functor { template @@ -212,6 +79,11 @@ struct ResizeBilinear { const int64 out_height = output.dimension(1); const int64 out_width = output.dimension(2); + const int64 in_row_size = in_width * channels; + const int64 in_batch_num_values = in_height * in_row_size; + const int64 out_row_size = out_width * channels; + const int64 out_batch_num_values = out_row_size * out_height; + // Handle no-op resizes efficiently. if (out_height == in_height && out_width == in_width) { output = images.template cast(); @@ -232,8 +104,15 @@ struct ResizeBilinear { xs[i].upper *= channels; } - resize_image(images, batch_size, in_height, in_width, out_height, - out_width, channels, xs, ys, output); + for (int b = 0; b < batch_size; ++b) { + crop_resize_single_image( + images.data() + (int64)b * in_batch_num_values, + in_height,in_width,out_height,out_width,channels, + 0,out_width-1,xs.data(), + 0,out_height-1,ys.data(), + 0.0f,false,false, + output.data() + (int64)b * out_batch_num_values); + } } }; } // namespace functor -- cgit v1.2.3 From 61fbae09afbdc225e2a603ee9045f6f298e802c3 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Thu, 12 Jul 2018 10:00:35 -0700 Subject: Several improvements required by downstream analyses, detailed below. Add additional structures to the CFG which track links for higher level statements (which usually aggregate multiple CFG nodes). These allow drawing edges between blocks like if statements and other CFG nodes. Add support for closures. This is dont by including function definition nodes into the graph. CFGs are built on a per-function basis and are generally independent (we ignore flow dur to function calls). These nodes allow connecting these independent CFG at dataflow analysis (e.g. the variables that a function closes over are live into the definition of the function). Additional minor updates: * changed the string representation to better comply with the DOT format * force visitor subclasses to define a state initializer * slightly modify the visitor interface to allow the user to manipulate state before the actual visit PiperOrigin-RevId: 204316850 --- tensorflow/contrib/autograph/pyct/cfg.py | 133 ++++++++++++---- tensorflow/contrib/autograph/pyct/cfg_test.py | 213 ++++++++++++++++++++++++-- 2 files changed, 302 insertions(+), 44 deletions(-) diff --git a/tensorflow/contrib/autograph/pyct/cfg.py b/tensorflow/contrib/autograph/pyct/cfg.py index 666328781f..8ef234745c 100644 --- a/tensorflow/contrib/autograph/pyct/cfg.py +++ b/tensorflow/contrib/autograph/pyct/cfg.py @@ -64,11 +64,17 @@ class Node(object): self.prev = frozenset(self.prev) def __repr__(self): + if isinstance(self.ast_node, gast.FunctionDef): + return 'def %s' % self.ast_node.name + elif isinstance(self.ast_node, gast.withitem): + return compiler.ast_to_source(self.ast_node.context_expr).strip() return compiler.ast_to_source(self.ast_node).strip() class Graph( - collections.namedtuple('Graph', ['entry', 'exit', 'error', 'index'])): + collections.namedtuple( + 'Graph', + ['entry', 'exit', 'error', 'index', 'stmt_prev', 'stmt_next'])): """A Control Flow Graph. The CFG maintains an index to allow looking up a CFG node by the AST node to @@ -82,6 +88,11 @@ class Graph( because these are shared, and wiring them would create a reverse path from normal control flow into the error nodes, which we want to avoid. + The graph also maintains edges corresponding to higher level statements + like for-else loops. A node is considered successor of a statement if there + is an edge from a node that is lexically a child of that statement to a node + that is not. Statement predecessors are analogously defined. + Attributes: entry: Node, the entry node exit: FrozenSet[Node, ...], the exit nodes @@ -89,6 +100,10 @@ class Graph( error (errors propagated from function calls are not accounted) index: Dict[ast.Node, Node], mapping AST nodes to the respective CFG node + stmt_prev: Dict[ast.Node, FrozenSet[Node, ...]], mapping statement AST + nodes to their predecessor CFG nodes + stmt_next: Dict[ast.Node, FrozenSet[Node, ...]], mapping statement AST + nodes to their successor CFG nodes """ def __repr__(self): @@ -96,9 +111,8 @@ class Graph( for node in self.index.values(): result += ' %s [label="%s"];\n' % (id(node), node) for node in self.index.values(): - if node.next: - result += ' %s -> {%s};\n' % (id(node), ', '.join( - repr(id(n)) for n in node.next)) + for next_ in node.next: + result += ' %s -> %s;\n' % (id(node), id(next_)) result += '}' return result @@ -130,25 +144,20 @@ class GraphVisitor(object): out: Dict[Node, Any], stores node-keyed state during a visit """ - def reset(self): - self.in_ = { - node: self.init_state(node) for node in self.graph.index.values() - } - self.out = { - node: self.init_state(node) for node in self.graph.index.values() - } + def __init__(self, graph): + self.graph = graph + self.reset() def init_state(self, node): """State initialization function. Optional to overload. An in/out state slot will be created for each node in the graph. Subclasses - may overload this to control what that is initialized to. + must overload this to control what that is initialized to. Args: node: Node """ - del node - return None + raise NotImplementedError('Subclasses must implement this.') def visit_node(self, node): """Visitor function. @@ -161,6 +170,14 @@ class GraphVisitor(object): """ raise NotImplementedError('Subclasses must implement this.') + def reset(self): + self.in_ = { + node: self.init_state(node) for node in self.graph.index.values() + } + self.out = { + node: self.init_state(node) for node in self.graph.index.values() + } + def _visit_internal(self, mode): """Visits the CFG, depth-first.""" assert mode in (_WalkMode.FORWARD, _WalkMode.REVERSE) @@ -169,7 +186,6 @@ class GraphVisitor(object): elif mode == _WalkMode.REVERSE: open_ = list(self.graph.exit) closed = set() - self.reset() while open_: node = open_.pop(0) @@ -186,12 +202,10 @@ class GraphVisitor(object): if should_revisit or next_ not in closed: open_.append(next_) - def visit_forward(self, graph): - self.graph = graph + def visit_forward(self): self._visit_internal(_WalkMode.FORWARD) - def visit_reverse(self, graph): - self.graph = graph + def visit_reverse(self): self._visit_internal(_WalkMode.REVERSE) @@ -244,8 +258,16 @@ class GraphBuilder(object): # TODO(mdan): Too many primitives. Use classes. self.leaves = set() + # Note: This mechanism requires that nodes are added in lexical order (top + # to bottom, depth first). + self.active_stmts = set() + self.owners = {} # type: Set[any] + self.forward_edges = set() # type: Tuple[Node, Node] # (from, to) + self.finally_sections = {} - self.finally_section_subgraphs = {} # Values are [begin_node, exit_nodes] + # Dict values represent (entry, exits) + self.finally_section_subgraphs = { + } # type: Dict[ast.AST, Tuple[Node, Set[Node]]] # Whether the guard section can be reached from the statement that precedes # it. self.finally_section_has_direct_flow = {} @@ -275,6 +297,7 @@ class GraphBuilder(object): if isinstance(first, Node): first.next.add(second) second.prev.add(first) + self.forward_edges.add((first, second)) else: for node in first: self._connect_nodes(node, second) @@ -285,6 +308,7 @@ class GraphBuilder(object): raise ValueError('%s added twice' % ast_node) node = Node(next_=set(), prev=set(), ast_node=ast_node) self.node_index[ast_node] = node + self.owners[node] = frozenset(self.active_stmts) if self.head is None: self.head = node @@ -299,6 +323,25 @@ class GraphBuilder(object): return node + def begin_statement(self, stmt): + """Marks the beginning of a statement. + + Args: + stmt: Hashable, a key by which the statement can be identified in + the CFG's stmt_prev and stmt_next attributes + """ + self.active_stmts.add(stmt) + + def end_statement(self, stmt): + """Marks the end of a statement. + + Args: + stmt: Hashable, a key by which the statement can be identified in + the CFG's stmt_prev and stmt_next attributes; must match a key + previously passed to begin_statement. + """ + self.active_stmts.remove(stmt) + def add_ordinary_node(self, ast_node): """Grows the graph by adding an ordinary CFG node. @@ -505,11 +548,35 @@ class GraphBuilder(object): for node in self.node_index.values(): node.freeze() + # Build the statement edges. + stmt_next = {} + stmt_prev = {} + for node, _ in self.forward_edges: + for stmt in self.owners[node]: + if stmt not in stmt_next: + stmt_next[stmt] = set() + if stmt not in stmt_prev: + stmt_prev[stmt] = set() + for first, second in self.forward_edges: + stmts_exited = self.owners[first] - self.owners[second] + for stmt in stmts_exited: + stmt_next[stmt].add(second) + stmts_entered = self.owners[second] - self.owners[first] + for stmt in stmts_entered: + stmt_prev[stmt].add(first) + for stmt in stmt_next: + stmt_next[stmt] = frozenset(stmt_next[stmt]) + for stmt in stmt_prev: + stmt_prev[stmt] = frozenset(stmt_prev[stmt]) + + # Construct the final graph object. result = Graph( entry=self.head, exit=self.leaves, error=self.errors, - index=self.node_index) + index=self.node_index, + stmt_prev=stmt_prev, + stmt_next=stmt_next) # Reset the state. self.reset() @@ -523,8 +590,6 @@ class AstToCfg(gast.NodeVisitor): A separate CFG will be constructed for each function. """ - # TODO(mdan): Figure out how to deal with closures. - def __init__(self): super(AstToCfg, self).__init__() @@ -577,6 +642,13 @@ class AstToCfg(gast.NodeVisitor): self.builder.add_continue_node(node, try_node, guards) def visit_FunctionDef(self, node): + # We also keep the FunctionDef node in the CFG. This allows us to determine + # things like reaching definitions via closure. Note that the function body + # will be stored in a separate graph, because function definitions are not + # the same as function calls. + if self.builder is not None: + self.builder.add_ordinary_node(node) + self.builder_stack.append(self.builder) self.builder = GraphBuilder(node) @@ -637,6 +709,7 @@ class AstToCfg(gast.NodeVisitor): # targets of jump statements like break/continue/etc. Since there is no # statement that can interrupt a conditional, we don't need to track their # lexical scope. That may change in the future. + self.builder.begin_statement(node) self.builder.enter_cond_section(node) self._process_basic_statement(node.test) @@ -650,8 +723,10 @@ class AstToCfg(gast.NodeVisitor): self.visit(stmt) self.builder.exit_cond_section(node) + self.builder.end_statement(node) def visit_While(self, node): + self.builder.begin_statement(node) self._enter_lexical_scope(node) self.builder.enter_section(node) @@ -670,8 +745,10 @@ class AstToCfg(gast.NodeVisitor): self.visit(stmt) self.builder.exit_section(node) + self.builder.end_statement(node) def visit_For(self, node): + self.builder.begin_statement(node) self._enter_lexical_scope(node) self.builder.enter_section(node) @@ -693,6 +770,7 @@ class AstToCfg(gast.NodeVisitor): self.visit(stmt) self.builder.exit_section(node) + self.builder.end_statement(node) def visit_Break(self, node): self._process_exit_statement(node, gast.While, gast.For) @@ -722,12 +800,13 @@ class AstToCfg(gast.NodeVisitor): def visit_With(self, node): # TODO(mdan): Mark the context manager's exit call as exit guard. - self._process_basic_statement(node.items) + for item in node.items: + self._process_basic_statement(item) for stmt in node.body: self.visit(stmt) def build(node): - builder = AstToCfg() - builder.visit(node) - return builder.cfgs + visitor = AstToCfg() + visitor.visit(node) + return visitor.cfgs diff --git a/tensorflow/contrib/autograph/pyct/cfg_test.py b/tensorflow/contrib/autograph/pyct/cfg_test.py index 00afadd521..9d0a85d615 100644 --- a/tensorflow/contrib/autograph/pyct/cfg_test.py +++ b/tensorflow/contrib/autograph/pyct/cfg_test.py @@ -25,9 +25,13 @@ from tensorflow.python.platform import test class CountingVisitor(cfg.GraphVisitor): - def __init__(self): + def __init__(self, graph): + super(CountingVisitor, self).__init__(graph) self.counts = {} + def init_state(self, _): + return None + def visit_node(self, node): self.counts[node.ast_node] = self.counts.get(node.ast_node, 0) + 1 return False # visit only once @@ -51,8 +55,8 @@ class GraphVisitorTest(test.TestCase): graphs, node = self._build_cfg(test_fn) graph, = graphs.values() - visitor = CountingVisitor() - visitor.visit_forward(graph) + visitor = CountingVisitor(graph) + visitor.visit_forward() fn_node = node.body[0] self.assertEqual(visitor.counts[fn_node.args], 1) @@ -74,8 +78,8 @@ class GraphVisitorTest(test.TestCase): graphs, node = self._build_cfg(test_fn) graph, = graphs.values() - visitor = CountingVisitor() - visitor.visit_reverse(graph) + visitor = CountingVisitor(graph) + visitor.visit_reverse() fn_node = node.body[0] self.assertEqual(visitor.counts[fn_node.args], 1) @@ -94,7 +98,7 @@ class AstToCfgTest(test.TestCase): return cfgs def _repr_set(self, node_set): - return set(repr(n) for n in node_set) + return frozenset(repr(n) for n in node_set) def _as_set(self, elements): if elements is None: @@ -110,14 +114,35 @@ class AstToCfgTest(test.TestCase): matched = False for cfg_node in graph.index.values(): if repr(cfg_node) == node_repr: - if (self._as_set(prev) == set(map(repr, cfg_node.prev)) and - self._as_set(next_) == set(map(repr, cfg_node.next))): + if (self._as_set(prev) == frozenset(map(repr, cfg_node.prev)) and + self._as_set(next_) == frozenset(map(repr, cfg_node.next))): matched = True break if not matched: self.fail( 'match failed for node "%s" in graph:\n%s' % (node_repr, graph)) + def assertStatementEdges(self, graph, edges): + """Tests whether the CFG contains the specified statement edges.""" + for prev_node_reprs, node_repr, next_node_reprs in edges: + matched = False + partial_matches = [] + self.assertSetEqual( + frozenset(graph.stmt_next.keys()), frozenset(graph.stmt_prev.keys())) + for stmt_ast_node in graph.stmt_next: + ast_repr = '%s:%s' % (stmt_ast_node.__class__.__name__, + stmt_ast_node.lineno) + if ast_repr == node_repr: + actual_next = frozenset(map(repr, graph.stmt_next[stmt_ast_node])) + actual_prev = frozenset(map(repr, graph.stmt_prev[stmt_ast_node])) + partial_matches.append((actual_prev, node_repr, actual_next)) + if (self._as_set(prev_node_reprs) == actual_prev and + self._as_set(next_node_reprs) == actual_next): + matched = True + break + if not matched: + self.fail('edges mismatch for %s: %s' % (node_repr, partial_matches)) + def test_straightline(self): def test_fn(a): @@ -171,7 +196,7 @@ class AstToCfgTest(test.TestCase): ), ) - def test_branch_straightline(self): + def test_if_straightline(self): def test_fn(a): if a > 0: @@ -189,6 +214,10 @@ class AstToCfgTest(test.TestCase): ('(a > 0)', 'a += -1', None), ), ) + self.assertStatementEdges( + graph, + (('a', 'If:2', None),), + ) def test_branch_nested(self): @@ -219,6 +248,14 @@ class AstToCfgTest(test.TestCase): ('(a > 2)', 'a = 4', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'If:2', None), + ('(a > 0)', 'If:3', None), + ('(a > 0)', 'If:8', None), + ), + ) def test_branch_straightline_semi(self): @@ -236,6 +273,10 @@ class AstToCfgTest(test.TestCase): ('(a > 0)', 'a = 1', None), ), ) + self.assertStatementEdges( + graph, + (('a', 'If:2', None),), + ) def test_branch_return(self): @@ -257,6 +298,10 @@ class AstToCfgTest(test.TestCase): ('a = 1', 'a = 2', None), ), ) + self.assertStatementEdges( + graph, + (('a', 'If:2', 'a = 2'),), + ) def test_branch_return_minimal(self): @@ -273,6 +318,10 @@ class AstToCfgTest(test.TestCase): ('(a > 0)', 'return', None), ), ) + self.assertStatementEdges( + graph, + (('a', 'If:2', None),), + ) def test_while_straightline(self): @@ -291,6 +340,10 @@ class AstToCfgTest(test.TestCase): ('(a > 0)', 'a = 2', None), ), ) + self.assertStatementEdges( + graph, + (('a', 'While:2', 'a = 2'),), + ) def test_while_else_straightline(self): @@ -312,6 +365,10 @@ class AstToCfgTest(test.TestCase): ('a = 2', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + (('a', 'While:2', 'a = 3'),), + ) def test_while_else_continue(self): @@ -339,6 +396,13 @@ class AstToCfgTest(test.TestCase): ('a = 2', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'While:2', 'a = 3'), + ('(a > 0)', 'If:3', ('a = 1', '(a > 0)')), + ), + ) def test_while_else_break(self): @@ -364,6 +428,13 @@ class AstToCfgTest(test.TestCase): (('break', 'a = 2'), 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'While:2', 'a = 3'), + ('(a > 0)', 'If:3', ('a = 1', 'a = 3')), + ), + ) def test_while_else_return(self): @@ -389,6 +460,13 @@ class AstToCfgTest(test.TestCase): ('a = 2', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'While:2', 'a = 3'), + ('(a > 0)', 'If:3', 'a = 1'), + ), + ) def test_while_nested_straightline(self): @@ -411,6 +489,13 @@ class AstToCfgTest(test.TestCase): ('(a > 0)', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'While:2', 'a = 3'), + ('(a > 0)', 'While:3', 'a = 2'), + ), + ) def test_while_nested_continue(self): @@ -437,6 +522,14 @@ class AstToCfgTest(test.TestCase): ('(a > 0)', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'While:2', 'a = 3'), + ('(a > 0)', 'While:3', 'a = 2'), + ('(a > 1)', 'If:4', ('a = 1', '(a > 1)')), + ), + ) def test_while_nested_break(self): @@ -451,16 +544,21 @@ class AstToCfgTest(test.TestCase): graph, = self._build_cfg(test_fn).values() - self.assertGraphMatches( + self.assertGraphMatches(graph, ( + (('a', 'a = 2'), '(a > 0)', ('(a > 1)', 'a = 3')), + (('(a > 0)', 'a = 1'), '(a > 1)', ('(a > 2)', 'a = 2')), + ('(a > 1)', '(a > 2)', ('break', 'a = 1')), + ('(a > 2)', 'break', 'a = 2'), + ('(a > 2)', 'a = 1', '(a > 1)'), + (('(a > 1)', 'break'), 'a = 2', '(a > 0)'), + ('(a > 0)', 'a = 3', None), + )) + self.assertStatementEdges( graph, ( - (('a', 'a = 2'), '(a > 0)', ('(a > 1)', 'a = 3')), - (('(a > 0)', 'a = 1'), '(a > 1)', ('(a > 2)', 'a = 2')), - ('(a > 1)', '(a > 2)', ('break', 'a = 1')), - ('(a > 2)', 'break', 'a = 2'), - ('(a > 2)', 'a = 1', '(a > 1)'), - (('(a > 1)', 'break'), 'a = 2', '(a > 0)'), - ('(a > 0)', 'a = 3', None), + ('a', 'While:2', 'a = 3'), + ('(a > 0)', 'While:3', 'a = 2'), + ('(a > 1)', 'If:4', ('a = 1', 'a = 2')), ), ) @@ -481,6 +579,10 @@ class AstToCfgTest(test.TestCase): ('range(0, a)', 'a = 2', None), ), ) + self.assertStatementEdges( + graph, + (('a', 'For:2', 'a = 2'),), + ) def test_for_else_straightline(self): @@ -502,6 +604,10 @@ class AstToCfgTest(test.TestCase): ('a = 2', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + (('a', 'For:2', 'a = 3'),), + ) def test_for_else_continue(self): @@ -530,6 +636,13 @@ class AstToCfgTest(test.TestCase): ('a = 2', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'For:2', 'a = 3'), + ('range(0, a)', 'If:3', ('a = 1', 'range(0, a)')), + ), + ) def test_for_else_break(self): @@ -555,6 +668,13 @@ class AstToCfgTest(test.TestCase): (('break', 'a = 2'), 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'For:2', 'a = 3'), + ('range(0, a)', 'If:3', ('a = 1', 'a = 3')), + ), + ) def test_for_else_return(self): @@ -580,6 +700,13 @@ class AstToCfgTest(test.TestCase): ('a = 2', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'For:2', 'a = 3'), + ('range(0, a)', 'If:3', 'a = 1'), + ), + ) def test_for_nested_straightline(self): @@ -602,6 +729,13 @@ class AstToCfgTest(test.TestCase): ('range(0, a)', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'For:2', 'a = 3'), + ('range(0, a)', 'For:3', 'a = 2'), + ), + ) def test_for_nested_continue(self): @@ -629,6 +763,14 @@ class AstToCfgTest(test.TestCase): ('range(0, a)', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'For:2', 'a = 3'), + ('range(0, a)', 'For:3', 'a = 2'), + ('range(1, a)', 'If:4', ('b += 1', 'range(1, a)')), + ), + ) def test_for_nested_break(self): @@ -655,6 +797,14 @@ class AstToCfgTest(test.TestCase): ('range(0, a)', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('a', 'For:2', 'a = 3'), + ('range(0, a)', 'For:3', 'a = 2'), + ('range(1, a)', 'If:4', ('b += 1', 'a = 2')), + ), + ) def test_complex(self): @@ -704,6 +854,17 @@ class AstToCfgTest(test.TestCase): ('range(1, a)', 'a = 3', None), ), ) + self.assertStatementEdges( + graph, + ( + ('b = 0', 'While:3', 'range(1, a)'), + ('(a > 0)', 'For:4', 'a = 2'), + ('range(0, a)', 'If:5', ('(a > 3)', 'a = 2')), + ('(a > 2)', 'If:7', ('b += 1', 'a = 2', 'range(0, a)')), + ('(a > 3)', 'If:8', ('a = 2', 'range(0, a)')), + ('(a > 0)', 'For:17', 'a = 3'), + ), + ) def test_finally_straightline(self): @@ -785,6 +946,24 @@ class AstToCfgTest(test.TestCase): ), ) + def test_with_straightline(self): + + def test_fn(a): + with max(a) as b: + a = 0 + return b + + graph, = self._build_cfg(test_fn).values() + + self.assertGraphMatches( + graph, + ( + ('a', 'max(a)', 'a = 0'), + ('max(a)', 'a = 0', 'return b'), + ('a = 0', 'return b', None), + ), + ) + if __name__ == '__main__': test.main() -- cgit v1.2.3 From 6bbc2dc43ed083b21d593341d497521f43ac1061 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 10:00:41 -0700 Subject: Automated rollback of commit bbc23f229eb01dcc285a5884954b0f0eebb0a68b PiperOrigin-RevId: 204316871 --- tensorflow/contrib/lite/kernels/fully_connected.cc | 8 ++++---- tensorflow/contrib/lite/kernels/fully_connected_test.cc | 12 ++++-------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/fully_connected.cc b/tensorflow/contrib/lite/kernels/fully_connected.cc index 6c9a845bd1..3b203dd480 100644 --- a/tensorflow/contrib/lite/kernels/fully_connected.cc +++ b/tensorflow/contrib/lite/kernels/fully_connected.cc @@ -152,10 +152,10 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } } - // Resize output to the same as input (except the last dimension which is - // determined by the number of units). - TfLiteIntArray* output_size_array = TfLiteIntArrayCopy(input->dims); - output_size_array->data[input->dims->size - 1] = num_units; + // Resize output. + TfLiteIntArray* output_size_array = TfLiteIntArrayCreate(2); + output_size_array->data[0] = batch_size; + output_size_array->data[1] = num_units; TF_LITE_ENSURE_OK(context, context->ResizeTensor(context, output, output_size_array)); return kTfLiteOk; diff --git a/tensorflow/contrib/lite/kernels/fully_connected_test.cc b/tensorflow/contrib/lite/kernels/fully_connected_test.cc index a6b6b2f497..ec94905697 100644 --- a/tensorflow/contrib/lite/kernels/fully_connected_test.cc +++ b/tensorflow/contrib/lite/kernels/fully_connected_test.cc @@ -207,7 +207,6 @@ class FloatFullyConnectedOpModel : public BaseFullyConnectedOpModel { } std::vector GetOutput() { return ExtractVector(output_); } - std::vector GetOutputSize() { return GetTensorShape(output_); } }; class QuantizedFullyConnectedOpModel : public BaseFullyConnectedOpModel { @@ -299,7 +298,6 @@ class HybridFullyConnectedOpModel : public SingleOpModel { void SetInput(const std::vector& f) { PopulateTensor(input_, f); } std::vector GetOutput() { return ExtractVector(output_); } - std::vector GetOutputSize() { return GetTensorShape(output_); } int input_size() { return input_size_; } int num_units() { return units_; } @@ -374,7 +372,6 @@ TEST_P(FloatFullyConnectedOpTest, SimpleTest) { m.Invoke(); - EXPECT_THAT(m.GetOutputSize(), ElementsAre(2, 3)); EXPECT_THAT(m.GetOutput(), ElementsAre(24, 25, 26, 58, 59, 60)); } @@ -393,7 +390,6 @@ TEST_P(FloatFullyConnectedOpTest, SimpleTest2) { m.Invoke(); - EXPECT_THAT(m.GetOutputSize(), ElementsAre(2, 1)); EXPECT_THAT(m.GetOutput(), ElementsAre(11, 9)); } @@ -580,10 +576,11 @@ TEST(HybridFullyConnectedOpTest, SimpleTestQuantized) { TEST_P(FloatFullyConnectedOpTest, SimpleTest4DInput) { // Note that it is not required that the first dimension be the number of - // batches. All we care is that the input size is the last dimension. + // batches. All we care is that the input can be evenly distributed in + // batches. In this case, we need the input to have multiples of '2'. FloatFullyConnectedOpModel m(GetRegistration(), /*units=*/3, /*batches=*/2, - /*input=*/{TensorType_FLOAT32, {1, 2, 1, 10}}); + /*input=*/{TensorType_FLOAT32, {4, 1, 5, 1}}); m.SetWeights({ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // u = 0 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // u = 1 @@ -598,7 +595,6 @@ TEST_P(FloatFullyConnectedOpTest, SimpleTest4DInput) { m.Invoke(); - EXPECT_THAT(m.GetOutputSize(), ElementsAre(1, 2, 1, 3)); EXPECT_THAT(m.GetOutput(), ElementsAreArray({ 24, 25, 26, // first batch 58, 59, 60, // second batch @@ -608,7 +604,7 @@ TEST_P(FloatFullyConnectedOpTest, SimpleTest4DInput) { TEST_P(QuantizedFullyConnectedOpTest, SimpleTest4dInputQuantized) { QuantizedFullyConnectedOpModel m( GetRegistration(), /*units=*/3, /*batches=*/2, - /*input=*/{TensorType_UINT8, {1, 2, 1, 10}, -63.5, 64}, + /*input=*/{TensorType_UINT8, {4, 1, 5, 1}, -63.5, 64}, /*output=*/{TensorType_UINT8, {}, -127, 128}); // input_product_scale < output_scale was not true. -- cgit v1.2.3 From 48c280f308ca01ba457f950d62e9e32d19e5eff5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 10:02:44 -0700 Subject: Remove testing assumption that outermost stack frame does not belong to the *_test.py file. This is not true in all test environments. PiperOrigin-RevId: 204317251 --- tensorflow/python/framework/traceable_stack_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/framework/traceable_stack_test.py b/tensorflow/python/framework/traceable_stack_test.py index 168f9083b0..3e7876f631 100644 --- a/tensorflow/python/framework/traceable_stack_test.py +++ b/tensorflow/python/framework/traceable_stack_test.py @@ -66,10 +66,12 @@ class TraceableObjectTest(test_util.TensorFlowTestCase): result = t_obj.set_filename_and_line_from_caller(offset=300) # We expect a heuristic to be used because we are not currently 300 frames - # down on the stack. The filename should be some wacky thing from the - # outermost stack frame -- definitely not equal to this filename. + # down on the stack. The filename and lineno of the outermost frame are not + # predictable -- in some environments the filename is this test file, but in + # other environments it is not (e.g. due to a test runner calling this + # file). Therefore we only test that the called function knows it applied a + # heuristic for the ridiculous stack offset. self.assertEqual(t_obj.HEURISTIC_USED, result) - self.assertNotEqual(_THIS_FILENAME, t_obj.filename) class TraceableStackTest(test_util.TensorFlowTestCase): -- cgit v1.2.3 From 25021d386cd989aedde11b72c5db36b7c1bfd2b4 Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Wed, 11 Jul 2018 17:57:38 +0000 Subject: [ROCm] Interface changes for StreamExecutor to support both CUDA and ROCm 1) StreamInterface::CudaStreamMemberHack() Despite the fact that StreamExecutor and GPU common runtime are largely orthogonal, this particular routine in StreamExecutor is used in GPU common runtime and a couple of other operators. In this commit it's renamed as StreamInterface::GpuStreamMemberHack() and their call sites are also changed. 2) StreamExecutorInterface::CudaContextHack() This member is renamed to StramExecutorInterface::GpuContextHack(). Changes introduced in this commit includes: - some StreamExecutor interfaces and CUDA implementation - GPU common runtime related to interface changes in StreamExecutor - operators affected by interface changes in StreamExecutor --- tensorflow/contrib/nccl/kernels/nccl_manager.cc | 2 +- .../custom_plugin_examples/inc_op_kernel.cu.cc | 2 +- .../contrib/tensorrt/kernels/trt_engine_op.cc | 4 +-- tensorflow/core/common_runtime/gpu/gpu_device.cc | 2 +- tensorflow/core/kernels/cuda_solvers.cc | 2 +- tensorflow/core/util/cuda_launch_config.h | 2 +- .../stream_executor/cuda/cuda_gpu_executor.cc | 2 +- .../stream_executor/cuda/cuda_gpu_executor.h | 2 +- tensorflow/stream_executor/cuda/cuda_stream.h | 4 +-- .../stream_executor/host/host_gpu_executor.h | 2 +- tensorflow/stream_executor/host/host_stream.h | 4 +-- .../stream_executor/stream_executor_internal.h | 34 ++++++++++++---------- 12 files changed, 32 insertions(+), 30 deletions(-) diff --git a/tensorflow/contrib/nccl/kernels/nccl_manager.cc b/tensorflow/contrib/nccl/kernels/nccl_manager.cc index b1cb89391c..99fecf9651 100644 --- a/tensorflow/contrib/nccl/kernels/nccl_manager.cc +++ b/tensorflow/contrib/nccl/kernels/nccl_manager.cc @@ -445,7 +445,7 @@ void NcclManager::LoopKernelLaunches(NcclStream* nccl_stream) { se::Stream* comm_stream = nccl_stream->stream.get(); ScopedActivateExecutorContext scoped_context(nccl_stream->executor); const cudaStream_t* cu_stream = reinterpret_cast( - comm_stream->implementation()->CudaStreamMemberHack()); + comm_stream->implementation()->GpuStreamMemberHack()); while (true) { // Find collective to run. diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_kernel.cu.cc b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_kernel.cu.cc index 988b35f74f..2de7973750 100644 --- a/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_kernel.cu.cc +++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/inc_op_kernel.cu.cc @@ -65,7 +65,7 @@ class IncPluginTRT : public OpKernel { reinterpret_cast(context->op_device_context() ->stream() ->implementation() - ->CudaStreamMemberHack())); + ->GpuStreamMemberHack())); IncrementKernel(input_tensor.flat().data(), inc_, output_tensor->flat().data(), input_shape.num_elements(), *stream); diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc index 8a17eb02f1..3daf810a4b 100644 --- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc +++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc @@ -230,7 +230,7 @@ void TRTEngineOp::ExecuteCalibration(tensorflow::OpKernelContext* ctx, reinterpret_cast(ctx->op_device_context() ->stream() ->implementation() - ->CudaStreamMemberHack())); + ->GpuStreamMemberHack())); calib_res->calibrator_->setBatch(input_data, *stream); VLOG(2) << "Passed calibration data"; ExecuteNativeSegment(ctx, helper); @@ -380,7 +380,7 @@ void TRTEngineOp::ComputeAsync(tensorflow::OpKernelContext* ctx, reinterpret_cast(ctx->op_device_context() ->stream() ->implementation() - ->CudaStreamMemberHack())); + ->GpuStreamMemberHack())); // TODO(jie): trt enqueue does not return error auto& trt_execution_context_ptr = engine_ctx_pair.second; diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc index 3cb51b0dbc..f38ccd0d5b 100644 --- a/tensorflow/core/common_runtime/gpu/gpu_device.cc +++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc @@ -856,7 +856,7 @@ void BaseGPUDevice::ReinitializeDevice(OpKernelContext* context, static_cast(device); DCHECK(concrete_device); const cudaStream_t* cuda_stream = reinterpret_cast( - streams_[stream_id]->compute->implementation()->CudaStreamMemberHack()); + streams_[stream_id]->compute->implementation()->GpuStreamMemberHack()); concrete_device->Reinitialize(context, cuda_stream, tf_gpu_id_, allocator, scratch_[stream_id]); } diff --git a/tensorflow/core/kernels/cuda_solvers.cc b/tensorflow/core/kernels/cuda_solvers.cc index a857bd3ce4..a59baaa96f 100644 --- a/tensorflow/core/kernels/cuda_solvers.cc +++ b/tensorflow/core/kernels/cuda_solvers.cc @@ -151,7 +151,7 @@ CudaSolver::CudaSolver(OpKernelContext* context) : context_(context) { reinterpret_cast(context->op_device_context() ->stream() ->implementation() - ->CudaStreamMemberHack())); + ->GpuStreamMemberHack())); cuda_stream_ = *cu_stream_ptr; HandleMap* handle_map = CHECK_NOTNULL(GetHandleMapSingleton()); auto it = handle_map->find(cuda_stream_); diff --git a/tensorflow/core/util/cuda_launch_config.h b/tensorflow/core/util/cuda_launch_config.h index 81df7a51d7..d0d95736d3 100644 --- a/tensorflow/core/util/cuda_launch_config.h +++ b/tensorflow/core/util/cuda_launch_config.h @@ -295,7 +295,7 @@ inline const cudaStream_t& GetCudaStream(OpKernelContext* context) { reinterpret_cast(context->op_device_context() ->stream() ->implementation() - ->CudaStreamMemberHack())); + ->GpuStreamMemberHack())); return *ptr; } diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index f11022ef1d..259c813c57 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -844,7 +844,7 @@ CUDAExecutor::GetTimerImplementation() { return std::unique_ptr(new CUDATimer(this)); } -void *CUDAExecutor::CudaContextHack() { return context_; } +void *CUDAExecutor::GpuContextHack() { return context_; } CudaContext* CUDAExecutor::cuda_context() { return context_; } diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h index 773cbfb8a1..f7c341c857 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h @@ -210,7 +210,7 @@ class CUDAExecutor : public internal::StreamExecutorInterface { std::unique_ptr GetTimerImplementation() override; - void *CudaContextHack() override; + void *GpuContextHack() override; CudaContext* cuda_context(); diff --git a/tensorflow/stream_executor/cuda/cuda_stream.h b/tensorflow/stream_executor/cuda/cuda_stream.h index 02edff6431..bb8bda4755 100644 --- a/tensorflow/stream_executor/cuda/cuda_stream.h +++ b/tensorflow/stream_executor/cuda/cuda_stream.h @@ -40,8 +40,8 @@ class CUDAStream : public internal::StreamInterface { // Note: teardown is handled by a parent's call to DeallocateStream. ~CUDAStream() override {} - void *CudaStreamHack() override { return cuda_stream_; } - void **CudaStreamMemberHack() override { + void *GpuStreamHack() override { return cuda_stream_; } + void **GpuStreamMemberHack() override { return reinterpret_cast(&cuda_stream_); } diff --git a/tensorflow/stream_executor/host/host_gpu_executor.h b/tensorflow/stream_executor/host/host_gpu_executor.h index e82f57569f..858396ef96 100644 --- a/tensorflow/stream_executor/host/host_gpu_executor.h +++ b/tensorflow/stream_executor/host/host_gpu_executor.h @@ -202,7 +202,7 @@ class HostExecutor : public internal::StreamExecutorInterface { return std::unique_ptr(new HostTimer()); } - void *CudaContextHack() override { return nullptr; } + void *GpuContextHack() override { return nullptr; } private: const PluginConfig plugin_config_; diff --git a/tensorflow/stream_executor/host/host_stream.h b/tensorflow/stream_executor/host/host_stream.h index 5d7b8a3782..be88f074cf 100644 --- a/tensorflow/stream_executor/host/host_stream.h +++ b/tensorflow/stream_executor/host/host_stream.h @@ -34,8 +34,8 @@ class HostStream : public internal::StreamInterface { bool EnqueueTask(std::function task); - void *CudaStreamHack() override { return nullptr; } - void **CudaStreamMemberHack() override { return nullptr; } + void *GpuStreamHack() override { return nullptr; } + void **GpuStreamMemberHack() override { return nullptr; } void BlockUntilDone(); diff --git a/tensorflow/stream_executor/stream_executor_internal.h b/tensorflow/stream_executor/stream_executor_internal.h index 9c989b971d..fb1b92cb84 100644 --- a/tensorflow/stream_executor/stream_executor_internal.h +++ b/tensorflow/stream_executor/stream_executor_internal.h @@ -100,19 +100,20 @@ class StreamInterface { // Default destructor for the abstract interface. virtual ~StreamInterface() {} - // Returns the CUDA stream associated with this platform's stream + // Returns the GPU stream associated with this platform's stream // implementation. // - // WARNING: checks that the underlying platform is, in fact, CUDA, causing a - // fatal error if it is not. This hack is made available solely for use from - // distbelief code, which temporarily has strong ties to CUDA as a platform. - virtual void *CudaStreamHack() { return nullptr; } - - // See the above comment on CudaStreamHack -- this further breaks abstraction - // for Eigen within distbelief, which has strong ties to CUDA as a platform, - // and a historical attachment to a programming model which takes a + // WARNING: checks that the underlying platform is, in fact, CUDA or ROCm, + // causing a fatal error if it is not. This hack is made available solely for + // use from distbelief code, which temporarily has strong ties to CUDA or + // ROCm as a platform. + virtual void *GpuStreamHack() { return nullptr; } + + // See the above comment on GpuStreamHack -- this further breaks abstraction + // for Eigen within distbelief, which has strong ties to CUDA or ROCm as a + // platform, and a historical attachment to a programming model which takes a // stream-slot rather than a stream-value. - virtual void **CudaStreamMemberHack() { return nullptr; } + virtual void **GpuStreamMemberHack() { return nullptr; } private: SE_DISALLOW_COPY_AND_ASSIGN(StreamInterface); @@ -324,13 +325,14 @@ class StreamExecutorInterface { virtual std::unique_ptr GetStreamImplementation() = 0; virtual std::unique_ptr GetTimerImplementation() = 0; - // Returns the CUDA context associated with this StreamExecutor platform - // implementation. + // Returns the CUDA or ROCm context associated with this StreamExecutor + // platform implementation. // - // WARNING: checks that the underlying platform is, in fact, CUDA, causing a - // fatal error if it is not. This hack is made available solely for use from - // distbelief code, which temporarily has strong ties to CUDA as a platform. - virtual void *CudaContextHack() { return nullptr; } + // WARNING: checks that the underlying platform is, in fact, CUDA or ROCm, + // causing a fatal error if it is not. This hack is made available solely for + // use from distbelief code, which temporarily has strong ties to CUDA or ROCm + // as a platform. + virtual void *GpuContextHack() { return nullptr; } private: SE_DISALLOW_COPY_AND_ASSIGN(StreamExecutorInterface); -- cgit v1.2.3 From cd00681ed2000e29eb60730a8b9aed9076188800 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Thu, 12 Jul 2018 10:06:59 -0700 Subject: Add an improved tree-scoped state to the transformer, to replace the existing local_scope mechanism which only supports one partitioning and is stringly typed. PiperOrigin-RevId: 204318158 --- tensorflow/contrib/autograph/pyct/transformer.py | 136 ++++++++++++++++++++- .../contrib/autograph/pyct/transformer_test.py | 77 ++++++++++++ 2 files changed, 210 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/autograph/pyct/transformer.py b/tensorflow/contrib/autograph/pyct/transformer.py index 7655811830..d9a157aead 100644 --- a/tensorflow/contrib/autograph/pyct/transformer.py +++ b/tensorflow/contrib/autograph/pyct/transformer.py @@ -59,6 +59,103 @@ class EntityInfo(object): self.owner_type = owner_type +class _StateStack(object): + """Typed stack abstraction. + + This class provides syntactic sugar for a stack of objects of known + type. It allows accessing attributes of the object at the top of the stack + directly against this object, which allows for very terse syntax. + + For example, this code: + + stack = _StateStack(Foo) + stack.enter() + stack.bar + + Is equivalent to: + + stack = [] + stack.append(Foo()) + foo = stack[-1] + foo.bar + + See _State for more on how this is used. + + Attributes: + type: Any, the type of objects that this stack holds + level: int, the current stack depth + value: Any, the instance of the object at the top of the stack + """ + + def __init__(self, type_): + # Because we override __setattr__, we need to attach these attributes using + # the superclass' setattr. + object.__setattr__(self, 'type', type_) + object.__setattr__(self, '_stack', []) + self.enter() + + def enter(self): + self._stack.append(self.type()) + + def exit(self): + return self._stack.pop() + + @property + def level(self): + return len(self._stack) + + @property + def value(self): + return self._stack[-1] + + def __getattr__(self, key): + return getattr(self._stack[-1], key) + + def __setattr__(self, key, value): + setattr(self._stack[-1], key, value) + + +class _State(object): + """Supporting class for nested scope variable space for converter.Base. + + This structure offers syntactic sugar over a dict of stacks of objects + of known type. These structures are useful to keep state during AST walks. + Multiple different scopes can be tracked in parallel. For example: + + s = _State() + + s[foo].enter() + s[bar].enter() # this will not affect s[foo] + + Element access has special semantics: + * keys are a data type + * element values are _StateStack(type=key) objects + * missing elements are automatically added, similarly to defaultdict + + For example, the following block : + + _State s + s[Foo] + + Is equivalent to: + + s = {} + if Foo not in s: + s[Foo] = Foo() + s[Foo] + + See Base for how it's used. + """ + + def __init__(self): + self._value = {} + + def __getitem__(self, key): + if key not in self._value: + self._value[key] = _StateStack(key) + return self._value[key] + + class Base(gast.NodeTransformer): """Base class for general-purpose code transformers transformers. @@ -71,6 +168,27 @@ class Base(gast.NodeTransformer): (possibly nested) scopes, use enter/exit_local_scope and set/get_local. You must call enter/exit_local_scope manually, but the transformer detects when they are not properly paired. + + The transformer allows keeping state across calls to visit_* that is local to + arbitrary nodes and their descendants, using the self.state attribute. + Multiple independent scopes are allowed and automatically constructed. + + For example, to keep track of the If node that encloses any Name node, one can + write: + + class FooType(object): + + def __init__(self): + self.foo_property = None + + class DummyTransformer(Base): + + def visit_If(self, node): + self.state[FooType].enter() + self.state[FooType].foo_property = node + + def visit_Name(self, node): + self.state[FooType].foo_property # will hold the innermost enclosing if """ # TODO(mdan): Document all extra features. @@ -92,6 +210,12 @@ class Base(gast.NodeTransformer): self._local_scope_state = [] self.enter_local_scope() + # Allows scoping of local variables to keep state across calls to visit_* + # methods. Multiple scope hierchies may exist and are keyed by tag. A scope + # is valid at one or more nodes and all its children. Scopes created in + # child nodes supersede their parent. Scopes are isolated from one another. + self.state = _State() + @property def enclosing_entities(self): return tuple(self._enclosing_entities) @@ -101,7 +225,9 @@ class Base(gast.NodeTransformer): return len(self._local_scope_state) def enter_local_scope(self, inherit=None): - """Marks entry into a new local scope. + """Deprecated. Use self.state instead. + + Marks entry into a new local scope. Args: inherit: Optional enumerable of variable names to copy from the @@ -116,7 +242,9 @@ class Base(gast.NodeTransformer): self._local_scope_state.append(scope_entered) def exit_local_scope(self, keep=None): - """Marks exit from the current local scope. + """Deprecated. Use self.state instead. + + Marks exit from the current local scope. Args: keep: Optional enumerable of variable names to copy into the @@ -133,9 +261,11 @@ class Base(gast.NodeTransformer): return scope_left def set_local(self, name, value): + """Deprecated. Use self.state instead.""" self._local_scope_state[-1][name] = value def get_local(self, name, default=None): + """Deprecated. Use self.state instead.""" return self._local_scope_state[-1].get(name, default) def debug_print(self, node): @@ -216,7 +346,7 @@ class Base(gast.NodeTransformer): node_destination = new_destination return results - # TODO(mdan): Once we have error tracing, we may be able to just go to SSA. + # TODO(mdan): Remove. def apply_to_single_assignments(self, targets, values, apply_fn): """Applies a function to each individual assignment. diff --git a/tensorflow/contrib/autograph/pyct/transformer_test.py b/tensorflow/contrib/autograph/pyct/transformer_test.py index baf04653ae..19b80b09ac 100644 --- a/tensorflow/contrib/autograph/pyct/transformer_test.py +++ b/tensorflow/contrib/autograph/pyct/transformer_test.py @@ -93,6 +93,83 @@ class TransformerTest(test.TestCase): inner_function, lambda_node), anno.getanno(lambda_expr, 'enclosing_entities')) + def assertSameAnno(self, first, second, key): + self.assertIs(anno.getanno(first, key), anno.getanno(second, key)) + + def assertDifferentAnno(self, first, second, key): + self.assertIsNot(anno.getanno(first, key), anno.getanno(second, key)) + + def test_state_tracking(self): + + class LoopState(object): + pass + + class CondState(object): + pass + + class TestTransformer(transformer.Base): + + def visit(self, node): + anno.setanno(node, 'loop_state', self.state[LoopState].value) + anno.setanno(node, 'cond_state', self.state[CondState].value) + return super(TestTransformer, self).visit(node) + + def visit_While(self, node): + self.state[LoopState].enter() + node = self.generic_visit(node) + self.state[LoopState].exit() + return node + + def visit_If(self, node): + self.state[CondState].enter() + node = self.generic_visit(node) + self.state[CondState].exit() + return node + + tr = TestTransformer(self._simple_source_info()) + + def test_function(a): + a = 1 + while a: + _ = 'a' + if a > 2: + _ = 'b' + while True: + raise '1' + if a > 3: + _ = 'c' + while True: + raise '1' + + node, _ = parser.parse_entity(test_function) + node = tr.visit(node) + + fn_body = node.body[0].body + outer_while_body = fn_body[1].body + self.assertSameAnno(fn_body[0], outer_while_body[0], 'cond_state') + self.assertDifferentAnno(fn_body[0], outer_while_body[0], 'loop_state') + + first_if_body = outer_while_body[1].body + self.assertDifferentAnno(outer_while_body[0], first_if_body[0], + 'cond_state') + self.assertSameAnno(outer_while_body[0], first_if_body[0], 'loop_state') + + first_inner_while_body = first_if_body[1].body + self.assertSameAnno(first_if_body[0], first_inner_while_body[0], + 'cond_state') + self.assertDifferentAnno(first_if_body[0], first_inner_while_body[0], + 'loop_state') + + second_if_body = outer_while_body[2].body + self.assertDifferentAnno(first_if_body[0], second_if_body[0], 'cond_state') + self.assertSameAnno(first_if_body[0], second_if_body[0], 'loop_state') + + second_inner_while_body = second_if_body[1].body + self.assertDifferentAnno(first_inner_while_body[0], + second_inner_while_body[0], 'cond_state') + self.assertDifferentAnno(first_inner_while_body[0], + second_inner_while_body[0], 'loop_state') + def test_local_scope_info_stack(self): class TestTransformer(transformer.Base): -- cgit v1.2.3 From d9cf45ba645d30bd782b45bc757a1e11f49a84b0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 10:07:04 -0700 Subject: Fix support for batch_normalization with mixed precision When the type of the input tensor `x` is not the same as the type of the parameters `mean`, `variance` and `offset` a cast is required. This mixed precision case occurs when using the BatchNormalization layer with a data type of float16 or bfloat16. PiperOrigin-RevId: 204318176 --- tensorflow/python/keras/layers/normalization.py | 6 +++++- tensorflow/python/keras/layers/normalization_test.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py index 58c8a8a66d..a7835bc0a2 100644 --- a/tensorflow/python/keras/layers/normalization.py +++ b/tensorflow/python/keras/layers/normalization.py @@ -370,7 +370,7 @@ class BatchNormalization(Layer): decay = ops.convert_to_tensor(1.0 - momentum, name='decay') if decay.dtype != variable.dtype.base_dtype: decay = math_ops.cast(decay, variable.dtype.base_dtype) - update_delta = (variable - value) * decay + update_delta = (variable - math_ops.cast(value, variable.dtype)) * decay return state_ops.assign_sub(variable, update_delta, name=scope) def _fused_batch_norm(self, inputs, training): @@ -619,6 +619,10 @@ class BatchNormalization(Layer): else: mean, variance = self.moving_mean, self.moving_variance + mean = math_ops.cast(mean, inputs.dtype) + variance = math_ops.cast(variance, inputs.dtype) + if offset is not None: + offset = math_ops.cast(offset, inputs.dtype) outputs = nn.batch_normalization(inputs, _broadcast(mean), _broadcast(variance), diff --git a/tensorflow/python/keras/layers/normalization_test.py b/tensorflow/python/keras/layers/normalization_test.py index b22f3bd152..a97b4cac46 100644 --- a/tensorflow/python/keras/layers/normalization_test.py +++ b/tensorflow/python/keras/layers/normalization_test.py @@ -95,6 +95,24 @@ class NormalizationLayersTest(test.TestCase): np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1) np.testing.assert_allclose(out.std(), 1.0, atol=1e-1) + def test_batchnorm_mixed_precision(self): + with self.test_session(): + model = keras.models.Sequential() + norm = keras.layers.BatchNormalization(input_shape=(10,), momentum=0.8) + model.add(norm) + model.compile(loss='mse', optimizer='sgd') + + # centered on 5.0, variance 10.0 + x = np.random.normal( + loc=5.0, scale=10.0, size=(1000, 10)).astype(np.float16) + model.fit(x, x, epochs=4, verbose=0) + out = model.predict(x) + out -= keras.backend.eval(norm.beta) + out /= keras.backend.eval(norm.gamma) + + np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1) + np.testing.assert_allclose(out.std(), 1.0, atol=1e-1) + def test_batchnorm_convnet(self): if test.is_gpu_available(cuda_only=True): with self.test_session(use_gpu=True): -- cgit v1.2.3 From 59863962b1df6ebba96be77b9308679bdbee4ed4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 10:08:28 -0700 Subject: Add safe static constructor functions to the sparse tensor library and convert all CHECKs to DCHECKs. PiperOrigin-RevId: 204318525 --- tensorflow/core/util/sparse/dim_comparator.h | 16 +- tensorflow/core/util/sparse/group_iterator.h | 6 +- tensorflow/core/util/sparse/sparse_tensor.h | 196 +++++++++++++++++----- tensorflow/core/util/sparse/sparse_tensor_test.cc | 91 ++++++---- 4 files changed, 225 insertions(+), 84 deletions(-) diff --git a/tensorflow/core/util/sparse/dim_comparator.h b/tensorflow/core/util/sparse/dim_comparator.h index b773b33008..0782e7e1a8 100644 --- a/tensorflow/core/util/sparse/dim_comparator.h +++ b/tensorflow/core/util/sparse/dim_comparator.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_UTIL_SPARSE_DIM_COMPARATOR_H_ -#define TENSORFLOW_UTIL_SPARSE_DIM_COMPARATOR_H_ +#ifndef TENSORFLOW_CORE_UTIL_SPARSE_DIM_COMPARATOR_H_ +#define TENSORFLOW_CORE_UTIL_SPARSE_DIM_COMPARATOR_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/kernels/bounds_check.h" @@ -49,11 +49,11 @@ class DimComparator { DimComparator(const TTypes::Matrix& ix, const VarDimArray& order, const VarDimArray& shape) : ix_(ix), order_(order), dims_(shape.size()) { - CHECK_GT(order.size(), size_t{0}) << "Must order using at least one index"; - CHECK_LE(order.size(), shape.size()) << "Can only sort up to dims"; + DCHECK_GT(order.size(), size_t{0}) << "Must order using at least one index"; + DCHECK_LE(order.size(), shape.size()) << "Can only sort up to dims"; for (size_t d = 0; d < order.size(); ++d) { - CHECK_GE(order[d], 0); - CHECK_LT(order[d], shape.size()); + DCHECK_GE(order[d], 0); + DCHECK_LT(order[d], shape.size()); } } @@ -97,7 +97,7 @@ class FixedDimComparator : DimComparator { FixedDimComparator(const TTypes::Matrix& ix, const VarDimArray& order, const VarDimArray& shape) : DimComparator(ix, order, shape) { - CHECK_EQ(order.size(), ORDER_DIM); + DCHECK_EQ(order.size(), ORDER_DIM); } inline bool operator()(const int64 i, const int64 j) const { bool value = false; @@ -116,4 +116,4 @@ class FixedDimComparator : DimComparator { } // namespace sparse } // namespace tensorflow -#endif // TENSORFLOW_UTIL_SPARSE_DIM_COMPARATOR_H_ +#endif // TENSORFLOW_CORE_UTIL_SPARSE_DIM_COMPARATOR_H_ diff --git a/tensorflow/core/util/sparse/group_iterator.h b/tensorflow/core/util/sparse/group_iterator.h index fb70318078..3fa8cb6116 100644 --- a/tensorflow/core/util/sparse/group_iterator.h +++ b/tensorflow/core/util/sparse/group_iterator.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_UTIL_SPARSE_GROUP_ITERATOR_H_ -#define TENSORFLOW_UTIL_SPARSE_GROUP_ITERATOR_H_ +#ifndef TENSORFLOW_CORE_UTIL_SPARSE_GROUP_ITERATOR_H_ +#define TENSORFLOW_CORE_UTIL_SPARSE_GROUP_ITERATOR_H_ #include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" @@ -143,4 +143,4 @@ typename TTypes::UnalignedVec Group::values() const { } // namespace sparse } // namespace tensorflow -#endif // TENSORFLOW_UTIL_SPARSE_GROUP_ITERATOR_H_ +#endif // TENSORFLOW_CORE_UTIL_SPARSE_GROUP_ITERATOR_H_ diff --git a/tensorflow/core/util/sparse/sparse_tensor.h b/tensorflow/core/util/sparse/sparse_tensor.h index 258ee418c1..0f04b65f60 100644 --- a/tensorflow/core/util/sparse/sparse_tensor.h +++ b/tensorflow/core/util/sparse/sparse_tensor.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_UTIL_SPARSE_SPARSE_TENSOR_H_ -#define TENSORFLOW_UTIL_SPARSE_SPARSE_TENSOR_H_ +#ifndef TENSORFLOW_CORE_UTIL_SPARSE_SPARSE_TENSOR_H_ +#define TENSORFLOW_CORE_UTIL_SPARSE_SPARSE_TENSOR_H_ #include #include @@ -26,8 +26,10 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/kernels/bounds_check.h" +#include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/sparse/dim_comparator.h" @@ -41,32 +43,88 @@ class SparseTensor { typedef typename gtl::ArraySlice VarDimArray; typedef typename gtl::InlinedVector ShapeArray; + static Status Create(Tensor ix, Tensor vals, const VarDimArray shape, + const VarDimArray order, SparseTensor* result) { + if (ix.dtype() != DT_INT64) { + return Status( + error::INVALID_ARGUMENT, + strings::StrCat("indices must be type int64 but got: ", ix.dtype())); + } + if (!TensorShapeUtils::IsVector(vals.shape())) { + return Status(error::INVALID_ARGUMENT, + strings::StrCat("vals must be a vec, but got: ", + vals.shape().DebugString())); + } + if (ix.shape().dim_size(0) != vals.shape().dim_size(0)) { + return Status(error::INVALID_ARGUMENT, + strings::StrCat("indices and values rows (indexing " + "dimension) must match. (indices = ", + ix.shape().dim_size(0), ", values = ", + vals.shape().dim_size(0), ")")); + } + int dims; + TF_RETURN_IF_ERROR(GetDimsFromIx(ix, &dims)); + if (order.size() != dims) { + return Status(error::INVALID_ARGUMENT, + "Order length must be SparseTensor rank."); + } + if (shape.size() != dims) { + return Status(error::INVALID_ARGUMENT, + "Shape rank must be SparseTensor rank."); + } + + *result = SparseTensor(ix, vals, shape, order); + return Status(); + } + + static Status Create(Tensor ix, Tensor vals, const TensorShape& shape, + SparseTensor* result) { + return Create(ix, vals, TensorShapeToVector(shape), + UndefinedOrder(TensorShapeToVector(shape)), result); + } + + static Status Create(Tensor ix, Tensor vals, const VarDimArray shape, + SparseTensor* result) { + return Create(ix, vals, shape, UndefinedOrder(shape), result); + } + + static Status Create(Tensor ix, Tensor vals, const TensorShape& shape, + const VarDimArray order, SparseTensor* result) { + return Create(ix, vals, TensorShapeToVector(shape), order, result); + } + + SparseTensor() : dims_(0) {} + + // DEPRECATED: use Create() functions instead of constructors directly. SparseTensor(Tensor ix, Tensor vals, const TensorShape& shape) : SparseTensor(ix, vals, TensorShapeToVector(shape), UndefinedOrder(TensorShapeToVector(shape))) {} + // DEPRECATED: use Create() functions instead of constructors directly. SparseTensor(Tensor ix, Tensor vals, const VarDimArray shape) : SparseTensor(ix, vals, shape, UndefinedOrder(shape)) {} + // DEPRECATED: use Create() functions instead of constructors directly. SparseTensor(Tensor ix, Tensor vals, const TensorShape& shape, const VarDimArray order) : SparseTensor(ix, vals, TensorShapeToVector(shape), order) {} + // DEPRECATED: use Create() functions instead of constructors directly. SparseTensor(Tensor ix, Tensor vals, const VarDimArray shape, const VarDimArray order) : ix_(ix), vals_(vals), shape_(shape.begin(), shape.end()), order_(order.begin(), order.end()), - dims_(GetDimsFromIx(ix)) { - CHECK_EQ(ix.dtype(), DT_INT64) + dims_(UnsafeGetDimsFromIx(ix)) { + DCHECK_EQ(ix.dtype(), DT_INT64) << "indices must be type int64 but got: " << ix.dtype(); - CHECK(TensorShapeUtils::IsVector(vals.shape())) + DCHECK(TensorShapeUtils::IsVector(vals.shape())) << "vals must be a vec, but got: " << vals.shape().DebugString(); - CHECK_EQ(ix.shape().dim_size(0), vals.shape().dim_size(0)) + DCHECK_EQ(ix.shape().dim_size(0), vals.shape().dim_size(0)) << "indices and values rows (indexing dimension) must match."; - CHECK_EQ(order.size(), dims_) << "Order length must be SparseTensor rank."; - CHECK_EQ(shape.size(), dims_) << "Shape rank must be SparseTensor rank."; + DCHECK_EQ(order.size(), dims_) << "Order length must be SparseTensor rank."; + DCHECK_EQ(shape.size(), dims_) << "Shape rank must be SparseTensor rank."; } SparseTensor(const SparseTensor& other) @@ -81,6 +139,16 @@ class SparseTensor { vals_ = other.vals_; shape_ = other.shape_; order_ = other.order_; + dims_ = other.dims_; + return *this; + } + + SparseTensor& operator=(SparseTensor&& other) { + ix_ = std::move(other.ix_); + vals_ = std::move(other.vals_); + shape_ = std::move(other.shape_); + order_ = std::move(other.order_); + dims_ = std::move(other.dims_); return *this; } @@ -126,11 +194,11 @@ class SparseTensor { // // See the README.md in this directory for more usage information. GroupIterable group(const VarDimArray& group_ix) const { - CHECK_LE(group_ix.size(), dims_); + DCHECK_LE(group_ix.size(), dims_); for (std::size_t di = 0; di < group_ix.size(); ++di) { - CHECK_GE(group_ix[di], 0) << "Group dimension out of range"; - CHECK_LT(group_ix[di], dims_) << "Group dimension out of range"; - CHECK_EQ(group_ix[di], order_[di]) + DCHECK_GE(group_ix[di], 0) << "Group dimension out of range"; + DCHECK_LT(group_ix[di], dims_) << "Group dimension out of range"; + DCHECK_EQ(group_ix[di], order_[di]) << "Group dimension does not match sorted order"; } return GroupIterable(ix_, vals_, dims_, group_ix); @@ -166,9 +234,16 @@ class SparseTensor { // isn't an integer multiple of split_dim, we add one extra dimension for // each slice. template + static Status Split(const SparseTensor& tensor, const int split_dim, + const int num_split, std::vector* result); + + // DEPRECATED: use the form of Split() that takes an output pointer and + // returns a status instead. + template static std::vector Split(const SparseTensor& tensor, const int split_dim, - const int num_split); + const int num_split, + Status* status = nullptr); // Slice() will slice the input SparseTensor into a SparseTensor based on // specified start and size. Both start and size are 1-D array with each @@ -189,9 +264,18 @@ class SparseTensor { } private: - static int GetDimsFromIx(const Tensor& ix) { - CHECK(TensorShapeUtils::IsMatrix(ix.shape())) - << "indices must be a matrix, but got: " << ix.shape().DebugString(); + static Status GetDimsFromIx(const Tensor& ix, int* result) { + if (!TensorShapeUtils::IsMatrix(ix.shape())) { + return Status(error::INVALID_ARGUMENT, + strings::StrCat("indices must be a matrix, but got: ", + ix.shape().DebugString())); + } + *result = UnsafeGetDimsFromIx(ix); + return Status(); + } + + static int UnsafeGetDimsFromIx(const Tensor& ix) { + DCHECK(TensorShapeUtils::IsMatrix(ix.shape())); return ix.dim_size(1); } @@ -251,8 +335,8 @@ class SparseTensor { // Helper for Split() that returns the slice index. static inline int GetSliceIndex(const int dim, const int split_size, const int residual) { - CHECK_GT(split_size, 0); - CHECK_GE(dim, 0); + DCHECK_GT(split_size, 0); + DCHECK_GE(dim, 0); if (residual == 0) return dim / split_size; const int offset = residual * (split_size + 1); if (dim < offset) { @@ -265,8 +349,8 @@ class SparseTensor { // Helper for Split() that returns the dimension in the slice. static inline int GetDimensionInSlice(const int dim, const int split_size, const int residual) { - CHECK_GT(split_size, 0); - CHECK_GE(dim, 0); + DCHECK_GT(split_size, 0); + DCHECK_GE(dim, 0); if (residual == 0) return dim % split_size; const int offset = residual * (split_size + 1); if (dim < offset) { @@ -279,8 +363,8 @@ class SparseTensor { // Helper for Split() that returns the shape given a slice index. static inline int GetSliceShape(const int slice_index, const int split_size, const int residual) { - CHECK_GT(split_size, 0); - CHECK_GE(slice_index, 0); + DCHECK_GT(split_size, 0); + DCHECK_GE(slice_index, 0); if (residual == 0) return split_size; if (slice_index < residual) { return split_size + 1; @@ -293,7 +377,7 @@ class SparseTensor { Tensor vals_; ShapeArray shape_; ShapeArray order_; - const int dims_; + int dims_; }; // This operation updates the indices and values Tensor rows, so it is @@ -301,9 +385,9 @@ class SparseTensor { // temporary space. template void SparseTensor::Reorder(const VarDimArray& order) { - CHECK_EQ(DataTypeToEnum::v(), dtype()) + DCHECK_EQ(DataTypeToEnum::v(), dtype()) << "Reorder requested with the wrong datatype"; - CHECK_EQ(order.size(), dims_) << "Order length must be SparseTensor rank"; + DCHECK_EQ(order.size(), dims_) << "Order length must be SparseTensor rank"; auto ix_t = ix_.matrix(); auto vals_t = vals_.vec(); @@ -360,13 +444,13 @@ void SparseTensor::Reorder(const VarDimArray& order) { template bool SparseTensor::ValidateAndInitializeToDense(Tensor* out, bool initialize) { - CHECK_EQ(DataTypeToEnum::v(), dtype()) + DCHECK_EQ(DataTypeToEnum::v(), dtype()) << "ToDense requested with the wrong datatype"; - CHECK_EQ(out->shape().dims(), dims_) + DCHECK_EQ(out->shape().dims(), dims_) << "Incompatible dimensions between SparseTensor and output"; - CHECK_EQ(out->dtype(), DataTypeToEnum::v()) + DCHECK_EQ(out->dtype(), DataTypeToEnum::v()) << "Output must be type: " << DataTypeToEnum::v() << " but got: " << out->dtype(); @@ -422,9 +506,9 @@ bool SparseTensor::ToDense(Tensor* out, bool initialize) { template SparseTensor SparseTensor::Concat( const gtl::ArraySlice& tensors) { - CHECK_GE(tensors.size(), size_t{1}) << "Cannot concat 0 SparseTensors"; + DCHECK_GE(tensors.size(), size_t{1}) << "Cannot concat 0 SparseTensors"; const int dims = tensors[0].dims_; - CHECK_GE(dims, 1) << "Cannot concat 0-dimensional SparseTensors"; + DCHECK_GE(dims, 1) << "Cannot concat 0-dimensional SparseTensors"; auto order_0 = tensors[0].order(); const int primary_dim = order_0[0]; ShapeArray final_order(order_0.begin(), order_0.end()); @@ -434,17 +518,17 @@ SparseTensor SparseTensor::Concat( bool fully_ordered = true; for (const SparseTensor& st : tensors) { - CHECK_EQ(st.dims_, dims) << "All SparseTensors must have the same rank."; - CHECK_EQ(DataTypeToEnum::v(), st.dtype()) + DCHECK_EQ(st.dims_, dims) << "All SparseTensors must have the same rank."; + DCHECK_EQ(DataTypeToEnum::v(), st.dtype()) << "Concat requested with the wrong data type"; - CHECK_GE(st.order()[0], 0) << "SparseTensor must be ordered"; - CHECK_EQ(st.order()[0], primary_dim) + DCHECK_GE(st.order()[0], 0) << "SparseTensor must be ordered"; + DCHECK_EQ(st.order()[0], primary_dim) << "All SparseTensors' order[0] must match. This is the concat dim."; if (st.order() != final_order) fully_ordered = false; const VarDimArray& st_shape = st.shape(); for (int d = 0; d < dims - 1; ++d) { const int cdim = (d < primary_dim) ? d : d + 1; - CHECK_EQ(final_shape[cdim], st_shape[cdim]) + DCHECK_EQ(final_shape[cdim], st_shape[cdim]) << "All SparseTensors' shapes must match except on the concat dim. " << "Concat dim: " << primary_dim << ", mismatched shape at dim: " << cdim @@ -494,7 +578,8 @@ SparseTensor SparseTensor::Concat( template std::vector SparseTensor::Split(const SparseTensor& input_tensor, const int split_dim, - const int num_split) { + const int num_split, + Status* status /* = nullptr */) { std::vector output_indices; std::vector output_values; std::vector output_shapes; @@ -514,12 +599,18 @@ std::vector SparseTensor::Split(const SparseTensor& input_tensor, const int split_dim_size = input_tensor.shape()[split_dim]; const int split_size = split_dim_size / num_split; - CHECK(num_split > 0 && num_split <= split_dim_size) << "num_split must be in " - "the interval (0, " - << split_dim_size << "]"; - CHECK(split_dim >= 0 && split_dim < num_dim) << "num_dim must be in " - "the interval [0, " - << num_dim << ")"; + if (!(num_split > 0 && num_split <= split_dim_size) && status != nullptr) { + *status = Status(error::INVALID_ARGUMENT, + strings::StrCat("num_split must be in the interval (0, ", + split_dim_size, "]")); + return {}; + } + if (!(split_dim >= 0 && split_dim < num_dim)) { + *status = Status( + error::INVALID_ARGUMENT, + strings::StrCat("num_dim must be in the interval [0, ", num_dim, ")")); + return {}; + } const int residual = split_dim_size % num_split; for (int i = 0; i < input_tensor.indices().dim_size(0); ++i) { @@ -559,12 +650,27 @@ std::vector SparseTensor::Split(const SparseTensor& input_tensor, std::vector output_tensors; output_tensors.reserve(num_split); for (int i = 0; i < num_split; ++i) { - output_tensors.emplace_back(output_indices[i], output_values[i], - output_shapes[i]); + SparseTensor tensor; + Status create_status = + Create(output_indices[i], output_values[i], output_shapes[i], &tensor); + if (!create_status.ok() && status != nullptr) { + *status = create_status; + return {}; + } + output_tensors.push_back(std::move(tensor)); } return output_tensors; } +template +Status SparseTensor::Split(const SparseTensor& input_tensor, + const int split_dim, const int num_split, + std::vector* result) { + Status status; + *result = Split(input_tensor, split_dim, num_split, &status); + return status; +} + template SparseTensor SparseTensor::Slice(const SparseTensor& input_tensor, const gtl::ArraySlice& start, @@ -643,4 +749,4 @@ SparseTensor SparseTensor::Slice(const SparseTensor& input_tensor, } // namespace sparse } // namespace tensorflow -#endif // TENSORFLOW_UTIL_SPARSE_SPARSE_TENSOR_H_ +#endif // TENSORFLOW_CORE_UTIL_SPARSE_SPARSE_TENSOR_H_ diff --git a/tensorflow/core/util/sparse/sparse_tensor_test.cc b/tensorflow/core/util/sparse/sparse_tensor_test.cc index 85de032085..5578e42625 100644 --- a/tensorflow/core/util/sparse/sparse_tensor_test.cc +++ b/tensorflow/core/util/sparse/sparse_tensor_test.cc @@ -94,9 +94,12 @@ TEST(SparseTensorTest, SparseTensorInvalidIndicesType) { const int NDIM = 3; Tensor ix(DT_INT32, TensorShape({N, NDIM})); Tensor vals(DT_STRING, TensorShape({N})); + SparseTensor result; - EXPECT_DEATH(SparseTensor(ix, vals, TensorShape({10, 10, 10}), {0, 1, 2}), - "indices must be type int64"); + EXPECT_EQ(SparseTensor::Create(ix, vals, TensorShape({10, 10, 10}), {0, 1, 2}, + &result) + .code(), + error::INVALID_ARGUMENT); } TEST(SparseTensorTest, SparseTensorInvalidIndicesShape) { @@ -104,9 +107,12 @@ TEST(SparseTensorTest, SparseTensorInvalidIndicesShape) { const int NDIM = 3; Tensor ix(DT_INT64, TensorShape({N, NDIM, 1})); Tensor vals(DT_STRING, TensorShape({N})); + SparseTensor result; - EXPECT_DEATH(SparseTensor(ix, vals, TensorShape({10, 10, 10}), {0, 1, 2}), - "indices must be a matrix"); + EXPECT_EQ(SparseTensor::Create(ix, vals, TensorShape({10, 10, 10}), {0, 1, 2}, + &result) + .code(), + error::INVALID_ARGUMENT); } TEST(SparseTensorTest, SparseTensorInvalidValues) { @@ -114,9 +120,12 @@ TEST(SparseTensorTest, SparseTensorInvalidValues) { const int NDIM = 3; Tensor ix(DT_INT64, TensorShape({N, NDIM})); Tensor vals(DT_STRING, TensorShape({N, 1})); + SparseTensor result; - EXPECT_DEATH(SparseTensor(ix, vals, TensorShape({10, 10, 10}), {0, 1, 2}), - "vals must be a vec"); + EXPECT_EQ(SparseTensor::Create(ix, vals, TensorShape({10, 10, 10}), {0, 1, 2}, + &result) + .code(), + error::INVALID_ARGUMENT); } TEST(SparseTensorTest, SparseTensorInvalidN) { @@ -124,9 +133,12 @@ TEST(SparseTensorTest, SparseTensorInvalidN) { const int NDIM = 3; Tensor ix(DT_INT64, TensorShape({N, NDIM})); Tensor vals(DT_STRING, TensorShape({N - 1})); + SparseTensor result; - EXPECT_DEATH(SparseTensor(ix, vals, TensorShape({10, 10, 10}), {0, 1, 2}), - "indices and values rows .* must match"); + EXPECT_EQ(SparseTensor::Create(ix, vals, TensorShape({10, 10, 10}), {0, 1, 2}, + &result) + .code(), + error::INVALID_ARGUMENT); } TEST(SparseTensorTest, SparseTensorInvalidOrder) { @@ -134,18 +146,24 @@ TEST(SparseTensorTest, SparseTensorInvalidOrder) { const int NDIM = 3; Tensor ix(DT_INT64, TensorShape({N, NDIM})); Tensor vals(DT_STRING, TensorShape({N})); + SparseTensor result; - EXPECT_DEATH(SparseTensor(ix, vals, TensorShape({10, 10, 10}), {0, 1}), - "Order length must be SparseTensor rank"); + EXPECT_EQ( + SparseTensor::Create(ix, vals, TensorShape({10, 10, 10}), {0, 1}, &result) + .code(), + error::INVALID_ARGUMENT); } TEST(SparseTensorTest, SparseTensorInvalidShape) { int N = 5; const int NDIM = 3; Tensor ix(DT_INT64, TensorShape({N, NDIM})); Tensor vals(DT_STRING, TensorShape({N})); + SparseTensor result; - EXPECT_DEATH(SparseTensor(ix, vals, TensorShape({10, 10}), {0, 1, 2}), - "Shape rank must be SparseTensor rank"); + EXPECT_EQ( + SparseTensor::Create(ix, vals, TensorShape({10, 10}), {0, 1, 2}, &result) + .code(), + error::INVALID_ARGUMENT); } TEST(SparseTensorTest, SparseTensorConstruction) { @@ -169,7 +187,8 @@ TEST(SparseTensorTest, SparseTensorConstruction) { TensorShape shape({10, 10, 10}); std::vector order{0, 1, 2}; - SparseTensor st(ix, vals, shape, order); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, order, &st)); Status st_indices_valid = st.IndicesValid(); EXPECT_FALSE(st_indices_valid.ok()); EXPECT_EQ("indices[2] = [2,0,0] is out of order", @@ -210,7 +229,8 @@ TEST(SparseTensorTest, EmptySparseTensorAllowed) { std::vector shape{10, 10, 10}; std::vector order{0, 1, 2}; - SparseTensor st(ix, vals, shape, order); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, order, &st)); TF_EXPECT_OK(st.IndicesValid()); EXPECT_EQ(st.order(), order); @@ -227,7 +247,8 @@ TEST(SparseTensorTest, SortingWorksCorrectly) { Tensor ix(DT_INT64, TensorShape({N, NDIM})); Tensor vals(DT_STRING, TensorShape({N})); TensorShape shape({1000, 1000, 1000, 1000}); - SparseTensor st(ix, vals, shape); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, &st)); auto ix_t = ix.matrix(); @@ -266,7 +287,8 @@ TEST(SparseTensorTest, ValidateIndicesFindsInvalid) { TensorShape shape({10, 10, 10}); std::vector order{0, 1, 2}; - SparseTensor st(ix, vals, shape, order); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, order, &st)); st.Reorder(order); Status st_indices_valid = st.IndicesValid(); @@ -302,7 +324,8 @@ TEST(SparseTensorTest, SparseTensorCheckBoundaries) { TensorShape shape({10, 10, 10}); std::vector order{0, 1, 2}; - SparseTensor st(ix, vals, shape, order); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, order, &st)); EXPECT_FALSE(st.IndicesValid().ok()); st.Reorder(order); @@ -351,7 +374,8 @@ TEST(SparseTensorTest, SparseTensorToDenseTensor) { TensorShape shape({4, 4, 5}); std::vector order{0, 1, 2}; - SparseTensor st(ix, vals, shape, order); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, order, &st)); Tensor dense(DT_STRING, TensorShape({4, 4, 5})); st.ToDense(&dense); @@ -390,7 +414,8 @@ TEST(SparseTensorTest, SparseTensorToLargerDenseTensor) { TensorShape shape({4, 4, 5}); std::vector order{0, 1, 2}; - SparseTensor st(ix, vals, shape, order); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, order, &st)); Tensor dense(DT_STRING, TensorShape({10, 10, 10})); st.ToDense(&dense); @@ -433,7 +458,8 @@ TEST(SparseTensorTest, SparseTensorGroup) { TensorShape shape({10, 10, 10}); std::vector order{0, 1, 2}; - SparseTensor st(ix, vals, shape, order); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, order, &st)); st.Reorder(order); std::vector > groups; @@ -521,7 +547,8 @@ TEST(SparseTensorTest, Concat) { TensorShape shape({10, 10, 10}); std::vector order{0, 1, 2}; - SparseTensor st(ix, vals, shape, order); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, order, &st)); EXPECT_FALSE(st.IndicesValid().ok()); st.Reorder(order); TF_EXPECT_OK(st.IndicesValid()); @@ -551,7 +578,9 @@ TEST(SparseTensorTest, Concat) { // Concat works if non-primary ix is out of order, but output order // is not defined - SparseTensor st_ooo(ix, vals, shape, {0, 2, 1}); // non-primary ix OOO + SparseTensor st_ooo; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, {0, 2, 1}, + &st_ooo)); // non-primary ix OOO SparseTensor conc_ooo = SparseTensor::Concat({st, st, st, st_ooo}); std::vector expected_ooo{-1, -1, -1}; EXPECT_EQ(conc_ooo.order(), expected_ooo); @@ -584,9 +613,11 @@ TEST(SparseTensorTest, Split) { vals.vec()(2) = 3; vals.vec()(3) = 4; - SparseTensor st(ids, vals, TensorShape({4, 3})); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ids, vals, TensorShape({4, 3}), &st)); - std::vector st_list = SparseTensor::Split(st, 0, 2); + std::vector st_list; + TF_ASSERT_OK(SparseTensor::Split(st, 0, 2, &st_list)); EXPECT_EQ(st_list.size(), 2); auto expected_shape = gtl::InlinedVector{2, 3}; @@ -633,7 +664,8 @@ TEST(SparseTensorTest, Slice) { vals.vec()(2) = 3; vals.vec()(3) = 4; - SparseTensor st(ids, vals, TensorShape({4, 3})); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ids, vals, TensorShape({4, 3}), &st)); std::vector start(2, 0); std::vector size(2); @@ -662,7 +694,8 @@ TEST(SparseTensorTest, Dim0SparseTensorToDenseTensor) { vals.scalar()() = 5; TensorShape shape({}); - SparseTensor st(ix, vals, shape); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, &st)); Tensor dense(DT_INT32, TensorShape({})); st.ToDense(&dense); @@ -699,7 +732,8 @@ static void BM_SparseReorderFloat(int iters, int N32, int NDIM32) { ix_t(i, d) = rnd.Rand64() % 1000; } } - SparseTensor st(ix, vals, shape, order); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, order, &st)); testing::StartTiming(); st.Reorder(reorder); @@ -740,7 +774,8 @@ static void BM_SparseReorderString(int iters, int N32, int NDIM32) { ix_t(i, d) = rnd.Rand64() % 1000; } } - SparseTensor st(ix, vals, shape, order); + SparseTensor st; + TF_ASSERT_OK(SparseTensor::Create(ix, vals, shape, order, &st)); testing::StartTiming(); st.Reorder(reorder); -- cgit v1.2.3 From e36a1d67c53f831eaeb898b2cf5e0b072b1994cd Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Thu, 12 Jul 2018 10:10:29 -0700 Subject: Add a few more annotation utilities. Prepare to consolidate the annotation keys with the ones found in static_analysis. PiperOrigin-RevId: 204318953 --- tensorflow/contrib/autograph/pyct/anno.py | 89 ++++++++++++++++++++++---- tensorflow/contrib/autograph/pyct/anno_test.py | 23 ++++++- 2 files changed, 100 insertions(+), 12 deletions(-) diff --git a/tensorflow/contrib/autograph/pyct/anno.py b/tensorflow/contrib/autograph/pyct/anno.py index 92f1370e05..1a52110ef3 100644 --- a/tensorflow/contrib/autograph/pyct/anno.py +++ b/tensorflow/contrib/autograph/pyct/anno.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Handling annotations on AST nodes. +"""AST node annotation support. Adapted from Tangent. """ @@ -21,36 +21,87 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from enum import Enum +import enum +# pylint:disable=g-bad-import-order +import gast +# pylint:enable=g-bad-import-order -class NoValue(Enum): + +# TODO(mdan): Shorten the names. +# These names are heavily used, and anno.blaa +# TODO(mdan): Replace the attr-dict mechanism with a more typed solution. + + +class NoValue(enum.Enum): def __repr__(self): return self.name class Basic(NoValue): - """Container for annotation keys. + """Container for basic annotation keys. The enum values are used strictly for documentation purposes. """ - QN = 'Qualified name, as it appeared in the code.' + QN = 'Qualified name, as it appeared in the code. See qual_names.py.' SKIP_PROCESSING = ( 'This node should be preserved as is and not processed any further.') INDENT_BLOCK_REMAINDER = ( - 'When a node is annotated with this, the remainder of the block should ' - 'be indented below it. The annotation contains a tuple ' - '(new_body, name_map), where `new_body` is the new indented block and ' - '`name_map` allows renaming symbols.') - ORIGIN = ('Contains OriginInfo objects specific to the annotated node. See ' - 'origin_information.py for definition.') + 'When a node is annotated with this, the remainder of the block should' + ' be indented below it. The annotation contains a tuple' + ' (new_body, name_map), where `new_body` is the new indented block and' + ' `name_map` allows renaming symbols.') + ORIGIN = ('Information about the source code that converted code originated' + ' from. See origin_information.py.') + + +class Static(NoValue): + """Container for static analysis annotation keys. + + The enum values are used strictly for documentation purposes. + """ + + # Deprecated - use reaching definitions instead. + # Symbols + # These flags are boolean. + IS_LOCAL = 'Symbol is local to the function scope being analyzed.' + IS_PARAM = 'Symbol is a parameter to the function being analyzed.' + + # Scopes + # Scopes are represented by objects of type activity.Scope. + SCOPE = 'The scope for the annotated node. See activity.py.' + # TODO(mdan): Drop these in favor of accessing the child's SCOPE. + ARGS_SCOPE = 'The scope for the argument list of a function call.' + COND_SCOPE = 'The scope for the test node of a conditional statement.' + BODY_SCOPE = ( + 'The scope for the main body of a statement (True branch for if ' + 'statements, main body for loops).') + ORELSE_SCOPE = ( + 'The scope for the orelse body of a statement (False branch for if ' + 'statements, orelse body for loops).') + + # Static analysis annotations. + DEFINITIONS = ( + 'Reaching definition information. See reaching_definitions.py.') + ORIG_DEFINITIONS = ( + 'The value of DEFINITIONS that applied to the original code before any' + ' conversion.') + DEFINED_VARS_IN = ( + 'Symbols defined when entering the node. See reaching_definitions.py.') + LIVE_VARS_OUT = ('Symbols live when exiting the node. See liveness.py.') FAIL = object() +def keys(node, field_name='___pyct_anno'): + if not hasattr(node, field_name): + return frozenset() + return frozenset(getattr(node, field_name).keys()) + + def getanno(node, key, default=FAIL, field_name='___pyct_anno'): if (default is FAIL or (hasattr(node, field_name) and (key in getattr(node, field_name)))): @@ -88,3 +139,19 @@ def copyanno(from_node, to_node, key, field_name='___pyct_anno'): key, getanno(from_node, key, field_name=field_name), field_name=field_name) + + +def dup(node, copy_map, field_name='___pyct_anno'): + """Recursively copies annotations in an AST tree. + + Args: + node: ast.AST + copy_map: Dict[Hashable, Hashable], maps a source anno key to a destination + key. All annotations with the source key will be copied to identical + annotations with the destination key. + field_name: str + """ + for n in gast.walk(node): + for k in copy_map: + if hasanno(n, k, field_name): + setanno(n, copy_map[k], getanno(n, k, field_name), field_name) diff --git a/tensorflow/contrib/autograph/pyct/anno_test.py b/tensorflow/contrib/autograph/pyct/anno_test.py index f2c0c8cf05..5ef4da61a3 100644 --- a/tensorflow/contrib/autograph/pyct/anno_test.py +++ b/tensorflow/contrib/autograph/pyct/anno_test.py @@ -32,22 +32,27 @@ class AnnoTest(test.TestCase): def test_basic(self): node = ast.Name() + self.assertEqual(anno.keys(node), set()) self.assertFalse(anno.hasanno(node, 'foo')) with self.assertRaises(AttributeError): anno.getanno(node, 'foo') anno.setanno(node, 'foo', 3) + + self.assertEqual(anno.keys(node), {'foo'}) self.assertTrue(anno.hasanno(node, 'foo')) self.assertEqual(anno.getanno(node, 'foo'), 3) self.assertEqual(anno.getanno(node, 'bar', default=7), 7) anno.delanno(node, 'foo') + + self.assertEqual(anno.keys(node), set()) self.assertFalse(anno.hasanno(node, 'foo')) with self.assertRaises(AttributeError): anno.getanno(node, 'foo') self.assertIsNone(anno.getanno(node, 'foo', default=None)) - def test_copyanno(self): + def test_copy(self): node_1 = ast.Name() anno.setanno(node_1, 'foo', 3) @@ -58,6 +63,22 @@ class AnnoTest(test.TestCase): self.assertTrue(anno.hasanno(node_2, 'foo')) self.assertFalse(anno.hasanno(node_2, 'bar')) + def test_duplicate(self): + node = ast.If( + test=ast.Num(1), + body=[ast.Expr(ast.Name('bar', ast.Load()))], + orelse=[]) + anno.setanno(node, 'spam', 1) + anno.setanno(node, 'ham', 1) + anno.setanno(node.body[0], 'ham', 1) + + anno.dup(node, {'spam': 'eggs'}) + + self.assertTrue(anno.hasanno(node, 'spam')) + self.assertTrue(anno.hasanno(node, 'ham')) + self.assertTrue(anno.hasanno(node, 'eggs')) + self.assertFalse(anno.hasanno(node.body[0], 'eggs')) + if __name__ == '__main__': test.main() -- cgit v1.2.3 From c529a1317a3c8e7c5c16ca559873a938cd54c446 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Thu, 12 Jul 2018 10:17:01 -0700 Subject: [Keras/Cloud TPU]: Place variables on the device. In order to improve performance, avoid transferring the weights from the host memory onto device memory every step, but instead leave weights resident on the device to be re-used on subsequent steps. PiperOrigin-RevId: 204320023 --- tensorflow/contrib/tpu/BUILD | 3 + tensorflow/contrib/tpu/python/tpu/keras_support.py | 68 +++++++++++++++++++--- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index 0d1c7fc75a..0044fde9d0 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -175,9 +175,12 @@ py_library( "//tensorflow/contrib/tpu/proto:compilation_result_proto_py", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", + "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", + "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", "//tensorflow/python:platform", + "//tensorflow/python:random_ops", "//tensorflow/python:session", "//tensorflow/python:tensor_spec", "//tensorflow/python:variable_scope", diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py index 6e9c607f2e..722e31abb2 100644 --- a/tensorflow/contrib/tpu/python/tpu/keras_support.py +++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py @@ -64,6 +64,7 @@ from tensorflow.contrib.tpu.python.tpu import tpu_optimizer from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session as tf_session from tensorflow.python.estimator import model_fn as model_fn_lib +from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_spec from tensorflow.python.keras import backend as K @@ -72,7 +73,9 @@ from tensorflow.python.keras import optimizers as keras_optimizers from tensorflow.python.keras.engine import base_layer from tensorflow.python.keras.layers import embeddings from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_linalg_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import random_ops from tensorflow.python.ops import variable_scope from tensorflow.python.platform import tf_logging as logging @@ -208,8 +211,51 @@ class TPURewriteContext(object): self._default_placeholder = array_ops.placeholder self._default_name_scope = ops.name_scope self._default_make_variable = base_layer.make_variable + self._default_random_normal = random_ops.random_normal + self._default_qr = gen_linalg_ops.qr array_ops.placeholder = _placeholder + + # Replace random_ops.random_normal with a dummy function because + # `random_normal` isn't yet implemented on the TPU. Because these + # initialized values are overwritten by the CPU values, this is okay. + def random_normal(shape, + mean=0.0, + stddev=1.0, + dtype=dtypes.float32, + seed=None, + name=None): + del mean + del stddev + del seed + return array_ops.zeros(shape, dtype=dtype, name=name) + + random_ops.random_normal = random_normal + + # Replace gen_linalg_ops.qr because QR decomposition is not yet implemented. + # TODO(saeta): Remove qr override once we confirm the qr implementation is + # ok. + # pylint: disable=redefined-builtin + def qr(input, full_matrices=False, name=None): + """Dummy implementation of qr decomposition.""" + del full_matrices # TODO(saeta): Properly handle the full matrix case. + input_shape = input.shape + if len(input_shape) < 2: + raise ValueError('Invalid shape passed to qr: %s' % input_shape) + p = min(input_shape[-1], input_shape[-2]) + if len(input_shape) == 2: + q = array_ops.zeros((p, p), name=name) + r = array_ops.zeros(input_shape, name=name) + return (r, q) + elif len(input_shape) == 3: + n = input_shape[0] + q = array_ops.zeros((n, p, p), name=name) + r = array_ops.zeros(input_shape, name=name) + return (r, q) + else: + raise ValueError('Invalid shape passed to qr: %s' % input_shape) + gen_linalg_ops.qr = qr + ops.name_scope = _name_scope base_layer.make_variable = variable_scope.get_variable logging.info('Overriding default placeholder.') @@ -219,6 +265,8 @@ class TPURewriteContext(object): array_ops.placeholder = self._default_placeholder ops.name_scope = self._default_name_scope base_layer.make_variable = self._default_make_variable + random_ops.random_normal = self._default_random_normal + gen_linalg_ops.qr = self._default_qr class TPUFunction(object): @@ -287,7 +335,9 @@ class TPUFunction(object): # Clone our CPU model, running within the TPU device context. with TPURewriteContext(tpu_input_map): - self._cloned_model = models.clone_model(self.model) + # TODO(power): Replicate variables. + with ops.device('/device:TPU:0'): + self._cloned_model = models.clone_model(self.model) # Create a copy of the optimizer for this graph. if isinstance(self.model.optimizer, keras_optimizers.TFOptimizer): @@ -529,14 +579,16 @@ class KerasTPUModel(models.Model): self._tpu_weights_initialized = False self._graph = ops.Graph() - cluster_resolver = tpu_cluster_resolver.TPUClusterResolver( + self._cluster_resolver = tpu_cluster_resolver.TPUClusterResolver( tpu_name_or_address) - cluster_spec = cluster_resolver.cluster_spec() + master = self._cluster_resolver.master() + cluster_spec = self._cluster_resolver.cluster_spec() self._session = tf_session.Session( graph=self._graph, - target=cluster_resolver.master(), + target=master, config=config_pb2.ConfigProto(isolate_session_state=True)) + # TODO(saeta): Confirm the lines below work in ClusterSpec propagation env. if cluster_spec: self._session.cluster_def.CopyFrom(cluster_spec.as_cluster_def()) @@ -669,10 +721,10 @@ class KerasTPUModel(models.Model): K.set_session(default_session) def shutdown(self): - logging.info('Shutting down TPU session.') - with self.tpu_session() as session: - session.run(tpu.shutdown_system()) - + # TODO(b/111364423): Actually shut down the system. + logging.info('Skipping shutting down TPU system.') + # with self.tpu_session() as session: + # session.run(tpu.shutdown_system()) self._session.close() -- cgit v1.2.3 From 582e8fd393add33a45a47c453cf68a94391ec0ec Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Thu, 12 Jul 2018 10:34:52 -0700 Subject: [XLA] Retire TransferManager::TransferBufferToInfeed This isn't called from anywhere but the CPUTransferManager, just remove it as it's holding back the implementation on all targets. PiperOrigin-RevId: 204323463 --- tensorflow/compiler/xla/service/cpu/cpu_transfer_manager.h | 5 +++-- tensorflow/compiler/xla/service/generic_transfer_manager.cc | 5 ----- tensorflow/compiler/xla/service/generic_transfer_manager.h | 3 --- tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc | 9 --------- tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h | 2 -- tensorflow/compiler/xla/service/transfer_manager.h | 10 ---------- 6 files changed, 3 insertions(+), 31 deletions(-) diff --git a/tensorflow/compiler/xla/service/cpu/cpu_transfer_manager.h b/tensorflow/compiler/xla/service/cpu/cpu_transfer_manager.h index 6dfc666f09..593575c0fd 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_transfer_manager.h +++ b/tensorflow/compiler/xla/service/cpu/cpu_transfer_manager.h @@ -39,13 +39,14 @@ class CpuTransferManager : public GenericTransferManager { Status TransferLiteralToInfeed(se::StreamExecutor* executor, const LiteralSlice& literal) override; - Status TransferBufferToInfeed(se::StreamExecutor* executor, int64 size, - const void* source) override; Status TransferLiteralFromOutfeed(se::StreamExecutor* executor, const Shape& literal_shape, Literal* literal) override; private: + Status TransferBufferToInfeed(se::StreamExecutor* executor, int64 size, + const void* source); + // Transfers infeed data to device. InfeedBuffer->Done() must be // called to clean up the memory allocated for InfeedBuffer. StatusOr TransferBufferToInfeedInternal( diff --git a/tensorflow/compiler/xla/service/generic_transfer_manager.cc b/tensorflow/compiler/xla/service/generic_transfer_manager.cc index 33730049c4..e314a469f0 100644 --- a/tensorflow/compiler/xla/service/generic_transfer_manager.cc +++ b/tensorflow/compiler/xla/service/generic_transfer_manager.cc @@ -158,11 +158,6 @@ Status GenericTransferManager::TransferLiteralToInfeed( return Unimplemented("Generic transfer to Infeed"); } -Status GenericTransferManager::TransferBufferToInfeed( - se::StreamExecutor* executor, int64 size, const void* source) { - return Unimplemented("Generic transfer to Infeed"); -} - Status GenericTransferManager::TransferLiteralFromOutfeed( se::StreamExecutor* executor, const Shape& literal_shape, Literal* literal) { diff --git a/tensorflow/compiler/xla/service/generic_transfer_manager.h b/tensorflow/compiler/xla/service/generic_transfer_manager.h index d216fe7d29..3cd002c1bf 100644 --- a/tensorflow/compiler/xla/service/generic_transfer_manager.h +++ b/tensorflow/compiler/xla/service/generic_transfer_manager.h @@ -61,9 +61,6 @@ class GenericTransferManager : public TransferManager { int64 GetByteSizeRequirement(const Shape& shape) const override; protected: - Status TransferBufferToInfeed(se::StreamExecutor* executor, int64 size, - const void* source) override; - Status WriteSingleTupleIndexTable( se::Stream* stream, tensorflow::gtl::ArraySlice elements, diff --git a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc index 63466539fa..1446401b19 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc @@ -73,15 +73,6 @@ Status GpuTransferManager::TransferLiteralToInfeed( return EnqueueBuffersToInfeed(executor, std::move(buffer_tree)); } -Status GpuTransferManager::TransferBufferToInfeed(se::StreamExecutor* executor, - int64 size, - const void* source) { - return InternalError( - "Attempted to transfer data to infeed on a GPU device using " - "TransferBufferToInfeed. This should be done using " - "TransferLiteralToInfeed instead."); -} - Status GpuTransferManager::EnqueueBuffersToInfeed( se::StreamExecutor* executor, ShapeTree buffers) { gpu::InfeedManager* infeed_manager = gpu::GetOrCreateInfeedManager(); diff --git a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h index 7a5fe6979f..8122c9d8c3 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h +++ b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h @@ -40,8 +40,6 @@ class GpuTransferManager : public GenericTransferManager { Status TransferLiteralToInfeed(se::StreamExecutor* executor, const LiteralSlice& literal) override; - Status TransferBufferToInfeed(se::StreamExecutor* executor, int64 size, - const void* source) override; Status TransferLiteralFromOutfeed(se::StreamExecutor* executor, const Shape& literal_shape, Literal* literal) override; diff --git a/tensorflow/compiler/xla/service/transfer_manager.h b/tensorflow/compiler/xla/service/transfer_manager.h index 249bdcc1f5..82c599e482 100644 --- a/tensorflow/compiler/xla/service/transfer_manager.h +++ b/tensorflow/compiler/xla/service/transfer_manager.h @@ -167,16 +167,6 @@ class TransferManager { const se::Platform* platform); protected: - // Transfer a memory block of the given size from 'source' buffer to the - // Infeed interface of the device using the given executor. - // - // size is the size to transfer from source in bytes. - // - // source is the source data that must be in the target-dependent layout that - // the Infeed HLO used in the computation expects. - virtual Status TransferBufferToInfeed(se::StreamExecutor* executor, - int64 size, const void* source) = 0; - // Transfer a memory block of the given size from the device source into the // 'destination' buffer. // -- cgit v1.2.3 From c634fe54b7e270ccf08331c15e98c76ef5810b61 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Thu, 12 Jul 2018 10:48:33 -0700 Subject: Delete OpKernelContext::is_output_dead It seems to me that OpKernelContext::is_output_dead is not necessary. We only set it for Recv nodes and in those cases the output tensor is unset as well. PiperOrigin-RevId: 204325763 --- tensorflow/compiler/tf2xla/graph_compiler.cc | 3 +-- tensorflow/core/common_runtime/executor.cc | 2 +- tensorflow/core/framework/op_kernel.h | 3 --- tensorflow/core/kernels/sendrecv_ops.cc | 1 - 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc index 4900af6df1..e1cea03865 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.cc +++ b/tensorflow/compiler/tf2xla/graph_compiler.cc @@ -161,9 +161,8 @@ Status GraphCompiler::Compile() { outputs.resize(n->num_outputs()); for (int o = 0; o < n->num_outputs(); ++o) { outputs[o] = op_context.release_output(o); - if (*op_context.is_output_dead() || outputs[o].tensor == nullptr) { + if (outputs[o].tensor == nullptr) { return errors::Internal("Missing xla_context ", o, "-th output from ", - (*op_context.is_output_dead() ? "(dead)" : ""), SummarizeNode(*n)); } } diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc index f7f2cdc14f..5f3809ddd6 100644 --- a/tensorflow/core/common_runtime/executor.cc +++ b/tensorflow/core/common_runtime/executor.cc @@ -1976,7 +1976,7 @@ Status ExecutorState::ProcessOutputs(const NodeItem& item, OpKernelContext* ctx, for (int i = 0; i < item.num_outputs; ++i) { const TensorValue val = ctx->release_output(i); - if (*ctx->is_output_dead() || val.tensor == nullptr) { + if (val.tensor == nullptr) { // Unless it's a Switch or a Recv, the node must produce a // tensor value at i-th output. if (!IsSwitch(node) && !IsRecv(node)) { diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h index 6c4c3a2ac1..d9fe42fcbb 100644 --- a/tensorflow/core/framework/op_kernel.h +++ b/tensorflow/core/framework/op_kernel.h @@ -1044,7 +1044,6 @@ class OpKernelContext { // For control flow. FrameAndIter frame_iter() const { return params_->frame_iter; } bool is_input_dead() const { return params_->is_input_dead; } - bool* is_output_dead() { return &is_output_dead_; } // May be used, e.g., to get GPU handles, etc. // TODO(tucker): Add example usage. @@ -1143,8 +1142,6 @@ class OpKernelContext { // Constructed only if record_tensor_accesses>. ManualConstructor referenced_tensors_ GUARDED_BY(mu_); - bool is_output_dead_ = false; - // The following data members are only used when allocation tracking is // enabled. mutable mutex stats_mu_; diff --git a/tensorflow/core/kernels/sendrecv_ops.cc b/tensorflow/core/kernels/sendrecv_ops.cc index 2f87057f4e..6521dcf932 100644 --- a/tensorflow/core/kernels/sendrecv_ops.cc +++ b/tensorflow/core/kernels/sendrecv_ops.cc @@ -160,7 +160,6 @@ Rendezvous::DoneCallback make_recv_callback(OpKernelContext* ctx, if (!is_dead) { ctx->set_output(0, val); } - *ctx->is_output_dead() = is_dead; } done(); }, -- cgit v1.2.3 From f8044c89287b1d90510ceace4b53ec94abaffa50 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 10:50:42 -0700 Subject: Automated rollback of commit 0ea6847c892497afdd20c1150fee1e532612ca17 PiperOrigin-RevId: 204326206 --- tensorflow/compiler/jit/xla_compilation_cache.cc | 18 +++- tensorflow/compiler/jit/xla_device_context.cc | 117 ++++++++++++++------- tensorflow/compiler/jit/xla_device_context.h | 5 +- tensorflow/compiler/jit/xla_tensor.cc | 4 +- tensorflow/compiler/xla/service/executable.cc | 13 ++- tensorflow/compiler/xla/service/hlo_runner.cc | 9 +- .../xla/tests/local_client_execute_test.cc | 4 + .../compiler/xla/tests/local_client_test_base.cc | 14 ++- .../compiler/xla/tests/xla_hlo_profile_test.cc | 1 + .../stream_executor/host/host_gpu_executor.cc | 2 +- tensorflow/stream_executor/stream.cc | 6 ++ 11 files changed, 143 insertions(+), 50 deletions(-) diff --git a/tensorflow/compiler/jit/xla_compilation_cache.cc b/tensorflow/compiler/jit/xla_compilation_cache.cc index 7ed609c437..54a41a4daa 100644 --- a/tensorflow/compiler/jit/xla_compilation_cache.cc +++ b/tensorflow/compiler/jit/xla_compilation_cache.cc @@ -40,7 +40,23 @@ namespace tensorflow { XlaCompilationCache::XlaCompilationCache(xla::LocalClient* client, DeviceType device_type) : client_(client), device_type_(std::move(device_type)) {} -XlaCompilationCache::~XlaCompilationCache() = default; +XlaCompilationCache::~XlaCompilationCache() { + // Ensure any use of our programs have completed by waiting for all stream + // executors to complete. + for (auto* executor : client_->backend().stream_executors()) { + bool ok = executor->SynchronizeAllActivity(); + if (!ok) { + LOG(ERROR) << "Error synchronizing activity while waiting for all " + "programs to complete"; + } + } + // TODO(b/110813685): Think about the program ownership model. Programs are + // currently owned by the compilation cache which means we must wait for + // program completion in the destructor. There are multiple compilation caches + // around, which complicates things a little. Perhaps having programs be + // shared_ptrs (an invasive change) would make the model easier to reason + // about? +} string XlaCompilationCache::DebugString() { return "XLA JIT compilation cache"; diff --git a/tensorflow/compiler/jit/xla_device_context.cc b/tensorflow/compiler/jit/xla_device_context.cc index 04778c0090..8cf198239c 100644 --- a/tensorflow/compiler/jit/xla_device_context.cc +++ b/tensorflow/compiler/jit/xla_device_context.cc @@ -74,43 +74,64 @@ Status XlaTransferManager::TransferLiteralToDevice( xla::Shape xla_shape; TF_RETURN_IF_ERROR(TensorShapeToXLAShape(host_tensor.dtype(), host_tensor.shape(), &xla_shape)); - xla::BorrowingLiteral literal( + // Create a reference to hold onto host_tensor until after the literal has + // been transferred. Also make sure the literal exists until the function + // asynchronously completes, as it will be wrapped in an xla::LiteralSlice. + TensorReference ref(host_tensor); + auto literal = std::make_shared( static_cast(DMAHelper::base(&host_tensor)), xla_shape); XlaTensor* xla_tensor = XlaTensor::FromTensor(device_tensor); const xla::ShapedBuffer& shaped_buffer = xla_tensor->shaped_buffer(); - VLOG(1) << "Transfer to device as literal: " << literal.ToString() << " " + VLOG(1) << "Transfer to device as literal: " << literal->ToString() << " " << shaped_buffer.ToString(); - TF_RETURN_IF_ERROR(transfer_manager_->TransferLiteralToDevice( - host_to_device_stream_, literal, shaped_buffer)); + if (UseMultipleStreams()) { + // Initially wait for the compute stream so that memory allocations are + // synchronized. + host_to_device_stream_->ThenWaitFor(stream_); + } + TF_RETURN_IF_ERROR(transfer_manager_->TransferLiteralToDeviceAsync( + host_to_device_stream_, *literal, shaped_buffer)); if (UseMultipleStreams()) { se::Event event(stream_->parent()); TF_RET_CHECK(event.Init()) << "Event failed to initialize!"; host_to_device_stream_->ThenRecordEvent(&event); xla_tensor->SetDefinedOn(host_to_device_stream_, std::move(event)); } + // Unref the host tensor, and capture the literal shared_ptr too so it goes + // out of scope when the lambda completes. + host_to_device_stream_->ThenDoHostCallback([ref, literal]() { ref.Unref(); }); return Status::OK(); } -Status XlaTransferManager::TransferLiteralFromDevice( - Tensor* host_tensor, const Tensor& device_tensor) const { +void XlaTransferManager::TransferLiteralFromDevice( + Tensor* host_tensor, const Tensor& device_tensor, + const StatusCallback& done) const { const xla::ShapedBuffer& shaped_buffer = XlaTensor::FromTensor(&device_tensor)->shaped_buffer(); - TF_ASSIGN_OR_RETURN(std::unique_ptr literal, - transfer_manager_->TransferLiteralFromDevice( - device_to_host_stream_, shaped_buffer)); - VLOG(1) << "Transfer from device as literal: " << literal->ToString() << " " - << shaped_buffer.ToString(); - Tensor tensor; - TF_RETURN_IF_ERROR( - LiteralToHostTensor(*literal, host_tensor->dtype(), &tensor)); - // Reshape the tensor back to its declared shape. - if (!host_tensor->CopyFrom(tensor, device_tensor.shape())) { - return errors::Internal( - "Tensor::CopyFrom failed when copying from XLA device to CPU"); - } - return Status::OK(); + TensorReference ref(device_tensor); + transfer_manager_->TransferLiteralFromDevice( + device_to_host_stream_, shaped_buffer, + [=, &shaped_buffer]( + xla::StatusOr > literal_or) { + ref.Unref(); + done([&]() -> Status { + TF_ASSIGN_OR_RETURN(auto literal, std::move(literal_or)); + VLOG(1) << "Transfer from device as literal: " << literal->ToString() + << " " << shaped_buffer.ToString(); + Tensor tensor; + TF_RETURN_IF_ERROR( + LiteralToHostTensor(*literal, host_tensor->dtype(), &tensor)); + // Reshape the tensor back to its declared shape. + Status status; + if (!host_tensor->CopyFrom(tensor, device_tensor.shape())) { + status = errors::Internal( + "Tensor::CopyFrom failed when copying from XLA device to CPU"); + } + return status; + }()); + }); } void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor, @@ -163,6 +184,12 @@ void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor, return; } status = TransferLiteralToDevice(reshaped_cpu_tensor, device_tensor); + if (status.ok()) { + xla_tensor->set_host_tensor(*cpu_tensor); + host_to_device_stream_->ThenDoHostCallback( + [done]() { done(Status::OK()); }); + return; + } } else { se::DeviceMemoryBase dev_dst_ptr = XlaTensor::DeviceMemoryFromTensor(*device_tensor); @@ -212,7 +239,8 @@ void XlaTransferManager::CopyDeviceTensorToCPU(const Tensor* device_tensor, Status status; if (transfer_as_literal_) { - status = TransferLiteralFromDevice(cpu_tensor, *device_tensor); + TransferLiteralFromDevice(cpu_tensor, *device_tensor, done); + return; } else { device_to_host_stream_->ThenMemcpy(dst_ptr, dev_src_ptr, total_bytes); // TODO(hpucha): Make this asynchronous. @@ -234,15 +262,15 @@ void XlaTransferManager::CopyDeviceTensorToDevice(const Tensor& src_tensor, << reinterpret_cast(src_tensor.tensor_data().data()) << " " << reinterpret_cast(dst_tensor->tensor_data().data()); - // TODO(phawkins): replace this code with an asynchronous implementation. - auto body = [&]() { + // Perform memory allocation now, and enqueue the device-to-device transfer. + Status status = [&]() -> Status { if (src_tensor.NumElements() == 0) { return Status::OK(); } // TODO(jmolloy): We co-opt the device_to_host stream for device to device // transfers; perhaps we should have a dedicated device to device stream? or // one per device? - auto device_to_device_stream = device_to_host_stream_; + auto device_to_device_stream = stream_; XlaTensor* xla_src = XlaTensor::FromTensor(&src_tensor); XlaTensor* xla_dst = XlaTensor::FromTensor(dst_tensor); CHECK(xla_src && xla_dst) @@ -254,29 +282,40 @@ void XlaTransferManager::CopyDeviceTensorToDevice(const Tensor& src_tensor, TF_RETURN_IF_ERROR( xla_dst->AllocateShapedBuffer(src_tensor.dtype(), shape, client_, stream_->parent()->device_ordinal())); + if (stream_ != device_to_device_stream) { + // Initially wait for the compute stream so that memory allocations are + // synchronized. + device_to_device_stream->ThenWaitFor(stream_); + } } if (se::Event* event = xla_src->GetDefinitionEvent(device_to_device_stream)) { device_to_device_stream->ThenWaitFor(event); xla_src->SetDefinedOn(device_to_device_stream); - TF_RETURN_IF_ERROR(device_to_device_stream->BlockHostUntilDone()); } - TF_RETURN_IF_ERROR( - xla_dst->shaped_buffer().buffers().ForEachMutableElementWithStatus( - [&](const xla::ShapeIndex& index, se::DeviceMemoryBase* buffer) { - const se::DeviceMemoryBase& from_buffer = - xla_src->shaped_buffer().buffers().element(index); - CHECK_EQ(buffer->size(), from_buffer.size()); - if (!stream_->parent()->SynchronousMemcpy(buffer, from_buffer, - buffer->size())) { - return errors::Internal("Device to device memcpy failed"); - } - return Status::OK(); - })); + + auto from_iter = xla_src->shaped_buffer().buffers().begin(); + auto to_iter = xla_dst->shaped_buffer().buffers().begin(); + for (auto end_iter = xla_src->shaped_buffer().buffers().end(); + from_iter != end_iter; ++from_iter, ++to_iter) { + device_to_device_stream->ThenMemcpyD2D( + &to_iter->second, from_iter->second, to_iter->second.size()); + } + + if (UseMultipleStreams()) { + se::Event event(stream_->parent()); + CHECK(event.Init()); + device_to_device_stream->ThenRecordEvent(&event); + xla_dst->SetDefinedOn(device_to_device_stream, std::move(event)); + } return Status::OK(); - }; - done(body()); + }(); + if (!status.ok()) { + return done(status); + } else { + stream_->ThenDoHostCallback([=]() { done(Status::OK()); }); + } } XlaDeviceContext::XlaDeviceContext( diff --git a/tensorflow/compiler/jit/xla_device_context.h b/tensorflow/compiler/jit/xla_device_context.h index c726495f96..912f8d779e 100644 --- a/tensorflow/compiler/jit/xla_device_context.h +++ b/tensorflow/compiler/jit/xla_device_context.h @@ -66,8 +66,9 @@ class XlaTransferManager { private: Status TransferLiteralToDevice(const Tensor& host_tensor, Tensor* device_tensor) const; - Status TransferLiteralFromDevice(Tensor* host_tensor, - const Tensor& device_tensor) const; + void TransferLiteralFromDevice(Tensor* host_tensor, + const Tensor& device_tensor, + const StatusCallback& done) const; bool UseMultipleStreams() const { return stream_ != host_to_device_stream_; } // The main compute stream of the device, used to synchronize the transfer diff --git a/tensorflow/compiler/jit/xla_tensor.cc b/tensorflow/compiler/jit/xla_tensor.cc index 5dff187fff..d777dfa5a3 100644 --- a/tensorflow/compiler/jit/xla_tensor.cc +++ b/tensorflow/compiler/jit/xla_tensor.cc @@ -92,10 +92,8 @@ se::Event* XlaTensor::GetDefinitionEvent(se::Stream* stream) { void XlaTensor::SetDefinedOn(se::Stream* stream, se::Event event) { mutex_lock lock(mu_); - CHECK(!definition_event_.has_value()) - << "SetDefinedOn must only be called once!"; definition_event_ = std::move(event); - streams_defined_on_.push_back(stream); + streams_defined_on_ = {stream}; } void XlaTensor::SetDefinedOn(se::Stream* stream) { diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc index 7cf2746947..fd75847d0c 100644 --- a/tensorflow/compiler/xla/service/executable.cc +++ b/tensorflow/compiler/xla/service/executable.cc @@ -82,7 +82,18 @@ StatusOr Executable::ExecuteOnStreamWrapper( StatusOr return_value = ExecuteOnStream(run_options, arguments, profile_ptr.get()); - TF_RETURN_IF_ERROR(return_value.status()); + if (!return_value.status().ok()) { + if (profile != nullptr) { + // Ensure the ThenStartTimer call has completed before we destroy timer. + // We already have a failure status to return, so just log this if it + // fails. + Status status = stream->BlockHostUntilDone(); + if (!status.ok()) { + LOG(ERROR) << "Failed to BlockHostUntilDone: " << status; + } + } + return return_value.status(); + } if (profile != nullptr) { VLOG(1) << "enqueueing 'stop timer' and blocking host until done..."; diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc index 4f0569f405..b2725e2918 100644 --- a/tensorflow/compiler/xla/service/hlo_runner.cc +++ b/tensorflow/compiler/xla/service/hlo_runner.cc @@ -180,8 +180,12 @@ StatusOr HloRunner::ExecuteWithDeviceBuffers( TF_ASSIGN_OR_RETURN(std::unique_ptr executable, CreateExecutable(std::move(module), run_hlo_passes)); - return executable->ExecuteOnStreamWrapper(&service_run_options, - /*profile=*/profile, arguments); + TF_ASSIGN_OR_RETURN( + ScopedShapedBuffer retval, + executable->ExecuteOnStreamWrapper(&service_run_options, + /*profile=*/profile, arguments)); + TF_RETURN_IF_ERROR(stream.BlockHostUntilDone()); + return std::move(retval); } StatusOr HloRunner::ExecuteWithDeviceBuffers( @@ -309,6 +313,7 @@ StatusOr>> HloRunner::ExecuteReplicated( std::vector> exec_results; for (int64 i = 0; i < options.num_replicas; ++i) { + TF_RETURN_IF_ERROR(streams[i]->BlockHostUntilDone()); TF_ASSIGN_OR_RETURN(std::unique_ptr literal, backend().transfer_manager()->TransferLiteralFromDevice( streams[i].get(), results[i])); diff --git a/tensorflow/compiler/xla/tests/local_client_execute_test.cc b/tensorflow/compiler/xla/tests/local_client_execute_test.cc index 2f4d197ae6..5c3498c84c 100644 --- a/tensorflow/compiler/xla/tests/local_client_execute_test.cc +++ b/tensorflow/compiler/xla/tests/local_client_execute_test.cc @@ -772,6 +772,10 @@ XLA_TEST_F(LocalClientExecuteTest, CompileExecutable) { ScopedShapedBuffer result = executable->Run({&x_array}, DefaultExecutableRunOptions()) .ConsumeValueOrDie(); + ASSERT_IS_OK(local_client_->mutable_backend() + ->BorrowStream(0) + .ValueOrDie() + ->BlockHostUntilDone()); LiteralTestUtil::ExpectR1Near( {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(result), error_spec_); diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.cc b/tensorflow/compiler/xla/tests/local_client_test_base.cc index 88797a7d0a..c31ba0e713 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.cc +++ b/tensorflow/compiler/xla/tests/local_client_test_base.cc @@ -189,7 +189,19 @@ StatusOr LocalClientTestBase::ExecuteLocally( TF_ASSIGN_OR_RETURN( std::unique_ptr executable, local_client_->Compile(computation, argument_layouts, build_options)); - return executable->Run(arguments, run_options); + TF_ASSIGN_OR_RETURN(auto ret, executable->Run(arguments, run_options)); + + auto device_ordinal = + build_options.device_ordinal() == -1 ? 0 : build_options.device_ordinal(); + auto* stream = run_options.stream(); + if (!stream) { + stream = local_client_->mutable_backend() + ->BorrowStream(device_ordinal) + .ValueOrDie() + .get(); + } + TF_RETURN_IF_ERROR(stream->BlockHostUntilDone()); + return std::move(ret); } } // namespace xla diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc index 4d4dd62a3f..c000ff4dc8 100644 --- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc +++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc @@ -172,6 +172,7 @@ void ExecuteAndFetchProfile(string* profile_output, LocalClient* client, auto execution_result, executable->ExecuteOnStream(&run_options, {&lhs_arg, &rhs_arg}, &hlo_execution_profile)); + TF_ASSERT_OK(stream_ptr->BlockHostUntilDone()); (void)execution_result; *profile_output = diff --git a/tensorflow/stream_executor/host/host_gpu_executor.cc b/tensorflow/stream_executor/host/host_gpu_executor.cc index 3cd97b3cf1..8adf739b17 100644 --- a/tensorflow/stream_executor/host/host_gpu_executor.cc +++ b/tensorflow/stream_executor/host/host_gpu_executor.cc @@ -93,7 +93,7 @@ bool HostExecutor::MemcpyDeviceToDevice(Stream *stream, // the nature of the HostExecutor) memcpy on the stream (HostStream) // associated with the HostExecutor. AsHostStream(stream)->EnqueueTask( - [src_mem, dst_mem, size]() { memcpy(src_mem, dst_mem, size); }); + [src_mem, dst_mem, size]() { memcpy(dst_mem, src_mem, size); }); return true; } diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc index 9369183133..b3ed7e4452 100644 --- a/tensorflow/stream_executor/stream.cc +++ b/tensorflow/stream_executor/stream.cc @@ -268,6 +268,12 @@ Stream::~Stream() { VLOG_CALL(); temporary_memory_manager_.ForceDeallocateAll(); + // Ensure the stream is completed. + auto status = BlockHostUntilDone(); + if (!status.ok()) { + LOG(WARNING) << "Error blocking host until done in stream destructor: " + << status; + } if (allocated_) { parent_->DeallocateStream(this); -- cgit v1.2.3 From f8ce8dc04f569b3bbd1ac7cc9e358a651530990d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 10:57:48 -0700 Subject: Automated rollback of commit f8044c89287b1d90510ceace4b53ec94abaffa50 PiperOrigin-RevId: 204327453 --- tensorflow/compiler/jit/xla_compilation_cache.cc | 18 +--- tensorflow/compiler/jit/xla_device_context.cc | 117 +++++++-------------- tensorflow/compiler/jit/xla_device_context.h | 5 +- tensorflow/compiler/jit/xla_tensor.cc | 4 +- tensorflow/compiler/xla/service/executable.cc | 13 +-- tensorflow/compiler/xla/service/hlo_runner.cc | 9 +- .../xla/tests/local_client_execute_test.cc | 4 - .../compiler/xla/tests/local_client_test_base.cc | 14 +-- .../compiler/xla/tests/xla_hlo_profile_test.cc | 1 - .../stream_executor/host/host_gpu_executor.cc | 2 +- tensorflow/stream_executor/stream.cc | 6 -- 11 files changed, 50 insertions(+), 143 deletions(-) diff --git a/tensorflow/compiler/jit/xla_compilation_cache.cc b/tensorflow/compiler/jit/xla_compilation_cache.cc index 54a41a4daa..7ed609c437 100644 --- a/tensorflow/compiler/jit/xla_compilation_cache.cc +++ b/tensorflow/compiler/jit/xla_compilation_cache.cc @@ -40,23 +40,7 @@ namespace tensorflow { XlaCompilationCache::XlaCompilationCache(xla::LocalClient* client, DeviceType device_type) : client_(client), device_type_(std::move(device_type)) {} -XlaCompilationCache::~XlaCompilationCache() { - // Ensure any use of our programs have completed by waiting for all stream - // executors to complete. - for (auto* executor : client_->backend().stream_executors()) { - bool ok = executor->SynchronizeAllActivity(); - if (!ok) { - LOG(ERROR) << "Error synchronizing activity while waiting for all " - "programs to complete"; - } - } - // TODO(b/110813685): Think about the program ownership model. Programs are - // currently owned by the compilation cache which means we must wait for - // program completion in the destructor. There are multiple compilation caches - // around, which complicates things a little. Perhaps having programs be - // shared_ptrs (an invasive change) would make the model easier to reason - // about? -} +XlaCompilationCache::~XlaCompilationCache() = default; string XlaCompilationCache::DebugString() { return "XLA JIT compilation cache"; diff --git a/tensorflow/compiler/jit/xla_device_context.cc b/tensorflow/compiler/jit/xla_device_context.cc index 8cf198239c..04778c0090 100644 --- a/tensorflow/compiler/jit/xla_device_context.cc +++ b/tensorflow/compiler/jit/xla_device_context.cc @@ -74,64 +74,43 @@ Status XlaTransferManager::TransferLiteralToDevice( xla::Shape xla_shape; TF_RETURN_IF_ERROR(TensorShapeToXLAShape(host_tensor.dtype(), host_tensor.shape(), &xla_shape)); - // Create a reference to hold onto host_tensor until after the literal has - // been transferred. Also make sure the literal exists until the function - // asynchronously completes, as it will be wrapped in an xla::LiteralSlice. - TensorReference ref(host_tensor); - auto literal = std::make_shared( + xla::BorrowingLiteral literal( static_cast(DMAHelper::base(&host_tensor)), xla_shape); XlaTensor* xla_tensor = XlaTensor::FromTensor(device_tensor); const xla::ShapedBuffer& shaped_buffer = xla_tensor->shaped_buffer(); - VLOG(1) << "Transfer to device as literal: " << literal->ToString() << " " + VLOG(1) << "Transfer to device as literal: " << literal.ToString() << " " << shaped_buffer.ToString(); - if (UseMultipleStreams()) { - // Initially wait for the compute stream so that memory allocations are - // synchronized. - host_to_device_stream_->ThenWaitFor(stream_); - } - TF_RETURN_IF_ERROR(transfer_manager_->TransferLiteralToDeviceAsync( - host_to_device_stream_, *literal, shaped_buffer)); + TF_RETURN_IF_ERROR(transfer_manager_->TransferLiteralToDevice( + host_to_device_stream_, literal, shaped_buffer)); if (UseMultipleStreams()) { se::Event event(stream_->parent()); TF_RET_CHECK(event.Init()) << "Event failed to initialize!"; host_to_device_stream_->ThenRecordEvent(&event); xla_tensor->SetDefinedOn(host_to_device_stream_, std::move(event)); } - // Unref the host tensor, and capture the literal shared_ptr too so it goes - // out of scope when the lambda completes. - host_to_device_stream_->ThenDoHostCallback([ref, literal]() { ref.Unref(); }); return Status::OK(); } -void XlaTransferManager::TransferLiteralFromDevice( - Tensor* host_tensor, const Tensor& device_tensor, - const StatusCallback& done) const { +Status XlaTransferManager::TransferLiteralFromDevice( + Tensor* host_tensor, const Tensor& device_tensor) const { const xla::ShapedBuffer& shaped_buffer = XlaTensor::FromTensor(&device_tensor)->shaped_buffer(); - TensorReference ref(device_tensor); - transfer_manager_->TransferLiteralFromDevice( - device_to_host_stream_, shaped_buffer, - [=, &shaped_buffer]( - xla::StatusOr > literal_or) { - ref.Unref(); - done([&]() -> Status { - TF_ASSIGN_OR_RETURN(auto literal, std::move(literal_or)); - VLOG(1) << "Transfer from device as literal: " << literal->ToString() - << " " << shaped_buffer.ToString(); - Tensor tensor; - TF_RETURN_IF_ERROR( - LiteralToHostTensor(*literal, host_tensor->dtype(), &tensor)); - // Reshape the tensor back to its declared shape. - Status status; - if (!host_tensor->CopyFrom(tensor, device_tensor.shape())) { - status = errors::Internal( - "Tensor::CopyFrom failed when copying from XLA device to CPU"); - } - return status; - }()); - }); + TF_ASSIGN_OR_RETURN(std::unique_ptr literal, + transfer_manager_->TransferLiteralFromDevice( + device_to_host_stream_, shaped_buffer)); + VLOG(1) << "Transfer from device as literal: " << literal->ToString() << " " + << shaped_buffer.ToString(); + Tensor tensor; + TF_RETURN_IF_ERROR( + LiteralToHostTensor(*literal, host_tensor->dtype(), &tensor)); + // Reshape the tensor back to its declared shape. + if (!host_tensor->CopyFrom(tensor, device_tensor.shape())) { + return errors::Internal( + "Tensor::CopyFrom failed when copying from XLA device to CPU"); + } + return Status::OK(); } void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor, @@ -184,12 +163,6 @@ void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor, return; } status = TransferLiteralToDevice(reshaped_cpu_tensor, device_tensor); - if (status.ok()) { - xla_tensor->set_host_tensor(*cpu_tensor); - host_to_device_stream_->ThenDoHostCallback( - [done]() { done(Status::OK()); }); - return; - } } else { se::DeviceMemoryBase dev_dst_ptr = XlaTensor::DeviceMemoryFromTensor(*device_tensor); @@ -239,8 +212,7 @@ void XlaTransferManager::CopyDeviceTensorToCPU(const Tensor* device_tensor, Status status; if (transfer_as_literal_) { - TransferLiteralFromDevice(cpu_tensor, *device_tensor, done); - return; + status = TransferLiteralFromDevice(cpu_tensor, *device_tensor); } else { device_to_host_stream_->ThenMemcpy(dst_ptr, dev_src_ptr, total_bytes); // TODO(hpucha): Make this asynchronous. @@ -262,15 +234,15 @@ void XlaTransferManager::CopyDeviceTensorToDevice(const Tensor& src_tensor, << reinterpret_cast(src_tensor.tensor_data().data()) << " " << reinterpret_cast(dst_tensor->tensor_data().data()); - // Perform memory allocation now, and enqueue the device-to-device transfer. - Status status = [&]() -> Status { + // TODO(phawkins): replace this code with an asynchronous implementation. + auto body = [&]() { if (src_tensor.NumElements() == 0) { return Status::OK(); } // TODO(jmolloy): We co-opt the device_to_host stream for device to device // transfers; perhaps we should have a dedicated device to device stream? or // one per device? - auto device_to_device_stream = stream_; + auto device_to_device_stream = device_to_host_stream_; XlaTensor* xla_src = XlaTensor::FromTensor(&src_tensor); XlaTensor* xla_dst = XlaTensor::FromTensor(dst_tensor); CHECK(xla_src && xla_dst) @@ -282,40 +254,29 @@ void XlaTransferManager::CopyDeviceTensorToDevice(const Tensor& src_tensor, TF_RETURN_IF_ERROR( xla_dst->AllocateShapedBuffer(src_tensor.dtype(), shape, client_, stream_->parent()->device_ordinal())); - if (stream_ != device_to_device_stream) { - // Initially wait for the compute stream so that memory allocations are - // synchronized. - device_to_device_stream->ThenWaitFor(stream_); - } } if (se::Event* event = xla_src->GetDefinitionEvent(device_to_device_stream)) { device_to_device_stream->ThenWaitFor(event); xla_src->SetDefinedOn(device_to_device_stream); + TF_RETURN_IF_ERROR(device_to_device_stream->BlockHostUntilDone()); } - - auto from_iter = xla_src->shaped_buffer().buffers().begin(); - auto to_iter = xla_dst->shaped_buffer().buffers().begin(); - for (auto end_iter = xla_src->shaped_buffer().buffers().end(); - from_iter != end_iter; ++from_iter, ++to_iter) { - device_to_device_stream->ThenMemcpyD2D( - &to_iter->second, from_iter->second, to_iter->second.size()); - } - - if (UseMultipleStreams()) { - se::Event event(stream_->parent()); - CHECK(event.Init()); - device_to_device_stream->ThenRecordEvent(&event); - xla_dst->SetDefinedOn(device_to_device_stream, std::move(event)); - } + TF_RETURN_IF_ERROR( + xla_dst->shaped_buffer().buffers().ForEachMutableElementWithStatus( + [&](const xla::ShapeIndex& index, se::DeviceMemoryBase* buffer) { + const se::DeviceMemoryBase& from_buffer = + xla_src->shaped_buffer().buffers().element(index); + CHECK_EQ(buffer->size(), from_buffer.size()); + if (!stream_->parent()->SynchronousMemcpy(buffer, from_buffer, + buffer->size())) { + return errors::Internal("Device to device memcpy failed"); + } + return Status::OK(); + })); return Status::OK(); - }(); - if (!status.ok()) { - return done(status); - } else { - stream_->ThenDoHostCallback([=]() { done(Status::OK()); }); - } + }; + done(body()); } XlaDeviceContext::XlaDeviceContext( diff --git a/tensorflow/compiler/jit/xla_device_context.h b/tensorflow/compiler/jit/xla_device_context.h index 912f8d779e..c726495f96 100644 --- a/tensorflow/compiler/jit/xla_device_context.h +++ b/tensorflow/compiler/jit/xla_device_context.h @@ -66,9 +66,8 @@ class XlaTransferManager { private: Status TransferLiteralToDevice(const Tensor& host_tensor, Tensor* device_tensor) const; - void TransferLiteralFromDevice(Tensor* host_tensor, - const Tensor& device_tensor, - const StatusCallback& done) const; + Status TransferLiteralFromDevice(Tensor* host_tensor, + const Tensor& device_tensor) const; bool UseMultipleStreams() const { return stream_ != host_to_device_stream_; } // The main compute stream of the device, used to synchronize the transfer diff --git a/tensorflow/compiler/jit/xla_tensor.cc b/tensorflow/compiler/jit/xla_tensor.cc index d777dfa5a3..5dff187fff 100644 --- a/tensorflow/compiler/jit/xla_tensor.cc +++ b/tensorflow/compiler/jit/xla_tensor.cc @@ -92,8 +92,10 @@ se::Event* XlaTensor::GetDefinitionEvent(se::Stream* stream) { void XlaTensor::SetDefinedOn(se::Stream* stream, se::Event event) { mutex_lock lock(mu_); + CHECK(!definition_event_.has_value()) + << "SetDefinedOn must only be called once!"; definition_event_ = std::move(event); - streams_defined_on_ = {stream}; + streams_defined_on_.push_back(stream); } void XlaTensor::SetDefinedOn(se::Stream* stream) { diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc index fd75847d0c..7cf2746947 100644 --- a/tensorflow/compiler/xla/service/executable.cc +++ b/tensorflow/compiler/xla/service/executable.cc @@ -82,18 +82,7 @@ StatusOr Executable::ExecuteOnStreamWrapper( StatusOr return_value = ExecuteOnStream(run_options, arguments, profile_ptr.get()); - if (!return_value.status().ok()) { - if (profile != nullptr) { - // Ensure the ThenStartTimer call has completed before we destroy timer. - // We already have a failure status to return, so just log this if it - // fails. - Status status = stream->BlockHostUntilDone(); - if (!status.ok()) { - LOG(ERROR) << "Failed to BlockHostUntilDone: " << status; - } - } - return return_value.status(); - } + TF_RETURN_IF_ERROR(return_value.status()); if (profile != nullptr) { VLOG(1) << "enqueueing 'stop timer' and blocking host until done..."; diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc index b2725e2918..4f0569f405 100644 --- a/tensorflow/compiler/xla/service/hlo_runner.cc +++ b/tensorflow/compiler/xla/service/hlo_runner.cc @@ -180,12 +180,8 @@ StatusOr HloRunner::ExecuteWithDeviceBuffers( TF_ASSIGN_OR_RETURN(std::unique_ptr executable, CreateExecutable(std::move(module), run_hlo_passes)); - TF_ASSIGN_OR_RETURN( - ScopedShapedBuffer retval, - executable->ExecuteOnStreamWrapper(&service_run_options, - /*profile=*/profile, arguments)); - TF_RETURN_IF_ERROR(stream.BlockHostUntilDone()); - return std::move(retval); + return executable->ExecuteOnStreamWrapper(&service_run_options, + /*profile=*/profile, arguments); } StatusOr HloRunner::ExecuteWithDeviceBuffers( @@ -313,7 +309,6 @@ StatusOr>> HloRunner::ExecuteReplicated( std::vector> exec_results; for (int64 i = 0; i < options.num_replicas; ++i) { - TF_RETURN_IF_ERROR(streams[i]->BlockHostUntilDone()); TF_ASSIGN_OR_RETURN(std::unique_ptr literal, backend().transfer_manager()->TransferLiteralFromDevice( streams[i].get(), results[i])); diff --git a/tensorflow/compiler/xla/tests/local_client_execute_test.cc b/tensorflow/compiler/xla/tests/local_client_execute_test.cc index 5c3498c84c..2f4d197ae6 100644 --- a/tensorflow/compiler/xla/tests/local_client_execute_test.cc +++ b/tensorflow/compiler/xla/tests/local_client_execute_test.cc @@ -772,10 +772,6 @@ XLA_TEST_F(LocalClientExecuteTest, CompileExecutable) { ScopedShapedBuffer result = executable->Run({&x_array}, DefaultExecutableRunOptions()) .ConsumeValueOrDie(); - ASSERT_IS_OK(local_client_->mutable_backend() - ->BorrowStream(0) - .ValueOrDie() - ->BlockHostUntilDone()); LiteralTestUtil::ExpectR1Near( {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(result), error_spec_); diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.cc b/tensorflow/compiler/xla/tests/local_client_test_base.cc index c31ba0e713..88797a7d0a 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.cc +++ b/tensorflow/compiler/xla/tests/local_client_test_base.cc @@ -189,19 +189,7 @@ StatusOr LocalClientTestBase::ExecuteLocally( TF_ASSIGN_OR_RETURN( std::unique_ptr executable, local_client_->Compile(computation, argument_layouts, build_options)); - TF_ASSIGN_OR_RETURN(auto ret, executable->Run(arguments, run_options)); - - auto device_ordinal = - build_options.device_ordinal() == -1 ? 0 : build_options.device_ordinal(); - auto* stream = run_options.stream(); - if (!stream) { - stream = local_client_->mutable_backend() - ->BorrowStream(device_ordinal) - .ValueOrDie() - .get(); - } - TF_RETURN_IF_ERROR(stream->BlockHostUntilDone()); - return std::move(ret); + return executable->Run(arguments, run_options); } } // namespace xla diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc index c000ff4dc8..4d4dd62a3f 100644 --- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc +++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc @@ -172,7 +172,6 @@ void ExecuteAndFetchProfile(string* profile_output, LocalClient* client, auto execution_result, executable->ExecuteOnStream(&run_options, {&lhs_arg, &rhs_arg}, &hlo_execution_profile)); - TF_ASSERT_OK(stream_ptr->BlockHostUntilDone()); (void)execution_result; *profile_output = diff --git a/tensorflow/stream_executor/host/host_gpu_executor.cc b/tensorflow/stream_executor/host/host_gpu_executor.cc index 8adf739b17..3cd97b3cf1 100644 --- a/tensorflow/stream_executor/host/host_gpu_executor.cc +++ b/tensorflow/stream_executor/host/host_gpu_executor.cc @@ -93,7 +93,7 @@ bool HostExecutor::MemcpyDeviceToDevice(Stream *stream, // the nature of the HostExecutor) memcpy on the stream (HostStream) // associated with the HostExecutor. AsHostStream(stream)->EnqueueTask( - [src_mem, dst_mem, size]() { memcpy(dst_mem, src_mem, size); }); + [src_mem, dst_mem, size]() { memcpy(src_mem, dst_mem, size); }); return true; } diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc index b3ed7e4452..9369183133 100644 --- a/tensorflow/stream_executor/stream.cc +++ b/tensorflow/stream_executor/stream.cc @@ -268,12 +268,6 @@ Stream::~Stream() { VLOG_CALL(); temporary_memory_manager_.ForceDeallocateAll(); - // Ensure the stream is completed. - auto status = BlockHostUntilDone(); - if (!status.ok()) { - LOG(WARNING) << "Error blocking host until done in stream destructor: " - << status; - } if (allocated_) { parent_->DeallocateStream(this); -- cgit v1.2.3 From d843a4eee58314320354cbd53e80a53a78449da6 Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Thu, 12 Jul 2018 11:06:21 -0700 Subject: Register DestroyResourceOp for XLA devices Before this change, we were not releasing device memory allocated by ResourceVariables. PiperOrigin-RevId: 204329027 --- tensorflow/compiler/jit/xla_device_ops.h | 3 +++ tensorflow/compiler/tests/BUILD | 2 +- tensorflow/compiler/tests/eager_test.py | 11 +++++++++ tensorflow/core/kernels/resource_variable_ops.cc | 29 ++++++++++-------------- tensorflow/core/kernels/resource_variable_ops.h | 9 ++++++++ 5 files changed, 36 insertions(+), 18 deletions(-) diff --git a/tensorflow/compiler/jit/xla_device_ops.h b/tensorflow/compiler/jit/xla_device_ops.h index a605335a94..134dcc1bb5 100644 --- a/tensorflow/compiler/jit/xla_device_ops.h +++ b/tensorflow/compiler/jit/xla_device_ops.h @@ -90,6 +90,9 @@ class XlaAssignVariableOp : public AsyncOpKernel { REGISTER_KERNEL_BUILDER( \ Name("ReadVariableOp").Device(DEVICE).HostMemory("resource"), \ ReadVariableOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("DestroyResourceOp").Device(DEVICE).HostMemory("resource"), \ + DestroyResourceOp); \ REGISTER_KERNEL_BUILDER(Name("Shape") \ .Device(DEVICE) \ .HostMemory("output") \ diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index e8e19f055e..080bed50e6 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -418,7 +418,7 @@ tf_xla_py_test( tf_xla_py_test( name = "eager_test", - size = "small", + size = "large", srcs = ["eager_test.py"], disabled_backends = [ # TODO(b/78199195) Support XLA CPU devices in eager runtime diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py index 3524666499..8a3ed382a1 100644 --- a/tensorflow/compiler/tests/eager_test.py +++ b/tensorflow/compiler/tests/eager_test.py @@ -177,6 +177,17 @@ class EagerTest(xla_test.XLATestCase): for _ in range(100): values.append(var.value()) + def testVariablesAreDeleted(self): + # This test makes sure that we release device (especially TPU) memory + # when resource variable is deleted. + with self.test_scope(): + # Create and destroy a 128MB variable 100 times. + # If we don't release device memory when python variable is deleted, + # this will eat over 13GB and OOM. + for _ in range(100): + # Create 128MiB variables + resource_variable_ops.ResourceVariable(array_ops.ones([32, 1024, 1024])) + # The shape, shape_n, size, and rank are tested here because their # execution kernels (as opposed to compilation only tf2xla kernels) # are distincts from tf2xla kernels. diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc index af921e4815..c5292e1ae1 100644 --- a/tensorflow/core/kernels/resource_variable_ops.cc +++ b/tensorflow/core/kernels/resource_variable_ops.cc @@ -174,25 +174,20 @@ REGISTER_KERNEL_BUILDER(Name("VariableShape") #endif // GOOGLE_CUDA -class DestroyResourceOp : public OpKernel { - public: - explicit DestroyResourceOp(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, - ctx->GetAttr("ignore_lookup_error", &ignore_lookup_error_)); - } +DestroyResourceOp::DestroyResourceOp(OpKernelConstruction* ctx) + : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, + ctx->GetAttr("ignore_lookup_error", &ignore_lookup_error_)); +} - void Compute(OpKernelContext* ctx) override { - const ResourceHandle& p = HandleFromInput(ctx, 0); - Status status = DeleteResource(ctx, p); - if (ignore_lookup_error_ && errors::IsNotFound(status)) { - return; - } - OP_REQUIRES_OK(ctx, status); +void DestroyResourceOp::Compute(OpKernelContext* ctx) { + const ResourceHandle& p = HandleFromInput(ctx, 0); + Status status = DeleteResource(ctx, p); + if (ignore_lookup_error_ && errors::IsNotFound(status)) { + return; } - - private: - bool ignore_lookup_error_; -}; + OP_REQUIRES_OK(ctx, status); +} REGISTER_KERNEL_BUILDER(Name("DestroyResourceOp").Device(DEVICE_CPU), DestroyResourceOp); diff --git a/tensorflow/core/kernels/resource_variable_ops.h b/tensorflow/core/kernels/resource_variable_ops.h index 8cae5d21f0..9b60106f13 100644 --- a/tensorflow/core/kernels/resource_variable_ops.h +++ b/tensorflow/core/kernels/resource_variable_ops.h @@ -28,6 +28,15 @@ class ReadVariableOp : public OpKernel { DataType dtype_; }; +class DestroyResourceOp : public OpKernel { + public: + explicit DestroyResourceOp(OpKernelConstruction* ctx); + void Compute(OpKernelContext* ctx) override; + + private: + bool ignore_lookup_error_; +}; + } // namespace tensorflow #endif // TENSORFLOW_CORE_KERNELS_RESOURCE_VARIABLE_OPS_H_ -- cgit v1.2.3 From c8a913c3d5930d388b4232809a16c730673472ff Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 11:14:32 -0700 Subject: changed the image link. PiperOrigin-RevId: 204330664 --- .../examples/generative_examples/image_captioning_with_attention.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb index 9c7004b049..1a5a186e7a 100644 --- a/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb +++ b/tensorflow/contrib/eager/python/examples/generative_examples/image_captioning_with_attention.ipynb @@ -1154,7 +1154,7 @@ }, "cell_type": "code", "source": [ - "image_url = 'https://tensorflow.org/images/imcap_prediction.png'\n", + "image_url = 'https://tensorflow.org/images/surf.jpg'\n", "image_extension = image_url[-4:]\n", "image_path = tf.keras.utils.get_file('image'+image_extension, \n", " origin=image_url)\n", -- cgit v1.2.3 From 365d2fc4d62540b2c6524500a7a58e7edab0dfa9 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Thu, 12 Jul 2018 11:17:14 -0700 Subject: [tf.data / Bigtable]: Set AlwaysRetryMutationPolicy Because the tf.data integration currently doesn't support setting client-side timestamps, using the AlwaysRetryMutationPolicy can make data loading more reliable. (This is safe-ish to do, because when reading TF always read Latest(1), so duplicate writes will not be visible to user programs.) PiperOrigin-RevId: 204331133 --- tensorflow/contrib/bigtable/kernels/bigtable_lib.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_lib.h b/tensorflow/contrib/bigtable/kernels/bigtable_lib.h index 12d8256dea..a2a5df1037 100644 --- a/tensorflow/contrib/bigtable/kernels/bigtable_lib.h +++ b/tensorflow/contrib/bigtable/kernels/bigtable_lib.h @@ -58,7 +58,8 @@ class BigtableTableResource : public ResourceBase { BigtableTableResource(BigtableClientResource* client, string table_name) : client_(client), table_name_(std::move(table_name)), - table_(client->get_client(), table_name_) { + table_(client->get_client(), table_name_, + google::cloud::bigtable::AlwaysRetryMutationPolicy()) { client_->Ref(); } -- cgit v1.2.3 From ee971f12623d22dd6d69829b36195f1712a6ab8f Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Thu, 12 Jul 2018 11:48:18 -0700 Subject: More AST utils. Includes support for persistent annotations and copying the multiple assignment walking code out of transformer.py (to be removed later). Persistent annotations require: * allow copy_clean to keep user-specified annotations * ensure rename_symbols does not destroy annotations PiperOrigin-RevId: 204336881 --- tensorflow/contrib/autograph/pyct/ast_util.py | 144 +++++++++++++++------ tensorflow/contrib/autograph/pyct/ast_util_test.py | 122 +++++++++++------ 2 files changed, 188 insertions(+), 78 deletions(-) diff --git a/tensorflow/contrib/autograph/pyct/ast_util.py b/tensorflow/contrib/autograph/pyct/ast_util.py index c4f82d1170..0cf87dd8d3 100644 --- a/tensorflow/contrib/autograph/pyct/ast_util.py +++ b/tensorflow/contrib/autograph/pyct/ast_util.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Copy an AST tree, discarding annotations.""" +"""AST manipulation utilities.""" from __future__ import absolute_import from __future__ import division @@ -26,47 +26,53 @@ from tensorflow.contrib.autograph.pyct import anno from tensorflow.contrib.autograph.pyct import parser -class CleanCopier(gast.NodeVisitor): - """Copies AST nodes. +class CleanCopier(object): + """NodeTransformer-like visitor that copies an AST.""" - The copied nodes will ignore almost all fields that are prefixed by '__'. - Exceptions make some annotations. - """ + def __init__(self, preserve_annos): + super(CleanCopier, self).__init__() + self.preserve_annos = preserve_annos - # TODO(mdan): Parametrize which annotations get carried over. + def copy(self, node): + """Returns a deep copy of node (excluding some fields, see copy_clean).""" + + if isinstance(node, list): + return [self.copy(n) for n in node] + elif isinstance(node, tuple): + return tuple(self.copy(n) for n in node) + elif not isinstance(node, (gast.AST, ast.AST)): + # Assuming everything that's not an AST, list or tuple is a value type + # and may simply be assigned. + return node + + assert isinstance(node, (gast.AST, ast.AST)) - def generic_visit(self, node): new_fields = {} for f in node._fields: - if f.startswith('__'): - continue - if not hasattr(node, f): - continue - v = getattr(node, f) - if isinstance(v, list): - v = [self.generic_visit(n) for n in v] - elif isinstance(v, tuple): - v = tuple(self.generic_visit(n) for n in v) - elif isinstance(v, (gast.AST, ast.AST)): - v = self.generic_visit(v) - else: - # Assume everything else is a value type. - pass - new_fields[f] = v + if not f.startswith('__') and hasattr(node, f): + new_fields[f] = self.copy(getattr(node, f)) new_node = type(node)(**new_fields) - if anno.hasanno(node, anno.Basic.SKIP_PROCESSING): - anno.setanno(new_node, anno.Basic.SKIP_PROCESSING, True) + + if self.preserve_annos: + for k in self.preserve_annos: + anno.copyanno(node, new_node, k) return new_node -def copy_clean(node): - copier = CleanCopier() - if isinstance(node, list): - return [copier.visit(n) for n in node] - elif isinstance(node, tuple): - return tuple(copier.visit(n) for n in node) - else: - return copier.visit(node) +def copy_clean(node, preserve_annos=None): + """Creates a deep copy of an AST. + + The copy will not include fields that are prefixed by '__', with the + exception of user-specified annotations. + + Args: + node: ast.AST + preserve_annos: Optional[Set[Hashable]], annotation keys to include in the + copy + Returns: + ast.AST + """ + return CleanCopier(preserve_annos).copy(node) class SymbolRenamer(gast.NodeTransformer): @@ -78,7 +84,11 @@ class SymbolRenamer(gast.NodeTransformer): def _process(self, node): qn = anno.getanno(node, anno.Basic.QN) if qn in self.name_map: - return gast.Name(str(self.name_map[qn]), node.ctx, None) + new_node = gast.Name(str(self.name_map[qn]), node.ctx, None) + # All annotations get carried over. + for k in anno.keys(node): + anno.copyanno(node, new_node, k) + return new_node return self.generic_visit(node) def visit_Name(self, node): @@ -92,6 +102,7 @@ class SymbolRenamer(gast.NodeTransformer): def rename_symbols(node, name_map): + """Renames symbols in an AST. Requires qual_names annotations.""" renamer = SymbolRenamer(name_map) if isinstance(node, list): return [renamer.visit(n) for n in node] @@ -101,6 +112,7 @@ def rename_symbols(node, name_map): def keywords_to_dict(keywords): + """Converts a list of ast.keyword objects to a dict.""" keys = [] values = [] for kw in keywords: @@ -110,10 +122,7 @@ def keywords_to_dict(keywords): class PatternMatcher(gast.NodeVisitor): - """Matches a node against a pattern represented by a node. - - The pattern may contain wildcards represented by the symbol '_'. - """ + """Matches a node against a pattern represented by a node.""" def __init__(self, pattern): self.pattern = pattern @@ -177,9 +186,68 @@ class PatternMatcher(gast.NodeVisitor): def matches(node, pattern): + """Basic pattern matcher for AST. + + The pattern may contain wildcards represented by the symbol '_'. A node + matches a pattern if for every node in the tree, either there is a node of + the same type in pattern, or a Name node with id='_'. + + Args: + node: ast.AST + pattern: ast.AST + Returns: + bool + """ if isinstance(pattern, str): pattern = parser.parse_expression(pattern) matcher = PatternMatcher(pattern) matcher.visit(node) return matcher.matches + +# TODO(mdan): Once we have error tracing, we may be able to just go to SSA. +def apply_to_single_assignments(targets, values, apply_fn): + """Applies a function to each individual assignment. + + This function can process a possibly-unpacked (e.g. a, b = c, d) assignment. + It tries to break down the unpacking if possible. In effect, it has the same + effect as passing the assigned values in SSA form to apply_fn. + + Examples: + + The following will result in apply_fn(a, c), apply_fn(b, d): + + a, b = c, d + + The following will result in apply_fn(a, c[0]), apply_fn(b, c[1]): + + a, b = c + + The following will result in apply_fn(a, (b, c)): + + a = b, c + + It uses the visitor pattern to allow subclasses to process single + assignments individually. + + Args: + targets: Union[List[ast.AST, ...], Tuple[ast.AST, ...], ast.AST, should be + used with the targets field of an ast.Assign node + values: ast.AST + apply_fn: Callable[[ast.AST, ast.AST], None], called with the + respective nodes of each single assignment + """ + if not isinstance(targets, (list, tuple)): + targets = (targets,) + for target in targets: + if isinstance(target, (gast.Tuple, gast.List)): + for i in range(len(target.elts)): + target_el = target.elts[i] + if isinstance(values, (gast.Tuple, gast.List)): + value_el = values.elts[i] + else: + idx = parser.parse_expression(str(i)) + value_el = gast.Subscript(values, gast.Index(idx), ctx=gast.Load()) + apply_to_single_assignments(target_el, value_el, apply_fn) + else: + apply_fn(target, values) diff --git a/tensorflow/contrib/autograph/pyct/ast_util_test.py b/tensorflow/contrib/autograph/pyct/ast_util_test.py index 3afa04a506..bd546c7f48 100644 --- a/tensorflow/contrib/autograph/pyct/ast_util_test.py +++ b/tensorflow/contrib/autograph/pyct/ast_util_test.py @@ -19,7 +19,10 @@ from __future__ import division from __future__ import print_function import ast +import collections +import textwrap +from tensorflow.contrib.autograph.pyct import anno from tensorflow.contrib.autograph.pyct import ast_util from tensorflow.contrib.autograph.pyct import compiler from tensorflow.contrib.autograph.pyct import parser @@ -29,53 +32,65 @@ from tensorflow.python.platform import test class AstUtilTest(test.TestCase): - def test_rename_symbols(self): - node = ast.Tuple([ - ast.Name('a', ast.Load()), - ast.Name('b', ast.Load()), - ast.Attribute(ast.Name('b', None), 'c', ast.Store()), - ast.Attribute( - ast.Attribute(ast.Name('b', None), 'c', ast.Load()), 'd', None) - ], None) + def setUp(self): + super(AstUtilTest, self).setUp() + self._invocation_counts = collections.defaultdict(lambda: 0) + + def test_rename_symbols_basic(self): + node = parser.parse_str('a + b') + node = qual_names.resolve(node) + + node = ast_util.rename_symbols( + node, {qual_names.QN('a'): qual_names.QN('renamed_a')}) + + self.assertIsInstance(node.body[0].value.left.id, str) + self.assertEqual(compiler.ast_to_source(node).strip(), 'renamed_a + b') + + def test_rename_symbols_attributes(self): + node = parser.parse_str('b.c = b.c.d') node = qual_names.resolve(node) + node = ast_util.rename_symbols( - node, { - qual_names.QN('a'): - qual_names.QN('renamed_a'), - qual_names.QN(qual_names.QN('b'), attr='c'): - qual_names.QN('renamed_b_c'), - }) - - self.assertEqual(node.elts[0].id, 'renamed_a') - self.assertTrue(isinstance(node.elts[0].ctx, ast.Load)) - self.assertEqual(node.elts[1].id, 'b') - self.assertEqual(node.elts[2].id, 'renamed_b_c') - self.assertTrue(isinstance(node.elts[2].ctx, ast.Store)) - self.assertEqual(node.elts[3].value.id, 'renamed_b_c') - self.assertTrue(isinstance(node.elts[3].value.ctx, ast.Load)) + node, {qual_names.from_str('b.c'): qual_names.QN('renamed_b_c')}) + + self.assertEqual( + compiler.ast_to_source(node).strip(), 'renamed_b_c = renamed_b_c.d') + + def test_rename_symbols_annotations(self): + node = parser.parse_str('a[i]') + node = qual_names.resolve(node) + anno.setanno(node, 'foo', 'bar') + orig_anno = anno.getanno(node, 'foo') + + node = ast_util.rename_symbols(node, + {qual_names.QN('a'): qual_names.QN('b')}) + + self.assertIs(anno.getanno(node, 'foo'), orig_anno) def test_copy_clean(self): - ret = ast.Return( - ast.BinOp( - op=ast.Add(), - left=ast.Name(id='a', ctx=ast.Load()), - right=ast.Num(1))) - setattr(ret, '__foo', 'bar') - node = ast.FunctionDef( - name='f', - args=ast.arguments( - args=[ast.Name(id='a', ctx=ast.Param())], - vararg=None, - kwarg=None, - defaults=[]), - body=[ret], - decorator_list=[], - returns=None) + node = parser.parse_str( + textwrap.dedent(""" + def f(a): + return a + 1 + """)) + setattr(node.body[0], '__foo', 'bar') new_node = ast_util.copy_clean(node) - self.assertFalse(node is new_node) - self.assertFalse(ret is new_node.body[0]) + self.assertIsNot(new_node, node) + self.assertIsNot(new_node.body[0], node.body[0]) self.assertFalse(hasattr(new_node.body[0], '__foo')) + def test_copy_clean_preserves_annotations(self): + node = parser.parse_str( + textwrap.dedent(""" + def f(a): + return a + 1 + """)) + anno.setanno(node.body[0], 'foo', 'bar') + anno.setanno(node.body[0], 'baz', 1) + new_node = ast_util.copy_clean(node, preserve_annos={'foo'}) + self.assertEqual(anno.getanno(new_node.body[0], 'foo'), 'bar') + self.assertFalse(anno.hasanno(new_node.body[0], 'baz')) + def test_keywords_to_dict(self): keywords = parser.parse_expression('f(a=b, c=1, d=\'e\')').keywords d = ast_util.keywords_to_dict(keywords) @@ -113,6 +128,33 @@ class AstUtilTest(test.TestCase): self.assertNoMatch('super(Foo, self).__init__()', 'super(Bar, _).__init__(_)') + def _mock_apply_fn(self, target, source): + target = compiler.ast_to_source(target).strip() + source = compiler.ast_to_source(source).strip() + self._invocation_counts[(target, source)] += 1 + + def test_apply_to_single_assignments_dynamic_unpack(self): + node = parser.parse_str('a, b, c = d') + node = node.body[0] + ast_util.apply_to_single_assignments(node.targets, node.value, + self._mock_apply_fn) + self.assertDictEqual(self._invocation_counts, { + ('a', 'd[0]'): 1, + ('b', 'd[1]'): 1, + ('c', 'd[2]'): 1, + }) + + def test_apply_to_single_assignments_static_unpack(self): + node = parser.parse_str('a, b, c = d, e, f') + node = node.body[0] + ast_util.apply_to_single_assignments(node.targets, node.value, + self._mock_apply_fn) + self.assertDictEqual(self._invocation_counts, { + ('a', 'd'): 1, + ('b', 'e'): 1, + ('c', 'f'): 1, + }) + if __name__ == '__main__': test.main() -- cgit v1.2.3 From 35cc95de26a70acd4a55588d8ae8da29fe44d9e6 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Thu, 12 Jul 2018 11:51:09 -0700 Subject: [TF:XLA] Bump open source llvm revision to r336887 PiperOrigin-RevId: 204337319 --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index b712954d6d..cd4f17a5ff 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -472,11 +472,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): tf_http_archive( name = "llvm", urls = [ - "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/d5d94ca3a7f8526c2e4e5f663f9dc79ae5d39d93.tar.gz", - "https://github.com/llvm-mirror/llvm/archive/d5d94ca3a7f8526c2e4e5f663f9dc79ae5d39d93.tar.gz", + "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/ae80745b73e435d07e7fb9c12589304ee29e7f59.tar.gz", + "https://github.com/llvm-mirror/llvm/archive/ae80745b73e435d07e7fb9c12589304ee29e7f59.tar.gz", ], - sha256 = "280fdc888e2eb88a3a8cc4e7d3034fffc87f98e3e686be31f8c719c6e5b67d2d", - strip_prefix = "llvm-d5d94ca3a7f8526c2e4e5f663f9dc79ae5d39d93", + sha256 = "de69b6f92a634b4d12b9e03ebd8eb34c28f997d9480c28358d6efd4c433fe853", + strip_prefix = "llvm-ae80745b73e435d07e7fb9c12589304ee29e7f59", build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"), ) -- cgit v1.2.3 From 0678f10d0f96b46ecabf129cd69a04de2df49a3d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 11:52:39 -0700 Subject: Internal changes. PiperOrigin-RevId: 204337595 --- .../examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb b/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb index fff673921a..d6a29ea1ec 100644 --- a/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb +++ b/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb @@ -303,7 +303,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 0, "metadata": { "colab": { "autoexec": { @@ -378,6 +378,7 @@ } ], "source": [ + "#@test {\"timeout\": 90} \n", "with tf.Graph().as_default():\n", " hp = tf.contrib.training.HParams(\n", " learning_rate=0.05,\n", @@ -504,7 +505,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 0, "metadata": { "colab": { "autoexec": { @@ -579,6 +580,7 @@ } ], "source": [ + "#@test {\"timeout\": 90} \n", "with context.eager_mode():\n", " durations = []\n", " for t in range(burn_ins + trials):\n", -- cgit v1.2.3 From c35bd2e9d3d9311bc7fb0f2463869faf1a8a7b50 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Thu, 12 Jul 2018 11:56:18 -0700 Subject: Internal Change. PiperOrigin-RevId: 204338153 --- tensorflow/BUILD | 2 +- tensorflow/contrib/cmake/python_modules.txt | 3 +- tensorflow/contrib/cmake/tf_python.cmake | 14 +- tensorflow/python/BUILD | 2 +- tensorflow/python/estimator/api/BUILD | 4 +- tensorflow/python/tools/api/generator/BUILD | 84 +++++ tensorflow/python/tools/api/generator/api_gen.bzl | 164 +++++++++ .../tools/api/generator/create_python_api.py | 408 +++++++++++++++++++++ .../tools/api/generator/create_python_api_test.py | 99 +++++ tensorflow/python/tools/api/generator/doc_srcs.py | 92 +++++ .../python/tools/api/generator/doc_srcs_test.py | 83 +++++ tensorflow/tools/api/generator/BUILD | 71 ---- tensorflow/tools/api/generator/api_gen.bzl | 164 --------- .../tools/api/generator/create_python_api.py | 408 --------------------- .../tools/api/generator/create_python_api_test.py | 99 ----- tensorflow/tools/api/generator/doc_srcs.py | 92 ----- tensorflow/tools/api/generator/doc_srcs_test.py | 83 ----- tensorflow/tools/pip_package/BUILD | 1 + 18 files changed, 944 insertions(+), 929 deletions(-) create mode 100644 tensorflow/python/tools/api/generator/BUILD create mode 100644 tensorflow/python/tools/api/generator/api_gen.bzl create mode 100644 tensorflow/python/tools/api/generator/create_python_api.py create mode 100644 tensorflow/python/tools/api/generator/create_python_api_test.py create mode 100644 tensorflow/python/tools/api/generator/doc_srcs.py create mode 100644 tensorflow/python/tools/api/generator/doc_srcs_test.py delete mode 100644 tensorflow/tools/api/generator/BUILD delete mode 100644 tensorflow/tools/api/generator/api_gen.bzl delete mode 100644 tensorflow/tools/api/generator/create_python_api.py delete mode 100644 tensorflow/tools/api/generator/create_python_api_test.py delete mode 100644 tensorflow/tools/api/generator/doc_srcs.py delete mode 100644 tensorflow/tools/api/generator/doc_srcs_test.py diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 51eea94847..518c2b0489 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -20,7 +20,7 @@ load( "tf_additional_binary_deps", ) load( - "//tensorflow/tools/api/generator:api_gen.bzl", + "//tensorflow/python/tools/api/generator:api_gen.bzl", "gen_api_init_files", # @unused ) diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt index 40041d9c88..75e00f3267 100644 --- a/tensorflow/contrib/cmake/python_modules.txt +++ b/tensorflow/contrib/cmake/python_modules.txt @@ -62,6 +62,8 @@ tensorflow/python/saved_model tensorflow/python/summary tensorflow/python/summary/writer tensorflow/python/tools +tensorflow/python/tools/api +tensorflow/python/tools/api/generator tensorflow/python/training tensorflow/python/training/checkpointable tensorflow/python/user_ops @@ -69,7 +71,6 @@ tensorflow/python/util tensorflow/python/util/protobuf tensorflow/tools tensorflow/tools/api -tensorflow/tools/api/generator tensorflow/tools/graph_transforms tensorflow/contrib tensorflow/contrib/all_reduce diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index 8a9172b43c..32b185f07b 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -736,8 +736,8 @@ endif() # Generate API __init__.py files. ######################################################## -# Parse tensorflow/tools/api/generator/BUILD to get list of generated files. -FILE(READ ${tensorflow_source_dir}/tensorflow/tools/api/generator/api_gen.bzl api_generator_BUILD_text) +# Parse tensorflow/python/tools/api/generator/BUILD to get list of generated files. +FILE(READ ${tensorflow_source_dir}/tensorflow/python/tools/api/generator/api_gen.bzl api_generator_BUILD_text) STRING(REGEX MATCH "# BEGIN GENERATED FILES.*# END GENERATED FILES" api_init_files_text ${api_generator_BUILD_text}) string(REPLACE "# BEGIN GENERATED FILES" "" api_init_files_text ${api_init_files_text}) string(REPLACE "# END GENERATED FILES" "" api_init_files_text ${api_init_files_text}) @@ -781,7 +781,7 @@ if (tensorflow_ENABLE_MKL_SUPPORT) # Run create_python_api.py to generate API init files. COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/tf_python PATH=${PY_RUNTIME_ENV} ${PYTHON_EXECUTABLE} - "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/tools/api/generator/create_python_api.py" + "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/tools/api/generator/create_python_api.py" "--root_init_template=${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/api_template.__init__.py" "--apidir=${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow" "--package=tensorflow.python" @@ -803,7 +803,7 @@ else (tensorflow_ENABLE_MKL_SUPPORT) # Run create_python_api.py to generate API init files. COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/tf_python ${PYTHON_EXECUTABLE} - "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/tools/api/generator/create_python_api.py" + "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/tools/api/generator/create_python_api.py" "--root_init_template=${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/api_template.__init__.py" "--apidir=${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow" "--package=tensorflow.python" @@ -824,8 +824,8 @@ add_dependencies(tf_python_api tf_python_ops) # Generate API __init__.py files for tf.estimator. ######################################################## -# Parse tensorflow/tools/api/generator/BUILD to get list of generated files. -FILE(READ ${tensorflow_source_dir}/tensorflow/tools/api/generator/api_gen.bzl api_generator_BUILD_text) +# Parse tensorflow/python/tools/api/generator/BUILD to get list of generated files. +FILE(READ ${tensorflow_source_dir}/tensorflow/python/tools/api/generator/api_gen.bzl api_generator_BUILD_text) STRING(REGEX MATCH "# BEGIN GENERATED ESTIMATOR FILES.*# END GENERATED ESTIMATOR FILES" api_init_files_text ${api_generator_BUILD_text}) string(REPLACE "# BEGIN GENERATED ESTIMATOR FILES" "" api_init_files_text ${api_init_files_text}) string(REPLACE "# END GENERATED ESTIMATOR FILES" "" api_init_files_text ${api_init_files_text}) @@ -849,7 +849,7 @@ add_custom_command( # Run create_python_api.py to generate API init files. COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/tf_python ${PYTHON_EXECUTABLE} - "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/tools/api/generator/create_python_api.py" + "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/tools/api/generator/create_python_api.py" "--apidir=${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/estimator/api" "--package=tensorflow.python.estimator" "--apiname=estimator" diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index d00debe1a1..924db54cbc 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -73,7 +73,7 @@ py_library( visibility = [ "//tensorflow:__pkg__", "//tensorflow/python/tools:__pkg__", - "//tensorflow/tools/api/generator:__pkg__", + "//tensorflow/python/tools/api/generator:__pkg__", ], deps = [ ":array_ops", diff --git a/tensorflow/python/estimator/api/BUILD b/tensorflow/python/estimator/api/BUILD index ceb9baef4d..a75fa7d0ae 100644 --- a/tensorflow/python/estimator/api/BUILD +++ b/tensorflow/python/estimator/api/BUILD @@ -6,8 +6,8 @@ package( licenses(["notice"]) # Apache 2.0 -load("//tensorflow/tools/api/generator:api_gen.bzl", "gen_api_init_files") -load("//tensorflow/tools/api/generator:api_gen.bzl", "ESTIMATOR_API_INIT_FILES") +load("//tensorflow/python/tools/api/generator:api_gen.bzl", "gen_api_init_files") +load("//tensorflow/python/tools/api/generator:api_gen.bzl", "ESTIMATOR_API_INIT_FILES") gen_api_init_files( name = "estimator_python_api_gen", diff --git a/tensorflow/python/tools/api/generator/BUILD b/tensorflow/python/tools/api/generator/BUILD new file mode 100644 index 0000000000..223d1281ba --- /dev/null +++ b/tensorflow/python/tools/api/generator/BUILD @@ -0,0 +1,84 @@ +# Description: +# Scripts used to generate TensorFlow Python API. + +licenses(["notice"]) # Apache 2.0 + +load("//tensorflow:tensorflow.bzl", "py_test") +load("//tensorflow/python/tools/api/generator:api_gen.bzl", "ESTIMATOR_API_INIT_FILES") +load("//tensorflow/python/tools/api/generator:api_gen.bzl", "TENSORFLOW_API_INIT_FILES") + +exports_files( + [ + "LICENSE", + "create_python_api.py", + ], +) + +py_binary( + name = "create_python_api", + srcs = ["//tensorflow/python/tools/api/generator:create_python_api.py"], + main = "//tensorflow/python/tools/api/generator:create_python_api.py", + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/python:no_contrib", + "//tensorflow/python/tools/api/generator:doc_srcs", + ], +) + +py_library( + name = "doc_srcs", + srcs = ["doc_srcs.py"], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/python:util", + ], +) + +py_test( + name = "create_python_api_test", + srcs = [ + "create_python_api.py", + "create_python_api_test.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":doc_srcs", + "//tensorflow/python:client_testlib", + "//tensorflow/python:no_contrib", + ], +) + +py_test( + name = "tensorflow_doc_srcs_test", + srcs = ["doc_srcs_test.py"], + args = [ + "--package=tensorflow.python", + "--api_name=tensorflow", + ] + TENSORFLOW_API_INIT_FILES, + main = "doc_srcs_test.py", + srcs_version = "PY2AND3", + deps = [ + ":doc_srcs", + "//tensorflow/python:client_testlib", + "//tensorflow/python:no_contrib", + ], +) + +py_test( + name = "estimator_doc_srcs_test", + srcs = ["doc_srcs_test.py"], + args = [ + "--package=tensorflow.python.estimator", + "--api_name=estimator", + ] + ESTIMATOR_API_INIT_FILES, + main = "doc_srcs_test.py", + srcs_version = "PY2AND3", + deps = [ + ":doc_srcs", + "//tensorflow/python:client_testlib", + "//tensorflow/python:no_contrib", + "//tensorflow/python/estimator:estimator_py", + ], +) diff --git a/tensorflow/python/tools/api/generator/api_gen.bzl b/tensorflow/python/tools/api/generator/api_gen.bzl new file mode 100644 index 0000000000..f9170610b9 --- /dev/null +++ b/tensorflow/python/tools/api/generator/api_gen.bzl @@ -0,0 +1,164 @@ +"""Targets for generating TensorFlow Python API __init__.py files.""" + +# keep sorted +TENSORFLOW_API_INIT_FILES = [ + # BEGIN GENERATED FILES + "__init__.py", + "app/__init__.py", + "bitwise/__init__.py", + "compat/__init__.py", + "data/__init__.py", + "debugging/__init__.py", + "distributions/__init__.py", + "distributions/bijectors/__init__.py", + "dtypes/__init__.py", + "errors/__init__.py", + "feature_column/__init__.py", + "gfile/__init__.py", + "graph_util/__init__.py", + "image/__init__.py", + "io/__init__.py", + "initializers/__init__.py", + "keras/__init__.py", + "keras/activations/__init__.py", + "keras/applications/__init__.py", + "keras/applications/densenet/__init__.py", + "keras/applications/inception_resnet_v2/__init__.py", + "keras/applications/inception_v3/__init__.py", + "keras/applications/mobilenet/__init__.py", + "keras/applications/nasnet/__init__.py", + "keras/applications/resnet50/__init__.py", + "keras/applications/vgg16/__init__.py", + "keras/applications/vgg19/__init__.py", + "keras/applications/xception/__init__.py", + "keras/backend/__init__.py", + "keras/callbacks/__init__.py", + "keras/constraints/__init__.py", + "keras/datasets/__init__.py", + "keras/datasets/boston_housing/__init__.py", + "keras/datasets/cifar10/__init__.py", + "keras/datasets/cifar100/__init__.py", + "keras/datasets/fashion_mnist/__init__.py", + "keras/datasets/imdb/__init__.py", + "keras/datasets/mnist/__init__.py", + "keras/datasets/reuters/__init__.py", + "keras/estimator/__init__.py", + "keras/initializers/__init__.py", + "keras/layers/__init__.py", + "keras/losses/__init__.py", + "keras/metrics/__init__.py", + "keras/models/__init__.py", + "keras/optimizers/__init__.py", + "keras/preprocessing/__init__.py", + "keras/preprocessing/image/__init__.py", + "keras/preprocessing/sequence/__init__.py", + "keras/preprocessing/text/__init__.py", + "keras/regularizers/__init__.py", + "keras/utils/__init__.py", + "keras/wrappers/__init__.py", + "keras/wrappers/scikit_learn/__init__.py", + "layers/__init__.py", + "linalg/__init__.py", + "logging/__init__.py", + "losses/__init__.py", + "manip/__init__.py", + "math/__init__.py", + "metrics/__init__.py", + "nn/__init__.py", + "nn/rnn_cell/__init__.py", + "profiler/__init__.py", + "python_io/__init__.py", + "quantization/__init__.py", + "resource_loader/__init__.py", + "strings/__init__.py", + "saved_model/__init__.py", + "saved_model/builder/__init__.py", + "saved_model/constants/__init__.py", + "saved_model/loader/__init__.py", + "saved_model/main_op/__init__.py", + "saved_model/signature_constants/__init__.py", + "saved_model/signature_def_utils/__init__.py", + "saved_model/tag_constants/__init__.py", + "saved_model/utils/__init__.py", + "sets/__init__.py", + "sparse/__init__.py", + "spectral/__init__.py", + "summary/__init__.py", + "sysconfig/__init__.py", + "test/__init__.py", + "train/__init__.py", + "train/queue_runner/__init__.py", + "user_ops/__init__.py", + # END GENERATED FILES +] + +# keep sorted +ESTIMATOR_API_INIT_FILES = [ + # BEGIN GENERATED ESTIMATOR FILES + "__init__.py", + "estimator/__init__.py", + "estimator/export/__init__.py", + "estimator/inputs/__init__.py", + # END GENERATED ESTIMATOR FILES +] + +# Creates a genrule that generates a directory structure with __init__.py +# files that import all exported modules (i.e. modules with tf_export +# decorators). +# +# Args: +# name: name of genrule to create. +# output_files: List of __init__.py files that should be generated. +# This list should include file name for every module exported using +# tf_export. For e.g. if an op is decorated with +# @tf_export('module1.module2', 'module3'). Then, output_files should +# include module1/module2/__init__.py and module3/__init__.py. +# root_init_template: Python init file that should be used as template for +# root __init__.py file. "# API IMPORTS PLACEHOLDER" comment inside this +# template will be replaced with root imports collected by this genrule. +# srcs: genrule sources. If passing root_init_template, the template file +# must be included in sources. +# api_name: Name of the project that you want to generate API files for +# (e.g. "tensorflow" or "estimator"). +# package: Python package containing the @tf_export decorators you want to +# process +# package_dep: Python library target containing your package. + +def gen_api_init_files( + name, + output_files = TENSORFLOW_API_INIT_FILES, + root_init_template = None, + srcs = [], + api_name = "tensorflow", + package = "tensorflow.python", + package_dep = "//tensorflow/python:no_contrib", + output_package = "tensorflow"): + root_init_template_flag = "" + if root_init_template: + root_init_template_flag = "--root_init_template=$(location " + root_init_template + ")" + + api_gen_binary_target = "create_" + package + "_api" + native.py_binary( + name = "create_" + package + "_api", + srcs = ["//tensorflow/python/tools/api/generator:create_python_api.py"], + main = "//tensorflow/python/tools/api/generator:create_python_api.py", + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], + deps = [ + package_dep, + "//tensorflow/python/tools/api/generator:doc_srcs", + ], + ) + + native.genrule( + name = name, + outs = output_files, + cmd = ( + "$(location :" + api_gen_binary_target + ") " + + root_init_template_flag + " --apidir=$(@D) --apiname=" + + api_name + " --package=" + package + " --output_package=" + + output_package + " $(OUTS)"), + srcs = srcs, + tools = [":" + api_gen_binary_target ], + visibility = ["//tensorflow:__pkg__"], + ) diff --git a/tensorflow/python/tools/api/generator/create_python_api.py b/tensorflow/python/tools/api/generator/create_python_api.py new file mode 100644 index 0000000000..e78fe4b738 --- /dev/null +++ b/tensorflow/python/tools/api/generator/create_python_api.py @@ -0,0 +1,408 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Generates and prints out imports and constants for new TensorFlow python api. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import collections +import importlib +import os +import sys + +from tensorflow.python.tools.api.generator import doc_srcs +from tensorflow.python.util import tf_decorator +from tensorflow.python.util import tf_export + +API_ATTRS = tf_export.API_ATTRS + +_DEFAULT_PACKAGE = 'tensorflow.python' +_GENFILES_DIR_SUFFIX = 'genfiles/' +_SYMBOLS_TO_SKIP_EXPLICITLY = { + # Overrides __getattr__, so that unwrapping tf_decorator + # would have side effects. + 'tensorflow.python.platform.flags.FLAGS' +} +_GENERATED_FILE_HEADER = """# This file is MACHINE GENERATED! Do not edit. +# Generated by: tensorflow/python/tools/api/generator/create_python_api.py script. +\"\"\"%s +\"\"\" + +from __future__ import print_function + +""" +_GENERATED_FILE_FOOTER = '\n\ndel print_function\n' + + +class SymbolExposedTwiceError(Exception): + """Raised when different symbols are exported with the same name.""" + pass + + +def format_import(source_module_name, source_name, dest_name): + """Formats import statement. + + Args: + source_module_name: (string) Source module to import from. + source_name: (string) Source symbol name to import. + dest_name: (string) Destination alias name. + + Returns: + An import statement string. + """ + if source_module_name: + if source_name == dest_name: + return 'from %s import %s' % (source_module_name, source_name) + else: + return 'from %s import %s as %s' % ( + source_module_name, source_name, dest_name) + else: + if source_name == dest_name: + return 'import %s' % source_name + else: + return 'import %s as %s' % (source_name, dest_name) + + +class _ModuleInitCodeBuilder(object): + """Builds a map from module name to imports included in that module.""" + + def __init__(self): + self.module_imports = collections.defaultdict( + lambda: collections.defaultdict(set)) + self._dest_import_to_id = collections.defaultdict(int) + # Names that start with underscore in the root module. + self._underscore_names_in_root = [] + + def add_import( + self, symbol_id, dest_module_name, source_module_name, source_name, + dest_name): + """Adds this import to module_imports. + + Args: + symbol_id: (number) Unique identifier of the symbol to import. + dest_module_name: (string) Module name to add import to. + source_module_name: (string) Module to import from. + source_name: (string) Name of the symbol to import. + dest_name: (string) Import the symbol using this name. + + Raises: + SymbolExposedTwiceError: Raised when an import with the same + dest_name has already been added to dest_module_name. + """ + import_str = format_import(source_module_name, source_name, dest_name) + + # Check if we are trying to expose two different symbols with same name. + full_api_name = dest_name + if dest_module_name: + full_api_name = dest_module_name + '.' + full_api_name + if (full_api_name in self._dest_import_to_id and + symbol_id != self._dest_import_to_id[full_api_name] and + symbol_id != -1): + raise SymbolExposedTwiceError( + 'Trying to export multiple symbols with same name: %s.' % + full_api_name) + self._dest_import_to_id[full_api_name] = symbol_id + + if not dest_module_name and dest_name.startswith('_'): + self._underscore_names_in_root.append(dest_name) + + # The same symbol can be available in multiple modules. + # We store all possible ways of importing this symbol and later pick just + # one. + self.module_imports[dest_module_name][full_api_name].add(import_str) + + def build(self): + """Get a map from destination module to __init__.py code for that module. + + Returns: + A dictionary where + key: (string) destination module (for e.g. tf or tf.consts). + value: (string) text that should be in __init__.py files for + corresponding modules. + """ + module_text_map = {} + for dest_module, dest_name_to_imports in self.module_imports.items(): + # Sort all possible imports for a symbol and pick the first one. + imports_list = [ + sorted(imports)[0] + for _, imports in dest_name_to_imports.items()] + module_text_map[dest_module] = '\n'.join(sorted(imports_list)) + + # Expose exported symbols with underscores in root module + # since we import from it using * import. + underscore_names_str = ', '.join( + '\'%s\'' % name for name in self._underscore_names_in_root) + # We will always generate a root __init__.py file to let us handle * + # imports consistently. Be sure to have a root __init__.py file listed in + # the script outputs. + module_text_map[''] = module_text_map.get('', '') + ''' +_names_with_underscore = [%s] +__all__ = [_s for _s in dir() if not _s.startswith('_')] +__all__.extend([_s for _s in _names_with_underscore]) +__all__.remove('print_function') +''' % underscore_names_str + + return module_text_map + + +def get_api_init_text(package, output_package, api_name): + """Get a map from destination module to __init__.py code for that module. + + Args: + package: Base python package containing python with target tf_export + decorators. + api_name: API you want to generate (e.g. `tensorflow` or `estimator`). + + Returns: + A dictionary where + key: (string) destination module (for e.g. tf or tf.consts). + value: (string) text that should be in __init__.py files for + corresponding modules. + """ + module_code_builder = _ModuleInitCodeBuilder() + + # Traverse over everything imported above. Specifically, + # we want to traverse over TensorFlow Python modules. + for module in list(sys.modules.values()): + # Only look at tensorflow modules. + if (not module or not hasattr(module, '__name__') or + module.__name__ is None or package not in module.__name__): + continue + # Do not generate __init__.py files for contrib modules for now. + if '.contrib.' in module.__name__ or module.__name__.endswith('.contrib'): + continue + + for module_contents_name in dir(module): + if (module.__name__ + '.' + module_contents_name + in _SYMBOLS_TO_SKIP_EXPLICITLY): + continue + attr = getattr(module, module_contents_name) + + # If attr is _tf_api_constants attribute, then add the constants. + if module_contents_name == API_ATTRS[api_name].constants: + for exports, value in attr: + for export in exports: + names = export.split('.') + dest_module = '.'.join(names[:-1]) + module_code_builder.add_import( + -1, dest_module, module.__name__, value, names[-1]) + continue + + _, attr = tf_decorator.unwrap(attr) + # If attr is a symbol with _tf_api_names attribute, then + # add import for it. + if (hasattr(attr, '__dict__') and + API_ATTRS[api_name].names in attr.__dict__): + for export in getattr(attr, API_ATTRS[api_name].names): # pylint: disable=protected-access + names = export.split('.') + dest_module = '.'.join(names[:-1]) + module_code_builder.add_import( + id(attr), dest_module, module.__name__, module_contents_name, + names[-1]) + + # Import all required modules in their parent modules. + # For e.g. if we import 'foo.bar.Value'. Then, we also + # import 'bar' in 'foo'. + imported_modules = set(module_code_builder.module_imports.keys()) + for module in imported_modules: + if not module: + continue + module_split = module.split('.') + parent_module = '' # we import submodules in their parent_module + + for submodule_index in range(len(module_split)): + if submodule_index > 0: + parent_module += ('.' + module_split[submodule_index-1] if parent_module + else module_split[submodule_index-1]) + import_from = output_package + if submodule_index > 0: + import_from += '.' + '.'.join(module_split[:submodule_index]) + module_code_builder.add_import( + -1, parent_module, import_from, + module_split[submodule_index], module_split[submodule_index]) + + return module_code_builder.build() + + +def get_module(dir_path, relative_to_dir): + """Get module that corresponds to path relative to relative_to_dir. + + Args: + dir_path: Path to directory. + relative_to_dir: Get module relative to this directory. + + Returns: + Name of module that corresponds to the given directory. + """ + dir_path = dir_path[len(relative_to_dir):] + # Convert path separators to '/' for easier parsing below. + dir_path = dir_path.replace(os.sep, '/') + return dir_path.replace('/', '.').strip('.') + + +def get_module_docstring(module_name, package, api_name): + """Get docstring for the given module. + + This method looks for docstring in the following order: + 1. Checks if module has a docstring specified in doc_srcs. + 2. Checks if module has a docstring source module specified + in doc_srcs. If it does, gets docstring from that module. + 3. Checks if module with module_name exists under base package. + If it does, gets docstring from that module. + 4. Returns a default docstring. + + Args: + module_name: module name relative to tensorflow + (excluding 'tensorflow.' prefix) to get a docstring for. + package: Base python package containing python with target tf_export + decorators. + api_name: API you want to generate (e.g. `tensorflow` or `estimator`). + + Returns: + One-line docstring to describe the module. + """ + # Module under base package to get a docstring from. + docstring_module_name = module_name + + doc_sources = doc_srcs.get_doc_sources(api_name) + + if module_name in doc_sources: + docsrc = doc_sources[module_name] + if docsrc.docstring: + return docsrc.docstring + if docsrc.docstring_module_name: + docstring_module_name = docsrc.docstring_module_name + + docstring_module_name = package + '.' + docstring_module_name + if (docstring_module_name in sys.modules and + sys.modules[docstring_module_name].__doc__): + return sys.modules[docstring_module_name].__doc__ + + return 'Public API for tf.%s namespace.' % module_name + + +def create_api_files( + output_files, package, root_init_template, output_dir, output_package, + api_name): + """Creates __init__.py files for the Python API. + + Args: + output_files: List of __init__.py file paths to create. + Each file must be under api/ directory. + package: Base python package containing python with target tf_export + decorators. + root_init_template: Template for top-level __init__.py file. + "#API IMPORTS PLACEHOLDER" comment in the template file will be replaced + with imports. + output_dir: output API root directory. + api_name: API you want to generate (e.g. `tensorflow` or `estimator`). + + Raises: + ValueError: if an output file is not under api/ directory, + or output_files list is missing a required file. + """ + module_name_to_file_path = {} + for output_file in output_files: + module_name = get_module(os.path.dirname(output_file), output_dir) + module_name_to_file_path[module_name] = os.path.normpath(output_file) + + # Create file for each expected output in genrule. + for module, file_path in module_name_to_file_path.items(): + if not os.path.isdir(os.path.dirname(file_path)): + os.makedirs(os.path.dirname(file_path)) + open(file_path, 'a').close() + + module_text_map = get_api_init_text(package, output_package, api_name) + + # Add imports to output files. + missing_output_files = [] + for module, text in module_text_map.items(): + # Make sure genrule output file list is in sync with API exports. + if module not in module_name_to_file_path: + module_file_path = '"%s/__init__.py"' % ( + module.replace('.', '/')) + missing_output_files.append(module_file_path) + continue + contents = '' + if module or not root_init_template: + contents = ( + _GENERATED_FILE_HEADER % + get_module_docstring(module, package, api_name) + + text + _GENERATED_FILE_FOOTER) + else: + # Read base init file + with open(root_init_template, 'r') as root_init_template_file: + contents = root_init_template_file.read() + contents = contents.replace('# API IMPORTS PLACEHOLDER', text) + with open(module_name_to_file_path[module], 'w') as fp: + fp.write(contents) + + if missing_output_files: + raise ValueError( + 'Missing outputs for python_api_gen genrule:\n%s.' + 'Make sure all required outputs are in the ' + 'tensorflow/tools/api/generator/api_gen.bzl file.' % + ',\n'.join(sorted(missing_output_files))) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'outputs', metavar='O', type=str, nargs='+', + help='If a single file is passed in, then we we assume it contains a ' + 'semicolon-separated list of Python files that we expect this script to ' + 'output. If multiple files are passed in, then we assume output files ' + 'are listed directly as arguments.') + parser.add_argument( + '--package', default=_DEFAULT_PACKAGE, type=str, + help='Base package that imports modules containing the target tf_export ' + 'decorators.') + parser.add_argument( + '--root_init_template', default='', type=str, + help='Template for top level __init__.py file. ' + '"#API IMPORTS PLACEHOLDER" comment will be replaced with imports.') + parser.add_argument( + '--apidir', type=str, required=True, + help='Directory where generated output files are placed. ' + 'gendir should be a prefix of apidir. Also, apidir ' + 'should be a prefix of every directory in outputs.') + parser.add_argument( + '--apiname', required=True, type=str, + choices=API_ATTRS.keys(), + help='The API you want to generate.') + parser.add_argument( + '--output_package', default='tensorflow', type=str, + help='Root output package.') + + args = parser.parse_args() + + if len(args.outputs) == 1: + # If we only get a single argument, then it must be a file containing + # list of outputs. + with open(args.outputs[0]) as output_list_file: + outputs = [line.strip() for line in output_list_file.read().split(';')] + else: + outputs = args.outputs + + # Populate `sys.modules` with modules containing tf_export(). + importlib.import_module(args.package) + create_api_files(outputs, args.package, args.root_init_template, + args.apidir, args.output_package, args.apiname) + + +if __name__ == '__main__': + main() diff --git a/tensorflow/python/tools/api/generator/create_python_api_test.py b/tensorflow/python/tools/api/generator/create_python_api_test.py new file mode 100644 index 0000000000..368b4c37e8 --- /dev/null +++ b/tensorflow/python/tools/api/generator/create_python_api_test.py @@ -0,0 +1,99 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Tests for create_python_api.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import imp +import sys + +from tensorflow.python.platform import test +from tensorflow.python.tools.api.generator import create_python_api +from tensorflow.python.util.tf_export import tf_export + + +@tf_export('test_op', 'test_op1') +def test_op(): + pass + + +@tf_export('TestClass', 'NewTestClass') +class TestClass(object): + pass + + +_TEST_CONSTANT = 5 +_MODULE_NAME = 'tensorflow.python.test_module' + + +class CreatePythonApiTest(test.TestCase): + + def setUp(self): + # Add fake op to a module that has 'tensorflow' in the name. + sys.modules[_MODULE_NAME] = imp.new_module(_MODULE_NAME) + setattr(sys.modules[_MODULE_NAME], 'test_op', test_op) + setattr(sys.modules[_MODULE_NAME], 'TestClass', TestClass) + test_op.__module__ = _MODULE_NAME + TestClass.__module__ = _MODULE_NAME + tf_export('consts._TEST_CONSTANT').export_constant( + _MODULE_NAME, '_TEST_CONSTANT') + + def tearDown(self): + del sys.modules[_MODULE_NAME] + + def testFunctionImportIsAdded(self): + imports = create_python_api.get_api_init_text( + package=create_python_api._DEFAULT_PACKAGE, + output_package='tensorflow', + api_name='tensorflow') + expected_import = ( + 'from tensorflow.python.test_module ' + 'import test_op as test_op1') + self.assertTrue( + expected_import in str(imports), + msg='%s not in %s' % (expected_import, str(imports))) + + expected_import = ('from tensorflow.python.test_module ' + 'import test_op') + self.assertTrue( + expected_import in str(imports), + msg='%s not in %s' % (expected_import, str(imports))) + + def testClassImportIsAdded(self): + imports = create_python_api.get_api_init_text( + package=create_python_api._DEFAULT_PACKAGE, + output_package='tensorflow', + api_name='tensorflow') + expected_import = ('from tensorflow.python.test_module ' + 'import TestClass') + self.assertTrue( + 'TestClass' in str(imports), + msg='%s not in %s' % (expected_import, str(imports))) + + def testConstantIsAdded(self): + imports = create_python_api.get_api_init_text( + package=create_python_api._DEFAULT_PACKAGE, + output_package='tensorflow', + api_name='tensorflow') + expected = ('from tensorflow.python.test_module ' + 'import _TEST_CONSTANT') + self.assertTrue(expected in str(imports), + msg='%s not in %s' % (expected, str(imports))) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/python/tools/api/generator/doc_srcs.py b/tensorflow/python/tools/api/generator/doc_srcs.py new file mode 100644 index 0000000000..ad1988494d --- /dev/null +++ b/tensorflow/python/tools/api/generator/doc_srcs.py @@ -0,0 +1,92 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Specifies sources of doc strings for API modules.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections + +from tensorflow.python.util import tf_export + + +# Specifies docstring source for a module. +# Only one of docstring or docstring_module_name should be set. +# * If docstring is set, then we will use this docstring when +# for the module. +# * If docstring_module_name is set, then we will copy the docstring +# from docstring source module. +DocSource = collections.namedtuple( + 'DocSource', ['docstring', 'docstring_module_name']) +# Each attribute of DocSource is optional. +DocSource.__new__.__defaults__ = (None,) * len(DocSource._fields) + +_TENSORFLOW_DOC_SOURCES = { + 'app': DocSource(docstring_module_name='platform.app'), + 'compat': DocSource(docstring_module_name='util.compat'), + 'distributions': DocSource( + docstring_module_name='ops.distributions.distributions'), + 'bitwise': DocSource(docstring_module_name='ops.bitwise_ops'), + 'errors': DocSource(docstring_module_name='framework.errors'), + 'gfile': DocSource(docstring_module_name='platform.gfile'), + 'graph_util': DocSource(docstring_module_name='framework.graph_util'), + 'image': DocSource(docstring_module_name='ops.image_ops'), + 'keras.estimator': DocSource(docstring_module_name='keras.estimator'), + 'linalg': DocSource(docstring_module_name='ops.linalg_ops'), + 'logging': DocSource(docstring_module_name='ops.logging_ops'), + 'losses': DocSource(docstring_module_name='ops.losses.losses'), + 'manip': DocSource(docstring_module_name='ops.manip_ops'), + 'math': DocSource(docstring_module_name='ops.math_ops'), + 'metrics': DocSource(docstring_module_name='ops.metrics'), + 'nn': DocSource(docstring_module_name='ops.nn_ops'), + 'nn.rnn_cell': DocSource(docstring_module_name='ops.rnn_cell'), + 'python_io': DocSource(docstring_module_name='lib.io.python_io'), + 'resource_loader': DocSource( + docstring_module_name='platform.resource_loader'), + 'sets': DocSource(docstring_module_name='ops.sets'), + 'sparse': DocSource(docstring_module_name='ops.sparse_ops'), + 'spectral': DocSource(docstring_module_name='ops.spectral_ops'), + 'strings': DocSource(docstring_module_name='ops.string_ops'), + 'sysconfig': DocSource(docstring_module_name='platform.sysconfig'), + 'test': DocSource(docstring_module_name='platform.test'), + 'train': DocSource(docstring_module_name='training.training'), + 'train.queue_runner': DocSource( + docstring_module_name='training.queue_runner'), +} + +_ESTIMATOR_DOC_SOURCES = { + 'estimator': DocSource( + docstring_module_name='estimator_lib'), + 'estimator.export': DocSource( + docstring_module_name='export.export_lib'), + 'estimator.inputs': DocSource( + docstring_module_name='inputs.inputs'), +} + + +def get_doc_sources(api_name): + """Get a map from module to a DocSource object. + + Args: + api_name: API you want to generate (e.g. `tensorflow` or `estimator`). + + Returns: + Map from module name to DocSource object. + """ + if api_name == tf_export.TENSORFLOW_API_NAME: + return _TENSORFLOW_DOC_SOURCES + if api_name == tf_export.ESTIMATOR_API_NAME: + return _ESTIMATOR_DOC_SOURCES + return {} diff --git a/tensorflow/python/tools/api/generator/doc_srcs_test.py b/tensorflow/python/tools/api/generator/doc_srcs_test.py new file mode 100644 index 0000000000..481d9874a4 --- /dev/null +++ b/tensorflow/python/tools/api/generator/doc_srcs_test.py @@ -0,0 +1,83 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Tests for tensorflow.python.tools.api.generator.doc_srcs.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import importlib +import sys + +from tensorflow.python.platform import test +from tensorflow.python.tools.api.generator import doc_srcs + + +FLAGS = None + + +class DocSrcsTest(test.TestCase): + + def testModulesAreValidAPIModules(self): + for module_name in doc_srcs.get_doc_sources(FLAGS.api_name): + # Convert module_name to corresponding __init__.py file path. + file_path = module_name.replace('.', '/') + if file_path: + file_path += '/' + file_path += '__init__.py' + + self.assertIn( + file_path, FLAGS.outputs, + msg='%s is not a valid API module' % module_name) + + def testHaveDocstringOrDocstringModule(self): + for module_name, docsrc in doc_srcs.get_doc_sources(FLAGS.api_name).items(): + self.assertFalse( + docsrc.docstring and docsrc.docstring_module_name, + msg=('%s contains DocSource has both a docstring and a ' + 'docstring_module_name. Only one of "docstring" or ' + '"docstring_module_name" should be set.') % (module_name)) + + def testDocstringModulesAreValidModules(self): + for _, docsrc in doc_srcs.get_doc_sources(FLAGS.api_name).items(): + if docsrc.docstring_module_name: + doc_module_name = '.'.join([ + FLAGS.package, docsrc.docstring_module_name]) + self.assertIn( + doc_module_name, sys.modules, + msg=('docsources_module %s is not a valid module under %s.' % + (docsrc.docstring_module_name, FLAGS.package))) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + 'outputs', metavar='O', type=str, nargs='+', + help='create_python_api output files.') + parser.add_argument( + '--package', type=str, + help='Base package that imports modules containing the target tf_export ' + 'decorators.') + parser.add_argument( + '--api_name', type=str, + help='API name: tensorflow or estimator') + FLAGS, unparsed = parser.parse_known_args() + + importlib.import_module(FLAGS.package) + + # Now update argv, so that unittest library does not get confused. + sys.argv = [sys.argv[0]] + unparsed + test.main() diff --git a/tensorflow/tools/api/generator/BUILD b/tensorflow/tools/api/generator/BUILD deleted file mode 100644 index 8c760e6f52..0000000000 --- a/tensorflow/tools/api/generator/BUILD +++ /dev/null @@ -1,71 +0,0 @@ -# Description: -# Scripts used to generate TensorFlow Python API. - -licenses(["notice"]) # Apache 2.0 - -load("//tensorflow/tools/api/generator:api_gen.bzl", "ESTIMATOR_API_INIT_FILES") -load("//tensorflow/tools/api/generator:api_gen.bzl", "TENSORFLOW_API_INIT_FILES") - -exports_files( - [ - "LICENSE", - "create_python_api.py", - ], -) - -py_library( - name = "doc_srcs", - srcs = ["doc_srcs.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/python:util", - ], -) - -py_test( - name = "create_python_api_test", - srcs = [ - "create_python_api.py", - "create_python_api_test.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":doc_srcs", - "//tensorflow/python:client_testlib", - "//tensorflow/python:no_contrib", - ], -) - -py_test( - name = "tensorflow_doc_srcs_test", - srcs = ["doc_srcs_test.py"], - args = [ - "--package=tensorflow.python", - "--api_name=tensorflow", - ] + TENSORFLOW_API_INIT_FILES, - main = "doc_srcs_test.py", - srcs_version = "PY2AND3", - deps = [ - ":doc_srcs", - "//tensorflow/python:client_testlib", - "//tensorflow/python:no_contrib", - ], -) - -py_test( - name = "estimator_doc_srcs_test", - srcs = ["doc_srcs_test.py"], - args = [ - "--package=tensorflow.python.estimator", - "--api_name=estimator", - ] + ESTIMATOR_API_INIT_FILES, - main = "doc_srcs_test.py", - srcs_version = "PY2AND3", - deps = [ - ":doc_srcs", - "//tensorflow/python:client_testlib", - "//tensorflow/python:no_contrib", - "//tensorflow/python/estimator:estimator_py", - ], -) diff --git a/tensorflow/tools/api/generator/api_gen.bzl b/tensorflow/tools/api/generator/api_gen.bzl deleted file mode 100644 index ed164bf9e4..0000000000 --- a/tensorflow/tools/api/generator/api_gen.bzl +++ /dev/null @@ -1,164 +0,0 @@ -"""Targets for generating TensorFlow Python API __init__.py files.""" - -# keep sorted -TENSORFLOW_API_INIT_FILES = [ - # BEGIN GENERATED FILES - "__init__.py", - "app/__init__.py", - "bitwise/__init__.py", - "compat/__init__.py", - "data/__init__.py", - "debugging/__init__.py", - "distributions/__init__.py", - "distributions/bijectors/__init__.py", - "dtypes/__init__.py", - "errors/__init__.py", - "feature_column/__init__.py", - "gfile/__init__.py", - "graph_util/__init__.py", - "image/__init__.py", - "io/__init__.py", - "initializers/__init__.py", - "keras/__init__.py", - "keras/activations/__init__.py", - "keras/applications/__init__.py", - "keras/applications/densenet/__init__.py", - "keras/applications/inception_resnet_v2/__init__.py", - "keras/applications/inception_v3/__init__.py", - "keras/applications/mobilenet/__init__.py", - "keras/applications/nasnet/__init__.py", - "keras/applications/resnet50/__init__.py", - "keras/applications/vgg16/__init__.py", - "keras/applications/vgg19/__init__.py", - "keras/applications/xception/__init__.py", - "keras/backend/__init__.py", - "keras/callbacks/__init__.py", - "keras/constraints/__init__.py", - "keras/datasets/__init__.py", - "keras/datasets/boston_housing/__init__.py", - "keras/datasets/cifar10/__init__.py", - "keras/datasets/cifar100/__init__.py", - "keras/datasets/fashion_mnist/__init__.py", - "keras/datasets/imdb/__init__.py", - "keras/datasets/mnist/__init__.py", - "keras/datasets/reuters/__init__.py", - "keras/estimator/__init__.py", - "keras/initializers/__init__.py", - "keras/layers/__init__.py", - "keras/losses/__init__.py", - "keras/metrics/__init__.py", - "keras/models/__init__.py", - "keras/optimizers/__init__.py", - "keras/preprocessing/__init__.py", - "keras/preprocessing/image/__init__.py", - "keras/preprocessing/sequence/__init__.py", - "keras/preprocessing/text/__init__.py", - "keras/regularizers/__init__.py", - "keras/utils/__init__.py", - "keras/wrappers/__init__.py", - "keras/wrappers/scikit_learn/__init__.py", - "layers/__init__.py", - "linalg/__init__.py", - "logging/__init__.py", - "losses/__init__.py", - "manip/__init__.py", - "math/__init__.py", - "metrics/__init__.py", - "nn/__init__.py", - "nn/rnn_cell/__init__.py", - "profiler/__init__.py", - "python_io/__init__.py", - "quantization/__init__.py", - "resource_loader/__init__.py", - "strings/__init__.py", - "saved_model/__init__.py", - "saved_model/builder/__init__.py", - "saved_model/constants/__init__.py", - "saved_model/loader/__init__.py", - "saved_model/main_op/__init__.py", - "saved_model/signature_constants/__init__.py", - "saved_model/signature_def_utils/__init__.py", - "saved_model/tag_constants/__init__.py", - "saved_model/utils/__init__.py", - "sets/__init__.py", - "sparse/__init__.py", - "spectral/__init__.py", - "summary/__init__.py", - "sysconfig/__init__.py", - "test/__init__.py", - "train/__init__.py", - "train/queue_runner/__init__.py", - "user_ops/__init__.py", - # END GENERATED FILES -] - -# keep sorted -ESTIMATOR_API_INIT_FILES = [ - # BEGIN GENERATED ESTIMATOR FILES - "__init__.py", - "estimator/__init__.py", - "estimator/export/__init__.py", - "estimator/inputs/__init__.py", - # END GENERATED ESTIMATOR FILES -] - -# Creates a genrule that generates a directory structure with __init__.py -# files that import all exported modules (i.e. modules with tf_export -# decorators). -# -# Args: -# name: name of genrule to create. -# output_files: List of __init__.py files that should be generated. -# This list should include file name for every module exported using -# tf_export. For e.g. if an op is decorated with -# @tf_export('module1.module2', 'module3'). Then, output_files should -# include module1/module2/__init__.py and module3/__init__.py. -# root_init_template: Python init file that should be used as template for -# root __init__.py file. "# API IMPORTS PLACEHOLDER" comment inside this -# template will be replaced with root imports collected by this genrule. -# srcs: genrule sources. If passing root_init_template, the template file -# must be included in sources. -# api_name: Name of the project that you want to generate API files for -# (e.g. "tensorflow" or "estimator"). -# package: Python package containing the @tf_export decorators you want to -# process -# package_dep: Python library target containing your package. - -def gen_api_init_files( - name, - output_files = TENSORFLOW_API_INIT_FILES, - root_init_template = None, - srcs = [], - api_name = "tensorflow", - package = "tensorflow.python", - package_dep = "//tensorflow/python:no_contrib", - output_package = "tensorflow"): - root_init_template_flag = "" - if root_init_template: - root_init_template_flag = "--root_init_template=$(location " + root_init_template + ")" - - api_gen_binary_target = "create_" + package + "_api" - native.py_binary( - name = "create_" + package + "_api", - srcs = ["//tensorflow/tools/api/generator:create_python_api.py"], - main = "//tensorflow/tools/api/generator:create_python_api.py", - srcs_version = "PY2AND3", - visibility = ["//visibility:public"], - deps = [ - package_dep, - "//tensorflow/tools/api/generator:doc_srcs", - ], - ) - - native.genrule( - name = name, - outs = output_files, - cmd = ( - "$(location :" + api_gen_binary_target + ") " + - root_init_template_flag + " --apidir=$(@D) --apiname=" + - api_name + " --package=" + package + " --output_package=" + - output_package + " $(OUTS)"), - srcs = srcs, - tools = [":" + api_gen_binary_target ], - visibility = ["//tensorflow:__pkg__"], - ) diff --git a/tensorflow/tools/api/generator/create_python_api.py b/tensorflow/tools/api/generator/create_python_api.py deleted file mode 100644 index 7f17360c91..0000000000 --- a/tensorflow/tools/api/generator/create_python_api.py +++ /dev/null @@ -1,408 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Generates and prints out imports and constants for new TensorFlow python api. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import collections -import importlib -import os -import sys - -from tensorflow.python.util import tf_decorator -from tensorflow.python.util import tf_export -from tensorflow.tools.api.generator import doc_srcs - -API_ATTRS = tf_export.API_ATTRS - -_DEFAULT_PACKAGE = 'tensorflow.python' -_GENFILES_DIR_SUFFIX = 'genfiles/' -_SYMBOLS_TO_SKIP_EXPLICITLY = { - # Overrides __getattr__, so that unwrapping tf_decorator - # would have side effects. - 'tensorflow.python.platform.flags.FLAGS' -} -_GENERATED_FILE_HEADER = """# This file is MACHINE GENERATED! Do not edit. -# Generated by: tensorflow/tools/api/generator/create_python_api.py script. -\"\"\"%s -\"\"\" - -from __future__ import print_function - -""" -_GENERATED_FILE_FOOTER = '\n\ndel print_function\n' - - -class SymbolExposedTwiceError(Exception): - """Raised when different symbols are exported with the same name.""" - pass - - -def format_import(source_module_name, source_name, dest_name): - """Formats import statement. - - Args: - source_module_name: (string) Source module to import from. - source_name: (string) Source symbol name to import. - dest_name: (string) Destination alias name. - - Returns: - An import statement string. - """ - if source_module_name: - if source_name == dest_name: - return 'from %s import %s' % (source_module_name, source_name) - else: - return 'from %s import %s as %s' % ( - source_module_name, source_name, dest_name) - else: - if source_name == dest_name: - return 'import %s' % source_name - else: - return 'import %s as %s' % (source_name, dest_name) - - -class _ModuleInitCodeBuilder(object): - """Builds a map from module name to imports included in that module.""" - - def __init__(self): - self.module_imports = collections.defaultdict( - lambda: collections.defaultdict(set)) - self._dest_import_to_id = collections.defaultdict(int) - # Names that start with underscore in the root module. - self._underscore_names_in_root = [] - - def add_import( - self, symbol_id, dest_module_name, source_module_name, source_name, - dest_name): - """Adds this import to module_imports. - - Args: - symbol_id: (number) Unique identifier of the symbol to import. - dest_module_name: (string) Module name to add import to. - source_module_name: (string) Module to import from. - source_name: (string) Name of the symbol to import. - dest_name: (string) Import the symbol using this name. - - Raises: - SymbolExposedTwiceError: Raised when an import with the same - dest_name has already been added to dest_module_name. - """ - import_str = format_import(source_module_name, source_name, dest_name) - - # Check if we are trying to expose two different symbols with same name. - full_api_name = dest_name - if dest_module_name: - full_api_name = dest_module_name + '.' + full_api_name - if (full_api_name in self._dest_import_to_id and - symbol_id != self._dest_import_to_id[full_api_name] and - symbol_id != -1): - raise SymbolExposedTwiceError( - 'Trying to export multiple symbols with same name: %s.' % - full_api_name) - self._dest_import_to_id[full_api_name] = symbol_id - - if not dest_module_name and dest_name.startswith('_'): - self._underscore_names_in_root.append(dest_name) - - # The same symbol can be available in multiple modules. - # We store all possible ways of importing this symbol and later pick just - # one. - self.module_imports[dest_module_name][full_api_name].add(import_str) - - def build(self): - """Get a map from destination module to __init__.py code for that module. - - Returns: - A dictionary where - key: (string) destination module (for e.g. tf or tf.consts). - value: (string) text that should be in __init__.py files for - corresponding modules. - """ - module_text_map = {} - for dest_module, dest_name_to_imports in self.module_imports.items(): - # Sort all possible imports for a symbol and pick the first one. - imports_list = [ - sorted(imports)[0] - for _, imports in dest_name_to_imports.items()] - module_text_map[dest_module] = '\n'.join(sorted(imports_list)) - - # Expose exported symbols with underscores in root module - # since we import from it using * import. - underscore_names_str = ', '.join( - '\'%s\'' % name for name in self._underscore_names_in_root) - # We will always generate a root __init__.py file to let us handle * - # imports consistently. Be sure to have a root __init__.py file listed in - # the script outputs. - module_text_map[''] = module_text_map.get('', '') + ''' -_names_with_underscore = [%s] -__all__ = [_s for _s in dir() if not _s.startswith('_')] -__all__.extend([_s for _s in _names_with_underscore]) -__all__.remove('print_function') -''' % underscore_names_str - - return module_text_map - - -def get_api_init_text(package, output_package, api_name): - """Get a map from destination module to __init__.py code for that module. - - Args: - package: Base python package containing python with target tf_export - decorators. - api_name: API you want to generate (e.g. `tensorflow` or `estimator`). - - Returns: - A dictionary where - key: (string) destination module (for e.g. tf or tf.consts). - value: (string) text that should be in __init__.py files for - corresponding modules. - """ - module_code_builder = _ModuleInitCodeBuilder() - - # Traverse over everything imported above. Specifically, - # we want to traverse over TensorFlow Python modules. - for module in list(sys.modules.values()): - # Only look at tensorflow modules. - if (not module or not hasattr(module, '__name__') or - module.__name__ is None or package not in module.__name__): - continue - # Do not generate __init__.py files for contrib modules for now. - if '.contrib.' in module.__name__ or module.__name__.endswith('.contrib'): - continue - - for module_contents_name in dir(module): - if (module.__name__ + '.' + module_contents_name - in _SYMBOLS_TO_SKIP_EXPLICITLY): - continue - attr = getattr(module, module_contents_name) - - # If attr is _tf_api_constants attribute, then add the constants. - if module_contents_name == API_ATTRS[api_name].constants: - for exports, value in attr: - for export in exports: - names = export.split('.') - dest_module = '.'.join(names[:-1]) - module_code_builder.add_import( - -1, dest_module, module.__name__, value, names[-1]) - continue - - _, attr = tf_decorator.unwrap(attr) - # If attr is a symbol with _tf_api_names attribute, then - # add import for it. - if (hasattr(attr, '__dict__') and - API_ATTRS[api_name].names in attr.__dict__): - for export in getattr(attr, API_ATTRS[api_name].names): # pylint: disable=protected-access - names = export.split('.') - dest_module = '.'.join(names[:-1]) - module_code_builder.add_import( - id(attr), dest_module, module.__name__, module_contents_name, - names[-1]) - - # Import all required modules in their parent modules. - # For e.g. if we import 'foo.bar.Value'. Then, we also - # import 'bar' in 'foo'. - imported_modules = set(module_code_builder.module_imports.keys()) - for module in imported_modules: - if not module: - continue - module_split = module.split('.') - parent_module = '' # we import submodules in their parent_module - - for submodule_index in range(len(module_split)): - if submodule_index > 0: - parent_module += ('.' + module_split[submodule_index-1] if parent_module - else module_split[submodule_index-1]) - import_from = output_package - if submodule_index > 0: - import_from += '.' + '.'.join(module_split[:submodule_index]) - module_code_builder.add_import( - -1, parent_module, import_from, - module_split[submodule_index], module_split[submodule_index]) - - return module_code_builder.build() - - -def get_module(dir_path, relative_to_dir): - """Get module that corresponds to path relative to relative_to_dir. - - Args: - dir_path: Path to directory. - relative_to_dir: Get module relative to this directory. - - Returns: - Name of module that corresponds to the given directory. - """ - dir_path = dir_path[len(relative_to_dir):] - # Convert path separators to '/' for easier parsing below. - dir_path = dir_path.replace(os.sep, '/') - return dir_path.replace('/', '.').strip('.') - - -def get_module_docstring(module_name, package, api_name): - """Get docstring for the given module. - - This method looks for docstring in the following order: - 1. Checks if module has a docstring specified in doc_srcs. - 2. Checks if module has a docstring source module specified - in doc_srcs. If it does, gets docstring from that module. - 3. Checks if module with module_name exists under base package. - If it does, gets docstring from that module. - 4. Returns a default docstring. - - Args: - module_name: module name relative to tensorflow - (excluding 'tensorflow.' prefix) to get a docstring for. - package: Base python package containing python with target tf_export - decorators. - api_name: API you want to generate (e.g. `tensorflow` or `estimator`). - - Returns: - One-line docstring to describe the module. - """ - # Module under base package to get a docstring from. - docstring_module_name = module_name - - doc_sources = doc_srcs.get_doc_sources(api_name) - - if module_name in doc_sources: - docsrc = doc_sources[module_name] - if docsrc.docstring: - return docsrc.docstring - if docsrc.docstring_module_name: - docstring_module_name = docsrc.docstring_module_name - - docstring_module_name = package + '.' + docstring_module_name - if (docstring_module_name in sys.modules and - sys.modules[docstring_module_name].__doc__): - return sys.modules[docstring_module_name].__doc__ - - return 'Public API for tf.%s namespace.' % module_name - - -def create_api_files( - output_files, package, root_init_template, output_dir, output_package, - api_name): - """Creates __init__.py files for the Python API. - - Args: - output_files: List of __init__.py file paths to create. - Each file must be under api/ directory. - package: Base python package containing python with target tf_export - decorators. - root_init_template: Template for top-level __init__.py file. - "#API IMPORTS PLACEHOLDER" comment in the template file will be replaced - with imports. - output_dir: output API root directory. - api_name: API you want to generate (e.g. `tensorflow` or `estimator`). - - Raises: - ValueError: if an output file is not under api/ directory, - or output_files list is missing a required file. - """ - module_name_to_file_path = {} - for output_file in output_files: - module_name = get_module(os.path.dirname(output_file), output_dir) - module_name_to_file_path[module_name] = os.path.normpath(output_file) - - # Create file for each expected output in genrule. - for module, file_path in module_name_to_file_path.items(): - if not os.path.isdir(os.path.dirname(file_path)): - os.makedirs(os.path.dirname(file_path)) - open(file_path, 'a').close() - - module_text_map = get_api_init_text(package, output_package, api_name) - - # Add imports to output files. - missing_output_files = [] - for module, text in module_text_map.items(): - # Make sure genrule output file list is in sync with API exports. - if module not in module_name_to_file_path: - module_file_path = '"%s/__init__.py"' % ( - module.replace('.', '/')) - missing_output_files.append(module_file_path) - continue - contents = '' - if module or not root_init_template: - contents = ( - _GENERATED_FILE_HEADER % - get_module_docstring(module, package, api_name) + - text + _GENERATED_FILE_FOOTER) - else: - # Read base init file - with open(root_init_template, 'r') as root_init_template_file: - contents = root_init_template_file.read() - contents = contents.replace('# API IMPORTS PLACEHOLDER', text) - with open(module_name_to_file_path[module], 'w') as fp: - fp.write(contents) - - if missing_output_files: - raise ValueError( - 'Missing outputs for python_api_gen genrule:\n%s.' - 'Make sure all required outputs are in the ' - 'tensorflow/tools/api/generator/api_gen.bzl file.' % - ',\n'.join(sorted(missing_output_files))) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - 'outputs', metavar='O', type=str, nargs='+', - help='If a single file is passed in, then we we assume it contains a ' - 'semicolon-separated list of Python files that we expect this script to ' - 'output. If multiple files are passed in, then we assume output files ' - 'are listed directly as arguments.') - parser.add_argument( - '--package', default=_DEFAULT_PACKAGE, type=str, - help='Base package that imports modules containing the target tf_export ' - 'decorators.') - parser.add_argument( - '--root_init_template', default='', type=str, - help='Template for top level __init__.py file. ' - '"#API IMPORTS PLACEHOLDER" comment will be replaced with imports.') - parser.add_argument( - '--apidir', type=str, required=True, - help='Directory where generated output files are placed. ' - 'gendir should be a prefix of apidir. Also, apidir ' - 'should be a prefix of every directory in outputs.') - parser.add_argument( - '--apiname', required=True, type=str, - choices=API_ATTRS.keys(), - help='The API you want to generate.') - parser.add_argument( - '--output_package', default='tensorflow', type=str, - help='Root output package.') - - args = parser.parse_args() - - if len(args.outputs) == 1: - # If we only get a single argument, then it must be a file containing - # list of outputs. - with open(args.outputs[0]) as output_list_file: - outputs = [line.strip() for line in output_list_file.read().split(';')] - else: - outputs = args.outputs - - # Populate `sys.modules` with modules containing tf_export(). - importlib.import_module(args.package) - create_api_files(outputs, args.package, args.root_init_template, - args.apidir, args.output_package, args.apiname) - - -if __name__ == '__main__': - main() diff --git a/tensorflow/tools/api/generator/create_python_api_test.py b/tensorflow/tools/api/generator/create_python_api_test.py deleted file mode 100644 index 1a7187463a..0000000000 --- a/tensorflow/tools/api/generator/create_python_api_test.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Tests for create_python_api.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import imp -import sys - -from tensorflow.python.platform import test -from tensorflow.python.util.tf_export import tf_export -from tensorflow.tools.api.generator import create_python_api - - -@tf_export('test_op', 'test_op1') -def test_op(): - pass - - -@tf_export('TestClass', 'NewTestClass') -class TestClass(object): - pass - - -_TEST_CONSTANT = 5 -_MODULE_NAME = 'tensorflow.python.test_module' - - -class CreatePythonApiTest(test.TestCase): - - def setUp(self): - # Add fake op to a module that has 'tensorflow' in the name. - sys.modules[_MODULE_NAME] = imp.new_module(_MODULE_NAME) - setattr(sys.modules[_MODULE_NAME], 'test_op', test_op) - setattr(sys.modules[_MODULE_NAME], 'TestClass', TestClass) - test_op.__module__ = _MODULE_NAME - TestClass.__module__ = _MODULE_NAME - tf_export('consts._TEST_CONSTANT').export_constant( - _MODULE_NAME, '_TEST_CONSTANT') - - def tearDown(self): - del sys.modules[_MODULE_NAME] - - def testFunctionImportIsAdded(self): - imports = create_python_api.get_api_init_text( - package=create_python_api._DEFAULT_PACKAGE, - output_package='tensorflow', - api_name='tensorflow') - expected_import = ( - 'from tensorflow.python.test_module ' - 'import test_op as test_op1') - self.assertTrue( - expected_import in str(imports), - msg='%s not in %s' % (expected_import, str(imports))) - - expected_import = ('from tensorflow.python.test_module ' - 'import test_op') - self.assertTrue( - expected_import in str(imports), - msg='%s not in %s' % (expected_import, str(imports))) - - def testClassImportIsAdded(self): - imports = create_python_api.get_api_init_text( - package=create_python_api._DEFAULT_PACKAGE, - output_package='tensorflow', - api_name='tensorflow') - expected_import = ('from tensorflow.python.test_module ' - 'import TestClass') - self.assertTrue( - 'TestClass' in str(imports), - msg='%s not in %s' % (expected_import, str(imports))) - - def testConstantIsAdded(self): - imports = create_python_api.get_api_init_text( - package=create_python_api._DEFAULT_PACKAGE, - output_package='tensorflow', - api_name='tensorflow') - expected = ('from tensorflow.python.test_module ' - 'import _TEST_CONSTANT') - self.assertTrue(expected in str(imports), - msg='%s not in %s' % (expected, str(imports))) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/tools/api/generator/doc_srcs.py b/tensorflow/tools/api/generator/doc_srcs.py deleted file mode 100644 index ad1988494d..0000000000 --- a/tensorflow/tools/api/generator/doc_srcs.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Specifies sources of doc strings for API modules.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections - -from tensorflow.python.util import tf_export - - -# Specifies docstring source for a module. -# Only one of docstring or docstring_module_name should be set. -# * If docstring is set, then we will use this docstring when -# for the module. -# * If docstring_module_name is set, then we will copy the docstring -# from docstring source module. -DocSource = collections.namedtuple( - 'DocSource', ['docstring', 'docstring_module_name']) -# Each attribute of DocSource is optional. -DocSource.__new__.__defaults__ = (None,) * len(DocSource._fields) - -_TENSORFLOW_DOC_SOURCES = { - 'app': DocSource(docstring_module_name='platform.app'), - 'compat': DocSource(docstring_module_name='util.compat'), - 'distributions': DocSource( - docstring_module_name='ops.distributions.distributions'), - 'bitwise': DocSource(docstring_module_name='ops.bitwise_ops'), - 'errors': DocSource(docstring_module_name='framework.errors'), - 'gfile': DocSource(docstring_module_name='platform.gfile'), - 'graph_util': DocSource(docstring_module_name='framework.graph_util'), - 'image': DocSource(docstring_module_name='ops.image_ops'), - 'keras.estimator': DocSource(docstring_module_name='keras.estimator'), - 'linalg': DocSource(docstring_module_name='ops.linalg_ops'), - 'logging': DocSource(docstring_module_name='ops.logging_ops'), - 'losses': DocSource(docstring_module_name='ops.losses.losses'), - 'manip': DocSource(docstring_module_name='ops.manip_ops'), - 'math': DocSource(docstring_module_name='ops.math_ops'), - 'metrics': DocSource(docstring_module_name='ops.metrics'), - 'nn': DocSource(docstring_module_name='ops.nn_ops'), - 'nn.rnn_cell': DocSource(docstring_module_name='ops.rnn_cell'), - 'python_io': DocSource(docstring_module_name='lib.io.python_io'), - 'resource_loader': DocSource( - docstring_module_name='platform.resource_loader'), - 'sets': DocSource(docstring_module_name='ops.sets'), - 'sparse': DocSource(docstring_module_name='ops.sparse_ops'), - 'spectral': DocSource(docstring_module_name='ops.spectral_ops'), - 'strings': DocSource(docstring_module_name='ops.string_ops'), - 'sysconfig': DocSource(docstring_module_name='platform.sysconfig'), - 'test': DocSource(docstring_module_name='platform.test'), - 'train': DocSource(docstring_module_name='training.training'), - 'train.queue_runner': DocSource( - docstring_module_name='training.queue_runner'), -} - -_ESTIMATOR_DOC_SOURCES = { - 'estimator': DocSource( - docstring_module_name='estimator_lib'), - 'estimator.export': DocSource( - docstring_module_name='export.export_lib'), - 'estimator.inputs': DocSource( - docstring_module_name='inputs.inputs'), -} - - -def get_doc_sources(api_name): - """Get a map from module to a DocSource object. - - Args: - api_name: API you want to generate (e.g. `tensorflow` or `estimator`). - - Returns: - Map from module name to DocSource object. - """ - if api_name == tf_export.TENSORFLOW_API_NAME: - return _TENSORFLOW_DOC_SOURCES - if api_name == tf_export.ESTIMATOR_API_NAME: - return _ESTIMATOR_DOC_SOURCES - return {} diff --git a/tensorflow/tools/api/generator/doc_srcs_test.py b/tensorflow/tools/api/generator/doc_srcs_test.py deleted file mode 100644 index dbff904abe..0000000000 --- a/tensorflow/tools/api/generator/doc_srcs_test.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Tests for tensorflow.tools.api.generator.doc_srcs.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -import importlib -import sys - -from tensorflow.python.platform import test -from tensorflow.tools.api.generator import doc_srcs - - -FLAGS = None - - -class DocSrcsTest(test.TestCase): - - def testModulesAreValidAPIModules(self): - for module_name in doc_srcs.get_doc_sources(FLAGS.api_name): - # Convert module_name to corresponding __init__.py file path. - file_path = module_name.replace('.', '/') - if file_path: - file_path += '/' - file_path += '__init__.py' - - self.assertIn( - file_path, FLAGS.outputs, - msg='%s is not a valid API module' % module_name) - - def testHaveDocstringOrDocstringModule(self): - for module_name, docsrc in doc_srcs.get_doc_sources(FLAGS.api_name).items(): - self.assertFalse( - docsrc.docstring and docsrc.docstring_module_name, - msg=('%s contains DocSource has both a docstring and a ' - 'docstring_module_name. Only one of "docstring" or ' - '"docstring_module_name" should be set.') % (module_name)) - - def testDocstringModulesAreValidModules(self): - for _, docsrc in doc_srcs.get_doc_sources(FLAGS.api_name).items(): - if docsrc.docstring_module_name: - doc_module_name = '.'.join([ - FLAGS.package, docsrc.docstring_module_name]) - self.assertIn( - doc_module_name, sys.modules, - msg=('docsources_module %s is not a valid module under %s.' % - (docsrc.docstring_module_name, FLAGS.package))) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - 'outputs', metavar='O', type=str, nargs='+', - help='create_python_api output files.') - parser.add_argument( - '--package', type=str, - help='Base package that imports modules containing the target tf_export ' - 'decorators.') - parser.add_argument( - '--api_name', type=str, - help='API name: tensorflow or estimator') - FLAGS, unparsed = parser.parse_known_args() - - importlib.import_module(FLAGS.package) - - # Now update argv, so that unittest library does not get confused. - sys.argv = [sys.argv[0]] + unparsed - test.main() diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index c9d53f46c3..ac252143d7 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -104,6 +104,7 @@ COMMON_PIP_DEPS = [ "//tensorflow/python/kernel_tests/testdata:self_adjoint_eig_op_test_files", "//tensorflow/python/saved_model:saved_model", "//tensorflow/python/tools:tools_pip", + "//tensorflow/python/tools/api/generator:create_python_api", "//tensorflow/python:test_ops", "//tensorflow/tools/dist_test/server:grpc_tensorflow_server", ] -- cgit v1.2.3 From 9ba6943a1dbbc415b72835517ad58808ca6a6a3d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 12:04:34 -0700 Subject: Support narrow_range attr on FakeQuant nodes PiperOrigin-RevId: 204339562 --- tensorflow/contrib/lite/builtin_op_data.h | 4 + tensorflow/contrib/lite/kernels/fake_quant.cc | 11 ++ .../lite/kernels/internal/quantization_util.h | 10 +- tensorflow/contrib/lite/kernels/register.cc | 2 +- tensorflow/contrib/lite/model.cc | 1 + tensorflow/contrib/lite/schema/schema.fbs | 4 + tensorflow/contrib/lite/schema/schema_generated.h | 23 ++++- tensorflow/contrib/lite/toco/BUILD | 3 +- tensorflow/contrib/lite/toco/export_tensorflow.cc | 3 + .../lite/toco/graph_transformations/dequantize.cc | 1 + .../graph_transformations/graph_transformations.h | 3 +- .../make_initial_dequantize_operator.cc | 11 +- .../propagate_fake_quant_num_bits.cc | 18 +--- .../graph_transformations/quantization_util.cc | 69 ++++++++----- .../toco/graph_transformations/quantization_util.h | 20 +--- .../lite/toco/graph_transformations/quantize.cc | 12 ++- .../toco/graph_transformations/quantize_weights.cc | 6 +- ...rray_minmax_and_narrow_range_from_fake_quant.cc | 78 ++++++++++++++ .../read_fake_quant_min_max.cc | 112 --------------------- .../resolve_constant_fake_quant.cc | 46 ++++++++- .../resolve_fake_quant_args_from_vars.cc | 80 +++++++++++++++ tensorflow/contrib/lite/toco/import_tensorflow.cc | 6 ++ tensorflow/contrib/lite/toco/model.h | 35 +++++++ tensorflow/contrib/lite/toco/tflite/operator.cc | 10 +- tensorflow/contrib/lite/toco/toco_tooling.cc | 3 +- 25 files changed, 367 insertions(+), 204 deletions(-) create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/read_array_minmax_and_narrow_range_from_fake_quant.cc delete mode 100644 tensorflow/contrib/lite/toco/graph_transformations/read_fake_quant_min_max.cc create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/resolve_fake_quant_args_from_vars.cc diff --git a/tensorflow/contrib/lite/builtin_op_data.h b/tensorflow/contrib/lite/builtin_op_data.h index a58dde9a7b..a24aaad7dd 100644 --- a/tensorflow/contrib/lite/builtin_op_data.h +++ b/tensorflow/contrib/lite/builtin_op_data.h @@ -268,9 +268,13 @@ typedef struct { } TfLiteShapeParams; typedef struct { + // Parameters supported by version 1: float min; float max; int num_bits; + + // Parameters supported by version 2: + bool narrow_range; } TfLiteFakeQuantParams; #ifdef __cplusplus diff --git a/tensorflow/contrib/lite/kernels/fake_quant.cc b/tensorflow/contrib/lite/kernels/fake_quant.cc index f8927a0799..0ef1a50b30 100644 --- a/tensorflow/contrib/lite/kernels/fake_quant.cc +++ b/tensorflow/contrib/lite/kernels/fake_quant.cc @@ -44,6 +44,17 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + const auto* params = + reinterpret_cast(node->builtin_data); + + if (params->narrow_range) { + context->ReportError( + context, + "narrow_range FakeQuant is not currently supported at runtime. " + "narrow_range is only meant to be applied to weights, not activations"); + return kTfLiteError; + } + OpContext op_context(context, node); TfLiteIntArray* output_dims = TfLiteIntArrayCopy(op_context.input->dims); op_context.output->type = op_context.input->type; diff --git a/tensorflow/contrib/lite/kernels/internal/quantization_util.h b/tensorflow/contrib/lite/kernels/internal/quantization_util.h index 525857a2e6..9b3f1823dc 100644 --- a/tensorflow/contrib/lite/kernels/internal/quantization_util.h +++ b/tensorflow/contrib/lite/kernels/internal/quantization_util.h @@ -28,8 +28,9 @@ namespace tflite { // Given the min and max values of a float array, return // reasonable quantization parameters to use for this array. template -QuantizationParams ChooseQuantizationParams(double rmin, double rmax) { - const T qmin = std::numeric_limits::min(); +QuantizationParams ChooseQuantizationParams(double rmin, double rmax, + bool narrow_range) { + const T qmin = std::numeric_limits::min() + (narrow_range ? 1 : 0); const T qmax = std::numeric_limits::max(); const double qmin_double = qmin; const double qmax_double = qmax; @@ -97,6 +98,11 @@ QuantizationParams ChooseQuantizationParams(double rmin, double rmax) { return quantization_params; } +template +QuantizationParams ChooseQuantizationParams(double rmin, double rmax) { + return ChooseQuantizationParams(rmin, rmax, false); +} + // Converts a floating-point number to an integer. For all inputs x where // static_cast(x) is legal according to the C++ standard, the result // is identical to that cast (i.e. the result is x with its fractional part diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc index 1994e85ce3..22a507e6a4 100644 --- a/tensorflow/contrib/lite/kernels/register.cc +++ b/tensorflow/contrib/lite/kernels/register.cc @@ -190,7 +190,7 @@ BuiltinOpResolver::BuiltinOpResolver() { AddBuiltin(BuiltinOperator_RSQRT, Register_RSQRT()); AddBuiltin(BuiltinOperator_SHAPE, Register_SHAPE()); AddBuiltin(BuiltinOperator_POW, Register_POW()); - AddBuiltin(BuiltinOperator_FAKE_QUANT, Register_FAKE_QUANT()); + AddBuiltin(BuiltinOperator_FAKE_QUANT, Register_FAKE_QUANT(), 1, 2); // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that // custom ops aren't always included by default. diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc index 93b3df98f3..71e38c3f13 100644 --- a/tensorflow/contrib/lite/model.cc +++ b/tensorflow/contrib/lite/model.cc @@ -715,6 +715,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type, params->min = schema_params->min(); params->max = schema_params->max(); params->num_bits = schema_params->num_bits(); + params->narrow_range = schema_params->narrow_range(); } *builtin_data = static_cast(params); break; diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs index 17ea26052d..64830b1dc3 100644 --- a/tensorflow/contrib/lite/schema/schema.fbs +++ b/tensorflow/contrib/lite/schema/schema.fbs @@ -526,9 +526,13 @@ table PowOptions { } table FakeQuantOptions { + // Parameters supported by version 1: min:float; max:float; num_bits:int; + + // Parameters supported by version 2: + narrow_range:bool; } // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h index 37489ebc68..c0b57039cb 100755 --- a/tensorflow/contrib/lite/schema/schema_generated.h +++ b/tensorflow/contrib/lite/schema/schema_generated.h @@ -5213,10 +5213,12 @@ struct FakeQuantOptionsT : public flatbuffers::NativeTable { float min; float max; int32_t num_bits; + bool narrow_range; FakeQuantOptionsT() : min(0.0f), max(0.0f), - num_bits(0) { + num_bits(0), + narrow_range(false) { } }; @@ -5225,7 +5227,8 @@ struct FakeQuantOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { enum { VT_MIN = 4, VT_MAX = 6, - VT_NUM_BITS = 8 + VT_NUM_BITS = 8, + VT_NARROW_RANGE = 10 }; float min() const { return GetField(VT_MIN, 0.0f); @@ -5236,11 +5239,15 @@ struct FakeQuantOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { int32_t num_bits() const { return GetField(VT_NUM_BITS, 0); } + bool narrow_range() const { + return GetField(VT_NARROW_RANGE, 0) != 0; + } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && VerifyField(verifier, VT_MIN) && VerifyField(verifier, VT_MAX) && VerifyField(verifier, VT_NUM_BITS) && + VerifyField(verifier, VT_NARROW_RANGE) && verifier.EndTable(); } FakeQuantOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const; @@ -5260,6 +5267,9 @@ struct FakeQuantOptionsBuilder { void add_num_bits(int32_t num_bits) { fbb_.AddElement(FakeQuantOptions::VT_NUM_BITS, num_bits, 0); } + void add_narrow_range(bool narrow_range) { + fbb_.AddElement(FakeQuantOptions::VT_NARROW_RANGE, static_cast(narrow_range), 0); + } explicit FakeQuantOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); @@ -5276,11 +5286,13 @@ inline flatbuffers::Offset CreateFakeQuantOptions( flatbuffers::FlatBufferBuilder &_fbb, float min = 0.0f, float max = 0.0f, - int32_t num_bits = 0) { + int32_t num_bits = 0, + bool narrow_range = false) { FakeQuantOptionsBuilder builder_(_fbb); builder_.add_num_bits(num_bits); builder_.add_max(max); builder_.add_min(min); + builder_.add_narrow_range(narrow_range); return builder_.Finish(); } @@ -7896,6 +7908,7 @@ inline void FakeQuantOptions::UnPackTo(FakeQuantOptionsT *_o, const flatbuffers: { auto _e = min(); _o->min = _e; }; { auto _e = max(); _o->max = _e; }; { auto _e = num_bits(); _o->num_bits = _e; }; + { auto _e = narrow_range(); _o->narrow_range = _e; }; } inline flatbuffers::Offset FakeQuantOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const FakeQuantOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) { @@ -7909,11 +7922,13 @@ inline flatbuffers::Offset CreateFakeQuantOptions(flatbuffers: auto _min = _o->min; auto _max = _o->max; auto _num_bits = _o->num_bits; + auto _narrow_range = _o->narrow_range; return tflite::CreateFakeQuantOptions( _fbb, _min, _max, - _num_bits); + _num_bits, + _narrow_range); } inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const { diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD index 209dce56cb..2c469c0e75 100644 --- a/tensorflow/contrib/lite/toco/BUILD +++ b/tensorflow/contrib/lite/toco/BUILD @@ -212,7 +212,7 @@ cc_library( "graph_transformations/quantization_util.h", "graph_transformations/quantize.cc", "graph_transformations/quantize_weights.cc", - "graph_transformations/read_fake_quant_min_max.cc", + "graph_transformations/read_array_minmax_and_narrow_range_from_fake_quant.cc", "graph_transformations/remove_final_dequantize_op.cc", "graph_transformations/remove_tensorflow_assert.cc", "graph_transformations/remove_tensorflow_identity.cc", @@ -245,6 +245,7 @@ cc_library( "graph_transformations/resolve_constant_strided_slice.cc", "graph_transformations/resolve_constant_transpose.cc", "graph_transformations/resolve_constant_unary.cc", + "graph_transformations/resolve_fake_quant_args_from_vars.cc", "graph_transformations/resolve_mean_attributes.cc", "graph_transformations/resolve_multiply_by_zero.cc", "graph_transformations/resolve_pad_attributes.cc", diff --git a/tensorflow/contrib/lite/toco/export_tensorflow.cc b/tensorflow/contrib/lite/toco/export_tensorflow.cc index a08cdbfba6..bf9a51a525 100644 --- a/tensorflow/contrib/lite/toco/export_tensorflow.cc +++ b/tensorflow/contrib/lite/toco/export_tensorflow.cc @@ -884,6 +884,9 @@ void ConvertFakeQuantOperator(const FakeQuantOperator& src_op, if (src_op.num_bits) { (*fakequant_op->mutable_attr())["num_bits"].set_i(src_op.num_bits); } + if (src_op.narrow_range) { + (*fakequant_op->mutable_attr())["narrow_range"].set_b(src_op.narrow_range); + } } void ConvertMaxPoolOperator(const MaxPoolOperator& src_op, diff --git a/tensorflow/contrib/lite/toco/graph_transformations/dequantize.cc b/tensorflow/contrib/lite/toco/graph_transformations/dequantize.cc index 2c7ffe4884..1688586733 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/dequantize.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/dequantize.cc @@ -159,6 +159,7 @@ bool DequantizeArray(const string& array_name, new_array.GetOrCreateMinMax() = array->GetMinMax(); fakequant_op->minmax.reset(new MinMax); *fakequant_op->minmax = array->GetMinMax(); + fakequant_op->narrow_range = array->narrow_range; if (must_insert_fakequant_before) { for (const auto& op : model->operators) { for (string& output : op->outputs) { diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h index 8cd1298bca..7cc9bb75d7 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h +++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h @@ -159,7 +159,7 @@ DECLARE_GRAPH_TRANSFORMATION(ResolveConstantBinaryOperator) DECLARE_GRAPH_TRANSFORMATION(ResolveConstantUnaryOperator) DECLARE_GRAPH_TRANSFORMATION(CreateIm2colArrays) DECLARE_GRAPH_TRANSFORMATION(DropIm2colArrays) -DECLARE_GRAPH_TRANSFORMATION(ReadFakeQuantMinMax) +DECLARE_GRAPH_TRANSFORMATION(ReadArrayMinmaxAndNarrowRangeFromFakeQuant) DECLARE_GRAPH_TRANSFORMATION(ReorderElementwiseUnary) DECLARE_GRAPH_TRANSFORMATION(ReorderReshapeTranspose) DECLARE_GRAPH_TRANSFORMATION(ResolveReorderAxes) @@ -194,6 +194,7 @@ DECLARE_GRAPH_TRANSFORMATION(ResolveMultiplyByZero) DECLARE_GRAPH_TRANSFORMATION(Dequantize) DECLARE_GRAPH_TRANSFORMATION(UnpartitionEmbeddingLookup) DECLARE_GRAPH_TRANSFORMATION(ShuffleFCWeights) +DECLARE_GRAPH_TRANSFORMATION(ResolveFakeQuantArgsFromVars) class PropagateDefaultMinMax : public GraphTransformation { public: diff --git a/tensorflow/contrib/lite/toco/graph_transformations/make_initial_dequantize_operator.cc b/tensorflow/contrib/lite/toco/graph_transformations/make_initial_dequantize_operator.cc index 45d9f73a1e..f684de08ab 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/make_initial_dequantize_operator.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/make_initial_dequantize_operator.cc @@ -85,15 +85,8 @@ bool AddDequantizeOperatorToInput(const string& input_name, const Operator* op, dequantized_input_minmax = input_minmax; auto& input_qparams = input_array.GetOrCreateQuantizationParams(); input_array.data_type = input_array.final_data_type; - if (input_array.data_type == ArrayDataType::kUint8) { - GetQuantizationParamsFromMinMax(input_minmax, - &input_qparams); - } else if (input_array.data_type == ArrayDataType::kInt16) { - GetQuantizationParamsFromMinMax(input_minmax, - &input_qparams); - } else { - LOG(FATAL) << "unhandled data type"; - } + ChooseQuantizationParamsForArrayAndQuantizedDataType( + input_array, input_array.data_type, &input_qparams); transformation->AddMessageF( "Created %s" diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fake_quant_num_bits.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fake_quant_num_bits.cc index 53fc87da7b..3ad6b0ec6f 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fake_quant_num_bits.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fake_quant_num_bits.cc @@ -66,24 +66,10 @@ bool ChangeArrayDataType(GraphTransformation* transformation, Array* array, "Rescaling min/max from %g,%g (%s) to %g,%g (%s)", array_minmax.min, array_minmax.max, ArrayDataTypeName(array->data_type), min, max, ArrayDataTypeName(new_data_type)); - array_minmax.min = min; array_minmax.max = max; - switch (new_data_type) { - case ArrayDataType::kUint8: - GetQuantizationParamsFromMinMax( - array_minmax, array->quantization_params.get()); - break; - case ArrayDataType::kInt16: - GetQuantizationParamsFromMinMax( - array_minmax, array->quantization_params.get()); - break; - default: - CHECK(false) << "Unsupported quantized data type: " - << ArrayDataTypeName(new_data_type); - return false; - } - + ChooseQuantizationParamsForArrayAndQuantizedDataType( + *array, new_data_type, array->quantization_params.get()); // Directly change the type as the array was already quantized. array->data_type = new_data_type; changed = true; diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.cc b/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.cc index d74cad9a62..44733391f5 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.cc @@ -74,46 +74,54 @@ ArrayDataType GetQuantizedDataType(const Array& array, } } -void GetQuantizationParams(ArrayDataType data_type, const MinMax& minmax, - QuantizationParams* quantization_params) { - switch (data_type) { +template +void ChooseQuantizationParamsForArrayAndQuantizedDataType( + const Array& array, QuantizationParams* quantization_params) { + *quantization_params = ::tflite::ChooseQuantizationParams>( + array.minmax->min, array.minmax->max, array.narrow_range); +} + +void ChooseQuantizationParamsForArrayAndQuantizedDataType( + const Array& array, ArrayDataType quantized_data_type, + QuantizationParams* quantization_params) { + switch (quantized_data_type) { case ArrayDataType::kInt8: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType< + ArrayDataType::kInt8>(array, quantization_params); break; case ArrayDataType::kUint8: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType< + ArrayDataType::kUint8>(array, quantization_params); break; case ArrayDataType::kInt16: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType< + ArrayDataType::kInt16>(array, quantization_params); break; case ArrayDataType::kUint16: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType< + ArrayDataType::kUint16>(array, quantization_params); break; case ArrayDataType::kInt32: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType< + ArrayDataType::kInt32>(array, quantization_params); break; case ArrayDataType::kUint32: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType< + ArrayDataType::kUint32>(array, quantization_params); break; case ArrayDataType::kInt64: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType< + ArrayDataType::kInt64>(array, quantization_params); break; case ArrayDataType::kUint64: - GetQuantizationParamsFromMinMax( - minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType< + ArrayDataType::kUint64>(array, quantization_params); break; case ArrayDataType::kFloat: case ArrayDataType::kNone: default: LOG(FATAL) << "Unhandled final quantization type " - << static_cast(data_type); + << static_cast(quantized_data_type); } } @@ -121,8 +129,8 @@ namespace { template std::unique_ptr QuantizeBuffer( - const GenericBuffer& buffer, - const QuantizationParams& quantization_params) { + const Array& array, const QuantizationParams& quantization_params) { + const GenericBuffer& buffer = *array.buffer; const auto inverse_scale = 1. / quantization_params.scale; CHECK(buffer.type == ArrayDataType::kFloat); const auto& float_buffer = @@ -140,8 +148,15 @@ std::unique_ptr QuantizeBuffer( } else { scaled_val = quantization_params.zero_point + inverse_scale * src_val; } - quantized_buffer->data[i] = - tflite::SafeCast>(std::round(scaled_val)); + auto integer_val = tflite::SafeCast>(std::round(scaled_val)); + // In addition to its effect on the choice of quantization params upstream + // of here, narrow_range also means nudge the min quantized value by +1, + // so e.g. uint8 values get constrained to [1, 255]. + if (integer_val == std::numeric_limits>::min() && + array.narrow_range) { + integer_val++; + } + quantized_buffer->data[i] = integer_val; } return std::unique_ptr(quantized_buffer); } @@ -155,7 +170,7 @@ void QuantizeArray(GraphTransformation* transformation, Model* model, CHECK(!array.quantization_params); array.GetOrCreateQuantizationParams() = quantization_params; if (array.buffer) { - array.buffer = QuantizeBuffer(*array.buffer, quantization_params); + array.buffer = QuantizeBuffer(array, quantization_params); } array.data_type = A; array.final_data_type = A; @@ -210,8 +225,8 @@ bool IsArrayQuantizedRangeSubset(GraphTransformation* transformation, } else { // Work around cases where we are asking for this prior to the Quantize // transformation having added the quantization_params. - GetQuantizationParams(quantized_data_type, *array.minmax, - &quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType( + array, quantized_data_type, &quantization_params); transformation->AddMessageF( "No quantization params - infering from data type %s with minmax " "%g,%g as zero_point=%g, scale=%g", diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h b/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h index 79a2ce7e50..cf093c6f17 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h +++ b/tensorflow/contrib/lite/toco/graph_transformations/quantization_util.h @@ -38,21 +38,11 @@ bool GetQuantizedDataTypeNumericalRange(ArrayDataType data_type, ArrayDataType GetQuantizedDataType(const Array& array, ArrayDataType default_type); -// Returns the quantization params for the array with the given data type and -// minmax. -void GetQuantizationParams(ArrayDataType data_type, const MinMax& minmax, - QuantizationParams* quantization_params); - -// Returns the quantization params for the data type and minmax values. -template -void GetQuantizationParamsFromMinMax(const MinMax& minmax, - QuantizationParams* quantization_params) { - using Integer = DataType; - const double rmin = minmax.min; - const double rmax = minmax.max; - *quantization_params = - ::tflite::ChooseQuantizationParams(rmin, rmax); -} +// Chooses the quantization params for a given array and a given target +// quantized data type (which may not be the array's current data type). +void ChooseQuantizationParamsForArrayAndQuantizedDataType( + const Array& array, ArrayDataType quantized_data_type, + QuantizationParams* quantization_params); // Quantizes an array by setting its data type and (if constant) quantizing // all values in the array. diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc index 58885b4950..5be2757479 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc @@ -212,13 +212,15 @@ bool ChooseQuantizationForOperatorInput( if (op.type == OperatorType::kLstmCell) { if (input_index == LstmCellOperator::PREV_STATE_INPUT) { *quantized_data_type = ArrayDataType::kInt16; - GetQuantizationParams(*quantized_data_type, minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType( + array, *quantized_data_type, quantization_params); return true; } } *quantized_data_type = GetQuantizedDataType(array, ArrayDataType::kUint8); - GetQuantizationParams(*quantized_data_type, minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType( + array, *quantized_data_type, quantization_params); transformation->AddMessageF( "For input array %s with min=%g, max=%g, chose to quantize as %s (f=%s) " "with zero_point=%d, scale=%g", @@ -358,12 +360,14 @@ bool ChooseQuantizationForOperatorOutput( if (output_index == LstmCellOperator::STATE_OUTPUT || output_index == LstmCellOperator::ACTIV_TEMP) { *quantized_data_type = ArrayDataType::kInt16; - GetQuantizationParams(*quantized_data_type, minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType( + array, *quantized_data_type, quantization_params); return true; } } *quantized_data_type = GetQuantizedDataType(array, ArrayDataType::kUint8); - GetQuantizationParams(*quantized_data_type, minmax, quantization_params); + ChooseQuantizationParamsForArrayAndQuantizedDataType( + array, *quantized_data_type, quantization_params); transformation->AddMessageF( "For output array %s with min=%g, max=%g" ", chose to quantize as %s with zero_point=%d" diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantize_weights.cc b/tensorflow/contrib/lite/toco/graph_transformations/quantize_weights.cc index 88ea0945e7..7a8515f6d1 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/quantize_weights.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/quantize_weights.cc @@ -36,10 +36,8 @@ void GetQuantizationParamsFromArray(const Array& array, const std::vector& float_vals = array.GetBuffer().data; auto minmax = std::minmax_element(float_vals.begin(), float_vals.end()); - MinMax toco_minmax; - toco_minmax.min = *minmax.first; - toco_minmax.max = *minmax.second; - GetQuantizationParams(ArrayDataType::kUint8, toco_minmax, params); + *params = tflite::ChooseQuantizationParams( + *minmax.first, *minmax.second, array.narrow_range); } } // namespace diff --git a/tensorflow/contrib/lite/toco/graph_transformations/read_array_minmax_and_narrow_range_from_fake_quant.cc b/tensorflow/contrib/lite/toco/graph_transformations/read_array_minmax_and_narrow_range_from_fake_quant.cc new file mode 100644 index 0000000000..5b41c49bfa --- /dev/null +++ b/tensorflow/contrib/lite/toco/graph_transformations/read_array_minmax_and_narrow_range_from_fake_quant.cc @@ -0,0 +1,78 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include +#include +#include + +#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" +#include "tensorflow/contrib/lite/toco/model.h" +#include "tensorflow/contrib/lite/toco/tooling_util.h" +#include "tensorflow/core/platform/logging.h" + +namespace toco { + +namespace { + +bool ApplyAttrsToArray(GraphTransformation* transformation, Model* model, + const FakeQuantOperator& fq_op, + const string& array_name) { + bool changed = false; + auto& annotated_array = model->GetArray(array_name); + if (!annotated_array.minmax) { + const MinMax& minmax = *fq_op.minmax; + annotated_array.GetOrCreateMinMax() = minmax; + transformation->AddMessageF( + "Read min/max annotation for array %s: min=%g, max=%g", array_name, + minmax.min, minmax.max); + changed = true; + } + if (fq_op.narrow_range && !annotated_array.narrow_range) { + annotated_array.narrow_range = true; + transformation->AddMessageF("Read narrow_range annotation for array %s", + array_name); + changed = true; + } + return changed; +} + +} // end namespace + +bool ReadArrayMinmaxAndNarrowRangeFromFakeQuant::Run(Model* model, + std::size_t op_index) { + const auto fakequant_it = model->operators.begin() + op_index; + auto* fakequant_base_op = fakequant_it->get(); + if (fakequant_base_op->type != OperatorType::kFakeQuant) { + return false; + } + auto* fq_op = static_cast(fakequant_base_op); + + if (!fq_op->minmax) { + // Need to be resolved first by ResolveFakeQuantArgsFromVars. + return false; + } + + // At this point, this FakeQuantOperator should have a MinMax + // attached to it, and should only have 1 input (it should not have + // 2nd and 3rd input arrays giving min and max anymore). + CHECK(fq_op->minmax); + CHECK_EQ(1, fq_op->inputs.size()); + + return ApplyAttrsToArray(this, model, *fq_op, fq_op->inputs[0]) || + ApplyAttrsToArray(this, model, *fq_op, fq_op->outputs[0]); +} + +} // namespace toco diff --git a/tensorflow/contrib/lite/toco/graph_transformations/read_fake_quant_min_max.cc b/tensorflow/contrib/lite/toco/graph_transformations/read_fake_quant_min_max.cc deleted file mode 100644 index bdcca5b7ca..0000000000 --- a/tensorflow/contrib/lite/toco/graph_transformations/read_fake_quant_min_max.cc +++ /dev/null @@ -1,112 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include -#include -#include -#include -#include - -#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" -#include "tensorflow/contrib/lite/toco/model.h" -#include "tensorflow/contrib/lite/toco/tooling_util.h" -#include "tensorflow/core/platform/logging.h" - -namespace toco { - -namespace { - -bool ApplyMinMaxToArray(GraphTransformation* transformation, Model* model, - const MinMax& minmax, const string& array_name) { - auto& annotated_array = model->GetArray(array_name); - if (annotated_array.minmax) { - return false; - } - annotated_array.GetOrCreateMinMax() = minmax; - transformation->AddMessageF( - "Read min/max annotation for array %s: min=%g, max=%g", array_name, - minmax.min, minmax.max); - return true; -} - -} // end namespace - -bool ReadFakeQuantMinMax::Run(Model* model, std::size_t op_index) { - const auto fakequant_it = model->operators.begin() + op_index; - auto* fakequant_base_op = fakequant_it->get(); - if (fakequant_base_op->type != OperatorType::kFakeQuant) { - return false; - } - auto* fakequant_op = static_cast(fakequant_base_op); - - bool changed = false; - - if (!fakequant_op->minmax) { - CHECK_EQ(fakequant_op->inputs.size(), 3); - // We need to yield until the min and max parameters have been - // resolved to constant arrays. - for (int i = 1; i <= 2; i++) { - if (!IsConstantParameterArray(*model, fakequant_op->inputs[1])) { - return false; - } - } - - // Obtain the final min/max values - const auto& min_array = model->GetArray(fakequant_op->inputs[1]); - const auto& max_array = model->GetArray(fakequant_op->inputs[2]); - CHECK_EQ(RequiredBufferSizeForShape(min_array.shape()), 1); - CHECK_EQ(RequiredBufferSizeForShape(max_array.shape()), 1); - fakequant_op->minmax.reset(new MinMax); - MinMax& minmax = *fakequant_op->minmax; - minmax.min = min_array.GetBuffer().data[0]; - minmax.max = max_array.GetBuffer().data[0]; - // We always want [min, max] to contain 0. - if (minmax.min > 0 || minmax.max < 0) { - LOG(ERROR) << "For " << LogName(*fakequant_op) << " the MinMax range " - << "[" << minmax.min << ", " << minmax.max - << "] does not contain 0. " - << "Proceeding by tweaking it to contain 0, which will result " - "in poor accuracy."; - } - minmax.min = std::min(minmax.min, 0.); - minmax.max = std::max(minmax.max, 0.); - - // We won't use the input arrays that provided these min and max - // values, anymore. Delete them unless they are used by something - // else. - for (int i = 1; i <= 2; i++) { - if (CountOpsWithInput(*model, fakequant_op->inputs[i]) == 1) { - model->EraseArray(fakequant_op->inputs[i]); - } - } - fakequant_op->inputs.resize(1); - changed = true; - } - - // At this point, this FakeQuantOperator should have a MinMax - // attached to it, and should only have 1 input (it should not have - // 2nd and 3rd input arrays giving min and max anymore). - CHECK(fakequant_op->minmax); - CHECK_EQ(1, fakequant_op->inputs.size()); - - const MinMax& minmax = *fakequant_op->minmax; - - // Record the MinMax info on the input and output arrays - changed |= ApplyMinMaxToArray(this, model, minmax, fakequant_op->inputs[0]); - changed |= ApplyMinMaxToArray(this, model, minmax, fakequant_op->outputs[0]); - - return changed; -} - -} // namespace toco diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_fake_quant.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_fake_quant.cc index efb7bb2184..058f314b33 100644 --- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_fake_quant.cc +++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_fake_quant.cc @@ -25,6 +25,37 @@ limitations under the License. namespace toco { +template +void GetBoundsForQuantizedDataType(double* min, double* max) { + using limits = std::numeric_limits>; + *min = limits::min(); + *max = limits::max(); +} + +void GetBoundsForQuantizedDataType(ArrayDataType quantized_data_type, + double* min, double* max) { + switch (quantized_data_type) { + case ArrayDataType::kUint8: + return GetBoundsForQuantizedDataType(min, max); + case ArrayDataType::kInt8: + return GetBoundsForQuantizedDataType(min, max); + case ArrayDataType::kUint16: + return GetBoundsForQuantizedDataType(min, max); + case ArrayDataType::kInt16: + return GetBoundsForQuantizedDataType(min, max); + case ArrayDataType::kUint32: + return GetBoundsForQuantizedDataType(min, max); + case ArrayDataType::kInt32: + return GetBoundsForQuantizedDataType(min, max); + case ArrayDataType::kUint64: + return GetBoundsForQuantizedDataType(min, max); + case ArrayDataType::kInt64: + return GetBoundsForQuantizedDataType(min, max); + default: + LOG(FATAL) << "unhandled quantized data type"; + } +} + bool ResolveConstantFakeQuant::Run(Model* model, std::size_t op_index) { const auto fakequant_it = model->operators.begin() + op_index; const auto* fakequant_base_op = fakequant_it->get(); @@ -76,14 +107,21 @@ bool ResolveConstantFakeQuant::Run(Model* model, std::size_t op_index) { const int size = input_buffer.data.size(); output_buffer.data.resize(size); QuantizationParams qparams; - GetQuantizationParamsFromMinMax(*fakequant_op->minmax, - &qparams); + ChooseQuantizationParamsForArrayAndQuantizedDataType( + output_array, quantized_data_type, &qparams); + double quantized_min, quantized_max; + GetBoundsForQuantizedDataType(quantized_data_type, &quantized_min, + &quantized_max); + if (fakequant_op->narrow_range) { + quantized_min++; + } + for (int i = 0; i < size; i++) { const double src_val = input_buffer.data[i]; const double unclamped_quantized_val = std::round(qparams.zero_point + src_val / qparams.scale); - const double quantized_val = - std::min(255., std::max(0., unclamped_quantized_val)); + const double quantized_val = std::min( + quantized_max, std::max(quantized_min, unclamped_quantized_val)); const double dst_val = qparams.scale * (quantized_val - qparams.zero_point); output_buffer.data[i] = dst_val; } diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_fake_quant_args_from_vars.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_fake_quant_args_from_vars.cc new file mode 100644 index 0000000000..0dda1fd0b3 --- /dev/null +++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_fake_quant_args_from_vars.cc @@ -0,0 +1,80 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include +#include +#include + +#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h" +#include "tensorflow/contrib/lite/toco/model.h" +#include "tensorflow/contrib/lite/toco/tooling_util.h" +#include "tensorflow/core/platform/logging.h" + +namespace toco { + +bool ResolveFakeQuantArgsFromVars::Run(Model* model, std::size_t op_index) { + const auto fakequant_it = model->operators.begin() + op_index; + auto* fakequant_base_op = fakequant_it->get(); + if (fakequant_base_op->type != OperatorType::kFakeQuant) { + return false; + } + auto* fakequant_op = static_cast(fakequant_base_op); + + if (fakequant_op->minmax) { + // Already resolved. + return false; + } + + CHECK_EQ(fakequant_op->inputs.size(), 3); + // We need to yield until the min and max parameters have been + // resolved to constant arrays. + for (int i = 1; i <= 2; i++) { + if (!IsConstantParameterArray(*model, fakequant_op->inputs[i])) { + return false; + } + } + + // Obtain the final min/max values + const auto& min_array = model->GetArray(fakequant_op->inputs[1]); + const auto& max_array = model->GetArray(fakequant_op->inputs[2]); + CHECK_EQ(RequiredBufferSizeForShape(min_array.shape()), 1); + CHECK_EQ(RequiredBufferSizeForShape(max_array.shape()), 1); + fakequant_op->minmax.reset(new MinMax); + MinMax& minmax = *fakequant_op->minmax; + minmax.min = min_array.GetBuffer().data[0]; + minmax.max = max_array.GetBuffer().data[0]; + // We always want [min, max] to contain 0. + if (minmax.min > 0 || minmax.max < 0) { + LOG(ERROR) << "For " << LogName(*fakequant_op) << " the MinMax range " + << "[" << minmax.min << ", " << minmax.max + << "] does not contain 0. " + << "Proceeding by tweaking it to contain 0, which will result " + "in poor accuracy."; + } + minmax.min = std::min(minmax.min, 0.); + minmax.max = std::max(minmax.max, 0.); + + // We won't use the input arrays that provided these min and max + // values, anymore. Delete them unless they are used by something + // else. + for (int i = 1; i <= 2; i++) { + DeleteArrayIfUsedOnce(fakequant_op->inputs[i], model); + } + fakequant_op->inputs.resize(1); + return true; +} + +} // namespace toco diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc index bc439a2feb..ab3762e7ea 100644 --- a/tensorflow/contrib/lite/toco/import_tensorflow.cc +++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc @@ -755,6 +755,9 @@ tensorflow::Status ConvertFakeQuantWithMinMaxArgs( op->outputs.push_back(node.name()); // tf.fake_quant_with_min_max_args num_bits defaults to 8. op->num_bits = HasAttr(node, "num_bits") ? GetIntAttr(node, "num_bits") : 8; + if (HasAttr(node, "narrow_range")) { + op->narrow_range = GetBoolAttr(node, "narrow_range"); + } model->operators.emplace_back(op); return tensorflow::Status::OK(); } @@ -774,6 +777,9 @@ tensorflow::Status ConvertFakeQuantWithMinMaxVars( } op->outputs.push_back(node.name()); op->num_bits = HasAttr(node, "num_bits") ? GetIntAttr(node, "num_bits") : 8; + if (HasAttr(node, "narrow_range")) { + op->narrow_range = GetBoolAttr(node, "narrow_range"); + } model->operators.emplace_back(op); return tensorflow::Status::OK(); } diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h index 8660464fdb..d06a30b638 100644 --- a/tensorflow/contrib/lite/toco/model.h +++ b/tensorflow/contrib/lite/toco/model.h @@ -791,6 +791,7 @@ struct FakeQuantOperator : Operator { FakeQuantOperator() : Operator(OperatorType::kFakeQuant) {} std::unique_ptr minmax; int num_bits = 8; + bool narrow_range = false; }; // Element-wise division operator. @@ -1854,6 +1855,40 @@ struct Array { // If this is non-null, then these quantization parameters are to be used // to assign a meaning as real numbers to the elements of this array. std::unique_ptr quantization_params; + // narrow_range is a detail of how toco handles FakeQuant operators with + // narrow_range, see + // https://www.tensorflow.org/api_docs/python/tf/fake_quant_with_min_max_vars + // + // For more context about what that is useful for, see the big comment in + // graph_transformations/ensure_uint8_weights_safe_for_fast_int8_kernels.cc + // + // The narrow_range flag applies only to quantized arrays, and changes + // their quantization in the following way when it is set to 'true': + // 1. The computation of {zero_point, scale} from {min, max} needs to be + // amended so that the real min value will get quantized to + // (min_quantized_value + 1) instead of just (min_quantized_value). + // E.g. for uint8 quantization, the real min value should get quantized to + // the uint8 value 1, not 0. + // 2. Quantized values should get clamped to the interval + // [min_quantized_value + 1, max_value]. Equivalently, the + // min_quantized_value should get nudged to (min_quantized_value + 1). + // The reason why 1. does not imply 2. is that real values may not belong to + // the stated [min, max] interval. Concretely, weights recorded at the last + // learning step may not fall in the [min, max] interval recorded over + // previous learning steps, as the values evolve across learning steps. + // + // Rationale why this is directly a field on Array: + // - This can't be just a field on FakeQuantOperator, because + // FakeQuantOperators are gone (DropFakeQuant) before we get to using that + // information (Quantize). We need a place to store that bit in the interim. + // - This can't be in QuantizationParams because we need to record this + // ahead of quantization, and QuantizationParams are only created during + // quantization. + // - This could be in MinMax, but that would be an abuse of what MinMax is + // about, and would break existing code that assumes that a MinMax is just + // a min and a max. Unlike MinMax which is agnostic as to the quantized + // data type, narrow_range refers to values in the quantized data type. + bool narrow_range = false; private: std::unique_ptr array_shape; diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc index 8377ba6a03..a791e60f91 100644 --- a/tensorflow/contrib/lite/toco/tflite/operator.cc +++ b/tensorflow/contrib/lite/toco/tflite/operator.cc @@ -290,8 +290,8 @@ class FakeQuant flatbuffers::Offset WriteOptions( const TocoOperator& op, flatbuffers::FlatBufferBuilder* builder) const override { - return ::tflite::CreateFakeQuantOptions(*builder, op.minmax->min, - op.minmax->max, op.num_bits); + return ::tflite::CreateFakeQuantOptions( + *builder, op.minmax->min, op.minmax->max, op.num_bits, op.narrow_range); } void ReadOptions(const TfLiteOptions& options, TocoOperator* op) const override { @@ -300,9 +300,13 @@ class FakeQuant minmax->max = options.max(); op->minmax.reset(minmax); op->num_bits = options.num_bits(); + op->narrow_range = options.narrow_range(); } - int GetVersion(const Operator& op) const override { return 1; } + int GetVersion(const Operator& op) const override { + const auto& fq_op = static_cast(op); + return fq_op.narrow_range ? 2 : 1; + } }; class FullyConnected diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc index 3ca36338eb..a4dc1bbe93 100644 --- a/tensorflow/contrib/lite/toco/toco_tooling.cc +++ b/tensorflow/contrib/lite/toco/toco_tooling.cc @@ -105,7 +105,8 @@ void MakeGeneralGraphTransformationsSet( transformations->Add(new IdentifyRelu1); transformations->Add(new IdentifyPRelu); transformations->Add(new RemoveTrivialBinaryOperator); - transformations->Add(new ReadFakeQuantMinMax); + transformations->Add(new ResolveFakeQuantArgsFromVars); + transformations->Add(new ReadArrayMinmaxAndNarrowRangeFromFakeQuant); transformations->Add(new ResolveSpaceToBatchNDAttributes); transformations->Add(new ResolveBatchToSpaceNDAttributes); transformations->Add(new ResolvePadAttributes); -- cgit v1.2.3 From 746a51b76742574d81783a4efe437e1824073d88 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 12:13:14 -0700 Subject: To package support for both 32 bit and 64 bit in single apk, we need different flag PiperOrigin-RevId: 204341142 --- tensorflow/contrib/lite/examples/android/app/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/lite/examples/android/app/README.md b/tensorflow/contrib/lite/examples/android/app/README.md index 3065a5f6ee..8e12bd04dd 100644 --- a/tensorflow/contrib/lite/examples/android/app/README.md +++ b/tensorflow/contrib/lite/examples/android/app/README.md @@ -4,10 +4,10 @@ 1. Follow the [Bazel steps for the TF Demo App](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#bazel). -2. Build the app with Bazel. The demo needs C++11: +2. Build the app with Bazel. The demo needs C++11. We configure the fat_apk_cpu flag to package support for 4 hardware variants. You may replace it with --config=android_arm64 on a 64-bit device and --config=android_arm for 32-bit device: ```shell - bazel build -c opt --cxxopt='--std=c++11' \ + bazel build -c opt --cxxopt='--std=c++11' --fat_apk_cpu=x86,x86_64,arm64-v8a,armeabi-v7a \ //tensorflow/contrib/lite/examples/android:tflite_demo ``` -- cgit v1.2.3 From e8a65666c6aadbbbd2b19b9322d841b1547dbd35 Mon Sep 17 00:00:00 2001 From: Alan Chiao Date: Thu, 12 Jul 2018 12:16:09 -0700 Subject: LSTM CHECK_OK on input tensor checks. PiperOrigin-RevId: 204341675 --- .../lite/kernels/bidirectional_sequence_lstm.cc | 56 +++++++++++++--------- tensorflow/contrib/lite/kernels/lstm.cc | 3 +- tensorflow/contrib/lite/kernels/lstm_test.cc | 3 ++ .../lite/kernels/unidirectional_sequence_lstm.cc | 3 +- 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/bidirectional_sequence_lstm.cc b/tensorflow/contrib/lite/kernels/bidirectional_sequence_lstm.cc index 3425288f02..14a19aeef3 100644 --- a/tensorflow/contrib/lite/kernels/bidirectional_sequence_lstm.cc +++ b/tensorflow/contrib/lite/kernels/bidirectional_sequence_lstm.cc @@ -276,27 +276,33 @@ TfLiteStatus CheckLstmTensorDimensions( TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context, TfLiteNode* node, int n_input, int n_output, int n_cell) { - CheckLstmTensorDimensions( - context, node, n_input, n_output, n_cell, kFwInputToInputWeightsTensor, - kFwInputToForgetWeightsTensor, kFwInputToCellWeightsTensor, - kFwInputToOutputWeightsTensor, kFwRecurrentToInputWeightsTensor, - kFwRecurrentToForgetWeightsTensor, kFwRecurrentToCellWeightsTensor, - kFwRecurrentToOutputWeightsTensor, kFwCellToInputWeightsTensor, - kFwCellToForgetWeightsTensor, kFwCellToOutputWeightsTensor, - kFwInputGateBiasTensor, kFwForgetGateBiasTensor, kFwCellGateBiasTensor, - kFwOutputGateBiasTensor, kFwProjectionWeightsTensor, - kFwProjectionBiasTensor); - - CheckLstmTensorDimensions( - context, node, n_input, n_output, n_cell, kBwInputToInputWeightsTensor, - kBwInputToForgetWeightsTensor, kBwInputToCellWeightsTensor, - kBwInputToOutputWeightsTensor, kBwRecurrentToInputWeightsTensor, - kBwRecurrentToForgetWeightsTensor, kBwRecurrentToCellWeightsTensor, - kBwRecurrentToOutputWeightsTensor, kBwCellToInputWeightsTensor, - kBwCellToForgetWeightsTensor, kBwCellToOutputWeightsTensor, - kBwInputGateBiasTensor, kBwForgetGateBiasTensor, kBwCellGateBiasTensor, - kBwOutputGateBiasTensor, kBwProjectionWeightsTensor, - kBwProjectionBiasTensor); + TF_LITE_ENSURE_OK( + context, + CheckLstmTensorDimensions( + context, node, n_input, n_output, n_cell, + kFwInputToInputWeightsTensor, kFwInputToForgetWeightsTensor, + kFwInputToCellWeightsTensor, kFwInputToOutputWeightsTensor, + kFwRecurrentToInputWeightsTensor, kFwRecurrentToForgetWeightsTensor, + kFwRecurrentToCellWeightsTensor, kFwRecurrentToOutputWeightsTensor, + kFwCellToInputWeightsTensor, kFwCellToForgetWeightsTensor, + kFwCellToOutputWeightsTensor, kFwInputGateBiasTensor, + kFwForgetGateBiasTensor, kFwCellGateBiasTensor, + kFwOutputGateBiasTensor, kFwProjectionWeightsTensor, + kFwProjectionBiasTensor)); + + TF_LITE_ENSURE_OK( + context, + CheckLstmTensorDimensions( + context, node, n_input, n_output, n_cell, + kBwInputToInputWeightsTensor, kBwInputToForgetWeightsTensor, + kBwInputToCellWeightsTensor, kBwInputToOutputWeightsTensor, + kBwRecurrentToInputWeightsTensor, kBwRecurrentToForgetWeightsTensor, + kBwRecurrentToCellWeightsTensor, kBwRecurrentToOutputWeightsTensor, + kBwCellToInputWeightsTensor, kBwCellToForgetWeightsTensor, + kBwCellToOutputWeightsTensor, kBwInputGateBiasTensor, + kBwForgetGateBiasTensor, kBwCellGateBiasTensor, + kBwOutputGateBiasTensor, kBwProjectionWeightsTensor, + kBwProjectionBiasTensor)); // Check if Forward and Backward tensors match along required dimensions. return kTfLiteOk; @@ -334,7 +340,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { const int n_fw_output = fw_recurrent_to_output_weights->dims->data[1]; // Check that input tensor dimensions matches with each other. - CheckInputTensorDimensions(context, node, n_input, n_fw_output, n_fw_cell); + TF_LITE_ENSURE_OK( + context, CheckInputTensorDimensions(context, node, n_input, n_fw_output, + n_fw_cell)); // Get the pointer to output, state and scratch buffer tensors. TfLiteTensor* fw_output = GetOutput(context, node, kFwOutputTensor); @@ -404,7 +412,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { const int n_bw_output = bw_recurrent_to_output_weights->dims->data[1]; // Check that input tensor dimensions matches with each other. - CheckInputTensorDimensions(context, node, n_input, n_bw_output, n_bw_cell); + TF_LITE_ENSURE_OK( + context, CheckInputTensorDimensions(context, node, n_input, n_bw_output, + n_bw_cell)); // Get the pointer to output, output_state and cell_state buffer tensors. TfLiteTensor* bw_output = GetOutput(context, node, kBwOutputTensor); diff --git a/tensorflow/contrib/lite/kernels/lstm.cc b/tensorflow/contrib/lite/kernels/lstm.cc index 3577ae6caa..4dfc891548 100644 --- a/tensorflow/contrib/lite/kernels/lstm.cc +++ b/tensorflow/contrib/lite/kernels/lstm.cc @@ -306,7 +306,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { const int n_output = recurrent_to_output_weights->dims->data[1]; // Check that input tensor dimensions matches with each other. - CheckInputTensorDimensions(context, node, n_input, n_output, n_cell); + TF_LITE_ENSURE_OK(context, CheckInputTensorDimensions(context, node, n_input, + n_output, n_cell)); // Get the pointer to output, activation_state and cell_state tensors. TfLiteTensor* output = GetOutput(context, node, kOutputTensor); diff --git a/tensorflow/contrib/lite/kernels/lstm_test.cc b/tensorflow/contrib/lite/kernels/lstm_test.cc index 0b7c56133e..0266f5fe57 100644 --- a/tensorflow/contrib/lite/kernels/lstm_test.cc +++ b/tensorflow/contrib/lite/kernels/lstm_test.cc @@ -13,6 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ // Unit test for TFLite LSTM op. +// +// TODO(alanchiao): add unit test with invalid input dimensions for this and its +// variants. #include #include diff --git a/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc b/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc index 32daf2bb02..c48b470f92 100644 --- a/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc +++ b/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc @@ -274,7 +274,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { const int n_output = recurrent_to_output_weights->dims->data[1]; // Check that input tensor dimensions matches with each other. - CheckInputTensorDimensions(context, node, n_input, n_output, n_cell); + TF_LITE_ENSURE_OK(context, CheckInputTensorDimensions(context, node, n_input, + n_output, n_cell)); // Get the pointer to output, output_state and cell_state buffer tensors. TfLiteTensor* output = GetOutput(context, node, kOutputTensor); -- cgit v1.2.3 From 35a29824aa196aa9348e1f6daf836d07d9e61156 Mon Sep 17 00:00:00 2001 From: Revan Sopher Date: Thu, 12 Jul 2018 12:20:34 -0700 Subject: Internal change. PiperOrigin-RevId: 204342372 --- tensorflow/contrib/tpu/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index 0044fde9d0..f59545f651 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -16,7 +16,6 @@ package( "//cloud/vmm/testing/tests/tpu:__subpackages__", "//learning/brain:__subpackages__", "//tensorflow:__subpackages__", - "//third_party/cloud_tpu:__subpackages__", ], ) -- cgit v1.2.3 From 86f84b066706db97f7b3fd184249fdbd54abb05e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 12:49:24 -0700 Subject: Remove exposure of HWNC and HWCN data formats PiperOrigin-RevId: 204347001 --- tensorflow/compiler/tests/conv2d_test.py | 3 - tensorflow/core/ops/compat/ops_history.v1.pbtxt | 1031 +---------------------- tensorflow/core/ops/ops.pbtxt | 28 - tensorflow/core/util/tensor_format.cc | 2 +- 4 files changed, 28 insertions(+), 1036 deletions(-) diff --git a/tensorflow/compiler/tests/conv2d_test.py b/tensorflow/compiler/tests/conv2d_test.py index 98d41ba7ed..f9db103f6d 100644 --- a/tensorflow/compiler/tests/conv2d_test.py +++ b/tensorflow/compiler/tests/conv2d_test.py @@ -33,12 +33,9 @@ from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import nn_ops from tensorflow.python.platform import googletest - DATA_FORMATS = ( ("_data_format_NHWC", "NHWC"), ("_data_format_NCHW", "NCHW"), - ("_data_format_HWNC", "HWNC"), - ("_data_format_HWCN", "HWCN"), ) diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index 6cdd03e6a0..be72ee8066 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -7680,66 +7680,6 @@ op { } } } -op { - name: "AvgPool" - input_arg { - name: "value" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "ksize" - type: "list(int)" - has_minimum: true - minimum: 4 - } - attr { - name: "strides" - type: "list(int)" - has_minimum: true - minimum: 4 - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_BFLOAT16 - type: DT_FLOAT - type: DT_DOUBLE - } - } - } -} op { name: "AvgPool3D" input_arg { @@ -8429,70 +8369,6 @@ op { } } } -op { - name: "AvgPoolGrad" - input_arg { - name: "orig_input_shape" - type: DT_INT32 - } - input_arg { - name: "grad" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "ksize" - type: "list(int)" - has_minimum: true - minimum: 4 - } - attr { - name: "strides" - type: "list(int)" - has_minimum: true - minimum: 4 - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_BFLOAT16 - type: DT_FLOAT - type: DT_DOUBLE - } - } - } -} op { name: "Barrier" output_arg { @@ -10554,61 +10430,6 @@ op { } } } -op { - name: "BiasAdd" - input_arg { - name: "value" - type_attr: "T" - } - input_arg { - name: "bias" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_FLOAT - type: DT_DOUBLE - type: DT_INT32 - type: DT_UINT8 - type: DT_INT16 - type: DT_INT8 - type: DT_COMPLEX64 - type: DT_INT64 - type: DT_QINT8 - type: DT_QUINT8 - type: DT_QINT32 - type: DT_BFLOAT16 - type: DT_UINT16 - type: DT_COMPLEX128 - type: DT_HALF - type: DT_UINT32 - type: DT_UINT64 - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } -} op { name: "BiasAddGrad" input_arg { @@ -10801,57 +10622,6 @@ op { } } } -op { - name: "BiasAddGrad" - input_arg { - name: "out_backprop" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_FLOAT - type: DT_DOUBLE - type: DT_INT32 - type: DT_UINT8 - type: DT_INT16 - type: DT_INT8 - type: DT_COMPLEX64 - type: DT_INT64 - type: DT_QINT8 - type: DT_QUINT8 - type: DT_QINT32 - type: DT_BFLOAT16 - type: DT_UINT16 - type: DT_COMPLEX128 - type: DT_HALF - type: DT_UINT32 - type: DT_UINT64 - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } -} op { name: "BiasAddV1" input_arg { @@ -13457,13 +13227,17 @@ op { } } op { - name: "Conv2D" + name: "Conv2DBackpropFilter" input_arg { name: "input" type_attr: "T" } input_arg { - name: "filter" + name: "filter_sizes" + type: DT_INT32 + } + input_arg { + name: "out_backprop" type_attr: "T" } output_arg { @@ -13476,9 +13250,7 @@ op { allowed_values { list { type: DT_HALF - type: DT_BFLOAT16 type: DT_FLOAT - type: DT_DOUBLE } } } @@ -13513,20 +13285,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } - attr { - name: "dilations" - type: "list(int)" - default_value { - list { - i: 1 - i: 1 - i: 1 - i: 1 } } } @@ -13555,6 +13313,7 @@ op { allowed_values { list { type: DT_HALF + type: DT_BFLOAT16 type: DT_FLOAT } } @@ -13593,147 +13352,6 @@ op { } } } -} -op { - name: "Conv2DBackpropFilter" - input_arg { - name: "input" - type_attr: "T" - } - input_arg { - name: "filter_sizes" - type: DT_INT32 - } - input_arg { - name: "out_backprop" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_BFLOAT16 - type: DT_FLOAT - } - } - } - attr { - name: "strides" - type: "list(int)" - } - attr { - name: "use_cudnn_on_gpu" - type: "bool" - default_value { - b: true - } - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - } - } - } - attr { - name: "dilations" - type: "list(int)" - default_value { - list { - i: 1 - i: 1 - i: 1 - i: 1 - } - } - } -} -op { - name: "Conv2DBackpropFilter" - input_arg { - name: "input" - type_attr: "T" - } - input_arg { - name: "filter_sizes" - type: DT_INT32 - } - input_arg { - name: "out_backprop" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_BFLOAT16 - type: DT_FLOAT - type: DT_DOUBLE - } - } - } - attr { - name: "strides" - type: "list(int)" - } - attr { - name: "use_cudnn_on_gpu" - type: "bool" - default_value { - b: true - } - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - } - } - } attr { name: "dilations" type: "list(int)" @@ -13808,8 +13426,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -14042,85 +13658,6 @@ op { } } } -op { - name: "Conv2DBackpropInput" - input_arg { - name: "input_sizes" - type: DT_INT32 - } - input_arg { - name: "filter" - type_attr: "T" - } - input_arg { - name: "out_backprop" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_BFLOAT16 - type: DT_FLOAT - type: DT_DOUBLE - } - } - } - attr { - name: "strides" - type: "list(int)" - } - attr { - name: "use_cudnn_on_gpu" - type: "bool" - default_value { - b: true - } - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } - attr { - name: "dilations" - type: "list(int)" - default_value { - list { - i: 1 - i: 1 - i: 1 - i: 1 - } - } - } -} op { name: "Conv3D" input_arg { @@ -18852,13 +18389,17 @@ op { } } op { - name: "DepthwiseConv2dNative" + name: "DepthwiseConv2dNativeBackpropFilter" input_arg { name: "input" type_attr: "T" } input_arg { - name: "filter" + name: "filter_sizes" + type: DT_INT32 + } + input_arg { + name: "out_backprop" type_attr: "T" } output_arg { @@ -18870,8 +18411,6 @@ op { type: "type" allowed_values { list { - type: DT_HALF - type: DT_BFLOAT16 type: DT_FLOAT type: DT_DOUBLE } @@ -18891,33 +18430,6 @@ op { } } } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } - attr { - name: "dilations" - type: "list(int)" - default_value { - list { - i: 1 - i: 1 - i: 1 - i: 1 - } - } - } } op { name: "DepthwiseConv2dNativeBackpropFilter" @@ -18961,6 +18473,19 @@ op { } } } + attr { + name: "data_format" + type: "string" + default_value { + s: "NHWC" + } + allowed_values { + list { + s: "NHWC" + s: "NCHW" + } + } + } } op { name: "DepthwiseConv2dNativeBackpropFilter" @@ -18985,133 +18510,7 @@ op { type: "type" allowed_values { list { - type: DT_FLOAT - type: DT_DOUBLE - } - } - } - attr { - name: "strides" - type: "list(int)" - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - } - } - } -} -op { - name: "DepthwiseConv2dNativeBackpropFilter" - input_arg { - name: "input" - type_attr: "T" - } - input_arg { - name: "filter_sizes" - type: DT_INT32 - } - input_arg { - name: "out_backprop" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_BFLOAT16 - type: DT_FLOAT - type: DT_DOUBLE - } - } - } - attr { - name: "strides" - type: "list(int)" - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - } - } - } - attr { - name: "dilations" - type: "list(int)" - default_value { - list { - i: 1 - i: 1 - i: 1 - i: 1 - } - } - } -} -op { - name: "DepthwiseConv2dNativeBackpropFilter" - input_arg { - name: "input" - type_attr: "T" - } - input_arg { - name: "filter_sizes" - type: DT_INT32 - } - input_arg { - name: "out_backprop" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_BFLOAT16 + type: DT_BFLOAT16 type: DT_FLOAT type: DT_DOUBLE } @@ -19211,8 +18610,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -19467,78 +18864,6 @@ op { } } } -op { - name: "DepthwiseConv2dNativeBackpropInput" - input_arg { - name: "input_sizes" - type: DT_INT32 - } - input_arg { - name: "filter" - type_attr: "T" - } - input_arg { - name: "out_backprop" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_BFLOAT16 - type: DT_FLOAT - type: DT_DOUBLE - } - } - } - attr { - name: "strides" - type: "list(int)" - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } - attr { - name: "dilations" - type: "list(int)" - default_value { - list { - i: 1 - i: 1 - i: 1 - i: 1 - } - } - } -} op { name: "Dequantize" input_arg { @@ -32233,85 +31558,6 @@ op { } } } -op { - name: "MaxPoolGrad" - input_arg { - name: "orig_input" - type_attr: "T" - } - input_arg { - name: "orig_output" - type_attr: "T" - } - input_arg { - name: "grad" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "ksize" - type: "list(int)" - has_minimum: true - minimum: 4 - } - attr { - name: "strides" - type: "list(int)" - has_minimum: true - minimum: 4 - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } - attr { - name: "T" - type: "type" - default_value { - type: DT_FLOAT - } - allowed_values { - list { - type: DT_FLOAT - type: DT_DOUBLE - type: DT_INT32 - type: DT_UINT8 - type: DT_INT16 - type: DT_INT8 - type: DT_INT64 - type: DT_BFLOAT16 - type: DT_UINT16 - type: DT_HALF - type: DT_UINT32 - type: DT_UINT64 - } - } - } -} op { name: "MaxPoolGradGrad" input_arg { @@ -32604,82 +31850,6 @@ op { } } } -op { - name: "MaxPoolGradGrad" - input_arg { - name: "orig_input" - type_attr: "T" - } - input_arg { - name: "orig_output" - type_attr: "T" - } - input_arg { - name: "grad" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "ksize" - type: "list(int)" - has_minimum: true - minimum: 4 - } - attr { - name: "strides" - type: "list(int)" - has_minimum: true - minimum: 4 - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_FLOAT - type: DT_DOUBLE - type: DT_INT32 - type: DT_UINT8 - type: DT_INT16 - type: DT_INT8 - type: DT_INT64 - type: DT_BFLOAT16 - type: DT_UINT16 - type: DT_HALF - type: DT_UINT32 - type: DT_UINT64 - } - } - } -} op { name: "MaxPoolGradGradV2" input_arg { @@ -32956,78 +32126,6 @@ op { } } } -op { - name: "MaxPoolGradGradV2" - input_arg { - name: "orig_input" - type_attr: "T" - } - input_arg { - name: "orig_output" - type_attr: "T" - } - input_arg { - name: "grad" - type_attr: "T" - } - input_arg { - name: "ksize" - type: DT_INT32 - } - input_arg { - name: "strides" - type: DT_INT32 - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_FLOAT - type: DT_DOUBLE - type: DT_INT32 - type: DT_UINT8 - type: DT_INT16 - type: DT_INT8 - type: DT_INT64 - type: DT_BFLOAT16 - type: DT_UINT16 - type: DT_HALF - type: DT_UINT32 - type: DT_UINT64 - } - } - } -} op { name: "MaxPoolGradGradWithArgmax" input_arg { @@ -33596,81 +32694,6 @@ op { } } } -op { - name: "MaxPoolGradV2" - input_arg { - name: "orig_input" - type_attr: "T" - } - input_arg { - name: "orig_output" - type_attr: "T" - } - input_arg { - name: "grad" - type_attr: "T" - } - input_arg { - name: "ksize" - type: DT_INT32 - } - input_arg { - name: "strides" - type: DT_INT32 - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "padding" - type: "string" - allowed_values { - list { - s: "SAME" - s: "VALID" - } - } - } - attr { - name: "data_format" - type: "string" - default_value { - s: "NHWC" - } - allowed_values { - list { - s: "NHWC" - s: "NCHW" - s: "HWNC" - s: "HWCN" - } - } - } - attr { - name: "T" - type: "type" - default_value { - type: DT_FLOAT - } - allowed_values { - list { - type: DT_FLOAT - type: DT_DOUBLE - type: DT_INT32 - type: DT_UINT8 - type: DT_INT16 - type: DT_INT8 - type: DT_INT64 - type: DT_BFLOAT16 - type: DT_UINT16 - type: DT_HALF - type: DT_UINT32 - type: DT_UINT64 - } - } - } -} op { name: "MaxPoolGradWithArgmax" input_arg { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 9a9f10f01f..76572061a4 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -2490,8 +2490,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -2674,8 +2672,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -3989,8 +3985,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -4040,8 +4034,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -5730,8 +5722,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -5809,8 +5799,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -5888,8 +5876,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -8592,8 +8578,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -8664,8 +8648,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -8736,8 +8718,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -15509,8 +15489,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -15588,8 +15566,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -15660,8 +15636,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } @@ -15803,8 +15777,6 @@ op { list { s: "NHWC" s: "NCHW" - s: "HWNC" - s: "HWCN" } } } diff --git a/tensorflow/core/util/tensor_format.cc b/tensorflow/core/util/tensor_format.cc index 33ab87aa78..a5f7ecf0d1 100644 --- a/tensorflow/core/util/tensor_format.cc +++ b/tensorflow/core/util/tensor_format.cc @@ -18,7 +18,7 @@ limitations under the License. namespace tensorflow { string GetConvnetDataFormatAttrString() { - return "data_format: { 'NHWC', 'NCHW', 'HWNC', 'HWCN' } = 'NHWC' "; + return "data_format: { 'NHWC', 'NCHW' } = 'NHWC' "; } string GetConvnet3dDataFormatAttrString() { -- cgit v1.2.3 From 546322104425cc1cc70afeb7c0cfc1ec36ed0b41 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Thu, 12 Jul 2018 12:51:23 -0700 Subject: Fix bug in SetResourceHandleShapeAndType. Prior to this change, captured resource variables in TF functions (or any captured resource tensors) would not have shape information. PiperOrigin-RevId: 204347306 --- tensorflow/c/python_api.cc | 2 +- tensorflow/python/framework/function_test.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tensorflow/c/python_api.cc b/tensorflow/c/python_api.cc index e18fdf6c57..8486b585c8 100644 --- a/tensorflow/c/python_api.cc +++ b/tensorflow/c/python_api.cc @@ -155,7 +155,7 @@ void SetResourceHandleShapeAndType(TF_Graph* graph, TF_Output output, tensorflow::shape_inference::ShapeHandle shape; status->status = ic->MakeShapeFromShapeProto(shape_and_type_proto.shape(), &shape); - if (status->status.ok()) return; + if (!status->status.ok()) return; shapes_and_types.emplace_back(shape, shape_and_type_proto.dtype()); } ic->set_output_handle_shapes_and_types(output.index, shapes_and_types); diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py index 15e41ba91f..1707f929b8 100644 --- a/tensorflow/python/framework/function_test.py +++ b/tensorflow/python/framework/function_test.py @@ -537,19 +537,25 @@ class FunctionTest(test.TestCase): def testResourceVarAsImplicitInput(self): g = ops.Graph() with g.as_default(), ops.device("cpu:0"): + expected_type = dtypes.float32 + expected_shape = tensor_shape.TensorShape((4, 4)) v = variable_scope.get_variable( - "var", (4, 4), dtypes.float32, use_resource=True) + "var", expected_shape, expected_type, use_resource=True) @function.Defun() def Foo(): - return array_ops.identity(v) + captured = array_ops.identity(v) + self.assertEqual(expected_type, captured.dtype) + self.assertEqual(expected_shape, captured.shape) + return captured, array_ops.shape(captured) - y = v.value() - z = Foo() + expected_val = v.value() + actual_val, actual_shape = Foo() with self.test_session(graph=g): v.initializer.run() - self.assertAllEqual(y.eval(), z.eval()) + self.assertAllEqual(expected_val.eval(), actual_val.eval()) + self.assertAllEqual(expected_shape, actual_shape.eval()) def testDefineErrors(self): with ops.Graph().as_default(): -- cgit v1.2.3 From 0b418d46a557983e125c161ad4541ced5e39e0be Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 12:52:48 -0700 Subject: Fix tf.svd example code PiperOrigin-RevId: 204347508 --- tensorflow/python/ops/linalg_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index a0dfa543f9..f4a93560be 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -401,7 +401,7 @@ def svd(tensor, full_matrices=False, compute_uv=True, name=None): import tensorflow as tf import numpy as np s, u, v = tf.linalg.svd(a) - tf_a_approx = tf.matmul(u, tf.matmul(tf.linalg.diag(s), v, adjoint_v=True)) + tf_a_approx = tf.matmul(u, tf.matmul(tf.linalg.diag(s), v, adjoint_b=True)) u, s, v_adj = np.linalg.svd(a, full_matrices=False) np_a_approx = np.dot(u, np.dot(np.diag(s), v_adj)) # tf_a_approx and np_a_approx should be numerically close. -- cgit v1.2.3 From 39d5c4770a397e98ba30102924bae0de2f0c8b5a Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Thu, 12 Jul 2018 12:59:06 -0700 Subject: Remove unused annotation tag and mark for deprecation. PiperOrigin-RevId: 204348472 --- tensorflow/contrib/autograph/pyct/static_analysis/annos.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/annos.py b/tensorflow/contrib/autograph/pyct/static_analysis/annos.py index b929b35b79..5eefecf278 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/annos.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/annos.py @@ -21,6 +21,9 @@ from __future__ import print_function from enum import Enum +# TODO(mdan): Remove. + + class NoValue(Enum): def __repr__(self): @@ -50,10 +53,3 @@ class NodeAnno(NoValue): ORELSE_SCOPE = ( 'The scope for the orelse body of a statement (False branch for if ' 'statements, orelse body for loops).') - - # Type and Value annotations - # Type annotations are represented by objects of type type_info.Type. - STATIC_INFO = ( - 'The type or value information that should be asserted about the entity ' - 'referenced by the symbol holding this annotation, irrespective of the ' - 'execution context.') -- cgit v1.2.3 From 9d41a9e66bb85cfe361507109ad896ee1ef5f6a8 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Thu, 12 Jul 2018 12:59:09 -0700 Subject: Add support for persistent annotations in template expansions. PiperOrigin-RevId: 204348481 --- tensorflow/contrib/autograph/pyct/templates.py | 87 +++++++++++++++++--------- 1 file changed, 56 insertions(+), 31 deletions(-) diff --git a/tensorflow/contrib/autograph/pyct/templates.py b/tensorflow/contrib/autograph/pyct/templates.py index 9c479ebc2f..9001e54e46 100644 --- a/tensorflow/contrib/autograph/pyct/templates.py +++ b/tensorflow/contrib/autograph/pyct/templates.py @@ -26,6 +26,7 @@ import textwrap import gast +from tensorflow.contrib.autograph.pyct import anno from tensorflow.contrib.autograph.pyct import ast_util from tensorflow.contrib.autograph.pyct import parser from tensorflow.contrib.autograph.pyct import qual_names @@ -43,39 +44,64 @@ class ReplaceTransformer(gast.NodeTransformer): """ self.replacements = replacements self.in_replacements = False + self.preserved_annos = { + anno.Basic.SKIP_PROCESSING, + anno.Static.ORIG_DEFINITIONS, + } + + def _prepare_replacement(self, replaced, key): + """Prepares a replacement AST that's safe to swap in for a node. + + Args: + replaced: ast.AST, the node being replaced + key: Hashable, the key of the replacement AST + Returns: + ast.AST, the replacement AST + """ + repl = self.replacements[key] + + new_nodes = ast_util.copy_clean(repl, preserve_annos=self.preserved_annos) + if isinstance(new_nodes, gast.AST): + new_nodes = [new_nodes] + + return new_nodes def visit_Expr(self, node): - if (isinstance(node.value, gast.Name) and - node.value.id in self.replacements): - return self.visit(node.value) - self.generic_visit(node) - return node + # When replacing a placeholder with an entire statement, the replacement + # must stand on its own and not be wrapped in an Expr. + new_value = self.visit(node.value) + if new_value is node.value: + return node + return new_value def visit_keyword(self, node): - if node.arg in self.replacements: - repl = self.replacements[node.arg] - if isinstance(repl, gast.keyword): - return repl - elif (isinstance(repl, (list, tuple)) and repl and - all(isinstance(r, gast.keyword) for r in repl)): - return repl - # TODO(mdan): We may allow replacing with a string as well. - # For example, if one wanted to replace foo with bar in foo=baz, then - # we could allow changing just node arg, so that we end up with bar=baz. - raise ValueError( - 'a keyword argument may only be replaced by another keyword or a ' - 'non-empty list of keywords. Found: %s' % repl) - return self.generic_visit(node) + if node.arg not in self.replacements: + return self.generic_visit(node) + + repl = self._prepare_replacement(node, node.arg) + if isinstance(repl, gast.keyword): + return repl + elif (repl and isinstance(repl, (list, tuple)) and + all(isinstance(r, gast.keyword) for r in repl)): + return repl + # TODO(mdan): We may allow replacing with a string as well. + # For example, if one wanted to replace foo with bar in foo=baz, then + # we could allow changing just node arg, so that we end up with bar=baz. + raise ValueError( + 'a keyword argument may only be replaced by another keyword or a ' + 'non-empty list of keywords. Found: %s' % repl) def visit_FunctionDef(self, node): node = self.generic_visit(node) - if node.name in self.replacements: - repl = self.replacements[node.name] - if not isinstance(repl, (gast.Name, ast.Name)): - raise ValueError( - 'a function name can only be replaced by a Name node. Found: %s' % - repl) - node.name = repl.id + if node.name not in self.replacements: + return node + + repl = self.replacements[node.name] + if not isinstance(repl, (gast.Name, ast.Name)): + raise ValueError( + 'a function name can only be replaced by a Name node. Found: %s' % + repl) + node.name = repl.id return node def _check_has_context(self, node): @@ -148,6 +174,7 @@ class ReplaceTransformer(gast.NodeTransformer): node = self.generic_visit(node) if node.attr not in self.replacements: return node + repl = self.replacements[node.attr] if not isinstance(repl, gast.Name): raise ValueError( @@ -159,9 +186,7 @@ class ReplaceTransformer(gast.NodeTransformer): if node.id not in self.replacements: return node - new_nodes = ast_util.copy_clean(self.replacements[node.id]) - if isinstance(new_nodes, gast.AST): - new_nodes = [new_nodes] + new_nodes = self._prepare_replacement(node, node.id) # Preserve the target context. for n in new_nodes: @@ -182,7 +207,7 @@ class ReplaceTransformer(gast.NodeTransformer): def _convert_to_ast(n): - """Convert from a known data type to AST.""" + """Converts from a known data type to AST.""" if isinstance(n, str): # Note: the node will receive the ctx value from the template, see # ReplaceTransformer.visit_Name. @@ -197,7 +222,7 @@ def _convert_to_ast(n): def replace(template, **replacements): - """Replace placeholders in a Python template. + """Replaces placeholders in a Python template. AST Name and Tuple nodes always receive the context that inferred from the template. However, when replacing more complex nodes (that can potentially -- cgit v1.2.3 From 0ef634190dc2e49e4002a841185fc850b80cc1b9 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 12 Jul 2018 13:10:15 -0700 Subject: [tf.data] Handling checkpointing of optimized input pipelines correctly. PiperOrigin-RevId: 204350306 --- .../kernel_tests/optimize_dataset_op_test.py | 6 ---- .../core/kernels/data/optimize_dataset_op.cc | 40 ++++++++++++++++------ 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/optimize_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/optimize_dataset_op_test.py index 3bb9723bbc..21eebccd11 100644 --- a/tensorflow/contrib/data/python/kernel_tests/optimize_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/optimize_dataset_op_test.py @@ -35,8 +35,6 @@ class OptimizeDatasetTest(test.TestCase): with self.test_session() as sess: graph = graph_pb2.GraphDef().FromString( sess.run(dataset._as_serialized_graph())) - self.assertTrue( - all([node.op != "MapAndBatchDatasetV2" for node in graph.node])) self.assertAllEqual([x * x for x in range(10)], sess.run(get_next)) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) @@ -50,8 +48,6 @@ class OptimizeDatasetTest(test.TestCase): with self.test_session() as sess: graph = graph_pb2.GraphDef().FromString( sess.run(dataset._as_serialized_graph())) - self.assertTrue( - all([node.op != "MapAndBatchDatasetV2" for node in graph.node])) self.assertAllEqual([x * x for x in range(10)], sess.run(get_next)) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) @@ -65,8 +61,6 @@ class OptimizeDatasetTest(test.TestCase): with self.test_session() as sess: graph = graph_pb2.GraphDef().FromString( sess.run(dataset._as_serialized_graph())) - self.assertTrue( - any([node.op == "MapAndBatchDatasetV2" for node in graph.node])) self.assertAllEqual([x * x for x in range(10)], sess.run(get_next)) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc index 81be69105e..276f5f89c8 100644 --- a/tensorflow/core/kernels/data/optimize_dataset_op.cc +++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc @@ -53,23 +53,30 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel { OP_REQUIRES_OK( ctx, ParseVectorArgument(ctx, "optimizations", &optimizations)); Dataset* dataset = - new Dataset(ctx, optimizations, output_types_, output_shapes_); - OP_REQUIRES_OK(ctx, dataset->Optimize(ctx, input)); + new Dataset(ctx, input, optimizations, output_types_, output_shapes_); + OP_REQUIRES_OK(ctx, dataset->Optimize(ctx)); *output = dataset; } private: class Dataset : public GraphDatasetBase { public: - Dataset(OpKernelContext* ctx, const std::vector& optimizations, + Dataset(OpKernelContext* ctx, const DatasetBase* input, + const std::vector& optimizations, const DataTypeVector& output_types, const std::vector& output_shapes) : GraphDatasetBase(ctx), + input_(input), optimizations_(optimizations), output_types_(output_types), - output_shapes_(output_shapes) {} + output_shapes_(output_shapes) { + input_->Ref(); + } - ~Dataset() override { input_->Unref(); } + ~Dataset() override { + input_->Unref(); + optimized_input_->Unref(); + } std::unique_ptr MakeIteratorInternal( const string& prefix) const override { @@ -77,15 +84,17 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel { new Iterator({this, strings::StrCat(prefix, "::Optimize")})); } - Status Optimize(OpKernelContext* ctx, const DatasetBase* input) { + Status Optimize(OpKernelContext* ctx) { GraphDefBuilder b; DatasetGraphDefBuilder db(&b); Node* input_node = nullptr; - TF_RETURN_IF_ERROR(db.AddParentDataset(ctx, input, &input_node)); + TF_RETURN_IF_ERROR(db.AddParentDataset(ctx, input_, &input_node)); string output_node = input_node->name(); GraphDef graph_def; TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def)); + VLOG(3) << "Before optimization: " << graph_def.DebugString(); TF_RETURN_IF_ERROR(ApplyOptimizations(ctx, &graph_def, &output_node)); + VLOG(3) << "After optimization: " << graph_def.DebugString(); flib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), graph_def.library())); Graph graph(OpRegistry::Global()); @@ -94,8 +103,9 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel { GraphRunner graph_runner(ctx->function_library()->device()); TF_RETURN_IF_ERROR(graph_runner.Run(&graph, ctx->function_library(), {}, {output_node}, &outputs)); - TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &input_)); - input_->Ref(); + TF_RETURN_IF_ERROR( + GetDatasetFromVariantTensor(outputs[0], &optimized_input_)); + optimized_input_->Ref(); return Status::OK(); } @@ -127,7 +137,8 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel { : DatasetIterator(params) {} Status Initialize(IteratorContext* ctx) override { - return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_); + return dataset()->optimized_input_->MakeIterator(ctx, prefix(), + &input_impl_); } Status GetNextInternal(IteratorContext* ctx, @@ -199,6 +210,12 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel { tensorflow::grappler::VirtualCluster cluster(device_map); // Run optimizer. + if (VLOG_IS_ON(2)) { + LOG(INFO) << "Performing the following optimizations:"; + for (const string& optimization : optimizations_) { + LOG(INFO) << " " << optimization; + } + } TF_RETURN_IF_ERROR(tensorflow::grappler::RunMetaOptimizer( *grappler_item, rewriter_config, ctx->device(), &cluster, graph_def)); @@ -213,8 +230,9 @@ class OptimizeDatasetOp : public UnaryDatasetOpKernel { return Status::OK(); } - DatasetBase* input_; + DatasetBase* optimized_input_; std::shared_ptr flib_def_; + const DatasetBase* input_; const std::vector optimizations_; const DataTypeVector output_types_; const std::vector output_shapes_; -- cgit v1.2.3 From c61c7f1ea6a5e6aa0af19eb21d03b351031d944c Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Thu, 12 Jul 2018 13:36:06 -0700 Subject: Add a test to eagerly run ops on multiple TPU cores PiperOrigin-RevId: 204354687 --- tensorflow/compiler/tests/eager_test.py | 47 +++++++++++++++++++++- .../python/examples/resnet50/resnet50_test.py | 24 ++++++++--- tensorflow/python/eager/function.py | 29 ++++++++----- 3 files changed, 83 insertions(+), 17 deletions(-) diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py index 8a3ed382a1..a8919d1afd 100644 --- a/tensorflow/compiler/tests/eager_test.py +++ b/tensorflow/compiler/tests/eager_test.py @@ -414,7 +414,7 @@ class EagerFunctionTest(xla_test.XLATestCase): def testSliceInDefun(self): with self.test_scope(): - @function.defun(compiled=True) + @function.defun def f(x, y): return x[0::2, y:, ...] @@ -429,6 +429,21 @@ class EagerFunctionTest(xla_test.XLATestCase): self.assertAllEqual(np.ones([1, 2, 4]), z.numpy()) self.assertAllEqual((2, 3, 4), dz.shape.as_list()) + def testNestedDefun(self): + with self.test_scope(): + + @function.defun + def times_two(x): + return 2 * x + + @function.defun + def two_x_plus_1(x): + return times_two(x) + 1 + + x = constant_op.constant([2, 3, 4]) + y = two_x_plus_1(x) + self.assertAllEqual([5, 7, 9], y.numpy()) + class ExcessivePaddingTest(xla_test.XLATestCase): """Test that eager execution works with TPU flattened tensors. @@ -481,6 +496,36 @@ class ExcessivePaddingTest(xla_test.XLATestCase): self.assertAllEqual(100 * [[36.0]], reduced) +def multiple_tpus(): + devices = context.context().devices() + return len([d for d in devices if 'device:TPU:' in d]) > 1 + + +class MultiDeviceTest(xla_test.XLATestCase): + """Test running TPU computation on more than one core.""" + + def testBasic(self): + if not multiple_tpus(): + self.skipTest('MultiDeviceTest requires multiple TPU devices.') + + # Compute 10 on TPU core 0 + with ops.device('device:TPU:0'): + two = constant_op.constant(2) + five = constant_op.constant(5) + ten = two * five + self.assertAllEqual(10, ten) + + # Compute 6 on TPU core 1 + with ops.device('device:TPU:1'): + two = constant_op.constant(2) + three = constant_op.constant(3) + six = two * three + self.assertAllEqual(6, six) + + # Copy 10 and 6 to CPU and sum them + self.assertAllEqual(16, ten + six) + + if __name__ == '__main__': ops.enable_eager_execution( config=config_pb2.ConfigProto(log_device_placement=True)) diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py index b14ef1df8f..07d8788882 100644 --- a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py +++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py @@ -29,6 +29,7 @@ import tensorflow.contrib.eager as tfe from tensorflow.contrib.eager.python.examples.resnet50 import resnet50 from tensorflow.contrib.summary import summary_test_util from tensorflow.python.client import device_lib +from tensorflow.python.eager import tape def device_and_data_format(): @@ -49,13 +50,21 @@ def random_batch(batch_size, data_format): return images, one_hot -def compute_gradients(model, images, labels): - with tf.GradientTape() as tape: +def compute_gradients(model, images, labels, num_replicas=1): + with tf.GradientTape() as grad_tape: logits = model(images, training=True) loss = tf.losses.softmax_cross_entropy( logits=logits, onehot_labels=labels) tf.contrib.summary.scalar(name='loss', tensor=loss) - return tape.gradient(loss, model.variables) + if num_replicas != 1: + loss /= num_replicas + + # TODO(b/110991947): We can mistakenly trace the gradient call in + # multi-threaded environment. Explicitly disable recording until + # this is fixed. + with tape.stop_recording(): + grads = grad_tape.gradient(loss, model.variables) + return grads def apply_gradients(model, optimizer, gradients): @@ -188,11 +197,14 @@ class ResNet50Benchmarks(tf.test.Benchmark): return (32,) return (16, 32) - def _report(self, label, start, num_iters, device, batch_size, data_format): + def _report(self, label, start, num_iters, device, batch_size, data_format, + num_replicas=1): avg_time = (time.time() - start) / num_iters dev = tf.DeviceSpec.from_string(device).device_type.lower() - name = '%s_%s_batch_%d_%s' % (label, dev, batch_size, data_format) - extras = {'examples_per_sec': batch_size / avg_time} + replica_str = '' if num_replicas == 1 else 'replicas_%d_' % num_replicas + name = '%s_%s_batch_%d_%s%s' % (label, dev, batch_size, + replica_str, data_format) + extras = {'examples_per_sec': (num_replicas * batch_size) / avg_time} self.report_benchmark( iters=num_iters, wall_time=avg_time, name=name, extras=extras) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index df83d673ad..29a3848bd8 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -21,6 +21,7 @@ from __future__ import print_function import collections import functools +import threading import numpy as np @@ -137,7 +138,7 @@ class CapturingGraph(ops.Graph): inputs[i] = self.capture(inp) return super(CapturingGraph, self).create_op( op_type, inputs, dtypes, input_types, name, attrs, op_def, - compute_shapes, compute_device) + compute_device=compute_device) # pylint: disable=invalid-name @@ -770,6 +771,11 @@ class _PolymorphicFunction(object): See the documentation for `defun` for more information on the semantics of defined functions. + + _PolymorphicFunction class is thread-compatible meaning that minimal + usage of defuns (defining and calling) is thread-safe, but if users call other + methods or invoke the base `python_function` themselves, external + synchronization is necessary. """ def __init__(self, python_function, name, compiled=False): @@ -787,6 +793,8 @@ class _PolymorphicFunction(object): self._arguments_to_functions = {} self._variables = [] + self._lock = threading.Lock() + def __get__(self, instance, owner): """Makes it possible to defun instance methods.""" del owner @@ -825,15 +833,16 @@ class _PolymorphicFunction(object): # signature so we don't improperly capture tensors such as variables. signature += tuple([context.executing_eagerly() or ops.get_default_graph()]) - if signature not in self._arguments_to_functions: - graph_function = _trace_and_define_function( - self._name, self._python_function, self._compiled, args, kwds) - self._arguments_to_functions[signature] = graph_function - self._variables.extend( - [v for v in graph_function.variables if v not in self._variables]) - return graph_function, inputs - else: - return self._arguments_to_functions[signature], inputs + with self._lock: + if signature not in self._arguments_to_functions: + graph_function = _trace_and_define_function( + self._name, self._python_function, self._compiled, args, kwds) + self._arguments_to_functions[signature] = graph_function + self._variables.extend( + [v for v in graph_function.variables if v not in self._variables]) + return graph_function, inputs + else: + return self._arguments_to_functions[signature], inputs def __call__(self, *args, **kwds): """Calls a graph function specialized for this input signature.""" -- cgit v1.2.3 From 7a3fa74736e4359c208dbb66db38c186d6cf6813 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 14:25:47 -0700 Subject: Fix support for seq2seq with mixed precision When the type of the input tensor `x` is not the same as the type of the hidden states cast is required. This mixed precision case occurs when using the seq2seq layer with a data type of float16 or bfloat16. PiperOrigin-RevId: 204364209 --- tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py b/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py index 5e7b422e3c..e742447208 100644 --- a/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py +++ b/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py @@ -625,11 +625,13 @@ def attention_decoder(decoder_inputs, v = [] attention_vec_size = attn_size # Size of query vectors for attention. for a in xrange(num_heads): - k = variable_scope.get_variable("AttnW_%d" % a, - [1, 1, attn_size, attention_vec_size]) + k = variable_scope.get_variable( + "AttnW_%d" % a, [1, 1, attn_size, attention_vec_size], + dtype=dtype) hidden_features.append(nn_ops.conv2d(hidden, k, [1, 1, 1, 1], "SAME")) v.append( - variable_scope.get_variable("AttnV_%d" % a, [attention_vec_size])) + variable_scope.get_variable( + "AttnV_%d" % a, [attention_vec_size], dtype=dtype)) state = initial_state @@ -647,11 +649,13 @@ def attention_decoder(decoder_inputs, with variable_scope.variable_scope("Attention_%d" % a): y = Linear(query, attention_vec_size, True)(query) y = array_ops.reshape(y, [-1, 1, 1, attention_vec_size]) + y = math_ops.cast(y, dtype) # Attention mask is a softmax of v^T * tanh(...). s = math_ops.reduce_sum(v[a] * math_ops.tanh(hidden_features[a] + y), [2, 3]) - a = nn_ops.softmax(s) + a = nn_ops.softmax(math_ops.cast(s, dtype=dtypes.float32)) # Now calculate the attention-weighted vector d. + a = math_ops.cast(a, dtype) d = math_ops.reduce_sum( array_ops.reshape(a, [-1, attn_length, 1, 1]) * hidden, [1, 2]) ds.append(array_ops.reshape(d, [-1, attn_size])) @@ -681,6 +685,7 @@ def attention_decoder(decoder_inputs, raise ValueError("Could not infer input size from input: %s" % inp.name) inputs = [inp] + attns + inputs = [math_ops.cast(e, dtype) for e in inputs] x = Linear(inputs, input_size, True)(inputs) # Run the RNN. cell_output, state = cell(x, state) @@ -693,6 +698,7 @@ def attention_decoder(decoder_inputs, attns = attention(state) with variable_scope.variable_scope("AttnOutputProjection"): + cell_output = math_ops.cast(cell_output, dtype) inputs = [cell_output] + attns output = Linear(inputs, output_size, True)(inputs) if loop_function is not None: -- cgit v1.2.3 From c5e563e57feee793499fae9c3ce28f5176404749 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 14:38:02 -0700 Subject: Utilities for converting TF objects to TF Lite counterparts. PiperOrigin-RevId: 204366431 --- tensorflow/contrib/lite/delegates/eager/BUILD | 35 ++++++++ tensorflow/contrib/lite/delegates/eager/util.cc | 47 ++++++++++ tensorflow/contrib/lite/delegates/eager/util.h | 35 ++++++++ .../contrib/lite/delegates/eager/util_test.cc | 100 +++++++++++++++++++++ 4 files changed, 217 insertions(+) create mode 100644 tensorflow/contrib/lite/delegates/eager/BUILD create mode 100644 tensorflow/contrib/lite/delegates/eager/util.cc create mode 100644 tensorflow/contrib/lite/delegates/eager/util.h create mode 100644 tensorflow/contrib/lite/delegates/eager/util_test.cc diff --git a/tensorflow/contrib/lite/delegates/eager/BUILD b/tensorflow/contrib/lite/delegates/eager/BUILD new file mode 100644 index 0000000000..11cc8185f6 --- /dev/null +++ b/tensorflow/contrib/lite/delegates/eager/BUILD @@ -0,0 +1,35 @@ +# +# This is a TF Lite delegate that is powered by TensorFlow's Eager. +# +package(default_visibility = [ + "//visibility:public", +]) + +licenses(["notice"]) # Apache 2.0 + +cc_library( + name = "util", + srcs = ["util.cc"], + hdrs = ["util.h"], + deps = [ + "//tensorflow/contrib/lite:framework", + "//tensorflow/contrib/lite:kernel_api", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + ], +) + +cc_test( + name = "util_test", + size = "small", + srcs = ["util_test.cc"], + tags = [ + "tflite_not_portable", + ], + deps = [ + ":util", + "//tensorflow/contrib/lite/testing:util", + "//tensorflow/core:lib", + "//testing/base/public:gunit", + ], +) diff --git a/tensorflow/contrib/lite/delegates/eager/util.cc b/tensorflow/contrib/lite/delegates/eager/util.cc new file mode 100644 index 0000000000..04a852e515 --- /dev/null +++ b/tensorflow/contrib/lite/delegates/eager/util.cc @@ -0,0 +1,47 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/delegates/eager/util.h" + +namespace tflite { + +TfLiteStatus ConvertStatus(TfLiteContext* context, + const tensorflow::Status& status) { + if (!status.ok()) { + context->ReportError(context, "%s", status.error_message().c_str()); + return kTfLiteError; + } + return kTfLiteOk; +} + +TfLiteStatus CopyShape(TfLiteContext* context, const tensorflow::Tensor& src, + TfLiteTensor* tensor) { + int num_dims = src.dims(); + TfLiteIntArray* shape = TfLiteIntArrayCreate(num_dims); + for (int j = 0; j < num_dims; ++j) { + // We need to cast from TensorFlow's int64 to TF Lite's int32. Let's + // make sure there's no overflow. + if (src.dim_size(j) >= std::numeric_limits::max()) { + context->ReportError(context, + "Dimension value in TensorFlow shape is larger than " + "supported by TF Lite"); + TfLiteIntArrayFree(shape); + return kTfLiteError; + } + shape->data[j] = static_cast(src.dim_size(j)); + } + return context->ResizeTensor(context, tensor, shape); +} + +} // namespace tflite diff --git a/tensorflow/contrib/lite/delegates/eager/util.h b/tensorflow/contrib/lite/delegates/eager/util.h new file mode 100644 index 0000000000..2696ca8d0d --- /dev/null +++ b/tensorflow/contrib/lite/delegates/eager/util.h @@ -0,0 +1,35 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_UTIL_H_ +#define TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_UTIL_H_ + +#include "tensorflow/contrib/lite/context.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tflite { + +// Converts a tensorflow:Status into a TfLiteStatus. If the original status +// represented an error, reports it using the given 'context'. +TfLiteStatus ConvertStatus(TfLiteContext* context, + const tensorflow::Status& status); + +// Copies the given shape of the given 'src' into a TF Lite 'tensor'. Logs an +// error and returns kTfLiteError if the shape can't be converted. +TfLiteStatus CopyShape(TfLiteContext* context, const tensorflow::Tensor& src, + TfLiteTensor* tensor); +} // namespace tflite + +#endif // TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_UTIL_H_ diff --git a/tensorflow/contrib/lite/delegates/eager/util_test.cc b/tensorflow/contrib/lite/delegates/eager/util_test.cc new file mode 100644 index 0000000000..563f82dec3 --- /dev/null +++ b/tensorflow/contrib/lite/delegates/eager/util_test.cc @@ -0,0 +1,100 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/lite/delegates/eager/util.h" + +#include + +#include +#include +#include "tensorflow/contrib/lite/testing/util.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAre; + +struct TestContext : public TfLiteContext { + string error; + std::vector new_size; +}; + +void ReportError(TfLiteContext* context, const char* format, ...) { + TestContext* c = static_cast(context); + const size_t kBufferSize = 1024; + char temp_buffer[kBufferSize]; + + va_list args; + va_start(args, format); + vsnprintf(temp_buffer, kBufferSize, format, args); + va_end(args); + + c->error = temp_buffer; +} + +TfLiteStatus ResizeTensor(TfLiteContext* context, TfLiteTensor* tensor, + TfLiteIntArray* new_size) { + TestContext* c = static_cast(context); + c->new_size.clear(); + for (int i = 0; i < new_size->size; ++i) { + c->new_size.push_back(new_size->data[i]); + } + TfLiteIntArrayFree(new_size); + return kTfLiteOk; +} + +TEST(UtilTest, ConvertStatus) { + TestContext context; + context.ReportError = ReportError; + + EXPECT_EQ(ConvertStatus(&context, tensorflow::errors::Internal("Some Error")), + kTfLiteError); + EXPECT_EQ(context.error, "Some Error"); + + context.error.clear(); + EXPECT_EQ(ConvertStatus(&context, tensorflow::Status()), kTfLiteOk); + EXPECT_TRUE(context.error.empty()); +} + +TEST(UtilTest, CopyShape) { + TestContext context; + context.ReportError = ReportError; + context.ResizeTensor = ResizeTensor; + + using tensorflow::DT_FLOAT; + using tensorflow::Tensor; + + TfLiteTensor dst; + + EXPECT_EQ(CopyShape(&context, Tensor(), &dst), kTfLiteOk); + EXPECT_THAT(context.new_size, ElementsAre(0)); + + EXPECT_EQ(CopyShape(&context, Tensor(DT_FLOAT, {1, 2}), &dst), kTfLiteOk); + EXPECT_THAT(context.new_size, ElementsAre(1, 2)); + + EXPECT_EQ(CopyShape(&context, Tensor(DT_FLOAT, {1LL << 44, 2}), &dst), + kTfLiteError); + EXPECT_EQ(context.error, + "Dimension value in TensorFlow shape is larger than supported by " + "TF Lite"); +} + +} // namespace +} // namespace tflite + +int main(int argc, char** argv) { + ::tflite::LogToStderr(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} -- cgit v1.2.3 From da798407b4ff72f1daa629e054ccd47b162c9d58 Mon Sep 17 00:00:00 2001 From: Anna R Date: Thu, 12 Jul 2018 14:46:39 -0700 Subject: Support passing TensorFlow API names as a separate v1 argument to tf_export. PiperOrigin-RevId: 204368026 --- tensorflow/python/estimator/estimator.py | 4 +- tensorflow/python/tools/api/generator/api_gen.bzl | 52 ++++++++++++---------- .../tools/api/generator/create_python_api.py | 33 ++++++++++---- .../tools/api/generator/create_python_api_test.py | 6 +-- tensorflow/python/training/quantize_training.i | 2 + tensorflow/python/util/py_checkpoint_reader.i | 1 + tensorflow/python/util/stat_summarizer.i | 25 +++-------- tensorflow/python/util/tf_export.py | 45 ++++++++++++++----- tensorflow/python/util/tf_export_test.py | 2 + .../tools/api/tests/api_compatibility_test.py | 42 ++++++++++++----- 10 files changed, 136 insertions(+), 76 deletions(-) diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index 350a95eea1..253716b43e 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -576,7 +576,9 @@ class Estimator(object): allowed_overrides = set([ '_call_input_fn', '_create_global_step', '_convert_train_steps_to_hooks', '_convert_eval_steps_to_hooks', - '_tf_api_names', '_estimator_api_names', '_estimator_api_constants', + '_tf_api_names', '_tf_api_names_v1', '_estimator_api_names', + '_estimator_api_names_v1', '_estimator_api_constants', + '_estimator_api_constants_v1', '_validate_features_in_predict_input', '_call_model_fn', '_add_meta_graph_for_mode' ]) diff --git a/tensorflow/python/tools/api/generator/api_gen.bzl b/tensorflow/python/tools/api/generator/api_gen.bzl index f9170610b9..2a32e8a893 100644 --- a/tensorflow/python/tools/api/generator/api_gen.bzl +++ b/tensorflow/python/tools/api/generator/api_gen.bzl @@ -102,37 +102,41 @@ ESTIMATOR_API_INIT_FILES = [ # END GENERATED ESTIMATOR FILES ] -# Creates a genrule that generates a directory structure with __init__.py -# files that import all exported modules (i.e. modules with tf_export -# decorators). -# -# Args: -# name: name of genrule to create. -# output_files: List of __init__.py files that should be generated. -# This list should include file name for every module exported using -# tf_export. For e.g. if an op is decorated with -# @tf_export('module1.module2', 'module3'). Then, output_files should -# include module1/module2/__init__.py and module3/__init__.py. -# root_init_template: Python init file that should be used as template for -# root __init__.py file. "# API IMPORTS PLACEHOLDER" comment inside this -# template will be replaced with root imports collected by this genrule. -# srcs: genrule sources. If passing root_init_template, the template file -# must be included in sources. -# api_name: Name of the project that you want to generate API files for -# (e.g. "tensorflow" or "estimator"). -# package: Python package containing the @tf_export decorators you want to -# process -# package_dep: Python library target containing your package. - def gen_api_init_files( name, output_files = TENSORFLOW_API_INIT_FILES, root_init_template = None, srcs = [], api_name = "tensorflow", + api_version = 2, package = "tensorflow.python", package_dep = "//tensorflow/python:no_contrib", output_package = "tensorflow"): + """Creates API directory structure and __init__.py files. + + Creates a genrule that generates a directory structure with __init__.py + files that import all exported modules (i.e. modules with tf_export + decorators). + + Args: + name: name of genrule to create. + output_files: List of __init__.py files that should be generated. + This list should include file name for every module exported using + tf_export. For e.g. if an op is decorated with + @tf_export('module1.module2', 'module3'). Then, output_files should + include module1/module2/__init__.py and module3/__init__.py. + root_init_template: Python init file that should be used as template for + root __init__.py file. "# API IMPORTS PLACEHOLDER" comment inside this + template will be replaced with root imports collected by this genrule. + srcs: genrule sources. If passing root_init_template, the template file + must be included in sources. + api_name: Name of the project that you want to generate API files for + (e.g. "tensorflow" or "estimator"). + api_version: TensorFlow API version to generate. Must be either 1 or 2. + package: Python package containing the @tf_export decorators you want to + process + package_dep: Python library target containing your package. + """ root_init_template_flag = "" if root_init_template: root_init_template_flag = "--root_init_template=$(location " + root_init_template + ")" @@ -156,8 +160,8 @@ def gen_api_init_files( cmd = ( "$(location :" + api_gen_binary_target + ") " + root_init_template_flag + " --apidir=$(@D) --apiname=" + - api_name + " --package=" + package + " --output_package=" + - output_package + " $(OUTS)"), + api_name + " --apiversion=" + str(api_version) + " --package=" + package + + " --output_package=" + output_package + " $(OUTS)"), srcs = srcs, tools = [":" + api_gen_binary_target ], visibility = ["//tensorflow:__pkg__"], diff --git a/tensorflow/python/tools/api/generator/create_python_api.py b/tensorflow/python/tools/api/generator/create_python_api.py index e78fe4b738..863c922216 100644 --- a/tensorflow/python/tools/api/generator/create_python_api.py +++ b/tensorflow/python/tools/api/generator/create_python_api.py @@ -29,6 +29,7 @@ from tensorflow.python.util import tf_decorator from tensorflow.python.util import tf_export API_ATTRS = tf_export.API_ATTRS +API_ATTRS_V1 = tf_export.API_ATTRS_V1 _DEFAULT_PACKAGE = 'tensorflow.python' _GENFILES_DIR_SUFFIX = 'genfiles/' @@ -159,13 +160,16 @@ __all__.remove('print_function') return module_text_map -def get_api_init_text(package, output_package, api_name): +def get_api_init_text(package, output_package, api_name, api_version): """Get a map from destination module to __init__.py code for that module. Args: package: Base python package containing python with target tf_export decorators. + output_package: Base output python package where generated API will + be added. api_name: API you want to generate (e.g. `tensorflow` or `estimator`). + api_version: API version you want to generate (`v1` or `v2`). Returns: A dictionary where @@ -173,6 +177,12 @@ def get_api_init_text(package, output_package, api_name): value: (string) text that should be in __init__.py files for corresponding modules. """ + if api_version == 1: + names_attr = API_ATTRS_V1[api_name].names + constants_attr = API_ATTRS_V1[api_name].constants + else: + names_attr = API_ATTRS[api_name].names + constants_attr = API_ATTRS[api_name].constants module_code_builder = _ModuleInitCodeBuilder() # Traverse over everything imported above. Specifically, @@ -193,7 +203,7 @@ def get_api_init_text(package, output_package, api_name): attr = getattr(module, module_contents_name) # If attr is _tf_api_constants attribute, then add the constants. - if module_contents_name == API_ATTRS[api_name].constants: + if module_contents_name == constants_attr: for exports, value in attr: for export in exports: names = export.split('.') @@ -205,9 +215,8 @@ def get_api_init_text(package, output_package, api_name): _, attr = tf_decorator.unwrap(attr) # If attr is a symbol with _tf_api_names attribute, then # add import for it. - if (hasattr(attr, '__dict__') and - API_ATTRS[api_name].names in attr.__dict__): - for export in getattr(attr, API_ATTRS[api_name].names): # pylint: disable=protected-access + if (hasattr(attr, '__dict__') and names_attr in attr.__dict__): + for export in getattr(attr, names_attr): # pylint: disable=protected-access names = export.split('.') dest_module = '.'.join(names[:-1]) module_code_builder.add_import( @@ -297,7 +306,7 @@ def get_module_docstring(module_name, package, api_name): def create_api_files( output_files, package, root_init_template, output_dir, output_package, - api_name): + api_name, api_version): """Creates __init__.py files for the Python API. Args: @@ -309,7 +318,9 @@ def create_api_files( "#API IMPORTS PLACEHOLDER" comment in the template file will be replaced with imports. output_dir: output API root directory. + output_package: Base output package where generated API will be added. api_name: API you want to generate (e.g. `tensorflow` or `estimator`). + api_version: API version to generate (`v1` or `v2`). Raises: ValueError: if an output file is not under api/ directory, @@ -326,7 +337,8 @@ def create_api_files( os.makedirs(os.path.dirname(file_path)) open(file_path, 'a').close() - module_text_map = get_api_init_text(package, output_package, api_name) + module_text_map = get_api_init_text( + package, output_package, api_name, api_version) # Add imports to output files. missing_output_files = [] @@ -384,6 +396,10 @@ def main(): '--apiname', required=True, type=str, choices=API_ATTRS.keys(), help='The API you want to generate.') + parser.add_argument( + '--apiversion', default=2, type=int, + choices=[1, 2], + help='The API version you want to generate.') parser.add_argument( '--output_package', default='tensorflow', type=str, help='Root output package.') @@ -401,7 +417,8 @@ def main(): # Populate `sys.modules` with modules containing tf_export(). importlib.import_module(args.package) create_api_files(outputs, args.package, args.root_init_template, - args.apidir, args.output_package, args.apiname) + args.apidir, args.output_package, args.apiname, + args.apiversion) if __name__ == '__main__': diff --git a/tensorflow/python/tools/api/generator/create_python_api_test.py b/tensorflow/python/tools/api/generator/create_python_api_test.py index 368b4c37e8..a565a49d96 100644 --- a/tensorflow/python/tools/api/generator/create_python_api_test.py +++ b/tensorflow/python/tools/api/generator/create_python_api_test.py @@ -59,7 +59,7 @@ class CreatePythonApiTest(test.TestCase): imports = create_python_api.get_api_init_text( package=create_python_api._DEFAULT_PACKAGE, output_package='tensorflow', - api_name='tensorflow') + api_name='tensorflow', api_version=1) expected_import = ( 'from tensorflow.python.test_module ' 'import test_op as test_op1') @@ -77,7 +77,7 @@ class CreatePythonApiTest(test.TestCase): imports = create_python_api.get_api_init_text( package=create_python_api._DEFAULT_PACKAGE, output_package='tensorflow', - api_name='tensorflow') + api_name='tensorflow', api_version=2) expected_import = ('from tensorflow.python.test_module ' 'import TestClass') self.assertTrue( @@ -88,7 +88,7 @@ class CreatePythonApiTest(test.TestCase): imports = create_python_api.get_api_init_text( package=create_python_api._DEFAULT_PACKAGE, output_package='tensorflow', - api_name='tensorflow') + api_name='tensorflow', api_version=1) expected = ('from tensorflow.python.test_module ' 'import _TEST_CONSTANT') self.assertTrue(expected in str(imports), diff --git a/tensorflow/python/training/quantize_training.i b/tensorflow/python/training/quantize_training.i index fb5e47efa0..54d6789616 100644 --- a/tensorflow/python/training/quantize_training.i +++ b/tensorflow/python/training/quantize_training.i @@ -73,6 +73,8 @@ def do_quantize_training_on_graphdef(input_graph, num_bits): do_quantize_training_on_graphdef._tf_api_names = [ 'train.do_quantize_training_on_graphdef'] +do_quantize_training_on_graphdef._tf_api_names_v1 = [ + 'train.do_quantize_training_on_graphdef'] %} %unignoreall diff --git a/tensorflow/python/util/py_checkpoint_reader.i b/tensorflow/python/util/py_checkpoint_reader.i index 8004898cbc..1c73f7f06f 100644 --- a/tensorflow/python/util/py_checkpoint_reader.i +++ b/tensorflow/python/util/py_checkpoint_reader.i @@ -166,6 +166,7 @@ def NewCheckpointReader(filepattern): return CheckpointReader(compat.as_bytes(filepattern), status) NewCheckpointReader._tf_api_names = ['train.NewCheckpointReader'] +NewCheckpointReader._tf_api_names_v1 = ['train.NewCheckpointReader'] %} %include "tensorflow/c/checkpoint_reader.h" diff --git a/tensorflow/python/util/stat_summarizer.i b/tensorflow/python/util/stat_summarizer.i index 73fa85494b..a5a7984d91 100644 --- a/tensorflow/python/util/stat_summarizer.i +++ b/tensorflow/python/util/stat_summarizer.i @@ -27,8 +27,8 @@ limitations under the License. %ignoreall -%unignore _NewStatSummarizer; -%unignore _DeleteStatSummarizer; +%unignore NewStatSummarizer; +%unignore DeleteStatSummarizer; %unignore tensorflow; %unignore tensorflow::StatSummarizer; %unignore tensorflow::StatSummarizer::StatSummarizer; @@ -43,20 +43,20 @@ limitations under the License. // TODO(ashankar): Remove the unused argument from the API. %{ -tensorflow::StatSummarizer* _NewStatSummarizer( +tensorflow::StatSummarizer* NewStatSummarizer( const string& unused) { return new tensorflow::StatSummarizer(tensorflow::StatSummarizerOptions()); } %} %{ -void _DeleteStatSummarizer(tensorflow::StatSummarizer* ss) { +void DeleteStatSummarizer(tensorflow::StatSummarizer* ss) { delete ss; } %} -tensorflow::StatSummarizer* _NewStatSummarizer(const string& unused); -void _DeleteStatSummarizer(tensorflow::StatSummarizer* ss); +tensorflow::StatSummarizer* NewStatSummarizer(const string& unused); +void DeleteStatSummarizer(tensorflow::StatSummarizer* ss); %extend tensorflow::StatSummarizer { void ProcessStepStatsStr(const string& step_stats_str) { @@ -76,16 +76,3 @@ void _DeleteStatSummarizer(tensorflow::StatSummarizer* ss); %include "tensorflow/core/util/stat_summarizer_options.h" %include "tensorflow/core/util/stat_summarizer.h" %unignoreall - -%insert("python") %{ - -# Wrapping NewStatSummarizer and DeletStatSummarizer because -# SWIG-generated functions are built-in functions and do not support -# setting _tf_api_names attribute. - -def NewStatSummarizer(unused): - return _NewStatSummarizer(unused) - -def DeleteStatSummarizer(stat_summarizer): - _DeleteStatSummarizer(stat_summarizer) -%} diff --git a/tensorflow/python/util/tf_export.py b/tensorflow/python/util/tf_export.py index e154ffb68a..c362d588ab 100644 --- a/tensorflow/python/util/tf_export.py +++ b/tensorflow/python/util/tf_export.py @@ -63,6 +63,15 @@ API_ATTRS = { '_estimator_api_constants') } +API_ATTRS_V1 = { + TENSORFLOW_API_NAME: _Attributes( + '_tf_api_names_v1', + '_tf_api_constants_v1'), + ESTIMATOR_API_NAME: _Attributes( + '_estimator_api_names_v1', + '_estimator_api_constants_v1') +} + class SymbolAlreadyExposedError(Exception): """Raised when adding API names to symbol that already has API names.""" @@ -78,13 +87,16 @@ class api_export(object): # pylint: disable=invalid-name Args: *args: API names in dot delimited format. **kwargs: Optional keyed arguments. - overrides: List of symbols that this is overriding + v1: Names for the TensorFlow V1 API. If not set, we will use V2 API + names both for TensorFlow V1 and V2 APIs. + overrides: List of symbols that this is overriding (those overrided api exports will be removed). Note: passing overrides has no effect on exporting a constant. - api_name: Name of the API you want to generate (e.g. `tensorflow` or + api_name: Name of the API you want to generate (e.g. `tensorflow` or `estimator`). Default is `tensorflow`. """ self._names = args + self._names_v1 = kwargs.get('v1', args) self._api_name = kwargs.get('api_name', TENSORFLOW_API_NAME) self._overrides = kwargs.get('overrides', []) @@ -102,24 +114,27 @@ class api_export(object): # pylint: disable=invalid-name and kwarg `allow_multiple_exports` not set. """ api_names_attr = API_ATTRS[self._api_name].names - + api_names_attr_v1 = API_ATTRS_V1[self._api_name].names # Undecorate overridden names for f in self._overrides: _, undecorated_f = tf_decorator.unwrap(f) delattr(undecorated_f, api_names_attr) + delattr(undecorated_f, api_names_attr_v1) _, undecorated_func = tf_decorator.unwrap(func) + self.set_attr(undecorated_func, api_names_attr, self._names) + self.set_attr(undecorated_func, api_names_attr_v1, self._names_v1) + return func + def set_attr(self, func, api_names_attr, names): # Check for an existing api. We check if attribute name is in # __dict__ instead of using hasattr to verify that subclasses have # their own _tf_api_names as opposed to just inheriting it. - if api_names_attr in undecorated_func.__dict__: + if api_names_attr in func.__dict__: raise SymbolAlreadyExposedError( 'Symbol %s is already exposed as %s.' % - (undecorated_func.__name__, getattr( - undecorated_func, api_names_attr))) # pylint: disable=protected-access - setattr(undecorated_func, api_names_attr, self._names) - return func + (func.__name__, getattr(func, api_names_attr))) # pylint: disable=protected-access + setattr(func, api_names_attr, names) def export_constant(self, module_name, name): """Store export information for constants/string literals. @@ -140,12 +155,20 @@ class api_export(object): # pylint: disable=invalid-name name: (string) Current constant name. """ module = sys.modules[module_name] - if not hasattr(module, API_ATTRS[self._api_name].constants): - setattr(module, API_ATTRS[self._api_name].constants, []) + api_constants_attr = API_ATTRS[self._api_name].constants + api_constants_attr_v1 = API_ATTRS_V1[self._api_name].constants + + if not hasattr(module, api_constants_attr): + setattr(module, api_constants_attr, []) # pylint: disable=protected-access - getattr(module, API_ATTRS[self._api_name].constants).append( + getattr(module, api_constants_attr).append( (self._names, name)) + if not hasattr(module, api_constants_attr_v1): + setattr(module, api_constants_attr_v1, []) + getattr(module, api_constants_attr_v1).append( + (self._names_v1, name)) + tf_export = functools.partial(api_export, api_name=TENSORFLOW_API_NAME) estimator_export = functools.partial(tf_export, api_name=ESTIMATOR_API_NAME) diff --git a/tensorflow/python/util/tf_export_test.py b/tensorflow/python/util/tf_export_test.py index b9e26ecb33..4ae1dc55e0 100644 --- a/tensorflow/python/util/tf_export_test.py +++ b/tensorflow/python/util/tf_export_test.py @@ -60,6 +60,8 @@ class ValidateExportTest(test.TestCase): for symbol in [_test_function, _test_function, TestClassA, TestClassB]: if hasattr(symbol, '_tf_api_names'): del symbol._tf_api_names + if hasattr(symbol, '_tf_api_names_v1'): + del symbol._tf_api_names_v1 def _CreateMockModule(self, name): mock_module = self.MockModule(name) diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py index 90375a794f..d1b34fb242 100644 --- a/tensorflow/tools/api/tests/api_compatibility_test.py +++ b/tensorflow/tools/api/tests/api_compatibility_test.py @@ -34,6 +34,13 @@ import sys import unittest import tensorflow as tf +# pylint: disable=g-import-not-at-top +try: + from tensorflow.compat import v1 as tf_v1 + # We import compat.v1 as tf_v1 instead. + del tf.compat.v1 +except ImportError: + tf_v1 = None from google.protobuf import message from google.protobuf import text_format @@ -46,6 +53,7 @@ from tensorflow.tools.api.lib import api_objects_pb2 from tensorflow.tools.api.lib import python_object_to_proto_visitor from tensorflow.tools.common import public_api from tensorflow.tools.common import traverse +# pylint: enable=g-import-not-at-top # FLAGS defined at the bottom: @@ -215,25 +223,19 @@ class ApiCompatibilityTest(test.TestCase): visitor.do_not_descend_map['tf'].append('contrib') traverse.traverse(tf, visitor) - @unittest.skipUnless( - sys.version_info.major == 2, - 'API compabitility test goldens are generated using python2.') - def testAPIBackwardsCompatibility(self): - # Extract all API stuff. + def checkBackwardsCompatibility(self, root, golden_file_pattern): + # Extract all API stuff. visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor() public_api_visitor = public_api.PublicAPIVisitor(visitor) public_api_visitor.do_not_descend_map['tf'].append('contrib') public_api_visitor.do_not_descend_map['tf.GPUOptions'] = ['Experimental'] - traverse.traverse(tf, public_api_visitor) + traverse.traverse(root, public_api_visitor) proto_dict = visitor.GetProtos() # Read all golden files. - expression = os.path.join( - resource_loader.get_root_dir_with_all_resources(), - _KeyToFilePath('*')) - golden_file_list = file_io.get_matching_files(expression) + golden_file_list = file_io.get_matching_files(golden_file_pattern) def _ReadFileToProto(filename): """Read a filename, create a protobuf from its contents.""" @@ -254,6 +256,26 @@ class ApiCompatibilityTest(test.TestCase): verbose=FLAGS.verbose_diffs, update_goldens=FLAGS.update_goldens) + @unittest.skipUnless( + sys.version_info.major == 2, + 'API compabitility test goldens are generated using python2.') + def testAPIBackwardsCompatibility(self): + golden_file_pattern = os.path.join( + resource_loader.get_root_dir_with_all_resources(), + _KeyToFilePath('*')) + self.checkBackwardsCompatibility(tf, golden_file_pattern) + + @unittest.skipUnless( + sys.version_info.major == 2, + 'API compabitility test goldens are generated using python2.') + def testAPIBackwardsCompatibilityV1(self): + if not tf_v1: + return + golden_file_pattern = os.path.join( + resource_loader.get_root_dir_with_all_resources(), + _KeyToFilePath('*')) + self.checkBackwardsCompatibility(tf_v1, golden_file_pattern) + if __name__ == '__main__': parser = argparse.ArgumentParser() -- cgit v1.2.3 From 4297b9d492327072c0d64b7243925eba192fe028 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Thu, 12 Jul 2018 14:49:02 -0700 Subject: Broad refactor(part 5): Add liveness and reaching definitions analyses based on the refactored CFG. Not yet enabled. PiperOrigin-RevId: 204368511 --- .../contrib/autograph/pyct/static_analysis/BUILD | 36 ++- .../autograph/pyct/static_analysis/liveness.py | 200 +++++++++++++++ .../pyct/static_analysis/liveness_test.py | 149 +++++++++++ .../pyct/static_analysis/reaching_definitions.py | 273 +++++++++++++++++++++ .../static_analysis/reaching_definitions_test.py | 221 +++++++++++++++++ 5 files changed, 878 insertions(+), 1 deletion(-) create mode 100644 tensorflow/contrib/autograph/pyct/static_analysis/liveness.py create mode 100644 tensorflow/contrib/autograph/pyct/static_analysis/liveness_test.py create mode 100644 tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py create mode 100644 tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions_test.py diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/BUILD b/tensorflow/contrib/autograph/pyct/static_analysis/BUILD index bcf2dacec2..25f78536e0 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/BUILD +++ b/tensorflow/contrib/autograph/pyct/static_analysis/BUILD @@ -19,8 +19,10 @@ py_library( srcs = [ "activity.py", "annos.py", - "cfg.py", + "cfg.py", # TODO(mdan): Remove. "live_values.py", + "liveness.py", + "reaching_definitions.py", "type_info.py", ], srcs_version = "PY2AND3", @@ -28,6 +30,7 @@ py_library( deps = [ "//tensorflow/contrib/autograph/pyct", "//tensorflow/contrib/autograph/utils", + "//tensorflow/python:util", "@gast_archive//:gast", ], ) @@ -70,6 +73,37 @@ py_test( ], ) +# TODO(mdan): Enable these tests once child change is in. +py_test( + name = "liveness_test", + srcs = ["liveness_test.py"], + srcs_version = "PY2AND3", + tags = [ + "manual", + "notap", + ], + deps = [ + ":static_analysis", + "//tensorflow/contrib/autograph/pyct", + "//tensorflow/python:client_testlib", + ], +) + +py_test( + name = "reaching_definitions_test", + srcs = ["reaching_definitions_test.py"], + srcs_version = "PY2AND3", + tags = [ + "manual", + "notap", + ], + deps = [ + ":static_analysis", + "//tensorflow/contrib/autograph/pyct", + "//tensorflow/python:client_testlib", + ], +) + py_test( name = "type_info_test", srcs = ["type_info_test.py"], diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/liveness.py b/tensorflow/contrib/autograph/pyct/static_analysis/liveness.py new file mode 100644 index 0000000000..bf29d868a2 --- /dev/null +++ b/tensorflow/contrib/autograph/pyct/static_analysis/liveness.py @@ -0,0 +1,200 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Live variable analysis. + +This analysis attaches a set containing the live symbols that are live at the +exit of control flow statements. + +Requires activity analysis. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gast + +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import cfg +from tensorflow.contrib.autograph.pyct import transformer +from tensorflow.contrib.autograph.pyct.static_analysis import annos + + +class Analyzer(cfg.GraphVisitor): + """CFG visitor that performs liveness analysis at statement level.""" + + def __init__(self, graph): + super(Analyzer, self).__init__(graph) + # This allows communicating that nodes generate extra symbols, + # e.g. those that a function definition closes over. + self.extra_gen = {} + + def init_state(self, _): + return set() + + def visit_node(self, node): + prev_live_in = self.in_[node] + + if anno.hasanno(node.ast_node, anno.Static.SCOPE): + node_scope = anno.getanno(node.ast_node, anno.Static.SCOPE) + + gen = node_scope.used | self.extra_gen.get(node.ast_node, frozenset()) + # TODO(mdan): verify whether composites' parents need to be added. + # E.g. if x.y is live whether x needs to be added. Theoretically the + # activity analysis should have both so that wouldn't be needed. + kill = node_scope.modified + + live_out = set() + for n in node.next: + live_out |= self.in_[n] + live_in = gen | (live_out - kill) + + else: + # Nodes that don't have a scope annotation are assumed not to touch any + # symbols. + # This Name node below is a literal name, e.g. False + assert isinstance(node.ast_node, + (gast.Name, gast.Continue, gast.Break)), type( + node.ast_node) + live_in = prev_live_in + live_out = live_in + + self.in_[node] = live_in + self.out[node] = live_out + + # TODO(mdan): Move this to the superclass? + return prev_live_in != live_in + + +class WholeTreeAnalyzer(transformer.Base): + """Runs liveness analysis on each of the functions defined in the AST. + + If a function defined other local functions, those will have separate CFGs. + However, dataflow analysis needs to tie up these CFGs to properly emulate the + effect of closures. In the case of liveness, the parent function's live + variables must account for the variables that are live at the entry of each + subfunction. For example: + + def foo(): + # baz is live here + def bar(): + print(baz) + + This analyzer runs liveness analysis on each individual function, accounting + for the effect above. + """ + + def __init__(self, source_info, graphs): + super(WholeTreeAnalyzer, self).__init__(source_info) + self.graphs = graphs + self.current_analyzer = None + self.analyzers = {} + + def visit_FunctionDef(self, node): + parent_analyzer = self.current_analyzer + subgraph = self.graphs[node] + + # Postorder tree processing makes this a bit complicated: + # 1. construct an analyzer object and put it on stack + # 2. recursively walk the subtree; this will initialize the analyzer's + # in_ state properly (done in a block below) + # 3. run the final analysis + analyzer = Analyzer(subgraph) + self.current_analyzer = analyzer + node = self.generic_visit(node) + analyzer.visit_reverse() + + if parent_analyzer is not None: + # Wire the state between the two subgraphs' analyzers. + child_in_state = analyzer.in_[subgraph.entry] + # Exception: symbols modified in the child function are local to it + body_scope = anno.getanno(node, annos.NodeAnno.BODY_SCOPE) + for qn in body_scope.modified: + # Note: a function modifying the symbol doesn't make that symbol + # live at the function's entry. In fact when that happens it is + # probably a case of undefined assignment, like this: + # + # bar = 0 + # def foo(): + # print(bar) # bar is undefined here! + # bar = 1 + # + # Hence we use discard and not remove below. + child_in_state.discard(qn) + parent_analyzer.extra_gen[node] = frozenset(child_in_state,) + + self.analyzers[node] = analyzer + self.current_analyzer = parent_analyzer + return node + + def visit_nonlocal(self, node): + raise NotImplementedError() + + def visit_global(self, node): + raise NotImplementedError() + + +class Annotator(transformer.Base): + """AST visitor that annotates each control flow block with live symbols.""" + + # Note: additional nodes may be added as needed. + + def __init__(self, source_info, cross_function_analyzer): + super(Annotator, self).__init__(source_info) + self.cross_function_analyzer = cross_function_analyzer + self.current_analyzer = None + + def visit_FunctionDef(self, node): + parent_analyzer = self.current_analyzer + self.current_analyzer = self.cross_function_analyzer.analyzers[node] + + node = self.generic_visit(node) + self.current_analyzer = parent_analyzer + return node + + def _aggregate_successors_live_in(self, node): + successors = self.current_analyzer.graph.stmt_next[node] + node_live_out = set() + for s in successors: + node_live_out.update(self.current_analyzer.in_[s]) + anno.setanno(node, anno.Static.LIVE_VARS_OUT, frozenset(node_live_out)) + node = self.generic_visit(node) + return node + + def visit_If(self, node): + return self._aggregate_successors_live_in(node) + + def visit_For(self, node): + return self._aggregate_successors_live_in(node) + + def visit_While(self, node): + return self._aggregate_successors_live_in(node) + + +def resolve(node, source_info, graphs): + """Resolves the live symbols at the exit of control flow statements. + + Args: + node: ast.AST + source_info: transformer.SourceInfo + graphs: Dict[ast.FunctionDef, cfg.Graph] + Returns: + ast.AST + """ + cross_function_analyzer = WholeTreeAnalyzer(source_info, graphs) + node = cross_function_analyzer.visit(node) + visitor = Annotator(source_info, cross_function_analyzer) + node = visitor.visit(node) + return node diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/liveness_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/liveness_test.py new file mode 100644 index 0000000000..d53adb28af --- /dev/null +++ b/tensorflow/contrib/autograph/pyct/static_analysis/liveness_test.py @@ -0,0 +1,149 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for liveness module.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import cfg +from tensorflow.contrib.autograph.pyct import parser +from tensorflow.contrib.autograph.pyct import qual_names +from tensorflow.contrib.autograph.pyct import transformer +from tensorflow.contrib.autograph.pyct.static_analysis import activity +from tensorflow.contrib.autograph.pyct.static_analysis import liveness +from tensorflow.python.platform import test + + +class LivenessTest(test.TestCase): + + def _parse_and_analyze(self, test_fn): + node, source = parser.parse_entity(test_fn) + entity_info = transformer.EntityInfo( + source_code=source, + source_file=None, + namespace={}, + arg_values=None, + arg_types=None, + owner_type=None) + node = qual_names.resolve(node) + node = activity.resolve(node, entity_info) + graphs = cfg.build(node) + liveness.resolve(node, entity_info, graphs) + return node + + def assertHasLiveOut(self, node, expected): + live_out = anno.getanno(node, anno.Static.LIVE_VARS_OUT) + live_out_str = set(str(v) for v in live_out) + if not expected: + expected = () + if not isinstance(expected, tuple): + expected = (expected,) + self.assertSetEqual(live_out_str, set(expected)) + + def test_stacked_if(self): + + def test_fn(x, a): + if a > 0: + x = 0 + if a > 1: + x = 1 + return x + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasLiveOut(fn_body[0], ('a', 'x')) + self.assertHasLiveOut(fn_body[1], 'x') + + def test_stacked_if_else(self): + + def test_fn(x, a): + if a > 0: + x = 0 + if a > 1: + x = 1 + else: + x = 2 + return x + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasLiveOut(fn_body[0], 'a') + self.assertHasLiveOut(fn_body[1], 'x') + + def test_for_basic(self): + + def test_fn(x, a): + for i in range(a): + x += i + return x + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasLiveOut(fn_body[0], 'x') + + def test_attributes(self): + + def test_fn(x, a): + if a > 0: + x.y = 0 + return x.y + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasLiveOut(fn_body[0], ('x.y', 'x')) + + def test_nested_functions(self): + + def test_fn(a, b): + if b: + a = [] + + def foo(): + return a + + foo() + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasLiveOut(fn_body[0], 'a') + + def test_nested_functions_isolation(self): + + def test_fn(b): + if b: + a = 0 # pylint:disable=unused-variable + + def child(): + max(a) # pylint:disable=used-before-assignment + a = 1 + return a + + child() + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasLiveOut(fn_body[0], 'max') + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py b/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py new file mode 100644 index 0000000000..4d79b0a56a --- /dev/null +++ b/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py @@ -0,0 +1,273 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Reaching definition analysis. + +This analysis attaches a set of a Definition objects to each symbol, one +for each distinct definition that may reach it. The Definition objects are +mutable and may be used by subsequent analyses to further annotate data like +static type and value information. +The analysis also attaches the set of the symbols defined at the entry of +control flow statements. + +Requires activity analysis. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gast + +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import cfg +from tensorflow.contrib.autograph.pyct import transformer +from tensorflow.contrib.autograph.pyct.static_analysis import annos + + +class Definition(object): + """Definition objects describe a unique definition of a variable. + + Subclasses of this may be used by passing an appropriate factory fuction to + resolve. + + Attributes: + param_of: Optional[ast.AST] + """ + + def __init__(self): + self.param_of = None + + def __repr__(self): + return '%s[%d]' % (self.__class__.__name__, id(self)) + + +class _NodeState(object): + """Abstraction for the state of the CFG walk for reaching definition analysis. + + This is a value type. Only implements the strictly necessary operators. + + Attributes: + value: Dict[qual_names.QN, Set[Definition, ...]], the defined symbols and + their possible definitions + """ + + def __init__(self, init_from=None): + if init_from: + if isinstance(init_from, _NodeState): + self.value = { + s: set(other_infos) for s, other_infos in init_from.value.items() + } + elif isinstance(init_from, dict): + self.value = {s: set((init_from[s],)) for s in init_from} + else: + assert False, init_from + else: + self.value = {} + + def __eq__(self, other): + if frozenset(self.value.keys()) != frozenset(other.value.keys()): + return False + ret = all(self.value[s] == other.value[s] for s in self.value) + return ret + + def __ne__(self, other): + return not self.__eq__(other) + + def __or__(self, other): + assert isinstance(other, _NodeState) + result = _NodeState(self) + for s, other_infos in other.value.items(): + if s in result.value: + result.value[s].update(other_infos) + else: + result.value[s] = set(other_infos) + return result + + def __sub__(self, other): + assert isinstance(other, set) + result = _NodeState(self) + for s in other: + result.value.pop(s, None) + return result + + def __repr__(self): + return 'NodeState[%s]=%s' % (id(self), repr(self.value)) + + +class Analyzer(cfg.GraphVisitor): + """CFG visitor that determines reaching definitions at statement level.""" + + def __init__(self, graph, definition_factory): + self._definition_factory = definition_factory + super(Analyzer, self).__init__(graph) + self.defs_by_ast_node = {} + # This allows communicating that nodes have extra reaching definitions, + # e.g. those that a function closes over. + self.extra_in = {} + + self.gen_map = {} + + def init_state(self, _): + return _NodeState() + + def visit_node(self, node): + prev_defs_out = self.out[node] + + defs_in = _NodeState(self.extra_in.get(node.ast_node, None)) + for n in node.prev: + defs_in |= self.out[n] + + if anno.hasanno(node.ast_node, anno.Static.SCOPE): + node_scope = anno.getanno(node.ast_node, anno.Static.SCOPE) + # The definition objects created by each node must be singletons because + # their ids are used in equality checks. + if node not in self.gen_map: + node_symbols = {} + for s in node_scope.modified: + def_ = self._definition_factory() + if s in node_scope.params: + def_.param_of = node_scope.params[s] + node_symbols[s] = def_ + self.gen_map[node] = _NodeState(node_symbols) + + gen = self.gen_map[node] + kill = node_scope.modified + defs_out = gen | (defs_in - kill) + + else: + # Nodes that don't have a scope annotation are assumed not to touch any + # symbols. + # This Name node below is a literal name, e.g. False + # This can also happen if activity.py forgot to annotate the node with a + # scope object. + assert isinstance(node.ast_node, + (gast.Name, gast.Break, gast.Continue)), (node.ast_node, + node) + defs_out = defs_in + + self.in_[node] = defs_in + self.out[node] = defs_out + self.defs_by_ast_node[node.ast_node] = defs_out.value + + # TODO(mdan): Move this to the superclass? + return prev_defs_out != defs_out + + +class WholeTreeAnalyzer(transformer.Base): + """AST visitor that annotates each symbol name with its reaching definitions. + + Simultaneously, the visitor runs the dataflow analysis on each function node, + accounting for the effect of closures. For example: + + def foo(): + bar = 1 + def baz(): + # bar = 1 reaches here + """ + + def __init__(self, source_info, graphs, definition_factory): + super(WholeTreeAnalyzer, self).__init__(source_info) + self.stmt_reaching_defs_info = None + self.graphs = graphs + self.current_analyzer = None + self.definition_factory = definition_factory + self.current_stmt_defs = None + + def visit_FunctionDef(self, node): + parent_analyzer = self.current_analyzer + subgraph = self.graphs[node] + + # Preorder tree processing: + # 1. if this is a child function, the parent was already analyzed and it + # has the proper state value for the subgraph's entry + # 2. analyze the current function body + # 2. recursively walk the subtree; child functions will be processed + analyzer = Analyzer(subgraph, self.definition_factory) + if parent_analyzer is not None: + # Wire the state between the two subgraphs' analyzers. + parent_out_state = parent_analyzer.out[parent_analyzer.graph.index[node]] + # Exception: symbols modified in the child function are local to it + body_scope = anno.getanno(node, annos.NodeAnno.BODY_SCOPE) + parent_out_state -= body_scope.modified + analyzer.extra_in[node.args] = parent_out_state + + # Complete the analysis for the local function and annotate its body. + analyzer.visit_forward() + + # Recursively process any remaining subfunctions. + self.current_analyzer = analyzer + node = self.generic_visit(node) + self.current_analyzer = parent_analyzer + + return node + + def visit_nonlocal(self, node): + raise NotImplementedError() + + def visit_global(self, node): + raise NotImplementedError() + + def visit_Name(self, node): + if self.current_analyzer is None: + # Names may appear outside function defs - for example in class + # definitions. + return node + + qn = anno.getanno(node, anno.Basic.QN) + assert self.current_stmt_defs is not None, ( + 'name node outside of any statement?') + anno.setanno(node, anno.Static.DEFINITIONS, + tuple(self.current_stmt_defs.get(qn, ()))) + return node + + def _aggregate_predecessors_defined_in(self, node): + preds = self.current_analyzer.graph.stmt_prev[node] + node_defined_in = set() + for p in preds: + node_defined_in |= set(self.current_analyzer.out[p].value.keys()) + anno.setanno(node, anno.Static.DEFINED_VARS_IN, frozenset(node_defined_in)) + node = self.generic_visit(node) + return node + + def visit_If(self, node): + return self._aggregate_predecessors_defined_in(node) + + def visit_For(self, node): + return self._aggregate_predecessors_defined_in(node) + + def visit_While(self, node): + return self._aggregate_predecessors_defined_in(node) + + def visit(self, node): + if (self.current_analyzer is not None and + node in self.current_analyzer.defs_by_ast_node): + self.current_stmt_defs = self.current_analyzer.defs_by_ast_node[node] + return super(WholeTreeAnalyzer, self).visit(node) + + +def resolve(node, source_info, graphs, definition_factory): + """Resolves reaching definitions for each symbol. + + Args: + node: ast.AST + source_info: transformer.SourceInfo + graphs: Dict[ast.FunctionDef, cfg.Graph] + definition_factory: Callable[[], Definition] + Returns: + ast.AST + """ + visitor = WholeTreeAnalyzer(source_info, graphs, definition_factory) + node = visitor.visit(node) + return node diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions_test.py new file mode 100644 index 0000000000..0410bb2a35 --- /dev/null +++ b/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions_test.py @@ -0,0 +1,221 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for reaching_definitions module.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import cfg +from tensorflow.contrib.autograph.pyct import parser +from tensorflow.contrib.autograph.pyct import qual_names +from tensorflow.contrib.autograph.pyct import transformer +from tensorflow.contrib.autograph.pyct.static_analysis import activity +from tensorflow.contrib.autograph.pyct.static_analysis import reaching_definitions +from tensorflow.python.platform import test + + +class DefinitionInfoTest(test.TestCase): + + def _parse_and_analyze(self, test_fn): + node, source = parser.parse_entity(test_fn) + entity_info = transformer.EntityInfo( + source_code=source, + source_file=None, + namespace={}, + arg_values=None, + arg_types=None, + owner_type=None) + node = qual_names.resolve(node) + node = activity.resolve(node, entity_info) + graphs = cfg.build(node) + node = reaching_definitions.resolve(node, entity_info, graphs, + reaching_definitions.Definition) + return node + + def assertHasDefs(self, node, num): + defs = anno.getanno(node, anno.Static.DEFINITIONS) + self.assertEqual(len(defs), num) + for r in defs: + self.assertIsInstance(r, reaching_definitions.Definition) + + def assertHasDefinedIn(self, node, expected): + defined_in = anno.getanno(node, anno.Static.DEFINED_VARS_IN) + defined_in_str = set(str(v) for v in defined_in) + if not expected: + expected = () + if not isinstance(expected, tuple): + expected = (expected,) + self.assertSetEqual(defined_in_str, set(expected)) + + def test_conditional(self): + + def test_fn(a, b): + a = [] + if b: + a = [] + return a + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasDefs(fn_body[0].targets[0], 1) + self.assertHasDefs(fn_body[1].test, 1) + self.assertHasDefs(fn_body[1].body[0].targets[0], 1) + self.assertHasDefs(fn_body[2].value, 2) + + self.assertHasDefinedIn(fn_body[1], ('a', 'b')) + + def test_while(self): + + def test_fn(a): + max(a) + while True: + a = a + a = a + return a + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasDefs(fn_body[0].value.args[0], 1) + self.assertHasDefs(fn_body[1].body[0].targets[0], 1) + self.assertHasDefs(fn_body[1].body[0].value, 1) + self.assertHasDefs(fn_body[1].body[1].targets[0], 1) + self.assertHasDefs(fn_body[1].body[1].value, 1) + # The loop does have an invariant test, but the CFG doesn't know that. + self.assertHasDefs(fn_body[2].value, 2) + + def test_while_else(self): + + def test_fn(x, i): + y = 0 + while x: + x += i + if i: + break + else: + y = 1 + return x, y + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasDefs(fn_body[0].targets[0], 1) + self.assertHasDefs(fn_body[1].test, 2) + self.assertHasDefs(fn_body[1].body[0].target, 1) + self.assertHasDefs(fn_body[1].body[1].test, 1) + self.assertHasDefs(fn_body[1].orelse[0].targets[0], 1) + self.assertHasDefs(fn_body[2].value.elts[0], 2) + self.assertHasDefs(fn_body[2].value.elts[1], 2) + + def test_for_else(self): + + def test_fn(x, i): + y = 0 + for i in x: + x += i + if i: + break + else: + continue + else: + y = 1 + return x, y + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasDefs(fn_body[0].targets[0], 1) + self.assertHasDefs(fn_body[1].target, 1) + self.assertHasDefs(fn_body[1].body[0].target, 1) + self.assertHasDefs(fn_body[1].body[1].test, 1) + self.assertHasDefs(fn_body[1].orelse[0].targets[0], 1) + self.assertHasDefs(fn_body[2].value.elts[0], 2) + self.assertHasDefs(fn_body[2].value.elts[1], 2) + + def test_nested_functions(self): + + def test_fn(a, b): + a = [] + if b: + a = [] + + def foo(): + return a + + foo() + + return a + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + def_of_a_in_if = fn_body[1].body[0].targets[0] + + self.assertHasDefs(fn_body[0].targets[0], 1) + self.assertHasDefs(fn_body[1].test, 1) + self.assertHasDefs(def_of_a_in_if, 1) + self.assertHasDefs(fn_body[2].value, 2) + + inner_fn_body = fn_body[1].body[1].body + self.assertHasDefs(inner_fn_body[0].value, 1) + self.assertTrue( + anno.getanno(inner_fn_body[0].value, anno.Static.DEFINITIONS)[0] is + anno.getanno(def_of_a_in_if, anno.Static.DEFINITIONS)[0]) + + def test_nested_functions_isolation(self): + + def test_fn(a): + a = 0 + + def child(): + a = 1 + return a + + child() + return a + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasDefs(fn_body[3].value, 1) + self.assertHasDefs(fn_body[1].body[1].value, 1) + + parent_return = fn_body[3] + child_return = fn_body[1].body[1] + # The assignment `a = 1` makes `a` local to `child`. + self.assertFalse( + anno.getanno(parent_return.value, anno.Static.DEFINITIONS)[0] is + anno.getanno(child_return.value, anno.Static.DEFINITIONS)[0]) + + def test_debug(self): + + def foo(_): + pass + + def test_fn(a): + with foo(a): + return a + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + self.assertHasDefs(fn_body[0].items[0].context_expr.func, 0) + self.assertHasDefs(fn_body[0].items[0].context_expr.args[0], 1) + + +if __name__ == '__main__': + test.main() -- cgit v1.2.3 From 755503c4d7efc997efef2b211ec8b5c3b1020392 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Thu, 12 Jul 2018 15:01:11 -0700 Subject: [XLA:GPU] Support infeed of nested tuples Outfeed already supports this and with the shared queue management code this became easy. PiperOrigin-RevId: 204370770 --- .../compiler/xla/service/gpu/infeed_thunk.cc | 81 +++++++++++++--------- tensorflow/compiler/xla/shape_util.h | 2 +- tensorflow/compiler/xla/shape_util_test.cc | 9 +++ 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/infeed_thunk.cc b/tensorflow/compiler/xla/service/gpu/infeed_thunk.cc index 964efd3657..fee6d2af3b 100644 --- a/tensorflow/compiler/xla/service/gpu/infeed_thunk.cc +++ b/tensorflow/compiler/xla/service/gpu/infeed_thunk.cc @@ -30,45 +30,64 @@ InfeedThunk::InfeedThunk( Status InfeedThunk::ExecuteOnStream(const BufferAllocations& buffer_allocations, se::Stream* stream, HloExecutionProfiler* profiler) { - VLOG(2) << "Infeeding to GPU "; + VLOG(2) << "Infeeding to GPU: " << hlo_instruction()->ToString(); auto op_profiler = profiler->MakeScopedInstructionProfiler(hlo_instruction()); - // First copy the infeed data which is element 0 of the infeed instruction's - // two-tuple output (the other element is a token). - se::DeviceMemoryBase data_address = - buffer_allocations.GetDeviceAddress(infeed_slices_.element({0})); - InfeedManager* infeed_manager = GetOrCreateInfeedManager(); - const Shape& data_shape = - ShapeUtil::GetTupleElementShape(hlo_instruction()->shape(), 0); ShapeTree infeed_buffers = - infeed_manager->BlockingGetNextDestination(); - if (ShapeUtil::IsTuple(data_shape)) { - CHECK(!ShapeUtil::IsNestedTuple(data_shape)); - // Transfer the tuple elements first. + GetOrCreateInfeedManager()->BlockingGetNextDestination(); + + { + // The infeed buffer has an extra outer tuple with a token. Adjust the index + // accordingly. + ShapeIndex index = {0}; + std::function*)> copy_tuple_contents = + [&](std::vector* tuple_element_addresses) { + const Shape& shape = ShapeUtil::GetSubshape(infeed_buffers.shape(), + ShapeIndexView(index, 1)); + // For the leaf buffers of the tuple copy the elements directly. + if (ShapeUtil::IsArray(shape)) { + const BufferAllocation::Slice& tuple_element_buffer = + infeed_slices_.element(index); + se::DeviceMemoryBase tuple_element_address = + buffer_allocations.GetDeviceAddress(tuple_element_buffer); + + InfeedBuffer* buffer = + infeed_buffers.mutable_element(ShapeIndexView(index, 1)); + stream->ThenMemcpy(&tuple_element_address, + *(buffer->device_memory()), buffer->length()); + tuple_element_addresses->push_back(tuple_element_address.opaque()); + return; + } + + const int64 tuple_element_count = ShapeUtil::TupleElementCount(shape); + index.push_back(0); + std::vector inner_tuple_element_addresses; + for (int64 i = 0; i < tuple_element_count; ++i) { + index.back() = i; + copy_tuple_contents(&inner_tuple_element_addresses); + } + index.pop_back(); + + // Create a buffer of pointers for non-leaf buffers. + CHECK_EQ(tuple_element_count, inner_tuple_element_addresses.size()); + auto host_size = inner_tuple_element_addresses.size() * sizeof(void*); + se::DeviceMemoryBase tuple_address = + buffer_allocations.GetDeviceAddress( + infeed_slices_.element(index)); + stream->ThenMemcpy(&tuple_address, + inner_tuple_element_addresses.data(), host_size); + tuple_element_addresses->push_back(tuple_address.opaque()); + }; + std::vector tuple_element_addresses; - for (int i = 0; i < ShapeUtil::TupleElementCount(data_shape); ++i) { - const BufferAllocation::Slice& tuple_element_buffer = - infeed_slices_.element({0, i}); - se::DeviceMemoryBase tuple_element_address = - buffer_allocations.GetDeviceAddress(tuple_element_buffer); - - InfeedBuffer* buffer = infeed_buffers.mutable_element({i}); - stream->ThenMemcpy(&tuple_element_address, *(buffer->device_memory()), - buffer->length()); - tuple_element_addresses.push_back(tuple_element_address.opaque()); - } - // Transfer the tuple outer buffer. - auto host_size = tuple_element_addresses.size() * sizeof(void*); - stream->ThenMemcpy(&data_address, tuple_element_addresses.data(), - host_size); - } else { - InfeedBuffer* buffer = infeed_buffers.mutable_element({}); - stream->ThenMemcpy(&data_address, *(buffer->device_memory()), - buffer->length()); + copy_tuple_contents(&tuple_element_addresses); + CHECK_EQ(1, tuple_element_addresses.size()); } // Construct top-level tuple of infeed containing the data and the token. Use // a nullptr for the token, it should never be dereferenced. + se::DeviceMemoryBase data_address = + buffer_allocations.GetDeviceAddress(infeed_slices_.element({0})); void* infeed_addresses[] = {data_address.opaque(), nullptr}; se::DeviceMemoryBase top_level_address = buffer_allocations.GetDeviceAddress(infeed_slices_.element({})); diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h index d576be724e..17c1d7b10a 100644 --- a/tensorflow/compiler/xla/shape_util.h +++ b/tensorflow/compiler/xla/shape_util.h @@ -110,7 +110,7 @@ class ShapeIndex { class ShapeIndexView { public: ShapeIndexView(const ShapeIndex& shape_index, int64 offset = 0) - : indices_(shape_index.data() + offset, shape_index.size()) { + : indices_(shape_index.data() + offset, shape_index.size() - offset) { CHECK_LE(offset, shape_index.size()); } ShapeIndexView(std::initializer_list indices) : indices_(indices) {} diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc index 6cdb46d674..ed2d16c0e9 100644 --- a/tensorflow/compiler/xla/shape_util_test.cc +++ b/tensorflow/compiler/xla/shape_util_test.cc @@ -31,6 +31,15 @@ namespace { using ::testing::ElementsAre; +TEST(ShapeUtilTest, ShapeIndexViewTest) { + ShapeIndex index = {1, 2, 3, 4}; + ShapeIndexView index_view(index, 1); + EXPECT_EQ(3, index_view.size()); + EXPECT_EQ(ShapeIndexView({2, 3, 4}), index_view); + EXPECT_EQ(ShapeIndexView({3, 4}), index_view.ConsumeFront()); + EXPECT_EQ(ShapeIndexView({2, 3}), index_view.ConsumeBack()); +} + TEST(ShapeUtilTest, GetDimensionHelperCanNegativeIndex) { Shape matrix = ShapeUtil::MakeShape(F32, {2, 3}); EXPECT_EQ(3, ShapeUtil::GetDimension(matrix, -1)); -- cgit v1.2.3 From fde3f09e3080a28b9a06ee219474957ba149a20d Mon Sep 17 00:00:00 2001 From: Revan Sopher Date: Thu, 12 Jul 2018 15:16:18 -0700 Subject: Automated rollback of commit 35a29824aa196aa9348e1f6daf836d07d9e61156 PiperOrigin-RevId: 204373793 --- tensorflow/contrib/tpu/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index f59545f651..0044fde9d0 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -16,6 +16,7 @@ package( "//cloud/vmm/testing/tests/tpu:__subpackages__", "//learning/brain:__subpackages__", "//tensorflow:__subpackages__", + "//third_party/cloud_tpu:__subpackages__", ], ) -- cgit v1.2.3 From 86f632e29810fa93db559f882567b9569dabfad5 Mon Sep 17 00:00:00 2001 From: gracehoney <31743510+aaroey@users.noreply.github.com> Date: Thu, 12 Jul 2018 15:22:03 -0700 Subject: Implement the input/output edge validaters --- .../contrib/tensorrt/convert/convert_graph.cc | 46 +++--- .../contrib/tensorrt/convert/convert_nodes.cc | 165 ++++++++++++++------- .../contrib/tensorrt/convert/convert_nodes.h | 26 +++- tensorflow/contrib/tensorrt/segment/segment.cc | 8 +- tensorflow/contrib/tensorrt/segment/segment.h | 4 +- .../contrib/tensorrt/segment/segment_test.cc | 21 ++- 6 files changed, 182 insertions(+), 88 deletions(-) diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc index 359fac36f5..ba01eaabc2 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc @@ -107,10 +107,8 @@ bool IsTensorRTCandidate(const tensorflow::Node* node) { // TODO(ben,jie): ... }; // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h) - if (!candidate_ops.count(node->type_string()) && - !PluginFactoryTensorRT::GetInstance()->IsPlugin(node->type_string())) { - return false; - } + return (candidate_ops.count(node->type_string()) || + PluginFactoryTensorRT::GetInstance()->IsPlugin(node->type_string())); } tensorflow::Status BuildNodeMap( @@ -280,7 +278,8 @@ tensorflow::Status GetEngineInfo( subgraph_node_ids.push_back(node_id); for (const auto edge : node->in_edges()) { auto input_node = edge->src(); - if (segment_nodes.count(input_node->name()) == 0) { + if (segment_nodes.count(input_node->name()) == 0 && + !edge->IsControlEdge() && !input_node->IsSource()) { // Add constant input node into the segment. We don't care if it has // other output edges going into other engines or TF nodes. Since we add // it only to the subsegment node list, not the subsegment itself, it @@ -288,7 +287,7 @@ tensorflow::Status GetEngineInfo( // will prune it out. if (input_node->type_string() == "Const") { subgraph_node_ids.push_back(input_node->id()); - } else if (!edge->IsControlEdge() && !input_node->IsSource()) { + } else { string s(input_node->name()); StrAppend(&s, ":", edge->src_output()); VLOG(1) << "Input edge = " << s; @@ -351,9 +350,9 @@ tensorflow::Status CreateTRTNode(tensorflow::Graph* graph, nvinfer1::IGpuAllocator* alloc, int max_batch_size) { const auto& info = infos.at(pos); - std::vector out_shapes; - std::vector input_shapes; - std::vector shapes; + std::vector output_shape_protos; + std::vector input_shape_protos; + std::vector input_shapes; std::vector inputs; std::vector out_types; VLOG(1) << "Processing " << info.engine_name; @@ -366,11 +365,11 @@ tensorflow::Status CreateTRTNode(tensorflow::Graph* graph, tensorflow::TensorShapeProto out_shape; // shape of the output node inside segment conn.inside_shape.AsProto(&out_shape); - if (out_shapes.size() <= conn.port_number) { - out_shapes.resize(conn.port_number + 1); + if (output_shape_protos.size() <= conn.port_number) { + output_shape_protos.resize(conn.port_number + 1); out_types.resize(conn.port_number + 1); } - out_shapes.at(conn.port_number) = out_shape; + output_shape_protos.at(conn.port_number) = out_shape; out_types.at(conn.port_number) = conn.connection_type; continue; } @@ -378,12 +377,12 @@ tensorflow::Status CreateTRTNode(tensorflow::Graph* graph, // Set the shapes and data types of input edge. tensorflow::TensorShapeProto in_shape; conn.outside_shape.AsProto(&in_shape); - if (input_shapes.size() <= conn.port_number) { + if (input_shape_protos.size() <= conn.port_number) { + input_shape_protos.resize(conn.port_number + 1); input_shapes.resize(conn.port_number + 1); - shapes.resize(conn.port_number + 1); } - input_shapes.at(conn.port_number) = in_shape; - shapes.at(conn.port_number) = conn.outside_shape; + input_shape_protos.at(conn.port_number) = in_shape; + input_shapes.at(conn.port_number) = conn.outside_shape; string input_node = conn.outside_node_name; int input_port = conn.outside_port; @@ -411,6 +410,8 @@ tensorflow::Status CreateTRTNode(tensorflow::Graph* graph, VLOG(1) << "Engine Input " << input_node << ":" << input_port << " -> " << info.engine_name << ":" << inputs.size(); // Skip duplicate inputs. + // TODO(aaroey): use std::find instead. GetEngineInfo already remove + // duplicate connections, so here we should never find any duplicate? bool new_input = true; for (const auto& inp : inputs) { if (inp.node == input_node && inp.index == input_port) { @@ -438,8 +439,8 @@ tensorflow::Status CreateTRTNode(tensorflow::Graph* graph, TF_RETURN_IF_ERROR(ConvertGraphDefToEngine( info.segment_graph_def, info.precision_mode == INT8MODE ? FP32MODE : info.precision_mode, - max_batch_size, info.max_workspace_size_bytes, shapes, &trt_logger, - alloc, /*calibrator=*/nullptr, &engine, + max_batch_size, info.max_workspace_size_bytes, input_shapes, + &trt_logger, alloc, /*calibrator=*/nullptr, &engine, /*convert_successfully=*/nullptr)); TrtUniquePtrType engine_data(engine->serialize()); segment_string = @@ -487,8 +488,8 @@ tensorflow::Status CreateTRTNode(tensorflow::Graph* graph, } tensorflow::NodeDef trt_node; tensorflow::Status status = - node_builder.Attr("input_shapes", input_shapes) - .Attr("output_shapes", out_shapes) + node_builder.Attr("input_shapes", input_shape_protos) + .Attr("output_shapes", output_shape_protos) .Attr("static_engine", info.engine_type == EngineInfo::EngineType::TRTStatic) .Attr("segment_funcdef_name", @@ -705,6 +706,7 @@ std::pair GetDeviceAndAllocator( } // Entry function from optimization pass. +// TODO(aaeory): parameter should use pointer type. tensorflow::Status ConvertAfterShapes(ConversionParams& params) { // Convert graphdef to graph. tensorflow::FunctionLibraryDefinition flib(tensorflow::OpRegistry::Global(), @@ -722,8 +724,8 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) { segment_options.minimum_segment_size = params.minimum_segment_size; tensorflow::tensorrt::segment::SegmentNodesVector initial_segments; TF_RETURN_IF_ERROR(tensorrt::segment::SegmentGraph( - &graph, IsTensorRTCandidate, IsTensorRTInputCandidate, - IsTensorRTOutputCandidate, segment_options, &initial_segments)); + &graph, IsTensorRTCandidate, InputEdgeValidator(*params.graph_properties), + OutputEdgeValidator(), segment_options, &initial_segments)); if (initial_segments.size() > 1) { VLOG(0) << "MULTIPLE tensorrt candidate conversion: " << initial_segments.size(); diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc index 8f6656e4ad..c49e26ea4e 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/contrib/tensorrt/convert/convert_nodes.h" #include +#include #include #include #include @@ -57,7 +58,6 @@ namespace tensorflow { namespace tensorrt { namespace convert { using ::tensorflow::str_util::Split; - using ::tensorflow::strings::StrAppend; using ::tensorflow::strings::StrCat; @@ -77,11 +77,63 @@ inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype, break; default: return tensorflow::errors::InvalidArgument( - "Unsupported data type " + tensorflow::DataTypeString(tf_dtype)); + "Unsupported data type ", tensorflow::DataTypeString(tf_dtype)); } return tensorflow::Status::OK(); } +void GetInputProperties(const grappler::GraphProperties& graph_properties, + const Node* outside_node, const int out_port, + PartialTensorShape* shape, + tensorflow::DataType* dtype) { + if (graph_properties.HasOutputProperties(outside_node->name())) { + auto output_params = + graph_properties.GetOutputProperties(outside_node->name()); + auto out_shape = output_params.at(out_port); + *dtype = out_shape.dtype(); + *shape = out_shape.shape(); + } else { + VLOG(0) << "Unknown output shape" << outside_node->name(); + *dtype = outside_node->output_type(out_port); + } +} + +void GetOutputProperties(const grappler::GraphProperties& graph_properties, + const Node* outside_node, const int in_port, + PartialTensorShape* shape, + tensorflow::DataType* dtype) { + if (graph_properties.HasInputProperties(outside_node->name())) { + auto input_params = + graph_properties.GetInputProperties(outside_node->name()); + auto in_shape = input_params.at(in_port); + *dtype = in_shape.dtype(); + *shape = in_shape.shape(); + } else { + *dtype = outside_node->input_type(in_port); + } +} + +tensorflow::Status ValidateInputProperties(const PartialTensorShape& shape, + const tensorflow::DataType dtype, + nvinfer1::DataType* trt_dtype) { + TF_RETURN_IF_ERROR(ConvertDType(dtype, trt_dtype)); + if (shape.dims() < 0) { + return tensorflow::errors::InvalidArgument( + "Input tensor rank is unknown."); + } + if (shape.dims() > 8) { + return tensorflow::errors::OutOfRange( + "Input tensor rank is greater than 8."); + } + for (int d = 1; d < shape.dims(); ++d) { + if (shape.dim_size(d) < 0) { + return tensorflow::errors::InvalidArgument( + "Input tensor has a unknow non-batch dimemension at dim ", d); + } + } + return Status::OK(); +} + inline nvinfer1::Dims GetTensorShape(const tensorflow::Tensor& tensor) { nvinfer1::Dims dims; dims.nbDims = tensor.dims(); @@ -2177,25 +2229,22 @@ tensorflow::Status ConvertGraphDefToEngine( (node_def.op() == "Placeholder")) { nvinfer1::DimsCHW input_dim_pseudo_chw; for (int i = 0; i < 8; i++) input_dim_pseudo_chw.d[i] = 0; - nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT); - auto type_status = - ConvertDType(node_def.attr().at("dtype").type(), &dtype); - if (type_status != tensorflow::Status::OK()) { - LOG(WARNING) << "Type conversion failed for " << node_name; - return type_status; - } int32 slot_number = -1; - if (!tensorflow::strings::safe_strto32(node_name.c_str() + 8, - &slot_number)) { - LOG(ERROR) << "Failed to parse slot number from " << node_name - << " +8= " << node_name.c_str() + 8; + if (!tensorflow::strings::safe_strto32( + node_name.c_str() + strlen(kInputPHName), &slot_number)) { + return tensorflow::errors::InvalidArgument( + "Failed to parse slot number from ", node_name); } + nvinfer1::DataType dtype; auto shape = input_shapes.at(slot_number); - if (shape.dims() > 8) { - LOG(ERROR) << "Tensor rank is greater than 8 for " << node_name - << " at input slot " << slot_number; - return tensorflow::errors::OutOfRange( - "Input tensor rank is greater than 8"); + auto status = ValidateInputProperties( + shape, node_def.attr().at("dtype").type(), &dtype); + if (!status.ok()) { + const string error_message = StrCat( + "Validation failed for ", node_name, " and input slot ", + slot_number, ": ", status.error_message()); + LOG(WARNING) << error_message; + return Status(status.code(), error_message); } if (VLOG_IS_ON(1)) { string dim_str("dims="); @@ -2226,10 +2275,10 @@ tensorflow::Status ConvertGraphDefToEngine( } else if (tensorflow::str_util::StartsWith(node_name, kOutputPHName) && (node_def.op() == "Identity")) { int32 slot_number = -1; - if (!tensorflow::strings::safe_strto32(node_name.c_str() + 9, - &slot_number)) { - LOG(ERROR) << "Failed to parse slot number from " << node_name - << " +9=" << node_name.c_str() + 9; + if (!tensorflow::strings::safe_strto32( + node_name.c_str() + strlen(kOutputPHName), &slot_number)) { + return tensorflow::errors::InvalidArgument( + "Failed to parse slot number from ", node_name); } if (output_tensors.size() <= slot_number) { output_tensors.resize(slot_number + 1); @@ -2288,38 +2337,20 @@ tensorflow::Status ConvertSegmentToGraphDef( "Cannot find node with id ", connection.outside_id, " in the graph."); } // Updates the shape and data types of input/output connections. - tensorflow::DataType input_type = tensorflow::DT_FLOAT; + tensorflow::DataType dtype; tensorflow::PartialTensorShape partial_shape; if (connection.is_input_edge) { - if (graph_properties.HasOutputProperties(connection.outside_node_name)) { - auto output_params = - graph_properties.GetOutputProperties(connection.outside_node_name); - auto out_shape = output_params.at(connection.outside_port); - input_type = out_shape.dtype(); - std::vector dims; - partial_shape = out_shape.shape(); - connection.outside_shape = partial_shape; - } else { - VLOG(0) << "Unknown output shape" << outside_node->name(); - input_type = graph->FindNodeId(connection.outside_id) - ->output_type(connection.outside_port); - } - connection.connection_type = input_type; - - } else { // output edge - if (graph_properties.HasInputProperties(connection.outside_node_name)) { - auto input_params = - graph_properties.GetInputProperties(connection.outside_node_name); - auto in_shape = input_params.at(connection.outside_port); - input_type = in_shape.dtype(); - partial_shape = in_shape.shape(); - connection.inside_shape = partial_shape; - } else { - input_type = graph->FindNodeId(connection.inside_id) - ->output_type(connection.outside_port); - } - connection.connection_type = input_type; + GetInputProperties(graph_properties, + graph->FindNodeId(connection.outside_id), + connection.outside_port, &partial_shape, &dtype); + + } else { + GetOutputProperties(graph_properties, + graph->FindNodeId(connection.outside_id), + connection.outside_port, &partial_shape, &dtype); } + connection.outside_shape = partial_shape; + connection.connection_type = dtype; // Add dummy input/output nodes to the segment graphdef. if (connection.is_input_edge) { @@ -2335,7 +2366,7 @@ tensorflow::Status ConvertSegmentToGraphDef( auto seg_node = segment_def->add_node(); tensorflow::NodeDefBuilder builder(node_name, "Placeholder"); auto status = builder.Attr("shape", partial_shape) - .Attr("dtype", input_type) + .Attr("dtype", dtype) .Finalize(seg_node); VLOG(1) << "Constructing input " << node_name << " for the edge " << connection.outside_node_name << ":" << connection.outside_port @@ -2353,7 +2384,7 @@ tensorflow::Status ConvertSegmentToGraphDef( marker_nodes.insert(node_name); auto seg_node = segment_def->add_node(); tensorflow::NodeDefBuilder builder(node_name, "Identity"); - auto status = builder.Input(connection.inside_node_name, 0, input_type) + auto status = builder.Input(connection.inside_node_name, 0, dtype) .Finalize(seg_node); VLOG(1) << "Constructing output " << node_name << " for the edge " << connection.inside_node_name << ":" << connection.inside_port @@ -2391,11 +2422,35 @@ tensorflow::Status ConvertSegmentToGraphDef( return tensorflow::Status::OK(); } -bool IsTensorRTInputCandidate(const tensorflow::Node* node) { +bool InputEdgeValidator::operator()(const tensorflow::Edge* in_edge) const { + if (in_edge->IsControlEdge()) return true; + PartialTensorShape shape; + tensorflow::DataType dtype; + GetInputProperties(graph_properties_, in_edge->src(), in_edge->src_output(), + &shape, &dtype); + nvinfer1::DataType trt_dtype; + Status status = ValidateInputProperties(shape, dtype, &trt_dtype); + if (!status.ok()) { + VLOG(2) << "--> Need to remove input node " << in_edge->dst()->name() + << ": " << status; + return false; + } + if (shape.dims() < 3 && in_edge->src()->type_string() != "Const") { + VLOG(2) << "--> Need to remove input node " << in_edge->dst()->name() + << " which has an input at port " << in_edge->dst_input() + << " with #dim<3 and is not a const: " << shape; + return false; + } return true; } -bool IsTensorRTOutputCandidate(const tensorflow::Node* node) { +bool OutputEdgeValidator::operator()(const tensorflow::Edge* out_edge) const { + if (out_edge->IsControlEdge()) return true; + if (out_edge->src()->type_string() == "Const") { + VLOG(2) << "--> Need to remove output node " << out_edge->src()->name() + << " which is a Const."; + return false; + } return true; } diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h index 872ba6a080..64337eee84 100644 --- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h +++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h @@ -104,6 +104,8 @@ struct EngineInfo { // topological order. // - segment_def: the output GraphDef, whose non-input/output nodedefs will be // sorted in topological order. +// +// TODO(aaroey): add tests to validate these properties. tensorflow::Status ConvertSegmentToGraphDef( const tensorflow::Graph* graph, const tensorflow::grappler::GraphProperties& graph_properties, @@ -128,9 +130,29 @@ tensorflow::Status ConvertGraphDefToEngine( TrtUniquePtrType* engine, bool* convert_successfully); -bool IsTensorRTInputCandidate(const tensorflow::Node* node); +// Helper class for the segmenter to determine whether an input edge to the TRT +// segment is valid. +class InputEdgeValidator { + public: + InputEdgeValidator(const grappler::GraphProperties& graph_properties) + : graph_properties_(graph_properties) {} + + // Return true if the specified edge is eligible to be an input edge of the + // TRT segment. + bool operator()(const tensorflow::Edge* in_edge) const; -bool IsTensorRTOutputCandidate(const tensorflow::Node* node); + private: + const grappler::GraphProperties& graph_properties_; +}; + +// Helper class for the segmenter to determine whether an output edge from the +// TRT segment is valid. +class OutputEdgeValidator { + public: + // Return true if the specified edge is eligible to be an output edge of the + // TRT segment. + bool operator()(const tensorflow::Edge* out_edge) const; +}; } // namespace convert } // namespace tensorrt diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc index 5c0898b29a..92807bed14 100644 --- a/tensorflow/contrib/tensorrt/segment/segment.cc +++ b/tensorflow/contrib/tensorrt/segment/segment.cc @@ -364,8 +364,8 @@ void ContractEdge(SimpleEdge* edge, SimpleGraph* graph, tensorflow::Status SegmentGraph( const tensorflow::Graph* tf_graph, const std::function& candidate_fn, - const std::function& input_candidate_fn, - const std::function& output_candidate_fn, + const std::function& input_candidate_fn, + const std::function& output_candidate_fn, const SegmentOptions& options, SegmentNodesVector* segments) { // Steps: // 1. run the segmentation algorithm to find all the segments, which uses @@ -526,7 +526,7 @@ tensorflow::Status SegmentGraph( for (const tensorflow::Edge* edge : node->in_edges()) { if (!edge->IsControlEdge() && !edge->src()->IsSource() && !segment_nodes.count(edge->src())) { // 'node' is an input node. - if (!input_candidate_fn(node)) { + if (!input_candidate_fn(edge)) { in_nodes_que.push_back(node); added = true; break; @@ -537,7 +537,7 @@ tensorflow::Status SegmentGraph( for (const tensorflow::Edge* edge : node->out_edges()) { if (!edge->dst()->IsSink() && !edge->IsControlEdge() && !segment_nodes.count(edge->dst())) { // 'node' is an output node. - if (!output_candidate_fn(node)) { + if (!output_candidate_fn(edge)) { out_nodes_que.push_back(node); break; } diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h index ab75135054..8c44eb782a 100644 --- a/tensorflow/contrib/tensorrt/segment/segment.h +++ b/tensorflow/contrib/tensorrt/segment/segment.h @@ -52,8 +52,8 @@ struct SegmentOptions { tensorflow::Status SegmentGraph( const tensorflow::Graph* tf_graph, const std::function& candidate_fn, - const std::function& input_candidate_fn, - const std::function& output_candidate_fn, + const std::function& input_candidate_fn, + const std::function& output_candidate_fn, const SegmentOptions& options, SegmentNodesVector* segments); } // namespace segment diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/contrib/tensorrt/segment/segment_test.cc index a43cf4f416..432e7b1c04 100644 --- a/tensorflow/contrib/tensorrt/segment/segment_test.cc +++ b/tensorflow/contrib/tensorrt/segment/segment_test.cc @@ -41,15 +41,30 @@ class SegmentTest : public ::testing::Test { }; } + std::function MakeInputEdgeCandidateFn( + const std::set& node_names) { + return [node_names](const tensorflow::Edge* in_edge) -> bool { + return node_names.find(in_edge->dst()->name()) != node_names.end(); + }; + } + + std::function MakeOutputEdgeCandidateFn( + const std::set& node_names) { + return [node_names](const tensorflow::Edge* out_edge) -> bool { + return node_names.find(out_edge->src()->name()) != node_names.end(); + }; + } + void RunTest(const tensorflow::Graph* graph, const std::set& candidates, const std::set& input_candidates, const std::set& output_candidates, const std::vector>& expected_segments) { SegmentNodesVector segments; - TF_EXPECT_OK(SegmentGraph( - graph, MakeCandidateFn(candidates), MakeCandidateFn(input_candidates), - MakeCandidateFn(output_candidates), default_options_, &segments)); + TF_EXPECT_OK(SegmentGraph(graph, MakeCandidateFn(candidates), + MakeInputEdgeCandidateFn(input_candidates), + MakeOutputEdgeCandidateFn(output_candidates), + default_options_, &segments)); ValidateSegment(segments, expected_segments); } -- cgit v1.2.3 From 5c0af75bb5b80b2a1f9c7a2e83e012899dd66016 Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Thu, 12 Jul 2018 22:22:12 +0000 Subject: [XLA][AMDGPU] use AddrSpaceCast to access GlobalVariable on AMDGPU In XLA kernels pointers are in default address space (0). On LLVM AMDGPU backend, all GlobalVariable instances must be in global address space (1). Therefore in an AddrSpaceCast is required for AMDGPU backend. To cope with existing NVPTX backend, use ConstantExpr::getPointerBitCastOrAddrSpaceCast in this commit. --- tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc index d420863b85..6f2a7e1850 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc @@ -145,7 +145,7 @@ llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, llvm::Value* typed_ir_value; if (llvm::isa(ir_value)) { - typed_ir_value = llvm::ConstantExpr::getBitCast( + typed_ir_value = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast( llvm::cast(ir_value), dest_type); } else { typed_ir_value = -- cgit v1.2.3 From bafe937359881ae9d9b61160f92a97d4dee1e4d0 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Thu, 12 Jul 2018 16:38:18 -0700 Subject: Remove redundant context.graph_mode() calls. PiperOrigin-RevId: 204387465 --- tensorflow/python/eager/function.py | 99 ++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 50 deletions(-) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 29a3848bd8..a34a6fc053 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -470,7 +470,7 @@ class GraphModeFunction(object): def _construct_backprop_function(self): """Constructs the backprop function object for this function.""" - with self._graph.as_default(), context.graph_mode(): + with self._graph.as_default(): c_known_ops = set() c_captured_tensors = set() @@ -657,55 +657,54 @@ def _deterministic_dict_values(kwds): def _trace_and_define_function(name, func, compiled, args, kwds): """Defines and returns graph-mode version of func.""" graph_key = ops.get_default_graph()._graph_key # pylint: disable=protected-access - with context.graph_mode(): - captures = {} - tmp_graph = CapturingGraph(captures) - # Inherit the graph key, since this is used for matching variables in - # optimizers. - tmp_graph._graph_key = graph_key # pylint: disable=protected-access - # Copy the graph collections to ensure summaries and other things work. This - # lets the function access (but not mutate) collections of the containing - # graph, such as the global step and the summary writer collections. - curr_graph = ops.get_default_graph() - for collection in curr_graph.collections: - tmp_graph.get_collection_ref(collection)[:] = curr_graph.get_collection( - collection) - with tmp_graph.as_default(), AutomaticControlDependencies() as a: - func_args = _get_defun_inputs(args) - func_kwds = _get_defun_inputs(kwds) - - def convert(x): - if x is None: - return None - x = ops.convert_to_tensor_or_indexed_slices(x) - x = a.mark_as_return(x) - return x - - this_tape = tape.push_new_tape() - try: - func_outputs = func(*func_args, **func_kwds) - func_outputs = nest.map_structure(convert, func_outputs) - finally: - tape.pop_tape(this_tape) - variables = this_tape.watched_variables() - - # Returning a closed-over tensor as an output does not trigger a - # call to convert_to_tensor, so we manually capture all such tensors. - outputs_list = _flatten(func_outputs) - func_def_outputs = [ - tmp_graph.capture(x) for x in outputs_list - if x is not None - ] - - ids = list(sorted(captures.keys())) - if ids: - extra_inputs, extra_placeholders = zip(* [captures[x] for x in ids]) - else: - extra_inputs = [] - extra_placeholders = [] - output_shapes = tuple( - x.shape if isinstance(x, ops.Tensor) else None - for x in func_def_outputs) + captures = {} + tmp_graph = CapturingGraph(captures) + # Inherit the graph key, since this is used for matching variables in + # optimizers. + tmp_graph._graph_key = graph_key # pylint: disable=protected-access + # Copy the graph collections to ensure summaries and other things work. This + # lets the function access (but not mutate) collections of the containing + # graph, such as the global step and the summary writer collections. + curr_graph = ops.get_default_graph() + for collection in curr_graph.collections: + tmp_graph.get_collection_ref(collection)[:] = curr_graph.get_collection( + collection) + with tmp_graph.as_default(), AutomaticControlDependencies() as a: + func_args = _get_defun_inputs(args) + func_kwds = _get_defun_inputs(kwds) + + def convert(x): + if x is None: + return None + x = ops.convert_to_tensor_or_indexed_slices(x) + x = a.mark_as_return(x) + return x + + this_tape = tape.push_new_tape() + try: + func_outputs = func(*func_args, **func_kwds) + func_outputs = nest.map_structure(convert, func_outputs) + finally: + tape.pop_tape(this_tape) + variables = this_tape.watched_variables() + + # Returning a closed-over tensor as an output does not trigger a + # call to convert_to_tensor, so we manually capture all such tensors. + outputs_list = _flatten(func_outputs) + func_def_outputs = [ + tmp_graph.capture(x) for x in outputs_list + if x is not None + ] + + ids = list(sorted(captures.keys())) + if ids: + extra_inputs, extra_placeholders = zip(* [captures[x] for x in ids]) + else: + extra_inputs = [] + extra_placeholders = [] + output_shapes = tuple( + x.shape if isinstance(x, ops.Tensor) else None + for x in func_def_outputs) func_kwds_values = _deterministic_dict_values(func_kwds) flat_inputs = [ -- cgit v1.2.3 From aace1be4dae9ec2a1d49f26783ae9cedcd9edc27 Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Thu, 12 Jul 2018 16:57:54 -0700 Subject: Disable eager nested defuns on TPU test Also delete test for variable deletion as it is super slow on some backends. PiperOrigin-RevId: 204390369 --- tensorflow/compiler/tests/eager_test.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tensorflow/compiler/tests/eager_test.py b/tensorflow/compiler/tests/eager_test.py index a8919d1afd..6ead15da13 100644 --- a/tensorflow/compiler/tests/eager_test.py +++ b/tensorflow/compiler/tests/eager_test.py @@ -177,17 +177,6 @@ class EagerTest(xla_test.XLATestCase): for _ in range(100): values.append(var.value()) - def testVariablesAreDeleted(self): - # This test makes sure that we release device (especially TPU) memory - # when resource variable is deleted. - with self.test_scope(): - # Create and destroy a 128MB variable 100 times. - # If we don't release device memory when python variable is deleted, - # this will eat over 13GB and OOM. - for _ in range(100): - # Create 128MiB variables - resource_variable_ops.ResourceVariable(array_ops.ones([32, 1024, 1024])) - # The shape, shape_n, size, and rank are tested here because their # execution kernels (as opposed to compilation only tf2xla kernels) # are distincts from tf2xla kernels. @@ -430,6 +419,7 @@ class EagerFunctionTest(xla_test.XLATestCase): self.assertAllEqual((2, 3, 4), dz.shape.as_list()) def testNestedDefun(self): + self.skipTest('Nested defuns do not work on TPU at the moment') with self.test_scope(): @function.defun -- cgit v1.2.3 From 9610dfb8b7a6f0d3c57574baf701868987fef3b1 Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Thu, 12 Jul 2018 23:41:08 +0000 Subject: [XLA:GPU] let GpuTransferManager take different StreamExecutor and LLVMCompiler GpuTransferManager is tied to CUDA StreamExecutor and LLVM NVPTX backend thus far. In this commit refactor its constructor so it's possible to take different StreamExecutor instances and LLVM backends. --- .../compiler/xla/service/gpu/gpu_transfer_manager.cc | 17 +++++++++-------- .../compiler/xla/service/gpu/gpu_transfer_manager.h | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc index 1446401b19..e291d74dd3 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc @@ -41,11 +41,9 @@ namespace gpu { // TODO(b/30467474) Once GPU infeed implementation settles, consider // folding back the cpu and gpu infeed implementations into a generic // one if possible. -GpuTransferManager::GpuTransferManager() - : GenericTransferManager( - se::cuda::kCudaPlatformId, - /*pointer_size=*/llvm::DataLayout(gpu::GpuCompiler::kDataLayout) - .getPointerSize(0 /* default address space */)) {} +GpuTransferManager::GpuTransferManager(se::Platform::Id id, + unsigned pointer_size) + : GenericTransferManager(id, pointer_size) {} Status GpuTransferManager::TransferLiteralToInfeed( se::StreamExecutor* executor, const LiteralSlice& literal) { @@ -179,13 +177,16 @@ Status GpuTransferManager::TransferLiteralFromOutfeed( } // namespace gpu } // namespace xla -static std::unique_ptr CreateGpuTransferManager() { - return xla::MakeUnique(); +static std::unique_ptr CreateNVPTXTransferManager() { + return xla::MakeUnique( + /*id=*/ stream_executor::cuda::kCudaPlatformId, + /*pointer_size=*/ llvm::DataLayout(xla::gpu::GpuCompiler::kDataLayout) + .getPointerSize(0 /* default address space */)); } static bool InitModule() { xla::TransferManager::RegisterTransferManager( - stream_executor::cuda::kCudaPlatformId, &CreateGpuTransferManager); + stream_executor::cuda::kCudaPlatformId, &CreateNVPTXTransferManager); return true; } static bool module_initialized = InitModule(); diff --git a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h index 8122c9d8c3..dceeb9e2eb 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h +++ b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.h @@ -35,7 +35,7 @@ namespace gpu { // handles GPU-specific infeed. class GpuTransferManager : public GenericTransferManager { public: - GpuTransferManager(); + GpuTransferManager(se::Platform::Id id, unsigned pointer_size); ~GpuTransferManager() override {} Status TransferLiteralToInfeed(se::StreamExecutor* executor, -- cgit v1.2.3 From b13b2c15d2e864270b28eb5c5d3ec9f53d3932a7 Mon Sep 17 00:00:00 2001 From: Yu-Cheng Ling Date: Thu, 12 Jul 2018 17:38:45 -0700 Subject: Fix the broekn BUILD rule under lite/delegates/eager. PiperOrigin-RevId: 204396267 --- tensorflow/contrib/lite/delegates/eager/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/delegates/eager/BUILD b/tensorflow/contrib/lite/delegates/eager/BUILD index 11cc8185f6..066b106215 100644 --- a/tensorflow/contrib/lite/delegates/eager/BUILD +++ b/tensorflow/contrib/lite/delegates/eager/BUILD @@ -30,6 +30,6 @@ cc_test( ":util", "//tensorflow/contrib/lite/testing:util", "//tensorflow/core:lib", - "//testing/base/public:gunit", + "@com_google_googletest//:gtest", ], ) -- cgit v1.2.3 From fb9992423c20ed716124f63b9fe7858a0198e897 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Thu, 12 Jul 2018 17:44:19 -0700 Subject: Cleaning up test proto for `tensorflow/contrib/proto` and refactoring test code. PiperOrigin-RevId: 204397051 --- tensorflow/contrib/proto/python/kernel_tests/BUILD | 53 ++- .../python/kernel_tests/decode_proto_fail_test.py | 68 ---- .../python/kernel_tests/decode_proto_op_test.py | 261 +------------ .../kernel_tests/decode_proto_op_test_base.py | 310 ++++++++++++++++ .../python/kernel_tests/encode_proto_op_test.py | 152 +------- .../kernel_tests/encode_proto_op_test_base.py | 177 +++++++++ .../python/kernel_tests/proto_op_test_base.py | 407 +++++++++++++++++++++ .../contrib/proto/python/kernel_tests/test_base.py | 407 --------------------- .../proto/python/kernel_tests/test_example.proto | 159 ++++---- 9 files changed, 1006 insertions(+), 988 deletions(-) delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py create mode 100644 tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py create mode 100644 tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test_base.py create mode 100644 tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py delete mode 100644 tensorflow/contrib/proto/python/kernel_tests/test_base.py diff --git a/tensorflow/contrib/proto/python/kernel_tests/BUILD b/tensorflow/contrib/proto/python/kernel_tests/BUILD index 3f53ef1707..3c6fde23d2 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/BUILD +++ b/tensorflow/contrib/proto/python/kernel_tests/BUILD @@ -9,34 +9,13 @@ load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object") load("//tensorflow/core:platform/default/build_config_root.bzl", "if_static") load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library") -tf_py_test( - name = "decode_proto_fail_test", - size = "small", - srcs = ["decode_proto_fail_test.py"], - additional_deps = [ - ":py_test_deps", - "//third_party/py/numpy", - "//tensorflow/contrib/proto:proto", - "//tensorflow/contrib/proto/python/ops:decode_proto_op_py", - ], - data = if_static( - [], - otherwise = [":libtestexample.so"], - ), - tags = [ - "no_pip", # TODO(b/78026780) - "no_windows", # TODO(b/78028010) - ], -) - tf_py_test( name = "decode_proto_op_test", size = "small", srcs = ["decode_proto_op_test.py"], additional_deps = [ + ":decode_proto_op_test_base", ":py_test_deps", - "@absl_py//absl/testing:parameterized", - "//third_party/py/numpy", "//tensorflow/contrib/proto:proto", "//tensorflow/contrib/proto/python/ops:decode_proto_op_py", ], @@ -55,9 +34,8 @@ tf_py_test( size = "small", srcs = ["encode_proto_op_test.py"], additional_deps = [ + ":encode_proto_op_test_base", ":py_test_deps", - "@absl_py//absl/testing:parameterized", - "//third_party/py/numpy", "//tensorflow/contrib/proto:proto", "//tensorflow/contrib/proto/python/ops:decode_proto_op_py", "//tensorflow/contrib/proto/python/ops:encode_proto_op_py", @@ -73,8 +51,9 @@ tf_py_test( ) py_library( - name = "test_base", - srcs = ["test_base.py"], + name = "proto_op_test_base", + testonly = 1, + srcs = ["proto_op_test_base.py"], deps = [ ":test_example_proto_py", "//tensorflow/python:client_testlib", @@ -82,13 +61,31 @@ py_library( ) py_library( - name = "py_test_deps", + name = "decode_proto_op_test_base", + testonly = 1, + srcs = ["decode_proto_op_test_base.py"], + deps = [ + ":proto_op_test_base", + ":test_example_proto_py", + "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", + ], +) + +py_library( + name = "encode_proto_op_test_base", + testonly = 1, + srcs = ["encode_proto_op_test_base.py"], deps = [ - ":test_base", + ":proto_op_test_base", ":test_example_proto_py", + "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", ], ) +py_library(name = "py_test_deps") + tf_proto_library( name = "test_example_proto", srcs = ["test_example.proto"], diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py deleted file mode 100644 index 3b982864bc..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_fail_test.py +++ /dev/null @@ -1,68 +0,0 @@ -# ============================================================================= -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -# Python3 preparedness imports. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - -from tensorflow.contrib.proto.python.kernel_tests import test_base -from tensorflow.contrib.proto.python.ops import decode_proto_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import errors -from tensorflow.python.platform import test - - -class DecodeProtoFailTest(test_base.ProtoOpTestBase): - """Test failure cases for DecodeToProto.""" - - def _TestCorruptProtobuf(self, sanitize): - """Test failure cases for DecodeToProto.""" - - # The goal here is to check the error reporting. - # Testing against a variety of corrupt protobufs is - # done by fuzzing. - corrupt_proto = 'This is not a binary protobuf' - - # Numpy silently truncates the strings if you don't specify dtype=object. - batch = np.array(corrupt_proto, dtype=object) - msg_type = 'tensorflow.contrib.proto.TestCase' - field_names = ['sizes'] - field_types = [dtypes.int32] - - with self.test_session() as sess: - ctensor, vtensor = decode_proto_op.decode_proto( - batch, - message_type=msg_type, - field_names=field_names, - output_types=field_types, - sanitize=sanitize) - with self.assertRaisesRegexp(errors.DataLossError, - 'Unable to parse binary protobuf' - '|Failed to consume entire buffer'): - _ = sess.run([ctensor] + vtensor) - - def testCorrupt(self): - self._TestCorruptProtobuf(sanitize=False) - - def testSanitizerCorrupt(self): - self._TestCorruptProtobuf(sanitize=True) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py index 2a07794499..934035ec4c 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py +++ b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test.py @@ -13,273 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================= -"""Table-driven test for decode_proto op. +"""Tests for decode_proto op.""" -This test is run once with each of the *.TestCase.pbtxt files -in the test directory. -""" # Python3 preparedness imports. from __future__ import absolute_import from __future__ import division from __future__ import print_function -from absl.testing import parameterized -import numpy as np - - -from google.protobuf import text_format - -from tensorflow.contrib.proto.python.kernel_tests import test_base -from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 +from tensorflow.contrib.proto.python.kernel_tests import decode_proto_op_test_base as test_base from tensorflow.contrib.proto.python.ops import decode_proto_op -from tensorflow.python.framework import dtypes from tensorflow.python.platform import test -class DecodeProtoOpTest(test_base.ProtoOpTestBase, parameterized.TestCase): - - def _compareValues(self, fd, vs, evs): - """Compare lists/arrays of field values.""" - - if len(vs) != len(evs): - self.fail('Field %s decoded %d outputs, expected %d' % - (fd.name, len(vs), len(evs))) - for i, ev in enumerate(evs): - # Special case fuzzy match for float32. TensorFlow seems to mess with - # MAX_FLT slightly and the test doesn't work otherwise. - # TODO(nix): ask on TF list about why MAX_FLT doesn't pass through. - if fd.cpp_type == fd.CPPTYPE_FLOAT: - # Numpy isclose() is better than assertIsClose() which uses an absolute - # value comparison. - self.assertTrue( - np.isclose(vs[i], ev), 'expected %r, actual %r' % (ev, vs[i])) - elif fd.cpp_type == fd.CPPTYPE_STRING: - # In Python3 string tensor values will be represented as bytes, so we - # reencode the proto values to match that. - self.assertEqual(vs[i], ev.encode('ascii')) - else: - # Doubles and other types pass through unscathed. - self.assertEqual(vs[i], ev) - - def _compareRepeatedPrimitiveValue(self, batch_shape, sizes, fields, - field_dict): - """Compare protos of type RepeatedPrimitiveValue. - - Args: - batch_shape: the shape of the input tensor of serialized messages. - sizes: int matrix of repeat counts returned by decode_proto - fields: list of test_example_pb2.FieldSpec (types and expected values) - field_dict: map from field names to decoded numpy tensors of values - """ - - # Check that expected values match. - for field in fields: - values = field_dict[field.name] - self.assertEqual(dtypes.as_dtype(values.dtype), field.dtype) - - fd = field.expected.DESCRIPTOR.fields_by_name[field.name] - - # Values has the same shape as the input plus an extra - # dimension for repeats. - self.assertEqual(list(values.shape)[:-1], batch_shape) - - # Nested messages are represented as TF strings, requiring - # some special handling. - if field.name == 'message_value': - vs = [] - for buf in values.flat: - msg = test_example_pb2.PrimitiveValue() - msg.ParseFromString(buf) - vs.append(msg) - evs = getattr(field.expected, field.name) - if len(vs) != len(evs): - self.fail('Field %s decoded %d outputs, expected %d' % - (fd.name, len(vs), len(evs))) - for v, ev in zip(vs, evs): - self.assertEqual(v, ev) - continue - - # This can be a little confusing. For testing we are using - # RepeatedPrimitiveValue in two ways: it's the proto that we - # decode for testing, and it's used in the expected value as a - # union type. The two cases are slightly different: this is the - # second case. - # We may be fetching the uint64_value from the test proto, but - # in the expected proto we store it in the int64_value field - # because TensorFlow doesn't support unsigned int64. - tf_type_to_primitive_value_field = { - dtypes.float32: - 'float_value', - dtypes.float64: - 'double_value', - dtypes.int32: - 'int32_value', - dtypes.uint8: - 'uint8_value', - dtypes.int8: - 'int8_value', - dtypes.string: - 'string_value', - dtypes.int64: - 'int64_value', - dtypes.bool: - 'bool_value', - # Unhandled TensorFlow types: - # DT_INT16 DT_COMPLEX64 DT_QINT8 DT_QUINT8 DT_QINT32 - # DT_BFLOAT16 DT_QINT16 DT_QUINT16 DT_UINT16 - } - tf_field_name = tf_type_to_primitive_value_field.get(field.dtype) - if tf_field_name is None: - self.fail('Unhandled tensorflow type %d' % field.dtype) - - self._compareValues(fd, values.flat, - getattr(field.expected, tf_field_name)) - - def _runDecodeProtoTests(self, fields, case_sizes, batch_shape, batch, - message_type, message_format, sanitize, - force_disordered=False): - """Run decode tests on a batch of messages. - - Args: - fields: list of test_example_pb2.FieldSpec (types and expected values) - case_sizes: expected sizes array - batch_shape: the shape of the input tensor of serialized messages - batch: list of serialized messages - message_type: descriptor name for messages - message_format: format of messages, 'text' or 'binary' - sanitize: whether to sanitize binary protobuf inputs - force_disordered: whether to force fields encoded out of order. - """ - - if force_disordered: - # Exercise code path that handles out-of-order fields by prepending extra - # fields with tag numbers higher than any real field. Note that this won't - # work with sanitization because that forces reserialization using a - # trusted decoder and encoder. - assert not sanitize - extra_fields = test_example_pb2.ExtraFields() - extra_fields.string_value = 'IGNORE ME' - extra_fields.bool_value = False - extra_msg = extra_fields.SerializeToString() - batch = [extra_msg + msg for msg in batch] - - # Numpy silently truncates the strings if you don't specify dtype=object. - batch = np.array(batch, dtype=object) - batch = np.reshape(batch, batch_shape) - - field_names = [f.name for f in fields] - output_types = [f.dtype for f in fields] - - with self.test_session() as sess: - sizes, vtensor = decode_proto_op.decode_proto( - batch, - message_type=message_type, - field_names=field_names, - output_types=output_types, - message_format=message_format, - sanitize=sanitize) - - vlist = sess.run([sizes] + vtensor) - sizes = vlist[0] - # Values is a list of tensors, one for each field. - value_tensors = vlist[1:] - - # Check that the repeat sizes are correct. - self.assertTrue( - np.all(np.array(sizes.shape) == batch_shape + [len(field_names)])) - - # Check that the decoded sizes match the expected sizes. - self.assertEqual(len(sizes.flat), len(case_sizes)) - self.assertTrue( - np.all(sizes.flat == np.array( - case_sizes, dtype=np.int32))) - - field_dict = dict(zip(field_names, value_tensors)) - - self._compareRepeatedPrimitiveValue(batch_shape, sizes, fields, - field_dict) - - @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) - def testBinary(self, case): - batch = [primitive.SerializeToString() for primitive in case.primitive] - self._runDecodeProtoTests( - case.field, - case.sizes, - list(case.shape), - batch, - 'tensorflow.contrib.proto.RepeatedPrimitiveValue', - 'binary', - sanitize=False) - - @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) - def testBinaryDisordered(self, case): - batch = [primitive.SerializeToString() for primitive in case.primitive] - self._runDecodeProtoTests( - case.field, - case.sizes, - list(case.shape), - batch, - 'tensorflow.contrib.proto.RepeatedPrimitiveValue', - 'binary', - sanitize=False, - force_disordered=True) - - @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) - def testPacked(self, case): - # Now try with the packed serialization. - # We test the packed representations by loading the same test cases - # using PackedPrimitiveValue instead of RepeatedPrimitiveValue. - # To do this we rely on the text format being the same for packed and - # unpacked fields, and reparse the test message using the packed version - # of the proto. - packed_batch = [ - # Note: float_format='.17g' is necessary to ensure preservation of - # doubles and floats in text format. - text_format.Parse( - text_format.MessageToString( - primitive, float_format='.17g'), - test_example_pb2.PackedPrimitiveValue()).SerializeToString() - for primitive in case.primitive - ] - - self._runDecodeProtoTests( - case.field, - case.sizes, - list(case.shape), - packed_batch, - 'tensorflow.contrib.proto.PackedPrimitiveValue', - 'binary', - sanitize=False) - - @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) - def testText(self, case): - # Note: float_format='.17g' is necessary to ensure preservation of - # doubles and floats in text format. - text_batch = [ - text_format.MessageToString( - primitive, float_format='.17g') for primitive in case.primitive - ] - - self._runDecodeProtoTests( - case.field, - case.sizes, - list(case.shape), - text_batch, - 'tensorflow.contrib.proto.RepeatedPrimitiveValue', - 'text', - sanitize=False) +class DecodeProtoOpTest(test_base.DecodeProtoOpTestBase): - @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) - def testSanitizerGood(self, case): - batch = [primitive.SerializeToString() for primitive in case.primitive] - self._runDecodeProtoTests( - case.field, - case.sizes, - list(case.shape), - batch, - 'tensorflow.contrib.proto.RepeatedPrimitiveValue', - 'binary', - sanitize=True) + def __init__(self, methodName='runTest'): # pylint: disable=invalid-name + super(DecodeProtoOpTest, self).__init__(decode_proto_op, methodName) if __name__ == '__main__': diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py new file mode 100644 index 0000000000..5f7f510352 --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py @@ -0,0 +1,310 @@ +# ============================================================================= +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Tests for decode_proto op.""" + +# Python3 preparedness imports. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized +import numpy as np + + +from google.protobuf import text_format + +from tensorflow.contrib.proto.python.kernel_tests import proto_op_test_base as test_base +from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors + + +class DecodeProtoOpTestBase(test_base.ProtoOpTestBase, parameterized.TestCase): + """Base class for testing proto decoding ops.""" + + def __init__(self, decode_module, methodName='runTest'): # pylint: disable=invalid-name + """DecodeProtoOpTestBase initializer. + + Args: + decode_module: a module containing the `decode_proto_op` method + methodName: the name of the test method (same as for test.TestCase) + """ + + super(DecodeProtoOpTestBase, self).__init__(methodName) + self._decode_module = decode_module + + def _compareValues(self, fd, vs, evs): + """Compare lists/arrays of field values.""" + + if len(vs) != len(evs): + self.fail('Field %s decoded %d outputs, expected %d' % + (fd.name, len(vs), len(evs))) + for i, ev in enumerate(evs): + # Special case fuzzy match for float32. TensorFlow seems to mess with + # MAX_FLT slightly and the test doesn't work otherwise. + # TODO(nix): ask on TF list about why MAX_FLT doesn't pass through. + if fd.cpp_type == fd.CPPTYPE_FLOAT: + # Numpy isclose() is better than assertIsClose() which uses an absolute + # value comparison. + self.assertTrue( + np.isclose(vs[i], ev), 'expected %r, actual %r' % (ev, vs[i])) + elif fd.cpp_type == fd.CPPTYPE_STRING: + # In Python3 string tensor values will be represented as bytes, so we + # reencode the proto values to match that. + self.assertEqual(vs[i], ev.encode('ascii')) + else: + # Doubles and other types pass through unscathed. + self.assertEqual(vs[i], ev) + + def _compareProtos(self, batch_shape, sizes, fields, field_dict): + """Compare protos of type TestValue. + + Args: + batch_shape: the shape of the input tensor of serialized messages. + sizes: int matrix of repeat counts returned by decode_proto + fields: list of test_example_pb2.FieldSpec (types and expected values) + field_dict: map from field names to decoded numpy tensors of values + """ + + # Check that expected values match. + for field in fields: + values = field_dict[field.name] + self.assertEqual(dtypes.as_dtype(values.dtype), field.dtype) + + fd = field.value.DESCRIPTOR.fields_by_name[field.name] + + # Values has the same shape as the input plus an extra + # dimension for repeats. + self.assertEqual(list(values.shape)[:-1], batch_shape) + + # Nested messages are represented as TF strings, requiring + # some special handling. + if field.name == 'message_value': + vs = [] + for buf in values.flat: + msg = test_example_pb2.PrimitiveValue() + msg.ParseFromString(buf) + vs.append(msg) + evs = getattr(field.value, field.name) + if len(vs) != len(evs): + self.fail('Field %s decoded %d outputs, expected %d' % + (fd.name, len(vs), len(evs))) + for v, ev in zip(vs, evs): + self.assertEqual(v, ev) + continue + + # This can be a little confusing. For testing we are using TestValue in + # two ways: it's the proto that we decode for testing, and it's used in + # the expected value as a union type. + # + # The two cases are slightly different: this is the second case. We may be + # fetching the uint64_value from the test proto, but in the expected proto + # we store it in the int64_value field because TensorFlow doesn't support + # unsigned int64. + tf_type_to_primitive_value_field = { + dtypes.float32: + 'float_value', + dtypes.float64: + 'double_value', + dtypes.int32: + 'int32_value', + dtypes.uint8: + 'uint8_value', + dtypes.int8: + 'int8_value', + dtypes.string: + 'string_value', + dtypes.int64: + 'int64_value', + dtypes.bool: + 'bool_value', + # Unhandled TensorFlow types: + # DT_INT16 DT_COMPLEX64 DT_QINT8 DT_QUINT8 DT_QINT32 + # DT_BFLOAT16 DT_QINT16 DT_QUINT16 DT_UINT16 + } + tf_field_name = tf_type_to_primitive_value_field.get(field.dtype) + if tf_field_name is None: + self.fail('Unhandled tensorflow type %d' % field.dtype) + + self._compareValues(fd, values.flat, + getattr(field.value, tf_field_name)) + + def _runDecodeProtoTests(self, fields, case_sizes, batch_shape, batch, + message_type, message_format, sanitize, + force_disordered=False): + """Run decode tests on a batch of messages. + + Args: + fields: list of test_example_pb2.FieldSpec (types and expected values) + case_sizes: expected sizes array + batch_shape: the shape of the input tensor of serialized messages + batch: list of serialized messages + message_type: descriptor name for messages + message_format: format of messages, 'text' or 'binary' + sanitize: whether to sanitize binary protobuf inputs + force_disordered: whether to force fields encoded out of order. + """ + + if force_disordered: + # Exercise code path that handles out-of-order fields by prepending extra + # fields with tag numbers higher than any real field. Note that this won't + # work with sanitization because that forces reserialization using a + # trusted decoder and encoder. + assert not sanitize + extra_fields = test_example_pb2.ExtraFields() + extra_fields.string_value = 'IGNORE ME' + extra_fields.bool_value = False + extra_msg = extra_fields.SerializeToString() + batch = [extra_msg + msg for msg in batch] + + # Numpy silently truncates the strings if you don't specify dtype=object. + batch = np.array(batch, dtype=object) + batch = np.reshape(batch, batch_shape) + + field_names = [f.name for f in fields] + output_types = [f.dtype for f in fields] + + with self.test_session() as sess: + sizes, vtensor = self._decode_module.decode_proto( + batch, + message_type=message_type, + field_names=field_names, + output_types=output_types, + message_format=message_format, + sanitize=sanitize) + + vlist = sess.run([sizes] + vtensor) + sizes = vlist[0] + # Values is a list of tensors, one for each field. + value_tensors = vlist[1:] + + # Check that the repeat sizes are correct. + self.assertTrue( + np.all(np.array(sizes.shape) == batch_shape + [len(field_names)])) + + # Check that the decoded sizes match the expected sizes. + self.assertEqual(len(sizes.flat), len(case_sizes)) + self.assertTrue( + np.all(sizes.flat == np.array( + case_sizes, dtype=np.int32))) + + field_dict = dict(zip(field_names, value_tensors)) + + self._compareProtos(batch_shape, sizes, fields, field_dict) + + @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) + def testBinary(self, case): + batch = [value.SerializeToString() for value in case.values] + self._runDecodeProtoTests( + case.fields, + case.sizes, + list(case.shapes), + batch, + 'tensorflow.contrib.proto.TestValue', + 'binary', + sanitize=False) + + @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) + def testBinaryDisordered(self, case): + batch = [value.SerializeToString() for value in case.values] + self._runDecodeProtoTests( + case.fields, + case.sizes, + list(case.shapes), + batch, + 'tensorflow.contrib.proto.TestValue', + 'binary', + sanitize=False, + force_disordered=True) + + @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) + def testPacked(self, case): + # Now try with the packed serialization. + # + # We test the packed representations by loading the same test case using + # PackedTestValue instead of TestValue. To do this we rely on the text + # format being the same for packed and unpacked fields, and reparse the + # test message using the packed version of the proto. + packed_batch = [ + # Note: float_format='.17g' is necessary to ensure preservation of + # doubles and floats in text format. + text_format.Parse( + text_format.MessageToString( + value, float_format='.17g'), + test_example_pb2.PackedTestValue()).SerializeToString() + for value in case.values + ] + + self._runDecodeProtoTests( + case.fields, + case.sizes, + list(case.shapes), + packed_batch, + 'tensorflow.contrib.proto.PackedTestValue', + 'binary', + sanitize=False) + + @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) + def testText(self, case): + # Note: float_format='.17g' is necessary to ensure preservation of + # doubles and floats in text format. + text_batch = [ + text_format.MessageToString( + value, float_format='.17g') for value in case.values + ] + + self._runDecodeProtoTests( + case.fields, + case.sizes, + list(case.shapes), + text_batch, + 'tensorflow.contrib.proto.TestValue', + 'text', + sanitize=False) + + @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) + def testSanitizerGood(self, case): + batch = [value.SerializeToString() for value in case.values] + self._runDecodeProtoTests( + case.fields, + case.sizes, + list(case.shapes), + batch, + 'tensorflow.contrib.proto.TestValue', + 'binary', + sanitize=True) + + @parameterized.parameters((False), (True)) + def testCorruptProtobuf(self, sanitize): + corrupt_proto = 'This is not a binary protobuf' + + # Numpy silently truncates the strings if you don't specify dtype=object. + batch = np.array(corrupt_proto, dtype=object) + msg_type = 'tensorflow.contrib.proto.TestCase' + field_names = ['sizes'] + field_types = [dtypes.int32] + + with self.test_session() as sess: + ctensor, vtensor = self._decode_module.decode_proto( + batch, + message_type=msg_type, + field_names=field_names, + output_types=field_types, + sanitize=sanitize) + with self.assertRaisesRegexp(errors.DataLossError, + 'Unable to parse binary protobuf' + '|Failed to consume entire buffer'): + _ = sess.run([ctensor] + vtensor) diff --git a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py index fb33660554..fc5cd25d43 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py +++ b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test.py @@ -13,164 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================= -"""Table-driven test for encode_proto op. +"""Tests for encode_proto op.""" -This test is run once with each of the *.TestCase.pbtxt files -in the test directory. - -It tests that encode_proto is a lossless inverse of decode_proto -(for the specified fields). -""" # Python3 readiness boilerplate from __future__ import absolute_import from __future__ import division from __future__ import print_function -from absl.testing import parameterized -import numpy as np - -from google.protobuf import text_format - -from tensorflow.contrib.proto.python.kernel_tests import test_base -from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 +from tensorflow.contrib.proto.python.kernel_tests import encode_proto_op_test_base as test_base from tensorflow.contrib.proto.python.ops import decode_proto_op from tensorflow.contrib.proto.python.ops import encode_proto_op -from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.platform import flags from tensorflow.python.platform import test -FLAGS = flags.FLAGS - -flags.DEFINE_string('message_text_file', None, - 'A file containing a text serialized TestCase protobuf.') - - -class EncodeProtoOpTest(test_base.ProtoOpTestBase, parameterized.TestCase): - - def testBadInputs(self): - # Invalid field name - with self.test_session(): - with self.assertRaisesOpError('Unknown field: non_existent_field'): - encode_proto_op.encode_proto( - sizes=[[1]], - values=[np.array([[0.0]], dtype=np.int32)], - message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', - field_names=['non_existent_field']).eval() - - # Incorrect types. - with self.test_session(): - with self.assertRaisesOpError( - 'Incompatible type for field double_value.'): - encode_proto_op.encode_proto( - sizes=[[1]], - values=[np.array([[0.0]], dtype=np.int32)], - message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', - field_names=['double_value']).eval() - - # Incorrect shapes of sizes. - with self.test_session(): - with self.assertRaisesOpError( - r'sizes should be batch_size \+ \[len\(field_names\)\]'): - sizes = array_ops.placeholder(dtypes.int32) - values = array_ops.placeholder(dtypes.float64) - encode_proto_op.encode_proto( - sizes=sizes, - values=[values], - message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', - field_names=['double_value']).eval(feed_dict={ - sizes: [[[0, 0]]], - values: [[0.0]] - }) - - # Inconsistent shapes of values. - with self.test_session(): - with self.assertRaisesOpError( - 'Values must match up to the last dimension'): - sizes = array_ops.placeholder(dtypes.int32) - values1 = array_ops.placeholder(dtypes.float64) - values2 = array_ops.placeholder(dtypes.int32) - (encode_proto_op.encode_proto( - sizes=[[1, 1]], - values=[values1, values2], - message_type='tensorflow.contrib.proto.RepeatedPrimitiveValue', - field_names=['double_value', 'int32_value']).eval(feed_dict={ - values1: [[0.0]], - values2: [[0], [0]] - })) - - def _testRoundtrip(self, in_bufs, message_type, fields): - - field_names = [f.name for f in fields] - out_types = [f.dtype for f in fields] - - with self.test_session() as sess: - sizes, field_tensors = decode_proto_op.decode_proto( - in_bufs, - message_type=message_type, - field_names=field_names, - output_types=out_types) - - out_tensors = encode_proto_op.encode_proto( - sizes, - field_tensors, - message_type=message_type, - field_names=field_names) - - out_bufs, = sess.run([out_tensors]) - - # Check that the re-encoded tensor has the same shape. - self.assertEqual(in_bufs.shape, out_bufs.shape) - - # Compare the input and output. - for in_buf, out_buf in zip(in_bufs.flat, out_bufs.flat): - in_obj = test_example_pb2.RepeatedPrimitiveValue() - in_obj.ParseFromString(in_buf) - - out_obj = test_example_pb2.RepeatedPrimitiveValue() - out_obj.ParseFromString(out_buf) - - # Check that the deserialized objects are identical. - self.assertEqual(in_obj, out_obj) - - # Check that the input and output serialized messages are identical. - # If we fail here, there is a difference in the serialized - # representation but the new serialization still parses. This could - # be harmless (a change in map ordering?) or it could be bad (e.g. - # loss of packing in the encoding). - self.assertEqual(in_buf, out_buf) - - @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) - def testRoundtrip(self, case): - in_bufs = [primitive.SerializeToString() for primitive in case.primitive] - - # np.array silently truncates strings if you don't specify dtype=object. - in_bufs = np.reshape(np.array(in_bufs, dtype=object), list(case.shape)) - return self._testRoundtrip( - in_bufs, 'tensorflow.contrib.proto.RepeatedPrimitiveValue', case.field) - @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) - def testRoundtripPacked(self, case): - # Now try with the packed serialization. - # We test the packed representations by loading the same test cases - # using PackedPrimitiveValue instead of RepeatedPrimitiveValue. - # To do this we rely on the text format being the same for packed and - # unpacked fields, and reparse the test message using the packed version - # of the proto. - in_bufs = [ - # Note: float_format='.17g' is necessary to ensure preservation of - # doubles and floats in text format. - text_format.Parse( - text_format.MessageToString( - primitive, float_format='.17g'), - test_example_pb2.PackedPrimitiveValue()).SerializeToString() - for primitive in case.primitive - ] +class EncodeProtoOpTest(test_base.EncodeProtoOpTestBase): - # np.array silently truncates strings if you don't specify dtype=object. - in_bufs = np.reshape(np.array(in_bufs, dtype=object), list(case.shape)) - return self._testRoundtrip( - in_bufs, 'tensorflow.contrib.proto.PackedPrimitiveValue', case.field) + def __init__(self, methodName='runTest'): # pylint: disable=invalid-name + super(EncodeProtoOpTest, self).__init__(decode_proto_op, encode_proto_op, + methodName) if __name__ == '__main__': diff --git a/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test_base.py b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test_base.py new file mode 100644 index 0000000000..07dfb924d3 --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/encode_proto_op_test_base.py @@ -0,0 +1,177 @@ +# ============================================================================= +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Table-driven test for encode_proto op. + +This test is run once with each of the *.TestCase.pbtxt files +in the test directory. + +It tests that encode_proto is a lossless inverse of decode_proto +(for the specified fields). +""" +# Python3 readiness boilerplate +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized +import numpy as np + +from google.protobuf import text_format + +from tensorflow.contrib.proto.python.kernel_tests import proto_op_test_base as test_base +from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops + + +class EncodeProtoOpTestBase(test_base.ProtoOpTestBase, parameterized.TestCase): + """Base class for testing proto encoding ops.""" + + def __init__(self, decode_module, encode_module, methodName='runTest'): # pylint: disable=invalid-name + """EncodeProtoOpTestBase initializer. + + Args: + decode_module: a module containing the `decode_proto_op` method + encode_module: a module containing the `encode_proto_op` method + methodName: the name of the test method (same as for test.TestCase) + """ + + super(EncodeProtoOpTestBase, self).__init__(methodName) + self._decode_module = decode_module + self._encode_module = encode_module + + def testBadInputs(self): + # Invalid field name + with self.test_session(): + with self.assertRaisesOpError('Unknown field: non_existent_field'): + self._encode_module.encode_proto( + sizes=[[1]], + values=[np.array([[0.0]], dtype=np.int32)], + message_type='tensorflow.contrib.proto.TestValue', + field_names=['non_existent_field']).eval() + + # Incorrect types. + with self.test_session(): + with self.assertRaisesOpError( + 'Incompatible type for field double_value.'): + self._encode_module.encode_proto( + sizes=[[1]], + values=[np.array([[0.0]], dtype=np.int32)], + message_type='tensorflow.contrib.proto.TestValue', + field_names=['double_value']).eval() + + # Incorrect shapes of sizes. + with self.test_session(): + with self.assertRaisesOpError( + r'sizes should be batch_size \+ \[len\(field_names\)\]'): + sizes = array_ops.placeholder(dtypes.int32) + values = array_ops.placeholder(dtypes.float64) + self._encode_module.encode_proto( + sizes=sizes, + values=[values], + message_type='tensorflow.contrib.proto.TestValue', + field_names=['double_value']).eval(feed_dict={ + sizes: [[[0, 0]]], + values: [[0.0]] + }) + + # Inconsistent shapes of values. + with self.test_session(): + with self.assertRaisesOpError( + 'Values must match up to the last dimension'): + sizes = array_ops.placeholder(dtypes.int32) + values1 = array_ops.placeholder(dtypes.float64) + values2 = array_ops.placeholder(dtypes.int32) + (self._encode_module.encode_proto( + sizes=[[1, 1]], + values=[values1, values2], + message_type='tensorflow.contrib.proto.TestValue', + field_names=['double_value', 'int32_value']).eval(feed_dict={ + values1: [[0.0]], + values2: [[0], [0]] + })) + + def _testRoundtrip(self, in_bufs, message_type, fields): + + field_names = [f.name for f in fields] + out_types = [f.dtype for f in fields] + + with self.test_session() as sess: + sizes, field_tensors = self._decode_module.decode_proto( + in_bufs, + message_type=message_type, + field_names=field_names, + output_types=out_types) + + out_tensors = self._encode_module.encode_proto( + sizes, + field_tensors, + message_type=message_type, + field_names=field_names) + + out_bufs, = sess.run([out_tensors]) + + # Check that the re-encoded tensor has the same shape. + self.assertEqual(in_bufs.shape, out_bufs.shape) + + # Compare the input and output. + for in_buf, out_buf in zip(in_bufs.flat, out_bufs.flat): + in_obj = test_example_pb2.TestValue() + in_obj.ParseFromString(in_buf) + + out_obj = test_example_pb2.TestValue() + out_obj.ParseFromString(out_buf) + + # Check that the deserialized objects are identical. + self.assertEqual(in_obj, out_obj) + + # Check that the input and output serialized messages are identical. + # If we fail here, there is a difference in the serialized + # representation but the new serialization still parses. This could + # be harmless (a change in map ordering?) or it could be bad (e.g. + # loss of packing in the encoding). + self.assertEqual(in_buf, out_buf) + + @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) + def testRoundtrip(self, case): + in_bufs = [value.SerializeToString() for value in case.values] + + # np.array silently truncates strings if you don't specify dtype=object. + in_bufs = np.reshape(np.array(in_bufs, dtype=object), list(case.shapes)) + return self._testRoundtrip( + in_bufs, 'tensorflow.contrib.proto.TestValue', case.fields) + + @parameterized.named_parameters(*test_base.ProtoOpTestBase.named_parameters()) + def testRoundtripPacked(self, case): + # Now try with the packed serialization. + # We test the packed representations by loading the same test cases using + # PackedTestValue instead of TestValue. To do this we rely on the text + # format being the same for packed and unpacked fields, and reparse the test + # message using the packed version of the proto. + in_bufs = [ + # Note: float_format='.17g' is necessary to ensure preservation of + # doubles and floats in text format. + text_format.Parse( + text_format.MessageToString( + value, float_format='.17g'), + test_example_pb2.PackedTestValue()).SerializeToString() + for value in case.values + ] + + # np.array silently truncates strings if you don't specify dtype=object. + in_bufs = np.reshape(np.array(in_bufs, dtype=object), list(case.shapes)) + return self._testRoundtrip( + in_bufs, 'tensorflow.contrib.proto.PackedTestValue', case.fields) diff --git a/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py b/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py new file mode 100644 index 0000000000..cbc7b3d3f8 --- /dev/null +++ b/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py @@ -0,0 +1,407 @@ +# ============================================================================= +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Test case base for testing proto operations.""" + +# Python3 preparedness imports. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import ctypes as ct +import os + +from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 +from tensorflow.core.framework import types_pb2 +from tensorflow.python.platform import test + + +class ProtoOpTestBase(test.TestCase): + """Base class for testing proto decoding and encoding ops.""" + + def __init__(self, methodName="runTest"): # pylint: disable=invalid-name + super(ProtoOpTestBase, self).__init__(methodName) + lib = os.path.join(os.path.dirname(__file__), "libtestexample.so") + if os.path.isfile(lib): + ct.cdll.LoadLibrary(lib) + + @staticmethod + def named_parameters(): + return ( + ("defaults", ProtoOpTestBase.defaults_test_case()), + ("minmax", ProtoOpTestBase.minmax_test_case()), + ("nested", ProtoOpTestBase.nested_test_case()), + ("optional", ProtoOpTestBase.optional_test_case()), + ("promote_unsigned", ProtoOpTestBase.promote_unsigned_test_case()), + ("ragged", ProtoOpTestBase.ragged_test_case()), + ("shaped_batch", ProtoOpTestBase.shaped_batch_test_case()), + ("simple", ProtoOpTestBase.simple_test_case()), + ) + + @staticmethod + def defaults_test_case(): + test_case = test_example_pb2.TestCase() + test_case.values.add() # No fields specified, so we get all defaults. + test_case.shapes.append(1) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "double_value_with_default" + field.dtype = types_pb2.DT_DOUBLE + field.value.double_value.append(1.0) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "float_value_with_default" + field.dtype = types_pb2.DT_FLOAT + field.value.float_value.append(2.0) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "int64_value_with_default" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(3) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "sfixed64_value_with_default" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(11) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "sint64_value_with_default" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(13) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "uint64_value_with_default" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(4) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "fixed64_value_with_default" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(6) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "int32_value_with_default" + field.dtype = types_pb2.DT_INT32 + field.value.int32_value.append(5) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "sfixed32_value_with_default" + field.dtype = types_pb2.DT_INT32 + field.value.int32_value.append(10) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "sint32_value_with_default" + field.dtype = types_pb2.DT_INT32 + field.value.int32_value.append(12) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "uint32_value_with_default" + field.dtype = types_pb2.DT_INT32 + field.value.int32_value.append(9) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "fixed32_value_with_default" + field.dtype = types_pb2.DT_INT32 + field.value.int32_value.append(7) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "bool_value_with_default" + field.dtype = types_pb2.DT_BOOL + field.value.bool_value.append(True) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "string_value_with_default" + field.dtype = types_pb2.DT_STRING + field.value.string_value.append("a") + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "bytes_value_with_default" + field.dtype = types_pb2.DT_STRING + field.value.string_value.append("a longer default string") + return test_case + + @staticmethod + def minmax_test_case(): + test_case = test_example_pb2.TestCase() + value = test_case.values.add() + value.double_value.append(-1.7976931348623158e+308) + value.double_value.append(2.2250738585072014e-308) + value.double_value.append(1.7976931348623158e+308) + value.float_value.append(-3.402823466e+38) + value.float_value.append(1.175494351e-38) + value.float_value.append(3.402823466e+38) + value.int64_value.append(-9223372036854775808) + value.int64_value.append(9223372036854775807) + value.sfixed64_value.append(-9223372036854775808) + value.sfixed64_value.append(9223372036854775807) + value.sint64_value.append(-9223372036854775808) + value.sint64_value.append(9223372036854775807) + value.uint64_value.append(0) + value.uint64_value.append(18446744073709551615) + value.fixed64_value.append(0) + value.fixed64_value.append(18446744073709551615) + value.int32_value.append(-2147483648) + value.int32_value.append(2147483647) + value.sfixed32_value.append(-2147483648) + value.sfixed32_value.append(2147483647) + value.sint32_value.append(-2147483648) + value.sint32_value.append(2147483647) + value.uint32_value.append(0) + value.uint32_value.append(4294967295) + value.fixed32_value.append(0) + value.fixed32_value.append(4294967295) + value.bool_value.append(False) + value.bool_value.append(True) + value.string_value.append("") + value.string_value.append("I refer to the infinite.") + test_case.shapes.append(1) + test_case.sizes.append(3) + field = test_case.fields.add() + field.name = "double_value" + field.dtype = types_pb2.DT_DOUBLE + field.value.double_value.append(-1.7976931348623158e+308) + field.value.double_value.append(2.2250738585072014e-308) + field.value.double_value.append(1.7976931348623158e+308) + test_case.sizes.append(3) + field = test_case.fields.add() + field.name = "float_value" + field.dtype = types_pb2.DT_FLOAT + field.value.float_value.append(-3.402823466e+38) + field.value.float_value.append(1.175494351e-38) + field.value.float_value.append(3.402823466e+38) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "int64_value" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(-9223372036854775808) + field.value.int64_value.append(9223372036854775807) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "sfixed64_value" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(-9223372036854775808) + field.value.int64_value.append(9223372036854775807) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "sint64_value" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(-9223372036854775808) + field.value.int64_value.append(9223372036854775807) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "uint64_value" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(0) + field.value.int64_value.append(-1) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "fixed64_value" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(0) + field.value.int64_value.append(-1) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "int32_value" + field.dtype = types_pb2.DT_INT32 + field.value.int32_value.append(-2147483648) + field.value.int32_value.append(2147483647) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "sfixed32_value" + field.dtype = types_pb2.DT_INT32 + field.value.int32_value.append(-2147483648) + field.value.int32_value.append(2147483647) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "sint32_value" + field.dtype = types_pb2.DT_INT32 + field.value.int32_value.append(-2147483648) + field.value.int32_value.append(2147483647) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "uint32_value" + field.dtype = types_pb2.DT_INT32 + field.value.int32_value.append(0) + field.value.int32_value.append(-1) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "fixed32_value" + field.dtype = types_pb2.DT_INT32 + field.value.int32_value.append(0) + field.value.int32_value.append(-1) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "bool_value" + field.dtype = types_pb2.DT_BOOL + field.value.bool_value.append(False) + field.value.bool_value.append(True) + test_case.sizes.append(2) + field = test_case.fields.add() + field.name = "string_value" + field.dtype = types_pb2.DT_STRING + field.value.string_value.append("") + field.value.string_value.append("I refer to the infinite.") + return test_case + + @staticmethod + def nested_test_case(): + test_case = test_example_pb2.TestCase() + value = test_case.values.add() + message_value = value.message_value.add() + message_value.double_value = 23.5 + test_case.shapes.append(1) + test_case.sizes.append(1) + field = test_case.fields.add() + field.name = "message_value" + field.dtype = types_pb2.DT_STRING + message_value = field.value.message_value.add() + message_value.double_value = 23.5 + return test_case + + @staticmethod + def optional_test_case(): + test_case = test_example_pb2.TestCase() + value = test_case.values.add() + value.bool_value.append(True) + test_case.shapes.append(1) + test_case.sizes.append(1) + field = test_case.fields.add() + field.name = "bool_value" + field.dtype = types_pb2.DT_BOOL + field.value.bool_value.append(True) + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "double_value" + field.dtype = types_pb2.DT_DOUBLE + field.value.double_value.append(0.0) + return test_case + + @staticmethod + def promote_unsigned_test_case(): + test_case = test_example_pb2.TestCase() + value = test_case.values.add() + value.fixed32_value.append(4294967295) + value.uint32_value.append(4294967295) + test_case.shapes.append(1) + test_case.sizes.append(1) + field = test_case.fields.add() + field.name = "fixed32_value" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(4294967295) + test_case.sizes.append(1) + field = test_case.fields.add() + field.name = "uint32_value" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(4294967295) + # Comes from an explicitly-specified default + test_case.sizes.append(0) + field = test_case.fields.add() + field.name = "uint32_value_with_default" + field.dtype = types_pb2.DT_INT64 + field.value.int64_value.append(9) + return test_case + + @staticmethod + def ragged_test_case(): + test_case = test_example_pb2.TestCase() + value = test_case.values.add() + value.double_value.append(23.5) + value.double_value.append(123.0) + value.bool_value.append(True) + value = test_case.values.add() + value.double_value.append(3.1) + value.bool_value.append(False) + test_case.shapes.append(2) + test_case.sizes.append(2) + test_case.sizes.append(1) + test_case.sizes.append(1) + test_case.sizes.append(1) + field = test_case.fields.add() + field.name = "double_value" + field.dtype = types_pb2.DT_DOUBLE + field.value.double_value.append(23.5) + field.value.double_value.append(123.0) + field.value.double_value.append(3.1) + field.value.double_value.append(0.0) + field = test_case.fields.add() + field.name = "bool_value" + field.dtype = types_pb2.DT_BOOL + field.value.bool_value.append(True) + field.value.bool_value.append(False) + return test_case + + @staticmethod + def shaped_batch_test_case(): + test_case = test_example_pb2.TestCase() + value = test_case.values.add() + value.double_value.append(23.5) + value.bool_value.append(True) + value = test_case.values.add() + value.double_value.append(44.0) + value.bool_value.append(False) + value = test_case.values.add() + value.double_value.append(3.14159) + value.bool_value.append(True) + value = test_case.values.add() + value.double_value.append(1.414) + value.bool_value.append(True) + value = test_case.values.add() + value.double_value.append(-32.2) + value.bool_value.append(False) + value = test_case.values.add() + value.double_value.append(0.0001) + value.bool_value.append(True) + test_case.shapes.append(3) + test_case.shapes.append(2) + for _ in range(12): + test_case.sizes.append(1) + field = test_case.fields.add() + field.name = "double_value" + field.dtype = types_pb2.DT_DOUBLE + field.value.double_value.append(23.5) + field.value.double_value.append(44.0) + field.value.double_value.append(3.14159) + field.value.double_value.append(1.414) + field.value.double_value.append(-32.2) + field.value.double_value.append(0.0001) + field = test_case.fields.add() + field.name = "bool_value" + field.dtype = types_pb2.DT_BOOL + field.value.bool_value.append(True) + field.value.bool_value.append(False) + field.value.bool_value.append(True) + field.value.bool_value.append(True) + field.value.bool_value.append(False) + field.value.bool_value.append(True) + return test_case + + @staticmethod + def simple_test_case(): + test_case = test_example_pb2.TestCase() + value = test_case.values.add() + value.double_value.append(23.5) + value.bool_value.append(True) + test_case.shapes.append(1) + test_case.sizes.append(1) + field = test_case.fields.add() + field.name = "double_value" + field.dtype = types_pb2.DT_DOUBLE + field.value.double_value.append(23.5) + test_case.sizes.append(1) + field = test_case.fields.add() + field.name = "bool_value" + field.dtype = types_pb2.DT_BOOL + field.value.bool_value.append(True) + return test_case diff --git a/tensorflow/contrib/proto/python/kernel_tests/test_base.py b/tensorflow/contrib/proto/python/kernel_tests/test_base.py deleted file mode 100644 index 1fc8c16786..0000000000 --- a/tensorflow/contrib/proto/python/kernel_tests/test_base.py +++ /dev/null @@ -1,407 +0,0 @@ -# ============================================================================= -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= -"""Test case base for testing proto operations.""" - -# Python3 preparedness imports. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import ctypes as ct -import os - -from tensorflow.contrib.proto.python.kernel_tests import test_example_pb2 -from tensorflow.core.framework import types_pb2 -from tensorflow.python.platform import test - - -class ProtoOpTestBase(test.TestCase): - """Base class for testing proto decoding and encoding ops.""" - - def __init__(self, methodName="runTest"): # pylint: disable=invalid-name - super(ProtoOpTestBase, self).__init__(methodName) - lib = os.path.join(os.path.dirname(__file__), "libtestexample.so") - if os.path.isfile(lib): - ct.cdll.LoadLibrary(lib) - - @staticmethod - def named_parameters(): - return ( - ("defaults", ProtoOpTestBase.defaults_test_case()), - ("minmax", ProtoOpTestBase.minmax_test_case()), - ("nested", ProtoOpTestBase.nested_test_case()), - ("optional", ProtoOpTestBase.optional_test_case()), - ("promote_unsigned", ProtoOpTestBase.promote_unsigned_test_case()), - ("ragged", ProtoOpTestBase.ragged_test_case()), - ("shaped_batch", ProtoOpTestBase.shaped_batch_test_case()), - ("simple", ProtoOpTestBase.simple_test_case()), - ) - - @staticmethod - def defaults_test_case(): - test_case = test_example_pb2.TestCase() - test_case.primitive.add() # No fields specified, so we get all defaults. - test_case.shape.append(1) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "double_default" - field.dtype = types_pb2.DT_DOUBLE - field.expected.double_value.append(1.0) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "float_default" - field.dtype = types_pb2.DT_FLOAT - field.expected.float_value.append(2.0) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "int64_default" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(3) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "sfixed64_default" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(11) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "sint64_default" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(13) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "uint64_default" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(4) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "fixed64_default" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(6) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "int32_default" - field.dtype = types_pb2.DT_INT32 - field.expected.int32_value.append(5) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "sfixed32_default" - field.dtype = types_pb2.DT_INT32 - field.expected.int32_value.append(10) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "sint32_default" - field.dtype = types_pb2.DT_INT32 - field.expected.int32_value.append(12) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "uint32_default" - field.dtype = types_pb2.DT_INT32 - field.expected.int32_value.append(-1) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "fixed32_default" - field.dtype = types_pb2.DT_INT32 - field.expected.int32_value.append(7) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "bool_default" - field.dtype = types_pb2.DT_BOOL - field.expected.bool_value.append(True) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "string_default" - field.dtype = types_pb2.DT_STRING - field.expected.string_value.append("a") - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "bytes_default" - field.dtype = types_pb2.DT_STRING - field.expected.string_value.append("a longer default string") - return test_case - - @staticmethod - def minmax_test_case(): - test_case = test_example_pb2.TestCase() - primitive = test_case.primitive.add() - primitive.double_value.append(-1.7976931348623158e+308) - primitive.double_value.append(2.2250738585072014e-308) - primitive.double_value.append(1.7976931348623158e+308) - primitive.float_value.append(-3.402823466e+38) - primitive.float_value.append(1.175494351e-38) - primitive.float_value.append(3.402823466e+38) - primitive.int64_value.append(-9223372036854775808) - primitive.int64_value.append(9223372036854775807) - primitive.sfixed64_value.append(-9223372036854775808) - primitive.sfixed64_value.append(9223372036854775807) - primitive.sint64_value.append(-9223372036854775808) - primitive.sint64_value.append(9223372036854775807) - primitive.uint64_value.append(0) - primitive.uint64_value.append(18446744073709551615) - primitive.fixed64_value.append(0) - primitive.fixed64_value.append(18446744073709551615) - primitive.int32_value.append(-2147483648) - primitive.int32_value.append(2147483647) - primitive.sfixed32_value.append(-2147483648) - primitive.sfixed32_value.append(2147483647) - primitive.sint32_value.append(-2147483648) - primitive.sint32_value.append(2147483647) - primitive.uint32_value.append(0) - primitive.uint32_value.append(4294967295) - primitive.fixed32_value.append(0) - primitive.fixed32_value.append(4294967295) - primitive.bool_value.append(False) - primitive.bool_value.append(True) - primitive.string_value.append("") - primitive.string_value.append("I refer to the infinite.") - test_case.shape.append(1) - test_case.sizes.append(3) - field = test_case.field.add() - field.name = "double_value" - field.dtype = types_pb2.DT_DOUBLE - field.expected.double_value.append(-1.7976931348623158e+308) - field.expected.double_value.append(2.2250738585072014e-308) - field.expected.double_value.append(1.7976931348623158e+308) - test_case.sizes.append(3) - field = test_case.field.add() - field.name = "float_value" - field.dtype = types_pb2.DT_FLOAT - field.expected.float_value.append(-3.402823466e+38) - field.expected.float_value.append(1.175494351e-38) - field.expected.float_value.append(3.402823466e+38) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "int64_value" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(-9223372036854775808) - field.expected.int64_value.append(9223372036854775807) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "sfixed64_value" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(-9223372036854775808) - field.expected.int64_value.append(9223372036854775807) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "sint64_value" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(-9223372036854775808) - field.expected.int64_value.append(9223372036854775807) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "uint64_value" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(0) - field.expected.int64_value.append(-1) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "fixed64_value" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(0) - field.expected.int64_value.append(-1) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "int32_value" - field.dtype = types_pb2.DT_INT32 - field.expected.int32_value.append(-2147483648) - field.expected.int32_value.append(2147483647) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "sfixed32_value" - field.dtype = types_pb2.DT_INT32 - field.expected.int32_value.append(-2147483648) - field.expected.int32_value.append(2147483647) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "sint32_value" - field.dtype = types_pb2.DT_INT32 - field.expected.int32_value.append(-2147483648) - field.expected.int32_value.append(2147483647) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "uint32_value" - field.dtype = types_pb2.DT_INT32 - field.expected.int32_value.append(0) - field.expected.int32_value.append(-1) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "fixed32_value" - field.dtype = types_pb2.DT_INT32 - field.expected.int32_value.append(0) - field.expected.int32_value.append(-1) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "bool_value" - field.dtype = types_pb2.DT_BOOL - field.expected.bool_value.append(False) - field.expected.bool_value.append(True) - test_case.sizes.append(2) - field = test_case.field.add() - field.name = "string_value" - field.dtype = types_pb2.DT_STRING - field.expected.string_value.append("") - field.expected.string_value.append("I refer to the infinite.") - return test_case - - @staticmethod - def nested_test_case(): - test_case = test_example_pb2.TestCase() - primitive = test_case.primitive.add() - message_value = primitive.message_value.add() - message_value.double_value = 23.5 - test_case.shape.append(1) - test_case.sizes.append(1) - field = test_case.field.add() - field.name = "message_value" - field.dtype = types_pb2.DT_STRING - message_value = field.expected.message_value.add() - message_value.double_value = 23.5 - return test_case - - @staticmethod - def optional_test_case(): - test_case = test_example_pb2.TestCase() - primitive = test_case.primitive.add() - primitive.bool_value.append(True) - test_case.shape.append(1) - test_case.sizes.append(1) - field = test_case.field.add() - field.name = "bool_value" - field.dtype = types_pb2.DT_BOOL - field.expected.bool_value.append(True) - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "double_value" - field.dtype = types_pb2.DT_DOUBLE - field.expected.double_value.append(0.0) - return test_case - - @staticmethod - def promote_unsigned_test_case(): - test_case = test_example_pb2.TestCase() - primitive = test_case.primitive.add() - primitive.fixed32_value.append(4294967295) - primitive.uint32_value.append(4294967295) - test_case.shape.append(1) - test_case.sizes.append(1) - field = test_case.field.add() - field.name = "fixed32_value" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(4294967295) - test_case.sizes.append(1) - field = test_case.field.add() - field.name = "uint32_value" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(4294967295) - # Comes from an explicitly-specified default - test_case.sizes.append(0) - field = test_case.field.add() - field.name = "uint32_default" - field.dtype = types_pb2.DT_INT64 - field.expected.int64_value.append(4294967295) - return test_case - - @staticmethod - def ragged_test_case(): - test_case = test_example_pb2.TestCase() - primitive = test_case.primitive.add() - primitive.double_value.append(23.5) - primitive.double_value.append(123.0) - primitive.bool_value.append(True) - primitive = test_case.primitive.add() - primitive.double_value.append(3.1) - primitive.bool_value.append(False) - test_case.shape.append(2) - test_case.sizes.append(2) - test_case.sizes.append(1) - test_case.sizes.append(1) - test_case.sizes.append(1) - field = test_case.field.add() - field.name = "double_value" - field.dtype = types_pb2.DT_DOUBLE - field.expected.double_value.append(23.5) - field.expected.double_value.append(123.0) - field.expected.double_value.append(3.1) - field.expected.double_value.append(0.0) - field = test_case.field.add() - field.name = "bool_value" - field.dtype = types_pb2.DT_BOOL - field.expected.bool_value.append(True) - field.expected.bool_value.append(False) - return test_case - - @staticmethod - def shaped_batch_test_case(): - test_case = test_example_pb2.TestCase() - primitive = test_case.primitive.add() - primitive.double_value.append(23.5) - primitive.bool_value.append(True) - primitive = test_case.primitive.add() - primitive.double_value.append(44.0) - primitive.bool_value.append(False) - primitive = test_case.primitive.add() - primitive.double_value.append(3.14159) - primitive.bool_value.append(True) - primitive = test_case.primitive.add() - primitive.double_value.append(1.414) - primitive.bool_value.append(True) - primitive = test_case.primitive.add() - primitive.double_value.append(-32.2) - primitive.bool_value.append(False) - primitive = test_case.primitive.add() - primitive.double_value.append(0.0001) - primitive.bool_value.append(True) - test_case.shape.append(3) - test_case.shape.append(2) - for _ in range(12): - test_case.sizes.append(1) - field = test_case.field.add() - field.name = "double_value" - field.dtype = types_pb2.DT_DOUBLE - field.expected.double_value.append(23.5) - field.expected.double_value.append(44.0) - field.expected.double_value.append(3.14159) - field.expected.double_value.append(1.414) - field.expected.double_value.append(-32.2) - field.expected.double_value.append(0.0001) - field = test_case.field.add() - field.name = "bool_value" - field.dtype = types_pb2.DT_BOOL - field.expected.bool_value.append(True) - field.expected.bool_value.append(False) - field.expected.bool_value.append(True) - field.expected.bool_value.append(True) - field.expected.bool_value.append(False) - field.expected.bool_value.append(True) - return test_case - - @staticmethod - def simple_test_case(): - test_case = test_example_pb2.TestCase() - primitive = test_case.primitive.add() - primitive.double_value.append(23.5) - primitive.bool_value.append(True) - test_case.shape.append(1) - test_case.sizes.append(1) - field = test_case.field.add() - field.name = "double_value" - field.dtype = types_pb2.DT_DOUBLE - field.expected.double_value.append(23.5) - test_case.sizes.append(1) - field = test_case.field.add() - field.name = "bool_value" - field.dtype = types_pb2.DT_BOOL - field.expected.bool_value.append(True) - return test_case diff --git a/tensorflow/contrib/proto/python/kernel_tests/test_example.proto b/tensorflow/contrib/proto/python/kernel_tests/test_example.proto index a2c88e372b..674d881220 100644 --- a/tensorflow/contrib/proto/python/kernel_tests/test_example.proto +++ b/tensorflow/contrib/proto/python/kernel_tests/test_example.proto @@ -1,6 +1,4 @@ // Test description and protos to work with it. -// -// Many of the protos in this file are for unit tests that haven't been written yet. syntax = "proto2"; @@ -8,54 +6,27 @@ import "tensorflow/core/framework/types.proto"; package tensorflow.contrib.proto; -// A TestCase holds a proto and a bunch of assertions -// about how it should decode. +// A TestCase holds a proto and assertions about how it should decode. message TestCase { - // A batch of primitives to be serialized and decoded. - repeated RepeatedPrimitiveValue primitive = 1; - // The shape of the batch. - repeated int32 shape = 2; + // Batches of primitive values. + repeated TestValue values = 1; + // The batch shapes. + repeated int32 shapes = 2; // Expected sizes for each field. repeated int32 sizes = 3; // Expected values for each field. - repeated FieldSpec field = 4; + repeated FieldSpec fields = 4; }; // FieldSpec describes the expected output for a single field. message FieldSpec { optional string name = 1; optional tensorflow.DataType dtype = 2; - optional RepeatedPrimitiveValue expected = 3; + optional TestValue value = 3; }; +// NOTE: This definition must be kept in sync with PackedTestValue. message TestValue { - optional PrimitiveValue primitive_value = 1; - optional EnumValue enum_value = 2; - optional MessageValue message_value = 3; - optional RepeatedMessageValue repeated_message_value = 4; - optional RepeatedPrimitiveValue repeated_primitive_value = 6; -} - -message PrimitiveValue { - optional double double_value = 1; - optional float float_value = 2; - optional int64 int64_value = 3; - optional uint64 uint64_value = 4; - optional int32 int32_value = 5; - optional fixed64 fixed64_value = 6; - optional fixed32 fixed32_value = 7; - optional bool bool_value = 8; - optional string string_value = 9; - optional bytes bytes_value = 12; - optional uint32 uint32_value = 13; - optional sfixed32 sfixed32_value = 15; - optional sfixed64 sfixed64_value = 16; - optional sint32 sint32_value = 17; - optional sint64 sint64_value = 18; -} - -// NOTE: This definition must be kept in sync with PackedPrimitiveValue. -message RepeatedPrimitiveValue { repeated double double_value = 1; repeated float float_value = 2; repeated int64 int64_value = 3; @@ -74,30 +45,31 @@ message RepeatedPrimitiveValue { repeated PrimitiveValue message_value = 19; // Optional fields with explicitly-specified defaults. - optional double double_default = 20 [default = 1.0]; - optional float float_default = 21 [default = 2.0]; - optional int64 int64_default = 22 [default = 3]; - optional uint64 uint64_default = 23 [default = 4]; - optional int32 int32_default = 24 [default = 5]; - optional fixed64 fixed64_default = 25 [default = 6]; - optional fixed32 fixed32_default = 26 [default = 7]; - optional bool bool_default = 27 [default = true]; - optional string string_default = 28 [default = "a"]; - optional bytes bytes_default = 29 [default = "a longer default string"]; - optional uint32 uint32_default = 30 [default = 4294967295]; - optional sfixed32 sfixed32_default = 31 [default = 10]; - optional sfixed64 sfixed64_default = 32 [default = 11]; - optional sint32 sint32_default = 33 [default = 12]; - optional sint64 sint64_default = 34 [default = 13]; + optional double double_value_with_default = 20 [default = 1.0]; + optional float float_value_with_default = 21 [default = 2.0]; + optional int64 int64_value_with_default = 22 [default = 3]; + optional uint64 uint64_value_with_default = 23 [default = 4]; + optional int32 int32_value_with_default = 24 [default = 5]; + optional fixed64 fixed64_value_with_default = 25 [default = 6]; + optional fixed32 fixed32_value_with_default = 26 [default = 7]; + optional bool bool_value_with_default = 27 [default = true]; + optional string string_value_with_default = 28 [default = "a"]; + optional bytes bytes_value_with_default = 29 + [default = "a longer default string"]; + optional uint32 uint32_value_with_default = 30 [default = 9]; + optional sfixed32 sfixed32_value_with_default = 31 [default = 10]; + optional sfixed64 sfixed64_value_with_default = 32 [default = 11]; + optional sint32 sint32_value_with_default = 33 [default = 12]; + optional sint64 sint64_value_with_default = 34 [default = 13]; } -// A PackedPrimitiveValue looks exactly the same as a RepeatedPrimitiveValue -// in the text format, but the binary serializion is different. -// We test the packed representations by loading the same test cases -// using this definition instead of RepeatedPrimitiveValue. -// NOTE: This definition must be kept in sync with RepeatedPrimitiveValue -// in every way except the packed=true declaration. -message PackedPrimitiveValue { +// A PackedTestValue looks exactly the same as a TestValue in the text format, +// but the binary serializion is different. We test the packed representations +// by loading the same test cases using this definition instead of TestValue. +// +// NOTE: This definition must be kept in sync with TestValue in every way except +// the packed=true declaration. +message PackedTestValue { repeated double double_value = 1 [packed = true]; repeated float float_value = 2 [packed = true]; repeated int64 int64_value = 3 [packed = true]; @@ -115,23 +87,53 @@ message PackedPrimitiveValue { repeated sint64 sint64_value = 18 [packed = true]; repeated PrimitiveValue message_value = 19; - optional double double_default = 20 [default = 1.0]; - optional float float_default = 21 [default = 2.0]; - optional int64 int64_default = 22 [default = 3]; - optional uint64 uint64_default = 23 [default = 4]; - optional int32 int32_default = 24 [default = 5]; - optional fixed64 fixed64_default = 25 [default = 6]; - optional fixed32 fixed32_default = 26 [default = 7]; - optional bool bool_default = 27 [default = true]; - optional string string_default = 28 [default = "a"]; - optional bytes bytes_default = 29 [default = "a longer default string"]; - optional uint32 uint32_default = 30 [default = 4294967295]; - optional sfixed32 sfixed32_default = 31 [default = 10]; - optional sfixed64 sfixed64_default = 32 [default = 11]; - optional sint32 sint32_default = 33 [default = 12]; - optional sint64 sint64_default = 34 [default = 13]; + optional double double_value_with_default = 20 [default = 1.0]; + optional float float_value_with_default = 21 [default = 2.0]; + optional int64 int64_value_with_default = 22 [default = 3]; + optional uint64 uint64_value_with_default = 23 [default = 4]; + optional int32 int32_value_with_default = 24 [default = 5]; + optional fixed64 fixed64_value_with_default = 25 [default = 6]; + optional fixed32 fixed32_value_with_default = 26 [default = 7]; + optional bool bool_value_with_default = 27 [default = true]; + optional string string_value_with_default = 28 [default = "a"]; + optional bytes bytes_value_with_default = 29 + [default = "a longer default string"]; + optional uint32 uint32_value_with_default = 30 [default = 9]; + optional sfixed32 sfixed32_value_with_default = 31 [default = 10]; + optional sfixed64 sfixed64_value_with_default = 32 [default = 11]; + optional sint32 sint32_value_with_default = 33 [default = 12]; + optional sint64 sint64_value_with_default = 34 [default = 13]; } +message PrimitiveValue { + optional double double_value = 1; + optional float float_value = 2; + optional int64 int64_value = 3; + optional uint64 uint64_value = 4; + optional int32 int32_value = 5; + optional fixed64 fixed64_value = 6; + optional fixed32 fixed32_value = 7; + optional bool bool_value = 8; + optional string string_value = 9; + optional bytes bytes_value = 12; + optional uint32 uint32_value = 13; + optional sfixed32 sfixed32_value = 15; + optional sfixed64 sfixed64_value = 16; + optional sint32 sint32_value = 17; + optional sint64 sint64_value = 18; +} + +// Message containing fields with field numbers higher than any field above. +// An instance of this message is prepended to each binary message in the test +// to exercise the code path that handles fields encoded out of order of field +// number. +message ExtraFields { + optional string string_value = 1776; + optional bool bool_value = 1777; +} + +// The messages below are for yet-to-be created tests. + message EnumValue { enum Color { RED = 0; @@ -171,12 +173,3 @@ message RepeatedMessageValue { repeated NestedMessageValue message_values = 11; } - -// Message containing fields with field numbers higher than any field above. An -// instance of this message is prepended to each binary message in the test to -// exercise the code path that handles fields encoded out of order of field -// number. -message ExtraFields { - optional string string_value = 1776; - optional bool bool_value = 1777; -} -- cgit v1.2.3 From 6468c599d4b3b5c6fa94d171c1ba6b4254286c23 Mon Sep 17 00:00:00 2001 From: Alan Chiao Date: Thu, 12 Jul 2018 17:55:49 -0700 Subject: Internal change. PiperOrigin-RevId: 204398429 --- .../contrib/lite/kernels/embedding_lookup.cc | 3 +- .../contrib/lite/kernels/embedding_lookup_test.cc | 36 +++++++++++----------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/embedding_lookup.cc b/tensorflow/contrib/lite/kernels/embedding_lookup.cc index 0ba170a4da..f550339d03 100644 --- a/tensorflow/contrib/lite/kernels/embedding_lookup.cc +++ b/tensorflow/contrib/lite/kernels/embedding_lookup.cc @@ -112,8 +112,9 @@ TfLiteStatus EvalHybrid(TfLiteContext* context, TfLiteNode* node, // TODO(alanchiao): refactor scalar multiply into separate function // for ease of adding a neon equivalent if ever necessary. for (int j = 0; j < col_size; j++) { + const int8_t* value_ptr = reinterpret_cast(value->data.uint8); output->data.f[j + i * col_size] = - value->data.uint8[j + idx * col_size] * scaling_factor; + value_ptr[j + idx * col_size] * scaling_factor; } } } diff --git a/tensorflow/contrib/lite/kernels/embedding_lookup_test.cc b/tensorflow/contrib/lite/kernels/embedding_lookup_test.cc index 04657fd863..4a88d168c6 100644 --- a/tensorflow/contrib/lite/kernels/embedding_lookup_test.cc +++ b/tensorflow/contrib/lite/kernels/embedding_lookup_test.cc @@ -107,9 +107,9 @@ TEST(HybridEmbeddingLookupHybridOpTest, Simple2DTest) { HybridEmbeddingLookupOpModel m({3}, {3, 8}); m.SetInput({1, 0, 2}); m.SetWeight({ - 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 - 1.00, 1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 - 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 + 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 + 1.00, -1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 + 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 }); m.Invoke(); @@ -117,9 +117,9 @@ TEST(HybridEmbeddingLookupHybridOpTest, Simple2DTest) { EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear( { - 1.00, 1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 - 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 - 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 + 1.00, -1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 + 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 + 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 }, 7.41e-03))); } @@ -128,9 +128,9 @@ TEST(HybridEmbeddingLookupHybridOpTest, Simple3DTest) { HybridEmbeddingLookupOpModel m({3}, {3, 2, 4}); m.SetInput({1, 0, 2}); m.SetWeight({ - 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 - 1.00, 1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 - 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 + 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 + 1.00, -1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 + 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 }); m.Invoke(); @@ -138,9 +138,9 @@ TEST(HybridEmbeddingLookupHybridOpTest, Simple3DTest) { EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear( { - 1.00, 1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 - 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 - 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 + 1.00, -1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 + 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 + 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 }, 7.41e-03))); } @@ -149,9 +149,9 @@ TEST(HybridEmbeddingLookupHybridOpTest, Simple4DTest) { HybridEmbeddingLookupOpModel m({3}, {3, 2, 2, 2}); m.SetInput({1, 0, 2}); m.SetWeight({ - 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 - 1.00, 1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 - 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 + 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 + 1.00, -1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 + 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 }); m.Invoke(); @@ -159,9 +159,9 @@ TEST(HybridEmbeddingLookupHybridOpTest, Simple4DTest) { EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear( { - 1.00, 1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 - 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 - 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 + 1.00, -1.01, 1.02, 1.03, 1.10, 1.11, 1.12, 1.13, // Row 1 + 0.00, 0.01, 0.02, 0.03, 0.10, 0.11, 0.12, 0.13, // Row 0 + 2.00, 2.01, 2.02, 2.03, 2.10, 2.11, 2.12, 2.13, // Row 2 }, 7.41e-03))); } -- cgit v1.2.3 From 445d26cf708b8e430549997573633a4dd1dc18fe Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Thu, 12 Jul 2018 18:06:11 -0700 Subject: Makes tfe.defun respect random seeds. PiperOrigin-RevId: 204399757 --- tensorflow/python/eager/function.py | 4 ++++ tensorflow/python/eager/function_test.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index a34a6fc053..a6906f9efd 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -669,6 +669,10 @@ def _trace_and_define_function(name, func, compiled, args, kwds): for collection in curr_graph.collections: tmp_graph.get_collection_ref(collection)[:] = curr_graph.get_collection( collection) + if context.executing_eagerly(): + tmp_graph.seed = context.global_seed() + else: + tmp_graph.seed = curr_graph.seed with tmp_graph.as_default(), AutomaticControlDependencies() as a: func_args = _get_defun_inputs(args) func_kwds = _get_defun_inputs(kwds) diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index a3e63c3153..13c4ee7f15 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -30,6 +30,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import function as tf_function from tensorflow.python.framework import ops +from tensorflow.python.framework import random_seed from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import test_util from tensorflow.python.layers import convolutional @@ -39,6 +40,7 @@ from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import random_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables @@ -136,6 +138,18 @@ class FunctionTest(test.TestCase): out = sq_op(t) self.assertAllEqual(out, math_ops.matmul(t, t).numpy()) + def testRandomSeed(self): + + @function.defun + def f(): + return random_ops.random_normal(()) + + random_seed.set_random_seed(1) + x = f() + self.assertNotEqual(x, f()) + random_seed.set_random_seed(1) + self.assertAllEqual(f(), x) + def testNestedInputsDefunOpGraphMode(self): matmul = function.defun(math_ops.matmul) -- cgit v1.2.3 From 7fe0ae12ec42eca1ea07d93bbd63de394743a018 Mon Sep 17 00:00:00 2001 From: Pradeep Banavara Date: Thu, 12 Jul 2018 21:52:13 -0400 Subject: Fix: #12686 SoftmaxCrossEntropyWithLogits Committing in a new PR as the old PR has too many commit files --- tensorflow/cc/gradients/nn_grad.cc | 94 +++++++++++++++++++++++++++++---- tensorflow/cc/gradients/nn_grad_test.cc | 29 ++++++++-- 2 files changed, 109 insertions(+), 14 deletions(-) diff --git a/tensorflow/cc/gradients/nn_grad.cc b/tensorflow/cc/gradients/nn_grad.cc index c73482d5f4..dc6477e59d 100644 --- a/tensorflow/cc/gradients/nn_grad.cc +++ b/tensorflow/cc/gradients/nn_grad.cc @@ -47,6 +47,81 @@ Status SoftmaxGrad(const Scope& scope, const Operation& op, } REGISTER_GRADIENT_OP("Softmax", SoftmaxGrad); +bool IsZero(const Scope& scope, Output grad) { + std::array zero_op_type_names{{"ZerosLike", "Zeros"}}; + string op_type_name = grad.op().node()->type_string(); + for (auto& zero_op_type_name: zero_op_type_names) { + if (op_type_name == zero_op_type_name) { + return true; + } + } + // the Operation we were provided is not named something obvious + // we need to actually look at its contents. + // the original python code did this by calling a utility function called + // tensor_util.constant_value. When you dig into tensor_tuil.constant_value + // it is a large number of 'if' statements that measure certain edge cases + // where it is possible to get the value of the tensor without actually + // evaluating it. There are many kinds of tensors that can not have this + // done. + // There is no C++ equivalent to tensor_util.constant_value so we do nothing + // for the moment. + return false; +} + +Output BroadcastMul(const Scope& scope, Output vec, Output mat) { + /* Multiply after broadcasting vec to match dimensions of mat. + Args: + vec: A 1-D tensor of dimension [D0] + mat: A 2-D tensor of dimesnion [D0, D1] + + Returns: + A tensor of dimension [D0, D1], the result fo vec * mat + we use an element for element multiply here. + */ + auto reshaped = ExpandDims(scope, vec, -1); + return Multiply(scope, reshaped, mat); +} + +Status SoftmaxCrossEntropyWithLogitsGrad(const Scope& scope, + const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + // Softmax gradient with cross entropy logits function + // We multiply the backprop for cost with the gradients - op.output[1] + // There is no gradient for labels + auto logits = + op.input(0); // the outputs of the network are at + // input index 0. The "truth" labels are at index 1. + auto softmax_grad = op.output(1); + + // The documentation for ops::SoftmaxCrossEntropyWithLogits says + // loss is the output at index 0, and backprop is the output at index 1 + auto grad_loss = grad_inputs[0]; + auto grad_grad = grad_inputs[1]; + + auto grad = BroadcastMul(scope, grad_loss, softmax_grad); + if (!IsZero(scope, grad_grad)) { + std::vector axis; + auto logitsSoftmax = Softmax(scope, logits); + + auto grad_gradExpand = ExpandDims(scope, grad_grad, 1); + auto logitsSoftMaxExpand = ExpandDims(scope, logitsSoftmax, 2); + auto matMulResult = + BatchMatMul(scope, grad_gradExpand, logitsSoftMaxExpand); + axis.push_back(1); + auto squeezeResult = Squeeze(scope, matMulResult, Squeeze::Axis(axis)); + auto subtractionResult = Subtract(scope, grad_grad, squeezeResult); + auto multiplyResult = Multiply(scope, subtractionResult, logitsSoftmax); + grad = Add(scope, grad, multiplyResult); + } + auto minusLogSoftmax = Multiply(scope, LogSoftmax(scope, logits), -1.0f); + grad_outputs->push_back(grad); + grad_outputs->push_back(BroadcastMul(scope, grad_loss, minusLogSoftmax)); + return scope.status(); +} +REGISTER_GRADIENT_OP("SoftmaxCrossEntropyWithLogits", + SoftmaxCrossEntropyWithLogitsGrad); + Status LogSoftmaxGrad(const Scope& scope, const Operation& op, const std::vector& grad_inputs, std::vector* grad_outputs) { @@ -195,9 +270,9 @@ Status MaxPool3DGradHelper(const Scope& scope, const Operation& op, TF_RETURN_IF_ERROR(GetNodeAttr(attrs, "padding", &padding)); TF_RETURN_IF_ERROR(GetNodeAttr(attrs, "data_format", &data_format)); MaxPool3DGrad::Attrs grad_attrs; - auto dx = MaxPool3DGrad(scope, op.input(0), op.output(0), grad_inputs[0], - ksize, strides, padding, - grad_attrs.DataFormat(data_format)); + auto dx = + MaxPool3DGrad(scope, op.input(0), op.output(0), grad_inputs[0], ksize, + strides, padding, grad_attrs.DataFormat(data_format)); grad_outputs->push_back(dx); return scope.status(); } @@ -216,10 +291,9 @@ Status AvgPoolGradHelper(const Scope& scope, const Operation& op, TF_RETURN_IF_ERROR(GetNodeAttr(attrs, "padding", &padding)); TF_RETURN_IF_ERROR(GetNodeAttr(attrs, "data_format", &data_format)); internal::AvgPoolGrad::Attrs grad_attrs; - auto dx = - internal::AvgPoolGrad(scope, Shape(scope, op.input(0)), grad_inputs[0], - ksize, strides, padding, - grad_attrs.DataFormat(data_format)); + auto dx = internal::AvgPoolGrad(scope, Shape(scope, op.input(0)), + grad_inputs[0], ksize, strides, padding, + grad_attrs.DataFormat(data_format)); grad_outputs->push_back(dx); return scope.status(); } @@ -238,9 +312,9 @@ Status AvgPool3DGradHelper(const Scope& scope, const Operation& op, TF_RETURN_IF_ERROR(GetNodeAttr(attrs, "padding", &padding)); TF_RETURN_IF_ERROR(GetNodeAttr(attrs, "data_format", &data_format)); AvgPool3DGrad::Attrs grad_attrs; - auto dx = AvgPool3DGrad(scope, Shape(scope, op.input(0)), grad_inputs[0], - ksize, strides, padding, - grad_attrs.DataFormat(data_format)); + auto dx = + AvgPool3DGrad(scope, Shape(scope, op.input(0)), grad_inputs[0], ksize, + strides, padding, grad_attrs.DataFormat(data_format)); grad_outputs->push_back(dx); return scope.status(); } diff --git a/tensorflow/cc/gradients/nn_grad_test.cc b/tensorflow/cc/gradients/nn_grad_test.cc index b4d457a9d1..f26a7e99e6 100644 --- a/tensorflow/cc/gradients/nn_grad_test.cc +++ b/tensorflow/cc/gradients/nn_grad_test.cc @@ -25,6 +25,8 @@ limitations under the License. namespace tensorflow { namespace { +using ops::AvgPool; +using ops::AvgPool3D; using ops::BiasAdd; using ops::Conv2D; using ops::Elu; @@ -33,11 +35,9 @@ using ops::FractionalMaxPool; using ops::L2Loss; using ops::LogSoftmax; using ops::LRN; -using ops::AvgPool; -using ops::AvgPool3D; using ops::MaxPool; -using ops::MaxPoolV2; using ops::MaxPool3D; +using ops::MaxPoolV2; using ops::Placeholder; using ops::Relu; using ops::Relu6; @@ -111,6 +111,27 @@ TEST_F(NNGradTest, SoftmaxGrad) { RunTest(x, shape, y, shape); } +TEST_F(NNGradTest, SoftmaxCrossEntropyWithLogitsGrad) { + TensorShape logitsShape( + {5, 3}); // batch size of 5,3 possible labels (classes), + // logits is what is produced by a network + // they are compared to labels which are the truth + TensorShape lossShape( + {5}); // batch size of 5, 1 value for each entry in the batch + // loss is the difference between logits and labels + + auto logits = Placeholder(scope_, DT_FLOAT, + Placeholder::Shape(logitsShape)); // estimation + auto labels = + Placeholder(scope_, DT_FLOAT, Placeholder::Shape(logitsShape)); // truth + auto y = + tensorflow::ops::SoftmaxCrossEntropyWithLogits(scope_, logits, labels); + // Please note the reversal of the backprop and loss orders. A separate issue + // #18734 has been opened for this. + RunTest({logits, labels}, {logitsShape, logitsShape}, {y.backprop, y.loss}, + {logitsShape, lossShape}); +} + TEST_F(NNGradTest, LogSoftmaxGrad) { TensorShape shape({5, 3}); auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); @@ -253,7 +274,7 @@ TEST_F(NNGradTest, AvgPool3DGradHelper) { RunTest(x, x_shape, y, y_shape); } -TEST_F(NNGradTest, LRN){ +TEST_F(NNGradTest, LRN) { TensorShape x_shape({1, 1, 2, 1}); auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); auto y = LRN(scope_, x); -- cgit v1.2.3 From b4654e6c936f03b058d361aa655ece84969742c4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 19:17:49 -0700 Subject: Replacing persistent tape with 2 different tapes for generator and discriminator. PiperOrigin-RevId: 204407120 --- tensorflow/contrib/eager/python/examples/gan/mnist.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/eager/python/examples/gan/mnist.py b/tensorflow/contrib/eager/python/examples/gan/mnist.py index cc9cf53410..b33243021b 100644 --- a/tensorflow/contrib/eager/python/examples/gan/mnist.py +++ b/tensorflow/contrib/eager/python/examples/gan/mnist.py @@ -214,7 +214,7 @@ def train_one_epoch(generator, discriminator, generator_optimizer, total_generator_loss = 0.0 total_discriminator_loss = 0.0 - for (batch_index, images) in enumerate(tfe.Iterator(dataset)): + for (batch_index, images) in enumerate(dataset): with tf.device('/cpu:0'): tf.assign_add(step_counter, 1) @@ -227,7 +227,10 @@ def train_one_epoch(generator, discriminator, generator_optimizer, maxval=1., seed=batch_index) - with tf.GradientTape(persistent=True) as g: + # we can use 2 tapes or a single persistent tape. + # Using two tapes is memory efficient since intermediate tensors can be + # released between the two .gradient() calls below + with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape: generated_images = generator(noise) tf.contrib.summary.image( 'generated_images', @@ -243,9 +246,10 @@ def train_one_epoch(generator, discriminator, generator_optimizer, generator_loss_val = generator_loss(discriminator_gen_outputs) total_generator_loss += generator_loss_val - generator_grad = g.gradient(generator_loss_val, generator.variables) - discriminator_grad = g.gradient(discriminator_loss_val, - discriminator.variables) + generator_grad = gen_tape.gradient(generator_loss_val, + generator.variables) + discriminator_grad = disc_tape.gradient(discriminator_loss_val, + discriminator.variables) generator_optimizer.apply_gradients( zip(generator_grad, generator.variables)) -- cgit v1.2.3 From 59bfc71a71df63c2859e17179a0d375e4bb96fb0 Mon Sep 17 00:00:00 2001 From: Smit Hinsu Date: Thu, 12 Jul 2018 19:45:23 -0700 Subject: Include NCCL in the list of requirements for TensorFlow with GPU support PiperOrigin-RevId: 204409167 --- tensorflow/docs_src/install/install_linux.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index f21c073a1b..541a55e184 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -511,6 +511,8 @@ on your system: list of supported GPU cards. * [GPU drivers](http://nvidia.com/drivers) that support your version of the CUDA Toolkit. +* NCCL 2.2 to use TensorFlow with multiple GPUs. For details, see [NVIDIA's + documentation](https://developer.nvidia.com/nccl). * The `libcupti-dev` library is the NVIDIA CUDA Profile Tools Interface. This library provides advanced profiling support. To install this library, use the following command for CUDA Toolkit >= 8.0: -- cgit v1.2.3 From 385bb78761489cfa6d6808f239fe884152e71653 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 20:18:46 -0700 Subject: Expose is_png similar to the method is_jpeg. PiperOrigin-RevId: 204411827 --- tensorflow/python/ops/image_ops_impl.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index 5b384fd596..9440bab9ee 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -1753,6 +1753,22 @@ def is_jpeg(contents, name=None): return math_ops.equal(substr, b'\xff\xd8\xff', name=name) +def _is_png(contents, name=None): + r"""Convenience function to check if the 'contents' encodes a PNG image. + + Args: + contents: 0-D `string`. The encoded image bytes. + name: A name for the operation (optional) + + Returns: + A scalar boolean tensor indicating if 'contents' may be a PNG image. + is_png is susceptible to false positives. + """ + with ops.name_scope(name, 'is_png'): + substr = string_ops.substr(contents, 0, 3) + return math_ops.equal(substr, b'\211PN', name=name) + + @tf_export('image.decode_image') def decode_image(contents, channels=None, dtype=dtypes.uint8, name=None): """Convenience function for `decode_bmp`, `decode_gif`, `decode_jpeg`, @@ -1830,8 +1846,8 @@ def decode_image(contents, channels=None, dtype=dtypes.uint8, name=None): def check_png(): """Checks if an image is PNG.""" - is_png = math_ops.equal(substr, b'\211PN', name='is_png') - return control_flow_ops.cond(is_png, _png, check_gif, name='cond_png') + return control_flow_ops.cond( + _is_png(contents), _png, check_gif, name='cond_png') def _jpeg(): """Decodes a jpeg image.""" -- cgit v1.2.3 From 4910a2bb53cda036ea78b3eba90c6f30a1fa67da Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Wed, 30 May 2018 16:14:42 +0800 Subject: third_party/repo: add TF_SYSTEM_LIBS to unbundle deps Signed-off-by: Jason Zaman --- third_party/repo.bzl | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/third_party/repo.bzl b/third_party/repo.bzl index 9cee1fcc4b..fb33c389ac 100644 --- a/third_party/repo.bzl +++ b/third_party/repo.bzl @@ -35,6 +35,15 @@ def _get_env_var(ctx, name): else: return None +# Checks if we should use the system lib instead of the bundled one +def _use_system_lib(ctx, name): + syslibenv = _get_env_var(ctx, "TF_SYSTEM_LIBS") + if syslibenv: + for n in syslibenv.strip().split(","): + if n.strip() == name: + return True + return False + # Executes specified command with arguments and calls 'fail' if it exited with # non-zero code def _execute_and_check_ret_code(repo_ctx, cmd_and_args): @@ -75,17 +84,28 @@ def _tf_http_archive(ctx): "Even if you don't have permission to mirror the file, please " + "put the correctly formatted mirror URL there anyway, because " + "someone will come along shortly thereafter and mirror the file.") - ctx.download_and_extract( - ctx.attr.urls, - "", - ctx.attr.sha256, - ctx.attr.type, - ctx.attr.strip_prefix) - if ctx.attr.delete: - _apply_delete(ctx, ctx.attr.delete) - if ctx.attr.patch_file != None: - _apply_patch(ctx, ctx.attr.patch_file) - if ctx.attr.build_file != None: + + use_syslib = _use_system_lib(ctx, ctx.attr.name) + if not use_syslib: + ctx.download_and_extract( + ctx.attr.urls, + "", + ctx.attr.sha256, + ctx.attr.type, + ctx.attr.strip_prefix) + if ctx.attr.delete: + _apply_delete(ctx, ctx.attr.delete) + if ctx.attr.patch_file != None: + _apply_patch(ctx, ctx.attr.patch_file) + + if use_syslib and ctx.attr.system_build_file != None: + # Use BUILD.bazel to avoid conflict with third party projects with + # BUILD or build (directory) underneath. + ctx.template("BUILD.bazel", ctx.attr.system_build_file, { + "%prefix%": ".." if _repos_are_siblings() else "external", + }, False) + + elif ctx.attr.build_file != None: # Use BUILD.bazel to avoid conflict with third party projects with # BUILD or build (directory) underneath. ctx.template("BUILD.bazel", ctx.attr.build_file, { @@ -102,6 +122,7 @@ tf_http_archive = repository_rule( "delete": attr.string_list(), "patch_file": attr.label(), "build_file": attr.label(), + "system_build_file": attr.label(), }) """Downloads and creates Bazel repos for dependencies. -- cgit v1.2.3 From e30f97697141255f52a62d191cb5ff81167fa1cf Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Wed, 30 May 2018 17:39:53 +0800 Subject: workspace: use TF_SYSTEM_LIBS to link to system libraries Signed-off-by: Jason Zaman --- tensorflow/workspace.bzl | 17 ++++++++++ third_party/systemlibs/BUILD | 0 third_party/systemlibs/curl.BUILD | 12 +++++++ third_party/systemlibs/flatbuffers.BUILD | 38 ++++++++++++++++++++++ third_party/systemlibs/gif.BUILD | 12 +++++++ third_party/systemlibs/grpc.BUILD | 54 ++++++++++++++++++++++++++++++++ third_party/systemlibs/jemalloc.BUILD | 30 ++++++++++++++++++ third_party/systemlibs/jpeg.BUILD | 12 +++++++ third_party/systemlibs/lmdb.BUILD | 12 +++++++ third_party/systemlibs/nasm.BUILD | 12 +++++++ third_party/systemlibs/pcre.BUILD | 12 +++++++ third_party/systemlibs/png.BUILD | 12 +++++++ third_party/systemlibs/re2.BUILD | 12 +++++++ third_party/systemlibs/six.BUILD | 11 +++++++ third_party/systemlibs/snappy.BUILD | 12 +++++++ third_party/systemlibs/sqlite.BUILD | 15 +++++++++ third_party/systemlibs/swig.BUILD | 23 ++++++++++++++ third_party/systemlibs/termcolor.BUILD | 12 +++++++ third_party/systemlibs/zlib.BUILD | 12 +++++++ 19 files changed, 320 insertions(+) create mode 100644 third_party/systemlibs/BUILD create mode 100644 third_party/systemlibs/curl.BUILD create mode 100644 third_party/systemlibs/flatbuffers.BUILD create mode 100644 third_party/systemlibs/gif.BUILD create mode 100644 third_party/systemlibs/grpc.BUILD create mode 100644 third_party/systemlibs/jemalloc.BUILD create mode 100644 third_party/systemlibs/jpeg.BUILD create mode 100644 third_party/systemlibs/lmdb.BUILD create mode 100644 third_party/systemlibs/nasm.BUILD create mode 100644 third_party/systemlibs/pcre.BUILD create mode 100644 third_party/systemlibs/png.BUILD create mode 100644 third_party/systemlibs/re2.BUILD create mode 100644 third_party/systemlibs/six.BUILD create mode 100644 third_party/systemlibs/snappy.BUILD create mode 100644 third_party/systemlibs/sqlite.BUILD create mode 100644 third_party/systemlibs/swig.BUILD create mode 100644 third_party/systemlibs/termcolor.BUILD create mode 100644 third_party/systemlibs/zlib.BUILD diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index cd4f17a5ff..4b8652b93c 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -161,6 +161,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): ], sha256 = "2f945446b71336e7f5a2bcace1abcf0b23fbba368266c6a1be33de3de3b3c912", strip_prefix = "re2-2018-04-01", + system_build_file = clean_dep("//third_party/systemlibs:re2.BUILD"), ) tf_http_archive( @@ -226,6 +227,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "63ec86477ad3f0f6292325fd89e1d93aea2e2fd490070863f17d48f7cd387011", strip_prefix = "nasm-2.13.03", build_file = clean_dep("//third_party:nasm.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:nasm.BUILD"), ) tf_http_archive( @@ -237,6 +239,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "1a17020f859cb12711175a67eab5c71fc1904e04b587046218e36106e07eabde", strip_prefix = "libjpeg-turbo-1.5.3", build_file = clean_dep("//third_party/jpeg:jpeg.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:jpeg.BUILD"), ) tf_http_archive( @@ -249,6 +252,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): strip_prefix = "libpng-1.6.34", build_file = clean_dep("//third_party:png.BUILD"), patch_file = clean_dep("//third_party:png_fix_rpi.patch"), + system_build_file = clean_dep("//third_party/systemlibs:png.BUILD"), ) tf_http_archive( @@ -260,6 +264,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "ad68c1216c3a474cf360c7581a4001e952515b3649342100f2d7ca7c8e313da6", strip_prefix = "sqlite-amalgamation-3240000", build_file = clean_dep("//third_party:sqlite.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:sqlite.BUILD"), ) tf_http_archive( @@ -271,6 +276,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1", strip_prefix = "giflib-5.1.4", build_file = clean_dep("//third_party:gif.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:gif.BUILD"), ) tf_http_archive( @@ -282,6 +288,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", strip_prefix = "six-1.10.0", build_file = clean_dep("//third_party:six.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:six.BUILD"), ) tf_http_archive( @@ -315,6 +322,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b", strip_prefix = "termcolor-1.1.0", build_file = clean_dep("//third_party:termcolor.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:termcolor.BUILD"), ) tf_http_archive( @@ -421,6 +429,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): ], strip_prefix = "pcre-8.42", build_file = clean_dep("//third_party:pcre.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:pcre.BUILD"), ) tf_http_archive( @@ -433,6 +442,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): ], strip_prefix = "swig-3.0.8", build_file = clean_dep("//third_party:swig.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:swig.BUILD"), ) tf_http_archive( @@ -444,6 +454,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): ], strip_prefix = "curl-7.60.0", build_file = clean_dep("//third_party:curl.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:curl.BUILD"), ) tf_http_archive( @@ -454,6 +465,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): ], sha256 = "50db9cf2221354485eb7c3bd55a4c27190caef7048a2a1a15fbe60a498f98b44", strip_prefix = "grpc-1.13.0", + system_build_file = clean_dep("//third_party/systemlibs:grpc.BUILD"), ) tf_http_archive( @@ -489,6 +501,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "f3927859882eb608868c8c31586bb7eb84562a40a6bf5cc3e13b6b564641ea28", strip_prefix = "lmdb-LMDB_0.9.22/libraries/liblmdb", build_file = clean_dep("//third_party:lmdb.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:lmdb.BUILD"), ) tf_http_archive( @@ -521,6 +534,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", strip_prefix = "zlib-1.2.11", build_file = clean_dep("//third_party:zlib.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:zlib.BUILD"), ) tf_http_archive( @@ -542,6 +556,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "3dfa02e873ff51a11ee02b9ca391807f0c8ea0529a4924afa645fbf97163f9d4", strip_prefix = "snappy-1.1.7", build_file = clean_dep("//third_party:snappy.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:snappy.BUILD"), ) tf_http_archive( @@ -612,6 +627,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "3c8f25c02e806c3ce0ab5fb7da1817f89fc9732709024e2a81b6b82f7cc792a8", strip_prefix = "jemalloc-4.4.0", build_file = clean_dep("//third_party:jemalloc.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:jemalloc.BUILD"), ) java_import_external( @@ -722,6 +738,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): "https://github.com/google/flatbuffers/archive/v1.9.0.tar.gz", ], build_file = clean_dep("//third_party/flatbuffers:flatbuffers.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:flatbuffers.BUILD"), ) native.new_http_archive( diff --git a/third_party/systemlibs/BUILD b/third_party/systemlibs/BUILD new file mode 100644 index 0000000000..e69de29bb2 diff --git a/third_party/systemlibs/curl.BUILD b/third_party/systemlibs/curl.BUILD new file mode 100644 index 0000000000..c5f125caa9 --- /dev/null +++ b/third_party/systemlibs/curl.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # MIT/X derivative license + +filegroup( + name = "COPYING", + visibility = ["//visibility:public"], +) + +cc_library( + name = "curl", + linkopts = ["-lcurl"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/flatbuffers.BUILD b/third_party/systemlibs/flatbuffers.BUILD new file mode 100644 index 0000000000..14fceada82 --- /dev/null +++ b/third_party/systemlibs/flatbuffers.BUILD @@ -0,0 +1,38 @@ +licenses(["notice"]) # Apache 2.0 + +filegroup( + name = "LICENSE.txt", + visibility = ["//visibility:public"], +) + +# Public flatc library to compile flatbuffer files at runtime. +cc_library( + name = "flatbuffers", + linkopts = ["-lflatbuffers"], + visibility = ["//visibility:public"], +) + +# Public flatc compiler library. +cc_library( + name = "flatc_library", + linkopts = ["-lflatbuffers"], + visibility = ["//visibility:public"], +) + +genrule( + name = "lnflatc", + outs = ["flatc.bin"], + cmd = "ln -s $$(which flatc) $@", +) + +# Public flatc compiler. +sh_binary( + name = "flatc", + srcs = ["flatc.bin"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "runtime_cc", + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/gif.BUILD b/third_party/systemlibs/gif.BUILD new file mode 100644 index 0000000000..5eb2c918ba --- /dev/null +++ b/third_party/systemlibs/gif.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # MIT + +filegroup( + name = "COPYING", + visibility = ["//visibility:public"], +) + +cc_library( + name = "gif", + linkopts = ["-lgif"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/grpc.BUILD b/third_party/systemlibs/grpc.BUILD new file mode 100644 index 0000000000..fd90eb0dd3 --- /dev/null +++ b/third_party/systemlibs/grpc.BUILD @@ -0,0 +1,54 @@ +licenses(["notice"]) # Apache v2 + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc", + linkopts = ["-lgrpc"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc++", + linkopts = ["-lgrpc++"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc_unsecure", + linkopts = ["-lgrpc_unsecure"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc++_unsecure", + linkopts = ["-lgrpc++_unsecure"], + visibility = ["//visibility:public"], +) + +genrule( + name = "ln_grpc_cpp_plugin", + outs = ["grpc_cpp_plugin.bin"], + cmd = "ln -s $$(which grpc_cpp_plugin) $@", +) + +sh_binary( + name = "grpc_cpp_plugin", + srcs = ["grpc_cpp_plugin.bin"], + visibility = ["//visibility:public"], +) + +genrule( + name = "ln_grpc_python_plugin", + outs = ["grpc_python_plugin.bin"], + cmd = "ln -s $$(which grpc_python_plugin) $@", +) + +sh_binary( + name = "grpc_python_plugin", + srcs = ["grpc_python_plugin.bin"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/jemalloc.BUILD b/third_party/systemlibs/jemalloc.BUILD new file mode 100644 index 0000000000..6a48d582ba --- /dev/null +++ b/third_party/systemlibs/jemalloc.BUILD @@ -0,0 +1,30 @@ +licenses(["notice"]) # BSD + +filegroup( + name = "COPYING", + visibility = ["//visibility:public"], +) + +cc_library( + name = "jemalloc_headers", + defines = [ + "jemalloc_posix_memalign=posix_memalign", + "jemalloc_malloc=malloc", + "jemalloc_realloc=realloc", + "jemalloc_free=free", + ], + visibility = ["//visibility:public"], +) + +cc_library( + name = "jemalloc_impl", + linkopts = ["-ljemalloc"], + defines = [ + "jemalloc_posix_memalign=posix_memalign", + "jemalloc_malloc=malloc", + "jemalloc_realloc=realloc", + "jemalloc_free=free", + ], + visibility = ["//visibility:public"], + deps = [":jemalloc_headers"], +) diff --git a/third_party/systemlibs/jpeg.BUILD b/third_party/systemlibs/jpeg.BUILD new file mode 100644 index 0000000000..f4f52da9bd --- /dev/null +++ b/third_party/systemlibs/jpeg.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # custom notice-style license, see LICENSE.md + +filegroup( + name = "LICENSE.md", + visibility = ["//visibility:public"], +) + +cc_library( + name = "jpeg", + linkopts = ["-ljpeg"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/lmdb.BUILD b/third_party/systemlibs/lmdb.BUILD new file mode 100644 index 0000000000..6177b095ec --- /dev/null +++ b/third_party/systemlibs/lmdb.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # OpenLDAP Public License + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +cc_library( + name = "lmdb", + linkopts = ["-llmdb"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/nasm.BUILD b/third_party/systemlibs/nasm.BUILD new file mode 100644 index 0000000000..10ef8d8832 --- /dev/null +++ b/third_party/systemlibs/nasm.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # BSD 2-clause + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +sh_binary( + name = "nasm", + srcs = ["nasm"], + visibility = ["@jpeg//:__pkg__"], +) diff --git a/third_party/systemlibs/pcre.BUILD b/third_party/systemlibs/pcre.BUILD new file mode 100644 index 0000000000..df74238847 --- /dev/null +++ b/third_party/systemlibs/pcre.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # BSD + +filegroup( + name = "LICENCE", + visibility = ["//visibility:public"], +) + +cc_library( + name = "pcre", + linkopts = ["-lpcre"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/png.BUILD b/third_party/systemlibs/png.BUILD new file mode 100644 index 0000000000..fc6b6f2d8b --- /dev/null +++ b/third_party/systemlibs/png.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # BSD/MIT-like license + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +cc_library( + name = "png", + linkopts = ["-lpng"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/re2.BUILD b/third_party/systemlibs/re2.BUILD new file mode 100644 index 0000000000..c18e252dbc --- /dev/null +++ b/third_party/systemlibs/re2.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # BSD/MIT-like license + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +cc_library( + name = "re2", + linkopts = ["-lre2"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/six.BUILD b/third_party/systemlibs/six.BUILD new file mode 100644 index 0000000000..ff9b1a540b --- /dev/null +++ b/third_party/systemlibs/six.BUILD @@ -0,0 +1,11 @@ +licenses(["notice"]) # MIT + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +py_library( + name = "six", + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/snappy.BUILD b/third_party/systemlibs/snappy.BUILD new file mode 100644 index 0000000000..fd2db9e2df --- /dev/null +++ b/third_party/systemlibs/snappy.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # BSD 3-Clause + +filegroup( + name = "COPYING", + visibility = ["//visibility:public"], +) + +cc_library( + name = "snappy", + linkopts = ["-lsnappy"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/sqlite.BUILD b/third_party/systemlibs/sqlite.BUILD new file mode 100644 index 0000000000..20ee1ebbef --- /dev/null +++ b/third_party/systemlibs/sqlite.BUILD @@ -0,0 +1,15 @@ +licenses(["unencumbered"]) # Public Domain + +# Production build of SQLite library that's baked into TensorFlow. +cc_library( + name = "org_sqlite", + linkopts = ["-lsqlite3"], + visibility = ["//visibility:public"], +) + +# This is a Copybara sync helper for Google. +py_library( + name = "python", + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/swig.BUILD b/third_party/systemlibs/swig.BUILD new file mode 100644 index 0000000000..4c9b74dadb --- /dev/null +++ b/third_party/systemlibs/swig.BUILD @@ -0,0 +1,23 @@ +licenses(["restricted"]) # GPLv3 + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +filegroup( + name = "templates", + visibility = ["//visibility:public"], +) + +genrule( + name = "lnswiglink", + outs = ["swiglink"], + cmd = "ln -s $$(which swig) $@", +) + +sh_binary( + name = "swig", + srcs = ["swiglink"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/termcolor.BUILD b/third_party/systemlibs/termcolor.BUILD new file mode 100644 index 0000000000..915eb621d5 --- /dev/null +++ b/third_party/systemlibs/termcolor.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # MIT + +filegroup( + name = "COPYING.txt", + visibility = ["//visibility:public"], +) + +py_library( + name = "termcolor", + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/zlib.BUILD b/third_party/systemlibs/zlib.BUILD new file mode 100644 index 0000000000..69462ae6cb --- /dev/null +++ b/third_party/systemlibs/zlib.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # BSD/MIT-like license (for zlib) + +filegroup( + name = "zlib.h", + visibility = ["//visibility:public"], +) + +cc_library( + name = "zlib", + linkopts = ["-lz"], + visibility = ["//visibility:public"], +) -- cgit v1.2.3 From 53c7c66b9afe42b3180bc6a9bbdeac2d92e12d1d Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Wed, 30 May 2018 21:40:45 +0800 Subject: third_party: Add astor systemlib Signed-off-by: Jason Zaman --- tensorflow/workspace.bzl | 1 + third_party/systemlibs/astor.BUILD | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 third_party/systemlibs/astor.BUILD diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 4b8652b93c..ca2fcefdac 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -300,6 +300,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "ff6d2e2962d834acb125cc4dcc80c54a8c17c253f4cc9d9c43b5102a560bb75d", strip_prefix = "astor-0.6.2", build_file = clean_dep("//third_party:astor.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:astor.BUILD"), ) tf_http_archive( diff --git a/third_party/systemlibs/astor.BUILD b/third_party/systemlibs/astor.BUILD new file mode 100644 index 0000000000..497ec4bcea --- /dev/null +++ b/third_party/systemlibs/astor.BUILD @@ -0,0 +1,12 @@ +licenses(["notice"]) # New BSD + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +py_library( + name = "astor", + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) -- cgit v1.2.3 From 17b1f8aa2949aa21c55cff91f0c1e793d7325079 Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Thu, 31 May 2018 14:37:28 +0800 Subject: third_party: Add cython system dep Signed-off-by: Jason Zaman --- tensorflow/workspace.bzl | 1 + third_party/systemlibs/cython.BUILD | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 third_party/systemlibs/cython.BUILD diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index ca2fcefdac..33643c1ca2 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -707,6 +707,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): strip_prefix = "cython-0.28.4", build_file = clean_dep("//third_party:cython.BUILD"), delete = ["BUILD.bazel"], + system_build_file = clean_dep("//third_party/systemlibs:cython.BUILD"), ) tf_http_archive( diff --git a/third_party/systemlibs/cython.BUILD b/third_party/systemlibs/cython.BUILD new file mode 100644 index 0000000000..1d52587676 --- /dev/null +++ b/third_party/systemlibs/cython.BUILD @@ -0,0 +1,13 @@ +licenses(["notice"]) # Apache-2.0 + +genrule( + name = "lncython", + outs = ["cython"], + cmd = "ln -s $$(which cython) $@", +) + +sh_binary( + name = "cython_binary", + srcs = ["cython"], + visibility = ["//visibility:public"], +) -- cgit v1.2.3 From b5499a3f84b218740d7bb36f0f99a920e1484d85 Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Tue, 26 Jun 2018 00:43:07 +0800 Subject: systemlibs: Add syslibs_configure repository_rule to generate if_system_lib macros Signed-off-by: Jason Zaman --- third_party/systemlibs/BUILD.tpl | 0 third_party/systemlibs/build_defs.bzl.tpl | 32 ++++++ third_party/systemlibs/syslibs_configure.bzl | 159 +++++++++++++++++++++++++++ 3 files changed, 191 insertions(+) create mode 100644 third_party/systemlibs/BUILD.tpl create mode 100644 third_party/systemlibs/build_defs.bzl.tpl create mode 100644 third_party/systemlibs/syslibs_configure.bzl diff --git a/third_party/systemlibs/BUILD.tpl b/third_party/systemlibs/BUILD.tpl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/third_party/systemlibs/build_defs.bzl.tpl b/third_party/systemlibs/build_defs.bzl.tpl new file mode 100644 index 0000000000..3faa46c581 --- /dev/null +++ b/third_party/systemlibs/build_defs.bzl.tpl @@ -0,0 +1,32 @@ +# -*- Python -*- +"""Skylark macros for system libraries. +""" + +SYSTEM_LIBS_ENABLED = %{syslibs_enabled} + +SYSTEM_LIBS_LIST = [ +%{syslibs_list} +] + + +def if_any_system_libs(a, b=[]): + """Conditional which evaluates to 'a' if any system libraries are configured.""" + if SYSTEM_LIBS_ENABLED: + return a + else: + return b + + +def if_system_lib(lib, a, b=[]): + """Conditional which evaluates to 'a' if we're using the system version of lib""" + + if SYSTEM_LIBS_ENABLED and lib in SYSTEM_LIBS_LIST: + return a + else: + return b + + +def if_not_system_lib(lib, a, b=[]): + """Conditional which evaluates to 'a' if we're using the system version of lib""" + + return if_system_lib(lib, b, a) diff --git a/third_party/systemlibs/syslibs_configure.bzl b/third_party/systemlibs/syslibs_configure.bzl new file mode 100644 index 0000000000..d2da333e60 --- /dev/null +++ b/third_party/systemlibs/syslibs_configure.bzl @@ -0,0 +1,159 @@ +# -*- Python -*- +"""Repository rule for system library autoconfiguration. + +`syslibs_configure` depends on the following environment variables: + + * `TF_SYSTEM_LIBS`: list of third party dependencies that should use + the system version instead +""" + +_TF_SYSTEM_LIBS="TF_SYSTEM_LIBS" + +VALID_LIBS=[ + "astor_archive", + "com_googlesource_code_re2", + "curl", + "cython", + "flatbuffers", + "gif_archive", + "grpc", + "jemalloc", + "jpeg", + "lmdb", + "nasm", + "org_sqlite", + "pcre", + "png_archive", + "six_archive", + "snappy", + "swig", + "termcolor_archive", + "zlib_archive", +] + + +def auto_configure_fail(msg): + """Output failure message when syslibs configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("\n%sSystem Library Configuration Error:%s %s\n" % (red, no_color, msg)) + + +def _is_windows(repository_ctx): + """Returns true if the host operating system is windows.""" + os_name = repository_ctx.os.name.lower() + if os_name.find("windows") != -1: + return True + return False + + +def _enable_syslibs(repository_ctx): + s = repository_ctx.os.environ.get(_TF_SYSTEM_LIBS, '').strip() + if not _is_windows(repository_ctx) and s != None and s != '': + return True + return False + + +def _get_system_lib_list(repository_ctx): + """Gets the list of deps that should use the system lib. + + Args: + repository_ctx: The repository context. + + Returns: + A string version of a python list + """ + if _TF_SYSTEM_LIBS not in repository_ctx.os.environ: + return [] + + libenv = repository_ctx.os.environ[_TF_SYSTEM_LIBS].strip() + libs = [] + + for l in list(libenv.split(',')): + l = l.strip() + if l == "": + continue + if l not in VALID_LIBS: + auto_configure_fail("Invalid system lib set: %s" % l) + return [] + libs.append(l) + + return libs + + +def _format_system_lib_list(repository_ctx): + """Formats the list of deps that should use the system lib. + + Args: + repository_ctx: The repository context. + + Returns: + A list of the names of deps that should use the system lib. + """ + libs = _get_system_lib_list(repository_ctx) + ret = '' + for l in libs: + ret += "'%s',\n" % l + + return ret + + +def _tpl(repository_ctx, tpl, substitutions={}, out=None): + if not out: + out = tpl.replace(":", "") + repository_ctx.template( + out, + Label("//third_party/systemlibs%s.tpl" % tpl), + substitutions, + False) + + +def _create_dummy_repository(repository_ctx): + """Creates the dummy repository to build with all bundled libraries.""" + + _tpl(repository_ctx, ":BUILD") + _tpl(repository_ctx, ":build_defs.bzl", + { + "%{syslibs_enabled}": 'False', + "%{syslibs_list}": '', + }) + + +def _create_local_repository(repository_ctx): + """Creates the repository to build with system libraries.""" + + _tpl(repository_ctx, ":BUILD") + _tpl(repository_ctx, ":build_defs.bzl", + { + "%{syslibs_enabled}": 'True', + "%{syslibs_list}": _format_system_lib_list(repository_ctx), + }) + + +def _syslibs_autoconf_impl(repository_ctx): + """Implementation of the syslibs_configure repository rule.""" + if not _enable_syslibs(repository_ctx): + _create_dummy_repository(repository_ctx) + else: + _create_local_repository(repository_ctx) + + +syslibs_configure = repository_rule( + implementation = _syslibs_autoconf_impl, + environ = [ + _TF_SYSTEM_LIBS, + ], +) + +"""Configures the build to link to system libraries +instead of using bundled versions. + +Add the following to your WORKSPACE FILE: + +```python +syslibs_configure(name = "local_config_syslibs") +``` + +Args: + name: A unique name for this workspace rule. +""" -- cgit v1.2.3 From fe0d32c8c29583e5b78f74df3400a62514dbcd84 Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Tue, 26 Jun 2018 00:45:16 +0800 Subject: workspace: configure system libs Signed-off-by: Jason Zaman --- tensorflow/workspace.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 33643c1ca2..2c20316a9c 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -8,6 +8,7 @@ load("//third_party/git:git_configure.bzl", "git_configure") load("//third_party/py:python_configure.bzl", "python_configure") load("//third_party/sycl:sycl_configure.bzl", "sycl_configure") +load("//third_party/systemlibs:syslibs_configure.bzl", "syslibs_configure") load("//third_party/toolchains/clang6:repo.bzl", "clang6_configure") load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compiler_configure") load("//third_party:repo.bzl", "tf_http_archive") @@ -35,6 +36,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): nccl_configure(name="local_config_nccl") git_configure(name="local_config_git") sycl_configure(name="local_config_sycl") + syslibs_configure(name="local_config_syslibs") python_configure(name="local_config_python") # For windows bazel build -- cgit v1.2.3 From 9cfb09e670557cb9f92a72295c0f1974f6e00108 Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Tue, 26 Jun 2018 00:46:02 +0800 Subject: pip_package: exclude grpc licenses when building system library The grpc license files are deeper in the heirarchy so are hard to nop out in the system version of the BUILD file. Signed-off-by: Jason Zaman --- tensorflow/tools/pip_package/BUILD | 13 +++++++++---- tensorflow/tools/pip_package/build_pip_package.sh | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index ac252143d7..abf631d05b 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -12,6 +12,7 @@ load( load("//third_party/mkl:build_defs.bzl", "if_mkl") load("//tensorflow:tensorflow.bzl", "if_cuda") load("@local_config_tensorrt//:build_defs.bzl", "if_tensorrt") +load("@local_config_syslibs//:build_defs.bzl", "if_not_system_lib") load("//tensorflow/core:platform/default/build_config_root.bzl", "tf_additional_license_deps") # This returns a list of headers of all public header libraries (e.g., @@ -145,7 +146,6 @@ filegroup( "@gast_archive//:PKG-INFO", "@gemmlowp//:LICENSE", "@gif_archive//:COPYING", - "@grpc//:LICENSE", "@highwayhash//:LICENSE", "@jemalloc//:COPYING", "@jpeg//:LICENSE.md", @@ -154,8 +154,6 @@ filegroup( "@lmdb//:LICENSE", "@local_config_nccl//:LICENSE", "@local_config_sycl//sycl:LICENSE.text", - "@grpc//third_party/nanopb:LICENSE.txt", - "@grpc//third_party/address_sorting:LICENSE", "@nasm//:LICENSE", "@nsync//:LICENSE", "@pcre//:LICENCE", @@ -169,7 +167,14 @@ filegroup( "@org_python_pypi_backports_weakref//:LICENSE", ] + if_mkl([ "//third_party/mkl:LICENSE", - ]) + tf_additional_license_deps(), + ]) + if_not_system_lib( + "grpc", + [ + "@grpc//:LICENSE", + "@grpc//third_party/nanopb:LICENSE.txt", + "@grpc//third_party/address_sorting:LICENSE", + ], + ) + tf_additional_license_deps(), ) sh_binary( diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh index b0089d3360..4101b34a11 100755 --- a/tensorflow/tools/pip_package/build_pip_package.sh +++ b/tensorflow/tools/pip_package/build_pip_package.sh @@ -27,7 +27,7 @@ function cp_external() { pushd . cd "$src_dir" - for f in `find . ! -type d ! -name '*.py' ! -path '*local_config_cuda*' ! -path '*local_config_tensorrt*' ! -path '*org_tensorflow*'`; do + for f in `find . ! -type d ! -name '*.py' ! -path '*local_config_cuda*' ! -path '*local_config_tensorrt*' ! -path '*local_config_syslibs*' ! -path '*org_tensorflow*'`; do mkdir -p "${dest_dir}/$(dirname ${f})" cp "${f}" "${dest_dir}/$(dirname ${f})/" done -- cgit v1.2.3 From 209e4d12a2f0237768d1306a5be411479a8d4b12 Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Tue, 26 Jun 2018 19:44:49 +0800 Subject: third_party: unbundle jsoncpp dep The jsoncpp headers are included with a different path so we have to symlink them so the are in the dir structure that is expected. Signed-off-by: Jason Zaman --- tensorflow/workspace.bzl | 1 + third_party/systemlibs/jsoncpp.BUILD | 37 ++++++++++++++++++++++++++++ third_party/systemlibs/syslibs_configure.bzl | 1 + 3 files changed, 39 insertions(+) create mode 100644 third_party/systemlibs/jsoncpp.BUILD diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 2c20316a9c..af21230ab5 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -516,6 +516,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "c49deac9e0933bcb7044f08516861a2d560988540b23de2ac1ad443b219afdb6", strip_prefix = "jsoncpp-1.8.4", build_file = clean_dep("//third_party:jsoncpp.BUILD"), + system_build_file = clean_dep("//third_party/systemlibs:jsoncpp.BUILD"), ) tf_http_archive( diff --git a/third_party/systemlibs/jsoncpp.BUILD b/third_party/systemlibs/jsoncpp.BUILD new file mode 100644 index 0000000000..cf91917cfb --- /dev/null +++ b/third_party/systemlibs/jsoncpp.BUILD @@ -0,0 +1,37 @@ +licenses(["unencumbered"]) # Public Domain or MIT + +filegroup( + name = "LICENSE", + visibility = ["//visibility:public"], +) + +HEADERS = [ + "include/json/autolink.h", + "include/json/config.h", + "include/json/features.h", + "include/json/forwards.h", + "include/json/json.h", + "include/json/reader.h", + "include/json/value.h", + "include/json/version.h", + "include/json/writer.h", +] + +genrule( + name = "link_headers", + outs = HEADERS, + cmd = """ + for i in $(OUTS); do + i=$${i##*/} + ln -vsf /usr/include/jsoncpp/json/$$i $(@D)/include/json/$$i + done + """, +) + +cc_library( + name = "jsoncpp", + hdrs = HEADERS, + includes = ["."], + linkopts = ["-ljsoncpp"], + visibility = ["//visibility:public"], +) diff --git a/third_party/systemlibs/syslibs_configure.bzl b/third_party/systemlibs/syslibs_configure.bzl index d2da333e60..2188ffce66 100644 --- a/third_party/systemlibs/syslibs_configure.bzl +++ b/third_party/systemlibs/syslibs_configure.bzl @@ -19,6 +19,7 @@ VALID_LIBS=[ "grpc", "jemalloc", "jpeg", + "jsoncpp_git", "lmdb", "nasm", "org_sqlite", -- cgit v1.2.3 From 8fc498b925d7fc0ebd0e00dba90fbc945e37a532 Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Thu, 12 Jul 2018 11:19:56 +0800 Subject: third_party/repo: add TF_SYSTEM_LIBS to environ so bazel knows to refresh Signed-off-by: Jason Zaman --- third_party/repo.bzl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/third_party/repo.bzl b/third_party/repo.bzl index fb33c389ac..5cb42691c5 100644 --- a/third_party/repo.bzl +++ b/third_party/repo.bzl @@ -123,7 +123,10 @@ tf_http_archive = repository_rule( "patch_file": attr.label(), "build_file": attr.label(), "system_build_file": attr.label(), - }) + }, + environ=[ + "TF_SYSTEM_LIBS", + ]) """Downloads and creates Bazel repos for dependencies. This is a swappable replacement for both http_archive() and -- cgit v1.2.3 From 1b765165987f5277e294251c118f321166c70932 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jul 2018 22:19:33 -0700 Subject: [XLA] Split out HloGatherInstruction as subclass from HloInstruction. PiperOrigin-RevId: 204421652 --- tensorflow/compiler/xla/service/BUILD | 1 + tensorflow/compiler/xla/service/hlo_instruction.cc | 109 ++++++------------ tensorflow/compiler/xla/service/hlo_instruction.h | 29 +---- .../compiler/xla/service/hlo_instruction_test.cc | 5 +- .../compiler/xla/service/hlo_instructions.cc | 89 +++++++++++++++ tensorflow/compiler/xla/service/hlo_instructions.h | 43 ++++++++ tensorflow/compiler/xla/service/hlo_parser.cc | 12 +- .../compiler/xla/service/shape_inference_test.cc | 122 +++++++++++---------- 8 files changed, 243 insertions(+), 167 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 85c6c632cd..989bb759e3 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -182,6 +182,7 @@ tf_cc_test( name = "shape_inference_test", srcs = ["shape_inference_test.cc"], deps = [ + ":hlo", ":shape_inference", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:test", diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 830ebfb125..19bee38790 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -386,6 +386,23 @@ StatusOr> HloInstruction::CreateFromProto( slice_sizes); break; } + case HloOpcode::kGather: { + TF_RET_CHECK(proto.operand_ids_size() == 2) + << "Gather instruction should have 2 operands but sees " + << proto.operand_ids_size(); + TF_RET_CHECK(proto.has_gather_dimension_numbers()) + << "Gather instruction should have GatherDimensionNumbers set."; + std::unique_ptr gather_dimension_numbers = + MakeUnique(proto.gather_dimension_numbers()); + std::vector gather_window_bounds; + for (int64 bound : proto.gather_window_bounds()) { + gather_window_bounds.push_back(bound); + } + instruction = + CreateGather(proto.shape(), operands(0), operands(1), + *gather_dimension_numbers, gather_window_bounds); + break; + } default: { instruction = WrapUnique(new HloInstruction(opcode, proto.shape())); for (const int64 operand_id : proto.operand_ids()) { @@ -427,13 +444,6 @@ StatusOr> HloInstruction::CreateFromProto( instruction->set_sharding(sharding); } - if (proto.has_gather_dimension_numbers()) { - instruction->gather_dimension_numbers_ = - MakeUnique(proto.gather_dimension_numbers()); - } - for (int64 bound : proto.gather_window_bounds()) { - instruction->gather_window_bounds_.push_back(bound); - } return std::move(instruction); } @@ -1036,34 +1046,8 @@ bool HloInstruction::HasSideEffect() const { const Shape& shape, HloInstruction* operand, HloInstruction* gather_indices, const GatherDimensionNumbers& gather_dim_numbers, tensorflow::gtl::ArraySlice window_bounds) { - std::unique_ptr instruction = - WrapUnique(new HloInstruction(HloOpcode::kGather, shape)); - instruction->AppendOperand(operand); - instruction->AppendOperand(gather_indices); - instruction->gather_dimension_numbers_ = - MakeUnique(gather_dim_numbers); - c_copy(window_bounds, std::back_inserter(instruction->gather_window_bounds_)); - return instruction; -} - -/* static */ GatherDimensionNumbers HloInstruction::MakeGatherDimNumbers( - tensorflow::gtl::ArraySlice output_window_dims, - tensorflow::gtl::ArraySlice elided_window_dims, - tensorflow::gtl::ArraySlice gather_dims_to_operand_dims, - int64 index_vector_dim) { - GatherDimensionNumbers gather_dim_numbers; - for (int64 output_window_dim : output_window_dims) { - gather_dim_numbers.add_output_window_dims(output_window_dim); - } - for (int64 elided_window_dim : elided_window_dims) { - gather_dim_numbers.add_elided_window_dims(elided_window_dim); - } - for (int64 gather_dim_to_input_dim : gather_dims_to_operand_dims) { - gather_dim_numbers.add_gather_dims_to_operand_dims(gather_dim_to_input_dim); - } - - gather_dim_numbers.set_index_vector_dim(index_vector_dim); - return gather_dim_numbers; + return MakeUnique(shape, operand, gather_indices, + gather_dim_numbers, window_bounds); } /* static */ std::unique_ptr HloInstruction::CreateDomain( @@ -1127,6 +1111,7 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( case HloOpcode::kPad: case HloOpcode::kDynamicSlice: case HloOpcode::kSort: + case HloOpcode::kGather: clone = CloneWithNewOperandsImpl(shape, new_operands, context); break; // Unary ops. @@ -1228,11 +1213,6 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( true_computation(), new_operands[2], false_computation()); break; - case HloOpcode::kGather: - CHECK_EQ(new_operands.size(), 2); - clone = CreateGather(shape, new_operands[0], new_operands[1], - *gather_dimension_numbers_, gather_window_bounds_); - break; case HloOpcode::kDomain: CHECK_EQ(new_operands.size(), 1); clone = @@ -1539,11 +1519,6 @@ bool HloInstruction::IdenticalSlowPath( return protobuf_util::ProtobufEquals(dot_dimension_numbers(), other.dot_dimension_numbers()); - case HloOpcode::kGather: - return protobuf_util::ProtobufEquals(gather_dimension_numbers(), - other.gather_dimension_numbers()) && - gather_window_bounds() == other.gather_window_bounds(); - // Remaining instructions with special values. case HloOpcode::kCall: return eq_computations(to_apply(), other.to_apply()); @@ -1590,6 +1565,7 @@ bool HloInstruction::IdenticalSlowPath( case HloOpcode::kHostCompute: case HloOpcode::kPad: case HloOpcode::kDynamicSlice: + case HloOpcode::kGather: LOG(FATAL) << "Base class impl called for opcode with subclass: " << opcode(); } @@ -1955,11 +1931,6 @@ std::vector HloInstruction::ExtraAttributesToString( if (dot_dimension_numbers_ != nullptr) { extra.push_back(DotDimensionNumbersToString()); } - if (gather_dimension_numbers_ != nullptr) { - extra.push_back(GatherDimensionNumbersToString()); - extra.push_back( - StrCat("window_bounds={", Join(gather_window_bounds(), ","), "}")); - } if (options.print_subcomputation_mode() == HloPrintOptions::PrintSubcomputationMode::kNameOnly) { @@ -2089,14 +2060,6 @@ HloInstructionProto HloInstruction::ToProto() const { if (dot_dimension_numbers_ != nullptr) { *proto.mutable_dot_dimension_numbers() = *dot_dimension_numbers_; } - if (gather_dimension_numbers_ != nullptr) { - *proto.mutable_gather_dimension_numbers() = *gather_dimension_numbers_; - } - if (opcode() == HloOpcode::kGather) { - for (int64 bound : gather_window_bounds()) { - proto.add_gather_window_bounds(bound); - } - } if (has_sharding()) { *proto.mutable_sharding() = sharding().ToProto(); @@ -2857,26 +2820,6 @@ std::ostream& operator<<(std::ostream& os, HloInstruction::FusionKind kind) { return os << ToString(kind); } -string HloInstruction::GatherDimensionNumbersToString() const { - CHECK_NE(gather_dimension_numbers_.get(), nullptr); - string output_window_dims = - StrCat("output_window_dims={", - Join(gather_dimension_numbers_->output_window_dims(), ","), "}"); - string elided_window_dims = - StrCat("elided_window_dims={", - Join(gather_dimension_numbers_->elided_window_dims(), ","), "}"); - string gather_dims_to_operand_dims = StrCat( - "gather_dims_to_operand_dims={", - Join(gather_dimension_numbers_->gather_dims_to_operand_dims(), ","), "}"); - string index_vector_dim = StrCat( - "index_vector_dim=", gather_dimension_numbers_->index_vector_dim()); - - return Join>( - {output_window_dims, elided_window_dims, gather_dims_to_operand_dims, - index_vector_dim}, - ", "); -} - bool HloInstruction::CouldBeBitcast() const { switch (opcode_) { case HloOpcode::kTranspose: @@ -3190,4 +3133,14 @@ int64 HloInstruction::slice_sizes(int64 dimension) const { const std::vector& HloInstruction::dynamic_slice_sizes() const { return Cast(this)->dynamic_slice_sizes(); } + +const GatherDimensionNumbers& HloInstruction::gather_dimension_numbers() const { + return Cast(this)->gather_dimension_numbers(); +} + +tensorflow::gtl::ArraySlice HloInstruction::gather_window_bounds() + const { + return Cast(this)->gather_window_bounds(); +} + } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index b392d65636..cbd78fa124 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -700,13 +700,6 @@ class HloInstruction { // when we plumb a primordial token from the entry computation. static std::unique_ptr CreateToken(); - // Creates an instance of GatherDimensionNumbers. - static GatherDimensionNumbers MakeGatherDimNumbers( - tensorflow::gtl::ArraySlice output_window_dims, - tensorflow::gtl::ArraySlice elided_window_dims, - tensorflow::gtl::ArraySlice gather_dims_to_operand_dims, - int64 index_vector_dim); - // Returns the opcode for this instruction. HloOpcode opcode() const { return opcode_; } @@ -1081,19 +1074,6 @@ class HloInstruction { // Returns the dump string of the dot dimension numbers. string DotDimensionNumbersToString() const; - const GatherDimensionNumbers& gather_dimension_numbers() const { - CHECK(gather_dimension_numbers_ != nullptr); - return *gather_dimension_numbers_; - } - - tensorflow::gtl::ArraySlice gather_window_bounds() const { - CHECK_EQ(opcode(), HloOpcode::kGather); - return gather_window_bounds_; - } - - // Returns the dump string of the gather dimension numbers. - string GatherDimensionNumbersToString() const; - // Clones the HLO instruction. The clone will have the same opcode, shape, and // operands. After creation the clone has no uses. "this" (the instruction // cloned from) is not changed. Suffix is the string to append to the name of @@ -1460,6 +1440,12 @@ class HloInstruction { // Delegates to HloDynamicSliceInstruction::dynamic_slice_sizes. const std::vector& dynamic_slice_sizes() const; + + // Delegates to HloGatherInstruction::gather_dimension_numbers. + const GatherDimensionNumbers& gather_dimension_numbers() const; + // Delegates to HloGatherInstruction::gather_window_bounds. + tensorflow::gtl::ArraySlice gather_window_bounds() const; + // Old methods kept for smooth subclassing transition END. protected: @@ -1603,9 +1589,6 @@ class HloInstruction { // Describes the dimension numbers used for a dot. std::unique_ptr dot_dimension_numbers_; - std::unique_ptr gather_dimension_numbers_; - std::vector gather_window_bounds_; - // Used to tag kCopy instructions that are eligible for copy elision. bool copy_elision_allowed_ = true; diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc index 87c048930f..b75a2bd34b 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/compiler/xla/protobuf_util.h" #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_instructions.h" #include "tensorflow/compiler/xla/service/hlo_parser.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/test.h" @@ -1369,7 +1370,7 @@ TEST_F(HloInstructionTest, StringifyGather_0) { HloInstruction* gather_instruction = builder.AddInstruction(HloInstruction::CreateGather( gather_result_shape, input, gather_indices, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1405,7 +1406,7 @@ TEST_F(HloInstructionTest, StringifyGather_1) { HloInstruction* gather_instruction = builder.AddInstruction(HloInstruction::CreateGather( gather_result_shape, input, gather_indices, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc index 7ea42caa7b..f333c489ed 100644 --- a/tensorflow/compiler/xla/service/hlo_instructions.cc +++ b/tensorflow/compiler/xla/service/hlo_instructions.cc @@ -1914,4 +1914,93 @@ HloDynamicSliceInstruction::CloneWithNewOperandsImpl( return MakeUnique( shape, new_operands[0], new_operands[1], dynamic_slice_sizes_); } + +HloGatherInstruction::HloGatherInstruction( + const Shape& shape, HloInstruction* operand, HloInstruction* gather_indices, + const GatherDimensionNumbers& gather_dim_numbers, + tensorflow::gtl::ArraySlice window_bounds) + : HloInstruction(HloOpcode::kGather, shape) { + AppendOperand(operand); + AppendOperand(gather_indices); + gather_dimension_numbers_ = + MakeUnique(gather_dim_numbers); + c_copy(window_bounds, std::back_inserter(gather_window_bounds_)); +} + +string HloGatherInstruction::GatherDimensionNumbersToString() const { + CHECK(gather_dimension_numbers_ != nullptr); + string output_window_dims = + StrCat("output_window_dims={", + Join(gather_dimension_numbers_->output_window_dims(), ","), "}"); + string elided_window_dims = + StrCat("elided_window_dims={", + Join(gather_dimension_numbers_->elided_window_dims(), ","), "}"); + string gather_dims_to_operand_dims = StrCat( + "gather_dims_to_operand_dims={", + Join(gather_dimension_numbers_->gather_dims_to_operand_dims(), ","), "}"); + string index_vector_dim = StrCat( + "index_vector_dim=", gather_dimension_numbers_->index_vector_dim()); + + return Join>( + {output_window_dims, elided_window_dims, gather_dims_to_operand_dims, + index_vector_dim}, + ", "); +} + +/* static */ GatherDimensionNumbers HloGatherInstruction::MakeGatherDimNumbers( + tensorflow::gtl::ArraySlice output_window_dims, + tensorflow::gtl::ArraySlice elided_window_dims, + tensorflow::gtl::ArraySlice gather_dims_to_operand_dims, + int64 index_vector_dim) { + GatherDimensionNumbers gather_dim_numbers; + for (int64 output_window_dim : output_window_dims) { + gather_dim_numbers.add_output_window_dims(output_window_dim); + } + for (int64 elided_window_dim : elided_window_dims) { + gather_dim_numbers.add_elided_window_dims(elided_window_dim); + } + for (int64 gather_dim_to_input_dim : gather_dims_to_operand_dims) { + gather_dim_numbers.add_gather_dims_to_operand_dims(gather_dim_to_input_dim); + } + + gather_dim_numbers.set_index_vector_dim(index_vector_dim); + return gather_dim_numbers; +} + +HloInstructionProto HloGatherInstruction::ToProto() const { + HloInstructionProto proto = HloInstruction::ToProto(); + *proto.mutable_gather_dimension_numbers() = gather_dimension_numbers(); + for (int64 bound : gather_window_bounds()) { + proto.add_gather_window_bounds(bound); + } + return proto; +} + +std::vector HloGatherInstruction::ExtraAttributesToStringImpl( + const HloPrintOptions& options) const { + return {GatherDimensionNumbersToString(), + StrCat("window_bounds={", Join(gather_window_bounds(), ","), "}")}; +} + +bool HloGatherInstruction::IdenticalSlowPath( + const HloInstruction& other, + const std::function& + eq_computations) const { + const auto& casted_other = static_cast(other); + return protobuf_util::ProtobufEquals( + gather_dimension_numbers(), + casted_other.gather_dimension_numbers()) && + gather_window_bounds() == casted_other.gather_window_bounds(); +} + +std::unique_ptr HloGatherInstruction::CloneWithNewOperandsImpl( + const Shape& shape, + tensorflow::gtl::ArraySlice new_operands, + HloCloneContext* context) const { + CHECK_EQ(new_operands.size(), 2); + return MakeUnique( + shape, new_operands[0], new_operands[1], gather_dimension_numbers(), + gather_window_bounds()); +} + } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h index e922d94234..65a93cdcf1 100644 --- a/tensorflow/compiler/xla/service/hlo_instructions.h +++ b/tensorflow/compiler/xla/service/hlo_instructions.h @@ -1148,6 +1148,49 @@ class HloDynamicSliceInstruction : public HloInstruction { // ('start' is specified dynamically in the second operand of the operation). std::vector dynamic_slice_sizes_; }; + +class HloGatherInstruction : public HloInstruction { + public: + explicit HloGatherInstruction( + const Shape& shape, HloInstruction* operand, + HloInstruction* gather_indices, + const GatherDimensionNumbers& gather_dim_numbers, + tensorflow::gtl::ArraySlice window_bounds); + const GatherDimensionNumbers& gather_dimension_numbers() const { + CHECK(gather_dimension_numbers_ != nullptr); + return *gather_dimension_numbers_; + } + tensorflow::gtl::ArraySlice gather_window_bounds() const { + return gather_window_bounds_; + } + // Returns the dump string of the gather dimension numbers. + string GatherDimensionNumbersToString() const; + // Returns a serialized representation of this instruction. + HloInstructionProto ToProto() const override; + + // Creates an instance of GatherDimensionNumbers. + static GatherDimensionNumbers MakeGatherDimNumbers( + tensorflow::gtl::ArraySlice output_window_dims, + tensorflow::gtl::ArraySlice elided_window_dims, + tensorflow::gtl::ArraySlice gather_dims_to_operand_dims, + int64 index_vector_dim); + + private: + std::vector ExtraAttributesToStringImpl( + const HloPrintOptions& options) const override; + bool IdenticalSlowPath( + const HloInstruction& other, + const std::function& + eq_computations) const override; + std::unique_ptr CloneWithNewOperandsImpl( + const Shape& shape, + tensorflow::gtl::ArraySlice new_operands, + HloCloneContext* context) const override; + + std::unique_ptr gather_dimension_numbers_; + std::vector gather_window_bounds_; +}; + } // namespace xla #endif // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_INSTRUCTIONS_H_ diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc index f162d52d3c..d387539350 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.cc +++ b/tensorflow/compiler/xla/service/hlo_parser.cc @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/service/hlo_domain_metadata.h" +#include "tensorflow/compiler/xla/service/hlo_instructions.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/service/hlo_sharding_metadata.h" #include "tensorflow/compiler/xla/shape_util.h" @@ -1192,11 +1193,12 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder, return false; } - GatherDimensionNumbers dim_numbers = HloInstruction::MakeGatherDimNumbers( - /*output_window_dims=*/*output_window_dims, - /*elided_window_dims=*/*elided_window_dims, - /*gather_dims_to_operand_dims=*/*gather_dims_to_operand_dims, - /*index_vector_dim=*/*index_vector_dim); + GatherDimensionNumbers dim_numbers = + HloGatherInstruction::MakeGatherDimNumbers( + /*output_window_dims=*/*output_window_dims, + /*elided_window_dims=*/*elided_window_dims, + /*gather_dims_to_operand_dims=*/*gather_dims_to_operand_dims, + /*index_vector_dim=*/*index_vector_dim); instruction = builder->AddInstruction(HloInstruction::CreateGather( shape, /*operand=*/operands[0], /*gather_indices=*/operands[1], diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc index bafe14d6f4..9b1ce143c6 100644 --- a/tensorflow/compiler/xla/service/shape_inference_test.cc +++ b/tensorflow/compiler/xla/service/shape_inference_test.cc @@ -17,6 +17,7 @@ limitations under the License. #include +#include "tensorflow/compiler/xla/service/hlo_instructions.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/test.h" #include "tensorflow/compiler/xla/test_helpers.h" @@ -1543,45 +1544,45 @@ class GatherShapeInferenceTest : public ShapeInferenceTest { }; TEST_F(GatherShapeInferenceTest, TensorFlowGather) { - TF_ASSERT_OK_AND_ASSIGN( - Shape gather_shape, - ShapeInference::InferGatherShape(matrix_64_48_, s64_vector_32_, - HloInstruction::MakeGatherDimNumbers( - /*output_window_dims=*/{0}, - /*elided_window_dims=*/{1}, - /*gather_dims_to_operand_dims=*/{1}, - /*index_vector_dim=*/1), - /*window_bounds=*/{64, 1})); + TF_ASSERT_OK_AND_ASSIGN(Shape gather_shape, + ShapeInference::InferGatherShape( + matrix_64_48_, s64_vector_32_, + HloGatherInstruction::MakeGatherDimNumbers( + /*output_window_dims=*/{0}, + /*elided_window_dims=*/{1}, + /*gather_dims_to_operand_dims=*/{1}, + /*index_vector_dim=*/1), + /*window_bounds=*/{64, 1})); EXPECT_TRUE( ShapeUtil::Equal(gather_shape, ShapeUtil::MakeShape(F32, {64, 32}))) << ShapeUtil::HumanString(gather_shape); } TEST_F(GatherShapeInferenceTest, TensorFlowGatherV2) { - TF_ASSERT_OK_AND_ASSIGN( - Shape gather_shape, - ShapeInference::InferGatherShape(matrix_64_48_, s64_vector_32_, - HloInstruction::MakeGatherDimNumbers( - /*output_window_dims=*/{1}, - /*elided_window_dims=*/{0}, - /*gather_dims_to_operand_dims=*/{0}, - /*index_vector_dim=*/1), - /*window_bounds=*/{1, 48})); + TF_ASSERT_OK_AND_ASSIGN(Shape gather_shape, + ShapeInference::InferGatherShape( + matrix_64_48_, s64_vector_32_, + HloGatherInstruction::MakeGatherDimNumbers( + /*output_window_dims=*/{1}, + /*elided_window_dims=*/{0}, + /*gather_dims_to_operand_dims=*/{0}, + /*index_vector_dim=*/1), + /*window_bounds=*/{1, 48})); EXPECT_TRUE( ShapeUtil::Equal(gather_shape, ShapeUtil::MakeShape(F32, {32, 48}))) << ShapeUtil::HumanString(gather_shape); } TEST_F(GatherShapeInferenceTest, TensorFlowGatherNd) { - TF_ASSERT_OK_AND_ASSIGN( - Shape gather_shape, - ShapeInference::InferGatherShape(matrix_64_48_, s64_4d_tensor_10_9_8_7_1_, - HloInstruction::MakeGatherDimNumbers( - /*output_window_dims=*/{4}, - /*elided_window_dims=*/{0}, - /*gather_dims_to_operand_dims=*/{0}, - /*index_vector_dim=*/4), - /*window_bounds=*/{1, 48})); + TF_ASSERT_OK_AND_ASSIGN(Shape gather_shape, + ShapeInference::InferGatherShape( + matrix_64_48_, s64_4d_tensor_10_9_8_7_1_, + HloGatherInstruction::MakeGatherDimNumbers( + /*output_window_dims=*/{4}, + /*elided_window_dims=*/{0}, + /*gather_dims_to_operand_dims=*/{0}, + /*index_vector_dim=*/4), + /*window_bounds=*/{1, 48})); EXPECT_TRUE(ShapeUtil::Equal(gather_shape, ShapeUtil::MakeShape(F32, {10, 9, 8, 7, 48}))) << ShapeUtil::HumanString(gather_shape); @@ -1592,7 +1593,7 @@ TEST_F(GatherShapeInferenceTest, TensorFlowBatchDynamicSlice) { Shape gather_shape, ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1609,7 +1610,7 @@ TEST_F(GatherShapeInferenceTest, NonDefaultGatherIndicesLeafDim_A) { Shape gather_shape, ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_5_7_6_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1627,7 +1628,7 @@ TEST_F(GatherShapeInferenceTest, NonDefaultGatherIndicesLeafDim_B) { Shape gather_shape, ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_5_10_9_7_6_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1646,7 +1647,7 @@ TEST_F(GatherShapeInferenceTest, NoOutputGatherDims) { Shape gather_shape, ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_vector_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{0, 1, 2, 3, 4}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1664,7 +1665,7 @@ TEST_F(GatherShapeInferenceTest, ScalarGatherIndices) { TF_ASSERT_OK_AND_ASSIGN(Shape gather_shape, ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_scalar_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{0, 1, 2, 3}, /*elided_window_dims=*/{0}, /*gather_dims_to_operand_dims=*/{0}, @@ -1679,10 +1680,11 @@ TEST_F(GatherShapeInferenceTest, ScalarGatherIndices) { TEST_F(GatherShapeInferenceTest, TupleShapedTensorInput) { StatusOr statusor = ShapeInference::InferGatherShape( tuple_shape_, s64_vector_32_, - HloInstruction::MakeGatherDimNumbers(/*output_window_dims=*/{0}, - /*elided_window_dims=*/{1}, - /*gather_dims_to_operand_dims=*/{1}, - /*index_vector_dim=*/1), + HloGatherInstruction::MakeGatherDimNumbers( + /*output_window_dims=*/{0}, + /*elided_window_dims=*/{1}, + /*gather_dims_to_operand_dims=*/{1}, + /*index_vector_dim=*/1), /*window_bounds=*/{64, 1}); ASSERT_FALSE(statusor.ok()); EXPECT_THAT(statusor.status().error_message(), @@ -1693,10 +1695,11 @@ TEST_F(GatherShapeInferenceTest, TupleShapedTensorInput) { TEST_F(GatherShapeInferenceTest, TupleShapedGatherIndicesInput) { StatusOr statusor = ShapeInference::InferGatherShape( s64_vector_32_, tuple_shape_, - HloInstruction::MakeGatherDimNumbers(/*output_window_dims=*/{0}, - /*elided_window_dims=*/{1}, - /*gather_dims_to_operand_dims=*/{1}, - /*index_vector_dim=*/0), + HloGatherInstruction::MakeGatherDimNumbers( + /*output_window_dims=*/{0}, + /*elided_window_dims=*/{1}, + /*gather_dims_to_operand_dims=*/{1}, + /*index_vector_dim=*/0), /*window_bounds=*/{64, 1}); ASSERT_FALSE(statusor.ok()); EXPECT_THAT(statusor.status().error_message(), @@ -1707,10 +1710,11 @@ TEST_F(GatherShapeInferenceTest, TupleShapedGatherIndicesInput) { TEST_F(GatherShapeInferenceTest, FloatingPointGatherIndicesInput) { StatusOr statusor = ShapeInference::InferGatherShape( s64_vector_32_, vector_32_, - HloInstruction::MakeGatherDimNumbers(/*output_window_dims=*/{0}, - /*elided_window_dims=*/{1}, - /*gather_dims_to_operand_dims=*/{1}, - /*index_vector_dim=*/0), + HloGatherInstruction::MakeGatherDimNumbers( + /*output_window_dims=*/{0}, + /*elided_window_dims=*/{1}, + /*gather_dims_to_operand_dims=*/{1}, + /*index_vector_dim=*/0), /*window_bounds=*/{64, 1}); ASSERT_FALSE(statusor.ok()); EXPECT_THAT(statusor.status().error_message(), @@ -1722,7 +1726,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_NonAscendingWindowIndices) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 8, 7}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1739,7 +1743,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_RepeatedWindowIndices) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 7}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1756,7 +1760,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_WindowIndexOutOfBounds) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 99, 100, 101}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1772,7 +1776,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_WindowIndexBarelyOutOfBounds) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 9}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1788,7 +1792,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_MismatchingElidedWindowDims) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{4}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1806,7 +1810,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_OutOfBoundsWindowToInputMapping) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{0, 1, 2, 3, 19}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1823,7 +1827,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_RepeatedWindowToInputMapping) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{0, 1, 2, 3, 3}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1841,7 +1845,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_MismatchingGatherToInputMapping) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3}, @@ -1860,7 +1864,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_OutOfBoundsGatherToInputMapping) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 7}, @@ -1878,7 +1882,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_RepeatedGatherToInputMapping) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 3}, @@ -1896,7 +1900,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_NonAscendingElidedWindowDims) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{2, 1}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1911,7 +1915,7 @@ TEST_F(GatherShapeInferenceTest, TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_WindowBoundsTooLarge) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7}, /*elided_window_dims=*/{2}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1928,7 +1932,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_MismatchingNumberOfWindowBounds) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1946,7 +1950,7 @@ TEST_F(GatherShapeInferenceTest, InvalidGatherDimNumbers_WindowBoundsNot1ForElidedDim) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_8_7_5_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7}, /*elided_window_dims=*/{1}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, @@ -1962,7 +1966,7 @@ TEST_F(GatherShapeInferenceTest, TEST_F(GatherShapeInferenceTest, OutOfBoundsGatherIndicesLeafDim) { StatusOr statusor = ShapeInference::InferGatherShape( f32_5d_tensor_50_49_48_47_46_, s64_4d_tensor_10_9_5_7_6_, - HloInstruction::MakeGatherDimNumbers( + HloGatherInstruction::MakeGatherDimNumbers( /*output_window_dims=*/{4, 5, 6, 7, 8}, /*elided_window_dims=*/{}, /*gather_dims_to_operand_dims=*/{0, 1, 2, 3, 4}, -- cgit v1.2.3 From 9be83d3d6e7d1e61d7c3cc4160d214c633665751 Mon Sep 17 00:00:00 2001 From: Nupur Garg Date: Fri, 13 Jul 2018 00:08:29 -0700 Subject: Fix import order in interpreter_wrapper.h. PiperOrigin-RevId: 204429340 --- .../contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h | 4 ++-- tensorflow/python/lib/core/numpy.h | 3 ++- tensorflow/python/lib/core/py_util.cc | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h index febfd2dc56..556ec7117a 100644 --- a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h +++ b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h @@ -15,13 +15,13 @@ limitations under the License. #ifndef TENSORFLOW_CONTRIB_LITE_PYTHON_INTERPRETER_WRAPPER_INTERPRETER_WRAPPER_H_ #define TENSORFLOW_CONTRIB_LITE_PYTHON_INTERPRETER_WRAPPER_INTERPRETER_WRAPPER_H_ +// Place `` before to avoid build failures in macOS. +#include #include #include #include -// Place `` before to avoid build failures in macOS. #include -#include // We forward declare TFLite classes here to avoid exposing them to SWIG. namespace tflite { diff --git a/tensorflow/python/lib/core/numpy.h b/tensorflow/python/lib/core/numpy.h index d4621d61ee..0098d938a0 100644 --- a/tensorflow/python/lib/core/numpy.h +++ b/tensorflow/python/lib/core/numpy.h @@ -30,9 +30,10 @@ limitations under the License. #endif // Place `` before to avoid build failure in macOS. -#include #include +#include + #include "numpy/arrayobject.h" #include "numpy/ufuncobject.h" diff --git a/tensorflow/python/lib/core/py_util.cc b/tensorflow/python/lib/core/py_util.cc index 6b6c82015f..2ee898ea1d 100644 --- a/tensorflow/python/lib/core/py_util.cc +++ b/tensorflow/python/lib/core/py_util.cc @@ -16,9 +16,10 @@ limitations under the License. #include "tensorflow/python/lib/core/py_util.h" // Place `` before to avoid build failure in macOS. -#include #include +#include + #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/strings/strcat.h" -- cgit v1.2.3 From e3e434d966ba5f800ba73ca688a851aa878c5463 Mon Sep 17 00:00:00 2001 From: Jie Date: Fri, 13 Jul 2018 01:02:12 -0700 Subject: [tftrt update] Added python tests for converter functions Added BUILD for python tests --- tensorflow/contrib/tensorrt/BUILD | 33 ++++ tensorflow/contrib/tensorrt/test/base_unit_test.py | 118 +++++++++++++ .../contrib/tensorrt/test/batch_matmul_test.py | 97 +++++++++++ .../contrib/tensorrt/test/biasadd_matmul_test.py | 116 +++++++++++++ .../test/binary_tensor_weight_broadcast_test.py | 148 +++++++++++++++++ .../contrib/tensorrt/test/concatenation_test.py | 87 ++++++++++ .../contrib/tensorrt/test/const_broadcast_test.py | 75 +++++++++ .../test/multi_connection_neighbor_engine_test.py | 101 +++++++++++ .../tensorrt/test/neighboring_engine_test.py | 78 +++++++++ tensorflow/contrib/tensorrt/test/run_test.py | 184 +++++++++++++++++++++ tensorflow/contrib/tensorrt/test/unary_test.py | 125 ++++++++++++++ tensorflow/contrib/tensorrt/test/unit_tests.py | 67 ++++++++ tensorflow/contrib/tensorrt/test/utilities.py | 30 ++++ .../contrib/tensorrt/test/vgg_block_nchw_test.py | 85 ++++++++++ tensorflow/contrib/tensorrt/test/vgg_block_test.py | 76 +++++++++ 15 files changed, 1420 insertions(+) create mode 100644 tensorflow/contrib/tensorrt/test/base_unit_test.py create mode 100644 tensorflow/contrib/tensorrt/test/batch_matmul_test.py create mode 100644 tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py create mode 100644 tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py create mode 100644 tensorflow/contrib/tensorrt/test/concatenation_test.py create mode 100644 tensorflow/contrib/tensorrt/test/const_broadcast_test.py create mode 100644 tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py create mode 100644 tensorflow/contrib/tensorrt/test/neighboring_engine_test.py create mode 100644 tensorflow/contrib/tensorrt/test/run_test.py create mode 100644 tensorflow/contrib/tensorrt/test/unary_test.py create mode 100644 tensorflow/contrib/tensorrt/test/unit_tests.py create mode 100644 tensorflow/contrib/tensorrt/test/utilities.py create mode 100644 tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py create mode 100644 tensorflow/contrib/tensorrt/test/vgg_block_test.py diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index adda0b758b..d957ca0861 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -341,6 +341,39 @@ py_test( ], ) +py_test( + name = "converter_unit_tests", + srcs = [ + "test/base_unit_test.py", + "test/batch_matmul_test.py", + "test/biasadd_matmul_test.py", + "test/binary_tensor_weight_broadcast_test.py", + "test/concatenation_test.py", + "test/const_broadcast_test.py", + "test/multi_connection_neighbor_engine_test.py", + "test/neighboring_engine_test.py", + "test/run_test.py", + "test/unary_test.py", + "test/unit_tests.py", + "test/utilities.py", + "test/vgg_block_nchw_test.py", + "test/vgg_block_test.py", + ], + main = "test/unit_tests.py", + srcs_version = "PY2AND3", + tags = [ + "manual", + "notap", + ], + deps = [ + ":init_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:layers", + "//tensorflow/python:training", + ], +) + cc_library( name = "utils", hdrs = ["convert/utils.h"], diff --git a/tensorflow/contrib/tensorrt/test/base_unit_test.py b/tensorflow/contrib/tensorrt/test/base_unit_test.py new file mode 100644 index 0000000000..8a6c648ab6 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/base_unit_test.py @@ -0,0 +1,118 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Base class to facilitate development of integration tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + + +class BaseUnitTest(object): + """Base class for unit tests in TF-TRT""" + + def __init__(self, log_file='log.txt'): + self.static_mode_list = {} + self.dynamic_mode_list = {} + self.dummy_input = None + self.get_network = None + self.expect_nb_nodes = None + self.test_name = None + self.log_file = log_file + self.ckpt = None + self.allclose_rtol = 0.01 + self.allclose_atol = 0.01 + self.allclose_equal_nan = True + # saves out graphdef + self.debug = False + # require node count check fail leads to test failure + self.check_node_count = False + + def run(self, run_test_context): + run_test_context.run_test(self.get_network, self.static_mode_list, + self.dynamic_mode_list, self.dummy_input, + self.ckpt) + return self.log_result(run_test_context) + + def log_result(self, run_test_result): + log = open(self.log_file, 'a') + log.write(("================= model: %s\n") % (self.test_name)) + + if self.debug: + open(self.test_name + "_native.pb", + 'wb').write(run_test_result.native_network.SerializeToString()) + all_success = True + if len(run_test_result.tftrt_conversion_flag) != 0: + log.write(" -- static_mode\n") + for static_mode in run_test_result.tftrt_conversion_flag: + if self.debug: + open(self.test_name + "_" + static_mode + ".pb", + 'wb').write(run_test_result.tftrt[static_mode].SerializeToString()) + log.write(" ----\n") + log.write((" mode: [%s]\n") % (static_mode)) + if run_test_result.tftrt_conversion_flag[static_mode]: + if run_test_result.tftrt_nb_nodes[static_mode] != self.expect_nb_nodes: + log.write( + ("[WARNING]: converted node number does not match (%d,%d,%d)!!!\n" + ) % (run_test_result.tftrt_nb_nodes[static_mode], + self.expect_nb_nodes, run_test_result.native_nb_nodes)) + if self.check_node_count: + all_success = False + + if np.array_equal(run_test_result.tftrt_result[static_mode], + run_test_result.native_result): + log.write(" output: equal\n") + elif np.allclose( + run_test_result.tftrt_result[static_mode], + run_test_result.native_result, + atol=self.allclose_atol, + rtol=self.allclose_rtol, + equal_nan=self.allclose_equal_nan): + log.write(" output: allclose\n") + else: + diff = run_test_result.tftrt_result[static_mode] - run_test_result.native_result + log.write("[ERROR]: output does not match!!!\n") + log.write("max diff: " + str(np.max(diff))) + log.write("\ntftrt:\n") + log.write(str(run_test_result.tftrt_result[static_mode])) + log.write("\nnative:\n") + log.write(str(run_test_result.native_result)) + log.write("\ndiff:\n") + log.write(str(diff)) + all_success = False + else: + log.write("[ERROR]: conversion failed!!!\n") + all_success = False + + if len(run_test_result.tftrt_dynamic_conversion_flag) != 0: + log.write(" -- dynamic_mode\n") + for dynamic_mode in run_test_result.tftrt_dynamic_conversion_flag: + log.write("\n ----\n") + log.write((" mode: [%s]\n") % (dynamic_mode)) + if run_test_result.tftrt_dynamic_conversion_flag[dynamic_mode]: + if np.array_equal(run_test_result.tftrt_dynamic_result[dynamic_mode], + run_test_result.native_result): + log.write(" output: equal\n") + elif np.allclose(run_test_result.tftrt_dynamic_result[dynamic_mode], + run_test_result.native_result): + log.write(" output: allclose\n") + else: + log.write("[ERROR]: output does not match!!!\n") + all_success = False + else: + log.write("[ERROR]: conversion failed!!!\n") + all_success = False + return all_success diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py new file mode 100644 index 0000000000..3c83a3a562 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py @@ -0,0 +1,97 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.client import session +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.utilities import get_all_variables + + +class BatchMatMulTest(BaseUnitTest): + """Testing BatchMatMul in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(BatchMatMulTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (12, 5, 8, 12) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.matmul_test + self.expect_nb_nodes = 16 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.ckpt = "./tmp.ckpt" + sess = session.Session() + + def matmul_test(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions() + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + b = constant_op.constant( + np.random.randn(12, 5, 12, 7), dtype=dtypes.float32) + x1 = math_ops.matmul(x, b) + b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32) + x1 = x1 + b + + var = variable_scope.get_variable( + "test", [12, 5, 12, 7], + dtype=dtypes.float32, + initializer=init_ops.truncated_normal_initializer) + x2 = math_ops.matmul(x, var) + b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32) + x2 = x2 * b + + var = variable_scope.get_variable( + "test2", [12, 84], + dtype=dtypes.float32, + initializer=init_ops.truncated_normal_initializer) + c = gen_array_ops.reshape(x, [12, 40, 12]) + b = gen_array_ops.reshape(var, [12, 12, 7]) + x3 = math_ops.matmul(c, b) + b = constant_op.constant(np.random.randn(40, 1), dtype=dtypes.float32) + x3 = x3 + b + x3 = gen_array_ops.reshape(x3, [12, 5, 8, 7]) + + out = x3 + x1 + array_ops.squeeze(out, name="output") + + with session.Session(config=sessconfig, graph=g) as sess: + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + sess.run(variables.global_variables_initializer()) + saver.save(sess, self.ckpt) + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py new file mode 100644 index 0000000000..1ac6f5cb6a --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py @@ -0,0 +1,116 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.layers import core +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class BiasaddMatMulTest(BaseUnitTest): + """Testing BiasAdd MatMul in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(BiasaddMatMulTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (48, 12) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.matmul_test + self.expect_nb_nodes = 53 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def matmul_test(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + b = constant_op.constant(np.random.randn(12, 4), dtype=dtypes.float32) + x1 = math_ops.matmul(x, b) + b = constant_op.constant(np.random.randn(1, 4), dtype=dtypes.float32) + x1 = x1 + b + + b = constant_op.constant(np.random.randn(48, 4), dtype=dtypes.float32) + x2 = math_ops.matmul(x, b, transpose_a=True) + x2 = gen_array_ops.reshape(x2, [48, 1]) + + b = constant_op.constant(np.random.randn(4, 12), dtype=dtypes.float32) + x3 = math_ops.matmul(x, b, transpose_b=True) + + b = constant_op.constant(np.random.randn(16, 48), dtype=dtypes.float32) + x4 = math_ops.matmul(x, b, transpose_b=True, transpose_a=True) + x4 = gen_array_ops.reshape(x4, [48, 4]) + + x5 = gen_array_ops.reshape(x, [4, 12, 12]) + x5 = core.flatten(x5) + b = constant_op.constant(np.random.randn(144, 48), dtype=dtypes.float32) + x5 = math_ops.matmul(x5, b) + b = constant_op.constant(np.random.randn(48), dtype=dtypes.float32) + x5 = nn.bias_add(x5, b) + x5 = gen_array_ops.reshape(x5, [48, 4]) + + x6 = gen_array_ops.reshape(x, [4, 12, 12]) + b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32) + x6 = nn.bias_add(x6, b, data_format="NHWC") + x6 = gen_array_ops.reshape(x6, [48, -1]) + + x7 = gen_array_ops.reshape(x, [4, 12, 3, 4]) + b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32) + x7 = nn.bias_add(x7, b, data_format="NHWC") + x7 = gen_array_ops.reshape(x7, [48, -1]) + + x8 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) + b = constant_op.constant(np.random.randn(2), dtype=dtypes.float32) + x8 = nn.bias_add(x8, b, data_format="NHWC") + x8 = gen_array_ops.reshape(x8, [48, -1]) + + x9 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2]) + b = constant_op.constant(np.random.randn(3), dtype=dtypes.float32) + x9 = nn.bias_add(x9, b, data_format="NCHW") + x9 = gen_array_ops.reshape(x9, [48, -1]) + + x10 = gen_array_ops.reshape(x, [4, 12, 3, 4]) + b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32) + x10 = nn.bias_add(x10, b, data_format="NCHW") + x10 = gen_array_ops.reshape(x10, [48, -1]) + + x11 = gen_array_ops.reshape(x, [4, 12, 12]) + b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32) + x11 = nn.bias_add(x11, b, data_format="NCHW") + x11 = gen_array_ops.reshape(x11, [48, -1]) + + out = array_ops.concat( + [x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11], axis=-1) + out = array_ops.squeeze(out, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py new file mode 100644 index 0000000000..5233a493d0 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py @@ -0,0 +1,148 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import math_ops +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class BinaryTensorWeightBroadcastTest(BaseUnitTest): + """unit tests for scale & elementwise layers in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(BinaryTensorWeightBroadcastTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (10, 24, 24, 20) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 35 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.1 + self.allclose_atol = 0.05 + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + # scale + a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant( + np.random.randn(24, 24, 20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # scale + a = constant_op.constant( + np.random.randn(24, 24, 20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 1, 1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 1, 1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 24, 1), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 24, 1), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 24, 20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant( + np.random.randn(1, 24, 24, 20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32) + f = a + x + x = math_ops.sigmoid(f) + + # elementwise + a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32) + f = x + a + x = math_ops.sigmoid(f) + + gen_array_ops.reshape(x, [5, -1], name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py new file mode 100644 index 0000000000..de0817d2e8 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/concatenation_test.py @@ -0,0 +1,87 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class ConcatenationTest(BaseUnitTest): + """Testing Concatenation in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(ConcatenationTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (2, 3, 3, 1) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 4 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + + # scale + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r1 = x / a + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r2 = a / x + a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32) + r3 = a + x + a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32) + r4 = x * a + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r5 = x - a + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r6 = a - x + a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) + r7 = x - a + a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) + r8 = a - x + a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32) + r9 = gen_math_ops.maximum(x, a) + a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32) + r10 = gen_math_ops.minimum(a, x) + a = constant_op.constant(np.random.randn(3), dtype=dtypes.float32) + r11 = x * a + a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32) + r12 = a * x + concat1 = array_ops.concat([r1, r2, r3, r4, r5, r6], axis=-1) + concat2 = array_ops.concat([r7, r8, r9, r10, r11, r12], axis=3) + x = array_ops.concat([concat1, concat2], axis=-1) + + gen_array_ops.reshape(x, [2, -1], name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py new file mode 100644 index 0000000000..74d39d9015 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py @@ -0,0 +1,75 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class ConstBroadcastTest(BaseUnitTest): + """Testing Constant broadcasting in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(ConstBroadcastTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (5, 12, 12, 2) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.conv_broadcast + self.expect_nb_nodes = 7 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.05 + self.allclose_atol = 0.05 + + def conv_broadcast(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + filt1 = constant_op.constant( + 1, shape=(3, 3, 2, 1), dtype=dtypes.float32, name='filt1') + y1 = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='SAME', name='y1') + z1 = nn.relu(y1, name='z1') + filt2 = constant_op.constant( + np.random.randn(9), + shape=(3, 3, 1, 1), + dtype=dtypes.float32, + name='filt2') + y2 = nn.conv2d(z1, filt2, strides=[1, 1, 1, 1], padding='SAME', name='y2') + z2 = nn.relu(y2, name='z') + filt3 = constant_op.constant( + np.random.randn(3, 3, 1, 1), + shape=(3, 3, 1, 1), + dtype=dtypes.float32, + name='filt3') + y3 = nn.conv2d(z2, filt3, strides=[1, 1, 1, 1], padding='SAME', name='y3') + z = nn.relu(y3, name='output') + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py new file mode 100644 index 0000000000..291b4d16c1 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py @@ -0,0 +1,101 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class MultiConnectionNeighborEngineTest(BaseUnitTest): + """Multi connection neighboring nodes wiring tests in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(MultiConnectionNeighborEngineTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (2, 3, 7, 5) + self.dummy_input = np.random.normal(1.0, 0.5, self.inp_dims) + self.get_network = self.neighboring_tensor_test + self.expect_nb_nodes = 7 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.05 + self.allclose_atol = 0.05 + + def neighboring_tensor_test(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + e = constant_op.constant( + np.random.normal(.05, .005, [3, 2, 3, 4]), + name="weights", + dtype=dtypes.float32) + conv = nn.conv2d( + input=x, + filter=e, + data_format="NCHW", + strides=[1, 1, 1, 1], + padding="VALID", + name="conv") + b = constant_op.constant( + np.random.normal(2.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + t = conv + b + + b = constant_op.constant( + np.random.normal(5.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + q = conv - b + edge = math_ops.sigmoid(q) + + b = constant_op.constant( + np.random.normal(5.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + d = b + conv + edge3 = math_ops.sigmoid(d) + + c = constant_op.constant( + np.random.normal(1.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + edge1 = gen_math_ops.tan(conv) + t = t - edge1 + q = q + edge + t = t + q + t = t + d + t = t - edge3 + array_ops.squeeze(t, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py new file mode 100644 index 0000000000..f916db3504 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py @@ -0,0 +1,78 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import gen_math_ops +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class NeighboringEngineTest(BaseUnitTest): + """Neighboring node wiring tests in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(NeighboringEngineTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (2, 3, 7, 5) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.neighboring_tensor_test + self.expect_nb_nodes = 5 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.allclose_rtol = 0.05 + self.allclose_atol = 0.05 + + def neighboring_tensor_test(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + e = constant_op.constant( + np.random.normal(.3, 0.05, [3, 2, 3, 4]), + name="weights", + dtype=dtypes.float32) + conv = nn.conv2d( + input=x, + filter=e, + data_format="NCHW", + strides=[1, 1, 1, 1], + padding="VALID", + name="conv") + b = constant_op.constant( + np.random.normal(1.0, 1.0, [1, 4, 1, 1]), + name="bias", + dtype=dtypes.float32) + t = conv * b + + e = gen_math_ops.tan(conv) + t = t - e + array_ops.squeeze(t, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/run_test.py b/tensorflow/contrib/tensorrt/test/run_test.py new file mode 100644 index 0000000000..4d109cc378 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/run_test.py @@ -0,0 +1,184 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""script to convert and execute TF-TensorRT graph.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 +from tensorflow.core.protobuf import rewriter_config_pb2 +from tensorflow.python.client import session +from tensorflow.python.framework import importer +from tensorflow.python.framework import ops +from tensorflow.python.training import training +from tensorflow.contrib.tensorrt.test.utilities import get_all_variables + +OUTPUT_NODE = "output" +INPUT_NODE = "input" +CALIB_COUNT = 5 # calibration iteration + + +class RunTest: + """base class to run TR-TRT conversion and execution""" + + def __init__(self): + self.clean() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.clean() + + def clean(self): + self.tftrt = {} + self.tftrt_conversion_flag = {} + self.tftrt_nb_nodes = {} + self.tftrt_result = {} + self.tftrt_dynamic_conversion_flag = {} + self.tftrt_dynamic_result = {} + self.check_file = None + self.native_network = None + + def run_test(self, + network, + static_mode_list, + dynamic_mode_list, + dummy_input, + file_name=None): + self.native_network = network() + success = True + initialization = False + if file_name != None: + initialization = True + self.check_file = file_name + self.native_result, self.native_nb_nodes = self.execute_graph( + self.native_network, dummy_input, initialization) + for mode in static_mode_list: + try: + self.run_static_convert_network(mode, dummy_input, initialization) + self.tftrt_conversion_flag[mode] = True + except Exception as inst: + self.tftrt_conversion_flag[mode] = False + success = False + for mode in dynamic_mode_list: + try: + self.run_dynamic_convert_network(mode, dummy_input, initialization) + self.tftrt_dynamic_conversion_flag[mode] = True + except Exception as inst: + self.tftrt_dynamic_conversion_flag[mode] = False + success = False + return success + + def run_dynamic_convert_network(self, mode, dummy_input, initialization=True): + inp_dims = dummy_input.shape + if mode == "FP32" or mode == "FP16": + opt_config = rewriter_config_pb2.RewriterConfig() + opt_config.optimizers.extend(["constfold", "layout"]) + custom_op = opt_config.custom_optimizers.add() + custom_op.name = "TensorRTOptimizer" + custom_op.parameter_map["minimum_segment_size"].i = 3 + custom_op.parameter_map["precision_mode"].s = mode + custom_op.parameter_map["max_batch_size"].i = inp_dims[0] + custom_op.parameter_map["max_workspace_size_bytes"].i = 1 << 25 + print(custom_op) + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + graph_options = config_pb2.GraphOptions(rewrite_options=opt_config) + sessconfig = config_pb2.ConfigProto( + gpu_options=gpu_options, graph_options=graph_options) + print(sessconfig) + g = ops.Graph() + ops.reset_default_graph() + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=self.native_network, return_elements=["input", "output"]) + inp = inp.outputs[0] + out = out.outputs[0] + with session.Session(config=sessconfig, graph=g) as sess: + if (initialization): + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + saver.restore(sess, self.check_file) + self.tftrt_dynamic_result[mode] = sess.run(out, {inp: dummy_input}) + else: + raise Exception("dynamic op mode: " + mode + " not supported") + + def run_static_convert_network(self, mode, dummy_input, initialization=True): + inp_dims = dummy_input.shape + if mode == "FP32" or mode == "FP16" or mode == "INT8": + trt_graph = trt.create_inference_graph( + input_graph_def=self.native_network, + outputs=[OUTPUT_NODE], + max_batch_size=inp_dims[0], + max_workspace_size_bytes=1 << 25, + precision_mode=mode, # TRT Engine precision "FP32","FP16" or "INT8" + minimum_segment_size=2 # minimum number of nodes in an engine + ) + if mode == "INT8": + _ = self.execute_calibration(trt_graph, dummy_input, initialization) + trt_graph = trt.calib_graph_to_infer_graph(trt_graph) + trt_result, nb_nodes = self.execute_graph(trt_graph, dummy_input, + initialization) + self.tftrt[mode] = trt_graph + self.tftrt_nb_nodes[mode] = nb_nodes + self.tftrt_result[mode] = trt_result + else: + raise Exception("mode: " + mode + " not supported") + + def execute_graph(self, gdef, dummy_input, initialization=True): + """Run given graphdef once.""" + gpu_options = config_pb2.GPUOptions() + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + ops.reset_default_graph() + g = ops.Graph() + nb_nodes = 0 + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") + nb_nodes = len(g.get_operations()) + inp = inp.outputs[0] + out = out.outputs[0] + with session.Session(config=sessconfig, graph=g) as sess: + if (initialization): + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + saver.restore(sess, self.check_file) + val = sess.run(out, {inp: dummy_input}) + return val, nb_nodes + + # Use real data that is representative of the inference dataset + # for calibration. For this test script it is random data. + def execute_calibration(self, gdef, dummy_input, initialization=True): + """Run given calibration graph multiple times.""" + gpu_options = config_pb2.GPUOptions() + ops.reset_default_graph() + g = ops.Graph() + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") + inp = inp.outputs[0] + out = out.outputs[0] + with session.Session( + config=config_pb2.ConfigProto(gpu_options=gpu_options), + graph=g) as sess: + if (initialization): + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + saver.restore(sess, self.check_file) + for _ in range(CALIB_COUNT): + val = sess.run(out, {inp: dummy_input}) + return val diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py new file mode 100644 index 0000000000..a054939ce2 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unary_test.py @@ -0,0 +1,125 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.client import session +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.training import training +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest +from tensorflow.contrib.tensorrt.test.utilities import get_all_variables + + +class UnaryTest(BaseUnitTest): + """Unit tests for unary operations in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(UnaryTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (12, 5, 8, 1, 1, 12) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.unary_test + self.expect_nb_nodes = 17 + self.log_file = log_file + self.test_name = self.__class__.__name__ + self.ckpt = "./tmp.ckpt" + + def unary_test(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + q = math_ops.abs(x) + q = q + 1.0 + q = gen_math_ops.exp(q) + q = gen_math_ops.log(q) + q = array_ops.squeeze(q, axis=-2) + q = math_ops.abs(q) + q = q + 2.2 + q = gen_math_ops.sqrt(q) + q = gen_math_ops.rsqrt(q) + q = math_ops.negative(q) + q = array_ops.squeeze(q, axis=3) + q = math_ops.abs(q) + q = q + 3.0 + a = gen_math_ops.reciprocal(q) + + x = constant_op.constant(np.random.randn(5, 8, 12), dtype=dtypes.float32) + q = math_ops.abs(x) + q = q + 2.0 + q = gen_math_ops.exp(q) + q = gen_math_ops.log(q) + q = math_ops.abs(q) + q = q + 2.1 + q = gen_math_ops.sqrt(q) + q = gen_math_ops.rsqrt(q) + q = math_ops.negative(q) + q = math_ops.abs(q) + q = q + 4.0 + b = gen_math_ops.reciprocal(q) + + # TODO(jie): this one will break, broadcasting on batch. + x = variable_scope.get_variable( + "test", [12, 40, 12], + dtype=dtypes.float32, + initializer=init_ops.truncated_normal_initializer) + x = gen_array_ops.reshape(x, [12, 5, 8, 1, 12, 1, 1]) + q = math_ops.abs(x) + q = q + 5.0 + q = gen_math_ops.exp(q) + q = array_ops.squeeze(q, axis=[-1, -2, 3]) + q = gen_math_ops.log(q) + q = math_ops.abs(q) + q = q + 5.1 + q = gen_array_ops.reshape(q, [12, 5, 1, 1, 8, 1, 12]) + q = array_ops.squeeze(q, axis=[5, 2, 3]) + q = gen_math_ops.sqrt(q) + q = math_ops.abs(q) + q = q + 5.2 + q = gen_math_ops.rsqrt(q) + q = math_ops.negative(q) + q = math_ops.abs(q) + q = q + 5.3 + c = gen_math_ops.reciprocal(q) + + q = a * b + q = q / c + array_ops.squeeze(q, name="output") + + with session.Session(config=sessconfig, graph=g) as sess: + names_var_list = get_all_variables(sess) + saver = training.Saver(names_var_list) + sess.run(variables.global_variables_initializer()) + saver.save(sess, self.ckpt) + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/unit_tests.py b/tensorflow/contrib/tensorrt/test/unit_tests.py new file mode 100644 index 0000000000..ac6e3b13ee --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/unit_tests.py @@ -0,0 +1,67 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Script to execute and log all integration tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.tensorrt.test.batch_matmul_test import BatchMatMulTest +from tensorflow.contrib.tensorrt.test.biasadd_matmul_test import BiasaddMatMulTest +from tensorflow.contrib.tensorrt.test.binary_tensor_weight_broadcast_test import BinaryTensorWeightBroadcastTest +from tensorflow.contrib.tensorrt.test.concatenation_test import ConcatenationTest +from tensorflow.contrib.tensorrt.test.multi_connection_neighbor_engine_test import MultiConnectionNeighborEngineTest +from tensorflow.contrib.tensorrt.test.neighboring_engine_test import NeighboringEngineTest +from tensorflow.contrib.tensorrt.test.unary_test import UnaryTest +from tensorflow.contrib.tensorrt.test.vgg_block_nchw_test import VGGBlockNCHWTest +from tensorflow.contrib.tensorrt.test.vgg_block_test import VGGBlockTest +from tensorflow.contrib.tensorrt.test.const_broadcast_test import ConstBroadcastTest + +from tensorflow.contrib.tensorrt.test.run_test import RunTest + +tests = 0 +passed_test = 0 + +failed_list = [] +test_list = [] + +test_list.append(BatchMatMulTest()) +test_list.append(BiasaddMatMulTest()) +test_list.append(BinaryTensorWeightBroadcastTest()) +test_list.append(ConcatenationTest()) +test_list.append(NeighboringEngineTest()) +test_list.append(UnaryTest()) +test_list.append(VGGBlockNCHWTest()) +test_list.append(VGGBlockTest()) +test_list.append(MultiConnectionNeighborEngineTest()) +test_list.append(ConstBroadcastTest()) + +for test in test_list: + test.debug = True + test.check_node_count = False + with RunTest() as context: + tests += 1 + if test.run(context): + passed_test += 1 + else: + failed_list.append(test.test_name) + print("Failed test: %s\n", test.test_name) + +if passed_test == tests: + print("Passed\n") +else: + print(("%d out of %d passed\n -- failed list:") % (passed_test, tests)) + for test in failed_list: + print(" - " + test) diff --git a/tensorflow/contrib/tensorrt/test/utilities.py b/tensorflow/contrib/tensorrt/test/utilities.py new file mode 100644 index 0000000000..0ea5f5b883 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/utilities.py @@ -0,0 +1,30 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities script for TF-TensorRT integration tests.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.ops import variables + + +def get_all_variables(sess): + var_names = sess.run(variables.report_uninitialized_variables()) + names_var_list = {} + for name in var_names: + names_var_list[name] = sess.graph.get_tensor_by_name(name + ":0") + print(var_names) + return names_var_list diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py new file mode 100644 index 0000000000..9a759eb994 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py @@ -0,0 +1,85 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class VGGBlockNCHWTest(BaseUnitTest): + """single vgg layer in NCHW unit tests in TF-TRT""" + + def __init__(self, log_file='log.txt'): + super(VGGBlockNCHWTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (5, 2, 8, 8) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 3 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + x, mean_x, var_x = nn_impl.fused_batch_norm( + x, + np.random.randn(2).astype(np.float32), + np.random.randn(2).astype(np.float32), + mean=np.random.randn(2).astype(np.float32), + variance=np.random.randn(2).astype(np.float32), + data_format="NCHW", + is_training=False) + e = constant_op.constant( + np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32) + conv = nn.conv2d( + input=x, + filter=e, + data_format="NCHW", + strides=[1, 1, 2, 2], + padding="SAME", + name="conv") + b = constant_op.constant( + np.random.randn(6), name="bias", dtype=dtypes.float32) + t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd") + relu = nn.relu(t, "relu") + idty = array_ops.identity(relu, "ID") + v = nn_ops.max_pool( + idty, [1, 1, 2, 2], [1, 1, 2, 2], + "VALID", + data_format="NCHW", + name="max_pool") + array_ops.squeeze(v, name="output") + + return g.as_graph_def() diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py new file mode 100644 index 0000000000..04176d58ca --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/vgg_block_test.py @@ -0,0 +1,76 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Model script to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import nn_impl +from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest + + +class VGGBlockTest(BaseUnitTest): + """single vgg layer test in TF-TRT conversion""" + + def __init__(self, log_file='log.txt'): + super(VGGBlockTest, self).__init__() + self.static_mode_list = {"FP32", "FP16"} + self.debug = True + self.dynamic_mode_list = {} + self.inp_dims = (5, 8, 8, 2) + self.dummy_input = np.random.random_sample(self.inp_dims) + self.get_network = self.get_simple_graph_def + self.expect_nb_nodes = 7 + self.log_file = log_file + self.test_name = self.__class__.__name__ + + def get_simple_graph_def(self): + g = ops.Graph() + gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) + sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) + with g.as_default(): + x = array_ops.placeholder( + dtype=dtypes.float32, shape=self.inp_dims, name="input") + x, mean_x, var_x = nn_impl.fused_batch_norm( + x, + np.random.randn(2).astype(np.float32), + np.random.randn(2).astype(np.float32), + mean=np.random.randn(2).astype(np.float32), + variance=np.random.randn(2).astype(np.float32), + is_training=False) + e = constant_op.constant( + np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32) + conv = nn.conv2d( + input=x, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv") + b = constant_op.constant( + np.random.randn(6), name="bias", dtype=dtypes.float32) + t = nn.bias_add(conv, b, name="biasAdd") + relu = nn.relu(t, "relu") + idty = array_ops.identity(relu, "ID") + v = nn_ops.max_pool( + idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool") + array_ops.squeeze(v, name="output") + + return g.as_graph_def() -- cgit v1.2.3 From e438b192d27f2881e9c627166d73a4fdafcfeb7d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 02:19:27 -0700 Subject: Make parallel_gpu_execute work on Windows PiperOrigin-RevId: 204441157 --- .../ci_build/gpu_build/parallel_gpu_execute.sh | 28 ++++++++++++++++++++-- .../ci_build/windows/gpu/pip/build_tf_windows.sh | 6 ++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/tensorflow/tools/ci_build/gpu_build/parallel_gpu_execute.sh b/tensorflow/tools/ci_build/gpu_build/parallel_gpu_execute.sh index d0816c92b7..75da9bb835 100755 --- a/tensorflow/tools/ci_build/gpu_build/parallel_gpu_execute.sh +++ b/tensorflow/tools/ci_build/gpu_build/parallel_gpu_execute.sh @@ -35,6 +35,30 @@ elif [[ ${BASH_VER_MAJOR} -eq 4 ]] && [[ ${BASH_VER_MINOR} -lt 2 ]]; then exit 1 fi +function is_absolute { + [[ "$1" = /* ]] || [[ "$1" =~ ^[a-zA-Z]:[/\\].* ]] +} + +RUNFILES_MANIFEST_FILE="${TEST_SRCDIR}/MANIFEST" +function rlocation() { + if is_absolute "$1" ; then + # If the file path is already fully specified, simply return it. + echo "$1" + elif [[ -e "$TEST_SRCDIR/$1" ]]; then + # If the file exists in the $TEST_SRCDIR then just use it. + echo "$TEST_SRCDIR/$1" + elif [[ -e "$RUNFILES_MANIFEST_FILE" ]]; then + # If a runfiles manifest file exists then use it. + echo "$(grep "^$1 " "$RUNFILES_MANIFEST_FILE" | sed 's/[^ ]* //')" + fi +} + +TEST_BINARY="$(rlocation $TEST_WORKSPACE/${1#./})" +shift + +# Make sure /var/lock exists, this may not be true under MSYS +mkdir -p /var/lock + TF_GPU_COUNT=${TF_GPU_COUNT:-8} for i in `seq 0 $((TF_GPU_COUNT-1))`; do @@ -45,8 +69,8 @@ for i in `seq 0 $((TF_GPU_COUNT-1))`; do # This export only works within the brackets, so it is isolated to one # single command. export CUDA_VISIBLE_DEVICES=$i - echo "Running test $* on GPU $CUDA_VISIBLE_DEVICES" - $@ + echo "Running test $TEST_BINARY $* on GPU $CUDA_VISIBLE_DEVICES" + "$TEST_BINARY" $@ ) return_code=$? flock -u "$lock_fd" diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh index fe3bce428f..36b2142d95 100644 --- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh @@ -105,14 +105,18 @@ create_python_test_dir "${PY_TEST_DIR}" PIP_NAME=$(ls ${PY_TEST_DIR}/tensorflow-*.whl) reinstall_tensorflow_pip ${PIP_NAME} +TF_GPU_COUNT=${TF_GPU_COUNT:-8} + # Define no_tensorflow_py_deps=true so that every py_test has no deps anymore, # which will result testing system installed tensorflow # GPU tests are very flaky when running concurrently, so set local_test_jobs=1 bazel test --announce_rc --config=opt -k --test_output=errors \ + --test_env=TF_GPU_COUNT \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ --define=no_tensorflow_py_deps=true --test_lang_filters=py \ --test_tag_filters=-no_pip,-no_windows,-no_windows_gpu,-no_gpu,-no_pip_gpu,-no_oss \ --build_tag_filters=-no_pip,-no_windows,-no_windows_gpu,-no_gpu,-no_pip_gpu,-no_oss --build_tests_only \ - --local_test_jobs=1 --test_timeout="300,450,1200,3600" \ + --local_test_jobs=$TF_GPU_COUNT --test_timeout="300,450,1200,3600" \ --flaky_test_attempts=3 \ //${PY_TEST_DIR}/tensorflow/python/... \ //${PY_TEST_DIR}/tensorflow/contrib/... -- cgit v1.2.3 From 895a7667884545a68480eb91916a5a23c2852308 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 04:51:48 -0700 Subject: Add initial support for interpolating filename and line number in error messages returned from C++. PiperOrigin-RevId: 204455158 --- tensorflow/python/BUILD | 5 +- tensorflow/python/framework/error_interpolation.py | 82 +++++++++++++++- .../python/framework/error_interpolation_test.py | 104 +++++++++++++++++++-- tensorflow/python/util/tf_stack.py | 6 ++ 4 files changed, 182 insertions(+), 15 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 924db54cbc..2fba3c2acb 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -705,7 +705,9 @@ py_library( "framework/error_interpolation.py", ], srcs_version = "PY2AND3", - deps = [], + deps = [ + ":util", + ], ) py_library( @@ -1040,6 +1042,7 @@ py_test( srcs_version = "PY2AND3", deps = [ ":client_testlib", + ":constant_op", ":error_interpolation", ], ) diff --git a/tensorflow/python/framework/error_interpolation.py b/tensorflow/python/framework/error_interpolation.py index 9ccae76147..519e0fda0a 100644 --- a/tensorflow/python/framework/error_interpolation.py +++ b/tensorflow/python/framework/error_interpolation.py @@ -29,6 +29,9 @@ import string import six +from tensorflow.python.util import tf_stack + + _NAME_REGEX = r"[A-Za-z0-9.][A-Za-z0-9_.\-/]*?" _FORMAT_REGEX = r"[A-Za-z0-9_.\-/${}:]+" _TAG_REGEX = r"\^\^({name}):({name}):({fmt})\^\^".format( @@ -38,6 +41,8 @@ _INTERPOLATION_PATTERN = re.compile(_INTERPOLATION_REGEX) _ParseTag = collections.namedtuple("_ParseTag", ["type", "name", "format"]) +_BAD_FILE_SUBSTRINGS = ["tensorflow/python", " + file: Replaced with the filename in which the node was defined. + line: Replaced by the line number at which the node was defined. + Args: message: String to parse @@ -72,9 +81,47 @@ def _parse_message(message): return seps, tags -# TODO(jtkeeling): Modify to actually interpolate format strings rather than -# echoing them. -def interpolate(error_message): +def _get_field_dict_from_traceback(tf_traceback, frame_index): + """Convert traceback elements into interpolation dictionary and return.""" + frame = tf_traceback[frame_index] + return { + "file": frame[tf_stack.TB_FILENAME], + "line": frame[tf_stack.TB_LINENO], + } + + +def _find_index_of_defining_frame_for_op(op): + """Return index in op._traceback with first 'useful' frame. + + This method reads through the stack stored in op._traceback looking for the + innermost frame which (hopefully) belongs to the caller. It accomplishes this + by rejecting frames whose filename appears to come from TensorFlow (see + error_interpolation._BAD_FILE_SUBSTRINGS for the list of rejected substrings). + + Args: + op: the Operation object for which we would like to find the defining + location. + + Returns: + Integer index into op._traceback where the first non-TF file was found + (innermost to outermost), or 0 (for the outermost stack frame) if all files + came from TensorFlow. + """ + # pylint: disable=protected-access + # Index 0 of tf_traceback is the outermost frame. + tf_traceback = tf_stack.convert_stack(op._traceback) + size = len(tf_traceback) + # pylint: enable=protected-access + filenames = [frame[tf_stack.TB_FILENAME] for frame in tf_traceback] + # We process the filenames from the innermost frame to outermost. + for idx, filename in enumerate(reversed(filenames)): + contains_bad_substrings = [ss in filename for ss in _BAD_FILE_SUBSTRINGS] + if not any(contains_bad_substrings): + return size - idx - 1 + return 0 + + +def interpolate(error_message, graph): """Interpolates an error message. The error message can contain tags of the form ^^type:name:format^^ which will @@ -82,11 +129,38 @@ def interpolate(error_message): Args: error_message: A string to interpolate. + graph: ops.Graph object containing all nodes referenced in the error + message. Returns: The string with tags of the form ^^type:name:format^^ interpolated. """ seps, tags = _parse_message(error_message) - subs = [string.Template(tag.format).safe_substitute({}) for tag in tags] + + node_name_to_substitution_dict = {} + for name in [t.name for t in tags]: + try: + op = graph.get_operation_by_name(name) + except KeyError: + op = None + + if op: + frame_index = _find_index_of_defining_frame_for_op(op) + # pylint: disable=protected-access + field_dict = _get_field_dict_from_traceback(op._traceback, frame_index) + # pylint: enable=protected-access + else: + field_dict = { + "file": "", + "line": "", + "func": "", + "code": None, + } + node_name_to_substitution_dict[name] = field_dict + + subs = [ + string.Template(tag.format).safe_substitute( + node_name_to_substitution_dict[tag.name]) for tag in tags + ] return "".join( itertools.chain(*six.moves.zip_longest(seps, subs, fillvalue=""))) diff --git a/tensorflow/python/framework/error_interpolation_test.py b/tensorflow/python/framework/error_interpolation_test.py index ad448deb62..091f0da2a2 100644 --- a/tensorflow/python/framework/error_interpolation_test.py +++ b/tensorflow/python/framework/error_interpolation_test.py @@ -18,31 +18,115 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import constant_op from tensorflow.python.framework import error_interpolation from tensorflow.python.platform import test +from tensorflow.python.util import tf_stack + + +def _make_frame_with_filename(op, idx, filename): + """Return a copy of an existing stack frame with a new filename.""" + stack_frame = list(op._traceback[idx]) + stack_frame[tf_stack.TB_FILENAME] = filename + return tuple(stack_frame) + + +def _modify_op_stack_with_filenames(op, num_user_frames, user_filename, + num_inner_tf_frames): + """Replace op._traceback with a new traceback using special filenames.""" + tf_filename = "%d" + error_interpolation._BAD_FILE_SUBSTRINGS[0] + user_filename = "%d/my_favorite_file.py" + + num_requested_frames = num_user_frames + num_inner_tf_frames + num_actual_frames = len(op._traceback) + num_outer_frames = num_actual_frames - num_requested_frames + assert num_requested_frames <= num_actual_frames, "Too few real frames." + + # The op's traceback has outermost frame at index 0. + stack = [] + for idx in range(0, num_outer_frames): + stack.append(op._traceback[idx]) + for idx in range(len(stack), len(stack)+num_user_frames): + stack.append(_make_frame_with_filename(op, idx, user_filename % idx)) + for idx in range(len(stack), len(stack)+num_inner_tf_frames): + stack.append(_make_frame_with_filename(op, idx, tf_filename % idx)) + op._traceback = stack class InterpolateTest(test.TestCase): + def setUp(self): + # Add nodes to the graph for retrieval by name later. + constant_op.constant(1, name="One") + constant_op.constant(2, name="Two") + three = constant_op.constant(3, name="Three") + self.graph = three.graph + + # Change the list of bad file substrings so that constant_op.py is chosen + # as the defining stack frame for constant_op.constant ops. + self.old_bad_strings = error_interpolation._BAD_FILE_SUBSTRINGS + error_interpolation._BAD_FILE_SUBSTRINGS = ["/ops.py", "/util"] + + def tearDown(self): + error_interpolation._BAD_FILE_SUBSTRINGS = self.old_bad_strings + + def testFindIndexOfDefiningFrameForOp(self): + local_op = constant_op.constant(42).op + user_filename = "hope.py" + _modify_op_stack_with_filenames(local_op, + num_user_frames=3, + user_filename=user_filename, + num_inner_tf_frames=5) + idx = error_interpolation._find_index_of_defining_frame_for_op(local_op) + # Expected frame is 6th from the end because there are 5 inner frames witih + # TF filenames. + expected_frame = len(local_op._traceback) - 6 + self.assertEqual(expected_frame, idx) + + def testFindIndexOfDefiningFrameForOpReturnsZeroOnError(self): + local_op = constant_op.constant(43).op + # Truncate stack to known length. + local_op._traceback = local_op._traceback[:7] + # Ensure all frames look like TF frames. + _modify_op_stack_with_filenames(local_op, + num_user_frames=0, + user_filename="user_file.py", + num_inner_tf_frames=7) + idx = error_interpolation._find_index_of_defining_frame_for_op(local_op) + self.assertEqual(0, idx) + def testNothingToDo(self): normal_string = "This is just a normal string" - interpolated_string = error_interpolation.interpolate(normal_string) + interpolated_string = error_interpolation.interpolate(normal_string, + self.graph) self.assertEqual(interpolated_string, normal_string) def testOneTag(self): - one_tag_string = "^^node:Foo:${file}^^" - interpolated_string = error_interpolation.interpolate(one_tag_string) - self.assertEqual(interpolated_string, "${file}") + one_tag_string = "^^node:Two:${file}^^" + interpolated_string = error_interpolation.interpolate(one_tag_string, + self.graph) + self.assertTrue(interpolated_string.endswith("constant_op.py"), + "interpolated_string '%s' did not end with constant_op.py" + % interpolated_string) + + def testOneTagWithAFakeNameResultsInPlaceholders(self): + one_tag_string = "^^node:MinusOne:${file}^^" + interpolated_string = error_interpolation.interpolate(one_tag_string, + self.graph) + self.assertEqual(interpolated_string, "") def testTwoTagsNoSeps(self): - two_tags_no_seps = "^^node:Foo:${file}^^^^node:Bar:${line}^^" - interpolated_string = error_interpolation.interpolate(two_tags_no_seps) - self.assertEqual(interpolated_string, "${file}${line}") + two_tags_no_seps = "^^node:One:${file}^^^^node:Three:${line}^^" + interpolated_string = error_interpolation.interpolate(two_tags_no_seps, + self.graph) + self.assertRegexpMatches(interpolated_string, "constant_op.py[0-9]+") def testTwoTagsWithSeps(self): - two_tags_with_seps = "123^^node:Foo:${file}^^456^^node:Bar:${line}^^789" - interpolated_string = error_interpolation.interpolate(two_tags_with_seps) - self.assertEqual(interpolated_string, "123${file}456${line}789") + two_tags_with_seps = ";;;^^node:Two:${file}^^,,,^^node:Three:${line}^^;;;" + interpolated_string = error_interpolation.interpolate(two_tags_with_seps, + self.graph) + expected_regex = "^;;;.*constant_op.py,,,[0-9]*;;;$" + self.assertRegexpMatches(interpolated_string, expected_regex) if __name__ == "__main__": diff --git a/tensorflow/python/util/tf_stack.py b/tensorflow/python/util/tf_stack.py index dacc1ce83e..fe4f4a63eb 100644 --- a/tensorflow/python/util/tf_stack.py +++ b/tensorflow/python/util/tf_stack.py @@ -21,6 +21,12 @@ from __future__ import print_function import linecache import sys +# Names for indices into TF traceback tuples. +TB_FILENAME = 0 +TB_LINENO = 1 +TB_FUNCNAME = 2 +TB_CODEDICT = 3 # Dictionary of Python interpreter state. + def extract_stack(extract_frame_info_fn=None): """A lightweight, extensible re-implementation of traceback.extract_stack. -- cgit v1.2.3 From 8e6fce0fc7bd541b3af1b27dac0e9cf682360b65 Mon Sep 17 00:00:00 2001 From: Dmitry Klimenkov Date: Fri, 13 Jul 2018 16:44:18 +0300 Subject: some rnn.py fixes were incorrect --- tensorflow/python/ops/rnn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py index aea01a8081..6e1c23f928 100644 --- a/tensorflow/python/ops/rnn.py +++ b/tensorflow/python/ops/rnn.py @@ -434,7 +434,7 @@ def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, with vs.variable_scope("bw") as bw_scope: inputs_reverse = _reverse( inputs, seq_lengths=sequence_length, - seq_axis=time_dim, batch_axis=batch_dim) + seq_dim=time_dim, batch_dim=batch_dim) tmp, output_state_bw = dynamic_rnn( cell=cell_bw, inputs=inputs_reverse, sequence_length=sequence_length, initial_state=initial_state_bw, dtype=dtype, @@ -443,7 +443,7 @@ def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, output_bw = _reverse( tmp, seq_lengths=sequence_length, - seq_axis=time_dim, batch_axis=batch_dim) + seq_dim=time_dim, batch_dim=batch_dim) outputs = (output_fw, output_bw) output_states = (output_state_fw, output_state_bw) -- cgit v1.2.3 From 98010279f40e4963512ba2f2f39c3d732aef7b93 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 07:21:37 -0700 Subject: TF SavedModel: Split off a reader from the loader module PiperOrigin-RevId: 204468340 --- tensorflow/cc/saved_model/BUILD | 30 ++++++ tensorflow/cc/saved_model/loader.cc | 70 ++----------- tensorflow/cc/saved_model/reader.cc | 88 +++++++++++++++++ tensorflow/cc/saved_model/reader.h | 39 ++++++++ tensorflow/cc/saved_model/reader_test.cc | 108 +++++++++++++++++++++ .../java/org/tensorflow/SavedModelBundleTest.java | 2 +- 6 files changed, 272 insertions(+), 65 deletions(-) create mode 100644 tensorflow/cc/saved_model/reader.cc create mode 100644 tensorflow/cc/saved_model/reader.h create mode 100644 tensorflow/cc/saved_model/reader_test.cc diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD index 06a3be18e0..730b1b669b 100644 --- a/tensorflow/cc/saved_model/BUILD +++ b/tensorflow/cc/saved_model/BUILD @@ -33,6 +33,35 @@ cc_library( hdrs = ["tag_constants.h"], ) +cc_library( + name = "reader", + srcs = ["reader.cc"], + hdrs = ["reader.h"], + deps = [ + ":constants", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + ], +) + +tf_cc_test( + name = "reader_test", + srcs = ["reader_test.cc"], + data = [ + ":saved_model_half_plus_two", + ], + linkstatic = 1, + deps = [ + ":constants", + ":reader", + ":tag_constants", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + cc_library( name = "loader", hdrs = ["loader.h"], @@ -54,6 +83,7 @@ cc_library( hdrs = ["loader.h"], deps = [ ":constants", + ":reader", ] + if_not_mobile([ "//tensorflow/core:core_cpu", "//tensorflow/core:framework", diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc index faa1e378d0..07807ed2f3 100644 --- a/tensorflow/cc/saved_model/loader.cc +++ b/tensorflow/cc/saved_model/loader.cc @@ -18,8 +18,10 @@ limitations under the License. #include #include "tensorflow/cc/saved_model/constants.h" +#include "tensorflow/cc/saved_model/reader.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/monitoring/counter.h" +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/protobuf_internal.h" @@ -43,56 +45,6 @@ auto* load_latency = monitoring::Counter<1>::New( constexpr char kLoadAttemptFail[] = "fail"; constexpr char kLoadAttemptSuccess[] = "success"; -Status ReadSavedModel(const string& export_dir, SavedModel* saved_model_proto) { - const string saved_model_pb_path = - io::JoinPath(export_dir, kSavedModelFilenamePb); - if (Env::Default()->FileExists(saved_model_pb_path).ok()) { - return ReadBinaryProto(Env::Default(), saved_model_pb_path, - saved_model_proto); - } - const string saved_model_pbtxt_path = - io::JoinPath(export_dir, kSavedModelFilenamePbTxt); - if (Env::Default()->FileExists(saved_model_pbtxt_path).ok()) { - return ReadTextProto(Env::Default(), saved_model_pbtxt_path, - saved_model_proto); - } - return Status(error::Code::NOT_FOUND, - "Could not find SavedModel .pb or .pbtxt at supplied export " - "directory path: " + - export_dir); -} - -string GetTagsAsString(const std::unordered_set& tags) { - string tags_as_string = "{ "; - for (const string& tag : tags) { - tags_as_string = strings::StrCat(tags_as_string, tag, " "); - } - tags_as_string = strings::StrCat(tags_as_string, "}"); - return tags_as_string; -} - -Status FindMetaGraphDefToLoad(const SavedModel& saved_model_proto, - const std::unordered_set& tags, - MetaGraphDef* meta_graph_def_to_load) { - for (const MetaGraphDef& meta_graph_def : saved_model_proto.meta_graphs()) { - // Get tags from the meta_graph_def. - std::unordered_set graph_tags; - for (const string& tag : meta_graph_def.meta_info_def().tags()) { - graph_tags.insert(tag); - } - // Match with the set of tags provided. - if (graph_tags == tags) { - *meta_graph_def_to_load = meta_graph_def; - return Status::OK(); - } - } - return Status(error::Code::NOT_FOUND, - "Could not find meta graph def matching supplied tags: " + - GetTagsAsString(tags) + - ". To inspect available tag-sets in the SavedModel, please " - "use the SavedModel CLI: `saved_model_cli`"); -} - Status LoadMetaGraphIntoSession(const MetaGraphDef& meta_graph_def, const SessionOptions& session_options, std::unique_ptr* session) { @@ -235,18 +187,8 @@ Status LoadSavedModelInternal(const SessionOptions& session_options, const string& export_dir, const std::unordered_set& tags, SavedModelBundle* const bundle) { - if (!MaybeSavedModelDirectory(export_dir)) { - return Status(error::Code::NOT_FOUND, - "SavedModel not found in export directory: " + export_dir); - } - LOG(INFO) << "Loading SavedModel with tags: " << GetTagsAsString(tags) - << "; from: " << export_dir; - - SavedModel saved_model_proto; - TF_RETURN_IF_ERROR(ReadSavedModel(export_dir, &saved_model_proto)); - - TF_RETURN_IF_ERROR( - FindMetaGraphDefToLoad(saved_model_proto, tags, &bundle->meta_graph_def)); + TF_RETURN_IF_ERROR(ReadMetaGraphDefFromSavedModel(export_dir, tags, + &bundle->meta_graph_def)); TF_RETURN_IF_ERROR(LoadMetaGraphIntoSession( bundle->meta_graph_def, session_options, &bundle->session)); @@ -288,8 +230,8 @@ Status LoadSavedModel(const SessionOptions& session_options, return end_microseconds - start_microseconds; }(); auto log_and_count = [&](const string& status_str) { - LOG(INFO) << "SavedModel load for tags " << GetTagsAsString(tags) - << "; Status: " << status_str << ". Took " + LOG(INFO) << "SavedModel load for tags { " << str_util::Join(tags, " ") + << " }; Status: " << status_str << ". Took " << load_latency_microsecs << " microseconds."; load_attempt_count->GetCell(export_dir, status_str)->IncrementBy(1); }; diff --git a/tensorflow/cc/saved_model/reader.cc b/tensorflow/cc/saved_model/reader.cc new file mode 100644 index 0000000000..2146c8a197 --- /dev/null +++ b/tensorflow/cc/saved_model/reader.cc @@ -0,0 +1,88 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/cc/saved_model/reader.h" + +#include + +#include "tensorflow/cc/saved_model/constants.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/protobuf/saved_model.pb.h" + +namespace tensorflow { +namespace { + +Status ReadSavedModel(const string& export_dir, SavedModel* saved_model_proto) { + LOG(INFO) << "Reading SavedModel from: " << export_dir; + + const string saved_model_pb_path = + io::JoinPath(export_dir, kSavedModelFilenamePb); + if (Env::Default()->FileExists(saved_model_pb_path).ok()) { + return ReadBinaryProto(Env::Default(), saved_model_pb_path, + saved_model_proto); + } + const string saved_model_pbtxt_path = + io::JoinPath(export_dir, kSavedModelFilenamePbTxt); + if (Env::Default()->FileExists(saved_model_pbtxt_path).ok()) { + return ReadTextProto(Env::Default(), saved_model_pbtxt_path, + saved_model_proto); + } + return Status(error::Code::NOT_FOUND, + "Could not find SavedModel .pb or .pbtxt at supplied export " + "directory path: " + + export_dir); +} + +Status FindMetaGraphDef(const SavedModel& saved_model_proto, + const std::unordered_set& tags, + MetaGraphDef* meta_graph_def) { + LOG(INFO) << "Reading meta graph with tags { " << str_util::Join(tags, " ") + << " }"; + for (const MetaGraphDef& graph_def : saved_model_proto.meta_graphs()) { + // Get tags from the graph_def. + std::unordered_set graph_tags; + for (const string& tag : graph_def.meta_info_def().tags()) { + graph_tags.insert(tag); + } + // Match with the set of tags provided. + if (graph_tags == tags) { + *meta_graph_def = graph_def; + return Status::OK(); + } + } + return Status( + error::Code::NOT_FOUND, + strings::StrCat( + "Could not find meta graph def matching supplied tags: { ", + str_util::Join(tags, " "), + " }. To inspect available tag-sets in the SavedModel, please " + "use the SavedModel CLI: `saved_model_cli`")); +} + +} // namespace + +Status ReadMetaGraphDefFromSavedModel(const string& export_dir, + const std::unordered_set& tags, + MetaGraphDef* const meta_graph_def) { + SavedModel saved_model_proto; + TF_RETURN_IF_ERROR(ReadSavedModel(export_dir, &saved_model_proto)); + TF_RETURN_IF_ERROR(FindMetaGraphDef(saved_model_proto, tags, meta_graph_def)); + return Status::OK(); +} + +} // namespace tensorflow diff --git a/tensorflow/cc/saved_model/reader.h b/tensorflow/cc/saved_model/reader.h new file mode 100644 index 0000000000..5815108df2 --- /dev/null +++ b/tensorflow/cc/saved_model/reader.h @@ -0,0 +1,39 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/// Functions to read the SavedModel proto, or parts of it. + +#ifndef TENSORFLOW_CC_SAVED_MODEL_READER_H_ +#define TENSORFLOW_CC_SAVED_MODEL_READER_H_ + +#include +#include + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/protobuf/meta_graph.pb.h" + +namespace tensorflow { + +// Reads the SavedModel proto from saved_model.pb(txt) in the given directory, +// finds the MetaGraphDef that matches the given set of tags and writes it to +// the `meta_graph_def` parameter. Returns a failure status when the SavedModel +// file does not exist or no MetaGraphDef matches the tags. +Status ReadMetaGraphDefFromSavedModel(const string& export_dir, + const std::unordered_set& tags, + MetaGraphDef* const meta_graph_def); + +} // namespace tensorflow + +#endif // TENSORFLOW_CC_SAVED_MODEL_READER_H_ diff --git a/tensorflow/cc/saved_model/reader_test.cc b/tensorflow/cc/saved_model/reader_test.cc new file mode 100644 index 0000000000..620e9c2eec --- /dev/null +++ b/tensorflow/cc/saved_model/reader_test.cc @@ -0,0 +1,108 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/cc/saved_model/reader.h" + +#include "tensorflow/cc/saved_model/constants.h" +#include "tensorflow/cc/saved_model/tag_constants.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +constexpr char kTestDataPbTxt[] = + "cc/saved_model/testdata/half_plus_two_pbtxt/00000123"; +constexpr char kTestDataSharded[] = + "cc/saved_model/testdata/half_plus_two/00000123"; + +class ReaderTest : public ::testing::Test { + protected: + ReaderTest() {} + + void CheckMetaGraphDef(const MetaGraphDef& meta_graph_def) { + const auto& tags = meta_graph_def.meta_info_def().tags(); + EXPECT_TRUE(std::find(tags.begin(), tags.end(), kSavedModelTagServe) != + tags.end()); + EXPECT_NE(meta_graph_def.meta_info_def().tensorflow_version(), ""); + EXPECT_EQ( + meta_graph_def.signature_def().at("serving_default").method_name(), + "tensorflow/serving/predict"); + } +}; + +TEST_F(ReaderTest, TagMatch) { + MetaGraphDef meta_graph_def; + + const string export_dir = + io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataSharded); + TF_ASSERT_OK(ReadMetaGraphDefFromSavedModel(export_dir, {kSavedModelTagServe}, + &meta_graph_def)); + CheckMetaGraphDef(meta_graph_def); +} + +TEST_F(ReaderTest, NoTagMatch) { + MetaGraphDef meta_graph_def; + + const string export_dir = + io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataSharded); + Status st = ReadMetaGraphDefFromSavedModel(export_dir, {"missing-tag"}, + &meta_graph_def); + EXPECT_FALSE(st.ok()); + EXPECT_TRUE(str_util::StrContains( + st.error_message(), + "Could not find meta graph def matching supplied tags: { missing-tag }")) + << st.error_message(); +} + +TEST_F(ReaderTest, NoTagMatchMultiple) { + MetaGraphDef meta_graph_def; + + const string export_dir = + io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataSharded); + Status st = ReadMetaGraphDefFromSavedModel( + export_dir, {kSavedModelTagServe, "missing-tag"}, &meta_graph_def); + EXPECT_FALSE(st.ok()); + EXPECT_TRUE(str_util::StrContains( + st.error_message(), + "Could not find meta graph def matching supplied tags: ")) + << st.error_message(); +} + +TEST_F(ReaderTest, PbtxtFormat) { + MetaGraphDef meta_graph_def; + + const string export_dir = + io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataPbTxt); + TF_ASSERT_OK(ReadMetaGraphDefFromSavedModel(export_dir, {kSavedModelTagServe}, + &meta_graph_def)); + CheckMetaGraphDef(meta_graph_def); +} + +TEST_F(ReaderTest, InvalidExportPath) { + MetaGraphDef meta_graph_def; + + const string export_dir = + io::JoinPath(testing::TensorFlowSrcRoot(), "missing-path"); + Status st = ReadMetaGraphDefFromSavedModel(export_dir, {kSavedModelTagServe}, + &meta_graph_def); + EXPECT_FALSE(st.ok()); +} + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/java/src/test/java/org/tensorflow/SavedModelBundleTest.java b/tensorflow/java/src/test/java/org/tensorflow/SavedModelBundleTest.java index 7922f3329c..b063b6f1cd 100644 --- a/tensorflow/java/src/test/java/org/tensorflow/SavedModelBundleTest.java +++ b/tensorflow/java/src/test/java/org/tensorflow/SavedModelBundleTest.java @@ -47,7 +47,7 @@ public class SavedModelBundleTest { fail("not expected"); } catch (org.tensorflow.TensorFlowException e) { // expected exception - assertTrue(e.getMessage().contains("SavedModel not found")); + assertTrue(e.getMessage().contains("Could not find SavedModel")); } } } -- cgit v1.2.3 From 3fc5dc5714d8ae8e5caed84c8f1751ed59b6ff35 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 07:27:00 -0700 Subject: Change TF op extension example to not use namespace "tensorflow", which does not belong to users. Users should write extensions in their own namespaces. PiperOrigin-RevId: 204468844 --- tensorflow/docs_src/extend/new_data_formats.md | 60 ++++++++++++++------------ 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/tensorflow/docs_src/extend/new_data_formats.md b/tensorflow/docs_src/extend/new_data_formats.md index d1d1f69766..abbf47910e 100644 --- a/tensorflow/docs_src/extend/new_data_formats.md +++ b/tensorflow/docs_src/extend/new_data_formats.md @@ -77,18 +77,24 @@ can be used as a starting point for your implementation: #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/shape_inference.h" -namespace tensorflow { +namespace myproject { namespace { -class MyReaderDatasetOp : public DatasetOpKernel { +using ::tensorflow::DT_STRING; +using ::tensorflow::PartialTensorShape; +using ::tensorflow::Status; + +class MyReaderDatasetOp : public tensorflow::DatasetOpKernel { public: - MyReaderDatasetOp(OpKernelConstruction* ctx) : DatasetOpKernel(ctx) { + MyReaderDatasetOp(tensorflow::OpKernelConstruction* ctx) + : DatasetOpKernel(ctx) { // Parse and validate any attrs that define the dataset using // `ctx->GetAttr()`, and store them in member variables. } - void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override { + void MakeDataset(tensorflow::OpKernelContext* ctx, + tensorflow::DatasetBase** output) override { // Parse and validate any input tensors 0that define the dataset using // `ctx->input()` or the utility function // `ParseScalarArgument(ctx, &arg)`. @@ -99,14 +105,14 @@ class MyReaderDatasetOp : public DatasetOpKernel { } private: - class Dataset : public GraphDatasetBase { + class Dataset : public tensorflow::GraphDatasetBase { public: - Dataset(OpKernelContext* ctx) : GraphDatasetBase(ctx) {} + Dataset(tensorflow::OpKernelContext* ctx) : GraphDatasetBase(ctx) {} - std::unique_ptr MakeIteratorInternal( + std::unique_ptr MakeIteratorInternal( const string& prefix) const override { - return std::unique_ptr( - new Iterator({this, strings::StrCat(prefix, "::MyReader")})); + return std::unique_ptr(new Iterator( + {this, tensorflow::strings::StrCat(prefix, "::MyReader")})); } // Record structure: Each record is represented by a scalar string tensor. @@ -114,8 +120,8 @@ class MyReaderDatasetOp : public DatasetOpKernel { // Dataset elements can have a fixed number of components of different // types and shapes; replace the following two methods to customize this // aspect of the dataset. - const DataTypeVector& output_dtypes() const override { - static DataTypeVector* dtypes = new DataTypeVector({DT_STRING}); + const tensorflow::DataTypeVector& output_dtypes() const override { + static auto* const dtypes = new tensorflow::DataTypeVector({DT_STRING}); return *dtypes; } const std::vector& output_shapes() const override { @@ -132,16 +138,16 @@ class MyReaderDatasetOp : public DatasetOpKernel { // Implement this method if you want to be able to save and restore // instances of this dataset (and any iterators over it). Status AsGraphDefInternal(DatasetGraphDefBuilder* b, - Node** output) const override { + tensorflow::Node** output) const override { // Construct nodes to represent any of the input tensors from this // object's member variables using `b->AddScalar()` and `b->AddVector()`. - std::vector input_tensors; + std::vector input_tensors; TF_RETURN_IF_ERROR(b->AddDataset(this, input_tensors, output)); return Status::OK(); } private: - class Iterator : public DatasetIterator { + class Iterator : public tensorflow::DatasetIterator { public: explicit Iterator(const Params& params) : DatasetIterator(params), i_(0) {} @@ -158,15 +164,15 @@ class MyReaderDatasetOp : public DatasetOpKernel { // return `Status::OK()`. // 3. If an error occurs, return an error status using one of the helper // functions from "tensorflow/core/lib/core/errors.h". - Status GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, + Status GetNextInternal(tensorflow::IteratorContext* ctx, + std::vector* out_tensors, bool* end_of_sequence) override { // NOTE: `GetNextInternal()` may be called concurrently, so it is // recommended that you protect the iterator state with a mutex. - mutex_lock l(mu_); + tensorflow::mutex_lock l(mu_); if (i_ < 10) { // Create a scalar string tensor and add it to the output. - Tensor record_tensor(ctx->allocator({}), DT_STRING, {}); + tensorflow::Tensor record_tensor(ctx->allocator({}), DT_STRING, {}); record_tensor.scalar()() = "MyReader!"; out_tensors->emplace_back(std::move(record_tensor)); ++i_; @@ -183,20 +189,20 @@ class MyReaderDatasetOp : public DatasetOpKernel { // // Implement these two methods if you want to be able to save and restore // instances of this iterator. - Status SaveInternal(IteratorStateWriter* writer) override { - mutex_lock l(mu_); + Status SaveInternal(tensorflow::IteratorStateWriter* writer) override { + tensorflow::mutex_lock l(mu_); TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("i"), i_)); return Status::OK(); } - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - mutex_lock l(mu_); + Status RestoreInternal(tensorflow::IteratorContext* ctx, + tensorflow::IteratorStateReader* reader) override { + tensorflow::mutex_lock l(mu_); TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("i"), &i_)); return Status::OK(); } private: - mutex mu_; + tensorflow::mutex mu_; int64 i_ GUARDED_BY(mu_); }; }; @@ -211,14 +217,14 @@ class MyReaderDatasetOp : public DatasetOpKernel { REGISTER_OP("MyReaderDataset") .Output("handle: variant") .SetIsStateful() - .SetShapeFn(shape_inference::ScalarShape); + .SetShapeFn(tensorflow::shape_inference::ScalarShape); // Register the kernel implementation for MyReaderDataset. -REGISTER_KERNEL_BUILDER(Name("MyReaderDataset").Device(DEVICE_CPU), +REGISTER_KERNEL_BUILDER(Name("MyReaderDataset").Device(tensorflow::DEVICE_CPU), MyReaderDatasetOp); } // namespace -} // namespace tensorflow +} // namespace myproject ``` The last step is to build the C++ code and add a Python wrapper. The easiest way -- cgit v1.2.3 From bef6212a1a69610053b713557d79f8a358eebdb7 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Fri, 13 Jul 2018 07:31:38 -0700 Subject: Broad refactor (part 6): Add several helpers to the base converter class. Not yet used. Testing will be covered by the individual converter tests. PiperOrigin-RevId: 204469361 --- tensorflow/contrib/autograph/core/converter.py | 120 +++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/tensorflow/contrib/autograph/core/converter.py b/tensorflow/contrib/autograph/core/converter.py index 54e6aa0f3b..a93e4a8064 100644 --- a/tensorflow/contrib/autograph/core/converter.py +++ b/tensorflow/contrib/autograph/core/converter.py @@ -64,15 +64,29 @@ from __future__ import division from __future__ import print_function import collections +from enum import Enum + from tensorflow.contrib.autograph.core import config from tensorflow.contrib.autograph.core import naming +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import ast_util +from tensorflow.contrib.autograph.pyct import cfg +from tensorflow.contrib.autograph.pyct import compiler +from tensorflow.contrib.autograph.pyct import qual_names from tensorflow.contrib.autograph.pyct import transformer +from tensorflow.contrib.autograph.pyct.static_analysis import activity +from tensorflow.contrib.autograph.pyct.static_analysis import live_values +from tensorflow.contrib.autograph.pyct.static_analysis import liveness +from tensorflow.contrib.autograph.pyct.static_analysis import reaching_definitions +from tensorflow.contrib.autograph.pyct.static_analysis import type_info # TODO(mdan): These contexts can be refactored into first class objects. # For example, we could define Program and Entity abstractions that hold on # to the actual entity and have conversion methods. +# TODO(mdan): Add a test specific to this converter. + class ProgramContext(object): """ProgramContext keeps track of converting function hierarchies. @@ -197,6 +211,46 @@ class Base(transformer.Base): self._used = False self._ast_depth = 0 + def get_definition_directive(self, node, directive, arg, default): + """Returns the unique directive for a symbol, or a default if none exist. + + See lang/directives.py for details on directives. + + Args: + node: ast.AST + directive: Callable[..., Any] + arg: str + default: Any + + Raises: + ValueError: if conflicting annotations have been found + """ + defs = anno.getanno(node, anno.Static.ORIG_DEFINITIONS, ()) + if not defs: + return default + + # TODO(mdan): Simplify this. + arg_values = [] + for def_ in defs: + if (directive not in def_.directives or + arg not in arg not in def_.directives[directive]): + continue + arg_value = def_.directives[directive][arg] + for prev_value in arg_values: + if not ast_util.matches(arg_value, prev_value): + qn = anno.getanno(node, anno.Basic.QN) + raise ValueError('%s has ambiguous annotations for %s(%s): %s, %s' % + (qn, directive.__name__, arg, + compiler.ast_to_source(arg_value).strip(), + compiler.ast_to_source(prev_value).strip())) + arg_values.append(arg_value) + + if not arg_values: + return default + + arg_value, = arg_values + return arg_value + def visit(self, node): if not self._ast_depth: if self._used: @@ -208,3 +262,69 @@ class Base(transformer.Base): return super(Base, self).visit(node) finally: self._ast_depth -= 1 + + +class AnnotatedDef(reaching_definitions.Definition): + + def __init__(self): + super(AnnotatedDef, self).__init__() + self.directives = {} + + +class AgAnno(Enum): + """Annotation labels specific to AutoGraph. See anno.py.""" + + DIRECTIVES = 'User directives associated with the annotated statement.' + + def __repr__(self): + return self.name + + +def standard_analysis(node, context, is_initial=False): + """Performs a complete static analysis of the given code. + + Args: + node: ast.AST + context: converter.EntityContext + is_initial: bool, whether this is the initial analysis done on the input + source code + + Returns: + ast.AST, same as node, with the static analysis annotations added + """ + # TODO(mdan): Clear static analysis here. + # TODO(mdan): Consider not running all analyses every time. + # TODO(mdan): Don't return a node because it's modified by reference. + graphs = cfg.build(node) + node = qual_names.resolve(node) + node = activity.resolve(node, context.info, None) + node = reaching_definitions.resolve(node, context.info, graphs, AnnotatedDef) + node = liveness.resolve(node, context.info, graphs) + node = live_values.resolve(node, context.info, config.PYTHON_LITERALS) + node = type_info.resolve(node, context.info) + # This second call allows resolving first-order class attributes. + node = live_values.resolve(node, context.info, config.PYTHON_LITERALS) + if is_initial: + anno.dup( + node, + { + anno.Static.DEFINITIONS: anno.Static.ORIG_DEFINITIONS, + }, + ) + return node + + +def apply_(node, context, converter_module): + """Applies a converter to an AST. + + Args: + node: ast.AST + context: converter.EntityContext + converter_module: converter.Base + + Returns: + ast.AST, the result of applying converter to node + """ + node = standard_analysis(node, context) + node = converter_module.transform(node, context) + return node -- cgit v1.2.3 From e8c44d765146d228ee88e46b1e1f1e8fb3894818 Mon Sep 17 00:00:00 2001 From: Dmitry Klimenkov Date: Fri, 13 Jul 2018 18:30:49 +0300 Subject: rename arguments from seq_dim to seq_axis, batch_dim to batch_axis in _reverse(...) function --- tensorflow/python/ops/rnn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py index 6e1c23f928..7521d59a60 100644 --- a/tensorflow/python/ops/rnn.py +++ b/tensorflow/python/ops/rnn.py @@ -423,18 +423,18 @@ def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, time_dim = 0 batch_dim = 1 - def _reverse(input_, seq_lengths, seq_dim, batch_dim): + def _reverse(input_, seq_lengths, seq_axis, batch_axis): if seq_lengths is not None: return array_ops.reverse_sequence( input=input_, seq_lengths=seq_lengths, - seq_axis=seq_dim, batch_axis=batch_dim) + seq_axis=seq_axis, batch_axis=batch_axis) else: - return array_ops.reverse(input_, axis=[seq_dim]) + return array_ops.reverse(input_, axis=[seq_axis]) with vs.variable_scope("bw") as bw_scope: inputs_reverse = _reverse( inputs, seq_lengths=sequence_length, - seq_dim=time_dim, batch_dim=batch_dim) + seq_axis=time_dim, batch_axis=batch_dim) tmp, output_state_bw = dynamic_rnn( cell=cell_bw, inputs=inputs_reverse, sequence_length=sequence_length, initial_state=initial_state_bw, dtype=dtype, @@ -443,7 +443,7 @@ def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, output_bw = _reverse( tmp, seq_lengths=sequence_length, - seq_dim=time_dim, batch_dim=batch_dim) + seq_axis=time_dim, batch_axis=batch_dim) outputs = (output_fw, output_bw) output_states = (output_state_fw, output_state_bw) -- cgit v1.2.3 From 23e26ec2a7521c924b2a9fc435c04761f97fe6ba Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 08:56:53 -0700 Subject: Adds the code that uses the recently added AutoGraph traceback rewriting logic. Removes the Try visit function in the single return detection visitor to allow generated function bodies to be wrapped in our error rewriting try/except blocks. PiperOrigin-RevId: 204478398 --- tensorflow/contrib/autograph/__init__.py | 7 +- tensorflow/contrib/autograph/converters/BUILD | 13 +++ .../contrib/autograph/converters/error_handlers.py | 52 ++++++++++++ .../autograph/converters/error_handlers_test.py | 61 ++++++++++++++ .../contrib/autograph/converters/single_return.py | 5 -- .../contrib/autograph/core/converter_testing.py | 3 + tensorflow/contrib/autograph/impl/api.py | 24 +++++- tensorflow/contrib/autograph/impl/api_test.py | 15 +++- tensorflow/contrib/autograph/impl/conversion.py | 9 +- .../contrib/autograph/impl/conversion_test.py | 8 +- tensorflow/contrib/autograph/pyct/ast_util.py | 31 ++++++- tensorflow/contrib/autograph/pyct/ast_util_test.py | 32 +++++-- tensorflow/contrib/autograph/pyct/cfg.py | 6 +- tensorflow/contrib/autograph/pyct/compiler.py | 97 ++++++++++++++++++++-- tensorflow/contrib/autograph/pyct/compiler_test.py | 4 +- tensorflow/contrib/autograph/pyct/origin_info.py | 56 ++++++++++++- tensorflow/contrib/autograph/pyct/templates.py | 1 + tensorflow/contrib/autograph/pyct/transformer.py | 3 +- 18 files changed, 384 insertions(+), 43 deletions(-) create mode 100644 tensorflow/contrib/autograph/converters/error_handlers.py create mode 100644 tensorflow/contrib/autograph/converters/error_handlers_test.py diff --git a/tensorflow/contrib/autograph/__init__.py b/tensorflow/contrib/autograph/__init__.py index 4f8ef2d8a1..7821c98f1c 100644 --- a/tensorflow/contrib/autograph/__init__.py +++ b/tensorflow/contrib/autograph/__init__.py @@ -30,7 +30,6 @@ from tensorflow.contrib.autograph.impl.api import do_not_convert from tensorflow.contrib.autograph.impl.api import RunMode from tensorflow.contrib.autograph.impl.api import to_code from tensorflow.contrib.autograph.core.errors import improved_errors -from tensorflow.contrib.autograph.core.errors import rewrite_graph_construction_error from tensorflow.contrib.autograph.core.errors import GraphConstructionError from tensorflow.contrib.autograph.core.errors import TfRuntimeError from tensorflow.contrib.autograph.impl.api import to_graph @@ -46,12 +45,14 @@ _allowed_symbols = [ 'convert', 'converted_call', 'do_not_convert', - 'improved_errors', 'to_code', 'to_graph', # Overloaded operators 'operators', - 'rewrite_graph_construction_error', + # Errors + 'improved_errors', + 'GraphConstructionError', + 'TfRuntimeError', # Python language "extensions" 'set_element_type', 'set_loop_options', diff --git a/tensorflow/contrib/autograph/converters/BUILD b/tensorflow/contrib/autograph/converters/BUILD index b2e2e27673..33d8d517a5 100644 --- a/tensorflow/contrib/autograph/converters/BUILD +++ b/tensorflow/contrib/autograph/converters/BUILD @@ -24,6 +24,7 @@ py_library( "continue_statements.py", "control_flow.py", "decorators.py", + "error_handlers.py", "ifexp.py", "list_comprehension.py", "lists.py", @@ -215,6 +216,18 @@ py_test( ], ) +py_test( + name = "error_handlers_test", + srcs = ["error_handlers_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":converters", + "//tensorflow/contrib/autograph/core:test_lib", + "//tensorflow/contrib/autograph/pyct", + "//tensorflow/python:client_testlib", + ], +) + py_test( name = "slices_test", srcs = ["slices_test.py"], diff --git a/tensorflow/contrib/autograph/converters/error_handlers.py b/tensorflow/contrib/autograph/converters/error_handlers.py new file mode 100644 index 0000000000..3f23662152 --- /dev/null +++ b/tensorflow/contrib/autograph/converters/error_handlers.py @@ -0,0 +1,52 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Wraps function bodies with a try/except to rewrite error tracebacks. + +Only adds try/except wrappers to functions that have the anno.Basic.ORIGIN +annotation because these are the functions originally written by the user. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.autograph.core import converter +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import templates + + +class ErrorRewritingTransformer(converter.Base): + """Possibly wraps the body of a function in a try/except. + + Only wraps functions that were originally defined by the user, detected by + checking for the anno.Basic.ORIGIN annotation. + """ + + def visit_FunctionDef(self, node): + node = self.generic_visit(node) + + if anno.hasanno(node, anno.Basic.ORIGIN): + template = """ + try: + body + except: + ag__.rewrite_graph_construction_error(ag_source_map__) + """ + node.body = templates.replace(template, body=node.body) + return node + + +def transform(node, ctx): + return ErrorRewritingTransformer(ctx).visit(node) diff --git a/tensorflow/contrib/autograph/converters/error_handlers_test.py b/tensorflow/contrib/autograph/converters/error_handlers_test.py new file mode 100644 index 0000000000..408e35b4b6 --- /dev/null +++ b/tensorflow/contrib/autograph/converters/error_handlers_test.py @@ -0,0 +1,61 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for error_handlers module.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.autograph.converters import error_handlers +from tensorflow.contrib.autograph.core import converter_testing +from tensorflow.contrib.autograph.core import errors +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import origin_info +from tensorflow.python.platform import test + + +class ErrorHandlersTest(converter_testing.TestCase): + + def compiled_fn(self, test_fn, add_origin=False): + node = self.parse_and_analyze(test_fn, {}) + if add_origin: + anno.setanno(node.body[0], anno.Basic.ORIGIN, + origin_info.OriginInfo(__file__, None, None, None, None)) + node = error_handlers.transform(node, self.ctx) + module = self.compiled(node,) + return module + + def test_no_origin_annotation(self): + + def test_fn(): + raise ValueError('Crash!') + + with self.compiled_fn(test_fn) as result: + with self.assertRaises(ValueError): + result.test_fn() + + def test_wraps_body(self): + + def test_fn(): + raise ValueError('Crash!') + + with self.compiled_fn(test_fn, add_origin=True) as result: + result.rewrite_graph_construction_error = None + with self.assertRaises(errors.GraphConstructionError): + result.test_fn() + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/autograph/converters/single_return.py b/tensorflow/contrib/autograph/converters/single_return.py index a351cd81b8..3b9c9a06d8 100644 --- a/tensorflow/contrib/autograph/converters/single_return.py +++ b/tensorflow/contrib/autograph/converters/single_return.py @@ -224,11 +224,6 @@ class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor): self.generic_visit(node) self.cant_return = False - def visit_Try(self, node): - self.cant_return = True - self.generic_visit(node) - self.cant_return = False - def visit_Return(self, node): if self.cant_return: raise ValueError( diff --git a/tensorflow/contrib/autograph/core/converter_testing.py b/tensorflow/contrib/autograph/core/converter_testing.py index 0e46aacc12..c47b70f15c 100644 --- a/tensorflow/contrib/autograph/core/converter_testing.py +++ b/tensorflow/contrib/autograph/core/converter_testing.py @@ -25,6 +25,7 @@ from tensorflow.contrib.autograph import operators from tensorflow.contrib.autograph import utils from tensorflow.contrib.autograph.core import config from tensorflow.contrib.autograph.core import converter +from tensorflow.contrib.autograph.core import errors from tensorflow.contrib.autograph.pyct import compiler from tensorflow.contrib.autograph.pyct import parser from tensorflow.contrib.autograph.pyct import pretty_printer @@ -89,6 +90,8 @@ class TestCase(test.TestCase): fake_ag = self.make_fake_mod('fake_ag', converted_call) fake_ag.__dict__.update(operators.__dict__) fake_ag.__dict__['utils'] = utils + fake_ag.__dict__['rewrite_graph_construction_error'] = ( + errors.rewrite_graph_construction_error) result.__dict__['ag__'] = fake_ag yield result except Exception: # pylint:disable=broad-except diff --git a/tensorflow/contrib/autograph/impl/api.py b/tensorflow/contrib/autograph/impl/api.py index c7401c7df1..f7fe3de5da 100644 --- a/tensorflow/contrib/autograph/impl/api.py +++ b/tensorflow/contrib/autograph/impl/api.py @@ -99,6 +99,7 @@ def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None): Returns: A decorator that wraps the original function. """ + def decorator(f): """Decorator implementation.""" @@ -109,8 +110,7 @@ def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None): @wraps(f) def py_func_wrapper(*args, **kwargs): if kwargs: - raise NotImplementedError( - 'RunMode.PY_FUNC does not yet support kwargs') + raise NotImplementedError('RunMode.PY_FUNC does not yet support kwargs') # TODO(mdan): Add support for kwargs. return py_func.wrap_py_func( f, return_dtypes, args, kwargs, use_dummy_return=not return_dtypes) @@ -231,7 +231,10 @@ def to_graph(e, Returns: A function with a signature identical to `o`, but which when executed it - creates TF a graph that has the same functionality as the original entity. + creates TF a graph that has the same functionality as the original entity. + Raises: + ValueError: If the converted function defines or refers to symbol names that + are reserved for AutoGraph. """ program_ctx = converter.ProgramContext( recursive=recursive, @@ -256,6 +259,19 @@ def to_graph(e, compiled_node.__dict__[key] = val compiled_fn = getattr(compiled_node, name) + # Need this so the source_mapping attribute is available for the context + # manager to access for runtime errors. + # + # Note that compiler.ast_to_object attaches the source map 'ag_source_map__' + # symbol to the compiled module. + source_map_attribute_name = 'ag_source_map' + if getattr(compiled_fn, source_map_attribute_name, None) is not None: + raise ValueError('cannot convert %s because is has an attribute ' + '"%s", which is reserved for AutoGraph.' % + (compiled_fn, source_map_attribute_name)) + setattr(compiled_fn, source_map_attribute_name, + compiled_node.__dict__['ag_source_map__']) + if verbose: logging.info('Compiled output of %s:\n\n%s\n', e, compiled_src) @@ -292,7 +308,7 @@ def to_code(e, conversion.entity_to_graph(e, program_ctx, arg_values, arg_types) code = '\n'.join( - compiler.ast_to_source(dep, indentation) + compiler.ast_to_source(dep, indentation)[0] for dep in reversed(tuple(six.itervalues(program_ctx.dependency_cache)))) return program_ctx.required_imports + '\n\n' + code diff --git a/tensorflow/contrib/autograph/impl/api_test.py b/tensorflow/contrib/autograph/impl/api_test.py index 9943093332..4de7df6572 100644 --- a/tensorflow/contrib/autograph/impl/api_test.py +++ b/tensorflow/contrib/autograph/impl/api_test.py @@ -206,8 +206,8 @@ class ApiTest(test.TestCase): return x with self.test_session() as sess: - x = api.converted_call( - test_fn, False, False, {}, constant_op.constant(-1)) + x = api.converted_call(test_fn, False, False, {}, + constant_op.constant(-1)) self.assertEqual(1, sess.run(x)) def test_converted_call_method(self): @@ -274,8 +274,8 @@ class ApiTest(test.TestCase): return self.x with self.test_session() as sess: - tc = api.converted_call( - TestClass, False, False, {}, constant_op.constant(-1)) + tc = api.converted_call(TestClass, False, False, {}, + constant_op.constant(-1)) # tc is now a converted object. x = tc.test_method() self.assertEqual(1, sess.run(x)) @@ -305,6 +305,13 @@ class ApiTest(test.TestCase): # Just check that it is parseable Python code. self.assertIsNotNone(parser.parse_str(compiled_code)) + def test_source_map_attribute_present(self): + + def test_fn(y): + return y**2 + + self.assertTrue(hasattr(api.to_graph(test_fn), 'ag_source_map')) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/autograph/impl/conversion.py b/tensorflow/contrib/autograph/impl/conversion.py index 776d19f672..bd14359356 100644 --- a/tensorflow/contrib/autograph/impl/conversion.py +++ b/tensorflow/contrib/autograph/impl/conversion.py @@ -31,6 +31,7 @@ from tensorflow.contrib.autograph.converters import call_trees from tensorflow.contrib.autograph.converters import continue_statements from tensorflow.contrib.autograph.converters import control_flow from tensorflow.contrib.autograph.converters import decorators +from tensorflow.contrib.autograph.converters import error_handlers from tensorflow.contrib.autograph.converters import ifexp from tensorflow.contrib.autograph.converters import lists from tensorflow.contrib.autograph.converters import logical_expressions @@ -40,8 +41,10 @@ from tensorflow.contrib.autograph.converters import single_return from tensorflow.contrib.autograph.converters import slices from tensorflow.contrib.autograph.core import config from tensorflow.contrib.autograph.core import converter +from tensorflow.contrib.autograph.core import errors from tensorflow.contrib.autograph.pyct import ast_util from tensorflow.contrib.autograph.pyct import inspect_utils +from tensorflow.contrib.autograph.pyct import origin_info from tensorflow.contrib.autograph.pyct import parser from tensorflow.contrib.autograph.pyct import qual_names from tensorflow.contrib.autograph.pyct import transformer @@ -231,6 +234,8 @@ def _add_self_references(namespace, autograph_module): ag_internal = imp.new_module('autograph') ag_internal.converted_call = autograph_module.converted_call ag_internal.utils = utils + ag_internal.rewrite_graph_construction_error = ( + errors.rewrite_graph_construction_error) # TODO(mdan): Add safeguards against name clashes. # We don't want to create a submodule because we want the operators to be # accessible as ag__. @@ -241,9 +246,10 @@ def _add_self_references(namespace, autograph_module): def function_to_graph(f, program_ctx, arg_values, arg_types, owner_type=None): """Specialization of `entity_to_graph` for callable functions.""" + node, source = parser.parse_entity(f) node = node.body[0] - + origin_info.resolve(node, source, f) namespace = inspect_utils.getnamespace(f) _add_self_references(namespace, program_ctx.autograph_module) namer = program_ctx.new_namer(namespace) @@ -319,4 +325,5 @@ def node_to_graph(node, context): node = _apply_transformer(node, context, logical_expressions) node = _apply_transformer(node, context, side_effect_guards) node = _apply_transformer(node, context, name_scopes) + node = _apply_transformer(node, context, error_handlers) return node diff --git a/tensorflow/contrib/autograph/impl/conversion_test.py b/tensorflow/contrib/autograph/impl/conversion_test.py index f5279298af..207225a1ac 100644 --- a/tensorflow/contrib/autograph/impl/conversion_test.py +++ b/tensorflow/contrib/autograph/impl/conversion_test.py @@ -79,10 +79,12 @@ class ConversionTest(test.TestCase): self.assertTrue(f in program_ctx.dependency_cache) self.assertTrue(g in program_ctx.dependency_cache) self.assertEqual('tf__f', program_ctx.dependency_cache[f].name) - # need the extra .body[0] in order to step past the with tf.name_scope('f') - # that is added automatically + # need one extra .body[0] in order to step past the try/except wrapper that + # is added automatically, the other for the with tf.name_scope('f') that is + # added automatically self.assertEqual( - 'tf__g', program_ctx.dependency_cache[f].body[0].body[0].value.func.id) + 'tf__g', + program_ctx.dependency_cache[f].body[0].body[0].body[0].value.func.id) self.assertEqual('tf__g', program_ctx.dependency_cache[g].name) def test_entity_to_graph_class_hierarchy(self): diff --git a/tensorflow/contrib/autograph/pyct/ast_util.py b/tensorflow/contrib/autograph/pyct/ast_util.py index 0cf87dd8d3..86e3f56a64 100644 --- a/tensorflow/contrib/autograph/pyct/ast_util.py +++ b/tensorflow/contrib/autograph/pyct/ast_util.py @@ -20,6 +20,7 @@ from __future__ import print_function import ast +import collections import gast from tensorflow.contrib.autograph.pyct import anno @@ -184,7 +185,6 @@ class PatternMatcher(gast.NodeVisitor): if v != p: return self.no_match() - def matches(node, pattern): """Basic pattern matcher for AST. @@ -251,3 +251,32 @@ def apply_to_single_assignments(targets, values, apply_fn): apply_to_single_assignments(target_el, value_el, apply_fn) else: apply_fn(target, values) + + +def iter_fields(node): + for field in sorted(node._fields): + try: + yield getattr(node, field) + except AttributeError: + pass + + +def iter_child_nodes(node): + for field in iter_fields(node): + if isinstance(field, gast.AST): + yield field + elif isinstance(field, list): + for item in field: + if isinstance(item, gast.AST): + yield item + + +def parallel_walk(node_a, node_b): + todo_a = collections.deque([node_a]) + todo_b = collections.deque([node_b]) + while todo_a and todo_b: + node_a = todo_a.popleft() + node_b = todo_b.popleft() + todo_a.extend(iter_child_nodes(node_a)) + todo_b.extend(iter_child_nodes(node_b)) + yield node_a, node_b diff --git a/tensorflow/contrib/autograph/pyct/ast_util_test.py b/tensorflow/contrib/autograph/pyct/ast_util_test.py index bd546c7f48..981e398b93 100644 --- a/tensorflow/contrib/autograph/pyct/ast_util_test.py +++ b/tensorflow/contrib/autograph/pyct/ast_util_test.py @@ -44,7 +44,8 @@ class AstUtilTest(test.TestCase): node, {qual_names.QN('a'): qual_names.QN('renamed_a')}) self.assertIsInstance(node.body[0].value.left.id, str) - self.assertEqual(compiler.ast_to_source(node).strip(), 'renamed_a + b') + source, _ = compiler.ast_to_source(node) + self.assertEqual(source.strip(), 'renamed_a + b') def test_rename_symbols_attributes(self): node = parser.parse_str('b.c = b.c.d') @@ -53,8 +54,8 @@ class AstUtilTest(test.TestCase): node = ast_util.rename_symbols( node, {qual_names.from_str('b.c'): qual_names.QN('renamed_b_c')}) - self.assertEqual( - compiler.ast_to_source(node).strip(), 'renamed_b_c = renamed_b_c.d') + source, _ = compiler.ast_to_source(node) + self.assertEqual(source.strip(), 'renamed_b_c = renamed_b_c.d') def test_rename_symbols_annotations(self): node = parser.parse_str('a[i]') @@ -129,9 +130,9 @@ class AstUtilTest(test.TestCase): 'super(Bar, _).__init__(_)') def _mock_apply_fn(self, target, source): - target = compiler.ast_to_source(target).strip() - source = compiler.ast_to_source(source).strip() - self._invocation_counts[(target, source)] += 1 + target, _ = compiler.ast_to_source(target) + source, _ = compiler.ast_to_source(source) + self._invocation_counts[(target.strip(), source.strip())] += 1 def test_apply_to_single_assignments_dynamic_unpack(self): node = parser.parse_str('a, b, c = d') @@ -155,6 +156,25 @@ class AstUtilTest(test.TestCase): ('c', 'f'): 1, }) + def test_parallel_walk(self): + ret = ast.Return( + ast.BinOp( + op=ast.Add(), + left=ast.Name(id='a', ctx=ast.Load()), + right=ast.Num(1))) + node = ast.FunctionDef( + name='f', + args=ast.arguments( + args=[ast.Name(id='a', ctx=ast.Param())], + vararg=None, + kwarg=None, + defaults=[]), + body=[ret], + decorator_list=[], + returns=None) + for child_a, child_b in ast_util.parallel_walk(node, node): + self.assertEqual(child_a, child_b) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/autograph/pyct/cfg.py b/tensorflow/contrib/autograph/pyct/cfg.py index 8ef234745c..9f060236f4 100644 --- a/tensorflow/contrib/autograph/pyct/cfg.py +++ b/tensorflow/contrib/autograph/pyct/cfg.py @@ -67,8 +67,10 @@ class Node(object): if isinstance(self.ast_node, gast.FunctionDef): return 'def %s' % self.ast_node.name elif isinstance(self.ast_node, gast.withitem): - return compiler.ast_to_source(self.ast_node.context_expr).strip() - return compiler.ast_to_source(self.ast_node).strip() + source, _ = compiler.ast_to_source(self.ast_node.context_expr) + return source.strip() + source, _ = compiler.ast_to_source(self.ast_node) + return source.strip() class Graph( diff --git a/tensorflow/contrib/autograph/pyct/compiler.py b/tensorflow/contrib/autograph/pyct/compiler.py index 24c4517afa..538481ff79 100644 --- a/tensorflow/contrib/autograph/pyct/compiler.py +++ b/tensorflow/contrib/autograph/pyct/compiler.py @@ -30,9 +30,49 @@ import tempfile import astor import gast +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import ast_util +from tensorflow.contrib.autograph.pyct import origin_info +from tensorflow.contrib.autograph.pyct import parser + + +def _build_source_map(node, code): + """Return the Python objects represented by given AST. + + Compiling the AST code this way ensures that the source code is readable by + e.g. `pdb` or `inspect`. + + Args: + node: An AST node of the original generated code, before the source code is + generated. + code: The string representation of the source code for the newly generated + code. + + Returns: + Dict[CodeLocation, OriginInfo], a mapping between the user and AutoGraph + generated code. + """ + # After we have the final generated code we reparse it to get the final line + # numbers. Then we walk through the generated and original ASTs in parallel + # to build the mapping between the user and generated code. + new_node = parser.parse_str(code) + origin_info.resolve(new_node, code) + source_mapping = {} + for before, after in ast_util.parallel_walk(node, new_node): + # Need both checks because if origin information is ever copied over to new + # nodes then we need to rely on the fact that only the original user code + # has the origin annotation. + if (anno.hasanno(before, anno.Basic.ORIGIN) and + anno.hasanno(after, anno.Basic.ORIGIN)): + source_info = anno.getanno(before, anno.Basic.ORIGIN) + new_line_number = anno.getanno(after, anno.Basic.ORIGIN).line_number + source_mapping[new_line_number] = source_info + return source_mapping + def ast_to_source(node, indentation=' '): """Return the source code of given AST.""" + original_node = node if isinstance(node, gast.AST): node = gast.gast_to_ast(node) generator = astor.codegen.SourceGenerator(indentation, False, @@ -42,11 +82,16 @@ def ast_to_source(node, indentation=' '): # In some versions of Python, literals may appear as actual values. This # ensures everything is string. code = map(str, generator.result) - return astor.source_repr.pretty_source(code).lstrip() + code = astor.source_repr.pretty_source(code).lstrip() + source_mapping = _build_source_map(original_node, code) + return code, source_mapping -def ast_to_object( - node, indentation=' ', source_prefix=None, delete_on_exit=True): + +def ast_to_object(node, + indentation=' ', + source_prefix=None, + delete_on_exit=False): """Return the Python objects represented by given AST. Compiling the AST code this way ensures that the source code is readable by @@ -56,15 +101,30 @@ def ast_to_object( node: The code to compile, as an AST object. indentation: The string to use for indentation. source_prefix: Optional string to print as-is into the source file. - delete_on_exit: Whether to delete the temporary file used for compilation - on exit. + delete_on_exit: Whether to delete the temporary file used for compilation on + exit. Returns: A module object containing the compiled source code. + Raises: + ValueError: If ag_source_map__ is already in the namespace of the compiled + node. """ - source = ast_to_source(node, indentation) + # code_source_mapping does not yet include the offsets from import statements. + source, code_source_mapping = ast_to_source(node, indentation=indentation) with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + # TODO(znado): move into an _offset_source_map() helper function. + # Need to offset the generated line numbers by the number of import lines. + if source_prefix: + num_import_lines = source_prefix.count('\n') + 1 + else: + num_import_lines = 0 + source_mapping = {} + for line_number, original_position in code_source_mapping.items(): + source_map_key = origin_info.CodeLocation( + file_path=f.name, line_number=line_number + num_import_lines) + source_mapping[source_map_key] = original_position module_name = os.path.basename(f.name[:-3]) if source_prefix: f.write(source_prefix) @@ -72,4 +132,27 @@ def ast_to_object( f.write(source) if delete_on_exit: atexit.register(lambda: os.remove(f.name)) - return imp.load_source(module_name, f.name), source + compiled_node = imp.load_source(module_name, f.name) + + # TODO(znado): Clean this up so we don't need to attach it to the namespace. + # TODO(znado): This does not work for classes because their methods share a + # namespace. + # This attaches the source map which is needed for error handling. Note that + # api.to_graph copies this source map into an attribute of the function. + # + # We need this so the ag_source_map__ variable is available to the call to + # rewrite_graph_construction_error in the except block inside each function + # that handles graph construction errors. + # + # We cannot get the rewritten function name until it is too late so templating + # is hard, and this cleanly fixes the + # issues encountered with nested functions because this is attached to the + # outermost one. + source_map_name = 'ag_source_map__' + if source_map_name in compiled_node.__dict__: + raise ValueError('cannot convert %s because is has namespace attribute ' + '"%s", which is reserved for AutoGraph.' % + (compiled_node, source_map_name)) + compiled_node.__dict__[source_map_name] = source_mapping + + return compiled_node, source diff --git a/tensorflow/contrib/autograph/pyct/compiler_test.py b/tensorflow/contrib/autograph/pyct/compiler_test.py index 98cdc1506b..e29fa9324c 100644 --- a/tensorflow/contrib/autograph/pyct/compiler_test.py +++ b/tensorflow/contrib/autograph/pyct/compiler_test.py @@ -59,14 +59,14 @@ class CompilerTest(test.TestCase): value=gast.Str('c')) ]) + source, _ = compiler.ast_to_source(node, indentation=' ') self.assertEqual( textwrap.dedent(""" if 1: a = b else: a = 'c' - """).strip(), - compiler.ast_to_source(node, indentation=' ').strip()) + """).strip(), source.strip()) def test_ast_to_object(self): node = gast.FunctionDef( diff --git a/tensorflow/contrib/autograph/pyct/origin_info.py b/tensorflow/contrib/autograph/pyct/origin_info.py index b3c6a43d37..614e346634 100644 --- a/tensorflow/contrib/autograph/pyct/origin_info.py +++ b/tensorflow/contrib/autograph/pyct/origin_info.py @@ -17,10 +17,16 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from collections import namedtuple +import collections +import gast -class CodeLocation(namedtuple('CodeLocation', ('file_path', 'line_number'))): +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.python.util import tf_inspect + + +class CodeLocation( + collections.namedtuple('CodeLocation', ('file_path', 'line_number'))): """Location of a line of code. Attributes: @@ -31,8 +37,9 @@ class CodeLocation(namedtuple('CodeLocation', ('file_path', 'line_number'))): class OriginInfo( - namedtuple('OriginInfo', ('file_path', 'function_name', 'line_number', - 'column_offset', 'source_code_line'))): + collections.namedtuple('OriginInfo', + ('file_path', 'function_name', 'line_number', + 'column_offset', 'source_code_line'))): """Container for information about the source code before conversion. Instances of this class contain information about the source code that @@ -50,3 +57,44 @@ class OriginInfo( """ return (self.file_path, self.line_number, self.function_name, self.source_code_line) + + +# TODO(znado): Consider refactoring this into a Visitor. +def resolve(node, source, function=None): + """Adds an origin information to all nodes inside the body of function. + + Args: + node: The AST node for the function whose body nodes will be annotated. + source: Text, the source code string for the function whose body nodes will + be annotated. + function: Callable, the function that will have all nodes inside of it + annotation with an OriginInfo annotation with key anno.Basic.ORIGIN. If + it is None then only the line numbers and column offset will be set in the + annotation, with the rest of the information being None. + + Returns: + A tuple of the AST node for function and a String containing its source + code. + """ + if function: + _, function_lineno = tf_inspect.getsourcelines(function) + function_filepath = tf_inspect.getsourcefile(function) + else: + function_lineno = None + function_filepath = None + source_lines = source.split('\n') + for n in gast.walk(node): + if hasattr(n, 'lineno'): + # n.lineno is relative to the start of the enclosing function, so need to + # offset it by the line of the function. + source_code_line = source_lines[n.lineno - 1] + if function: + source_lineno = n.lineno + function_lineno - 1 + function_name = function.__name__ + else: + source_lineno = n.lineno + function_name = None + anno.setanno( + n, anno.Basic.ORIGIN, + OriginInfo(function_filepath, function_name, source_lineno, + n.col_offset, source_code_line)) diff --git a/tensorflow/contrib/autograph/pyct/templates.py b/tensorflow/contrib/autograph/pyct/templates.py index 9001e54e46..72d1d3b269 100644 --- a/tensorflow/contrib/autograph/pyct/templates.py +++ b/tensorflow/contrib/autograph/pyct/templates.py @@ -45,6 +45,7 @@ class ReplaceTransformer(gast.NodeTransformer): self.replacements = replacements self.in_replacements = False self.preserved_annos = { + anno.Basic.ORIGIN, anno.Basic.SKIP_PROCESSING, anno.Static.ORIG_DEFINITIONS, } diff --git a/tensorflow/contrib/autograph/pyct/transformer.py b/tensorflow/contrib/autograph/pyct/transformer.py index d9a157aead..bbdfefc50a 100644 --- a/tensorflow/contrib/autograph/pyct/transformer.py +++ b/tensorflow/contrib/autograph/pyct/transformer.py @@ -396,7 +396,8 @@ class Base(gast.NodeTransformer): def _get_source(self, node): try: - return compiler.ast_to_source(node) + source, _ = compiler.ast_to_source(node) + return source except AssertionError: return '' -- cgit v1.2.3 From ee7c2fc000d5640468343ec93b4878d1334d481c Mon Sep 17 00:00:00 2001 From: Dmitry Klimenkov Date: Fri, 13 Jul 2018 19:21:10 +0300 Subject: changed time_dim, batch_dim to time_axis, batch_axis accordingly --- tensorflow/python/ops/rnn.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py index 7521d59a60..7096e0dd84 100644 --- a/tensorflow/python/ops/rnn.py +++ b/tensorflow/python/ops/rnn.py @@ -417,11 +417,11 @@ def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, # Backward direction if not time_major: - time_dim = 1 - batch_dim = 0 + time_axis = 1 + batch_axis = 0 else: - time_dim = 0 - batch_dim = 1 + time_axis = 0 + batch_axis = 1 def _reverse(input_, seq_lengths, seq_axis, batch_axis): if seq_lengths is not None: @@ -434,7 +434,7 @@ def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, with vs.variable_scope("bw") as bw_scope: inputs_reverse = _reverse( inputs, seq_lengths=sequence_length, - seq_axis=time_dim, batch_axis=batch_dim) + seq_axis=time_axis, batch_axis=batch_axis) tmp, output_state_bw = dynamic_rnn( cell=cell_bw, inputs=inputs_reverse, sequence_length=sequence_length, initial_state=initial_state_bw, dtype=dtype, @@ -443,7 +443,7 @@ def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, output_bw = _reverse( tmp, seq_lengths=sequence_length, - seq_axis=time_dim, batch_axis=batch_dim) + seq_axis=time_axis, batch_axis=batch_axis) outputs = (output_fw, output_bw) output_states = (output_state_fw, output_state_bw) -- cgit v1.2.3 From feba399acb6f3c89ed5cd2602c454c179fcb792b Mon Sep 17 00:00:00 2001 From: Shashi Shekhar Date: Fri, 13 Jul 2018 09:43:10 -0700 Subject: Update documentation. PiperOrigin-RevId: 204484476 --- tensorflow/contrib/lite/g3doc/models.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tensorflow/contrib/lite/g3doc/models.md b/tensorflow/contrib/lite/g3doc/models.md index c1c8ef049f..4e7d33a1b6 100644 --- a/tensorflow/contrib/lite/g3doc/models.md +++ b/tensorflow/contrib/lite/g3doc/models.md @@ -39,22 +39,22 @@ single thread large core. Model Name | Paper_Model_Files | Model_Size | Top-1 Accuracy | Top-5 Accuracy | TF Lite Performance ------------------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------: | ---------: | -------------: | -------------: | ------------------: -Mobilenet_0.25_128_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.25_128_quant.tgz) | 0.5 Mb | 39.9% | 65.8% | 3.7 ms -Mobilenet_0.25_160_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.25_160_quant.tgz) | 0.5 Mb | 43.5% | 69.1% | 5.5 ms -Mobilenet_0.25_192_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.25_192_quant.tgz) | 0.5 Mb | 45.8% | 71.9% | 7.9 ms -Mobilenet_0.25_224_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.25_224_quant.tgz) | 0.5 Mb | 48.2% | 73.8% | 10.4 ms -Mobilenet_0.50_128_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.5_128_quant.tgz) | 1.4 Mb | 54.9% | 78.9% | 8.8 ms -Mobilenet_0.50_160_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.5_160_quant.tgz) | 1.4 Mb | 57.7% | 81.3% | 13.0 ms -Mobilenet_0.50_192_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.5_192_quant.tgz) | 1.4 Mb | 60.4% | 83.2% | 18.3 ms -Mobilenet_0.50_224_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.5_224_quant.tgz) | 1.4 Mb | 62.2% | 84.5% | 24.7 ms -Mobilenet_0.75_128_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.75_128_quant.tgz) | 2.6 Mb | 59.8% | 82.8% | 16.2 ms -Mobilenet_0.75_160_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.75_160_quant.tgz) | 2.6 Mb | 63.9% | 85.5% | 24.3 ms -Mobilenet_0.75_192_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.75_192_quant.tgz) | 2.6 Mb | 66.2% | 87.1% | 33.8 ms -Mobilenet_0.75_224_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_0.75_224_quant.tgz) | 2.6 Mb | 67.9% | 88.1% | 45.4 ms -Mobilenet_1.0_128_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_128_quant.tgz) | 4.3 Mb | 64.0% | 85.5% | 24.9 ms -Mobilenet_1.0_160_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_160_quant.tgz) | 4.3 Mb | 67.3% | 87.7% | 37.4 ms -Mobilenet_1.0_192_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_192_quant.tgz) | 4.3 Mb | 69.0% | 88.9% | 51.9 ms -Mobilenet_1.0_224_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224_quant.tgz) | 4.3 Mb | 69.7% | 89.5% | 70.2 ms +Mobilenet_0.25_128_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.25_128_quant.tgz) | 0.5 Mb | 39.7% | 65.8% | 3.7 ms +Mobilenet_0.25_160_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.25_160_quant.tgz) | 0.5 Mb | 41.9% | 69.1% | 5.5 ms +Mobilenet_0.25_192_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.25_192_quant.tgz) | 0.5 Mb | 45.3% | 71.9% | 7.9 ms +Mobilenet_0.25_224_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.25_224_quant.tgz) | 0.5 Mb | 46.4% | 73.8% | 10.4 ms +Mobilenet_0.50_128_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.5_128_quant.tgz) | 1.4 Mb | 54.1% | 78.9% | 8.8 ms +Mobilenet_0.50_160_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.5_160_quant.tgz) | 1.4 Mb | 57.6% | 81.3% | 13.0 ms +Mobilenet_0.50_192_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.5_192_quant.tgz) | 1.4 Mb | 59.1% | 83.2% | 18.3 ms +Mobilenet_0.50_224_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.5_224_quant.tgz) | 1.4 Mb | 61.0% | 84.5% | 24.7 ms +Mobilenet_0.75_128_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.75_128_quant.tgz) | 2.6 Mb | 52.5% | 82.8% | 16.2 ms +Mobilenet_0.75_160_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.75_160_quant.tgz) | 2.6 Mb | 63.6% | 85.5% | 24.3 ms +Mobilenet_0.75_192_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.75_192_quant.tgz) | 2.6 Mb | 61.1% | 87.1% | 33.8 ms +Mobilenet_0.75_224_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_0.75_224_quant.tgz) | 2.6 Mb | 66.7% | 88.1% | 45.4 ms +Mobilenet_1.0_128_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_1.0_128_quant.tgz) | 4.3 Mb | 62.7% | 85.5% | 24.9 ms +Mobilenet_1.0_160_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_1.0_160_quant.tgz) | 4.3 Mb | 66.6% | 87.7% | 37.4 ms +Mobilenet_1.0_192_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_1.0_192_quant.tgz) | 4.3 Mb | 69.2% | 88.9% | 51.9 ms +Mobilenet_1.0_224_quant | [paper](https://arxiv.org/pdf/1712.05877.pdf), [tflite&pb](http://download.tensorflow.org/models/mobilenet_v1_2018_07_12/mobilenet_v1_1.0_224_quant.tgz) | 4.3 Mb | 69.3% | 89.5% | 70.2 ms ## Other models -- cgit v1.2.3 From 8e7d3dc7326bb81ef55175c48f51436408219c4a Mon Sep 17 00:00:00 2001 From: Smit Hinsu Date: Fri, 13 Jul 2018 10:38:15 -0700 Subject: Automated rollback of commit 895a7667884545a68480eb91916a5a23c2852308 PiperOrigin-RevId: 204493360 --- tensorflow/python/BUILD | 5 +- tensorflow/python/framework/error_interpolation.py | 82 +--------------- .../python/framework/error_interpolation_test.py | 104 ++------------------- tensorflow/python/util/tf_stack.py | 6 -- 4 files changed, 15 insertions(+), 182 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 2fba3c2acb..924db54cbc 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -705,9 +705,7 @@ py_library( "framework/error_interpolation.py", ], srcs_version = "PY2AND3", - deps = [ - ":util", - ], + deps = [], ) py_library( @@ -1042,7 +1040,6 @@ py_test( srcs_version = "PY2AND3", deps = [ ":client_testlib", - ":constant_op", ":error_interpolation", ], ) diff --git a/tensorflow/python/framework/error_interpolation.py b/tensorflow/python/framework/error_interpolation.py index 519e0fda0a..9ccae76147 100644 --- a/tensorflow/python/framework/error_interpolation.py +++ b/tensorflow/python/framework/error_interpolation.py @@ -29,9 +29,6 @@ import string import six -from tensorflow.python.util import tf_stack - - _NAME_REGEX = r"[A-Za-z0-9.][A-Za-z0-9_.\-/]*?" _FORMAT_REGEX = r"[A-Za-z0-9_.\-/${}:]+" _TAG_REGEX = r"\^\^({name}):({name}):({fmt})\^\^".format( @@ -41,8 +38,6 @@ _INTERPOLATION_PATTERN = re.compile(_INTERPOLATION_REGEX) _ParseTag = collections.namedtuple("_ParseTag", ["type", "name", "format"]) -_BAD_FILE_SUBSTRINGS = ["tensorflow/python", " - file: Replaced with the filename in which the node was defined. - line: Replaced by the line number at which the node was defined. - Args: message: String to parse @@ -81,47 +72,9 @@ def _parse_message(message): return seps, tags -def _get_field_dict_from_traceback(tf_traceback, frame_index): - """Convert traceback elements into interpolation dictionary and return.""" - frame = tf_traceback[frame_index] - return { - "file": frame[tf_stack.TB_FILENAME], - "line": frame[tf_stack.TB_LINENO], - } - - -def _find_index_of_defining_frame_for_op(op): - """Return index in op._traceback with first 'useful' frame. - - This method reads through the stack stored in op._traceback looking for the - innermost frame which (hopefully) belongs to the caller. It accomplishes this - by rejecting frames whose filename appears to come from TensorFlow (see - error_interpolation._BAD_FILE_SUBSTRINGS for the list of rejected substrings). - - Args: - op: the Operation object for which we would like to find the defining - location. - - Returns: - Integer index into op._traceback where the first non-TF file was found - (innermost to outermost), or 0 (for the outermost stack frame) if all files - came from TensorFlow. - """ - # pylint: disable=protected-access - # Index 0 of tf_traceback is the outermost frame. - tf_traceback = tf_stack.convert_stack(op._traceback) - size = len(tf_traceback) - # pylint: enable=protected-access - filenames = [frame[tf_stack.TB_FILENAME] for frame in tf_traceback] - # We process the filenames from the innermost frame to outermost. - for idx, filename in enumerate(reversed(filenames)): - contains_bad_substrings = [ss in filename for ss in _BAD_FILE_SUBSTRINGS] - if not any(contains_bad_substrings): - return size - idx - 1 - return 0 - - -def interpolate(error_message, graph): +# TODO(jtkeeling): Modify to actually interpolate format strings rather than +# echoing them. +def interpolate(error_message): """Interpolates an error message. The error message can contain tags of the form ^^type:name:format^^ which will @@ -129,38 +82,11 @@ def interpolate(error_message, graph): Args: error_message: A string to interpolate. - graph: ops.Graph object containing all nodes referenced in the error - message. Returns: The string with tags of the form ^^type:name:format^^ interpolated. """ seps, tags = _parse_message(error_message) - - node_name_to_substitution_dict = {} - for name in [t.name for t in tags]: - try: - op = graph.get_operation_by_name(name) - except KeyError: - op = None - - if op: - frame_index = _find_index_of_defining_frame_for_op(op) - # pylint: disable=protected-access - field_dict = _get_field_dict_from_traceback(op._traceback, frame_index) - # pylint: enable=protected-access - else: - field_dict = { - "file": "", - "line": "", - "func": "", - "code": None, - } - node_name_to_substitution_dict[name] = field_dict - - subs = [ - string.Template(tag.format).safe_substitute( - node_name_to_substitution_dict[tag.name]) for tag in tags - ] + subs = [string.Template(tag.format).safe_substitute({}) for tag in tags] return "".join( itertools.chain(*six.moves.zip_longest(seps, subs, fillvalue=""))) diff --git a/tensorflow/python/framework/error_interpolation_test.py b/tensorflow/python/framework/error_interpolation_test.py index 091f0da2a2..ad448deb62 100644 --- a/tensorflow/python/framework/error_interpolation_test.py +++ b/tensorflow/python/framework/error_interpolation_test.py @@ -18,115 +18,31 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.framework import constant_op from tensorflow.python.framework import error_interpolation from tensorflow.python.platform import test -from tensorflow.python.util import tf_stack - - -def _make_frame_with_filename(op, idx, filename): - """Return a copy of an existing stack frame with a new filename.""" - stack_frame = list(op._traceback[idx]) - stack_frame[tf_stack.TB_FILENAME] = filename - return tuple(stack_frame) - - -def _modify_op_stack_with_filenames(op, num_user_frames, user_filename, - num_inner_tf_frames): - """Replace op._traceback with a new traceback using special filenames.""" - tf_filename = "%d" + error_interpolation._BAD_FILE_SUBSTRINGS[0] - user_filename = "%d/my_favorite_file.py" - - num_requested_frames = num_user_frames + num_inner_tf_frames - num_actual_frames = len(op._traceback) - num_outer_frames = num_actual_frames - num_requested_frames - assert num_requested_frames <= num_actual_frames, "Too few real frames." - - # The op's traceback has outermost frame at index 0. - stack = [] - for idx in range(0, num_outer_frames): - stack.append(op._traceback[idx]) - for idx in range(len(stack), len(stack)+num_user_frames): - stack.append(_make_frame_with_filename(op, idx, user_filename % idx)) - for idx in range(len(stack), len(stack)+num_inner_tf_frames): - stack.append(_make_frame_with_filename(op, idx, tf_filename % idx)) - op._traceback = stack class InterpolateTest(test.TestCase): - def setUp(self): - # Add nodes to the graph for retrieval by name later. - constant_op.constant(1, name="One") - constant_op.constant(2, name="Two") - three = constant_op.constant(3, name="Three") - self.graph = three.graph - - # Change the list of bad file substrings so that constant_op.py is chosen - # as the defining stack frame for constant_op.constant ops. - self.old_bad_strings = error_interpolation._BAD_FILE_SUBSTRINGS - error_interpolation._BAD_FILE_SUBSTRINGS = ["/ops.py", "/util"] - - def tearDown(self): - error_interpolation._BAD_FILE_SUBSTRINGS = self.old_bad_strings - - def testFindIndexOfDefiningFrameForOp(self): - local_op = constant_op.constant(42).op - user_filename = "hope.py" - _modify_op_stack_with_filenames(local_op, - num_user_frames=3, - user_filename=user_filename, - num_inner_tf_frames=5) - idx = error_interpolation._find_index_of_defining_frame_for_op(local_op) - # Expected frame is 6th from the end because there are 5 inner frames witih - # TF filenames. - expected_frame = len(local_op._traceback) - 6 - self.assertEqual(expected_frame, idx) - - def testFindIndexOfDefiningFrameForOpReturnsZeroOnError(self): - local_op = constant_op.constant(43).op - # Truncate stack to known length. - local_op._traceback = local_op._traceback[:7] - # Ensure all frames look like TF frames. - _modify_op_stack_with_filenames(local_op, - num_user_frames=0, - user_filename="user_file.py", - num_inner_tf_frames=7) - idx = error_interpolation._find_index_of_defining_frame_for_op(local_op) - self.assertEqual(0, idx) - def testNothingToDo(self): normal_string = "This is just a normal string" - interpolated_string = error_interpolation.interpolate(normal_string, - self.graph) + interpolated_string = error_interpolation.interpolate(normal_string) self.assertEqual(interpolated_string, normal_string) def testOneTag(self): - one_tag_string = "^^node:Two:${file}^^" - interpolated_string = error_interpolation.interpolate(one_tag_string, - self.graph) - self.assertTrue(interpolated_string.endswith("constant_op.py"), - "interpolated_string '%s' did not end with constant_op.py" - % interpolated_string) - - def testOneTagWithAFakeNameResultsInPlaceholders(self): - one_tag_string = "^^node:MinusOne:${file}^^" - interpolated_string = error_interpolation.interpolate(one_tag_string, - self.graph) - self.assertEqual(interpolated_string, "") + one_tag_string = "^^node:Foo:${file}^^" + interpolated_string = error_interpolation.interpolate(one_tag_string) + self.assertEqual(interpolated_string, "${file}") def testTwoTagsNoSeps(self): - two_tags_no_seps = "^^node:One:${file}^^^^node:Three:${line}^^" - interpolated_string = error_interpolation.interpolate(two_tags_no_seps, - self.graph) - self.assertRegexpMatches(interpolated_string, "constant_op.py[0-9]+") + two_tags_no_seps = "^^node:Foo:${file}^^^^node:Bar:${line}^^" + interpolated_string = error_interpolation.interpolate(two_tags_no_seps) + self.assertEqual(interpolated_string, "${file}${line}") def testTwoTagsWithSeps(self): - two_tags_with_seps = ";;;^^node:Two:${file}^^,,,^^node:Three:${line}^^;;;" - interpolated_string = error_interpolation.interpolate(two_tags_with_seps, - self.graph) - expected_regex = "^;;;.*constant_op.py,,,[0-9]*;;;$" - self.assertRegexpMatches(interpolated_string, expected_regex) + two_tags_with_seps = "123^^node:Foo:${file}^^456^^node:Bar:${line}^^789" + interpolated_string = error_interpolation.interpolate(two_tags_with_seps) + self.assertEqual(interpolated_string, "123${file}456${line}789") if __name__ == "__main__": diff --git a/tensorflow/python/util/tf_stack.py b/tensorflow/python/util/tf_stack.py index fe4f4a63eb..dacc1ce83e 100644 --- a/tensorflow/python/util/tf_stack.py +++ b/tensorflow/python/util/tf_stack.py @@ -21,12 +21,6 @@ from __future__ import print_function import linecache import sys -# Names for indices into TF traceback tuples. -TB_FILENAME = 0 -TB_LINENO = 1 -TB_FUNCNAME = 2 -TB_CODEDICT = 3 # Dictionary of Python interpreter state. - def extract_stack(extract_frame_info_fn=None): """A lightweight, extensible re-implementation of traceback.extract_stack. -- cgit v1.2.3 From 2449da260ccd7dc075b890729f7a6cbb08e10882 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 10:57:14 -0700 Subject: Internal changes. PiperOrigin-RevId: 204496471 --- .../examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb b/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb index d6a29ea1ec..a64e266f6a 100644 --- a/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb +++ b/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb @@ -378,7 +378,7 @@ } ], "source": [ - "#@test {\"timeout\": 90} \n", + "#@test {\"timeout\": 90}\n", "with tf.Graph().as_default():\n", " hp = tf.contrib.training.HParams(\n", " learning_rate=0.05,\n", @@ -580,7 +580,7 @@ } ], "source": [ - "#@test {\"timeout\": 90} \n", + "#@test {\"timeout\": 90}\n", "with context.eager_mode():\n", " durations = []\n", " for t in range(burn_ins + trials):\n", @@ -628,10 +628,6 @@ "colab": { "collapsed_sections": [], "default_view": {}, - "last_runtime": { - "build_target": "", - "kind": "local" - }, "name": "Autograph vs. Eager MNIST benchmark", "provenance": [ { -- cgit v1.2.3 From b5b5cc0e248b6fbd5025765f592a43af158a1cb2 Mon Sep 17 00:00:00 2001 From: Andrew Selle Date: Fri, 13 Jul 2018 10:59:05 -0700 Subject: Make an option that allows making the memory plan not reuse space. This is called 'allow_intermediates'. In the future we should have a way for users to enable this functionality from the interpreter API. Also, preserve_inputs is now better commented. PiperOrigin-RevId: 204496777 --- tensorflow/contrib/lite/arena_planner.cc | 21 ++++++++++++--------- tensorflow/contrib/lite/arena_planner.h | 9 ++++++++- tensorflow/contrib/lite/arena_planner_test.cc | 2 +- tensorflow/contrib/lite/interpreter.cc | 2 +- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/tensorflow/contrib/lite/arena_planner.cc b/tensorflow/contrib/lite/arena_planner.cc index 4257e754ad..16a0e71624 100644 --- a/tensorflow/contrib/lite/arena_planner.cc +++ b/tensorflow/contrib/lite/arena_planner.cc @@ -36,12 +36,13 @@ struct AllocationInfo { ArenaPlanner::ArenaPlanner(TfLiteContext* context, std::unique_ptr graph_info, - bool preserve_inputs) + bool preserve_inputs, bool preserve_intermediates) : context_(context), graph_info_(std::move(graph_info)), arena_(kDefaultArenaAlignment), persistent_arena_(kDefaultArenaAlignment), - preserve_inputs_(preserve_inputs) {} + preserve_inputs_(preserve_inputs), + preserve_intermediates_(preserve_intermediates) {} ArenaPlanner::~ArenaPlanner() {} int64_t ArenaPlanner::BasePointer(TfLiteAllocationType type) { @@ -164,13 +165,15 @@ TfLiteStatus ArenaPlanner::PlanAllocations() { // Then update the ref-counts of the node's inputs, and if necessary queue // them for deallocation. - TfLiteIntArray* node_inputs = node.inputs; - for (int j = 0; j < node_inputs->size; ++j) { - int tensor_index = node_inputs->data[j]; - if (tensor_index != kOptionalTensor) { - refcounts[tensor_index]--; - if (refcounts[tensor_index] == 0) { - TF_LITE_ENSURE_STATUS(deallocate(i, tensor_index)); + if (!preserve_intermediates_) { + TfLiteIntArray* node_inputs = node.inputs; + for (int j = 0; j < node_inputs->size; ++j) { + int tensor_index = node_inputs->data[j]; + if (tensor_index != kOptionalTensor) { + refcounts[tensor_index]--; + if (refcounts[tensor_index] == 0) { + TF_LITE_ENSURE_STATUS(deallocate(i, tensor_index)); + } } } } diff --git a/tensorflow/contrib/lite/arena_planner.h b/tensorflow/contrib/lite/arena_planner.h index 1d84950e91..82c866734f 100644 --- a/tensorflow/contrib/lite/arena_planner.h +++ b/tensorflow/contrib/lite/arena_planner.h @@ -47,7 +47,7 @@ class ArenaPlanner : public MemoryPlanner { // graph will not share memory with any other tensor, effectively preserving // them until the end of inference. ArenaPlanner(TfLiteContext* context, std::unique_ptr graph_info, - bool preserve_inputs); + bool preserve_inputs, bool preserve_intermediates); ~ArenaPlanner() override; ArenaPlanner(const ArenaPlanner&) = delete; ArenaPlanner& operator=(const ArenaPlanner&) = delete; @@ -104,7 +104,14 @@ class ArenaPlanner : public MemoryPlanner { // declared as kTfLiteArenaRwPersistent. SimpleMemoryArena persistent_arena_; + // Ensure that the memory self-allocated for inputs is never reused by the + // allocator. This allows for example, multiple runs without getting + // unpredictable results. bool preserve_inputs_; + + // If true, then no overlapping of memory areas is done, meaning intermediates + // results can be queried after running (modulo running delegates). + bool preserve_intermediates_; }; } // namespace tflite diff --git a/tensorflow/contrib/lite/arena_planner_test.cc b/tensorflow/contrib/lite/arena_planner_test.cc index f5bd1932f9..1adb426d58 100644 --- a/tensorflow/contrib/lite/arena_planner_test.cc +++ b/tensorflow/contrib/lite/arena_planner_test.cc @@ -156,7 +156,7 @@ class ArenaPlannerTest : public ::testing::Test { context_.ReportError = ReportError; planner_.reset(new ArenaPlanner( &context_, std::unique_ptr(new TestGraphInfo(graph)), - preserve_inputs)); + preserve_inputs, /*preserve intermediates*/ false)); CHECK(planner_->ResetAllocations() == kTfLiteOk); CHECK(planner_->PlanAllocations() == kTfLiteOk); } diff --git a/tensorflow/contrib/lite/interpreter.cc b/tensorflow/contrib/lite/interpreter.cc index 0641a08636..d103786694 100644 --- a/tensorflow/contrib/lite/interpreter.cc +++ b/tensorflow/contrib/lite/interpreter.cc @@ -593,7 +593,7 @@ TfLiteStatus Interpreter::PrepareOpsAndTensors() { if (!memory_planner_) { memory_planner_.reset(new ArenaPlanner( &context_, std::unique_ptr(new InterpreterInfo(this)), - /*preserve_inputs=*/true)); + /*preserve_inputs=*/true, /*preserve_intermediates*/ false)); memory_planner_->PlanAllocations(); } -- cgit v1.2.3 From d98b99d1cd4337ee11e7cbc4c9b6324f0e381502 Mon Sep 17 00:00:00 2001 From: Reed Wanderman-Milne Date: Fri, 13 Jul 2018 11:05:09 -0700 Subject: Add version of SessionFactory::NewSession that returns Status. This causes DirectSession to report a better error message if there is an error initializing GPUs. PiperOrigin-RevId: 204498143 --- tensorflow/core/common_runtime/direct_session.cc | 14 ++++++-------- tensorflow/core/common_runtime/session.cc | 20 +++++++++++++------- tensorflow/core/common_runtime/session_factory.h | 7 ++++++- tensorflow/core/common_runtime/session_test.cc | 6 ++++-- .../core/distributed_runtime/rpc/grpc_session.cc | 15 ++++++--------- tensorflow/core/public/session.h | 2 +- 6 files changed, 36 insertions(+), 28 deletions(-) diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc index f903faf1bd..b1f0e0b6e7 100644 --- a/tensorflow/core/common_runtime/direct_session.cc +++ b/tensorflow/core/common_runtime/direct_session.cc @@ -146,18 +146,15 @@ class DirectSessionFactory : public SessionFactory { return options.target.empty(); } - Session* NewSession(const SessionOptions& options) override { + Status NewSession(const SessionOptions& options, + Session** out_session) override { // Must do this before the CPU allocator is created. if (options.config.graph_options().build_cost_model() > 0) { EnableCPUAllocatorFullStats(true); } std::vector devices; - const Status s = DeviceFactory::AddDevices( - options, "/job:localhost/replica:0/task:0", &devices); - if (!s.ok()) { - LOG(ERROR) << s; - return nullptr; - } + TF_RETURN_IF_ERROR(DeviceFactory::AddDevices( + options, "/job:localhost/replica:0/task:0", &devices)); DirectSession* session = new DirectSession(options, new DeviceMgr(devices), this); @@ -165,7 +162,8 @@ class DirectSessionFactory : public SessionFactory { mutex_lock l(sessions_lock_); sessions_.push_back(session); } - return session; + *out_session = session; + return Status::OK(); } Status Reset(const SessionOptions& options, diff --git a/tensorflow/core/common_runtime/session.cc b/tensorflow/core/common_runtime/session.cc index 4a9248171b..8c30beeec2 100644 --- a/tensorflow/core/common_runtime/session.cc +++ b/tensorflow/core/common_runtime/session.cc @@ -53,27 +53,33 @@ Status Session::PRun(const string& handle, Session* NewSession(const SessionOptions& options) { SessionFactory* factory; - const Status s = SessionFactory::GetFactory(options, &factory); + Status s = SessionFactory::GetFactory(options, &factory); if (!s.ok()) { LOG(ERROR) << s; return nullptr; } - return factory->NewSession(options); + Session* out_session; + s = NewSession(options, &out_session); + if (!s.ok()) { + LOG(ERROR) << "Failed to create session: " << s; + return nullptr; + } + return out_session; } Status NewSession(const SessionOptions& options, Session** out_session) { SessionFactory* factory; - const Status s = SessionFactory::GetFactory(options, &factory); + Status s = SessionFactory::GetFactory(options, &factory); if (!s.ok()) { *out_session = nullptr; LOG(ERROR) << s; return s; } - *out_session = factory->NewSession(options); - if (!*out_session) { - return errors::Internal("Failed to create session."); + s = factory->NewSession(options, out_session); + if (!s.ok()) { + *out_session = nullptr; } - return Status::OK(); + return s; } Status Reset(const SessionOptions& options, diff --git a/tensorflow/core/common_runtime/session_factory.h b/tensorflow/core/common_runtime/session_factory.h index df3198a70d..81c172c6ae 100644 --- a/tensorflow/core/common_runtime/session_factory.h +++ b/tensorflow/core/common_runtime/session_factory.h @@ -30,7 +30,12 @@ struct SessionOptions; class SessionFactory { public: - virtual Session* NewSession(const SessionOptions& options) = 0; + // Creates a new session and stores it in *out_session, or fails with an error + // status if the Session could not be created. Caller takes ownership of + // *out_session if this returns Status::OK(). + virtual Status NewSession(const SessionOptions& options, + Session** out_session) = 0; + virtual bool AcceptsOptions(const SessionOptions& options) = 0; // Abort and close all existing sessions, disconnecting their resources from diff --git a/tensorflow/core/common_runtime/session_test.cc b/tensorflow/core/common_runtime/session_test.cc index feaf29c7bb..1fa5aad60c 100644 --- a/tensorflow/core/common_runtime/session_test.cc +++ b/tensorflow/core/common_runtime/session_test.cc @@ -47,8 +47,10 @@ class FakeSessionFactory : public SessionFactory { return str_util::StartsWith(options.target, "fake"); } - Session* NewSession(const SessionOptions& options) override { - return nullptr; + Status NewSession(const SessionOptions& options, + Session** out_session) override { + *out_session = nullptr; + return Status::OK(); } }; class FakeSessionRegistrar { diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc index fd1c150fa7..fdce1b10e0 100644 --- a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc +++ b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc @@ -452,15 +452,12 @@ class GrpcSessionFactory : public SessionFactory { return str_util::StartsWith(options.target, kSchemePrefix); } - Session* NewSession(const SessionOptions& options) override { - std::unique_ptr ret; - Status s = GrpcSession::Create(options, &ret); - if (s.ok()) { - return ret.release(); - } else { - LOG(ERROR) << "Error during session construction: " << s.ToString(); - return nullptr; - } + Status NewSession(const SessionOptions& options, + Session** out_session) override { + std::unique_ptr session; + TF_RETURN_IF_ERROR(GrpcSession::Create(options, &session)); + *out_session = session.release(); + return Status::OK(); } // Invokes the session specific static method to reset containers. diff --git a/tensorflow/core/public/session.h b/tensorflow/core/public/session.h index d58c877cfd..cc8596ef3d 100644 --- a/tensorflow/core/public/session.h +++ b/tensorflow/core/public/session.h @@ -237,7 +237,7 @@ class Session { /// If session creation succeeds, the new `Session` will be stored in /// `*out_session`, the caller will take ownership of the returned /// `*out_session`, and this function will return `OK()`. Otherwise, this -/// function will return an error status. +/// function will return an error status and set *out_session to nullptr. Status NewSession(const SessionOptions& options, Session** out_session); /// \brief Resets resource containers associated with a target. -- cgit v1.2.3 From 9c2a75ac753696f7f3a0d1cbd187baa743055454 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Fri, 13 Jul 2018 11:11:59 -0700 Subject: Internal Change. PiperOrigin-RevId: 204499384 --- tensorflow/python/estimator/BUILD | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD index 8ee38d35cc..6c415b1bf2 100644 --- a/tensorflow/python/estimator/BUILD +++ b/tensorflow/python/estimator/BUILD @@ -706,6 +706,14 @@ py_library( visibility = ["//visibility:public"], ) +py_library( + name = "expect_h5py_installed", + # This is a dummy rule used as a numpy dependency in open-source. + # We expect h5py to already be installed on the system, e.g. via + # `pip install h5py' + visibility = ["//visibility:public"], +) + py_library( name = "expect_six_installed", # This is a dummy rule used as a numpy dependency in open-source. -- cgit v1.2.3 From 3c3b9578bfcae1df34f7e885b086001fdb691993 Mon Sep 17 00:00:00 2001 From: Toby Boyd Date: Fri, 13 Jul 2018 11:16:24 -0700 Subject: Add NCCL 2.x to docker files. PiperOrigin-RevId: 204500106 --- tensorflow/tools/docker/Dockerfile.devel-gpu | 2 ++ tensorflow/tools/docker/Dockerfile.gpu | 1 + 2 files changed, 3 insertions(+) diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu index 5ec43b8cb8..2818b822b8 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu @@ -15,6 +15,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ libcudnn7=7.1.4.18-1+cuda9.0 \ libcudnn7-dev=7.1.4.18-1+cuda9.0 \ + libnccl2=2.2.13-1+cuda9.0 \ + libnccl-dev=2.2.13-1+cuda9.0 \ libcurl3-dev \ libfreetype6-dev \ libhdf5-serial-dev \ diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu index 9197651ff4..28d4371da3 100644 --- a/tensorflow/tools/docker/Dockerfile.gpu +++ b/tensorflow/tools/docker/Dockerfile.gpu @@ -13,6 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cuda-cusparse-9-0 \ curl \ libcudnn7=7.1.4.18-1+cuda9.0 \ + libnccl2=2.2.13-1+cuda9.0 \ libfreetype6-dev \ libhdf5-serial-dev \ libpng12-dev \ -- cgit v1.2.3 From baf95402b1f5b559c3488710d911cbc00ade0416 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Fri, 13 Jul 2018 12:00:51 -0700 Subject: [TF:XLA] Bump open source llvm revision to r336991 PiperOrigin-RevId: 204506936 --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index cd4f17a5ff..ed654c3285 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -472,11 +472,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): tf_http_archive( name = "llvm", urls = [ - "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/ae80745b73e435d07e7fb9c12589304ee29e7f59.tar.gz", - "https://github.com/llvm-mirror/llvm/archive/ae80745b73e435d07e7fb9c12589304ee29e7f59.tar.gz", + "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/bd8c8d759852871609ba2e4e79868420f751949d.tar.gz", + "https://github.com/llvm-mirror/llvm/archive/bd8c8d759852871609ba2e4e79868420f751949d.tar.gz", ], - sha256 = "de69b6f92a634b4d12b9e03ebd8eb34c28f997d9480c28358d6efd4c433fe853", - strip_prefix = "llvm-ae80745b73e435d07e7fb9c12589304ee29e7f59", + sha256 = "0c63e8583b213543309e8577ffe87a0cf34cc22269630d2c5c2f0a2345fda4a8", + strip_prefix = "llvm-bd8c8d759852871609ba2e4e79868420f751949d", build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"), ) -- cgit v1.2.3 From 63e6b9bf43049472b33393df74de271b6aa33863 Mon Sep 17 00:00:00 2001 From: Smit Hinsu Date: Fri, 13 Jul 2018 12:46:24 -0700 Subject: Update default NCCL version while installing from source to 2.2 This is to keep the default configuration consistent with prebuilt TensorFlow. PiperOrigin-RevId: 204513386 --- configure.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/configure.py b/configure.py index 8930c3a1f1..d411214817 100644 --- a/configure.py +++ b/configure.py @@ -35,7 +35,7 @@ except ImportError: _DEFAULT_CUDA_VERSION = '9.0' _DEFAULT_CUDNN_VERSION = '7' -_DEFAULT_NCCL_VERSION = '1.3' +_DEFAULT_NCCL_VERSION = '2.2' _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,5.2' _DEFAULT_CUDA_PATH = '/usr/local/cuda' _DEFAULT_CUDA_PATH_LINUX = '/opt/cuda' @@ -1097,8 +1097,10 @@ def set_tf_nccl_install_path(environ_cp): raise ValueError('Currently NCCL is only supported on Linux platforms.') ask_nccl_version = ( - 'Please specify the NCCL version you want to use. ' - '[Leave empty to default to NCCL %s]: ') % _DEFAULT_NCCL_VERSION + 'Please specify the NCCL version you want to use. If NCCL %s is not ' + 'installed, then you can use version 1.3 that can be fetched ' + 'automatically but it may have worse performance with multiple GPUs. ' + '[Default is %s]: ') % (_DEFAULT_NCCL_VERSION, _DEFAULT_NCCL_VERSION) for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): tf_nccl_version = get_from_env_or_user_or_default( -- cgit v1.2.3 From bb7541b96c49b06b5c13775f3666ae2b8450a457 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 13:09:46 -0700 Subject: Automated rollback of commit 57527f7e47e3e67966b432065f510a601a4d8647 PiperOrigin-RevId: 204516578 --- tensorflow/python/keras/callbacks.py | 28 ++---------- tensorflow/python/keras/callbacks_test.py | 76 ------------------------------- 2 files changed, 4 insertions(+), 100 deletions(-) diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py index 5d66db232a..53d907a2cc 100644 --- a/tensorflow/python/keras/callbacks.py +++ b/tensorflow/python/keras/callbacks.py @@ -32,10 +32,8 @@ import numpy as np import six from tensorflow.python.keras import backend as K -from tensorflow.python.keras import optimizers from tensorflow.python.keras.utils.generic_utils import Progbar from tensorflow.python.ops import array_ops -from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Variable from tensorflow.python.platform import tf_logging as logging from tensorflow.python.summary import summary as tf_summary from tensorflow.python.util.tf_export import tf_export @@ -644,35 +642,17 @@ class LearningRateScheduler(Callback): self.verbose = verbose def on_epoch_begin(self, epoch, logs=None): - # TODO(yashkatariya): Change the property checking when the learning - # rate attribute is unified across all TF Optimizers. - if isinstance(self.model.optimizer, optimizers.TFOptimizer): - if not hasattr(self.model.optimizer.optimizer, '_lr') and not hasattr( - self.model.optimizer.optimizer, '_learning_rate'): - raise ValueError( - 'TF Optimizer must have a "_lr" or "_learning_rate" attribute.') - else: - opt = self.model.optimizer.optimizer - if hasattr(opt, '_lr'): - opt_lr = Variable(opt._lr) # pylint: disable=protected-access - elif hasattr(opt, '_learning_rate'): - opt_lr = Variable(opt._learning_rate) # pylint: disable=protected-access - else: - if not hasattr(self.model.optimizer, 'lr'): - raise ValueError('Optimizer must have a "lr" attribute.') - else: - opt = self.model.optimizer - opt_lr = opt.lr - + if not hasattr(self.model.optimizer, 'lr'): + raise ValueError('Optimizer must have a "lr" attribute.') try: # new API - lr = float(K.get_value(opt_lr)) + lr = float(K.get_value(self.model.optimizer.lr)) lr = self.schedule(epoch, lr) except TypeError: # Support for old API for backward compatibility lr = self.schedule(epoch) if not isinstance(lr, (float, np.float32, np.float64)): raise ValueError('The output of the "schedule" function ' 'should be float.') - K.set_value(opt_lr, lr) + K.set_value(self.model.optimizer.lr, lr) if self.verbose > 0: print('\nEpoch %05d: LearningRateScheduler reducing learning ' 'rate to %s.' % (epoch + 1, lr)) diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py index 244d48591c..45598cafd3 100644 --- a/tensorflow/python/keras/callbacks_test.py +++ b/tensorflow/python/keras/callbacks_test.py @@ -29,16 +29,10 @@ import numpy as np from tensorflow.core.framework import summary_pb2 from tensorflow.python import keras -from tensorflow.python.eager import context -from tensorflow.python.framework import test_util from tensorflow.python.keras import testing_utils -from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Variable from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging as logging from tensorflow.python.summary.writer import writer_cache -from tensorflow.python.training.adam import AdamOptimizer -from tensorflow.python.training.gradient_descent import GradientDescentOptimizer - try: import h5py # pylint:disable=g-import-not-at-top @@ -376,76 +370,6 @@ class KerasCallbacksTest(test.TestCase): float(keras.backend.get_value( model.optimizer.lr)) - 0.01 / 4) < keras.backend.epsilon() - @test_util.run_in_graph_and_eager_modes - def test_TF_LearningRateScheduler_Adam(self): - with self.test_session(): - with context.eager_mode(): - np.random.seed(1337) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=TRAIN_SAMPLES, - test_samples=TEST_SAMPLES, - input_shape=(INPUT_DIM,), - num_classes=NUM_CLASSES) - y_test = keras.utils.to_categorical(y_test) - y_train = keras.utils.to_categorical(y_train) - model = keras.models.Sequential() - model.add( - keras.layers.Dense( - NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu')) - model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax')) - model.compile( - loss='categorical_crossentropy', - optimizer=AdamOptimizer(), - metrics=['accuracy']) - cbks = [keras.callbacks.LearningRateScheduler(lambda x: 1. / (1. + x))] - model.fit( - x_train, - y_train, - batch_size=BATCH_SIZE, - validation_data=(x_test, y_test), - callbacks=cbks, - epochs=5, - verbose=0) - opt_lr = model.optimizer.optimizer._lr - self.assertLess( - float(keras.backend.get_value( - Variable(opt_lr))) - 0.2, keras.backend.epsilon()) - - @test_util.run_in_graph_and_eager_modes - def test_TF_LearningRateScheduler_GradientDescent(self): - with self.test_session(): - with context.eager_mode(): - np.random.seed(1337) - (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( - train_samples=TRAIN_SAMPLES, - test_samples=TEST_SAMPLES, - input_shape=(INPUT_DIM,), - num_classes=NUM_CLASSES) - y_test = keras.utils.to_categorical(y_test) - y_train = keras.utils.to_categorical(y_train) - model = keras.models.Sequential() - model.add( - keras.layers.Dense( - NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu')) - model.add(keras.layers.Dense(NUM_CLASSES, activation='softmax')) - model.compile( - loss='categorical_crossentropy', - optimizer=GradientDescentOptimizer(1e-3), - metrics=['accuracy']) - cbks = [keras.callbacks.LearningRateScheduler(lambda x: 1. / (1. + x))] - model.fit( - x_train, - y_train, - batch_size=BATCH_SIZE, - validation_data=(x_test, y_test), - callbacks=cbks, - epochs=5, - verbose=0) - opt_lr = model.optimizer.optimizer._learning_rate - self.assertLess( - float(keras.backend.get_value( - Variable(opt_lr))) - 0.2, keras.backend.epsilon()) - def test_ReduceLROnPlateau(self): with self.test_session(): np.random.seed(1337) -- cgit v1.2.3 From 1210d97118ecf10e1e5b43fef970709a5909d03b Mon Sep 17 00:00:00 2001 From: Smit Hinsu Date: Fri, 13 Jul 2018 13:19:22 -0700 Subject: Change size of linear_operator_low_rank_update_test to large as it is currently timing out in windows PiperOrigin-RevId: 204518052 --- tensorflow/python/kernel_tests/linalg/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD index 69d3aa4017..487418e694 100644 --- a/tensorflow/python/kernel_tests/linalg/BUILD +++ b/tensorflow/python/kernel_tests/linalg/BUILD @@ -197,7 +197,7 @@ cuda_py_test( cuda_py_test( name = "linear_operator_low_rank_update_test", - size = "medium", + size = "large", srcs = ["linear_operator_low_rank_update_test.py"], additional_deps = [ "//tensorflow/python/ops/linalg", -- cgit v1.2.3 From 34c427aecd0fcc23816bc04399c6d40022ca4480 Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Fri, 13 Jul 2018 13:21:13 -0700 Subject: Reorder estimator guides in index and nav based on feedback. PiperOrigin-RevId: 204518322 --- tensorflow/docs_src/guide/index.md | 15 ++++++--------- tensorflow/docs_src/guide/leftnav_files | 4 ++-- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/tensorflow/docs_src/guide/index.md b/tensorflow/docs_src/guide/index.md index eefdb9ceae..f78dfc9a89 100644 --- a/tensorflow/docs_src/guide/index.md +++ b/tensorflow/docs_src/guide/index.md @@ -16,15 +16,12 @@ works. The units are as follows: ## Estimators -* @{$estimators} provides an introduction. -* @{$premade_estimators}, introduces Estimators for machine learning. -* @{$custom_estimators}, which demonstrates how to build and train models you - design yourself. -* @{$feature_columns}, which shows how an Estimator can handle a variety of input - data types without changes to the model. -* @{$datasets_for_estimators} describes using tf.data with estimators. -* @{$checkpoints}, which explains how to save training progress and resume where - you left off. +* @{$estimators}, learn how to use Estimators for machine learning. +* @{$premade_estimators}, the basics of premade Estimators. +* @{$checkpoints}, save training progress and resume where you left off. +* @{$feature_columns}, handle a variety of input data types without changes to the model. +* @{$datasets_for_estimators}, use `tf.data` to input data. +* @{$custom_estimators}, write your own Estimator. ## Accelerators diff --git a/tensorflow/docs_src/guide/leftnav_files b/tensorflow/docs_src/guide/leftnav_files index 357a2a1cb9..b3324278c1 100644 --- a/tensorflow/docs_src/guide/leftnav_files +++ b/tensorflow/docs_src/guide/leftnav_files @@ -8,10 +8,10 @@ datasets.md ### Estimators estimators.md: Introduction to Estimators premade_estimators.md -custom_estimators.md +checkpoints.md feature_columns.md datasets_for_estimators.md -checkpoints.md +custom_estimators.md ### Accelerators using_gpu.md -- cgit v1.2.3 From 98b9a4e45a559217bc89960b889c130af95c1d1a Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 13 Jul 2018 13:29:28 -0700 Subject: tfdbg: remove Experimental tags and obsolete library * debug_gateway and the related node_outputs_callback are not used and hence are removed in this CL. PiperOrigin-RevId: 204519574 --- tensorflow/core/common_runtime/direct_session.cc | 3 +- tensorflow/core/common_runtime/executor.cc | 21 - tensorflow/core/common_runtime/executor.h | 3 - tensorflow/core/debug/BUILD | 55 -- tensorflow/core/debug/debug_gateway.cc | 122 --- tensorflow/core/debug/debug_gateway.h | 83 -- tensorflow/core/debug/debug_gateway_test.cc | 1011 ---------------------- tensorflow/core/distributed_runtime/graph_mgr.cc | 2 +- tensorflow/core/ops/debug_ops.cc | 2 +- tensorflow/core/protobuf/debug.proto | 4 +- 10 files changed, 5 insertions(+), 1301 deletions(-) delete mode 100644 tensorflow/core/debug/debug_gateway.cc delete mode 100644 tensorflow/core/debug/debug_gateway.h delete mode 100644 tensorflow/core/debug/debug_gateway_test.cc diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc index b1f0e0b6e7..4c670820be 100644 --- a/tensorflow/core/common_runtime/direct_session.cc +++ b/tensorflow/core/common_runtime/direct_session.cc @@ -1186,12 +1186,11 @@ Status DirectSession::CreateExecutors( delete kernel; } }; - params.node_outputs_cb = node_outputs_callback_; optimizer.Optimize(lib, options_.env, device, &iter->second, /*shape_map=*/nullptr); - // EXPERIMENTAL: tfdbg inserts debug nodes in the graph. + // TensorFlow Debugger (tfdbg) inserts debug nodes in the graph. const DebugOptions& debug_options = options.callable_options.run_options().debug_options(); if (!debug_options.debug_tensor_watch_opts().empty()) { diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc index 5f3809ddd6..8096139d90 100644 --- a/tensorflow/core/common_runtime/executor.cc +++ b/tensorflow/core/common_runtime/executor.cc @@ -1966,14 +1966,6 @@ Status ExecutorState::ProcessOutputs(const NodeItem& item, OpKernelContext* ctx, device_context = device_context_map_[node->id()]; } - // Experimental: debugger (tfdb) access to intermediate node completion. - if (item.num_outputs == 0 && impl_->params_.node_outputs_cb != nullptr) { - // If the node has no output, invoke the callback with output slot set to - // -1, signifying that this is a no-output node. - s.Update(impl_->params_.node_outputs_cb(item.node->name(), -1, nullptr, - false, ctx)); - } - for (int i = 0; i < item.num_outputs; ++i) { const TensorValue val = ctx->release_output(i); if (val.tensor == nullptr) { @@ -2018,13 +2010,6 @@ Status ExecutorState::ProcessOutputs(const NodeItem& item, OpKernelContext* ctx, LogMemory::RecordTensorOutput(ctx->op_kernel().name(), ctx->step_id(), i, to_log); } - - // Experimental: debugger (tfdb) access to intermediate node - // outputs. - if (impl_->params_.node_outputs_cb != nullptr) { - s.Update(impl_->params_.node_outputs_cb(item.node->name(), i, - out->ref, true, ctx)); - } } else { // NOTE that std::move is used here, so val.tensor goes to // uninitialized state (val.tensor->IsInitialized return false). @@ -2036,12 +2021,6 @@ Status ExecutorState::ProcessOutputs(const NodeItem& item, OpKernelContext* ctx, LogMemory::RecordTensorOutput(ctx->op_kernel().name(), ctx->step_id(), i, *out->val); } - - // Experimental: debugger access to intermediate node outputs. - if (impl_->params_.node_outputs_cb != nullptr) { - s.Update(impl_->params_.node_outputs_cb( - item.node->name(), i, out->val.get(), false, ctx)); - } } } else { s.Update(errors::Internal("Output ", i, " of type ", diff --git a/tensorflow/core/common_runtime/executor.h b/tensorflow/core/common_runtime/executor.h index e5d7b7c53c..cd01b43aea 100644 --- a/tensorflow/core/common_runtime/executor.h +++ b/tensorflow/core/common_runtime/executor.h @@ -103,7 +103,6 @@ class Executor { const Tensor* tensor, const bool is_ref, OpKernelContext* ctx)> NodeOutputsCallback; - NodeOutputsCallback node_outputs_cb = nullptr; }; typedef std::function DoneCallback; virtual void RunAsync(const Args& args, DoneCallback done) = 0; @@ -139,8 +138,6 @@ struct LocalExecutorParams { // when the executor is deleted. std::function create_kernel; std::function delete_kernel; - - Executor::Args::NodeOutputsCallback node_outputs_cb; }; ::tensorflow::Status NewLocalExecutor(const LocalExecutorParams& params, std::unique_ptr graph, diff --git a/tensorflow/core/debug/BUILD b/tensorflow/core/debug/BUILD index 36e9b3455a..591c22b8f6 100644 --- a/tensorflow/core/debug/BUILD +++ b/tensorflow/core/debug/BUILD @@ -81,25 +81,6 @@ cc_library( alwayslink = 1, ) -tf_cuda_library( - name = "debug_gateway_internal", - srcs = ["debug_gateway.cc"], - hdrs = ["debug_gateway.h"], - copts = tf_copts(), - linkstatic = 1, - deps = [ - ":debug", - "//tensorflow/core:core_cpu_internal", - "//tensorflow/core:direct_session_internal", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:proto_text", - "//tensorflow/core:protos_all_cc", - ], - alwayslink = 1, -) - tf_cuda_library( name = "debugger_state_impl", srcs = ["debugger_state_impl.cc"], @@ -187,42 +168,6 @@ tf_cuda_library( ], ) -# TODO(cais): Fix flakiness on GPU and change this back to a tf_cc_test_gpu. -# See b/34081273. -tf_cc_test( - name = "debug_gateway_test", - size = "small", - srcs = ["debug_gateway_test.cc"], - args = ["--heap_check=local"], - linkstatic = tf_kernel_tests_linkstatic(), - tags = [ - "no_cuda_on_cpu_tap", - "no_gpu", - ], - deps = [ - ":debug", - ":debug_gateway_internal", - ":debug_graph_utils", - "//tensorflow/cc:cc_ops", - "//tensorflow/core:all_kernels", - "//tensorflow/core:core_cpu", - "//tensorflow/core:core_cpu_internal", - "//tensorflow/core:direct_session", - "//tensorflow/core:direct_session_internal", - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:gpu_runtime", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - "//tensorflow/core/kernels:debug_ops", - "//tensorflow/core/kernels:ops_util", - ], -) - tf_cc_test( name = "debug_io_utils_test", size = "small", diff --git a/tensorflow/core/debug/debug_gateway.cc b/tensorflow/core/debug/debug_gateway.cc deleted file mode 100644 index 2e1aabd1cc..0000000000 --- a/tensorflow/core/debug/debug_gateway.cc +++ /dev/null @@ -1,122 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/debug/debug_gateway.h" - -#include - -#include "tensorflow/core/common_runtime/device_factory.h" -#include "tensorflow/core/common_runtime/session_factory.h" -#include "tensorflow/core/framework/tensor.h" - -namespace tensorflow { - -DebugGateway::DebugGateway(DirectSession* session) : session_(session) { - session_->node_outputs_callback_ = - [this](const string& node_name, const int output_slot, - const Tensor* tensor, const bool is_ref, OpKernelContext* ctx) { - if (comp_cb_ != nullptr && output_slot <= 0) { - // The node completion callback is invoked once for a node regardless - // of whether the node has zero, one or more outputs. - // The output_slot can be negative (-1, or kControlSlot) if - // node_outputs_callback_ is invoked for a node with no output. If - // that is the case, notify the callback that the node in question has - // no output. - comp_cb_(node_name, output_slot == 0); - } - - // Copy tensor values (e.g., from GPU to host) only if the - // value callback is not nullptr. - if (val_cb_ != nullptr && output_slot >= 0) { - CopyTensor(node_name, output_slot, tensor, ctx, - [this, node_name, output_slot, - is_ref](const Tensor* copied_tensor) { - val_cb_(node_name, output_slot, *copied_tensor, is_ref); - }); - } - - return Status::OK(); - }; -} - -DebugGateway::~DebugGateway() { - if (session_ != nullptr) { - session_->node_outputs_callback_ = nullptr; - } -} - -void DebugGateway::SetNodeCompletionCallback(NodeCompletionCallback callback) { - comp_cb_ = std::move(callback); -} - -void DebugGateway::SetNodeValueCallback(NodeValueCallback callback) { - val_cb_ = std::move(callback); -} - -void DebugGateway::CopyTensor(const string& node_name, const int output_slot, - const Tensor* src_tensor, OpKernelContext* ctx, - CopyDoneCallback copy_done_cb) { - Device* device = static_cast(ctx->device()); - - // Determine if the tensor is initialized properly. - // The second part of the check is necessary because in some cases, a - // tensor can pass the IsInitialized() check, but the dtype is not set, - // e.g., tf.FIFOQueue. - if (src_tensor->IsInitialized() && DataTypeSize(src_tensor->dtype()) > 0) { - // Tensor is initialized. - - string tensor_tag = strings::StrCat(node_name, ":", output_slot); - - // Create copied tensor on host - Allocator* cpu_allocator = tensorflow::cpu_allocator(); - Tensor cpu_tensor(cpu_allocator, src_tensor->dtype(), src_tensor->shape()); - - // Determine if the tensor is on device (GPU) or host (CPU). - // The second part of the check is necessary because even an OpKernel on - // may have output tensors allocated on CPU. - if ((device->name().find("GPU:") != string::npos || - device->name().find("SYCL:") != string::npos) && - !ctx->output_alloc_attr(output_slot).on_host()) { - // GPU tensors: Copy it to host (CPU). - DeviceContext* device_ctxt = ctx->op_device_context(); - - // Copy device (e.g., GPU) tensor to host and when done, invoke the - // callback. - device_ctxt->CopyDeviceTensorToCPU( - src_tensor, "TensorCopy", device, &cpu_tensor, - [node_name, cpu_tensor, copy_done_cb](const Status& s) { - if (s.ok()) { - copy_done_cb(&cpu_tensor); - } else { - LOG(ERROR) << "Copying of device Tensor " << node_name - << " to CPU for debugging failed."; - } - }); - } else { - // For CPU tensors, copy the source tensor and own the copy, because the - // value callback may outlive the life time of the tensor and the tensor - // may shared the underlying buffer with other tensors. - cpu_tensor.UnsafeCopyFromInternal(*src_tensor, src_tensor->dtype(), - src_tensor->shape()); - - copy_done_cb(&cpu_tensor); - } - } else { - // Tensor is not initialized: No need to copy. - copy_done_cb(src_tensor); - } -} - -} // namespace tensorflow diff --git a/tensorflow/core/debug/debug_gateway.h b/tensorflow/core/debug/debug_gateway.h deleted file mode 100644 index bf5b6e08db..0000000000 --- a/tensorflow/core/debug/debug_gateway.h +++ /dev/null @@ -1,83 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_DEBUG_DEBUG_SESSION_H_ -#define TENSORFLOW_DEBUG_DEBUG_SESSION_H_ - -#include - -#include "tensorflow/core/common_runtime/direct_session.h" -#include "tensorflow/core/common_runtime/executor.h" - -namespace tensorflow { - -// Experimental. tfdb (TensorFlow Debugger): Gateway to intermediate node -// outputs during Session Run calls. Currently limited to DirectSession. -class DebugGateway { - public: - DebugGateway(DirectSession* session); - virtual ~DebugGateway(); - - // Callback for node completion. This callback is invoked only once for - // a node regardless of whether it has one or more outputs. The value(s) of - // the output tensor(s) are not necessarily available when this callback is - // invoked. They may need to be asynchronously copied from device (e.g., - // GPU) to host, hence the need for the NodeValueCallback below. - // - // Args: - // node_name: Name of the node that has just completed execution - // any_output: Whether the node has any output(s) - typedef std::function - NodeCompletionCallback; - void SetNodeCompletionCallback(NodeCompletionCallback callback); - - // Callback for node value. This is invoked when the value of a node's - // output tensor is available on the host, possibly after copying from - // a device (e.g., GPU). - // - // Args: - // node_name: Name of the node of which the output has become available - // output_slot: Output slot number of the output Tensor - // tensor_value: Reference to the tensor value - // is_ref: Whether the output of the reference type - typedef std::function - NodeValueCallback; - void SetNodeValueCallback(NodeValueCallback callback); - - // TODO(cais): Add whitelists for ops/tensors (e.g., {"A:0", "B:0"}) - // for node completion callback (whitelist_comp_) and node value callback - // (whitelist_val_). If whitelist_comp_ is non-empty, the gateway will - // invoke the NodeCompletionCallback only for the nodes specified in the - // whitelist. And so forth for whitelist_val_. - - private: - DirectSession* session_; - // TODO(cais): DebugGateway currently supports only DirectSession. Add - // support for GrpcSession. - - NodeCompletionCallback comp_cb_ = nullptr; - NodeValueCallback val_cb_ = nullptr; - - typedef std::function CopyDoneCallback; - - void CopyTensor(const string& node_name, const int output_slot, - const Tensor* src_tensor, OpKernelContext* ctx, - CopyDoneCallback copy_done_cb); -}; - -} // end namespace tensorflow - -#endif // TENSORFLOW_DEBUG_DEBUG_SESSION_H_ diff --git a/tensorflow/core/debug/debug_gateway_test.cc b/tensorflow/core/debug/debug_gateway_test.cc deleted file mode 100644 index b1bbd3f698..0000000000 --- a/tensorflow/core/debug/debug_gateway_test.cc +++ /dev/null @@ -1,1011 +0,0 @@ -/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/debug/debug_gateway.h" - -#include -#include -#include -#include - -#include "tensorflow/core/debug/debug_graph_utils.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/graph/testlib.h" -#include "tensorflow/core/lib/core/notification.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/protobuf/rewriter_config.pb.h" - -namespace tensorflow { -namespace { - -std::unique_ptr CreateSession() { - SessionOptions options; - // Turn off graph optimizer so we can observe intermediate node states. - options.config.mutable_graph_options() - ->mutable_optimizer_options() - ->set_opt_level(OptimizerOptions_Level_L0); - options.config.mutable_graph_options() - ->mutable_rewrite_options() - ->set_constant_folding(RewriterConfig::OFF); - options.config.mutable_graph_options() - ->mutable_rewrite_options() - ->set_dependency_optimization(RewriterConfig::OFF); - - return std::unique_ptr( - dynamic_cast(NewSession(options))); -} - -class SessionDebugMinusAXTest : public ::testing::Test { - public: - void Initialize(std::initializer_list a_values) { - Graph graph(OpRegistry::Global()); - -#if GOOGLE_CUDA - const string kDeviceName = "/job:localhost/replica:0/task:0/device:GPU:0"; -#elif defined(TENSORFLOW_USE_SYCL) - const string kDeviceName = "/job:localhost/replica:0/task:0/device:SYCL:0"; -#else - const string kDeviceName = "/job:localhost/replica:0/task:0/device:CPU:0"; -#endif - - Tensor a_tensor(DT_FLOAT, TensorShape({2, 2})); - test::FillValues(&a_tensor, a_values); - Node* a = test::graph::Constant(&graph, a_tensor); - a->set_assigned_device_name(kDeviceName); - a_ = a->name(); - - Tensor x_tensor(DT_FLOAT, TensorShape({2, 1})); - test::FillValues(&x_tensor, {1, 1}); - Node* x = test::graph::Constant(&graph, x_tensor); - x->set_assigned_device_name(kDeviceName); - x_ = x->name(); - - // y = A * x - Node* y = test::graph::Matmul(&graph, a, x, false, false); - y->set_assigned_device_name(kDeviceName); - y_ = y->name(); - - Node* y_neg = test::graph::Unary(&graph, "Neg", y); - y_neg_ = y_neg->name(); - y_neg->set_assigned_device_name(kDeviceName); - - test::graph::ToGraphDef(&graph, &def_); - } - - string a_; - string x_; - string y_; - string y_neg_; - GraphDef def_; -}; - -TEST_F(SessionDebugMinusAXTest, RunSimpleNetwork) { - Initialize({3, 2, -1, 0}); - auto session = CreateSession(); - ASSERT_TRUE(session != nullptr); - - DebugGateway debug_gateway(session.get()); - - // Supply completion and value callbacks - mutex mu; - // Completed nodes with and without outputs - std::vector completed_nodes_w_outputs; - std::vector completed_nodes_wo_outputs; - - Notification callbacks_done; - debug_gateway.SetNodeCompletionCallback( - [&mu, &completed_nodes_w_outputs, &completed_nodes_wo_outputs]( - const string& node_name, const bool any_output) { - mutex_lock l(mu); - if (any_output) { - completed_nodes_w_outputs.push_back(node_name); - } else { - completed_nodes_wo_outputs.push_back(node_name); - } - }); - - std::vector tensors_initialized; - std::unordered_map tensor_vals; - // output_slot values recorded in value callbacks - std::vector output_slots_val; - // is_ref values recorded in value callbacks - std::vector is_refs_val; - - debug_gateway.SetNodeValueCallback( - [this, &mu, &tensors_initialized, &tensor_vals, &output_slots_val, - &is_refs_val, - &callbacks_done](const string& node_name, const int output_slot, - const Tensor& tensor_value, const bool is_ref) { - mutex_lock l(mu); - tensors_initialized.push_back(tensor_value.IsInitialized()); - tensor_vals.insert(std::make_pair(node_name, tensor_value)); - output_slots_val.push_back(output_slot); - is_refs_val.push_back(is_ref); - - // Set the notification once we have the value from the target node. - if (node_name == y_neg_ && !callbacks_done.HasBeenNotified()) { - callbacks_done.Notify(); - } - }); - - TF_ASSERT_OK(session->Create(def_)); - - std::vector> inputs; - - // Request two targets: one fetch output and one non-fetched output. - std::vector output_names = {y_ + ":0"}; - std::vector target_nodes = {y_neg_}; - std::vector outputs; - Status s = session->Run(inputs, output_names, target_nodes, &outputs); - TF_ASSERT_OK(s); - - // Wait for callbacks to complete. - callbacks_done.WaitForNotification(); - - ASSERT_EQ(1, outputs.size()); - // The first output should be initialized and have the correct - // output. - auto mat = outputs[0].matrix(); - ASSERT_TRUE(outputs[0].IsInitialized()); - EXPECT_FLOAT_EQ(5.0, mat(0, 0)); - - // Verify the calling history of the completion callback - // The following verifies each node with output(s) invoked the callback - // exactly once. - ASSERT_GE(completed_nodes_w_outputs.size(), 4); // There may be added nodes. - - ASSERT_EQ(1, std::count(completed_nodes_w_outputs.begin(), - completed_nodes_w_outputs.end(), a_)); - ASSERT_EQ(1, std::count(completed_nodes_w_outputs.begin(), - completed_nodes_w_outputs.end(), x_)); - ASSERT_EQ(1, std::count(completed_nodes_w_outputs.begin(), - completed_nodes_w_outputs.end(), y_)); - ASSERT_EQ(1, std::count(completed_nodes_w_outputs.begin(), - completed_nodes_w_outputs.end(), y_neg_)); - - // Apart from nodes with outputs, there are also no-output (control) nodes. - // They ought to be captured by the DebugGateway through - // NodeOutputCallback as well. - ASSERT_GT(completed_nodes_wo_outputs.size(), 0); - - // The DebugGateway should have captured the _SOURCE node. - ASSERT_LE(1, std::count(completed_nodes_wo_outputs.begin(), - completed_nodes_wo_outputs.end(), "_SOURCE")); - - // Verify the calling history of the value callabck - ASSERT_EQ(completed_nodes_w_outputs.size(), tensors_initialized.size()); - - // In this graph, there is no uninitialized node value. - ASSERT_EQ( - tensors_initialized.end(), - std::find(tensors_initialized.begin(), tensors_initialized.end(), false)); - - ASSERT_EQ(completed_nodes_w_outputs.size(), tensor_vals.size()); - ASSERT_EQ(completed_nodes_w_outputs.size(), output_slots_val.size()); - ASSERT_EQ(completed_nodes_w_outputs.size(), is_refs_val.size()); - - // Verify the intermediate tensor values captured through the value callback - auto mat_a = tensor_vals[a_].matrix(); - ASSERT_EQ(3.0, mat_a(0, 0)); - ASSERT_EQ(2.0, mat_a(0, 1)); - ASSERT_EQ(-1.0, mat_a(1, 0)); - ASSERT_EQ(0.0, mat_a(1, 1)); - - auto mat_x = tensor_vals[x_].matrix(); - ASSERT_EQ(1.0, mat_x(0, 0)); - ASSERT_EQ(1.0, mat_x(1, 0)); - - auto mat_y = tensor_vals[y_].matrix(); - ASSERT_EQ(5.0, mat_y(0, 0)); - ASSERT_EQ(-1.0, mat_y(1, 0)); - - auto mat_y_neg = tensor_vals[y_neg_].matrix(); - ASSERT_EQ(-5.0, mat_y_neg(0, 0)); - ASSERT_EQ(1.0, mat_y_neg(1, 0)); - - // In this graph, all outputs are on the first slot - ASSERT_EQ(output_slots_val.size(), - std::count_if(output_slots_val.begin(), output_slots_val.end(), - [](int slot) { return slot == 0; })); - - // In this graph, there is no ref-type tensor. - ASSERT_EQ(is_refs_val.end(), - std::find(is_refs_val.begin(), is_refs_val.end(), true)); -} - -TEST_F(SessionDebugMinusAXTest, RunSimpleNetworkWithTwoDebugNodesInserted) { - // Tensor contains one count of NaN - Initialize({3, std::numeric_limits::quiet_NaN(), -1, 0}); - auto session = CreateSession(); - ASSERT_TRUE(session != nullptr); - - DebugGateway debug_gateway(session.get()); - - // Create debug tensor watch options with two debug ops: - // DebugIdentity and DebugNanCount - RunOptions run_opts; - run_opts.set_output_partition_graphs(true); - - const string debug_identity = "DebugIdentity"; - const string debug_nan_count = "DebugNanCount"; - DebugTensorWatch* tensor_watch_opts = - run_opts.mutable_debug_options()->add_debug_tensor_watch_opts(); - tensor_watch_opts->set_node_name(y_); - tensor_watch_opts->set_output_slot(0); - tensor_watch_opts->add_debug_ops(debug_identity); - tensor_watch_opts->add_debug_ops(debug_nan_count); - - // Expected name of the inserted debug node - string debug_identity_node_name = DebugNodeInserter::GetDebugNodeName( - strings::StrCat(y_, ":", 0), 0, debug_identity); - string debug_nan_count_node_name = DebugNodeInserter::GetDebugNodeName( - strings::StrCat(y_, ":", 0), 1, debug_nan_count); - - // Supply completion and value callbacks - mutex mu; - // Completed nodes with and without outputs - std::vector completed_debug_nodes; - - Notification callbacks_done; - debug_gateway.SetNodeCompletionCallback( - [&mu, &debug_identity_node_name, &debug_nan_count_node_name, - &completed_debug_nodes](const string& node_name, const bool any_output) { - mutex_lock l(mu); - if (any_output && (node_name == debug_identity_node_name || - node_name == debug_nan_count_node_name)) { - completed_debug_nodes.push_back(node_name); - } - }); - - std::vector watched_tensor_vals; - std::vector debug_identity_tensor_vals; - std::vector debug_nan_count_tensor_vals; - - debug_gateway.SetNodeValueCallback( - [this, &mu, &debug_identity_node_name, &debug_nan_count_node_name, - &watched_tensor_vals, &debug_identity_tensor_vals, - &debug_nan_count_tensor_vals, - &callbacks_done](const string& node_name, const int output_slot, - const Tensor& tensor_value, const bool is_ref) { - mutex_lock l(mu); - if (node_name == y_) { - watched_tensor_vals.push_back(tensor_value); - } else if (node_name == debug_identity_node_name && output_slot == 0) { - // output_slot == 0 carries the debug signal. Same below. - debug_identity_tensor_vals.push_back(tensor_value); - } else if (node_name == debug_nan_count_node_name && output_slot == 0) { - debug_nan_count_tensor_vals.push_back(tensor_value); - } - - // Set the notification once we have the value from the target node. - if (node_name == y_neg_ && !callbacks_done.HasBeenNotified()) { - callbacks_done.Notify(); - } - }); - - TF_ASSERT_OK(session->Create(def_)); - - std::vector> inputs; - - // Request two targets: one fetch output and one non-fetched output. - std::vector output_names = {y_ + ":0"}; - std::vector target_nodes = {y_neg_}; - std::vector outputs; - - RunMetadata run_metadata; - Status s = session->Run(run_opts, inputs, output_names, target_nodes, - &outputs, &run_metadata); - TF_ASSERT_OK(s); - -// Verify the correct number of partition graphs (GraphDefs) outputted -// through RunMetadata, given whether GPU is involved. -#if GOOGLE_CUDA - ASSERT_EQ(2, run_metadata.partition_graphs().size()); -#elif defined(TENSORFLOW_USE_SYCL) - ASSERT_EQ(2, run_metadata.partition_graphs().size()); -#else - ASSERT_EQ(1, run_metadata.partition_graphs().size()); -#endif - - // Wait for callbacks to complete. - callbacks_done.WaitForNotification(); - - // Verify that each of the two debug nodes has completed exactly once. - ASSERT_EQ(2, completed_debug_nodes.size()); - ASSERT_EQ( - 1, std::count(completed_debug_nodes.begin(), completed_debug_nodes.end(), - debug_identity_node_name)); - ASSERT_EQ( - 1, std::count(completed_debug_nodes.begin(), completed_debug_nodes.end(), - debug_nan_count_node_name)); - - // Verify that the tensor values from the watched node and the identity - // debug node are received and they are equal (owing to the debug op being - // "DebugIdentity") - ASSERT_EQ(1, watched_tensor_vals.size()); - ASSERT_EQ(1, debug_identity_tensor_vals.size()); - auto mat_y = watched_tensor_vals[0].matrix(); - auto mat_identity = debug_identity_tensor_vals[0].matrix(); - // ASSERT_EQ doesn't work for nan == nan - ASSERT_TRUE(std::isnan(mat_y(0, 0))); - ASSERT_TRUE(std::isnan(mat_identity(0, 0))); - ASSERT_EQ(-1, mat_identity(1, 0)); - - // Verify that the output from the NaN-count debug node indicates exactly - // one NaN. - ASSERT_EQ(1, debug_nan_count_tensor_vals.size()); - ASSERT_EQ(1, debug_nan_count_tensor_vals[0].scalar()()); -} - -#if !defined(GOOGLE_CUDA) && !defined(TENSORFLOW_USE_SYCL) -// TODO(cais): Reinstate the following test for concurrent debugged runs on -// a GPU once the root cause of the ~0.5% flakiness has been addressed. -// (b/34081273) -TEST_F(SessionDebugMinusAXTest, - RunSimpleNetworkConcurrentlyWithDifferentDebugTensorWatches) { - // Test concurrent Run() calls on a graph with different debug watches. - - Initialize({3, 2, -1, 0}); - auto session = CreateSession(); - ASSERT_TRUE(session != nullptr); - TF_ASSERT_OK(session->Create(def_)); - - // Number of concurrent Run() calls to launch. - const int kConcurrentRuns = 3; - thread::ThreadPool* tp = - new thread::ThreadPool(Env::Default(), "test", kConcurrentRuns); - - std::vector output_names = {y_ + ":0"}; - std::vector target_nodes = {y_neg_}; - - mutex mu; - DebugGateway debug_gateway(session.get()); - std::unordered_map debug_identity_tensor_vals; - - const string debug_identity = "DebugIdentity"; - - const string a_debug_identity_node_name = DebugNodeInserter::GetDebugNodeName( - strings::StrCat(a_, ":", 0), 0, debug_identity); - const string x_debug_identity_node_name = DebugNodeInserter::GetDebugNodeName( - strings::StrCat(x_, ":", 0), 0, debug_identity); - const string y_debug_identity_node_name = DebugNodeInserter::GetDebugNodeName( - strings::StrCat(y_, ":", 0), 0, debug_identity); - - Notification callbacks_done; - volatile int val_callback_count = 0; - - debug_gateway.SetNodeValueCallback( - [this, &mu, &val_callback_count, &a_debug_identity_node_name, - &x_debug_identity_node_name, &y_debug_identity_node_name, - &debug_identity_tensor_vals, &callbacks_done, - &kConcurrentRuns](const string& node_name, const int output_slot, - const Tensor& tensor_value, const bool is_ref) { - mutex_lock l(mu); - - if (node_name == a_debug_identity_node_name && output_slot == 0) { - debug_identity_tensor_vals["a"] = tensor_value; - val_callback_count++; - } else if (node_name == x_debug_identity_node_name && - output_slot == 0) { - // output_slot == 0 carries the debug signal. - debug_identity_tensor_vals["x"] = tensor_value; - val_callback_count++; - } else if (node_name == y_debug_identity_node_name && - output_slot == 0) { - debug_identity_tensor_vals["y"] = tensor_value; - val_callback_count++; - } - - // Set the notification once we have the value from the callbacks from - // all the concurrent Run() calls. - if (val_callback_count == kConcurrentRuns && - !callbacks_done.HasBeenNotified()) { - callbacks_done.Notify(); - } - }); - - int run_counter = 0; - mutex run_lock; - - // Function to be executed concurrently. - auto fn = [this, &run_lock, &run_counter, &session, output_names, - target_nodes, &debug_identity]() { - // Create unique debug tensor watch options for each of the concurrent - // run calls. - RunOptions run_opts; - run_opts.set_output_partition_graphs(true); - - DebugTensorWatch* tensor_watch_opts = - run_opts.mutable_debug_options()->add_debug_tensor_watch_opts(); - tensor_watch_opts->set_output_slot(0); - tensor_watch_opts->add_debug_ops(debug_identity); - - { - // Let the concurrent runs watch different tensors. - - mutex_lock l(run_lock); - - if (run_counter == 0) { - // Let the 1st concurrent run watch a. - tensor_watch_opts->set_node_name(a_); - } else if (run_counter == 1) { - // Let the 2nd concurrent watch x. - tensor_watch_opts->set_node_name(x_); - } else if (run_counter == 2) { - // Let the 3rd concurrent watch y. - tensor_watch_opts->set_node_name(y_); - } - - run_counter++; - } - - // Run the graph. - RunMetadata run_metadata; - std::vector> inputs; - std::vector outputs; - Status s = session->Run(run_opts, inputs, output_names, target_nodes, - &outputs, &run_metadata); - TF_ASSERT_OK(s); - - ASSERT_EQ(1, run_metadata.partition_graphs().size()); - - ASSERT_EQ(1, outputs.size()); - ASSERT_TRUE(outputs[0].IsInitialized()); - ASSERT_EQ(TensorShape({2, 1}), outputs[0].shape()); - auto mat = outputs[0].matrix(); - EXPECT_FLOAT_EQ(5.0, mat(0, 0)); - EXPECT_FLOAT_EQ(-1.0, mat(1, 0)); - }; - - for (int i = 0; i < kConcurrentRuns; ++i) { - tp->Schedule(fn); - } - - // Wait for the debug callbacks to finish. - callbacks_done.WaitForNotification(); - - // Wait for the concurrent functions with Run() calls to finish. - delete tp; - - { - mutex_lock l(mu); - - ASSERT_EQ(kConcurrentRuns, val_callback_count); - ASSERT_EQ(kConcurrentRuns, debug_identity_tensor_vals.size()); - - ASSERT_EQ(TensorShape({2, 2}), debug_identity_tensor_vals["a"].shape()); - auto a_mat_identity = debug_identity_tensor_vals["a"].matrix(); - ASSERT_EQ(3.0, a_mat_identity(0, 0)); - ASSERT_EQ(2.0, a_mat_identity(0, 1)); - ASSERT_EQ(-1.0, a_mat_identity(1, 0)); - ASSERT_EQ(0.0, a_mat_identity(1, 1)); - - ASSERT_EQ(TensorShape({2, 1}), debug_identity_tensor_vals["x"].shape()); - auto x_mat_identity = debug_identity_tensor_vals["x"].matrix(); - ASSERT_EQ(1.0, x_mat_identity(0, 0)); - ASSERT_EQ(1.0, x_mat_identity(1, 0)); - - ASSERT_EQ(TensorShape({2, 1}), debug_identity_tensor_vals["y"].shape()); - auto y_mat_identity = debug_identity_tensor_vals["y"].matrix(); - ASSERT_EQ(5.0, y_mat_identity(0, 0)); - ASSERT_EQ(-1.0, y_mat_identity(1, 0)); - } -} -#endif - -class SessionDebugOutputSlotWithoutOutgoingEdgeTest : public ::testing::Test { - public: - void Initialize() { - Graph graph(OpRegistry::Global()); - -#if GOOGLE_CUDA - const string kDeviceName = "/job:localhost/replica:0/task:0/device:GPU:0"; -#elif defined(TENSORFLOW_USE_SYCL) - const string kDeviceName = "/job:localhost/replica:0/task:0/device:SYCL:0"; -#else - const string kDeviceName = "/job:localhost/replica:0/task:0/device:CPU:0"; -#endif - - Tensor a_tensor(DT_FLOAT, TensorShape({1, 1})); - test::FillValues(&a_tensor, {42.0}); - Node* a = test::graph::Constant(&graph, a_tensor); - a->set_assigned_device_name(kDeviceName); - - Node* c = test::graph::Constant(&graph, a_tensor); - c->set_assigned_device_name(kDeviceName); - c_ = c->name(); - - // Node c will be executed only because of the control edge from c to y. - // Its output slot (slot 0) does not have an outgoing edge. This test - // is for testing that the debugger can watch that slot properly. - Node* y = test::graph::NoOp(&graph, {c}); - y->set_assigned_device_name(kDeviceName); - y_ = y->name(); - - test::graph::ToGraphDef(&graph, &def_); - } - - string c_; - string y_; - GraphDef def_; -}; - -TEST_F(SessionDebugOutputSlotWithoutOutgoingEdgeTest, - WatchSlotWithoutOutgoingEdge) { - Initialize(); - auto session = CreateSession(); - ASSERT_TRUE(session != nullptr); - - DebugGateway debug_gateway(session.get()); - - // Supply completion and value callbacks - mutex mu; - - string debug_identity_node_name = DebugNodeInserter::GetDebugNodeName( - strings::StrCat(c_, ":", 0), 0, "DebugIdentity"); - - Notification callbacks_done; - - std::vector debug_identity_tensor_vals; - debug_gateway.SetNodeValueCallback( - [this, &mu, &callbacks_done, &debug_identity_node_name, - &debug_identity_tensor_vals]( - const string& node_name, const int output_slot, - const Tensor& tensor_value, const bool is_ref) { - mutex_lock l(mu); - - if (node_name == debug_identity_node_name && output_slot == 0) { - debug_identity_tensor_vals.push_back(tensor_value); - - if (!callbacks_done.HasBeenNotified()) { - callbacks_done.Notify(); - } - } - }); - - // Add DebugIdentity watch on c:0, which does not have an outgoing edge. - RunOptions run_opts; - run_opts.set_output_partition_graphs(true); - - DebugTensorWatch* tensor_watch_opts = - run_opts.mutable_debug_options()->add_debug_tensor_watch_opts(); - tensor_watch_opts->set_node_name(c_); - tensor_watch_opts->set_output_slot(0); - tensor_watch_opts->add_debug_ops("DebugIdentity"); - - TF_ASSERT_OK(session->Create(def_)); - - // Invoke Session::Run() on y. - std::vector> inputs; - std::vector output_names; - std::vector target_nodes = {y_}; - std::vector outputs; - - RunMetadata run_metadata; - Status s = session->Run(run_opts, inputs, output_names, target_nodes, - &outputs, &run_metadata); - TF_ASSERT_OK(s); - - // Wait for callbacks to complete. - callbacks_done.WaitForNotification(); - - // Assert that DebugIdentity node watching the control edge has been run. - ASSERT_EQ(1, debug_identity_tensor_vals.size()); - auto mat_identity = debug_identity_tensor_vals[0].matrix(); - ASSERT_EQ(42.0, mat_identity(0, 0)); -} - -class SessionDebugVariableTest : public ::testing::Test { - public: - void Initialize() { - Graph graph(OpRegistry::Global()); - -#if GOOGLE_CUDA - const string kDeviceName = "/job:localhost/replica:0/task:0/device:GPU:0"; -#elif defined(TENSORFLOW_USE_SYCL) - const string kDeviceName = "/job:localhost/replica:0/task:0/device:SYCL:0"; -#else - const string kDeviceName = "/job:localhost/replica:0/task:0/device:CPU:0"; -#endif - - // Define variable node. - var_node_name_ = "var"; - Node* var = - test::graph::Var(&graph, DT_FLOAT, TensorShape({3}), var_node_name_); - var->set_assigned_device_name(kDeviceName); - - // Define the initial value and the initial-value node. - Tensor nan_nan_seven(DT_FLOAT, TensorShape({3})); - nan_nan_seven.flat()(0) = std::numeric_limits::quiet_NaN(); - nan_nan_seven.flat()(1) = std::numeric_limits::quiet_NaN(); - nan_nan_seven.flat()(2) = 7.0; - - init_val_node_name_ = "init_val"; - Node* init_val = - test::graph::Constant(&graph, nan_nan_seven, init_val_node_name_); - init_val->set_assigned_device_name(kDeviceName); - - // Define node for variable value initialization - Node* init = test::graph::Assign(&graph, var, init_val); - init->set_assigned_device_name(kDeviceName); - init_node_name_ = init->name(); - - // Define new value node - Tensor nan_eight_eight(DT_FLOAT, TensorShape({3})); - nan_eight_eight.flat()(0) = std::numeric_limits::quiet_NaN(); - nan_eight_eight.flat()(1) = 8.0; - nan_eight_eight.flat()(2) = 8.0; - - Node* new_val = test::graph::Constant(&graph, nan_eight_eight); - new_val->set_assigned_device_name(kDeviceName); - new_val_node_name_ = new_val->name(); - - // Define node for assigning new value - Node* assign = test::graph::Assign(&graph, var, new_val); - assign->set_assigned_device_name(kDeviceName); - assign_node_name_ = assign->name(); - - test::graph::ToGraphDef(&graph, &def_); - } - - string var_node_name_; - string init_val_node_name_; - string init_node_name_; - string new_val_node_name_; - string assign_node_name_; - GraphDef def_; -}; - -TEST_F(SessionDebugVariableTest, WatchUninitializedVariableWithDebugOps) { - Initialize(); - auto session = CreateSession(); - ASSERT_TRUE(session != nullptr); - - DebugGateway debug_gateway(session.get()); - - TF_ASSERT_OK(session->Create(def_)); - - // Set up DebugTensorWatch for an uninitialized tensor (in node var). - RunOptions run_opts; - const string debug_identity = "DebugIdentity"; - DebugTensorWatch* tensor_watch_opts = - run_opts.mutable_debug_options()->add_debug_tensor_watch_opts(); - tensor_watch_opts->set_node_name(var_node_name_); - tensor_watch_opts->set_output_slot(0); - tensor_watch_opts->add_debug_ops(debug_identity); - - // Expected name of the inserted debug node - string debug_identity_node_name = DebugNodeInserter::GetDebugNodeName( - strings::StrCat(var_node_name_, ":", 0), 0, debug_identity); - - // Supply completion and value callbacks - mutex mu; - // Completed nodes with and without outputs - std::vector completed_debug_nodes; - - Notification callbacks_done; - debug_gateway.SetNodeCompletionCallback( - [this, &mu, &debug_identity_node_name, &completed_debug_nodes, - &callbacks_done](const string& node_name, const bool any_output) { - mutex_lock l(mu); - if (any_output && (node_name == debug_identity_node_name)) { - completed_debug_nodes.push_back(node_name); - } - }); - - std::vector debug_identity_tensor_vals; - - debug_gateway.SetNodeValueCallback( - [this, &mu, &debug_identity_node_name, &debug_identity_tensor_vals, - &callbacks_done](const string& node_name, const int output_slot, - const Tensor& tensor_value, const bool is_ref) { - mutex_lock l(mu); - if (node_name == debug_identity_node_name && output_slot == 0) { - // output_slot == 0 carries the debug signal. Same below. - debug_identity_tensor_vals.push_back(tensor_value); - } - - // Set the notification once we have the value from the target node. - if (node_name == init_node_name_ && !callbacks_done.HasBeenNotified()) { - callbacks_done.Notify(); - } - }); - - // First run the initialization op - std::vector> inputs_init; - std::vector outputs_init; - - RunMetadata run_metadata; - Status s = session->Run(run_opts, inputs_init, {init_node_name_}, {}, - &outputs_init, &run_metadata); - TF_ASSERT_OK(s); - - callbacks_done.WaitForNotification(); - - ASSERT_EQ(1, completed_debug_nodes.size()); - ASSERT_EQ( - 1, std::count(completed_debug_nodes.begin(), completed_debug_nodes.end(), - debug_identity_node_name)); - - // Assert the output reflects the uninitialized nature of var's tensor. - ASSERT_EQ(1, debug_identity_tensor_vals.size()); - ASSERT_FALSE(debug_identity_tensor_vals[0].IsInitialized()); - ASSERT_EQ(DT_FLOAT, debug_identity_tensor_vals[0].dtype()); - ASSERT_EQ(TensorShape({3}), debug_identity_tensor_vals[0].shape()); -} - -TEST_F(SessionDebugVariableTest, VariableAssignWithDebugOps) { - // Tensor contains one count of NaN - Initialize(); - auto session = CreateSession(); - ASSERT_TRUE(session != nullptr); - - DebugGateway debug_gateway(session.get()); - - TF_ASSERT_OK(session->Create(def_)); - - // First run the initialization op - std::vector> inputs_init; - std::vector outputs_init; - Status s = session->Run(inputs_init, {init_node_name_}, {}, &outputs_init); - TF_ASSERT_OK(s); - - // Create debug tensor watch options with two ref-type debug ops: - // DebugIdentity and DebugNanCount - RunOptions run_opts; - run_opts.set_output_partition_graphs(true); - const string debug_identity = "DebugIdentity"; - const string debug_nan_count = "DebugNanCount"; - DebugTensorWatch* tensor_watch_opts = - run_opts.mutable_debug_options()->add_debug_tensor_watch_opts(); - tensor_watch_opts->set_node_name(var_node_name_); - tensor_watch_opts->set_output_slot(0); - tensor_watch_opts->add_debug_ops(debug_identity); - tensor_watch_opts->add_debug_ops(debug_nan_count); - - char tempdir_template[] = "/tmp/tfdbg_XXXXXX"; - string temp_dir(mkdtemp(tempdir_template)); - tensor_watch_opts->add_debug_urls(strings::StrCat("file://", temp_dir)); - - // Expected name of the inserted debug node - string debug_identity_node_name = DebugNodeInserter::GetDebugNodeName( - strings::StrCat(var_node_name_, ":", 0), 0, debug_identity); - string debug_nan_count_node_name = DebugNodeInserter::GetDebugNodeName( - strings::StrCat(var_node_name_, ":", 0), 1, debug_nan_count); - - // Supply completion and value callbacks - mutex mu; - // Completed nodes with and without outputs - std::vector completed_debug_nodes; - - Notification callbacks_done; - debug_gateway.SetNodeCompletionCallback( - [this, &mu, &debug_identity_node_name, &debug_nan_count_node_name, - &completed_debug_nodes, - &callbacks_done](const string& node_name, const bool any_output) { - mutex_lock l(mu); - if (any_output && (node_name == debug_identity_node_name || - node_name == debug_nan_count_node_name)) { - completed_debug_nodes.push_back(node_name); - } - }); - - std::vector debug_identity_tensor_vals; - std::vector debug_nan_count_tensor_vals; - - debug_gateway.SetNodeValueCallback( - [this, &mu, &debug_identity_node_name, &debug_nan_count_node_name, - &debug_identity_tensor_vals, &debug_nan_count_tensor_vals, - &callbacks_done](const string& node_name, const int output_slot, - const Tensor& tensor_value, const bool is_ref) { - mutex_lock l(mu); - if (node_name == debug_identity_node_name && output_slot == 0) { - // output_slot == 0 carries the debug signal. Same below. - debug_identity_tensor_vals.push_back(tensor_value); - } else if (node_name == debug_nan_count_node_name && output_slot == 0) { - debug_nan_count_tensor_vals.push_back(tensor_value); - } - - // Set the notification once we have the value from the target node. - if (node_name == assign_node_name_ && - !callbacks_done.HasBeenNotified()) { - callbacks_done.Notify(); - } - }); - - // // Request two targets: one fetch output and one non-fetched output. - std::vector> inputs; - std::vector output_names = {assign_node_name_ + ":0"}; - std::vector target_nodes = {assign_node_name_}; - std::vector outputs; - - // Run with RunOptions that has tensor watches - RunMetadata run_metadata; - s = session->Run(run_opts, inputs, output_names, target_nodes, &outputs, - &run_metadata); - TF_ASSERT_OK(s); - -#if GOOGLE_CUDA - ASSERT_EQ(2, run_metadata.partition_graphs().size()); -#elif defined(TENSORFLOW_USE_SYCL) - ASSERT_EQ(2, run_metadata.partition_graphs().size()); -#else - ASSERT_EQ(1, run_metadata.partition_graphs().size()); -#endif - - // Wait for callbacks to complete. - callbacks_done.WaitForNotification(); - - // Verify that the update has happened properly. - ASSERT_EQ(1, outputs.size()); - ASSERT_TRUE(std::isnan(outputs[0].vec()(0))); - ASSERT_EQ(8.0, outputs[0].vec()(1)); // Expect new value - ASSERT_EQ(8.0, outputs[0].vec()(2)); // Expect new value - - // Verify that each of the two debug nodes has completed exactly once. - ASSERT_EQ(2, completed_debug_nodes.size()); - ASSERT_EQ( - 1, std::count(completed_debug_nodes.begin(), completed_debug_nodes.end(), - debug_identity_node_name)); - ASSERT_EQ( - 1, std::count(completed_debug_nodes.begin(), completed_debug_nodes.end(), - debug_nan_count_node_name)); - - // Verify that the values from the ref identity node reflects the value - // before the new assign. - ASSERT_EQ(1, debug_identity_tensor_vals.size()); - - auto vec_identity = debug_identity_tensor_vals[0].vec(); - ASSERT_TRUE(std::isnan(vec_identity(0))); - ASSERT_TRUE(std::isnan(vec_identity(1))); - ASSERT_EQ(7.0, vec_identity(2)); - - // Verify that the output from the NaN-count debug node indicates exactly - // two NaNs, i.e., reflecting the value before the new assign. - ASSERT_EQ(1, debug_nan_count_tensor_vals.size()); - ASSERT_EQ(2, debug_nan_count_tensor_vals[0].scalar()()); -} - -#if defined(GOOGLE_CUDA) || defined(TENSORFLOW_USE_SYCL) -class SessionDebugGPUSwitchTest : public ::testing::Test { - public: - void Initialize() { - Graph graph(OpRegistry::Global()); - -#ifdef GOOGLE_CUDA - const string kDeviceName = "/job:localhost/replica:0/task:0/device:GPU:0"; -#elif TENSORFLOW_USE_SYCL - const string kDeviceName = "/job:localhost/replica:0/task:0/device:SYCL:0"; -#endif - - Tensor vb(DT_BOOL, TensorShape({})); - vb.scalar()() = true; - Tensor vi(DT_INT64, TensorShape({})); - vi.scalar()() = 42; - // So vi is expected to be forwarded to the second output port of sw. - - Node* pred = test::graph::Constant(&graph, vb); - pred->set_assigned_device_name(kDeviceName); - pred_node_name_ = pred->name(); - - Node* value = test::graph::Constant(&graph, vi); - pred->set_assigned_device_name(kDeviceName); - value_node_name_ = value->name(); - - Node* sw = test::graph::Switch(&graph, value, pred); - sw->set_assigned_device_name(kDeviceName); - sw_node_name_ = sw->name(); - - Node* z = test::graph::Identity(&graph, sw, 1); - sw->set_assigned_device_name(kDeviceName); - z_node_name_ = z->name(); - - test::graph::ToGraphDef(&graph, &def_); - } - - string pred_node_name_; - string value_node_name_; - string sw_node_name_; - string z_node_name_; - GraphDef def_; -}; - -// Test for debug-watching tensors marked as HOST_MEMORY on GPU. -TEST_F(SessionDebugGPUSwitchTest, RunSwitchWithHostMemoryDebugOp) { - Initialize(); - auto session = CreateSession(); - ASSERT_TRUE(session != nullptr); - - DebugGateway debug_gateway(session.get()); - - RunOptions run_opts; - run_opts.set_output_partition_graphs(true); - // This is the name of the boolean tensor fed as pred to the Switch node. - // On GPU, this edge is HOST_MEMORY. - const string watched_tensor = strings::StrCat(pred_node_name_, "/_1"); - - const string debug_identity = "DebugIdentity"; - DebugTensorWatch* tensor_watch_opts = - run_opts.mutable_debug_options()->add_debug_tensor_watch_opts(); - tensor_watch_opts->set_node_name(watched_tensor); - tensor_watch_opts->set_output_slot(0); - tensor_watch_opts->add_debug_ops(debug_identity); - - // Expected name of the inserted debug node - string debug_identity_node_name = DebugNodeInserter::GetDebugNodeName( - strings::StrCat(watched_tensor, ":", 0), 0, debug_identity); - - // Supply completion and value callbacks - mutex mu; - // Completed nodes with and without outputs - std::vector completed_nodes_w_outputs; - std::vector completed_nodes_wo_outputs; - - Notification callbacks_done; - debug_gateway.SetNodeCompletionCallback( - [&mu, &completed_nodes_w_outputs, &completed_nodes_wo_outputs]( - const string& node_name, const bool any_output) { - mutex_lock l(mu); - if (any_output) { - completed_nodes_w_outputs.push_back(node_name); - } else { - completed_nodes_wo_outputs.push_back(node_name); - } - }); - - std::vector debug_identity_tensor_vals; - - debug_gateway.SetNodeValueCallback( - [this, &mu, &debug_identity_node_name, &debug_identity_tensor_vals, - &callbacks_done](const string& node_name, const int output_slot, - const Tensor& tensor_value, const bool is_ref) { - mutex_lock l(mu); - if (node_name == debug_identity_node_name && output_slot == 0) { - debug_identity_tensor_vals.push_back(tensor_value); - } - - // Set the notification once we have the value from the target node. - if (node_name == z_node_name_ && !callbacks_done.HasBeenNotified()) { - callbacks_done.Notify(); - } - }); - - TF_ASSERT_OK(session->Create(def_)); - - std::vector> inputs; - - // Request two targets: one fetch output and one non-fetched output. - std::vector output_names = {z_node_name_ + ":0"}; - std::vector target_nodes = {z_node_name_}; - std::vector outputs; - - RunMetadata run_metadata; - Status s = session->Run(run_opts, inputs, output_names, target_nodes, - &outputs, &run_metadata); - TF_ASSERT_OK(s); - - ASSERT_EQ(2, run_metadata.partition_graphs().size()); - - // Wait for callbacks to complete. - callbacks_done.WaitForNotification(); - - ASSERT_EQ(1, debug_identity_tensor_vals.size()); - ASSERT_TRUE(debug_identity_tensor_vals[0].scalar()()); -} -#endif // GOOGLE_CUDA - -} // end namespace -} // end namespace tensorflow diff --git a/tensorflow/core/distributed_runtime/graph_mgr.cc b/tensorflow/core/distributed_runtime/graph_mgr.cc index e2f13df19f..6c146036ae 100644 --- a/tensorflow/core/distributed_runtime/graph_mgr.cc +++ b/tensorflow/core/distributed_runtime/graph_mgr.cc @@ -261,7 +261,7 @@ Status GraphMgr::InitItem(const string& session, const GraphDef& gdef, optimizer.Optimize(lib, worker_env_->env, params.device, &subgraph, /*shape_map=*/nullptr); - // EXPERIMENTAL: tfdbg inserts debug nodes (i.e., probes) to the graph. + // TensorFlow Debugger (tfdbg) inserts debug nodes in the graph. if (!debug_options.debug_tensor_watch_opts().empty()) { TF_RETURN_IF_ERROR(DecorateAndPublishGraphForDebug( debug_options, subgraph.get(), params.device)); diff --git a/tensorflow/core/ops/debug_ops.cc b/tensorflow/core/ops/debug_ops.cc index 5aebdca1ea..2d9b4360de 100644 --- a/tensorflow/core/ops/debug_ops.cc +++ b/tensorflow/core/ops/debug_ops.cc @@ -20,7 +20,7 @@ limitations under the License. namespace tensorflow { -// EXPERIMENTAL: tfdbg debugger-inserted ops. +// TensorFlow Debugger-inserted ops. // These ops are used only internally by tfdbg. There is no API for users to // direct create them. Users can create them indirectly by using // RunOptions.debug_options during Session::Run() call. See tfdbg documentation diff --git a/tensorflow/core/protobuf/debug.proto b/tensorflow/core/protobuf/debug.proto index 499900f965..811cf406b9 100644 --- a/tensorflow/core/protobuf/debug.proto +++ b/tensorflow/core/protobuf/debug.proto @@ -7,7 +7,7 @@ option java_multiple_files = true; option java_package = "org.tensorflow.framework"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; -// EXPERIMENTAL. Option for watching a node. +// Option for watching a node in TensorFlow Debugger (tfdbg). message DebugTensorWatch { // Name of the node to watch. string node_name = 1; @@ -51,7 +51,7 @@ message DebugTensorWatch { bool tolerate_debug_op_creation_failures = 5; } -// EXPERIMENTAL. Options for initializing DebuggerState. +// Options for initializing DebuggerState in TensorFlow Debugger (tfdbg). message DebugOptions { // Debugging options repeated DebugTensorWatch debug_tensor_watch_opts = 4; -- cgit v1.2.3 From 4277f432f0a677d2b73f7640e34abdca7b261b82 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 13:35:09 -0700 Subject: Re-enabling delete on exit true by default. PiperOrigin-RevId: 204520442 --- tensorflow/contrib/autograph/pyct/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/autograph/pyct/compiler.py b/tensorflow/contrib/autograph/pyct/compiler.py index 538481ff79..c172ab21f6 100644 --- a/tensorflow/contrib/autograph/pyct/compiler.py +++ b/tensorflow/contrib/autograph/pyct/compiler.py @@ -91,7 +91,7 @@ def ast_to_source(node, indentation=' '): def ast_to_object(node, indentation=' ', source_prefix=None, - delete_on_exit=False): + delete_on_exit=True): """Return the Python objects represented by given AST. Compiling the AST code this way ensures that the source code is readable by -- cgit v1.2.3 From dcfaef189162694a645678f76f9cf54a4b705231 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 13:53:26 -0700 Subject: [XLA] Don't call VisibleDeviceCount from GetSupportedPlatforms PiperOrigin-RevId: 204523318 --- tensorflow/compiler/xla/service/platform_util.cc | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tensorflow/compiler/xla/service/platform_util.cc b/tensorflow/compiler/xla/service/platform_util.cc index 7c63c0acc7..39fe3c7835 100644 --- a/tensorflow/compiler/xla/service/platform_util.cc +++ b/tensorflow/compiler/xla/service/platform_util.cc @@ -75,19 +75,6 @@ PlatformUtil::GetSupportedPlatforms() { auto* platform = platform_pair.second; auto compiler_status = Compiler::GetForPlatform(platform); if (compiler_status.ok()) { - if (platform->VisibleDeviceCount() > 0) { - LOG(INFO) << "platform " << platform->Name() << " present with " - << platform->VisibleDeviceCount() << " visible devices"; - } else { - LOG(WARNING) << "platform " << platform->Name() << " present but no " - << "visible devices found"; - } - // Note: currently we call zero device platforms "supported" on the basis - // that, if the platform support was linked in, it was probably intended - // to be used for execution, and this way we can flag an error. - // - // TODO(b/33730287) If we want an alternative version of this behavior we - // could add an --xla_fallback_to_host flag. platforms.push_back(platform); } else { LOG(INFO) << "platform " << platform->Name() << " present but no " -- cgit v1.2.3 From 4555d63e01f66e241359660de39148eaab2a0dd2 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Fri, 13 Jul 2018 14:05:49 -0700 Subject: Disable flaky random seed test PiperOrigin-RevId: 204525365 --- tensorflow/python/eager/function_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index 13c4ee7f15..cdd9fe1760 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -138,7 +138,7 @@ class FunctionTest(test.TestCase): out = sq_op(t) self.assertAllEqual(out, math_ops.matmul(t, t).numpy()) - def testRandomSeed(self): + def disabled_testRandomSeed(self): @function.defun def f(): -- cgit v1.2.3 From 13a3185eb7f8b13a846b25f0913eaa793b37fb1a Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Fri, 13 Jul 2018 14:12:32 -0700 Subject: Allow non-index slices to be used without conversion, instead of rejecting them - they are widely used in TF code. PiperOrigin-RevId: 204526510 --- tensorflow/contrib/autograph/converters/slices.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/contrib/autograph/converters/slices.py b/tensorflow/contrib/autograph/converters/slices.py index 3f5fc57125..de04cc9184 100644 --- a/tensorflow/contrib/autograph/converters/slices.py +++ b/tensorflow/contrib/autograph/converters/slices.py @@ -56,8 +56,7 @@ class SliceTransformer(converter.Base): def visit_Subscript(self, node): node = self.generic_visit(node) if not isinstance(node.slice, gast.Index): - # TODO(mdan): It might make more sense to wave them through. - raise NotImplementedError('non-index slice') + return node if not isinstance(node.ctx, gast.Load): # Index writes are handled at a higher level, one at which the rvalue is -- cgit v1.2.3 From 3d949e2016a967e303b8ddbd3cbe3bd3408320e8 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Fri, 13 Jul 2018 14:16:02 -0700 Subject: Disable flaky tests on windows. PiperOrigin-RevId: 204527084 --- tensorflow/contrib/cmake/tf_tests.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake index eb9482dc25..b2330c4e34 100644 --- a/tensorflow/contrib/cmake/tf_tests.cmake +++ b/tensorflow/contrib/cmake/tf_tests.cmake @@ -193,6 +193,7 @@ if (tensorflow_BUILD_PYTHON_TESTS) # flaky test "${tensorflow_source_dir}/tensorflow/python/profiler/internal/run_metadata_test.py" "${tensorflow_source_dir}/tensorflow/python/profiler/model_analyzer_test.py" + "${tensorflow_source_dir}/tensorflow/python/data/kernel_tests/map_dataset_op_test.py" # Fails because uses data dependencies with bazel "${tensorflow_source_dir}/tensorflow/python/saved_model/saved_model_test.py" "${tensorflow_source_dir}/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py" @@ -216,7 +217,8 @@ if (tensorflow_BUILD_PYTHON_TESTS) ${tensorflow_source_dir}/tensorflow/python/kernel_tests/duplicate_op_test.py ${tensorflow_source_dir}/tensorflow/python/kernel_tests/invalid_op_test.py ${tensorflow_source_dir}/tensorflow/python/kernel_tests/ackermann_test.py - + # Tests too large to run. + ${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py ) if (WIN32) set(tf_test_src_py_exclude -- cgit v1.2.3 From 590af170ca85a4921db0c28e4fa2785462bdcebd Mon Sep 17 00:00:00 2001 From: Jeremy Lau Date: Fri, 13 Jul 2018 14:29:12 -0700 Subject: TPUEstimator: Run tpu.initialize_system() in its own graph whenever the main graph is finalized. PiperOrigin-RevId: 204529164 --- tensorflow/contrib/tpu/python/tpu/tpu_context.py | 10 +- tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 101 ++++++++++++++++++--- 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py index 211c59cb90..e54395f05d 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py @@ -234,7 +234,7 @@ class _InternalTPUContext(object): def mode(self): return self._assert_mode() - def _get_master_address(self): + def master_address(self): mode = self._assert_mode() config = self._config master = ( @@ -244,7 +244,7 @@ class _InternalTPUContext(object): def _get_tpu_system_metadata(self): """Gets the (maybe cached) TPU system metadata.""" - master = self._get_master_address() + master = self.master_address() tpu_system_metadata = self._lazy_tpu_system_metadata_dict.get(master) if tpu_system_metadata is not None: return tpu_system_metadata @@ -261,7 +261,7 @@ class _InternalTPUContext(object): def _get_device_assignment(self): """Gets the (maybe cached) TPU device assignment.""" - master = self._get_master_address() + master = self.master_address() device_assignment = self._lazy_device_assignment_dict.get(master) if device_assignment is not None: return device_assignment @@ -589,7 +589,7 @@ class _InternalTPUContext(object): 'model-parallelism, the total number of TPU cores should be ' 'num_cores_per_replica * num_replicas. Please set it ' 'accordingly or leave it as `None`'.format( - self._get_master_address(), num_replicas, + self.master_address(), num_replicas, user_provided_num_replicas)) raise ValueError(message) @@ -644,7 +644,7 @@ class _OneCoreTPUContext(_InternalTPUContext): def _get_tpu_system_metadata(self): """Gets the (maybe cached) TPU system metadata.""" - master = self._get_master_address() + master = self.master_address() tpu_system_metadata = self._lazy_tpu_system_metadata_dict.get(master) if tpu_system_metadata is not None: return tpu_system_metadata diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 74157a6193..aa407cf4d8 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -43,6 +43,7 @@ from tensorflow.contrib.training.python.training import hparam from tensorflow.core.framework import variable_pb2 from tensorflow.core.framework.summary_pb2 import Summary from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.client import session as session_lib from tensorflow.python.data.ops import dataset_ops from tensorflow.python.estimator import estimator as estimator_lib from tensorflow.python.estimator import model_fn as model_fn_lib @@ -67,6 +68,7 @@ from tensorflow.python.saved_model import tag_constants from tensorflow.python.summary import summary from tensorflow.python.training import basic_session_run_hooks from tensorflow.python.training import evaluation +from tensorflow.python.training import monitored_session from tensorflow.python.training import session_run_hook from tensorflow.python.training import training from tensorflow.python.training import training_util @@ -382,7 +384,14 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook): def begin(self): logging.info('TPU job name %s', self._master_job) self._iterations_per_loop_var = _create_or_get_iterations_per_loop() - self._init_ops = [tpu.initialize_system(job=self._master_job)] + self._init_ops = [] + # For distributed sessions, we can't run initialize_system in a separate + # graph here because 'begin' is only invoked when the MonitoredSession is + # created. We need to reinitialize the system every time MonitoredSession + # creates an underlying tf.Session, so we initialize from Scaffold.finalize. + # See _get_and_wrap_scaffold for more details. + if self._master_job is None: + self._init_ops.append(tpu.initialize_system(job=self._master_job)) self._finalize_ops = [tpu.shutdown_system(job=self._master_job)] summary_writer_init_ops = contrib_summary.summary_writer_initializer_op() @@ -484,7 +493,7 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook): return _OpQueueContext(name=name, target=target, args=args) def after_create_session(self, session, coord): - logging.info('Init TPU system') + logging.info('Running init_ops') session.run(self._init_ops, options=config_pb2.RunOptions(timeout_in_ms=5 * 60 * 1000)) @@ -2700,7 +2709,7 @@ def _eval_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn): outputs_from_all_shards=False, device_assignment=ctx.device_assignment) - scaffold = _get_scaffold(captured_scaffold_fn) + scaffold = _get_and_wrap_scaffold(captured_scaffold_fn, ctx) return loss, host_calls, scaffold @@ -2723,7 +2732,7 @@ def _train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn): outputs_from_all_shards=False, device_assignment=ctx.device_assignment) - scaffold = _get_scaffold(captured_scaffold_fn) + scaffold = _get_and_wrap_scaffold(captured_scaffold_fn, ctx) return loss, host_call, scaffold @@ -2751,7 +2760,7 @@ def _predict_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn): num_shards=num_cores, outputs_from_all_shards=False) - scaffold = _get_scaffold(captured_scaffold_fn) + scaffold = _get_and_wrap_scaffold(captured_scaffold_fn, ctx) return dummy_predict_op, host_calls, scaffold @@ -2841,8 +2850,20 @@ class _CapturedObject(object): return self._object -def _get_scaffold(captured_scaffold_fn): - """Retrieves the Scaffold from `captured_scaffold_fn`.""" +def _get_and_wrap_scaffold(captured_scaffold_fn, ctx): + """Retrieves the Scaffold from `captured_scaffold_fn`. + + Also wraps the scaffold's finalize method to initialize the TPU after the + graph is finalized. + + Args: + captured_scaffold_fn: a `_CapturedObject` containing a scaffold_fn. + ctx: A `_InternalTPUContext` instance used to initialize the TPU. + + Returns: + The Scaffold produced by captured_scaffold_fn, wrapped to initialize the TPU + after the graph is finalized. + """ with _CapturingContext(message='Inside scaffold_fn'): scaffold_fn = captured_scaffold_fn.get() if scaffold_fn: @@ -2853,14 +2874,64 @@ def _get_scaffold(captured_scaffold_fn): else: scaffold = None - if scaffold: - wrapped_finalize = scaffold.finalize - - def _finalize(): - with _CapturingContext('Inside Scaffold.finalize'): - wrapped_finalize() - - scaffold.finalize = _finalize + if scaffold is None: + # When master_address is None, we are using DirectSession, so we can't + # invoke initialize_system from finalize. See comments below. + if ctx.master_address() is None: + return scaffold + scaffold = monitored_session.Scaffold() + + wrapped_finalize = scaffold.finalize + + def _finalize(): + """Invoke wrapped_finalize and initialize the TPU.""" + with _CapturingContext('Inside Scaffold.finalize'): + wrapped_finalize() + # Run tpu.initialize_system in its own graph after finalizing the main graph + # for distributed sessions. This is necessary because the TPU must be + # initialized before the TPU graph rewrite pass runs. We can't put the + # initialization op in the main graph because the main graph also contains + # replicate ops created by tpu.shard. If we tried to run initialization from + # the main graph, the TPU graph rewrite pass would rewrite the replicate ops + # before actually evaluating the initialization ops. + # + # For distributed sessions, the master may independently restart. After a + # master restarts, the rewrite pass runs again when any op in the main graph + # runs, so we must reinitialize the system every time the main graph is + # finalized. + # + # Special case: When master_address is unset, we're using DirectSession. + # DirectSession resets device state between sessions, and uses + # place_pruned_graph. Initialization currently passes state to replication + # through the TPU_SYSTEM resource manager. Under DirectSession, this + # resource manager gets reset when init_session is closed, so DirectSession + # can't initialize here, and must instead initialize from the main graph's + # init_ops. This is possible with DirectSession because it uses + # place_pruned_graph, which removes unreferenced ops before invoking the + # rewrite pass. This makes it possible to run init_ops from the main graph, + # which contains both tpu.initialize_system and tpu.shard ops, without first + # triggering the TPU graph rewrite. We can't do this for distributed + # sessions because they don't support place_pruned_graph. + # + # TODO(b/110943344) Clean this up as part of the initialize_system dataflow + # cleanup. It should be possible to remove the special case for + # DirectSession and the other call to initialize_system from + # _obtain_topology, when topology info is always explicitly passed from + # tpu.initialize_system to tpu.shard, though this requires editing or + # rebuilding the main graph each time the master restarts. + if ctx.master_address() is None: + return + with ops.Graph().as_default(): + logging.info('Init TPU system master_address %s', ctx.master_address()) + with session_lib.Session( + ctx.master_address(), + config=ctx.config.session_config) as init_session: + run_options = config_pb2.RunOptions(timeout_in_ms=5 * 60 * 1000) + init_session.run( + tpu.initialize_system(job=ctx.master_job), options=run_options) + logging.info('TPU system initialized') + + scaffold.finalize = _finalize return scaffold -- cgit v1.2.3 From b63be23fe975040a8d8b8fd5d701dc4bdf3c26dd Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Fri, 13 Jul 2018 15:30:09 -0700 Subject: Add a few TODOs based on discussion with Andrew. PiperOrigin-RevId: 204538445 --- tensorflow/contrib/autograph/pyct/cfg.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/contrib/autograph/pyct/cfg.py b/tensorflow/contrib/autograph/pyct/cfg.py index 9f060236f4..cef6e95206 100644 --- a/tensorflow/contrib/autograph/pyct/cfg.py +++ b/tensorflow/contrib/autograph/pyct/cfg.py @@ -124,6 +124,8 @@ class _WalkMode(Enum): REVERSE = 2 +# TODO(mdan): Rename to DataFlowAnalyzer. +# TODO(mdan): Consider specializations that use gen/kill/transfer abstractions. class GraphVisitor(object): """Base class for a CFG visitors. @@ -161,6 +163,7 @@ class GraphVisitor(object): """ raise NotImplementedError('Subclasses must implement this.') + # TODO(mdan): Rename to flow? def visit_node(self, node): """Visitor function. -- cgit v1.2.3 From 0cc772c00fdffad5d8416fcde6b9fd3f566cca8e Mon Sep 17 00:00:00 2001 From: Karmel Allison Date: Fri, 13 Jul 2018 15:47:14 -0700 Subject: Add support for metric_ops that are Operations when exporting TRAIN or EVAL SavedModels. PiperOrigin-RevId: 204540825 --- tensorflow/python/estimator/estimator_test.py | 37 ++++++++++++++++++++++ .../python/estimator/export/export_output.py | 11 ++++++- .../python/estimator/export/export_output_test.py | 15 +++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/estimator/estimator_test.py b/tensorflow/python/estimator/estimator_test.py index 2a0e4e7617..495d019f26 100644 --- a/tensorflow/python/estimator/estimator_test.py +++ b/tensorflow/python/estimator/estimator_test.py @@ -2304,6 +2304,43 @@ class EstimatorExportTest(test.TestCase): with self.assertRaisesRegexp(ValueError, err_regex): est._export_all_saved_models(export_dir_base, input_receiver_fn_map) + def test_export_all_saved_models_metric_operation(self): + """Ensures metrics ops.Operations can be expoerted (b/109740581).""" + + def _model_fn(features, labels, mode): + del features, labels # Unused + metrics = {'metrics': (constant_op.constant([0]), + control_flow_ops.no_op())} + return model_fn_lib.EstimatorSpec( + mode, + predictions=constant_op.constant(10.), + loss=constant_op.constant(1.), + train_op=state_ops.assign_add(training.get_global_step(), 1), + eval_metric_ops=metrics) + + tmpdir = tempfile.mkdtemp() + est = estimator.Estimator(model_fn=_model_fn) + est.train(input_fn=dummy_input_fn, steps=1) + + # Perform the export. + export_dir_base = os.path.join( + compat.as_bytes(tmpdir), compat.as_bytes('metric_operation_export')) + + input_receiver_fn_map = { + model_fn_lib.ModeKeys.EVAL: _get_supervised_input_receiver_fn()} + + export_dir = est._export_all_saved_models( + export_dir_base, input_receiver_fn_map) + + # Restore, to validate that the export was well-formed. + with ops.Graph().as_default() as graph: + with session.Session(graph=graph) as sess: + meta_graph = loader.load(sess, [tag_constants.EVAL], export_dir) + sig_outputs = meta_graph.signature_def[ + model_fn_lib.ModeKeys.EVAL].outputs + self.assertEqual( + sig_outputs['metrics/update_op'].name, 'metric_op_wrapper:0') + def test_export_savedmodel_with_saveables_proto_roundtrip(self): tmpdir = tempfile.mkdtemp() est = estimator.Estimator( diff --git a/tensorflow/python/estimator/export/export_output.py b/tensorflow/python/estimator/export/export_output.py index 6c26d29985..20382a58d8 100644 --- a/tensorflow/python/estimator/export/export_output.py +++ b/tensorflow/python/estimator/export/export_output.py @@ -23,6 +23,7 @@ import abc import six +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.saved_model import signature_def_utils @@ -338,8 +339,16 @@ class _SupervisedOutput(ExportOutput): raise ValueError( '{} update_op must be a Tensor or Operation; got {}.'.format( key, metric_op)) + + # We must wrap any ops in a Tensor before export, as the SignatureDef + # proto expects tensors only. See b/109740581 + metric_op_tensor = metric_op + if isinstance(metric_op, ops.Operation): + with ops.control_dependencies([metric_op]): + metric_op_tensor = constant_op.constant([], name='metric_op_wrapper') + outputs[val_name] = metric_val - outputs[op_name] = metric_op + outputs[op_name] = metric_op_tensor return outputs diff --git a/tensorflow/python/estimator/export/export_output_test.py b/tensorflow/python/estimator/export/export_output_test.py index b21ba91b0f..d94c764fd7 100644 --- a/tensorflow/python/estimator/export/export_output_test.py +++ b/tensorflow/python/estimator/export/export_output_test.py @@ -24,8 +24,10 @@ from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.python.estimator.export import export_output as export_output_lib from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops from tensorflow.python.platform import test from tensorflow.python.saved_model import signature_constants @@ -335,5 +337,18 @@ class SupervisedOutputTest(test.TestCase): self.assertTrue("predictions/output1" in sig_def.outputs) self.assertTrue("features" in sig_def.inputs) + def test_metric_op_is_operation(self): + """Tests that ops.Operation is wrapped by a tensor for metric_ops.""" + loss = {"my_loss": constant_op.constant([0])} + predictions = {u"output1": constant_op.constant(["foo"])} + metrics = {"metrics": (constant_op.constant([0]), control_flow_ops.no_op())} + + outputter = MockSupervisedOutput(loss, predictions, metrics) + self.assertEqual(outputter.metrics["metrics/value"], metrics["metrics"][0]) + self.assertEqual( + outputter.metrics["metrics/update_op"].name, "metric_op_wrapper:0") + self.assertTrue( + isinstance(outputter.metrics["metrics/update_op"], ops.Tensor)) + if __name__ == "__main__": test.main() -- cgit v1.2.3 From b3137565ac28ca6cd9f17342cb4213206e7ec5c8 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Fri, 13 Jul 2018 15:55:05 -0700 Subject: [XLA] Update bad-ptxas-version warning to warn for anything less than 9.2.88. We have hit another ptxas bug, which appears to be fixed in 9.2.88. We're no longer testing 8.x, so we can't say those versions work either. Just warn if it's less than 9.2.88. Also update the warning to suggest that people can cherry-pick the new ptxas if they can't upgrade to 9.2.88 properly. PiperOrigin-RevId: 204541857 --- .../compiler/xla/service/gpu/gpu_compiler.cc | 24 +++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index e1da8d940c..6a441548ca 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -357,13 +357,19 @@ void WarnIfBadPtxasVersion(const string& ptxas_path) { // ptxas 9.0 before 9.0.276 and ptxas 9.1 before 9.1.121 miscompile some // address calculations with large offsets (e.g. "load ptr + large_constant"), // b/70245379. - if ((vmaj == 9 && vmin == 0 && vdot < 276) || - (vmaj == 9 && vmin == 1 && vdot < 121)) { - LOG(WARNING) << "*** WARNING *** You are using ptxas " << vmaj << "." - << vmin << "." << vdot - << ", which is in range [9.0.0, 9.0.276) + [9.1.0, 9.1.121). " - "These versions are known to miscompile XLA code, leading " - "to incorrect results or invalid-address errors."; + // + // ptxas 9.1.121 miscompiles some large multioutput fusions, again in a way + // that appears related to address calculations. ptxas 9.2.88 appears to + // work, as far as we can tell. + if ((vmaj < 9 || vmin < 2 || vdot < 88)) { + LOG(WARNING) + << "*** WARNING *** You are using ptxas " << vmaj << "." << vmin << "." + << vdot + << ", which older than 9.2.88. XLA doesn't support ptxas 8.x, and " + "ptxas 9.x before 9.2.88 is known to miscompile XLA code, leading " + "to incorrect results or invalid-address errors.\n\nYou do not need " + "to update to CUDA 9.2.88; cherry-picking the ptxas binary is " + "sufficient."; } } @@ -391,6 +397,10 @@ void WarnIfBadDriverJITVersion() { // - 384.x before 384.108 // - 387.x before 387.40 // - 390.x before 390.10. + // + // TODO(jlebar): This list does not cover the address-calculation bug we've + // observed in ptxas 9.1.121. Need to get a new safe range from nvidia + // corresponding to ptxas >= 9.2.88. auto vmaj = std::get<0>(version); auto vmin = std::get<1>(version); if ((vmaj == 384 && vmin < 108) || // -- cgit v1.2.3 From 2936833c7e22c102ff2b82e3f4e261b94602fbcc Mon Sep 17 00:00:00 2001 From: Reed Wanderman-Milne Date: Fri, 13 Jul 2018 16:14:41 -0700 Subject: Automated rollback of commit d98b99d1cd4337ee11e7cbc4c9b6324f0e381502 PiperOrigin-RevId: 204544587 --- tensorflow/core/common_runtime/direct_session.cc | 14 ++++++++------ tensorflow/core/common_runtime/session.cc | 20 +++++++------------- tensorflow/core/common_runtime/session_factory.h | 7 +------ tensorflow/core/common_runtime/session_test.cc | 6 ++---- .../core/distributed_runtime/rpc/grpc_session.cc | 15 +++++++++------ tensorflow/core/public/session.h | 2 +- 6 files changed, 28 insertions(+), 36 deletions(-) diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc index 4c670820be..1732553abd 100644 --- a/tensorflow/core/common_runtime/direct_session.cc +++ b/tensorflow/core/common_runtime/direct_session.cc @@ -146,15 +146,18 @@ class DirectSessionFactory : public SessionFactory { return options.target.empty(); } - Status NewSession(const SessionOptions& options, - Session** out_session) override { + Session* NewSession(const SessionOptions& options) override { // Must do this before the CPU allocator is created. if (options.config.graph_options().build_cost_model() > 0) { EnableCPUAllocatorFullStats(true); } std::vector devices; - TF_RETURN_IF_ERROR(DeviceFactory::AddDevices( - options, "/job:localhost/replica:0/task:0", &devices)); + const Status s = DeviceFactory::AddDevices( + options, "/job:localhost/replica:0/task:0", &devices); + if (!s.ok()) { + LOG(ERROR) << s; + return nullptr; + } DirectSession* session = new DirectSession(options, new DeviceMgr(devices), this); @@ -162,8 +165,7 @@ class DirectSessionFactory : public SessionFactory { mutex_lock l(sessions_lock_); sessions_.push_back(session); } - *out_session = session; - return Status::OK(); + return session; } Status Reset(const SessionOptions& options, diff --git a/tensorflow/core/common_runtime/session.cc b/tensorflow/core/common_runtime/session.cc index 8c30beeec2..4a9248171b 100644 --- a/tensorflow/core/common_runtime/session.cc +++ b/tensorflow/core/common_runtime/session.cc @@ -53,33 +53,27 @@ Status Session::PRun(const string& handle, Session* NewSession(const SessionOptions& options) { SessionFactory* factory; - Status s = SessionFactory::GetFactory(options, &factory); + const Status s = SessionFactory::GetFactory(options, &factory); if (!s.ok()) { LOG(ERROR) << s; return nullptr; } - Session* out_session; - s = NewSession(options, &out_session); - if (!s.ok()) { - LOG(ERROR) << "Failed to create session: " << s; - return nullptr; - } - return out_session; + return factory->NewSession(options); } Status NewSession(const SessionOptions& options, Session** out_session) { SessionFactory* factory; - Status s = SessionFactory::GetFactory(options, &factory); + const Status s = SessionFactory::GetFactory(options, &factory); if (!s.ok()) { *out_session = nullptr; LOG(ERROR) << s; return s; } - s = factory->NewSession(options, out_session); - if (!s.ok()) { - *out_session = nullptr; + *out_session = factory->NewSession(options); + if (!*out_session) { + return errors::Internal("Failed to create session."); } - return s; + return Status::OK(); } Status Reset(const SessionOptions& options, diff --git a/tensorflow/core/common_runtime/session_factory.h b/tensorflow/core/common_runtime/session_factory.h index 81c172c6ae..df3198a70d 100644 --- a/tensorflow/core/common_runtime/session_factory.h +++ b/tensorflow/core/common_runtime/session_factory.h @@ -30,12 +30,7 @@ struct SessionOptions; class SessionFactory { public: - // Creates a new session and stores it in *out_session, or fails with an error - // status if the Session could not be created. Caller takes ownership of - // *out_session if this returns Status::OK(). - virtual Status NewSession(const SessionOptions& options, - Session** out_session) = 0; - + virtual Session* NewSession(const SessionOptions& options) = 0; virtual bool AcceptsOptions(const SessionOptions& options) = 0; // Abort and close all existing sessions, disconnecting their resources from diff --git a/tensorflow/core/common_runtime/session_test.cc b/tensorflow/core/common_runtime/session_test.cc index 1fa5aad60c..feaf29c7bb 100644 --- a/tensorflow/core/common_runtime/session_test.cc +++ b/tensorflow/core/common_runtime/session_test.cc @@ -47,10 +47,8 @@ class FakeSessionFactory : public SessionFactory { return str_util::StartsWith(options.target, "fake"); } - Status NewSession(const SessionOptions& options, - Session** out_session) override { - *out_session = nullptr; - return Status::OK(); + Session* NewSession(const SessionOptions& options) override { + return nullptr; } }; class FakeSessionRegistrar { diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc index fdce1b10e0..fd1c150fa7 100644 --- a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc +++ b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc @@ -452,12 +452,15 @@ class GrpcSessionFactory : public SessionFactory { return str_util::StartsWith(options.target, kSchemePrefix); } - Status NewSession(const SessionOptions& options, - Session** out_session) override { - std::unique_ptr session; - TF_RETURN_IF_ERROR(GrpcSession::Create(options, &session)); - *out_session = session.release(); - return Status::OK(); + Session* NewSession(const SessionOptions& options) override { + std::unique_ptr ret; + Status s = GrpcSession::Create(options, &ret); + if (s.ok()) { + return ret.release(); + } else { + LOG(ERROR) << "Error during session construction: " << s.ToString(); + return nullptr; + } } // Invokes the session specific static method to reset containers. diff --git a/tensorflow/core/public/session.h b/tensorflow/core/public/session.h index cc8596ef3d..d58c877cfd 100644 --- a/tensorflow/core/public/session.h +++ b/tensorflow/core/public/session.h @@ -237,7 +237,7 @@ class Session { /// If session creation succeeds, the new `Session` will be stored in /// `*out_session`, the caller will take ownership of the returned /// `*out_session`, and this function will return `OK()`. Otherwise, this -/// function will return an error status and set *out_session to nullptr. +/// function will return an error status. Status NewSession(const SessionOptions& options, Session** out_session); /// \brief Resets resource containers associated with a target. -- cgit v1.2.3 From 88a87310753f3d73c926fd9162cc871ec0a4440e Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Fri, 13 Jul 2018 16:30:17 -0700 Subject: [XLA] Use shfl.sync.down instead of shfl.sync. shfl.down is deprecated and ptxas 9.2 emits a loud warning when you use it. Convert XLA to shfl.sync.down. This change makes XLA:GPU require CUDA 9. PiperOrigin-RevId: 204546742 --- tensorflow/compiler/xla/service/gpu/gpu_compiler.cc | 20 ++++++++++++++------ .../compiler/xla/service/gpu/ir_emission_utils.cc | 19 +++++++++++-------- .../compiler/xla/service/gpu/ir_emission_utils.h | 16 ++++++++++------ .../compiler/xla/service/gpu/ir_emitter_unnested.cc | 18 ++++++++++++------ .../service/gpu/llvm_gpu_backend/gpu_backend_lib.cc | 2 +- 5 files changed, 48 insertions(+), 27 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index 6a441548ca..5e5d893582 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -354,6 +354,9 @@ void WarnIfBadPtxasVersion(const string& ptxas_path) { return; } + // We need ptxas >= 9.0 as a hard requirement, because we compile targeting + // PTX 6.0. An older ptxas will just fail to compile any of our code. + // // ptxas 9.0 before 9.0.276 and ptxas 9.1 before 9.1.121 miscompile some // address calculations with large offsets (e.g. "load ptr + large_constant"), // b/70245379. @@ -361,15 +364,20 @@ void WarnIfBadPtxasVersion(const string& ptxas_path) { // ptxas 9.1.121 miscompiles some large multioutput fusions, again in a way // that appears related to address calculations. ptxas 9.2.88 appears to // work, as far as we can tell. - if ((vmaj < 9 || vmin < 2 || vdot < 88)) { + if (vmaj < 9) { + LOG(ERROR) + << "You are using ptxas 8.x, but XLA requires ptxas 9.x (and strongly " + "prefers >= 9.2.88). Compilation of XLA kernels below will likely " + "fail.\n\nYou do not need to update CUDA; cherry-picking the ptxas " + "binary is sufficient."; + } else if ((vmaj < 9 || vmin < 2 || vdot < 88)) { LOG(WARNING) << "*** WARNING *** You are using ptxas " << vmaj << "." << vmin << "." << vdot - << ", which older than 9.2.88. XLA doesn't support ptxas 8.x, and " - "ptxas 9.x before 9.2.88 is known to miscompile XLA code, leading " - "to incorrect results or invalid-address errors.\n\nYou do not need " - "to update to CUDA 9.2.88; cherry-picking the ptxas binary is " - "sufficient."; + << ", which older than 9.2.88. ptxas 9.x before 9.2.88 is known to " + "miscompile XLA code, leading to incorrect results or " + "invalid-address errors.\n\nYou do not need to update to CUDA " + "9.2.88; cherry-picking the ptxas binary is sufficient."; } } diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc index 388aa35d7d..2799baab41 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc @@ -242,15 +242,17 @@ llvm::Value* EmitPrintf(tensorflow::StringPiece fmt, arguments_ptr}); } -llvm::Value* EmitShuffleDown(llvm::Value* value, llvm::Value* offset, - llvm::IRBuilder<>* builder) { +llvm::Value* EmitFullWarpShuffleDown(llvm::Value* value, llvm::Value* offset, + llvm::IRBuilder<>* builder) { int bit_width = value->getType()->getPrimitiveSizeInBits(); + llvm::Value* all_warps_mask = builder->getInt32(-1); // Special case for efficiency if (value->getType()->isFloatTy() && bit_width == 32) { return llvm_ir::EmitCallToIntrinsic( - llvm::Intrinsic::nvvm_shfl_down_f32, - {value, offset, builder->getInt32(kWarpSize - 1)}, {}, builder); + llvm::Intrinsic::nvvm_shfl_sync_down_f32, + {all_warps_mask, value, offset, builder->getInt32(kWarpSize - 1)}, {}, + builder); } // We must split values wider than 32 bits as the "shfl" instruction operates @@ -264,10 +266,11 @@ llvm::Value* EmitShuffleDown(llvm::Value* value, llvm::Value* offset, for (int i = 0; i < num_segments; ++i) { x = builder->CreateInsertElement( x, - llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_shfl_down_i32, - {builder->CreateExtractElement(x, i), - offset, builder->getInt32(kWarpSize - 1)}, - {}, builder), + llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::nvvm_shfl_sync_down_i32, + {all_warps_mask, builder->CreateExtractElement(x, i), offset, + builder->getInt32(kWarpSize - 1)}, + {}, builder), i); } return builder->CreateBitCast( diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h index 59455f389e..9bb4c42b15 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h @@ -125,13 +125,17 @@ llvm::Value* EmitPrintf(tensorflow::StringPiece fmt, llvm::IRBuilder<>* builder); // Emits code to shuffle data between threads of a warp. This has the same -// semantics as the PTX "shfl.down" instruction [0] but works for values of any -// size. The last operand of the emitted "shfl" is `kWarpSize - 1`. +// semantics as the PTX "shfl.sync.down" instruction but works for values that +// aren't 32 bits in size. The last operand of the emitted "shfl" is +// `kWarpSize - 1`. // -// [0] -// http://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-shfl -llvm::Value* EmitShuffleDown(llvm::Value* value, llvm::Value* offset, - llvm::IRBuilder<>* builder); +// This function emits a "full-warp" shuffle, which all threads of a warp +// participate in. *Do not use this function from a divergent context:* You +// can't correctly do so on both Volta and earlier GPUs. +// +// https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-shfl-sync +llvm::Value* EmitFullWarpShuffleDown(llvm::Value* value, llvm::Value* offset, + llvm::IRBuilder<>* builder); } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index 673ba530df..75bbbbe8ef 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -918,10 +918,13 @@ Status IrEmitterUnnested::EmitReductionToScalar( ir_builder_.CreateBitCast(partial_reduction_result_addresses[i], shuffle_ir_type->getPointerTo()), "partial_reduction_result"); + CHECK_EQ(launch_dimensions.threads_per_block() % kWarpSize, 0) + << "Requires block size a multiple of the warp size, otherwise we " + "will read undefined elements."; ir_builder_.CreateStore( - EmitShuffleDown(partial_reduction_result, - ir_builder_.getInt32(shuffle_distance), - &ir_builder_), + EmitFullWarpShuffleDown(partial_reduction_result, + ir_builder_.getInt32(shuffle_distance), + &ir_builder_), ir_builder_.CreateBitCast(result_from_other_lane, shuffle_ir_type->getPointerTo())); TF_RETURN_IF_ERROR(EmitCallToNestedComputation( @@ -1498,10 +1501,13 @@ Status IrEmitterUnnested::EmitRowReduction( ir_builder_.CreateBitCast(partial_reduction_result_addresses[i], shuffle_ir_type->getPointerTo()), "partial_reduction_result"); + CHECK_EQ(launch_dimensions.threads_per_block() % kWarpSize, 0) + << "Requires block size a multiple of the warp size, otherwise we " + "will read undefined elements."; ir_builder_.CreateStore( - EmitShuffleDown(partial_reduction_result, - ir_builder_.getInt32(shuffle_distance), - &ir_builder_), + EmitFullWarpShuffleDown(partial_reduction_result, + ir_builder_.getInt32(shuffle_distance), + &ir_builder_), ir_builder_.CreateBitCast(result_from_other_lane, shuffle_ir_type->getPointerTo())); TF_RETURN_IF_ERROR(EmitCallToNestedComputation( diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc index a4e4e85bf3..2b0d6924a2 100644 --- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc +++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc @@ -206,7 +206,7 @@ std::unique_ptr GetTargetMachine( codegen_opt_level = CodeGenOpt::None; } return WrapUnique(target->createTargetMachine( - triple.str(), llvm_ir::AsStringRef(cpu_name), "+ptx42", target_options, + triple.str(), llvm_ir::AsStringRef(cpu_name), "+ptx60", target_options, Optional(RelocModel), Optional(CMModel), codegen_opt_level)); } -- cgit v1.2.3 From 9b792c16ebdf6837e70b570a9a30e4dcb394f237 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Fri, 13 Jul 2018 17:23:28 -0700 Subject: [Cloud TPU / Keras]: Support experimental tf.data integration. PiperOrigin-RevId: 204552928 --- tensorflow/contrib/tpu/BUILD | 1 + tensorflow/contrib/tpu/python/tpu/keras_support.py | 508 ++++++++++++++++++--- 2 files changed, 439 insertions(+), 70 deletions(-) diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index 0044fde9d0..ba93dbbd65 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -184,6 +184,7 @@ py_library( "//tensorflow/python:session", "//tensorflow/python:tensor_spec", "//tensorflow/python:variable_scope", + "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/estimator:model_fn", "//tensorflow/python/keras:backend", "//tensorflow/python/keras:engine", diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py index 722e31abb2..8292c920fc 100644 --- a/tensorflow/contrib/tpu/python/tpu/keras_support.py +++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py @@ -45,6 +45,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import abc import collections import contextlib import re @@ -63,9 +64,11 @@ from tensorflow.contrib.tpu.python.tpu import tpu_function from tensorflow.contrib.tpu.python.tpu import tpu_optimizer from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session as tf_session +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_spec from tensorflow.python.keras import backend as K from tensorflow.python.keras import models @@ -202,7 +205,6 @@ class TPURewriteContext(object): caller_obj = caller_frame.f_locals.get('self') if (caller_obj is not None and isinstance(caller_obj, base_layer.Layer) and name is not None): - logging.info('Intercepted name_scope: %s', caller_obj) return variable_scope.variable_scope( name, default_name, values, reuse=variable_scope.AUTO_REUSE) @@ -269,6 +271,329 @@ class TPURewriteContext(object): gen_linalg_ops.qr = self._default_qr +class SizedInfeed(collections.namedtuple('SizedInfeed', + ['sharded_infeed_tensors', + 'infeed_ops'])): + """Represents an instantiation of the infeed ops for a concrete input shape. + + sharded_infeed_tensors: A data structure of Tensors used to represent the + placeholder tensors that must be fed when using feed_dicts. + + infeed_ops: the set of ops that will be run to drive infeed for a single step. + """ + pass + + +class TPUInfeedInstance(object): + """TPUInfeedInstance represents the logic to manage feeding in a single step. + + See the comments on the `TPUInfeedManager` for a description for how infeed + is managed. + """ + + @abc.abstractmethod + def make_input_specs(self, input_tensors): + """Constructs the infeed_specs for the given Infeed instance. + + Args: + input_tensors: The inputs to the model. + + Returns: + A list of + """ + pass + + def make_feed_dict(self, tpu_model_op): + """Constructs a feed_dict for this instance, given the tpu_model_op. + + Args: + tpu_model_op: A `TPUModelOp` representing the TPU Model for this + instance's input spec. + + Returns: + A dictionary to use as the feed_dict of a `session.run` call. + """ + pass + + +class TPUInfeedManager(object): + """TPUInfeedManager manages the data infeeding of data to a TPU computation. + + Because there are multiple data sources (e.g. in-memory NumPy arrays, + `tf.data.Dataset`s), we abstract the different logic behind a single + interface: the `TPUInfeedManager`. + + (1) A `TPUFunction` is called with a set of inputs. Based on the inputs, + `TPUFunction` retrieves the corresponding `TPUInfeedManager` (or constructs a + new one if required). + + (2) The `TPUFunction` calls `make_infeed_instance` on the `TPUInfeedManager` + which returns a `TPUInfeedInstance`. + + (3) The `TPUFunction` checks in the shape cache for a pre-compiled instance of + the model based on the returned `input_specs` from `TPUInfeedInstance`. + + (4) [Optional.] If the model has not already been instantiated for the given + input spec, the `TPUFunction` compiles the model for the input spec (using the + `TPUInfeedManager`). + + (5) The `TPUInfeedInstance` constructs the session.run's feed_dict given the + compiled model instance corresponding to its shape. + """ + + @abc.abstractmethod + def make_infeed_instance(self, inputs): + """Given a single step's input, construct a `TPUInfeedInstance`. + + Args: + inputs: The inputs to a given step. + + Returns: + A subclass of `TPUInfeedInstance`. + """ + pass + + @abc.abstractmethod + def build_infeed_from_input_specs(self, input_specs, execution_mode): + """For a given input specification (size, type), construct the infeed ops. + + This is called only once for a given input specification and builds the + graph ops. It does not have a pointer to the actual infeed data. + + Args: + input_specs: TODO(saeta): Document me! + execution_mode: TODO(saeta): Document me! + + Returns: + A `SizedInfeed` instance. + """ + pass + + +class TPUNumpyInfeedManager(TPUInfeedManager): + """TPU Infeed manager for Numpy inputs.""" + + class NumpyInfeedInstance(TPUInfeedInstance): + """Infeed instance for Numpy inputs.""" + + def __init__(self, sharded_inputs): + self._sharded_inputs = sharded_inputs + + def make_input_specs(self, input_tensors): + # Compute an input specification (used to generate infeed enqueue and + # dequeue operations). We use the shape from our input array and the + # dtype from our model. A user may pass in a float64 for a float32 + # input: for model compatibility we still must generate a float32 infeed. + input_specs = [] + # We use the shape and dtype from the first shard to compute the input + # metadata (`input_specs`); all replicas have the same type and shape. + for tensor, ary in zip(input_tensors, self._sharded_inputs[0]): + input_specs.append( + tensor_spec.TensorSpec(ary.shape, tensor.dtype, + _valid_name(tensor.name))) + + return input_specs + + def make_feed_dict(self, tpu_model_op): + infeed_dict = {} + for infeed_tensors, inputs in zip(tpu_model_op.infeed_tensors, + self._sharded_inputs): + for tensor, value in zip(infeed_tensors, inputs): + infeed_dict[tensor] = value + return infeed_dict + + def __init__(self, distribution_strategy): + self._strategy = distribution_strategy + + def _split_tensors(self, inputs): + """Split input data across shards. + + Each input is sliced along the batch axis. + + Args: + inputs: List of Numpy arrays to run on the TPU. + + Returns: + List of lists containing the input to feed to each TPU shard. + """ + if self._strategy.num_towers == 1: + return [inputs] + + batch_size = inputs[0].shape[0] + assert batch_size % self._strategy.num_towers == 0, ( + 'batch_size must be divisible by strategy.num_towers (%s vs %s)' % + (batch_size, self._strategy.num_towers)) + shard_size = batch_size // self._strategy.num_towers + input_list = [] + for index in range(self._strategy.num_towers): + shard_inputs = [ + x[index * shard_size:(index + 1) * shard_size] for x in inputs + ] + input_list.append(shard_inputs) + return input_list + + def make_infeed_instance(self, inputs): + sharded_inputs = self._split_tensors(inputs) + return self.NumpyInfeedInstance(sharded_inputs) + + def build_infeed_from_input_specs(self, input_specs, execution_mode): + infeed_op = [] + shard_infeed_tensors = [] + + for shard_id in range(self._strategy.num_towers): + with ops.device('/device:TPU:%d' % shard_id): + infeed_tensors = [] + for spec in input_specs: + # Construct placeholders for each of the inputs. + infeed_tensors.append( + array_ops.placeholder( + dtype=spec.dtype, + shape=spec.shape, + name='infeed-enqueue-%s-%d' % (spec.name, shard_id))) + shard_infeed_tensors.append(infeed_tensors) + + infeed_op.append( + tpu_ops.infeed_enqueue_tuple( + infeed_tensors, [spec.shape for spec in input_specs], + name='infeed-enqueue-%s-%d' % (execution_mode, shard_id))) + return SizedInfeed(infeed_ops=infeed_op, + sharded_infeed_tensors=shard_infeed_tensors) + + +class TPUDatasetInfeedManager(TPUInfeedManager): + """Manages infeed for a `tf.data.Dataset` into a TPU computation. + """ + + class DatasetInfeedInstance(TPUInfeedInstance): + """An instance of the TPU infeed.""" + + def __init__(self, input_specs): + self._input_specs = input_specs + + def make_input_specs(self, input_tensors): + # TODO(saeta): Do error checking here! + return self._input_specs + + def make_feed_dict(self, tpu_model_op): + # TODO(saeta): Verify tpu_model_op is as expected! + return {} + + def __init__(self, dataset, distribution_strategy, tpu_session): + """Constructs a TPUDatasetInfeedManager. + + Must be called within a `KerasTPUModel.tpu_session` context! + + Args: + dataset: A `tf.data.Dataset` to infeed. + distribution_strategy: The `TPUDistributionStrategy` used to configure the + Keras TPU model. + tpu_session: The `tf.Session` object used for running the TPU model. + """ + self._verify_dataset_shape(dataset) + self._dataset = dataset + self._strategy = distribution_strategy + dummy_x_shape = dataset.output_shapes[0].as_list() + dummy_x_shape[0] *= distribution_strategy.num_towers + dummy_y_shape = dataset.output_shapes[1].as_list() + dummy_y_shape[0] *= distribution_strategy.num_towers + self._iterator = dataset.make_initializable_iterator() + tpu_session.run(self._iterator.initializer) + + self._get_next_ops = [] + ctrl_deps = [] + for i in range(distribution_strategy.num_towers): + with ops.control_dependencies(ctrl_deps): # Ensure deterministic + # TODO(saeta): Ensure correct placement! + get_next_op = self._iterator.get_next() + self._get_next_ops.append(get_next_op) + ctrl_deps.extend(get_next_op) + + # Use dummy numpy inputs for the rest of Keras' shape checking. We + # intercept them when building the model. + self._dummy_x = np.zeros(dummy_x_shape, + dtype=dataset.output_types[0].as_numpy_dtype) + self._dummy_y = np.zeros(dummy_y_shape, + dtype=dataset.output_types[1].as_numpy_dtype) + + input_specs = [] + if isinstance(self._iterator.output_shapes, tuple): + assert isinstance(self._iterator.output_types, tuple) + assert len(self._iterator.output_shapes) == len( + self._iterator.output_types) + for i in range(len(self._iterator.output_shapes)): + spec = tensor_spec.TensorSpec(self._iterator.output_shapes[i], + self._iterator.output_types[i]) + input_specs.append(spec) + elif isinstance(self._iterator.output_shapes, tensor_shape.TensorShape): + spec = tensor_spec.TensorSpec(self._iterator.output_shapes, + self._iterator.output_types) + input_specs.append(spec) + + self._infeed_instance = self.DatasetInfeedInstance(input_specs) + + def _verify_dataset_shape(self, dataset): + """Verifies a dataset is of an appropriate shape for TPUs.""" + if not isinstance(dataset, dataset_ops.Dataset): + raise ValueError('The function passed as the `x` parameter did not ' + 'return a `tf.data.Dataset`.') + if not isinstance(dataset.output_classes, tuple): + raise ValueError('The dataset must return a tuple of tf.Tensors, ' + 'instead it returns: %s' % dataset.output_classes) + if len(dataset.output_classes) != 2: + raise ValueError( + 'The dataset must return a 2-element tuple, got ' + '%s output classes instead.' % (dataset.output_classes,)) + for i, cls in enumerate(dataset.output_classes): + if cls != ops.Tensor: + raise ValueError('The dataset returned a non-Tensor type (%s) at ' + 'index %d.' % (cls, i)) + for i, shape in enumerate(dataset.output_shapes): + if not shape: + raise ValueError('The dataset returns a scalar tensor in ' + 'tuple index %d. Did you forget to batch? ' + '(Output shapes: %s).' % (i, + dataset.output_shapes)) + for j, dim in enumerate(shape): + if dim.value is None: + if j == 0: + hint = (' Hint: did you use `ds.batch(BATCH_SIZE, ' + 'drop_remainder=True)`?') + else: + hint = '' + raise ValueError( + 'The Keras-TPU integration for `tf.data` ' + 'currently requires static shapes. The provided ' + 'dataset only has a partially defined shape. ' + '(Dimension %d of output tensor %d is not statically known ' + 'for output shapes: %s.%s)' % (i, j, dataset.output_shapes, hint)) + + @property + def dummy_x(self): + return self._dummy_x + + @property + def dummy_y(self): + return self._dummy_y + + def make_infeed_instance(self, inputs): + # TODO(saeta): Verify inputs is as expected. + return self._infeed_instance + + def build_infeed_from_input_specs(self, input_specs, execution_mode): + shard_infeed_tensors = self._get_next_ops + assert len(shard_infeed_tensors) == self._strategy.num_towers + infeed_ops = [] + for shard_id in range(self._strategy.num_towers): + with ops.device('/device:TPU:%d' % shard_id): + infeed_ops.append( + tpu_ops.infeed_enqueue_tuple( + shard_infeed_tensors[shard_id], + [spec.shape for spec in input_specs], + name='infeed-enqueue-%s-%d' % (execution_mode, shard_id))) + return SizedInfeed(infeed_ops=infeed_ops, + sharded_infeed_tensors=shard_infeed_tensors) + + class TPUFunction(object): """K.function compatible interface for invoking a TPU compiled function. @@ -294,7 +619,7 @@ class TPUFunction(object): if not isinstance(self.model.optimizer, keras_optimizers.TFOptimizer): self._optimizer_config = self.model.optimizer.get_config() - def _specialize_model(self, input_specs): + def _specialize_model(self, input_specs, infeed_manager): """Specialize `self.model` (a Keras model) for the given input shapes.""" # Re-create our input and output layers inside our subgraph. They will be # attached to the true computation when we clone our model in `tpu_fn`. @@ -320,8 +645,8 @@ class TPUFunction(object): name='infeed-%s' % self.execution_mode) assert len(infeed_tensors) == len(infeed_layers), ( - 'Infeed inputs did not match model: %s vs %s', (infeed_layers, - infeed_tensors)) + 'Infeed inputs did not match model: %s vs %s' % (infeed_layers, + infeed_tensors)) tpu_targets = [] tpu_input_map = {} @@ -410,26 +735,12 @@ class TPUFunction(object): # Generate CPU side operations to enqueue features/labels and dequeue # outputs from the model call. - infeed_op = [] + sized_infeed = infeed_manager.build_infeed_from_input_specs( + input_specs, self.execution_mode) + # Build output ops. outfeed_op = [] - shard_infeed_tensors = [] - for shard_id in range(self._strategy.num_towers): with ops.device('/device:TPU:%d' % shard_id): - infeed_tensors = [] - for spec in input_specs: - infeed_tensors.append( - array_ops.placeholder( - dtype=spec.dtype, - shape=spec.shape, - name='infeed-enqueue-%s-%d' % (spec.name, shard_id))) - shard_infeed_tensors.append(infeed_tensors) - - infeed_op.append( - tpu_ops.infeed_enqueue_tuple( - infeed_tensors, [spec.shape for spec in input_specs], - name='infeed-enqueue-%s-%d' % (self.execution_mode, shard_id))) - outfeed_op.extend( tpu_ops.outfeed_dequeue_tuple( dtypes=[spec.dtype for spec in self._outfeed_spec], @@ -439,8 +750,8 @@ class TPUFunction(object): return TPUModelOp( compile_op, execute_op, - infeed_tensors=shard_infeed_tensors, - infeed_op=infeed_op, + infeed_tensors=sized_infeed.sharded_infeed_tensors, + infeed_op=sized_infeed.infeed_ops, outfeed_op=outfeed_op) def _test_model_compiles(self, tpu_model_ops): @@ -459,36 +770,17 @@ class TPUFunction(object): logging.info('Finished compiling. Time elapsed: %s secs', end_time - start_time) - def _split_tensors(self, inputs): - """Split input data across shards. - - Each input is sliced along the batch axis. - - Args: - inputs: List of Numpy arrays to run on the TPU. - - Returns: - List of lists containing the input to feed to each TPU shard. - """ - if self._strategy.num_towers == 1: - return [inputs] - - batch_size = inputs[0].shape[0] - assert batch_size % self._strategy.num_towers == 0, ( - 'batch_size must be divisible by strategy.num_towers (%s vs %s)' % - (batch_size, self._strategy.num_towers)) - shard_size = batch_size // self._strategy.num_towers - input_list = [] - for index in range(self._strategy.num_towers): - shard_inputs = [ - x[index * shard_size:(index + 1) * shard_size] for x in inputs - ] - input_list.append(shard_inputs) - return input_list - def __call__(self, inputs): assert isinstance(inputs, list) + infeed_manager = None + for x, mgr in self.model._numpy_to_infeed_manager_list: + if inputs[0] is x: + infeed_manager = mgr + break + if infeed_manager is None: + infeed_manager = TPUNumpyInfeedManager(self.model._strategy) + # Strip sample weight from inputs if (self.execution_mode == model_fn_lib.ModeKeys.TRAIN or self.execution_mode == model_fn_lib.ModeKeys.EVAL): @@ -497,21 +789,9 @@ class TPUFunction(object): else: input_tensors = self.model._feed_inputs - shard_inputs = self._split_tensors(inputs) + infeed_instance = infeed_manager.make_infeed_instance(inputs) del inputs # To avoid accident usage. - - # Compute an input specification (used to generate infeed enqueue and - # dequeue operations). We use the shape from our input array and the - # dtype from our model. A user may pass in a float64 for a float32 - # input: for model compatibility we still must generate a float32 infeed. - input_specs = [] - - # We use the shape and dtype from the first shard to compute the input - # metadata (`input_specs`); all replicas have the same type and shape. - for tensor, ary in zip(input_tensors, shard_inputs[0]): - input_specs.append( - tensor_spec.TensorSpec(ary.shape, tensor.dtype, - _valid_name(tensor.name))) + input_specs = infeed_instance.make_input_specs(input_tensors) # XLA requires every operation in the graph has a fixed shape. To # handle varying batch sizes we recompile a new sub-graph for each @@ -522,7 +802,8 @@ class TPUFunction(object): with self.model.tpu_session(): logging.info('New input shapes; (re-)compiling: mode=%s, %s', self.execution_mode, input_specs) - new_tpu_model_ops = self._specialize_model(input_specs) + new_tpu_model_ops = self._specialize_model(input_specs, + infeed_manager) self._compilation_cache[shape_key] = new_tpu_model_ops self._test_model_compiles(new_tpu_model_ops) @@ -530,11 +811,7 @@ class TPUFunction(object): self.model._initialize_weights(self._cloned_model) tpu_model_ops = self._compilation_cache[shape_key] - infeed_dict = {} - for infeed_tensors, inputs in zip(tpu_model_ops.infeed_tensors, - shard_inputs): - for tensor, value in zip(infeed_tensors, inputs): - infeed_dict[tensor] = value + infeed_dict = infeed_instance.make_feed_dict(tpu_model_ops) with self.model.tpu_session() as session: _, _, outfeed_outputs = session.run([ @@ -568,6 +845,11 @@ class KerasTPUModel(models.Model): name=cpu_model.name, ) + # Create a mapping from numpy arrays to infeed managers. + # Note: uses a list of tuples instead of a map because numpy arrays are + # not hashable. + self._numpy_to_infeed_manager_list = [] + self.predict_function = None self.test_function = None self.train_function = None @@ -640,6 +922,92 @@ class KerasTPUModel(models.Model): sample_weight_mode, weighted_metrics, target_tensors, **kwargs) + def fit(self, + x=None, + y=None, + batch_size=None, + epochs=1, + verbose=1, + callbacks=None, + validation_split=0., + validation_data=None, + shuffle=True, + class_weight=None, + sample_weight=None, + initial_epoch=0, + steps_per_epoch=None, + validation_steps=None, + **kwargs): + assert not self._numpy_to_infeed_manager_list # Ensure empty. + + infeed_managers = [] # Managers to clean up at the end of the fit call. + if isinstance(x, dataset_ops.Dataset): + # TODO(b/111413240): Support taking a tf.data.Dataset directly. + raise ValueError( + 'Taking a Dataset directly is not yet supported. Please ' + 'wrap your dataset construction code in a function and ' + 'pass that to fit instead. For examples, see: ' + 'https://github.com/tensorflow/tpu/tree/master/models/experimental' + '/keras') + if callable(x): + with self.tpu_session() as sess: + dataset = x() + if steps_per_epoch is None: + raise ValueError('When using tf.data as input to a model, you ' + 'should specify the steps_per_epoch argument.') + if y is not None: + raise ValueError('When using tf.data as input to a model, y must be ' + 'None') + infeed_manager = TPUDatasetInfeedManager(dataset, self._strategy, sess) + # Use dummy numpy inputs for the rest of Keras' shape checking. We + # intercept them when building the model. + x = infeed_manager.dummy_x + y = infeed_manager.dummy_y + infeed_managers.append((x, infeed_manager)) + + if isinstance(validation_data, dataset_ops.Dataset): + # TODO(b/111413240): Support taking a tf.data.Dataset directly. + raise ValueError( + 'Taking a Dataset directly is not yet supported. Please ' + 'wrap your dataset construction code in a function and ' + 'pass that to fit instead. For examples, see: ' + 'https://github.com/tensorflow/tpu/tree/master/models/experimental' + '/keras') + if callable(validation_data): + with self.tpu_session() as sess: + dataset = validation_data() + if validation_steps is None: + raise ValueError('When using tf.data as validation for a model, you ' + 'should specify the validation_steps argument.') + infeed_manager = TPUDatasetInfeedManager(dataset, self._strategy, sess) + # Use dummy numpy inputs for the rest of Keras' shape checking. We + # intercept them when building the model. + val_x = infeed_manager.dummy_x + val_y = infeed_manager.dummy_y + infeed_managers.append((val_x, infeed_manager)) + validation_data = (val_x, val_y) + + self._numpy_to_infeed_manager_list = infeed_managers + try: + return super(KerasTPUModel, self).fit( + x, + y, + batch_size, + epochs, + verbose, + callbacks, + validation_split, + validation_data, + shuffle, + class_weight, + sample_weight, + initial_epoch, + steps_per_epoch, + validation_steps, + **kwargs) + finally: + self._numpy_to_infeed_manager_list = [] + def _make_train_function(self): if not self.train_function: self.train_function = TPUFunction( -- cgit v1.2.3 From 687a3d8b7784fe1e13867fd62e6bf560154ad90d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 17:45:54 -0700 Subject: Qualify all names fully in registration macro. Otherwise the macro cannot be used from other namespaces without additional contortions. PiperOrigin-RevId: 204554899 --- tensorflow/compiler/tf2xla/xla_op_registry.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h index 2d4593ea49..fc14834ca6 100644 --- a/tensorflow/compiler/tf2xla/xla_op_registry.h +++ b/tensorflow/compiler/tf2xla/xla_op_registry.h @@ -279,7 +279,7 @@ class XlaOpRegistrar { #define REGISTER_XLA_OP_UNIQ(CTR, BUILDER, OP) \ static ::tensorflow::XlaOpRegistrar xla_op_registrar__body__##CTR##__object( \ - XlaOpRegistrationBuilder::BUILDER.Build( \ + ::tensorflow::XlaOpRegistrationBuilder::BUILDER.Build( \ [](::tensorflow::OpKernelConstruction* context) \ -> ::tensorflow::OpKernel* { return new OP(context); })); -- cgit v1.2.3 From 1824f82565f9a289e992b9e85ca2c54f84807c8d Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Fri, 13 Jul 2018 17:49:18 -0700 Subject: Add linear estimator colab entry to tutorials. FIx other colab links. PiperOrigin-RevId: 204555150 --- tensorflow/docs_src/tutorials/_index.yaml | 2 +- tensorflow/docs_src/tutorials/_toc.yaml | 2 ++ tensorflow/docs_src/tutorials/eager/custom_training_walkthrough.md | 2 +- tensorflow/docs_src/tutorials/estimators/linear.md | 3 +++ tensorflow/docs_src/tutorials/keras/basic_classification.md | 2 +- tensorflow/docs_src/tutorials/keras/basic_regression.md | 2 +- tensorflow/docs_src/tutorials/keras/basic_text_classification.md | 2 +- tensorflow/docs_src/tutorials/keras/overfit_and_underfit.md | 2 +- tensorflow/docs_src/tutorials/keras/save_and_restore_models.md | 2 +- 9 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 tensorflow/docs_src/tutorials/estimators/linear.md diff --git a/tensorflow/docs_src/tutorials/_index.yaml b/tensorflow/docs_src/tutorials/_index.yaml index 07d561b8a2..c74fe58089 100644 --- a/tensorflow/docs_src/tutorials/_index.yaml +++ b/tensorflow/docs_src/tutorials/_index.yaml @@ -175,7 +175,7 @@ landing_page: Estimators guide.

    -
  1. Premade Estimators guide
    2. +
  2. Build a linear model with Estimators
  3. Wide and deep learning with Estimators
  4. Boosted trees
  5. How to build a simple text classifier with TF-Hub
    6. diff --git a/tensorflow/docs_src/tutorials/_toc.yaml b/tensorflow/docs_src/tutorials/_toc.yaml index 4db97e35fc..d33869af6e 100644 --- a/tensorflow/docs_src/tutorials/_toc.yaml +++ b/tensorflow/docs_src/tutorials/_toc.yaml @@ -44,6 +44,8 @@ toc: - title: ML at production scale style: accordion section: + - title: Linear model with Estimators + path: /tutorials/estimators/linear - title: Wide and deep learning path: https://github.com/tensorflow/models/tree/master/official/wide_deep status: external diff --git a/tensorflow/docs_src/tutorials/eager/custom_training_walkthrough.md b/tensorflow/docs_src/tutorials/eager/custom_training_walkthrough.md index b45fbefac0..b564a27ecf 100644 --- a/tensorflow/docs_src/tutorials/eager/custom_training_walkthrough.md +++ b/tensorflow/docs_src/tutorials/eager/custom_training_walkthrough.md @@ -1,3 +1,3 @@ # Custom training: walkthrough -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/eager.ipynb) +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/tutorials/eager/custom_training_walkthrough.ipynb) diff --git a/tensorflow/docs_src/tutorials/estimators/linear.md b/tensorflow/docs_src/tutorials/estimators/linear.md new file mode 100644 index 0000000000..067a33ac03 --- /dev/null +++ b/tensorflow/docs_src/tutorials/estimators/linear.md @@ -0,0 +1,3 @@ +# Build a linear model with Estimators + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/tutorials/estimators/linear.ipynb) diff --git a/tensorflow/docs_src/tutorials/keras/basic_classification.md b/tensorflow/docs_src/tutorials/keras/basic_classification.md index 91bbd85b24..e028af99b9 100644 --- a/tensorflow/docs_src/tutorials/keras/basic_classification.md +++ b/tensorflow/docs_src/tutorials/keras/basic_classification.md @@ -1,3 +1,3 @@ # Basic Classification -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_classification.ipynb) +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/tutorials/keras/basic_classification.ipynb) diff --git a/tensorflow/docs_src/tutorials/keras/basic_regression.md b/tensorflow/docs_src/tutorials/keras/basic_regression.md index a535f22f5a..8721b7aca1 100644 --- a/tensorflow/docs_src/tutorials/keras/basic_regression.md +++ b/tensorflow/docs_src/tutorials/keras/basic_regression.md @@ -1,3 +1,3 @@ # Basic Regression -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_regression.ipynb) +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/tutorials/keras/basic_regression.ipynb) diff --git a/tensorflow/docs_src/tutorials/keras/basic_text_classification.md b/tensorflow/docs_src/tutorials/keras/basic_text_classification.md index 7c5d4f7896..c2a16bdd20 100644 --- a/tensorflow/docs_src/tutorials/keras/basic_text_classification.md +++ b/tensorflow/docs_src/tutorials/keras/basic_text_classification.md @@ -1,3 +1,3 @@ # Basic Text Classification -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/basic_text_classification.ipynb) +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/tutorials/keras/basic_text_classification.ipynb) diff --git a/tensorflow/docs_src/tutorials/keras/overfit_and_underfit.md b/tensorflow/docs_src/tutorials/keras/overfit_and_underfit.md index e5b5ae7b5a..f07f3addd8 100644 --- a/tensorflow/docs_src/tutorials/keras/overfit_and_underfit.md +++ b/tensorflow/docs_src/tutorials/keras/overfit_and_underfit.md @@ -1,3 +1,3 @@ # Overfitting and Underfitting -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/overfit_and_underfit.ipynb) +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/tutorials/keras/overfit_and_underfit.ipynb) diff --git a/tensorflow/docs_src/tutorials/keras/save_and_restore_models.md b/tensorflow/docs_src/tutorials/keras/save_and_restore_models.md index 44b3772945..a799b379a0 100644 --- a/tensorflow/docs_src/tutorials/keras/save_and_restore_models.md +++ b/tensorflow/docs_src/tutorials/keras/save_and_restore_models.md @@ -1,3 +1,3 @@ # Save and restore Models -[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/save_and_restore_models.ipynb) +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/tutorials/keras/save_and_restore_models.ipynb) -- cgit v1.2.3 From d722c3e93fa180e4dad7678cf32868ed18f6ef84 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 17:51:35 -0700 Subject: Exclude util/stats_calculator.* from :framework_internal_impl Otherwise both :framework_internal_impl and :stats_calculator_portable compile this file and cause multiple definition. PiperOrigin-RevId: 204555332 --- tensorflow/core/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index dbe87a6dbb..8a43220ec5 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -2464,6 +2464,7 @@ tf_cuda_library( "framework/resource_handle.cc", "util/memmapped_file_system.*", "util/memmapped_file_system_writer.*", + "util/stats_calculator.*", "util/version_info.cc", ], ) + select({ @@ -2490,6 +2491,7 @@ tf_cuda_library( ":protos_all_proto_text", ":error_codes_proto_text", ":protos_all_cc", + ":stats_calculator_portable", ":version_lib", "//tensorflow/core/platform/default/build_config:platformlib", "//tensorflow/core/kernels:bounds_check", -- cgit v1.2.3 From 4424e3270e4056ef7318fbdd83727cb93bec6858 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Fri, 13 Jul 2018 18:18:12 -0700 Subject: [XLA] Move implementation of ThreeFry stateless PRNG into xla/client/lib PiperOrigin-RevId: 204557470 --- tensorflow/compiler/tf2xla/kernels/BUILD | 1 + .../tf2xla/kernels/stateless_random_ops.cc | 168 +++++---------------- tensorflow/compiler/xla/client/lib/BUILD | 15 ++ tensorflow/compiler/xla/client/lib/prng.cc | 150 ++++++++++++++++++ tensorflow/compiler/xla/client/lib/prng.h | 34 +++++ 5 files changed, 234 insertions(+), 134 deletions(-) create mode 100644 tensorflow/compiler/xla/client/lib/prng.cc create mode 100644 tensorflow/compiler/xla/client/lib/prng.h diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD index 5a335aa43c..d88a34dfd9 100644 --- a/tensorflow/compiler/tf2xla/kernels/BUILD +++ b/tensorflow/compiler/tf2xla/kernels/BUILD @@ -127,6 +127,7 @@ tf_kernel_library( "//tensorflow/compiler/xla/client/lib:constants", "//tensorflow/compiler/xla/client/lib:math", "//tensorflow/compiler/xla/client/lib:numeric", + "//tensorflow/compiler/xla/client/lib:prng", "//tensorflow/compiler/xla/client/xla_client:xla_builder", "//tensorflow/core:framework", "//tensorflow/core:image_ops_op_lib", diff --git a/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc b/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc index a6f5769e7b..cc4b13d3b9 100644 --- a/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/lib/constants.h" #include "tensorflow/compiler/xla/client/lib/math.h" #include "tensorflow/compiler/xla/client/lib/numeric.h" +#include "tensorflow/compiler/xla/client/lib/prng.h" #include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" @@ -33,134 +34,6 @@ limitations under the License. namespace tensorflow { namespace { -// Rotates a 32-bit integer 'v' left by 'distance' bits. -xla::XlaOp RotateLeftS32(xla::XlaBuilder* builder, const xla::XlaOp& v, - int distance) { - return xla::Or( - xla::ShiftLeft(v, xla::ConstantR0(builder, distance)), - xla::ShiftRightLogical(v, xla::ConstantR0(builder, 32 - distance))); -} - -using ThreeFry2x32State = std::array; - -// Implements the ThreeFry counter-based PRNG algorithm. -// Salmon et al. SC 2011. Parallel random numbers: as easy as 1, 2, 3. -// http://www.thesalmons.org/john/random123/papers/random123sc11.pdf -ThreeFry2x32State ThreeFry2x32(xla::XlaBuilder* builder, - ThreeFry2x32State input, ThreeFry2x32State key) { - // Rotation distances specified by the Threefry2x32 algorithm. - constexpr std::array rotations = {13, 15, 26, 6, 17, 29, 16, 24}; - ThreeFry2x32State x; - - std::array ks; - // 0x1BD11BDA is a parity constant specified by the ThreeFry2x32 algorithm. - ks[2] = xla::ConstantR0(builder, 0x1BD11BDA); - for (int i = 0; i < 2; ++i) { - ks[i] = key[i]; - x[i] = input[i]; - ks[2] = xla::Xor(ks[2], key[i]); - } - - x[0] = xla::Add(x[0], ks[0]); - x[1] = xla::Add(x[1], ks[1]); - - // Performs a single round of the Threefry2x32 algorithm, with a rotation - // amount 'rotation'. - auto round = [builder](ThreeFry2x32State v, int rotation) { - v[0] = xla::Add(v[0], v[1]); - v[1] = RotateLeftS32(builder, v[1], rotation); - v[1] = xla::Xor(v[0], v[1]); - return v; - }; - - // There are no known statistical flaws with 13 rounds of Threefry2x32. - // We are conservative and use 20 rounds. - x = round(x, rotations[0]); - x = round(x, rotations[1]); - x = round(x, rotations[2]); - x = round(x, rotations[3]); - x[0] = xla::Add(x[0], ks[1]); - x[1] = xla::Add(xla::Add(x[1], ks[2]), xla::ConstantR0(builder, 1)); - - x = round(x, rotations[4]); - x = round(x, rotations[5]); - x = round(x, rotations[6]); - x = round(x, rotations[7]); - x[0] = xla::Add(x[0], ks[2]); - x[1] = xla::Add(xla::Add(x[1], ks[0]), xla::ConstantR0(builder, 2)); - - x = round(x, rotations[0]); - x = round(x, rotations[1]); - x = round(x, rotations[2]); - x = round(x, rotations[3]); - x[0] = xla::Add(x[0], ks[0]); - x[1] = xla::Add(xla::Add(x[1], ks[1]), xla::ConstantR0(builder, 3)); - - x = round(x, rotations[4]); - x = round(x, rotations[5]); - x = round(x, rotations[6]); - x = round(x, rotations[7]); - x[0] = xla::Add(x[0], ks[1]); - x[1] = xla::Add(xla::Add(x[1], ks[2]), xla::ConstantR0(builder, 4)); - - x = round(x, rotations[0]); - x = round(x, rotations[1]); - x = round(x, rotations[2]); - x = round(x, rotations[3]); - x[0] = xla::Add(x[0], ks[2]); - x[1] = xla::Add(xla::Add(x[1], ks[0]), xla::ConstantR0(builder, 5)); - - return x; -} - -// Returns a tensor of 'shape' random values uniformly distributed in the range -// [minval, maxval) -xla::XlaOp RandomUniform(xla::XlaBuilder* builder, const xla::XlaOp& seed, - const TensorShape& shape, double minval, - double maxval) { - // Split the seed into two 32-bit scalars to form a key. - auto seed0 = xla::Reshape(xla::Slice(seed, {0}, {1}, {1}), {}); - auto seed1 = xla::Reshape(xla::Slice(seed, {1}, {2}, {1}), {}); - ThreeFry2x32State key = {seed0, seed1}; - const int64 size = shape.num_elements(); - - const int64 half_size = MathUtil::CeilOfRatio(size, 2); - const bool size_is_odd = (half_size * 2 != size); - - // Fill the generator inputs with unique counter values. - ThreeFry2x32State inputs; - inputs[0] = xla::Iota(builder, xla::S32, half_size); - inputs[1] = xla::Add(inputs[0], xla::ConstantR0(builder, half_size)); - ThreeFry2x32State outputs = ThreeFry2x32(builder, inputs, key); - - if (size_is_odd) { - outputs[1] = xla::Slice(outputs[1], {0}, {half_size - 1}, {1}); - } - - auto bits = - xla::Reshape(xla::ConcatInDim(builder, outputs, 0), shape.dim_sizes()); - - // Form 22 random mantissa bits, with a leading 1 bit. The leading 1 bit - // forces the random bits into the mantissa. - constexpr int kFloatBits = 32; - constexpr int kMantissaBits = 23; - bits = xla::Or( - xla::ShiftRightLogical( - bits, xla::ConstantR0(builder, kFloatBits - kMantissaBits)), - xla::ConstantR0(builder, bit_cast(1.0f))); - auto floats = xla::BitcastConvertType(bits, xla::F32); - - // We have a floating point number in the range [1.0, 2.0). - // Subtract 1.0f to shift to the range [0.0, 1.0) - floats = xla::Sub(floats, xla::ConstantR0(builder, 1.0f)); - // Multiply and add to shift to the range [minval, maxval). - floats = xla::Mul(floats, xla::ConstantR0(builder, maxval - minval)); - floats = xla::Add(floats, xla::ConstantR0(builder, minval)); - return floats; -} - -} // namespace - class StatelessRandomUniformOp : public XlaOpKernel { public: explicit StatelessRandomUniformOp(OpKernelConstruction* ctx) @@ -177,7 +50,17 @@ class StatelessRandomUniformOp : public XlaOpKernel { errors::InvalidArgument("seed must have shape [2], not ", seed_shape.DebugString())); xla::XlaOp seed = ctx->Input(1); - ctx->SetOutput(0, RandomUniform(builder, seed, shape, 0.0, 1.0)); + + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(DT_FLOAT, shape, &xla_shape)); + + auto seed0 = xla::Reshape(xla::Slice(seed, {0}, {1}, {1}), {}); + auto seed1 = xla::Reshape(xla::Slice(seed, {1}, {2}, {1}), {}); + + auto uniform = xla::StatelessRngUniform( + {seed0, seed1}, xla_shape, xla::ConstantR0(builder, 0.0), + xla::ConstantR0(builder, 1.0)); + ctx->SetOutput(0, uniform); } private: @@ -206,8 +89,16 @@ class StatelessRandomNormalOp : public XlaOpKernel { seed_shape.DebugString())); xla::XlaOp seed = ctx->Input(1); xla::XlaBuilder* builder = ctx->builder(); - auto uniform = - RandomUniform(builder, seed, shape, std::nextafter(-1.0f, 0.0f), 1.0); + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(DT_FLOAT, shape, &xla_shape)); + + auto seed0 = xla::Reshape(xla::Slice(seed, {0}, {1}, {1}), {}); + auto seed1 = xla::Reshape(xla::Slice(seed, {1}, {2}, {1}), {}); + + auto uniform = xla::StatelessRngUniform( + {seed0, seed1}, xla_shape, + xla::ConstantR0(builder, std::nextafter(-1.0f, 0.0f)), + xla::ConstantR0(builder, 1.0)); // Convert uniform distribution to normal distribution by computing // sqrt(2) * erfinv(x) auto normal = @@ -240,10 +131,18 @@ class StatelessTruncatedNormalOp : public XlaOpKernel { errors::InvalidArgument("seed must have shape [2], not ", seed_shape.DebugString())); xla::XlaOp seed = ctx->Input(1); - xla::XlaBuilder* b = ctx->builder(); + xla::XlaBuilder* builder = ctx->builder(); + + auto seed0 = xla::Reshape(xla::Slice(seed, {0}, {1}, {1}), {}); + auto seed1 = xla::Reshape(xla::Slice(seed, {1}, {2}, {1}), {}); + + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(DT_FLOAT, shape, &xla_shape)); + auto uniform = xla::StatelessRngUniform( + {seed0, seed1}, xla_shape, + xla::ConstantR0(builder, std::numeric_limits::min()), + xla::ConstantR0(builder, 1.0)); - auto uniform = - RandomUniform(b, seed, shape, std::numeric_limits::min(), 1.0); ctx->SetOutput(0, TruncatedNormal(uniform)); } @@ -257,4 +156,5 @@ REGISTER_XLA_OP(Name("StatelessTruncatedNormal") .TypeConstraint("Tseed", DT_INT32), StatelessTruncatedNormalOp); +} // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD index 6933e9a838..ece5a885b5 100644 --- a/tensorflow/compiler/xla/client/lib/BUILD +++ b/tensorflow/compiler/xla/client/lib/BUILD @@ -118,6 +118,21 @@ xla_test( ], ) +cc_library( + name = "prng", + srcs = ["prng.cc"], + hdrs = ["prng.h"], + deps = [ + ":constants", + ":math", + ":numeric", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/client/xla_client:xla_builder", + "//tensorflow/core:lib", + ], +) + cc_library( name = "testing", srcs = ["testing.cc"], diff --git a/tensorflow/compiler/xla/client/lib/prng.cc b/tensorflow/compiler/xla/client/lib/prng.cc new file mode 100644 index 0000000000..299a6ac2b6 --- /dev/null +++ b/tensorflow/compiler/xla/client/lib/prng.cc @@ -0,0 +1,150 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/xla/client/lib/constants.h" +#include "tensorflow/compiler/xla/client/lib/math.h" +#include "tensorflow/compiler/xla/client/lib/numeric.h" +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/lib/core/casts.h" + +namespace xla { +namespace { + +// Rotates a 32-bit integer 'v' left by 'distance' bits. +XlaOp RotateLeftS32(XlaOp v, int distance) { + return (v << ConstantR0(v.builder(), distance)) | + ShiftRightLogical(v, ConstantR0(v.builder(), 32 - distance)); +} + +using ThreeFry2x32State = std::array; + +// Implements the ThreeFry counter-based PRNG algorithm. +// Salmon et al. SC 2011. Parallel random numbers: as easy as 1, 2, 3. +// http://www.thesalmons.org/john/random123/papers/random123sc11.pdf +ThreeFry2x32State ThreeFry2x32(ThreeFry2x32State input, ThreeFry2x32State key) { + XlaBuilder* builder = input[0].builder(); + // Rotation distances specified by the Threefry2x32 algorithm. + constexpr std::array rotations = {13, 15, 26, 6, 17, 29, 16, 24}; + ThreeFry2x32State x; + + std::array ks; + // 0x1BD11BDA is a parity constant specified by the ThreeFry2x32 algorithm. + ks[2] = ConstantR0(builder, 0x1BD11BDA); + for (int i = 0; i < 2; ++i) { + ks[i] = key[i]; + x[i] = input[i]; + ks[2] = ks[2] ^ key[i]; + } + + x[0] = x[0] + ks[0]; + x[1] = x[1] + ks[1]; + + // Performs a single round of the Threefry2x32 algorithm, with a rotation + // amount 'rotation'. + auto round = [builder](ThreeFry2x32State v, int rotation) { + v[0] = v[0] + v[1]; + v[1] = RotateLeftS32(v[1], rotation); + v[1] = v[0] ^ v[1]; + return v; + }; + + // There are no known statistical flaws with 13 rounds of Threefry2x32. + // We are conservative and use 20 rounds. + x = round(x, rotations[0]); + x = round(x, rotations[1]); + x = round(x, rotations[2]); + x = round(x, rotations[3]); + x[0] = x[0] + ks[1]; + x[1] = x[1] + ks[2] + ConstantR0(builder, 1); + + x = round(x, rotations[4]); + x = round(x, rotations[5]); + x = round(x, rotations[6]); + x = round(x, rotations[7]); + x[0] = x[0] + ks[2]; + x[1] = x[1] + ks[0] + ConstantR0(builder, 2); + + x = round(x, rotations[0]); + x = round(x, rotations[1]); + x = round(x, rotations[2]); + x = round(x, rotations[3]); + x[0] = x[0] + ks[0]; + x[1] = x[1] + ks[1] + ConstantR0(builder, 3); + + x = round(x, rotations[4]); + x = round(x, rotations[5]); + x = round(x, rotations[6]); + x = round(x, rotations[7]); + x[0] = x[0] + ks[1]; + x[1] = x[1] + ks[2] + ConstantR0(builder, 4); + + x = round(x, rotations[0]); + x = round(x, rotations[1]); + x = round(x, rotations[2]); + x = round(x, rotations[3]); + x[0] = x[0] + ks[2]; + x[1] = x[1] + ks[0] + ConstantR0(builder, 5); + + return x; +} + +} // namespace + +XlaOp StatelessRngUniform(std::array seeds, const Shape& shape, + XlaOp minval, XlaOp maxval) { + XlaBuilder* builder = seeds[0].builder(); + if (shape.element_type() != F32) { + return builder->ReportError(Unimplemented( + "Types other than F32 are not implemented by StatelessRngUniform.")); + } + ThreeFry2x32State key = seeds; + const int64 size = ShapeUtil::ElementsIn(shape); + + const int64 half_size = CeilOfRatio(size, 2); + const bool size_is_odd = (half_size * 2 != size); + + // Fill the generator inputs with unique counter values. + ThreeFry2x32State inputs; + inputs[0] = Iota(builder, S32, half_size); + inputs[1] = inputs[0] + ConstantR0(builder, half_size); + ThreeFry2x32State outputs = ThreeFry2x32(inputs, key); + + if (size_is_odd) { + outputs[1] = Slice(outputs[1], {0}, {half_size - 1}, {1}); + } + + auto bits = Reshape(ConcatInDim(builder, outputs, 0), + AsInt64Slice(shape.dimensions())); + + // Form 23 random mantissa bits, with a leading 1 bit. The leading 1 bit + // forces the random bits into the mantissa. + constexpr int kFloatBits = 32; + constexpr int kMantissaBits = 23; + bits = ShiftRightLogical( + bits, ConstantR0(builder, kFloatBits - kMantissaBits)) | + ConstantR0(builder, tensorflow::bit_cast(1.0f)); + auto floats = BitcastConvertType(bits, F32); + + // We have a floating point number in the range [1.0, 2.0). + // Subtract 1.0f to shift to the range [0.0, 1.0) + floats = floats - ConstantR0(builder, 1.0f); + // Multiply and add to shift to the range [minval, maxval). + return floats * (maxval - minval) + minval; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/prng.h b/tensorflow/compiler/xla/client/lib/prng.h new file mode 100644 index 0000000000..ac86390239 --- /dev/null +++ b/tensorflow/compiler/xla/client/lib/prng.h @@ -0,0 +1,34 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_LIB_PRNG_H_ +#define TENSORFLOW_COMPILER_XLA_CLIENT_LIB_PRNG_H_ + +#include + +#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" + +namespace xla { + +// Returns a tensor containing 'shape' random values uniformly distributed in +// the range [minval, maxval). Requires 2 32-bit integer seeds. +// Currently only 'shape's of type F32 are implemented. +XlaOp StatelessRngUniform(std::array seeds, const Shape& shape, + XlaOp minval, XlaOp maxval); + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_PRNG_H_ -- cgit v1.2.3 From 04c2ea507fc3135667d2b933df8ee4832c0f9593 Mon Sep 17 00:00:00 2001 From: Li Liangbin Date: Fri, 6 Jul 2018 21:44:00 +0800 Subject: Make protocol used in estimator customizable. Example code as follow: config = tf.estimator.RunConfig(protocol='grpc+verbs') nn = tf.estimator.Estimator(model_fn=model_fn, model_dir=model_dir, params=params, config=config) --- .../learn/python/learn/estimators/run_config.py | 4 ++++ tensorflow/core/protobuf/tensorflow_server.proto | 2 +- tensorflow/python/estimator/run_config.py | 25 +++++++++++++++++----- tensorflow/python/estimator/training.py | 3 ++- tensorflow/python/estimator/training_test.py | 4 ++++ tensorflow/python/training/server_lib.py | 9 ++++---- .../golden/tensorflow.estimator.-run-config.pbtxt | 6 +++++- 7 files changed, 41 insertions(+), 12 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/estimators/run_config.py b/tensorflow/contrib/learn/python/learn/estimators/run_config.py index 14ee2ba609..7cb87619d9 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/run_config.py +++ b/tensorflow/contrib/learn/python/learn/estimators/run_config.py @@ -240,6 +240,7 @@ class RunConfig(ClusterConfig, core_run_config.RunConfig): keep_checkpoint_max=5, keep_checkpoint_every_n_hours=10000, log_step_count_steps=100, + protocol=None, evaluation_master='', model_dir=None, session_config=None): @@ -289,6 +290,8 @@ class RunConfig(ClusterConfig, core_run_config.RunConfig): session_config: a ConfigProto used to set session parameters, or None. Note - using this argument, it is easy to provide settings which break otherwise perfectly good models. Use with care. + protocol: An optional argument which specifies the protocol used when + starting server. None means default to grpc. """ # Neither parent class calls super().__init__(), so here we have to # manually call their __init__() methods. @@ -313,6 +316,7 @@ class RunConfig(ClusterConfig, core_run_config.RunConfig): self._save_summary_steps = save_summary_steps self._save_checkpoints_secs = save_checkpoints_secs self._log_step_count_steps = log_step_count_steps + self._protocol = protocol self._session_config = session_config if save_checkpoints_secs == RunConfig._USE_DEFAULT: if save_checkpoints_steps is None: diff --git a/tensorflow/core/protobuf/tensorflow_server.proto b/tensorflow/core/protobuf/tensorflow_server.proto index be25804a1b..2bf48d50e1 100644 --- a/tensorflow/core/protobuf/tensorflow_server.proto +++ b/tensorflow/core/protobuf/tensorflow_server.proto @@ -46,6 +46,6 @@ message ServerDef { // The protocol to be used by this server. // - // Acceptable values include: "grpc". + // Acceptable values include: "grpc", "grpc+verbs". string protocol = 5; } diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py index aa594af2e4..b495c4884d 100644 --- a/tensorflow/python/estimator/run_config.py +++ b/tensorflow/python/estimator/run_config.py @@ -48,7 +48,8 @@ _DEFAULT_REPLACEABLE_LIST = [ 'keep_checkpoint_every_n_hours', 'log_step_count_steps', 'train_distribute', - 'device_fn' + 'device_fn', + 'protocol' ] _SAVE_CKPT_ERR = ( @@ -288,6 +289,10 @@ def _validate_properties(run_config): message='device_fn must be callable with exactly' ' one argument "op".') + _validate('protocol', + lambda protocol: protocol in (None, "grpc", "grpc+verbs"), + message='protocol should be grpc or grpc+verbs') + class TaskType(object): MASTER = 'master' @@ -312,7 +317,8 @@ class RunConfig(object): keep_checkpoint_every_n_hours=10000, log_step_count_steps=100, train_distribute=None, - device_fn=None): + device_fn=None, + protocol=None): """Constructs a RunConfig. All distributed training related properties `cluster_spec`, `is_chief`, @@ -436,7 +442,7 @@ class RunConfig(object): the feature. log_step_count_steps: The frequency, in number of global steps, that the global step/sec and the loss will be logged during training. - train_distribute: an optional instance of + train_distribute: An optional instance of `tf.contrib.distribute.DistributionStrategy`. If specified, then Estimator will distribute the user's model during training, according to the policy specified by that strategy. @@ -444,6 +450,8 @@ class RunConfig(object): `Operation` and returns the device string. If `None`, defaults to the device function returned by `tf.train.replica_device_setter` with round-robin strategy. + protocol: An optional argument which specifies the protocol used when + starting server. None means default to grpc. Raises: ValueError: If both `save_checkpoints_steps` and `save_checkpoints_secs` @@ -481,7 +489,8 @@ class RunConfig(object): keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours, log_step_count_steps=log_step_count_steps, train_distribute=train_distribute, - device_fn=device_fn) + device_fn=device_fn, + protocol=protocol) self._init_distributed_setting_from_environment_var(tf_config) @@ -754,6 +763,11 @@ class RunConfig(object): """ return self._train_distribute + @property + def protocol(self): + """Returns the optional protocol value.""" + return self._protocol + def replace(self, **kwargs): """Returns a new instance of `RunConfig` replacing specified properties. @@ -769,7 +783,8 @@ class RunConfig(object): - `keep_checkpoint_every_n_hours`, - `log_step_count_steps`, - `train_distribute`, - - `device_fn`. + - `device_fn`, + - `protocol`. In addition, either `save_checkpoints_steps` or `save_checkpoints_secs` can be set (should not be both). diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py index f5ac79ced2..a01b2300dd 100644 --- a/tensorflow/python/estimator/training.py +++ b/tensorflow/python/estimator/training.py @@ -732,7 +732,8 @@ class _TrainingExecutor(object): job_name=config.task_type, task_index=config.task_id, config=session_config, - start=False) + start=False, + protocol=config.protocol) server.start() return server diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py index 6bee7cbe83..dc106c7d3b 100644 --- a/tensorflow/python/estimator/training_test.py +++ b/tensorflow/python/estimator/training_test.py @@ -472,6 +472,7 @@ class _TrainingExecutorTrainingTest(object): job_name=mock_est.config.task_type, task_index=mock_est.config.task_id, config=test.mock.ANY, + protocol=None, start=False) self.assertTrue(mock_server_instance.start.called) @@ -502,6 +503,7 @@ class _TrainingExecutorTrainingTest(object): job_name=mock_est.config.task_type, task_index=mock_est.config.task_id, config=test.mock.ANY, + protocol=None, start=False) self.assertTrue(mock_server_instance.start.called) @@ -729,6 +731,7 @@ class TrainingExecutorRunMasterTest(test.TestCase): job_name=mock_est.config.task_type, task_index=mock_est.config.task_id, config=test.mock.ANY, + protocol=None, start=False) self.assertTrue(mock_server_instance.start.called) @@ -1481,6 +1484,7 @@ class TrainingExecutorRunPsTest(test.TestCase): job_name=mock_est.config.task_type, task_index=mock_est.config.task_id, config=test.mock.ANY, + protocol=None, start=False) self.assertTrue(mock_server_instance.start.called) diff --git a/tensorflow/python/training/server_lib.py b/tensorflow/python/training/server_lib.py index 2f421d1cc0..58cf5277fe 100644 --- a/tensorflow/python/training/server_lib.py +++ b/tensorflow/python/training/server_lib.py @@ -42,8 +42,8 @@ def _make_server_def(server_or_cluster_def, job_name, task_index, protocol, Defaults to the value in `server_or_cluster_def`, if specified. Otherwise defaults to 0 if the server's job has only one task. protocol: (Optional.) Specifies the protocol to be used by the server. - Acceptable values include `"grpc"`. Defaults to the value in - `server_or_cluster_def`, if specified. Otherwise defaults to `"grpc"`. + Acceptable values include `"grpc", "grpc+verbs"`. Defaults to the value + in `server_or_cluster_def`, if specified. Otherwise defaults to `"grpc"`. config: (Options.) A `tf.ConfigProto` that specifies default configuration options for all sessions that run on this server. @@ -129,8 +129,9 @@ class Server(object): job. Defaults to the value in `server_or_cluster_def`, if specified. Otherwise defaults to 0 if the server's job has only one task. protocol: (Optional.) Specifies the protocol to be used by the server. - Acceptable values include `"grpc"`. Defaults to the value in - `server_or_cluster_def`, if specified. Otherwise defaults to `"grpc"`. + Acceptable values include `"grpc", "grpc+verbs"`. Defaults to the + value in `server_or_cluster_def`, if specified. Otherwise defaults to + `"grpc"`. config: (Options.) A `tf.ConfigProto` that specifies default configuration options for all sessions that run on this server. start: (Optional.) Boolean, indicating whether to start the server diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt index c8da55d802..5aa4b3d4fb 100644 --- a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt @@ -50,6 +50,10 @@ tf_class { name: "num_worker_replicas" mtype: "" } + member { + name: "protocol" + mtype: "" + } member { name: "save_checkpoints_secs" mtype: "" @@ -88,7 +92,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\', \'train_distribute\', \'device_fn\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'100\', \'\', \'\', \'None\', \'5\', \'10000\', \'100\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\', \'train_distribute\', \'device_fn\', \'protocol\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'100\', \'\', \'\', \'None\', \'5\', \'10000\', \'100\', \'None\', \'None\', \'None\'], " } member_method { name: "replace" -- cgit v1.2.3 From 5061a440b38bf64f4bd6d18a61b2e639f845cb6c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 19:00:43 -0700 Subject: Internal change. PiperOrigin-RevId: 204560026 --- .../contrib/lite/kernels/internal/kernel_utils.cc | 23 +++---- .../internal/optimized/neon_tensor_utils.cc | 71 ++++++++++++++++++++++ .../kernels/internal/optimized/neon_tensor_utils.h | 4 ++ .../kernels/internal/optimized/tensor_utils_impl.h | 6 ++ .../internal/reference/portable_tensor_utils.cc | 7 +++ .../internal/reference/portable_tensor_utils.h | 10 +++ .../contrib/lite/kernels/internal/tensor_utils.h | 4 ++ .../lite/kernels/internal/tensor_utils_test.cc | 16 +++++ 8 files changed, 127 insertions(+), 14 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/kernel_utils.cc b/tensorflow/contrib/lite/kernels/internal/kernel_utils.cc index a0e382edb6..200f2f1515 100644 --- a/tensorflow/contrib/lite/kernels/internal/kernel_utils.cc +++ b/tensorflow/contrib/lite/kernels/internal/kernel_utils.cc @@ -255,14 +255,6 @@ void LstmStep( output_state_ptr); } -// TODO(alanchiao): move this to tensor_utils. -void VectorMultiply(const int8_t* vector, const int v_size, const float scale, - float* result) { - for (int i = 0; i < v_size; ++i) { - *result++ = scale * *vector++; - } -} - void LstmStep( const float* input_ptr_batch, const int8_t* input_to_input_weights_ptr, float input_to_input_weights_scale, @@ -415,8 +407,9 @@ void LstmStep( // For each batch and cell: update input gate. if (!use_cifg) { if (use_peephole && !is_cell_state_all_zeros) { - VectorMultiply(cell_to_input_weights_ptr, n_cell, - cell_to_input_weights_scale, recovered_cell_weights); + tensor_utils::VectorScalarMultiply(cell_to_input_weights_ptr, n_cell, + cell_to_input_weights_scale, + recovered_cell_weights); tensor_utils::VectorBatchVectorCwiseProductAccumulate( recovered_cell_weights, n_cell, cell_state_ptr, n_batch, input_gate_scratch); @@ -427,8 +420,9 @@ void LstmStep( // For each batch and cell: update forget gate. if (use_peephole && !is_cell_state_all_zeros) { - VectorMultiply(cell_to_forget_weights_ptr, n_cell, - cell_to_forget_weights_scale, recovered_cell_weights); + tensor_utils::VectorScalarMultiply(cell_to_forget_weights_ptr, n_cell, + cell_to_forget_weights_scale, + recovered_cell_weights); tensor_utils::VectorBatchVectorCwiseProductAccumulate( recovered_cell_weights, n_cell, cell_state_ptr, n_batch, forget_gate_scratch); @@ -459,8 +453,9 @@ void LstmStep( tensor_utils::IsZeroVector(cell_state_ptr, n_batch * n_cell); // For each batch and cell: update the output gate. if (use_peephole && !is_cell_state_all_zeros) { - VectorMultiply(cell_to_output_weights_ptr, n_cell, - cell_to_output_weights_scale, recovered_cell_weights); + tensor_utils::VectorScalarMultiply(cell_to_output_weights_ptr, n_cell, + cell_to_output_weights_scale, + recovered_cell_weights); tensor_utils::VectorBatchVectorCwiseProductAccumulate( recovered_cell_weights, n_cell, cell_state_ptr, n_batch, output_gate_scratch); diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/neon_tensor_utils.cc b/tensorflow/contrib/lite/kernels/internal/optimized/neon_tensor_utils.cc index 8c57c987d7..420bc68b43 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/neon_tensor_utils.cc +++ b/tensorflow/contrib/lite/kernels/internal/optimized/neon_tensor_utils.cc @@ -342,6 +342,77 @@ void NeonClipVector(const float* vector, int v_size, float abs_limit, } } +void NeonVectorScalarMultiply(const int8_t* vector, const int v_size, + const float scale, float* result) { + // Here the assumption is that each buffer is 4-byte aligned. + const int kWeightsPerUint32 = 4; + TFLITE_CHECK_EQ((intptr_t)(&vector[0]) & (kWeightsPerUint32 - 1), 0); + // If v_size is not divisible by kWeightsPerNeonLane, we cannot use the main + // vectorized loop, and we need to process sequentially. postamble_start shows + // the start index where this should happen. + const int kWeightsPerNeonLane = 16; + const int postamble_start = v_size - (v_size & (kWeightsPerNeonLane - 1)); + + // Create a vector of 4 floats with the scale value. + const float32x4_t scale_f32x4 = vdupq_n_f32(scale); + int v = 0; + for (; v < postamble_start; v += kWeightsPerNeonLane) { + // Load int8 values, sixteen at a time. + const int8x16_t v_i8x16 = vld1q_s8(vector + v); + // Split it into two components of size eight. + const int8x8_t v0_i8x8 = vget_low_s8(v_i8x16); + const int8x8_t v1_i8x8 = vget_high_s8(v_i8x16); + // Convert both components to int16 first. + const int16x8_t v0_i16x8 = vmovl_s8(v0_i8x8); + const int16x8_t v1_i16x8 = vmovl_s8(v1_i8x8); + // Split each of them into two components each. + const int16x4_t v0_i16x4 = vget_low_s16(v0_i16x8); + const int16x4_t v1_i16x4 = vget_high_s16(v0_i16x8); + const int16x4_t v2_i16x4 = vget_low_s16(v1_i16x8); + const int16x4_t v3_i16x4 = vget_high_s16(v1_i16x8); + // Convert these to int32 and then to float. + float32x4_t v0_f32x4 = vcvtq_f32_s32(vmovl_s16(v0_i16x4)); + float32x4_t v1_f32x4 = vcvtq_f32_s32(vmovl_s16(v1_i16x4)); + float32x4_t v2_f32x4 = vcvtq_f32_s32(vmovl_s16(v2_i16x4)); + float32x4_t v3_f32x4 = vcvtq_f32_s32(vmovl_s16(v3_i16x4)); + // Vector multiply four floats at a time. + v0_f32x4 = vmulq_f32(v0_f32x4, scale_f32x4); + v1_f32x4 = vmulq_f32(v1_f32x4, scale_f32x4); + v2_f32x4 = vmulq_f32(v2_f32x4, scale_f32x4); + v3_f32x4 = vmulq_f32(v3_f32x4, scale_f32x4); + // Store the results. + vst1q_f32(result + v, v0_f32x4); + vst1q_f32(result + v + 4, v1_f32x4); + vst1q_f32(result + v + 8, v2_f32x4); + vst1q_f32(result + v + 12, v3_f32x4); + } + + if (v_size - postamble_start >= (kWeightsPerNeonLane >> 1)) { + // Load eight int8 values, if there is at least eight remaining. + const int8x8_t v_i8x8 = vld1_s8(vector + v); + // Convert them to int16 first. + const int16x8_t v_i16x8 = vmovl_s8(v_i8x8); + // Split it into two components. + const int16x4_t v0_i16x4 = vget_low_s16(v_i16x8); + const int16x4_t v1_i16x4 = vget_high_s16(v_i16x8); + // Convert the components two floats. + float32x4_t v0_f32x4 = vcvtq_f32_s32(vmovl_s16(v0_i16x4)); + float32x4_t v1_f32x4 = vcvtq_f32_s32(vmovl_s16(v1_i16x4)); + // Vector multiply four floats at a time. + v0_f32x4 = vmulq_f32(v0_f32x4, scale_f32x4); + v1_f32x4 = vmulq_f32(v1_f32x4, scale_f32x4); + // Store the results. + vst1q_f32(result + v, v0_f32x4); + vst1q_f32(result + v + 4, v1_f32x4); + v += (kWeightsPerNeonLane >> 1); + } + + // Postamble loop. + for (; v < v_size; v++) { + result[v] = scale * vector[v]; + } +} + void NeonSymmetricQuantizeFloats(const float* values, const int size, int8_t* quantized_values, float* min, float* max, float* scaling_factor) { diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/neon_tensor_utils.h b/tensorflow/contrib/lite/kernels/internal/optimized/neon_tensor_utils.h index 7a5a8fc541..45c9f65b64 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/neon_tensor_utils.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/neon_tensor_utils.h @@ -105,6 +105,10 @@ bool IsZeroVector(const float* vector, int v_size) { return NEON_OR_PORTABLE(IsZeroVector, vector, v_size); } +void VectorScalarMultiply(const int8_t* vector, int v_size, float scale, + float* result) { + NEON_OR_PORTABLE(VectorScalarMultiply, vector, v_size, scale, result); +} void ClipVector(const float* vector, int v_size, float abs_limit, float* result) { NEON_OR_PORTABLE(ClipVector, vector, v_size, abs_limit, result); diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/tensor_utils_impl.h b/tensorflow/contrib/lite/kernels/internal/optimized/tensor_utils_impl.h index f14667090f..db7926df9a 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/tensor_utils_impl.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/tensor_utils_impl.h @@ -124,6 +124,12 @@ void PortableCopyVector(const float* vector, int v_size, float* result); // Fill vector with 0.f. void PortableZeroVector(float* vector, int v_size); +// Multiply all elements of vector with a scalar. +void PortableVectorScalarMultiply(const int8_t* vector, int v_size, float scale, + float* result); +void NeonVectorScalarMultiply(const int8_t* vector, int v_size, float scale, + float* result); + // Limit a float input f between +abs_limit and -abs_limit. float PortableClip(float f, float abs_limit); diff --git a/tensorflow/contrib/lite/kernels/internal/reference/portable_tensor_utils.cc b/tensorflow/contrib/lite/kernels/internal/reference/portable_tensor_utils.cc index ccf112c990..7ead449ca8 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/portable_tensor_utils.cc +++ b/tensorflow/contrib/lite/kernels/internal/reference/portable_tensor_utils.cc @@ -195,6 +195,13 @@ void PortableZeroVector(float* vector, int v_size) { memset(vector, 0, v_size * sizeof(float)); } +void PortableVectorScalarMultiply(const int8_t* vector, const int v_size, + const float scale, float* result) { + for (int v = 0; v < v_size; ++v) { + *result++ = scale * *vector++; + } +} + void PortableClipVector(const float* vector, int v_size, float abs_limit, float* result) { for (int v = 0; v < v_size; v++) { diff --git a/tensorflow/contrib/lite/kernels/internal/reference/portable_tensor_utils.h b/tensorflow/contrib/lite/kernels/internal/reference/portable_tensor_utils.h index d2e1fecd25..d3a4fa8507 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/portable_tensor_utils.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/portable_tensor_utils.h @@ -96,6 +96,10 @@ void PortableSub1Vector(const float* vector, int v_size, float* result); // Fill vector with 0.f. void PortableZeroVector(float* vector, int v_size); +// Multiply all elements of vector with a scalar. +void PortableVectorScalarMultiply(const int8_t* vector, int v_size, float scale, + float* result); + // Clip elements of a vector using a abs_limit value. void PortableClipVector(const float* vector, int v_size, float abs_limit, float* result); @@ -199,6 +203,12 @@ void ZeroVector(float* vector, int v_size) { PortableZeroVector(vector, v_size); } +// Multiply all elements of vector with a scalar. +void VectorScalarMultiply(const int8_t* vector, int v_size, float scale, + float* result) { + PortableVectorScalarMultiply(vector, v_size, scale, result); +} + void ClipVector(const float* vector, int v_size, float abs_limit, float* result) { PortableClipVector(vector, v_size, abs_limit, result); diff --git a/tensorflow/contrib/lite/kernels/internal/tensor_utils.h b/tensorflow/contrib/lite/kernels/internal/tensor_utils.h index 5160e22307..82f4503127 100644 --- a/tensorflow/contrib/lite/kernels/internal/tensor_utils.h +++ b/tensorflow/contrib/lite/kernels/internal/tensor_utils.h @@ -124,6 +124,10 @@ void Sub1Vector(const float* vector, int v_size, float* result); // Fill vector with 0.f. void ZeroVector(float* vector, int v_size); +// Multiply all elements of vector with a scalar. +void VectorScalarMultiply(const int8_t* vector, int v_size, float scale, + float* result); + // Clip elements of a vector using a abs_limit value. void ClipVector(const float* vector, int v_size, float abs_limit, float* result); diff --git a/tensorflow/contrib/lite/kernels/internal/tensor_utils_test.cc b/tensorflow/contrib/lite/kernels/internal/tensor_utils_test.cc index aa0d49ae4d..372a6efec5 100644 --- a/tensorflow/contrib/lite/kernels/internal/tensor_utils_test.cc +++ b/tensorflow/contrib/lite/kernels/internal/tensor_utils_test.cc @@ -32,6 +32,22 @@ TEST(uKernels, ClipTest) { {0.0, -0.5, 1.0, -1.5, 2.0, -2.0, 2.0, -2.0, 2.0, -2.0}))); } +TEST(uKernels, VectorScalarMultiply) { + constexpr int kVectorSize = 29; + static int8_t input[kVectorSize]; + for (int i = 0; i < 29; ++i) { + input[i] = static_cast(i - 14); + } + const float scale = 0.1f; + std::vector output(kVectorSize, 0.0f); + VectorScalarMultiply(input, kVectorSize, scale, output.data()); + EXPECT_THAT(output, + ElementsAreArray(ArrayFloatNear( + {-1.4, -1.3, -1.2, -1.1, -1.0, -0.9, -0.8, -0.7, -0.6, -0.5, + -0.4, -0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3, 0.4, 0.5, + 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4}))); +} + TEST(uKernels, IsZeroTest) { constexpr int kVectorSize = 21; static float zeros[kVectorSize] = {0.0}; -- cgit v1.2.3 From 97b58e82e637d5e614b452c058d6b15768a7be3b Mon Sep 17 00:00:00 2001 From: Revan Sopher Date: Fri, 13 Jul 2018 19:01:47 -0700 Subject: Automated rollback of commit fde3f09e3080a28b9a06ee219474957ba149a20d PiperOrigin-RevId: 204560078 --- tensorflow/contrib/tpu/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index ba93dbbd65..ef6c752851 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -16,7 +16,6 @@ package( "//cloud/vmm/testing/tests/tpu:__subpackages__", "//learning/brain:__subpackages__", "//tensorflow:__subpackages__", - "//third_party/cloud_tpu:__subpackages__", ], ) -- cgit v1.2.3 From f48c9b64c93969eeba833ee83336d1bef45e5231 Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Sat, 14 Jul 2018 02:11:40 +0000 Subject: [XLA:GPU] avoid hard coded CUDA PlatformId --- tensorflow/compiler/jit/kernels/xla_launch_op.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/jit/kernels/xla_launch_op.cc b/tensorflow/compiler/jit/kernels/xla_launch_op.cc index 338fb5a6f0..c5d0e4f8fb 100644 --- a/tensorflow/compiler/jit/kernels/xla_launch_op.cc +++ b/tensorflow/compiler/jit/kernels/xla_launch_op.cc @@ -51,7 +51,11 @@ XlaLocalLaunchBase::XlaLocalLaunchBase(OpKernelConstruction* ctx, if (device_type_ == DeviceType(DEVICE_CPU)) { platform_id_ = se::host::kHostPlatformId; } else if (device_type_ == DeviceType(DEVICE_GPU)) { - platform_id_ = se::cuda::kCudaPlatformId; + platform_id_ = ctx->device() + ->tensorflow_gpu_device_info() + ->stream->parent() + ->platform() + ->id(); } else { platform_id_ = nullptr; } -- cgit v1.2.3 From 274cce5990ff279e2ce293ae9ce1c1d0445d3242 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Fri, 13 Jul 2018 19:59:51 -0700 Subject: Broad refactor (part 7): Swap in the new CFG implementation, along with the new directives support. Simplify the process by which transformers are applied and their testing. Changes to converters: * all tests have been refactored to use the new helpers; functionally they remain the same * several converters have been renamed to be more consistent with the overall naming patterns: single_return.py -> return_statements.py; list_comprehension.py -> list_comprehensions.py; if_exp.py -> conditional_expressions.py * conditional expression converter has been rewritten to use local functions instead of lambdas, which are not yet supported by the static analyzers * the handling of if statements in control_flow.py now uses both the liveness and reaching definitions analyses to more robustly detect output variables * a new directives.py converter is introduced, that transforms user directives (see lang/directives.py) into AST annotations and removes the function calls from the code * list_comprehensions.py is slightly refactored for better readability; it is still not yet enabled * slices.py and lists.py now use the new directives mechanism to obtain list type information * side_effects_guards_test.py has been refactored to avoid flakiness Changes to AG core: * conversion.py now uses the helpers from converter.py and removes the redundant function * converter_testing.py now offers a more unified helper that can be used to test most use cases with less boilerplate * the new directives module now completely replaces the obsolete type_hints.py Changes to static analysis: * activity.py no longer considers index mutation to modify the parent (e.g. a[0] = 1 is not considered to modify a) * activity.py no longer detects the local symbol changes (the IS_MODIFIED_SINCE_ENTRY annotation), which is now handled by reaching_definitions.py * activity.py marks the created field as obsolete, to be removed once the loops in control_flow are migrated to use the dataflow analyses * the obsolete annos.py extension has been removes in favor of the consolidated tags in anno.py * the new CFG implementation now completely replaces the old version * live_values.py and type_info.py also use the symbol definitions from reaching_definitions.py PiperOrigin-RevId: 204563046 --- tensorflow/contrib/autograph/converters/BUILD | 55 +-- tensorflow/contrib/autograph/converters/asserts.py | 8 +- .../contrib/autograph/converters/asserts_test.py | 4 +- .../autograph/converters/break_statements.py | 35 +- .../autograph/converters/break_statements_test.py | 62 +-- .../autograph/converters/builtin_functions_test.py | 60 +-- .../autograph/converters/call_trees_test.py | 76 ++-- .../converters/conditional_expressions.py | 129 ++++++ .../converters/conditional_expressions_test.py | 53 +++ .../converters/continue_statements_test.py | 48 +-- .../contrib/autograph/converters/control_flow.py | 165 ++++---- .../autograph/converters/control_flow_test.py | 211 ++++------ .../autograph/converters/decorators_test.py | 15 +- .../contrib/autograph/converters/directives.py | 108 +++++ .../autograph/converters/directives_test.py | 78 ++++ .../autograph/converters/error_handlers_test.py | 30 +- tensorflow/contrib/autograph/converters/ifexp.py | 49 --- .../contrib/autograph/converters/ifexp_test.py | 106 ----- .../autograph/converters/list_comprehension.py | 77 ---- .../converters/list_comprehension_test.py | 75 ---- .../autograph/converters/list_comprehensions.py | 82 ++++ .../converters/list_comprehensions_test.py | 61 +++ tensorflow/contrib/autograph/converters/lists.py | 30 +- .../contrib/autograph/converters/lists_test.py | 78 ++-- .../converters/logical_expressions_test.py | 13 +- .../autograph/converters/name_scopes_test.py | 90 ++--- .../autograph/converters/return_statements.py | 317 +++++++++++++++ .../autograph/converters/return_statements_test.py | 167 ++++++++ .../converters/side_effect_guards_test.py | 132 +++--- .../contrib/autograph/converters/single_return.py | 312 -------------- .../autograph/converters/single_return_test.py | 189 --------- tensorflow/contrib/autograph/converters/slices.py | 7 +- .../contrib/autograph/converters/slices_test.py | 47 ++- tensorflow/contrib/autograph/core/annos.py | 39 -- .../contrib/autograph/core/converter_testing.py | 57 +-- .../autograph_vs_eager_mnist_benchmark.ipynb | 217 +++------- tensorflow/contrib/autograph/impl/conversion.py | 53 +-- tensorflow/contrib/autograph/pyct/cfg.py | 2 +- .../contrib/autograph/pyct/static_analysis/BUILD | 23 -- .../autograph/pyct/static_analysis/activity.py | 226 +++++------ .../pyct/static_analysis/activity_test.py | 76 +--- .../contrib/autograph/pyct/static_analysis/cfg.py | 446 --------------------- .../autograph/pyct/static_analysis/cfg_test.py | 303 -------------- .../autograph/pyct/static_analysis/live_values.py | 28 +- .../pyct/static_analysis/live_values_test.py | 5 + .../pyct/static_analysis/reaching_definitions.py | 7 +- .../autograph/pyct/static_analysis/type_info.py | 48 +-- .../pyct/static_analysis/type_info_test.py | 5 + tensorflow/contrib/autograph/utils/BUILD | 1 - tensorflow/contrib/autograph/utils/__init__.py | 1 - tensorflow/contrib/autograph/utils/type_hints.py | 41 -- 51 files changed, 1767 insertions(+), 2780 deletions(-) create mode 100644 tensorflow/contrib/autograph/converters/conditional_expressions.py create mode 100644 tensorflow/contrib/autograph/converters/conditional_expressions_test.py create mode 100644 tensorflow/contrib/autograph/converters/directives.py create mode 100644 tensorflow/contrib/autograph/converters/directives_test.py delete mode 100644 tensorflow/contrib/autograph/converters/ifexp.py delete mode 100644 tensorflow/contrib/autograph/converters/ifexp_test.py delete mode 100644 tensorflow/contrib/autograph/converters/list_comprehension.py delete mode 100644 tensorflow/contrib/autograph/converters/list_comprehension_test.py create mode 100644 tensorflow/contrib/autograph/converters/list_comprehensions.py create mode 100644 tensorflow/contrib/autograph/converters/list_comprehensions_test.py create mode 100644 tensorflow/contrib/autograph/converters/return_statements.py create mode 100644 tensorflow/contrib/autograph/converters/return_statements_test.py delete mode 100644 tensorflow/contrib/autograph/converters/single_return.py delete mode 100644 tensorflow/contrib/autograph/converters/single_return_test.py delete mode 100644 tensorflow/contrib/autograph/core/annos.py delete mode 100644 tensorflow/contrib/autograph/pyct/static_analysis/cfg.py delete mode 100644 tensorflow/contrib/autograph/pyct/static_analysis/cfg_test.py delete mode 100644 tensorflow/contrib/autograph/utils/type_hints.py diff --git a/tensorflow/contrib/autograph/converters/BUILD b/tensorflow/contrib/autograph/converters/BUILD index 33d8d517a5..7cbba71683 100644 --- a/tensorflow/contrib/autograph/converters/BUILD +++ b/tensorflow/contrib/autograph/converters/BUILD @@ -21,17 +21,18 @@ py_library( "break_statements.py", "builtin_functions.py", "call_trees.py", + "conditional_expressions.py", "continue_statements.py", "control_flow.py", "decorators.py", + "directives.py", "error_handlers.py", - "ifexp.py", - "list_comprehension.py", + "list_comprehensions.py", "lists.py", "logical_expressions.py", "name_scopes.py", + "return_statements.py", "side_effect_guards.py", - "single_return.py", "slices.py", ], srcs_version = "PY2AND3", @@ -95,6 +96,17 @@ py_test( ], ) +py_test( + name = "conditional_expressions_test", + srcs = ["conditional_expressions_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":converters", + "//tensorflow/contrib/autograph/core:test_lib", + "//tensorflow/python:client_testlib", + ], +) + py_test( name = "continue_statements_test", srcs = ["continue_statements_test.py"], @@ -132,6 +144,18 @@ py_test( ], ) +py_test( + name = "directives_test", + srcs = ["directives_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":converters", + "//tensorflow/contrib/autograph/core:test_lib", + "//tensorflow/contrib/autograph/lang", + "//tensorflow/python:client_testlib", + ], +) + py_test( name = "name_scopes_test", srcs = ["name_scopes_test.py"], @@ -144,8 +168,8 @@ py_test( ) py_test( - name = "list_comprehension_test", - srcs = ["list_comprehension_test.py"], + name = "list_comprehensions_test", + srcs = ["list_comprehensions_test.py"], srcs_version = "PY2AND3", deps = [ ":converters", @@ -180,11 +204,6 @@ py_test( name = "side_effect_guards_test", srcs = ["side_effect_guards_test.py"], srcs_version = "PY2AND3", - tags = [ - # TODO(mdan): Fix. - "flaky", - "notap", - ], deps = [ ":converters", "//tensorflow/contrib/autograph/core:test_lib", @@ -193,20 +212,8 @@ py_test( ) py_test( - name = "single_return_test", - srcs = ["single_return_test.py"], - srcs_version = "PY2AND3", - deps = [ - ":converters", - "//tensorflow/contrib/autograph/core:test_lib", - "//tensorflow/contrib/autograph/pyct", - "//tensorflow/python:client_testlib", - ], -) - -py_test( - name = "ifexp_test", - srcs = ["ifexp_test.py"], + name = "return_statements_test", + srcs = ["return_statements_test.py"], srcs_version = "PY2AND3", deps = [ ":converters", diff --git a/tensorflow/contrib/autograph/converters/asserts.py b/tensorflow/contrib/autograph/converters/asserts.py index e664a403a5..af2f20f267 100644 --- a/tensorflow/contrib/autograph/converters/asserts.py +++ b/tensorflow/contrib/autograph/converters/asserts.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Converts Assert statements to their corresponding TF calls.""" +"""Converts assert statements to their corresponding TF calls.""" from __future__ import absolute_import from __future__ import division @@ -24,8 +24,8 @@ from tensorflow.contrib.autograph.core import converter from tensorflow.contrib.autograph.pyct import templates -class AssertsTransformer(converter.Base): - """Transforms Print nodes to Call so they can be handled as functions.""" +class AssertTransformer(converter.Base): + """Transforms Assert nodes to Call so they can be handled as functions.""" def visit_Assert(self, node): self.generic_visit(node) @@ -46,4 +46,4 @@ class AssertsTransformer(converter.Base): def transform(node, ctx): - return AssertsTransformer(ctx).visit(node) + return AssertTransformer(ctx).visit(node) diff --git a/tensorflow/contrib/autograph/converters/asserts_test.py b/tensorflow/contrib/autograph/converters/asserts_test.py index 2cd0e626bc..9c58ae3acc 100644 --- a/tensorflow/contrib/autograph/converters/asserts_test.py +++ b/tensorflow/contrib/autograph/converters/asserts_test.py @@ -32,8 +32,8 @@ class AssertsTest(converter_testing.TestCase): def test_fn(a): assert a > 0 - node = self.parse_and_analyze(test_fn, {}) - node = asserts.transform(node, self.ctx) + node, ctx = self.prepare(test_fn, {}) + node = asserts.transform(node, ctx) self.assertTrue(isinstance(node.body[0].body[0].value, gast.Call)) diff --git a/tensorflow/contrib/autograph/converters/break_statements.py b/tensorflow/contrib/autograph/converters/break_statements.py index a990e359a2..2a60750bda 100644 --- a/tensorflow/contrib/autograph/converters/break_statements.py +++ b/tensorflow/contrib/autograph/converters/break_statements.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Canonicalizes break statements by de-sugaring into a control boolean.""" +"""Lowers break statements to conditionals.""" from __future__ import absolute_import from __future__ import division @@ -24,17 +24,22 @@ from tensorflow.contrib.autograph.pyct import templates from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno -# Tags for local state. -BREAK_USED = 'break_used' -CONTROL_VAR_NAME = 'control_var_name' +class _Break(object): + def __init__(self): + self.used = False + self.control_var_name = None -class BreakStatementTransformer(converter.Base): + def __repr__(self): + return 'used: %s, var: %s' % (self.used, self.control_var_name) + + +class BreakTransformer(converter.Base): """Canonicalizes break statements into additional conditionals.""" def visit_Break(self, node): - self.set_local(BREAK_USED, True) - var_name = self.get_local(CONTROL_VAR_NAME) + self.state[_Break].used = True + var_name = self.state[_Break].control_var_name # TODO(mdan): This will fail when expanded inside a top-level else block. template = """ var_name = True @@ -57,12 +62,12 @@ class BreakStatementTransformer(converter.Base): block=block) return node - def _track_body(self, nodes, break_var): - self.enter_local_scope() - self.set_local(CONTROL_VAR_NAME, break_var) + def _process_body(self, nodes, break_var): + self.state[_Break].enter() + self.state[_Break].control_var_name = break_var nodes = self.visit_block(nodes) - break_used = self.get_local(BREAK_USED, False) - self.exit_local_scope() + break_used = self.state[_Break].used + self.state[_Break].exit() return nodes, break_used def visit_While(self, node): @@ -70,7 +75,7 @@ class BreakStatementTransformer(converter.Base): break_var = self.ctx.namer.new_symbol('break_', scope.referenced) node.test = self.visit(node.test) - node.body, break_used = self._track_body(node.body, break_var) + node.body, break_used = self._process_body(node.body, break_var) # A break in the else clause applies to the containing scope. node.orelse = self.visit_block(node.orelse) @@ -101,7 +106,7 @@ class BreakStatementTransformer(converter.Base): node.target = self.visit(node.target) node.iter = self.visit(node.iter) - node.body, break_used = self._track_body(node.body, break_var) + node.body, break_used = self._process_body(node.body, break_var) # A break in the else clause applies to the containing scope. node.orelse = self.visit_block(node.orelse) @@ -138,4 +143,4 @@ class BreakStatementTransformer(converter.Base): def transform(node, ctx): - return BreakStatementTransformer(ctx).visit(node) + return BreakTransformer(ctx).visit(node) diff --git a/tensorflow/contrib/autograph/converters/break_statements_test.py b/tensorflow/contrib/autograph/converters/break_statements_test.py index dcff1c54c2..c26ca2946c 100644 --- a/tensorflow/contrib/autograph/converters/break_statements_test.py +++ b/tensorflow/contrib/autograph/converters/break_statements_test.py @@ -25,7 +25,11 @@ from tensorflow.python.platform import test class BreakCanonicalizationTest(converter_testing.TestCase): - def test_basic_while(self): + def assertTransformedEquivalent(self, test_fn, *inputs): + with self.converted(test_fn, break_statements, {}) as result: + self.assertEqual(test_fn(*inputs), result.test_fn(*inputs)) + + def test_while_loop(self): def test_fn(x): v = [] @@ -36,15 +40,11 @@ class BreakCanonicalizationTest(converter_testing.TestCase): v.append(x) return v - node = self.parse_and_analyze(test_fn, {}) - node = break_statements.transform(node, self.ctx) - - with self.compiled(node) as result: - self.assertEqual([], result.test_fn(0)) - self.assertEqual([], result.test_fn(1)) - self.assertEqual([3], result.test_fn(4)) + self.assertTransformedEquivalent(test_fn, 0) + self.assertTransformedEquivalent(test_fn, 1) + self.assertTransformedEquivalent(test_fn, 4) - def test_basic_for(self): + def test_for_loop(self): def test_fn(a): v = [] @@ -55,18 +55,12 @@ class BreakCanonicalizationTest(converter_testing.TestCase): v.append(x) return v - node = self.parse_and_analyze(test_fn, {}) - node = break_statements.transform(node, self.ctx) - - with self.compiled(node) as result: + with self.converted(test_fn, break_statements, {}) as result: # The break is incompletely canonicalized. The loop will not interrupt, # but the section following the break will be skipped. - self.assertEqual([], result.test_fn([])) - self.assertEqual([3, 3], result.test_fn([4, 4])) - self.assertEqual([3], result.test_fn([4, 5])) self.assertEqual([3], result.test_fn([5, 4])) - def test_deeply_nested(self): + def test_nested(self): def test_fn(x): v = [] @@ -83,13 +77,9 @@ class BreakCanonicalizationTest(converter_testing.TestCase): v.append(x) return v, u, w - node = self.parse_and_analyze(test_fn, {}) - node = break_statements.transform(node, self.ctx) - - with self.compiled(node) as result: - self.assertEqual(([], [], []), result.test_fn(0)) - self.assertEqual(([2, 1], [2], [0]), result.test_fn(3)) - self.assertEqual(([10, 9, 8, 7], [10, 8], [6]), result.test_fn(11)) + self.assertTransformedEquivalent(test_fn, 0) + self.assertTransformedEquivalent(test_fn, 3) + self.assertTransformedEquivalent(test_fn, 11) def test_nested_loops(self): @@ -109,16 +99,12 @@ class BreakCanonicalizationTest(converter_testing.TestCase): v.append(x) return v, u - node = self.parse_and_analyze(test_fn, {}) - node = break_statements.transform(node, self.ctx) - - with self.compiled(node) as result: - self.assertEqual(([], []), result.test_fn(0)) - self.assertEqual(([1], []), result.test_fn(2)) - self.assertEqual(([2, 1], [1]), result.test_fn(3)) - self.assertEqual(([4, 3, 2, 1], [3, 1]), result.test_fn(5)) + self.assertTransformedEquivalent(test_fn, 0) + self.assertTransformedEquivalent(test_fn, 2) + self.assertTransformedEquivalent(test_fn, 3) + self.assertTransformedEquivalent(test_fn, 5) - def test_loop_else(self): + def test_loop_orelse(self): def test_fn(x): v = [] @@ -134,13 +120,9 @@ class BreakCanonicalizationTest(converter_testing.TestCase): v.append(x) return v, u - node = self.parse_and_analyze(test_fn, {}) - node = break_statements.transform(node, self.ctx) - - with self.compiled(node) as result: - self.assertEqual(([], []), result.test_fn(0)) - self.assertEqual(([], [1]), result.test_fn(2)) - self.assertEqual(([2], [1]), result.test_fn(3)) + self.assertTransformedEquivalent(test_fn, 0) + self.assertTransformedEquivalent(test_fn, 2) + self.assertTransformedEquivalent(test_fn, 3) if __name__ == '__main__': diff --git a/tensorflow/contrib/autograph/converters/builtin_functions_test.py b/tensorflow/contrib/autograph/converters/builtin_functions_test.py index e9000e518c..d5c3e2c250 100644 --- a/tensorflow/contrib/autograph/converters/builtin_functions_test.py +++ b/tensorflow/contrib/autograph/converters/builtin_functions_test.py @@ -18,8 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import sys - import six from tensorflow.contrib.autograph.converters import builtin_functions @@ -36,55 +34,39 @@ class BuiltinFunctionsTest(converter_testing.TestCase): def test_fn(a): return len(a) - node = self.parse_and_analyze(test_fn, {'len': len}) - node = builtin_functions.transform(node, self.ctx) - - with self.compiled(node, array_ops.shape) as result: + with self.converted(test_fn, builtin_functions, {'len': len}, + array_ops.shape) as result: with self.test_session() as sess: - self.assertEqual(3, - sess.run( - result.test_fn(constant_op.constant([0, 0, 0])))) - - self.assertEqual(3, result.test_fn([0, 0, 0])) + ops = result.test_fn(constant_op.constant([0, 0, 0])) + self.assertEqual(sess.run(ops), 3) def test_print(self): - def test_fn(a): - print(a) + if six.PY2: + return - node = self.parse_and_analyze(test_fn, {'print': print}) - node = builtin_functions.transform(node, self.ctx) + def test_fn(a): + return print(a) - with self.compiled(node) as result: + with self.converted(test_fn, builtin_functions, {'print': print}) as result: with self.test_session() as sess: - try: - out_capturer = six.StringIO() - sys.stdout = out_capturer - result.test_fn(constant_op.constant('a')) - sess.run(sess.graph.get_operations()) - self.assertEqual(out_capturer.getvalue(), 'a\n') - finally: - sys.stdout = sys.__stdout__ + with self.assertPrints('a\n'): + sess.run(result.test_fn('a')) - def test_print_with_op_multiple_values(self): + def test_print_multiple_values(self): - def test_fn(a, b, c): - print(a, b, c) + if six.PY2: + return - node = self.parse_and_analyze(test_fn, {'print': print}) - node = builtin_functions.transform(node, self.ctx) + def test_fn(a, b, c): + return print(a, b, c) - with self.compiled(node) as result: + with self.converted(test_fn, builtin_functions, {'print': print}) as result: with self.test_session() as sess: - try: - out_capturer = six.StringIO() - sys.stdout = out_capturer - result.test_fn( - constant_op.constant('a'), constant_op.constant(1), [2, 3]) - sess.run(sess.graph.get_operations()) - self.assertEqual(out_capturer.getvalue(), 'a 1 [2, 3]\n') - finally: - sys.stdout = sys.__stdout__ + with self.assertPrints('a 1 [2, 3]\n'): + sess.run( + result.test_fn( + constant_op.constant('a'), constant_op.constant(1), [2, 3])) if __name__ == '__main__': diff --git a/tensorflow/contrib/autograph/converters/call_trees_test.py b/tensorflow/contrib/autograph/converters/call_trees_test.py index 27d8281b85..8cdba659ee 100644 --- a/tensorflow/contrib/autograph/converters/call_trees_test.py +++ b/tensorflow/contrib/autograph/converters/call_trees_test.py @@ -36,37 +36,34 @@ class CallTreesTest(converter_testing.TestCase): def test_fn_1(_): raise ValueError('This should not be called in the compiled version.') - def renamed_test_fn_1(a): + def other_test_fn_1(a): return a + 1 def test_fn_2(a): return test_fn_1(a) + 1 - node = self.parse_and_analyze(test_fn_2, {'test_fn_1': test_fn_1}) - node = call_trees.transform(node, self.ctx) + ns = {'test_fn_1': test_fn_1} + node, ctx = self.prepare(test_fn_2, ns) + node = call_trees.transform(node, ctx) - with self.compiled(node) as result: - # Only test_fn_2 is transformed, so we'll insert renamed_test_fn_1 - # manually. - result.renamed_test_fn_1 = renamed_test_fn_1 - self.assertEquals(3, result.test_fn_2(1)) + with self.compiled(node, ns) as result: + new_name, _ = ctx.namer.compiled_function_name(('test_fn_1',)) + setattr(result, new_name, other_test_fn_1) + self.assertEquals(result.test_fn_2(1), 3) def test_dynamic_function(self): def test_fn_1(): - raise ValueError('This should be masked by the mock.') + raise ValueError('This should be masked by the mock in self.compiled.') def test_fn_2(f): return f() + 3 - node = self.parse_and_analyze(test_fn_2, {}) - node = call_trees.transform(node, self.ctx) - - with self.compiled(node) as result: + with self.converted(test_fn_2, call_trees, {}) as result: # 10 = 7 (from the mock) + 3 (from test_fn_2) self.assertEquals(10, result.test_fn_2(test_fn_1)) - def test_simple_methods(self): + def test_basic_method(self): class TestClass(object): @@ -76,49 +73,43 @@ class CallTreesTest(converter_testing.TestCase): def test_fn_2(self, a): return self.test_fn_1(a) + 1 - node = self.parse_and_analyze( - TestClass.test_fn_2, {'TestClass': TestClass}, + ns = {'TestClass': TestClass} + node, ctx = self.prepare( + TestClass.test_fn_2, + ns, namer=converter_testing.FakeNoRenameNamer(), arg_types={'self': (TestClass.__name__, TestClass)}) - node = call_trees.transform(node, self.ctx) + node = call_trees.transform(node, ctx) - with self.compiled(node) as result: + with self.compiled(node, ns) as result: tc = TestClass() self.assertEquals(3, result.test_fn_2(tc, 1)) - def test_py_func_wrap_no_retval(self): + def test_py_func_no_retval(self): def test_fn(a): setattr(a, 'foo', 'bar') - node = self.parse_and_analyze(test_fn, {'setattr': setattr}) - node = call_trees.transform(node, self.ctx) - - with self.compiled(node) as result: + with self.converted(test_fn, call_trees, {'setattr': setattr}) as result: with self.test_session() as sess: - # The function has no return value, so we do some tricks to grab the - # generated py_func node and ensure its effect only happens at graph - # execution. class Dummy(object): pass a = Dummy() result.test_fn(a) + py_func_op, = sess.graph.get_operations() self.assertFalse(hasattr(a, 'foo')) - sess.run(sess.graph.get_operations()[0]) + sess.run(py_func_op) self.assertEquals('bar', a.foo) - def test_py_func_wrap_known_function(self): + def test_py_func_known_function(self): def test_fn(): return np.random.binomial(2, 0.5) - node = self.parse_and_analyze(test_fn, {'np': np}) - node = call_trees.transform(node, self.ctx) - - with self.compiled(node, dtypes.int64) as result: - result.np = np + with self.converted(test_fn, call_trees, {'np': np}, + dtypes.int64) as result: with self.test_session() as sess: self.assertTrue(isinstance(result.test_fn(), ops.Tensor)) self.assertIn(sess.run(result.test_fn()), (0, 1, 2)) @@ -130,22 +121,17 @@ class CallTreesTest(converter_testing.TestCase): a = math_ops.add(a, constant_op.constant(1)) return a - node = self.parse_and_analyze( - test_fn, { - 'math_ops': math_ops, - 'constant_op': constant_op - }, + ns = {'math_ops': math_ops, 'constant_op': constant_op} + node, ctx = self.prepare( + test_fn, + ns, arg_types=set(((math_ops.__name__,), (constant_op.__name__,)))) - node = call_trees.transform(node, self.ctx) + node = call_trees.transform(node, ctx) - with self.compiled(node) as result: - result.math_ops = math_ops - result.constant_op = constant_op + with self.compiled(node, ns) as result: with self.test_session() as sess: - # Not renamed, because the converter doesn't rename the definition - # itself (the caller is responsible for that). result_tensor = result.test_fn(constant_op.constant(1)) - self.assertEquals(3, sess.run(result_tensor)) + self.assertEquals(sess.run(result_tensor), 3) if __name__ == '__main__': diff --git a/tensorflow/contrib/autograph/converters/conditional_expressions.py b/tensorflow/contrib/autograph/converters/conditional_expressions.py new file mode 100644 index 0000000000..63f649dfdf --- /dev/null +++ b/tensorflow/contrib/autograph/converters/conditional_expressions.py @@ -0,0 +1,129 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Converts the ternary conditional operator.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.autograph.core import converter +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import templates +from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno + + +class _FunctionDefs(object): + + def __init__(self): + self.nodes = [] + + +class _Statement(object): + + def __init__(self): + self.scope = None + + +class ConditionalExpressionTransformer(converter.Base): + """Converts contitional expressions to functional form.""" + + def _postprocess_statement(self, node): + """Inserts any separate functions that node may use.""" + replacements = [] + for def_node in self.state[_FunctionDefs].nodes: + replacements.extend(def_node) + replacements.append(node) + node = replacements + # The corresponding enter is called by self.visit_block (see _process_block) + self.state[_FunctionDefs].exit() + return node, None + + def _create_branch(self, expr, name_stem): + scope = self.state[_Statement].scope + name = self.ctx.namer.new_symbol(name_stem, scope.referenced) + template = """ + def name(): + return expr, + """ + node = templates.replace(template, name=name, expr=expr) + self.state[_FunctionDefs].nodes.append(node) + return name + + def visit_IfExp(self, node): + if anno.hasanno(node.test, anno.Basic.QN): + name_root = anno.getanno(node.test, anno.Basic.QN).ssf() + else: + name_root = 'ifexp' + + true_fn_name = self._create_branch(node.body, '%s_true' % name_root) + false_fn_name = self._create_branch(node.orelse, '%s_false' % name_root) + + return templates.replace_as_expression( + 'ag__.utils.run_cond(test, true_fn_name, false_fn_name)', + test=node.test, + true_fn_name=true_fn_name, + false_fn_name=false_fn_name) + + def _process_block(self, scope, block): + self.state[_Statement].enter() + self.state[_Statement].scope = scope + block = self.visit_block( + block, + before_visit=self.state[_FunctionDefs].enter, + after_visit=self._postprocess_statement) + self.state[_Statement].exit() + return block + + def visit_FunctionDef(self, node): + node.args = self.generic_visit(node.args) + node.decorator_list = self.visit_block(node.decorator_list) + node.body = self._process_block( + anno.getanno(node, anno.Static.SCOPE), node.body) + return node + + def visit_For(self, node): + node.target = self.visit(node.target) + node.body = self._process_block( + anno.getanno(node, NodeAnno.BODY_SCOPE), node.body) + node.orelse = self._process_block( + anno.getanno(node, NodeAnno.ORELSE_SCOPE), node.orelse) + return node + + def visit_While(self, node): + node.test = self.visit(node.test) + node.body = self._process_block( + anno.getanno(node, NodeAnno.BODY_SCOPE), node.body) + node.orelse = self._process_block( + anno.getanno(node, NodeAnno.ORELSE_SCOPE), node.orelse) + return node + + def visit_If(self, node): + node.test = self.visit(node.test) + node.body = self._process_block( + anno.getanno(node, NodeAnno.BODY_SCOPE), node.body) + node.orelse = self._process_block( + anno.getanno(node, NodeAnno.ORELSE_SCOPE), node.orelse) + return node + + def visit_With(self, node): + node.items = self.visit_block(node.items) + node.body = self._process_block( + anno.getanno(node, NodeAnno.BODY_SCOPE), node.body) + return node + + +def transform(node, ctx): + node = ConditionalExpressionTransformer(ctx).visit(node) + return node diff --git a/tensorflow/contrib/autograph/converters/conditional_expressions_test.py b/tensorflow/contrib/autograph/converters/conditional_expressions_test.py new file mode 100644 index 0000000000..95a3108741 --- /dev/null +++ b/tensorflow/contrib/autograph/converters/conditional_expressions_test.py @@ -0,0 +1,53 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for conditional_expressions module.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.autograph.converters import conditional_expressions +from tensorflow.contrib.autograph.core import converter_testing +from tensorflow.python.platform import test + + +class ConditionalExpressionsTest(converter_testing.TestCase): + + def assertTransformedEquivalent(self, test_fn, *inputs): + ns = {} + with self.converted(test_fn, conditional_expressions, ns) as result: + self.assertEqual(test_fn(*inputs), result.test_fn(*inputs)) + + def test_basic(self): + + def test_fn(x): + return 1 if x else 0 + + self.assertTransformedEquivalent(test_fn, 0) + self.assertTransformedEquivalent(test_fn, 3) + + def test_nested_orelse(self): + + def test_fn(x): + y = x * x if x > 0 else x if x else 1 + return y + + self.assertTransformedEquivalent(test_fn, -2) + self.assertTransformedEquivalent(test_fn, 0) + self.assertTransformedEquivalent(test_fn, 2) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/autograph/converters/continue_statements_test.py b/tensorflow/contrib/autograph/converters/continue_statements_test.py index 2ce1837972..3a7c7d1486 100644 --- a/tensorflow/contrib/autograph/converters/continue_statements_test.py +++ b/tensorflow/contrib/autograph/converters/continue_statements_test.py @@ -25,7 +25,11 @@ from tensorflow.python.platform import test class ContinueCanonicalizationTest(converter_testing.TestCase): - def test_basic_continue(self): + def assertTransformedEquivalent(self, test_fn, *inputs): + with self.converted(test_fn, continue_statements, {}) as result: + self.assertEqual(test_fn(*inputs), result.test_fn(*inputs)) + + def test_basic(self): def test_fn(x): v = [] @@ -36,17 +40,12 @@ class ContinueCanonicalizationTest(converter_testing.TestCase): v.append(x) return v - node = self.parse_and_analyze(test_fn, {}) - node = continue_statements.transform(node, self.ctx) - - with self.compiled(node) as result: - self.assertEqual(test_fn(0), result.test_fn(0)) - self.assertEqual(test_fn(1), result.test_fn(1)) - self.assertEqual(test_fn(2), result.test_fn(2)) - self.assertEqual(test_fn(3), result.test_fn(3)) - self.assertEqual(test_fn(4), result.test_fn(4)) + self.assertTransformedEquivalent(test_fn, 0) + self.assertTransformedEquivalent(test_fn, 1) + self.assertTransformedEquivalent(test_fn, 3) + self.assertTransformedEquivalent(test_fn, 4) - def test_basic_continue_for_loop(self): + def test_for_loop(self): def test_fn(a): v = [] @@ -57,16 +56,12 @@ class ContinueCanonicalizationTest(converter_testing.TestCase): v.append(x) return v - node = self.parse_and_analyze(test_fn, {}) - node = continue_statements.transform(node, self.ctx) + self.assertTransformedEquivalent(test_fn, []) + self.assertTransformedEquivalent(test_fn, [1]) + self.assertTransformedEquivalent(test_fn, [2]) + self.assertTransformedEquivalent(test_fn, [1, 2, 3]) - with self.compiled(node) as result: - self.assertEqual(test_fn([]), result.test_fn([])) - self.assertEqual(test_fn([1]), result.test_fn([1])) - self.assertEqual(test_fn([2]), result.test_fn([2])) - self.assertEqual(test_fn([1, 2, 3]), result.test_fn([1, 2, 3])) - - def test_continue_deeply_nested(self): + def test_nested(self): def test_fn(x): v = [] @@ -83,15 +78,10 @@ class ContinueCanonicalizationTest(converter_testing.TestCase): v.append(x) return v, u, w - node = self.parse_and_analyze(test_fn, {}) - node = continue_statements.transform(node, self.ctx) - - with self.compiled(node) as result: - self.assertEqual(test_fn(0), result.test_fn(0)) - self.assertEqual(test_fn(1), result.test_fn(1)) - self.assertEqual(test_fn(2), result.test_fn(2)) - self.assertEqual(test_fn(3), result.test_fn(3)) - self.assertEqual(test_fn(4), result.test_fn(4)) + self.assertTransformedEquivalent(test_fn, 0) + self.assertTransformedEquivalent(test_fn, 1) + self.assertTransformedEquivalent(test_fn, 3) + self.assertTransformedEquivalent(test_fn, 4) if __name__ == '__main__': diff --git a/tensorflow/contrib/autograph/converters/control_flow.py b/tensorflow/contrib/autograph/converters/control_flow.py index f4a8710627..a25232f713 100644 --- a/tensorflow/contrib/autograph/converters/control_flow.py +++ b/tensorflow/contrib/autograph/converters/control_flow.py @@ -25,8 +25,7 @@ from tensorflow.contrib.autograph.pyct import anno from tensorflow.contrib.autograph.pyct import ast_util from tensorflow.contrib.autograph.pyct import parser from tensorflow.contrib.autograph.pyct import templates -from tensorflow.contrib.autograph.pyct.static_analysis import cfg -from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno +from tensorflow.contrib.autograph.pyct.static_analysis import annos class SymbolNamer(object): @@ -47,6 +46,7 @@ class SymbolNamer(object): class ControlFlowTransformer(converter.Base): """Transforms control flow structures like loops an conditionals.""" + def _create_cond_branch(self, body_name, aliased_orig_names, aliased_new_names, body, returns): if aliased_orig_names: @@ -90,55 +90,51 @@ class ControlFlowTransformer(converter.Base): return templates.replace( template, test=test, body_name=body_name, orelse_name=orelse_name) - def visit_If(self, node): - self.generic_visit(node) + def _fmt_symbol_list(self, symbol_set): + if not symbol_set: + return 'no variables' + return ', '.join(map(str, symbol_set)) - body_scope = anno.getanno(node, NodeAnno.BODY_SCOPE) - orelse_scope = anno.getanno(node, NodeAnno.ORELSE_SCOPE) - body_defs = body_scope.created | body_scope.modified - orelse_defs = orelse_scope.created | orelse_scope.modified - live = anno.getanno(node, 'live_out') - - # We'll need to check if we're closing over variables that are defined - # elsewhere in the function - # NOTE: we can only detect syntactic closure in the scope - # of the code passed in. If the AutoGraph'd function itself closes - # over other variables, this analysis won't take that into account. - defined = anno.getanno(node, 'defined_in') - - # We only need to return variables that are - # - modified by one or both branches - # - live (or has a live parent) at the end of the conditional - modified = [] - for def_ in body_defs | orelse_defs: - def_with_parents = set((def_,)) | def_.support_set - if live & def_with_parents: - modified.append(def_) - - # We need to check if live created variables are balanced - # in both branches - created = live & (body_scope.created | orelse_scope.created) - - # The if statement is illegal if there are variables that are created, - # that are also live, but both branches don't create them. - if created: - if created != (body_scope.created & live): - raise ValueError( - 'The main branch does not create all live symbols that the else ' - 'branch does.') - if created != (orelse_scope.created & live): - raise ValueError( - 'The else branch does not create all live symbols that the main ' - 'branch does.') - - # Alias the closure variables inside the conditional functions - # to avoid errors caused by the local variables created in the branch - # functions. + def visit_If(self, node): + node = self.generic_visit(node) + + body_scope = anno.getanno(node, annos.NodeAnno.BODY_SCOPE) + orelse_scope = anno.getanno(node, annos.NodeAnno.ORELSE_SCOPE) + defined_in = anno.getanno(node, anno.Static.DEFINED_VARS_IN) + live_out = anno.getanno(node, anno.Static.LIVE_VARS_OUT) + + modified_in_cond = body_scope.modified | orelse_scope.modified + returned_from_cond = set() + for s in modified_in_cond: + if s in live_out: + returned_from_cond.add(s) + elif s.is_composite(): + # Special treatment for compound objects: if any of their owner entities + # are live, then they are outputs as well. + if any(owner in live_out for owner in s.owner_set): + returned_from_cond.add(s) + + need_alias_in_body = body_scope.modified & defined_in + need_alias_in_orelse = orelse_scope.modified & defined_in + + created_in_body = body_scope.modified & returned_from_cond - defined_in + created_in_orelse = orelse_scope.modified & returned_from_cond - defined_in + + if created_in_body != created_in_orelse: + raise ValueError( + 'if statement may not initialize all variables: the true branch' + ' creates %s, while the false branch creates %s. Make sure all' + ' these variables are initialized either in both' + ' branches or before the if statement.' % + (self._fmt_symbol_list(created_in_body), + self._fmt_symbol_list(created_in_orelse))) + + # Alias the closure variables inside the conditional functions, to allow + # the functions access to the respective variables. # We will alias variables independently for body and orelse scope, # because different branches might write different variables. - aliased_body_orig_names = tuple(body_scope.modified - body_scope.created) - aliased_orelse_orig_names = tuple(orelse_scope.modified - - orelse_scope.created) + aliased_body_orig_names = tuple(need_alias_in_body) + aliased_orelse_orig_names = tuple(need_alias_in_orelse) aliased_body_new_names = tuple( self.ctx.namer.new_symbol(s.ssf(), body_scope.referenced) for s in aliased_body_orig_names) @@ -153,58 +149,47 @@ class ControlFlowTransformer(converter.Base): node_body = ast_util.rename_symbols(node.body, alias_body_map) node_orelse = ast_util.rename_symbols(node.orelse, alias_orelse_map) - if not modified: + returned_from_cond = tuple(returned_from_cond) + if returned_from_cond: + if len(returned_from_cond) == 1: + # TODO(mdan): Move this quirk into the operator implementation. + cond_results = returned_from_cond[0] + else: + cond_results = gast.Tuple([s.ast() for s in returned_from_cond], None) + + returned_from_body = tuple( + alias_body_map[s] if s in need_alias_in_body else s + for s in returned_from_cond) + returned_from_orelse = tuple( + alias_orelse_map[s] if s in need_alias_in_orelse else s + for s in returned_from_cond) + + else: # When the cond would return no value, we leave the cond called without # results. That in turn should trigger the side effect guards. The # branch functions will return a dummy value that ensures cond # actually has some return value as well. - results = None - elif len(modified) == 1: - results = modified[0] - else: - results = gast.Tuple([s.ast() for s in modified], None) + cond_results = None + # TODO(mdan): This doesn't belong here; it's specific to the operator. + returned_from_body = templates.replace_as_expression('1') + returned_from_orelse = templates.replace_as_expression('1') body_name = self.ctx.namer.new_symbol('if_true', body_scope.referenced) orelse_name = self.ctx.namer.new_symbol('if_false', orelse_scope.referenced) - if modified: - - def build_returns(aliased_names, alias_map, scope): - """Builds list of return variables for a branch of a conditional.""" - returns = [] - for s in modified: - if s in aliased_names: - returns.append(alias_map[s]) - else: - if s not in scope.created | defined: - raise ValueError( - 'Attempting to return variable "%s" from the true branch of ' - 'a conditional, but it was not closed over, or created in ' - 'this branch.' % str(s)) - else: - returns.append(s) - return tuple(returns) - - body_returns = build_returns(aliased_body_orig_names, alias_body_map, - body_scope) - orelse_returns = build_returns(aliased_orelse_orig_names, - alias_orelse_map, orelse_scope) - - else: - body_returns = orelse_returns = templates.replace('tf.ones(())')[0].value body_def = self._create_cond_branch( body_name, - aliased_orig_names=tuple(aliased_body_orig_names), - aliased_new_names=tuple(aliased_body_new_names), + aliased_orig_names=aliased_body_orig_names, + aliased_new_names=aliased_body_new_names, body=node_body, - returns=body_returns) + returns=returned_from_body) orelse_def = self._create_cond_branch( orelse_name, - aliased_orig_names=tuple(aliased_orelse_orig_names), - aliased_new_names=tuple(aliased_orelse_new_names), + aliased_orig_names=aliased_orelse_orig_names, + aliased_new_names=aliased_orelse_new_names, body=node_orelse, - returns=orelse_returns) - cond_expr = self._create_cond_expr(results, node.test, body_name, + returns=returned_from_orelse) + cond_expr = self._create_cond_expr(cond_results, node.test, body_name, orelse_name) return body_def + orelse_def + cond_expr @@ -212,11 +197,11 @@ class ControlFlowTransformer(converter.Base): def visit_While(self, node): self.generic_visit(node) - body_scope = anno.getanno(node, NodeAnno.BODY_SCOPE) + body_scope = anno.getanno(node, annos.NodeAnno.BODY_SCOPE) body_closure = body_scope.modified - body_scope.created all_referenced = body_scope.referenced - cond_scope = anno.getanno(node, NodeAnno.COND_SCOPE) + cond_scope = anno.getanno(node, annos.NodeAnno.COND_SCOPE) cond_closure = set() for s in cond_scope.referenced: for root in s.support_set: @@ -277,7 +262,7 @@ class ControlFlowTransformer(converter.Base): def visit_For(self, node): self.generic_visit(node) - body_scope = anno.getanno(node, NodeAnno.BODY_SCOPE) + body_scope = anno.getanno(node, annos.NodeAnno.BODY_SCOPE) body_closure = body_scope.modified - body_scope.created all_referenced = body_scope.referenced @@ -331,7 +316,5 @@ class ControlFlowTransformer(converter.Base): def transform(node, ctx): - cfg.run_analyses(node, cfg.Liveness(ctx.info)) - cfg.run_analyses(node, cfg.Defined(ctx.info)) node = ControlFlowTransformer(ctx).visit(node) return node diff --git a/tensorflow/contrib/autograph/converters/control_flow_test.py b/tensorflow/contrib/autograph/converters/control_flow_test.py index 735eb92a0d..6670b8a66f 100644 --- a/tensorflow/contrib/autograph/converters/control_flow_test.py +++ b/tensorflow/contrib/autograph/converters/control_flow_test.py @@ -20,16 +20,22 @@ from __future__ import print_function from tensorflow.contrib.autograph.converters import control_flow from tensorflow.contrib.autograph.core import converter_testing +from tensorflow.contrib.autograph.pyct import transformer from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import control_flow_ops from tensorflow.python.platform import test class ControlFlowTest(converter_testing.TestCase): - def test_simple_while(self): + def assertTransformedResult(self, test_fn, inputs, expected): + if not isinstance(inputs, tuple): + inputs = (inputs,) + with self.converted(test_fn, control_flow, {}) as result: + with self.test_session() as sess: + self.assertEqual(sess.run(result.test_fn(*inputs)), expected) + + def test_while_basic(self): def test_fn(n): i = 0 @@ -39,29 +45,18 @@ class ControlFlowTest(converter_testing.TestCase): i += 1 return s, i, n - node = self.parse_and_analyze(test_fn, {}) - node = control_flow.transform(node, self.ctx) - - with self.compiled(node) as result: - with self.test_session() as sess: - self.assertEqual((10, 5, 5), - sess.run(result.test_fn(constant_op.constant(5)))) + self.assertTransformedResult(test_fn, constant_op.constant(5), (10, 5, 5)) - def test_while_single_var(self): + def test_while_single_output(self): def test_fn(n): while n > 0: n -= 1 return n - node = self.parse_and_analyze(test_fn, {}) - node = control_flow.transform(node, self.ctx) + self.assertTransformedResult(test_fn, constant_op.constant(5), 0) - with self.compiled(node) as result: - with self.test_session() as sess: - self.assertEqual(0, sess.run(result.test_fn(constant_op.constant(5)))) - - def test_simple_if(self): + def test_if_basic(self): def test_fn(n): a = 0 @@ -72,114 +67,85 @@ class ControlFlowTest(converter_testing.TestCase): b = 2 * n return a, b - node = self.parse_and_analyze(test_fn, {}) - node = control_flow.transform(node, self.ctx) + self.assertTransformedResult(test_fn, constant_op.constant(1), (-1, 0)) + self.assertTransformedResult(test_fn, constant_op.constant(-1), (0, -2)) + + def test_if_complex_outputs(self): + + class TestClass(object): - with self.compiled(node) as result: + def __init__(self, a, b): + self.a = a + self.b = b + + def test_fn(n, obj): + obj.a = 0 + obj.b = 0 + if n > 0: + obj.a = -n + else: + obj.b = 2 * n + return obj + + with self.converted(test_fn, control_flow, {}) as result: with self.test_session() as sess: - self.assertEqual((-1, 0), - sess.run(result.test_fn(constant_op.constant(1)))) - self.assertEqual((0, -2), - sess.run(result.test_fn(constant_op.constant(-1)))) + res_obj = result.test_fn(constant_op.constant(1), TestClass(0, 0)) + self.assertEqual(sess.run((res_obj.a, res_obj.b)), (-1, 0)) + res_obj = result.test_fn(constant_op.constant(-1), TestClass(0, 0)) + self.assertEqual(sess.run((res_obj.a, res_obj.b)), (0, -2)) - def test_if_single_var(self): + def test_if_single_output(self): def test_fn(n): if n > 0: n = -n return n - node = self.parse_and_analyze(test_fn, {}) - node = control_flow.transform(node, self.ctx) + self.assertTransformedResult(test_fn, constant_op.constant(1), -1) - with self.compiled(node) as result: - with self.test_session() as sess: - self.assertEqual(-1, sess.run(result.test_fn(constant_op.constant(1)))) - - def test_imbalanced_aliasing(self): + def test_if_semi(self): def test_fn(n): if n > 0: n = 3 return n - node = self.parse_and_analyze(test_fn, {}) - node = control_flow.transform(node, self.ctx) - - with self.compiled(node, control_flow_ops.cond) as result: - with self.test_session() as sess: - self.assertEqual(3, sess.run(result.test_fn(constant_op.constant(2)))) - self.assertEqual(-3, sess.run(result.test_fn(constant_op.constant(-3)))) + self.assertTransformedResult(test_fn, constant_op.constant(2), 3) + self.assertTransformedResult(test_fn, constant_op.constant(-3), -3) - def test_ignore_unread_variable(self): + def test_if_local_var(self): def test_fn(n): - b = 3 # pylint: disable=unused-variable if n > 0: b = 4 + n = b + 1 return n - node = self.parse_and_analyze(test_fn, {}) - node = control_flow.transform(node, self.ctx) + self.assertTransformedResult(test_fn, constant_op.constant(1), 5) + self.assertTransformedResult(test_fn, constant_op.constant(-1), -1) - with self.compiled(node, control_flow_ops.cond, array_ops.ones) as result: - with self.test_session() as sess: - self.assertEqual(3, sess.run(result.test_fn(constant_op.constant(3)))) - self.assertEqual(-3, sess.run(result.test_fn(constant_op.constant(-3)))) + def test_if_no_outputs(self): - def test_handle_temp_variable(self): + def test_fn(n): + if n > 0: + b = 4 # pylint:disable=unused-variable + return n - def test_fn_using_temp(x, y, w): - if x < y: - z = x + y - else: - w = 2 - tmp = w - z = x - tmp - return z, w + # Without side effect guards, the if statement will stage a cond, + # but that will be pruned at execution. + self.assertTransformedResult(test_fn, constant_op.constant(1), 1) + self.assertTransformedResult(test_fn, constant_op.constant(-1), -1) - node = self.parse_and_analyze(test_fn_using_temp, {}) - node = control_flow.transform(node, self.ctx) + def test_if_imbalanced_outputs(self): - with self.compiled(node, control_flow_ops.cond, array_ops.ones) as result: - with self.test_session() as sess: - z, w = sess.run( - result.test_fn_using_temp( - constant_op.constant(-3), constant_op.constant(3), - constant_op.constant(3))) - self.assertEqual(0, z) - self.assertEqual(3, w) - z, w = sess.run( - result.test_fn_using_temp( - constant_op.constant(3), constant_op.constant(-3), - constant_op.constant(3))) - self.assertEqual(1, z) - self.assertEqual(2, w) - - def test_fn_ignoring_temp(x, y, w): - if x < y: - z = x + y - else: - w = 2 - tmp = w - z = x - tmp - return z + def test_fn(n): + if n > 0: + b = 4 + return b - node = self.parse_and_analyze(test_fn_ignoring_temp, {}) - node = control_flow.transform(node, self.ctx) - - with self.compiled(node, control_flow_ops.cond, array_ops.ones) as result: - with self.test_session() as sess: - z = sess.run( - result.test_fn_ignoring_temp( - constant_op.constant(-3), constant_op.constant(3), - constant_op.constant(3))) - self.assertEqual(0, z) - z = sess.run( - result.test_fn_ignoring_temp( - constant_op.constant(3), constant_op.constant(-3), - constant_op.constant(3))) - self.assertEqual(1, z) + node, ctx = self.prepare(test_fn, {}) + with self.assertRaises(transformer.AutographParseError): + control_flow.transform(node, ctx) def test_simple_for(self): @@ -191,22 +157,11 @@ class ControlFlowTest(converter_testing.TestCase): s2 += e * e return s1, s2 - node = self.parse_and_analyze(test_fn, {}) - node = control_flow.transform(node, self.ctx) + self.assertTransformedResult(test_fn, constant_op.constant([1, 3]), (4, 10)) + empty_vector = constant_op.constant([], shape=(0,), dtype=dtypes.int32) + self.assertTransformedResult(test_fn, empty_vector, (0, 0)) - with self.compiled(node) as result: - with self.test_session() as sess: - l = [1, 2, 3] - self.assertEqual( - test_fn(l), sess.run(result.test_fn(constant_op.constant(l)))) - l = [] - self.assertEqual( - test_fn(l), - sess.run( - result.test_fn( - constant_op.constant(l, shape=(0,), dtype=dtypes.int32)))) - - def test_for_single_var(self): + def test_for_single_output(self): def test_fn(l): s = 0 @@ -214,22 +169,11 @@ class ControlFlowTest(converter_testing.TestCase): s += e return s - node = self.parse_and_analyze(test_fn, {}) - node = control_flow.transform(node, self.ctx) + self.assertTransformedResult(test_fn, constant_op.constant([1, 3]), 4) + empty_vector = constant_op.constant([], shape=(0,), dtype=dtypes.int32) + self.assertTransformedResult(test_fn, empty_vector, 0) - with self.compiled(node) as result: - with self.test_session() as sess: - l = [1, 2, 3] - self.assertEqual( - test_fn(l), sess.run(result.test_fn(constant_op.constant(l)))) - l = [] - self.assertEqual( - test_fn(l), - sess.run( - result.test_fn( - constant_op.constant(l, shape=(0,), dtype=dtypes.int32)))) - - def test_for_with_iterated_expression(self): + def test_for_iterated_expression(self): eval_count = [0] @@ -243,14 +187,13 @@ class ControlFlowTest(converter_testing.TestCase): s += e return s - node = self.parse_and_analyze(test_fn, {'count_evals': count_evals}) - node = control_flow.transform(node, self.ctx) + ns = {'count_evals': count_evals} + node, ctx = self.prepare(test_fn, ns) + node = control_flow.transform(node, ctx) - with self.compiled(node) as result: - result.count_evals = count_evals - self.assertEqual(test_fn(5), result.test_fn(5)) - # count_evals ran twice, once for test_fn and another for result.test_fn - self.assertEqual(eval_count[0], 2) + with self.compiled(node, ns) as result: + self.assertEqual(result.test_fn(5), 10) + self.assertEqual(eval_count[0], 1) if __name__ == '__main__': diff --git a/tensorflow/contrib/autograph/converters/decorators_test.py b/tensorflow/contrib/autograph/converters/decorators_test.py index d41c7fde24..095abc5edc 100644 --- a/tensorflow/contrib/autograph/converters/decorators_test.py +++ b/tensorflow/contrib/autograph/converters/decorators_test.py @@ -61,13 +61,13 @@ class DecoratorsTest(converter_testing.TestCase): 'simple_decorator': simple_decorator, 'converter_testing': converter_testing, } - node = self.parse_and_analyze( + node, ctx = self.prepare( f, namespace, recursive=False, autograph_decorators=autograph_decorators) - node = decorators.transform(node, self.ctx) - import_line = '\n'.join(self.ctx.program.additional_imports) + node = decorators.transform(node, ctx) + import_line = '\n'.join(ctx.program.additional_imports) result, _ = compiler.ast_to_object(node, source_prefix=import_line) return getattr(result, f.__name__) @@ -76,11 +76,8 @@ class DecoratorsTest(converter_testing.TestCase): def test_fn(a): return a - node = self.parse_and_analyze(test_fn, {}) - node = decorators.transform(node, self.ctx) - result, _ = compiler.ast_to_object(node) - - self.assertEqual(1, result.test_fn(1)) + with self.converted(test_fn, decorators, {}) as result: + self.assertEqual(1, result.test_fn(1)) def test_function(self): @@ -124,7 +121,7 @@ class DecoratorsTest(converter_testing.TestCase): return b + 11 return inner_fn(a) - # Expected to fail because simple_decorator cannot be imported. + # Expected to fail because simple_decorator could not be imported. with self.assertRaises(transformer.AutographParseError): test_fn(1) diff --git a/tensorflow/contrib/autograph/converters/directives.py b/tensorflow/contrib/autograph/converters/directives.py new file mode 100644 index 0000000000..ccdf79d47b --- /dev/null +++ b/tensorflow/contrib/autograph/converters/directives.py @@ -0,0 +1,108 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Handles directives. + +This converter removes the directive functions from the code and moves the +information they specify into AST annotations. It is a specialized form of +static analysis, one that is specific to AutoGraph. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gast + +from tensorflow.contrib.autograph.core import converter +from tensorflow.contrib.autograph.lang import directives +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.python.util import tf_inspect + +ENCLOSING_LOOP = 'enclosing_loop' + + +def _map_args(call_node, function): + """Maps AST call nodes to the actual function's arguments. + + Args: + call_node: ast.Call + function: Callable[..., Any], the actual function matching call_node + Returns: + Dict[Text, ast.AST], mapping each of the function's argument names to + the respective AST node. + """ + args = call_node.args + kwds = {kwd.arg: kwd.value for kwd in call_node.keywords} + return tf_inspect.getcallargs(function, *args, **kwds) + + +class DirectivesTransformer(converter.Base): + """Parses compiler directives and converts them into AST annotations.""" + + def _process_symbol_directive(self, call_node, directive): + if len(call_node.args) < 1: + raise ValueError('"%s" requires a positional first argument' + ' as the target' % directive.__name__) + target = call_node.args[0] + defs = anno.getanno(target, anno.Static.ORIG_DEFINITIONS) + for def_ in defs: + def_.directives[directive] = _map_args(call_node, directive) + return call_node + + def _process_statement_directive(self, call_node, directive): + if self.local_scope_level < 1: + raise ValueError( + '"%s" must be used inside a statement' % directive.__name__) + target = self.get_local(ENCLOSING_LOOP) + node_anno = anno.getanno(target, converter.AgAnno.DIRECTIVES, {}) + node_anno[directive] = _map_args(call_node, directive) + anno.setanno(target, converter.AgAnno.DIRECTIVES, node_anno) + return call_node + + def visit_Expr(self, node): + if isinstance(node.value, gast.Call): + call_node = node.value + if anno.hasanno(call_node.func, 'live_val'): + live_val = anno.getanno(call_node.func, 'live_val') + + if live_val is directives.set_element_type: + call_node = self._process_symbol_directive(call_node, live_val) + elif live_val is directives.set_loop_options: + call_node = self._process_statement_directive(call_node, live_val) + else: + return self.generic_visit(node) + + return None # Directive calls are not output in the generated code. + return self.generic_visit(node) + + # TODO(mdan): This will be insufficient for other control flow. + # That means that if we ever have a directive that affects things other than + # loops, we'll need support for parallel scopes, or have multiple converters. + def _track_and_visit_loop(self, node): + self.enter_local_scope() + self.set_local(ENCLOSING_LOOP, node) + node = self.generic_visit(node) + self.exit_local_scope() + return node + + def visit_While(self, node): + return self._track_and_visit_loop(node) + + def visit_For(self, node): + return self._track_and_visit_loop(node) + + +def transform(node, ctx): + return DirectivesTransformer(ctx).visit(node) diff --git a/tensorflow/contrib/autograph/converters/directives_test.py b/tensorflow/contrib/autograph/converters/directives_test.py new file mode 100644 index 0000000000..5f798a5b76 --- /dev/null +++ b/tensorflow/contrib/autograph/converters/directives_test.py @@ -0,0 +1,78 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for directives module.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.autograph.converters import directives as directives_converter +from tensorflow.contrib.autograph.core import converter_testing +from tensorflow.contrib.autograph.core.converter import AgAnno +from tensorflow.contrib.autograph.lang import directives +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.python.platform import test + + +class DirectivesTest(converter_testing.TestCase): + + def test_local_target(self): + + def test_fn(): + l = [] + string_var = 0 + directives.set_element_type(l, 'a', string_var) + + node, ctx = self.prepare(test_fn, {'directives': directives}) + node = directives_converter.transform(node, ctx) + + def_, = anno.getanno(node.body[0].body[0].targets[0], + anno.Static.DEFINITIONS) + d = def_.directives[directives.set_element_type] + self.assertEqual(d['dtype'].s, 'a') + self.assertEqual(d['shape'].id, 'string_var') + + def test_argument_target(self): + + def test_fn(a): + directives.set_element_type(a, 1, shape=2) + + node, ctx = self.prepare(test_fn, {'directives': directives}) + node = directives_converter.transform(node, ctx) + + def_, = anno.getanno(node.body[0].args.args[0], anno.Static.DEFINITIONS) + d = def_.directives[directives.set_element_type] + self.assertEqual(d['dtype'].n, 1) + self.assertEqual(d['shape'].n, 2) + + def test_loop_target(self): + + def test_fn(): + a = True + while True: + directives.set_loop_options(parallel_iterations=10, back_prop=a) + + node, ctx = self.prepare(test_fn, {'directives': directives}) + node = directives_converter.transform(node, ctx) + + d = anno.getanno(node.body[0].body[1], AgAnno.DIRECTIVES) + d = d[directives.set_loop_options] + self.assertEqual(d['parallel_iterations'].n, 10) + self.assertEqual(d['back_prop'].id, 'a') + self.assertEqual(d['swap_memory'], directives.UNSPECIFIED) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/autograph/converters/error_handlers_test.py b/tensorflow/contrib/autograph/converters/error_handlers_test.py index 408e35b4b6..878526c8b4 100644 --- a/tensorflow/contrib/autograph/converters/error_handlers_test.py +++ b/tensorflow/contrib/autograph/converters/error_handlers_test.py @@ -28,32 +28,26 @@ from tensorflow.python.platform import test class ErrorHandlersTest(converter_testing.TestCase): - def compiled_fn(self, test_fn, add_origin=False): - node = self.parse_and_analyze(test_fn, {}) - if add_origin: - anno.setanno(node.body[0], anno.Basic.ORIGIN, - origin_info.OriginInfo(__file__, None, None, None, None)) - node = error_handlers.transform(node, self.ctx) - module = self.compiled(node,) - return module - - def test_no_origin_annotation(self): + def test_basic(self): def test_fn(): - raise ValueError('Crash!') + raise ValueError() - with self.compiled_fn(test_fn) as result: - with self.assertRaises(ValueError): + node, ctx = self.prepare(test_fn, {}) + anno.setanno(node.body[0], anno.Basic.ORIGIN, + origin_info.OriginInfo('test_path', None, None, None, None)) + node = error_handlers.transform(node, ctx) + with self.compiled(node, {}) as result: + with self.assertRaises(errors.GraphConstructionError): result.test_fn() - def test_wraps_body(self): + def test_no_origin_annotation(self): def test_fn(): - raise ValueError('Crash!') + raise ValueError() - with self.compiled_fn(test_fn, add_origin=True) as result: - result.rewrite_graph_construction_error = None - with self.assertRaises(errors.GraphConstructionError): + with self.converted(test_fn, error_handlers, {}) as result: + with self.assertRaises(ValueError): result.test_fn() diff --git a/tensorflow/contrib/autograph/converters/ifexp.py b/tensorflow/contrib/autograph/converters/ifexp.py deleted file mode 100644 index e996138498..0000000000 --- a/tensorflow/contrib/autograph/converters/ifexp.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Canonicalizes the ternary conditional operator.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.autograph.core import converter -from tensorflow.contrib.autograph.pyct import templates - - -class IfExp(converter.Base): - """Canonicalizes all IfExp nodes into plain conditionals.""" - - def visit_IfExp(self, node): - template = """ - ag__.utils.run_cond(test, lambda: (body,), lambda: (orelse,)) - """ - desugared_ifexp = templates.replace_as_expression( - template, test=node.test, body=node.body, orelse=node.orelse) - return desugared_ifexp - - -def transform(node, ctx): - """Desugar IfExp nodes into plain conditionals. - - Args: - node: ast.AST, the node to transform - ctx: converter.EntityContext - - Returns: - new_node: an AST with no IfExp nodes, only conditionals. - """ - - node = IfExp(ctx).visit(node) - return node diff --git a/tensorflow/contrib/autograph/converters/ifexp_test.py b/tensorflow/contrib/autograph/converters/ifexp_test.py deleted file mode 100644 index cdd5a2f591..0000000000 --- a/tensorflow/contrib/autograph/converters/ifexp_test.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for ifexp module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.autograph import utils -from tensorflow.contrib.autograph.converters import ifexp -from tensorflow.contrib.autograph.core import converter_testing -from tensorflow.python.platform import test - - -class IfExpTest(converter_testing.TestCase): - - def compiled_fn(self, test_fn, *args): - node = self.parse_and_analyze(test_fn, {}) - node = ifexp.transform(node, self.ctx) - module = self.compiled(node, *args) - return module - - def test_simple(self): - - def test_fn(x): - return 1 if x else 0 - - with self.compiled_fn(test_fn) as result: - result.autograph_util = utils - for x in [0, 1]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_fn(self): - - def f(x): - return 3 * x - - def test_fn(x): - y = f(x * x if x > 0 else x) - return y - - with self.compiled_fn(test_fn) as result: - result.autograph_util = utils - result.f = f - for x in [-2, 2]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_exp(self): - - def test_fn(x): - return x * x if x > 0 else x - - with self.compiled_fn(test_fn) as result: - result.autograph_util = utils - for x in [-2, 2]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_nested(self): - - def test_fn(x): - return x * x if x > 0 else x if x else 1 - - with self.compiled_fn(test_fn) as result: - result.autograph_util = utils - for x in [-2, 0, 2]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_in_cond(self): - - def test_fn(x): - if x > 0: - return x * x if x < 5 else x * x * x - return -x - - with self.compiled_fn(test_fn) as result: - result.autograph_util = utils - for x in [-2, 2, 5]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_assign_in_cond(self): - - def test_fn(x): - if x > 0: - x = -x if x < 5 else x - return x - - with self.compiled_fn(test_fn) as result: - result.autograph_util = utils - for x in [-2, 2, 5]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/autograph/converters/list_comprehension.py b/tensorflow/contrib/autograph/converters/list_comprehension.py deleted file mode 100644 index c4a13ee822..0000000000 --- a/tensorflow/contrib/autograph/converters/list_comprehension.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Canonicalizing list comprehensions into for and if statements. - -e.g. -result = [x * x for x in xs] - -becomes - -result = [] -for x in xs: - elt = x * x - result.append(elt) -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gast - -from tensorflow.contrib.autograph.core import converter -from tensorflow.contrib.autograph.pyct import parser -from tensorflow.contrib.autograph.pyct import templates - - -class ListCompCanonicalizationTransformer(converter.Base): - """NodeTransformer to canonicalize list comprehensions.""" - - def make_update_list_node(self, list_, elt): - return templates.replace('list_.append(elt)', list_=list_, elt=elt)[0] - - def instantiate_list_node(self): - return parser.parse_str('[]').body[0].value - - def visit_Assign(self, node): - if not isinstance(node.value, gast.ListComp): - return node - if len(node.targets) > 1: - raise ValueError('Only support single assignment.') - return self.canonicalize_listcomp(node.targets[0], node.value) - - def canonicalize_listcomp(self, result_node, list_comp_node): - - make_list = templates.replace( - 'list_ = create_list', - list_=result_node, - create_list=self.instantiate_list_node()) - loop_body = self.make_update_list_node(result_node, list_comp_node.elt) - - for gen in reversed(list_comp_node.generators): - for gen_if in reversed(gen.ifs): - loop_body = templates.replace( - 'if test: loop_body', test=gen_if, loop_body=loop_body) - loop_body = templates.replace( - 'for target in iter_: loop_body', - iter_=gen.iter, - target=gen.target, - loop_body=loop_body) - - return make_list + loop_body - - -def transform(node, ctx): - return ListCompCanonicalizationTransformer(ctx).visit(node) diff --git a/tensorflow/contrib/autograph/converters/list_comprehension_test.py b/tensorflow/contrib/autograph/converters/list_comprehension_test.py deleted file mode 100644 index 2bbee93412..0000000000 --- a/tensorflow/contrib/autograph/converters/list_comprehension_test.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for list_comprehension module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.autograph.converters import list_comprehension -from tensorflow.contrib.autograph.core import converter_testing -from tensorflow.python.platform import test - - -class ListCompTest(converter_testing.TestCase): - - def test_basic(self): - - def test_fn(l): - s = [e * e for e in l] - return s - - node = self.parse_and_analyze(test_fn, {}) - node = list_comprehension.transform(node, self.ctx) - - with self.compiled(node) as result: - l = [1, 2, 3] - self.assertEqual(test_fn(l), result.test_fn(l)) - l = [] - self.assertEqual(test_fn(l), result.test_fn(l)) - - def test_multiple_generators(self): - - def test_fn(l): - s = [e * e for sublist in l for e in sublist] - return s - - node = self.parse_and_analyze(test_fn, {}) - node = list_comprehension.transform(node, self.ctx) - - with self.compiled(node) as result: - l = [[1], [2], [3]] - self.assertEqual(test_fn(l), result.test_fn(l)) - l = [] - self.assertEqual(test_fn(l), result.test_fn(l)) - - def test_conds(self): - - def test_fn(l): - s = [e * e for e in l if e > 1] - return s - - node = self.parse_and_analyze(test_fn, {}) - node = list_comprehension.transform(node, self.ctx) - - with self.compiled(node) as result: - l = [1, 2, 3] - self.assertEqual(test_fn(l), result.test_fn(l)) - l = [] - self.assertEqual(test_fn(l), result.test_fn(l)) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/autograph/converters/list_comprehensions.py b/tensorflow/contrib/autograph/converters/list_comprehensions.py new file mode 100644 index 0000000000..ecf4628816 --- /dev/null +++ b/tensorflow/contrib/autograph/converters/list_comprehensions.py @@ -0,0 +1,82 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Lowers list comprehensions into for and if statements. + +Example: + + result = [x * x for x in xs] + +becomes + + result = [] + for x in xs: + elt = x * x + result.append(elt) +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gast + +from tensorflow.contrib.autograph.core import converter +from tensorflow.contrib.autograph.pyct import templates + + +# TODO(mdan): This should covert directly to operator calls. + + +class ListCompTransformer(converter.Base): + """Lowers list comprehensions into standard control flow.""" + + def visit_Assign(self, node): + if not isinstance(node.value, gast.ListComp): + return self.generic_visit(node) + if len(node.targets) > 1: + raise NotImplementedError('multiple assignments') + + target, = node.targets + list_comp_node = node.value + + template = """ + target = [] + """ + initialization = templates.replace(template, target=target) + + template = """ + target.append(elt) + """ + body = templates.replace(template, target=target, elt=list_comp_node.elt) + + for gen in reversed(list_comp_node.generators): + for gen_if in reversed(gen.ifs): + template = """ + if test: + body + """ + body = templates.replace(template, test=gen_if, body=body) + template = """ + for target in iter_: + body + """ + body = templates.replace( + template, iter_=gen.iter, target=gen.target, body=body) + + return initialization + body + + +def transform(node, ctx): + return ListCompTransformer(ctx).visit(node) diff --git a/tensorflow/contrib/autograph/converters/list_comprehensions_test.py b/tensorflow/contrib/autograph/converters/list_comprehensions_test.py new file mode 100644 index 0000000000..59b5ce9ca0 --- /dev/null +++ b/tensorflow/contrib/autograph/converters/list_comprehensions_test.py @@ -0,0 +1,61 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for list_comprehensions module.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.autograph.converters import list_comprehensions +from tensorflow.contrib.autograph.core import converter_testing +from tensorflow.python.platform import test + + +class ListCompTest(converter_testing.TestCase): + + def assertTransformedEquivalent(self, test_fn, *inputs): + with self.converted(test_fn, list_comprehensions, {}) as result: + self.assertEqual(test_fn(*inputs), result.test_fn(*inputs)) + + def test_basic(self): + + def test_fn(l): + s = [e * e for e in l] + return s + + self.assertTransformedEquivalent(test_fn, []) + self.assertTransformedEquivalent(test_fn, [1, 2, 3]) + + def test_multiple_generators(self): + + def test_fn(l): + s = [e * e for sublist in l for e in sublist] + return s + + self.assertTransformedEquivalent(test_fn, []) + self.assertTransformedEquivalent(test_fn, [[1], [2], [3]]) + + def test_cond(self): + + def test_fn(l): + s = [e * e for e in l if e > 1] + return s + + self.assertTransformedEquivalent(test_fn, []) + self.assertTransformedEquivalent(test_fn, [1, 2, 3]) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/autograph/converters/lists.py b/tensorflow/contrib/autograph/converters/lists.py index d77a044798..a02fc827b8 100644 --- a/tensorflow/contrib/autograph/converters/lists.py +++ b/tensorflow/contrib/autograph/converters/lists.py @@ -33,6 +33,7 @@ from __future__ import print_function import gast from tensorflow.contrib.autograph.core import converter +from tensorflow.contrib.autograph.lang import directives from tensorflow.contrib.autograph.pyct import anno from tensorflow.contrib.autograph.pyct import parser from tensorflow.contrib.autograph.pyct import templates @@ -88,12 +89,12 @@ class ListTransformer(converter.Base): scope = anno.getanno(node, NodeAnno.ARGS_SCOPE) target_node = node.func.value - # Attempt to use a related name if can get one. Otherwise use something + # Attempt to use a related name if one exists. Otherwise use something # generic. if anno.hasanno(target_node, anno.Basic.QN): target_name = anno.getanno(target_node, anno.Basic.QN).ssf() else: - target_name = 'list' + target_name = 'list_' pop_var_name = self.ctx.namer.new_symbol(target_name, scope.referenced) pop_uses = self.get_local(POP_USES, []) @@ -104,9 +105,10 @@ class ListTransformer(converter.Base): def _replace_stack_call(self, node): assert len(node.args) == 1 - dtype = anno.getanno( + dtype = self.get_definition_directive( node.args[0], - 'element_type', + directives.set_element_type, + 'dtype', default=templates.replace_as_expression('None')) template = """ ag__.list_stack( @@ -134,7 +136,10 @@ class ListTransformer(converter.Base): node = self._replace_append_call(node) elif func_name == 'pop' and (len(node.args) <= 1): node = self._replace_pop_call(node) - elif func_name == 'stack' and (len(node.args) == 1): + elif (func_name == 'stack' and (len(node.args) == 1) and + (not node.keywords or node.keywords[0].arg == 'strict')): + # This avoids false positives with keyword args. + # TODO(mdan): handle kwargs properly. node = self._replace_stack_call(node) return node @@ -146,15 +151,22 @@ class ListTransformer(converter.Base): pop_element = original_call_node.args[0] else: pop_element = parser.parse_expression('None') + # The call will be something like "target.pop()", and the dtype is hooked to # target, hence the func.value. - dtype = anno.getanno( + # TODO(mdan): For lists of lists, this won't work. + # The reason why it won't work is because it's unclear how to annotate + # the list as a "list of lists with a certain element type" when using + # operations like `l.pop().pop()`. + dtype = self.get_definition_directive( original_call_node.func.value, - 'element_type', + directives.set_element_type, + 'dtype', default=templates.replace_as_expression('None')) - shape = anno.getanno( + shape = self.get_definition_directive( original_call_node.func.value, - 'element_shape', + directives.set_element_type, + 'shape', default=templates.replace_as_expression('None')) template = """ diff --git a/tensorflow/contrib/autograph/converters/lists_test.py b/tensorflow/contrib/autograph/converters/lists_test.py index ea04097b28..447a88bbe2 100644 --- a/tensorflow/contrib/autograph/converters/lists_test.py +++ b/tensorflow/contrib/autograph/converters/lists_test.py @@ -18,9 +18,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.autograph import utils from tensorflow.contrib.autograph.converters import lists from tensorflow.contrib.autograph.core import converter_testing +from tensorflow.contrib.autograph.lang import directives +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import parser from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -28,6 +30,9 @@ from tensorflow.python.ops import list_ops from tensorflow.python.platform import test +tf = None # Will be replaced by a mock. + + class ListTest(converter_testing.TestCase): def test_empty_list(self): @@ -35,10 +40,7 @@ class ListTest(converter_testing.TestCase): def test_fn(): return [] - node = self.parse_and_analyze(test_fn, {}) - node = lists.transform(node, self.ctx) - - with self.compiled(node) as result: + with self.converted(test_fn, lists, {}) as result: tl = result.test_fn() # Empty tensor lists cannot be evaluated or stacked. self.assertTrue(isinstance(tl, ops.Tensor)) @@ -49,10 +51,7 @@ class ListTest(converter_testing.TestCase): def test_fn(): return [1, 2, 3] - node = self.parse_and_analyze(test_fn, {}) - node = lists.transform(node, self.ctx) - - with self.compiled(node) as result: + with self.converted(test_fn, lists, {}) as result: with self.test_session() as sess: tl = result.test_fn() r = list_ops.tensor_list_stack(tl, dtypes.int32) @@ -66,10 +65,7 @@ class ListTest(converter_testing.TestCase): l.append(3) return l - node = self.parse_and_analyze(test_fn, {}) - node = lists.transform(node, self.ctx) - - with self.compiled(node) as result: + with self.converted(test_fn, lists, {}) as result: with self.test_session() as sess: tl = result.test_fn() r = list_ops.tensor_list_stack(tl, dtypes.int32) @@ -79,23 +75,19 @@ class ListTest(converter_testing.TestCase): def test_fn(): l = [1, 2, 3] - utils.set_element_type(l, dtypes.int32, ()) s = l.pop() return s, l - node = self.parse_and_analyze( - test_fn, - { - 'utils': utils, - 'dtypes': dtypes - }, - include_type_analysis=True, - ) - node = lists.transform(node, self.ctx) - - with self.compiled(node) as result: - result.utils = utils - result.dtypes = dtypes + node, ctx = self.prepare(test_fn, {}) + def_, = anno.getanno(node.body[0].body[0].targets[0], + anno.Static.ORIG_DEFINITIONS) + def_.directives[directives.set_element_type] = { + 'dtype': parser.parse_expression('tf.int32'), + 'shape': parser.parse_expression('()'), + } + node = lists.transform(node, ctx) + + with self.compiled(node, {}, dtypes.int32) as result: with self.test_session() as sess: ts, tl = result.test_fn() r = list_ops.tensor_list_stack(tl, dtypes.int32) @@ -108,10 +100,7 @@ class ListTest(converter_testing.TestCase): s = l.pop().pop() return s - node = self.parse_and_analyze(test_fn, {}) - node = lists.transform(node, self.ctx) - - with self.compiled(node) as result: + with self.converted(test_fn, lists, {}) as result: test_input = [1, 2, [1, 2, 3]] # TODO(mdan): Pass a list of lists of tensor when we fully support that. # For now, we just pass a regular Python list of lists just to verify that @@ -120,29 +109,24 @@ class ListTest(converter_testing.TestCase): def test_list_stack(self): - tf = None # Will be replaced with a mock. - def test_fn(): l = [1, 2, 3] - utils.set_element_type(l, dtypes.int32) return tf.stack(l) - node = self.parse_and_analyze( - test_fn, - { - 'utils': utils, - 'dtypes': dtypes - }, - include_type_analysis=True, - ) - node = lists.transform(node, self.ctx) - - with self.compiled(node, array_ops.stack, dtypes.int32) as result: - result.utils = utils - result.dtypes = dtypes + node, ctx = self.prepare(test_fn, {}) + def_, = anno.getanno(node.body[0].body[0].targets[0], + anno.Static.ORIG_DEFINITIONS) + def_.directives[directives.set_element_type] = { + 'dtype': parser.parse_expression('tf.int32') + } + node = lists.transform(node, ctx) + + with self.compiled(node, {}, array_ops.stack, dtypes.int32) as result: with self.test_session() as sess: self.assertAllEqual(sess.run(result.test_fn()), [1, 2, 3]) + # TODO(mdan): Add a test with tf.stack with axis kwarg. + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/autograph/converters/logical_expressions_test.py b/tensorflow/contrib/autograph/converters/logical_expressions_test.py index 48186024a9..ca07de5e8a 100644 --- a/tensorflow/contrib/autograph/converters/logical_expressions_test.py +++ b/tensorflow/contrib/autograph/converters/logical_expressions_test.py @@ -31,10 +31,8 @@ class GradientsFunctionTest(converter_testing.TestCase): def test_fn(a, b): return a == b - node = self.parse_and_analyze(test_fn, {}) - node = logical_expressions.transform(node, self.ctx) - - with self.compiled(node, math_ops.equal) as result: + with self.converted(test_fn, logical_expressions, {}, + math_ops.equal) as result: with self.test_session() as sess: self.assertTrue(sess.run(result.test_fn(1, 1))) self.assertFalse(sess.run(result.test_fn(1, 2))) @@ -44,11 +42,8 @@ class GradientsFunctionTest(converter_testing.TestCase): def test_fn(a, b, c): return (a or b) and (a or b or c) - node = self.parse_and_analyze(test_fn, {}) - node = logical_expressions.transform(node, self.ctx) - - with self.compiled(node, math_ops.logical_or, - math_ops.logical_and) as result: + with self.converted(test_fn, logical_expressions, {}, math_ops.logical_or, + math_ops.logical_and) as result: with self.test_session() as sess: self.assertTrue(sess.run(result.test_fn(True, False, True))) diff --git a/tensorflow/contrib/autograph/converters/name_scopes_test.py b/tensorflow/contrib/autograph/converters/name_scopes_test.py index 444d0bcd46..a329b0db70 100644 --- a/tensorflow/contrib/autograph/converters/name_scopes_test.py +++ b/tensorflow/contrib/autograph/converters/name_scopes_test.py @@ -31,17 +31,13 @@ class FunctionNameScopeTransformer(converter_testing.TestCase): def test_fn(l): """This should stay here.""" - a = 5 + a = 1 l += a return l - node = self.parse_and_analyze(test_fn, {}) - node = name_scopes.transform(node, self.ctx) - - with self.compiled(node, ops.name_scope) as result: + with self.converted(test_fn, name_scopes, {}, ops.name_scope) as result: result_op = result.test_fn(constant_op.constant(1)) self.assertIn('test_fn/', result_op.op.name) - self.assertEqual('This should stay here.', result.test_fn.__doc__) def test_long_docstring(self): @@ -54,13 +50,12 @@ class FunctionNameScopeTransformer(converter_testing.TestCase): Returns: l """ - return l - - node = self.parse_and_analyze(test_fn, {}) - node = name_scopes.transform(node, self.ctx) + return l + 1 - with self.compiled(node, ops.name_scope) as result: - self.assertIn('Multi-line', result.test_fn.__doc__) + with self.converted(test_fn, name_scopes, {}, ops.name_scope) as result: + result_op = result.test_fn(constant_op.constant(1)) + self.assertIn('test_fn/', result_op.op.name) + self.assertIn('Multi-line docstring.', result.test_fn.__doc__) self.assertIn('Returns:', result.test_fn.__doc__) def test_nested_functions(self): @@ -68,21 +63,16 @@ class FunctionNameScopeTransformer(converter_testing.TestCase): def test_fn(l): def inner_fn(i): - return i ** 2 - - l += 4 - return inner_fn(l) + return i + 1 - node = self.parse_and_analyze(test_fn, {}) - node = name_scopes.transform(node, self.ctx) + l += 1 + return l, inner_fn(l) - with self.compiled(node, ops.name_scope) as result: - result_op = result.test_fn(constant_op.constant(1)) - first_result_input_name = result_op.op.inputs[0].name - second_result_input_name = result_op.op.inputs[1].name - self.assertIn('test_fn/', first_result_input_name) - self.assertNotIn('inner_fn', first_result_input_name) - self.assertIn('test_fn/inner_fn/', second_result_input_name) + with self.converted(test_fn, name_scopes, {}, ops.name_scope) as result: + first, second = result.test_fn(constant_op.constant(1)) + self.assertIn('test_fn/', first.op.name) + self.assertNotIn('inner_fn', first.op.name) + self.assertIn('test_fn/inner_fn/', second.op.name) def test_method(self): @@ -91,48 +81,20 @@ class FunctionNameScopeTransformer(converter_testing.TestCase): def test_fn(self, l): def inner_fn(i): - return i ** 2 - - l += 4 - return inner_fn(l) + return i + 1 - # Note that 'TestClass' was needed in the namespace here. - node = self.parse_and_analyze( - TestClass, {'TestClass': TestClass}, owner_type=TestClass) - node = name_scopes.transform(node, self.ctx) + l += 1 + return l, inner_fn(l) - with self.compiled(node, ops.name_scope) as result: - result_op = result.TestClass().test_fn(constant_op.constant(1)) - first_result_input_name = result_op.op.inputs[0].name - second_result_input_name = result_op.op.inputs[1].name - self.assertIn('TestClass/test_fn/', first_result_input_name) - self.assertNotIn('inner_fn', first_result_input_name) - self.assertIn('TestClass/test_fn/inner_fn/', second_result_input_name) + ns = {'TestClass': TestClass} + node, ctx = self.prepare(TestClass, ns, owner_type=TestClass) + node = name_scopes.transform(node, ctx) - def test_operator(self): - - class TestClass(object): - - def __call__(self, l): - - def inner_fn(i): - return i ** 2 - - l += 4 - return inner_fn(l) - - # Note that 'TestClass' was needed in the namespace here. - node = self.parse_and_analyze( - TestClass.__call__, {'TestClass': TestClass}, owner_type=TestClass) - node = name_scopes.transform(node, self.ctx) - - with self.compiled(node, ops.name_scope) as result: - result_op = result.__call__(TestClass(), constant_op.constant(1)) - first_result_input_name = result_op.op.inputs[0].name - second_result_input_name = result_op.op.inputs[1].name - self.assertIn('call__/', first_result_input_name) - self.assertNotIn('inner_fn', first_result_input_name) - self.assertIn('call__/inner_fn/', second_result_input_name) + with self.compiled(node, {}, ops.name_scope) as result: + first, second = result.TestClass().test_fn(constant_op.constant(1)) + self.assertIn('TestClass/test_fn/', first.op.name) + self.assertNotIn('inner_fn', first.op.name) + self.assertIn('TestClass/test_fn/inner_fn/', second.op.name) if __name__ == '__main__': diff --git a/tensorflow/contrib/autograph/converters/return_statements.py b/tensorflow/contrib/autograph/converters/return_statements.py new file mode 100644 index 0000000000..a351cd81b8 --- /dev/null +++ b/tensorflow/contrib/autograph/converters/return_statements.py @@ -0,0 +1,317 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Canonicalizes functions with multiple returns to use just one.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gast + +from tensorflow.contrib.autograph.core import converter +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import ast_util +from tensorflow.contrib.autograph.pyct import templates +from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno + + +# TODO(mdan): Move this logic into transformer_base. +class BodyVisitor(converter.Base): + """Walks breadth- or depth-first the list-of-nodes bodies of AST nodes.""" + + def __init__(self, ctx, depth_first=False): + super(BodyVisitor, self).__init__(ctx) + self.depth_first = depth_first + self.changes_made = False + + def visit_nodelist(self, nodelist): + for node in nodelist: + if isinstance(node, list): + node = self.visit_nodelist(node) + else: + node = self.generic_visit(node) + return nodelist + + def visit_If(self, node): + if self.depth_first: + node = self.generic_visit(node) + node.body = self.visit_nodelist(node.body) + node.orelse = self.visit_nodelist(node.orelse) + if not self.depth_first: + node = self.generic_visit(node) + return node + + def visit_For(self, node): + if self.depth_first: + node = self.generic_visit(node) + node.body = self.visit_nodelist(node.body) + node.orelse = self.visit_nodelist(node.orelse) + if not self.depth_first: + node = self.generic_visit(node) + return node + + def visit_While(self, node): + if self.depth_first: + node = self.generic_visit(node) + node.body = self.visit_nodelist(node.body) + node.orelse = self.visit_nodelist(node.orelse) + if not self.depth_first: + node = self.generic_visit(node) + return node + + def visit_Try(self, node): + if self.depth_first: + node = self.generic_visit(node) + node.body = self.visit_nodelist(node.body) + node.orelse = self.visit_nodelist(node.orelse) + node.finalbody = self.visit_nodelist(node.finalbody) + for i in range(len(node.handlers)): + node.handlers[i].body = self.visit_nodelist(node.handlers[i].body) + if not self.depth_first: + node = self.generic_visit(node) + return node + + def visit_With(self, node): + if self.depth_first: + node = self.generic_visit(node) + node.body = self.visit_nodelist(node.body) + if not self.depth_first: + node = self.generic_visit(node) + return node + + def visit_FunctionDef(self, node): + if self.depth_first: + node = self.generic_visit(node) + node.body = self.visit_nodelist(node.body) + self.generic_visit(node) + if not self.depth_first: + node = self.generic_visit(node) + return node + + +class FoldElse(BodyVisitor): + + def visit_nodelist(self, nodelist): + for i in range(len(nodelist)): + node = nodelist[i] + if isinstance(node, gast.If): + true_branch_returns = isinstance(node.body[-1], gast.Return) + false_branch_returns = len(node.orelse) and isinstance( + node.orelse[-1], gast.Return) + # If the last node in the if body is a return, + # then every line after this if statement effectively + # belongs in the else. + if true_branch_returns and not false_branch_returns: + for j in range(i + 1, len(nodelist)): + nodelist[i].orelse.append(ast_util.copy_clean(nodelist[j])) + if nodelist[i + 1:]: + self.changes_made = True + return nodelist[:i + 1] + elif not true_branch_returns and false_branch_returns: + for j in range(i + 1, len(nodelist)): + nodelist[i].body.append(ast_util.copy_clean(nodelist[j])) + if nodelist[i + 1:]: + self.changes_made = True + return nodelist[:i + 1] + elif true_branch_returns and false_branch_returns: + if nodelist[i + 1:]: + raise ValueError( + 'Unreachable code after conditional where both branches return.' + ) + return nodelist + elif isinstance(node, gast.Return) and nodelist[i + 1:]: + raise ValueError( + 'Cannot have statements after a return in the same basic block') + return nodelist + + +def contains_return(node): + for n in gast.walk(node): + if isinstance(n, gast.Return): + return True + return False + + +class LiftReturn(converter.Base): + """Move return statements out of If and With blocks.""" + + def __init__(self, ctx): + super(LiftReturn, self).__init__(ctx) + self.changes_made = False + self.common_return_name = None + + def visit_If(self, node): + # Depth-first traversal of if statements + node = self.generic_visit(node) + + # We check if both branches return, and if so, lift the return out of the + # conditional. We don't enforce that the true and false branches either + # both return or both do not, because FoldElse might move a return + # into a branch after this transform completes. FoldElse and LiftReturn + # are alternately run until the code reaches a fixed point. + true_branch_returns = isinstance(node.body[-1], gast.Return) + false_branch_returns = len(node.orelse) and isinstance( + node.orelse[-1], gast.Return) + if true_branch_returns and false_branch_returns: + node.body[-1] = templates.replace( + 'a = b', a=self.common_return_name, b=node.body[-1].value)[0] + node.orelse[-1] = templates.replace( + 'a = b', a=self.common_return_name, b=node.orelse[-1].value)[0] + return_node = templates.replace('return a', a=self.common_return_name)[0] + self.changes_made = True + return [node, return_node] + else: + return node + + def visit_With(self, node): + # Depth-first traversal of syntax + node = self.generic_visit(node) + + # If the with statement returns, lift the return + if isinstance(node.body[-1], gast.Return): + node.body[-1] = templates.replace( + 'a = b', a=self.common_return_name, b=node.body[-1].value)[0] + return_node = templates.replace('return a', a=self.common_return_name)[0] + node = self.generic_visit(node) + self.changes_made = True + return [node, return_node] + else: + return node + + def visit_FunctionDef(self, node): + # Ensure we're doing depth-first traversal + last_return_name = self.common_return_name + body_scope = anno.getanno(node, NodeAnno.BODY_SCOPE) + referenced_names = body_scope.referenced + self.common_return_name = self.ctx.namer.new_symbol('return_', + referenced_names) + node = self.generic_visit(node) + self.common_return_name = last_return_name + return node + + +class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor): + """Throws an error if code returns inside loops or try/except.""" + + # First, throw an error if we detect a return statement in a loop. + # TODO(alexbw): we need to learn to handle returns inside a loop, + # but don't currently have the TF constructs to do so (need something + # that looks vaguely like a goto). + + def __init__(self): + self.cant_return = False + super(DetectReturnInUnsupportedControlFlow, self).__init__() + + def visit_While(self, node): + self.cant_return = True + self.generic_visit(node) + self.cant_return = False + + def visit_For(self, node): + self.cant_return = True + self.generic_visit(node) + self.cant_return = False + + def visit_Try(self, node): + self.cant_return = True + self.generic_visit(node) + self.cant_return = False + + def visit_Return(self, node): + if self.cant_return: + raise ValueError( + '`return` statements are not supported in loops. ' + 'Try assigning to a variable in the while loop, and returning ' + 'outside of the loop') + + +class DetectReturnInConditional(gast.NodeVisitor): + """Assert that no return statements are present in conditionals.""" + + def __init__(self): + self.cant_return = False + super(DetectReturnInConditional, self).__init__() + + def visit_If(self, node): + self.cant_return = True + self.generic_visit(node) + self.cant_return = False + + def visit_Return(self, node): + if self.cant_return: + raise ValueError( + 'After transforms, a conditional contained a `return `statement, ' + 'which is not allowed. This is a bug, and should not happen.') + + +class DetectReturnInFunctionDef(gast.NodeVisitor): + + def visit_FunctionDef(self, node): + self.generic_visit(node) + if not contains_return(node): + raise ValueError( + 'Each function definition should contain at least one return.') + + +def transform(node, ctx): + """Ensure a function has only a single return. + + This transforms an AST node with multiple returns successively into containing + only a single return node. + There are a few restrictions on what we can handle: + - An AST being transformed must contain at least one return. + - No returns allowed in loops. We have to know the type of the return value, + and we currently don't have either a type inference system to discover it, + nor do we have a mechanism for late type binding in TensorFlow. + - After all transformations are finished, a Return node is not allowed inside + control flow. If we were unable to move a return outside of control flow, + this is an error. + + Args: + node: ast.AST + ctx: converter.EntityContext + + Returns: + new_node: an AST with a single return value + + Raises: + ValueError: if the AST is structured so that we can't perform the + transform. + """ + # Make sure that the function has at least one return statement + # TODO(alexbw): turning off this assertion for now -- + # we need to not require this in e.g. class constructors. + # DetectReturnInFunctionDef().visit(node) + + # Make sure there's no returns in unsupported locations (loops, try/except) + DetectReturnInUnsupportedControlFlow().visit(node) + + while True: + + # Try to lift all returns out of if statements and with blocks + lr = LiftReturn(ctx) + node = lr.visit(node) + changes_made = lr.changes_made + fe = FoldElse(ctx) + node = fe.visit(node) + changes_made = changes_made or fe.changes_made + + if not changes_made: + break + + # Make sure we've scrubbed all returns from conditionals + DetectReturnInConditional().visit(node) + + return node diff --git a/tensorflow/contrib/autograph/converters/return_statements_test.py b/tensorflow/contrib/autograph/converters/return_statements_test.py new file mode 100644 index 0000000000..3c7c8c8a25 --- /dev/null +++ b/tensorflow/contrib/autograph/converters/return_statements_test.py @@ -0,0 +1,167 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for return_statements module.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.autograph.converters import return_statements +from tensorflow.contrib.autograph.core import converter_testing +from tensorflow.python.framework import ops +from tensorflow.python.platform import test + + +class SingleReturnTest(converter_testing.TestCase): + + def assertTransformedEquivalent(self, test_fn, *inputs): + ns = {'ops': ops} + with self.converted(test_fn, return_statements, ns) as result: + self.assertEqual(test_fn(*inputs), result.test_fn(*inputs)) + + def test_straightline(self): + + def test_fn(x): + return x * x + + self.assertTransformedEquivalent(test_fn, 2) + + def test_conditional(self): + + def test_fn(x): + if x > 0: + return x + else: + return x * x + + self.assertTransformedEquivalent(test_fn, 2) + self.assertTransformedEquivalent(test_fn, -2) + + def test_missing_orelse(self): + + def test_fn(x): + if x > 0: + return x + + node, ctx = self.prepare(test_fn, {}) + with self.assertRaises(ValueError): + return_statements.transform(node, ctx) + + def test_missing_orelse_recovrable(self): + + def test_fn(x): + if x > 0: + return x + return x * x + + self.assertTransformedEquivalent(test_fn, 2) + self.assertTransformedEquivalent(test_fn, -2) + + def test_missing_branch_return_recoverable(self): + + def test_fn(x): + if x < 0: + x *= x + else: + return x + return x + + self.assertTransformedEquivalent(test_fn, 2) + self.assertTransformedEquivalent(test_fn, -2) + + def test_conditional_nested(self): + + def test_fn(x): + if x > 0: + if x < 5: + return x + else: + return x * x + else: + return x * x * x + + self.assertTransformedEquivalent(test_fn, 2) + self.assertTransformedEquivalent(test_fn, -2) + self.assertTransformedEquivalent(test_fn, 5) + + def test_context_manager(self): + + def test_fn(x): + with ops.name_scope(''): + return x * x + + self.assertTransformedEquivalent(test_fn, 2) + self.assertTransformedEquivalent(test_fn, -2) + + def test_context_manager_in_conditional(self): + + def test_fn(x): + if x > 0: + with ops.name_scope(''): + return x * x + else: + return x + + self.assertTransformedEquivalent(test_fn, 2) + self.assertTransformedEquivalent(test_fn, -2) + + def text_conditional_in_context_manager(self): + + def test_fn(x): + with ops.name_scope(''): + if x > 0: + return x * x + else: + return x + + self.assertTransformedEquivalent(test_fn, 2) + self.assertTransformedEquivalent(test_fn, -2) + + def test_no_return(self): + + def test_fn(x): + x *= x + + self.assertTransformedEquivalent(test_fn, 2) + + def test_nested_functions(self): + + def test_fn(x): + + def inner_fn(y): + if y > 0: + return y * y + else: + return y + + return inner_fn(x) + + self.assertTransformedEquivalent(test_fn, 2) + self.assertTransformedEquivalent(test_fn, -2) + + def test_loop(self): + + def test_fn(x): + for _ in range(10): + return x + return x + + node, ctx = self.prepare(test_fn, {}) + with self.assertRaises(ValueError): + return_statements.transform(node, ctx) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/autograph/converters/side_effect_guards_test.py b/tensorflow/contrib/autograph/converters/side_effect_guards_test.py index a7ad8efed4..de1874321e 100644 --- a/tensorflow/contrib/autograph/converters/side_effect_guards_test.py +++ b/tensorflow/contrib/autograph/converters/side_effect_guards_test.py @@ -25,140 +25,138 @@ from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import state_ops -from tensorflow.python.ops import variables +from tensorflow.python.ops import variable_scope from tensorflow.python.platform import test +tf = None # Will be replaced by a mock. + + class SideEffectGuardsTest(converter_testing.TestCase): def test_side_effect_on_return_only_variable(self): - tf = None - def test_fn(a): tf.assign(a, a + 1) return a - node = self.parse_and_analyze(test_fn, {}) - node = side_effect_guards.transform(node, self.ctx) + node, ctx = self.prepare(test_fn, {}) + node = side_effect_guards.transform(node, ctx) - with self.compiled(node, state_ops.assign) as result: - self.assertEqual(len(node.body[0].body), 1) + self.assertEqual(len(node.body[0].body), 1) + + with self.compiled(node, {}, state_ops.assign) as result: with self.test_session() as sess: - v = variables.Variable(2) + v = variable_scope.get_variable('test', initializer=2) sess.run(v.initializer) - # NOTE: We don't expect the assignment to execute in this case, because - # variables cannot be reliably guarded. - self.assertEqual(2, sess.run(result.test_fn(v))) + sess.run(result.test_fn(v)) + # TODO(mdan): Add support for this use case. + # Right now the variable `a` is not conditioned on the `assign` because + # there's no way to add control dependencies to a variable object. + self.assertEqual(2, sess.run(v)) def test_side_effect_on_used_variable(self): - tf = None - def test_fn(a): tf.assign(a, a + 1) return a + 1 - node = self.parse_and_analyze(test_fn, {}) - node = side_effect_guards.transform(node, self.ctx) + node, ctx = self.prepare(test_fn, {}) + node = side_effect_guards.transform(node, ctx) - with self.compiled(node, state_ops.assign) as result: - self.assertEqual(len(node.body[0].body), 1) + self.assertEqual(len(node.body[0].body), 1) + + with self.compiled(node, {}, state_ops.assign) as result: with self.test_session() as sess: - v = variables.Variable(2) + v = variable_scope.get_variable('test', initializer=2) sess.run(v.initializer) - # NOTE: Unlike test_side_effect_on_return_only_variable, the variable - # was used in the local scope and so we could catch the assign's side - # effect. - self.assertEqual(4, sess.run(result.test_fn(v))) + sess.run(result.test_fn(v)) + # TODO(mdan): Ensure the result of test_fn(v) is also deterministic. + # Right now it's 3 or 4 based on whether the read is synchronized. + self.assertEqual(3, sess.run(v)) def test_side_effect_on_tensor(self): - tf = None - def test_fn(a): tf.Assert(a > 0, ['expected in throw']) return a - node = self.parse_and_analyze(test_fn, {}) - node = side_effect_guards.transform(node, self.ctx) + node, ctx = self.prepare(test_fn, {}) + node = side_effect_guards.transform(node, ctx) - with self.compiled(node, control_flow_ops.Assert) as result: - self.assertEqual(len(node.body[0].body), 1) + self.assertEqual(len(node.body[0].body), 1) + + with self.compiled(node, {}, control_flow_ops.Assert) as result: with self.test_session() as sess: - # NOTE: In this case we can also capture the side effect because the - # argument is a tensor ans we can wrap it inside an identity. with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, 'expected in throw'): sess.run(result.test_fn(constant_op.constant(-1))) def test_multiline_block(self): - tf = None - def test_fn(a): - tf.assign(a, a + 1) + tf.assign_add(a, 1) b = a + 1 - tf.assign(a, b + 1) - c = b + 1 - d = c + 1 - return d + tf.assign_add(a, 1) + b += 1 + return b - node = self.parse_and_analyze(test_fn, {}) - node = side_effect_guards.transform(node, self.ctx) + node, ctx = self.prepare(test_fn, {}) + node = side_effect_guards.transform(node, ctx) - with self.compiled(node, state_ops.assign) as result: - self.assertEqual(len(node.body[0].body), 1) + self.assertEqual(len(node.body[0].body), 1) + + with self.compiled(node, {}, state_ops.assign_add) as result: with self.test_session() as sess: - v = variables.Variable(2) + v = variable_scope.get_variable('test', initializer=2) sess.run(v.initializer) - self.assertEqual(6, sess.run(result.test_fn(v))) + sess.run(result.test_fn(v)) + # TODO(mdan): Ensure the result of test_fn(v) is also deterministic. + self.assertEqual(4, sess.run(v)) def test_multiline_nested_block(self): - tf = None - def test_fn(a): with tf.name_scope('foo'): tf.assign(a, a + 1) b = a + 1 - c = b + 1 - d = c + 1 - return d + return b - node = self.parse_and_analyze(test_fn, {}) - node = side_effect_guards.transform(node, self.ctx) + node, ctx = self.prepare(test_fn, {}) + node = side_effect_guards.transform(node, ctx) - with self.compiled(node, state_ops.assign, ops.name_scope) as result: - self.assertEqual(len(node.body[0].body[0].body), 1) + self.assertEqual(len(node.body[0].body[0].body), 1) + + with self.compiled(node, {}, state_ops.assign, ops.name_scope) as result: with self.test_session() as sess: - v = variables.Variable(2) + v = variable_scope.get_variable('test', initializer=2) sess.run(v.initializer) - self.assertEqual(6, sess.run(result.test_fn(v))) + sess.run(result.test_fn(v)) + # TODO(mdan): Ensure the result of test_fn(v) is also deterministic. + self.assertEqual(3, sess.run(v)) def test_multiline_block_unsafe(self): - tf = None - def test_fn(a): tf.assign(a, a + 1) b = a + 1 - tf.assign(a, a + 1) + tf.assign_add(a, 1) c = b + 1 - d = c + 1 - return d + return c + + node, ctx = self.prepare(test_fn, {}) + node = side_effect_guards.transform(node, ctx) - node = self.parse_and_analyze(test_fn, {}) - node = side_effect_guards.transform(node, self.ctx) + self.assertEqual(len(node.body[0].body), 1) - with self.compiled(node, state_ops.assign) as result: - self.assertEqual(len(node.body[0].body), 1) + with self.compiled(node, {}, state_ops.assign, + state_ops.assign_add) as result: with self.test_session() as sess: - v = variables.Variable(2) + v = variable_scope.get_variable('test', initializer=2) sess.run(v.initializer) - # NOTE: This intentionally highlights the flakiness. The test should be - # tightened down once that is solved. - self.assertTrue(sess.run(result.test_fn(v)) in (6, 7)) + sess.run(result.test_fn(v)) + # TODO(mdan): Ensure the result of test_fn(v) is also deterministic. + self.assertEqual(4, sess.run(v)) if __name__ == '__main__': diff --git a/tensorflow/contrib/autograph/converters/single_return.py b/tensorflow/contrib/autograph/converters/single_return.py deleted file mode 100644 index 3b9c9a06d8..0000000000 --- a/tensorflow/contrib/autograph/converters/single_return.py +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Canonicalizes functions with multiple returns to use just one.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import gast - -from tensorflow.contrib.autograph.core import converter -from tensorflow.contrib.autograph.pyct import anno -from tensorflow.contrib.autograph.pyct import ast_util -from tensorflow.contrib.autograph.pyct import templates -from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno - - -# TODO(mdan): Move this logic into transformer_base. -class BodyVisitor(converter.Base): - """Walks breadth- or depth-first the list-of-nodes bodies of AST nodes.""" - - def __init__(self, ctx, depth_first=False): - super(BodyVisitor, self).__init__(ctx) - self.depth_first = depth_first - self.changes_made = False - - def visit_nodelist(self, nodelist): - for node in nodelist: - if isinstance(node, list): - node = self.visit_nodelist(node) - else: - node = self.generic_visit(node) - return nodelist - - def visit_If(self, node): - if self.depth_first: - node = self.generic_visit(node) - node.body = self.visit_nodelist(node.body) - node.orelse = self.visit_nodelist(node.orelse) - if not self.depth_first: - node = self.generic_visit(node) - return node - - def visit_For(self, node): - if self.depth_first: - node = self.generic_visit(node) - node.body = self.visit_nodelist(node.body) - node.orelse = self.visit_nodelist(node.orelse) - if not self.depth_first: - node = self.generic_visit(node) - return node - - def visit_While(self, node): - if self.depth_first: - node = self.generic_visit(node) - node.body = self.visit_nodelist(node.body) - node.orelse = self.visit_nodelist(node.orelse) - if not self.depth_first: - node = self.generic_visit(node) - return node - - def visit_Try(self, node): - if self.depth_first: - node = self.generic_visit(node) - node.body = self.visit_nodelist(node.body) - node.orelse = self.visit_nodelist(node.orelse) - node.finalbody = self.visit_nodelist(node.finalbody) - for i in range(len(node.handlers)): - node.handlers[i].body = self.visit_nodelist(node.handlers[i].body) - if not self.depth_first: - node = self.generic_visit(node) - return node - - def visit_With(self, node): - if self.depth_first: - node = self.generic_visit(node) - node.body = self.visit_nodelist(node.body) - if not self.depth_first: - node = self.generic_visit(node) - return node - - def visit_FunctionDef(self, node): - if self.depth_first: - node = self.generic_visit(node) - node.body = self.visit_nodelist(node.body) - self.generic_visit(node) - if not self.depth_first: - node = self.generic_visit(node) - return node - - -class FoldElse(BodyVisitor): - - def visit_nodelist(self, nodelist): - for i in range(len(nodelist)): - node = nodelist[i] - if isinstance(node, gast.If): - true_branch_returns = isinstance(node.body[-1], gast.Return) - false_branch_returns = len(node.orelse) and isinstance( - node.orelse[-1], gast.Return) - # If the last node in the if body is a return, - # then every line after this if statement effectively - # belongs in the else. - if true_branch_returns and not false_branch_returns: - for j in range(i + 1, len(nodelist)): - nodelist[i].orelse.append(ast_util.copy_clean(nodelist[j])) - if nodelist[i + 1:]: - self.changes_made = True - return nodelist[:i + 1] - elif not true_branch_returns and false_branch_returns: - for j in range(i + 1, len(nodelist)): - nodelist[i].body.append(ast_util.copy_clean(nodelist[j])) - if nodelist[i + 1:]: - self.changes_made = True - return nodelist[:i + 1] - elif true_branch_returns and false_branch_returns: - if nodelist[i + 1:]: - raise ValueError( - 'Unreachable code after conditional where both branches return.' - ) - return nodelist - elif isinstance(node, gast.Return) and nodelist[i + 1:]: - raise ValueError( - 'Cannot have statements after a return in the same basic block') - return nodelist - - -def contains_return(node): - for n in gast.walk(node): - if isinstance(n, gast.Return): - return True - return False - - -class LiftReturn(converter.Base): - """Move return statements out of If and With blocks.""" - - def __init__(self, ctx): - super(LiftReturn, self).__init__(ctx) - self.changes_made = False - self.common_return_name = None - - def visit_If(self, node): - # Depth-first traversal of if statements - node = self.generic_visit(node) - - # We check if both branches return, and if so, lift the return out of the - # conditional. We don't enforce that the true and false branches either - # both return or both do not, because FoldElse might move a return - # into a branch after this transform completes. FoldElse and LiftReturn - # are alternately run until the code reaches a fixed point. - true_branch_returns = isinstance(node.body[-1], gast.Return) - false_branch_returns = len(node.orelse) and isinstance( - node.orelse[-1], gast.Return) - if true_branch_returns and false_branch_returns: - node.body[-1] = templates.replace( - 'a = b', a=self.common_return_name, b=node.body[-1].value)[0] - node.orelse[-1] = templates.replace( - 'a = b', a=self.common_return_name, b=node.orelse[-1].value)[0] - return_node = templates.replace('return a', a=self.common_return_name)[0] - self.changes_made = True - return [node, return_node] - else: - return node - - def visit_With(self, node): - # Depth-first traversal of syntax - node = self.generic_visit(node) - - # If the with statement returns, lift the return - if isinstance(node.body[-1], gast.Return): - node.body[-1] = templates.replace( - 'a = b', a=self.common_return_name, b=node.body[-1].value)[0] - return_node = templates.replace('return a', a=self.common_return_name)[0] - node = self.generic_visit(node) - self.changes_made = True - return [node, return_node] - else: - return node - - def visit_FunctionDef(self, node): - # Ensure we're doing depth-first traversal - last_return_name = self.common_return_name - body_scope = anno.getanno(node, NodeAnno.BODY_SCOPE) - referenced_names = body_scope.referenced - self.common_return_name = self.ctx.namer.new_symbol('return_', - referenced_names) - node = self.generic_visit(node) - self.common_return_name = last_return_name - return node - - -class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor): - """Throws an error if code returns inside loops or try/except.""" - - # First, throw an error if we detect a return statement in a loop. - # TODO(alexbw): we need to learn to handle returns inside a loop, - # but don't currently have the TF constructs to do so (need something - # that looks vaguely like a goto). - - def __init__(self): - self.cant_return = False - super(DetectReturnInUnsupportedControlFlow, self).__init__() - - def visit_While(self, node): - self.cant_return = True - self.generic_visit(node) - self.cant_return = False - - def visit_For(self, node): - self.cant_return = True - self.generic_visit(node) - self.cant_return = False - - def visit_Return(self, node): - if self.cant_return: - raise ValueError( - '`return` statements are not supported in loops. ' - 'Try assigning to a variable in the while loop, and returning ' - 'outside of the loop') - - -class DetectReturnInConditional(gast.NodeVisitor): - """Assert that no return statements are present in conditionals.""" - - def __init__(self): - self.cant_return = False - super(DetectReturnInConditional, self).__init__() - - def visit_If(self, node): - self.cant_return = True - self.generic_visit(node) - self.cant_return = False - - def visit_Return(self, node): - if self.cant_return: - raise ValueError( - 'After transforms, a conditional contained a `return `statement, ' - 'which is not allowed. This is a bug, and should not happen.') - - -class DetectReturnInFunctionDef(gast.NodeVisitor): - - def visit_FunctionDef(self, node): - self.generic_visit(node) - if not contains_return(node): - raise ValueError( - 'Each function definition should contain at least one return.') - - -def transform(node, ctx): - """Ensure a function has only a single return. - - This transforms an AST node with multiple returns successively into containing - only a single return node. - There are a few restrictions on what we can handle: - - An AST being transformed must contain at least one return. - - No returns allowed in loops. We have to know the type of the return value, - and we currently don't have either a type inference system to discover it, - nor do we have a mechanism for late type binding in TensorFlow. - - After all transformations are finished, a Return node is not allowed inside - control flow. If we were unable to move a return outside of control flow, - this is an error. - - Args: - node: ast.AST - ctx: converter.EntityContext - - Returns: - new_node: an AST with a single return value - - Raises: - ValueError: if the AST is structured so that we can't perform the - transform. - """ - # Make sure that the function has at least one return statement - # TODO(alexbw): turning off this assertion for now -- - # we need to not require this in e.g. class constructors. - # DetectReturnInFunctionDef().visit(node) - - # Make sure there's no returns in unsupported locations (loops, try/except) - DetectReturnInUnsupportedControlFlow().visit(node) - - while True: - - # Try to lift all returns out of if statements and with blocks - lr = LiftReturn(ctx) - node = lr.visit(node) - changes_made = lr.changes_made - fe = FoldElse(ctx) - node = fe.visit(node) - changes_made = changes_made or fe.changes_made - - if not changes_made: - break - - # Make sure we've scrubbed all returns from conditionals - DetectReturnInConditional().visit(node) - - return node diff --git a/tensorflow/contrib/autograph/converters/single_return_test.py b/tensorflow/contrib/autograph/converters/single_return_test.py deleted file mode 100644 index 1f0de4310e..0000000000 --- a/tensorflow/contrib/autograph/converters/single_return_test.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for single_return module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.autograph.converters import single_return -from tensorflow.contrib.autograph.core import converter_testing -from tensorflow.python.framework.ops import name_scope -from tensorflow.python.platform import test - - -class SingleReturnTest(converter_testing.TestCase): - - def compiled_fn(self, test_fn, *args): - node = self.parse_and_analyze(test_fn, {}) - node = single_return.transform(node, self.ctx) - module = self.compiled(node, *args) - return module - - def test_noop(self): - # Noop - def test_fn(x): - return x - - with self.compiled_fn(test_fn) as result: - self.assertEqual(test_fn(2.0), result.test_fn(2.0)) - - def test_return_expression(self): - # ANF - def test_fn(x): - return x * x - - with self.compiled_fn(test_fn) as result: - x = 2 - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_merge(self): - # Simple merge - def test_fn(x): - if x > 0: - return x - else: - return x * x - - with self.compiled_fn(test_fn) as result: - for x in [-2, 2]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_orphan_branch(self): - - def test_fn(x): - if x > 0: - return x - - with self.assertRaises(ValueError): - self.compiled_fn(test_fn) - - def test_lift_body_into_false_branch(self): - - def test_fn(x): - if x > 0: - return x - return x * x - - with self.compiled_fn(test_fn) as result: - for x in [-2, 2]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_lift_body_into_true_branch(self): - - def test_fn(x): - if x < 0: - x *= x - else: - # TODO(alexbw): linter bug here that requires us suppress this warning. - return x # pylint: disable=undefined-loop-variable - return x - - with self.compiled_fn(test_fn) as result: - for x in [-2, 2]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_nested_if(self): - - def test_fn(x): - if x > 0: - if x < 5: - return x - else: - return x * x - else: - return x * x * x - - with self.compiled_fn(test_fn) as result: - for x in [-2, 2, 5]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_context_manager(self): - - def test_fn(x): - - with name_scope(''): - return x * x - - with self.compiled_fn(test_fn) as result: - result.name_scope = name_scope - for x in [-2, 2]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_context_manager_in_conditional(self): - - def test_fn(x): - if x > 0: - with name_scope(''): - return x * x - else: - return x - - with self.compiled_fn(test_fn, name_scope) as result: - result.name_scope = name_scope - for x in [-2, 2]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def text_conditional_in_context_manager(self): - - def test_fn(x): - with name_scope(''): - if x > 0: - return x * x - else: - return x - - with self.compiled_fn(test_fn) as result: - result.name_scope = name_scope - for x in [-2, 2]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_no_return(self): - - def test_fn(x): - x *= x - - with self.compiled_fn(test_fn) as result: - self.assertEqual(test_fn(2), result.test_fn(2)) - - def test_nested_functiondefs(self): - - def test_fn(x): - - def inner_fn(y): - if y > 0: - return y * y - else: - return y - - return inner_fn(x) - - with self.compiled_fn(test_fn) as result: - for x in [-2, 2]: - self.assertEqual(test_fn(x), result.test_fn(x)) - - def test_loop(self): - - def test_fn(x): - for _ in range(10): - return x - return x - - with self.assertRaises(ValueError): - self.compiled_fn(test_fn) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/autograph/converters/slices.py b/tensorflow/contrib/autograph/converters/slices.py index de04cc9184..9cfa066672 100644 --- a/tensorflow/contrib/autograph/converters/slices.py +++ b/tensorflow/contrib/autograph/converters/slices.py @@ -21,7 +21,7 @@ from __future__ import print_function import gast from tensorflow.contrib.autograph.core import converter -from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.lang import directives from tensorflow.contrib.autograph.pyct import templates @@ -63,9 +63,10 @@ class SliceTransformer(converter.Base): # also available. return node - dtype = anno.getanno( + dtype = self.get_definition_directive( node.value, - 'element_type', + directives.set_element_type, + 'dtype', default=templates.replace_as_expression('None')) template = """ diff --git a/tensorflow/contrib/autograph/converters/slices_test.py b/tensorflow/contrib/autograph/converters/slices_test.py index df9a4c8bab..3c0f81e8bc 100644 --- a/tensorflow/contrib/autograph/converters/slices_test.py +++ b/tensorflow/contrib/autograph/converters/slices_test.py @@ -18,9 +18,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.autograph import utils from tensorflow.contrib.autograph.converters import slices from tensorflow.contrib.autograph.core import converter_testing +from tensorflow.contrib.autograph.lang import directives +from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import parser +from tensorflow.contrib.autograph.pyct import transformer from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.ops import list_ops @@ -32,28 +35,42 @@ class SliceTest(converter_testing.TestCase): def test_index_access(self): def test_fn(l): - utils.set_element_type(l, dtypes.int32) return l[1] - node = self.parse_and_analyze( - test_fn, - { - 'utils': utils, - 'dtypes': dtypes - }, - include_type_analysis=True, - ) - node = slices.transform(node, self.ctx) - - with self.compiled(node, dtypes.int32) as result: - result.utils = utils - result.dtypes = dtypes + node, ctx = self.prepare(test_fn, {}) + def_, = anno.getanno(node.body[0].args.args[0], anno.Static.DEFINITIONS) + def_.directives[directives.set_element_type] = { + 'dtype': parser.parse_expression('tf.int32') + } + node = slices.transform(node, ctx) + + with self.compiled(node, {}, dtypes.int32) as result: with self.test_session() as sess: tl = list_ops.tensor_list_from_tensor( [1, 2], element_shape=constant_op.constant([], dtype=dtypes.int32)) y = result.test_fn(tl) self.assertEqual(2, sess.run(y)) + def test_index_access_multiple_definitions(self): + + def test_fn(l): + if l: + l = [] + return l[1] + + node, ctx = self.prepare(test_fn, {}) + def_, = anno.getanno(node.body[0].args.args[0], anno.Static.DEFINITIONS) + def_.directives[directives.set_element_type] = { + 'dtype': parser.parse_expression('tf.int32') + } + def_, = anno.getanno(node.body[0].body[0].body[0].targets[0], + anno.Static.DEFINITIONS) + def_.directives[directives.set_element_type] = { + 'dtype': parser.parse_expression('tf.float32') + } + with self.assertRaises(transformer.AutographParseError): + slices.transform(node, ctx) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/autograph/core/annos.py b/tensorflow/contrib/autograph/core/annos.py deleted file mode 100644 index b8937ce36a..0000000000 --- a/tensorflow/contrib/autograph/core/annos.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Annotations specific to AutoGraph.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from enum import Enum - - -class NoValue(Enum): - - def __repr__(self): - return self.name - - -class NodeAnno(NoValue): - """Additional annotations used by AutoGraph converters. - - These are in addition to the basic annotations declared in pyct/anno.py and - pyct/static_analysis/annos.py. - """ - - # The directives collection - see directives.py - DIRECTIVES = ( - 'Dict depicting static directive calls. See the directives converter.') diff --git a/tensorflow/contrib/autograph/core/converter_testing.py b/tensorflow/contrib/autograph/core/converter_testing.py index c47b70f15c..2025e32817 100644 --- a/tensorflow/contrib/autograph/core/converter_testing.py +++ b/tensorflow/contrib/autograph/core/converter_testing.py @@ -20,6 +20,9 @@ from __future__ import print_function import contextlib import imp +import sys + +import six from tensorflow.contrib.autograph import operators from tensorflow.contrib.autograph import utils @@ -29,11 +32,7 @@ from tensorflow.contrib.autograph.core import errors from tensorflow.contrib.autograph.pyct import compiler from tensorflow.contrib.autograph.pyct import parser from tensorflow.contrib.autograph.pyct import pretty_printer -from tensorflow.contrib.autograph.pyct import qual_names from tensorflow.contrib.autograph.pyct import transformer -from tensorflow.contrib.autograph.pyct.static_analysis import activity -from tensorflow.contrib.autograph.pyct.static_analysis import live_values -from tensorflow.contrib.autograph.pyct.static_analysis import type_info from tensorflow.python.platform import test @@ -75,7 +74,17 @@ class TestCase(test.TestCase): """Base class for unit tests in this module. Contains relevant utilities.""" @contextlib.contextmanager - def compiled(self, node, *symbols): + def assertPrints(self, expected_result): + try: + out_capturer = six.StringIO() + sys.stdout = out_capturer + yield + self.assertEqual(out_capturer.getvalue(), expected_result) + finally: + sys.stdout = sys.__stdout__ + + @contextlib.contextmanager + def compiled(self, node, namespace, *symbols): source = None self.dynamic_calls = [] @@ -93,6 +102,8 @@ class TestCase(test.TestCase): fake_ag.__dict__['rewrite_graph_construction_error'] = ( errors.rewrite_graph_construction_error) result.__dict__['ag__'] = fake_ag + for k, v in namespace.items(): + result.__dict__[k] = v yield result except Exception: # pylint:disable=broad-except if source is None: @@ -101,6 +112,13 @@ class TestCase(test.TestCase): print('Offending compiled code:\n%s' % source) raise + @contextlib.contextmanager + def converted(self, entity, converter_module, namespace, *tf_symbols): + node, ctx = self.prepare(entity, namespace) + node = converter_module.transform(node, ctx) + with self.compiled(node, namespace, *tf_symbols) as result: + yield result + def make_fake_mod(self, name, *symbols): fake_mod = imp.new_module(name) for s in symbols: @@ -117,17 +135,15 @@ class TestCase(test.TestCase): for k, v in ns.items(): setattr(module, k, v) - def parse_and_analyze(self, - test_fn, - namespace, - namer=None, - arg_types=None, - include_type_analysis=True, - owner_type=None, - recursive=True, - autograph_decorators=()): + def prepare(self, + test_fn, + namespace, + namer=None, + arg_types=None, + owner_type=None, + recursive=True, + autograph_decorators=()): node, source = parser.parse_entity(test_fn) - if namer is None: namer = FakeNamer() program_ctx = converter.ProgramContext( @@ -144,12 +160,5 @@ class TestCase(test.TestCase): arg_types=arg_types, owner_type=owner_type) ctx = converter.EntityContext(namer, entity_info, program_ctx) - - node = qual_names.resolve(node) - node = activity.resolve(node, entity_info) - node = live_values.resolve(node, entity_info, {}) - if include_type_analysis: - node = type_info.resolve(node, entity_info) - node = live_values.resolve(node, entity_info, {}) - self.ctx = ctx - return node + node = converter.standard_analysis(node, ctx, is_initial=True) + return node, ctx diff --git a/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb b/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb index a64e266f6a..18eb84cca9 100644 --- a/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb +++ b/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb @@ -15,18 +15,12 @@ }, "outputs": [], "source": [ - "from __future__ import absolute_import\n", - "from __future__ import division\n", - "from __future__ import print_function\n", - "\n", "import os\n", "import time\n", "\n", - "import tensorflow as tf\n", - "\n", - "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import six\n", + "import tensorflow as tf\n", "\n", "from tensorflow.contrib import autograph\n", "from tensorflow.contrib.eager.python import tfe\n", @@ -155,7 +149,7 @@ "# Test-only parameters. Test checks successful completion not correctness. \n", "burn_ins = 1\n", "trials = 1\n", - "max_steps = 2" + "max_steps = 2\n" ] }, { @@ -176,7 +170,7 @@ "#@test {\"skip\": true} \n", "burn_ins = 3\n", "trials = 10\n", - "max_steps = 500" + "max_steps = 500\n" ] }, { @@ -275,16 +269,16 @@ "def train(train_ds, test_ds, hp):\n", " m = mlp_model((28 * 28,))\n", " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", + "\n", " train_losses = []\n", - " train_losses = autograph.utils.set_element_type(train_losses, tf.float32)\n", " test_losses = []\n", - " test_losses = autograph.utils.set_element_type(test_losses, tf.float32)\n", " train_accuracies = []\n", - " train_accuracies = autograph.utils.set_element_type(train_accuracies,\n", - " tf.float32)\n", " test_accuracies = []\n", - " test_accuracies = autograph.utils.set_element_type(test_accuracies,\n", - " tf.float32)\n", + " autograph.set_element_type(train_losses, tf.float32)\n", + " autograph.set_element_type(test_losses, tf.float32)\n", + " autograph.set_element_type(train_accuracies, tf.float32)\n", + " autograph.set_element_type(test_accuracies, tf.float32)\n", + "\n", " i = tf.constant(0)\n", " while i \u003c hp.max_steps:\n", " train_x, train_y = get_next_batch(train_ds)\n", @@ -296,27 +290,28 @@ " test_losses.append(step_test_loss)\n", " train_accuracies.append(step_train_accuracy)\n", " test_accuracies.append(step_test_accuracy)\n", + "\n", " i += 1\n", - " return (autograph.stack(train_losses), autograph.stack(test_losses), autograph.stack(train_accuracies),\n", - " autograph.stack(test_accuracies))\n" + " return (autograph.stack(train_losses), autograph.stack(test_losses),\n", + " autograph.stack(train_accuracies), autograph.stack(test_accuracies))\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 10, "metadata": { "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, - "height": 789 + "height": 220 }, "colab_type": "code", "executionInfo": { - "elapsed": 11529, + "elapsed": 12896, "status": "ok", - "timestamp": 1531163743912, + "timestamp": 1531534784996, "user": { "displayName": "", "photoUrl": "", @@ -325,56 +320,25 @@ "user_tz": 240 }, "id": "K1m8TwOKjdNd", - "outputId": "59db8f19-23a5-413a-e9d0-fb756b0e4757" + "outputId": "2ee3ff78-9aae-4fac-a1fd-32bf3b2f18f4" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Duration: 0.592790126801\n", - "Duration: 0.594069957733\n", - "Duration: 0.591835975647\n", - "Duration: 0.592386007309\n", - "Duration: 0.595040082932\n", - "Duration: 0.594245910645\n", - "Duration: 0.624264001846\n", - "Duration: 0.6021900177\n", - "Duration: 0.592960119247\n", - "Duration: 0.599496841431\n", - "Mean duration: 0.597927904129 +/- 0.0093268291102\n" + "('Duration:', 0.7540969848632812)\n", + "('Duration:', 0.7829370498657227)\n", + "('Duration:', 0.7111489772796631)\n", + "('Duration:', 0.6126768589019775)\n", + "('Duration:', 0.6143529415130615)\n", + "('Duration:', 0.6174650192260742)\n", + "('Duration:', 0.6425611972808838)\n", + "('Duration:', 0.6188449859619141)\n", + "('Duration:', 0.6388339996337891)\n", + "('Duration:', 0.6235959529876709)\n", + "('Mean duration:', 0.66165139675140383, '+/-', 0.060382254849383483)\n" ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEcCAYAAAAydkhNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd8FGX+wPHPbMum90IKvQSQ3jtSbYCAqHee9TxPT0VF\njztRT+9UzvMOsdzPUxTO3gURsYsgTRBFmvROQkJ63T7z+2OS3Wx2EwIkBC7f9+vFi+zO7Mwzz84+\n33nKPKNomqYhhBBCBGFo7gQIIYQ4d0mQEEIIUScJEkIIIeokQUIIIUSdJEgIIYSokwQJIYQQdZIg\nIYQQok4SJISow6ZNm7j44oubOxknlZWVRWZmJqqqNndSxP8gCRLilI0ZM4YePXpQXFzs9/6UKVPI\nzMwkOzsbgD//+c9kZmaybds27zpHjhwhMzPT+/raa6/lgw8+8L5+4YUXGDt2LH379mX06NHMmjUL\ngMsuu4y+ffvSt29funXrRs+ePenTpw99+/ZlwYIFAWn897//zezZs8/oOPv3789nn312Sp958cUX\nmT9/Phs3bmTUqFFntP9qtfMoGEVRGmVfQtRmau4EiPNTeno6y5cv55prrgFgz549OBwOv8JKURRi\nYmJ4+umnWbhwod/7wSxZsoRly5bx6quvkp6eTkFBAStWrADgk08+8a537bXXcvnllzN9+vQzOgZN\n0xq9cF21ahX33XcfLpdLCm7xP0FqEuK0TJkyhSVLlnhfL1myhKlTpwasN3XqVHbv3s2mTZtOus3t\n27czfPhw0tPTAYiPj2fGjBlB161vNpnVq1fzwgsv8Omnn9KnTx8uv/xyQA8u8+fP51e/+hW9e/fm\n2LFjLF68mEsuuYS+ffsyfvx43n33Xe92atcGxowZw6JFi5g8eTIDBgxg1qxZOJ1O7/LS0lIOHz5M\nt27duOWWWzhx4oS3tpOXl4emaSxYsIDx48czePBg7rnnHkpLSwFwOp388Y9/ZNCgQQwYMIAZM2ZQ\nWFjI/Pnz+fHHH3n00Ufp27cvjz322Enz8cSJE9x2220MGjSIiRMn8v7773uXbd26lenTp9OvXz+G\nDx/OP/7xj3r3D1BeXs4DDzzA8OHDGTVqFE8//bQ3/48cOcK1115L//79GTJkiLfmJ/53SE1CnJZe\nvXqxdOlSDhw4QNu2bfn888956623mD9/vt96VquVW2+9laeeeoq33nrrpNt8/PHHSUpKYtCgQXTr\n1g2D4dSvY0aMGMGtt97KkSNHePLJJ/2WLVu2jJdeeol27dqhqirx8fEsWLCA9PR0Nm3axM0330zP\nnj3p2rUrEFjr+fzzz1m0aBEWi4Wrr76aJUuWcNVVVwGwZs0aBg8ejNVq5aWXXmL27NmsXLnS+9lX\nXnmFFStW8OabbxIbG8tjjz3GX//6V+bNm8eSJUsoLy9n9erVmM1mdu7cSUhICPfccw8//fQTU6ZM\n4YorrmjQ8c+aNYsuXbrw7LPPsn//fm688UYyMjIYPHgwc+fO5frrr2fy5MnYbDb27t0LUOf+AWbP\nnk1SUhLffPMNFRUV3HrrraSmpnLllVfyzDPPMHz4cF5//XWcTifbt28/5e9LnNukJiFO25QpU/jo\no49Yu3Yt7du3JykpKeh6V155JcePH2f16tX1bm/y5Mk89NBDrF27lmuvvZahQ4cG7W84E1OnTqVD\nhw4YDAZMJhOjRo3y1lz69+/PsGHD6q31XHfddSQkJBAVFcWFF17Izp07vctWrlxZbz/Ee++9x913\n301SUhJms5nbb7+dL774AlVVMZlMFBcXc/DgQRRFoVu3boSHh5/y8R0/fpzNmzdz3333YTabyczM\nZMaMGSxduhQAk8nEkSNHKCoqIjQ0lJ49e3rfD7b/goICVq9ezZw5cwgJCSEuLo7rr7+e5cuXez+X\nlZVFbm4uFouFvn37nnKaxblNahLitE2ePJnf/OY3HDt2jClTptS5nsVi4Q9/+APPPPMM8+bNq3eb\nl112GZdddhkej4evv/6ae++9l+7duzNs2LBGSXNKSorf61WrVvH8889z6NAhVFXFbrfTpUuXOj8f\nHx/v/Ts0NJS8vDxAb/5at24d999/f52fzc7O5o477vDWjjRNw2QykZ+fz5QpU8jJyWHWrFmUlZUx\nadIkZs2ahdFoPKXjy8vLIzo6mtDQUO97qamp7NixA4C5c+fyzDPPcPHFF5ORkcHtt9/O6NGjA/Y/\nefJk7rnnHrKysnC73QwfPtybZk3TaNWqFaDXMp5++mmuuOIKYmJiuOGGG864r0icWyRIiNOWmppK\nWloa3333HXPnzq133WnTpvHyyy/z1VdfNWjbRqORiRMnsmDBAvbu3dtoQaJm85HT6eSuu+7in//8\nJ2PHjsVgMHD77bfX299Rl23btpGenk5sbGzAfqq1atWKuXPn0qdPn6DbuP3227n99tvJzs7md7/7\nHe3bt2f69Omn1AGelJRESUkJlZWVhIWFAXrtorqW17p1a2+g/uKLL5g5cyYbN27EarUG7L9du3aM\nHDmSkJAQNmzYEDQd8fHxPProowD8+OOP3HjjjQwcOJCMjIwGp1mc26S5SZyRuXPn8uqrr2K1Wutd\nz2g0cscdd/DSSy/Vuc6SJUtYtWoVFRUVaJrGqlWr2L9/v7dJ5FQkJCSQlZVVb4HvcrlwuVzExsZi\nMBhYtWoVa9euPeV9gd7UNHLkSO/r+Ph4iouLKS8v97531VVX8dRTT3mHCBcWFvLNN98AsGHDBvbs\n2YOqqoSFhWEymby1iISEBI4ePVrv/quPMyUlhT59+vDUU0/hdDrZtWsXH3zwAZMnTwbg448/9nZI\nR0ZGoigKBoOhzv0nJiYybNgw5s6dS3l5OZqmcfToUX744QdA76PJzc0FICoqCoPBcFr9SOLc1aQ1\niTlz5rBy5Uri4+NZtmyZ9/3XX3+dN998E7PZzKhRo7jvvvuaMhmikdW8oqx9xVjfVe9ll13GggUL\nKCsrC7p+REQEL7zwAgcOHMDj8ZCamsojjzwS0M7dkCvriy66iI8//phBgwaRnp7O4sWLAz4XHh7O\nAw88wF133YXL5eLCCy9k7NixdW6zvv2uWrWKv/3tb97X7du359JLL2Xs2LFomsby5cu5/vrrAbjp\nppvIy8sjPj6eiy++mLFjx5Kfn8/DDz9Mbm4u4eHhXHLJJd6C/brrruNPf/oT77zzDpMnT+aBBx6o\nN23z5s3j4YcfZsSIEURHR3PXXXcxZMgQQB/59cQTT2C320lLS2P+/PlYLJZ69/+Pf/yDf/3rX1x6\n6aVUVlaSkZHBzTffDOg1qOoAkpCQwAMPPEBaWlq93404vyhN+WS6TZs2ER4ezuzZs71BYsOGDbz4\n4ossWLAAk8lEYWEhcXFxTZUEIZpcQUEBl19++Uk75oU4HzVpvbB///5ERUX5vff222/zu9/9DpNJ\nr8RIgBDnu7Kysno7rIU4n531xsNDhw6xadMmrrzySq699lq/KRuEOB+1bduWSy65pLmTIUSTOOuj\nmzweD6Wlpbz33nts3bqVu+++29t5J4QQ4txy1msSKSkpTJgwAYCePXtiMBgoKio66eeasOtECCFE\nHZq8JlG7cB83bhzr169nwIABHDx4ELfb7R1bXh9FUcjLKzvpei1BYmKk5EUVyQsfyQsfyQufxMTI\nM/p8kwaJe++9lw0bNlBcXMzo0aO58847mT59Ovfffz+TJk3CbDZ7JxgTQghx7mnSIbCNTa4MdHKV\n5CN54SN54SN54XOmNQm5NVIIIUSdJEgIIYSokwQJIYQQdZIgIYQQok4SJM4zTo+LVza/T7GjpFG3\nuyn3Z37I2dyo22xqdredV356jzJn+clXPgUbjv/ITye2Nuo2m1qlq5JXfnqPSldlo253bfYGtubt\naNRtNrUyZzmvbH4fu9veqNv97tg6dhTsbtRtng8kSJxndhXu4dM9K1ifffJnRp+K/+54i1d+ebtR\nt9nUtufv5NO937Ix56dG3e5rO99l4fY3GnWbTe3nvO18uvdbfjyxpdG2qWoqb+36kBe3vdpo2zwb\nfszdwqd7VrDlFIJbeXk5S5Z8UOdyl8fFu3s+4vktCwOWzZ59NxUVDb9QWbRoAe+8c/6cXxIkzjO2\nqqujEmdpo23To3q8f2eX57CnaD+bT2zD5razLf+XRttPY7N5qvLC0Xh5YXc7vH8fLctmT9F+tuRt\np9JVyY6CXY22n8bmPS8aMS8qatRKjpQeY0/Rfrbl/0K5s4KdhXsabT+N7XTyoqyslCVL3g+6TFVV\nSp2+4bQHS46wt2g/2/N3UuIo47d/voPw8IgzS/Q5TJ5Md55xePRCrDELgzKX7yro8Y1PBSy/reeN\nXJDQtdH211iqC/STBUxN0xr8dLfSGtt64oenA5bf0/c2Osa0O4VUnh32Jjgvam7rH5ueDVh+/4C7\nSY9MbbT9NRbvb+QULqReeOHfZGdncdNN19C//yCGDBnGf//7EvHxCezbt4e//vtJDr69FVeJg9+5\nryVhSAbx/fRj3/nUOhYufJ1Qzcp9982kR4/ebN++hcTEZJ54Yh4Wi6XO/e7du5t//esJHA4HaWlp\n3H//w0RERPD++++wdOliTCYTbdu245FHHmfz5h959tl5Veeywv/930t+j6ltKhIkzpCmaVS6bYSb\nw87K/uwN+AFUuioJNYX6P6rT40QDQoyBJ+zJCpYiR3GD0na28sLudLN8/WGUVjag/vS//90vfLbu\nOM/eNZKIUDMADo8TBbAEzYv6b8CqeUVZH1VTsbvthDVSXmiaRoXd7T2G2nwFY93pq3BVBnw3drcD\ng2LAYgzc7skK2XJXxcmSDVTnhYMwc8MKtPdW7OOHXScatG4wFW4LDvcoVmwzs/HrdQAMyEziyjEd\nfevUyovbbruTAwf38+LLr2I2mNi8+Ud27vyF119/j+jYBP67bgWtL++KMdSM6vKw98VNRHdLxBRq\nBgUq3TZCjVaOHTvKX//6d/70pwe4/e57eP29pfz2NzP88sKtur2vH3vsEWbN+hO9evVm4cIX+e9/\nF3DnnbN4881X+eCDZZhMJm9T1jvvvMG99/6ZCy7oid1urzf4NCZpbjpDa7K/Z/bqR065KeLjNQf5\neV/+Ke/P4XECUFpHYXao9Ah/XP0Inx9a4ff+X7//J/ev+Rvf/nSM1Vuy/ZadLEgYGnCafL8jh/9b\ns5TZqx9hX/HBk65/Jj5YuZ/l6w+zeV8OUHfBvadoHyvdr2BMPszOw75JJB9c+zgPrw8+HczJCsaa\nP/D6fHrwK/64+hGOlB1r0Pons2brcWY+s7rOc8ZRVasqDfJdbj9YwIsrVzB79SOsy97ot+yPqx9m\nbpDaI5w8YFYHppNZvO8T/rj6YXIqcimtcFJS3rDPna7qSSTUOiaT2JSzmdmrH+HH3J+976mayvGK\nHP616d/e97p1605KSgqb9+ax+VAWeeuPsvv5jex96UdcpQ6cBbaqHYLNpTdxpbRKJTw2FVXTyCqP\n5ONv/fuI3t71IV8c/pZKVyUVFeVUVJTTq1dvAC666FJ+/lkfPNKxYyceeeQBvvzyMwwG/TG2PXr0\n4tlnn+KDD96hrKz0rD0mVmoSp2HBxzvYe6yEf/5hqLcw/jF3C93jMxv0+bJKJx+t0QvSRX8ec0r7\ndtRoYlE1lez8SkLMRr79KYvIcDPuhJ0AfHLwCy5u53sUZ/VoqDdW/ozmDGNEr1Q278nj25+z6DOo\n/hExFQ0YMbNg2S9Y+3+PYoCteTvqbJL5YuMRsvMruOHizAY3AdWWX2KvSpcNrHUHuR9z9R+oOW0/\n+SU27/uV7uoaSBnRIf5TFgQrZGtqSF4AfHZIn/5+V+FeWkemN+gz9fl84xEA3vxyN+1aRREd7n8V\nWV3DLA4S5J56dwuWjpsxxsFnB74lM6IncVFW3KobVVPJsxVQ6bIFXOmXniRgNjQvvj26BoADJYd5\n+TW9M7m+8/7KMR39rvpP1YKtr7Ilfwfx1jj+NvTPActXZ30PwMpj6+iXrBfQDrd+8XWsPNt7IWa1\nWvn+lxxe/mQnDsMhyg8W0emW/hhMBvYt+gnV7evLq3BV6udihYc5C77n4RsGgKKA6vFr7lx3XH82\neHFVAK5rVqR//vMZfv75J9asWcUrr7zMG2+8z29+cwNDh45g/fo1/P73N/L008/TunWb086nhpKa\nxClwq26e2byAH078SEGpHYfT472aCjGGeNf75sh3vLj1Vb8TQFU13B6VnIpc/rX5GZRQ31Xatz8d\n4+/vr+TR7+dxsOQIa7cdx+HST8D/W7KNj1Yf8K5bXRiomkqFq5K/vPYdf1n9L778ZQuLt6zh88O+\nGsTS/Z+xL6uEQ8d9P3Zr7+8wZeyiuNzBc4u3sf1AIUeK6q/RbMr9mbkb5/sNNV26/zNe/+U9lu7/\njEXb3gJAMah6Xph8efHujs94euN/ASgstfPuin2s3nqcrPwKjpZl8/iGpzhRmee3v6Nl2Ty2YR7Z\n5Tne93KLKtm63z+dmuL25ond7aDAVsjcjfM5UnqM9cc3sSZ7g54uk4tNRWsB/5rAnLWPsmz/537b\nDFbI1rT++A/8bd1T/HI01/veB3s+5u3di/lw7zLe3Ok/QsZa47xYduALXv/lvaDbPVhymMc3PEWB\nzVfj+WTdIf65dAWPbZiH26TnfUGpg4cXbQz4fHXBVu6swKN6yK04weMbniK7PAdj4hGMcXp6C50F\nzPnoTUBvgqz2x9UP89lB/+e6nKyG+d2xdTy56Tm/zv63d33Ih3uX8fbuxby3Z6nf+jWbOt/asZS3\ndy8Out09RfuYu3F+wP73Fu3n8Q1PUWiv/9EC3tq2sxRVVckqP151nuWz4sh37CvRL9AOlBxi5VH9\nvDBZTagO/Td376qHvKPElq87DIDqKccYasZgMmDPq6DymH/aVmWt48Utr+Fw6efXL4cKMcYfxxBV\nwKJt7/DRvk/91jcaDISHRxAVFcX8T57jw73L+OKLT+ndW3+ee25uDn369GPUlRPILTpBXkk+WVnH\naNeuPddccz2p7TKY99WzJ63tNQapSZyCnIoT7Cnah6U92PLTKat0en8gZoOvTXfxvk8Avc020hKB\nzeHmLws3UlLhoP3wneQ78jC3dePcOQiA17/cQ0i39RgqS3hr2yfsX5OJ060yuFsyP+7O48fdeVw6\npC1mk4Gf9h2HcH0/eeXFmNP2Ywgvw9JpM4rZ6ZfeLw9/y9KNIaCohA7wvW9udYgdBwu9r/fmnIB6\nmjezKo4DekHWM7G7d9t+DON8f6tGlq8/xMSBrfkuV1+v0mnn281ZvjxadYATSV9S5D7Bkn2f8vue\n13uXvbztNfLthXx5+Ftu6P4rAF77fDe7jhTx9J3Dvc0ImsFX4Jc6S1m6/3Oyyo/z6s53yanwFeIA\nOZaf0bRf+RWMAJ8fXsGkDhfhcquYjArlzvrb2bPK9bx4atkqXv7DlWiaxrfH1vitM639ZO/fRsXo\n21dV7eKarlfgdmuYTAYMVVeYL29/g2JHCV8c/oZfZ16h59F3B7D2/Rqlwo0lej/kdtKPtcL/GMBX\nw9TQKHOV89buD8muyOHdPUuwtPNv/jOm70ZVNW9hWu2Tg18wvvWFAJiMBspO0udwtFxvtnzmkzUc\nO2TmsVsGegNztRmdfHlRZvPtb22uXjhf3XlqQI3y+S2LcKluvj26hovbTCTErOfh05tfBGD98U1c\n2m48ALuPFHGi2MaInr4O9OoLKZfqZua/vyWmzyaKXAUs/Gkxx5z7/Pb1/t6lDE8dginUQljraHb/\n3wYiO8VT1qmQeHcYefl6HkR2TKR4j8bu5zcSkhBGeEaNxzIr+nmhOt1g0M/N3ccKMISXgtnDT/l6\nE1JbbaD3I9UjCufMeYTbH7oV1a1iNbbnuXlP4Ha7+dvfHqKiooKs8mzihqSyrWwnP727nq9WriUq\nwoojvpLWrbuy5siPXNppNJt2nWDBsh08evMgkmMbt09QgsQpcKo1flRmOw98thBTsn5SHMwug05Q\nXKO99VD+CY45NnAiK4SC0qorPbu+XDHbMbfZQZG9n/66qmbhcIIhooifStYRltMdU9pe3Fkd2X20\niHatoqh02akudnLKCqGqoNRUI8Eab4xJhzHGBnYCHsotxZSxG09eGnllpZjiT378Jc5SPKqHd/d8\nFLDM0vlH79/rtmdxfKf/8v15eazKWYkhIhq1PI6f9+UTYq3EEAZF9iLe3r2YSe0mEm4OI9+uB7Cs\nvEp2F+7jYMkR9pcWYmxVRG5RX28h6cHlS5ujzFerMwSPeB/t/oYDFYFDNw8cL+KJFW9zUYcRVIQ2\nrAlFMTsoLKvk2R9eC1h2z/JnMMbofwdrt88qLuDR5R8S7kjjst59GNsv3VvDyass4I1fPmRC+jhA\nQzFVfb9oGKLzUKwVaPZw5q/8kLEZo+nZQf/i7DX2U+Io9QaNEEMIwXy460t2lwQOb571/CqsbfZx\n54gplDkaNvZ/b24uqjOR//z8SsCy/+54y/v3m9/8ArT2W17hquTrI6vol9yLjMg0QC/cAbZnH+az\nH/7Db/tNZc+RUm+7h8vjYmveDoocJbz31TGchnKslssYkJkUkBc2tQJPeQWGEDiUXYkpITD9d727\nEEvCCdpc0d3v/ZFdb+azQ1/iyu6AMcRD+2t7BT3+rvcM1f8IM9P+sutRy9zsMn1B0jD/Y/3Pj29h\nSoKUC9sxrKsejDt27ESnW/rrad00jle2f8rvYy7i+edfBuD2FbMBOFh0lMjR7chQ+4ECof2/AuBY\nYREbsn7m5bVb0KLMvPnTl8wae3nQdJ4uCRKnoGYbrDltH6YkX6dkYUUFJ4pt/PmF9YRWXTA89+UK\nzK33YHEkAnowsLtcYAKD1YbBepR3dn0EhlQUo95UU+YuIaTbIQ6ocOLQLsxpZaglCRzMLtVHthh9\n7aB5FUUo1a89vivWmixtdwZ9/5eyzZhbHcQYdxzNHt6g4/9h3xGiLJGsrXW1CGCM8jUBFJTrV18H\nskuh6nlSH21ZiydxN0nx6VT8kkxZpQsUPcAeLc/maHk2HtXD5R18z4o+WnKCZ39eAICSEYrZamNP\n3lGKyvRCwK25vO2lr3+zBWua/v3UbO6q6evsL4O+P3/VYsytDvFVXgGt4qKCrlObYnHw+sZvyVMO\nBCwzxviaz9b+coztG7fSKsF3dffNgQ2YUw9gK6zgza/CGd6jlXfZnuL9wH427shHsfj6MjymckK6\n6PtS7WHsUyvZvszC87dfQojZSIXTd3dxsaOU/LJyUPxruDWtzAn+yGBH/C94Ig7z+MpFGMwulAZc\nlCoWB8bEoxy1B+ZFzZv7vOcqqve99cd/4KsjKylxljI67lKS43z9IsfdBzElw8vfL8dTmIK1p/7+\nCVs+Xx1ZqW+pVSgWq421u7oxIDOJVT9nkVdS5i3ZFIsDxVhV4/TUUdyl7CWwbgafHfsUU0o2isUB\nJleQNQIpZgemlEN+v4dqNcuL6lpczYsIU9IR8i17WHUskl9lTvf77M8F+gwAxqSuqGUx3vcLHYW8\ntns9lnagOqzsx065czwRlob9phvC+MgjjzzSaFtrYpWVwb7Ks+dAyWG25usdb4Zw/zZJm1bGVxty\nUEwuTAl6NVwJsaGYXHg0D1HGeLSoXNyR2WDw/UgqXXYcDg1jjN7e7tZcKFVVVqemH6/mslBuc7P8\nl42Ya5xoRwrz8eBGsThQ7WEYQho+DYGt3IwSVqIX1KoBxXLyvM0vUNlS9JNfM08wSkglmtuMS7Hh\njDwKQKm7BMXkIizESNu4ZE549BpO9bGCHkC//fkITqte81EMqrdgqb6iLiyAwjIbhpgTmBJ9zVfl\n7grsaiWqwYmzPAynoeFj5F12M4awcjTViN3lRDOePC80l4Vi6x4weOpdr9RVQk6ek30ncjDF630s\n+bZCPIoTNAXNGcIvBbsoVI56gyaAR1XB6PYWNqqmevNAqSqwNEcoTs3OcechNuf7RuqU2MvJrSxA\nMahY3LGUeAoanBeax4jBagO3BUxO7z7rz4sQTMlH/L7LYBSLHTx6Xlf3kezNO45mcIJqYOmqY3y+\ncxPG6Nrp1VDMToyR+lBsg6J4h996a1r2cBIS4J0f1uKO9J0XitGNIVyvpau2CO/fDaPvV3OEYbBW\nevvc6mM1hKElHORkYzKK7MWEmUM5VHrUOzKyurywGq2YFCM7Cnazq2hvwGcNFjuGCH0gSnGpSw9i\nVOWFAqnhKWSX5fHV7p8IDzWSEZ9yCsccSB46dAq+PrKKJfuWn/X9qg4ritlR74/Q6orHbm54YeAp\niccYXYBqCwODekoBpjmp9lCUEDuKUndeeMpivAVKQ1Sv7ymLwWCtQDE37KrxbNI0Agoe1R6GElJZ\nb4HkKY0NelVbF9UWjiG0Ak9REoaofG8N95yiKX4BFcDgCkc119+Hcqp5obnMKGYX7rw0jAlZJy34\nm4OmKgHlQrgSQ4Wmn/9RxlhevmLuGe1DRjedgoYO+atLvJKBY08f7FtH0J0JeAqT/ZarDmvQzxlC\n7CgGDdVWdxXSHO7f9q1WBD6NynW8LVHGOAAUa9WxeMz1Bgj7z6Ow/zwKrUZzln3bUFxHO3tfR+YN\nDPbRevVI6EYXzzguib0ex56+eIoS/ZZrzuBNRgarDUXRSAlLqnPbhhD/7ynCEBOwjiu7Pao9tGqb\nVeurxnoDhG3zaD0vavwm7VuH48rq4H2tHAvebl0fd0GKngc7RhJybBCeEv8OIs0ZErSAMlj1AFHf\neWGw+heccdbA58m7jnX05rdiqToXFK3eAGH/eRT2LSP939s6HFe2b+iz89Cp36Xvzk/Fsacv9q3D\n6eIZj6fUP72aMyQgQADeAFH9nQYTHu3/G9GcgX1XriNd0NxmvfCtOhcUs7PeAGH/eSSGPaP93vvL\n4D8yMm2I97Vy/DTy4kS6Ny8ce/qilvs3haoOa9ALx+oA4cpqT6ZnwinvtzYJEqegooF3mNZU86Tt\nndaRSHcGwzp15IL4bqiV/gW5pyBwigPV7msUbu0aVOd+as+EqlZEB6wzoFM6mQn6j9gQot8rUF1t\nrYvmDNWX+CEEAAAgAElEQVT/ufUflKcwGc0WhafYV6ifONywdvwQzXe87aPbMHP8BC7t050h6T1R\nbf5z37jz0wI+H2uJ8/49o/OUOvdTu+ksxRqYr5rL4s2j6lFhgc0ctbisaM5QcJur0tgKzR7BbWP1\nTkjNY6TyRFx9W/AyeXzHO65rT4a36Y2zIozi7FjC8Q9qwfIiMdQXSK7ObHheVHcO16S5Qogx6kG3\nunmvZr9KMJozFM0Rhqbqpaf7RDqaPQK1TC/UFbcVQ3lyfZvwqvkbUUvjUIuT0OwRzBw/HrPH/zcS\nLC9q/kZcR+oujO2a/8WDWhl43mquENTKSL/C9+R5EUZFse+i5sKM4SSHJdI+uq2+H1s4tvyGnRf+\neRFPa2sHnrxxAjFqBqrD/2LAk19/eeHOaUduzplXf5o0SMyZM4ehQ4cyadKkgGULFy4kMzOT4uKG\nNws0txK7HiSc+3riPHBBwHJ3rm80g0kxMrHNGK7v6RtpkBgew5O3DuWGSzIZkJmEBf9eQa0yArVC\nP3E1txlrcRfu7nsLIYVdcB7O5OpBQ7ilx3WMib7K70q+pgszhhObeyFakE66CIsVax2dusE4dumj\nLtISw/l1l2l0CumD65g+DFOzReI82B37lhGgBu7LneO7ycegmXBldaBnuO/KKtri+4EO79kKzeWf\nLrU8mhBVL8Q1p4XBCcO4p9/vmdhmDFd1nkpmXCda20Zh3zGE0OIuQdPvym6HfcdgIq1Bri5VI3ER\nDe/cc+z0jSF2HuhBO1MvrulxGXfP6EnvVp0w5/TAsW243pZfy7jWo3wvPGZcWR3I0Hp73+qYlEy/\nLr6g2ybBv1allsZ5a5kxIdFc1HYsd/a+hQltLuSazCsY1aE3N3b7FTd1vhV3bkbQ9F/Udix/GjAz\n6LQsqEZaxTb8OciOX3wXK7+94BouTB+BO1uvTd13yQRMORdwRer1PH3b2IDPTmzju4lOc1kYlz6W\nXtG+mmiUJZKkmFCuv0j/Tnu39Q8KanECmks/BtVhxZXVAeeuAbiy2+Hc34P/u2kGkzMux75tGO78\nVgRzabvx3D/gbvp2DGyr1zzGOgeBBGPfMbjqLwXnvl70iBjo/b77JffCmNMdx85BaM7AVoKaNVDV\nYcV1tBOeE74y5OaL+vDAdf2Ij7bStXVsQO26Z2J3NLf+21PtYSQ5ejHAPEnPi329MCsh5BbZOFNN\nGiSmTZvGwoWBU+vm5OSwbt06UlPPvcnB6lNaNSTQU5jCr/sF/gA8RUmYNP1kGNyqP5M7XESneF9h\nGW2Jwlw1Nj40xMTYnv53lY7r0554RV/ffbwdtw28gi4pqTw44RpmjppCu1bR9Eq8gFsvGs2Q1L4B\n+7cYLUzpcAndEzuCGniiR4eFBi8k6qCWJtA5PZpHfzuIEe17cfewX9Exwfej9eRloDmCF7Ttw3wF\n99jWI7ip7xTGd/cNMYwO8QWJLq1juWxArbvV3RZaW/Uf0YiUkVzbcwrxobFM7nARI9P1YHPjsDG0\nj23NNf0D7941ahbcxzqhVcQQExaYxoGdU+nRNjHg/eAU1DLflbtakkTfiFGM7NqJnh0SUBQFa2lH\nvZYRZCBy/2RfQBiRPJJ0T18m9KiZF5H0aB/PA9f2487pPRjQwX/opOa2oBbrV/qXthvPpPYTiQ+N\nZUqHixmaqhew/VP6MDyzG+68wLu7oy1RXNZuAq0j0/1u7vPyGIkNb9jYejNW1HJfE1C/lJ5c0XlS\n1bFDZps4nvn1dYy+oAOh5sB9DWrVz/t3iqMPUztPJMnqazq8uF9nnrh1CKN66+dZp2T/gtyshGK2\n6e+5j3bGndUJzRmK+1gXopztCbEYubDtIDRbJJ4TgQEzMTSei9uOIz0ylVCTr+D22N3kbzwGqpGo\nBk6al2hNJC1Mz++iA2tw5SUwIX0CMSH6xY1BMeDJbcvR1Yvo0yrwvHDXSJ/7aGfcxzv41ajbJSRi\nrJp646qxnRjS2f/u6mtG98BToo/p7RcxkocvvoabxvfjotYTGNiqNw9e15/fXdatQcdSnyYNEv37\n9ycqKrBKN3fuXGbPnt2Uu24Sdre9KnIb6N0xAeeBC3Adb+tdrrlCvEMOnarenhlr9TUd1CwYAdKi\n/augQ7q05t5xk8iwdqBP0gW0T9XXj4uy0qO9fzt1sKvjtlGtMRtMXD6iHWN6tw1YnhQdGXQiN4BB\nyb4r5R4J3fDs0a8WQ0P8awmpCXqB2yY5kumj2gNgtRgZETfB7yqxf4cMbz+GBzeDu6eQFOYbpB5l\n8b9ybZ/oX2DfNLEXv+47ngviM5nQZQDBJMWE8vQ9o2mbFNjOnmpNp/r0DgtSWA3skkqYOXgfUBK+\nK7wL4rsywBTYnFM7Xyod+iibET1bcUWnyd6bvQCiatSaYqPM/OWGAXRKTA1Y3iEtmj6dEompdZ7E\nhUXgzm1N97jMeqd+iQg1c8flvQPe7xjTznvDWkiQIHHLZb2wBAsewAU19ucpTmRm35v58zV9uWRw\nG349rpN32dSR7bk6yFQal3e4xG9Yc3SN731YD72wbxXh++7jQ/2bSaNqTZvisBm5od/FhDvTCPf4\n1xTiIvVjsFTdfKcFuVDqUDMvatSqPTYXBRuzQDXSvU3w/q4orSpgadA9PpPf9fwNf/vtQB67eRCu\nnO/p3ymW1sn+zaa3XX4BUWEWRvdJ49J245na4TLvsh6tfenXtKo01xiOXvO8iQg1k5nu33wXbg7D\nfbwdnqIkojTftqaN7MDvJnUnIymCzDaBv41Tddbvk1ixYgWtWrWiS5fgTQTnMpvHjuYxcdnQNsRG\nhnD7qEvILark43J9UrBhXdpy1LoHm60Mp0cPEgbFF4drF4yt4xOgxs2wVpOVWGsMfx76+5OmRfUE\nxvekqnbq0BATaXHRUKsp1WwwB5351J2bQaeu7diQq88rc2vPG3js500cKC7lRLF/dTU1Xj+JI8LM\nXDSoNf0zk/zu8PyialqQoZltWZZnxoXHe5ezuUaAql0QJkf6t8P3aJNCpCWC23rdVHcmVAl2TB0T\n0zhgVOjRPh6rMbBJM8RkCRowx2SMQHWGcCJ3PwC39bqRNVuP8x07SY4LI7ewsupY/PPfVhUkwqwm\nLswYDsDyg/oNT5E1xqxXXzyEmnxBPrrWeRFaK3g98KuhoBmJiTh5U2HbpFjwv6mYpDBfIRwsSMRH\nhpNlD8yLi9qOxaN62F6wC0014NzTj/bTWkMMdM7w/74mDW0bND3j24wG4KP9+rQU1hpX7waT3u6f\nFOEryOLC/M8Lk1KroHeb6ZPRgT4Zd1FYaueTdYcoLHOwdX8B7VNr9cMFCRLJfnnhO2+Of70fZ5GN\nrG9e5aeSNjAklBNrjlC8IxfNozFw6BC6TEjj+2NZZL2/i0r1GBvUL7n++pspLMynoqyIH5bPZ+/a\nGJ555j/e7V7QLp7UhHAsJiOXtBvPV199zu4X9PuMLhiXAfGgqRrH132GpaKcUKuJskojiUMyWLZk\nid904Rf/fprfsViMFq4Y2Jf3V0YzaNiZzw9Wl7MaJOx2Oy+88AKLFi3yvncejcDFoerjvKunCejd\nSb8yzv2lH9sLdnLjhT35pdDK81sW+rVDD08dxPaCXQFBIikyBqshDLtaidUYQmxIYGdzXRKjwqFW\n2RcT4vvhBmtW0DQ1aHNT/05paJr/SJZRvVI5kF1KXJR/gZWaqBd4kaFmjAZDwBQAvRN7cLj0KKHm\nEG7ocSUvbXuNETVGeQxI7sPBksN+hSRArDWWMFMolW4bEebwU5pu3BLkhrGksASeu2sgRqPiDX41\nGRVj0OASbg7HYFGgxqweQy9IoazSyYCuScz+z3oAbE7/+wc6pkWz+2gxrZN833G3uC4U2IswKAau\n73Y1r/7yDgNTfM2EvRIvIK8y3y94AqSEJWMxWnB6nMSERBMdFtrgyRCDHVPNGlywPqkQoyXoeRFu\nDvNODKmgMLBr3SPKTqZDdDtcVQHy6i7TeGf3Ynom6E1ukeEWPMUJ+n00VjOL933C5hPbAPzOS01V\nCOm1iofW1Zi7KgrUCI3YeA87Qow8tE7Pp5ThGk63G1utAVo1A2bN30ir8R2wn6jgv68uZEPuJj78\n5kMchZV0/v0ANE3jxNLjxOxPpPRoAdaYMP47T7+TvLKygrCwcN59922ee+7FoC0n1fLz83nhhX8z\n8s5LiYyIZNfrW+macgE/VmynbXQoz73yKgZF4R9rn8YaFsabz/pPF+4xaygoaGjeAQgXDWrNhX3T\nsFqarig/q0HiyJEjZGVlMWXKFDRNIzc3l+nTp/P+++8TH3/yeSESExveudbYVE3FqTrQPGHExYb5\npeXeUTd7/05K6s/ozP5+n52ZeEOd231txrzTSs8V4zP5oNaDtNITEr3piqoMbI4Kj7R4O4Nr6tI6\nkXCr7weTmBjJ1LGdiYsNo0fHBOKjfdsaGGml87pDDOuTHvT7mDPmD96/xycOYXy3IX7L/zj6ljqP\n6ZXpwaesPplWyYFV6tZJyaSn6UEzrDwwiERFhxBvCPxBJ8fGUOH01Z6qj/G6SfpAhTk3DODtL3cz\nYUg7IsJ8BetDNw/mx125XNgvw1ugPzL+bu/ySxNHcWmPGh3YwANjbq/jiCJ544pn6lhWv7TkwFE0\nHVulk5igH4fxROBFWWxsGDFB+pZaxcXjKKq6i91i4qFfDQlYp6H+fpGveXla4nim9fY1x4VHWnHu\n0X8z6VfHEFZhwWioDopGEsPi0DQoKncQGm6qsaxqDYOC2WQIeM9kMmOrNWo9M60NidF6XriPBd40\nmZYcR3hZCGX7CynfX8ie/2xE0yDaGEFFUQmhyRGc+Oogr722gFGjRtG/v55ugwHi48OJiQn8TZjN\nRmJjw8jOPsDQoUN4YsajAHxQ8QH79+/nrdue54rPruClBc8yatQo/jnlQRRF4Xfv7+Lvf3+YcePG\nMW7cOMLCwnj3qucbkNuNq8mDRM2aQufOnVm7dq339ZgxY1iyZAnR0Q27gm7Om+mqH4mIx4Tb4W7W\ntCQmRlJQEDivTqIh2Zsui0sv2N0FKWSkWjjuOEKoO5KcysB5nBw2D1FVtZBu8V282+jeOgbVGXis\nf/61fjXc3Dc3gp4X+fmBeRGlxnrTF1o1jHJoqwHsKThKvjMHg91KaVngyA9bhYd4q3612S+pV8Ax\ndkyJ5KHr+mOrcGCr8B9336NNbNC0nC2JiZEUFQQek8UZ7j2OKEUPqGMyRrA9fycnbPm4yg0Ulgam\n21WpkWLR+076J/Vusu+7ZhlRUWbjorQJXJR2ZuP7ExMjOZ5bxF0r5/i9b7BZyat6/kiCUf+eL2k7\njq926M2kFSUuCstKQdNIG92RqD567emuPr+nwFbIG7ve56a/3Ul8bgT/+Mc/GThwMDfccDOqqlFQ\nUI7LFdjE5XJ5KCqqpKSkEpvN6c3HsjI7lZVOHA6FhQvfZMOG9fz3v6+yZMnH3H//X3j88Xne6cKf\ne+7fvPHG+6f1DIkzvbhu0iBx7733smHDBoqLixk9ejR33nkn06f75iRRFOW8aW6yVwUJzW0ixNLw\nIXJnw+UdLqFnQjeSw33NAe2i22DfPhStMoJx3buSlqGQHpnK4bKjAZ83Goy0i27DnwfcRXI9N6md\nD67oNJnu8V38bhrLjOvEnwbMJD0iFafHRYG9kMSweOwnAm8iNBmMdI3rzJ/6z6RVxJlNZ9AcajZL\n/brLdDrHdiTC7Ksl9Erozp8GzCQjIo1L2o2jyF5CdEik9/w2G8zeZiGTwUSPuM7M7n8naRHBh5M2\ndppNxsYbS2My+Iq367peRYeYdn79UANS+tAqIpmMiDT6RffkD69sw2qyYnc7iOwYT/7KI4R3j8do\nMWIrrqBLdHt+1/4auqR0JrRXKKGhVj77TJ+BISwsnIqKCqKi6r7g7dbtAp599ilKS0sID4/g66+/\n4IorrqakpBiz2cyoUReSmprG3//+V8A3XXiPHr34+usvsNkqm+VZ2k0aJObNq78p5Ztvgk8ydi6q\nflANHrN39MS5IsoS6RcgqoW4Y7HjIToslIxIvRmiui8gMTQeDci3FXgL1GA3Wp1vokOi/Nqdq1U/\n+MdqCvEWeNWFZ5vIDPJtBVS4K70d6q2jmq4j8GyJsUaTGObfjKsoijcvQk2hhEbo50N1f1mH6Lbs\nLtqHhkakJRJFUWgTFfzei8aUnhiuT/rYROKsMSSE+jfFGRSDNy9S4lPo3asv119/NfFdWhE5OI5E\nezRbXtoEwHPxOfztkb/jPGHj1odvwlDVnHXfffcDMHny5dx330wSEhL9Oq7BFwTj4xP4/e9v5847\n9YEpQ4YMZ/jwkezbt5e5c/+KpqkoisKtt97pN104aFx11TXNEiBAZoFtsOrmJs1j8nZcnyvqmvX0\n4RsHsHV/gd8wuD6JPbi6y1R6JlyAqnnYUbCLXgndg37+fBT0PoA6DEkdgEtz0z+pNw6Pg91F++kc\ne/pPRDvXBBvJVJcxGSMwGUwMbtWPMmc5B0uOkBF59u5j+ttv655NoDHU9Rup6S9/0fsKXKqb1Vnr\nGTZqEEW/LeJYWTb9U/oAkJqaxsCBgwM+O336VUyfflXQ7T777Avev8eNm8i4cRP9lnfs2IlFi94I\n+Fz1dOHNTYJEA9mqaxJuE9ZzrLnJbAj+NSbHhjG+v/8oIUVR/EYbDU8LPOHPZ6dyR7lBMTA6fRgA\nEYQzNLRhUyecL04lYBoNRu/Q3VBTaNDa2PnsVPLCbDAxJmMEACnhyaSEN2x6kf9VMndTA9WsSZxr\nzU1K0McNtUwmRa57qp1KTeJ/neTF6ZMg0UB27+gmMyHmcyvbGjqGviWQvPAxGyVgVpMgcfrOrdLu\nHFZUdVOR5gpp0htXTsWENvrso9Wdby3ZqKpmo8TQIM+nbGEGt9LH7tcc1dRS9U3qiclgqnM6GnFy\n8tChBlqw9VW25O/A9tOFvHj3BMym5mtySkyM9OaFqql+U3+0NJIXPpIXPpIXPmd6n0TLzblTdLQ0\nB8VjRvFYGnUs95lqySd/bZIXPpIXPpIXZ0ZyrwFKKuwU2AtxV4bTNiVa2r2FEC2GBIkGyC4tQDFo\naI5QxvQ9/284E0KIhpIg0QA2pz4RWKg5hCEXnH9TNQghxOmSINEAdrc+XUByTDgGaWoSQrQgEiQa\nwO7yTXgmhBAtiQSJBnC4q56sFuThNkII8b9MgkQDVNck5A5WIURLI0GiAZxVfRIWCRJCiBZGgkQD\nODzVQUKam4QQLYsEiQZweqQmIYRomSRINIDT7QYgxGQ5yZpCCPG/RYJEA1TXJELM0twkhGhZmjRI\nzJkzh6FDhzJp0iTve08++SQXX3wxU6ZM4c4776S8vLwpk3DaDpcepdJVCYDLo9ckrNInIYRoYZo0\nSEybNo2FCxf6vTd8+HCWL1/O0qVLadOmDS+++GJTJuG0fPLjLzy56Tn+svopnl+yDZeqBwmL1CSE\nEC1MkwaJ/v37ExUV5ffe0KFDMRj03fbu3ZucnJymTMJpWbJ2NwA2Stm0O88bJKzSJyGEaGGatU/i\ngw8+YOTIkc2ZhAZxVt1xHWaRmoQQomVptjGd//nPfzCbzX79FSdzpk9Yaij/ViUNZ1WfREpi7FlL\nw8mcK+k4F0he+Ehe+EheNI5mCRJLlixh1apVvPbaa6f0ubP1+FLFqPpemFzeaTls5U7ylOZ7hGq1\nmo9mbOkkL3wkL3wkL3zONFg2eZCo/Qjt7777jpdffpk33ngDi+XcbOM3GHxBQrHYcHpcmACzzAIr\nhGhhmrTUu/fee9mwYQPFxcWMHj2aO++8kxdffBGXy8VNN90EQK9evXjkkUeaMhmnTFVUb2eNEmKH\nqqAhU4ULIVqaJi315s2bF/De9OnTm3KXjULV3N4gYQipBKU6SEjHtRCiZZE7rmtRNQ0Vj/e1ElKJ\nUlWTMBuMzZUsIYRoFtJ+UovHo3qblwAUayWKUR/dJM1NQoiWRkq9Wlxuzdu8BGCMLvD+LUFCCNHS\nSHNTLW6P6m1eqs2gSHYJIVoWKfVqcXtUMOh9EjVH75qVc3O4rhBCNCUJErW4PKqvucntG800NfU3\nzZQiIYRoPhIkanG7fc1NmttXe+iakdBcSRJCiGYjQaIWt8fXca3VqElYTSHNlSQhhGg2EiRqcdUc\nAlsjSFgM0ichhGh5JEjU4qnZcV2juckiT6UTQrRAEiRqcXlUlCDNTTL8VQjREknJV4vbrXmbm6YP\n69rMqRFCiOYlQaIWd40+iQhzWDOnRgghmpcEiVqq75NQMMiIJiFEiydBohb9PgkPRsVIiFGChBCi\nZZMgUYvbo4LRjVmxYJbnRwghWjgJErW4PBqK0U2IIQRFae7UCCFE85IgUYvL7QGjG4tBmpqEEEKC\nRC02pxPFoGE1WkkJTwagT2KPZk6VEEI0jyZ9is6cOXNYuXIl8fHxLFu2DICSkhLuuecesrKySE9P\n5+mnnyYyMrIpk3FKyl2VYIBQs5UoSyT/HPFXGeUkhGixmrQmMW3aNBYuXOj33oIFCxgyZAhffPEF\ngwYN4sUXX2zKJJyyCqcNgDBzqPd/udtaCNFSNWnp179/f6Kiovze++abb5g6dSoAU6dO5euvv27K\nJJyySpcdgAhLaDOnRAghmt9Zv0QuLCwkIUF/NkNiYiJFRUVnOwn1srklSAghRLUm7ZNobImJTd93\n4dIcACTHxZ6V/Z2uczltZ5vkhY/khY/kReM460EiPj6e/Px8EhISyMvLIy4ursGfzcsra8KU6aqb\nm9x25azs73QkJkaes2k72yQvfCQvfCQvfM40WDZ5c5OmaX6vx4wZw+LFiwFYsmQJY8eObeoknBKn\nqtckQk3WZk6JEEI0vyYNEvfeey9XX301Bw8eZPTo0Xz44YfccsstrFu3jokTJ7J+/XpuueWWpkzC\nKatubgo1SpAQQogmbW6aN29e0PdfeeWVptztaXO5VTSDC4AwmSZcCCHkjuuabA43mKqChElGNwkh\nhASJGmxON0p1kDBLkBBCCAkSNdgcbjC6QFOwyrMkhBBCgkRNNrtekzArISgyT7gQQkiQqKnS4UEx\nurEoMrJJCCFAgoSfSrsLTC55bKkQQlSRIFFDudOOYlAJNUqntRBCgAQJP2WOCgDCTHKPhBBCgAQJ\nP2VOPUhEWCRICCEESJDwU+4sByA6RGaPFEIIkCDhp8Kj1yRiQ6ObOSVCCHFukCBRg60qSMSFRp1k\nTSGEaBkkSNTg0CoBiJUgIYQQgAQJPy7FBkifhBBCVJMgUYPHoD+VLtIc0cwpEUKIc0ODgsSnn35K\nebk+8ueZZ57ht7/9Ldu3b2/ShDUH1WgDjxmz0dzcSRFCiHNCg4LEf/7zHyIiIti6dStr1qzh8ssv\n57HHHmvqtJ1VTo8LzVKOySn9EUIIUa1BQcJk0h9gt3btWmbMmMGkSZNwOBxNmrCzLacyFxQwu2Oa\nOylCCHHOaFCQUBSFjz/+mOXLlzNkyBAAXC5XkybsbMsqzwEgxCNBQgghqjUoSDz44IN8/vnnzJgx\ng4yMDA4dOsSgQYPOaMevvPIKl112GZMmTeLee+/F6XSe0fbOVKGtCIAQTUY2CSFEtQYFib59+/L8\n889z/fXXA9C2bVseeuih095pbm4ur7/+OosXL2bZsmV4PB4+/fTT095eY3B53ACYDKZmTYcQQpxL\nGhQknnjiCcrKynC73fz617+md+/eLF269Ix2rKoqNpsNt9uN3W4nKSnpjLZ3plweDwBmo7FZ0yGE\nEOeSBgWJdevWERkZyZo1a0hOTuaLL75g0aJFp73T5ORkbrzxRkaPHs3IkSOJjIxk6NChp729xiBB\nQgghAp1S28oPP/zA+PHjSU5OPqNnQJeWlvLNN9/w7bffEhkZycyZM1m2bBmTJk2q93OJiU3XX2C0\n6McTHmpt0v00lvMhjWeL5IWP5IWP5EXjaFCQiI+P58EHH2Tt2rXccsstuN1uPFVX3qdj3bp1ZGRk\nEBOjjyQaP348mzdvPmmQyMsrO+19nkx5hT6kV3VrTbqfxpCYGHnOp/FskbzwkbzwkbzwOdNg2aDm\npnnz5tGxY0fmz59PdHQ0OTk53Hjjjae909TUVLZs2YLD4UDTNL7//ns6dOhw2ttrDG5V77i2mKTj\nWgghqjWoRIyLi+M3v/kNBw8eZN++fbRt25Zp06ad9k579uzJxIkTufzyyzGZTHTr1o0rr7zytLfX\nGNyq9EkIIURtDQoS27ZtY+bMmVgsFjRNw+1289xzz9G9e/fT3vEdd9zBHXfccdqfb2zVQcJilJqE\nEEJUa1CJ+PjjjzN37lzv3dbff/89jz76KO+8806TJu5s8kiQEEKIAA3qk7DZbN4AATB48GBsNluT\nJao5eFQVkD4JIYSoqUFBIjQ0lO+//977euPGjYSGhjZZopqDt7lJgoQQQng1qEScM2cOd911FxaL\nBdAn93v22WebNGFnm0eT5iYhhKitQSViz549+fLLLzl48CCaptGuXTsmTJjAypUrmzh5Z4+3ucks\nQUIIIao1uEQ0m8107tzZ+1rTtCZJUHOprkmESHOTEEJ4nfYzrs9kWo5zkaqpaBqESE1CCCG86i0R\n9+3bV+cyt9vd6IlpTh7NA5qC2XTacVMIIf7n1BskbrnlljqXhYSENHpimpOqqaAZJEgIIUQN9QaJ\nFStWnK10NDsVVa9JGCVICCFENSkRq+g1CWluEkKImqRErKJR3dwkE/wJIUQ1CRJV9NFNUpMQQoia\npESsokmfhBBCBJASsUp1kDCZ/rfu/xBCiDMhQaKKhgoYMBokS4QQopqUiFU0RUOR7BBCCD9SKlbR\nUFE0aWoSQoiaJEhUU1SpSQghRC3NViqWlZUxc+ZMLr74Yi699FK2bNnSXEmpomFQJEgIIURNzTbl\n6eOPP86oUaN49tlncbvd2O325kqKfre1gtQkhBCilmYpFcvLy9m0aRPTp08HwGQyERER0RxJAcCj\n6eCf+HkAABL8SURBVA8cMkiQEEIIP81SKh47dozY2Fjuv/9+pk6dykMPPdSsNQmn2wUgzU1CCFGL\nojXDI+a2b9/OVVddxTvvvEOPHj14/PHHiYyMZObMmWc7KQDc/I9PKW27jAhXOot+80CzpEEIIc5F\nzdInkZKSQkpKCj169ABg4sSJvPzyyyf9XF5eWZOkJ7ewgtC24HY13T4aU2Ji5HmRzrNB8sJH8sJH\n8sInMTHyjD7fLO0rCQkJtGrVioMHDwLw/fff06FDh+ZIik7R+yQczv+t53YLIcSZarbRTQ8++CD3\n3XcfbrebjIwM/v73vzdXUlAUPTi43BIkhBCipmYLEpmZmXz44YfNtXt/VUEiKvR/65GsQghxpmQ4\nD2C16v/3bJ/YvAkRQohzjAQJwGPQh9/GhUU3c0qEEOLc0uKDhKZp3iARZWm+G/qEEOJc1OKDhKpp\nYHICEGk5s6FiQgjxv6bFBwm3R0MxOwCIkiAhhBB+WnyQ8HjUGkFCmpuEEKKmFh8kXB4NxSzNTUII\nEUyLDxJ6TcKJQTMRYrQ0d3KEEOKc0uKDhNujgtGNEXNzJ0UIIc45LT5IuDwaisGDQYKEEEIEaPFB\nwuNRweDB2HwzlAghxDmrxQcJt0cDgweTIjUJIYSorcUHCYfbhWLQMCpSkxBCiNpafJCwu/R7JEzS\nJyGEEAEkSLj1eyRMBqlJCCFEbS0+SDiqgoRZkXskhBCithYfJOwevbnJbJDmJiGEqK3FBwmnxwVI\nkBBCiGBafJDwNjdJkBBCiADNGiRUVWXq1KnceuutzZYGp6oHCZm3SQghAjVrkHjttdfo0KFDcybB\n29xkMUpNQgghamu2IJGTk8OqVauYMWNGcyUB8NUkLMaQZk2HEEKci5otSMydO5fZs2ejKEpzJQEA\npyo1CSGEqEuz3EG2cuVKEhIS6Nq1Kxs2bGjw5xITG/+hQIrRA25IiI5qku03lfMprU1N8sJH8sJH\n8qJxNEuQ+Omnn1ixYgWrVq3C4XBQUVHB7NmzefLJJ+v9XF5eWaOnpcJhB8DtUJtk+00hMTHyvElr\nU5O88JG88JG88DnTYNksQWLWrFnMmjULgI0bN7Jo0aKTBoim4lLdYACrWUY3CSFEbS3+PgmP6gbA\napIgIYQQtTX7rHYDBw5k4MCBzbZ/t+YBIFRqEkIIEaDF1yTc1TUJCRJCCBGgxQcJD3qQCLVIkBBC\niNpafJBQpblJCCHqJEECPUiEyM10QggRoMUHCY+mNzfJk+mEECJQiw8SGh5QDc0+PYgQQpyLWnyQ\nUBUPaC0+G4QQIqgWXzpqeFA0Y3MnQwghzkkSJBQVBQkSQggRTIsPEigqBgkSQggRVIsOEm6PCgZp\nbhJCiLq06CDhcqtgUDEoEiSEECKYFh0knC4PikHFKM1NQggRVIsOEjaX/nxroyI30gkhRDAtOkjY\nXfrzrY3S3CSEEEG16CBRWqk/utRskHmbhBAimPMmSLg8LrLLcwBQNRW724GqqWe0zRKbDYAQkwQJ\nIYQI5rxpjL9l8UNUqCVYK9NwhRTgMdoxYuKC+K5M7XQJiWHxp7zNUptek5AgIYQQwTVLTSInJ4fr\nrruOSy65hEmTJvHaa6+d9DMVagkA9rAsPEY7amUELruZLQXbeHLj8xTaik45HcW2cgDCLaGn/Fkh\nhGgJmqUmYTQauf/+++natSsVFRVMmzaNYcOG0aFDhzo/MzTqUoa37cH3ew6SEBnJ/7d390FR1f8e\nwN+7KynyoCIrGJKDOPhTygdMsOCiFwkMQXYn0IlxakbNMgt5SMKdUeeq6Uw4zNRtHDMrs7g5eUt/\nU/izudH4dMW1SLQGLdExWIpdEZAnZV32c//gsoayiLl4kH2//trztPs9n+Hw3u+ec76nuXEIrDc7\n8H3NEbQF/YatJ7cjL+qVe+pRNLY3AQD8ho28730iIhqMFAkJrVYLrVYLAPDy8kJoaCgsFkuvIZH1\nbDKuXGnGeH+/bvP/vSEIm//nv9Dm/xt2nP4Mhqdeg0bdebWSpa0OgGDMcK1jfXOrBaOGjcIjGg80\nW5uAR4DRwxkSREQ9UfzEtclkwvnz5zF16tS/tb121HBk/dsi2K+NRm17DTaWFuJi42WUmc9gk3Eb\n/uNkAX6trwQAXL5WhY3GbfjvC/8EALTYOn9uCvBmSBAR9UTRE9etra3IzMyEwWCAl5fX336fkLG+\nmD92IQ5W/wt1o2tR+NP2bsvfLd+JkCHTUWW+BowG/vePU8j4RxpuSCsAQOvFkCAi6oliIWGz2ZCZ\nmYnU1FTEx8f3aRut1sfpsuUpkZhW8RgKSoogI6vQcU0LsQ4D7GoMefQSLtnPAD4e6Hr+nHXITdyw\nd4ZE6LggDBsy9H536YHqrRbuhrW4hbW4hbVwDZWIiBIfnJeXh1GjRmHt2rV93ubKlea7rtN24yaq\nLS34s74NlaZrmBg0Akfr/wWz6rfuK3YMATQ2qMUD/znvrXttvqK0Wp8+1cIdsBa3sBa3sBa33G9Y\nKtKTKCsrw9dff42wsDDodDqoVCpkZ2cjNjb2vt97+DAPTHpsFCY9NgpzpwcBAOzVk/Dlhc6QiBoT\nBaPFCGhsAABP9d//mYuIaLBTJCRmzpyJc+fOPbDPG+sV4HidEBLTGRL/z8+T5yOIiJx5aO64vh9h\nI0Mxd1w0IgMjoPX077Ys0MfPyVZEROQWIaFRa5AeluqYHj1sFK7e6LxDe8RQntwiInJG8fsklGCI\nzHG8fkTziIItISIa2NwyJP56uatCF3cRET0U3DIkACBxfBwAIHz0JIVbQkQ0cLnFOYmeJE9IwJxx\n0TwnQUTUC7ftSahVagYEEdFduG1IEBHR3TEkiIjIKYYEERE5xZAgIiKnGBJEROQUQ4KIiJxiSBAR\nkVMMCSIicoohQURETjEkiIjIKYYEERE5xZAgIiKnFAuJo0ePYv78+UhMTMTOnTuVagYREfVCkZCw\n2+3YtGkTPvzwQ3zzzTcoLi7GxYsXlWgKERH1QpGQOHv2LMaPH4+goCB4eHhgwYIFKCkpUaIpRETU\nC0VCwmw2Y+zYsY7pgIAAWCwWJZpCRES9UCQk+FxpIqKHgyKPLw0MDMQff/zhmDabzRgzZsxdt9Nq\n+SS5LqzFLazFLazFLayFayjSk3jiiSdQVVWFmpoaWK1WFBcXY968eUo0hYiIeqFIT0Kj0WDdunVY\nunQpRARpaWkIDQ1VoilERNQLlfAEAREROcE7romIyCmGBBEROcWQICIipwZ8SLjjGE8GgwFPP/00\nUlJSHPOuXbuGpUuXIjExEcuWLUNzc7Nj2ebNm5GQkIDU1FScO3dOiSb3i9raWrzwwgtISkpCSkoK\n9uzZA8A9a2G1WpGeng6dToeUlBS89957AACTyYRFixYhMTEROTk5sNlsjvWzs7ORkJCAxYsXd7vk\nfLCw2+3Q6/V45ZVXALhvLeLi4rBw4ULodDqkpaUBcPExIgNYR0eHxMfHi8lkEqvVKgsXLpTKykql\nm9XvfvjhB6moqJDk5GTHvLffflt27twpIiLvv/++FBQUiIjI4cOH5aWXXhIRkfLycklPT3/wDe4n\nFotFKioqRESkpaVFEhISpLKy0i1rISLS1tYmIiI2m03S09OlvLxcVq9eLQcPHhQRkfXr18vnn38u\nIiJFRUWyYcMGEREpLi6WrKwsRdrcnz7++GPJzc2Vl19+WUTEbWsRFxcnjY2N3ea58hgZ0D0Jdx3j\n6cknn4Svr2+3eSUlJdDr9QAAvV7vqENJSQl0Oh0AYNq0aWhubkZdXd2DbXA/0Wq1mDx5MgDAy8sL\noaGhMJvNblkLAPD09ATQ+c3YZrNBpVLBaDQiMTERQGctvvvuOwDd/14SExNRWlqqTKP7SW1tLY4c\nOYL09HTHvJMnT7plLUQEdru92zxXHiMDOiQ4xtMt9fX18Pf3B9D5z7O+vh4AYLFYEBgY6FgvICAA\nZrNZkTb2J5PJhPPnz2PatGm4evWqW9bCbrdDp9MhOjoa0dHRCA4Ohq+vL9TqzsM4MDDQsb9/rYVG\no4Gvry8aGxsVa7urbdmyBXl5eVCpVACAhoYGjBgxwi1roVKpsGzZMjz33HPYt28fALj0GFHkZrq+\nEt7CcVc91ajrwBksWltbkZmZCYPBAC8vL6f7N9hroVarceDAAbS0tGDVqlU9Dq/ftb+310JEBk0t\nDh8+DH9/f0yePBlGoxFA5/7dvs/uUAsA2Lt3ryMIli5dipCQEJceIwM6JP7uGE+D0ejRo1FXVwd/\nf39cuXIFfn5+ADq/CdTW1jrWq62tHVQ1stlsyMzMRGpqKuLj4wG4by26eHt7Y9asWThz5gyamppg\nt9uhVqu77W9XLQICAtDR0YGWlhaMGDFC4Za7xk8//YTvv/8eR44cQXt7O1pbW7FlyxY0Nze7XS2A\nzp4CAPj5+SE+Ph5nz5516TEyoH9ucucxnm5P/Li4OHz11VcAgP379zvqMG/ePBw4cAAAUF5eDl9f\nX0c3czAwGAyYOHEiXnzxRcc8d6xFfX294wqVGzduoLS0FBMnTkRUVBQOHToEoHst4uLisH//fgDA\noUOHMHv2bGUa3g9ycnJw+PBhlJSUoLCwEFFRUdi2bZtb1uL69etobW0FALS1teH48eMICwtz6TEy\n4IflOHr0KN566y3HGE8rVqxQukn9Ljc3F0ajEY2NjfD398frr7+O+Ph4rF69Gn/++SceffRRvPPO\nO46T2xs3bsSxY8fg6emJrVu3Ijw8XOE9cI2ysjIsWbIEYWFhUKlUUKlUyM7OxtSpU5GVleVWtfj1\n11+Rn58Pu90Ou92OpKQkrFy5EtXV1cjJyUFTUxMmT56MgoICeHh4wGq1Ys2aNTh37hxGjhyJwsJC\njBs3TundcLlTp07ho48+wo4dO9yyFtXV1XjttdegUqnQ0dGBlJQUrFixAo2NjS47RgZ8SBARkXIG\n9M9NRESkLIYEERE5xZAgIiKnGBJEROQUQ4KIiJxiSBARkVMMCXroLFq0CHq9HgsWLEB4eDj0ej30\nej0MBsM9v9fy5cv7NHT02rVrUV5e/neae08qKirw7bff9vvnEPUV75Ogh1ZNTQ3S0tJ6HdWza5iG\nh8W+fftQWlqKwsJCpZtCBGCAj91EdK9KS0tRUFCA6dOno6KiAqtWrUJ9fT2KioocD6HJz89HZGQk\nAGDOnDnYvXs3QkJCkJGRgRkzZuD06dOwWCxITk5GVlYWACAjIwOvvvoqYmJisGbNGnh7e+PixYsw\nm82IiIjA1q1bAXSOhZOXl4eGhgYEBwejo6MDcXFxWLx4cbd21tXVITc3Fw0NDQCAmJgYLF++HNu3\nb0dbWxv0ej2ioqKQn5+P06dPo7CwENevXwcAZGZmIjY2FlVVVcjIyEBycjLKyspgtVqxYcMGRERE\nPJBak5u4n4ddECnJZDLJ7Nmzu807ceKETJkyRX7++WfHvL8+kKWyslLmzp3rmI6NjZVLly6JiMjz\nzz8vubm5IiLS1NQkkZGRYjKZHMuOHTsmIiJvvPGGLFmyRG7evCnt7e0yf/58MRqNIiKycuVK+eCD\nD0REpLq6WmbMmCF79+69o+27du2S9evXO6abmppEROSLL76QnJycbm3X6XRy9epVERGpra2V2NhY\naWlpkd9//10mTZokxcXFjn2fO3eu2Gy2vheR6C7Yk6BBZ8KECXj88ccd05cvX8a7774Li8UCjUYD\ni8WCxsZGjBw58o5tn332WQCAj48PQkJCUFVVhaCgoDvWe+aZZzBkSOfhM2XKFFRVVSEyMhJGoxGb\nN28GAIwbN87RY7nd9OnT8dlnn2Hbtm2YNWsWYmJielyvrKwMJpMJy5Ytcwz6qNFoUF1djeHDh8PT\n0xNJSUkAgKeeegoajQaXL19GaGhoX8tF1CuGBA06Xl5e3aazs7OxYcMGzJkzB3a7HVOnTkV7e3uP\n2w4dOtTxWq1Wo6Oj457W6+tzCmbOnIn9+/fjxIkT+PLLL7Fr1y58+umnd6wnIggPD8fu3bvvWFZV\nVXXHPLvdPqielUDKe3jO6BH1QPpw3UVLS4tj1M+9e/c6/cfvCpGRkY4hmmtqanDq1Kke1zOZTPD2\n9kZSUhLy8/Pxyy+/AOh8VsRfH1ofERGByspK/Pjjj455Z8+edby+fv06Dh48CKDz8Z0AMH78eNfu\nFLk19iToodaXb80GgwErVqzA2LFjERUVBR8fnx63v/29nC3rbb1169bhzTffRHFxMSZMmICIiIhu\nn9eltLQUe/bsgUajgYhg06ZNAIDo6Gh88skn0Ol0mD17NvLz87F9+3YUFBSgubkZN2/eRHBwMHbs\n2AEA8Pf3x4ULF5Ceng6r1YrCwkJoNJq71oSor3gJLJELtbe3w8PDA2q1GmazGenp6SgqKkJwcLDL\nP6vr6qbjx4+7/L2JurAnQeRCly5dwtq1ayEisNvtyM7O7peAIHpQ2JMgIiKneOKaiIicYkgQEZFT\nDAkiInKKIUFERE4xJIiIyCmGBBEROfV/smX5vm0Z6kkAAAAASUVORK5CYII=\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f970d490590\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "test_accuracy 0.1\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXl4FFW6/79V1Vv2BEhIAG/AuCAIsgjoCEFgFDSsio7I\n6Dg4l/GODgpu4wxcnWHEHYXBDQVllJ/LRUAhDKCgYd+XsCVhS0IWOnvSSXqtqt8f1V1d1V2d7iwN\nIbyf5+Ghu6q66tRJ1fmedznnMKIoiiAIgiCIILCXuwAEQRDElQEJBkEQBBESJBgEQRBESJBgEARB\nECFBgkEQBEGEBAkGQRAEERIkGARBEERIkGAQVx0HDhzAPffcc7mL0eEZOHAgioqKLncxiDaEBIOQ\nGT16NPr164eamhrV9kmTJqF3794oKSkBAPzlL39B7969cezYMfmYwsJC9O7dW/7+yCOPYNWqVfL3\njz76CGPGjMGgQYNw5513Ys6cOQCA8ePHY9CgQRg0aBD69OmD/v37Y+DAgRg0aBCWLl3qV8YlS5bg\nhRdeaNV93nrrrfjPf/7TrN98/PHHePfdd7Fv3z6MHDmyVdf34FtHHY3Dhw+jR48el7sYRBuiu9wF\nINoXPXr0QGZmJqZPnw4AyMvLg91uB8Mw8jEMwyA+Ph7vvfceli1bptquxZo1a7Bu3TqsWLECPXr0\nQGVlJbZu3QoAWL9+vXzcI488gsmTJ+P+++9v1T2IohiwLC0lKysLzz33HJxOZ5ufu73C8zw4jrvc\nxSDaEWRhEComTZqENWvWyN/XrFmDKVOm+B03ZcoU5Obm4sCBA0HPefz4cQwfPlzubXbu3BkPPPCA\n5rFNzVSzfft2fPTRR9iwYQMGDhyIyZMnA5CE5t1338W0adMwYMAAFBUVYfXq1bj33nsxaNAg3HXX\nXfjmm2/k8/haCaNHj8by5csxceJEDBkyBHPmzIHD4ZD319XVoaCgAH369MHMmTNRVlYmW0Hl5eUQ\nRRFLly7FXXfdhdtuuw2zZ89GXV0dAMDhcOD555/HsGHDMGTIEDzwwAOoqqrCu+++i4MHD2L+/PkY\nNGgQ/vnPf2re89NPP43hw4djyJAheOSRR3DmzBl5n91ux+uvv47Ro0djyJAhmD59ulzuAwcO4KGH\nHsKQIUMwatQorF27Vq4rpVWzZs0aPPzww/L33r17Y+XKlRg7dizGjh0LAHj11Vdx5513YvDgwbj/\n/vtVf3NBEPDRRx/hrrvukvebzWb5XBcuXJDr4Y033sCoUaMwfPhwvPLKK3JZq6ur8cQTT2DIkCEY\nNmwYfvvb3wZ8BojLCwkGoeKWW25BQ0MDzp07B0EQsHHjRkycONGvITeZTHjiiSewcOHCkM65du1a\nLFu2DMePH4cgCC0q24gRI/DEE0/g3nvvxeHDh+VGEADWrVuHf/7znzh06BBSUlLQuXNnLF26FIcO\nHcJrr72G1157DadOnZKP97USNm7ciOXLl2PLli3IyclRieaOHTtw2223wWQy4ZNPPkFSUhIOHz6M\nQ4cOITExEStWrMDWrVuxcuVKbN++HbGxsfj73/8OQGqQ6+vrsX37duzbtw9///vfYTQaMXv2bAwe\nPBjz5s3DoUOHMHfuXM17HjlyJH788Ufs2rULffr0wXPPPSfve/3113Hy5El888032LdvH55//nkw\nDIPS0lLMnDkTjz76KPbs2YO1a9eq3IW++NbF1q1bsWrVKmzYsAEA0L9/f/zwww/Yv38/JkyYgGee\neUZu7JcvX44NGzbg008/xcGDB7FgwQKYTCa/87711lsoKCjADz/8gM2bN8NsNuP9998HAHz22WdI\nTk7G3r17sWvXLsyePTtgWYnLCwkG4cekSZOwdu1a7Ny5E9deey2SkpI0j3vwwQdRWlqK7du3N3m+\niRMnYt68edi5cyceeeQR/OpXv9KMT7SGKVOmIC0tDSzLQqfTYeTIkbJFc+utt+KOO+5o0hp69NFH\n0aVLF8TGxmLUqFEqcfnll1+ajFt8++23eOaZZ5CUlAS9Xo8nn3wSmzZtgiAI0Ol0qKmpwfnz58Ew\nDPr06YOoqKiQ7+u+++5DRESEfN6cnBzU19dDFEWsXr0ac+fORWJiIhiGwYABA6DX67Fu3Trccccd\nuPfee8FxHOLi4poUDF/++Mc/IiYmBgaDAQAwYcIExMbGgmVZPPbYY3A4HDh//jwAYNWqVZg9ezZS\nU1MBADfeeCPi4uIAqK3FVatW4aWXXkJMTAwiIyMxc+ZM2R2p0+lQXl6OoqIicByHwYMHh1xW4tJC\nMQzCj4kTJ+K3v/0tioqKMGnSpIDHGQwG/OlPf8KiRYvwzjvvNHnO8ePHY/z48eB5Hj/99BOeffZZ\n9O3bF3fccUeblDk5OVn1PSsrCx988AHy8/MhCAJsNhtuvPHGgL/v3Lmz/DkiIgLl5eUApEZv165d\neOmllwL+tqSkBE899RRYlpV/o9PpUFFRgUmTJuHixYuYM2cOLBYLJkyYgDlz5oQUGxAEAQsXLsSm\nTZtQXV0NhmHAMAyqq6vhcDjgcDhwzTXX+P2utLRUc3uo+Nbl8uXLsWrVKrlOGhoaUF1dDQC4ePFi\n0GtVVVXBarWqYlOCIMiC8vjjj2PJkiWYMWMGGIbBAw88gJkzZ7a4/ET4IAuD8KNbt27o3r07tm3b\nhrvvvrvJY++77z5YLBb8+OOPIZ2b4ziMHTsWN954I06fPt0WxQWgdn84HA48/fTT+MMf/oDdu3dj\n//79SE9PbzI+Eohjx46hR48eSEhI8LuOh5SUFHzyySfYt28f9u3bh/379+PIkSNISkqCTqfDk08+\niczMTHz99df45ZdfZFdasOD5unXr8PPPP2PFihU4cOAAtm7dKt9DQkICjEYjCgsLNcujtR0AIiMj\nYbPZ5O8eEVCiLNeBAwfw6aefYvHixdi/fz/279+P6OhouRzJyckBr+UhISEBERERWL9+vVxHBw4c\nwMGDBwEAUVFRePHFF/HTTz/ho48+wueff449e/Y0eU7i8kCCQWiyYMECrFixQvZHB4LjODz11FP4\n5JNPAh6zZs0aZGVloaGhAaIoIisrC2fPnkX//v2bXa4uXbqguLi4ycbf6XTC6XQiISEBLMsiKysL\nO3fubPa1AMkdlZ6eLn/v3LkzampqUF9fL2/7zW9+g4ULF8ppx1VVVdiyZQsAYO/evcjLy4MgCIiM\njIROp5Otiy5dushBYS0aGhpgMBgQGxuLxsZGvPPOO3JjzjAM7rvvPrz++usoKyuDIAg4cuQInE4n\nJkyYgN27d2Pjxo3geR41NTXIyckBIAWiN2/eDJvNhoKCAnz33XdN3n9DQwN0Oh3i4+PhcDiwZMkS\nNDQ0yPsfeOABLFq0CAUFBQCA3Nxc1NbWqs7hsRoWLFiAqqoqAIDZbMaOHTvkOvaITmRkJDiOo+ys\ndkrYBWPbtm0YN24cxo4dG9BvvWHDBmRkZGDChAmqoB5xaVH2LK+55hr07dtXc58v48ePR1JSkl/q\nrYfo6Gh89NFHcjbPO++8g1deeQWDBg0KeP1AjBs3DqIoYtiwYbjvvvs0fxcVFYW//e1vePrppzF0\n6FBs2LABY8aMCXjOpq6blZWlil9ce+21yMjIwJgxYzB06FCUl5fjd7/7HcaMGYMZM2Zg8ODBeOih\nh5CdnQ0AqKiowKxZszB48GCMHz8ew4YNw8SJEwFIcZONGzdi2LBhePXVV/2uPXnyZKSkpCA9PR3j\nx4/HwIEDVftffPFF3HDDDZg6dSqGDRuGd955B6IoIiUlBUuXLsXy5csxdOhQTJkyRRaMxx57DHq9\nHnfccQdeeuklTJgwocm6GDFiBEaMGIGxY8dizJgxiIiIULmsfv/73+Oee+6R733u3LmyBaM813PP\nPYfU1FQ8+OCDuPXWWzFjxgzk5+cDAPLz8/HYY49h4MCBmDZtGqZPn44hQ4YE/JsQlw8mnCvuCYKA\nsWPH4vPPP0dSUhKmTp2KhQsXIi0tTT6moKAAs2fPxr///W9ER0ejqqoKnTp1CleRCCJkKisrMXny\n5KBBfYK4WgirhZGdnY3U1FR0794der0eGRkZsqnu4dtvv8XDDz+M6OhoACCxINoNFoulyWA3QVxt\nhDVLymw2IyUlRf7etWtX1XQSAGSzdNq0aRBFEU8++SRGjBgRzmIRREj07NkTPXv2vNzFIIh2Q1gF\nIxRvF8/zKCwsxMqVK1FSUoLp06cjMzNTtjgIgiCI9kFYXVLJycly5gggWRy+g8C6du2KMWPGgGVZ\n9OjRA7169ZKtjkCEMexCEARBBCCsFka/fv1QWFiI4uJiJCYmIjMz028qiV//+tfIzMzE5MmTUVVV\nhYKCgqADgRiGQXm5JZxFv2JITIyhunBDdeGF6sIL1YWXxMSYVv0+rILBcRzmzZuHGTNmQBRFTJ06\nFWlpaVi8eDH69euHUaNGYcSIEdi5cycyMjLAcRxeeOEFeWoBgiAIov0Q1rTacEI9BgnqPXmhuvBC\ndeGF6sJLay0MGulNEARBhAQJBkEQBBESJBgEQRBESJBgEARBECFBgkEQBEGEBAkGQRAEERIkGARB\nEERIkGAQBEEQIUGCQRAEQYQECQZBEAQREiQYBEEQREiQYBAEQRAhQYJBEARBhAQJBoA6hwX/PvkN\nNpz/8XIXhSAIot0S1vUwrhROVuZi78WDAIDR16TDpDNe5hIRBEG0P8jCAMCLvPxZEIXLWBKCIIj2\nCwkGAEGxhhQJBkEQhDYkGFCLhAASDIIgCC1IMOAjGGRhEARBaEKCAUBUiAQvkGAQBEFocVUKRo29\nFg7eKX/nFYIhkkuKIAhCk6tOMGwuO/6281W8tv9deZuoCHrz5JIiCILQ5OoTDN4GAChrrJC3KQPd\nIgkGQRCEJledYDAat6y0KsjCIAiC0KZDC0aj04ptRbtU8QqtGIWoypIS/fYTBEEQHXxqkK9yv8Oh\nsmzUOiyYcO1YANpZUOqBe7zffoIgCKKDWxhFlhIAQLkiXqFlYQhkYRAEQQSlQwuGRwhYxnubWjEK\nGrhHEAQRnLALxrZt2zBu3DiMHTsWS5cu9du/Zs0a3H777ZgyZQqmTJmCVatWtdm1BUjWglIwtARB\nmSVFgkEQBKFNWGMYgiBg/vz5+Pzzz5GUlISpU6dizJgxSEtLUx2XkZGBuXPntvn1PeMrGDDeMmla\nGDT5IEEQRDDCamFkZ2cjNTUV3bt3h16vR0ZGBrZs2eJ3nBimuIHXJeUVDK2pzMklRRAEEZywCobZ\nbEZKSor8vWvXrigrK/M7bvPmzZg0aRKefvppXLx4sc2u73E1MQFcUk7B5beNpywpgiAITcIqGKFY\nDqNHj8bWrVvx/fff4/bbb8eLL77Y5tcPFMNw8A73cd5tDXbvmA2CIAjCS1hjGMnJySgpKZG/m81m\nJCUlqY6Ji4uTPz/44IN4++23Qzp3YmJM8IPcnqjICIN8fAVM8u7YeCO6RMVAf46Tt8XEGEM7dzvi\nSitvOKG68EJ14YXqom0Iq2D069cPhYWFKC4uRmJiIjIzM7Fw4ULVMeXl5UhMTAQAbNmyBdddd11I\n5y4vtwQ9hhck95Ld5pKPr6qul/dfLK+G2KhHo9Uhb6uubQjp3O2FxMSYK6q84YTqwgvVhReqCy+t\nFc6wCgbHcZg3bx5mzJgBURQxdepUpKWlYfHixejXrx9GjRqFL774Alu3boVOp0NcXBxee+21Nrt+\nsHEYDncMQzmYT6D1MAiCIDQJ+9Qg6enpSE9PV22bNWuW/HnOnDmYM2dOWK7tSZdlGO20WqcgWRZK\nEXEJFPQmCILQokOP9BY9A/cQKOgtBbhpxT2CIIjgdFjBqKi1yu4lRjUOQ2lhSIIhqBZQIguDIAhC\niw4rGC98uFsWBzbASG+tcRgusjAIgiA06bCCAQAKw0JGaxyGQC4pgiCIoHRowfDgmYQQ8LUmtEZ6\nk2AQBEFocXUIRgBB8GREKbdRWi1BEIQ2V51gKFfUcwpO/FSYhdM1Z+VtFPQmCILQpkMu0eobhxAD\nTF/uElxYf35zk78lCIIgJDqkhWF3qK0E5QJJvEYMQwnFMAiCILTpkIJh8xGMBpt3riilteHUEAyK\nYRAEQWhzVQjGmeIa+TOvimFoWRgUwyAIgtDiqhCMQEuwarukwrP6H0EQxJVOBxUMtRAEWoJV2yVF\nFgZBEIQWHVQw1I0+H3Achv/qehT0JgiC0KaDCoYLUIzuFgNYGC5Rw8IgwSAIgtCkQwqG3cEDjFIk\ntGMYTp7SagmCIEKlQwqGwyUAjLZIBLMwRBIMgiAITTrkSG8XLwCst+G3O1348PtjiE7LwxnLGXm7\ndlotCQZBEIQWHVIweF5UuaTAiDhwthCmmL2q47TSaimGQRAEoU2HdEm5BAEMq8iU0lgXQzqOBIMg\nCCJUOqZg8KLKJRVl4jSP0xyHQQP3CIIgNOmYguESAIWFkdQpAso0WwBgwcpreitRTlRIEARBeOmY\ngiGIYBQWBsOIqqwpAICokxdQUkIuKYIgCG06pmDwagtDSzAYgdMc6U2CQRAEoU0HFQwX2Ngq+ftF\n60UweofqGJHnNGMYNA6DIAhCmw4pGMW6Q9CnnJe/23g7jDftUx0j8tqBcAEU9CYIgtCiQwqGhSsO\negzv1B6CQhYGQRCENh1SMBhRH/QY0aUtGJQlRRAEoU2HFAwI2u4mJSIfQDDIwiAIgtAk7IKxbds2\njBs3DmPHjsXSpUsDHrdx40b07t0bJ06caPU1GSGEGU98BCMlojtEgYFIMQyCIAhNwioYgiBg/vz5\nWLZsGdavX4/MzEycPXvW77iGhgZ8+eWXGDBgQBtdOLhgKC2MCFs3PNzzdwAYimEQBEEEIKyCkZ2d\njdTUVHTv3h16vR4ZGRnYsmWL33GLFi3Cf//3f0OvDx57CIkA7iYPDBjVMRwM0LEcIDKUJUUQBBGA\nsAqG2WxGSkqK/L1r164oKytTHXPq1ClcvHgRI0eObLPrBkqZ9aBnjBAVcQ6O4cCyDCAyECnoTRAE\noUlYpzcXg0zkJ4oiFixYgDfeeCPk33hITIwJfN4gv43QG1GvEBW9jkOXztEAGDBM0+duj1xp5Q0n\nVBdeqC68UF20DWEVjOTkZJSUlMjfzWYzkpKS5O8NDQ04c+YMHnnkEYiiiIqKCvzpT3/Chx9+iL59\n+zZ57vJyS8B9vMYcUUoMPhaGKDCorWkERAa8wDd57vZGYmLMFVXecEJ14YXqwgvVhZfWCmdYBaNf\nv34oLCxEcXExEhMTkZmZiYULF8r7o6OjsXv3bvn7I488gpdeegl9+vRp1XWDTVEeoTepUm85hiWX\nFEEQRBDCKhgcx2HevHmYMWMGRFHE1KlTkZaWhsWLF6Nfv34YNWqU6niGYUJ2STVFsLEUPWJScFpp\nhIgsWIaBKFJaLUEQRCDCvkRreno60tPTVdtmzZqleey///3vNrmmCAEMgFE9huPnoh2qfcmRSXjg\nhonY+mOm4geMZGGALAyCIIhAdMiR3h6X1K+6DfXbd0+vX8PA6QFVJhUDTnZJkYVBEAShRYcTDFEU\nZSuBZfxvz7NNFfT2WBgkGARBEAHpcILh4r0NPsswfvs5t2BE6k3ejQIjHSsyCJ6USxAEcXXSAQVD\nkFfXa8rCeO0Pw+Vtouh2SVEMgyAIIiAdTjB4wbscK6NxeywjuaJiIg3ejeSSIgiCCEqHEwwXL8Dj\nVmrKJaVEEL1Bb9+1vwmCIAiJDicYgsLCYBkWg5NuUe3XEozrr4kiC4MgCCIIHU4weEEEoxCMGTdP\nx8ged8j7PS4pADByklsqMtojEgxAMQyCIAhNOpxgCILXQmDcLikd6xUJZSA8UhcJAGhwNkrHgwEY\nWnWPIAhCi6CCYTabL0U52gxl0Jt1356eUax9oRCMLhGdAACe2Ug8QfIXtr8SdAJDgiCIq42ggnH/\n/ffjz3/+s2qSwPaMSjDcFgYXwMJ45KbfYHDSLZiUNs69RTre6rLB4qy/NAUmCIK4QggqGFu3bsWY\nMWPw3nvv4d5778XKlStRX99+G1PJJaUeh6FjvRaGUjA6RyRgxs3TkWCKB6BOw9Uaw0EQBHE1E7RV\nNBgMmDx5Mr755hv885//xCeffIL09HTMnz8flZWVl6KMzUI1DkNDMJTWhi8svGm4bTFrLkEQREci\npG50cXEx3nnnHTz77LO4/fbb8emnn6Jz5854/PHHw12+ZqNKq3ULgC5ADMMfr2BQ4JsgCEJN0OnN\nn3jiCeTl5eGhhx7C6tWrkZCQAAAYNGgQNmzYEPYCNhdeUA7ck8Qh2hAl72/K1cSChSfUzZNgEARB\nqAgqGJMmTcLdd98NjvN35axfvz4shWoNXguDkdNqY/TR8v6mBINR7BNEypIiCIJQEtQlFRcXh8bG\nRvl7XV1du86Y4kVp4B6jcC/FGryCwTGBYxgMuaQIgiACElQw3nzzTURHexvc6OhovPnmm2EtVGvg\nec/Eg97GP8bgXfi8qRiG0voglxRBEISaoIIhiqLs2gEAlmXB8+3XXeNxSSlTZCN03rUvmnRJKUSG\nJ5cUQRCEiqCCERUVhaNHj8rfjx49isjIyLAWqjXw7nEYysZfJXhNuKRYKGMYZGEQBEEoCRr0fv75\n5/Hkk0/iuuuuAwCcOXMGS5YsCXvBWoogui0MjanNAe0pzz2og94kGARBEEqCCsbAgQORmZmJI0eO\nQBRFDBw4EHFxcZeibC3CM3CP9TGeJl47Dudq85sMeivFhOaSIgiCUBNUMAApU2rkyJHhLkubIGi4\npABgbM/RQX+rtEoo6E0QBKEmaAwjJycHv/nNb3DLLbfgpptukv+1VyQLAwFdUk2iWG3vqy25KC5v\nv3NmEQRBXGqCCsYrr7yCZ555BqmpqcjKysLMmTMxe/bsS1G2FqGVJRUqjEIwiivqsXxDTlsWjSAI\n4oomaKvqcDhw++23QxRFJCUlYfbs2di+ffulKFuL8GRJNRXcDoTqJ4wIp4vcUgRBEB6CCgbLSofE\nxcUhJycH1dXVKC4uDnvBWopnidaWWBhKlxQY0T0vFUEQBAGEEPTOyMhAdXU1Zs6ciWnTpkEQBMya\nNetSlK1FCK2yMETVZ8+ocYIgCCKIYAiCgNtvvx0JCQlIT0/Hvn37YLfbVVOFBGPbtm1YsGABRFHE\n/fffj5kzZ6r2f/3111i5ciU4jkNUVBT+8Y9/IC0trWV3A/dstYzYsgWQfFxSZGEQBEF4abJVZVkW\nf/vb3+Tver2+WWIhCALmz5+PZcuWYf369cjMzMTZs2dVx0yYMAHr1q3D2rVr8fjjj+O1115r5i34\nXrMVQW8oXVICXAJZGARBEB6CtqppaWkoKipq0cmzs7ORmpqK7t27Q6/XIyMjA1u2bFEdExXlXaui\nsbFRjpm0FHngXossDJ8YBrmkCIIgZILGMKqqqjBx4kQMHjxYNYfUokWLgp7cbDYjJSVF/t61a1cc\nO3bM77iVK1fi888/h8vlwooVK0ItuyaC6Fk8qXXjMMCI4LucxvGKLri5S/sdd0IQBHGpCCnonZGR\n0aKTh7ou9vTp0zF9+nRkZmbigw8+wOuvvx70N4mJMZrbjUY94BCh1+kCHhMInY4BXNJnhnMBKXn4\nMDsH3/7mw2ad51LT3PvsyFBdeKG68EJ10TYEFYwpU6a0+OTJyckoKSmRv5vNZiQlJQU8/t5778XL\nL78c0rnLyy2a2y31dgAiRCHwMYHgeUWQW+cIeq32QGJiTLsu36WE6sIL1YUXqgsvrRXOoIIxa9Ys\nzWk2QnFJ9evXD4WFhSguLkZiYiIyMzOxcOFC1TEFBQVITU0FAPz888/o2bNniEXXRhBEgG1ZWq3I\niJ7lwMHonK0qB0EQREcjqGCMGjVK/my327Fp06aQ0145jsO8efMwY8YMiKKIqVOnIi0tDYsXL0a/\nfv0watQofPnll9i9ezf0ej1iY2PxxhtvtPxuALhEAQzT9EJJAcurmMmWBIMgCEJNs11S9913H/7n\nf/4n5Aukp6cjPT1dtU058E+ZttsWCO6xEy0RjP6mdGyuKwTD8SqXFEEQBBFCWq0vDMO0OM32UuCS\nBaP5LqkYXRwceYMBAAwJBkEQhIpmxTBEUURubi5uv/32sBespbTGwuBYBhCleyWXFEEQhJpmxTA4\njsOMGTMwYMCAsBaqNXim8+BaKBiiRzD0JBgEQRBKwppWezngW2Fh6DhWtjAIgiAINUFb1WnTpqG2\ntlb+XlNTg+nTp4e1UK3B5V5atSVTjBgNHCD6/47W9yYIgghBMBobGxEXFyd/j4+PR319+126VGiF\nS8pk4DQtDJdIgkEQBBG0VRUEAY2NjfL3hoYG8Hz7bUB5d+POsVyQI/0xGXTagiG4Wl0ugiCIK52g\nMYzx48djxowZmDZtGgDgq6++wsSJE8NesJbiEqV0WBNnaPZvA1kYToEC4ARBEEEF449//COSkpKw\ndetWiKKIhx56CJMnT74UZWsRLlFq3I06Y7N/azJwUK+i5D4nWRgEQRDBBQOQMqWulGwpwT3dbEst\nDJFcUgRBEJoEjWH8+c9/Rk1Njfy9uroaTz/9dFgL1RpccFsYXPMtjEBZUk4SDIIgiOCCceHCBcTH\nx8vfExISUFhYGNZCtQaXKDXuhhZYGByrPQ7DwVMMgyAIIqhg8DyvyopyOp1wONrvPEs8pLIZWyAY\nADQFw+4kwSAIgggawxg+fDhmz56NRx99FACwYsUKv9ln2xOeGEZLLAwAmoJhdbZfgSQIgrhUBBWM\nOXPm4OOPP5aXTR01ahSGDRsW9oK1FF6OYbRUMPyNLhIMgiCIEFxSer0eTz31FN5//33cdddd+OGH\nH/DXv/71UpStRQiMZGG0JOgtwUAU1NXicJFLiiAIokkLw+VyYevWrfjuu+9w5MgRuFwuLFu2rF3P\nVutxSbXYwgAAgQNY7/reNhIMgiCIwBbGa6+9hjvvvBNff/01xo8fj6ysLMTFxbVrsQCUFkbLBOOv\nvx2MCL36txabDau3nUNtA7mmCIK4egloYXz11VcYOHAgZs6cidtuuw0A5IWU2jMi4wKDlge9r+sR\nh+hCI2yoDujAAAAgAElEQVS2BnnbgbwSVJ5lcMFswdMP3NJGJSUIgriyCCgYO3bswLp16/Dmm2+i\ntrYWkydPbteTDnoQWUkwWh7DUE6NzgAQYbHbAACVdfZWl48gCOJKJaBLKjY2FtOnT8fq1avx/vvv\no7a2FjabDdOnT8fXX399KcvYLES29TEMxl0tBkYSHY+b6wowsAiCIMJGSItG9O7dG3PnzsX27dsx\nffp0bNmyJdzlahGCKIJheUBkWjS9uQfWrQwGxgQA0jmhNS0hQRDE1UNIkw960Ov1uPfee3HvvfeG\nqzytgudFgBHAiC0XC8C7vKue1QMCAM49lxQpBkEQVzHNX5auHePiBYAVwKJ1gsG4lUHHSHoqWxjk\nkyII4iqmQwkGL7gtjFYKhsclpWP07g28e3urTksQBHFF0+EEg2EFsK10SXmC3hzLQRQBcLy8hyAI\n4mqlYwmGxyXFtI2FAYiAwIFhKUuKIAiiQwmGy+2SanUMg/FUiwjwOtnCIMEgCOJqJuyCsW3bNowb\nNw5jx47F0qVL/fZ//vnnyMjIwKRJk/D73/8epaWlLb6WZGHwbRb0FhkRosAp0mpJMQiCuHoJq2AI\ngoD58+dj2bJlWL9+PTIzM3H27FnVMX369MHq1avx/fff4+6778abb77Z4us5XTwYBuCaly3sh8ol\nxXNyWi1ZGARBXM2EVTCys7ORmpqK7t27Q6/XIyMjw2/Q39ChQ2E0SiOqBwwYALPZ3OLr2d1LqbY2\nhqF0SYkC586SEsm+IAjiqiasgmE2m5GSkiJ/79q1K8rKygIev2rVqlat5md3L3TEtTbo7XFJuWMY\nDAMpXZdMDIIgrmJa57sJgiiKIR/7/fff48SJE/jiiy9COj4xMcZvm6myAgBg1Bk194fKr3oNQk71\naQxM6Y8LxYekjRwPg0HXqvOGi/ZYpssF1YUXqgsvVBdtQ1gFIzk5GSUlJfJ3s9mMpKQkv+N27dqF\npUuX4ssvv4Rerw/p3OXlFr9tFVW1AABRYDT3h8rAuIH429Du4BzR+F48LG1kBDidrladNxwkJsa0\nuzJdLqguvFBdeKG68NJa4QyrS6pfv34oLCxEcXExHA4HMjMzMWbMGNUxJ0+exMsvv4wPP/wQCQkJ\nrbqeJ4aha/U4DBbdopOh43SAe7lWhlxSBEFc5YTVwuA4DvPmzcOMGTMgiiKmTp2KtLQ0LF68GP36\n9cOoUaPw1ltvwWq14umnn4YoiujWrRs++OCDFl3PyUvZTJ45oFqLjmMA0T3V+fWHwdti2+S8BEEQ\nVyJhFQwASE9P9wtkz5o1S/782Weftdm1HG4Lg2Pb5rY4jpUFg42yoNywF0DLg/IEQRBXMh1qpLfD\nbWHo20owWAai4K0igXG2yXkJgiCuRDqUYDgFdwyjDQXDY2FIUAyDIIirlw4lGK42jmFwHCMHvQFp\napAfC37Bq3sXwim42uQaBEEQVwodSjAcotslxbWNYLAMA4hKq4JBTtVplDRcRJ2d0vQIgri66FCC\n4XIHvfVsaGM5gsEwjHoxJhGwOOsBAA7B0SbXIAiCuFLoUIJhd0mCYdS1jWAAUM98KzKoc0iWhZ23\nt9k12gKn4EJ+XWGzRtd3VBy8EwV1Fy53MdoFdt6Bwrqiy12MdoHNZcMFS/HlLsYVTYcSjEaH1OuP\ndk9m2BawiioSIaLe0QAAcPDty8JYeWoV3jqwBMcqTl7uolx2lp9YiTcP/At51WeDH9zB+fDocrxx\nYDE1lADePfQRXt+/CGWN5Ze7KFcsHUowrC6pEY+JMLXZOZUz3wqMQ5qQEFLPrT1xwCxNYZJPPWtZ\nNIuokcTpmnMAgNKGls8C3VEoqpemKaqwVl3mkly5dCjBsDsll1SbCobCJcVzXjdUexMMz7QlHkEj\nQDVBaELPRcvpUIJhc1sYkQZDm51TKRgi6xWJ9hbDoNUA/SHx9EKxLSVUFy2lQwmGZ2qQthq4BwRe\nW6PdWRju/6lhIIimoXek5XQYwRBFUZ58sK3SaoHAq/fll1Uj70JNm12n1dBMun5Qw+CFrC0vVBct\np8MIht3JQwAPoO1Gekvn0haMPaeK8frKQ6i3to/5pbyrkNPLQPhDT4UXXhQudxGuWDqMYNRbnQAr\nPQhtNdIbCOySYlhJnBrajWA0HfQ+W1KLV784gGqLN/bCCzyWHPkUe0sP4oKlGG8fWNKhMkiaI55O\nwYVFh5fiUFk2ztcW4O0D76PGXhvG0l1axGY0knbegXcPfYhjFSeRV30W7xx8X04nvxQIogi7kw/b\n+V3NmNan0WnFOwc/wKmqPJyszMXCgx/A6rKGrWztnbBPb36psDl4MIz0UrRpDIMNsBgTJz3Q4Xyw\nm4XHJRWgjVy8KhuWRic27C7A9LtvAACUNJhxqioPp6rykBLVFaUNZvxw9j+YcfP0S1ToMNOMbvXZ\nmvPIqz6DvOoziDXEoM5hwab8n/GbGyeHr3yXkOY0kscqTuJMzXmcqTkPlmEhiAKyincho9ddYSyh\nl7e/Ooycwhosff5O6Li279M2Zx64/ebDOFebjyVHPpW37bt4GCN7/KrNy3Ul0GEsDLuDly2MtnRJ\ncYHO5bYwbI52IhhuAvWqBcF/u15DDHmxfd1Pa2iO60HZyfDUYUeqi+Y0klodLkG4dHWRUyjFBsP1\nbrmE0L0CWi7pjvRcNJcrVjB4gVdlKtkcPMBIf8i2Wg8D8HdJiU4pZZdxWxiOdmJhsB6XlChqZnBJ\n8R0RDCOZ2YBk+su/Z6RHwdPIWl02CFe4r9cpODVH5Dt4h3xvnrpQ/p09nwW5Lqwdti7s7roQRVGu\nC2XSiP9zYb1kyQROV3jq3Mlr14XNZYcoihBFUXY76Tn/BBrls3O1JVZcsYLxjz1vYU7WXPm7zeEC\nWAEMmMBupBZQWmFTfRftEdIHzuW+bvsQDE/Y++eiHZiTNRfnawvkPQ3ORqD/f6C/NhvFTDae3/4y\nTlXlwSV6e52cu2EQRAE2lw3Pbftf/Ethhl+JbCrYitlZc1FcXypvq7XXYXbWXHyduwbrz23G89tf\nxrnafFWvkVXUhcVRj+e2vYxPjn1xycvflqw7twmzs+aqpsWosFZhTtZcrD2zAavPrMfz219GcX2p\n3PkAvB0RQRRQbavBc9texoqTX1+SMjtd4Xm3vjuzHrOz5qLa5s1yLG0w49lt87Dh/I/4Kvc7PLft\nZZQ1VgSwtgRcbCjD89tfxjd5a8NSxvbKFSsYFTYpOOtRe5uDB8MK6skC24CGRvVDK7oMEGwRYE31\nAMIbnGsOvlm1+y4ekj97gre6LqUoZrIBAEfKj4MXlI2kt1dd655gMa/6TDiLfMk4XHZM/lzWWAEA\n2FmyF//J/wkAcKIy16cuPL1qHuVW6fjsihOXqrhh5WRlnvzZM1XGlgvbsPXCdgBAXvVZVUdCfi4g\nyPNR7XdPQxNuHGGyMDycrc33fq45DwDYkP8TdpbsAwAU1F3QTBYQIOCc+7fbi3eHtYztjStWMDxY\n3Wa05JIS2jR+AQBjh/RUfTewRojWGDB6J6B3tBvB8A1dWF1ey0hpNovuxAAWjCoQyroVhxcFzTHj\nVrsLLl798giiiCWrj+GnA+17/iqboi60rE8GjE8j6XXvNbXKYqHZglc+24eKWv+smUZb+8ie80X5\nXJg4/0k6GTBwKcRTaXkyl3isTyCXlCiK+GDtcWzcW9iq8yvrItYQ47efYdR14UFyz12d456uSMHI\nyfemflocVjh4J/KteQArgGvD+AUATBl+neq7iTNBaJQeLjbCIgXbFdTYa+XJ7wRRwKGybFUansVR\nj1NVedCiuL4UedVncaTsWLN8o5W1NjTa1eVQvgzKxtAFyXfr+zJ4PgsiD9+XodHmxJPvbsPSH9S9\nbEujE4fyynEoT3v2zypbNU5U5gKQYk4HzUdhc3nTemvtloBWTKGlCKerzyK7vPU9e1VdaAR/pbrw\nbmfg9ds3NeXK+2uOodBcj9Xbzqm27zlxEU+9tx0HcsrkbRXWSvnv7hRcOGg+qvKj19hrcbpafR4P\nBXUXcLr6LI5XnGrqNkPCynufRa26YBnfjoR2XRRcbPkCYubGcvnv7uAdOGg+KgflRVEEY7CCja7W\nFIzztQXINufhUOkJfPtz6yxgm+od8RcG346E5/4FgW8z8SxtMOOM27qxuew4aD6qsnYrrJU4X6st\njGdqziOv+ixyqk63SVlC4YoUjL9+uFP+XN3YgG/z1uKQYyNYU2ObWxg6nx5plF4hGJEWPwvj9f2L\n8FH25yipv4jdJfux7PiX+PLU/8n7Fx78AEuOfIpCi/8aBQv2vYtFhz/GJ8e/aNbU3BfK6jUsDG/D\n4BkBr4QFq3oZPI1XlcXmZ4afKZZcWgdy1cJQ4x7TESiO8/c9b+GDo8tQbavBz0U7sPzESvzf6e/l\n/Qv2LcSiw0s1p5t+Y/9ivHf4Y3x8bEWr13NQ1YVWIwlWJZ4WmycpQPBz9YmiiHU7z+NMUS3sTqme\nDDr1M/LzYcl1s+Wgt9wv734DS458ikanFRvP/4TlJ1Zi3blN3v27Xsd7hz9Crc9KjqIo4s0D/8J7\nhz/Gh9mftXpqbmUjqVUXDMOqGk9PxpggqJ+Jv3++v8Vl+Meet7Do8FLwAo81ZzZg+YmV2FzwMwDA\nxQswDciCsc9eNDjUlptLcOHtg+9j6cllMN54END5z+dmd/BY+WOearxRXmE11mzzf58aVe+Iv0XI\nMqymq9IpuNrMvvjn3nfw7qEPAQBf567B8hMrsU3h5np59xt4++ASv6SLRmcj3j30IRYd/hj/OvLJ\nJZuq6IoUDGXPo87WgNOKxrUtB+0B0kMyb9hz8vdoYyREqyQYTKRFbixdvIB6qxMWh7QiX3Fthezz\n9UwxDQBlbp94sCVeq+yhTzui0/k/vqH1qr0vg+eBq6hthM2pfnnyA/QmaxuklzKQW85z3TqHBWer\npSC8Mhhf75QGgzU41Q2D78tR66jTPH+oWF02rNych882nNJMqfTtVTfYbe5y8H6B1+LyBqzZfh4L\nvjwoZ8gZ9OrXyKCXBETLB2912WTfuXKRJ08jbePVSRYOn/L61lVzaQzyXPi6Kj3PhUt0+fWqeaF1\nMQYbb8cZ97vhSUxQdj58BUP5TAPeTEUPdQ0OZO7Jx5aDRVj0f0fl7c8u2oZ1u8/7XT+QFe5h+9FS\nVV14EiPsgqPNJ/t08k7kVkuWwoYjx5F1RD01v83n3n3rwsE7UFzRgL98vBtF5fVtWjYlV6RgKLHY\nG1UpkW2ZUushOSpJ/hylj8DYW24ECx3YCIvcaHy49jhmLdouH/fxumMoqZL+cKFYPXaXuofg8R2H\ngsMp+LlUg70MDMOAV7wMckPFiKi1qh/G/FJJMDrFqn3eNfVSmW0OHrwgYM/Ji35xDkBaCfHwGck9\no5V14tuI+84EHGg+r1BpdNmw5VARtmeXajaSosioc+vdDREvCigqV4ulskHzCKVRry6fQSf97Rwa\nWT523i5fS6su/BsGdaNpaXTg2LlKv99poZUKHNzCYFS9aqf7byNZoL6uytDHdmhhddnkZ9Mz3kHp\n4m1wNt1IesZCAcCR0xV45l87sHmfJMLFFT4j0xn/uvB07gDtusg+VwGHhnVudzlk67qtsPI2ud7r\n6l1YsTEXh097rUnfe7f5vCN23oGVm3NRVm3FF5ty27RsSq5IwdD18FaIxdaA+kbvw6CVN92WROhM\neHDUDegW1RVMRD1+OVqEilorjpaehq6HNzbBsALqGqU/slag1bdhqKhXP+AsGDgFFz47ugqf/LRb\nsyH2YLW7/F4Ia5CGgQULp6KR9BzPRtXhZIXXJ7rmTCYuOiWrwGRQN3A19W4Lw8Fj874LWPrDScx8\n6xdsOH4Qmec2y8eVVNXJ5dOaasX3ZWj0szh4OHgnvsldi4sNZWgKrdiPxe6tW6tTYyyCg1fHMNwD\nQE9V5aHI6vUfr8r7AWdqvdas51K+o5E9FobTKeBkZS425W/1Xt9lky27UOrC9zn5dMMxvLvqID49\n8g0qrE0Lh1bA1mPVSfv9rS1RFDU7GEfKjyO/zlsX+tSTOFUh1cWBnDKV++1gbjl+3O+fCHGs4iR+\nKsySv1tdVjhd0rUEQRIjq8N77UZno+r3vnUBlkdlfR1e+XEZ1uyR4oYeq473HajKagmGtzOgaYWz\nAup8Ok8AsN98CHkV3vv7Kuc7Vd2UVjbgi025ftZpWXUj/r0pF1a7C4fKsvHLBa9r/bON2bB6LHtR\nqot/rT4i728M8o44eAecsELf8zhELnxLL1yRU4Pou3nNy3qnDQLPyHdivASCAQBdIrqgqKEYjN6O\n1VnnYOyzV30gy0trjBu8vSdlY2b1cT1U1qt7snbeiUMlOThQuQ+uslSkHb0Gowf10CyTJ0NMtY23\nSWZuYR2+2HUSSPH/DR+h3UPcXu5t4H4qzAK6AigY5zdI0WNhNNpdqpl7M8u+UR13zlwll88TE1L2\nfn0byYp6tUm9YvNJpPevwbayXTDpjJiUdo9muStrrfIU90osTgukIA+Depv/y9RodyEmUtuttqdq\nm/z556Id7k/jVMf4irnn71xRa8P7R5ep9lldVtnC4FgOlbU2REcxiv0+DYNfQ2EHl1CPw1XH0MPc\nBeN6jtEsNwAsyzwORKu3VVqrpMCyjxvOQ1b2BQy6qZPm+Tac/1H+rOtaiBVnPsOGLVNRaJb+XqMH\ndQfDMHh/jZTGPGZwD7Cs994+yv5cdT6ry4Z6uwNggNOFdUB/oMZqUe1X3buPtcWwPD7ZsQXlhlw4\nXAyAnprllg72FwzlvGmaU6cwAsw12nNo5dgOyp93lOzFjpK9eHXofPyw4zx+OSKlK3dPjFK9s59t\nyEHuhRpwLINd3Jeq82UXmGG8yQWGAURB6oAwesXAZJ9793Vd2nk7GkwF0MUXwVbnfdkdTh42B4/Y\nqLZZI+iKtDCUNDqsqh68oY1jGL6Y9JJbJlLvXtWP5REVoSFSnEsSDHhdD0pXi+/LUNWgbiR3nSzC\nsi173OdyykG8eqsTe0+a1eJjd4Jh/XvWFxvLcPR0pWYjeaGsXrMH2hS+PnmlWX70bODe7skLZsBd\nPo7hsOt4KaoavHEJ37ow16on/attbMSmbClbqqC8StOKKDRb8Ng/NuOrrf7muAAejEnqrTbY/evi\nXGk1LBo9yVBxOH3E2uHJOPMvp2RhSI2Tpd6F5z/chR8Pn1HtBwBzdSP2nTLLaeMeGJYHEyk1qr4N\nqBKeF7A/76Lf9kaXVR6Xo2V55ptrYXWEHkD1iAXgH8uqD5JabHVZIbpnZ6ixSGWptgUWDD8Lg+NR\naZcsTkbX9LUYDcGoddTJFpfm1CmsgBP5FU2eV8mcJTtlsdDC8zyczPef4JPhXN53WHQ3y3rt9sLh\n5HHkXKny57DzDjj0UqdNUCz09q/Vx/DMv3bgfGmdKmuvpVz5guGywil6K6gt18LQwshJSm3SuRWb\n42EyaOT2cy44eG9PEpCCvx58X4Zaq1owTpdUgYmwyOdqtEsP9NIfTuDjH05g9wmpMcgvrcPq7drp\nhcX1pdLvNF4WvV47ttEUdQ0OrPwxT0468AS9g/7O1ii/sJW1Dny6/hS+yjou71fWhdXuwsqtJ9Un\n4FzgoqX6OVFYhiNn/F/i00VSI7gtW3sdb09dNmg0hgVldfjpYMtz+n1dD02N/le6pKrqpEYu6+R5\nxX4raurtePXfB/HR9yewJ9fnfjgXWLdgWJ2BRa6i1ia71nzxBJg1G0lGQKWlZTPT+sY0ausd+H8/\n5uGERgMJSHUhwlvGaosdtQrBsPE2rN1+Tvbl1/uJpwt2TmokGc7/XmwOlzdBJkBdlNRL75FTa34p\nRoCIlo+z8nWLJcRInc3Sykb/g5Xld78rSgtD+Y5k7i7AtuPq5zWvuAINkOq5rM6CI6eld+TEeWnb\nu98exQdrj6O1hN0ltW3bNixYsACiKOL+++/HzJkzVfsPHDiABQsWIDc3F++++y7uvvvuZp3/jOMw\nlIO723KmWi0i3BaG0T3oSZ9yDj/VHQcbpT5Ol3Ieojt4esFSjKd/fglDEofJ+60uG+oaHWAZBrwg\nYt3eMzBerzgB620YuE5l2C0sR3XW7TgbfxCmgQJOFMUhB1uxP6cM+l7aZf3i1LcwGDqBYRP99rkE\nV7MsDOMtv8B+dCS2HCxCgXgQxdwRONleAG4AIMLQex+E+ni4im70+62u+xk5o6VKLILp1hKcr0sD\nIjx14W0IDuSUQWDULy/D8oBJskh0XUqxrPAd3G8aj435W8AyLP46dDZ2NayDvpcDjF5bxIzXHwFf\nlwCro7f/TlaA3eVEqF0NY7/tsB8bIZWnRy726jeh9KfBEC6m4flpt6C002bohAS4iq/3++33ZzfI\nAcs6w3mYBhfAXuP9A1pdNhw7Vymvs5JbXA50UZaVBxshieeu0n3YZz6EKddlIPPcZhg4A/42dA6W\nHf8SUboEGK7L0Sz/h9mfoW/n3ugWlaxZF1WW0DOxjH13wX5Cmrn1q1NrcarhMLiuN4I398T6PeeQ\nzfyAnTu6YX6nh/1++395P4CH2wrvWohX9v8DaRE3y/uPnr+Ig6WRAIDlfxmN/Xm+4snDZagFA0CX\nXAAu8QKcF26EvsdpiC499uddj12N30N3DQuuk1mz/IsOf4wBXfrByET67WMYIaDQaGHovReOHOkd\n1/c6hu/rNsFUPAUjut8GO+9AbsT34JK6gS/7L//f9vI25vpu56HrWgi+3OvO2pNbhBuib0ZCjBFl\nNVY/gVy39wz0PS3u35/D0vx34Np+A0yD8yA6TKg/fgdMhtZ3psNqYQiCgPnz52PZsmVYv349MjMz\ncfasOh+6W7dueP311zFhwoRWX++G+DQMS7m11efRwp43CC7zNegaIWVMeSwNrpMZbJR/2qlvyp9L\n5LGr2Ju73ui04pnFO/Dy8n3IL63zewAYnROMydvTY1gRObb9gN4ORu9Efv15HCw7CrZTKXRd1Oap\nEoe+SvOhd/Au2TUi8sEfA9Zok3tBhY35ACOC6+R2eXAucLHV0Hc7j+t7xPn91rcuGFaAI8abXqvs\nPVXX2wHORzD0dvA6b12IjID/nP8J9c4G1DksyK0+g4uu89AlFoOLD+xC4GKrNd0tyoYhpLqIaPAG\n8eMqAUZEfmMeThfV4qKlBi5jFfTdz/pllQH+2S0MJ4CP8/YWyyx1qKhRWFw+bifWaAVj8J7DJbiw\n4fyPsqsppyoPOdWncbB8H9jowOnIJypzAvjtRdS6kzVEPnh2GhtVB88goOOVORAhgkuQGucD5/Kl\n/d1y8NyHO/x+6+uHd8GJHIuiF+zzHJRb1O8ZG1EPRqdIVuAE6LufAaNzgTVZsWLndhTWF0Kfkg/W\nGFgEj1Qcw84TGpYpKwKMdG+iEDyNlout9n6OL4MIUR54WmQpgUNXC0PPU2prwlN2nc/7z/HgOnvd\nWycKzXj2/Z1wuni4XALg44JjI+tUbmmG4+WOGhvRACbCgi5xEUHvIRhhFYzs7Gykpqaie/fu0Ov1\nyMjIwJYtW1THdOvWDTfccEObjJx8etAf0bezfw+3LRBqkuAs6CtnwHgEozkweu8fuapBevirLXZY\nGp1+DxEbVecXl1D+vhL5Aa/j55Zj/R9QycJwC4ZDepBEgYVgjfI7Vr6+u4zy/6ZGgOFVYhdhCvhz\nFSKnMLcVDcfFqkY/8WSj/RcyanB5zfpD5qN+++XruNQWp+8YEwAAI4DxNAzuuhDsJgj2Jm7GU0Z3\n3TKR9QAEfLjOm9kSGer7qXj59+YUyVONdIo1wi74pBhH+4/PaVBkEy3d/qPffg++dRHIDVNRJ51P\ndEj3LzTEQHQ10Tt1p7d6FhWTLGNRTk8GoNlIaqGMRSifA0EQYfcRGK26UL4jus6BO1IRnM/fVsuS\nUFoYLkn8+drOclBaGxFRJp2cgexx/ymTPIb01U4q8Lu84l489feXj/fgTEmtxjvSdF2wkRYkxof4\ncjZBWAXDbDYjJcUbse/atSvKylofeLmceATD0ALBUFLeUA02tgKMwYq9F05IvVYFWo2kEq5T4Hrk\nGE41sEjrYSqutOBAgWTteRoGhhUAIXCvkjFYwRisgLs3xDAAl1AGNtrbs7JFNX9UdqW1CvNXb8Ci\n7/di/4WT4CLUPt5gdXG0iYkBRd4nFVjQqDfOG0iW60LnbLoujFZAb5N7hgwrgI2rQJldEfSMD9xg\nBTyvqRF7L5wAY7QisUcDGGPz6qKp50J0qZ/Z0gZ/N01kBMC4XV6eqfzBuZq0Njx14WnUGJ0LbEwV\n2Cjvc+exOpoDY2oAG1sB6G34n6XfoYFXW0zB6yLwNRle3XhKk4mqiYpSbHd5ljVwAU3UBWu0Ys70\n3rJ1VOuoQ2FdkSrtNqVn8wfWsRH1YGMrUG2rhYUtAWPwFc+m64KNq4CDa/l0Lh7C6vC/lHPF63h/\nH2RbMvvBW3DsXCW6dZauY9SYuK051KMKxt5SQOocAF2IvVHRqYco6Jo0sR28EzpGB6coPbRKU9mD\nrvNFeLZ6GkmgaTeEsY80i6eyh2W4Tt27v2DcieZSaCkC4otwEYBBI8QQCMFuAsMKquBgMCysf+aQ\nrou3kRed0t+V4XiITQiGqa80fYPSVWG88ZDqmMq4fSGXywMXUwOu9wEAQCEAXXTTx3sQbJFgdA4/\n14b6IHX/UDlbqwdnfL43JOhpJHUur3hoYOrn/zc33qSeOsTQq/lzgnFxVeDimr9ksGiNhj7Kpu1y\nc+OwsYBCM7Tcd474s3KPWnAYwEUC0DkhChwYaGdlGW/ZhrePbVNNKfPGgcWqY368+J9Qb0WGi69o\n0t0aiO7RKSiuvwhd54vIRyZ8U8KbS1gFIzk5GSUl3pfRbDYjKSmpiV+0DMFuwi2GSUhM9J9xsq0Y\nnRiD0cN6yt+TXP6+eiXOwhsh2KJgvEFqRETROwW57sKtiO3sRKUuR+WPBgDHuZthuDZwNoNgjQFn\nvtSov7cAACAASURBVAlOUxn013gH2Ik8B74mCbrOpRDAQ+ARsv2oFIymetUeGFaA0BArWRz6ptMZ\nAcBR0Bs6PhrstVJDqKyLJ2/9A1Zs2QNLVK5PaiQDR/5NMPQ86X9CNzpnHKyFqTAm1AIp3nRa0aUH\n6juDib8ITic0Z6VWVcNo0hkRTIoYVgRviQcXaVG7YLTOLQLOwt4Q7ZHyc+EeHgJRBBynByE6wQpH\npzxV3EfkWbiKr4f+vwKP4BUaYjGwy63o05fFN8d/kLcnmOJQYdaBi6tE5zgjqp2h9269FoYTsIXW\nIePrOoGNqZZdfAHPLbBwXrgRoj3C7x0xsHpYcm4GG1EPXfezqmwv0aWD62JP6HtImYEMGL9VJjl7\nAl4YOxmL1/2M+jjvuyTYIgBeDzaqDnanCF+vVJPlVVgYTYmnqhwNSeCjgntUInQmTOs/CZ0i4vH2\nzo+l67nrIkofiT8OmY4SixnfHFunulfRYYSrops8Lk0UWL/MuL7J1+PhpEnIyjmF265XT6TaEsLq\nkurXrx8KCwtRXFwMh8OBzMxMjBkTeKBRSy0SobYLOhvjUV5uuWT/rPWKF9qp9u/ytZ1xW9KvINR4\nxVGwJLgPZlB/sTMcRb3AOvy7jwZLapM+UrExBjd06olZI+4DX9VV3j6g8wCIivhDc6rSxHobA18X\nTsByOI3ISNUeQKc6t7UbeHNPRNq7y9s8daFnDDh+gEV5bg+IPg0SyzB444HfeK+ncT8pkcmIZ1Ng\nvdALfI03lchV3h2dTPEAgGYnhijcNjGmEM0+pxGdGwcGPYyvSgZv7ql6LroaesjXFWqSMOPWCUiO\n7uLzQz1cZdd4v2uIumiNwU0JaUhPGo4Ywftc3J48RLaaWK5575fcSDJo0tpS/cZuguvCDUGP4ytT\nwJtTVXXRKyYVABBtiMZN8b0xrtdoxBli1ed3mFR1EaX3FzK9Mw7dddfgGnEghAbv7/mya7wrZmqk\nmjd5Xwr3XCidKgBwNJjgLAreSA9NHoTB8YPRy5gmb/O8I9G6aKSZrseIxOGINqjji4ItEryiLvSi\nvwJ24rqgp+la/G5ABm6Man18N6yCwXEc5s2bhxkzZmD8+PHIyMhAWloaFi9ejJ9/lmanPHbsGEaO\nHImNGzfi5ZdfbnG2VJe41gd0moMy6M3XqV/waKMRv79H7VsRLFKgSyeaIIoMymtsMDD+DRLLMPID\nyYoaq301xiA6Qo8eSdEqYYmLiAr5QfYlWq8QrhDP0aNTPPp3D5DPq8CzJnqUydtyJ+ok8eAEEyxW\nqQ+v1WvrFBPhnRvMpbFfn4iYCPd2RV2kxMXhum5SfWtNW90UUTrvS9k1LjSLVXTp0SMmJehxqUnx\nuHvINZh4R09527VxUh167j8hxoR4k8Z1FX8XTlC4EN3TSAiNMeifJj2HKZ28jWSEzoTru0uNT3PW\nslaWCUCTfnsVvB6CNQQ/msZz1ruL1LjGGmLw7EMDcV96GjpHxvodp/xtjMF7LUaQnhUjL93vrwf3\nQJTB6zoWeZ13UFwz0mUBAO6gP8MKIYun4OL8OkJaaK7q524vYo3eZyFa75uQwqjKYlLk9nv+dt2j\ngz+XzSHsA/fS09OxadMmbN68WR6DMWvWLIwaNQqAZIVkZWXh8OHD2LNnD9atW9fsa/CVKUiMb33K\nWHNQCUZliqoH7OKlqRfmPupN8R3V93p0jUhCDOMVlwifwRsjuqbD4RLkjBQjEwlnaU8AwJCkWyEK\nDIT6BMRGGRAdofc+/ACiDRFBsjcAoV7bjZZolHp5fHViyIJxXUoXJEcGdy96Jg5U+nRv6poK0R4B\nV0M0KmvdKZxOdUxo4rWSrzVSF+neb8CAToOk+yi/BqLAontUD0SapJdNVNTFqFt6IjFW+h0fQDDi\nWe2yj+s7AADgqkhBbERoz5TI69A3uWfQ43p1jcNDY67H5BHXytuu75wK0WGE0Cg1jF3iTH695ju6\njgDAyFlOejECrkqpIeDLu0PkOSQZU+SBYUadV5xNughc00X6uwcadxPPddHcLtRK22+JH4S+qf5j\nebQQeZ2qVx/wOI1nNS2uJ6L1UeihaOSUsULe5kL5zxYfwYjBzZ1vAgBwlu4QXTpEuweu9E5NwH8l\nSfd+7suj4K2C97qMdl0IARr4G+Ikq8lZ2rN54lmfEPQwrcHGQl0niC49ronpJm9jfSYkdZlTAV7x\nt2Yj0TO6p3TpmkQwvEF7rE0ruKJHeg/pOgjWQ6MhWDpfBgvD+yALNYmwHR4tN8icTlKPa7t5X5zr\nuyXghSFPYWrqA/K2CL23QbIeGo1f9xgNp0uQXUscy8B14Ub0a5yGaTfeB9uRURBtUeiRGC1NeKd4\n6WKMkSoB0cKeeytsx37lF9hOik6A9eAYOM4MDCn3HpB6rnpOj7dG/N1vn+PczZK/GN6pv5Vz2XSK\nikavugxYTvVFbqE7k8Z9LybOhNeH/y9+/V8jAQBdIjoDkHrnM/o/gLdGvILOllthO3InkqO7yIKh\nrIsInSnoiP+HUh/FX4fOhu8MrCP7XIdOhRMx7YapoSc28Dpc0zkefx86129XVNlQGFnpPFqLeyVG\nxeKGhkkYl5KBJc+MQIRRJzcMXSI647Xh8/DwgLsAeEVVzxjhPNcP1oNj4Mzvg/8d+iJenj5cPqey\nxxqpM8lWmmYaLYCpPR7Bi0Nm+W0fe8tNGGB7GI8PeACdogKnW6vgdYDLiGHib/12/bHf7+TPCdH+\nYhxnjMXLt72AqTdM8tvXPbI7os6MgL40H+/NGiH3tmP0Ufjvfo/gzeEvgynuB9vRkRjRN1X+nefe\nr/3tLWC5qKAWxq3Mfegv+ns5pgwdAOvBMXBduDF095xLB9ERAduRkX777DlD5M+cxrLSoiMCMQV3\na86bdnPnm/Dszc9BqO6qeuc5wYTZt85Er4oH4Mzvi8TSe2DStS45x5crcvJBDxzLyq6K+Ji2rZhg\nqNNqGcQao5GSHI/C+lokd/YvC8fqYNKZ0LdnEgApkBtl1EOeecBlQKRRauQ8QWi7aAXAIMoQCaNe\nJ99rjyS3Ga54WCL1EcGtA14H0RorNa6KoGqXmEhvT0UI7ZHwTMIYqfd/8QVrtFyW7okm9L61BzJu\n74m/uudnjDQacH23KJw6Xw8R0pQJid1icUGQLBGlmyHW/bnR1QiO5RDJRqJrQhRKKqyIidAj0ugu\nr+i99widSTVOQ4teyQmIjtCDY1iVFWIy6DH/Manx/f5saFMpiLwe8TFGxEb6u82eGT8ciw+fhF2w\nq6aTl6+nM+GZ+wMPNlUuHSo6jUBEAwTWLv3t3YMMk+PiVb9RTvFvUghGIPdccnwskqP9e9YPjvb6\n30NNI/dYxz27dMFen+nFukWnyEHqO25OwVqfBQQjdCbN5wkA9DoO0ZX7UV5Wimee+j2c3VmwPU3I\n/PJbXEg9hTNn8vC/ry3Fq/94Cf/O/RKfOhx44IFp0PWU7v3Uwl1IHdUfMXE8sv+1B7GpXWAprIQ+\n1oieD/cH656S/vF7+sPBO/H40s9gzsqHyIvgInWIfNMhWQwuB4o374Wt6iLAMEi+sxfi+iSi7nQl\nLv50DqIoQhepR9pjA1F+6AAYZz5uHT4eJQByl+xFr9/eAkBE1c8rwZ8R0XihDqNe+hXefvt15Oae\nxPnKAsT1TUJc3Gjc0ee/cDo3F4sX///2zjwgynJ7/J+ZYdgZkE1kEVEUccEdUMktrpgrXEWvZup1\nrVxyqUS+Zd+ytG96vdXtdjXNTLMsb9mvm7bp1dJETZOstMUV0QABkX0GmOf3xzADA4MMCiLwfP5i\n3vV5D+/7nOc85zzn/I2iomIydVn4P9SVr/7+EYOejDTJ5vzmU/iO6YitnwIbpQ2d/Dw5cyGP0IDW\nFiR5ZzRthaFQsmZeJEXaUlMd5rtF1YV7M0Z05nBBeSoGC4kAjVla1TZKEh/qQ+r1fHKdfuZcpSzQ\nduU5qYwjyTJRitpGiZ+n+eiuTXlo76Du/hy5blgx7WBT+5SUh8aBrNxibO2gcr48D03FR9on2IfT\nxbWXfDQqDIuUVkyX6SljSrS5E9TTxQkP/4pOblAPX0q8s7iSWt3sdrUzWGk6fUW8Ukd/N36+mI2P\nh2MNFoZDrXVRnMsTRtY0ZQV1WJxZamO6XlVcbJ1NI35LU0KWLCFjidiq77TQGd4LPbdeBFe5/oqj\njQM25fewVB8DwENjX+vC2arWVklKCGXZ1ac7nNVO4KjiP99eorjYfGT94i9nKSoeBMAXP1W00TgC\n/0yfxoPR5lM4xvdBqVDwyCMLuXTpAlu27OC15M18d/IYWZczmLd6AT4+hrb8428v4eLiglarZc6c\naQxYEGO4kALG39eZayWlnMguos3kQHzHBnPpg5+4eSaDVmEVz2KjVOEU6EbHuQZFnnXyGh/9+11a\nufTlt+N7UTnbEzLfkAKkrLiU0gIdqZ/8QvCsPti62VNWntbF+E56t3LkWnkbjGRl/EH7sb3wHx2C\nq4cb8+bNx8XFhUf3PcH5raeYMdONmHB/pj44gVWr/o+QkM68cHgdV4vT6TiwG/v37SXhwVno8q+z\n/AclDq2daeVm6D9G929HgJczYcEeNf9Db5MmrjBUtG7VsOsvakKpUBLh04fDxw2LYezUSpNSqGz6\nz+gymYOp3xLSqiKvULCfK8F+ruSXuHH2xq/8/p3hZVUplSyOD+M/R+2xdSlhVPtoggZ2MCmSByLa\noivRm+ovuDk7QnmNFQcbe/Q3PdHnu1Ka1g6hszdLuT6gTTgTB0VSWqbn8W8/N20vve6Lb1jFiN5W\neetO0sfRG7VKTadWFaPPv4T8mTNZv9LFI4Sfs85yXOtoUl6VE9xNDhnP0T9O0MmzLcK94uvx93Im\nOHAoF3IvE99xrNn9YtoN43LuFTPTfHi/AAb39MXBzgbHcme6qDIl1ce7J99eO84D7e7HwcaeV069\nYdpvn1sRjVKZKN8Is9+1TUn5OvmgVKiY8ZcHTJ37hI5juXDzEu1d23E+5yLOaieLU0ITO8WSfP0n\nPByqz3GP6zCC9MIMpnSeYNo2d0wXvjljh975JJqcXmTXsA4AzKtO2tvYE+UXwanrpxnXYSR//yDZ\nUN60nJI/2pkWo1ZmqH+U2W/7WmRhTLBpo7JBWa7wHNQOlOnLsFGqKNWXoUCBQqFACPNAWDuVPXpR\natH5G99pHFt+3sGDnSdAXoXC+0tIHJfPXsC+c6hJWQB88MG7HDpkqLmRkZFBQWaeYb2FgB5BrelU\npOFD97dZ/KcFFJYU8sKh59HdMPjRHihPFa9UKNHdLOba++cozdfiqHTgYuB5fPsNJjnzdwL6RWH8\n8FT2NuT+molzOzdsy1dSt/MKRKlQkVpyBYUSfNwdKLnQAVHyPX1ahYO6mJutU3D0MwyGSvQl7N//\nBZ988jE5RTmU5WhxFHmkXrmMp6cXISGGAJq/9pzK9rPvMzU+nmUPL2D+/MVs2vQO8eMmkuacR2z5\nN6JUKujVyTqfU11pkgrDaNZWHY3ebaZ1mcTBTwy1I1QqpWkkV3nBUD+fXvTzsRxy6ax24vG+85nz\n34OmbWEdPMujXQZUOz5+qHmIXuVRtIONPQvjelGk7c7mMwZbf1HPubyabOgoHww1dD5qmwqZ6S52\npex6ABonW8I6eHD6fJbBF3KL2Zy2Gn+md/mL2bb7/CK5zy/S9Pfxvf81WRiVZRHlF0GUX3mnrDQ4\neDNvFhPg7YSrnSNP9l1Y7X4aWxce77vAbJtSqcChfCpKbSxeJMwVhqPagYR+j5m2ze72EJt/2g6A\nT1F4tfvM6DK52v/pllYU0N41kMmdx5ttGxoQxdCAKNPfQCULo0IWg/0HMNi/+v8YoJW9WzWfQmRX\nHyK7+gAD2fbFr4Ah99Hi+LBq59tUeS9cbJ3L/TWgv3kV3cUu2AYZpkVn9x5f7fy53afRw6ub2Tb7\nKrJQt/3VbF3IsID7GN+x9gjHxMOruKnLY0CbcPb/P0Mk0HMzBtSY58jb0dP0f0zLq1g57+ngwZTO\n49mZXFFX4tSpk3z//QneeGMrtra2LFw4j8rGmL2NA2pbG3w0renUyjBoCPUM4ec0w/cyun2M6dir\ne37De2BbEiclUHw5j7fe2kRg6/LpwSrTtlVDvrt6hjKmfQzjNvwPQgjUNipKr3ZEX6xioO9AvFzt\nSXLYh1KhRC/0ZKZfZ+/OXbz55nacnJxZvfpZdDpttev6OvuwvFwW/fpFcOjQQQ4c2MfmzdtxcWm4\nNWiVaZpO7/LBaWMrjMooFFQaSVqfNlylvP1nqNox9OroxYBubVgzL5LnZlbvFKuiLzS8ZPa2Khb8\nuTuLJoTRJeDWkU9WO4ItWBhVeWp6X5ZP6YV3fViJVaakqu2uNPVkmsaqhKXww9oUhrWysLmN9+JW\nGG0zJ3sbUyhtZdQK8/ei+gUqeqLw0Orz3LcnC+um7yrLomuQQWG0stL/6OjoSGFhzaOZgoJ8XFxc\nsLW15fLlS/z8808mqx/AVmmMqKvcE1tem6LXlmGjscPPuQ2fffYpAKP6B9ItrA/ZP1WsWi8rKsEp\nQEP+pRx0OYbsC/oiw/95YJ/OaG9epXt7d4pvplJSdANHu4piakZZFBYW4uDggKOjE9nZWRw9egSA\nwMB2ZGVl8ssvZ03H6cvrqI8ePY6XX15HaGjXu6YsoIlaGEoUlCEslrhsLOzUKtztDPPybna3XgVe\nlRHhbU3RRHXB3MKo6CSN03QXbt46Umh8eA/Ss3SmKa6ewZ5cyr21s9jGQrnZqswaFcpnqRe4QSau\ndjW/zBpHWzRt66cSWGULw1JkiLFzF0JhSA5XhdaO1U14S4qnMtY6gr0cPEjJSzVz5t8Jrs6G+3q3\nstw+o6WrVqqrTfMsndSDXT/kkImluH4D7vbVp8nqS3l6O3qRVXwDZ1tHpsX3oEwvrB40aTSudO/e\ng+nT/0JExAD69x9otj8iYgAff/whM2ZMoW3bQLp1627qIxQKpclPU9lfY5yCrfr/bz20HZd3/khi\n0jK6dOlGWtofONjZ8NcZs0l8IYFfXzsGSgU+Q4NwDfXCf2xnLr33I0JAgfc1xr0+ikUz47n221Ge\nXj6Pm8Vu2Dp5mSxjhUKBj6M3KXmpBAa1I7tjCA89NAlfXz/CwnoAYGNjw7PPruHvf38JrVaLvb09\nL7/8Ovb29oSEdMbJyYlRo+48y3ddaJIKw7D0lHrJcHunPDWtLz9fzCLA2xlvj6GUCT2D/PvX6RqV\no1HqgrFjUCqU2FpwngZp2jKyXTTdvbqYbV/Ycw452ptEtqk+l19bJ1lioQRqVQZ2b0Ovzg/y5eUD\npmmZhiKgtaET9nXXcB3DXLsly7OLRwhlf3SgJNMHx7AKWT3aYyZFJUUW667X1kmKGpzIVZnYKZZW\n9m6mUOE7JaZfW7S6Mob29rO43+jDsNT+bkEehAaO49OLDkT49DHbN7f7dMpEmUX51fZeqKy09h8K\nnciBK4cZHjgUpVJhVsLVGlauXGX2u1evimdQq9WsW2eet2nvxa84e/EiUStGotG4otG48vbbO037\nl89NYM/FrxhYxX+1dMISbOJVhHl1Ndvu6a7BN3I0duX5voxoOnqg6WhwMj/Y2RA6b+jg/wnAzBcN\nU9e+bQzrKt5+eyfZxTf4JjWJoQH38UBitMXn7dw5lI0b36q2PTPzOkII+vWLtHBWw9E0FcY9RHtf\njWm9hb2NHbHBI+/avdXloycHG8tRLgqFglHtqxek6uxevbCPkdo6Sa0VCgMM4bZ3QxZd27mzYmpv\nrul/4YNz39XYsSkVSsZ3Gsl7V36nf9cKJ2lXj5qzHdbWSWr11iU9dLZ1Ii54lFXHWoOdraqaP6sy\nxiipmtqvUqosxvf3qNI5Vqa290Jn5SpyVzvNXf1GjBaWQ03huiq1xfb09q7uGwJDgMbU+7uy6+oJ\ni/vBfPrTyEsP96ekSu13d/tWtyWLzz/fw6ZN/2LRoqV1PvdOaZIKw5i6+25mw70XsVEZRsrVcvvf\nAbVdS1dmXVnWu0lHfzdupBmmFm7VsUX39WdYHz+rp0BqVZ6l1mfJvZsYpypra39dqH0gca/Kov6/\nkVB/L2PMgUV0FmThWY+ZKEaMGMWIEfU3AKkL947XuA40/kTUvUFDdAxqlWW/h/HDC27V3uL+xsbY\nvqrRPJVRKBR1CjKoKZTU6LsIcg20uL+xsbmLCsMoi7Yu/hb3NzYNIosarBXjtHAb5/pNx3Ev0SQt\nDGNioqppjVsatU093CkrIx6nVJRRUFJAB9cgLuam0P6e7SQN03OO9dgxVPZrPNd/BcVlxRSUFNDe\ntR2Xcq/QwbVdvd2rPmmITrKyU/v5AYkUlhZRUFJIkGsgKbmpdHBrV2/3qk8qZFF/30hla+W5/ivQ\nlmkpKCkgUNOWq/nX7tmBRH3QJBWG0cJo6QrjVs7N+sDN3s0sXDLYrfbstI2FNRbGneBu72bmJ7q3\nZVH/70XlZ29l70YrKlbq36vKAhreCq+68LI5KwtoslNS5S9vy9YXDTJ6qoy1kS/3AkZZODaQLO6F\niDxrMc3bN5AsmhINoTBaMk2nR7BAS7cw3O1b4ah2IMDFcnjl7RLqbsj9dC+tc6kNTwd37G3s6l0W\nHVzbWQxZvpfxcvTEzsaOAGff2g+uA/7Ovrio62ctye2Qn5/P7t3/rtM53o5e2Kls8Xfx5YMP3kOr\nrZ+gDU8HDzzt3evlWk0JhWiCoUbTPlxMcanW6nQEzRl3D0eys2692K6uCCHQC73FtQn3MlIWFTSU\nLBozJc8ff1xj+fIlbNv2fp3OM8oiPn4sb765HY2mbgtrLWFM5FhXWZSVlaFSNd67dKdlrJuoD0M6\nvY00REemUCialHVhRMqigoaShaIRYxQ3bHiNa9euMnPmg/TtG8Gjjy7i3Xe3c+DAV5SUlDJo0BBm\nzpxLcXExK1cmcP16Bnq9noULF3DpUiqZmddZuPBh3NzceOWVf5lde+vWzXz77SF0Oi3duoXxxBOJ\nAFy9msratavJyclBpVKxatWL+Pr68d672/nyy89QKpVERg5k3rz5LFw4jwULlhAS0pmbN3OYPXsa\nu3Z9wmeffcqRI4fR6bQUF2t58cW/kZCwjPz8PEpLS5kz52Giosoz9n72KTt37kCpVNChQ0eWLl3O\n9OmT2bnzI1QqFYWFBeW/dzeK4mmSCqOS11sikTQCH537lFMZP9brNXt5d+fPwaNr3F85vTnAd98d\nJTU1hU2btiGEYPnypfzwQzI5Odl4enrx0ksvA+DgoKBvX8H777/HP/6xEY2mekXA8eMnMWPGbABW\nrVrJkSOHGTAgimeffYpp0/5KVNRgSkpK0Ov1HD16hMOHv2HTpm3Y2tqSl5dXQ4srlOvPP//Itm3v\n4+zsjF6vZ82adTg6OnLzZg7z5hmuf+HCed55Zyv/+tcWNBoNeXl5ODo60rt3H5KSDhMVNZh9+75k\nyJD7G81KaZIKQ1oYEonk+PFjfPfdcWbOfBAhBEVFxaSmphAW1pN//vMVNmx4jf79o4iOvo+iojwM\nI0zLfcbJk8d5993taLXF5OXl0b59B3r27E1m5nXT6F+tNviyTpw4zqhRY7C1NUQQWpP8r1+/CJyd\nDf4fvV7Pxo2vkZx8CqVSQWbmdW7cyObUqRMMGXK/SaEZrzt69DjefXc7UVGD2bv3PyxfXr2y492i\nSSoMiUTSuPw5ePQtrYG7gRCChx6awdixcdX2vfnmOyQlfcvGja/x228/Eh//UI3X0el0rF//Elu2\nvIOnpxdbtryBTqejJuVicPtWn5pTqVSm/GKG8ytwqFQf/quvPicnJ4e33tqBUqkkPn4sWq2uxswV\n3bv3IC3t/0hO/h69Xk9QUOMtnm2SUVJyRkoiaXlUTW8eERHJnj2fUFRkSCtuGKnfIDMzEzs7O4YP\nH8HkyVM5c+ZM+flOFBQUVLuuTqdDoTBkwy0sLOTgwf2m4729W3Po0EEASkpK0GqLCQ833FerNRRe\nys3NBaBNGz9++cVwrwMH9tX4HPn5+bRq5Y5SqeT770+Qlmao89GnTzgHDuwjN/em2XUBYmJG8r//\n+z+MGjXW4jXvFk3Swujg3o7T6WfxtJCGWSKRNE+qpjd/9NFFXLp0iYcf/itgUChPP72K1NQr/POf\nr6BUKrCxUfPCC4YMt2PHxvL444vw9PQyc3o7OzszZkwc06ZNok0bX0JDK5IwPvXUs6xdu5rNmzei\nVqtZtepFIiL6c+7cb8yaNQ1bWzWRkQOZO/dRJk9+kKefXsEXX3xGnz79anyO4cNHsHz5UubMmUZw\ncAiBgYZFoEFB7Zk2bSYLFsxFpVLRsWMIiYnPlJ/zAJs3byA6unoy0btJkwyrzdXm89WZIwzw7Wex\nrGNLwsvLhevXa3K6tSykLCqQsqigOcjiwIF9fPvtIZ566tk7uk6LDKvV2DnXueaERCKRNEVefnkt\nR48msW7dK43dlKapMCQSiaSlsHjxE43dBBNN0uktkUgkkruPVBgSiUQisQqpMCQSiURiFQ2uML75\n5htGjBhBTEwMb7zxRrX9Op2OJUuWMHz4cCZNmsS1a9caukkSiUQiuQ0aVGHo9XpWrVrFm2++yaef\nfsqePXs4f/682TH//ve/cXV15csvv2T69OmsXbu2IZskkUgkktukQRXG6dOnCQwMxM/PD7VazahR\no9i/f7/ZMfv37ycuzrC0PyYmhqSkpIZskkQikUhukwZVGOnp6bRp08b0u3Xr1mRkZJgdk5GRgY+P\noWi6SqVCo9GQk5PTkM2SSCQSyW3QoArDmkXkVY8RQjSpcpgSiUTSUmjQhXs+Pj5mTuz09HS8vb2r\nHZOWlkbr1q0pKysjPz8fV9faK2Ld6RL35oSURQVSFhVIWVQgZVE/NKiF0b17d1JSUrh69So6nY49\ne/Zw//33mx0zdOhQdu/eDcDnn39OZGRkQzZJIpFIJLdJgycf/Oabb3jhhRcQQjBhwgTmzp3Ls8Qs\nGwAACZ5JREFUq6++Svfu3Rk6dCg6nY4nnniCs2fP4ubmxvr16/H392/IJkkkEonkNmiS2WolEolE\ncveRK70lEolEYhVSYUgkEonEKqTCkEgkEolVNDmFUVtuquZGYmIiAwYMYMyYMaZtN2/eZObMmcTE\nxDBr1izy8iqqiT3//PMMHz6ccePGcfbs2cZocoOQlpbGtGnTGDlyJGPGjGHbtm1Ay5SFTqcjPj6e\n2NhYxowZw2uvvQZAamoqEydOJCYmhqVLl1JaWmo6vrnna9Pr9cTFxfHwww8DLVcWw4YNY+zYscTG\nxjJhwgSgnr8R0YQoKysT0dHRIjU1Veh0OjF27Fhx7ty5xm5Wg/Ldd9+JM2fOiNGjR5u2vfTSS+KN\nN94QQgixceNGsXbtWiGEEAcPHhRz5swRQgiRnJws4uPj736DG4iMjAxx5swZIYQQ+fn5Yvjw4eLc\nuXMtUhZCCFFYWCiEEKK0tFTEx8eL5ORk8dhjj4m9e/cKIYRYuXKleO+994QQQuzYsUM888wzQggh\n9uzZIxYvXtwobW5I3nrrLbFs2TIxb948IYRosbIYNmyYyMnJMdtWn99Ik7IwrMlN1dzo27cvGo3G\nbFvl/FtxcXEmGezfv5/Y2FgAevToQV5eHpmZmXe3wQ2El5cXoaGhADg5OdGhQwfS09NbpCwAHBwc\nAMOIubS0FIVCwbFjx4iJiQEMsti3bx/Q/PO1paWl8fXXXxMfH2/advTo0RYpCyEEer3ebFt9fiNN\nSmFYk5uqJZCdnY2npydg6Eizs7MB87xcYJBPenp6o7SxIUlNTeWXX36hR48eZGVltUhZ6PV6YmNj\nGThwIAMHDiQgIACNRoNSafikfXx8TM/b3PO1rV69mieffNKUUujGjRu4urq2SFkoFApmzZrF+PHj\n2bVrF0C9fiNNqqa3kEtGbokl+TS3vFwFBQUsWrSIxMREnJycany+5i4LpVLJxx9/TH5+PvPnz69W\nNgAqnreqLEQzytd28OBBPD09CQ0N5dixY4Dh+ao+c0uQBcDOnTtNSmHmzJkEBQXV6zfSpBSGNbmp\nWgIeHh5kZmbi6enJ9evXcXd3BwwjhLS0NNNxaWlpzUo+paWlLFq0iHHjxhEdHQ20XFkYcXZ2pl+/\nfvzwww/k5uai1+tRKpVmz2uURV3ztTUFvv/+e/773//y9ddfo9VqKSgoYPXq1eTl5bU4WYDBggBw\nd3cnOjqa06dP1+s30qSmpKzJTdUcqToSGDZsGB999BEAu3fvNsng/vvv5+OPPwYgOTkZjUZjMkWb\nA4mJiQQHBzN9+nTTtpYoi+zsbFOkS3FxMUlJSQQHBxMREcHnn38OmMti2LBhzTZf29KlSzl48CD7\n9+9n/fr1REREsG7duhYpi6KiIgoKCgAoLCzk8OHDdOrUqV6/kSaXGsRSbqrmzLJlyzh27Bg5OTl4\nenqycOFCoqOjeeyxx/jjjz/w9fXllVdeMTnGn3vuOQ4dOoSDgwNr1qyha9eujfwE9cPJkyeZOnUq\nnTp1QqFQoFAoWLJkCWFhYSxevLhFyeLXX38lISEBvV6PXq9n5MiRPPLII1y5coWlS5eSm5tLaGgo\na9euRa1Wt5h8bcePH2fLli1s2LChRcriypUrLFiwAIVCQVlZGWPGjGHu3Lnk5OTU2zfS5BSGRCKR\nSBqHJjUlJZFIJJLGQyoMiUQikViFVBgSiUQisQqpMCQSiURiFVJhSCQSicQqpMKQSCQSiVVIhSFp\n0kycOJG4uDhGjRpF165diYuLIy4ujsTExDpfa/bs2Valu16xYgXJycm309w6cebMGb744osGv49E\nYi1yHYakWXD16lUmTJhwy+yjxlQRTYVdu3aRlJTE+vXrG7spEgnQxHJJSSR1ISkpibVr19KzZ0/O\nnDnD/Pnzyc7OZseOHaaCOgkJCYSHhwMwePBgtm7dSlBQEFOmTKFXr16cOnWKjIwMRo8ezeLFiwGY\nMmUKjz76KFFRUTzxxBM4Oztz/vx50tPT6d27N2vWrAEMuXmefPJJbty4QUBAAGVlZQwbNoxJkyaZ\ntTMzM5Nly5Zx48YNAKKiopg9ezavv/46hYWFxMXFERERQUJCAqdOnWL9+vUUFRUBsGjRIgYNGkRK\nSgpTpkxh9OjRnDx5Ep1OxzPPPEPv3r3viqwlLYQ7KdYhkdwrpKamisjISLNtR44cEV26dBE//vij\naVvl4jLnzp0TQ4YMMf0eNGiQuHDhghBCiMmTJ4tly5YJIYTIzc0V4eHhIjU11bTv0KFDQgghHn/8\ncTF16lRRUlIitFqtGDFihDh27JgQQohHHnlEbNq0SQghxJUrV0SvXr3Ezp07q7V98+bNYuXKlabf\nubm5QgghPvjgA7F06VKztsfGxoqsrCwhhBBpaWli0KBBIj8/X1y+fFmEhISIPXv2mJ59yJAhorS0\n1HohSiS1IC0MSbOmffv2dOvWzfT70qVLvPrqq2RkZKBSqcjIyCAnJwc3N7dq5z7wwAMAuLi4EBQU\nREpKCn5+ftWO+9Of/oSNjeFT6tKlCykpKYSHh3Ps2DGef/55APz9/U2WTFV69uzJO++8w7p16+jX\nrx9RUVEWjzt58iSpqanMmjXLlJBSpVJx5coVHB0dcXBwYOTIkQD0798flUrFpUuX6NChg7Xikkhu\niVQYkmaNk5OT2e8lS5bwzDPPMHjwYPR6PWFhYWi1Wovn2tnZmf5WKpWUlZXV6Thr6yz06dOH3bt3\nc+TIET788EM2b97M9u3bqx0nhKBr165s3bq12r6UlJRq2/R6fbOq9SBpfJqOB1AiqQVhRfxGfn6+\nKTvpzp07a1QC9UF4eLgprfTVq1c5fvy4xeNSU1NxdnZm5MiRJCQk8NNPPwGGWhfGNOYAvXv35ty5\nc5w4ccK07fTp06a/i4qK2Lt3L2AoUQoQGBhYvw8ladFIC0PSbLBmNJ2YmMjcuXNp06YNERERuLi4\nWDy/6rVq2ner455++mmWL1/Onj17aN++Pb179za7n5GkpCS2bduGSqVCCMGqVasAGDhwIG+//Tax\nsbFERkaSkJDA66+/ztq1a8nLy6OkpISAgAA2bNgAgKenJ7///jvx8fHodDrWr1+PSqWqVSYSibXI\nsFqJpIHQarWo1WqUSiXp6enEx8ezY8cOAgIC6v1exiipw4cP1/u1JRIj0sKQSBqICxcusGLFCoQQ\n6PV6lixZ0iDKQiK5W0gLQyKRSCRWIZ3eEolEIrEKqTAkEolEYhVSYUgkEonEKqTCkEgkEolVSIUh\nkUgkEquQCkMikUgkVvH/AcQ/YGad+SX7AAAAAElFTkSuQmCC\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f971b401110\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" } ], "source": [ @@ -387,41 +351,25 @@ " train_ds = setup_mnist_data(True, hp, 500)\n", " test_ds = setup_mnist_data(False, hp, 100)\n", " tf_train = autograph.to_graph(train)\n", - " (train_losses_, test_losses_, train_accuracies_,\n", - " test_accuracies_) = tf_train(train_ds, test_ds, hp)\n", + " losses = tf_train(train_ds, test_ds, hp)\n", "\n", " with tf.Session() as sess:\n", " durations = []\n", " for t in range(burn_ins + trials):\n", " sess.run(tf.global_variables_initializer())\n", + "\n", " start = time.time()\n", " (train_losses, test_losses, train_accuracies,\n", - " test_accuracies) = sess.run([train_losses_, \n", - " test_losses_, \n", - " train_accuracies_,\n", - " test_accuracies_])\n", + " test_accuracies) = sess.run(losses)\n", + "\n", " if t \u003c burn_ins:\n", " continue\n", + "\n", " duration = time.time() - start\n", " durations.append(duration)\n", " print('Duration:', duration)\n", "\n", - " print('Mean duration:', np.mean(durations), '+/-', np.std(durations))\n", - " plt.title('MNIST train/test losses')\n", - " plt.plot(train_losses, label='train loss')\n", - " plt.plot(test_losses, label='test loss')\n", - " plt.legend()\n", - " plt.xlabel('Training step')\n", - " plt.ylabel('Loss')\n", - " plt.show()\n", - " plt.title('MNIST train/test accuracies')\n", - " plt.plot(train_accuracies, label='train accuracy')\n", - " plt.plot(test_accuracies, label='test accuracy')\n", - " print('test_accuracy', test_accuracies[-1])\n", - " plt.legend(loc='lower right')\n", - " plt.xlabel('Training step')\n", - " plt.ylabel('Accuracy')\n", - " plt.show()\n" + " print('Mean duration:', np.mean(durations), '+/-', np.std(durations))\n" ] }, { @@ -476,10 +424,12 @@ "def train(ds, hp):\n", " m = mlp_model((28 * 28,))\n", " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", + "\n", " train_losses = []\n", " test_losses = []\n", " train_accuracies = []\n", " test_accuracies = []\n", + "\n", " i = 0\n", " train_test_itr = tfe.Iterator(ds)\n", " for (train_x, train_y), (test_x, test_y) in train_test_itr:\n", @@ -487,8 +437,10 @@ " train_y = tf.one_hot(tf.squeeze(train_y), 10)\n", " test_x = tf.to_float(tf.reshape(test_x, (-1, 28 * 28)))\n", " test_y = tf.one_hot(tf.squeeze(test_y), 10)\n", + "\n", " if i \u003e hp.max_steps:\n", " break\n", + "\n", " with tf.GradientTape() as tape:\n", " step_train_loss, step_train_accuracy = predict(m, train_x, train_y)\n", " grad = tape.gradient(step_train_loss, m.variables)\n", @@ -499,26 +451,27 @@ " test_losses.append(step_test_loss)\n", " train_accuracies.append(step_train_accuracy)\n", " test_accuracies.append(step_test_accuracy)\n", + "\n", " i += 1\n", " return train_losses, test_losses, train_accuracies, test_accuracies\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": 13, "metadata": { "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, - "height": 789 + "height": 220 }, "colab_type": "code", "executionInfo": { - "elapsed": 56025, + "elapsed": 53945, "status": "ok", - "timestamp": 1531163800231, + "timestamp": 1531534839296, "user": { "displayName": "", "photoUrl": "", @@ -527,56 +480,25 @@ "user_tz": 240 }, "id": "plv_yrn_t8Dy", - "outputId": "68be955d-61dd-43e4-b540-3794e3c8f990" + "outputId": "93f2f468-7191-430c-88d2-948b4ce1ea06" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Duration: 4.2232978344\n", - "Duration: 4.2386469841\n", - "Duration: 4.24286484718\n", - "Duration: 4.24036884308\n", - "Duration: 4.25758385658\n", - "Duration: 4.23242998123\n", - "Duration: 4.4213449955\n", - "Duration: 4.29613113403\n", - "Duration: 4.28209114075\n", - "Duration: 4.24192905426\n", - "Mean duration: 4.26766886711 +/- 0.055508619589\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXdgFGX6x78zW7KbTSE9JIA0pQkIooCgqBx2qiK/O0XU\n8zyFAw/w8MSCFcuJCHqKoFiwIHIgIgooaGjSCU1aaCEJ6W1btszM74/ZmZ2tWchuQjbP55/s7szO\nvDPZeb/vU97nZQRBEEAQBEEQ9cA2dQMIgiCI5gEJBkEQBBESJBgEQRBESJBgEARBECFBgkEQBEGE\nBAkGQRAEERIkGARBEERIkGAQRIjs3r0bt99+e1M3o14KCwvRtWtX8Dzf1E0hogwSDKLB3HzzzejZ\nsyeqq6s9Ph85ciS6du2KoqIiAMC///1vdO3aFQcPHpT3yc/PR9euXeX348ePx/Lly+X3CxYswNCh\nQ9G3b1/ceOONmDZtGgDgrrvuQt++fdG3b190794dvXr1Qp8+fdC3b18sXLjQp43vvfceZsyY0aDr\n7NevH3766acL+s6HH36IuXPnYufOnRgyZEiDzi/hfY/8wTBMWM5FEErUTd0AIjpo06YN1qxZg/vu\nuw8AcPz4cdhsNo+Oi2EYtGrVCu+88w4+/vhjj8/9sXLlSqxevRqfffYZ2rRpg4qKCmzcuBEA8MMP\nP8j7jR8/HqNGjcLdd9/doGsQBCHsHW1OTg6efPJJOBwO6sSJZg9ZGERYGDlyJFauXCm/X7lyJUaP\nHu2z3+jRo3Hs2DHs3r273mMeOnQIgwcPRps2bQAAKSkpGDt2rN99g1W42bx5MxYsWIAff/wRffr0\nwahRowCIQjN37lz8+c9/xlVXXYWCggKsWLECd9xxB/r27Ythw4bhm2++kY/jbSXcfPPNWLx4MUaM\nGIFrrrkG06ZNg91ul7fX1tbi7Nmz6N69Ox599FGUlpbKVlBZWRkEQcDChQsxbNgwDBgwAFOnTkVt\nbS0AwG6341//+hf69++Pa665BmPHjkVlZSXmzp2LPXv24OWXX0bfvn3xyiuv1HsfS0tL8fjjj6N/\n//649dZb8e2338rbDhw4gLvvvhtXX301Bg8ejDfeeCPo+QHAZDLhmWeeweDBgzFkyBC888478v3P\nz8/H+PHj0a9fPwwcOFC2CInogCwMIiz07t0bq1atwqlTp9C+fXusXbsWX331FebOneuxn06nw2OP\nPYa3334bX331Vb3HfPXVV5Geno7+/fuje/fuYNkLH+Ncf/31eOyxx5Cfn48333zTY9vq1auxaNEi\ndOjQATzPIyUlBQsXLkSbNm2we/duPPLII+jVqxe6desGwNcaWrt2LRYvXgytVov/+7//w8qVKzFu\n3DgAwJYtWzBgwADodDosWrQIM2bMwG+//SZ/99NPP8XGjRvx5ZdfIikpCa+88gpefPFFzJkzBytX\nroTJZMLmzZuh0Whw5MgRxMTEYOrUqdi7dy9GjhyJe+65J6TrnzZtGrp06YL58+fj5MmTeOihh9C2\nbVsMGDAAs2fPxoQJEzBixAhYrVacOHECAAKeHwBmzJiB9PR0bNiwAWazGY899hiysrJw7733Yt68\neRg8eDCWLFkCu92OQ4cOXfD/i7h0IQuDCBsjR47Ed999h61bt6Jjx45IT0/3u9+9996L8+fPY/Pm\nzUGPN2LECDz33HPYunUrxo8fj+uuu85vfKIhjB49Gp06dQLLslCr1RgyZIhs0fTr1w+DBg0Kag09\n8MADSE1NRUJCAm666SYcOXJE3vbbb78FjVssW7YM//znP5Geng6NRoNJkyZh3bp14HkearUa1dXV\nOH36NBiGQffu3WEwGC74+s6fP499+/bhySefhEajQdeuXTF27FisWrUKAKBWq5Gfn4+qqiro9Xr0\n6tVL/tzf+SsqKrB582bMnDkTMTExSE5OxoQJE7BmzRr5e4WFhSgpKYFWq0Xfvn0vuM3EpQtZGETY\nGDFiBO6//34UFBRg5MiRAffTarWYOHEi5s2bhzlz5gQ95l133YW77roLHMfhl19+wfTp09GjRw8M\nGjQoLG3OzMz0eJ+Tk4P3338fZ86cAc/zqKurQ5cuXQJ+PyUlRX6t1+tRVlYGQHSRbdu2DU8//XTA\n7xYVFeEf//iHbDUJggC1Wo3y8nKMHDkSxcXFmDZtGoxGI4YPH45p06ZBpVJd0PWVlZUhMTERer1e\n/iwrKwuHDx8GAMyePRvz5s3D7bffjrZt22LSpEm48cYbfc4/YsQITJ06FYWFhXA6nRg8eLDcZkEQ\n0Lp1awCi9fHOO+/gnnvuQatWrfDggw82OLZEXDqQYBBhIysrC9nZ2di0aRNmz54ddN8xY8bgo48+\nws8//xzSsVUqFW699VYsXLgQJ06cCJtgKF1MdrsdTzzxBP7zn/9g6NChYFkWkyZNChofCcTBgwfR\npk0bJCUl+ZxHonXr1pg9ezb69Onj9xiTJk3CpEmTUFRUhL/97W/o2LEj7r777gsKnqenp6OmpgYW\niwWxsbEARKtDsv7atWsni/a6deswZcoU7Ny5Ezqdzuf8HTp0wA033ICYmBjs2LHDbztSUlLw8ssv\nAwD27NmDhx56CNdeey3atm0bcpuJSxdySRFhZfbs2fjss8+g0+mC7qdSqfCPf/wDixYtCrjPypUr\nkZOTA7PZDEEQkJOTg5MnT8pukwshNTUVhYWFQTt/h8MBh8OBpKQksCyLnJwcbN269YLPBYjuqBtu\nuEF+n5KSgurqaphMJvmzcePG4e2335bTjisrK7FhwwYAwI4dO3D8+HHwPI/Y2Fio1WrZukhNTcW5\nc+eCnl+6zszMTPTp0wdvv/027HY7jh49iuXLl2PEiBEAgO+//14OZsfHx4NhGLAsG/D8aWlpGDRo\nEGbPng2TyQRBEHDu3Dns2rULgBjTKSkpAQAkJCSAZdmLijsRlyYRtTCKi4sxY8YMlJeXQ6VSYezY\nsXjggQc89tm5cycmTpwoj0CGDRuGiRMnRrJZRJhRjjS9R5LBRsN33XUXFi5cCKPR6Hf/uLg4LFiw\nAKdOnQLHccjKysILL7zg4xcPZcR922234fvvv0f//v3Rpk0brFixwud7BoMBzzzzDJ544gk4HA7c\ndNNNGDp0aMBjBjtvTk4OXnrpJfl9x44dceedd2Lo0KEQBAFr1qzBhAkTAAAPP/wwysrKkJKSgttv\nvx1Dhw5FeXk5Zs2ahZKSEhgMBtxxxx1yJ//AAw/gqaeewtKlSzFixAg888wzQds2Z84czJo1C9df\nfz0SExPxxBNPYODAgQDEDLLXX38ddXV1yM7Oxty5c6HVaoOe/4033sBbb72FO++8ExaLBW3btsUj\njzwCQLSsJDFJTU3FM888g+zs7KD/G6L5wERyxb2ysjKUl5ejW7duMJvNGDNmDN5//3106tRJ3mfn\nzp1YvHgxFixYEKlmEESjUlFRgVGjRtUb1CeI5kZEbcW0tDQ5HdFgMKBTp04oLS2N5CkJoskxGo1B\ng90E0VxptKB3QUEBjh496tf/nJubi1GjRiE9PR0zZsxA586dG6tZBBF22rdvj/bt2zd1Mwgi7ETU\nJSVhNpsxfvx4TJw4EX/60598trEsC71ej5ycHMyePRvr1q2LdJMIgiCICyTi6QtOpxNTpkzByJEj\nfcQCEF1VUo74kCFD4HA4fIrYedMIGkcQBEF4EXGX1MyZM9G5c2c5I8Sb8vJypKamAhDr2gBAq1at\ngh6TYRiUlRmD7tNSSEuLp3vhgu6FG7oXbuheuElLi2/Q9yMqGHv27MHq1atxxRVXYNSoUWAYBlOn\nTkVRUREYhsG4ceOwbt06fP3111Cr1dDpdD61hwiCIIhLg0aJYUQCGjGI0OjJDd0LN3Qv3NC9cNNQ\nC4OmYBIEQRAhQYJBEARBhAQJBkEQBBESJBgEQRBESJBgEARBECFBgkEQBKHAZDJh5crlF/XdGTP+\nCbPZVP+OLhYvXoilS7+4qHM1BSQYBEEQCozGWqxc+a3fbTzPB/3um2++A4MhLhLNuiSgFfcIgiAU\nLFjwHoqKCvHww/ehX7/+GDhwED75ZBFSUlKRl3ccS5Ysw9NPP4myslLY7TaMHftnDB8+CgAwduwI\nfPzxElgsFjz55BT07HkVDh3aj7S0DLz++hxotdqA5z1x4hjeeut12Gw2ZGdn4+mnZyEuLg7ffrsU\nq1atgFqtRvv2HfDCC69i3749mD9/jmvdEwb//e8ij2V4IwUJBkEQlyzLNuZh19GGLYmgUjHgOPf8\n5Gu6puPemwNXxH788ck4c+YUFi/+EgCwb98eHDnyB5YsWSavAT9z5izEx8fDZrPhb397AEOG3IyE\nhAQA7oWrCgrO4cUXX8NTTz2D559/Gr/9thG33HJbwPO+8soLmDbtKfTufRU+/vhDfPLJQkyePA1f\nfvkZli9fDbVaLbu7li79AtOn/xtXXtkLdXV1QYUonDRLl9T3m042dRMIgmhBdO/eQxYLAFi27Cs8\n+OBf8Pe/P4TS0lIUFOS7triFqXXrLHTqJApTly5dUVxcFPD4ZrMJZrMJvXtfBQC47bY7kZu7DwDQ\nufPleOGFZ7B+/U9gWXGZ3p49e2P+/LexfPlSGI21jbYMbrO0MBatOoTOrQcirVXkTTCCIJqOe2/u\nHNQaCIVwlAZRrlG/b98e7N27GwsXfgqtVovJk/8Ou93u8x3lqJ9lVX73URKoStN//jMPubl7sWVL\nDj799CN88cW3uP/+B3Hdddfj99+34O9/fwjvvPM+2rW77CKvLnSapYUBAAVloWciEARBhEpsbCws\nFkvA7WazCfHx8dBqtTh79gwOHz7kd78LKdNnMMQhISEBBw7kAgDWrfsRV10lrl1fUlKMPn2uxuOP\nT4HZbILVakFhYQE6duyE++6bgC5duiE//0zoF9gAmqWFAQAFZWb0uTytqZtBEESUkZCQiJ49e2PC\nhP9D//7XYeDAQR7b+/e/Dt999z88+OBf0K7dZbjyyp6Kre4YhhiQDp2ZM1/AW2+9BpvNhqysbMyc\nOQtOpxMvvfQczGYzAAHjxt0HgyEOixZ9gL17d0OlUqF9+44YMGBQvccPB82yWu3w6atwbbd0PDby\nyqZuSpNDlTjd0L1wQ/fCDd0LNy2yWq1Wo0JJpbWpm0EQBNGiaJYuqYTsMpTXNk4aGUEQBCHSLC0M\nc/pO8J23wGbnmropBEEQLYZmKRgSFbV1Td0EgiCIFgMJBkEQBBESzVIwkrViOm1JTW0Tt4QgCKLl\n0CwFIzsuGwBQYa5p4pYQBBFtNKS8OQAsW/Y1bDab322TJ/8dx44dvehjNzXNUjCS9YkAgCorWRgE\nQYSXYOXNQ+Hbb7+GzRad7vJmmVabFp8EAKi1U3kQgiDCi3d584kTp+Crr5bg119/hsPhxA033IiH\nH34UdXV1eP75f6OsrBQ8z2PChEdQWVmO8vIyTJ78GFq1aoV58z4IeJ6ff16LL774FAAwYMAgPP74\nZPA8j9dffxnHjh0BwODOO0fg3nv/7LfEeVPQLAWjdWIyAMDoJJcUQUQzK/J+wL7Sgw06hoplwPHu\nghZ90ntiTOe7Au7vXd58167tKCjIx6JFn0MQBDz11DTs35+L6upKpKam4c033wEAWCxmxMYa8M03\nX+Pddz90lTv3T3l5ORYseA+ffPIl4uLiMXXqJGzZkoO0tAyUlZXis8+WAoBcztxfifOmoFm6pDol\ntwUAmJmKJm4JQRDRzs6dO7Br1048/PB9ePjh+5CffxYFBfno2LEzdu/eiQUL3sP+/bmIjTW4viFA\nWebcH0ePHkbfvv2QkJAIlmUxbNhtyM3dh6ysbJw/X4R33nkLO3b8Lh/TX4nzpqBZWhiZ8ekAp4Zd\nU9nUTSEIIoKM6XxXUGsgFBpaS0oQBIwf/yBGjBjts+3jj7/A779vxYcfvodrrx2ABx98JORj+ivj\nFx8fj08//Ro7dvyOFSuWYePGn/H008/7LXHeWGtgKGmWFgbLsNA6kiHEmGB1RGdwiSCIpsG7vHn/\n/gOwZs33sFrF+nXl5WWoqqpCeXk5YmJicMstt+HPf74fx48fc33f4KouG5ju3a/E/v37UFtbA47j\n8Msv63DVVX1RU1MNnucwZMhNeOSRx3HihHhMfyXOm4JmaWEAgA7xsKMUJaZKtE/KaurmEAQRJXiX\nN584cQrOnDmDxx57CIAoKM899zIKCs7hv/+dB5ZloFZr8OSTTwMARowYhSefnILU1DSfoLdU8jwl\nJRV///skTJ78dwDAwIGDMXjwDcjLO4HZs1+EIPBgGAaPPTY5YInzpqBZljcHgH9+tQBFqv24r8ME\nXNehR1M3p8mg0s1u6F64oXvhhu6FmxZZ3hwAWunECy+urWrilhAEQbQMmq1gpBrEyXtlJkqtJQiC\naAyarWBkJoiT96rqyNQkCIJoDJqtYGS3EgXDaCfBIAiCaAyarWBkuMqDWLmmSS8jCIJoaTRbwTBo\nYwEBcIDW9iYIgmgMmq1gsAwLhteCY2x+Z0wSBEEQ4SWiglFcXIwHHngAd9xxB4YPH47PP//c736v\nvPIKbrnlFowcORJHjhwJ+fhq6AC1A1Ybre1NEAQRaSI601ulUuHpp59Gt27dYDabMWbMGAwaNAid\nOnWS98nJyUF+fj7Wr1+P/fv3Y9asWVi2bFlIx9cyMbCrjKgx2xCra7aT1gmCIJoFEbUw0tLS0K1b\nNwCAwWBAp06dUFpa6rHPhg0bMGrUKABA7969YTQaUV5eHtLxdaweDCug3EiZUgRBEJGm0WIYBQUF\nOHr0KHr16uXxeWlpKTIzM+X3GRkZKCkpCemYsepYAEC5iVbeIwiCiDSN4scxm82YMmUKZs6cCYPB\n4LHNX8BaKtAVjLS0eCTHJeBcNWCFrcE1UpozLfnavaF74YbuhRu6F+Eh4oLhdDoxZcoUjBw5En/6\n0598tmdkZKC4uFh+X1xcjPT09HqPW1ZmRCyrAwAUlFe02OJiVFjNDd0LN3Qv3NC9cHPJFx+cOXMm\nOnfujAkTJvjdPnToUHz33XcAgNzcXCQkJCA1NTWkYycbxCUQq620tjdBEESkiaiFsWfPHqxevRpX\nXHEFRo0aBYZhMHXqVBQVFYFhGIwbNw5DhgxBTk4Ohg0bBr1ej9deey3k46fFiYJRS+VBCIIgIk5E\nBePqq68OaV7F888/f1HHz4gTLRETRxVrCYIgIk2znekNAGl6UTDsrJFmexMEQUSYZi0YerUOKl4H\nQWuGxeZs6uYQBEFENc1aMABAJySAibHCaLU1dVMIgiCimuYvGEwcGEZAhYUm7xEEQUSSZi8YWlYL\nALDY6pq4JQRBENFN8xcMlQYAYCLBIAiCiCjNXjBiVC4Lw06CQRAEEUmav2CoYwAAFoe9iVtCEAQR\n3TR7wdCpRQvD6iALgyAIIpI0e8HQa0QLo85JabUEQRCRpNkLRqwsGOSSIgiCiCTNXjAkC8NGgkEQ\nBBFRmr1gGGLENTFsPAkGQRBEJIkawbBzJBgEQRCRpNkLRrxWFAwH72jilhAEQUQ3zV4w9FoxhkEW\nBkEQRGRp9oKhkyfu2cDTmhgEQRARo9kLhlR8UGCcqKihyXsEQRCRotkLhlRLCiyP8xXmpm0MQRBE\nFNPsBUPFqsBCBUblREmltambQxAEEbU0e8EAgBiVDlA7YKVlWgmCICJGVAiGXqUDo3LQut4EQRAR\nJCoEI1YTC6idsNhoLgZBEESkiArBMGhiwTACzDaKYRAEQUSKqBCMeK0eAGB2UlotQRBEpIgKwTBo\nDQAAq9PSxC0hCIKIXqJCMGLVooVhJQuDIAgiYkSFYBg0sQAAG08xDIIgiEgRFYIhWRh2gZZpJQiC\niBRRIRgJMfEAAE5lAcfzTdwagiCI6CQqBCPLkAkAYPUmWG1cE7eGIAgiOokKwUjQxkMlxICJNVJ5\nEIIgiAgRFYLBMAwMSAITY0GNlVJrCYIgIkFUCAYAxLFJYBig1FTR1E0hCIKISqJGMOLVCQCACmt1\nE7eEIAgiOomoYMycORPXXXcdhg8f7nf7zp070a9fP4wePRqjR4/G+++/f9HnStSKglFlI8EgCIKI\nBOpIHnzMmDEYP348ZsyYEXCffv36YcGCBQ0+V6uYVoAZqLHXNvhYBEEQhC8RtTD69euHhISESJ5C\nJlnfCgBgcpJgEARBRIImj2Hk5uZi1KhRePTRR5GXl3fRx0mLTQIAmDljuJpGEARBKIioS6o+evTo\ngV9//RV6vR45OTmYNGkS1q1bd1HHitfpIDjVsKmonhRBEEQkaFLBMBgM8ushQ4bgxRdfRHV1NVq1\nalXvd9PS4j3eMxo1hF0aOFQ2n23RTku73mDQvXBD98IN3YvwEHHBEAQh4Lby8nKkpqYCAA4cOAAA\nIYkFAJSVebqerDYnwGnghNlnWzSTlhbfoq43GHQv3NC9cEP3wk1DhTOigjF9+nTs2LED1dXVuPHG\nGzF58mQ4HA4wDINx48Zh3bp1+Prrr6FWq6HT6TB37tyLPpdOq4Lg1EBgONg5B7QqTRivhCAIgoio\nYMyZMyfo9vvuuw/33XdfWM7FMAzUgg4CAIvTAq0qMSzHJQiCIESaPEsqnGjZGACA2UH1pAiCIMJN\nVAmGjhVX3qutMzVxSwiCIKKPqBIMaeW99w4sgpOnMucEQRDhJLoEQxsjv66xUVYEQRBEOIkqwcjW\nXSa/5gRaeY8gCCKcRJVgZMalwlnSFgAJBkEQRLiJKsGI02sBgQEAcDwJBkEQRDiJKsHQaVUQBPGS\nyMIgCIIIL1ElGDEaldvCIMEgCIIIK1ElGFoNC0gWBrmkCIIgwkpIgvHjjz/CZBInw82bNw9//etf\ncejQoYg27GIQLQzxkpxkYRAEQYSVkATjgw8+QFxcHA4cOIAtW7Zg1KhReOWVVyLdtgtGq1EBPAW9\nCYIgIkFIgqFWizUKt27dirFjx2L48OGw2WwRbdjFEKNhKehNEAQRIUISDIZh8P3332PNmjUYOHAg\nAMDhcES0YReD1iPozTdxawiCIKKLkATj2Wefxdq1azF27Fi0bdsWZ86cQf/+/SPdtgtGrWLBgoLe\nBEEQkSCk9TD69u2L999/X37fvn17PPfccxFrVENQs+IlUdCbIAgivIRkYbz++uswGo1wOp34y1/+\ngquuugqrVq2KdNsuCjWrAgDwZGEQBEGElZAEY9u2bYiPj8eWLVuQkZGBdevWYfHixZFu20WhUYmC\nQRYGQRBEeLmgiXu7du3CsGHDkJGRAYZhItWmBqFWiS4pypIiCIIILyEJRkpKCp599ln8+OOPGDRo\nEJxOJzju0uyQta4YBgW9CYIgwktIgjFnzhx07twZc+fORWJiIoqLi/HQQw9Fum0XhUYtuqTsHK24\nRxAEEU5CEozk5GTcf//9MBgMyMvLQ2ZmJsaMGRPptl0UGpdLyu4kwSAIgggnIaXVHjx4EFOmTIFW\nq4UgCHA6nXj33XfRo0ePSLfvgolRawAANuelN7GQIAiiOROSYLz66quYPXu2PMt7+/btePnll7F0\n6dKINu5i0GlEwSALgyAIIryE5JKyWq2yWADAgAEDYLVaI9aohhCjcbmkKIZBEAQRVkISDL1ej+3b\nt8vvd+7cCb1eH7FGNQS9RgsAsJGFQRAEEVZCcknNnDkTTzzxBLRasTN2OByYP39+RBt2scRqtYAd\ncJBgEARBhJWQBKNXr15Yv349Tp8+DUEQ0KFDB9xyyy347bffIty8C0cfowZMgIMnwSAIgggnIQkG\nAGg0GlxxxRXye0EQItKghhKrjQEAOCiGQRAEEVYuek3vS7U0iCHG5Ta7RGeiEwRBNFeCWhh5eXkB\ntzkv0RhBrEswnFQahCAIIqwEFYxHH3004LaYmJiwNyYcGHTiPAwnxTAIgiDCSlDB2LhxY2O1I2wY\nXEJG1WoJgiDCy0XHMC5VpBgGVaslCIIIL1EnGFqNCgLPws6aYeeonhRBEES4iDrBAABVdTvwagt2\nFu9p6qYQBEFEDREVjJkzZ+K6667D8OHDA+7zyiuv4JZbbsHIkSNx5MiRsJzXYGsDAKi1G8NyPIIg\nCCLCgjFmzBh8/PHHAbfn5OQgPz8f69evx0svvYRZs2aF5bx6jQ4AYHPaw3I8giAIIsKC0a9fPyQk\nJATcvmHDBowaNQoA0Lt3bxiNRpSXlzf4vAaXYJjtdQ0+FkEQBCHSpDGM0tJSZGZmyu8zMjJQUlLS\n4OPG6UgwCIIgwk2TCoa/elThKDkSFyOWXrc6bA0+FkEQBCEScvHBSJCRkYHi4mL5fXFxMdLT00P6\nblpafMBtrVNbAeWAE46g+wXCzjnwxf4VGNbperRNzLrg7zc2F3ON0QrdCzd0L9zQvQgPEReMYFVt\nhw4dii+//BJ33HEHcnNzkZCQgNTU1JCOW1YWOANKzbMQBMBkswbdLxCbCrZh7YnfsOXMLrxxfXgC\n8ZEiLS3+oq4xGqF74YbuhRu6F24aKpwRFYzp06djx44dqK6uxo033ojJkyfD4XCAYRiMGzcOQ4YM\nQU5ODoYNGwa9Xo/XXnstLOeN02sAXgU7d3FZUmaHuPysyWEOS3sIgiCigYgKxpw5c+rd5/nnnw/7\neRMMWoBTw666OMHgqQ4VQRCED1E50zsxLgYCr4JDcJcGKbdWYMf50GZ+8wIPAGCZqLw9BEEQF0WT\nBr0jRaJBC3AqcIJV/mzW728AANrGZyMrLjPQVwEAnCQYuDQXiSIIgmgKonIIHaNRgRXUEBgnzhkL\nPSrXStZDMHiQhUEQBOFNVFoYAKBmtHAywOu75uGWy26SPw9lnQy3S0oVsfYRBEE0N6J2CK1hNfLr\nXcX75NehLN0qCYaKLAyCIAiZqO0RdSq9/Fq5XKuDr3+NDCmGEY5Z5wRBENFC1ApGK1WK/FoZtwhl\nrW+BLAyCIAgforZHTNG6S4ywrPsyQxEMjmIYBEEQPkStYGTo3amzVoc7vdYRimDwlCVFEAThTdT2\niK1iDXCWtwYAOBWZUaEIhtMV5yCXFEEQhJuo7RENOg0c57r4fB6KS0oKjDMkGARBEDJR2yMaXAUI\nvXGGkCXCp1jIAAAgAElEQVRld4kKS1lSBEEQMtErGDo1IPheXiguKYerym3gwuwEQRAtj+gVDL0G\n4C9OMOwuK4T3M8nvaOUJvLN3AaxOq882giCIaCZ6BUOnBsAAgqdb6UJiGP7KiLybuwgnqk9hx/m9\nYWknQRBEcyFqBUPFstDHqHzcUiEJBud07Ru4jIhADiuCIFoYUSsYgJgp5R34DqU0iCQqoRQqJAiC\naClEvWAIvLdLKvTig1yQUuhkYRAE0dKIbsHQqyF4Bb5DKz7Iefz1i0CCQRBEyyK6BUOnkWMYrOtS\nL6SWlL8sKQmSC4IgWhrRLRiK1NoYVgcgxFpSLsvCKXAQAlgS5JIiCKKlEd2CoZy8x4kLKtVnYQiC\n4FEOPZQlXRsbk92M8+aSpm4GQRAtjKgWDH2MGoJLMBw2FipGVa+F4S0Qmwp/xz9/m4kamzFi7bxQ\nntn2Kl7ZMScka4kgCCJcRLVg1Nk5MBobAMBm0ULNqOsNentnRi0/8T0cvBMHyw97fB7IVdUYSFaS\ng6s/gE8QBBEuolowEmI1YLRiCQ/epocKGtQ564J+hw+QGcXg0itESPNECIJoTKJaMG7qmw2GFS0B\nwa6H4FTDytXBZuewcPVhFJabfb4TbO6Fkksh6B1KxhdBEES4iGrBUCmWZtVy8bDZWFiddVi78yy2\nHy7Bf77yrQcVapC7KV1SEqFMQiQIgggXUS0YADD96okY1u5GXNHqcjhsLHiBh91Vvtxo8YwBcDyH\n1afWhXTcxnIH8QKPUkuZX4FyCmRhEATReES9YHRMbI9Rne9AWis9BE4NAMgXDoBtVeLjVPr9/C5s\nLdoR0nG5Rhrdrz/7K17c/h/8fn63zzZySQG1diNKzKVN3QyCaBFEvWBIxMaoAac4F+MUvxsxV+zz\n2cdo941pSHjHLEKNdTSUPSX7AQCHKo74bCPBAJ7e8jJe2vFWUzeDIFoELUYwDDqNbGHIsKFbCd7x\ngkshQ4kEw82lEFMiiGinxQhGrE4NeAkGE+NtUQTudLznbzS6YPiLYVDQW+ZSEHCCiHZalGAIrvIg\nEqw+sAvKG+/OubE6a4YJPP+Dgt5uSDwJIvK0HMGIUfsYEIzOgnOlJuUn8ist6ykuzqa2MPzQWKVB\nmoO7J9CES4IgwkeLEQxlqXMZlQOzFu9UfODuGPVqnceuDq/RPMc3blFCf112fTGMrYU78N/cj33m\nlhypOI5lx1eFJARFpmL849ensOP8ngtpbqPTWEkIBNGSaTGCEatTgyvPgiP/Ctj+6A8AYNSi1eCv\n89ep9R7vvTtnrpHdQf5mltfnhsktP4Q/Ko/B7LB4fP7e/o+QU7AVxZb601G3nRcF9ZvjKy+gtY0P\nJQAQROSJuGBs2rQJt912G2699VYsXLjQZ/vKlSsxcOBAjB49GqNHj8by5csj0o5YnRoAC2dxR/B1\nBgAAoxI7mYpam+/+3hYG5y0YTT+ira+TtDnFCYoNcZ9J1gnLqOrZs2m5FP4fBBHtqOvf5eLheR4v\nv/wyPv30U6Snp+Oee+7B0KFD0alTJ4/97rzzTjz77LORbApiNIoOz5UtpTaYwPTKwYdbq9Cv9ZUo\nV1nlXXReguEdYG6siXvBqC/obeNEIQxkiYRyDbzLbaViLm1j9FKIKRFEtBNRwThw4AAuu+wyZGdn\nAxCFYcOGDT6C0RhBVYZhMLhnazh5HtsPl0DgVECMGSyAYuzAD9U74CjqCE2WuL9PDMPHJdVIWVJy\nIF68R8p4RL0WhkswuAD72UNY31wKJrOXumBcAgJOENFORHuBkpIStG7dWn6fkZGB0lJfv/n69esx\ncuRIPPHEEyguLo5Yex6+sxsevqOb+MZ7Eh8ANq5Kfu0tGD4xjAvooOqcdTAFmEXu5J04VXMmYNFD\n76Ra7gIEo04SjADHDmU9DU52SV3igkEWBkFEnIhaGKFYDjfffDPuuusuaDQaLF26FE899RQ+++yz\ner+XlhbfoLapoAUPz9iFKsEtGMnxCR7bzJwJKSkG+b2d57D5UDEcTh7jhnUJeq57v5kBAFg27gOf\nbR/vWYp1eTl4/JrxuKnjdT7b1WrRlabRqpGWFo86h3s9D61O3BboXkgWRHxiDNKSxX2U/xN9vLre\n+6g96Tq/StXgex5JEhJFgb+U29jY0L1wQ/ciPERUMDIzM1FUVCS/LykpQXp6usc+iYmJ8ut7770X\nb70VWl2gsrKLXzJ1wfQhmL//CE7XBj6GYPcM8p6qysdHO5a53xdV4cgffwAAbr4qK6Tz+mvzznNi\nrah9547gyviePtudnDjCt9ucKCszwuJwx1lWHlmLUd1uhana11LgBR42pyiI5ZW1iOfEc9tclXoB\noLyqFmWa4Pex1iJaRoLANOieR5qyylp0TmnY76K5wws8BEGAihXFvSXfCyV0L9w0VDgj6mfo2bMn\n8vPzUVhYCLvdjjVr1mDo0KEe+5SVlcmvN2zYgM6dO0eySQAArUaFWI0+6D7eLikA+CU/x/2GcY/U\nrTYnVm05jYIyk8936kNy9YSa5ePtevnm0Gq/+9kVwqAMeludVr/7BMLqWqHwUgx6K914NHEPeGPX\nfEzNiWzyCNGyiaiFoVKp8Nxzz+Hhhx+GIAi455570KlTJ8yfPx89e/bETTfdhCVLlmDjxo1Qq9VI\nTEzEa6+9FskmyfgTBCU6dUzwAzDuzmrN72fx4/az+P1wMV7/+8CAXymrsiAtKdbjM6kjrr/DEwXK\nWzACxSGk+IX3d6yKJWrrW99cuf+lOM9BKRj1TaS0OutgcViRok+KdLOajAJTUf07EUQDiKhgAMAN\nN9yAG264weOzKVOmyK+nTZuGadOmRboZPsRpDEG3/7b3PBDMCFFYGL/uKwQAlFZZfXZTjuif+nAb\npo/rix4dkuXPWFkwgge9pbN575cS678DVLqeLE4rquqqkaRr5SEY9hCC3lL7bSFYI42NRwJAPSnG\nL/7+JowOE+bdOBtq1v2zLzGXotRajp6p3SPWzsamOZRyIZonl56foZFI0rUKuj2voB6fp8LCsNrc\nnVVRdSU2F/4OO+dAta0GT26apfiOgEOnKzwOU59geOM9klZ2fkqk+AUAfHzoCzy7bTZMDvNFWxj+\nBEMQBNQpjtfYKDPV6nPpGR2iu9A7PfqlHW9hwYFPfWbD+8PqtOLzP75BhbXyIlrbeDjJPUdEiBYr\nGCm65OA7CAycFZngqlMx54aX8UC3cR6bGdZ/B/W/wxux9NhKzN37Ac6bSzw3MoJP9VlVkBhGjckG\nh9Pzc2/XVSBXkY3znb2+pXAHamw18vtQ0mqlOIeDd/iI2rLjqzB90/Mot1b4+2rEUbraQk1zDnS/\nQonnfH9yHXYU78HHh74MrYFNxKXoPiSig4i7pC5Vkr0sDN6mAxujHC0zcJy8CgCgZbXINHhmd0Hl\n+VBe2y0duXnlyCs7DyQA+cYC6FRecRKWh3e1cqnkhj8LY+p7WxHT3Qw2zl1LyltYvEuWSNT5EYzV\np9Z6vA9l4p5ytGrn7B4z4DcVbgMAnKw+g1R9Sr3HCjceMYwQR9UNEQyTy0qxOOu3RpoSEgwiUrRY\nCyNZ5+n7F2zeAQu3H7isxoo4lWc6GqPiPPbpnJ2I7NQ42AWl6HjXU/f1LdfvkmI8DiUJRvuEdgAC\nlzgPJeZQn2BwPOfRrkDHbKpJc6FaGMoONFCZFBtf//2S7r3qEq+rVZ9gVFirkF9b0EitIZqaalsN\nqhWehYbQYgXDO+gt2Dyzl27sm4Ur2ohzRJ7+cDvmfn3U9yAKK6NVXAwMejWgltJQVT6dE8PwqLN7\nfsa6TI5QO13JJRWj0gIAnAHcSsoYRiCCuaQsDisWH/Z0vfhzcwFNt3iRMp4T7P4p4xPK4LiHGDpD\nEAzXdarYS08wlIHu+mJTXx/7H+bt+zDSTWpSjlXm4VD5kaZuxiXBM1tfxTNbXw3LsVqsYDAMg8d6\nPSi/97YwDDo1+nV1u6GKyn3dEIzKgZf+ei1u6puN3p1TEafXgNG4C/6dr/QKnDMCrHWeoz+3heFp\nfQRKE5U6Rq1LMCQLw8bZsf7Mr6hzCUWgzl1JMAtjw7lNyC075PFZiaUMB8oOB2xTY6M8b7BAr1Iw\njledxObC3wF4phjbQ7IwXIJxCVoYSvGrb2GtKlsN6jibj1X2R8UxbMjfFJH2NTbzcxfigwOfNHUz\noo4WKxgAPFIp777OM62ydWoskhM8YxBclVccQ+1Em7Q4jL+lCzRqFgadCtCIHQ/DAHvzvIPePCw2\nz4dZFSCGYbJ4duZyDMMlJLKF4Xrolx9fhVWnfsL3p34C4D+GIZERmwYAcChcTA7OgV/yc2C0i356\nf4Kz4MCn+PDgZzhZfcbj86bymXtM3Ati5Zgd7jpey45/h6XHVsLssHgISSguPOmeMD4Vvpoeh4fb\nLfj/w+K6bm9h+e/+j7Ei7wd50EEQ3rRowVCSluBZO0rNAu3S46BWsejdKQWPDu8OR35X8DYdeKvo\nzrqqS6LHdzQ6p0dQ21jnNS+DEXDwVAXyCmtQWi1uqzWLwuAtGEYvwTBZXYs9ebmkpIf+ZM0ZAJBT\nPoN1gFL8xq7oMH49twUr89bg08Nfi00N0imWeC28FErAOBJ4xDCCpNWa/KTMztj8AnaV7JPfhyIY\nUgFJK+c736bYXIKcgm1NNgdC6WoLZmEIgiALRqC5K6GkGEcLds7hMVcq2gj377HFZkn5Y1BWf2wt\n2gEAiFHHILWVHv+degM0alFX+1xxO5Zu6AxD+3P4teRnDL4q1eP7lapTgKKfL60xAcpYOcNDEIDZ\nS/YgwaDFv/7cB+fKjFAleqbLWm1O5Jd6urOksiOSsGjlGIYTJocZJRaxxIqaVcPisOCcsTDgdcZp\n4qBlNXLHAQCVtmoAwDmT7/c0rMbDL27xesCCWTNKzhmLkBGbKre9oXi4pIJYGJYAHeCPp3+WX4ci\netJcDmU9L4k3d78LG2dHRmwauiZfXu+xwo0zRAvDwTtk951yP2XHYnKYomZGfH3p1s9ufRVmpwX/\nvfnNRmpR46J8RsIhHi3ewojXxgEAErTx+EvXu/HMtdNwW/uh6JHSFQBksQDERZgm3NYV2UliSm4d\n79lRnnEcgsCpwBld2707IUWWlIk9j+NFJfJndo7DkvXHsOdYGWYu3I6PfvAM2Dk5Hk6Od1sYrGRh\nOPDt8VXyflW2Gry+az6OVeUFvOZYjR7JuiRU1rmr80rBd3/ZWglazwyxyjpRXCQrxBrC5L2ztefw\n+q53sPDg5363VxltKKu+sJGeMs7zw+l1KLf4n1Bn5epvX7CYjyAIKLGUyddpddb5PHyShVJsrn/Z\n20igFMxgguGRAKDYz6xIFfZnkYWTs7XnsOb0zxGzxpTHrS8T0NyAFOkySwUWHVyCKtfzcCmiHOiF\nI9bY4i2MGf0m40TVKXRKbA8AyIrLRFZcZtDv6F2FCyVTluM5LD22AkauGoI1EXCIdagY1hWgFmJh\nZyxol2HA2VPA4GvisYdZi2+LDgCMmJ1VXmvGmX2F+HVvYMvgx9/PIqOD+ANgoQEgPvSVio6y0lol\nj4SVZBkyUWQW1xqJVeuRrE9CsaUU5dZKrMxbA6tD7PAkwVC6pBK0caioc5+jwjVRT8Wq4OSdIc32\nliygI5XH/W6f/t+tAIDF/7653mNJeE9iXH74R9zdfqTPfqH45INZGHtKcvHJH1/L7zmBg513yG5B\nQExe4AUelbYqf4eIOMpFsoK5pJTW4cZzm+HknfhL13tQY6uVP1fGfCLBm7vfBQB0T+6CDontwn58\np8e9qH+uESA+wxea/fbl0W9xovoUVAyLh6+874K+21jYudB+F6HS4i2MZF0S+re+2mcGdjCk9b6l\nEefRqjxsO78LACDYdRAE1211CUasVtz/nps64L1/3oD4BHEExGhtYFwlRjiNCWCD/UMFfLflNL76\n5RgAYOfhcgDixD2T3YxkXRK6Jl3uVywA4OqM3vLrU/kWJKpFK2jx4S+RW3YQx6rF43I8L8dLJOK9\nLAyp85dmqQeyMEwOM/aUiOXbA5Uw8b7GMkvgWeMlljIsObJMdjF5xy10AVxddSFZGIEF41DFMZ/P\njHZPl2FyjHg/KwOMNo9UHsdHB5eIAl9XhUkbZ2BL4fZ62xUqTiE0C0Ppnssp2IatRTvh4J0egmEK\nQTA2FfyOY5WBrdhQECDAyTuxMX8TamzhKz+uFIlgqeOemWWe+31z7Du8vef9oOepC1I2JxR4gceB\nssMRTRpRXlc4ztPiBeNi0KtFC0N6+DSKkYmejQMEl/i4BEOvliwOAbE6NZLjFSm8CjeVpp17rsdl\nGfEY3Ku1z34Wm/jjLCqzQsWo4OCdMNqNiNfGITkmcLmTGJW7+u7BE0YcPyUe52ztOY/9HByHqe9u\n8VjqL8HltpMos1bAZDfL0xKVMYw6uxN1dvGHOW/vh1h8+EscrTwBdZBUVMmFoG57DC9sfyOgFfLR\nwSXYfn431p39FYCviR2r9V8tMpQ5KcqHnuM5HCz/Qy7O6G8sccbrvsVqREuxIkCZlPdyP8K+soPI\nNxZgvys1+etjK+ptV6g4L8LCkKhz1qFaaWEEWB0SEDvg38/vxjfHV2J+7sKQyssEghd4bC3aif/l\n/YBFAVyVF4PSDRXMwlC6IVfk/YBFB5fI7w+UH8bJmjNBxUD6/VucVvzvxOoLDp7nFGzDhwc/w/IT\n/pcoCAeOEO9FqJBgXARS3ENKQVVWfb217+W4tovo0hJngwN6jdhZSx1cWqJSMNyjHFXKeQzskYGJ\no67ErIeuwd1DFGufS8Ii/RVYsGBRW2eEU+CQoI2Dw+w5+VBJnMbd6QucBtZa/6NxhhXAXnbAY/Tl\nbWEAwKmaM3JnoXRJTXx7E6bM2wxe4GUXmNlhCVpcUSreqE4XO+HD5X4mScI98pUsGu+AZqBg+rmK\nwD7m7DhRlJWdx8Zzm7HgwKdYmbcGgKd7Ttr/lCsrTULqpPy5v5QWmCAAWpUmYHsuFs+02sAdg9lP\nwP7fW17C/rKD8vtgFsaPZ37BF0fcC4ntU3zvQrFxdnkGcr4xfDPPleVygsUwlMkLW4t2IrfsIIrN\npR7t8rYk/XGq5gw2ntss/15C5axrtv3hCv+/93CgFHRySTUR0ixx95wF9yhEzaoQqxM7LrVa7CSl\ntTUkF4pe5+6ADLHukTdvTMLfhveQJwwmGrRITxbdWR2yXJ22LBgMnE5GDvSWlfPYsivwjztGUMxs\nd2qgsicG3FedVugRRFXOio9jxJpRO4r3yHNDvF1STk7A3L0L5PecwHn8WJXBdgAwSi4wTrwXoUyi\nA3wD9NJs7RqTDUvWHYOlzoFqkw1nSgPHFeQUY8WDddbVeW0q3IbcskMe7spOiR2gYlTIry2A1WmV\nvyfNafE3Ij1ZfVp+beftcsJCOAk1SypQHaxDik4rmGB4lxTJU1zbhWLjbBdcrTkUPF1SF2Ztvbzj\nLfx6bov8vjaIYAhepX9MfiyzIlMx9pX6F1XJMyH9vywOK7459l2DqiGXmEux/syv8v20k0uq6VGz\nasSq9XK8QBkwbROXJfv2e14udsrSyFcaESs7z1idCq1ixP3aZvq6VGK00r+IR0KsBnBZLe0zWoHn\nGFmEzhU5INTF+XxfQnC4JyEKPAsVFxd0EakCxahco1LLbazKF91eylngUozA5nCLjHIE/sXGQ/hh\n+0n5/XPbPBfJkiYpCrxLMAK4Obw9QzUWT6GycXYsP/49nt3+MjYVbcXKLSfF+SyqwA9Ksi4JDBiY\nHWbwAo8jFZ7usEUHPwerOHOsRo9WMYk4XZuPJzfNwqeuYLjUZn9ip3Rf2TmHd4WxsBCyS8qPhSEh\n/U4tF1Cy/nTNWZ/PVp9ah8m//tuvi0b5v7VxdlkwvDvfhhCqGybQvZAqAQBA7QXEVjR+LMdXd76N\njw75z6RSueJ60v9u7dkN2FS4DYsPfxXyOb15ffd8rDr1E/5wxd3IJXWJEK+N97EwhrW7EV2TL5dn\nb0sTo6RsGs5P/jsncFCzamhYNTR+PBXSSIETeKQk6sGoxXON6H8FILj/fYJDi9v6XBGwvTaz4uBO\nDVgwaBvfJuD+J8vd2Vqck8Hw9L+g7vBAOM939D02Zwcv8O7Z6V7B+zrOiqLKwMvXykF2WTA8O90q\now3Lfs0DL/UprpjH4TOe8QK7045DFUfAs3ZoLzuKU/wu1JrtYLwE4/FeD8mv4zVxSNYlodRajg35\nm/De/o+wr/SAx/5KC0PLajwqHR921SuSihfaOLtPuqgyTnS2tAq/7PXtZC+UIlMxlhxZJrvAQg16\nB0sjzTZkQqvSXpAv/ry5xCdLbu2ZDeAFHqf8iIkyA8vO2SNSZsXOhSgYAa5TmQAQzMLwRs0ETuyo\nqPO1ciWxlP5fVod4H6v87Bsq0rMjXZuHS6oB8SYJEoyLJF5rgNlhAcdz8j/p8iQx5iA9BNLnsoXh\n6vyVI0CO56BiVNCyWr8/bqnGlCAI6N05Ra5VlRrXCjq1wrXhiEHrFIOniHDuhzG/xIwXBjyFbNMQ\nCPZYgAF6p/YIeH2M3v1gf/5THhb+7zQEcyJ8x/kitVYL3l1xANDYoO/3i+ex1A45xVjim40n8NbS\nfdifV+4jGJVmz07tvysPYu2OfNSaxftZXiM+WBVG8aEQHKIYHj5bCqvdAYFTga+LRbHqEEqM1T4W\nhkHjjvXEavRIj02F0W7yqZ2luAL5lValRasYt2BwAg+O5+SHkRd4n7pWysmQq7efxKlid4fgLS5O\njkdFTf0j/PdyP8L287uxqUAsMa8UiR/P/AKjzb9AW4NYGMm6JMSq9bA4rDhaeQKTNs6o1+UkQEB5\nABeKt+sRgFc5FltY1op38k6PtWeUz1GwVSUDueeU1s6FCEawisf+3ExS3Ez6vbBs4LVxLhZPC4Nc\nUk1GvDYeAgSYHBb5Hy9ZEtJD4BYMsUOT6h15+JsFJ1QMC41K43cugKCwMG7vfxlSUljX+eOg07qt\nBsGuQ2ZyLFSu+RlcTQrq9ruXxt19tBSp+mTE2kSrwmx14I+9Bqh5/5lFUsBePLjnz8Rxzncm88b9\nZ5FfYoIqsVz+7NrMvuILlcMjuA8A63adxR9nqjBv+QG5DIrkkjpTKprvuSfK8d6KgzhVVOvxXXOd\nuH+VSez8JKsnv7QKRrsJgtUAriwbYAScNp72sTCUgqFX6eTaWkWm837vhTK4rlVpEKd1f1/sMCs8\nOhnl/5EXeE/fNst5LL7lHfP46ufj+NcH23D6vOc1eyN1ZFKGmrdVsTV/N+ycHW/smo/NivRdY5AM\nqDitAVomBhanFd+d/BEAsO7sxoD7S6nSyjk6Skot5T6f1Sg6YBtnv6B09kB8/sc3eGXHHLnGWUNd\nUkqCxjC8xD5QRQEAKLX63gvJOpS8CNJAM5yCYQ/RVRkqJBgXSbxGypQyyg+9JBisK5gljTiklFZ/\nJRmszjqoWRW0XuU3JKQfDy9w0KhZpCQzYBkWsWo9DFq3hSHYY5CRrIeaEQVDcMQAzhjYT/cA8gai\ntNqKFZtO4dAp8eGuNtmx+0gljLuvB28LHMsQD8bg0eHd8ehwsUCj83xHOMtbe+xSXF3jOq9bxDJi\nxeA9o3YA3isUKiyOo/mukSgv/hwZlsO+E2WY/78D2Hu8zKc5VpuYumu0ivdXcLrOqXaAUfEQnFpw\ntWLZlmO2XWDUTgic+6fOcu77tvC7E+CtogAEyqhR/l9ES9DzwSu2eLZRKRhWZx0ECLK7gmF5D/H0\nniT3W26R2O780GYPS0LlLRh6tQ7Hq04i31iApcdWYHdJLowWO44WBp6JrmV0OF/mgNXhDuaXmMvw\n780v+V0/o7Xr/+s9ek5yWWD+Zr1vK9opv7b5qZh7MewpFef6nDWKrj9HiC6pGntgUZYyIWvtRtTY\narH65Np6Kxr4EwzJ7VTmRzyVrjzR0yAlAISv+rPTI+hNLqkmQ6q1U2otlwVDK1sYomBIIyzvGIZ3\nh6NiVNCoNKixG7Hx3GaPbbwsGOJfo92EOI0BLMNCrxCMB4f1RnysFipB7DylUTVX1hYPDL4O+hgV\n1vx+1qeMOsACXPBJdXcO6IABPTKRKqcDM4hTeWZZ5Ve4On2lMNh0EHgWsQbBQyDE/dzvD5wtAiC4\nM8BYDut2es5zUFJSZcX+vAq54xVc7We04ohRx+qRrE4DBAZmeI1+BeDpD/a43zq1yDsWPGvpWKH7\nYeccLK5OFydBSpaJdzFGKY7x+dqjWL5dLHAYKy3AxXIe114TYATr5HhY6hx44ZOd+GW3773wHpl7\n19Kqc9o8XEKfHP4Kp4trRGsvAJxdDTjVAAOYXPG5irpKGB0mfHlEnDOitKQyDRkAgJ/zc7DkyDL5\nNyp1kiavSaS8wONg+R/y82F12MI7ac3121aOqpceW4kKi/+YQHVd4EWFLotvCzWrRq3NhC+OfIu1\nZzdi+Ynvsfrk2oBVAcyKmAgvCHj3fweghvjb8mepKEvWmJ2WiFgYlFZ7idAmLgsAUGAsCuiSktCy\n7iypM7X5+PbEKo/tKpeFAQD/O7Haw+8qPaBSR2+0m+XRj+QSiFFpcX3PtgDcJUOULqWMZD06tPas\nxqtEUAhG2/g2MDCey9cmxYsWkkHv3q+Dqg+cFa3hrBDnnJQbxc5BGatYsek04NRAq3P6rIF+x3Vt\nkJKgAxNjgb7vr8joc0zuSBmVE8fPBR9hf/j9YberidNA4FkwWvEBTI1LRFK8HoLdbTk5Czu7rlUD\nTtkUpxrxbBKuSOoc8FzKTBmjmccVSZ3wxuBZGNnxDgDAqpM/eexv42z442wVck4cwU6H+L82Vrvi\nSSznIarKAKdUYBIA6uwczhYbkV9iwle/nMD5Cv+uJOlzKcGiZ2o3AKJlU+iaByNxrqpCtPYCYLOq\nIHDi78c7OJ5fVo3P1h7FiQL3/0XKyqu1G7H9/G6UWsphcVhlq8nG2cHxPH7dWwCrzYlauxGcwKF9\ngp5oOOIAACAASURBVPhb3ZtXHHQdk/rILzHi87W+cxi8rYrdhQd89gEQdBW6VrpEJGjjUWs3oszl\nTtp+fjfWnt0ou+u8UVoYNSY79p0oh80h/l/8xVKUc3bqnHXyIMDJcXByDRMNwY94kmA0IZJgnDMV\nyqmUsmB41aSJUQS939nru9KZaGG4R7nVNvdDyStcUg7OgTquTnaHSSM8pR/8xvbXAABu63a1/Jle\n67kYlA+u2IGW1eDf10xB70zPGEVGsmhZGHRud9O9N3ZFRu110NSJrh9G5cSgnplIT3Ffh7MqHQKn\nhsBy0Lg+7p7cBQDQ/8pUvPH4QDx0t3gfazVnkJHsmhGvtfmMhNtlxEGjEn+ukoBJGWOJOgPAq8Cw\n4kPSp0MWurZLguBw3ffaJDgrxPPI7isXAqfG+UoLLk8ILBjSGicAUFhqxQ/bzmDOV39g8Xdn/O5+\ntrQam/cXQRXvtm4cVnd9MaV4StkzgiDg+Y/d7poft5/F5+vcJUmKK7zcHa6B/t7jZeAFQR5JsjZR\n7MtrjSg0esZkfij6Fow6SMqthREtDH+onMjJLYLD6bYwtu2rggbuCgKv75qHf22eJcdV6pw2/LQ9\nH0vWH8dHPx3Egv3igkaJajE12+qou6BUT++YwSuf75FdeEq8j2nQxsLisOD1XfOw8ODn2FYklvGp\nCiIYBnUsErTxqLJV+0zGlFxw3u2p42zILTuE/NoCVyKHIKfB+0u3Vrqk6jibey4KI3gMHi4Gd4IN\nzcO4JIjTGtAqJhGFxiJ5wphkSXinCmoVLil/D4hoYbgfVGU9ImVareRzTYgR3RvXZw8AALR2CQcA\n3NbxBjzVbwqGX+Eu4hejVWFI7yz8c6y7nhQA9OjgKiXisjCkH1m7BK90W9dzEatztzG9lR4v/bU/\n+nfJdl2EExNu64qh/cTYRrZpiChEvAoO3o6+XcRzSQFnB28HyzAex9QoPEOsoQa9O6Wg7xVpmP/E\n9XjhoWsRFyvu0L2Dq/S2a7TMcFqPjLAEXRwG9cyUG945qxX6XS7eIz2rR5u0OGRX34K72t6FyzPT\nUVJpwffrA1s0jEIwNueWYMWmUzhbbISxxn/n+vnPf+Do2SpZsADI1k5KksYjhiH9rw+f9g0cl1S5\nXRw1Fs8OR9lXlVVbsf246LbatV+0hn7cfgJnazw7U9bg6xaRKisDgMkICLz/a2Jj6qBuc8xzXXpe\nBdbhntTp/du2cXbkl4jnPG07iHMmsT0qh8s9p+LgVEysyyuowRfrj+G3fYU+9czeWroP73zraSl4\nj8Klxcm800ctDisOVRzFOWMh9pcdwpdHv8X6M7/6WBgjO90uv47V6OUqzd712Q5VHMU3x1bKMTQA\ncvHSRQc/xxu756OwugxgBDByNWpfwVC6pGzO8LrnpGOFGs8JFRKMBtAmLgs1diPK6yqhYdWyZeHj\nkvLKkvJGxag8ROb387vk0YskGIIgyJN/pKBia0MGXrp5Oib2flj+LsMwaJfQBizDYtq9vXHnwMuQ\naNCCYRj06pSCuf8YhFZxYkd2W3+xUqjUmUkxlnZe8zPaxouioFb5/lx6tBU74lv6t4ZaxcqB41v6\ndUC7jDhkJsXDzjnkY0uCIVlFyrIbyh83a6jBwCsz8Y8xPRGn97IKwGPGn/ugXZY4ur3vph5gBPf9\ni9fGISMpFhkpomWk02rwyJ09wTIsumW3xkt/vRYzx/wJt19+Azq71m231/qWP5HvqZc7rX/3DDw3\noR9iNQGSBVgOtRbPQL+0BHDHbIOHu7DCWonCcjPeXrY/4PkBYEXOKZRWiVZGrcXunpMCYMfhEhTX\n1Hich9GbwTMBgvjn28uv7Sf6yq/PFtqgjg2c6aPJOu3hchR4Vs7K80ed04bjLheWco7Clj01oguR\n5WB1uNs4+8td2Li3EJ+vO4YPvjsEQRDwyY9HsGTdMfxxpgoHT1WgpDJw+77fKqYAe5ezt9itOFzu\nWUBylWtlSo+qzGp3XM6gjgXnDNw9bir83cPF0ye9F2IYd8ZhbuU+j1iVOFdJQG5eOTieh51zeIhI\nHWfzOJ7VduGuui0H3BalFOBWpjF7u04vBhKMBtA2XnRzVNZVedQx8rEwWLdLyh8qRgWHYvWz3SW5\n2OvK/JBiF5zAyyZ0kmLiWNe0znJ5C2+u7JiCu4d08giQJsbFYOb4qzH13t7o0T4ZN/fNxg0dPS2P\nrDh3BtT8G19DnNY9ipw0+kpMH3eV/D5BL25jY0SzXfLVJsXG4oWHrkVKnAECBNn8lgRD2i/QDGVG\nZ8aVHVL8XpeTd6LrZUnQ6DioGRWu6pSJtqnuhz1VJ1ozIy+/BQBwU9vroVVpMfmqRzCm83CPY/W5\nXAxcQ1BhSpfpSFSL91LgGXBVab4n59T489DL0aF1AjKT3fdFa0uFI190t0mxlaREdycpxYlsvN1j\nguahgkJ89pPSDy8AfmY9m6wOvP7lXgDAb/sK3fswYhVjyT0nCQYbJ7q6HIWd4Ch01yRL4LPAlSkG\nBAoXXU0N0DnG/b/1hxQnAgDwKhhNQWZoMwJqXbPxVayiq3HEiGVgWA4HTisyzBQd7JGzVfjw+8PY\nfOA8ck7lQttlF8A68dOOfJQHWTfF5uDkOSh3d74LALD7RAF2FfiuRQ8AYy6/S3596A/Fb9GmwoFz\ngRMvAICHYrKknYWzzu2eO2M+6XE9ds6OX3YXYP7yA/jqtwN4dturHseqc9o8LACr7cKsDUEQsPhH\n9xo6UhJErSN8VYABEowGIcUxAPeoH3BniUhIMYxApcfVrMrHjD5SeQIAwMMdw3BbGIHrQIVCaqIe\nPTuKnfH9t3TB2GuuFdvhEjoNq8bjvR7C9Ksn+cRjru6S7nZlQbRGtCotDpQdhiAIsq9WsqqkYL4U\nRI2VBUOaGe32Dzt5p3xt6rQiLM37xm/7pYfB7LDAoIkFwzAea1Ok6sX29UnviblDXkGPFLEjvyKp\ns89Kch2zEtCvSxr+b+jl6JKdgSSdmBwg2HXgje7rFATAfrInBJsBCQat6z66LYwJfYaDd3XWOh3Q\nKk6L63q5V2S8vpM4SdLBOaBz1RLjbTowGjvyCsWBABNjgf7adVBnncTrfx/gc93VJjtKq61Yt/Oc\nPDKWrB9GbYeKUePqTqIYMCrX78acgPuvHiofQy3o5fRlEQa8yfV74tS4odOVuFn1NzhL2vqcH3DF\nlyR4Vly22BK4JI00adIuKOam1BmgZrSAygmrXXE8ZSYd60Su8ycwhhrEdNkDVWIFVClF2LS/CDMW\nuEt3eMByWPDdIew4kQ8A6JjYAQBwpCwPjNbXJZSh6oCuSe543bY97s71fKkDzoLgKydyikHemUIr\nHA7FhD+hwmP+j5134Gi+6HbcVrJNHvnzZvH3dryoHFa7u42WOv/WYbG5FL8VbAXnyqKTz+e1pPNJ\n1+RQk90ENatGliH4Gj+hQoLRAC5LcD9UbeLd4uHdyUrvt5/f7fc48Zo4H/+lVE5CDnpDkJdRVVoY\n4UCr0uLpa/6J5wf8S/7sytRu6Jh4WQjf1aBnSjeU11Uip3CbbDlIQiFZXmaHBQwYxLpKw0uCoSyN\n7uAdiNfGyYK7p3S/h59Z6iTdxdosMLgKI2YrrCKdokZWfcvBsgyDiaN74pZrxP9lgk48HqOxe8RF\nBEsCuIps/N/N7uB4crwOzhLRrdcpuS2GDxBH8tdfnYSX/tpf7gCnXz0RE/7UGxpWDTvngJQNLdh1\ngNoOyVro3Uf8q2mTh/SkWEwfdxWm3NPLo70vf7oLVpsTLCvei/atXZMI1Q7EaWLx+IjeHgMWwRaL\nHm2y5ffdsjIxqKf7t9o+Mx4JRTfCunsYAAadshPx/+2deVwV57nHfzNzVg5nAQ77JqsiKosKLkQR\nCbihUEEbkza9as1iNKJZDPfT2BtTc29MbZO0ualNW5PWW1vbmn760U+allSjDcFoJGpQEzSKGAHZ\nZD/bvPePOTPMcEBRIQq833/kzHZmXs+8z/u8z/P+nonR/nBcSgSxCyNm4lAjtKNnEah0bcKB2LyQ\nZ30Yhro02M5NBukVNBen37odwv9znCMLcGqg5TTCPlk8R+69qAIvgfO5Bm1Cec9+WYbX1cYOT80B\nlsdn5xtBVN0gThWOfCp00Jyx7/gU59LDSyW0n2A0e674/se14Nv84LjiKYUjIvcwyk8rZWoYlgdr\nUK7zOPWVkH7tsvd4daLBOHSyGqdk3laHree9IITg9FeNuN5uw39/8ir2fvFX7PzgQzzx08NobhOO\n612t8kRVvbBWyd4OL9YAa5fyd3S7UINxB8g77iCvniyk3lNS/ckf6DihY5sTniHN/U+0jke0eQyu\ndtTBxbsU6zDEvHq5NzNYhBlD4Kfvv57GjciLnge9So/3L37gIYciehodjg6oWZW0XXxeeYaXg3dC\nzaolowIA+y/8Q1rcJV+kxhMeXc5ueLmrH4qyLHdKsv9EAO6OThYAJg4tCu6LQk5aT4U4J8/DcSkB\ntmM5MGq8kZuYBL1Kj+MNn0KrgSx7Tuh4NawGNt4OTkVAeAZwaIRaGyoHOJZBcoxyCiw0hMMvq18G\nF9ijydTR7cS89AhpiifIXweDTgVG7YC3xuD2tnqmRhKCQ+Bn6mlPs16PlfenwnI9Gd2fT0eo1YA1\niydImXI+Ri1iQk1ISwiAUSecp2I0eHC2ctoSALzcWQrTxgfC4owCf90ffKcyfXvt0gSEWg1wQmiL\nU+e63OfqBM0xWZxHrhIgBtcZlhfaCnAnORCwxkb856/+7TFxJ8ZXGLUdxKHBB5/UKfbbKtNhvzRO\n+nypxoGNPz2G7opZsJ1JUxwrZdO5biBFL4/nuFRwNgiG2KoR+gLWolyf4xINjCxxQBrocE5FfZc/\nHDqHr6624vi5a1i/cz/+96v/wVN7/iBNW50i74PRt+HVsv/D2x8cx4ef9coWY3gcOXkVzV1taGkB\nPjndf0bYrUANxh2SEiBYbrm30dtA9Bdj2Dj5MZSkFcNP7yv9EERxOwKC6/ZWRfC7svEcfLQWqZO8\nV/D38kOUKQLX7W3SQjRRuVM0EDaXHQQ9noetjykpAgI1q1ZU5/vo6lH84tTbwn638XQSJzocnSAg\nkocxzicWLMNibrjnSPhWmBqUgimBybBfHA9Xqy/CtNGwXxoH+4UJCPBR1htJTwgEwGBFttAJ6VQ6\nTAlMRrujA1931ErZc+J0mU6lQ7ezG3odAxWrgr9RmAoK8ldhTLARKq5noOHiXSit/lBow8gz7pgE\nD4Bgycwo6Tgn78SP1qSB4ZySDL3ObTC81QY8tUxIsxYHGeK6hycz85EcEoOiObEIdD9XsJ/wL8ey\neHTJBPga3OnbZnOfv+HiolQ8/70p8DXpoFa5f/O8crBkNnGIDjH1dK7u/RaDl5AGLXbyYBA/vu9p\nGDUjPA+r6wQXcBnahE+gDvvS80CGF9pIZQfH6wGeAyE9XgPf5Q1X3RjpsxhXInYvmLz0eHZFirTP\n18uIZx5I8UjD7o8N30rF+sw8lKQVY1PaI9CwGqj8lOtgGLcop+gpWfV+SAoRPFaGcyoMEKPtxNa3\nj+Hn+07BbrootIOswBqjckI38d+4pjqDcvu7ioC38F08/lZeBbA8DGoDHsicOKDnuBnUYNwhDycs\nx6bJjyNeNsKVT0k9NXktNJwGP83c5nFugN4qjTASfYVOJ94nBmatMEprsV33kH0eihrIg0GgQRgd\n17QJQntioF/DypMBWMmA/O3Ce/jHpYNSpyiiYlWSYRCpbDwHF++SYhdO3ikV3BFXW3upvbC78DUU\nxC68o+dgGRb/kbgCrvoIwKHDipgHhU7GqUVyrFVxbEyoGT8vnoU5KT1TPqKnea2rUbagU+u+Rz06\nHZ1w8k7o1RpMiRXiDQ8tiMJTy1MUWTPvnj+gWPWvHV8Ofdr7CJh6AloNJ02e2HkHCCecJxoMsY1N\nssJXa5NXIdFvHLLC7wMABPh4Yd3SSTAZNPDWq7FtzTT853d61u4I1xE6S51KA2+1AepeZXYtBh3G\nBAm/VdFgsHaD4hiby4as1DBpPQJxqcCxjJRhpvfiwTEcfHUWNNtasLZgAnpj1rsNmU89NGMqhe/x\nEbyHmRN65ubjI41YlhMBhgGCzT4AGCllnCVqyVtI9he+g3Tr8eO1M1GYGYONy5IwNsJHkrbZtCwV\n8eEWxaJWAODbTXDWek7VhvlZMDFaeJ9NGiMWx8zzOCY8SI+XH50uxXXWJa9GpNVtiDnl4lbOKFud\n7u4COK5v3S1GY4M1uhaKZAmGl1brR/pZMXPczaeXBwI1GHeImlMj2p2DLSKfkopyxwF6v2yA0rAs\niV2ADSmPYEZImlR7QszRH+sTi0luZdlYS/9zqncTUTdKlFUWn1deXW7FuEJp9AugzxWzalYlZZPN\nDElHSsAkt8hjh7Sa2cm7cNadFDBWtkKbY7lBEbMDgJhQoSO0WnSYkxqKB7LjoNV4SnHrtSrFd4oB\n94auRpmWmNCBe6n0sPMOdDm7oWbVUgfvQDe0Gk6hVdRbIkakjalX1qJ2OaQ4j2ggwtyDEPkUY7Ah\nEI8nrZRUAnoT5OsFL51yND0teCqMam+M840DwzAeK4XlnuDksYLhnheRg3mRWZg/Rgi021x2RAYZ\nERUidPrfnjMO6wsnSW2i0bmgZlUwaUxotbchJd4q1H2RxTbkXqgIq+1GTFq1lBoOACZvFcbHCt/j\nbxA8Ko4I3yMmU0QGGvG9xBVYFLgMq+6bDR+jFgumRSIiUGi7ZMcyeF9YiCBfL7Asg4mRymCxr7cR\n8yZ7LvIUi6SJzAnP8DhmyewIWC16RITo3OfoEC4aDFYZz2Hdiz6FMs3C70vFMSB83122JvQr7Fg3\nQ/qs1fasHwo0WqBTaW8azxsINxYRotwW/cUsfpC+CTaXHS8fex2AMptKzaqkeXipWJHbYIij3s+u\nnUZqwOAErwYbeQxHw6qlTlT+I51gTYCaVSEveh4qG8/ifK8ypwCgYtWSweAYVqon3mpvV3gYVS0X\noGI4D2M9WDz17RR02Zww6NT4Ts7YAZ/nrxeyz75ur8Wl60I2k9o9DSfGZq7bWxFkCJQMhqi51OXq\nCVwa1F6KHPpoc6RUX6Ld0aGQyhc1y/y9BA/oO+OXY1xQNCJ1Y275ueVMD56C6cFTpM8+WsELKIpf\nAgaMwoPJmBiMyEAjwgK8wTKxKHOvphZrS+j1AGzA3ORIcCyH02c10rN4qw0waY3gW3l0ODqx+Tsp\n2PX5WVx2O1ydzi6EegfjSi814a9RiRBrj0fjghPN7sSQaP9ATP3WePyr9Qucb70AG+lC8bIkRAYa\noWZVmJ84BX2xZpEyVrN4egxe6ZEeQ0yAFeGWQMCtWB9likSzrUURNxJ5dNL3sPPUO4izRONccxXO\nd55BCuLgbQDQIigla9zTUHERBrQ4HHAQb7R1OMB4CVO7S2ZG4WKZHo0QastzLAO5/23RmkEIQYej\nE1rZLXAckZIIArwFoxRqUAqG3g7UYHyDiFIeWeH39VtDAAAs7ikp0cNgGAYaTo2pQSn9nnO3iTZH\nSh2KXaHu2jNqFUeV88ZkwaI19Wkw1CwnqXWyDCtTDW2VgoI2lx1NthYEewcNSX1sANCqOWjVt17c\nx1fvCwaMpKAK9AgFymNPk6zjpfUtf/zir4qYBeAp4xBhDEOEMQwHa/6N5u4WmZClQ5LOFo2VmlVh\nSUIOrl0b3Bz8J5JXobajHskBnvPhDMNIo3QAUjXHvV/+FVMCk9HtErwq0auWDyTaHR0wu41Pq70N\nH1w9jMv2nhgFT3iYtSbJYCRZE/FZw+fSuSJO3olmt6Cgj9aMlAh/uOqm4fznFxBhDJNSyW8F+X3O\nH5ONtKAUWPV+aOpuRrfThrzoXOn5ezPROh6vZm5DafWHONdchX9dPoLJAUm40n4VWk4DjuXAEhYc\nw4FwDrAuHmqiAt+tAWtswv88NhV+Zh3iIr3ReFWs1kjAdxjBaLvAqJyIMIbBwTtwpukLhVy70ZtD\nu7t2jtifPJmy5pafvzfUYAwBNxNUWxqXd8P9ooch1hlgh8HMIcdyyI6cjb1fKIUV+3ODk/0n4lhd\nBc40KUuiBhoCEGWKxNnmLxHo5S9Ne8hLXIoSKfJU2nsFNatCiHeQx2gYgJTCCQi1QkRj6uSd+MMX\n7yqOtbnsYMBIMSwtp5WmPeTV29rs7ah3y6sHeCljLINNkCFQGvTcjLG+cbBozWixXcf2Y6+jobtJ\n4VH3HpGbNEKndt3Wio9rPdPP5ZlzsZYoWHQWHKr5N2plhZPsLofkYfi4g/STA5Lga/GGH26gpXYD\nQgxBWBh1P8b6xCHGMkbanhM5Z0Dn916T9crxnwPomc5jGAZmrQkttuvgCQ8dp0VSaDhOtzXBxrYB\nMEqGQPwtJIWNwXV7C6o7LiPMGIKGLiGlV66N5WfRYoyPBcebAbO7P+mrhOytMuQ90Ycffoh58+Yh\nNzcXO3fu9Nhvt9tRXFyMnJwcLF++HF9/7SkmNty4U41/i9YMHafD541CVkR/dRruNWaHzsCCMdko\nilsibZN7GHJ0Ki2eSF7tsT3ZfwJWTXgQK8YuRUboNMnD6KsmcvAAO69vmm+P/Vaf2+WdXoDeCj+9\nLzZP3dDvdWaGpkt/q1gOZnen2tDZk/Pfam/DsboKsAw7JOnWt4tepcP65O8DABrcAx957EXbayBh\n1vZ4GH1fr6fttCot/NwGQV6LxME7PBa3MgyD9LAUKZHkVmEYBgui7lcYi1ul3eGpNCz3IEXD2uHo\nhIpVYVyQkHFZ11mP67ZWnGxQrlL3NRjh6yU8X5h3iCRGKn9HCFzSlJTlNp+9L4bUw+B5Hlu3bsWu\nXbsQEBCAwsJCzJ07FzExPRlFf/rTn2A2m/H+++/jwIED2L59O37yk58M5W0NOao+Aty3AsuwCDT4\nS4v36js9iwjdizAMg4XROYptfB8yF3JK0orh5J041XAGHY5OWN3TKmJnKc6Tn7h2yuPcCX7jPLbd\nC0SbI/Fa5kt45vAPFenWetmUlDg1E24MQV50Lg5f+VgKXmtYNUxaE+aEZUDHafHP6kNI9Bsnqab+\nq+aIx3fGW2I8FozebQINAZgbPgttjnYcrf1UsU9uMNSsWurQ93/1jz6vZVB7warzRUN3E8waEwxq\nYVBWJyvSVN1Wg5r2r8Ey7G0biKFgbsQs1HXW41TDGagYzmMGQjRuLuJCp7NLSlr49ef/1+f1vFR6\n+GgtONd8HtHmSKl/+MMX+6Rjvmy5AAAesaY7ZUgNxsmTJxEZGYnQUCHtcOHChSgtLVUYjNLSUqxf\nvx4AkJubixdeeGEob+kbId4nBvMis5B8BwFqcdQAAMvi8wfjtu4KvUuQ9kacVpJ3rHLE/P86mdFc\nHp+PWWEz+jz+XoFjObx83w8VUxJsP/XQ542Zi/sjMrH+4HMAgB/P3goGDBiGwZKY+cgKvw9mrUmK\ne8lH4WaNCQ7eccfpxEPFt+IWgSc8jtZ+ijhZhp98hP1f05+FQe2FsT6xONdc5XENHadDRkg67gud\nhsrGL5DoN06q4d3bePKExxhThMdU0N3EpDHi0Un/geu2Vmg5DT68UqaQ6rDIpH4SfOMRY4mCXqVH\nl7NvzayxPnGItUQhK/w+cCwneeF9VQQ0qL0GdSAxpAajrq4OwcE988yBgYE4dUo5Uqyvr0dQkNB4\nHMfBZDKhpaUFFsu9417fKizDIq+PPOxboTBuMbScBkvjFkvu+nAkziIsMhvonG9veqeBzh+Tfc8b\nC5H+XlT59Ir8WLPGBC2nURoZ2WjZr9fiudzIrD7z/e81WIbFj2dthUrWHvLqdOLzPZG8GicbKlHb\nUY+/XXhP2r9ywoOSqsKMEGEhYn/TkfMis5DZR0rrvYD4nDd6Fx4cVwiGYfB40kqcvPY5ciIz8fTh\nHwIQ4ikTrAmI8xEMr5i+31+qNADMCp0+SHcvMKQGo3eBkYEcQwgZtFz64Yy/lx9WTnjwbt/GHWPV\n++GnmdskYcPbYWlcHv785d+wZuJ3pfUow5G04Mm40lGL2f28xFtnPHfD3z7DMJgckITj9Z9hScx8\nzAm7NzvGvui9TiHRbyzeu1gqZRkBgmFJ9p8A+ANTA1PgIi64iKtP48AwDMb5xOFss5BNtSw+H+lB\nqQodseGCWH9mTliG9P8fbY6UtNz+O+N5qFiVlHnWmyiTclGeWWPCjJCpmBqYgkDD7QX7+4MhA+nV\nb5OKigq8/vrr+NWvfgUAUtB7zZqe9K7Vq1dj3bp1SEpKgsvlQkZGBsrK+lGjpFAoFMpdY0gn+iZO\nnIjq6mpcuXIFdrsd+/fvx9y5cxXHzJkzB/v2CcGa9957D9Omeco6UygUCuXuM6QeBiCk1f7oRz8C\nIQSFhYVYs2YNXnvtNUycOBFz5syB3W7H008/jTNnzsBisWDHjh0ICwu7+YUpFAqF8o0y5AaDQqFQ\nKCODeyf3jEKhUCj3NNRgUCgUCmVAUINBoVAolAEx7AzGzbSpRholJSWYMWMG8vJ6BAuvX7+OlStX\nIjc3F6tWrUJbW8/K3xdffBE5OTlYsmQJzpw5czdueUiora3Fd7/7XSxYsAB5eXl45513AIzOtrDb\n7SgqKkJ+fj7y8vLws5/9DABQU1ODZcuWITc3Fxs3boTT6ZSOH2l6bb3heR4FBQV49NFHAYzetsjK\nysLixYuRn5+PwsJCAIP8jpBhhMvlItnZ2aSmpobY7XayePFiUlVVdbdva0j55JNPSGVlJVm0aJG0\n7eWXXyY7d+4khBDyi1/8gmzfvp0QQsjBgwfJ97//fUIIIRUVFaSoqOibv+Ehor6+nlRWVhJCCGlv\nbyc5OTmkqqpqVLYFIYR0dnYSQghxOp2kqKiIVFRUkCeffJIcOHCAEELI888/T37/+98TQgjZvXs3\n2bJlCyGEkP3795MNGzbclXseSn7zm9+QTZs2kUceeYQQQkZtW2RlZZGWlhbFtsF8R4aVhyHXT7us\nDgAACDZJREFUplKr1ZI21UhmypQpMJmUQmqlpaUoKCgAABQUFEhtUFpaivx8QXcqKSkJbW1taGho\n+GZveIjw9/dHQkICAMBgMCAmJgZ1dXWjsi0AQK8X5EXsdjucTicYhkF5eTlyc4WV0wUFBfjnP/8J\nQPl7yc3NHXELY2tra3Ho0CEUFRVJ2z7++ONR2RaEEPC8ssTxYL4jw8pg9KVNVV9ff4MzRiZNTU2w\nWoXaB/7+/mhqEkTp5LpcgNA+dXV1fV5jOFNTU4OzZ88iKSkJjY2No7IteJ5Hfn4+Zs6ciZkzZyI8\nPBwmkwksK7zSQUFB0vP2p9c2Uti2bRueeeYZSVajubkZZrN5VLYFwzBYtWoVli5dir179wLAoL4j\nw6qAEqFLRm5IX+0z0nS5Ojo6sH79epSUlMBgMPT7fCO9LViWxbvvvov29nasXbsW58+f9zhGfN7e\nbUFGkF7bwYMHYbVakZCQgPLycgDC8/V+5tHQFgCwZ88eySisXLkSUVFRg/qODCuDERQUpAhS1dXV\nISBgcMW1hgN+fn5oaGiA1WrFtWvX4OvrC0AYIdTW1krH1dbWjqj2cTqdWL9+PZYsWYLs7GwAo7ct\nRLy9vTF16lR89tlnaG1tBc/zYFlW8bxiWwQGBsLlcqG9vR1ms/kmVx4efPrpp/jggw9w6NAh2Gw2\ndHR0YNu2bWhraxt1bQEIHgQA+Pr6Ijs7GydPnhzUd2RYTUkNRJtqJNJ7JJCVlYW//OUvAIB9+/ZJ\nbTB37ly8+65Q6rOiogImk0lyRUcCJSUliI2NxcMPPyxtG41t0dTUJGW6dHd3o6ysDLGxsUhPT8d7\n7wmy4PK2yMrKGrF6bRs3bsTBgwdRWlqKHTt2ID09Ha+88sqobIuuri50dAjV/To7O3HkyBHEx8cP\n6jsy7KRB+tKmGsls2rQJ5eXlaGlpgdVqxbp165CdnY0nn3wSV69eRUhICF599VUpMP7CCy/g8OHD\n0Ov1eOmll5CYOHzlwOUcP34cDz30EOLj48EwQnGh4uJiTJo0CRs2bBhVbXHu3Dls3rwZPM+D53ks\nWLAAjz32GC5fvoyNGzeitbUVCQkJ2L59O9Rq9ajRazt69Ch+/etf48033xyVbXH58mU88cQTYBgG\nLpcLeXl5WLNmDVpaWgbtHRl2BoNCoVAod4dhNSVFoVAolLsHNRgUCoVCGRDUYFAoFAplQFCDQaFQ\nKJQBQQ0GhUKhUAYENRgUCoVCGRDUYFCGNcuWLUNBQQEWLlyIxMREFBQUoKCgACUlJbd8rdWrVw9I\n7vq5555DRUXF7dzuLVFZWYm///3vQ/49FMpAoeswKCOCK1euoLCw8Ibqo6JUxHBh7969KCsrw44d\nO+72rVAoAIaZlhSFciuUlZVh+/btSE5ORmVlJdauXYumpibs3r1bKqizefNmpKWlAQBmz56NXbt2\nISoqCitWrEBKSgpOnDiB+vp6LFq0CBs2bAAArFixAo8//jgyMjLw9NNPw9vbG+fPn0ddXR1SU1Px\n0ksvARC0eZ555hk0NzcjPDwcLpcLWVlZWL58ueI+GxoasGnTJjQ3NwMAMjIysHr1arzxxhvo7OxE\nQUEB0tPTsXnzZpw4cQI7duxAV1cXAGD9+vWYNWsWqqursWLFCixatAjHjx+H3W7Hli1bkJqa+o20\nNWWUcCfFOiiUe4Wamhoybdo0xbaPPvqIjB8/npw6dUraJi8uU1VVRTIzM6XPs2bNIhcuXCCEEPLA\nAw+QTZs2EUIIaW1tJWlpaaSmpkbad/jwYUIIIU899RR56KGHiMPhIDabjcybN4+Ul5cTQgh57LHH\nyC9/+UtCCCGXL18mKSkpZM+ePR73/tZbb5Hnn39e+tza2koIIeSPf/wj2bhxo+Le8/PzSWNjIyGE\nkNraWjJr1izS3t5OLl26RMaOHUv2798vPXtmZiZxOp0Db0QK5SZQD4MyoomOjsaECROkzxcvXsRr\nr72G+vp6cByH+vp6tLS0wGKxeJw7f/58AIDRaERUVBSqq6sRGhrqcdz9998PlUp4lcaPH4/q6mqk\npaWhvLwcL774IgAgLCxM8mR6k5ycjN/97nd45ZVXMHXqVGRkZPR53PHjx1FTU4NVq1ZJgpQcx+Hy\n5cvw8vKCXq/HggULAADTp08Hx3G4ePEiYmJiBtpcFMoNoQaDMqIxGAyKz8XFxdiyZQtmz54Nnucx\nadIk2Gy2Ps/VarXS3yzLwuVy3dJxA62zMHnyZOzbtw8fffQR/vznP+Ott97Cb3/7W4/jCCFITEzE\nrl27PPZVV1d7bON5fkTVeqDcfYZPBJBCuQlkAPkb7e3tkjrpnj17+jUCg0FaWpokK33lyhUcPXq0\nz+Nqamrg7e2NBQsWYPPmzTh9+jQAodaFKGMOAKmpqaiqqsKxY8ekbSdPnpT+7urqwoEDBwAIJUoB\nIDIycnAfijKqoR4GZcQwkNF0SUkJ1qxZg+DgYKSnp8NoNPZ5fu9r9bfvRsf94Ac/wLPPPov9+/cj\nOjoaqampiu8TKSsrwzvvvAOO40AIwdatWwEAM2fOxNtvv438/HxMmzYNmzdvxhtvvIHt27ejra0N\nDocD4eHhePPNNwEAVqsVX375JYqKimC327Fjxw5wHHfTNqFQBgpNq6VQhgibzQa1Wg2WZVFXV4ei\noiLs3r0b4eHhg/5dYpbUkSNHBv3aFIoI9TAolCHiwoULeO6550AIAc/zKC4uHhJjQaF8U1APg0Kh\nUCgDgga9KRQKhTIgqMGgUCgUyoCgBoNCoVAoA4IaDAqFQqEMCGowKBQKhTIgqMGgUCgUyoD4f001\n1ZxdsABYAAAAAElFTkSuQmCC\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f96f1241810\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "test_accuracy tf.Tensor(0.99, shape=(), dtype=float32)\n" + "('Duration:', 4.146992206573486)\n", + "('Duration:', 4.107615947723389)\n", + "('Duration:', 4.07602596282959)\n", + "('Duration:', 4.113464832305908)\n", + "('Duration:', 4.100026845932007)\n", + "('Duration:', 4.145462989807129)\n", + "('Duration:', 4.11216402053833)\n", + "('Duration:', 4.094243049621582)\n", + "('Duration:', 4.095034837722778)\n", + "('Duration:', 4.11162805557251)\n", + "('Mean duration:', 4.1102658748626713, '+/-', 0.020919605607527668)\n" ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXdgFGXex79TtiabZJNsGoGEBEihhRJAQBQQQaqABQue\n4h3HqYeK7ThFz0NRz8N2nHIoqKe+dyIKJ4KA0gQpofcaSO/JJluSrTPvH7PTtiSBJIIwn792Z6c8\n88zs83t+9SFYlmWhoKCgoKDQAuSVboCCgoKCwq8DRWAoKCgoKLQKRWAoKCgoKLQKRWAoKCgoKLQK\nRWAoKCgoKLQKRWAoKCgoKLQKRWAoKCgoKLQKRWAoXHfs378ft91225VuxjVPv379UFJScqWbodCO\nKAJDQWDUqFHo3bs36uvrZdunTJmCzMxMlJWVAQD+9Kc/ITMzE8eOHRP2KSoqQmZmpvB95syZWLVq\nlfB96dKlGD16NPr374+bb74Z8+bNAwBMnDgR/fv3R//+/ZGdnY0+ffqgX79+6N+/P5YtWxbQxiVL\nluDZZ59t030OHDgQ33///SUd869//Qtvv/028vLycNNNN7Xp+jz+fXStcejQISQnJ1/pZii0I/SV\nboDC1UVycjLWrVuH++67DwBw9uxZOJ1OEAQh7EMQBKKiovDOO+9g+fLlsu3BWL16NdauXYtPP/0U\nycnJqK2txZYtWwAA3333nbDfzJkzcfvtt2P69OltugeWZUO25XLZvn07nn76abjd7nY/99WK1+sF\nRVFXuhkKVxGKhqEgY8qUKVi9erXwffXq1Zg6dWrAflOnTsWZM2ewf//+Fs95/PhxDB8+XJhtxsTE\n4M477wy6b3OVanbs2IGlS5di/fr16NevH26//XYAnKB5++23cc899yAnJwclJSX45ptvMH78ePTv\n3x9jxozBl19+KZzHX0sYNWoUVqxYgcmTJyM3Nxfz5s2Dy+USfrdYLCgsLER2djZmz56NqqoqQQuq\nrq4Gy7JYtmwZxowZgyFDhuDJJ5+ExWIBALhcLjzzzDMYPHgwcnNzceedd6Kurg5vv/02Dhw4gIUL\nF6J///545ZVXgt7z448/juHDhyM3NxczZ87E+fPnhd+cTidef/11jBo1Crm5ubjvvvuEdu/fvx8z\nZsxAbm4uRo4ciTVr1gh9JdVqVq9ejXvvvVf4npmZiS+++AJjx47F2LFjAQCvvvoqbr75ZgwYMADT\np0+XPXOGYbB06VKMGTNG+L2yslI4V3FxsdAPb7zxBkaOHInhw4fjL3/5i9BWs9mMOXPmIDc3F4MH\nD8b9998f8h1QuLIoAkNBRt++fWG323HhwgUwDIMNGzZg8uTJAQO5VqvFnDlz8NZbb7XqnGvWrMHy\n5ctx/PhxMAxzWW278cYbMWfOHIwfPx6HDh0SBkEAWLt2LV555RUcPHgQiYmJiImJwbJly3Dw4EG8\n9tpreO2113Dq1Clhf38tYcOGDVixYgU2b96M06dPy4Tmzp07MWTIEGi1Wnz44YeIi4vDoUOHcPDg\nQZhMJnz66afYsmULvvjiC+zYsQMRERF4+eWXAXADss1mw44dO5CXl4eXX34ZGo0GTz75JAYMGIAF\nCxbg4MGDeOGFF4Le80033YQffvgBu3btQnZ2Np5++mnht9dffx0nT57El19+iby8PDzzzDMgCALl\n5eWYPXs2HnjgAezZswdr1qyRmQv98e+LLVu2YNWqVVi/fj0AoE+fPvj222+xb98+TJo0CU888YQw\n2K9YsQLr16/HRx99hAMHDmDRokXQarUB533zzTdRWFiIb7/9Fps2bUJlZSX++c9/AgA+/vhjJCQk\nYO/evdi1axeefPLJkG1VuLIoAkMhgClTpmDNmjX4+eefkZaWhri4uKD73XXXXSgvL8eOHTuaPd/k\nyZOxYMEC/Pzzz5g5cyaGDh0a1D/RFqZOnYr09HSQJAmapnHTTTcJGs3AgQMxbNiwZrWhBx54ALGx\nsYiIiMDIkSNlwmXbtm3N+i1WrlyJJ554AnFxcVCpVHj00UexceNGMAwDmqZRX1+PixcvgiAIZGdn\nIywsrNX3NW3aNOh0OuG8p0+fhs1mA8uy+Oabb/DCCy/AZDKBIAjk5ORApVJh7dq1GDZsGMaPHw+K\nohAZGdmswPDn97//PQwGA9RqNQBg0qRJiIiIAEmSePDBB+FyuXDx4kUAwKpVq/Dkk08iJSUFAJCR\nkYHIyEgAcm1x1apVmD9/PgwGA/R6PWbPni2YI2maRnV1NUpKSkBRFAYMGNDqtir8sig+DIUAJk+e\njPvvvx8lJSWYMmVKyP3UajUeeeQRvPvuu1i8eHGz55w4cSImTpwIr9eLH3/8EU899RR69uyJYcOG\ntUubExISZN+3b9+O999/HwUFBWAYBg6HAxkZGSGPj4mJET7rdDpUV1cD4Aa9Xbt2Yf78+SGPLSsr\nw2OPPQaSJIVjaJpGTU0NpkyZgoqKCsybNw9WqxWTJk3CvHnzWuUbYBgGb731FjZu3Aiz2QyCIEAQ\nBMxmM1wuF1wuFzp37hxwXHl5edDtrcW/L1esWIFVq1YJfWK322E2mwEAFRUVLV6rrq4OTU1NMt8U\nwzCCQHn44YexZMkSzJo1CwRB4M4778Ts2bMvu/0KHYeiYSgEkJSUhE6dOuGnn37Crbfe2uy+06ZN\ng9VqxQ8//NCqc1MUhbFjxyIjIwPnzp1rj+YCkJs/XC4XHn/8cfz2t7/F7t27sW/fPowYMaJZ/0go\njh07huTkZBiNxoDr8CQmJuLDDz9EXl4e8vLysG/fPhw+fBhxcXGgaRqPPvoo1q1bh//+97/Ytm2b\nYEpryXm+du1abN26FZ9++in279+PLVu2CPdgNBqh0WhQVFQUtD3BtgOAXq+Hw+EQvvNCQIq0Xfv3\n78dHH32E9957D/v27cO+ffsQHh4utCMhISHktXiMRiN0Oh2+++47oY/279+PAwcOAADCwsLw3HPP\n4ccff8TSpUvxySefYM+ePc2eU+HKoAgMhaAsWrQIn376qWCPDgVFUXjsscfw4Ycfhtxn9erV2L59\nO+x2O1iWxfbt25Gfn48+ffpccrtiY2NRWlra7ODvdrvhdrthNBpBkiS2b9+On3/++ZKvBXDmqBEj\nRgjfY2JiUF9fD5vNJmy7++678dZbbwlhx3V1ddi8eTMAYO/evTh79iwYhoFerwdN04J2ERsbKziF\ng2G326FWqxEREYHGxkYsXrxYGMwJgsC0adPw+uuvo6qqCgzD4PDhw3C73Zg0aRJ2796NDRs2wOv1\nor6+HqdPnwbAOaI3bdoEh8OBwsJCfP31183ev91uB03TiIqKgsvlwpIlS2C324Xf77zzTrz77rso\nLCwEAJw5cwYNDQ2yc/Baw6JFi1BXVwcAqKysxM6dO4U+5oWOXq8HRVFKdNZVSocKjD//+c8YOnQo\nJk2aFHKfV155BbfeeiumTJkisxsr/PJIZ5adO3dGz549g/7mz8SJExEXFxcQessTHh6OpUuXCtE8\nixcvxl/+8hf0798/5PVDMW7cOLAsi8GDB2PatGlBjwsLC8Pzzz+Pxx9/HIMGDcL69esxevTokOds\n7rrbt2+X+S/S0tIwYcIEjB49GoMGDUJ1dTV+85vfYPTo0Zg1axYGDBiAGTNm4OjRowCAmpoazJ07\nFwMGDMDEiRMxePBgTJ48GQDnN9mwYQMGDx6MV199NeDat99+OxITEzFixAhMnDgR/fr1k/3+3HPP\noUePHrjjjjswePBgLF68GCzLIjExEcuWLcOKFSswaNAgTJ06VRAYDz74IFQqFYYNG4b58+cH/Df9\n++LGG2/EjTfeiLFjx2L06NHQ6XQyk9VDDz2E2267Tbj3F154QdBgpOd6+umnkZKSgrvuugsDBw7E\nrFmzUFBQAAAoKCjAgw8+iH79+uGee+7Bfffdh9zc3JDPROHKQXTkinv79+9HWFgYnn32Waxduzbg\n9+3bt+OLL77AsmXLcOTIEbz66qtYuXJlRzVHQeGSqK2txe23396iU19B4XqhQzWMgQMHIiIiIuTv\nmzdvFmLp+/btC6vVipqamo5skoJCq7Farc06uxUUrjeuaJRUVVWVTL2Nj49HZWUlYmNjr2CrFBQ4\nUlNTkZqaeqWboaBw1XBFnd7BrGHXS9kFBQUFhV8bV1TDiI+PR0VFhfC9oqIiZJKYlI6oFdSefHpo\nFdad3QwtrcG/p79zWee468s/CJ+jdVFYOvm19mpeh1FurcLj618CAKy8+wMAwHu7V2Bn0T7EhcVg\nycTg5S8W/7wMe0sOAQA+mvI3RGgNwm9bLvyMpfs+DzjmnxNfgSksJmD71cxft76D41Vn0D2mK169\nhSugWGGtwlxfn/nzybS3oFfphO8sy+LulY8I33vG9cBLI6/erOjT1efx4hZ5fg7/Xizc9g6OVZ5B\nt+hULBrzHABgT/FBvLXrQ2hoDT6+/e+4d9UfAQDzhv4OQzrLAyQ2nNuGFQe/RDAitRH4cMob7X07\nbeJ8bQH+/CPXpi/vel82fj2z8VUU1otVfVfe/YHw/3/hprnok5CFnYX78N6eFQCA/9y5BBQpRpF9\neWwtvj65Puh1aZLGx1MXQ0Or2+U+OlxgNOdTHz16NL744guMHz8ehw8fRkRERKvMUQRBoLra2p7N\nbFdcDi8AwMN4L6udDU6L7HtaRGrI85hMhqumL87WivH4fJtcLq4v3J7gffG//O8FYQEAP58/jAh1\nOPIqDuGejGm4UFUa9FrPblyEV4Y+DzWlEra1R194GS/+e+YbDIjPQWZ090s+nmVZfHZqJeocZmhp\nLYYkDEBOXG8AgMvtey/cDKqrrWhwWvDS7tAD2+5zR0CRFI5UH8eMjGmosFfJfj9RdRZPr38FRk0U\nHup5L1Tt3Bdt5Xx5YGlzvk1a6Ll96grww8ndyDH1wt4CLrLM6XHi6e/FqLG3dn2Il4Y8ixJbGc6Y\nz+PuHrejrC60r7PBYUFpRS3UlBpWlw3fFHyLMUmjkBSeEPKYULi8bnx+aiVGdxmBlIjLS4Yss1Xg\nrYMfCN/nb3wD47uOQVZ0DwCAjhAnBUZNlOy5LdnzKSLVEahsFJ/9/gsnUeeoR4GlCNO7TUJhbVnI\na3sYD/acP4rsmAysPLsGjw6beVn3wNOhJqmnnnoKM2bMwMWLF3HzzTfj66+/xn//+1+hEBxfvmHM\nmDF48cUX8dJLwWdavzYogutWL+O9rONP18kT2rSUps1t+iWoaaoN2Eb6XjEWgRMHs6Memwq3Bpzj\n3UPLsLt8H87W56O2iYvbD6P1sv3s7kYUWppPGLscTtSexq7yffjH4dB5Jc3R4LJgb8UBnKu/gGM1\nJ/Hh8c9C7ru1eCfcjDvk7yW2Mrx/ZAV+LstDsbUUp+vOBuxTZC3FkZoTOGM+H+QMV5bqIO8Dj0Ed\nLnz+8Ni/Acjf+zJ7hWz/U3Vnsfz459hZugd1DjOsruDCMCEsHgBQ43tvVp9fh7ySw/jPmW8u6x4O\nVh3Bgaoj+Nv+f1zW8QDwwdGP0eRpEr5faCjEksMfCd/rnGbhc7TWCDfjEb43uhtRaC2Gw+sUtlXa\nq/Hxif/D1uKdsLhsqG2qAwECNCmf/9/ShQsHr2qsgcPjxPaSXZd9DzwdKjAWL16MnTt34vjx49i2\nbRumT5+OGTNm4O677xb2efHFF/HDDz/g22+/lcX9/5ohfeoiCxb1Ti6JqcRaJggQs6MeDU75C+/y\nulBm4/4kp83cH+d3vbjZgJdl4GE8KLaGnklcaRiWwa7yfQHbSV54sty9syyLQksxGJbB2gsbA/b3\nSIRsibUMlY3VoEkaTw98LGDfat+g0J6YnQ0t79QMlhADGQCwLF90kROeUnOTFL7PaiT3d67+Ak76\nBMYfc34XcMy6i5uEgabOYYa5qW33cbkwLIMiSwk8jAe7g7wP/D35TyCqG2tR66hDplGu1fGDoFSA\n1DTVweLiEief6Pd7YfsNibkYkjDAtw8nrEps/H+mddkDdRYHzFZxcFZToinnWM1JVNirUGGvwvGa\nUzhWcxIuryvYaQBwE8Yia0nISUFtkxkMy6CuyYyUiM5Qkyq4GTfOmy8AAAbG5+Dtm1+FilTJjiuQ\nTJRqHbWobqpFrC4a83OfELaPSx2N7lFpALixxeqyoT1Qakl1ABQh2hef//lVPNL3Ybx/ZDluSMzF\n/Vl34oVdiwAA/xz1N2G/5ce/wPHaU3h24B9xvv4iwlVh6GzoBIAbbL8v2IwNBZvx/KB5l6VadzTH\na06h2MqZj6QaEW+q5U2TJ2pP44OjHyNKEykIUym8YAGANfmcXTZBHyczPWkpDRxeJ2qbmcFeLv7m\nwEuluT9mo2+Wyc8WvSGq9kZrolDrMMs0ttXn1wHgZtAmnei7yTB2wxnzeRRZS/Fz6V7c3HkY3j20\nDGEaHZ7tP7dN93IplFTb8PW2fPTKteObC2sQ7e6GelXg831p9+v4x8jX4fKKg2iMNhr5DVwxwz6m\nnrjQUACXb5A1aiJR3VSLY9UnhP2rm2phddlAExQiNWLYvppSIUYXDQAotVRj87bDqDJxpiuPn7Zf\nU98EmiYRFS7X3p9+n5uFr/jTKADA7hOioFp69JOA+xmfegsmpAUvn/Ppyf/iQNWRoL8BwIu7X8Or\nw56Hh/UiRmtEbVMdah11WHKE0z6KS904qKqGs5EGqRX7a2fZXuFzibUcNrcdnQ2doKXFe1GTavyY\nVw5oASfjgtXdPuZJpTRIB0CT8rIGxVbOlhtsxsVzvJbLcr/QUAizox6JYfGCY4thGRyrOQkAqG66\nOvNUpIObRjIrc/pmYIxvdl3la38wYQFwNld/0iJTZbMs3uzQnMnjcuEFRqSac7x7GQb/XH0Mu49X\nNHeYgKUZgcELE6uLqzZ74Dw3+52UNhYP97offWI5DZsiKURro4Ka+LKM3WW+Cj2tw9RuEwBwfepl\nvKhpqkVhfQnqHOaA49uTH/cX46utnCns3a+O4Eh+LfZc4LSgGoZ7529LvSXgOLu7USYwaJIW3odY\nXQxujRDXw6B8GkaDRHOrbaqD1WVFuDoc63aJfhI1qUa4iqsEfKqkCscuVgmze/++fHbpbsxbIi8X\n4/aIAtza6EJxlQ2H85t/7rVN9SF/a05Y8Jgd3H2HqcKgIlWwuxuF30rKXVjyzTGA4cYBLREecPze\n4uMAuH7TkKLAyC9qxLHz3LmdXmez7+WloGgYbeRg1VEcrjqG32TPEAZ4f0e/nhZNDwv3/F34vDZ/\nA1SUGuNSRwnbVp37FgD3AvCaitnRgFJbOYDmB6S28lPJLhRbS3FfVvDFjZpD2q4GlxX/Pvkl7s+6\nEw4PN5vmBYaOEmtTqSk1Mo3dcbRGnD1KNQyeblFdZfbZWF00SqylMpPNt/kbcOrAGUxOvQ1ZMT0u\nuf08dc564R7mb34LpohwHL+QjANnqtE9ORKxUeKzLKm24b+bz+GeW3qgUyw3UAWzrf/f6VWYkTEN\nNjdXg8nubsT8D3fDbKgHHQ/0NfVCYlg8DldxS94SBIlYXUxQv0QPYzpUkr6w2LzYcsgFJHOai9Ut\nPofTdecwNGlQ0Ps8U2TGpn3F+O3EbOg0wYeB7y5sxL7yY4jWRYAFgyaPAwzLQEfr0MXQCet/5Aaw\niUNTUW/jJgaNbANAAKSGKw+SY+qF7wt+lJ3X4rLCzXD7EwyNysYqwTz57bZSnDsHqLOiQBnqg/rv\nCswVsLhsiNfHYceuKuh81dBVlApNjZxK64EThEo0LTV6mvDmyjzcOSIT8UbRH3ax3IIIvRoxkVqY\nbU7Qnc4BXhrnSnrjbHE9QDbvh9xbuR+RWgOmpN+GAksRNhRsxm+y72l1octKO1f8sbLGBcbrN3+n\n5JOnRisF0k9mXLRdAEFx/4ldR8VCkkXlTQDDTdzqbHYcbWgfc7aiYbSRTYVbcaDqCCokUQz+g56W\nFgdJ6X4bCrdg7YUNcHsDbZycwOAeD6+uA8EHpPbiy7NrsKt8nzC4Xwr87FnnE457Kw6g2FoKh5cb\nOPg+kTr0RiYPl5magOAaRh9TT9ASM5+aVCMxLB6ltjLBhry1ZCeKGkqxvfTyigzy1DvEGaOFqEC+\n9TzUXblZ3PnSBtTbnCiv5Qb+XccqcLLAjAUf7QXjGyD4fsiNF8NAfy7LQ4W9SuhXFiyqnZXCgPDN\n1iJsPViCKenj0cXQCQ9l34P+cYGFGRl7BGpKw1FRLXGA1rpQWcP1gcPjkJnE/IMnpCz+8jAOnavB\njqPlQX93ed34vmAzapxVOFt/HufqL6DEVoYyewXyGy5ia8lOYd+iSiu8DHf/UgcuADQ1EtDV9ZJt\nyztbJGgYXpf8+Z8r4N4X98VeiNMmYKZk8kKAAOuhcc56Gm7GDT0ihdk3AOw8VIn3vuQ09WqrBYRK\n7l84XVGK91cfR53ER7Hw0/145oNd8DIMzBYHVJ3yoepyBkv/dxyb9hWDoFoOXNlUuBV/+7+DePvA\nv3Cs5hS2l+zCgdLgdfFU9Wny+63mBvIT5y0wN8jffY1Fvi/rDRTsfPs276rDFz+Iz7vW7BH2P3i+\nAjtPFrR4H61BERhtwOayo8TniJaqvP720pbIbygI2BatjRKcn1KkA0JNUy12Fe2XRWC0B8EG7Zaw\n+GykRk2kuM1lFTQMN+NBdWOtYCKY3fs3mJw+DoTfK9jolt/LY31/Cx2tlcWdqygaGdHd4WG9OF/P\nCVM+qv2c+cJlR6fxbfaHiqoBaBfqbS7MW/Iznv9wLxiGhcPF9xOLjScOg2VZ4fgp6eOQFpkqnKPc\nXik/Z0QNCJ/AOHDSjM82nQXp0YM8PwKEMxLDOw1BnF4eYu48MRSfb8jHwk8PCNusNq8wMDS6HbL2\nHyo/jcIKuU/m+MVavL3yCDxeboCvqGvEiYI6vPb5ASz6/AAaHW7YHW68vqZ15epBMDhUykdvMSDU\n8uf3zdZiNBWnwmsxCtvWHzgnmCrh8csP8H1nHeGYYnoAbnsYWA93f16nFt4GsU+OHyUBVsxnqKl3\ng/VwAsjqsoOK5sxJjJObxNCmUphtTahsaAARJjeJ/ufHc9h58ajYDC8n3DUarp+cpwI1NakScbqo\nHh7fu731cAE+3x184mI52w2sS9SaTpRx1YpZLw2WEf8LjsMjMH2oXNDCK/4HXOf7wms2Cd+rKuX/\nI9ZLi8KU9Mq0rbagCIw2cMZ8Toj2kJpH/DWMlgawiw2FAdvi9LEy5zn/WWr6ef/Ix3hn93JsKNhy\n6Y1vBvdlCAyrywY1qUKYSlT36xz1cHjEtRe+OP2VIDDUPp8EyZfr9g35/iY3qSOPhyZpZBi7AeCE\nLcuywozV4XWi1B581hwMl9uLd746ggNnquD2umXhi1JUyeewcqtoIqq3OYWZKt35DL6r+g92leeh\nys6FOIarw2W+nHJflE+vmCzuvg31osnB98d+e+VhnCo04/3VnGmqpyEnRKvFQdLjJgCfwLA4GoX+\nI1gKDOXEe9+JA1d5rR1vfXkExy6Ik5tth0qx+L+Hca6kAedLGlBQYcWWAyUoaQxddl2EharLKexo\n+hpUdDkItQMEKY6iLEPgTIEVTU4PwrSiJkGonGh0OcEyZOCsmSXRN51z6m8+UIxV2/KF+2UdeoS7\nk4RdGUuMrC/AUABDgWUIUFE1oBO4/xVjjQIA0PFFIDudwr/2rYIme7dMuG07fwQHveuE7zf04drg\nZrn3ShPENEawcl8ly3LDqdnuAGkI5T8i4a0XB/oGt28/6QAPYGhGV4zom4S//eEGDO7MCY5ErZgH\nwro1gEU8D+uUR9yxXhpd431CmvSgU6f2GeoVH0YbkKr8cg1DPuC2NAC7/MLuJqeNQ2pEF5lpKEoT\nGRB/bvY5NWsvMby0zuJAk9OD+Gg9ln17AgMz4zAoK17S3tC5AaGwumwwqA0yX0NNUy0cXid0tA5N\nniY0eRxCX/D78WY3kiDhZb0B0RxScx6PiqCFaJgGpwVuxi0L07S57LL9nS4viqqs6J4cFXCus8X1\nOJpfi6P5tfjDnWkBv/P4z9BqLQ7UWZzQqCgQJs7xes58AYXWYrD2CLhdrExgFPpsyEZtFDcrptyI\nCKdg81LgB72Saq7dZqsTRZVWrF9LgYwcANajBtwhMnUZCnFGPSxeCnZXk/B+UA4jPLoaWJkGeBkG\nBEHgpRV5Ie+Px9bkxs5j5SBi7MF3YAmA8PU1yYCK5e6LNJi5dkrQUlo4fPemogkIUwfahXp7I8BQ\niNLr4K/TZaYYcSS/FicKuPdbN4h7HxmrEXfn3gzSmAG3U4UP8+SJjNyASwBeFUCK5ijGFgXEcpMI\nOqEIrEsDggBuH52AtIiu6GQKw6INX0F6x+kpWuw+CpiiVagD0D0pBv4GPoIQ37pbB3XCT17f5Idk\nQGgb4Q9xhsuLcBdmgfWooEq6CELLXVWn0iI+NhJl7lqwXhI39U0GSRCIjdRhZu8pGGTuiazoHnhs\nK6ddPjV9ENSMAbVkHzRaaHgjo/HFD2KeTlJUBIb2TMLXZgIE7YGZKQ/QWC8HRcNoA6fN56H1OXGr\nmzFJ8Xb8UHhZr0ybGBDPzSwJyexJRakQrgpDfkOBEDHFz3qks/L8sgb87f8Owtoot9+6PQxOFNSB\nZVk8/+FeLFieh1XHf8AR51Ys/d8J2b5v7l+CmqZabC3eiY+OfSY48BiWxbJvT2DHEbkDbfme71Hv\nbADrVqOkUvyjbCneAZvbjnh9LGhGhxJbGTYUcAsL8b4LXnDwUVBVjfIoMA2pxrELtWh0SKNqVIjw\nJX5ZXVaYG+V/Tj589Wh+LQ6fr8Fnm87gtc8P4uDZwNXl+EEapAcf5zeTrOfngDxX0oCSahv0nYtA\n0Nxv52vKQBAsvA0xOFVYL3umpyqLxPtmKNAqBjodEKbWYdygLrJzO1xefLnlPAACTIMJrD0SrEuH\nOGNg3gbLUOjVNRrw0mh0O1Bj40wtTWYuyotVNaKm3oFNecWCGQqkB+qMfSCjKqHJ3g1N9m6A4vq3\nuMoGc/hR0DEVINjA4UHmyyUYkL5XlI4vgiFFrpWEq8W1yylKslaK2gm7ywmCodApNjDyJ6NLoGAH\nOIGRlhiNiARjAAAgAElEQVSJgQn9kJucHbQv/jxzAGjItQHGHin7Tqg54b+h9kswhgosPbUU9uij\nsn2i4zxIufEw6lScmJh2Y2AghXSScnP/RMBnUgozNoEgGTBN4v3H601gHb57ZSl4azlNiX93/jAp\nB0nRvhBhhkZEuCh8KZJCdkyGrJxISkws0pOMGJTQHzf36INeadGytv12Qm9kpxpBsiqQYRa4GBcy\njZcfDMKjaBiXSaO7CXUOM7JjMpBffxE2yaDtZeWDC2/HDwXDMKBICl4vJ2j4wZAgCFAEBS/rBU1Q\n6GFMx4GqIzhafQK9Y7MFDUQ6K3/3q6OwNbmxYW8RGJZFo8ODh8ZnYdW2fPywvxgP3pYJp69ExY7a\nzaDjAHexfK3remcDPjr2GYp9SU9uxg01pUZdgwN7TlZiz8lK3NiXe+Evlluwr/IQKANQcSEKlMEM\nSv7uIpyKhMtVC1KiLPCCYnzXMSi3V2JK+m34+4F/BvRNQVkTlnzFLf6j85mRfz5aCcvFElAEBYvL\nhq+2nwb04P6wJINGdxOKq2x45yt5WOPhczXo38Mk21ZYyfUdGVkDgg6tWRF+AoMzlbBwxB4TttU5\n6kCouBntP1cfQ+oQ8bmwak4wqUgVWC8FimLg8Dpg0IRhZGYnbMiTZ62fKgw0afxhSi98vP4Uiqok\nZjuGwg09E7DnqApNHgd2nSkEogDGZgRwEaS2EQs/3Y9GJ9f+4X0ScaTuEDyRtaAixUkOGV4PpsGE\nw/kVUKVyiWNhap0Q2QUAzrP9oe4mlnEB6QVJEuCnRy6dPAQ1QiMOmCBYIXeOCrMAlBfGsHDQtDgI\nTus6Fd179hKimAgAzz8wEIdrSWw8fRCM1YiYSO4loqlAYXZzn87o1ikSUfnhqPVYfecgsHTuVHx9\nmsDO0j0BCYPBcisA4GTtaVQ5xfsxGcICdyLEc4XrKcGf4lJx/co0GkDquP5TU2poVBSanL4EVpdc\nc9artMLEjPXSiAoLrlHOyJiGgoYiIbhEaF+UDkOy40HrR8GlL0fn6BiQBIkovR5mJzd5TDYkBj3n\npaBoGJdJjYN7KUy6GE71lti+Q2kYj/R9OOi5vCwjMz9Js0t5kw1FUrgnk1thjtco3D7BJHWEe3U1\nAOVG3qlKbMwrxo6j5SivteOH/dzs78CZIGs4awKd5sU2UYvgHZR86CShbsLn+3/A01+sxMJ/54HU\nNIFx6OCtTBUcd6zEQWfwJskcdgDw2ffn8b+dF2FQh+OJ/nNkdXqkM7MlXwVGm1TWObExrxiMS40G\npxVHL3CmCcb3JzxfWY2SahtIQ60wcwaAXQUn8PK/d+F/h/bjpf9swtnqYpwvqYdeQ6Nr99AZuwBA\n6KwgdFb86T4x+onQy40phIq7Vve4JBAEUFYbGAJ9rsgGlqHgVdlgc9uhpTUwRekw766+sv3UKhK3\n5op9Mu+uvkhJMODpe/phwW8GijsyFExROhjDwgDKAw/BvWs9Yrr62miBQ10F0liJxFQr7hndHdOH\nB840CQ2npZU7xbwGqbAYn3oL0BAvP4bywItAcyuvGetVOrzwwEB0T46EXis+f0JnA6FyIUytFfx9\nKRGdMbrrDegSb4BGTeGlB3Px1mPDkJYUgYmZN8KdnwMVTTdbdHRoTy7RNT5STOZ7ov8cJMVGYEbG\nVDwuyQpviQq7/H+iJpsv3udm3IAvYoklfNFwjWIRTQ2lxh+n90Fmlyj8/ZGhePauQbJwey2lgdM3\nhoSrtVCrgi9Re2OnIZiZfVdAP5AEgdmTe2LWkHGY0+chIWBGahb1FzKXgyIwLhPeyR2ri4GW1sq0\nCH8fRpPP8SsNDZXiZtwhI5NI3zE0QUNLaUGTtCAgeGd6k8cBt9eNqsZqoNtuaHruQq1FbM/zH4qZ\noaU1gYMYoWls1jH//PJdyDtViTordx+qlJPYbfkBTYn7QcWWglA7wTp9zm7ejMGSoF2cKWDrT06w\njPzezxZZsX6P6OyXRoSxMnt9kAHCJ5S8Tq64HENyAzU/ayuqqcOpmnPQZO2DOp3TMgi9BZqsPFQl\nr8Em80rUxP+Id4/9A7UWJ/p2i4FHVxV4HR80qwFBstD2/hlpSRHolRaNznHhmDou0HRCgMC8qUPx\ntzlDkRyvD/j91MUGwUkNQEg0M0nMTS88MBB/fXgwpgzvKmwz6Lk+Cdep0DVRHBBZhoRBr4LJYABB\nMoiOYaGjtHjzkVugp/WgDPXQZOVB0/0Q6uN+RoOnFlpt4N9+UA6n1VKRokmwn6m35MYIvP77IaBI\n8XmQ+sCIMpqkkeoT/lGaSKQlRWD+/QNkznAeNaUS3nv//0ZKggGRvixstYrCX2cNwt/m3CDbZ/bk\nbAzKEqtb82ZNad0xaR5H18guCKP1AXXJgiEt9gdAFqUXDIfXGRCCKzWFqSk1uiZG4Nl7+yM6QovM\nFCNidaIqrqXFSWdStNyE1hakk8/2qEmnCIzLhHdyx2qjoaU1Mj8FP2uK13PmD16YSF+6EZ1uwH2Z\ndwCAEBabFJaAv94gX+GNn0jQJAWCIGBQhcPisoJhGVk01v78Ylh9zl5S24RQtXPqfILEaBBfHlLT\nhLe/OhR0fwCwO51Y+r8TKKvhzk/oxJknGc7lLagZ32zKN5jr1DSeGfQHOI4PBevSyQZJAABLwu1h\nsOVgCfJLG3D8oiQT1z/U0h+GxLBeCWDdajDwAj6HNC8wqiwWnK7jIpqoKG4AJOhQGgSLzO5aVDXV\nIDs6A0/0myMrvQEAsWGSPzDBYN5dOXh51iCwFPfMe0WJORNRmkioSBoxkVoYwoJYfBlKJjzvzZwO\nAIiJ4NquVpFIS4pAXJQOOg0tPCfp8/I/H0EQQnCAxWOGQcMN/iZ9YPn3Wkd9UBOph+ImEmRELQiW\nwpP9/4D7/RI4Y6N0MvlNRnGz8Elp44RtYbQeD/e6Hw/1vBeT08XtwXJ7VKRKeIcpsnnreHJcuCBA\neIZkJ2DOFDH0lNdspIUNpUETNEnjqQGPYG6/2SGvw9+LVLtqDXyFgM7hSfhN9gzM7vUg+sSLpt5g\nUVaxkvdMS2vh9D0XLRUY6HG5SDWMYAEkl4oiMC4TqYaho7RwMx5htsSbpLr5in/xAkHqBM2K7iGU\nz270aSBJ4QmI0Ynx6oCYNU6RFEqrbdCSeljdNhw6L58B/XvzMdQ1ijM+QheoScy81WeKIBhQ3UWt\nQ9XlDC7EfBX6Zn3Zrkfya+Efa0+buPpREwdk4ZHbewl2XRVNITk6Gl189bBovwJqg3pw9tTPN53F\nq58dwFtfiv4Gf23En37d45FkCuNCCwFouh/mjvMJDIZwwUKKobXanK3QxAQvI9I9RY+vyrnaPT1j\nMtHdmCYk4fFInbcurwuFlmK8lvcOCiycmW9wsigwpLNGKkgeDRhKMM+pSBpRvrwVmiLx6u8G4405\nQ2W7L3x4EF56MBcRIWzaN/l8SfzskQULg4oT3v6CDwDeP7IcR6qPB2yvcdQBKgdIvQ3xqmR0i+oK\nLa0Vssr5d1caiBEex/lZhiQOEO4jTKWHURuFgfE5gvYkPV4adq2mVMJ26cDWVuQCQz5Qx4fFoVN4\naFv+gLi+IX9rDkFgGJIxKKE/+sZlY+508VyaICYtqcBQS/4f0j5qKxpFw7g64J3cERoDNL6Xklcp\necGh80l0fjtNUvhjzu/QP64PsmMyBDOMwydQgtlJGZ+mQBM0FizPQ0kZZ756f+0B2X5u1oll6w4L\n3wltI/qkiy/kotlDcHM/bvAmdFbYabmDMpjJQPyNExiFFVYx1r5JvlZ7H1M2BmSYkBjLvex8WfPo\nCK5vKFY+gzRFBnEi+khLMGJk8nCMTBgtbJtwQ4rweVBWHBKM+oAY/pRon3mCdsvMJYTaCZguIhgT\nRkcJs9x+vnUr/GfDpETQO70u/OPwRyixlQlVhWO0opCPUIt263sypiMrugf6SbK2WS8lxNtThLz9\niTFhiPQTDHqtCikJBoRiYAbnV5AODHzQRLTWGPQYvt1SKu1V0Edxs+qsBLGvn+z/B/SOzcJNyUMD\njnEwTYhQGxCliRRm0KHs5A9k340exm6YkTFN2Kajdbg/6y5kRffAjIypIe+xJWb1vBc5pl5CUU6D\n5BkEGyQJgsCtKSODnitSI+/rgfE5mJJ+GwBgbs5spEWmIi0yBX/M+R2eGvCosB+fQBoqdDU2iPDO\nMfVC5/AkDO80BARB4MGe9yI7JkOoDdYe6CTmN52iYVw5mnxCQEdphZeSV/X5AYhXLfnkNYqgkBnd\nHQ/3uh80SYvJeE7O4Rh0VSyJhgFAmFUTanmoLkEygCTKJyNNh8nDuoKMqgQVUwozSsTw2yDhkoHX\nFWeS94xJ40IojRWCOcpIibbjGzvdgISweBAEgbQk7g/HC0OSt3n7JTnRFPc9XKfC/Pv7y+oZdUuM\nxh09JmNaFlcFlCCA6Tely47vEm+QtREApt+UBS2lBaltBEF5oWODD5hSLG4uDHV81zFC5VP/SBq3\npIS10+uUZdbTBCUTEtI/ZYzOiMdyfitoWQA4k5RPw6DIdvj7+bpAWpCQHzCpED4zf/qaeoEFi/65\nnKAMV4uDTEpEZ8zp81DIwUa89+ZLtncKT8Tj/WbLZvcGdThiddF4LOe3goZyOQyIz8Hvej8gvHMR\nEg3Df40IninptwlFLKWoSJXMn3JH98mCcMmI7oanBjyCpwY8iszo7kiLTMFknwmLD3XPjA4euhps\nMa6UiM7406AncI9PiCaExeHRvg/LNKS2EtGMtnU5KALjMmBYFnU2K1QkDYqkBNugU9AwuLwKq110\nSgOBVWzF4oKcH0BH63DgTBVe/mSfkHfAD16sLymIL33gLzCmjOgiC/3M7haGiCgvND0OQZ1+DEsO\nf4Q3+UVgWiioBshfrthoNdSpp6DpfhiaFC45qHuMOAuVvuB8e/k/r94nCFS0/FUb2b8TeqfF4Jl7\n+qF7chT+8lCu8JtKkgX+tzk3YPGjw2THEgAXXukn+LQqDQzqMCHOPkmdgpao8/W9NGLFX8OQJlZW\nNsqjZwxqg6ySribIn1IqUITkMoQOgmgNyeGcKSpGy5nApNopb0LrFtU18MAg9PVVyT1v4TSP5gaW\n3Ph+su/8s/f6+kztZ3r0R9rPsn5pR1o74PJtkZrZCIKAXmIS0rcQWcQLaqvbBj2tQye/pQf4SaNs\n0vALIu2LYH6US6XD8zB++uknLFq0CCzLYvr06Zg9W+5wKisrw5///GfU1dUhKioKb775JuLjAyX/\n1cSP+0tQZbFCreW6T+d7ELxg8LAeUASFH/LKoE4VTVLSGd+u4+X4Pu8ikCqag1irEZuPl6CwworC\nCiuyUqMFH4aHlwW8fd8vL8Bk1GDUoDjsKOOcvRaXFS5G7tw0O+u5UhwSgTGmy83IieuFN/cvke2r\nV+kER76bcYMy+ZKytJypZ2DXFOT5cp2ksxh+sOVLfky7KR3WRjf0XapwSJKQHqFX40lJKKlJUgVW\nWpBQWh2Wh5//d4oxQOrJoQgK6ZFdhSTKPp07o1Nxb/zkDlz1LtPYHafN54QS4M0JDGnme4lNXnbE\noA6Xze51QRyW/KAOcFFNvJ+nJUdvczzZfw6qm+oEE4ha1gbufcyM7o75uU9AS2tAgMCLu18Peq7U\nSC5xsMHF2eGbs3XPyJwGFiz2VnAmUX7QZ4Xn3rwQlPazQdV+M2kprRVEYT5tiCAILLxhvtB2Pa2D\nxWWFhlK3GB0lnSwYg9R/e3HIM/CynhbP01FI+yJYbbpLpUM1DIZhsHDhQixfvhzfffcd1q1bh/z8\nfNk+b7zxBqZOnYpvv/0Wjz76KBYvXhzibFcPRZVWEJQXXrevTr1Pw3j74AdwMx54GS9nw/dFDPE+\nDelL8/X2CyitFjOUWYbEN9+bcbqIm/Gabb6y4L7h8WKZzXcObpDhtQn+pf/3qS+xs2y3cD6ryxa0\nxIe622GAFAdEkiCRGtElYD+pw1ZaD4onQmLrldqMeQHHx4lHhqkx944+0Gha/6q1NEvlX/zBWfLZ\nnNPrlKn+iYY4GLVRQvE6KXwJdEFgSEwp/jMx6aBQ5icwItThsnLjwSJRpNFKM0ZmIj6aO39bNAwt\nrUVng1hXSaphSEMpkw1JiNXFCKVUghGtNcp8D81F06hIWgjmAAI1DLKZPAlA/h/oMA2jlYJI77Pv\n0yQNozZK8F/w70Jr8hakzz7Y/URqDCF9Sb8E7WneAjpYYBw9ehQpKSno1KkTVCoVJkyYgM2bN8v2\nyc/Px5AhQwAAgwcPDvj9aoQiCYASywdLVfgfj52Gm/Fwhcj8on1on5OTYVgu81Zig2cdepmdf2Ne\nMSrqGsH4ykY3WDkBMaA7p311T+FeBOlAJ7W9W11W2SI1PISxXKZhhKobJZ2NVNgDcxSkL2JQDcPv\n1Wrt+gAAoAoRMTM3ZzZyTL3RO5YrC6Gm5YJlYFIf9I7NQnZMBrJjMpAemYKMzlEy53jPmExMShsr\nzKJ5k5Q0MmV27wdk5324l7igD78uSW58P6REdMaQxFxZXwWbnUsHkv7dEpAcz5mMyHacdUo1jFCm\nh9vTxwfdriJpmKQ5AS2YLqSDJP8eiBpG64eU9h7MeCiSwm2po3F/ZvPruuTG90NqRBdM6ipfMY+f\n4Blb4VeRTiY66n7agqGdhXKHmqQqKyuRmCg6ueLj43Hs2DHZPpmZmdi0aRNmzpyJTZs2obGxEQ0N\nDYiMbL/klUuhsrEalfYq9DGJ64t7GS/2Vx5GuDoMMVojKMpXh943EDklTtGvd5xBeGYTAEJWrhgA\nVqw7DQpqJETr4XR5uX1YX66Fnz2+uMqGPy/bA20uCwJiJcy4yDDADnTtpENBCaBX64FG0dbDVUoN\nw4WGQlQ1BWZ1A8DtNyfj+1IuoopfT0JF0rIiidI/fqldHlEFyGdx4ZLPjJ8PQ9h+CQIjlIaREd0N\nGdHdhO/SGfp9mXdCRamgpbV4VJJRn5KgRVykATUOTksalzoKaZGp2F/J3b/ZGejDSDYkYU6fB4Wy\nEfF6E+7Pugufn1opmLumpN/GFRL0I9jsXNoXakoFb4hktbYgHbjUIQTumJSbsbFwK5o8TQhXhcly\nDWJ1MSjyLbHbUjQNHWRW7b0MgRGh6RgNAwAmpo1tcZ+smB5BF9sq9i1Z0NmQ3OI51NTVLTDaW4vr\nUIHRmlnls88+i4ULF2L16tUYOHAg4uPjQVEt/5FMpvbriItlDVjwr134y+9uwF/3vAkA+GjKm3j1\no0PomhSJrr3N+PepL4X9E2unAQkAvDRMJgN6etOxmq98Tbnh8no4E4Ff2Ofhs3UB0UJC9U82uCrP\na/jJJgOevnsk8puOA2UAVNwf1D8qJVytR6zeCKvLhnUXNwU9Z2Q0CXBjA4Z0zYHJZICaVsPtEgXG\n6G7D8PmRbwAAlY3ytRwM6jAkxEchOSIRJZZypCclQqviBpnBKX1wpPo4hnfNlT2jwal9sK/yINdF\nJNXs84uOCm/V842yiKG5Rl+YbrDjjPow1Dg44RkfEwWT0YB4t9xM0DnehCideGysVwwbNpkMiHOI\nExiKINEtuVPQwTEhxhi0DSZ9NKob65CcEIvhzoE4Xnsao7oNbbf3ONYptjc+Jkpotz86lYYTGBq9\nIDBMJgO6xCTiYBXnlEoyRcMUEbpdsW7xt86mOJhMBtzSbTjWnNqIYWn9Wrwnoy4S5qYGdE1MANke\nkWKt4FL6+Zb04fghfwduzRwGU2zzx0nfk0RjbLuOS+1BNMNpzjG64O/lpdKhAiMhIQFlZWJNosrK\nSsTFxcn2iYuLwz/+wUXvNDY2YtOmTQgPb1lSV1e338pzH6w6ggabC/9ceRjwBTOcLazCyYt1OHmx\nDr0ZefnwixW10CYAjJdCdbUViVQyRnW+EVuKd3DVJwkGHg9Au4xgGULMcfDTIoZkx+MwSwBgYdBp\nkZMdj9QEg69SqRwtTUNPE2iycyakeht3/2F+AiNMFY4Hs+7DS7tfR4MzeB9VmH2z5LTb0FWTjupq\nK2i/V2FI9GAQWTQ+O7Uy4DxhqnBUV1vxZM4f0ORxwFrvhhVcu/oY+uL5QQlICIuTPaNMfRaeHzQP\nakoFHa1r9vlZrc5WPd8muyjgGm3c9YMdR7HiLNBmcaPaY4XTLndsN1oYuG3isVZJaZXqaiscdtGM\np6f1qK0JngnssHmDtuFPA59Ak8eBhjoHeoX3xoLBTyFeH9du77HDJravyeoB4oP3Bf+c9ZQerw9/\nEQC3n54V/3N2iwfVId4dALBbRTOmysU9y9EJI9Ensg/i1aYW7+n53Hlwel2orb20bOrLxWQyXFI/\nT0geh2GmoTCyMS0eJ+0L0qVq13GpvXh12PPQUGpUV1vbLDQ6VLz37t0bRUVFKC0thcvlwrp16zB6\n9GjZPmazWdBE/vWvf2H69Okd2aQAVp37FsWxawCwcHvFQaTW0gQirB7agRtxplae9KXpxS1KI3Wm\npho4xzHd6TwI2g2vh0CSMdJXNZRH1CLuvaU7Fxnk29TFFInfT+6JsYO6IJjfUKfmrsWbA/jIK38N\nI0IVjhitUWai8IefWXaWhPr5JywRBCH7HRAzhw2+DF4trQ0wyxAEgaTwhIDZN789VhfTYiZr0Azp\nFvZrLgpFaibizV3+Ic6qFiKWpH6BULkG/tfy3873FUEQQt5Ke6FqhQ8DENdmMagNMKjDBTNKrFZS\npqKF0hRSk5RRw90TSZBCKZyW0NG6NuVddDQqShW0rEowpObTjnLit5UoTWS7FB4EOlhgUBSFBQsW\nYNasWZg4cSImTJiA9PR0vPfee9i6dSsAIC8vD+PGjcO4ceNQV1eHOXPmdGSTAthavBNeqgmg3Sis\nEGcHtRY7VMnnQJAsqCi5L4CvYc94SWw7VAqHywPGJzxI38IpnppOiI7QyipW8lnPE25IwS0DO0Or\nocAHiUoHsHf+OBzvPX6j/JqEPHafTxL0H7wM6nAQBCFzRPeKycSwJHGJSb54oXSQeajnvUJsP480\n8oYAgUlpY9E9Kg03JcvzItqb1trBpWGpzfkDpEEJ/D13MSRjeNJgDEro36rMWqmturnY/PYov3A5\nSNsXyocBiD4rg0qeac9nIhMgWizTIRWuVypc9GohKTwBA+L6IsfUC10jW877+bXT4XkYI0aMwIgR\nI2Tb5s6dK3weO3Ysxo5t2UHV4RByE0V1gz1gm6c2EXSMGFZJ6uz498YzWLPjAqyohdZXB411q+Gt\nTEF0sgasVRxA/v7IMFSZGxEbyQ04Xi8raBjSMhF8ZdL3Hr8Rz+3eAACIjfCtSyxoGJwTV6+WD168\nw9KgNqDWFzI6NGkw+pp6IlITifUXf8AZM2fykg4ycXoTHup5DxbuFcOapb+rKBUGxOcIizt1JC3F\n8vNIhWxzA5c0N4IXgiRB4p7M0Nqs/9xfrmGE1pDao8Db5SAV7s0N+E6GExj+CYZGbSQogoKaUrWo\n+VxKAMO1Dk3SmNXrvivdjF8MJdPbB+GX/Xy+3MyV25DANsp9K14z54+xNLoBiXmKcHEDSpIpDGlx\n8toycUa9UC4jK9WIYBoGT7hOkhRk8C0cwwsMn4bhb97hB05pxAavNvsPJP61q/gQPF7TkIdqtl9x\nuFBkGrkcigR9XAt7ckgTIZvTMKTFA1syPfGYfAlx6ZFctrQ0ciiYhsGbWH6JfgpGa58VHzLqXwyP\nJEikRCQjTteyWSnKV0KlV0zm5TRV4VeMsuIej59wKKpqgLqrXIjwdZwAgLFFwlstht2xXvEP27dL\nF4wZOBApCQYYk3pg2fFdQS+ZnhQJgltMLmTNGx6+fEGAhqGSz2j5QTRYPR3/gcTfzxGm0uOVoX8W\nqoxKZ9XBqm22N4/0nYUGl6XViU5SgdFc1jQ/6ANotd8gShOJhUPnC3ZpaRhxMB/Gi0OegcPjaJds\n2stB+iyb81/xBCth8oc+s1p1LaM2CouGLQgIuFC49rkuNYyLDUV4ctvzOFEtqdrpX1+JZBCml89a\npYvcM049ZIYLSQhtosGErokRIAkCRq28qmsoWhQYvA+D9PNh+Jmk+MFemrDDzz79naHSWSmPURsl\n2PlJghTs4cEGmPaGIqlLyoqVmqGac5R3jQzMZG8N0Vqj8FykgiaYhqGh1ELxwiuB1G/RnFDk7ydY\nNrRepWvWoS8lUmNo8Z1VuPa4Lp/4tpKdcDFufHTsc2EbQXrlNUpJBrSKBXw5eRRB4Y+398Oyk1wu\ngX8mM0CgK3JhiG3EIEmBttYm86iIVgoM3358VrdUCPSOzcaozjcGXJefcfo7Q5tzjvJoKQ1cXleL\nS1ReCaRmvOYGL5qkcX/WXSFXNbxULiVr/Zeitaa2ZwY8hr0VBzDwF/BFKVx7XJcCIymMq0HkgmQt\naz8NIz5aI8t81tIaGHSi+Sc90YiTkrJYqQkGPD1qVMC1WiswWjtb899PRYnfH8i6S9AOpCF+oU1S\nLV9TFeLYqwFpoEBLpbxvSBzY7O+tgVu73YFGT+Aa6Fea1prCkg1JSDYktbyjgkIQrkuTVNAoD5LB\ntBGirXvCsM6yOkvcetrioKShxcFq4cOD8Nx9/YNeix+sW4o7D+b0ljXPZ/7yH+SlA7lUY5CaHEST\nVKCjsyV438nVKDDkGkbHh3dOSBsDgFs/4mqlNf4LBYXL5brSMHYfr0DnuPCgBfe0WmDs4GR8v923\ngWACNAxaMqPVSArfRUdooVGFHrAWj1jYYjJaixoGQQTdL0IbqEkAwZ3e0t8Xj1jY/PWEywa/7tWA\nLHGvHesyhWJk8nDkxve7KmsGAcDfR7z8i/SDwvXL1TcKdBAWuwsffsetijVheqDAYDsfxpGabOG7\nw+uUVX/VUhrZoCmtlKpRN/8nbc1KV6H+6FpKA4fXKeQS+O8XFaKAm8zp7Zt1EhKFsrWrb/ECg8HV\nZ7eXmaR+AQ2DIIirVlgArSvHraDQFq4bgeFwS0t6B3d+fnzi/4TP58wXZL/paJ3MHCQ1SbW0BkBr\nCPOJaOkAACAASURBVKWBPDXgUewqy0NuAudIlwktUgWtSouHsu9Bo9+aFTpaC5qg4GG9wjHJ4YmY\nlDYWWSGWkQwG79y/Gh29MpOUMrNWUOhwrnmB4fYwUNEkHE5RSJTUNLR4HL9GL0+YSi8brLWq9rUV\nEyEERlJ4Au7oMVn4Lh0keS1iYEK/gOO42bABVpdV8FUQBIFxqaMD9m2+Xb6lYf1WobsakIXVXoUm\nMwWFa41r2um9+3gFfv/3bThxsQ4Ol6hhnCura+YoDtbPBKOndbLBWku3rxO4tQllJEEGXew+GCkR\nnZEYZKH7S4G/1tVokqJbmemtoKDQPlzT07LvdhcAALYfKcONfRK50FnCK5T8YL0UtxBSK9CpdDKn\nt1atgpCk0Q6QAdWLQsOvatfSalqzet4bIPguFT5K6qrUMCRC4kplWCsoXE9c0wJDit3phDZnGwja\nDdbLDTSsR9VqgRFG62UmEJqkMWt8ulCBtq2EMkk1R0sO2PZwBMfoolFiK7uiWcyhkN5fe5YKV1BQ\nCM51IzAsDjsImouOEoSE34p4BAjZjJwmaSE7WK/SyWaxNEnjhj6JaC8uRcPg8V+voiOYkTEVJl0M\nbk0Z2eHXulQUrUJB4ZfluvnHNbnk5iOWIQG/Nbf91zKWRkX51w9qb5v55WgYWdHd27UNwYhQGzC1\n24QWFz1SUFC49rluBQYYEp4KMbO7d2x2gMCQRkX5F2Vr77j/SzGpdI9KQ7gqTFj0RkFBQeGXoMNN\nUj/99BMWLVoElmUxffp0zJ49W/Z7eXk5nnvuOVitVjAMg3nz5uGmm25ql2uzhAf8ehNNbr9kPYZC\ndlQvPDLiTqF0xuv73gVgFnaROrlV5KXXYboULsUk9Xi/3wuObwUFBYVfig7VMBiGwcKFC7F8+XJ8\n9913WLduHfLz82X7fPDBBxg/fjxWr16Nt956Cy+//HK7XLvR3YiGtG+hSjsKsCwcnkCTVFyUDhpa\nDYIgQBBEwBKl0sJ+ejr4uhNtha//dCkmH4IgrvulMRUUFH55OlTDOHr0KFJSUtCpE+ecnTBhAjZv\n3oz09HRhH4IgYLNxa0xbLBbEx7ctb4CnxMYtpUrHlsNTx8LrdsvuNkKnxR3D02XH3J1xO0z6GKy9\nsBEAV8jt2YF/xMWGIsToomX7tlexu+cGzsWR6uPIjslol/Ndb/y218wW63QpKCi0Dx0qMCorK5GY\nKEYSxcfH49ixY7J9HnvsMcyaNQufffYZHA4HPv7443a5tsVlFT5X06cR5omS/R4drg8oGKim1BiX\nOhqbCrfC6XWBJmikRHRGSkTngPO3VzG+hLA4JIQFlkVXaB394npf6SYoKFw3dKjAaE39oXXr1mH6\n9Ol48MEHcfjwYTzzzDNYt25di8eZTM0nrTmq7cLnuoj9sJcMBiSH6DTakOfQqrRwel3QazUh94mN\njoApuvk2/FK01BfXE0pfiCh9IaL0RfvQoQIjISEBZWVlwvfKykrExcXJ9lm1ahWWL18OAMjJyYHT\n6URdXR2io+UmIH+qq63N/l5YUyb7bnXZIXVbE14y5DlI1ldwz0OE3MdS70C1t/k2/BKYTIYW++J6\nQekLEaUvRJS+EGmr4OxQ42/v3r1RVFSE0tJSuFwurFu3DqNHy4vfJSUlYdeuXQCA/Px8uFyuFoVF\na6h1mGXfNTp5RrfUoe0PnxDWnNlJSRpTUFC43uhQDYOiKCxYsACzZs0Cy7K44447kJ6ejvfeew+9\ne/fGyJEj8dxzz+GFF17AJ598ApIk8cYbb7TLtZ1eeVRUdrcwnBCtVOgcHjpLmmqFwGhrjSYFBQWF\nXxsdnocxYsQIjBgxQrZt7ty5wuf09HT85z//affrerxyjYJWuwGJwMhsZk0IoRx4u7dKQUFB4dfL\nNWtXaWiULyjkJpwAAJMuBrG6GKQGiXziEUp6B0mOG5k8XDiPgoKCwvXENVt80O50Qerltrs59WJq\nt4noa+rZ7LG8wPAGERh39JiM6d0nKdVRFRQUrjuuSQ3D4fLAw8hNUjafwGhN/kRzGgaglNJWUFC4\nPrkmBUZJlR0g5E5pm5vLJm9NlVmqBYGhoKCgcD1yTQqM4iorCEI+2PNRU63RMAhFYCgoKCgEcE0K\nDLPNGaBh8LSmBhTVjA9DQUFB4XrlmhQYDTYXQLAwaePw0pBnZb+1hw9DQUFB4Xrk2hQYdhdAMFBT\ndLOLIoWiiyEZAJBsSGphTwUFBYXrh2syrLbB7gKMLGiSChQYRMu3PL7rGMSHxaGfSamEqqCgoMBz\nTQoMi90FgmBAkRRokoaaUsN1CU5vNaXCDYkDO7qZCgoKCr8qrjmTFMOysNidACE6r/W0uB63Slmp\nTkFBQeGyuOYEhr3JLUQ38cuoSgVGey18pKCgoHC9cc0JDN7hDUBY91q6XnZ7rcWtoKCgcL1xjQoM\nLgfDX8NQkbQgRBQUFBQULo1rTmBYbFKBwd2emuKqEOppfcjjFBQUFBSa55oTGMFMUg4vV+pcr9KF\nPE5BQUFBoXk63AP8008/YdGiRWBZFtOnT8fs2bNlv7/22mvYu3cvCIJAY2MjzGYz8vLyLvt69TYn\nCD+TVKO7CYDc+a2goKCgcGl0qMBgGAYLFy7EJ598gri4ONxxxx0YPXo00tPThX3mz58vfP78889x\n6tSpNl3TItEw+BIf3aPSkN9QgF6xWW06t4KCgsL1TIcKjKNHjyIlJQWdOnHrZ0+YMAGbN2+WCQwp\n3333HR5//PE2XVPu9OYExviuY5Ae1RVZzSzLqqCgoKDQPB3qw6isrERiYqLwPT4+HlVVVUH3LSsr\nQ2lpKYYMGdKma9bbHdBm7QMg+jAokkJ2TIay8JGCgoJCG+hQDYNlg5cYD8a6deswduzYVg/qJpMh\n6Hartw5Qcet3h+t1Ife7lrge7rG1KH0hovSFiNIX7UOHCoyEhASUlZUJ3ysrKxEXFxd03/Xr1+Ol\nl15q9bmrq60B29weBvZGBny5QZfDG3S/awmTyXDN32NrUfpCROkLEaUvRNoqODvUJNW7d28UFRWh\ntLQULpcL69atw+jRowP2u3DhAiwWC3Jyctp0PWujS/ad92EoKCgoKLSdDtUwKIrCggULMGvWLLAs\nizvuuAPp6el477330Lt3b4wcORIAp11MmDChzdeTOrwBgFSyuhUUFBTajQ7PwxgxYgRGjBgh2zZ3\n7lzZ98cee6xdrtVgkwsMpW6UgoKCQvtxTdls6u1OIQcDUExSCgoKCu3JNTWiXii1yE1SisBQUFBQ\naDeumRGVYVkcvVALvVa8JTfjuYItUlBQULi2uGYExsot52Gxu9A1SQwbczPuK9giBQUFhWuLa0Zg\n7G/4Cepuh3BzPzGz3O1VBIaCgoJCe3HNrFfaFHUGFAC1WswUVzQMBQUFhfbjmtAwjteIFW6bPA7h\ns0sRGAoKCgrtxq9eYJRbq/HB0Y+F702eJuFzj6jgVXEVFP6/vTsPbKpKHz7+TdK0LC2b3QCZikVB\nsAqoLMKUdYChBVoBFas4U6SAQNlEFgXGqQNYmAr8FBVBQUBRXwGFMOpYQUAqKIIwLDrgQGmRlq3Q\njaTJPe8fLSmhQFJoUtM+n79yb05Ozn2g98k5595zhRDl5/UJI+PsRYftgpIeRmTjjrQLbVsZTRJC\niCrJacLIysryRDtumk6vOWxv+PVzAG73byTLmQshRAVymjAGDhzI2LFjSUtL80R7yu16V0LJOlJC\nCFGxnCaMr7/+mh49erBgwQL69u3L6tWrycvL80TbXHLJZr7mflkWRAghKpbTs6qvry8xMTF8+OGH\nvPzyy7z99ttERkaSlJTE2bNnPdHGGzJbr93DkIQhhBAVy6WzamZmJv/85z+ZNGkSHTt2ZOnSpdx2\n220MGzbM3e1zymIrfgaGKvJ12C8r1QohRMVyeuPeyJEj+eWXX3j88cdZu3Yt9evXB6Bt27Zs2rTJ\n7Q10xlwyh6Hl18VQ77R9vyw8KIQQFctpwhgwYAC9evXCYCj7i33jxo1Ov2Dr1q3Mnj0bpRQDBw4k\nISGhTJlNmzbx+uuvo9frad68OfPnz3ex+Vf2MIwO+w0y6S2EEBXKacKoW7cuBQUFBAQUL+p38eJF\nDhw4QMeOHZ1WrmkaSUlJLF++nODgYAYNGkSPHj0IDy+9oe748eMsXbqUDz/8EH9/f86dO1euA7CU\n9DCU1c9hvwxJCSFExXI6bpOcnIy/v79929/fn+TkZJcq37dvH2FhYTRu3Bij0UhUVBSpqakOZT76\n6COeeOIJ+3c0aNCgPO2nSCt5jvdVcxgyJCWEEBXL6VlVKeVwA5xer8dms7lUeVZWFg0blq4eGxIS\nQnZ2tkOZY8eO8b///Y8hQ4bw+OOPs23bNlfbDpSuF9Whxe0O+6WHIYQQFcvpkFTt2rX56aefuP/+\n+wH46aefqFWrlkuVK6WclrHZbKSnp7N69WpOnjxJXFwcJpPJoVdzI5dv3PMzOA5JSQ9DCCEqltOE\nMXnyZEaPHk2zZs0AOHLkCK+99ppLlYeGhnLy5En7dlZWFsHBwQ5lQkJCaNOmDXq9nttvv52mTZty\n7Ngx7r333hvWHRRU8qAkHw3McGeDO6gd/Ef+fbS4hxLYIICgBgE3qKHqsMdCSCyuILEoJbGoGE4T\nRps2bTCZTOzduxelFG3atKFu3bouVR4REUF6ejqZmZkEBQVhMplISUlxKNOzZ09MJhMxMTGcO3eO\n48eP06RJE6d1nz6dC0ChuXixwaJLiu53dbUnjIsXLnHalutSO71ZUFCAPRbVncSilMSilMSi1K0m\nTpceoFS3bl26dOlS7soNBgMzZswgPj4epRSDBg0iPDycRYsWERERQbdu3fjjH//It99+S1RUFAaD\ngeeff97lhARgVcXP7a7h44tRX3o4cqe3EEJULKcJ4/Dhw8yaNYvDhw9jsVjs+w8dOnSDT5WKjIwk\nMjLSYV9iYqLD9tSpU5k6dapL9V3NqornMHx9jPjoSg9HL5PeQghRoZz+DP/b3/7G+PHjCQsL45tv\nviEhIYEJEyZ4om0usSorStPhZ/DBx6GHIQlDCCEqktOEYbFY6NixI0opgoODmTBhQrkvfXUnqyoC\nzYDBoHe4/NeglyEpIYSoSE7PqvqSE2/dunU5fPgw58+fJzMz0+0Nc5WN4oThY3A8FOlhCCFExXI6\nhxEVFcX58+dJSEhgyJAhaJpWZg6iMtlUEcrmg4/B8el6ch+GEEJUrBsmDE3T6NixI/Xr1ycyMpJd\nu3ZhNptdvqnOE2xYQfPFUKaHIQlDCCEq0g3Pqnq9nhdeeMG+bTQaf1fJQlMams6KshnK9DBkSEoI\nISqW05/h4eHhZGRkeKIt5VakFd+DgWbAoJchKSGEcCencxjnzp2jf//+PPDAAw5rSC1cuNCtDXOF\nueR53sVzGI4JQhKGEEJULJcmvaOiojzRlnIzW0tuJLziKqm764XzS85Rh0tshRBC3DqnCSM2NtYT\n7bgpFq00YVwekkpsk4CmtEpslRBCVE1OE0ZiYuI1f63/voakSnsYOp1OJryFEMINnCaMbt262V+b\nzWa++OILh0esViaz7XIPwweDQYaghBDCnco9JPXII48watQotzWoPC4nDJ1mQC9zFkII4VblvpRI\np9P9bi6ztZQkDKPeWMktEUKIqq9ccxhKKX7++Wc6duzo9oa54pK1eA4joIZrj4wVQghx88o1h2Ew\nGIiPj6d169ZubZSrLhQUAFDPxWeMCyGEuHluv6x269atzJ49G6UUAwcOJCEhweH9devWkZycTGho\nKABxcXEMGjTIpbovFBQCUN+/5i21UQghhHNO5zCGDBnChQsX7Ns5OTnExcW5VLmmaSQlJbFs2TI2\nbtyIyWTi6NGjZcpFRUWxbt061q1b53KyALhYWDwk1cBfehhCCOFuThNGQUGBwzO269WrR15enkuV\n79u3j7CwMBo3bozRaCQqKorU1NQy5ZRS5WhyqcKi4oRRt1aNm/q8EEII1zlNGJqmUVAyVwCQn5+P\nzWZzqfKsrCwaNmxo3w4JCSE7O7tMuS+//JIBAwYwbtw4Tp065VLdAFatuB1+Bqcja0IIIW6R0zNt\ndHQ08fHxDBkyBIAPPviA/v37u1S5Kz2H7t27Ex0djdFoZM2aNUyZMoUVK1a4VL+1ZLVaPx8/l8oL\nIYS4eU4TxogRIwgODubrr79GKcXjjz9OTEyMS5WHhoZy8uRJ+3ZWVhbBwcEOZa4c7nr00UeZP3++\nS3UHBQWAQYENgm+rU7xdTVXnY7+axKKUxKKUxKJiuDSWExsbe1NXS0VERJCenk5mZiZBQUGYTCZS\nUlIcypw+fZqgoCAAUlNTadasmUt1nz6di7nIAnowFxRx+nRuudtXFQQFBVTbY7+axKKUxKKUxKLU\nrSZOp3MYY8eOJScnx759/vx5xo0b51LlBoOBGTNmEB8fT3R0NFFRUYSHh7No0SI2b94MwMqVK4mO\njiYmJoZVq1YxZ84clxtvU8VzGDWMcqe3EEK4m9MexokTJ6hXr559u379+qSnp7v8BZGRkURGRjrs\nS0xMtL+eOHEiEydOdLm+K11OGH4+vjf1eSGEEK5z2sOw2WwOV0UVFRVhsVjc2ihXaap40lt6GEII\n4X5OexidO3dmwoQJDB06FIAVK1aU6TFUFhsyJCWEEJ7iNGFMnDiRt956i7lz5wLFa0u1b9/e7Q1z\nhYYNpekx+sgDk4QQwt2cDkkZjUbGjBnD66+/zp/+9Cc+++wzpk+f7om2OaVhA01vfzyrEEII97lh\nD8NqtfL111/zySefsHfvXqxWK8uWLfvdrFar0EDpr/kIWSGEEBXruj2MOXPm0LVrV9asWUN0dDTf\nfPMNdevW/d0kCwCFDZ0q9zOghBBC3ITr9jA++OAD2rRpQ0JCAh06dAD43f2SVzoNNJm/EEIIT7hu\nwti+fTsbNmwgOTmZCxcuEBMT4/Kig56idDZ0yBVSQgjhCdcdz6lTpw5xcXGsXbuW119/nQsXLnDp\n0iXi4uJYs2aNJ9t4fToNPdLDEEIIT3BpAqBFixa8+OKLbNu2jbi4uGs+06JS6DR0ShKGEEJ4Qrke\nJGE0Gunbty99+/Z1V3tcpikNdEp6GEII4SFee4lRka0IQBKGEEJ4iNcmjEtFJQlDJwlDCCE8wWsT\nRmFR8QKIBulhCCGER3htwrhklR6GEEJ4ktcmDHPJkJSPrlzz9kIIIW6S2xPG1q1b6dOnD71792bJ\nkiXXLff555/TokULDhw44FK9l6wlQ1J66WEIIYQnuDVhaJpGUlISy5YtY+PGjZhMJo4ePVqmXH5+\nPqtWrSrXOlXSwxBCCM9ya8LYt28fYWFhNG7cGKPRSFRU1DVv+lu4cCHDhw/HWI4HIZlLLqv1kR6G\nEEJ4hFsTRlZWFg0bNrRvh4SEkJ2d7VDm0KFDnDp1ii5dupSrbnPJpLdBLz0MIYTwBLeebZVSTt+f\nPXs2r7zyisufuczoV5zravv5ERQUcPONrAKq+/FfSWJRSmJRSmJRMdyaMEJDQzl58qR9Oysri+Dg\nYPt2fn4+R44c4amnnkIpxZkzZ3j22Wd54403aNWq1Q3rPncxHwBl03H6dK57DsALBAUFVOvjv5LE\nopTEopTEotStJk63JoyIiAjS09PJzMwkKCgIk8lESkqK/X1/f3/S0tLs20899RTTpk2jZcuWTusu\nKhmSMsqQlBBCeIRbz7YGg4EZM2YQHx+PUopBgwYRHh7OokWLiIiIoFu3bg7ldTqdy0NSFs0KgNEg\nCUMIITzB7WfbyMhIIiMjHfYlJiZes+x7773ncr0W2+UehjxASQghPMFr7/S2lvQwfKWHIYQQHuG1\nCaPIVpIwfKSHIYQQnuC9CUN6GEII4VFemzAuD0n5SQ9DCCE8wusThgxJCSGEZ3hvwlDFCaOGJAwh\nhPAIr00YNs0GgJ+PbyW3RAghqgevTRiXexgyhyGEEJ7htQnDpop7GDXKsSS6EEKIm1cFEoYMSQkh\nhCd4bcLQkB6GEEJ4kvcmjJI5jJoy6S2EEB7hvQkDDaXA6CN3egshhCd4ccKwgfLa5gshhNfx2jOu\n0mnoNENlN0MIIaoN700Y0sMQQgiPcvsZd+vWrfTp04fevXuzZMmSMu+vWbOGfv36ERMTQ1xcHEeP\nHnWpXqWThCGEEJ7k1jOupmkkJSWxbNkyNm7ciMlkKpMQ+vXrx4YNG1i/fj3Dhg1jzpw5LtWtdBo6\nJUNSQgjhKW5NGPv27SMsLIzGjRtjNBqJiooiNTXVoUzt2rXtrwsKCtDrXWySTkPnvSNqQgjhddx6\nTWpWVhYNGza0b4eEhLB///4y5VavXs3y5cuxWq2sWLHCtcp1NulhCCGEB7k1YSilXCoXFxdHXFwc\nJpOJxYsXM3fuXOcf0mnodQaCggJusZXeT2JQSmJRSmJRSmJRMdyaMEJDQzl58qR9Oysri+Dg4OuW\n79u3L7NmzXJar02zgQ50Ss/p07kV0lZvFRQUUO1jcJnEopTEopTEotStJk63TgJERESQnp5OZmYm\nFosFk8lEjx49HMocP37c/nrz5s3ccccdTustshUByByGEEJ4kFt7GAaDgRkzZhAfH49SikGDBhEe\nHs6iRYuIiIigW7durFq1irS0NIxGI3Xq1OGVV15xWq+lJGHokTkMIYTwFLcvxBQZGUlkZKTDvsTE\nRPvrF154odx1mq2SMIQQwtO8ckznUpEFkIQhhBCe5JUJw1xUvLS5JAwhhPAcr0wYl6zFPQyDThKG\nEEJ4ilcmDPschiQMIYTwGO9MGCVzGAYZkhJCCI/xzoRhLZ7DMOjkaXtCCOEpXpkwLPY5DEkYQgjh\nKd6ZMGyXexgyJCWEEJ7ilQnDXHKnt49eehhCCOEpXpkwikqukpIehhBCeI5XJozLQ1LSwxBCCM/x\n0oRRPOktCUOI6iMvL4916/7fTX32+efHk5+fV8Etqn68MmEUXe5hyJCUENVGbu5F1q37+JrvaZp2\nw88mJy+gdm1/dzTrlrn6oLnfA6/8iX55eXOjwVjJLRFCeMqbb77GyZOZxMfH8eCD7enYsRPvvvs2\nt90WyJEjv7By5UdMm/Ycp09nY7GYGTx4CP36xQAweHB/li1bSUFBAc89l0hERGv+85+fCAoKYe7c\nf+Lr6+vwXd9+u40VK5ZhtVqpW7cuM2e+TP369SksLOTVV5P5+edD6HR6/vrX4XTp0o3vvtvBkiWL\n0TSNevXqsWDBYt55Zwm1atXi8cefBGDo0MdITl4IKJ57LpE2bR7kwIH9zJkzn5Url/Pzzwcxm810\n7dqD+PgEAA4dOsCiRf+ksPASvr6+LFiwmMmTxzFhwvM0a3YXAKNGDWPy5GnceWczt/8beHfC0EvC\nEKIyfPT1Eb4/nF2hdT7UIphHu1//pDdq1FiOHfuVd95ZDcCePbs5dOggK1d+RGhoKADTp88iICAA\ns9nM8OFD6dKle8lT5nT2ejIyTvDSS3OYMuUFZs6cxpYtX9OrVx+H77r//jYsWbIcgI0b1/P+++8x\nevQ4li9fSkBAACtWrAGKh8lycnJITv4HixcvIzQ0lNzcaz/dT6crbcOJE+m88MLfmDRpCgAjRowm\nICAATdMYN24Uv/56hD/84Q5mzZpOUtIrNG/egoKCAvz8/OjXL4ZNmz4jMXESJ06kY7UWeSRZgJcm\njCKteEjKKHMYQlRrLVu2sicLgI8+ep9t274BIDs7m4yMdMLDGwOlwz4NGzYiPLz4BNu8eQtOnTrJ\n1bKzTzFz5gLOnj2D1WqlYcNGAPzwwy7+/vc59nL+/v58++022rRpa29HQMC1H4N65dBTSEgo99zT\nyr6dmvoFn322HpvNxrlzZ/nf//4HQGBgEM2btwCgVq1aAHTr1oPly5cxevR4TKbP+POf+7kYrVvn\n9jPu1q1bmT17NkopBg4cSEJCgsP7y5cv5+OPP8bHx4cGDRowe/ZsGjZseMM6i0omvWVISojK8Wj3\nZjfsDXhKjRo17K/37NnNjz/+wJIly/H19WXs2BFYLJYyn7ly+EmvN1yzzKuvzmPIkKd4+OHO7Nmz\nm3fffRu49nzD9eYgDAYDmlb63pXfU7NmTfvr3347yZo1q1m2bCW1a/sze/ZLWCxmrje14edXg4ce\nas+2bVvYvPkrli5dee2CbuDWSW9N00hKSmLZsmVs3LgRk8nE0aNHHcq0bNmStWvX8umnn9KrVy+S\nk5Od1nu5h+ErQ1JCVBu1atWioKDguu/n5+cREBCAr68vx48f48CB/1yznCuTzPn5+QQGBgLwr39t\ntO9v164Dn3zyoX07NzeXe++9j71793Dq1G8AXLx4ESjuyfzyy2EAfv75ML/9VtqTubIN+fn51KxZ\nk1q1anPu3Fm++24HAGFhd3D27BkOHz4EQEFBgX1yPzp6AAsWzOeee1pdt0fjDm7tYezbt4+wsDAa\nN24MQFRUFKmpqYSHh9vLtGvXzv66devWbNiwwWm9Vq14DsNXehhCVBt16tQlIuJ+nn76cdq3f5iO\nHTs5vN++/cOsX/8Jf/nLE/zhD2Hce2/EFe+Wzh9cOZdwPfHxw3nxxSkEB4fQsuW99mTw9NPDSEl5\nhaFDH8NgMPDXvyYQGdmV559/genTn0MpRf36DUhJeY0uXbrz+ecm4uPjaNGiJU2ahF2zDc2a3cVd\ndzXnqaceo1Gjxtx33/0A+Pj48NJLc3j11WTMZjM1atRgwYLF1KhRg+bNW1C7dm2iojw3HAWgU268\npuuLL75g+/btJCUlAfDpp5+yf/9+XnzxxWuWT0pKIigoiJEjR96w3omfzifj0lGGhIyhc6s/VHi7\nvUlQUACnT197kq26kViUkliUqoqxOHPmNImJI3n//U/K9bniCwBunlt7GOXJRZ9++ikHDhxg5Urn\n43GXexiB9evccgCqAolBKYlFKYlFqaoUi/Xr17Nw4UKmTZvm8eNya8IIDQ3l5MnScbusrCyCMd1e\n6gAAERdJREFUg4PLlNuxYwdLlixh1apVGI3Oh5msyopSUJhXVOV+OZRXVfz1dLMkFqUkFqWqWiw6\ndepBp049AMp9XLeaYNw66R0REUF6ejqZmZlYLBZMJhM9evRwKHPw4EFmzZrFG2+8Qf369V2q16pZ\nQTPg6yt3egshhKe4tYdhMBiYMWMG8fHxKKUYNGgQ4eHhLFq0iIiICLp168a8efMoLCxk3LhxKKVo\n1KgRixcvvmG9xQlDTw1JGEII4TFuvw8jMjKSyMhIh32JiYn21++++26567SqIpRmoIZREoYQQniK\nVy4+aFNWUHr8pIchhBAe45UJQ8MGmoEavrI0iBDVxa0sbw7w0UcfYDabK7BF1Y+XJgyZwxCiurnR\n8uau+PjjDzCbL1Vgi8rPZrNV6vffKq/8ia50GigDPgavzHdCiJtw9fLmzz6byPvvr2Tz5n9TVGQl\nMrIr8fEJXLp0iZkzp3L6dDaapjF27BiOHcvgzJnTjB07knr16rFw4RsOdS9fvpRvv92GxWLm3nvv\nY/Lk6QBkZmYwb95scnJyMBgMJCXNpVGjxqxevYIvv/wXer2eDh06MWLEaMaOHcGYMRNo3rwFFy7k\n8MwzQ/n448/41782smPHdiwWM5cumZk7959MnTqJvLxcrFYrw4ePpHPnLkDxMiRr1qxGr9cRHn4X\nEydO4emnh7BmzVoMBgMFBfkl2+swGDz/g9krEwaAXknvQojKsvbIRvZk76/QOtsER/BIs+jrvn/1\n8ubff/8dGRnpvP32eyilmDJlIj/9tJecnHMEBgaRnLwAgJo1dTz4oOLDDz/g//7vLerUqVOm7oED\nH+Mvf3kGgKSkmezYsZ2HH+7MSy+9yNChf6Vz5y4UFRWhaRrffbeD7du38vbb7+Hr63vd5cyvXI7k\nwIH9vPfeh/j7+6NpGnPmzKdWrVpcuJDDiBHF9f/661FWrVrOG2+8Q506dcjNzaVWrVq0bfsAaWnb\n6dy5C1999SVdu/aolGQBXpwwDPK0PSGqtV27dvL997uIj49DKUVh4SUyMtK5777WvP76Qt588zU6\nduxMz55/pLAwl+Ilzq+9+sTu3bt4//2VmM2XyM3N5c47w2ndui1nzpy2//q/fFPxDz/sIiqqn33V\nW1cW/3voofb4+xc/8U/TNN566zX27t2DXq/jzJnTnD9/jj17fqBr1x72hHa53ujoAbz//ko6d+7C\npk0bmDLl2ksreYLXJgy9ThYeFKKyPNIs+oa9AU9QSvHUU3+hf//YMu8tW7aKtLRveeut1/jll/0M\nHvzUdeuxWCykpCTzzjurCAwM4p13lpQsRX7t5FK85FHZBQwNBgNKafY6r3Tlcub//vfn5OTk8O67\nq9Hr9Qwe3B+z2XLdpZQiIu7n1KlX2Lv3RzRNo2nTO697LO7mtZMAvqqm80JCiCrj6uXN27fvgMn0\nGYWFhQAlv9TPc+bMGfz8/OjVqw9DhjzJwYMHSz5fm/z8/DL1WiwWdLri1XALCgrYsiXVXj44OIRt\n27YAUFRUhNl8iXbtir/38gR66XLmjTl8uPi7Nm/+6rrHkZeXR/36DdDr9fz44w/2lXAfeKAdmzd/\nxcWLFxzqBejduy9/+9sLREX1L3/gKpBX9jDMhx/iD/XvqOxmCCE86OrlzZ99NpFjx44xcuRfgeKE\nMmNGEhkZJ3j99YXo9Tp8fIz84x/Fq2X37x/Dc88lEhgY5DDp7e/vT79+sQwd+hgNGzZyeBLeiy++\nxLx5s1m69C2MRiNJSXNp374jR478wrBhQ/H1NdKhQycSEp5lyJA4ZsyYxhdf/IsHHnjousfRq1cf\npkyZyPDhQ2nWrDlhYU0BaNr0ToYOjWfMmAQMBgN33dWc6dNnlXzmzyxd+iY9e/aq8LiWh1uXN3eX\nfpM+pc1dgYwdeF9lN6XSVbWF1W6FxKKUxKJUVYjF5s1f8e2323jxxZduqZ7f9fLm7iR3eQshqoMF\nC+bx3XdpzJ+/sLKb4r0JI6Cmr/NCQgjh5caPn1zZTbDz2knv+gF+ld0EIYSoVrw2YdTzlx6GEEJ4\nktcmDOlhCCGEZ7k9YWzdupU+ffrQu3dvlixZUub9H374gUceeYRWrVrx5ZdfulxvPX9JGEII4Ulu\nTRiappGUlMSyZcvYuHEjJpOJo0ePOpRp1KgRc+fOpV+/fuWqu570MIQQwqPcepXUvn37CAsLo3Hj\nxgBERUWRmppKeHi4vUyjRo0A0OnK3mp/PQ1vq42fPG1PCCE8yq09jKysLBo2bGjfDgkJITs7+5br\nfXVCl1uuQwghRPm4NWG46yby2jVl4UEhhPA0tw5JhYaGcvLkSft2VlYWwcHBFVL3rd7iXpVILEpJ\nLEpJLEpJLCqGW3sYERERpKenk5mZicViwWQy0aNHj+uW98JlrYQQotpw++KDW7du5R//+AdKKQYN\nGkRCQgKLFi0iIiKCbt26sX//fsaMGcPFixfx8/MjKCiIDRs2uLNJQgghboJXrlYrhBDC87z2Tm8h\nhBCeJQlDCCGESyRhCCGEcInXJQxna1NVNdOnT+fhhx92WDrlwoULxMfH07t3b4YNG0ZubunTxF5+\n+WV69erFgAEDOHToUGU02S1OnTrF0KFD6du3L/369eO9994DqmcsLBYLgwcPJiYmhn79+vHaa68B\nkJGRwaOPPkrv3r2ZOHEiVqvVXn7ChAn06tWLxx57zOFS96pC0zRiY2MZOXIkUH1j0b17d/r3709M\nTAyDBg0CKvhvRHkRm82mevbsqTIyMpTFYlH9+/dXR44cqexmudX333+vDh48qKKjo+37kpOT1ZIl\nS5RSSr311ltq3rx5SimltmzZooYPH66UUmrv3r1q8ODBnm+wm2RnZ6uDBw8qpZTKy8tTvXr1UkeO\nHKmWsVBKqYKCAqWUUlarVQ0ePFjt3btXjRs3Tm3atEkppdTMmTPVBx98oJRSavXq1WrWrFlKKaVM\nJpMaP358pbTZnd599101adIkNWLECKWUqrax6N69u8rJyXHYV5F/I17Vw7hybSqj0Whfm6oqe/DB\nB6lTp47DvtTUVGJjYwGIjY21xyA1NZWYmBgA7r//fnJzczlz5oxnG+wmQUFB3HPPPQDUrl2b8PBw\nsrKyqmUsAGrWrAkU/2K2Wq3odDp27txJ7969geJYfPXVV4Dj/5fevXuTlpZWOY12k1OnTvHNN98w\nePBg+77vvvuuWsZCKYWmaQ77KvJvxKsShrvWpvI2586dIzAwECg+kZ47dw6A7OxsQkND7eVCQkLI\nysqqlDa6U0ZGBocPH+b+++/n7Nmz1TIWmqYRExNDp06d6NSpE02aNKFOnTro9cV/0qGhofbjvTIW\nBoOBOnXqkJOTU2ltr2izZ8/m+eefty9gev78eerWrVstY6HT6Rg2bBgDBw7k448/BqjQvxGveqa3\nkltGbuha8SnPKsDeID8/n8TERKZPn07t2rWve3xVPRZ6vZ7169eTl5fH6NGjyzw2AEqP9+pYKKWq\nTCy2bNlCYGAg99xzDzt37gSKj+/qY64OsQBYs2aNPSnEx8fTtGnTCv0b8aqE4c61qbzJbbfdxpkz\nZwgMDOT06dM0aNAAKP6FcOrUKXu5U6dOVan4WK1WEhMTGTBgAD179gSqbywu8/f356GHHuKnn37i\n4sWLaJqGXq93ON7LsQgJCcFms5GXl0fdunUrueUV48cff+Trr7/mm2++wWw2k5+fz+zZs8nNza12\nsYDiHgRAgwYN6NmzJ/v27avQvxGvGpIq79pUVcXVvwS6d+/O2rVrAVi3bp09Bj169GD9+vUA7N27\nlzp16ti7olXB9OnTadasGU8//bR9X3WMxblz5+xXuly6dIm0tDSaNWtG+/bt+fzzzwHHWHTv3p11\n69YB8Pnnn9OhQ4fKabgbTJw4kS1btpCamkpKSgrt27dn/vz51TIWhYWF5OfnA1BQUMD27du5++67\nK/RvxOuWBrnW2lRV2aRJk9i5cyc5OTkEBgYyduxYevbsybhx4/jtt99o1KgRCxcutE+M//3vf2fb\ntm3UrFmTOXPm0KpVq0o+goqxe/dunnzySe6++250Oh06nY4JEyZw3333MX78+GoVi59//pmpU6ei\naRqaptG3b19GjRrFiRMnmDhxIhcvXuSee+5h3rx5GI1GLBYLkydP5tChQ9SrV4+UlBRuv/32yj6M\nCrdr1y7eeecd3nzzzWoZixMnTjBmzBh0Oh02m41+/fqRkJBATk5Ohf2NeF3CEEIIUTm8akhKCCFE\n5ZGEIYQQwiWSMIQQQrhEEoYQQgiXSMIQQgjhEkkYQgghXCIJQ3i1Rx99lNjYWKKiomjVqhWxsbHE\nxsYyffr0ctf1zDPPuLTc9bRp09i7d+/NNLdcDh48yBdffOH27xHCVXIfhqgSMjMzGTRo0A1XH728\nVIS3+Pjjj0lLSyMlJaWymyIE4GVrSQlRHmlpacybN4/WrVtz8OBBRo8ezblz51i9erX9gTpTp06l\nXbt2AHTp0oXly5fTtGlTnnjiCdq0acOePXvIzs4mOjqa8ePHA/DEE0/w7LPP0rlzZyZPnoy/vz9H\njx4lKyuLtm3bMmfOHKB4bZ7nn3+e8+fP06RJE2w2G927d+exxx5zaOeZM2eYNGkS58+fB6Bz5848\n88wzLF68mIKCAmJjY2nfvj1Tp05lz549pKSkUFhYCEBiYiKRkZGkp6fzxBNPEB0dze7du7FYLMya\nNYu2bdt6JNaimriVh3UI8XuRkZGhOnTo4LBvx44dqmXLlmr//v32fVc+XObIkSOqa9eu9u3IyEj1\n66+/KqWUGjJkiJo0aZJSSqmLFy+qdu3aqYyMDPt727ZtU0op9dxzz6knn3xSFRUVKbPZrPr06aN2\n7typlFJq1KhR6u2331ZKKXXixAnVpk0btWbNmjJtX7p0qZo5c6Z9++LFi0oppT766CM1ceJEh7bH\nxMSos2fPKqWUOnXqlIqMjFR5eXnq+PHjqnnz5spkMtmPvWvXrspqtboeRCGckB6GqNLuvPNO7r33\nXvv2sWPHWLRoEdnZ2RgMBrKzs8nJyaFevXplPvvnP/8ZgICAAJo2bUp6ejqNGzcuU+5Pf/oTPj7F\nf0otW7YkPT2ddu3asXPnTl5++WUAbr/9dntP5mqtW7dm1apVzJ8/n4ceeojOnTtfs9zu3bvJyMhg\n2LBh9gUpDQYDJ06coFatWtSsWZO+ffsC0LFjRwwGA8eOHSM8PNzVcAlxQ5IwRJVWu3Zth+0JEyYw\na9YsunTpgqZp3HfffZjN5mt+1s/Pz/5ar9djs9nKVc7V5yw88MADrFu3jh07dvDJJ5+wdOlSVq5c\nWaacUopWrVqxfPnyMu+lp6eX2adpWpV61oOofN4zAyiEE8qF6zfy8vLsq5OuWbPmukmgIrRr186+\nrHRmZia7du26ZrmMjAz8/f3p27cvU6dO5T//+Q9Q/KyLy8uYA7Rt25YjR47www8/2Pft27fP/rqw\nsJBNmzYBxY8oBQgLC6vYgxLVmvQwRJXhyq/p6dOnk5CQQMOGDWnfvj0BAQHX/PzVdV3vvRuVmzFj\nBlOmTMFkMnHnnXfStm1bh++7LC0tjffeew+DwYBSiqSkJAA6derEihUriImJoUOHDkydOpXFixcz\nb948cnNzKSoqokmTJrz55psABAYG8t///pfBgwdjsVhISUnBYDA4jYkQrpLLaoVwE7PZjNFoRK/X\nk5WVxeDBg1m9ejVNmjSp8O+6fJXU9u3bK7xuIS6THoYQbvLrr78ybdo0lFJomsaECRPckiyE8BTp\nYQghhHCJTHoLIYRwiSQMIYQQLpGEIYQQwiWSMIQQQrhEEoYQQgiXSMIQQgjhkv8PZHg4l1eLyCQA\nAAAASUVORK5CYII=\n", - "text/plain": [ - "\u003cmatplotlib.figure.Figure at 0x7f96f7389490\u003e" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" } ], "source": [ @@ -594,33 +516,20 @@ " start = time.time()\n", " (train_losses, test_losses, train_accuracies,\n", " test_accuracies) = train(ds, hp)\n", - " if t \u003c burn_ins:\n", - " continue\n", + " \n", " train_losses[-1].numpy()\n", " test_losses[-1].numpy()\n", " train_accuracies[-1].numpy()\n", " test_accuracies[-1].numpy()\n", + "\n", + " if t \u003c burn_ins:\n", + " continue\n", + "\n", " duration = time.time() - start\n", " durations.append(duration)\n", " print('Duration:', duration)\n", "\n", - "\n", - " print('Mean duration:', np.mean(durations), '+/-', np.std(durations))\n", - " plt.title('MNIST train/test losses')\n", - " plt.plot(train_losses, label='train loss')\n", - " plt.plot(test_losses, label='test loss')\n", - " plt.legend()\n", - " plt.xlabel('Training step')\n", - " plt.ylabel('Loss')\n", - " plt.show()\n", - " plt.title('MNIST train/test accuracies')\n", - " plt.plot(train_accuracies, label='train accuracy')\n", - " plt.plot(test_accuracies, label='test accuracy')\n", - " print('test_accuracy', test_accuracies[-1])\n", - " plt.legend(loc='lower right')\n", - " plt.xlabel('Training step')\n", - " plt.ylabel('Accuracy')\n", - " plt.show()\n" + " print('Mean duration:', np.mean(durations), '+/-', np.std(durations))\n" ] } ], @@ -628,6 +537,10 @@ "colab": { "collapsed_sections": [], "default_view": {}, + "last_runtime": { + "build_target": "", + "kind": "local" + }, "name": "Autograph vs. Eager MNIST benchmark", "provenance": [ { diff --git a/tensorflow/contrib/autograph/impl/conversion.py b/tensorflow/contrib/autograph/impl/conversion.py index bd14359356..1c768b659f 100644 --- a/tensorflow/contrib/autograph/impl/conversion.py +++ b/tensorflow/contrib/autograph/impl/conversion.py @@ -28,16 +28,17 @@ from tensorflow.contrib.autograph.converters import asserts from tensorflow.contrib.autograph.converters import break_statements from tensorflow.contrib.autograph.converters import builtin_functions from tensorflow.contrib.autograph.converters import call_trees +from tensorflow.contrib.autograph.converters import conditional_expressions from tensorflow.contrib.autograph.converters import continue_statements from tensorflow.contrib.autograph.converters import control_flow from tensorflow.contrib.autograph.converters import decorators +from tensorflow.contrib.autograph.converters import directives from tensorflow.contrib.autograph.converters import error_handlers -from tensorflow.contrib.autograph.converters import ifexp from tensorflow.contrib.autograph.converters import lists from tensorflow.contrib.autograph.converters import logical_expressions from tensorflow.contrib.autograph.converters import name_scopes +from tensorflow.contrib.autograph.converters import return_statements from tensorflow.contrib.autograph.converters import side_effect_guards -from tensorflow.contrib.autograph.converters import single_return from tensorflow.contrib.autograph.converters import slices from tensorflow.contrib.autograph.core import config from tensorflow.contrib.autograph.core import converter @@ -48,9 +49,6 @@ from tensorflow.contrib.autograph.pyct import origin_info from tensorflow.contrib.autograph.pyct import parser from tensorflow.contrib.autograph.pyct import qual_names from tensorflow.contrib.autograph.pyct import transformer -from tensorflow.contrib.autograph.pyct.static_analysis import activity -from tensorflow.contrib.autograph.pyct.static_analysis import live_values -from tensorflow.contrib.autograph.pyct.static_analysis import type_info from tensorflow.python.util import tf_inspect @@ -278,16 +276,6 @@ def function_to_graph(f, program_ctx, arg_values, arg_types, owner_type=None): return node, new_name, namespace -def _apply_transformer(node, context, converter_module): - # TODO(mdan): Clear static analysis here. - node = qual_names.resolve(node) - node = activity.resolve(node, context.info, None) - node = live_values.resolve(node, context.info, config.PYTHON_LITERALS) - node = type_info.resolve(node, context.info) - node = converter_module.transform(node, context) - return node - - def node_to_graph(node, context): """Convert Python code to equivalent TF graph mode code. @@ -301,29 +289,32 @@ def node_to_graph(node, context): * deps: A set of strings, the fully qualified names of entity dependencies that this node has. """ - # TODO(mdan): Verify arguments for correctness. + # TODO(mdan): Insert list_comprehensions somewhere. - node = _apply_transformer(node, context, ifexp) + node = converter.standard_analysis(node, context, is_initial=True) # Past this point, line numbers are no longer accurate so we ignore the # source. # TODO(mdan): Is it feasible to reconstruct intermediate source code? context.info.source_code = None - node = _apply_transformer(node, context, decorators) - node = _apply_transformer(node, context, break_statements) - node = _apply_transformer(node, context, asserts) + + node = converter.apply_(node, context, decorators) + node = converter.apply_(node, context, directives) + node = converter.apply_(node, context, break_statements) + node = converter.apply_(node, context, asserts) # Note: sequencing continue canonicalization before for loop one avoids # dealing with the extra loop increment operation that the for # canonicalization creates. - node = _apply_transformer(node, context, continue_statements) + node = converter.apply_(node, context, continue_statements) context.info.namespace['len'] = len - node = _apply_transformer(node, context, single_return) - node = _apply_transformer(node, context, lists) - node = _apply_transformer(node, context, slices) - node = _apply_transformer(node, context, builtin_functions) - node = _apply_transformer(node, context, call_trees) - node = _apply_transformer(node, context, control_flow) - node = _apply_transformer(node, context, logical_expressions) - node = _apply_transformer(node, context, side_effect_guards) - node = _apply_transformer(node, context, name_scopes) - node = _apply_transformer(node, context, error_handlers) + node = converter.apply_(node, context, return_statements) + node = converter.apply_(node, context, lists) + node = converter.apply_(node, context, slices) + node = converter.apply_(node, context, builtin_functions) + node = converter.apply_(node, context, call_trees) + node = converter.apply_(node, context, control_flow) + node = converter.apply_(node, context, conditional_expressions) + node = converter.apply_(node, context, logical_expressions) + node = converter.apply_(node, context, side_effect_guards) + node = converter.apply_(node, context, name_scopes) + node = converter.apply_(node, context, error_handlers) return node diff --git a/tensorflow/contrib/autograph/pyct/cfg.py b/tensorflow/contrib/autograph/pyct/cfg.py index cef6e95206..25fec7fd53 100644 --- a/tensorflow/contrib/autograph/pyct/cfg.py +++ b/tensorflow/contrib/autograph/pyct/cfg.py @@ -699,7 +699,7 @@ class AstToCfg(gast.NodeVisitor): ) if try_node is None: raise ValueError('%s that is not enclosed by any FunctionDef' % node) - self.builder.add_error_node(node, try_node, guards) + self.builder.add_error_node(node, guards) def visit_Assert(self, node): # Ignoring the effect of exceptions. diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/BUILD b/tensorflow/contrib/autograph/pyct/static_analysis/BUILD index 25f78536e0..92eacba3fd 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/BUILD +++ b/tensorflow/contrib/autograph/pyct/static_analysis/BUILD @@ -19,7 +19,6 @@ py_library( srcs = [ "activity.py", "annos.py", - "cfg.py", # TODO(mdan): Remove. "live_values.py", "liveness.py", "reaching_definitions.py", @@ -48,19 +47,6 @@ py_test( ], ) -py_test( - name = "cfg_test", - srcs = ["cfg_test.py"], - srcs_version = "PY2AND3", - tags = ["no_windows"], - deps = [ - ":static_analysis", - "//tensorflow/contrib/autograph/pyct", - "//tensorflow/python:client_testlib", - "@gast_archive//:gast", - ], -) - py_test( name = "live_values_test", srcs = ["live_values_test.py"], @@ -73,15 +59,10 @@ py_test( ], ) -# TODO(mdan): Enable these tests once child change is in. py_test( name = "liveness_test", srcs = ["liveness_test.py"], srcs_version = "PY2AND3", - tags = [ - "manual", - "notap", - ], deps = [ ":static_analysis", "//tensorflow/contrib/autograph/pyct", @@ -93,10 +74,6 @@ py_test( name = "reaching_definitions_test", srcs = ["reaching_definitions_test.py"], srcs_version = "PY2AND3", - tags = [ - "manual", - "notap", - ], deps = [ ":static_analysis", "//tensorflow/contrib/autograph/pyct", diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/activity.py b/tensorflow/contrib/autograph/pyct/static_analysis/activity.py index 4d7b0cbb7b..a0182da9d1 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/activity.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/activity.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Activity analysis.""" +"""Activity analysis. + +Requires qualified name annotations (see qual_names.py). +""" from __future__ import absolute_import from __future__ import division @@ -59,9 +62,10 @@ class Scope(object): self.parent = parent self.add_unknown_symbols = add_unknown_symbols self.modified = set() + # TODO(mdan): Completely remove this. self.created = set() self.used = set() - self.params = set() + self.params = {} self.returned = set() # TODO(mdan): Rename to `locals` @@ -106,37 +110,23 @@ class Scope(object): self.modified |= other.modified self.created |= other.created self.used |= other.used - self.params |= other.params + self.params.update(other.params) self.returned |= other.returned def has(self, name): - if name in self.modified or name in self.params: + if name in self.modified: return True elif self.parent is not None: return self.parent.has(name) return False - def is_modified_since_entry(self, name): - if name in self.modified: - return True - elif self.parent is not None and not self.isolated: - return self.parent.is_modified_since_entry(name) - return False - - def is_param(self, name): - if name in self.params: - return True - elif self.parent is not None and not self.isolated: - return self.parent.is_param(name) - return False - def mark_read(self, name): self.used.add(name) if self.parent is not None and name not in self.created: self.parent.mark_read(name) - def mark_param(self, name): - self.params.add(name) + def mark_param(self, name, owner): + self.params[name] = owner def mark_creation(self, name, writes_create_symbol=False): """Mark a qualified name as created.""" @@ -226,37 +216,56 @@ class ActivityAnalyzer(transformer.Base): elif isinstance(node.ctx, gast.Param): # Param contexts appear in function defs, so they have the meaning of # defining a variable. - # TODO(mdan): This may be incorrect with nested functions. - # For nested functions, we'll have to add the notion of hiding args from - # the parent scope, not writing to them. - self.scope.mark_creation(qn) - self.scope.mark_param(qn) + self.scope.mark_write(qn) + self.scope.mark_param(qn, self.enclosing_entities[-1]) else: raise ValueError('Unknown context %s for node %s.' % (type(node.ctx), qn)) anno.setanno(node, NodeAnno.IS_LOCAL, self.scope.has(qn)) - anno.setanno(node, NodeAnno.IS_MODIFIED_SINCE_ENTRY, - self.scope.is_modified_since_entry(qn)) - anno.setanno(node, NodeAnno.IS_PARAM, self.scope.is_param(qn)) if self._in_return_statement: self.scope.mark_returned(qn) + def _enter_scope(self, isolated): + self.scope = Scope(self.scope, isolated=isolated) + + def _exit_scope(self): + self.scope = self.scope.parent + + def _process_statement(self, node): + self._enter_scope(False) + node = self.generic_visit(node) + anno.setanno(node, anno.Static.SCOPE, self.scope) + self._exit_scope() + return node + + def visit_Expr(self, node): + return self._process_statement(node) + + def visit_Return(self, node): + self._in_return_statement = True + node = self._process_statement(node) + self._in_return_statement = False + return node + + def visit_Assign(self, node): + return self._process_statement(node) + def visit_AugAssign(self, node): # Special rules for AugAssign. In Assign, the target is only written, # but in AugAssig (e.g. a += b), the target is both read and written. self._in_aug_assign = True - self.generic_visit(node) + node = self._process_statement(node) self._in_aug_assign = False return node def visit_Name(self, node): - self.generic_visit(node) + node = self.generic_visit(node) self._track_symbol(node) return node def visit_Attribute(self, node): - self.generic_visit(node) + node = self.generic_visit(node) if self._in_constructor and self._node_sets_self_attribute(node): self._track_symbol( node, composite_writes_alter_parent=True, writes_create_symbol=True) @@ -265,44 +274,38 @@ class ActivityAnalyzer(transformer.Base): return node def visit_Subscript(self, node): - self.generic_visit(node) + node = self.generic_visit(node) # Subscript writes (e.g. a[b] = "value") are considered to modify # both the element itself (a[b]) and its parent (a). - self._track_symbol(node, composite_writes_alter_parent=True) + self._track_symbol(node) return node def visit_Print(self, node): - current_scope = self.scope - args_scope = Scope(current_scope) - self.scope = args_scope - for n in node.values: - self.visit(n) - anno.setanno(node, NodeAnno.ARGS_SCOPE, args_scope) - self.scope = current_scope + self._enter_scope(False) + node.values = self.visit_block(node.values) + anno.setanno(node, anno.Static.SCOPE, self.scope) + anno.setanno(node, NodeAnno.ARGS_SCOPE, self.scope) + self._exit_scope() return node + def visit_Assert(self, node): + return self._process_statement(node) + def visit_Call(self, node): - current_scope = self.scope - args_scope = Scope(current_scope, isolated=False) - self.scope = args_scope - for n in node.args: - self.visit(n) + self._enter_scope(False) + node.args = self.visit_block(node.args) + node.keywords = self.visit_block(node.keywords) # TODO(mdan): Account starargs, kwargs - for n in node.keywords: - self.visit(n) - anno.setanno(node, NodeAnno.ARGS_SCOPE, args_scope) - self.scope = current_scope - self.visit(node.func) + anno.setanno(node, NodeAnno.ARGS_SCOPE, self.scope) + self._exit_scope() + node.func = self.visit(node.func) return node def _process_block_node(self, node, block, scope_name): - current_scope = self.scope - block_scope = Scope(current_scope, isolated=False) - self.scope = block_scope - for n in block: - self.visit(n) - anno.setanno(node, scope_name, block_scope) - self.scope = current_scope + self._enter_scope(False) + block = self.visit_block(block) + anno.setanno(node, scope_name, self.scope) + self._exit_scope() return node def _process_parallel_blocks(self, parent, children): @@ -321,94 +324,75 @@ class ActivityAnalyzer(transformer.Base): self.scope.merge_from(after_child) return parent + def visit_arguments(self, node): + return self._process_statement(node) + def visit_FunctionDef(self, node): - if self.scope: - qn = qual_names.QN(node.name) - self.scope.mark_write(qn) - current_scope = self.scope - body_scope = Scope(current_scope, isolated=True) - self.scope = body_scope - self.generic_visit(node) - anno.setanno(node, NodeAnno.BODY_SCOPE, body_scope) - self.scope = current_scope + # The FunctionDef node itself has a Scope object that tracks the creation + # of its name, along with the usage of any decorator accompany it. + self._enter_scope(False) + node.decorator_list = self.visit_block(node.decorator_list) + self.scope.mark_write(qual_names.QN(node.name)) + anno.setanno(node, anno.Static.SCOPE, self.scope) + self._exit_scope() + + # A separate Scope tracks the actual function definition. + self._enter_scope(True) + node.args = self.visit(node.args) + + # Track the body separately. This is for compatibility reasons, it may not + # be strictly needed. + self._enter_scope(False) + node.body = self.visit_block(node.body) + anno.setanno(node, NodeAnno.BODY_SCOPE, self.scope) + self._exit_scope() + + self._exit_scope() return node def visit_With(self, node): - current_scope = self.scope - with_scope = Scope(current_scope, isolated=False) - self.scope = with_scope - self.generic_visit(node) - anno.setanno(node, NodeAnno.BODY_SCOPE, with_scope) - self.scope = current_scope + self._enter_scope(False) + node = self.generic_visit(node) + anno.setanno(node, NodeAnno.BODY_SCOPE, self.scope) + self._exit_scope() return node - def visit_If(self, node): - current_scope = self.scope - cond_scope = Scope(current_scope, isolated=False) - self.scope = cond_scope - self.visit(node.test) - anno.setanno(node, NodeAnno.COND_SCOPE, cond_scope) - self.scope = current_scope + def visit_withitem(self, node): + return self._process_statement(node) + def visit_If(self, node): + self._enter_scope(False) + node.test = self.visit(node.test) + anno.setanno(node, NodeAnno.COND_SCOPE, self.scope) + anno.setanno(node.test, anno.Static.SCOPE, self.scope) + self._exit_scope() node = self._process_parallel_blocks(node, ((node.body, NodeAnno.BODY_SCOPE), (node.orelse, NodeAnno.ORELSE_SCOPE))) return node def visit_For(self, node): - self.visit(node.target) - self.visit(node.iter) + self._enter_scope(False) + node.target = self.visit(node.target) + node.iter = self.visit(node.iter) + anno.setanno(node.iter, anno.Static.SCOPE, self.scope) + self._exit_scope() node = self._process_parallel_blocks(node, ((node.body, NodeAnno.BODY_SCOPE), (node.orelse, NodeAnno.ORELSE_SCOPE))) return node def visit_While(self, node): - current_scope = self.scope - cond_scope = Scope(current_scope, isolated=False) - self.scope = cond_scope - self.visit(node.test) - anno.setanno(node, NodeAnno.COND_SCOPE, cond_scope) - self.scope = current_scope - + self._enter_scope(False) + node.test = self.visit(node.test) + anno.setanno(node, NodeAnno.COND_SCOPE, self.scope) + anno.setanno(node.test, anno.Static.SCOPE, self.scope) + self._exit_scope() node = self._process_parallel_blocks(node, ((node.body, NodeAnno.BODY_SCOPE), (node.orelse, NodeAnno.ORELSE_SCOPE))) return node - def visit_Return(self, node): - self._in_return_statement = True - node = self.generic_visit(node) - self._in_return_statement = False - return node - - -def get_read(node, context): - """Return the variable names as QNs (qual_names.py) read by this statement.""" - analyzer = ActivityAnalyzer(context, None, True) - analyzer.visit(node) - return analyzer.scope.used - - -def get_updated(node, context): - """Return the variable names created or mutated by this statement. - - This function considers assign statements, augmented assign statements, and - the targets of for loops, as well as function arguments. - For example, `x[0] = 2` will return `x`, `x, y = 3, 4` will return `x` and - `y`, `for i in range(x)` will return `i`, etc. - Args: - node: An AST node - context: An EntityContext instance - - Returns: - A set of variable names (QNs, see qual_names.py) of all the variables - created or mutated. - """ - analyzer = ActivityAnalyzer(context, None, True) - analyzer.visit(node) - return analyzer.scope.created | analyzer.scope.modified - def resolve(node, context, parent_scope=None): return ActivityAnalyzer(context, parent_scope).visit(node) diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py index bc22be0a27..e940516190 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py @@ -52,18 +52,18 @@ class ScopeTest(test.TestCase): other = activity.Scope(None) other.copy_from(scope) - self.assertTrue(QN('foo') in other.created) + self.assertTrue(QN('foo') in other.modified) scope.mark_write(QN('bar')) scope.copy_from(other) - self.assertFalse(QN('bar') in scope.created) + self.assertFalse(QN('bar') in scope.modified) scope.mark_write(QN('bar')) scope.merge_from(other) - self.assertTrue(QN('bar') in scope.created) - self.assertFalse(QN('bar') in other.created) + self.assertTrue(QN('bar') in scope.modified) + self.assertFalse(QN('bar') in other.modified) def test_copy_of(self): scope = activity.Scope(None) @@ -157,7 +157,8 @@ class ActivityAnalyzerTest(test.TestCase): """Assert the scope contains specific used, modified & created variables.""" self.assertSymbolSetsAre(used, scope.used, 'read') self.assertSymbolSetsAre(modified, scope.modified, 'modified') - self.assertSymbolSetsAre(created, scope.created, 'created') + # Created is deprecated, we're no longer verifying it. + # self.assertSymbolSetsAre(created, scope.created, 'created') def test_print_statement(self): @@ -215,12 +216,6 @@ class ActivityAnalyzerTest(test.TestCase): (), (), ) - self.assertScopeIsRmc( - anno.getanno(call_node, NodeAnno.ARGS_SCOPE).parent, - ('a', 'a.b', 'a.c', 'a.d', 'foo'), - ('a.c',), - ('a',), - ) def test_call_args_subscripts(self): @@ -241,12 +236,6 @@ class ActivityAnalyzerTest(test.TestCase): (), (), ) - self.assertScopeIsRmc( - anno.getanno(call_node, NodeAnno.ARGS_SCOPE).parent, - ('a', 'a[0]', 'a[b]', 'a[c]', 'b', 'c', 'foo'), - ('b', 'c'), - ('a', 'b', 'c'), - ) def test_while(self): @@ -362,20 +351,20 @@ class ActivityAnalyzerTest(test.TestCase): self.assertScopeIsRmc( anno.getanno(if_node, NodeAnno.BODY_SCOPE), ('a', 'b', 'c', 'a[c]'), - ('a', 'a[b]', 'd'), + ('a[b]', 'd'), ('d',), ) # TODO(mdan): Should subscript writes (a[0] = 1) be considered to read "a"? self.assertScopeIsRmc( anno.getanno(if_node, NodeAnno.ORELSE_SCOPE), ('a', 'e'), - ('a', 'a[0]', 'd'), + ('a[0]', 'd'), ('d',), ) self.assertScopeIsRmc( anno.getanno(if_node, NodeAnno.ORELSE_SCOPE).parent, ('a', 'b', 'c', 'd', 'e', 'a[c]'), - ('a', 'd', 'a[b]', 'a[0]'), + ('d', 'a[b]', 'a[0]'), ('a', 'b', 'c', 'd', 'e'), ) @@ -415,10 +404,6 @@ class ActivityAnalyzerTest(test.TestCase): node, _ = self._parse_and_analyze(test_fn) fn_def_node = node.body[0].body[0] - self.assertScopeIsRmc( - anno.getanno(fn_def_node, - NodeAnno.BODY_SCOPE).parent, ('b', 'i', 'f', 'c', 'a'), - ('f', 'b', 'c', 'i'), ('f', 'a', 'b', 'c', 'i')) self.assertScopeIsRmc( anno.getanno(fn_def_node, NodeAnno.BODY_SCOPE), ('x', 'y'), ('y',), ( 'x', @@ -452,7 +437,7 @@ class ActivityAnalyzerTest(test.TestCase): self.assertScopeIsRmc( anno.getanno(fn_node, NodeAnno.BODY_SCOPE), ('a', 'a[0]'), - ('a', 'a[0]'), + ('a[0]',), ('a',), ) @@ -518,47 +503,6 @@ class ActivityAnalyzerTest(test.TestCase): anno.getanno(fn_node, NodeAnno.BODY_SCOPE), ('b',), (('')), (('a', 'b'))) - def test_get_read(self): - - def test_fn(x, y): - z = test_fn(x, y) - return z - - node, ctx = self._parse_and_analyze(test_fn) - node = node.body[0].body[0] - read_vars = activity.get_read(node, ctx) - self.assertEqual(read_vars, set(map(qual_names.QN, ('test_fn', 'x', 'y')))) - - def test_fn2(x, y, z): - z += test_fn2(x, y, z) - return z - - node, ctx = self._parse_and_analyze(test_fn2) - node = node.body[0].body[0] - read_vars = activity.get_read(node, ctx) - self.assertEqual(read_vars, - set(map(qual_names.QN, ('test_fn2', 'x', 'y', 'z')))) - - def test_get_updated(self): - - def test_fn(x, y): - z = test_fn(x, y) - return z - - node, ctx = self._parse_and_analyze(test_fn) - node = node.body[0].body[0] - updated_vars = activity.get_updated(node, ctx) - self.assertEqual(updated_vars, set(map(qual_names.QN, ('z')))) - - def test_fn2(x, y, z): - z += test_fn2(x, y, z) - return z - - node, ctx = self._parse_and_analyze(test_fn2) - node = node.body[0].body[0] - updated_vars = activity.get_updated(node, ctx) - self.assertEqual(updated_vars, set(map(qual_names.QN, ('z')))) - if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/cfg.py b/tensorflow/contrib/autograph/pyct/static_analysis/cfg.py deleted file mode 100644 index 4acc4ed66a..0000000000 --- a/tensorflow/contrib/autograph/pyct/static_analysis/cfg.py +++ /dev/null @@ -1,446 +0,0 @@ -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Control flow graph analysis. - -Given a Python AST we construct a control flow graph, with edges both to the -next and previous statements (so it can easily walk the graph both ways). Its -nodes contain the AST of the statements. It can then perform forward or backward -analysis on this CFG. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from collections import namedtuple -import functools -import operator - -import gast - -from tensorflow.contrib.autograph.pyct import anno -from tensorflow.contrib.autograph.pyct.static_analysis import activity - - -class CfgNode(object): - """A node in the CFG.""" - __slots__ = ['next', 'value', 'prev'] - - def __init__(self, value): - self.next = set() - self.prev = set() - self.value = value - - -class Cfg(namedtuple('Cfg', ['entry', 'exit'])): - """A Control Flow Graph. - - Each statement is represented as a node. For control flow statements such - as conditionals and loops the conditional itself is a node which either - branches or cycles, respectively. - Attributes: - entry: The entry node, which contains the `gast.arguments` node of the - function definition. - exit: The exit node. This node is special because it has no value (i.e. no - corresponding AST node). This is because Python functions can have - multiple return statements. - """ - pass - - -class CfgBuilder(gast.NodeVisitor): - """Construct a control flow graph. - - Construct a CFG starting from a FunctionDef node. - Usage: - cfg_obj = CfgBuilder().build_cfg(fndef_node) - """ - - def __init__(self): - # The current leaves of the CFG - self.current_leaves = [] - # TODO(alexbw): generalize to break, return, continue, yield, etc. - # A stack of lists, tracking continue statements - self.continue_ = [] - # A stack of lists tracking break nodes - self.break_ = [] - - def set_current_leaves(self, cfg_node): - """Link this cfg_node to the current leaves. - - This is the central function for building the CFG. It links the current - head cfg_nodes to the passed cfg_node. It then resets the head to the - passed cfg_node. - - Args: - cfg_node: A CfgNode instance. - """ - for head in self.current_leaves: - head.next.add(cfg_node) - # While we're linking the CFG forward, add backlinks - cfg_node.prev.add(head) - self.current_leaves = [cfg_node] - - def build_cfg(self, node): - """Build a CFG for a function. - - Implementation of building a CFG for dataflow analysis. See, e.g.: - https://www.seas.harvard.edu/courses/cs252/2011sp/slides/Lec02-Dataflow.pdf - - Args: - node: A function definition the body of which to analyze. - Returns: - A CFG object. - Raises: - TypeError: If the input is not a function definition. - """ - if not isinstance(node, gast.FunctionDef): - raise TypeError('input must be a function definition') - entry_cfg_node = CfgNode(node.args) - self.current_leaves = [entry_cfg_node] - self.visit_statements(node.body) - exit_cfg_node = CfgNode(None) - self.set_current_leaves(exit_cfg_node) - return Cfg(entry_cfg_node, exit_cfg_node) - - def visit_statements(self, nodes): - for node in nodes: - # Check for control flow - if isinstance(node, (gast.For, gast.While, gast.If, gast.Try, gast.Break, - gast.Continue, gast.With)): - self.visit(node) - else: - expr = CfgNode(node) - self.set_current_leaves(expr) - - def generic_visit(self, node): - raise ValueError('unknown control flow') - - def visit_If(self, node): - # TODO(alexbw): change this to use immutable tuples instead of lists - # The current head will hold the conditional - test = CfgNode(node.test) - self.set_current_leaves(test) - # Handle the body - self.visit_statements(node.body) - body_exit = self.current_leaves - self.current_leaves = [test] - # Handle the orelse - self.visit_statements(node.orelse) - self.current_leaves.extend(body_exit) - - def visit_While(self, node): - test = CfgNode(node.test) - self.set_current_leaves(test) - # Start a new level of nesting - self.break_.append([]) - self.continue_.append([]) - # Handle the body - self.visit_statements(node.body) - body_exit = self.current_leaves - self.current_leaves.extend(self.continue_.pop()) - self.set_current_leaves(test) - # Handle the orelse - self.visit_statements(node.orelse) - # The break statements and the test go to the next node - self.current_leaves.extend(self.break_.pop()) - # Body and orelse statements can reach out of the loop - self.current_leaves.extend(body_exit) - - def visit_For(self, node): - iter_ = CfgNode(node.iter) - self.set_current_leaves(iter_) - self.break_.append([]) - self.continue_.append([]) - self.visit_statements(node.body) - body_exit = self.current_leaves - self.current_leaves.extend(self.continue_.pop()) - self.set_current_leaves(iter_) - # Handle the orelse - self.visit_statements(node.orelse) - # The break statements and the test go to the next node - self.current_leaves.extend(self.break_.pop()) - # Body and orelse statements can reach out of the loop - self.current_leaves.extend(body_exit) - - def visit_Break(self, node): - self.break_[-1].extend(self.current_leaves) - self.current_leaves[:] = [] - - def visit_Continue(self, node): - self.continue_[-1].extend(self.current_leaves) - self.current_leaves[:] = [] - - def visit_Try(self, node): - self.visit_statements(node.body) - body = self.current_leaves - handlers = [] - for handler in node.handlers: - self.current_leaves = body[:] - self.visit_statements(handler.body) - handlers.extend(self.current_leaves) - self.current_leaves = body - self.visit_statements(node.orelse) - self.current_leaves = handlers + self.current_leaves - self.visit_statements(node.finalbody) - - def visit_With(self, node): - for item in node.items: - self.set_current_leaves(CfgNode(item)) - self.visit_statements(node.body) - - -# TODO(alexbw): once CFG analysis occurs at a block level, -# this extra class will not be necessary -class PropagateAnalysis(gast.NodeVisitor): - """Port analysis annotations from statements to their enclosing blocks.""" - - def __init__(self, analysis): - self.transfer_fn = analysis.transfer_fn - self.in_label = analysis.in_label - self.out_label = analysis.out_label - super(PropagateAnalysis, self).__init__() - - def visit_If(self, node): - # Depth-first. - self.generic_visit(node) - incoming = anno.getanno(node.body[0], self.in_label) - incoming |= anno.getanno(node.test, self.in_label) - outgoing = anno.getanno(node.body[-1], self.out_label) - outgoing |= anno.getanno(node.test, self.out_label) - if node.orelse: - orelse_outgoing = anno.getanno(node.orelse[-1], self.out_label) - outgoing = self.transfer_fn(outgoing, orelse_outgoing) - anno.setanno(node, self.in_label, incoming) - anno.setanno(node, self.out_label, outgoing) - - def visit_For(self, node): - self.generic_visit(node) - incoming = set(anno.getanno(node.body[0], self.in_label)) - incoming -= set((anno.getanno(node.target, anno.Basic.QN),)) - outgoing = anno.getanno(node.body[-1], self.out_label) - if node.orelse: - orelse_outgoing = anno.getanno(node.orelse[-1], self.out_label) - outgoing = self.transfer_fn(outgoing, orelse_outgoing) - anno.setanno(node, self.in_label, frozenset(incoming)) - anno.setanno(node, self.out_label, outgoing) - - def visit_While(self, node): - self.generic_visit(node) - incoming = anno.getanno(node.body[0], self.in_label) - incoming |= anno.getanno(node.test, self.in_label) - outgoing = anno.getanno(node.body[-1], self.out_label) - if node.orelse: - orelse_outgoing = anno.getanno(node.orelse[-1], self.out_label) - outgoing = self.transfer_fn(outgoing, orelse_outgoing) - anno.setanno(node, self.in_label, incoming) - anno.setanno(node, self.out_label, outgoing) - - def visit_With(self, node): - self.generic_visit(node) - incoming = anno.getanno(node.body[0], self.in_label) - for item in node.items: - incoming |= anno.getanno(item, self.in_label) - outgoing = anno.getanno(node.body[-1], self.out_label) - anno.setanno(node, self.in_label, incoming) - anno.setanno(node, self.out_label, outgoing) - - -# TODO(alexbw): Abstract the CFG walking machinery into a superclass -# which is parameterized on which fields it selects when walking. -# TODO(alexbw): Abstract the application of dataflow analysis -class Forward(object): - """Forward analysis on CFG. - - Args: - label: A name for this analysis e.g. 'active' for activity analysis. The AST - nodes in the CFG will be given annotations 'name_in', 'name_out', - 'name_gen' and 'name_kill' which contain the incoming values, outgoing - values, values generated by the statement, and values deleted by the - statement respectively. - transfer_fn: Either the AND or OR operator. If the AND operator is used it - turns into forward must analysis (i.e. a value will only be carried - forward if it appears on all incoming paths). The OR operator means that - forward may analysis is done (i.e. the union of incoming values will be - taken). - """ - - def __init__(self, label, source_info, transfer_fn=operator.or_): - self.transfer_fn = transfer_fn - self.source_info = source_info - self.out_label = label + '_out' - self.in_label = label + '_in' - self.gen_label = label + '_gen' - self.kill_label = label + '_kill' - - # TODO(alexbw): see if we can simplify by visiting breadth-first - def visit(self, node): - """Depth-first walking the CFG, applying dataflow info propagation.""" - # node.value is None only for the exit CfgNode. - if not node.value: - return - - if anno.hasanno(node.value, self.out_label): - before = hash(anno.getanno(node.value, self.out_label)) - else: - before = None - preds = [ - anno.getanno(pred.value, self.out_label) - for pred in node.prev - if anno.hasanno(pred.value, self.out_label) - ] - if preds: - incoming = functools.reduce(self.transfer_fn, preds[1:], preds[0]) - else: - incoming = frozenset() - anno.setanno(node.value, self.in_label, incoming) - gen, kill = self.get_gen_kill(node, incoming) - anno.setanno(node.value, self.gen_label, gen) - anno.setanno(node.value, self.kill_label, kill) - anno.setanno(node.value, self.out_label, (incoming - kill) | gen) - - if hash(anno.getanno(node.value, self.out_label)) != before: - for succ in node.next: - self.visit(succ) - - def get_gen_kill(self, cfg_node, incoming): - """Calculate Gen and Kill properties of a CFG node in dataflow analysis. - - A function which takes the CFG node as well as a set of incoming - values. It must return a set of newly generated values by the statement as - well as a set of deleted (killed) values. - - Args: - cfg_node: A CfgNode instance. - incoming: - """ - raise NotImplementedError() - - -class Backward(Forward): - """Backward analysis on CFG.""" - - def visit(self, cfg_node): - # cfg_node.value is None for the exit node, which will be visited only once - if not cfg_node.value: - for pred in cfg_node.prev: - self.visit(pred) - return - - if anno.hasanno(cfg_node.value, self.in_label): - before = hash(anno.getanno(cfg_node.value, self.in_label)) - else: - before = None - succs = [ - anno.getanno(succ.value, self.in_label) - for succ in cfg_node.next - if anno.hasanno(succ.value, self.in_label) - ] - if succs: - incoming = functools.reduce(self.transfer_fn, succs[1:], succs[0]) - else: - incoming = frozenset() - anno.setanno(cfg_node.value, self.out_label, incoming) - gen, kill = self.get_gen_kill(cfg_node, incoming) - anno.setanno(cfg_node.value, self.gen_label, gen) - anno.setanno(cfg_node.value, self.kill_label, kill) - anno.setanno(cfg_node.value, self.in_label, (incoming - kill) | gen) - if hash(anno.getanno(cfg_node.value, self.in_label)) != before: - for pred in cfg_node.prev: - self.visit(pred) - - -def run_analyses(node, analyses): - """Perform dataflow analysis on all functions within an AST. - - Args: - node: An AST node on which to run dataflow analysis. - analyses: Either an instance of the Forward or Backward dataflow analysis - class, or a list or tuple of them. - - Returns: - node: The node, but now with annotations on the AST nodes containing the - results of the dataflow analyses. - """ - if not isinstance(analyses, (tuple, list)): - analyses = (analyses,) - for analysis in analyses: - if not isinstance(analysis, (Forward, Backward)): - raise TypeError('not a valid forward analysis object') - - for child_node in gast.walk(node): - if isinstance(child_node, gast.FunctionDef): - cfg_obj = CfgBuilder().build_cfg(child_node) - for analysis in analyses: - if isinstance(analysis, Backward): - analysis.visit(cfg_obj.exit) - elif isinstance(analysis, Forward): - analysis.visit(cfg_obj.entry) - for analysis in analyses: - PropagateAnalysis(analysis).visit(node) - return node - - -class Liveness(Backward): - """Perform a liveness analysis. - - Each statement is annotated with a set of variables that may be used - later in the program. - """ - - def __init__(self, source_info): - super(Liveness, self).__init__('live', source_info) - - def get_gen_kill(self, node, _): - # A variable's parents are live if it is live - # e.g. x is live if x.y is live. This means gen needs to return - # all parents of a variable (if it's an Attribute or Subscript). - # This doesn't apply to kill (e.g. del x.y doesn't affect liveness of x) - gen = activity.get_read(node.value, self.source_info) - gen = functools.reduce(lambda left, right: left | right.support_set, gen, - gen) - kill = activity.get_updated(node.value, self.source_info) - return gen, kill - - -class ReachingDefinitions(Forward): - """Perform reaching definition analysis. - - Each statement is annotated with a set of (variable, definition) pairs. - """ - - def __init__(self, source_info): - super(ReachingDefinitions, self).__init__('definitions', source_info) - - def get_gen_kill(self, node, incoming): - definitions = activity.get_updated(node.value, self.source_info) - gen = frozenset((id_, node.value) for id_ in definitions) - kill = frozenset(def_ for def_ in incoming if def_[0] in definitions) - return gen, kill - - -class Defined(Forward): - """Perform defined variable analysis. - - Each statement is annotated with a set of variables which are guaranteed to - be defined at that point. - """ - - def __init__(self, source_info): - super(Defined, self).__init__( - 'defined', source_info, transfer_fn=operator.and_) - - def get_gen_kill(self, node, _): - gen = activity.get_updated(node.value, self.source_info) - return gen, frozenset() diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/cfg_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/cfg_test.py deleted file mode 100644 index 428ebbedca..0000000000 --- a/tensorflow/contrib/autograph/pyct/static_analysis/cfg_test.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for cfg module.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import functools - -import gast - -from tensorflow.contrib.autograph.pyct import anno -from tensorflow.contrib.autograph.pyct import parser -from tensorflow.contrib.autograph.pyct import qual_names -from tensorflow.contrib.autograph.pyct import transformer -from tensorflow.contrib.autograph.pyct.static_analysis import cfg -from tensorflow.python.platform import test - - -class CFGTest(test.TestCase): - - def _parse_and_analyze(self, test_fn): - node, source = parser.parse_entity(test_fn) - entity_info = transformer.EntityInfo( - source_code=source, - source_file=None, - namespace={}, - arg_values=None, - arg_types=None, - owner_type=None) - node = qual_names.resolve(node) - return node, entity_info - - def _check_anno_matches(self, node, anno_name, var_names): - if isinstance(var_names, str): - var_names = (var_names,) - qual_vars = set() - for var_name in var_names: - if isinstance(var_name, str): - if '[' in var_name or ']' in var_name: - raise ValueError('Annotation matching not supported with subscript.') - if '.' not in var_name: - qual_vars.add(qual_names.QN(var_name)) - else: - attrs = var_name.split('.') - this_qn = functools.reduce(qual_names.QN, attrs[1:], - qual_names.QN(attrs[0])) - qual_vars.add(this_qn) - self.assertEqual(anno.getanno(node, anno_name), qual_vars) - - def test_reaching(self): - - def f(x): - print(x) - while True: - x = x - x = x - return x - - node, ctx = self._parse_and_analyze(f) - cfg.run_analyses(node, cfg.ReachingDefinitions(ctx)) - body = node.body[0].body - # Only the argument reaches the expression - def_in = anno.getanno(body[0], 'definitions_in') - # One element, x, from arguments - self.assertEqual(set(type(d[1]) for d in def_in), set((gast.arguments,))) - - while_body = body[1].body - def_in = anno.getanno(while_body[0], 'definitions_in') - # One definition, two possible sources. - # - One from an assignment (if the loop is entered) - # - The other from the arguments (if loop is not entered) - self.assertEqual( - set(type(d[1]) for d in def_in), set((gast.arguments, gast.Assign))) - - def_in = anno.getanno(while_body[1], 'definitions_in') - # If we've reached this line, the only reaching definition of x is the - # Assign node in previous line - self.assertEqual(set(type(d[1]) for d in def_in), set((gast.Assign,))) - - def_in = anno.getanno(body[2], 'definitions_in') - # Same situation as while_body[0] - self.assertEqual( - set(type(d[1]) for d in def_in), set((gast.arguments, gast.Assign))) - - def test_defined(self): - - def f(x): - if x: - y = 2 # pylint: disable=unused-variable - return x - - node, ctx = self._parse_and_analyze(f) - cfg.run_analyses(node, cfg.Defined(ctx)) - body = node.body[0].body - # only x is for sure defined at the end - self._check_anno_matches(body[1], 'defined_in', 'x') - # at the end of the if body both x and y are defined - if_body = body[0].body - self._check_anno_matches(if_body[0], 'defined_out', ('x', 'y')) - - def _get_live_annotated_fnbody(self, f): - node, ctx = self._parse_and_analyze(f) - cfg.run_analyses(node, cfg.Liveness(ctx)) - body = node.body[0].body - return body - - def test_live_straightline(self): - - def f1(x): - a = g(x) # pylint: disable=undefined-variable - b = h(a) # pylint: disable=undefined-variable, unused-variable - return x - - body = self._get_live_annotated_fnbody(f1) - self._check_anno_matches(body[1], 'live_in', ('a', 'h', 'x')) - self._check_anno_matches(body[2], 'live_in', ('x')) - self._check_anno_matches(body[0], 'live_in', ('g', 'h', 'x')) - self._check_anno_matches(body[2], 'live_out', ()) - - def test_live_stacked_conds_with_else(self): - - def f2(x, a): # pylint: disable=unused-argument - if a > 0: # x should not be live - x = 0 - if a > 1: - x = 1 - else: - x = 2 - - body = self._get_live_annotated_fnbody(f2) - self._check_anno_matches(body[0], 'live_in', ('a')) - self._check_anno_matches(body[1], 'live_in', ('a')) - - def test_live_stacked_conds(self): - - def f3(x, a): - if a > 0: # x and a should be live - x = 0 - if a > 1: # x and a should be live_in - x = 1 - return x # x should be live - - body = self._get_live_annotated_fnbody(f3) - self._check_anno_matches(body[0], 'live_in', ('a', 'x')) - self._check_anno_matches(body[1], 'live_in', ('a', 'x')) - self._check_anno_matches(body[2], 'live_in', ('x')) - - def test_live_possibly_unused_cond(self): - - def f4(x, a): - if a > 0: # x should be live - x = 0 - x += 1 - - body = self._get_live_annotated_fnbody(f4) - self._check_anno_matches(body[0], 'live_in', ('x', 'a')) - self._check_anno_matches(body[1], 'live_in', ('x')) - - def test_live_attribute_in_cond(self): - - def f5(x, a): - if a > 0: # x.y should be live - x.y = 0 - return x.y - - body = self._get_live_annotated_fnbody(f5) - self._check_anno_matches(body[0], 'live_in', ('x', 'x.y', 'a')) - - def test_live_noop(self): - - def f6(x): - return x # should this cause x.* to be live? - - body = self._get_live_annotated_fnbody(f6) - self._check_anno_matches(body[0], 'live_in', ('x')) - - def test_live_loop(self): - - def f7(x, n): - for i in range(n): - x += i - return x - - body = self._get_live_annotated_fnbody(f7) - self._check_anno_matches(body[0], 'live_in', ('x', 'n', 'range')) - self._check_anno_matches(body[1], 'live_in', ('x')) - - def test_live_context_manager(self): - - def f8(x, f): - with f: - x += 1 - - body = self._get_live_annotated_fnbody(f8) - self._check_anno_matches(body[0], 'live_in', ('f', 'x')) - - def test_node_equality(self): - node_a = gast.parse('y = x').body[0] - node_b = gast.parse('y = x').body[0] - self.assertNotEqual(node_a, node_b) - - def test_nested_functions_defined(self): - - def f(x): - y = x * 2 - - def g(z): - return z + y - - return g(x) - - node, ctx = self._parse_and_analyze(f) - cfg.run_analyses(node, cfg.Defined(ctx)) - - body = node.body[0].body - self.assertEqual( - anno.getanno(body[2], 'defined_in'), - frozenset(map(qual_names.QN, ('g', 'x', 'y')))) - - # TODO(alexbw): CFG analysis doesn't currently cross FunctionDef boundaries. - # NOTE: 'z' is easy to find, but 'y' is not identified as - # defined, because CFG analysis is applied with each function separately. - # fndef_body = body[1].body - # self.assertEqual( - # anno.getanno(fndef_body[0], 'defined_in'), - # frozenset(map(qual_names.QN, ('z', 'y')))) - - def test_nested_functions_dont_leak_definitions(self): - - def f(x): - print(x) - - def g(): - y = 2 - return y - - return g() # y is not defined here - - node, ctx = self._parse_and_analyze(f) - cfg.run_analyses(node, cfg.Defined(ctx)) - body = node.body[0].body - self.assertEqual( - anno.getanno(body[2], 'defined_in'), - frozenset(map(qual_names.QN, ('x', 'g')))) - - def test_loop_else(self): - - # Disabling useless-else-on-loop error, because 'break' and 'continue' - # canonicalization are a separate analysis pass, and here we test - # the CFG analysis in isolation. - def for_orelse(x): - y = 0 - for i in range(len(x)): - x += i - else: # pylint: disable=useless-else-on-loop - y = 1 - return x, y - - def while_orelse(x, i): - y = 0 - while x < 10: - x += i - else: # pylint: disable=useless-else-on-loop - y = 1 - return x, y - - for f in (for_orelse, while_orelse): - node, ctx = self._parse_and_analyze(f) - cfg.run_analyses(node, cfg.ReachingDefinitions(ctx)) - body = node.body[0].body - return_node = body[-1] - reaching_defs = anno.getanno(return_node, 'definitions_in') - - # Y could be defined by Assign(Num(0)) or Assign(Num(1)) - # X could be defined as an argument or an AugAssign. - y_defs = [node for var, node in reaching_defs if str(var) == 'y'] - x_defs = [node for var, node in reaching_defs if str(var) == 'x'] - - self.assertEqual(set((gast.Assign,)), set(type(def_) for def_ in y_defs)) - self.assertEqual(set((0, 1)), set(def_.value.n for def_ in y_defs)) - self.assertEqual(len(y_defs), 2) - self.assertEqual( - set((gast.arguments, gast.AugAssign)), - set(type(def_) for def_ in x_defs)) - self.assertEqual(len(x_defs), 2) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py b/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py index 9ccb98f79a..32802069ba 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py @@ -16,7 +16,7 @@ Live values are extracted from the known execution context. -Requires activity analysis annotations. +Requires activity and reaching definitions analyses. """ from __future__ import absolute_import @@ -45,14 +45,12 @@ class LiveValueResolver(transformer.Base): def visit_Name(self, node): self.generic_visit(node) if isinstance(node.ctx, gast.Load): - assert anno.hasanno(node, NodeAnno.IS_LOCAL), node - symbol_is_local = anno.getanno(node, NodeAnno.IS_LOCAL) - assert anno.hasanno(node, NodeAnno.IS_MODIFIED_SINCE_ENTRY), node - symbol_is_modified = anno.getanno(node, NodeAnno.IS_MODIFIED_SINCE_ENTRY) - assert anno.hasanno(node, NodeAnno.IS_PARAM), node - symbol_is_param = anno.getanno(node, NodeAnno.IS_PARAM) - - if not symbol_is_local and not symbol_is_param: + defs = anno.getanno(node, anno.Static.DEFINITIONS, ()) + + is_defined = bool(defs) + has_single_def = len(defs) == 1 + + if not is_defined: if node.id in self.literals: anno.setanno(node, 'live_val', self.literals[node.id]) elif node.id in self.entity_info.namespace: @@ -79,11 +77,13 @@ class LiveValueResolver(transformer.Base): # TODO(mdan): Attempt to trace its value through the local chain. # TODO(mdan): Use type annotations as fallback. - if not symbol_is_modified: - if node.id in self.entity_info.arg_values: - obj = self.entity_info.arg_values[node.id] - anno.setanno(node, 'live_val', obj) - anno.setanno(node, 'fqn', (obj.__class__.__name__,)) + if has_single_def: + def_, = defs + if def_.param_of is self.enclosing_entities[0]: + if node.id in self.entity_info.arg_values: + obj = self.entity_info.arg_values[node.id] + anno.setanno(node, 'live_val', obj) + anno.setanno(node, 'fqn', (obj.__class__.__name__,)) return node def visit_Attribute(self, node): diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py index 38af792777..fe3051179c 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py @@ -21,11 +21,13 @@ from __future__ import print_function import six from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import cfg from tensorflow.contrib.autograph.pyct import parser from tensorflow.contrib.autograph.pyct import qual_names from tensorflow.contrib.autograph.pyct import transformer from tensorflow.contrib.autograph.pyct.static_analysis import activity from tensorflow.contrib.autograph.pyct.static_analysis import live_values +from tensorflow.contrib.autograph.pyct.static_analysis import reaching_definitions from tensorflow.contrib.autograph.pyct.static_analysis import type_info from tensorflow.python.framework import constant_op from tensorflow.python.platform import test @@ -48,7 +50,10 @@ class LiveValuesResolverTest(test.TestCase): arg_types=arg_types, owner_type=None) node = qual_names.resolve(node) + graphs = cfg.build(node) node = activity.resolve(node, entity_info) + node = reaching_definitions.resolve(node, entity_info, graphs, + reaching_definitions.Definition) node = live_values.resolve(node, entity_info, literals) node = type_info.resolve(node, entity_info) node = live_values.resolve(node, entity_info, literals) diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py b/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py index 4d79b0a56a..4ea7fd93cd 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py @@ -152,9 +152,10 @@ class Analyzer(cfg.GraphVisitor): # This Name node below is a literal name, e.g. False # This can also happen if activity.py forgot to annotate the node with a # scope object. - assert isinstance(node.ast_node, - (gast.Name, gast.Break, gast.Continue)), (node.ast_node, - node) + assert isinstance( + node.ast_node, + (gast.Name, gast.Break, gast.Continue, gast.Raise)), (node.ast_node, + node) defs_out = defs_in self.in_[node] = defs_in diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/type_info.py b/tensorflow/contrib/autograph/pyct/static_analysis/type_info.py index a229c288a8..835d5199fa 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/type_info.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/type_info.py @@ -43,9 +43,8 @@ from __future__ import print_function import gast -from tensorflow.contrib.autograph import utils from tensorflow.contrib.autograph.pyct import anno -from tensorflow.contrib.autograph.pyct import parser +from tensorflow.contrib.autograph.pyct import ast_util from tensorflow.contrib.autograph.pyct import transformer from tensorflow.python.util import tf_inspect @@ -166,7 +165,6 @@ class TypeInfoResolver(transformer.Base): definition = self.scope.getval(qn) anno.copyanno(definition, node, 'type') anno.copyanno(definition, node, 'type_fqn') - anno.setanno(node, 'definition', definition) # TODO(mdan): Remove this when the directives module is in. anno.copyanno(definition, node, 'element_type') @@ -198,52 +196,18 @@ class TypeInfoResolver(transformer.Base): def visit_With(self, node): for item in node.items: if item.optional_vars is not None: - self.apply_to_single_assignments((item.optional_vars,), - item.context_expr, - self._process_variable_assignment) + ast_util.apply_to_single_assignments((item.optional_vars,), + item.context_expr, + self._process_variable_assignment) self.generic_visit(node) return node def visit_Assign(self, node): self.generic_visit(node) - self.apply_to_single_assignments( - node.targets, node.value, self._process_variable_assignment) + ast_util.apply_to_single_assignments(node.targets, node.value, + self._process_variable_assignment) return node - # TODO(mdan): Remove as soon as the new directives module is ready. - def visit_Call(self, node): - if anno.hasanno(node.func, 'live_val'): - # Symbols targeted by the "set_type" marker function are assigned the data - # type that it specified. - if anno.getanno(node.func, 'live_val') is utils.set_element_type: - - if len(node.args) < 2 or len(node.args) > 3: - raise ValueError('"%s" must have either two or three parameters' - % self.context.type_annotation_func) - if len(node.args) == 2: - target_arg, type_arg = node.args - shape_arg = parser.parse_expression('None') - else: - target_arg, type_arg, shape_arg = node.args - if not anno.hasanno(target_arg, anno.Basic.QN): - raise ValueError('the first argument of "%s" must by a symbol' % - utils.set_element_type) - # TODO(mdan): This is vulnerable to symbol renaming. - element_type = type_arg - element_shape = shape_arg - - target_symbol = anno.getanno(target_arg, anno.Basic.QN) - # Find the definition of this symbol and annotate it with the given - # data type. That in turn will cause future uses of the symbol - # to receive the same type annotation. - definition = self.scope.getval(target_symbol) - anno.setanno(node, 'element_type', element_type) - anno.setanno(node, 'element_shape', element_shape) - anno.setanno(definition, 'element_type', element_type) - anno.setanno(definition, 'element_shape', element_shape) - # TODO(mdan): Should we update references between definition and here? - return self.generic_visit(node) - def resolve(node, context): return TypeInfoResolver(context).visit(node) diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/type_info_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/type_info_test.py index 32b1148ab2..404311ba24 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/type_info_test.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/type_info_test.py @@ -19,11 +19,13 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.autograph.pyct import anno +from tensorflow.contrib.autograph.pyct import cfg from tensorflow.contrib.autograph.pyct import parser from tensorflow.contrib.autograph.pyct import qual_names from tensorflow.contrib.autograph.pyct import transformer from tensorflow.contrib.autograph.pyct.static_analysis import activity from tensorflow.contrib.autograph.pyct.static_analysis import live_values +from tensorflow.contrib.autograph.pyct.static_analysis import reaching_definitions from tensorflow.contrib.autograph.pyct.static_analysis import type_info from tensorflow.python.client import session from tensorflow.python.platform import test @@ -69,7 +71,10 @@ class TypeInfoResolverTest(test.TestCase): arg_types=arg_types, owner_type=None) node = qual_names.resolve(node) + graphs = cfg.build(node) node = activity.resolve(node, entity_info) + node = reaching_definitions.resolve(node, entity_info, graphs, + reaching_definitions.Definition) node = live_values.resolve(node, entity_info, {}) node = type_info.resolve(node, entity_info) node = live_values.resolve(node, entity_info, {}) diff --git a/tensorflow/contrib/autograph/utils/BUILD b/tensorflow/contrib/autograph/utils/BUILD index d82c17bf2a..d2b399f19b 100644 --- a/tensorflow/contrib/autograph/utils/BUILD +++ b/tensorflow/contrib/autograph/utils/BUILD @@ -28,7 +28,6 @@ py_library( "tensor_list.py", "testing.py", "type_check.py", - "type_hints.py", ], srcs_version = "PY2AND3", visibility = ["//tensorflow:__subpackages__"], diff --git a/tensorflow/contrib/autograph/utils/__init__.py b/tensorflow/contrib/autograph/utils/__init__.py index 817d4126d1..57b5f74741 100644 --- a/tensorflow/contrib/autograph/utils/__init__.py +++ b/tensorflow/contrib/autograph/utils/__init__.py @@ -30,4 +30,3 @@ from tensorflow.contrib.autograph.utils.py_func import wrap_py_func from tensorflow.contrib.autograph.utils.tensor_list import dynamic_list_append from tensorflow.contrib.autograph.utils.testing import fake_tf from tensorflow.contrib.autograph.utils.type_check import is_tensor -from tensorflow.contrib.autograph.utils.type_hints import set_element_type diff --git a/tensorflow/contrib/autograph/utils/type_hints.py b/tensorflow/contrib/autograph/utils/type_hints.py deleted file mode 100644 index aeb9e54561..0000000000 --- a/tensorflow/contrib/autograph/utils/type_hints.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""No-op utilities that provide static type hints. - -These are used when the data type is not known at creation, for instance in the -case of empty lists. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -def set_element_type(entity, dtype, shape=None): - """Indicates that the entity is expected hold items of specified type. - - This function is a no-op. Its presence merely marks the data type of its - argument. The staged TensorFlow ops will reflect and assert this data type. - - Args: - entity: A Tensor or TensorArray. - dtype: TensorFlow dtype value to assert for entity. - shape: Optional shape to assert for entity. - Returns: - The value of entity, unchanged. - """ - del dtype - del shape - return entity -- cgit v1.2.3 From 7ff7013598717a4c21034fdfb30462442a0888f8 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Fri, 13 Jul 2018 21:19:01 -0700 Subject: Add all keep nodes to output lists --- tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc index ec9dbfa13b..84eb8aebe9 100644 --- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc +++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc @@ -232,8 +232,18 @@ tensorflow::Status TRTOptimizationPass::Optimize( tensorflow::grappler::GraphProperties static_graph_properties(item); TF_RETURN_IF_ERROR(static_graph_properties.InferStatically(true)); tensorflow::tensorrt::convert::ConversionParams cp; + + std::vector nodes_to_preserve; + for (const auto& n : item.NodesToPreserve()) { + auto tokens = str_util::Split(n, ":"); + string s = tokens.at(0); + for (size_t t = 1; t < tokens.size() - 1; ++t) { + StrAppend(&s, ":", tokens.at(t)); + } + nodes_to_preserve.push_back(s); + } cp.input_graph_def = &item.graph; - cp.output_names = &item.fetch; + cp.output_names = &nodes_to_preserve; cp.max_batch_size = maximum_batch_size_; cp.max_workspace_size_bytes = maximum_workspace_size_; cp.output_graph_def = optimized_graph; -- cgit v1.2.3 From 2de246accd1f7b8bc02dc5895aa9eff88ef7f6c6 Mon Sep 17 00:00:00 2001 From: Rodrigo Silveira Date: Fri, 13 Jul 2018 22:20:01 -0600 Subject: Fix typo in bounding box example --- tensorflow/core/api_def/base_api/api_def_DrawBoundingBoxes.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/api_def/base_api/api_def_DrawBoundingBoxes.pbtxt b/tensorflow/core/api_def/base_api/api_def_DrawBoundingBoxes.pbtxt index 6c3ae09f5d..35c916e269 100644 --- a/tensorflow/core/api_def/base_api/api_def_DrawBoundingBoxes.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_DrawBoundingBoxes.pbtxt @@ -30,7 +30,7 @@ height of the underlying image. For example, if an image is 100 x 200 pixels (height x width) and the bounding box is `[0.1, 0.2, 0.5, 0.9]`, the upper-left and bottom-right coordinates of -the bounding box will be `(40, 10)` to `(100, 50)` (in (x,y) coordinates). +the bounding box will be `(40, 10)` to `(180, 50)` (in (x,y) coordinates). Parts of the bounding box may fall outside the image. END -- cgit v1.2.3 From 88b656acd480f6956894e3bb8c8f0c52fe033bc4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jul 2018 21:31:49 -0700 Subject: Add wrapper to help export model trained with estimator as SavedModel for TPU. PiperOrigin-RevId: 204568222 --- tensorflow/contrib/tpu/__init__.py | 2 +- tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 44 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/tpu/__init__.py b/tensorflow/contrib/tpu/__init__.py index d62338680e..d5484e9032 100644 --- a/tensorflow/contrib/tpu/__init__.py +++ b/tensorflow/contrib/tpu/__init__.py @@ -42,10 +42,10 @@ @@TPUEstimator @@TPUEstimatorSpec +@@export_estimator_savedmodel @@RunConfig @@InputPipelineConfig @@TPUConfig - @@bfloat16_scope """ diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index aa407cf4d8..be6a5dc57d 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -3320,3 +3320,47 @@ def _add_item_to_params(params, key, value): else: # Now params is Python dict. params[key] = value + + +def export_estimator_savedmodel(estimator, + export_dir_base, + serving_input_receiver_fn, + assets_extra=None, + as_text=False, + checkpoint_path=None, + strip_default_attrs=False): + """Export `Estimator` trained model for TPU inference. + + Args: + estimator: `Estimator` with which model has been trained. + export_dir_base: A string containing a directory in which to create + timestamped subdirectories containing exported SavedModels. + serving_input_receiver_fn: A function that takes no argument and + returns a `ServingInputReceiver` or `TensorServingInputReceiver`. + assets_extra: A dict specifying how to populate the assets.extra directory + within the exported SavedModel, or `None` if no extra assets are needed. + as_text: whether to write the SavedModel proto in text format. + checkpoint_path: The checkpoint path to export. If `None` (the default), + the most recent checkpoint found within the model directory is chosen. + strip_default_attrs: Boolean. If `True`, default-valued attributes will be + removed from the NodeDefs. + + Returns: + The string path to the exported directory. + """ + # `TPUEstimator` requires `tpu_config.RunConfig`, so we cannot use + # `estimator.config`. + config = tpu_config.RunConfig(model_dir=estimator.model_dir) + est = TPUEstimator( + estimator._model_fn, # pylint: disable=protected-access + config=config, + params=estimator.params, + use_tpu=True, + train_batch_size=2048, # Does not matter. + eval_batch_size=2048, # Does not matter. + ) + return est.export_savedmodel(export_dir_base, serving_input_receiver_fn, + assets_extra, + as_text, + checkpoint_path, + strip_default_attrs) -- cgit v1.2.3 From 511ce2e7eb8f220c443f09382d09d14f0758e8ba Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Fri, 13 Jul 2018 22:35:47 -0700 Subject: Remove unnecessary for loop --- tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc index 84eb8aebe9..5bb0ffc797 100644 --- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc +++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc @@ -236,11 +236,7 @@ tensorflow::Status TRTOptimizationPass::Optimize( std::vector nodes_to_preserve; for (const auto& n : item.NodesToPreserve()) { auto tokens = str_util::Split(n, ":"); - string s = tokens.at(0); - for (size_t t = 1; t < tokens.size() - 1; ++t) { - StrAppend(&s, ":", tokens.at(t)); - } - nodes_to_preserve.push_back(s); + nodes_to_preserve.push_back(tokens.at(0)); } cp.input_graph_def = &item.graph; cp.output_names = &nodes_to_preserve; -- cgit v1.2.3 From 8aa4179ffae2d0a3724a70bea32ce35e2d88751a Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Sat, 14 Jul 2018 00:15:49 -0700 Subject: [Java]: Support ConfigProto and RunOptions when loading SavedModels. Fixes #18143 Fixes #20769 (Similar to #18716 by @raintung) PiperOrigin-RevId: 204575441 --- .../main/java/org/tensorflow/SavedModelBundle.java | 73 +++++++++++++++++++++- .../java/src/main/native/saved_model_bundle_jni.cc | 15 ++++- .../java/src/main/native/saved_model_bundle_jni.h | 4 +- .../java/org/tensorflow/SavedModelBundleTest.java | 54 ++++++++++++++++ 4 files changed, 141 insertions(+), 5 deletions(-) diff --git a/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java b/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java index c8b9126f03..49594e6b47 100644 --- a/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java +++ b/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java @@ -25,18 +25,86 @@ package org.tensorflow; * protocol buffer). */ public class SavedModelBundle implements AutoCloseable { + /** Options for loading a SavedModel. */ + public static final class Loader { + /** Load a SavedModelBundle with the configured options. */ + public SavedModelBundle load() { + return SavedModelBundle.load(exportDir, tags, configProto, runOptions); + } + + /** + * Sets options to use when executing model initialization operations. + * + * @param options Serialized RunOptions + * protocol buffer. + */ + public Loader withRunOptions(byte[] options) { + this.runOptions = options; + return this; + } + + /** + * Set configuration of the Session object created when loading the model. + * + * @param configProto Serialized ConfigProto + * protocol buffer. + */ + public Loader withConfigProto(byte[] configProto) { + this.configProto = configProto; + return this; + } + + /** + * Sets the set of tags that identify the specific graph in the saved model to load. + * + * @param tags the tags identifying the specific MetaGraphDef to load. + */ + public Loader withTags(String... tags) { + this.tags = tags; + return this; + } + + private Loader(String exportDir) { + this.exportDir = exportDir; + } + + private String exportDir = null; + private String[] tags = null; + private byte[] configProto = null; + private byte[] runOptions = null; + } /** * Load a saved model from an export directory. The model that is being loaded should be created * using the Saved Model * API. * + *

    This method is a shorthand for: + * + * 

    {@code
+   * SavedModelBundler.loader().withTags(tags).load();
+   * }
    + * * @param exportDir the directory path containing a saved model. * @param tags the tags identifying the specific metagraphdef to load. * @return a bundle containing the graph and associated session. */ public static SavedModelBundle load(String exportDir, String... tags) { - return load(exportDir, tags, null); + return loader(exportDir).withTags(tags).load(); + } + + /** + * Load a saved model. + * + *

    Returns a Loader object that can set configuration options before actually + * loading the model, + * + * @param exportDir the directory path containing a saved model. + */ + public static Loader loader(String exportDir) { + return new Loader(exportDir); } /** @@ -95,7 +163,8 @@ public class SavedModelBundle implements AutoCloseable { return new SavedModelBundle(graph, session, metaGraphDef); } - private static native SavedModelBundle load(String exportDir, String[] tags, byte[] runOptions); + private static native SavedModelBundle load( + String exportDir, String[] tags, byte[] config, byte[] runOptions); static { TensorFlow.init(); diff --git a/tensorflow/java/src/main/native/saved_model_bundle_jni.cc b/tensorflow/java/src/main/native/saved_model_bundle_jni.cc index de6382a79c..68999fb2da 100644 --- a/tensorflow/java/src/main/native/saved_model_bundle_jni.cc +++ b/tensorflow/java/src/main/native/saved_model_bundle_jni.cc @@ -22,12 +22,25 @@ limitations under the License. JNIEXPORT jobject JNICALL Java_org_tensorflow_SavedModelBundle_load( JNIEnv* env, jclass clazz, jstring export_dir, jobjectArray tags, - jbyteArray run_options) { + jbyteArray config, jbyteArray run_options) { TF_Status* status = TF_NewStatus(); jobject bundle = nullptr; // allocate parameters for TF_LoadSessionFromSavedModel TF_SessionOptions* opts = TF_NewSessionOptions(); + if (config != nullptr) { + size_t sz = env->GetArrayLength(config); + if (sz > 0) { + jbyte* config_data = env->GetByteArrayElements(config, nullptr); + TF_SetConfig(opts, static_cast(config_data), sz, status); + env->ReleaseByteArrayElements(config, config_data, JNI_ABORT); + if (!throwExceptionIfNotOK(env, status)) { + TF_DeleteSessionOptions(opts); + TF_DeleteStatus(status); + return nullptr; + } + } + } TF_Buffer* crun_options = nullptr; if (run_options != nullptr) { size_t sz = env->GetArrayLength(run_options); diff --git a/tensorflow/java/src/main/native/saved_model_bundle_jni.h b/tensorflow/java/src/main/native/saved_model_bundle_jni.h index 6cce6a81bd..a4b05d0409 100644 --- a/tensorflow/java/src/main/native/saved_model_bundle_jni.h +++ b/tensorflow/java/src/main/native/saved_model_bundle_jni.h @@ -26,10 +26,10 @@ extern "C" { * Class: org_tensorflow_SavedModelBundle * Method: load * Signature: - * (Ljava/lang/String;[Ljava/lang/String;[B)Lorg/tensorflow/SavedModelBundle; + * (Ljava/lang/String;[Ljava/lang/String;[B;[B)Lorg/tensorflow/SavedModelBundle; */ JNIEXPORT jobject JNICALL Java_org_tensorflow_SavedModelBundle_load( - JNIEnv *, jclass, jstring, jobjectArray, jbyteArray); + JNIEnv *, jclass, jstring, jobjectArray, jbyteArray, jbyteArray); #ifdef __cplusplus } // extern "C" diff --git a/tensorflow/java/src/test/java/org/tensorflow/SavedModelBundleTest.java b/tensorflow/java/src/test/java/org/tensorflow/SavedModelBundleTest.java index b063b6f1cd..7d936867a7 100644 --- a/tensorflow/java/src/test/java/org/tensorflow/SavedModelBundleTest.java +++ b/tensorflow/java/src/test/java/org/tensorflow/SavedModelBundleTest.java @@ -50,4 +50,58 @@ public class SavedModelBundleTest { assertTrue(e.getMessage().contains("Could not find SavedModel")); } } + + @Test + public void loader() { + try (SavedModelBundle bundle = SavedModelBundle.loader(SAVED_MODEL_PATH) + .withTags("serve") + .withConfigProto(sillyConfigProto()) + .withRunOptions(sillyRunOptions()) + .load()) { + assertNotNull(bundle.session()); + assertNotNull(bundle.graph()); + assertNotNull(bundle.metaGraphDef()); + } + } + + private static byte[] sillyRunOptions() { + // Ideally this would use the generated Java sources for protocol buffers + // and end up with something like the snippet below. However, generating + // the Java files for the .proto files in tensorflow/core:protos_all is + // a bit cumbersome in bazel until the proto_library rule is setup. + // + // See https://github.com/bazelbuild/bazel/issues/52#issuecomment-194341866 + // https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251515362 + // https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251692558 + // + // For this test, for now, the use of specific bytes suffices. + return new byte[] {0x08, 0x03}; + /* + return org.tensorflow.framework.RunOptions.newBuilder() + .setTraceLevel(RunOptions.TraceLevel.FULL_TRACE) + .build() + .toByteArray(); + */ + } + + public static byte[] sillyConfigProto() { + // Ideally this would use the generated Java sources for protocol buffers + // and end up with something like the snippet below. However, generating + // the Java files for the .proto files in tensorflow/core:protos_all is + // a bit cumbersome in bazel until the proto_library rule is setup. + // + // See https://github.com/bazelbuild/bazel/issues/52#issuecomment-194341866 + // https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251515362 + // https://github.com/bazelbuild/rules_go/pull/121#issuecomment-251692558 + // + // For this test, for now, the use of specific bytes suffices. + return new byte[] {0x10, 0x01, 0x28, 0x01}; + /* + return org.tensorflow.framework.ConfigProto.newBuilder() + .setInterOpParallelismThreads(1) + .setIntraOpParallelismThreads(1) + .build() + .toByteArray(); + */ + } } -- cgit v1.2.3 From e21702f871fe849b09e726f9d42eff0e6d5a44d8 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Sat, 14 Jul 2018 01:37:39 -0700 Subject: Disable broken model_analyzer_test PiperOrigin-RevId: 204579728 --- tensorflow/python/profiler/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/profiler/BUILD b/tensorflow/python/profiler/BUILD index 0654104a34..52f6f248a3 100644 --- a/tensorflow/python/profiler/BUILD +++ b/tensorflow/python/profiler/BUILD @@ -58,6 +58,7 @@ cuda_py_test( "//tensorflow/python:variables", ], tags = [ + "no_gpu", "no_pip", "oss_serial", ], -- cgit v1.2.3 From ee9a16b2032c8cb96180b0e81fbae3076b54a883 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Sat, 14 Jul 2018 02:06:06 -0700 Subject: [XLA:AMDGPU] Enable the AMDGPU backend for open source builds Merge pull request #20749 PiperOrigin-RevId: 204581011 --- .../xla/service/gpu/llvm_gpu_backend/BUILD | 1 + third_party/llvm/llvm.autogenerated.BUILD | 26 ++++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD index 7de8f9e1ee..da31c65b7e 100644 --- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD +++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD @@ -34,6 +34,7 @@ cc_library( "//tensorflow/compiler/xla/service/llvm_ir:llvm_util", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", + "@llvm//:amdgpu_code_gen", "@llvm//:analysis", "@llvm//:bit_reader", "@llvm//:bit_writer", diff --git a/third_party/llvm/llvm.autogenerated.BUILD b/third_party/llvm/llvm.autogenerated.BUILD index bf9f9ca9cf..c3b9ec4c25 100644 --- a/third_party/llvm/llvm.autogenerated.BUILD +++ b/third_party/llvm/llvm.autogenerated.BUILD @@ -28,9 +28,7 @@ llvm_host_triple = "x86_64-unknown-linux_gnu" llvm_targets = [ "AArch64", - # Uncomment to enable the AMDGPU backend. - # TODO(phawkins): use a configure-time test. - # "AMDGPU", + "AMDGPU", "ARM", "NVPTX", "PowerPC", @@ -256,13 +254,31 @@ llvm_target_list = [ ("-gen-dag-isel", "lib/Target/AMDGPU/AMDGPUGenDAGISel.inc"), ("-gen-callingconv", "lib/Target/AMDGPU/AMDGPUGenCallingConv.inc"), ("-gen-subtarget", "lib/Target/AMDGPU/AMDGPUGenSubtargetInfo.inc"), - ("-gen-tgt-intrinsic", "lib/Target/AMDGPU/AMDGPUGenIntrinsics.inc"), + ("-gen-tgt-intrinsic-impl", "lib/Target/AMDGPU/AMDGPUGenIntrinsicImpl.inc"), + ("-gen-tgt-intrinsic-enums", "lib/Target/AMDGPU/AMDGPUGenIntrinsicEnums.inc"), ("-gen-emitter", "lib/Target/AMDGPU/AMDGPUGenMCCodeEmitter.inc"), ("-gen-dfa-packetizer", "lib/Target/AMDGPU/AMDGPUGenDFAPacketizer.inc"), ("-gen-asm-writer", "lib/Target/AMDGPU/AMDGPUGenAsmWriter.inc"), ("-gen-asm-matcher", "lib/Target/AMDGPU/AMDGPUGenAsmMatcher.inc"), ("-gen-disassembler", "lib/Target/AMDGPU/AMDGPUGenDisassemblerTables.inc"), ("-gen-pseudo-lowering", "lib/Target/AMDGPU/AMDGPUGenMCPseudoLowering.inc"), + ("-gen-searchable-tables", "lib/Target/AMDGPU/AMDGPUGenSearchableTables.inc"), + ("-gen-global-isel", "lib/Target/AMDGPU/AMDGPUGenGlobalISel.inc"), + ], + }, + { + "name": "AMDGPU", + "lower_name": "amdgpu_r600", + "short_name": "R600", + "tbl_outs": [ + ("-gen-asm-writer", "lib/Target/AMDGPU/R600GenAsmWriter.inc"), + ("-gen-callingconv", "lib/Target/AMDGPU/R600GenCallingConv.inc"), + ("-gen-dag-isel", "lib/Target/AMDGPU/R600GenDAGISel.inc"), + ("-gen-dfa-packetizer", "lib/Target/AMDGPU/R600GenDFAPacketizer.inc"), + ("-gen-instr-info", "lib/Target/AMDGPU/R600GenInstrInfo.inc"), + ("-gen-emitter", "lib/Target/AMDGPU/R600GenMCCodeEmitter.inc"), + ("-gen-register-info", "lib/Target/AMDGPU/R600GenRegisterInfo.inc"), + ("-gen-subtarget", "lib/Target/AMDGPU/R600GenSubtargetInfo.inc"), ], }, { @@ -670,6 +686,7 @@ cc_library( ]), copts = llvm_copts + ["-Iexternal/llvm/lib/Target/AMDGPU"], deps = [ + ":amdgpu_r600_target_gen", ":amdgpu_target_gen", ":config", ":core", @@ -692,6 +709,7 @@ cc_library( ]), copts = llvm_copts + ["-Iexternal/llvm/lib/Target/AMDGPU"], deps = [ + ":amdgpu_r600_target_gen", ":amdgpu_target_gen", ":config", ":core", -- cgit v1.2.3 From 85aae3795775bf648d2e8baa56331f952d12e3e0 Mon Sep 17 00:00:00 2001 From: Toby Boyd Date: Sat, 14 Jul 2018 08:47:49 -0700 Subject: Link NCCL 2.x lib and header where ./configure expects them. PiperOrigin-RevId: 204596391 --- tensorflow/tools/docker/Dockerfile.devel-gpu | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu index 2818b822b8..44120bf274 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu @@ -35,6 +35,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ find /usr/local/cuda-9.0/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \ rm /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a +# Link NCCL libray and header where the build script expects them. +RUN mkdir /usr/local/cuda-9.0/lib && \ + ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \ + ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h + +# TODO(tobyboyd): Remove after license is excluded from BUILD file. +RUN gunzip /usr/share/doc/libnccl2/NCCL-SLA.txt.gz && \ + cp /usr/share/doc/libnccl2/NCCL-SLA.txt /usr/local/cuda/ + RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py && \ rm get-pip.py @@ -93,10 +102,13 @@ RUN git clone --branch=r1.9 --depth=1 https://github.com/tensorflow/tensorflow.g ENV CI_BUILD_PYTHON python ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH ENV TF_NEED_CUDA 1 -ENV TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1 +ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0 ENV TF_CUDA_VERSION=9.0 ENV TF_CUDNN_VERSION=7 +# NCCL 2.x +ENV TF_NCCL_VERSION=2 + RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \ tensorflow/tools/ci_build/builds/configured GPU \ -- cgit v1.2.3 From fe7d1d9447a31562acb26aad7a9ffca60686c38a Mon Sep 17 00:00:00 2001 From: Smit Hinsu Date: Sat, 14 Jul 2018 13:16:58 -0700 Subject: Update default cuda compute capability while installing from sources to cover up to Volta PiperOrigin-RevId: 204606836 --- configure.py | 2 +- tensorflow/docs_src/install/install_sources.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.py b/configure.py index d411214817..df6259778e 100644 --- a/configure.py +++ b/configure.py @@ -36,7 +36,7 @@ except ImportError: _DEFAULT_CUDA_VERSION = '9.0' _DEFAULT_CUDNN_VERSION = '7' _DEFAULT_NCCL_VERSION = '2.2' -_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,5.2' +_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0' _DEFAULT_CUDA_PATH = '/usr/local/cuda' _DEFAULT_CUDA_PATH_LINUX = '/opt/cuda' _DEFAULT_CUDA_PATH_WIN = ('C:/Program Files/NVIDIA GPU Computing ' diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index fc1f6d05bd..5caf36eed1 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -262,7 +262,7 @@ Please specify the location where cuDNN 7 library is installed. Refer to README. Please specify a list of comma-separated CUDA compute capabilities you want to build with. You can find the compute capability of your device at: https://developer.nvidia.com/cuda-gpus. Please note that each additional compute capability significantly increases your build time and binary size. -[Default is: "3.5,5.2"]: 3.0 +[Default is: "3.5,7.0"]: 6.0,7.0 Do you wish to build TensorFlow with MPI support? [y/N] MPI support will not be enabled for TensorFlow Configuration finished -- cgit v1.2.3 From a398d1c1a47ad58ab642dc70d1d281f5b77b07de Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 14 Jul 2018 23:22:31 +0000 Subject: Fix bug inside boston.py in boosted_trees with python 3 This fix tries to address the issue raised in 20776 where run the sample with boston.py in boosted_trees throws out error with python 3: ``` $ python3 boston.py \ > --batch_size=404 --output_dir="/tmp/boston" --depth=4 --learning_rate=0.1 \ > --num_eval_steps=1 --num_trees=500 --l2=0.001 \ > --vmodule=training_ops=1 ...... ...... Traceback (most recent call last): File "boston.py", line 169, in tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) ...... ...... File "/usr/lib/python3.5/genericpath.py", line 145, in _check_arg_types raise TypeError("Can't mix strings and bytes in path components") from None TypeError: Can't mix strings and bytes in path components ``` The reason for the error was because in python 3, the export_dir was returned as bytes. This fix as the `compat.as_bytes` which is consistent with other places in tensorflow. This fix fixes 20776 Signed-off-by: Yong Tang --- tensorflow/contrib/boosted_trees/examples/boston.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/boosted_trees/examples/boston.py b/tensorflow/contrib/boosted_trees/examples/boston.py index e9dbdb0fd7..babf1af1de 100644 --- a/tensorflow/contrib/boosted_trees/examples/boston.py +++ b/tensorflow/contrib/boosted_trees/examples/boston.py @@ -45,6 +45,7 @@ from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientB from tensorflow.contrib.boosted_trees.proto import learner_pb2 from tensorflow.contrib.layers.python.layers import feature_column from tensorflow.contrib.learn import learn_runner +from tensorflow.python.util import compat _BOSTON_NUM_FEATURES = 13 @@ -79,7 +80,7 @@ def _convert_fn(dtec, sorted_feature_names, num_dense, num_sparse_float, num_sparse_int, export_dir, unused_eval_result): universal_format = custom_export_strategy.convert_to_universal_format( dtec, sorted_feature_names, num_dense, num_sparse_float, num_sparse_int) - with tf.gfile.GFile(os.path.join(export_dir, "tree_proto"), "w") as f: + with tf.gfile.GFile(os.path.join(compat.as_bytes(export_dir), compat.as_bytes("tree_proto")), "w") as f: f.write(str(universal_format)) -- cgit v1.2.3 From 43d622778f2cc9d089e864d6feca9125d2803783 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 14 Jul 2018 23:30:41 +0000 Subject: Fix bug in custom_export_strategy.py for python 3 by add compat.as_bytes Signed-off-by: Yong Tang --- .../contrib/boosted_trees/estimator_batch/custom_export_strategy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py index 62f1f4122b..0cbd8f6501 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py @@ -32,6 +32,7 @@ from tensorflow.python.framework import ops from tensorflow.python.platform import gfile from tensorflow.python.saved_model import loader as saved_model_loader from tensorflow.python.saved_model import tag_constants +from tensorflow.python.util import compat _SPARSE_FLOAT_FEATURE_NAME_TEMPLATE = "%s_%d" @@ -88,9 +89,9 @@ def make_custom_export_strategy(name, len(sparse_float_indices), len(sparse_int_indices)) sorted_by_importance = sorted( feature_importances.items(), key=lambda x: -x[1]) - assets_dir = os.path.join(result_dir, "assets.extra") + assets_dir = os.path.join(compat.as_bytes(result_dir), compat.as_bytes("assets.extra")) gfile.MakeDirs(assets_dir) - with gfile.GFile(os.path.join(assets_dir, "feature_importances"), + with gfile.GFile(os.path.join(compat.as_bytes(assets_dir), compat.as_bytes("feature_importances")), "w") as f: f.write("\n".join("%s, %f" % (k, v) for k, v in sorted_by_importance)) return result_dir -- cgit v1.2.3 From 08921095829788973d08848ebdc4d6b082eeebbe Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 14 Jul 2018 23:34:36 +0000 Subject: Pylint fix Signed-off-by: Yong Tang --- .../boosted_trees/estimator_batch/custom_export_strategy.py | 8 +++++--- tensorflow/contrib/boosted_trees/examples/boston.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py index 0cbd8f6501..78232fa0a6 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py @@ -89,10 +89,12 @@ def make_custom_export_strategy(name, len(sparse_float_indices), len(sparse_int_indices)) sorted_by_importance = sorted( feature_importances.items(), key=lambda x: -x[1]) - assets_dir = os.path.join(compat.as_bytes(result_dir), compat.as_bytes("assets.extra")) + assets_dir = os.path.join( + compat.as_bytes(result_dir), compat.as_bytes("assets.extra")) gfile.MakeDirs(assets_dir) - with gfile.GFile(os.path.join(compat.as_bytes(assets_dir), compat.as_bytes("feature_importances")), - "w") as f: + with gfile.GFile(os.path.join( + compat.as_bytes(assets_dir), + compat.as_bytes("feature_importances")), "w") as f: f.write("\n".join("%s, %f" % (k, v) for k, v in sorted_by_importance)) return result_dir diff --git a/tensorflow/contrib/boosted_trees/examples/boston.py b/tensorflow/contrib/boosted_trees/examples/boston.py index babf1af1de..54c4ff059e 100644 --- a/tensorflow/contrib/boosted_trees/examples/boston.py +++ b/tensorflow/contrib/boosted_trees/examples/boston.py @@ -80,7 +80,8 @@ def _convert_fn(dtec, sorted_feature_names, num_dense, num_sparse_float, num_sparse_int, export_dir, unused_eval_result): universal_format = custom_export_strategy.convert_to_universal_format( dtec, sorted_feature_names, num_dense, num_sparse_float, num_sparse_int) - with tf.gfile.GFile(os.path.join(compat.as_bytes(export_dir), compat.as_bytes("tree_proto")), "w") as f: + with tf.gfile.GFile(os.path.join( + compat.as_bytes(export_dir), compat.as_bytes("tree_proto")), "w") as f: f.write(str(universal_format)) -- cgit v1.2.3 From 081a8a19531898dbcf2f243e55afc585ed950379 Mon Sep 17 00:00:00 2001 From: Nafis Sadat Date: Sat, 14 Jul 2018 19:57:55 -0700 Subject: Adding MKL DNN license from Intel's repo: https://github.com/intel/mkl-dnn/blob/master/LICENSE --- third_party/mkl_dnn/LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 third_party/mkl_dnn/LICENSE diff --git a/third_party/mkl_dnn/LICENSE b/third_party/mkl_dnn/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/third_party/mkl_dnn/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -- cgit v1.2.3 From 94b19f2a168c9cbd6e577a73580495a202738f9b Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 15 Jul 2018 16:10:58 +0000 Subject: Use FastBoundsCheck in ArgMax kernel op This fix updates ArgMax kernel implementation to use FastBoundsCheck for improved performance, and keep consistency with other places in tf. Signed-off-by: Yong Tang --- tensorflow/core/kernels/argmax_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/argmax_op.cc b/tensorflow/core/kernels/argmax_op.cc index 49cd997fed..adc573e40c 100644 --- a/tensorflow/core/kernels/argmax_op.cc +++ b/tensorflow/core/kernels/argmax_op.cc @@ -59,7 +59,7 @@ class ArgOp : public OpKernel { int axis = dim < 0 ? dim + input_dims : dim; - OP_REQUIRES(context, axis >= 0 && axis < input_dims, + OP_REQUIRES(context, FastBoundsCheck(axis, input_dims), errors::InvalidArgument("Expected dimension in the range [", -input_dims, ", ", input_dims, "), but got ", dim)); -- cgit v1.2.3 From e6ce9ea5a156873c5b927e99d8935e32122538b9 Mon Sep 17 00:00:00 2001 From: Pavithra Vijay Date: Sun, 15 Jul 2018 11:28:49 -0700 Subject: Partial update of tf.keras to the Keras 2.2.0 API. Changes included are: - Embedding visualization is added to TensorBoard callback (from older Keras API.) - Fix: learning phase info being left out in multi-input models (from older Keras API.) - Fix: Tensorboard callback only supports logging Embeddings layer weights - Fix: Tensorboard callback with layer with multiple outputs PiperOrigin-RevId: 204659796 --- tensorflow/python/keras/callbacks.py | 137 ++++++++++++++++++++- .../tensorflow.keras.callbacks.-tensor-board.pbtxt | 2 +- 2 files changed, 135 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py index 53d907a2cc..0857a3279f 100644 --- a/tensorflow/python/keras/callbacks.py +++ b/tensorflow/python/keras/callbacks.py @@ -31,11 +31,16 @@ import time import numpy as np import six +from tensorflow.python.framework import dtypes from tensorflow.python.keras import backend as K +from tensorflow.python.keras.engine.training_utils import standardize_input_data from tensorflow.python.keras.utils.generic_utils import Progbar from tensorflow.python.ops import array_ops +from tensorflow.python.ops import state_ops +from tensorflow.python.ops import variables from tensorflow.python.platform import tf_logging as logging from tensorflow.python.summary import summary as tf_summary +from tensorflow.python.training import saver from tensorflow.python.util.tf_export import tf_export @@ -697,7 +702,9 @@ class TensorBoard(Callback): write_images: whether to write model weights to visualize as image in TensorBoard. embeddings_freq: frequency (in epochs) at which selected embedding - layers will be saved. + layers will be saved. If set to 0, embeddings won't be computed. + Data to be visualized in TensorBoard's Embedding tab must be passed + as `embeddings_data`. embeddings_layer_names: a list of names of layers to keep eye on. If None or empty list all the embedding layer will be watched. embeddings_metadata: a dictionary which maps layer name to a file name @@ -705,6 +712,10 @@ class TensorBoard(Callback): [details](https://www.tensorflow.org/how_tos/embedding_viz/#metadata_optional) about metadata files format. In case if the same metadata file is used for all embedding layers, string can be passed. + embeddings_data: data to be embedded at layers specified in + `embeddings_layer_names`. Numpy array (if the model has a single + input) or list of Numpy arrays (if the model has multiple inputs). + Learn [more about embeddings](https://www.tensorflow.org/programmers_guide/embedding) """ # pylint: enable=line-too-long @@ -715,7 +726,11 @@ class TensorBoard(Callback): batch_size=32, write_graph=True, write_grads=False, - write_images=False): + write_images=False, + embeddings_freq=0, + embeddings_layer_names=None, + embeddings_metadata=None, + embeddings_data=None): super(TensorBoard, self).__init__() self.log_dir = log_dir self.histogram_freq = histogram_freq @@ -727,6 +742,10 @@ class TensorBoard(Callback): self._current_batch = 0 # abstracted writer class to be able to stub for testing self._writer_class = tf_summary.FileWriter + self.embeddings_freq = embeddings_freq + self.embeddings_layer_names = embeddings_layer_names + self.embeddings_metadata = embeddings_metadata + self.embeddings_data = embeddings_data def set_model(self, model): """Sets Keras model and creates summary ops.""" @@ -778,7 +797,11 @@ class TensorBoard(Callback): tf_summary.histogram('{}_grad'.format(mapped_weight_name), grads) if hasattr(layer, 'output'): - tf_summary.histogram('{}_out'.format(layer.name), layer.output) + if isinstance(layer.output, list): + for i, output in enumerate(layer.output): + tf_summary.histogram('{}_out_{}'.format(layer.name, i), output) + else: + tf_summary.histogram('{}_out'.format(layer.name), layer.output) self.merged = tf_summary.merge_all() if self.write_graph: @@ -786,6 +809,74 @@ class TensorBoard(Callback): else: self.writer = self._writer_class(self.log_dir) + # If both embedding_freq and embeddings_data are available, we will + # visualize embeddings. + if self.embeddings_freq and self.embeddings_data is not None: + self.embeddings_data = standardize_input_data(self.embeddings_data, + model.input_names) + + # If embedding_layer_names are not provided, get all of the embedding + # layers from the model. + embeddings_layer_names = self.embeddings_layer_names + if not embeddings_layer_names: + embeddings_layer_names = [ + layer.name + for layer in self.model.layers + if type(layer).__name__ == 'Embedding' + ] + + self.assign_embeddings = [] + embeddings_vars = {} + + self.batch_id = batch_id = array_ops.placeholder(dtypes.int32) + self.step = step = array_ops.placeholder(dtypes.int32) + + for layer in self.model.layers: + if layer.name in embeddings_layer_names: + embedding_input = self.model.get_layer(layer.name).output + embedding_size = np.prod(embedding_input.shape[1:]) + embedding_input = array_ops.reshape(embedding_input, + (step, int(embedding_size))) + shape = (self.embeddings_data[0].shape[0], int(embedding_size)) + embedding = variables.Variable( + array_ops.zeros(shape), name=layer.name + '_embedding') + embeddings_vars[layer.name] = embedding + batch = state_ops.assign(embedding[batch_id:batch_id + step], + embedding_input) + self.assign_embeddings.append(batch) + + self.saver = saver.Saver(list(embeddings_vars.values())) + + # Create embeddings_metadata dictionary + if isinstance(self.embeddings_metadata, str): + embeddings_metadata = { + layer_name: self.embeddings_metadata + for layer_name in embeddings_vars.keys() + } + else: + # If embedding_metadata is already a dictionary + embeddings_metadata = self.embeddings_metadata + + try: + from tensorboard.plugins import projector + except ImportError: + raise ImportError('Failed to import TensorBoard. Please make sure that ' + 'TensorBoard integration is complete."') + + # TODO(psv): Add integration tests to test embedding visualization + # with TensorBoard callback. We are unable to write a unit test for this + # because TensorBoard dependency assumes TensorFlow package is installed. + config = projector.ProjectorConfig() + for layer_name, tensor in embeddings_vars.items(): + embedding = config.embeddings.add() + embedding.tensor_name = tensor.name + + if (embeddings_metadata is not None and + layer_name in embeddings_metadata): + embedding.metadata_path = embeddings_metadata[layer_name] + + projector.visualize_embeddings(self.writer, config) + def _fetch_callback(self, summary): self.writer.add_summary( summary, @@ -833,6 +924,46 @@ class TensorBoard(Callback): if self.merged in self.model.test_function.fetch_callbacks: self.model.test_function.fetch_callbacks.pop(self.merged) + if self.embeddings_data is None and self.embeddings_freq: + raise ValueError('To visualize embeddings, embeddings_data must ' + 'be provided.') + + if self.embeddings_freq and self.embeddings_data is not None: + if epoch % self.embeddings_freq == 0: + # We need a second forward-pass here because we're passing + # the `embeddings_data` explicitly. This design allows to pass + # arbitrary data as `embeddings_data` and results from the fact + # that we need to know the size of the `tf.Variable`s which + # hold the embeddings in `set_model`. At this point, however, + # the `validation_data` is not yet set. + + embeddings_data = self.embeddings_data + n_samples = embeddings_data[0].shape[0] + i = 0 + while i < n_samples: + step = min(self.batch_size, n_samples - i) + batch = slice(i, i + step) + + if isinstance(self.model.input, list): + feed_dict = { + model_input: embeddings_data[idx][batch] + for idx, model_input in enumerate(self.model.input) + } + else: + feed_dict = {self.model.input: embeddings_data[0][batch]} + + feed_dict.update({self.batch_id: i, self.step: step}) + + if self.model.uses_learning_phase: + feed_dict[K.learning_phase()] = False + + self.sess.run(self.assign_embeddings, feed_dict=feed_dict) + self.saver.save(self.sess, + os.path.join(self.log_dir, 'keras_embedding.ckpt'), + epoch) + + i += self.batch_size + for name, value in logs.items(): if name in ['batch', 'size']: continue diff --git a/tensorflow/tools/api/golden/tensorflow.keras.callbacks.-tensor-board.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.callbacks.-tensor-board.pbtxt index 2f52464315..e58ba18c1c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.callbacks.-tensor-board.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.callbacks.-tensor-board.pbtxt @@ -5,7 +5,7 @@ tf_class { is_instance: "" member_method { name: "__init__" - argspec: "args=[\'self\', \'log_dir\', \'histogram_freq\', \'batch_size\', \'write_graph\', \'write_grads\', \'write_images\'], varargs=None, keywords=None, defaults=[\'./logs\', \'0\', \'32\', \'True\', \'False\', \'False\'], " + argspec: "args=[\'self\', \'log_dir\', \'histogram_freq\', \'batch_size\', \'write_graph\', \'write_grads\', \'write_images\', \'embeddings_freq\', \'embeddings_layer_names\', \'embeddings_metadata\', \'embeddings_data\'], varargs=None, keywords=None, defaults=[\'./logs\', \'0\', \'32\', \'True\', \'False\', \'False\', \'0\', \'None\', \'None\', \'None\'], " } member_method { name: "on_batch_begin" -- cgit v1.2.3 From e5945c00148186808e337b4946cf0fa6460f6803 Mon Sep 17 00:00:00 2001 From: Max Galkin Date: Sun, 15 Jul 2018 13:11:22 -0700 Subject: Comment cleanup. PiperOrigin-RevId: 204663736 --- tensorflow/python/ops/control_flow_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py index 04545cceb7..888075ba2e 100644 --- a/tensorflow/python/ops/control_flow_ops.py +++ b/tensorflow/python/ops/control_flow_ops.py @@ -3146,7 +3146,7 @@ def while_loop(cond, happen is that the thread updating `x` can never get ahead of the counter thread because the thread incrementing `x` depends on the value of the counter. - + ```python import tensorflow as tf -- cgit v1.2.3 From 006b8faeb79c8b9329bd600390dbda888e9df226 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Sun, 15 Jul 2018 23:36:37 +0200 Subject: [tfgan] Fix assertion in regularization unittest `self.assertTrue(3.0)` always evaluates to `True`. This PR fixes assertion so it will correctly check the loss value. --- tensorflow/contrib/gan/python/train_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/gan/python/train_test.py b/tensorflow/contrib/gan/python/train_test.py index 3ebbe55d05..e905a911cf 100644 --- a/tensorflow/contrib/gan/python/train_test.py +++ b/tensorflow/contrib/gan/python/train_test.py @@ -447,8 +447,8 @@ class GANLossTest(test.TestCase): reg_loss_gen_np = reg_loss.generator_loss.eval() reg_loss_dis_np = reg_loss.discriminator_loss.eval() - self.assertTrue(3.0, reg_loss_gen_np - no_reg_loss_gen_np) - self.assertTrue(3.0, reg_loss_dis_np - no_reg_loss_dis_np) + self.assertEqual(3.0, reg_loss_gen_np - no_reg_loss_gen_np) + self.assertEqual(2.0, reg_loss_dis_np - no_reg_loss_dis_np) def test_regularization_gan(self): self._test_regularization_helper(get_gan_model) -- cgit v1.2.3 From 6c3c766dcabff3b5fa41dbfd491c9e8062a77b07 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sun, 15 Jul 2018 16:04:03 -0700 Subject: [XLA] Enable the semantic for cross-modeul AllReduce. PiperOrigin-RevId: 204670087 --- .../compiler/xla/client/xla_client/xla_builder.cc | 8 ++-- tensorflow/compiler/xla/service/hlo_instruction.cc | 6 +++ tensorflow/compiler/xla/service/hlo_instruction.h | 3 ++ .../compiler/xla/service/hlo_instructions.cc | 2 - .../xla/service/hlo_module_group_metadata.cc | 47 ++++++++++++++++++++-- .../xla/service/hlo_module_group_metadata.h | 16 ++++++-- .../compiler/xla/service/hlo_module_group_util.cc | 24 ++++++++--- 7 files changed, 86 insertions(+), 20 deletions(-) diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc index aac7df4383..3b4f9e1407 100644 --- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc @@ -1845,10 +1845,6 @@ XlaOp XlaBuilder::CrossReplicaSum( tensorflow::gtl::ArraySlice replica_group_ids, const tensorflow::gtl::optional& channel_id) { return ReportErrorOrReturn([&]() -> StatusOr { - if (channel_id.has_value()) { - return Unimplemented("channel_id is not supported in AllReduce"); - } - HloInstructionProto instr; TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand)); TF_ASSIGN_OR_RETURN( @@ -1858,6 +1854,10 @@ XlaOp XlaBuilder::CrossReplicaSum( instr.add_replica_group_ids(replica_group_id); } + if (channel_id.has_value()) { + instr.set_all_reduce_id(channel_id->handle()); + } + AddCalledComputation(computation, &instr); return AddInstruction(std::move(instr), HloOpcode::kCrossReplicaSum, diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 19bee38790..02139facdb 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -988,6 +988,8 @@ bool HloInstruction::HasSideEffectNoRecurse() const { case HloOpcode::kTrace: case HloOpcode::kHostCompute: return true; + case HloOpcode::kCrossReplicaSum: + return all_reduce_id().has_value(); default: return false; } @@ -1839,6 +1841,10 @@ bool HloInstruction::IsElementwiseImpl( } } +bool HloInstruction::IsCrossModuleAllReduce() const { + return opcode() == HloOpcode::kCrossReplicaSum && all_reduce_id(); +} + string HloInstruction::ToStringWithCanonicalNameMap( const HloPrintOptions& options, CanonicalNameMap* canonical_name_map) const { diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index cbd78fa124..180b2fb359 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -1128,6 +1128,9 @@ class HloInstruction { // Returns true if this instruction is elementwise on all its operands. bool IsElementwise() const; + // Returns true if this is an cross module all-reduce instrucion. + bool IsCrossModuleAllReduce() const; + // Returns true if this elementwise instruction implicitly broadcasts operand // `operand_idx`. // diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc index f333c489ed..702f808449 100644 --- a/tensorflow/compiler/xla/service/hlo_instructions.cc +++ b/tensorflow/compiler/xla/service/hlo_instructions.cc @@ -291,8 +291,6 @@ HloAllReduceInstruction::HloAllReduceInstruction( replica_group_ids_(replica_group_ids.begin(), replica_group_ids.end()), cross_replica_sum_barrier_(barrier.begin(), barrier.end()), all_reduce_id_(all_reduce_id) { - // TODO(b/79737069): Remove the CHECK when supported. - CHECK(!all_reduce_id_); for (auto operand : operands) { AppendOperand(operand); } diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc index 6bcd7b042d..3ffac2f413 100644 --- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc +++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc @@ -75,10 +75,23 @@ Status HloModuleGroupMetadata::Build() { if (tracked == nullptr) { return Status::OK(); } - // Add the parent computation of this channel instruction and its peer - // computation (both must be while computations) as companions. + + std::vector peers; if (IsChannelInstruction(hlo)) { - HloComputation* peer_computation = PeerComputation(hlo); + peers.push_back(PeerComputation(hlo)); + } else if (hlo->IsCrossModuleAllReduce()) { + for (HloInstruction* instr : GetAllReduceGroup(*hlo->all_reduce_id())) { + if (instr == hlo) { + continue; + } + peers.push_back(instr->parent()); + } + } + + // Add the parent computation of this channel (or all-reduce) instruction + // and its peer computation(s) (both must be while computations) as + // companions. + for (HloComputation* peer_computation : peers) { const TrackedInstruction* peer_tracked = GetTrackedInstruction(peer_computation); TF_RET_CHECK(peer_tracked != nullptr) @@ -175,7 +188,8 @@ bool HloModuleGroupMetadata::IsCompanionInstruction(HloInstruction* hlo) const { bool HloModuleGroupMetadata::InstructionCommunicates( HloInstruction* hlo) const { - return IsChannelInstruction(hlo) || IsCompanionInstruction(hlo); + return IsChannelInstruction(hlo) || IsCompanionInstruction(hlo) || + hlo->IsCrossModuleAllReduce(); } const HloModuleGroupMetadata::Channel& HloModuleGroupMetadata::GetChannel( @@ -200,6 +214,13 @@ HloComputation* HloModuleGroupMetadata::PeerComputation( } } +const std::vector& HloModuleGroupMetadata::GetAllReduceGroup( + int64 all_reduce_id) const { + auto it = all_reduce_map_.find(all_reduce_id); + CHECK(it != all_reduce_map_.end()); + return it->second; +} + std::vector HloModuleGroupMetadata::GetCompanionsPath(const HloInstruction* hlo) const { std::vector path; @@ -278,10 +299,27 @@ Status HloModuleGroupMetadata::RecordInstructions() { tracked_instructions_[hlo->to_apply()] = TrackedInstruction(hlo, ComputationKind::kCallFunction); } + + // Group cross module all-reduce instructions by the all_reduce id. + if (hlo->IsCrossModuleAllReduce()) { + TF_RET_CHECK(channel_id_map_.find(*hlo->all_reduce_id()) == + channel_id_map_.end()) + << "all_reduce_id " << *hlo->all_reduce_id() + << " is already used by a send/recv instruction"; + all_reduce_map_[*hlo->all_reduce_id()].push_back(hlo); + max_channel_id_ = std::max(max_channel_id_, *hlo->all_reduce_id()); + return Status::OK(); + } + if (!IsChannelInstruction(hlo)) { return Status::OK(); } + TF_RET_CHECK(all_reduce_map_.find(hlo->channel_id()) == + all_reduce_map_.end()) + << "channel id " << hlo->channel_id() + << " is already used by an all-reduce instruction"; + // Add a new channel if needed. if (channel_id_map_.find(hlo->channel_id()) == channel_id_map_.end()) { channels_.emplace_back(); @@ -324,6 +362,7 @@ Status HloModuleGroupMetadata::RecordInstructions() { } } VLOG(2) << "Created " << channels_.size() << " channels"; + VLOG(2) << "Created " << all_reduce_map_.size() << " all-reduce groups"; return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.h b/tensorflow/compiler/xla/service/hlo_module_group_metadata.h index ffde3a332d..9eea5c6a3d 100644 --- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.h +++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.h @@ -118,13 +118,17 @@ class HloModuleGroupMetadata { // comment above on companion instructions. bool IsCompanionInstruction(HloInstruction* hlo) const; - // Returns true if the instruction is either a channel instruction or a - // companion instruction. + // Returns true if the instruction is either a channel instruction, a + // cross-module all-reduce instruction, or a companion instruction. bool InstructionCommunicates(HloInstruction* hlo) const; // Returns the Channel instance for the given channel id. const Channel& GetChannel(int64 channel_id) const; + // Returns the all-reduce instructions with the same all_reduce_id. + const std::vector& GetAllReduceGroup( + int64 all_reduce_id) const; + // Returns the computation that contains the peer channel instructions for // the given instruction. // @@ -187,13 +191,14 @@ class HloModuleGroupMetadata { // Returns all channels in the module group. const std::vector& channels() const { return channels_; } - // Returns the maximum channel id used in the module group. + // Returns the maximum channel id or all_reduce_id used in the module group. int64 max_channel_id() const { return max_channel_id_; } private: Status Build(); - // Record all channel instructions and While instructions. + // Record all channel instructions, cross-module AllReduce instructions, and + // While/Conditional/Call instructions. Status RecordInstructions(); // Verifies the given HloModules are well-formed and follow the specification, @@ -255,6 +260,9 @@ class HloModuleGroupMetadata { // Map from channel ids to the index in channels_. tensorflow::gtl::FlatMap channel_id_map_; + // Map from all-reduce ids to the all reduce instructions. + tensorflow::gtl::FlatMap> all_reduce_map_; + // The maximum channel id used in the module group. int64 max_channel_id_ = -1; diff --git a/tensorflow/compiler/xla/service/hlo_module_group_util.cc b/tensorflow/compiler/xla/service/hlo_module_group_util.cc index df1d562048..9fd0ade153 100644 --- a/tensorflow/compiler/xla/service/hlo_module_group_util.cc +++ b/tensorflow/compiler/xla/service/hlo_module_group_util.cc @@ -56,12 +56,17 @@ std::vector HloModuleGroupUtil::GlobalPredecessors( }; // If the given instruction is a companion instruction, we need to find the - // predecessors of all of its companion instructions. + // predecessors of all of its companion instructions. If the instruction is an + // all-reduce, we need to find the predecessors of all the peer all-reduce + // instructions. std::vector instruction_group; if (metadata_.IsCompanionInstruction(instruction)) { for (HloInstruction* companion : metadata_.Companions(instruction)) { instruction_group.push_back(companion); } + } else if (instruction->IsCrossModuleAllReduce()) { + instruction_group = + metadata_.GetAllReduceGroup(*instruction->all_reduce_id()); } else { instruction_group.push_back(instruction); } @@ -112,12 +117,17 @@ std::vector HloModuleGroupUtil::GlobalSuccessors( }; // If the given instruction is a companion instruction, we need to find the - // successors of all of its companion instructions. + // successors of all of its companion instructions. If the instruction is an + // all-reduce, we need to find the successors of all its peer all-reduce + // instructions. std::vector instruction_group; if (metadata_.IsCompanionInstruction(instruction)) { for (HloInstruction* companion : metadata_.Companions(instruction)) { instruction_group.push_back(companion); } + } else if (instruction->IsCrossModuleAllReduce()) { + instruction_group = + metadata_.GetAllReduceGroup(*instruction->all_reduce_id()); } else { instruction_group.push_back(instruction); } @@ -170,15 +180,17 @@ Status HloModuleGroupUtil::VisitTopologicalOrder( HloInstruction* hlo = stack.top(); // Find the instruction group of the currently visited instruction. The - // instruction group represents all companion instructions of the - // current instruction, and are considered to be a single entity for the - // purpose of the traversal (i.e., they must always be in the same visit - // state). + // instruction group represents all companion instructions of the current + // instruction, or all the all-reduce instructions that belong to the same + // group, or are considered to be a single entity for the purpose of the + // traversal (i.e., they must always be in the same visit state). std::vector instruction_group; if (metadata_.IsCompanionInstruction(hlo)) { for (HloInstruction* companion : metadata_.Companions(hlo)) { instruction_group.push_back(companion); } + } else if (hlo->IsCrossModuleAllReduce()) { + instruction_group = metadata_.GetAllReduceGroup(*hlo->all_reduce_id()); } else { instruction_group.push_back(hlo); } -- cgit v1.2.3 From 05f41cb39598fcdcd4510cdf1c1bf4b4c9fe4868 Mon Sep 17 00:00:00 2001 From: Jon Perl Date: Sun, 15 Jul 2018 19:57:10 -0400 Subject: Test the timer in InMemoryEvaluatorHook This failing test demonstrates a bug that all iterations are running --- tensorflow/contrib/estimator/python/estimator/hooks_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/contrib/estimator/python/estimator/hooks_test.py b/tensorflow/contrib/estimator/python/estimator/hooks_test.py index 95ae971852..e094dade6a 100644 --- a/tensorflow/contrib/estimator/python/estimator/hooks_test.py +++ b/tensorflow/contrib/estimator/python/estimator/hooks_test.py @@ -102,6 +102,7 @@ class InMemoryEvaluatorHookTest(test.TestCase): self.assertTrue(os.path.isdir(estimator.eval_dir())) step_keyword_to_value = summary_step_keyword_to_value_mapping( estimator.eval_dir()) + # 4.5 = sum(range(10))/10 # before training self.assertEqual(4.5, step_keyword_to_value[0]['mean_of_features']) @@ -110,6 +111,7 @@ class InMemoryEvaluatorHookTest(test.TestCase): self.assertEqual(4.5, step_keyword_to_value[8]['mean_of_features']) # end self.assertEqual(4.5, step_keyword_to_value[10]['mean_of_features']) + self.assertEqual([0, 4, 8, 10], list(step_keyword_to_value.keys())) def test_uses_latest_variable_value(self): -- cgit v1.2.3 From fe666a2bd2d99dfbdfe696e9d0424eb1aa236ece Mon Sep 17 00:00:00 2001 From: Jon Perl Date: Sun, 15 Jul 2018 19:58:24 -0400 Subject: Fix timer in InMemoryEvaluatorHook --- tensorflow/contrib/estimator/python/estimator/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/estimator/python/estimator/hooks.py b/tensorflow/contrib/estimator/python/estimator/hooks.py index ddd6aa442f..caadafdfa6 100644 --- a/tensorflow/contrib/estimator/python/estimator/hooks.py +++ b/tensorflow/contrib/estimator/python/estimator/hooks.py @@ -189,7 +189,7 @@ class InMemoryEvaluatorHook(training.SessionRunHook): init_fn=feed_variables, copy_from_scaffold=self._scaffold) with self._graph.as_default(): - return self._estimator._evaluate_run( + self._estimator._evaluate_run( checkpoint_path=None, scaffold=scaffold, update_op=self._update_op, -- cgit v1.2.3 From eadcdf91aa9e8ba6a196791ee349fd3474ffab76 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sun, 15 Jul 2018 20:04:46 -0700 Subject: add int32 support for sub PiperOrigin-RevId: 204681037 --- .../kernels/internal/optimized/optimized_ops.h | 13 +++++ .../kernels/internal/reference/reference_ops.h | 9 ++-- tensorflow/contrib/lite/kernels/sub.cc | 62 ++++++++++++++-------- tensorflow/contrib/lite/kernels/sub_test.cc | 58 ++++++++++++++++++++ .../lite/testing/generated_examples_zip_test.cc | 1 - 5 files changed, 115 insertions(+), 28 deletions(-) diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h index c857fdf699..2f73036e03 100644 --- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h @@ -3318,6 +3318,19 @@ inline void Sub(const float* input1_data, const Dims<4>& input1_dims, } } +inline void Sub(const int32* input1_data, const Dims<4>& input1_dims, + const int32* input2_data, const Dims<4>& input2_dims, + int32 output_activation_min, int32 output_activation_max, + int32* output_data, const Dims<4>& output_dims) { + gemmlowp::ScopedProfilingLabel label("Sub/int32"); + const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims); + for (int i = 0; i < flat_size; ++i) { + output_data[i] = ActivationFunctionWithMinMax( + input1_data[i] - input2_data[i], output_activation_min, + output_activation_max); + } +} + // TODO(jiawen): We can implement BroadcastSub on buffers of arbitrary // dimensionality if the runtime code does a single loop over one dimension // that handles broadcasting as the base case. The code generator would then diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h index 2d40f1769b..080b4e2d03 100644 --- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h @@ -1666,10 +1666,11 @@ inline void Div(const float* input1_data, const Dims<4>& input1_dims, } } -inline void Sub(const float* input1_data, const Dims<4>& input1_dims, - const float* input2_data, const Dims<4>& input2_dims, - float output_activation_min, float output_activation_max, - float* output_data, const Dims<4>& output_dims) { +template +inline void Sub(const T* input1_data, const Dims<4>& input1_dims, + const T* input2_data, const Dims<4>& input2_dims, + T output_activation_min, T output_activation_max, + T* output_data, const Dims<4>& output_dims) { const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims); for (int i = 0; i < flat_size; ++i) { output_data[i] = ActivationFunctionWithMinMax( diff --git a/tensorflow/contrib/lite/kernels/sub.cc b/tensorflow/contrib/lite/kernels/sub.cc index 1247525d41..541c85f756 100644 --- a/tensorflow/contrib/lite/kernels/sub.cc +++ b/tensorflow/contrib/lite/kernels/sub.cc @@ -78,29 +78,44 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } template -void EvalFloat(TfLiteContext* context, TfLiteNode* node, - TfLiteSubParams* params, const OpData* data, - const TfLiteTensor* input1, const TfLiteTensor* input2, - TfLiteTensor* output) { - float output_activation_min, output_activation_max; - CalculateActivationRange(params->activation, &output_activation_min, - &output_activation_max); -#define TF_LITE_SUB(type, opname) \ - type::opname(GetTensorData(input1), GetTensorDims(input1), \ - GetTensorData(input2), GetTensorDims(input2), \ - output_activation_min, output_activation_max, \ - GetTensorData(output), GetTensorDims(output)) - if (kernel_type == kReference) { - if (data->requires_broadcast) { - TF_LITE_SUB(reference_ops, BroadcastSub); +void EvalSub(TfLiteContext* context, TfLiteNode* node, TfLiteSubParams* params, + const OpData* data, const TfLiteTensor* input1, + const TfLiteTensor* input2, TfLiteTensor* output) { +#define TF_LITE_SUB(type, opname, data_type) \ + data_type output_activation_min, output_activation_max; \ + CalculateActivationRange(params->activation, &output_activation_min, \ + &output_activation_max); \ + type::opname(GetTensorData(input1), GetTensorDims(input1), \ + GetTensorData(input2), GetTensorDims(input2), \ + output_activation_min, output_activation_max, \ + GetTensorData(output), GetTensorDims(output)) + if (output->type == kTfLiteInt32) { + if (kernel_type == kReference) { + if (data->requires_broadcast) { + TF_LITE_SUB(reference_ops, BroadcastSub, int32_t); + } else { + TF_LITE_SUB(reference_ops, Sub, int32_t); + } } else { - TF_LITE_SUB(reference_ops, Sub); + if (data->requires_broadcast) { + TF_LITE_SUB(optimized_ops, BroadcastSub, int32_t); + } else { + TF_LITE_SUB(optimized_ops, Sub, int32_t); + } } - } else { - if (data->requires_broadcast) { - TF_LITE_SUB(optimized_ops, BroadcastSub); + } else if (output->type == kTfLiteFloat32) { + if (kernel_type == kReference) { + if (data->requires_broadcast) { + TF_LITE_SUB(reference_ops, BroadcastSub, float); + } else { + TF_LITE_SUB(reference_ops, Sub, float); + } } else { - TF_LITE_SUB(optimized_ops, Sub); + if (data->requires_broadcast) { + TF_LITE_SUB(optimized_ops, BroadcastSub, float); + } else { + TF_LITE_SUB(optimized_ops, Sub, float); + } } } #undef TF_LITE_SUB @@ -171,14 +186,15 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); - if (output->type == kTfLiteFloat32) { - EvalFloat(context, node, params, data, input1, input2, output); + if (output->type == kTfLiteFloat32 || output->type == kTfLiteInt32) { + EvalSub(context, node, params, data, input1, input2, output); } else if (output->type == kTfLiteUInt8) { EvalQuantized(context, node, params, data, input1, input2, output); } else { context->ReportError( - context, "output type %d is not supported, requires float|uint8 types.", + context, + "output type %d is not supported, requires float|uint8|int32 types.", output->type); return kTfLiteError; } diff --git a/tensorflow/contrib/lite/kernels/sub_test.cc b/tensorflow/contrib/lite/kernels/sub_test.cc index ff07aeec49..5978c574d3 100644 --- a/tensorflow/contrib/lite/kernels/sub_test.cc +++ b/tensorflow/contrib/lite/kernels/sub_test.cc @@ -52,6 +52,13 @@ class FloatSubOpModel : public BaseSubOpModel { std::vector GetOutput() { return ExtractVector(output_); } }; +class IntegerSubOpModel : public BaseSubOpModel { + public: + using BaseSubOpModel::BaseSubOpModel; + + std::vector GetOutput() { return ExtractVector(output_); } +}; + class QuantizedSubOpModel : public BaseSubOpModel { public: using BaseSubOpModel::BaseSubOpModel; @@ -125,6 +132,57 @@ TEST(FloatSubOpModel, WithBroadcast) { } } +TEST(IntegerSubOpModel, NoActivation) { + IntegerSubOpModel m({TensorType_INT32, {1, 2, 2, 1}}, + {TensorType_INT32, {1, 2, 2, 1}}, {TensorType_INT32, {}}, + ActivationFunctionType_NONE); + m.PopulateTensor(m.input1(), {-20, 2, 7, 8}); + m.PopulateTensor(m.input2(), {1, 2, 3, 5}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({-21, 0, 4, 3})); +} + +TEST(IntegerSubOpModel, ActivationRELU_N1_TO_1) { + IntegerSubOpModel m({TensorType_INT32, {1, 2, 2, 1}}, + {TensorType_INT32, {1, 2, 2, 1}}, {TensorType_INT32, {}}, + ActivationFunctionType_RELU_N1_TO_1); + m.PopulateTensor(m.input1(), {-20, 2, 7, 8}); + m.PopulateTensor(m.input2(), {1, 2, 3, 5}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({-1, 0, 1, 1})); +} + +TEST(IntegerSubOpModel, VariousInputShapes) { + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + IntegerSubOpModel m({TensorType_INT32, test_shapes[i]}, + {TensorType_INT32, test_shapes[i]}, + {TensorType_INT32, {}}, ActivationFunctionType_NONE); + m.PopulateTensor(m.input1(), {-20, 2, 7, 8, 11, 20}); + m.PopulateTensor(m.input2(), {1, 2, 3, 5, 11, 1}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), ElementsAreArray({-21, 0, 4, 3, 0, 19})) + << "With shape number " << i; + } +} + +TEST(IntegerSubOpModel, WithBroadcast) { + std::vector> test_shapes = { + {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}}; + for (int i = 0; i < test_shapes.size(); ++i) { + IntegerSubOpModel m({TensorType_INT32, test_shapes[i]}, + {TensorType_INT32, {}}, // always a scalar + {TensorType_INT32, {}}, ActivationFunctionType_NONE); + m.PopulateTensor(m.input1(), {-20, 2, 7, 8, 11, 20}); + m.PopulateTensor(m.input2(), {1}); + m.Invoke(); + EXPECT_THAT(m.GetOutput(), + ElementsAreArray(ArrayFloatNear({-21, 1, 6, 7, 10, 19}))) + << "With shape number " << i; + } +} + TEST(QuantizedSubOpModel, QuantizedTestsNoActivation) { float kQuantizedTolerance = GetTolerance(-1.0, 1.0); std::vector> inputs1 = { diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc index 58f6bb5382..ba36017baf 100644 --- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc +++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc @@ -54,7 +54,6 @@ tensorflow::Env* env = tensorflow::Env::Default(); // TODO(ahentz): make sure we clean this list up frequently. std::map kBrokenTests = { {R"(^\/div.*int32)", "68808744"}, - {R"(^\/sub.*int32)", "68808744"}, // Pad and PadV2 only supports 4D tensors. {R"(^\/pad.*,input_shape=\[.,.\],paddings=\[\[.,.\],\[.,.\]\])", -- cgit v1.2.3 From 5f173dbf165991306f9327ee2499dc4321c40e94 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 03:29:31 -0700 Subject: Make the HloDomainRemover pass more configurable Previously we had two different function to normalize instructions within a domain where one of them was specified inside the metadata while the other one is passed into the domain remover. This change unifies them to use the externally passed in function for both usecase to make it possible to rewrite both of them from the caller of the domain remover (to add special logic). PiperOrigin-RevId: 204715075 --- .../compiler/xla/service/hlo_domain_metadata.h | 6 --- .../compiler/xla/service/hlo_domain_remover.cc | 4 +- .../compiler/xla/service/hlo_domain_remover.h | 11 +++-- tensorflow/compiler/xla/service/hlo_domain_test.cc | 21 ++++---- .../compiler/xla/service/hlo_sharding_metadata.cc | 56 ++++++++++------------ .../compiler/xla/service/hlo_sharding_metadata.h | 23 +++++---- 6 files changed, 56 insertions(+), 65 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_domain_metadata.h b/tensorflow/compiler/xla/service/hlo_domain_metadata.h index aa0308100a..f855f2a1fc 100644 --- a/tensorflow/compiler/xla/service/hlo_domain_metadata.h +++ b/tensorflow/compiler/xla/service/hlo_domain_metadata.h @@ -71,12 +71,6 @@ class DomainMetadata { // Returns a string representation of the metadata. virtual string ToString() const = 0; - - // Given a reachable set (the set of instructions which are reachable from - // each other via user/operand pathways, without crossing a kDomain - // instruciton), makes sure that all of them have metadata attributes which - // are coherent with this metadata object. - virtual Status NormalizeInstructions(const Domain& domain) const = 0; }; } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_domain_remover.cc b/tensorflow/compiler/xla/service/hlo_domain_remover.cc index e2e820002b..67fad0769f 100644 --- a/tensorflow/compiler/xla/service/hlo_domain_remover.cc +++ b/tensorflow/compiler/xla/service/hlo_domain_remover.cc @@ -47,12 +47,12 @@ Status HloDomainRemover::RunContext::VerifyAndNormalizeDomain( HloDomainVerifier::VerifyDomain(domain)); if (ref_metadata != nullptr) { VLOG(4) << "Applying domain normalization: " << ref_metadata->ToString(); - TF_RETURN_IF_ERROR(ref_metadata->NormalizeInstructions(domain)); + TF_RETURN_IF_ERROR(remover_->normalizer_(domain, ref_metadata)); } else { // No kDomain instruction was present within this domain, so call the // generic normalization functions and have them apply their heuristic. VLOG(2) << "Applying domain-less normalization"; - TF_RETURN_IF_ERROR(remover_->normalizer_(domain)); + TF_RETURN_IF_ERROR(remover_->normalizer_(domain, nullptr)); } return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/hlo_domain_remover.h b/tensorflow/compiler/xla/service/hlo_domain_remover.h index 0c71dd34fd..c859e05f02 100644 --- a/tensorflow/compiler/xla/service/hlo_domain_remover.h +++ b/tensorflow/compiler/xla/service/hlo_domain_remover.h @@ -35,9 +35,10 @@ class HloDomainRemover : public HloPassInterface { // instructions in it with the same attributes (ie, sharding), a normalizer // function is tasked at applying attribute normalization on the instructions // within such domain. - HloDomainRemover( - tensorflow::StringPiece kind, - std::function normalizer) + HloDomainRemover(tensorflow::StringPiece kind, + std::function + normalizer) : kind_(kind.ToString()), normalizer_(std::move(normalizer)) {} tensorflow::StringPiece name() const override { return "domain_remover"; } @@ -48,7 +49,9 @@ class HloDomainRemover : public HloPassInterface { class RunContext; string kind_; - std::function normalizer_; + std::function + normalizer_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_domain_test.cc b/tensorflow/compiler/xla/service/hlo_domain_test.cc index 00b2c860a7..ffc18a0f88 100644 --- a/tensorflow/compiler/xla/service/hlo_domain_test.cc +++ b/tensorflow/compiler/xla/service/hlo_domain_test.cc @@ -97,12 +97,6 @@ class OpNameMetadata : public DomainMetadata { string ToString() const override { return opname_; } - Status NormalizeInstructions( - const DomainMetadata::Domain& domain) const override { - // For the purposes of this test, nothing to do. - return Status::OK(); - } - static tensorflow::StringPiece KindName() { return "opname"; } private: @@ -124,7 +118,8 @@ std::unique_ptr OpNameDomainCreator(HloInstruction* instruction, std::move(user_side_metadata)); } -Status OpNameDomainNormalizer(const DomainMetadata::Domain& domain) { +Status OpNameDomainNormalizer(const DomainMetadata::Domain& domain, + const DomainMetadata* metadata) { // Nothing to do for the particular use this test make of the OpName domains. return Status::OK(); } @@ -159,7 +154,7 @@ ENTRY entry { EXPECT_FALSE(HasDomainEdge(module, "e", "d")); HloDomainRemover remover(ShardingMetadata::KindName(), - NormalizeShardingDomain); + ShardingMetadata::NormalizeShardingDomain); TF_ASSERT_OK_AND_ASSIGN(bool remover_changed, remover.Run(module)); EXPECT_TRUE(remover_changed); @@ -227,7 +222,7 @@ ENTRY entry { EXPECT_FALSE(HasDomainEdge(module, "e", "d")); HloDomainRemover remover(ShardingMetadata::KindName(), - NormalizeShardingDomain); + ShardingMetadata::NormalizeShardingDomain); TF_ASSERT_OK_AND_ASSIGN(bool remover_changed, remover.Run(module)); EXPECT_TRUE(remover_changed); @@ -277,7 +272,7 @@ ENTRY entry { LOG(INFO) << "Original module:\n" << module->ToString(); HloDomainRemover remover(ShardingMetadata::KindName(), - NormalizeShardingDomain); + ShardingMetadata::NormalizeShardingDomain); TF_ASSERT_OK_AND_ASSIGN(bool remover_changed, remover.Run(module)); EXPECT_FALSE(remover_changed); @@ -324,7 +319,7 @@ ENTRY entry { EXPECT_FALSE(HasDomainEdge(module, "e", "d")); HloDomainRemover sharding_remover(ShardingMetadata::KindName(), - NormalizeShardingDomain); + ShardingMetadata::NormalizeShardingDomain); TF_ASSERT_OK_AND_ASSIGN(bool sharding_remover_changed, sharding_remover.Run(module)); EXPECT_TRUE(sharding_remover_changed); @@ -411,7 +406,7 @@ ENTRY entry { } HloDomainRemover remover(ShardingMetadata::KindName(), - NormalizeShardingDomain); + ShardingMetadata::NormalizeShardingDomain); TF_ASSERT_OK_AND_ASSIGN(bool remover_changed, remover.Run(module)); EXPECT_TRUE(remover_changed); @@ -465,7 +460,7 @@ ENTRY entry { TF_EXPECT_OK(module->entry_computation()->RemoveInstruction(tuple)); HloDomainRemover remover(ShardingMetadata::KindName(), - NormalizeShardingDomain); + ShardingMetadata::NormalizeShardingDomain); TF_ASSERT_OK_AND_ASSIGN(bool remover_changed, remover.Run(module)); EXPECT_TRUE(remover_changed); diff --git a/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc b/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc index 4f91d619ef..94f5a3b273 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc +++ b/tensorflow/compiler/xla/service/hlo_sharding_metadata.cc @@ -245,21 +245,6 @@ StatusOr ApplyDomainShardingPass(const DomainMetadata::Domain& domain, Status ApplyDomainSharding(const DomainMetadata::Domain& domain, const HloSharding& sharding) { - // Here is the place to call external sharding normalizers, which are - // implemented in other modules (ie, spatial partitioning). - // The signature of the external normalizer function should be something - // like: - // - // StatusOr Normalizer(const DomainMetadata::Domain&, - // const HloSharding& sharding); - // - // The function should return true if it has processed the domain - // normalization, false if domain was not one recognized by it, or an error. - // We will call the functions in order below, and fall back to local code if - // none of the external normalizers acted on the domain. - // External normalizers should not handle the cases that are already handled - // locally. - // None of the external normalizers handled the domain sharding, try to see // whether this is a single sharding first. auto single_sharding = sharding.ExtractSingleSharding(); @@ -390,25 +375,36 @@ string ShardingMetadata::ToString() const { return sharding_ != nullptr ? sharding_->ToString() : "{}"; } -Status ShardingMetadata::NormalizeInstructions( - const DomainMetadata::Domain& domain) const { - if (sharding_ != nullptr) { - VLOG(4) << "Normalizing sharding to " << sharding_->ToString() << ":"; - TF_RETURN_IF_ERROR(ApplyDomainSharding(domain, *sharding_)); - TF_RETURN_IF_ERROR(FixupPassThroughDomainLinks(domain, *sharding_)); +/*static*/ StatusOr +ShardingMetadata::ToShardingMetadata(const DomainMetadata* metadata) { + if (metadata->Kind() != ShardingMetadata::KindName()) { + return Status( + tensorflow::error::INVALID_ARGUMENT, + "ShardingMetadata normalizer called with incorrect domain metadata"); } - return Status::OK(); + return static_cast(metadata); } -Status NormalizeShardingDomain(const DomainMetadata::Domain& domain) { - TF_ASSIGN_OR_RETURN(std::unique_ptr sharding, - ExtractOriginalCommonSharding(domain.instructions)); - if (sharding != nullptr) { - VLOG(4) << "Normalizing sharding-less domain to " << sharding->ToString() - << ":"; - TF_RETURN_IF_ERROR(ApplyDomainSharding(domain, *sharding)); +Status ShardingMetadata::NormalizeShardingDomain( + const DomainMetadata::Domain& domain, const DomainMetadata* metadata) { + if (metadata != nullptr) { + TF_ASSIGN_OR_RETURN(const auto& sharding_metadata, + ToShardingMetadata(metadata)); + const HloSharding* sharding = sharding_metadata->sharding(); + if (sharding != nullptr) { + VLOG(4) << "Normalizing sharding to " << sharding->ToString() << ":"; + TF_RETURN_IF_ERROR(ApplyDomainSharding(domain, *sharding)); + TF_RETURN_IF_ERROR(FixupPassThroughDomainLinks(domain, *sharding)); + } } else { - VLOG(1) << "Unable to find common sharding"; + TF_ASSIGN_OR_RETURN(std::unique_ptr sharding, + ExtractOriginalCommonSharding(domain.instructions)); + if (sharding != nullptr) { + VLOG(4) << "Normalizing sharding-less domain to " << sharding->ToString(); + TF_RETURN_IF_ERROR(ApplyDomainSharding(domain, *sharding)); + } else { + VLOG(1) << "Unable to find common sharding"; + } } return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/hlo_sharding_metadata.h b/tensorflow/compiler/xla/service/hlo_sharding_metadata.h index ec162c3490..5e01fc0e22 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding_metadata.h +++ b/tensorflow/compiler/xla/service/hlo_sharding_metadata.h @@ -38,23 +38,26 @@ class ShardingMetadata : public DomainMetadata { string ToString() const override; - Status NormalizeInstructions( - const DomainMetadata::Domain& domain) const override; + const HloSharding* sharding() const { return sharding_.get(); } static tensorflow::StringPiece KindName() { return "sharding"; } + static StatusOr ToShardingMetadata( + const DomainMetadata* metadata); + + // Apply the specified domain metadata onto the specified domain. If no + // metadata is specified then apply sharding heuristics and normalize the + // instructions whose sharding deviates from the one which is inferred as to + // be the original one. Policy wise, HLO passes are allowed to create new + // unassigned instructions, but if they do create assigned ones, they have to + // conform to the ones around. + static Status NormalizeShardingDomain(const DomainMetadata::Domain& domain, + const DomainMetadata* metadata); + private: std::unique_ptr sharding_; }; -// Within a set of instructions which had common sharding attributes before -// entring the HLO passes pipeline, apply sharding heuristics and normalize the -// instructions whose sharding deviates from the one which is inferred as to be -// the original one. -// Policy wise, HLO passes are allowed to create new unassigned instructions, -// but if they do create assigned ones, they have to conform to the ones around. -Status NormalizeShardingDomain(const DomainMetadata::Domain& domain); - // Given an HLO graph edge between instruction and one of its operands, creates // a ShardingMetadata based kDomain instruction if the sharding between // instruction and operand changes. Returns nullptr if there is no need for a -- cgit v1.2.3 From 00ac59bb3e6808da419488046e66d76043316b62 Mon Sep 17 00:00:00 2001 From: James Keeling Date: Mon, 16 Jul 2018 06:22:57 -0700 Subject: Add experimental config field to output interpolatable error messages This is part of our effort to improve Python error messages by allowing the runtime to output formatted messages for the Python layer to interpolate. This will be gated by this config field to begin with. PiperOrigin-RevId: 204731230 --- tensorflow/core/common_runtime/placer.cc | 5 +++++ tensorflow/core/common_runtime/placer.h | 1 + tensorflow/core/protobuf/config.proto | 3 +++ .../tools/api/golden/tensorflow.-config-proto.-experimental.pbtxt | 6 ++++++ tensorflow/tools/api/golden/tensorflow.-config-proto.pbtxt | 6 ++++++ 5 files changed, 21 insertions(+) diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc index 1f0773d387..0be44662dd 100644 --- a/tensorflow/core/common_runtime/placer.cc +++ b/tensorflow/core/common_runtime/placer.cc @@ -938,4 +938,9 @@ void Placer::LogDeviceAssignment(const Node* node) const { } } +bool Placer::ClientHandlesErrorFormatting() const { + return options_ != nullptr && + options_->config.experimental().client_handles_error_formatting(); +} + } // namespace tensorflow diff --git a/tensorflow/core/common_runtime/placer.h b/tensorflow/core/common_runtime/placer.h index 75dce7c7fe..1f8b450103 100644 --- a/tensorflow/core/common_runtime/placer.h +++ b/tensorflow/core/common_runtime/placer.h @@ -87,6 +87,7 @@ class Placer { // placement if the SessionOptions entry in 'options_' requests it. void AssignAndLog(int assigned_device, Node* node) const; void LogDeviceAssignment(const Node* node) const; + bool ClientHandlesErrorFormatting() const; Graph* const graph_; // Not owned. const DeviceSet* const devices_; // Not owned. diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto index 5b6aa47b93..77639461d9 100644 --- a/tensorflow/core/protobuf/config.proto +++ b/tensorflow/core/protobuf/config.proto @@ -389,6 +389,9 @@ message ConfigProto { message Experimental { // Task name for group resolution. string collective_group_leader = 1; + // Whether the client will format templated errors. For example, the string: + // "The node was defined on ^^node:Foo:${file}:${line}^^". + bool client_handles_error_formatting = 2; }; Experimental experimental = 16; diff --git a/tensorflow/tools/api/golden/tensorflow.-config-proto.-experimental.pbtxt b/tensorflow/tools/api/golden/tensorflow.-config-proto.-experimental.pbtxt index 9e09a8d48e..ef9fe096a1 100644 --- a/tensorflow/tools/api/golden/tensorflow.-config-proto.-experimental.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.-config-proto.-experimental.pbtxt @@ -8,5 +8,11 @@ tf_proto { label: LABEL_OPTIONAL type: TYPE_STRING } + field { + name: "client_handles_error_formatting" + number: 2 + label: LABEL_OPTIONAL + type: TYPE_BOOL + } } } diff --git a/tensorflow/tools/api/golden/tensorflow.-config-proto.pbtxt b/tensorflow/tools/api/golden/tensorflow.-config-proto.pbtxt index 4af4ed70ef..eeef15515d 100644 --- a/tensorflow/tools/api/golden/tensorflow.-config-proto.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.-config-proto.pbtxt @@ -131,6 +131,12 @@ tf_proto { label: LABEL_OPTIONAL type: TYPE_STRING } + field { + name: "client_handles_error_formatting" + number: 2 + label: LABEL_OPTIONAL + type: TYPE_BOOL + } } } } -- cgit v1.2.3 From 5c6d6eb67c76c33022908c447033c5e7f9b4f10e Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Mon, 16 Jul 2018 06:43:51 -0700 Subject: Enable test that was accidentally misnamed. PiperOrigin-RevId: 204734160 --- tensorflow/contrib/autograph/pyct/templates_test.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tensorflow/contrib/autograph/pyct/templates_test.py b/tensorflow/contrib/autograph/pyct/templates_test.py index a01f8bf04c..a8bbc5a4de 100644 --- a/tensorflow/contrib/autograph/pyct/templates_test.py +++ b/tensorflow/contrib/autograph/pyct/templates_test.py @@ -151,17 +151,13 @@ class TemplatesTest(test.TestCase): self.assertEqual(node.func.id, 'bar') self.assertEqual(node.func.args[0].id, 'baz') - def replace_as_expression_restrictions(self): + def test_replace_as_expression_restrictions(self): template = """ foo(a) bar(b) """ with self.assertRaises(ValueError): templates.replace_as_expression(template) - with self.assertRaises(ValueError): - templates.replace('') - with self.assertRaises(ValueError): - templates.replace('a = b') if __name__ == '__main__': -- cgit v1.2.3 From 7dc3233bdf9cb3f03807b2c892d965b37e3de105 Mon Sep 17 00:00:00 2001 From: wangershi Date: Mon, 16 Jul 2018 21:52:17 +0800 Subject: Omit some operations if only one para is kLogZero If log_prob_1 is kLogZero and log_prob_2 is not kLogZero or log_prob_2 is kLogZero and log_prob_1 is not kLogZero, just return another parameters, this judge can omit some operations(log1pf() and expf()). The mathematical equation is: ln(e^(-inf)+e^(x))=ln(0+e^(x))=x --- tensorflow/core/util/ctc/ctc_loss_util.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/util/ctc/ctc_loss_util.h b/tensorflow/core/util/ctc/ctc_loss_util.h index 9c71f58e23..3c77396881 100644 --- a/tensorflow/core/util/ctc/ctc_loss_util.h +++ b/tensorflow/core/util/ctc/ctc_loss_util.h @@ -31,8 +31,10 @@ const float kLogZero = -std::numeric_limits::infinity(); inline float LogSumExp(float log_prob_1, float log_prob_2) { // Always have 'b' be the smaller number to avoid the exponential from // blowing up. - if (log_prob_1 == kLogZero && log_prob_2 == kLogZero) { - return kLogZero; + if (log_prob_1 == kLogZero) { + return log_prob_2; + } else if (log_prob_2 == kLogZero){ + return log_prob_1; } else { return (log_prob_1 > log_prob_2) ? log_prob_1 + log1pf(expf(log_prob_2 - log_prob_1)) -- cgit v1.2.3 From d4f5f85e677b9efe0397c92a64b4d114d03e3a36 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 07:06:24 -0700 Subject: Missing import and pip install of tf-nightly in workshop notebook PiperOrigin-RevId: 204736577 --- .../autograph/examples/notebooks/workshop.ipynb | 24 +++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb b/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb index e8f16b431d..4643656ff4 100644 --- a/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb +++ b/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb @@ -1,5 +1,23 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "yFn4b8J0CKQV" + }, + "outputs": [], + "source": [ + "pip install -U tf-nightly" + ] + }, { "cell_type": "code", "execution_count": 0, @@ -15,10 +33,10 @@ }, "outputs": [], "source": [ + "import os\n", + "import matplotlib.pyplot as plt\n", "import tensorflow as tf\n", - "from tensorflow.contrib import autograph\n", - "\n", - "import matplotlib.pyplot as plt" + "from tensorflow.contrib import autograph" ] }, { -- cgit v1.2.3 From e3aa44ec207bfdc798e26e92a38e80c3f9c5453b Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Mon, 16 Jul 2018 07:55:58 -0700 Subject: Bring the dev summit notebook to date. PiperOrigin-RevId: 204741684 --- .../examples/notebooks/dev_summit_2018_demo.ipynb | 1493 ++++++++++---------- 1 file changed, 755 insertions(+), 738 deletions(-) diff --git a/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb index 0702273fac..86e38c3490 100644 --- a/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb +++ b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb @@ -1,49 +1,20 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "Dev Summit 2018 - Autograph", - "version": "0.3.2", - "views": {}, - "default_view": {}, - "provenance": [ - { - "file_id": "1wCZUh73zTNs1jzzYjqoxMIdaBWCdKJ2K", - "timestamp": 1522238054357 - }, - { - "file_id": "1_HpC-RrmIv4lNaqeoslUeWaX8zH5IXaJ", - "timestamp": 1521743157199 - }, - { - "file_id": "1mjO2fQ2F9hxpAzw2mnrrUkcgfb7xSGW-", - "timestamp": 1520522344607 - } - ], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python2", - "display_name": "Python 2" - } - }, "cells": [ { + "cell_type": "markdown", "metadata": { - "id": "g7nGs4mzVUHP", - "colab_type": "text" + "colab_type": "text", + "id": "g7nGs4mzVUHP" }, - "cell_type": "markdown", "source": [ - "# Experimental: TF Autograph\n", + "# Experimental: TF AutoGraph\n", "**TensorFlow Dev Summit, 2018.**\n", "\n", - "This interactive notebook demonstrates **autograph**, an experimental source-code transformation library to automatically convert TF.Eager and Python code to TensorFlow graphs.\n", + "This interactive notebook demonstrates **AutoGraph**, an experimental source-code transformation library to automatically convert Python, TensorFlow and NumPy code to TensorFlow graphs.\n", "\n", "**Note: this is pre-alpha software!** The notebook works best with Python 2, for now.\n", "\n", - "> ![alt text](https://lh3.googleusercontent.com/QOvy0clmg7siaVKzwmSPAjicWWNQ0OeyaB16plDjSJMf35WD3vLjF6mz4CGrhSHw60HnlZPJjkyDCBzw5XOI0oBGSewyYw=s688)\n", + "\u003e ![alt text](https://lh3.googleusercontent.com/QOvy0clmg7siaVKzwmSPAjicWWNQ0OeyaB16plDjSJMf35WD3vLjF6mz4CGrhSHw60HnlZPJjkyDCBzw5XOI0oBGSewyYw=s688)\n", "\n", "### Table of Contents\n", "1. _Write Eager code that is fast and scalable._\n", @@ -53,37 +24,39 @@ ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "uFcgBENZqkB2", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "uFcgBENZqkB2" }, - "cell_type": "code", + "outputs": [], "source": [ "# Install TensorFlow; note that Colab notebooks run remotely, on virtual\n", "# instances provided by Google.\n", "!pip install -U -q tf-nightly" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "Pa2qpEmoVOGe", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "Pa2qpEmoVOGe" }, - "cell_type": "code", + "outputs": [], "source": [ "import os\n", "import time\n", @@ -96,170 +69,172 @@ "import six\n", "\n", "from google.colab import widgets" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "ZVKfj5ttVkqz", - "colab_type": "text" + "colab_type": "text", + "id": "ZVKfj5ttVkqz" }, - "cell_type": "markdown", "source": [ "# 1. Write Eager code that is fast and scalable\n", "\n", "TF.Eager gives you more flexibility while coding, but at the cost of losing the benefits of TensorFlow graphs. For example, Eager does not currently support distributed training, exporting models, and a variety of memory and computation optimizations.\n", "\n", - "Autograph gives you the best of both worlds: write your code in an Eager style, and we will automatically transform it into the equivalent TF graph code. The graph code can be executed eagerly (as a single op), included as part of a larger graph, or exported." + "AutoGraph gives you the best of both worlds: you can write your code in an Eager style, and we will automatically transform it into the equivalent TF graph code. The graph code can be executed eagerly (as a single op), included as part of a larger graph, or exported." ] }, { + "cell_type": "markdown", "metadata": { - "id": "snaZRFdWd9ym", - "colab_type": "text" + "colab_type": "text", + "id": "snaZRFdWd9ym" }, - "cell_type": "markdown", "source": [ - "For example, autograph can convert a function like this:" + "For example, AutoGraph can convert a function like this:" ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "9__n8cSIeDnD", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "9__n8cSIeDnD" }, - "cell_type": "code", + "outputs": [], "source": [ "def g(x):\n", - " if x > 0:\n", + " if x \u003e 0:\n", " x = x * x\n", " else:\n", " x = 0\n", " return x" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "gq0eQcuReHET", - "colab_type": "text" + "colab_type": "text", + "id": "gq0eQcuReHET" }, - "cell_type": "markdown", "source": [ "... into a TF graph-building function:" ] }, { + "cell_type": "code", + "execution_count": 4, "metadata": { - "id": "sELSn599ePUF", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, - "output_extras": [ - {} - ], - "base_uri": "https://localhost:8080/", - "height": 413 + "height": 431 }, - "outputId": "bb0c7216-1ca3-4da1-d1fb-589902cdcd1a", + "colab_type": "code", "executionInfo": { + "elapsed": 69, "status": "ok", - "timestamp": 1522345737505, - "user_tz": 240, - "elapsed": 243, + "timestamp": 1531750911837, "user": { - "displayName": "Dan Moldovan", - "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", - "userId": "112023154726779574577" - } - } + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "sELSn599ePUF", + "outputId": "2858bde5-ae05-4c32-be01-7770ac914f02" }, - "cell_type": "code", - "source": [ - "print(autograph.to_code(g))" - ], - "execution_count": 0, "outputs": [ { + "name": "stdout", "output_type": "stream", "text": [ "from __future__ import print_function\n", "import tensorflow as tf\n", - "from tensorflow.contrib.autograph.impl import api as autograph_api\n", - "from tensorflow.contrib.autograph import utils as autograph_utils\n", "\n", "def tf__g(x):\n", - " with tf.name_scope('g'):\n", + " try:\n", + " with tf.name_scope('g'):\n", "\n", - " def if_true():\n", - " with tf.name_scope('if_true'):\n", - " x_1, = x,\n", - " x_1 = x_1 * x_1\n", - " return x_1,\n", + " def if_true():\n", + " with tf.name_scope('if_true'):\n", + " x_1, = x,\n", + " x_1 = x_1 * x_1\n", + " return x_1,\n", "\n", - " def if_false():\n", - " with tf.name_scope('if_false'):\n", - " x_1, = x,\n", - " x_1 = 0\n", - " return x_1,\n", - " x = autograph_utils.run_cond(tf.greater(x, 0), if_true, if_false)\n", - " return x\n", + " def if_false():\n", + " with tf.name_scope('if_false'):\n", + " x_2, = x,\n", + " x_2 = 0\n", + " return x_2,\n", + " x = ag__.utils.run_cond(tf.greater(x, 0), if_true, if_false)\n", + " return x\n", + " except:\n", + " ag__.rewrite_graph_construction_error(ag_source_map__)\n", "\n" - ], - "name": "stdout" + ] } + ], + "source": [ + "print(autograph.to_code(g))" ] }, { + "cell_type": "markdown", "metadata": { - "id": "j74n-8hEe6dk", - "colab_type": "text" + "colab_type": "text", + "id": "j74n-8hEe6dk" }, - "cell_type": "markdown", "source": [ "You can then use the converted function as you would any regular TF op -- you can pass `Tensor` arguments and it will return `Tensor`s:" ] }, { + "cell_type": "code", + "execution_count": 5, "metadata": { - "id": "AkVaY0-dfEbH", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, - "output_extras": [ - {} - ], - "base_uri": "https://localhost:8080/", "height": 53 }, - "outputId": "4ffe3757-c44d-424c-c2a8-7ddc973bfcce", + "colab_type": "code", "executionInfo": { + "elapsed": 83, "status": "ok", - "timestamp": 1522345737841, - "user_tz": 240, - "elapsed": 257, + "timestamp": 1531750911965, "user": { - "displayName": "Dan Moldovan", - "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", - "userId": "112023154726779574577" - } - } + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "AkVaY0-dfEbH", + "outputId": "f04541ad-b1d3-4663-bf27-4d902648283d" }, - "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g(9) = 81\n", + "tf_g(9) = 81\n" + ] + } + ], "source": [ "tf_g = autograph.to_graph(g)\n", "\n", @@ -272,77 +247,72 @@ "\n", " print('g(9) = %s' % g(9))\n", " print('tf_g(9) = %s' % tf_g_result)" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "g(9) = 81\n", - "tf_g(9) = 81\n" - ], - "name": "stdout" - } ] }, { + "cell_type": "markdown", "metadata": { - "id": "trrHQBM1VnD0", - "colab_type": "text" + "colab_type": "text", + "id": "trrHQBM1VnD0" }, - "cell_type": "markdown", "source": [ "# 2. Case study: complex control flow\n", "\n", - "Autograph can convert a large chunk of the Python language into graph-equivalent code, and we're adding new supported language features all the time. In this section, we'll give you a taste of some of the functionality in autograph.\n", - "Autograph will automatically convert most Python control flow statements into their correct graph equivalent.\n", + "Autograph can convert a large subset of the Python language into graph-equivalent code, and we're adding new supported language features all the time. In this section, we'll give you a taste of some of the functionality in AutoGraph.\n", + "AutoGraph will automatically convert most Python control flow statements into their graph equivalent.\n", " " ] }, { + "cell_type": "markdown", "metadata": { - "id": "u0YG3DPgZxoW", - "colab_type": "text" + "colab_type": "text", + "id": "u0YG3DPgZxoW" }, - "cell_type": "markdown", "source": [ "We support common statements like `while`, `for`, `if`, `break`, `return` and more. You can even nest them as much as you like. Imagine trying to write the graph version of this code by hand:" ] }, { + "cell_type": "code", + "execution_count": 6, "metadata": { - "id": "xJYDzOcrZ8pI", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, - "output_extras": [ - {} - ], - "base_uri": "https://localhost:8080/", "height": 35 }, - "outputId": "6c244ee4-b141-4ad6-eefa-cfffa71f33c6", + "colab_type": "code", "executionInfo": { + "elapsed": 169, "status": "ok", - "timestamp": 1522345738402, - "user_tz": 240, - "elapsed": 483, + "timestamp": 1531750912183, "user": { - "displayName": "Dan Moldovan", - "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", - "userId": "112023154726779574577" - } - } + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "xJYDzOcrZ8pI", + "outputId": "f392b475-bf87-4d90-919d-44f895ee9fc7" }, - "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sum of even numbers: 42\n" + ] + } + ], "source": [ "def sum_even(numbers):\n", " s = 0\n", " for n in numbers:\n", - " if n % 2 > 0:\n", + " if n % 2 \u003e 0:\n", " continue\n", " s += n\n", " return s\n", @@ -358,77 +328,74 @@ " \n", "# Uncomment the line below to print the generated graph code\n", "# print(autograph.to_code(sum_even))" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Sum of even numbers: 42\n" - ], - "name": "stdout" - } ] }, { + "cell_type": "markdown", "metadata": { - "id": "_YXo4KOcbKrn", - "colab_type": "text" + "colab_type": "text", + "id": "_YXo4KOcbKrn" }, - "cell_type": "markdown", "source": [ "Try replacing the `continue` in the above code with `break` -- Autograph supports that as well!" ] }, { + "cell_type": "markdown", "metadata": { - "id": "xHmC0rBIavW_", - "colab_type": "text" + "colab_type": "text", + "id": "xHmC0rBIavW_" }, - "cell_type": "markdown", "source": [ "The Python code above is much more readable than the matching graph code. Autograph takes care of tediously converting every piece of Python code into the matching TensorFlow graph version for you, so that you can quickly write maintainable code, but still benefit from the optimizations and deployment benefits of graphs." ] }, { + "cell_type": "markdown", "metadata": { - "id": "UEHWGpBXbS7g", - "colab_type": "text" + "colab_type": "text", + "id": "UEHWGpBXbS7g" }, - "cell_type": "markdown", "source": [ "Let's try some other useful Python constructs, like `print` and `assert`. We automatically convert Python `assert` statements into the equivalent `tf.Assert` code. " ] }, { + "cell_type": "code", + "execution_count": 7, "metadata": { - "id": "qUU57xlEbauI", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, - "output_extras": [ - {} - ], - "base_uri": "https://localhost:8080/", "height": 53 }, - "outputId": "add3db4a-2077-4dd5-f7a7-a5b5a4529c26", + "colab_type": "code", "executionInfo": { + "elapsed": 56, "status": "ok", - "timestamp": 1522345738697, - "user_tz": 240, - "elapsed": 253, + "timestamp": 1531750912292, "user": { - "displayName": "Dan Moldovan", - "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", - "userId": "112023154726779574577" - } - } + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "qUU57xlEbauI", + "outputId": "c9cd536a-4a95-4eb0-98c0-aafce5d79580" }, - "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Got error message: assertion failed: [Do not pass zero!]\n", + "\t [[Node: f/Assert/Assert = Assert[T=[DT_STRING], summarize=3, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](f/NotEqual, f/Assert/Assert/data_0)]]\n" + ] + } + ], "source": [ "def f(x):\n", " assert x != 0, 'Do not pass zero!'\n", @@ -444,61 +411,35 @@ " \n", "# Uncomment the line below to print the generated graph code\n", "# print(autograph.to_code(f))" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Got error message: assertion failed: [Do not pass zero!]\n", - "\t [[Node: f/Assert/Assert = Assert[T=[DT_STRING], summarize=3, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](f/NotEqual, f/Assert/Assert/data_0)]]\n" - ], - "name": "stdout" - } ] }, { + "cell_type": "markdown", "metadata": { - "id": "w5hBZaVJbck4", - "colab_type": "text" + "colab_type": "text", + "id": "w5hBZaVJbck4" }, - "cell_type": "markdown", "source": [ "You can also use `print` functions in-graph:" ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "6NdzRKLEboRv", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 - }, - "output_extras": [ - {} - ], - "base_uri": "https://localhost:8080/", - "height": 35 - }, - "outputId": "fb82dfc3-790f-4127-87f6-361805be9e9b", - "executionInfo": { - "status": "ok", - "timestamp": 1522345739013, - "user_tz": 240, - "elapsed": 247, - "user": { - "displayName": "Dan Moldovan", - "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", - "userId": "112023154726779574577" } - } + }, + "colab_type": "code", + "id": "6NdzRKLEboRv" }, - "cell_type": "code", + "outputs": [], "source": [ "def print_sign(n):\n", - " if n >= 0:\n", + " if n \u003e= 0:\n", " print(n, 'is positive!')\n", " else:\n", " print(n, 'is negative!')\n", @@ -512,62 +453,58 @@ " \n", "# Uncomment the line below to print the generated graph code\n", "# print(autograph.to_code(print_sign))" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "1 is positive!\n" - ], - "name": "stdout" - } ] }, { + "cell_type": "markdown", "metadata": { - "id": "9u_Z3i3AivLA", - "colab_type": "text" + "colab_type": "text", + "id": "9u_Z3i3AivLA" }, - "cell_type": "markdown", "source": [ - "We can convert lists to TensorArray, so appending to lists also works, with a few modifications:" + "Appending to lists also works, with a few modifications:" ] }, { + "cell_type": "code", + "execution_count": 9, "metadata": { - "id": "MjhCQJVuiTNR", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, - "output_extras": [ - {} - ], - "base_uri": "https://localhost:8080/", "height": 35 }, - "outputId": "dc320b87-595b-4392-d29c-994486fd8a0a", + "colab_type": "code", "executionInfo": { + "elapsed": 148, "status": "ok", - "timestamp": 1522345744470, - "user_tz": 240, - "elapsed": 5391, + "timestamp": 1531750912595, "user": { - "displayName": "Dan Moldovan", - "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", - "userId": "112023154726779574577" - } - } + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "MjhCQJVuiTNR", + "outputId": "96bf9131-c7c1-4359-ee82-9c38575e7ab4" }, - "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[0 1 2 3 4]\n" + ] + } + ], "source": [ "def f(n):\n", " numbers = []\n", " # We ask you to tell us about the element dtype.\n", - " autograph.utils.set_element_type(numbers, tf.int32)\n", + " autograph.set_element_type(numbers, tf.int32)\n", " for i in range(n):\n", " numbers.append(i)\n", " return autograph.stack(numbers) # Stack the list so that it can be used as a Tensor\n", @@ -580,65 +517,62 @@ " \n", "# Uncomment the line below to print the generated graph code\n", "# print(autograph.to_code(f))" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "[0 1 2 3 4]\n" - ], - "name": "stdout" - } ] }, { + "cell_type": "markdown", "metadata": { - "id": "UdG8ZFrkTAF2", - "colab_type": "text" + "colab_type": "text", + "id": "UdG8ZFrkTAF2" }, - "cell_type": "markdown", "source": [ "And all of these functionalities, and more, can be composed into more complicated code:\n" ] }, { + "cell_type": "code", + "execution_count": 10, "metadata": { - "id": "DVs6wt8NKaGQ", - "colab_type": "code", + "cellView": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, - "output_extras": [ - {} - ], - "base_uri": "https://localhost:8080/", "height": 53 }, - "cellView": "code", - "outputId": "0a4b8d08-8f65-4bbc-85ba-dc4c60563519", + "colab_type": "code", "executionInfo": { + "elapsed": 555, "status": "ok", - "timestamp": 1522345745186, - "user_tz": 240, - "elapsed": 658, + "timestamp": 1531750913176, "user": { - "displayName": "Dan Moldovan", - "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", - "userId": "112023154726779574577" - } - } + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "DVs6wt8NKaGQ", + "outputId": "8729229c-4f08-4640-d3a1-0d3f9c697a87" }, - "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The prime numbers less than 50 are:\n", + "[ 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47]\n" + ] + } + ], "source": [ "def print_primes(n):\n", " \"\"\"Returns all the prime numbers less than n.\"\"\"\n", - " assert n > 0\n", + " assert n \u003e 0\n", " \n", " primes = []\n", - " autograph.utils.set_element_type(primes, tf.int32)\n", + " autograph.set_element_type(primes, tf.int32)\n", " for i in range(2, n):\n", " is_prime = True\n", " for k in range(2, i):\n", @@ -663,45 +597,36 @@ " \n", "# Uncomment the line below to print the generated graph code\n", "# print(autograph.to_code(print_primes))" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "The prime numbers less than 50 are:\n", - "[ 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47]\n" - ], - "name": "stdout" - } ] }, { + "cell_type": "markdown", "metadata": { - "id": "JQ8kQT99VqDk", - "colab_type": "text" + "colab_type": "text", + "id": "JQ8kQT99VqDk" }, - "cell_type": "markdown", "source": [ "# 3. Case study: training MNIST with Keras\n", "\n", - "As we've seen, writing control flow in Autograph is easy. So running a training loop in graph should be easy as well!\n", + "As we've seen, writing control flow in AutoGraph is easy. So running a training loop in graph should be easy as well!\n", "\n", "Here, we show an example of such a training loop for a simple Keras model that trains on MNIST." ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "0CrtGWgwuLJr", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "0CrtGWgwuLJr" }, - "cell_type": "code", + "outputs": [], "source": [ "import gzip\n", "import shutil\n", @@ -754,66 +679,67 @@ "\n", "def mnist_test(directory):\n", " return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "2zu1U9Nqir6L", - "colab_type": "text" + "colab_type": "text", + "id": "2zu1U9Nqir6L" }, - "cell_type": "markdown", "source": [ "First, we'll define a small three-layer neural network using the Keras API" ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "x_MU13boiok2", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "x_MU13boiok2" }, - "cell_type": "code", + "outputs": [], "source": [ "def mlp_model(input_shape):\n", - " model = tf.keras.Sequential([\n", + " model = tf.keras.Sequential((\n", " tf.keras.layers.Dense(100, activation='relu', input_shape=input_shape),\n", " tf.keras.layers.Dense(100, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')])\n", + " tf.keras.layers.Dense(10, activation='softmax'),\n", + " ))\n", " model.build()\n", " return model" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "Wuqg3H8mi0Xj", - "colab_type": "text" + "colab_type": "text", + "id": "Wuqg3H8mi0Xj" }, - "cell_type": "markdown", "source": [ "Let's connect the model definition (here abbreviated as `m`) to a loss function, so that we can train our model." ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "W51sfbONiz_5", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "W51sfbONiz_5" }, - "cell_type": "code", + "outputs": [], "source": [ "def predict(m, x, y):\n", " y_p = m(x)\n", @@ -822,63 +748,63 @@ " accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n", " accuracy = tf.reduce_mean(accuracies)\n", " return l, accuracy" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "035tNWQki9tr", - "colab_type": "text" + "colab_type": "text", + "id": "035tNWQki9tr" }, - "cell_type": "markdown", "source": [ "Now the final piece of the problem specification (before loading data, and clicking everything together) is backpropagating the loss through the model, and optimizing the weights using the gradient." ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "CsAD0ajbi9iZ", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "CsAD0ajbi9iZ" }, - "cell_type": "code", + "outputs": [], "source": [ "def fit(m, x, y, opt):\n", " l, accuracy = predict(m, x, y)\n", " opt.minimize(l)\n", " return l, accuracy" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "PcVRIacKjSwb", - "colab_type": "text" + "colab_type": "text", + "id": "PcVRIacKjSwb" }, - "cell_type": "markdown", "source": [ "These are some utility functions to download data and generate batches for training" ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "RVw57HdTjPzi", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "RVw57HdTjPzi" }, - "cell_type": "code", + "outputs": [], "source": [ "def setup_mnist_data(is_training, hp, batch_size):\n", " if is_training:\n", @@ -896,16 +822,14 @@ " x = tf.to_float(tf.reshape(image, (-1, 28 * 28)))\n", " y = tf.one_hot(tf.squeeze(label), 10)\n", " return x, y" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "2zEJH5XNjgFz", - "colab_type": "text" + "colab_type": "text", + "id": "2zEJH5XNjgFz" }, - "cell_type": "markdown", "source": [ "This function specifies the main training loop. We instantiate the model (using the code above), instantiate an optimizer (here we'll use SGD with momentum, nothing too fancy), and we'll instantiate some lists to keep track of training and test loss and accuracy over time.\n", "\n", @@ -913,33 +837,35 @@ ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "UUI0566FjZPx", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "UUI0566FjZPx" }, - "cell_type": "code", + "outputs": [], "source": [ "def train(train_ds, test_ds, hp):\n", " m = mlp_model((28 * 28,))\n", " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", + "\n", " train_losses = []\n", - " train_losses = autograph.utils.set_element_type(train_losses, tf.float32)\n", + " autograph.set_element_type(train_losses, tf.float32)\n", " test_losses = []\n", - " test_losses = autograph.utils.set_element_type(test_losses, tf.float32)\n", + " autograph.set_element_type(test_losses, tf.float32)\n", " train_accuracies = []\n", - " train_accuracies = autograph.utils.set_element_type(train_accuracies,\n", - " tf.float32)\n", + " autograph.set_element_type(train_accuracies, tf.float32)\n", " test_accuracies = []\n", - " test_accuracies = autograph.utils.set_element_type(test_accuracies,\n", - " tf.float32)\n", - " i = tf.constant(0)\n", - " while i < hp.max_steps:\n", + " autograph.set_element_type(test_accuracies, tf.float32)\n", + "\n", + " i = 0\n", + " while i \u003c hp.max_steps:\n", " train_x, train_y = get_next_batch(train_ds)\n", " test_x, test_y = get_next_batch(test_ds)\n", " step_train_loss, step_train_accuracy = fit(m, train_x, train_y, opt)\n", @@ -956,173 +882,144 @@ " return (autograph.stack(train_losses), autograph.stack(test_losses),\n", " autograph.stack(train_accuracies),\n", " autograph.stack(test_accuracies))" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "cYiUQ1ppkHzk", - "colab_type": "text" + "colab_type": "text", + "id": "cYiUQ1ppkHzk" }, - "cell_type": "markdown", "source": [ "Everything is ready to go, let's train the model and plot its performance!" ] }, { + "cell_type": "code", + "execution_count": 17, "metadata": { - "id": "K1m8TwOKjdNd", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, - "output_extras": [ - {}, - {}, - {} - ], - "base_uri": "https://localhost:8080/", - "height": 988 + "height": 585 }, - "outputId": "f9d3eef3-5bea-45c1-ddf9-4edee73e4436", + "colab_type": "code", "executionInfo": { + "elapsed": 17094, "status": "ok", - "timestamp": 1522345800262, - "user_tz": 240, - "elapsed": 52391, + "timestamp": 1531750930585, "user": { - "displayName": "Dan Moldovan", - "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", - "userId": "112023154726779574577" - } - } + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "K1m8TwOKjdNd", + "outputId": "9f63da19-c3bf-498b-cf00-29090bf3b4f0" }, - "cell_type": "code", - "source": [ - "with tf.Graph().as_default():\n", - " hp = tf.contrib.training.HParams(\n", - " learning_rate=0.05,\n", - " max_steps=500,\n", - " )\n", - " train_ds = setup_mnist_data(True, hp, 50)\n", - " test_ds = setup_mnist_data(False, hp, 1000)\n", - " tf_train = autograph.to_graph(train)\n", - " (train_losses, test_losses, train_accuracies,\n", - " test_accuracies) = tf_train(train_ds, test_ds, hp)\n", - "\n", - " with tf.Session() as sess:\n", - " sess.run(tf.global_variables_initializer())\n", - " (train_losses, test_losses, train_accuracies,\n", - " test_accuracies) = sess.run([train_losses, test_losses, train_accuracies,\n", - " test_accuracies])\n", - " plt.title('MNIST train/test losses')\n", - " plt.plot(train_losses, label='train loss')\n", - " plt.plot(test_losses, label='test loss')\n", - " plt.legend()\n", - " plt.xlabel('Training step')\n", - " plt.ylabel('Loss')\n", - " plt.show()\n", - " plt.title('MNIST train/test accuracies')\n", - " plt.plot(train_accuracies, label='train accuracy')\n", - " plt.plot(test_accuracies, label='test accuracy')\n", - " plt.legend(loc='lower right')\n", - " plt.xlabel('Training step')\n", - " plt.ylabel('Accuracy')\n", - " plt.show()" - ], - "execution_count": 0, "outputs": [ { - "output_type": "stream", - "text": [ - "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/train-images-idx3-ubyte.gz to /tmp/autograph_mnist_data/train-images-idx3-ubyte.gz\n", - "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/train-labels-idx1-ubyte.gz to /tmp/autograph_mnist_data/train-labels-idx1-ubyte.gz\n", - "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-images-idx3-ubyte.gz to /tmp/autograph_mnist_data/t10k-images-idx3-ubyte.gz\n", - "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels-idx1-ubyte.gz to /tmp/autograph_mnist_data/t10k-labels-idx1-ubyte.gz\n", - "Step 0 train loss: 2.244329 test loss: 2.2499208 train accuracy: 0.12 test accuracy: 0.161\n", - "Step 50 train loss: 0.64771986 test loss: 0.56013924 train accuracy: 0.82 test accuracy: 0.836\n", - "Step 100 train loss: 0.49011207 test loss: 0.42143965 train accuracy: 0.84 test accuracy: 0.879\n", - "Step 150 train loss: 0.3768609 test loss: 0.39319593 train accuracy: 0.88 test accuracy: 0.883\n", - "Step 200 train loss: 0.36007702 test loss: 0.37089333 train accuracy: 0.9 test accuracy: 0.881\n", - "Step 250 train loss: 0.182115 test loss: 0.28543878 train accuracy: 0.94 test accuracy: 0.915\n", - "Step 300 train loss: 0.2119576 test loss: 0.22305593 train accuracy: 0.92 test accuracy: 0.93\n", - "Step 350 train loss: 0.12932214 test loss: 0.29057172 train accuracy: 0.96 test accuracy: 0.906\n", - "Step 400 train loss: 0.22937602 test loss: 0.2200287 train accuracy: 0.92 test accuracy: 0.925\n", - "Step 450 train loss: 0.23444137 test loss: 0.19857481 train accuracy: 0.94 test accuracy: 0.94\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAFnCAYAAACPasF4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3XmAFNW9Pvynlt5mYdhmQMHggnGN\nS9zCD0ElKug1edUY9ZoQTYze3GuiRk1uYjRqRHNj4n5NrhKjiUYlbihGQFRUFDSoKIvgICAO6+xL\n711V5/2jlq7qZaZnpnumZ3g+/zjTXV1dXSP91PecU+dIQggBIiIiGjLkwT4AIiIi6h2GNxER0RDD\n8CYiIhpiGN5ERERDDMObiIhoiGF4ExERDTEMb6JeOOigg3DllVdmPf6rX/0KBx10kGe766+/3rPN\ne++9h9mzZwMAtm3bhkMPPdR57osvvsCPfvQjzJw5EzNnzsTZZ5+NV199FQBw0003YdasWZg1axYO\nO+wwnHLKKc7v4XDY8x7JZBLz58/v9edavXo1Lr300oK2XbBgAebMmdPn97J19/rZs2fjhRde6PO+\niYY7hjdRL3366aee0Ewmk1izZk3WditXrsQnn3xS0D6vu+46TJs2DYsXL8bixYtxyy234LrrrsPO\nnTtxyy23YNGiRVi0aBHGjRuH3//+987vVVVVnv188sknfQrUI444Ag8//HBB2y5fvhxTpkzp83vZ\n+vt6oj0Zw5uol0444QQsWbLE+f3tt9/GV77ylaztrrnmGtx+++0F7bO+vh5HHnmk8/uRRx6JxYsX\nY/z48QUfV3NzM3784x/jo48+wkUXXQTAbAF48MEHMXPmTOi6jlWrVuHcc8/FrFmzcOaZZ2L58uUA\nzFaB0047DQBw//334ze/+Q2uuOIKfP3rX8d5552HxsZG533ee+89HHzwwVnv9cEHH+Bb3/oWTjvt\nNJx//vloaGgAAOzevRsXX3wxzjzzTJx66qm4++67cx5rPu+99x7OOecczJo1C9/+9redC6Vc++3u\ncSEE/vd//xczZ87EKaecgjlz5kDXdQDAwoULcdZZZ+GMM87AN77xDbz33nsFn3eiwcDwJuqlM844\nAy+99JLz+z//+U/MmjUr53ZCCCxatKjHfU6fPh1XXnkl/va3v2HTpk0AgHHjxkGSpIKPa+zYsbjm\nmmtw1FFH4YknnnAeF0Jg8eLFUBQFv/71r3HppZdi0aJFuPzyy3HTTTfl3NeiRYtw/fXX49VXX8WY\nMWPw7LPPAgA2bdqE2tpaTJgwwfNe4XAY//mf/4lrrrkGS5Yswfe+9z1cddVVAIBHH30Uxx13HF5+\n+WUsWLAADQ0NMAwj57FmikQiuOqqq3DDDTdg0aJF+OEPf4jrrrsOhmHk3G9jY2Pex1944QUsWrQI\nzzzzDJYsWYKGhgY8+eSTAIBbbrkFDz74IBYuXIibbroJr7/+esHnnWgwMLyJeun444/Hxo0b0dLS\nglgshlWrVmHKlCk5t73++uvxhz/8AYlEott9/v73v8d3vvMdLFiwAGeddRZmzJjhBEt/nXzyyc7P\n8+fPxxlnnAEAOOaYY5zqONOxxx6LCRMmQJIkHHLIIdi5cycAYMWKFTk/6wcffIBx48Zh6tSpAICz\nzjoLX3zxBXbs2IExY8bg7bffxvvvvw+/34+77roLdXV1BR376tWrMX78eBxzzDEAgJkzZ6KtrQ3b\nt2/Pu998jy9duhTf+ta3UF1dDVVV8e1vfxuvvPIKAGDMmDF46qmnsH37dhx77LH45S9/WdjJJRok\n6mAfANFQoygKTj/9dCxcuBCjR4/GiSeeCFXN/U/psMMOw3HHHYdHHnkERx99dN59BgIBXHrppbj0\n0kvR2dmJRYsW4fbbb8fEiRMxbdq0fh3vyJEjnZ8XLFiAv/3tb4hEIjAMA/mWNqiurnZ+VhTFaV5+\n5513cMkll2Rt39nZiYaGBk8LhN/vR2trKy655BIYhoFbbrkFjY2N+M53voOf/OQnBR17a2srRowY\nkXVsLS0tefeb7/Guri48/PDDmDdvHgBA13WMHj0aAPCnP/0Jf/rTn3Duuedir732wvXXX4/jjz++\noGMkGgwMb6I+OPPMM3H33Xdj1KhRPfbZ/vSnP8W5556LiRMn5ny+tbUV69evd6rWESNG4Pzzz8ey\nZctQX1/f7/C27d69GzfccAOefvppHHLIIfj8888xc+bMgl+vaRrWrFmT8yKkrq4O+++/P5577rmc\nr7388stx+eWXY8uWLbjsssucSronY8aMQXt7u/O7EAIdHR0YM2YMVFXNud+pU6fmfLyurg4zZszA\nd7/73az3+dKXvoTf/va3MAwD8+fPx7XXXotly5YVeGaIBh6bzYn64Oijj0ZjYyM2btzYY4VWV1eH\n73znO7j//vtzPh+Px3HllVd6wmLr1q34+OOPceyxx/bquFRVRTgczllRt7a2oqKiAvvvvz80TXMq\n0EgkUtC+V69ejYMOOgh+vz/rvY488kg0NTXh448/BgA0NDTgZz/7GYQQ+PWvf4133nkHgBmSY8eO\nhSRJ3R6r7YgjjkBzczNWrVoFwBxfMH78eEycODHvfvM9/vWvfx0vvPACYrEYAOCpp57C888/j9bW\nVnz/+99HOByGLMs48sgjezXWgGgwsPIm6gNJknDaaachFotBlnu+Bv7BD36Ap59+Oudze++9N/70\npz/hvvvuw5w5cyCEQFVVFX75y196RqAX4phjjsEf/vAHTJs2DW+++abnuYMPPhjTp0/HzJkzMWbM\nGPziF7/Ahx9+iNmzZ+O///u/e9y3fYtYvve67777cOuttyISicDn8+Gqq66CJEm48MIL8etf/xq3\n3norhBCYMWMGpkyZgh07dnheryhK1ntWVFTgnnvuwa233opoNIrRo0fjrrvu6na/I0eOzPk4AGzc\nuBHnnHMOADPYb7vtNowePRrTpk3Dt771LSiKAp/Ph9tuu61X551ooElcz5uIiGhoYbM5ERHREMPw\nJiIiGmIY3kREREMMw5uIiGiIYXgTERENMUPmVrGmpq6i7m/UqAq0tUWLus89Ec9j//Ec9h/PYXHw\nPPZfsc9hbW11zsf32MpbVbPvKaXe43nsP57D/uM5LA6ex/4bqHO4x4Y3ERHRUMXwJiIiGmIY3kRE\nREMMw5uIiGiIYXgTERENMQxvIiKiIYbhTURENMQwvImIaNh6443XCt723nvvxI4d23vc7sMP38cN\nN/y8P4fVbwxvIiIalnbu3IFXX11c8PZXXXUt9t57QgmPqHiGzPSoREREvXHXXb/D+vXr8Mgjc2EY\nBnbs2I6dO3fgnnv+iN/+9jdoampELBbDD35wOaZOnYYf//hyXHPNz7F06WuIRML44out2L59G668\n8lpMmTI153u89toSzJv3dyiKgoMOOgS33XYL6us34M47fwefzwe/349bbvktdu7cnvVYdXXuqU8L\nsceGd0c4gfc3NOLYg+sG+1CIiIa9f7z+GVZuaCzqPo87uA7nz5ic9/l///fZeO65f+D7378MDz/8\nIDQthT/+8c9oa2vF8cd/DWeccRa2b9+GG2/8BaZOneZ5bWPjbvzhD/fh3XeX44UXns0Z3tFoFA89\n9AAeeeQJVFRU4Oc//yneffddvPzyyzjnnPMwa9a/4YMPVqK1tQUvv7wg6zGGdx9ceecbaO2M46ZL\njsOk8X0/gURENDQccshhAIDq6hFYv34dXnzxOUiSjM7OjqxtjzjiKABAXV0dwuFwzv01NHyBiRO/\nhIqKCgDA0Ucfg/Xr1+PEE0/CH/7wP2ho+AJf//ppmDRp35yP9cceGd5b23YiPOFNSMnD0dwRZ3gT\nEZXY+TMmd1slDwSfzwcAWLJkETo7O/HAA39GZ2cnfvjD2VnbKkp6gREhRM79SZL3OU1LQZJCOPbY\n4/HnP/8Ny5cvw5w5N+PHP74652Nf/eqxff4se2R4f7ztCyjVbTBG70RLZ3ywD4eIiEpAlmXoup71\neHt7O/baa2/Isow333wdqVSqT/vfZ59J2LbtC0SjEVRUVGLVqg9x1VU/xrPPzsOUKSfi9NPPgBAC\n9fUbsGXLpqzHGN69dPykA7G4CZArO9DSwfAmIhqOJk3aD59+ugH33XcnKiurnMdPPnkGfvGLa/DJ\nJ2vxb//2TdTV1eGRR+b2ev+hUAhXXHEVrr32J5AkGUcccRSOPfZY7NzZghtv/AWqqqrg8/lw/fU3\nob7+06zH+kMS+doDykxTU1dR93fjit+ipTOCQyLn4yfnHlHUfe9Jamuri/632dPwHPYfz2Fx8Dz2\nX7HPYW1t7m7dPfY+7y+P2Q+SL4mmcOtgHwoREVGv7LHhPbFmPACgLdk2yEdCRETUO3tseI8JjQIA\nxBFGPKkN8tEQEREVbs8N74rRAADJH+egNSIiGlL22PAeW2FW3pI/xtvFiIhoSNljw3uME96svImI\naGjZY8M75AvCLwcg+eNoZuVNRDQs9WZJUNtHH32ItjbvnUjlsAyo2x4b3gAwMlDDypuIaJjq7ZKg\ntn/+88Ws8C43e+QMa7a6ijFojDWiqSt7UnoiIhra3EuCXnDBRbj99lvQ1dUFXddx9dU/w+TJB+Lx\nxx/Fm28uhSzLmDp1Gg455FAsW/YGtmzZjDlz7sD48eOz9pu5DOjVV1/nLANaWRkCIJdkGVC3PTy8\nxwItQKfWPtiHQkQ0rD332UtY1bimqPs8uu4rOHfyWXmfdy8J+uijf8YJJ/w/fOMbZ2PLls24994/\n4J57/oinnnoc8+cvgqIomD//WRx33NcwefKXcc01P88Z3LmWAf3ww/fx1ltLcc4552H27AuxaNHr\nJVkG1G2PDu/a0FgAQAysvImIhrM1a1ajvb0Nixe/DABIJMzu0pNP/jquvvq/cNpps3D66bN63E+u\nZUDr6zc4S362tOzClCknlWQZULc9OrzrKszwTildMISALEmDfERERMPTuZPP6rZKLjWfT8VPf/oz\nHH64dy2L6677JbZu/Ryvv74EP/nJf+Chh/7a7X5yLQMaCAScJT/XrFlZsmVA3fboAWt25Y1gFNE4\nZ1kjIhpO3EuCHnro4XjrrTcAAFu2bMZTTz2OcDiMRx6Zi0mT9sX3v38ZqqtrEI1G8i4lCniXAQWA\nVas+xEEHHYpnn52Hzs4OfPOb38QFF1yE+voNzmOnn36G81ix7NGV96hgDSQhQw5EEYmnUBXyDfYh\nERFRkbiXBP3hD3+E2267Gf/1Xz+EYRi4+urrUFVVhfb2Nlx22fcQClXg8MOPwIgRNTjqqK/ihhv+\nG7/97Z3Yf/8DPPvMtQzokUcehVgsihtv/AVGjaoBIJdkGVC3PXZJUHvZtp++fgvicQM/O+pa7L/3\niKK+x56ASwj2H89h//EcFgfPY/9xSdABEpCCkNQUIvHUYB8KERFRQfb48A4qIUiqhs4oJ2ohIqKh\nYY8P7wo1BABoj4YH+UiIiIgKs8eHd6XPvFevIxEZ5CMhIiIqzB4f3iMClQCAjjjDm4iIhoY9PrxH\nVZgj+Xa1tw3ykRARERVmjw/v0RXm7WE7OjrQHk4M8tEQERH1bI8P70qfOWBNUpNYvallkI+GiIio\nZwxvn9nnDSWFpvbY4B4MERFRAUo6Peodd9yBDz74AJqm4T/+4z9w+umnO88tX74cd911FxRFwfTp\n03HFFVeU8lDysm8Vk9QUWjvZbE5EROWvZOH97rvvYuPGjZg3bx7a2tpwzjnneMJ7zpw5ePjhhzFu\n3Dh897vfxcyZMzF58uRSHU5eITVo/qBoaOviRC1ERFT+Shbexx13HI44wlx6bcSIEYjFYtB1HYqi\noKGhATU1Ndhrr70AACeddBJWrFgxKOHtV/wAAJ9foK2NlTcREZW/koW3oijOYuXPPPMMpk+fDkVR\nAABNTU0YPXq0s+3o0aPR0NDQ7f5GjaqAqipFPcba2mqM1M3K2+8XaI8kMXZsFSSu690r+SbOp8Lx\nHPYfz2Fx8Dz230Ccw5IvCfrqq6/imWeewV/+8pd+7aetLVqkIzLZK78IISBLMiTFQCKpY+u2NlQG\nuTRoobgKUf/xHPYfz2Fx8Dz237BYVWzZsmX4v//7P8ydOxfV1ekDqKurQ3Nzs/P77t27UVdXV8pD\nyUuSJPhlP2TVXHi9jYPWiIiozJUsvLu6unDHHXfgwQcfxMiRIz3PTZw4EeFwGNu2bYOmaVi6dCmm\nTp1aqkPpkV/xAbIZ3h2R5KAdBxERUSFK1mz+8ssvo62tDVdffbXz2AknnICDDjoIp512Gm6++WZc\ne+21AIAzzzwT++23X6kOpUd+xY9kyhxpHo5xXW8iIipvJQvvCy64ABdccEHe54877jjMmzevVG/f\nKwHFjw6YS4IyvImIqNzt8TOsAYBf9kMXZmhHGN5ERFTmGN4w+7wNGIBksPImIqKyx/BGeqIWyDrC\ncYY3ERGVN4Y3zD5vAGZ4s/ImIqIyx/CG2ecNAKrPYJ83ERGVPYY3rPu8AYRCEitvIiIqewxvpPu8\nQyEgHNMG+WiIiIi6x/BGus87GABiCQ26YQzyEREREeXH8Ea68g5YS3tH46y+iYiofDG8Afhls89b\nVc2KO57UB/NwiIiIusXwRrryllUBwGw6JyIiKlcMbwABJQAAzrKgrLyJiKicMbwBhFQzvCXVrLjj\nSVbeRERUvhjeAIKKNVJNNu/xjiVYeRMRUflieAMIWpW3IZkVd4yVNxERlTGGN4CQGgIAGJJZecdZ\neRMRURljeAMIWgPWdCQBsM+biIjKG8MbgCqrUCQFmhXe7PMmIqJyxvAGIEkSgmoAKWGFNytvIiIq\nYwxvS1AJImkkAABxTtJCRERljOFtCaoBJHQrvDlJCxERlTGGtyWkBpHQk1BkNpsTEVF5Y3hbgkoQ\nAgKBoOCtYkREVNYY3hZ7opZgCIiyz5uIiMoYw9sSVM0pUitCQCSWGuSjISIiyo/hbQlZ85sHQwJJ\nzUAixaZzIiIqTwxvi115B4IGAFbfRERUvhjeFrvP2+c3wzvM8CYiojLF8LbYzeYqw5uIiMocw9ti\nV96yz+zrZngTEVG5YnhbglblLavmbWIMbyIiKlcMb4tdeUNheBMRUXljeFtC1mhzQzJDm+FNRETl\niuFtCTK8iYhoiGB4W+w+b3tNb85vTkRE5YrhbfHJKmRJdtb0TunGIB8RERFRbgxviyRJCClBZ01v\nneFNRERliuHtElQDiGlxKLLEypuIiMoWw9slqAYR1xJQFRmaJgb7cIiIiHJieLsErWZzRQE0Vt5E\nRFSmGN4uITUAAQHVLxjeRERUthjeLj7Fb/5XNRjeRERUthjeLn7ZBwCQVYGUzj5vIiIqTwxvF5+s\nAgAU1YCm9b/ybutK4MEX16G5I9bvfREREdkY3i4+xay8FaU4fd5PvFqP9z7Zjb8u3NDvfREREdkY\n3i4+u9ncZ0ArQrN5PKl7/ktERFQMDG8Xu89bUQwYQsAw2O9NRETlh+HtYjebS4rZZM5Z1oiIqBwx\nvF2c0eayGdq8XYyIiMoRw9vF7vO2K+9i9HsTEREVG8PbxWk2tyvvItwuRkREVGwlDe/6+nqceuqp\nePzxx7OemzFjBi666CLMnj0bs2fPxu7du0t5KAWxK2/I5ujwfjebC1buRERUfGqpdhyNRnHrrbdi\nypQpebeZO3cuKisrS3UIvebPCG8OWCMionJUssrb7/dj7ty5qKurK9VbFF1Ws3mxwlsqzm6IiIiA\nElbeqqpCVbvf/U033YTt27fjmGOOwbXXXgtJGtyUs6dHFZLdbM5mbyIiKj8lC++eXHnllZg2bRpq\nampwxRVXYPHixZg1a1be7UeNqoCqKkU9htraas/vcf9IAIBqLi6Gqqpg1ja94fObp9enKv3aT7kb\nzp9toPAc9h/PYXHwPPbfQJzDQQvvs88+2/l5+vTpqK+v7za829qiRX3/2tpqNDV1eR4Lx1IAgJSW\nBAA0t4TRVBPo83ukkpq1Pz3rvYaLXOeReofnsP94DouD57H/in0O810IDMqtYl1dXbj00kuRTJoh\nuXLlShx44IGDcSge9mhzQ+KANSIiKl8lq7zXrl2L3/3ud9i+fTtUVcXixYsxY8YMTJw4Eaeddhqm\nT5+OCy64AIFAAIceemi3VfdA8St2n7dZMevs8yYiojJUsvA+/PDD8dhjj+V9/uKLL8bFF19cqrfv\nE6fyBitvIiIqX5xhzUWRFEiQYMCsvDnDGhERlSOGt4skSfApPqfy7uk+7x3hXXjsk38grsUH4vCI\niIgADOJo83Lll33QhTVKvIc+7/s+eghdyTDGVdTi9H1PGYjDIyIiYuWdKagEkDQSAAC9m8p7W2MY\nXckwACBpJAfk2IiIiACGd5bairGIGREEv/oqtic3593ulfcbnJ8lzn9KREQDiOGdYXyFORe7pGpY\nrb2af0N3i/ogT+tKRER7FoZ3hnGV6YVUVPjzbifAe8CJiGhwMLwzjK+oTf8iCquoZTabExHRAGJ4\nZxhfOc75OYEINEPLvaGn8GZ4ExHRwGF4Z6j2V+EHX/4h9I4xgCTQGm/r8TXs8iYiooHE8M5h/5pJ\nMLpGAQCaYq05t/GMV8tTebNXnIiISoHhnYOqSBApc7BaLJV7KVLhSmbeKkZERAOJ4Z2DqsiAYU4+\nl8g7AYsnvYmIiAYMwzsHVZEhDAUAkNBzh3chzeZERESlwPDOQVUkQDfDO5knvHuD4U5ERMXE8M5B\nkiQo1pot21o6cm8kvNsTERENFIZ3Hgp8AICV9TuwsyWS9TxHkhMR0WBheOdhhzdkHZ2R7pvO2SxO\nREQDieGdhyqlwzsX4bpXzBD5lw4lIiIqNoZ3HnZ4S0qe6VFd3EFORERUagzvPHyKz5yIRdaR1Lqv\nrA2w8iYiooHD8M5DlRXAUCApOpKp7KZzd7HNZnMiIhpIDO88fKp1r7esI5nqofJmszkREQ0ghnce\n5ixrKiRFQ0LLUXm7f2blTUREA4jhnYeqyAVX3nqe8GZBTkREpcDwzkOWJXN+c1lHIpljxLn7VjEO\nWCMiogHE8M7DMIQ5YE0WSGiprOe9zeY9lNicw4WIiIqI4Z2HYQhAN+c3j2mJ7rdlnzcREQ0ghnce\nuiGcZUFjqXj2Bp5bxdi5TUREA6eg8F67di2WLl0KALj77rtx8cUX4/333y/pgQ023RAQyQAAIKKH\nu92Wfd5ERDSQCgrvOXPmYL/99sP777+PNWvW4MYbb8R9991X6mMbVIYhIBIhAEBMdGU9z1vFiIho\nsBQU3oFAAPvuuy9ee+01nH/++Zg8eTJkeXi3uJuVdzfh7VmYhM3mREQ0cApK4FgshoULF+LVV1/F\niSeeiPb2dnR2dpb62AaVIQREMggASCLXet7pwK7f1pZzxDkXLCEiolIoKLyvueYaLFiwAD/96U9R\nVVWFxx57DJdcckmJD21w6a5m86Scq8873VS+uy2CpvZY9hZ2djPDiYioiNRCNvra176Gww8/HFVV\nVWhubsaUKVPw1a9+tdTHNqgMwwAMFUJToSvRrOfdlTckkQ5q9zZW5c0KnIiIiqmgyvvWW2/FwoUL\n0d7ejgsvvBCPP/44br755hIf2uD60rhqAIBIhKCrkawAzry3O3ezub1taY6RiIj2TAWF9yeffIJv\nf/vbWLhwIc455xzcc8892Lp1a6mPbVBdcsbB+N7Mg+DTqwFZR0cyo49fSieyxMqbiIgGUEHhbYfP\nG2+8gRkzZgAAkslk6Y6qDFQGfTj56AkIiBEAgMZok+d54b63WxI5VyGxA53ZTURExVRQeO+33344\n88wzEYlEcMghh2D+/Pmoqakp9bGVhZAwP+fOsDe8vROziJwBzcqbiIhKoaABa3PmzEF9fT0OOOAA\nAMDkyZNxxx13lPTAykW1MgotALZ37fY8nll557rXO+lrARSJfd5ERFRUBYV3PB7H66+/jnvvvReS\nJOGoo47C5MmTS31sZWGkbzSAXM3mmaPNvQm9qf1ztO31OvwVYyBaTy71YRIR0R6koGbzG2+8EeFw\nGBdeeCHOP/98NDc344Ybbij1sZWFmmAVhACi1uIkH3zaiBfe3gJkNJvruje869s2AQCUmhb2eRMR\nUVEVVHk3Nzfjrrvucn4/5ZRTMHv27JIdVDmpCKpAVIJm6ACAB55fCwA4cLLrukcS6EqGcdt7D+Oc\nyf+GQ8cchNZ4KwBApHzs8yYioqIqeHrUWCw9g1g0GkUi0f0a18NFZVAFhAxN1z2Pp9y/S8DqjlXY\nEdmFBz5+GADQEm8DAIhkiH3eRERUVAVV3hdccAHOOOMMHH744QCAdevW4aqrrirpgZWLiqAPEBI0\n4Q3vpK65fhMQGQndaoe3prLyJiKioioovM877zxMnToV69atgyRJuPHGG/HYY4+V+tjKgll5S9AN\n74xqKS0d3lLGgDUhhFN5QzYY3kREVFQFhTcA7LXXXthrr72c31evXl2SAyo3duWti+6azYUnoBN6\n0pk+VZJ1DlgjIqKi6vOi3HtKNVkZVCGEbC5U4pIy3GEunAFtABDX4+mnFH2POVdERDQw+hzekiQV\n8zjKVkVQBSBBhze8tYzKO2GkB/DFtHR4S7LOAWtERFRU3Tabn3TSSTlDWgiBtra2kh1UOamw+ryF\n8PZdp3QNAdd2yTzhzT5vIiIqtm7D+4knnhio4yhbiixDEjIMpKC5J2KR3TOsGXkrb7DPm4iIiqzb\n8J4wYcJAHUdZkyUJAgZSWrqpXJJdt4pJQMod3qmoazsDBpjeRERUPH3u8y5EfX09Tj31VDz++ONZ\nzy1fvhznnXceLrjgAjzwwAOlPIx+kyADEEhprn5v1R3eAkmRDu+2RIfn9ULSQEREVCwlC+9oNIpb\nb70VU6ZMyfn8nDlzcP/99+PJJ5/EO++8g88++6xUh9JviiRDSAaSrvD2VN4QSBnp9c2d8DbM0ysk\n721mRERE/VGy8Pb7/Zg7dy7q6uqynmtoaEBNTQ322msvyLKMk046CStWrCjVofSbLCkABOJJVwgr\n3klaUsIV3vF28wctCAAQYOVNRETFU7LwVlUVwWAw53NNTU0YPXq08/vo0aPR1NSUc9tyoMgyJFmg\nI5JuGpdUb+Wtwd1sboV3yhwf+2w3AAAgAElEQVSPzsqbiIiKqeAZ1gbbqFEVUFWlqPusra0uaDtV\nMU+TUFzXOlblLTQVkj/puQu8PWk1m1vhDVkv+L2GouH82QYKz2H/8RwWB89j/w3EORyU8K6rq0Nz\nc7Pz++7du3M2r7u1tUW7fb63amur0dTUVdC2spABCdi2M31vu2SHt+5zqvB9qvZGQ3gHuhJh87lU\nABIAQ9IKfq+hpjfnkXLjOew/nsPi4Hnsv2Kfw3wXAiUdbZ7PxIkTEQ6HsW3bNmiahqVLl2Lq1KmD\ncSgFUWTzNHVE0/3akDXz/m3NvP6pwlhMm+AdnCecZnP2eRMRUfGUrPJeu3Ytfve732H79u1QVRWL\nFy/GjBkzMHHiRJx22mm4+eabce211wIAzjzzTOy3336lOpR+UxUF0IHOqGvaU1UDdBWQzHu4fQhA\nldOn0y/7ENet5nb2eRMRURGVLLwPP/zwbpcNPe644zBv3rxSvX1RqbIV3rH0oDQoGoSuOn3fighA\nkdN98j7Fh6iumE0bDG8iIiqiQWk2H2pUK5S7oq7R5opZeUtOePuhSunwViU1fZ+3zGZzIiIqHoZ3\nAXyKGcrhuN3nLZzK2x6spghvs7kqqxCaFeYyK28iIioehncB/NatYl0xK7xlA5IkzD5v2A/5PM3m\nqqxA6NbvisaVxYiIqGgY3gXwWfeX68KqoJ3bxNzh7Tebyi2qpMIwrN9lnUuTEBFR0TC8C1Dh91k/\nmRHszGvuCm9J+Jy+cQBmFW5V3pJVeXdGk7j/2dXY1hgekOMmIqLhieFdAL/PCm/JmkdNsSpwwzXj\nm65mNJur6cpcMdf0/ufyrVi1sRn3Pbt6AI6aiIiGK4Z3ARTJOk2SgCSlK293szkMNavZ3A53STYr\nb3s98GSKA9iIiKjvGN4FUOxbwCSByqAPvoDVg+2qvCXd22yuuprNoegw2OlNRERFMmQWJhlMslV5\nS5JAZciHsGrAACB0BYmNR0EZ2QRZqvbcKqZIKgAZQpedypuIiKgYWHkXIN1sbuDEr4yHL2D1fRsq\njLbxSG35CoRhB7b9GsXZxu7zdkjSwBw4ERENSwzvAshWEP/7qZNx5tcmweczk9i5jxuArouMZnPV\n2UbKvM+bVTgREfUDw7sAduU9fmwIkiRB8dmVtyu8hchoNndV3jL7vImIqHgY3gWQrSVBDWGGtqxa\no8Vdo811XXjmNreb0IWuAIoGwzDSO2SzORER9QPDuwB2Fa1b4S1Z93m7m80NQzgD2wCkg1xXIUlA\nyuDiJEREVBwcbV4AO5S/6NyGz9o3A0rKfMJwVd6GAclVUctOs7n537iWXguceuetj3dgQm0lDti7\nZrAPhYioLDC8C2D3eS/e+joAQLZWGfMMWDPSk7CYr/Fuc+fqe3AELhqQ4x1OYgkNjy7cAAD4yy9m\nDPLREBGVBzabF0B29WUDgJCyJ2nRDYHHXql3frfDW1LNKj2hJyBghntnJIkHnlsDg6POe6TpRs8b\nERHtYRjeBVAk72kSMMy7vQxvn/fGhnbXa8zntN2T0tsgXZl/UN+Enc2REh0xERENZwzvAmSGNwAr\nuNN93PGk7unztkebG51jobWMN3+Gd05znfeP9YhniIgoG8O7ALKsZD+oe4cLRGIpz+8KXK+xKnQD\n3hHnDO+esWeBiCgbw7sAco7KWxgZ/eAwB1c5r5HdK45Z94lL3srbYHj3iOdoePvX+t247I6l2N0a\nHexDIRpSGN4FyN9s7hV2Vd+K69TaQS/YbN5rDO/h7c8vfQLdEFi2eudgHwrRkMLwLkDmaHMATjXt\n5g7j5vZk+glhbtssbYLkT1cYKY6k7hFH5A9v/PMS9Q3DuwC5Ku9RVaFuX7P4vW3pX6yg362uQ/Co\nt5yHUymGd08Y3kRE2RjeBcjV5z1+VBWqQj4AQCjQw1w3OZrYAVbehWCzORFRNoZ3AZQczeaqrDrL\nfFZX+Lp9vcjRxA4AyZSe83FKY3jvGbhWD1HvMLwLIOf4ZnEv/1kdyg7vQyeNxrUXHoWvHTouu/KW\nNQACb3e8jHe2vwcAWPDOFsxd8ElRj3s4YHYTEWVjeBcgksq+jcW9/Gd1hT/r+ZHVfhy272jzOeE9\nzZI/DskfxxfJT/HEp88CAJ5ftgUr1u0q8pEPnLWbW7BibfGPn5U3EVE2hncBJo3YBwDw5VGTncfM\nZnPz5xGV6fAWmlmFj6kYCQCQ5exmcykQg+RPrzJmrxMOwGmKz2f+ss34+LPmPnyK0rrrHx9j7kvF\nbznggDUiomwM7wJU+6vwwIw7cOa+pzqPqa5Z1/yq7Axei6+ZisTGo3DUhP0BABUBNavZXArEIAVi\nzu+NkXQYdxdWndEkXnznc9z7zOr+faAS6unio7fKObxffGeLs+IZEdFAYnj3guIKbFVWPfNu71NX\nBQCoCYzAdbPOwKTx1QCAiqAv655wSUlB8qfD+4GP/+KsEa7p3YR3JJn3uXKR1Io7gr6cm83nL9uC\ntz7eMdiHMaSV8bUZUVljePeCu59blVQ4y2ZIwMGTRgEAamtCzs8AUBlSs/q8IQlP5d2aaIW692YA\ngN7N7WPhaCrvc+Wi2CPoyzm8iYgGC8O7F7Iqbye7JZxxwpfwzan74rJvHOp5TWXQlzUPOmTDCe+v\n7zPd3F9tAyBrWZV3Uk9i4ZZX0Z7oQGe0/CvvRLHD23U6/ufvH6KxPZZ/40HCC4y+4y1iRH3D8O4F\n9/3e7j5vSQJURcbZ0/ZH7UjvzGuVOZrNIRmQ/HEoIoBzDzwLB4a+AknVIPnj0DIq73/Uv4CXtryC\nFzctQke4/MM7WeRZ49x93vUN7Zj32sai7r8YONlO37HZnKhvGN69oHbT551PZUjN7vOWDEi+JFQj\nCAAwdOt5SUBzVXHtiQ6s2LnS+b0jo897xY6VWLBpUS8/RWkVu/IWGVVtOS7mknnBRURUagzvXvBU\n3pKCQtK7MuiDENnN5lBSkI0AAEDT0o+7+7zvWzU3/RJJdgashQLm/h7f8DQWbX0dulE+M7UVu887\nM6zLsYk6VeRBekREPWF490L2aHMzSLrrtzNvFcucpCUBSQIk3QzvlDUOTZIM6Fafd2ckieZYi/Oa\nqBZzKu+qjBndolr59AMnSthsnuv3cqAxvPuNfd9EvcPw7gXPaHO5h8VILLIsZd/n7TMnaJE0c3IX\n3S5WJQOaYQbBtX98C7rQceDIAwAAsVQMHZEEAMCvKp77qbuS4d5/mCJyH0vxR5tn/l4e4a27Dox9\n3n0nCup8IqJMDO9eUFyBrcqq606xHsoG4X1e8pshbM/GJgzreUkgkdTxxqrt0GWzyq5UK+BX/Ihq\nMcQTZjDqhkBcT8/QFklF+vyZisHdtF30Pu/MyrtMwlvT0sfR3b35ROXglZUNWL+1bbAPg4qI4d0L\n7nW9PQPWemzyywhvnxnMwqq8Dd16Xjbw7Fub8bfFn0JSzI5wvxxEhRpCTIs5FZ4hBLqS6cAO55h7\nfSC5w9tdeacMDXd/+Ces2LEy18sKkt1s3uddFZW72uaANSpniZSOp17biN8/uWqwD4WKiOHdC1kD\n1iw9ZXfdqFDOx42kWXk74S0Z2N5kNoFLqtkRHrDCO6rFnYFRhiEQTrnDe5Arb1d4ufu8t3Y24LP2\nLXh8w9N933eZjjZ3BzYHrFE5K5fWKiouhncvSK5RNYprkpae3Pz94zC+60QkPj3G83gqYTbD233e\nkiTgXApY06X65QBCaghxLY6UZm5oCOFpKh/sZnMtT+WtGVquzXsl84unXAasuQepsc+7H6w/Z+bY\nBiLqHsO7j3yyAvf0qN0J+lVMCnwZRkcthKv/OxaVYRgCuqvytkmqGXw+KYAKXxACAklh9pUbRmaz\n+WBX3q4+by0d3rmWUu2tzLDOvO97sLgvWDjavP/K5aJsOOK5HZ4Y3n2UOT1qTxTFOtVGeluR8iMS\nT0HX0n3e6RdYlbcUQIVaYT1vPmYIIJxKjzAPJ7NDMvMfbCKl4911u5zqvZjczebJZPrnLtcxLnrv\niz7tO/N7Rx/gLyJNN7BkZQOice+88u7AZp93/7Fpt3TKpauJiovh3UfmwiSmQu5R9dnh7VqkROgq\nWjsTcPLUGm0OpPu8VRFASA1ab2pW45l93pnN5l/s7sIPf7cUb3603XnsuTc346EFn2D+si0FfT63\ndVta8doH2/I+7xlt7ro4CLtuYVv4Xu/fF8jRbD7AX0TPv7UZT762EX9fUu953N1Uzmbz/mN4l065\ntFZRcTG8+6jQ+7xtimIlvHuFMV3BLY+uRDhid3obTsVsh7cCPypUc8CbZFXjhiE8TdI7Irs8s6wt\nX7sLAPDU6585jzU0dgEANu3o7NVxA8Cd8z7C35fU521+y9fn7b7/3JD7tiJa1gxrA/w9tHFbBwCg\ntTPheZwD1orD/nOyabd0mN3DE8O7j1RZ6dWiCnblLQz7vxKc028FuiS5m83NKlsRfgTUgPVYesBa\nXDPv8z669itoT3Rgbct656V2FSP7EqhvMwPcp5qj4/vTbJ6vOvKMNk+6wtvVIiAUb/gV/J5Z93kP\nbFC2dZnHPbI64Hnc22wuEI2nsO7z1gE9tuGEAVM6bNUYnhjefaRIhU2PalNVO6itjQ1X5S7Sk7TY\n7CpbNgLpJnopfatYzArv0yedAgB4a9sK57VOv/A+a3DvqofwcdNa+K33T/ajSszXt5tvkhZ35d3X\n8M5s8hvoUcntYfO4R1T4PY+nXIP0NM3AnfM+xp1PfYT6hvYBPb6hzv6nM9AXZW5CCCz9cBt2tQ7u\nfAmlwlaN4Ynh3UeqrOLkoycAAA760qgCtvc2m8vCHd7Wn8E1YE3yJyAMCbLwwWc10UtyepKWmBaD\nLBQ89sIuHDhyf2xo24hdkd3m7uzAC5lBMn/Ty/D5zPdI9WPu8XwDX9yjzd39v+5BdYbct+VMM9/S\nEAKabmTNvFYq9mfOfD8tY5KWLTvN7oimMlxvvJw5zeaD2POweWcnHnulHr+a++7gHUQJsfIenhje\nfeSTFXzntC/jjh9NwWH7ju5xe6fytprNZeSqvN3hHYNIhqDrrv512Wo2N4CYFofQfdi8vRMnjDfv\nH/+0bZP5vN1vnjJHqTdGmyGpZgWZ7EezuZ5jGlAhhGeeb/e0oRHXKHhD6Wt4Zw9Yu/z3b2DO3z7o\n0/56w90FkDkoTcszYI0LbPRNb6vD9zc04sEX1xWlqozEzC6q4VqgsvIenhjefaTKKmRJwtiRuWdP\ny9o+Y7S5e7S63Q9uN5srqg7Jn4RIhKDpBnyKtYqY5K6844BuTtEaUioBAM+/vRHN7bF0FaOkB4nF\nVXOFMvfgqriW7hMvRGbl3RZvx/ee+ylWtb0PqGY4u0MtYbgCW+1bs3n2gDXzd7vSLaXWrvT88cmM\nFgv3eXT/LGekdyyhIZbo/2Q1A03TDWzd1TVg79fb6vCP89fivU92Y3cBTd2vf7itV5/FMATunPeR\n526NoYyV9/DE8O4j91SphVCt0eb2JC2q5FrW0x6wZjWLT5xo7lskQkhpRlazOWCGt6GZj8swt4+m\n4nhx+efpK20lHRqblXcANenp835iwzO4d9VD+KhpbUGfQc+oPjd1fI6ElsCyliUIHvkG4Is74W0I\nA5qhQYHVV9zHyjuzz3sgFwGx108Huq+8NU/l7Q3vK+5+C1fc/VaJjrB0HlrwCW55dOWA9eH3tTp0\n5k/Io7E9hsdfqcctj+afXz+ztaSxPYZ1W1rx10Wf9umYyg2ze3gqaXjffvvtuOCCC3DhhRdi9erV\nnudmzJiBiy66CLNnz8bs2bOxe/fuUh5K0Vz+le/hrP1metb2LoRdeUtWda1KKn71PWu61IxmcyVo\n9puKRAU03Ug3m9vN6rIBXegwUnZ4p5/fvKPTudIWcgp1FWMBAElEoY773FMl2iPUN1rN7T3pbrIH\nSTGgjGx0giypm8EXRJV1Avp2q1jSSHpaEIq95Gh3uqLp982cRU3zDFhzDTTsY7N5S6wVN7xzOza0\nbuzbDors/Q2NAFBQZVsMfa0OMy8oMxXy/0vmn2y4dX2w8h6eShbe//rXv7B161bMmzcPt912G267\n7basbebOnYvHHnsMjz32GMaNG1eqQymqI2sPxxn7fb3Xr3Oaza3wliDjgL1roMiS0w/ujDb3m1+Y\nduWdsjPErrytMBO6VZFb64VLio4dzRGzipEMQNYxOjAKFx30LfP5jACt9JnN7YVOr6plfAnYI95t\nysgmZxR2QrdmiDMqrffuW+W9UnseoWNegz20qbezRdW3fYZH1j2BVB/mWe+KuirvjLECqTxzm/e1\ngnz1izfRlmjH3DWPFbS9EAJPLKnHui2lvT2tKuTreaMi6Gu+9HSPfUE5LHX765DHPu/hqWThvWLF\nCpx66qkAgAMOOAAdHR0Ih8M9vGr4UuzR5s7tZVbftyJD2KPN7T5t1aq8k0GkdAML3ramFrUGrNnL\nhcIKbwjF83wkrjkBH1KDOGj0gZ7nbVU+c0BbOJk7vNvi7Xh03ZOQrIuJzCrHvtf8lJFnw4hXQK5u\ncyrUlNXfrYgghC47y6D2VhhmOMk1zX16/b2rHsL7uz/CxwV2Dbh1uirvzJDwDNJzN6FrfWz+tVpy\nNFHYRcb2pghe/WAb7pz3UZ/erzvukfWZF2yl4q4OOyNJrN3ckndbz/H11I3ShzJ6uGXdnlh5G0Lg\nd3//EP9c8flgH0rJ9G6asF5obm7GYYcd5vw+evRoNDU1oaqqynnspptuwvbt23HMMcfg2muvzeov\ndBs1qgKq2rum6p7U1lYXdX/dGdlsNT/azeaKitraavhUGYmUt887FJKBlFlZ+/wqWtpTwCjXJC5W\neAvdrIpGjrA+hxXOmiGchU1GVY/AXnXWrWzW/u3PXREIAl1AzIjmPBfLPnkbK3evQuAIGfH3T0f1\niJBnu+QXZrhVh6ogkgHIwSg0w0BtbTVi7eaAMkX2QST9gJrM+R7vbVuFUcEafHns/p7Ho/EUKoLp\nqk8Zux1GR61nm978/Xyh3v+9U64vPUOSPK/3B9LHJrv6XYMVfmc7dytBT+8dCJj/FDVDK+g4O+Lp\nC7Fi/3/c1pluUQm5Pk8pqT7FeZ9fPLQEja1R3Hftydhv75qsbSOx9EVVVXXQeV2u44y7rrnyfY6a\n1phnm5he+N9tKGgKpy+cC/k8w+Ezh6NJfNrQjk8b2nHJN78y4O8/IP9mSv4Olsz7ZK+88kpMmzYN\nNTU1uOKKK7B48WLMmjUr7+vb2orb91ZbW42mpoEbTdvVZX1BWAEqdKCpqQuyLOWYpMX6YhYyOrvi\nUCUFCddrneZva8BaW6s5ktsO/65Iup9Y0hR0tVnPWzO0NTZ2QpIkdMbMintXuAlf7GyCX/Z5+vI7\nwzFnv1IwjJaWCJpC5nsmkjpefOdTqOOARFQ4rQApI4mmpi7s6jAHOukpCdD8kIKRrPNtCAN3vvMQ\nAOCBGXc4j2/Y2oY7nlyF807e32yokAClphkpyfBML9ubv19LR5dn+22NYUgSMKG2yrPdZ9s7cO/T\nH+Pq849EY0u6RSIWT3le3+EKuIireb2tPepsF0+mq+jujvWtxmVY9Nkbzu/23ydTMqXj3U9247iD\n69DWnv73UKz/j1es24VdLVEctl/61sfWtuiA/DuJu85vo9XPvnFLC6p82Y2Dja576Ztawmiq9uf9\n99zSkm7ty/c5OjLOZVNzz68ZSlpb0/8f9/R5Bvp7sVQiroWEBvrzFPsc5rsQKFmzeV1dHZqb002d\njY2NqK1NV05nn302xowZA1VVMX36dNTX1+fazbDh3ELk6vMGkNHnbQ1YU+1FjmVougG/Yo3Ylg0E\nfIrTbG73ecPwNpvHk5qzTYUagk/2eZ5/7q3NeGfNTqfPOqEncd1bv8b1Cx/CZ9s7nGN2z58uBSOe\npuKuWNJpAZAMn3MsQtagG4bTbA5dgdB8kBQdCc3bdJ7Qc98+ttIaLLVw5RanA1JSNchVfR/5HE15\nJ0/59V/+hRsf/lfWds8s/QyRuIZnlm5yms1HVPg8zea6YeCL3el/nO4+b/e98O6R/d01Xc5bu8Dz\ne0TLfaG6YPnneHThBjz56saSNO3OXfAJFiz/HM0d6XM1UPO25+qXzddk7668e1qOtZAxEpnvM9xW\n4ervx+mIJPHBp03FOZgBMtz+hrmULLynTp2KxYsXAwDWrVuHuro6p8m8q6sLl156KZJJ88t85cqV\nOPDAA0t1KGUhff+vNe847D5vKV1NWuEtK1Z1bshIaQYCavo+74BPTt8CZjWb6xrM6t0K51hCd6rz\nkBqCIiuQhAzJev6fK7bi4X+uzxpwFg5twd3/SPehulcFU0Y1oiWRHhxlGMK5QJCFz6m8JUWDpgkk\nrNHmwlAgUubFR1vcezUaTaXf372wiv1FLqvmY0I3L07kEd5+UPMiQcsK5lw6k7mvhA1hYHc0/cVk\nT6aj6Qa6oklUhXwI+BVPiK1YuxtrXQPFtDyD19yz2el5phAzRPbjLbHcg9B2tZih/vmu0t7j3tKR\n/ruUMrzdrXG5Lm7yjST3hHcPo80LGayV+d7D7YvffQ76MjPh//z9Qzzw/JohNfVvrgmlhpuShfdX\nv/pVHHbYYbjwwgsxZ84c3HTTTXjuueewZMkSVFdXY/r06c5tZKNHj+62yXw4kK0Ba3a/tV15T6yt\nAmA1nctWVW6FNwwFKV0gqKbv8w74FSek7cldNN2AJBSn2Tye0JyAt5cTlaB41wuHQEJPoNJeKxyA\n0FTPVbq78lZrt+PvDQ8imdJR39BuTlpih7er8oaiIaUbSOr2iHgZ0Mzwrm/x3pIW1dKh61772/4y\ntS8OjC6zGVcOeQc8aprAw2sfw8+W3YRIKuoJccMQePHtLc5kOJ3J3IG3cMur+M27v3fudbfvCtB0\nga5oCtUVPvhUb3hv3tHh2UfKM2DNtba5PUJd0pHQcg9Ei2vZrQ/5Rv/b/w/phijpl1OLq0uglMud\nunMkZ3jnCdGwK7x7Or5CgjgrvHvY519eXo///r/lPe63XLg/X19Gntu3Cw6lqX/zXSwPJyXt877u\nuus8vx988MHOzxdffDEuvvjiUr59WXEqbzugrdHml5xxMPbfewdeiSsw7AFp9n+FDE0zYOj23Oe6\n2Wwup8MdsKoj4Qp1pG/NspcTlYXqHW2uaBAQ2K9mknO/t4hXpudgR+4Q+cvL6/Gv9Y046/9NgqRo\nELoC3YCn8tZ1A0nDWr5UV2AkzGOYt+kZHD/hSAStVdJirvDuSHRiZMAcnGR/v8jWoDsRr4DQFUhB\n7/GkdANrms1j//mymwEAPzjsIhwz7ij8a/1uzH97C0LHCEABOhPp4HdXa+/sMCfvWLV7DY6qPdwJ\n70RKRziWwoSxlYgndU94B/2utdxlAy0j/gUlPgJSMIIW3Q/AHHyXTBmAZCB45FuYv6kT3z3sW1nn\nM7P1AzAHreVi37FgGKLHirM/8lXeiZSOrmgSY2sKm1WwJ+4gyZWx+YI3Ek+fn54uYgoZaZ35Pj0F\n/turd1rbGVDk8p/nyhPehkAP89rkNZRaJIbSsfZV+f+fN0yMqQlaP3mbzasr/Pi3KftClc1wkkc2\nIpwKQ7Kq8ZRuIJGy/keUzD5vJ9ytyjulGea93q5wloLm1fLY0Bjzd6E4zeZAetWyoBLElV/5sfmg\nrHtmrAqnIhgd9C668q/1Zn/0xoYOc1CcrkI3BISRWXlbzea6DL3xSxBJM7Cjrv5cd3gv3rrUqdad\nudntixFdhYhXWp/JfZtQdoDtipjHZ37BC2cZ1Q5X5e2e6tReS317s9msbs+E12atJmZW3rInxGLW\nQLTbLjsBoTFtSFR/Dv8Bq+GbsAmrxItOU3hS0yH545D8CWzsyD0NbVw3gzKoBHDKPicCQM570qOp\nmPP31Q0BrciVhbs5tdm1drm7JeGOJ1bh539a4Zl5rj/0HirCfBcoUdd0sz1V3oWFd+ZtgIV98Wd2\nKQgh8PFnzWU3Ha773PYn1IbSLWdsNqeiGVUdwG2XneBUywq8k18okgJJ1RD48ofYEdllzaomIaUZ\nSKYEhCFDkg34fa6QFq5lPo2McA5EASFhTMhscpZyVN4AsGZjJ3738GdQjRAgG051J4RAOBVBlTWR\nS6akpluVt2p++en2RDEaNF044W1oMiBk6O3mYEU7oAHvILKPm9bipS3mGIms6V11FUas0hz17k+/\nRtMMyJL3f2FP8Lmmh+1IdDrv51621J4AJ2m9zl533V6UpLrSD58qw3AtwBK3ngv61Zwz7dmfMakZ\nkHxmOLfEWz2f3WZX3idNnIqJVXtnfwaYs9XNee9ObAq+CkBYK6sV98vJHUSeytsVjvZ88u5m6/5w\nh0GuUMn3GfU83RQ5t+1L5a27jyv//jOPb/naXbj3mdX466INPb7nQHJ/hP5c8w2lanYoXWj0FcN7\nAO01phIHpk6D3joOU+r+n+e5zBHGftkHVTFHmyeSulllywZURXaazYWr2VwYCiRfylkARA5GIeuh\n9LzoIqMyt5qk7XlzzIsD3ak8E3oSmqGhUq1AcrN5n6Q9hzoAJDTdDEddhaYJT5/3X15ej664GZS6\nZq+mZr7WDnXAW3kDwEeNa8xNnT5vb+UNAFIo3XSe0DSnYjyq1jzGlGEHp56ezAaAgMD7uz92nks/\nYU9ba430z2hTHFFhhjeQDji7sgr6FShq9rdh0khCNwxsbwxD8iec998VzZ4C2J7oJqQGnb9VKiPk\nP2hcjY5kJ8LyLsjVbVafd3Er77hrBTXPimnWZ3avsFasL3F3tZ85h735Prk/o2dq2j40my98dytW\nb2rJu02+VfIyZVbe9iDGTdtLv2hOb3gGBvbjNoVi/z9XSkPpQqOvGN4D7IpZU3H1cT/AtMO+1O12\nqqzCp5qVdyJlhndNtYq6kaH0wDO72Vw3zIFhAEJfXQqoCUj+BJRUFYQQ+OPzaxCNCUiygNPsbM8X\n7p5iVU734dn93RVqJTsCVIYAACAASURBVPTmCdA7R8OADsBuEk5Bks3Qjic1T5/3Z9s6sGqTGVS6\nZlXyVmVu94UDQDSjv7cl3obGaHO6/9OpvBWIlNns7p5mNZyMQkDgyNrD8c39Z5rnwqpaw7GU83q9\nrQ5CAG82LEdjW8QTRPY99kKyBt9l3F49wmo2B9Jf1vGEBglAwK84I+LdknoKTy/dhKde/wzwpZug\nd4R3ZW1rV95BNQjVuqVPM7zhvXLXh87PytjtVp934V9OWzq+wJ/XPo5wMpJ3tHE8zxzg9mduaDSv\n8tQJG/HytpcKfu/u9NRsnm+kuztce2w2z9hvNK7h6Tc24Z6nP05v002fd+b+3ecvc8rcZmtA11in\ni6w86D20cBS8nyE09Vyxu5XKEcN7gPl9Cg6eNKrb2eQAwCer8Cmy1WyuQ5FVCDkJQ04BkvWl4fR5\n604VDgDKSPP+eilZiVhCw/ufNmXdCy75zdCwQ1FYfeZ25b0zYgbNCL81QYDzeiu8EXFeH4lrnsob\nSFfYuqZ4Xp9wVd6fN5mVynXH/Bhn7GtOpdueaE9/Qcqu+9l17/EDQJc1rWuVr8IJPrvyjsRS6dHq\nsSroLXthV2wXfvXss3jh7S3pE23tLwnzizfzy7raVXl/vH0zVu/YjFhSRzCgQJYkyEqu8E5imTWo\nya68ge7DO6QE0pV3RrN5S7wN1b4qyEKFXNmRNWBtV2Q33tuZf33zf9TPx6rG1bjx5b/i9sdzbxfP\n009rn4+GRnNMgG/CJqxu/xDhWApPLKnvVxO6O0dyZUrmMqw276IwvWs2D8ey++u7azbPvIBwH1Pm\nc01Wd8OoEYFuj2mgGT3cklfwfoZQNbsn9HkP2Axr1DuqYt5fHEtqSGoGAlAQTnXhXflvgGwu4iKE\nq9lcl5wFFeRqs0lQSoUQtkfmWkHv+9IGpD4/DFLAXrnMHDls6GZzvH070spdq8ztIxMAtDmVM2Qd\nMFTEpU4oMEeoR42Uq/K2mrqtCwwtZVW2OZrN12zdBXUsMMJf5dzSFtPi6S8J2T52NT0JnSss7TnZ\nK32V8Cne4AvHNE+fubbjAKhjd0Ie0YpVG9OTB9n3w8eMMO54/37ElNEA9nKe19ROrK94BsrY/fBk\nwyLz/RJnOyPOJVflbUSrIFeEkdCTqAgoiCU0p88bAHZEssM77qq884V3OBnBmNAoSMlKdIR2Q0fK\nEzi3vncnAKAmMAL71UxCwJ7Uxz4uawBdomIbNm34ctYxAN5mczc7HKMJLf33ADD3pbVYs6kNmiHw\nvZkH5XxtT/L1ecuSBEOIvCuCefq8ezlgzT1ffa73BrxVW+b+3bPmuS/0hBDOQL5yC7nM0eZ9NZQC\nkc3mNGh8soqAT0VXxPyySfc3Cydw7C/7ZMqAcC2bKVeYVZIwZIStLys7PNW6bZBrmiFb4W3Ezfu8\nDatvWlENGMLA6uZPEMIIvPCKNWGIYc+/bo149pnNqCJe4am81boGyKN2Oc3Qeko2mxFzhLd7xLs7\nvJ2Kxj2TnD0Lnavytu9Dr/RVOLPI2f3F4VjK2b/QVAgtPdGNh7WNhhS2djag0f+x5+k2YxeSUgT+\n/dMLm8STOkKBdDcBACTWHwe9dbzzGYP281blXaFU5q68dbvPO2Teiw/vrWIpQ0Ncj6PaV4VqqRaS\nBBiBzpxNyvd/NBd3rLwPQgi8/uE2ayY2gdZ4m3ksqubchZApka/Z3AooTRee127aaf5/YfSjedId\nJO4+b7v1J6nluaDoplk7U+aXeFeOkfLdNptrmeGt53zOfftavhYDIQSefXOTZxbDgeAZbd6Ppu+B\nWqSmGPaE+7wZ3mXi9Emn4OBRBzr3OvtkFUG/4vzDE3L6S8eerGREyAy8aELzNM/KlWZ4G7qcbtbU\nXaOiJQEpEDOraWsCFfteclk2oBk6UkYKWjQEZ35Su9ncqnxl64vciFeiPZxIr3AGwDfhMwgrZFMp\nCaOqA1AlMzyT1rSpumFO8iKEOUNb0BXedsUl5HSfd2azPwBENKvyViucCxk7+CKxFKCmK3e4lk1N\nnwcjPSFOPkp2c3IslUDQnx5dDwBC8zvvsaO1AxV2ePviEJoPtYFx6Eh2eia+AbwD1pZ+YIb79tb0\ngCd7lrsqfyWqYN72JwKdeQcP7Yo2YmP7Jjz+Sj2WvN+AjmSXZzIcuTJ7MFVcS2Bly/KsVeeAdEBp\nugHZdZ99NGVdlAT7vmSonmcglWKHd54Q9FbehQ9YE0KgM9q7ZvPsyts1sM89IY/r4iffRcfnu7rw\nzxVbcftj+bs4SsGd17kGBvbEPb/AUFGs1oZyxvAuE//fAWfgJ0dfZt0iBqiy2WxuS0npL2A7qGsq\nrfCOa5B82TN1CUN2+vjcfeKSrEMKRK0mc8nZFgBk1XACUNcl1768fd72hCkiXoH2cDIdrjCb0u3K\nW+gKVEXGiKDZPG/fLhWOpszPofmh6cKpvONaHAnNACCQ8rUDwgxGu9nefTucHUqVvgrzVjtIzoC4\ncCzlDG4TKX+Oyl3Af9BKz2fPJQVr/vdPj4HeYYYnanYh4LcvqlyD6qxz8MTrG8zKXElBCkZhRKsw\n2mfeKpdZfcdc4b16o1khN7anJ5SxZ56r9lVBMsygFJLebRW0rvlT5+ftHebAwUlV5gBJKZQ9Teyz\nG1/EB13L4Nvn06zn7PBKaYZ5+6HFvmjpzz3NIk+zuT1oMpmnP7uvfd66Yc6alykz4LuvvHM3m7tb\nLqJGFz5sXJ31Pok8XROl1t8Ba3Z4r9ncUvKpeYuluwuw7vx10QZnEp5yx/AuM4p137JPVhH0eatl\nN2HIGGmFdyyhQW8dl7Uvs/K2vmxEOoilQBSSqkEkXTNluSpbu8/VHinuft4OTykQMydesSZnCfpV\nnDzCmkFMNiAk3ZqaVIJPlVFTYb5XJGmGVWs4BikQgxGvQDJleJrNkykdck0z9EA7KhP7mHO4Z1T+\nABC1Ku8qfyUkSYJPVhFJxLHgnS1WeNvN5n4AMoQhpcPfl4AywgxLvWW8s09ZT48U3m+vaucWPpEM\nQsTMufn9B6xG20irerIH1bmqe8jmjGxyVbvZzN01GmN85t9nc8fnnr+RHd6bGiLOHPbuPm+7X7/K\nX+lZtz39hZT9ZRxOpPvZd0fN/v0vjzBnN5QrO7NGnO+KmhPb5Ap2O7xSuuFtclfSa8c/99YmvPbB\ntqzX9iTfaPN05e0Nu2ff3ISXln+OsNTsdH/0ps9b13NX3lpGuOVbqx0AYnmazd2tBLvHvYSH1z6O\n19au97z2b1/8H/wHZy+GUyhDCM/FQ8Gv62cVav89GhrD+M2j72PTjg7c98xqRON9v3ArplxjI/py\nwZJM6Xjzox34y8vre964DDC8y4w9baoqKZ7KO4sho6bKbPKOJjSkPj8MX459A0YsPamK0CWn8naW\nEQUgWc3u9qxn9v4As9nZvlXJXXlnjVZXXCPMAVQEVewd2MfaRoMOzemHVhUZFX4zFCNWsOzobIYk\nCYh4BZKajpCSEd6VZr9gZXQ/81hzNJt3psyFEsYEzYrYp/iwszWM55dtMf/B+lyVN2BeaNjH75rn\nXa4I43jpAnNbpB8/cOJIp5lbaH7rIsDU5WswH5fTg+Ls1gF17834rKUBcrV5cWCER2Kczzw3G9q8\nM63t6mqF0GX88dkNCPnM/bv7vO1b9qp9Va7WAyNdfarZYbR5d5vzc7u1GEyNOhpGIgg51JX1ZaZI\n9oWZAXWfDVDqvjB/l1zN5prhad2RrM8djafw0vKt+PuSeuyM7Ma8T5/POV97Lkae+7ztqYTXbmnF\nR5+lBxf+c8VWvLBuGT6vfhnq3uY8+T32eXtmFzN6rLx1XXQ72txTeWu5K2+7p+mJ1z51LpQMYaAj\n1QZlRO5FZwpx3zOr8V93vdVta4emG57lMM337t993plTwP7+yVX46LNmvPnR9l7vq9iefXMTfnTn\nm9jZ4p06ubtBh24bt7Wjrcv8/zVfyLd0xEs6HXFfMbzLjF15GxCe8E6PJbcYCmoqrfCOpwChoEau\n9TRf64bkDFjzfPFat4l5mrqtn9uqV6fvv3Y1J2eFp6x7Xl8RUOFTFXMOckWH4QlvCRX/P3vfGW9H\nVa/9TN/19H5OzknvIR0SEjpEulIFiShYLyI2BEQR9PpD5aJX5d5XQbHAtYAIypULWABpIXRIg5De\nc0pO3XXKej+sMmv2npOQkJAE5vlAOHvKXrNm9jzr356/wfTMWax0xyDt5EUKSdiOhxjTYM+5eRSY\nJjgdgxEYq+w2H3D7YGkmKkxqERuqESB33eR9z/k51OD4GZztI2F5FehIjxDu/mRMxylzRvgxascA\nsf34rgGLndIG8VhnODZGNZaFPvlpkZvgZSphKnG0pVqwrm+DiGl7xENPoQskT5vT8LwDuXXqoBTz\nlhdQPO6rsAXKmMqRmBc/EwCwTYqZ9xeY7CuJg2QroJhF9GT7AxKnQo42MQijeQN0Rt5xU4fteMg5\nebylPwa10idSYXnnfCL58Su348mtS/D0tucwHAghIg8j2DDD30f+/Cf3Bd3PWgO18DU2Fsfx8Pra\nnmFL1uRzOR4JlXYtzXrfXZ33cAlroXFuhYhEtqCG/b4RAReWkRvHlOJbv34Bn//RUwGyGS488Xah\nlogfcC/Du9Uudnd4aMlGAMCK9cFFUVAlL/yaO3uz+O7/vIyb734RQDjJ9/Tn8dWfPov/vPe1sm0H\nGxF5H2Lgcp8e8QJu85OSl+CyKR8RWeeEqEjFDWiqItxXhq4G4rfE8RPWvLxvkYsabznWy+uwY9vx\n5zUPBT4D4Mufqi4AAqjB2vKERevS4eo0EU3xyds0NCTMIHl35ujLl1reXiDmXXRcEVsnpCRhTvXd\nxUNuH+pitejpz+P7v30Zjh20qDXTptYwczcTT2rqwv51drbD3dXC+qYbgOohldBx2xePRW1lDBk7\nA0MxAaIGLG9DYeSt+AI1gfkC/GQ3x4DjemhPt8EhDr551xMAgK5cD4jiwcumEDM1DGR4jbwjXLfC\n8jZTUtzft7x5XH989RjUaC1iO8cga4X6qwfXw8vSmv2/L1+BL972NJ5bSePv/YX+wHGinaylwXE9\nPLVlCfr1jZClCbjl3ZcpAIoL64h/iYXGa10rUIqubA8Gi0P4zSNv4KofP4Wt3ZlhNbdLX7YD2SJW\n71oLrWETVOba91gI47W1PfjRH1/Dd38d7o4W51IdvLTzFQxk/UUst4qD3+1hlf009DYa/y+zvAsS\nebthbnP/M0X1hFUnt9flHqF9xe6M561d9HmRPQHDLYzeLrRS5aJDEKW6GYFF2zBW86ad9J70MC3/\nsORH3tt+1cbesm0HGxF5H2KQyduSyLs53YA5jTNgKiwm66mIWzoqkqZI7DE0FfUVPkm7riLI29ky\nDsUNkwFIwiEy2Uj//0bvWwDoAsHfTv/fHLNMxHJlyzwRM6DrKrW8VQdEcaEpPB6uIcXc5isHX8fD\n6/+B3iJzKRcSKNgutnXmoCoqNnX3omh7Qq5UfAdhMWtOiEYBLhxk+k1c87MleHNzHwaGXMhtTxW9\n6LvMAboA4W5zVodOHJal7hIYGvMU6P6POGNnYaksN0Amb7Yw8VAU4QNSQt6KXqTKdVDgekR0U4Pq\nghCC7Sx5jeTSKBRdZLJ+jTx/6QvL20j6fd+lmDe3vFNGCiopDy2IVquOKch7xY4NAIA7HlwJQgj6\nCiWlSxqXf6WWt0tCrErNRUXSxMBQEUosCzVGX3KqomJd/4aApekRD//x4m345Yrf4cnXaDLQ+m0D\nw8a8S8l73dYB/PjV22GOXClkfVHiiXpdcq/L4C9xo/0N3Lf+TxhM+fFM/rIujXFvJstgtKwXf3O8\n2rUcAwU/L6DUba5YGcSP/Jv/5ap/H4ekKgO5MmRf8HZ6cpcuSDj25DbfuGMA/3X/soDrfTjyPpRy\nuEuH+HZi3p0lLU7DKjhMQyv77FBBRN6HGHi3MZd4Abd5OkGJQybvmKkFpBh1XUVrbYX4m3gaeofY\nKp9ocDtHiAYn/Bz+viGPQpjbHIA1eSn9n5KYNz1GBzQXRHVE85WYqSNp+eP86/q/+brmtonnV+7E\nt3/zIlxbw2CBveTYGF23xDvAPuelal2d0o+LaDSBTKUdxVy1ECBcriInn58vWlzPg6XSfXVDJu8M\nYiodu6gVB+ApLC9AsUXSXqAcDzSWbjHCdlxPiKcomgvXI9gyRInMy6XYi1AR96efuXeHbE7eKSGB\nC9XzrT5meadNSu6EoKScboiGDYgKkqXPhpyYtq2/F45EzqpnsnI6D6ahwnY9IfIiQzc8NFbHA+1n\nTxt5MmY3TAfgl8ABtAFNxsliTd86keCn6wrk05JhyAYA1m4rr4tWNV8NcHcQOvkshGEnt4ltnHxl\nK01uHSvvs7p3LX6+7C68YD8ItXon9Kb1cFwPHgsDFG0XetPGkkF6tIwS/n0EEBDuKRsv8fZIzm8n\nbC1n4e+N5X3lfzyOl1d34bkV5Tr85eMoP9ef1/zfbtX+9hV7Gnep5e0GLO/wY3mf8mSMl5mW73co\nl5lF5H2IgVvepJS848wFzcmbqIgZQfI2NEVYhAAATxUPKIXix39RalmHrDBD3OoyFATd5lTpTYei\nuVAU1gwFQNzUUBEP9oC2XZ6lreI1Fssjju5b1oxcbVt+80iWM3P9J9W0fz2uCkUliM/5BxQzD6J4\nAVc37bxGaDy9pDOb6xKYjFw1g373uv4NsD0HMS3BxufPnY0C8k4BLmxhvYfNkaHQc7oegcoFDVUX\nRdtFX44nDkpa2GyBMsAWXZt6ekA8BU7RD4koiivkTLmLO2kkqTEqhwYA5NwMVI/OPfdCFIlv+e0c\npLHCZHYU8svno4KwzHvdEfK8A5LL18vTcyUTCpK8xpuPQU+IOZRlcLn17xFPJPFpqor/emCZf97d\nSHgOZotlOR+q7ore67sDf4nzygoS8/MBuFXtegSKlYU17Um83hOMsfMXf3+BHpdVemGNewVG+5so\n2g4efHo9rvrxU1i5cRfUip7AsUrA8vYTqoazvPNOAf/+3K349crf7/aa9qTbrVZ24U3mPQPefsxb\nnvdk3F+YD7eYKP246Nr4+6YncNeqe3Y7vr2F63n42h1LcO9j4W11AZQ6YgJW9HCaCNt76LuxtoL+\n/sLc6283Uc0jBKs29r6jxi97i4i8DzHwhDW3JObNLW9D5a5XD5apo7bSJ0VdV8vIuxQyAQXIhoQ8\nCnsgd/m7EjFK3rL1yWO0MVNDMhaU7LSJLb6DJxEpngHD8jCiIeWXAknhQSK3PWX/5vNAR2MaJ8xs\nDYxXYdnqPMnMMjV/PjSnrDOb43rCbc47hf34lTsAUPUzOugY7E1UCtT2CljeQ12w3mBN4FwyTGbN\nP7NsOx54YpMYe9HxROa9fJ9UaIDioj9TxKadg+jNDQKOibVbB0RCG1RPlCxx8t64Nc+6z0neBcVD\nkRRgEDZ+fq3En1SejY5CEqZdA5PF8hXNgc403Tlx0fmk280YEd4WbnnHtLjwLshKev1539LnBNfd\nnwsmj9VtwD82/QuATzAXnzyOnstxxaLW3jyOnch5W+QtkvGYkp6iEtF5T7a89Za1UONZPLL1kcDx\nolQupMd63inikefpPX3hra1Q48GMZyhyzFsi72Es78c2P4nOXDde3PkqVvS8OSxp7l6m1IM14SX8\nz9q7sXGAVkS83WzzjTv9+/R2Er5K8wGyUmfEMG/NviKTc9DVl8fGnYPIOTlsGiwvS1R3Y3kPN/4u\nFs/mW12XAEYe8SMfwWObnsTTr2/H7//xVuixpXjxjU6ahf9WePjmQCAi70MMgZh3wG1OicVQfOvN\nKnWba6qwfADfsm6o9gl+WPIOURIjw7jNOUbUVfqHqwomtlcFysc42cRMXciJcnDxEz6GuKVjYmsD\nHGLjcxeNRnUFHWdBTiJ2Zbc3V3BTkU4YaKyOB65HJOUxy7syYYpriM96TKjQnbuQkoHjEphsMdLX\n9Dh6cr2iZGt2zZHivM6OUXD7a2ATGy/upPrvXBa11G0OACZbbK3fPuhnzGsOirYrVMrkudVVHVA9\nPL9qJ2761QtQDBq394h0P1TXLxdi5L1lexH5okv34d4JVmGQGeT3UQFxNbjwJ7UvT4nZKRiIW5oY\nLzQbpk7H1S/FefkCSTdcn7wNej5L8cm74PrWZU/Od3trjLxL1dOMjlV4YM1DcDwXhAAT26tw1GRa\nG19winCJC7evDs72MaKigTeMCUPfUAGPv7JVkJDcjc4cuRJqRXfA8pZDQNLFis5hgYQzBqphH1zA\naH3t0gLDldzmEnlb4eQtJ/r9v9fuxLLulaH7DVceV3RtaHV+WODxzc8AKPdqEELws9d/jf9dG1yo\n9PTnWRc8EiDm4cgvb7t4+LmNWL2ZlmxmbT+GvCvfF3rMvoCX5xUdF//96p34/gs/KRM7KvXWBGr3\nh1ns8NACfw4c1xNVDH9a81f86onnsKnTv++7s8K3sERBfr/fDUTkfYhBVcOzzbmVoTGZUUV1ETM0\n1ErkPba1UsiE0pPQ40c2paXPJHeYbJmXan5Lx7O9yzaPb6kVC4yBrI3KlIXjjmgX23lTkpipiZcc\nh6uxlxnLJm+pTeDoFkqSd6+6V4xHdpvLMWthgbsa0gkDDdWJwPWUlsNVJM3A9ei1NN5cnaIuccfz\nhCeBqDbuXf0AvcaqMWhPjQheuEv3W9e3EaYSA8nR+f33y+eXzRG3vAH4Cxtmeedt+sKvTib8/TUD\niurhjU19gOJC0VwQx4TrefA8iJg4J29uUaowaRmT7DYXjVmkBZur0wx5Bp4QZ+cNxC1dkLeiOeLe\n9hcly7tAnzdV84TbnBOXqcZD3ea9WXo8IUy6Vy+KEkYK/9nryXK3ugKTkXPe4wsxQ1wDNCcQ81ZT\nvbjnzT8LBb8f3PMq7n70TSxdxWK3hpSAVbMT1sQXBQm6HvGrGmRIuQWDdjl5F11byMNyD4ilxv3K\nDtUT3gWZvLmGQSl6MoMwSAIN8ToAwxPgcKpyj21+MqDBz/NKZPL9xV9XYe2urVjWvRKPbHxMfH7/\nk2tx9zPPIT7zceitawLqdjIxquke4bnY3p3BH59Yi+/9lraslWV4d2a7Qse4L8ixDP9C0cP6Aerp\nKG3y44sJ2VjbtwHLi0/AGE1DII7r4anXt+FP/1obOIaHRGQJYPkdEZu6BErCf/YzuxGl6WFW/HCS\nvgcCEXkfYjihbSEA4JSO42GZ5daAcFUzy7u1jr4oJnVUY1RzhbAeAQh3LHe5A74rm26XasI7R8Dp\nbIPT2eZvl9zQJJeCN1SJRNHvuGVqJs4/kVoZU0ZS17HIqAZQLPjkHbf0gIAMjAIjW7pPU20Ccxpn\nYFrdJKzr34AhbTs7h3TxcsyaK615GtIJk1qBsopcSUb9rPH1qEun/HMxxboYUzVzXSL01wFfxtXQ\nDJhG8GfCSSTjZGEp/uKptT6FUpiaLITj66sXbQ95FhNorvGTDGO6IVnOvshMNu/QlzBbwMiWNyGA\n6rG+6l65d0K27ImniVp2wE+kKuboPbKYWA50h1U7EAzZQ1DsONA5GvYW1pVMc0SiD0+aMxCDxa5X\nJm/umvcGqJiOVrsNg3JrTql0atsQJVtVVYVlXfAYKTAvCl3EObAMSU9/1HI8ufVZPLLhn3A9V5RM\n9bA2nUqImI3sNlfCyrcUV7yMB/dgeXMPSEz1PUCK6olqD368O1gFNZ4RLm0ZWTuHQk7DB0efAcDv\nA5ArOPjt31eL/Rw3PKntjV1BFy8PXcge7KGcjd89/1TZsX99diNyBiVEo3VtoFc5J38lNgRr0guw\nJtM6/lK1uqyUUf+/ax8WvyEZL+54Bc9L/elLUXSL5fr/kuXN8asVv8OPX75dhMe4Vfzwhn/ihy//\nP2zxVkKv2waoDhzPw6/+7w08tGRjwAshpH+55e2RMg+kKpP3btrfdrPnbLgGPwcCEXkfYphcOwE/\nOf67mNVwBCyj/PYYnGBUDzFTQ1XKwq1XHI0vf5hm+fK4LQBBvgH3ouwelC1vosHeMBVeVs5WD24v\nrJyPKUnfhWxqBj588nh89zPzMGMctRZiElnlmfEbs6jbvLBsIYobJ4rtinR+njRycvvx9OsUjyXE\nlMfdrSnPCsubeNTytgzNT3aDH1fk1xC3NMwd7y88eDcxUzOggCa1aFKHXMI8Dbyvugw5NGCqscC2\ns0afiqq+2eLvQHvOgHyqi6JXBCEKWup8z0jcNH0vCCccx0Qmb9MsbE+lMe+Cr3QH1wAhCLjNdU3y\nTngl91+aJy7/6hQMxE0NMS3OzmvT5iu6DZe4UPKViO+a5hOo6kiWN53LJ1/y+8bLMW+e8ObsGAli\nGzDa30Rfwbcqq6sly3Dlb2BNfRquPgRNVaEqCoqk3PImqitCSfLcPrrxMVz1xNegN9FSL9cjwoPh\nZYOLq0DCWgi5Q/WEBSqTN/GYfKtr+6ED9jwljARkHf3BnA3Xc7FhYBNMLyUWMLe8eBs2D/ou7oJt\nU8liR0c2xyxCRn6PLN0kyc8SbM9tx5cfvwlPrH8hMNwEy81wulqhQRM6/4E4t+KhR6WJX2rp619a\n5PXYdBHVP1TwNQXYgpiXBZaSmWx5bx7ahqU7yrPOf7Xy9/jNyj+Iv4u2G9B8v/Wl/8Y1T92EXfle\n/HrFH7Az0yme9VIZ1NV9a6HX00UQJ+A1fesD+6iJwYDbnH+XJ2nYc0+G63riPnLIev6lynUyIvKO\nAADQVPYjUspdedzyVlRPuNJrKmJCwjBgeTOrNpDMEaKqxlGZNANxW0teCDA0VfrkbmoGFEVBY7Xv\n9pXJm7+EYqbGXKBKILNazlavTtPjWpK+znhpkhx3hauJId8t7lLL2zTUACnpsWLgHKauBRc2jNhM\nzYSmqXC8oOXNyVtX9fLYqpQ3YKlWYNOpI09ERW6cv13zr5d7PbTa7XijawOKjg14KtobZcvbpN4F\nkDLL2/OISNrjpMFnvQAAIABJREFULwlFt0EcAx4jb3gaFIVlC3P3eWkSIRfaAc1GB2huQNzS/fun\nOYiZmp87UIhTS5yoILaBIjJSzLsI4mpYsqwbdz9MXZM524/r8nixl6mE09kORSEYcCh5X/Ghqaiu\nCU6vmhhCwaRuV8NQy8ibXoODdFJ+PoOWqNEuNVlhiwsu7AJQFz63vF3J8k5qKcQd2kRGUT3YbJ4H\n7SHEtBgaN58HZ/toAIBDbL8Gmn1Hykj4vyvFQ6Ho4q3e9cg5eaSdVnj9dWIMXUO+KtiWHuZKdw0M\nZei4RJMdRhp6y1rE5vwdD+/6LYrI4c8rngxc85Cdpde1fip01RALKNntrbe+BcegnhAufyyseOn3\ns9T5E17dvAFf+q9nxCLHigWJqbQ3Oq9l93rpb3ht34bAdtlb4HgObNfG13/+HP7th/8Sn29l5ZM3\nPPtdvLDzZfxr6xLkmOVdCHNJMw8cH2NNrDqwWUkMBkrF1vZuxuceuwbLu94Qn8ltb6EHr4mrJAI0\ncY4QgnvefACvdvnhCdvx0McSE4frQX8gEJH3IYzm2gROmNWKL15whPgskE0eAqOEcDVVCcgbkuEs\nbwC1lbEAoafiQWICgNYaP0lNjudyWJLb3M821/06TEk0hYu4AEBVih4XsFRLM+Cl97OaYGVWsuWt\ny+QddJsbuio0vGWYqgFNU6h2t7SY4Mk3pmqUkbec9CeTsxibNN+xEMtbjWXxt/7f04Q1TxPudtNQ\nYeq+Z0V0RXNMZPIOtSCY5e1fqA04BmzH893mAGJxlLnNP3bqBEHufFvOy9JnytMRs3RhvampPjqn\njLy9giU8EKQYR8YblFzGRX9O2Hf15XwrLONkaEzZMcR+3EqLWRo8zd83zix/T6X3z9BUOGD3UnwH\n/d5kXAqTlHTVcwerpG1sIWeb8DIVYpz8he95HqAX4RXi+FjHlTAddqzqsg531PJOm0nYDoSHpugV\nxQKAex8qrKT4XXHPx7Iu6vKOF5vhDVXD3joGAPDcm742+NZeupghjo6BQWZpMsubX6Wa3hUoAyxk\ng7+/jJ1hc6RAgyFCF778bT/05vVAMQEvkxZiQaVqfRxLNvtlc+NHVOH8U/zcDzW1C/IP8ub/eQld\ng9TFbO8YgYQex/qSJjy25xPj7ct+gy8/eQN2ubQpztbuTKjVammmkKQNI0au9Oc4dCxyxjtA3d6y\nbsCDq/8BAPjjW38Wn/FjHdcLvEMAQElI5J230Vvow5Nbl+Dny+4Sn+8azIuZiCzvCACoxfzRRRNw\nxBh/tT4hdQTcwSoU3pwdekwpucdMLaiQJJM3CZKZkDdlkBOpONpqq6T9yxcSlaZvRfKXWMyULT//\n/LpE3tzy1lTNJ9mSxUVx3RHwWMIUb0nKY96moQlXOOCX9nDi0jU1IBwiX4OuKtjUOYTfPbpOfM7L\no/RQ8vYXKLy0SoYWIG95MVOSw6C6UIkuLNiKhCnlNDjCCiCOQd3mhJM3LwVzoai0tj5XcFDgbnMA\nlim7zTUoACxDCyTNAUCB5JDQaC5CwtIRN+j86rU70KmsgWKypKd8TMwDKcThEgdEp5nJil70xXDY\ngi3LuscRQtDv7PLbz7Lvz7t0e6ezCTuTVNr0kxMvw1ktF9Ahs/71pqHCUYKVA2JRKB5PAhhFjEx3\n4OjmuXRqpC588iKosHIezGKNyDsAaLKiYhQB2/QXSACgUMvbIx6G7AzSZgq27Sc2Op4jkTf9jsp4\nWlLCo9v6cixbv0jnmeTpwAekBc6OPhZGcA08vIS6yGWyo99B5X7zrx1L50hx8M+XttA+5ZkiuocG\nxBzpii5i5tzw1iq7oShAYeN4EMeEogDZYkHEkkvj/tttP8FLU5WA0Iw1+XmYE18AJ/A1W/rx6rrt\nbJ4NjKrsQHd+FwaKfqWCHMte2fMmrftnv+MbfrEUP/uLb81yFN2i0DRwPSJCFv7AWNWJ68sJK0RF\n7oVF9JqsbMBtvnEbnfNdhV6YE16A3voWc6F71I3O3iFpvQLENpnbnC0M8g5eWeu3C/WIhy2dQ3h+\nVac/3ihhLcJwiGtxFFfNg9dfH7q9lLwtUyuxvOWENXr7501uxPc/O5+2/pMs7/GtJf5MMMuCn1sr\nt7wbEv5Cg1tIPGv5e5+Zh7PmjQ0dK7e8AYiM5dIMYJJP4SOTzg1+oWR5u92tZePxX8R+9q0MQzWk\nemH/+3gs2ND0snri0fX+NRoh5C3PtxJI6C8JA6gudMUQZXQVSRNtKRqX16q6AuSbZZa3r89OAuSe\nLTjCbQ4Ahkl8kvc0aBpLAJOS5gACm+TAeSNmakjr/uIrhz5R1uQWLDEPXoH1ZlcGac9yzRPhEL5Y\n4q1fu3O7YKMgLF5ueXsKJYo3B/3yqE1bbdz10Dq2ncdXM7Br3gKgCNLjC4BYjL2U9SIUBUjoSVw0\n4TwoTgyKbqOphu4vXP/FGEBUmAodP7f+O6uepIsgT4Preb4YDot5d2W74REP9fE62K4nFp02sSWl\nO2Z5mwkpt4Fu4yEEl1Vf8DnK2QUUbRe/fGgVVm1hjXpcXWyX8wb4dRLHFGI7ikoT2VZv7sMdf10B\nWymI+VWhS25ztsBgdegkmxZz2DkwhKLtQW9eC60mqKrW7/o1y6qqBKRhAdAOaZK1nnPZ78s10Jyg\nZX49OT80kA35/cmu+tfWlau6FdyicJtD8VhIyQfPc+GLqEwxA89mioKuCkVzgqV10m9Qq+wRTXgc\nhzDLm97Hj7Z/Bl6mIuClsl0Pv3/CL9+7c/n/4DtLbsMDT/niMZHlHWFYqHu4Y2aJNWwZWlD3N1Aq\nRh/kptoE6qvi0FQ1QO6TO2rF/08fU4svXzjdj8ejNL5OURvzCZ94Kl08MJd5Q3UCR030CVa4iAGk\nErIrmrfwLL/YMXV+TJx386pImNA1Bc7WsdQqIeUxfuIRVFoVpaeDqRmi5EgJqXU3VCMgvfjZD07B\n5R+YIf7Ww8hb2t+TpEcntNUFd9RtqIqOuGR5z2+eCxAFesNmP6Pe1ZHJ2zR2yaw6a+ozfptXx0Au\nHyTvbGIjI2h6nzVNoeTLXtqJlEvdpooHz6afxS0dST2Jwlv0+lzFFpa3V4gL0RauVpbxBkUSk8hl\nYN+/dscu/Owvy/HEm5ScSYaFW3jZGrdwTN+789yrg4J8XOY2R7IXUF2MVuaCFBOB73hsgJYUcosx\nriawoycLt6hDt1xRiREgb0BkxOftIjziIR9jFmM+IfIKAFoOt2pjL/64lNbzt6aaqQyqwlu32uVu\n81hSkD/XyOdVBbativtJPy/gsZe34ull27F5Fy2Rq02m/aQ/j7vNFYgcCFsqeWT3dzBrY8122mKX\ne0Dyeep2J4SAe43VWAbEU0AKcXGN37l7KdZvH4AxgvUzcHTkXliEpNMIBzZ4GZ+mKgErmkP+zdhM\nuY84hig5zEreroxdImID3+1N57A8abDgFkTCGkqSyVJGUhCrIyzvrK+q6OqA7gT7juvhSWe268F1\nCfNuqMgXJE8Zu8ai7QbG8GrXcmjpXvEbEfu8S4jI+z2GcLe5VPIVEvMOWJYSudek/Bfr/KlNmDra\nJ3MAAUEYDpnc4WkBlzk9p2+5VyV88RiZ8MQChBGVSGarS6IuLnkDPA26ptDEKkUBoIAUEtAhuarZ\nS8ojBIs6TkBV/wwa72OQyZk37pARqJsHVZKrkhYBCb085i27zT2pfj5VojKnKABcFQ3VCUwbXYu5\nExtQHauC5VZBiQ8FMuppqZhfh6omhkQmLHENDOVtFGzfbd6XXI7p09l99VToqsK6zrH5GPMMVNZb\nmj8TBduFoijwhmhoxEYOipmHQhTAlmLezPLut/tgxIPkzc/Vn83h+VWd+PsKSt6lljePLTpMMCb/\n+kLs7CmIuHavthHfff5HUHU6B5br51qIlynJQzFzIt5tKQlk8g6Ia8BVCjBYtUYpefNcjbybD1iD\nzrYxrByPC9FQ8n19K81gbk01U8ubu80JJe+EpQuXdtKK+d4PI0jeXPdAdPDzCsiKen3672lzx4rf\nnS2XWqksROKYoGI7qng+sgUnQJwAkMl6ICBwPMePeceyIIUEAFXyDri49wnfclR0ByAqFM8MzLWm\nKqHlcmD3UYkPQq3sogtqT4NC6Dhkb1fGLre8lVjWJz+jnFjzbkGUivE5cnc14gOpy2CqlvjMcT04\nnoO8mxeqisTVoaiOmGN6fSXfwd3ujkcXAJoNuAZts+zySgJ/n7LjpTkChkmqO0CIyPsww5566IZa\n3oGYd3mdNycbz/MCCWvyQqBUfrB0eygUglhJrbpsrVt6+PG8QQh/iR49tQkfP20irr5oBkzNpCtu\nNv50wixrSiCTN38RVqUs6KqO6sKkQHtUUzVEOdCYmjZ888jrMK5q9LDXaGhqoJZ9XHMdPnDkCNx0\n2Vz/slUFhVVzoearsKDZF27hgh4yKhMJ6JqKL104HfOnUq+CiQR9YRh+0h0tFSOB8h7enAWOgf6h\nYHY9APSxzm10kaMGLG8A0OtYwhT7bOG0ZurZYQRAyTEPnSQAKNA1BQumNWFEFQ3Z7Mr3wUqweHKJ\n5a0YBVimImWrJ6Brip/YptlQFF+qlQghGVVoxW8Z2ibmwHN8qV23zw8ZKUZRWN4WErSch32HbrJY\nrsVkMJnHIGXRf9/Y0oVfPkL7NDudbSDFOAtNcPJmI0pQi7Ml2URj3sxtTsnbRdzSka70UBmjrV1L\nyZ+7r4u8RxD7DRb1XRhwugHFhcZEgyqsBGrScRAiuc0VOW4vJe0x0ti4Y1C4r0lJ7kHRsyl560W6\nwGChB+Fh01ykkiE04AYXWSqzvHUvjuKGSbC3jaLbmSWqN9MFDsnR3vSKS8chk3e2pH4bAPSGLYjN\noNnmhuWTYFKpggIFBafot2FlY/EKcShOnC7CJLc5j6kHLG/NoUTMUGrdK6oHKJS4HZewcj0ahvIX\nOPQ7MnlbkLfT3QJnRzubA7o9bmmR5R1heBT3QN56iaVoGcGENeLJ2xXpv1wmskSqkyGsLWCY5Q0A\nY6voD5vYFsa0BF3VMtEamoFvf+JI3HrF0SXnpeTI5V1jpoZjp7eIuHiVVcmuRRMNW2QYEnlfcfZ0\nfP7caRjTSo/RNZW9YPh1aSJO1VSTQGOqBhWmbJkH57M0/p00E/jwiePQ3ugfo6kKvMFaJDedgOqY\n/7nc7IGjpabc2rcU+oJVOem4GnIFF7miCy3mZ1VziyWQxyBZ+l051vCFuc0NPRgWEYlVnobPnD0F\nNRUxen+IBuJqKHg5KLotXsS6ruITZ0zGNefT+9WT3wU9zsnbEucCAK1yF5qmv+W77l0dLbVJsVDQ\n67Yj1rLJF3MJkZYFAM9gCnBF+twkYjq83iZUDbIKDL0o9NKTajWyeceP+zJLTjGZNeZpuOXf5iPN\nyHv11h68vtFPsgJo8ppX4vZWrBwsNYZ7/rYJBP4C1CU0YU2zCsg4Q+iobGGeJhWEKFA1Rt6eDVM1\nYLOsZrHAqejBC7gPWsNmaJXsGow4aiuo0EvOLhey4fFuIkkFr9vZi9gUKpzCFy5y3NzziEgMEwtX\nISTjoChJ2Y6qYhnlJeENVSXoLw4grqThdnb4izUtaBUX3jgyMA5ZMjUjZYIHcmMAQPEEeRc3TMIM\n71xYmomiWxAxb1GD7eooFF0YqinKHh2X+Cp2IrFRh6J5yBSkeWTkrQ42wB1gZWWqC9vxqKdDs0Fc\ng3lwuOVNv38wa4v5cLvaUB1jybvsGY9behTzjjA89mR5lwovWKaGie30ITthVmvoS5KngHgEQQlR\nibiUEPIu7fTEccX0T6Bi2wkgmUqcefTIYcdqqDra6lOoqYiVfS6j1Hr33dYkKNTBj5dUz2pSScwc\n71tqhqYG6n0BX7iBZ30nDD9cUGZ5l2Sex8JKxbgbnhDELR1nzO/AvMmNWDituWxfSyuPmXPyVrhl\nzd2sRRfElPpCM3KXQx2Vdf6LWGQrexp0VaVubzlhx2KuVtfXntfZfSaOgSFniMqzshc5d5vH9TgS\nehy7cr2iJI+/zD+4YIw4f6eyRri947qJz35oqug0BwBoXYmCW4ACJZDMJ5frOBq1evM5Rt5snKpD\nCXj2tCTMup3wCnGkvUYqYcleuq/iL4DqUPJm46urjPueE83P6OcvfM8jovWqxvu6aw5yWQVLWJtM\ngy1aHTh0MZ2gNdod6XamSgfAU6FoLJud2DA1U2RNBxfQ/iKNz21N2gI8Ddtz23HHsrsgMvqlccLT\nxMKoM+tnO8ulcIBP3rwlqli4Sm5z3oa3Wm3CN46/KuCh4ffC1bLwiIdxDS04YVYrxrXUse22P5eA\nOG57Fx1vgLyZZXzVjE/jqhmfDswBvRd8gRLDUMaFpZk0YU3EvNn8cfJWTJFQ5rgeduWpp8kX86H/\nDhX8MSi6DS+TRmbVLH8Bwo7f5qyHogDeUGXAbS5yC3K2mA/iGJg7voWek2kimHpkeUfYDfbU67fU\nhZyMGWitT+G2Lx6DxaeMLy9XAkRrPyrmIFnGEonK33v6yJNRH68NxH5lWJqJq886ATd8bI7I+JXB\nm6+UeglKt/NyH8sILjgqmeWt6DbSyZBac0XOXA9+h6YpActbBpf7TOp+LH5PlndY9yQeo+eqcecd\nNwafPnsKmmuTqDQqA/uGldvFWekWfzEeN82vr1WkF7/O483SgqxdnVZ2PuL6lrdcIy7I3/W158e3\nV2H+lEY0pqtEaRBPaNOlhUtNrBo9+V7U1zOyZy/CU+YEdeCJ4oB4KmaOa0RTTQLfuuyowPa8W4Cl\nmaiuKF8EAUBRoyV7OUbeosENK9dzk53wFAfurkbs7M0hm7eF29RGHlrtdroAkcSBYixPQdHcMnf0\n8nW70DfAXMUa70jmBBZIwuOkuFTVLk5Jo6OizV9oen5M2iU2DCk8M5yXAQDqE3WoqYgJ1/1rXcuR\nJf2wJlBJUd/y1oU7l3sv7O0j4Q0wi5aR8zPbn4dLiBAb4QtX/syYY19FhtDx1+ktSFspGLoKj7e5\nZZamrdHjm5J1+OiiCWirpgaBYmVhjnsZWsUudk56n555lWaqb+rpFdfG3ea5IR1JI/heUHQbnsEs\nZ9tAf6YIS7NQCIl5wzGQLzq+qJLqoug6uH/NX+k1MhU7fo0i1q7QOm6xuJcaBdmOh60OFW5xu1tD\nLe+hrB2o8EjH6Hvig8eOwHc/PQ+WoUUx7wjDY97kJswaX4+vLZ4Vup1nVNdZ9Zg2uhZnL6Qu7GSM\nJWaFkTezvUvbBcqiJvKmM0Yvwk3zrw0mp5WgtjKGUc3h5M7JebiYOd8u9MdLkt7SJn0BKbqDdLyc\nvGXLu1RIxtBUv+SoBJwYZMtbLyFXTmAfnXQhxlaNwsiKkqYlAM6Y34HT5rXjU2dNLtv2hWlXBWr0\nwzL2E5rk1lc0jGzyCb+m+xhfI54n+kj3tEFvxw+O/ffgCT1VinlL99RgbnfPz3jXVBWfOmsK6lL+\nvXOKLAFLWrjUxqphezZ67R7qvuS92y3//GkjhYq0CsXTce6xNI9A04KLy135PliaiY+dOhFnzO8o\nmwvCwgCdPVRHnN8jTmI7MszqtC1s685Qy1tWA+WhBdvCSbOobn/CYG1Nx7wu+otzwn9pdZcIJ2ga\nK8nTnMACSSgPcnI26QJjRLoVlsmS+jwNHmhfe9uz0dMnJToNoxxYvXURLM2kSoeyVCk2+vMhW95C\n598RcyD2Y8f/c9OT6NPXQIlTD0ap5a1oHvQxNJuee5Fk8halWCo9vi5OibE6Sc9jtKyHVs3ugfQc\ncm8N11bo7suhM0sJ/Sf3vFn221fTPSC1G0CKFrxsBdZvH0BPn4OcUxAiLVzbgdgx5G0XGgwxxgIZ\nQme2G9PrpooWvdzyzhTZ74Qt1OIaj/v7mgeO6yGDXpCiBZJL0wx1fs/ZImkoZ0uuewOVTGggFgcq\nUxZMQ0XRdvdoYO0vhJs+EQ5ZWKaGK88tt644UkYS35p/HSrMVHhMOqS1J3/Zlbb+k6340pZ77wS+\n5R1O/pogb/riLiXvuJThHeY214gpvqd0gUHJV8XE+Cx0NFQFtnELf3duc+5Wntc8B/Oa54SOP27p\nuOD4saHbUrFYwPIPu0dJPQk4/vbqtH+9llcJe/0UWJOfB1GZFSBZhZahIaZbSOoJP8bo6dBUBTFL\ng9vTgqJuw+zw5SHh6mVd32TLyCkyYpeItyZO44V9hX5U6JXg7RsURUFh9UxY41+B7dlIWAZqrKQI\njWglnouck0M6UYdpo2sxbXQtHlqyEYWVR6FuyhoMEhazJ4Bjq+hoTQjPByfvXqaRbqoxbO/JImbq\nouc44MtbLpzSjounUNnadExanNWzpL1Aq1z6Ha45CKj1rCpAmmON11mzeD57oSeNhL/wJCo8uEhY\nOlzVLbG2gwsYTt5xRp7phBH4neaJn+XtDXBi8suYeLlVIJ9B+v+COgjVGoKXjwsPguyB4z9z/rsy\nNBW9/QRWo+/9Kap0DPUsVl2TKM/VCHw/I/8iyeP1tT348f8+g9gR6+EO1ACuIQiZQ6vugqIAxS3j\nAU9HvujCLChQzaLoC6ym+kEIdWsXii7i4HF5FzZT4RvoKxeEGsxnAZjQ0rS6okKvQhcQ0DywHQ8O\n8ZUCs3lb/K54eCJrboNVxWrfPRWVcbqIzjssVBUbgDbiDeSKp5XNzYFAZHm/B1EXrxk2mexblx9V\n9hmnZbIbgi61yt8JRFx+mFOqSnncXkZCcmvL7U55rJlnm4dZtfzlf3TNSTh7zKmh35PYjdv8ncLU\ntYALN8z7kNb9a7JUU7jhAZoMN29C0NqX3eomW4BUxZi1ThSAKNBUBcmYgfqqONydI6ES2UrSAhYz\nAD+jHxAvYpng5Xr+ilgSs8bXi0XloglzUUmakXcLyDm5gJiPripCHpQjVhL394aqMdM8xf/A1QEo\naK5Jipp8txict7pUBTp7cxjMFuF2t+KktuMB+PKWDekKUXWRNMt/G7LkbYJtH4qvg9FOFzky2QkJ\nYJ6Mp9iIaVbwuXV05L0cYpVDVMSmpLJDBifvGHvuUnED8o8jD2r1FlbPFORbEaf7ajXbpQ575RoO\nAPWsKUYRhCWr3fjxuThnQfnikqvrmYYq7jl3mxdUujyrZ5Z3Q0U5eZeqNxJPRc7JY9naHiEA43bS\nZ3XZup7gHHDPgOwV83QoCi+1I1CT/SC5FFRCyZ9b3takpbAVapWv3iDVkrt8AcF6rTdSQZZRFvOI\nSZZ10XHhoCg8BnLuBPds8GeBjRhxg3fQo+SdSa6F0bwBm3qD7UoPFCLyfp9hREMKN867BjcvuEF8\nxt08Lvt3XP+5uOGoqwPH7U/y5pa1S8KTO9QSy7s05t2UpOpN9bHaQO05fzlzyzuMGPnLP6xjG/+e\nZIjlffHJ4zB+RFVACW5fQL9fETHN/kJ5b2ceFgAASw+St6oquPC4oDu+QrIkeQ9s/pKloQefMK5f\nPBvzpjQibUrk7Oplcyxn3E8f2YxPnDEJx83wBXZ4xj9A5+vKc6dhFksMvPCEsRjZQL8/5+QDSXma\npsLZOg7FdVP9awxJ2otp/vg4cTbX+Za36ypI6v51N1VVwiME67cPQFVUnNi+AIBfTiff0zE1I/zs\neAauugbQOefQG1g3L4mYDFWjiyJmeXuqLeLoHPa2MSDwUGxaRj/gmvNmubdJMQsgRKEd5cDIW/N/\nG0VlqGwMHfXU82GOXAU11ReYJ/n7AKDIiJfY1LXb0ZTG+NagZgPgaxYQ4ru9eYJWERkYqiEWdfXp\n8pBYaSIeHAMFL49ETIfCeoDzkM+ytUHyVpmSn+w14Za8Yuap7oHmwstUwjI1arm7vuVcTG1mx0jN\nhQIxawIt3Y8RqTbUWLWB8Sqai6ydp78VtmjpzxQDx1tSCaCzg4Z3+D1/bPNTeHzz037mPQn3KO5v\nROT9PkRDog6VVvnKmbvGLSWFpmRDYFtIXtY+QxXkHX5SlcfaWcy71FoZXdmBzx7xcXxp9hXB47ik\nNCPv0pp3gFs1VIq0FMJtHmJ5nzJnBK67ZFawZn4foCgKrr5oBmbWURWz0sQdAEiYMVHrbGkmkjFd\nkLKmBUkLAJKmFONn19DMFjgcfOlVmbLw6bOmoDImddjytLJEx3qplOeoiW1YMK1ZzB2AwPOTCLsG\naQ5ly5vfS/klHUbeimv6nhNO3jVJcbzreUhLY2itoZ6ATN5BIqajwkoHqiHkMVZYaehvLkL+9YXi\nM/k+xIzdh5scj8BQLKiJQSjxAXhKOXl7fY1I6Wm4Zl/g+NLKCQFXF2JKybgRVC5TWaxXImdu9QF+\nXH+4RLiiTscwsq4ON1xKQz1hiZIJk96zwWxRsjqZ2xw5VJgp8ZzIz5x8DYCfsEk8DXllANvct0SN\nNo9Db9hRrtYG0JJD/qyLOTviaRH+IPkkYqaGfNHBUM6fI0/3NQ9Kx2N0rGJzRFATr0KMe5mE5e0K\nsR5O/rmCFPPWHDom1QOxTdibJtFxSc/tfW89KMJYils+twcCEXm/j/GlC6ejrT6JY46gJQ+cvGWC\nGsvqoxtq4uUn2EfwOHRYpjbgW+Ya++2kQmq5p9VNLluAcLe5RuiPKszyPml2G7568UyMaPDJ6+On\nTUR7QwpjWqk1EbS8939ayOSRNbjsiPNxycTzsajjxLLtluRaNzUqQiMatygKNFULvDiC5E3nroy8\nSxwnSSNoeZeiPu6Tt0zEHBVSA5rQBUiAvP2xcs+HHDoI08h3XCI8LPwl3taQFDK6iZiBasn676j3\n3fiJmA5VUZHQ/WssXfAYugqST6Hw5mwU101FXaU/3ngIecvEWSi6mKgfDUVzoTdtgIci4iELkLRR\nCaIEO7uVhif880slmpoaIG9P9ZOkOJRA1QDvXS/FsaUua45OiW/e+A7RwS6szDNlMNlbqVaeyt8S\n2MghLXljShd79BqY5rzJa8jpta8k//TVANl5t3aXS6USxwCIhoZqdi8kmeOWDno9Zx45HjGTajP0\nZf2ySRKKQnq/AAAbb0lEQVQbCJxfHo+i29CbN7BrTIjx8Xtijl6OdblV9CBXFzkAckKbodN7Egif\nlNxzy6I/suaaYEXJgUJE3u9jTBtdi29/4ihhhXLXuKym9qULp+Nri2dhTMv+eyC55T0cefPt6YSO\nb19+JCrfpqv6iDHUHdZWzVyKIdZFzNQxqaM68PI5dnoLbrr8SBh6iOUdco79AV3VcXTLkaFjNAzV\n713NSJeTN19Y8aoCUzUQtyTVOmF5S33R6ZkCf6UMyfIOJW/frZowysm7UnqRlxJj6THyNXLrkkus\nAggo1vE6esfxfCEPRkS1FTGcMa8DC6Y14XPnTEW15Sccjm32a/m5cE9ausbSaxC1+P31cLvbxLMD\nhJO3vMAp2C7aY+MB0HI7opAyyxsAKqQmL9yKa2sIL1NEqbWmlv825PvUW/Sbhvglf/52N6RxkRyO\naU+34YNjTgsQZMqS480avEwaWsUuaPVbQBQvcDwAjKroQEqpFp2+xjbV4rR57SK0YW8eL/bV0n1C\nOnU48FBGQzUTKUr6IaW8QRu3jKqvQ8LSkc07yG0ZAQyy6xS96/05mDLC9x7yTPWkkfS9H9K+y7LP\n0jE4BkawBQ4kt7mha9QL4egY1ZzGly+cXrbodPUcTM1EOvHOQmtvFxF5RxDglrfspo5bOsa1VQ13\nyD5B3UPMW1jm8IZ/2YXg46dNxFXnHYGFU6hs4R7lW4eBoRnCZbuv53gnMHVNvMy5vCTPOOfZ2jUx\npg6lKIhJ8WruNi9VsCq1vNvSkmBMyAtVJtQwy1te1ISRe3wYy1tkrEtWolySyMvRbMcT18jjoYqi\nIBEz8IkzJqO5NonqmL+gbK3zn9EjxtJrr5LIPVGywCgNxUwf689XmEuYuJqwVfNFF0nTAnE1EVMP\nI++URN7cyjv32NE4//gxZfsSV99ziZFENos6ThL/z/UQZOudZCqRe/HkQGy/QiJfRVGwqOMEJGzf\nQ5OWyRsKiutoAqLesLnseAC4es7nME+7UCwARjdX4YLjx4pyPrenFflXj5PGT3uNDwfujeGWt73N\nn6ch0Bh52kxhVEsFXI+gp9dD88CxwXOwRe8lp4zHF844DpW5CQAANUkt85SRRJznHYQtJFzd98rx\nksGqbrg1a2jioWvgklMmYOro2jLvQ09uV6gH5kAhIu8IAsfPpAlJs8aHtxvdXxhTORJAmHVIMb1u\nCh1P24K9Om/M1DFjXB3SZhKWZgaSrvYWPEZ6INzme4IpWd5claqmgvc7py8MTmxFtwhLiqNazHug\nqzqumvFptPfRspVSWhhd6ddUf2NxeQWCjHgIecsItbyHiXnLXp20Qq1dXv8L+Ja37bpoSNDnkBBg\nzsRgDgYAVPMFDILiOVzJri3V4o+x1PKWyPu4GS1oqfWvIWUl8K3510IfkhY4ro6PnEItyUVzR9De\n6LYpuqrFJaW9tnrqrm9IBpvoANSDcvq8Dnz9yC/jxBHHBM4vo3YoqONAXE2QCQBMrh+Lr5TkfJSF\nPzxdlNQBCP09aNLzLSc+AsBXPngs4GqC+NIhxxMoYlw8h+Wy0yYKWWRSjAnLXHZpA0DlllNwztgz\n/HOxPIiKBPME9jYF8hIA6k2Z2O7f95baCmiuf2/5d+SLDlRFRYfH+ruzkreUZHmHClY5BkYIqWMF\nbj99RovJbfQjVy+rfvHnItwDc6AQ1XlHEDh9XgcWTmt+227qfcWF4z+ICdVjMatxeuj2SbXj8b2F\n3wyWK+0FNFXDNXM+H3AN7y0Sehx9hf6DYnkbuubXm7JabRHzZqRTKxGX/DIxpSz6CTVjYbkZAD1l\n7D0i5WeOj24O96x8ceZnsHFwS5m7tBRhZYmyNR6WkAYA7eZErCg8g+aUb/0J8nY8zG2cgXV9GxDL\njMTZx00qO77GCo77psvmYiBbFHM1qmoEsCV8DHweZ42vx8dOnRgcu6WjLl6LOSNH47lupn3u6Zg/\npQknzaZCL6+s7mJSpdQzInsqvrZ4Nrr6cuhVN4nvT5oWvvrJo0TYoCXVhIUtR+GxzU/R87s6IHHC\nCGUatrxYg/icf9APSohGVYKJi6RE2lh87vj3Jox8PdZ61elpQoKFX266bC5Wb+7D5JE1UFdUwovv\nYseXPwfHTm/GP5dyOWBK3o01CXz90jlYuWEX/vPe10DsGHXtl1xDtVGHk9tnYkzlKNz94t+xsZMu\ndqaMqsH0TbUY0ZjCX5esByGK8C6kzBTGj/AXXifPbsPmt6rQyYVYuDgMlzw2LZCiJRZZKTOJmDK8\n5U1cHUdPbcIf/klbpBbfnIPYnL/BNQbE9pgxvOt/uGf9QCCyvCMIKIpywIkboC7Go5pn79aqTUuZ\nrfuCpmQjUua+kT9A3c5pI1VWc/5uwNJV1roRqGSJYaUxb9EUAcGyt9KSLz6DpIS9Dc3AhOqxGF9V\n7sLlGFc9Bie3Hzfsdo6w8IdcBx6WkAYAU5Nz8NkjPo6zR/v19kdNpkQ+sb0auqrjkknn47w5c0Q+\nggzZbQ4A7Y1pTB3lx65HVUqysiXPkio66ZW7qrkVP76+zf/Q1f0sZQCmqQUy5mXhoLilo70xHcgb\nOG56G1rqgs9jXPYGuDpOnOV/XzpBVdZ4YlmYlRhIOvR8adLW+iTOOYY1B5LqpsPCHzWDM+H2NqCt\nOE/McXtjGiczmVut6Lv+wyz3uso4KhL02r2S52DyyBpceMJYv1lKyTV091PCHVXZjqnG8aLne1NN\nAl+4YDqV2iWqyI8wVB2WZiIVN/CJMybhyxdOR3tjGpMa/C6AHz91Mlrrk2KR1TdUCCRHUstbCx0P\nAMAxkIobuP6js3Hy7DYACohtwVN5A52g5X3JxPOFpxAID58cKESWd4QIIfjY5ItQcIvvaAGxr9B1\nFW5nG2yjgCvPOx8AUJP21a+AYDKWXH5klpK3SJ0t/56rZn66/MO9wHnjzsIDax7CxJpxZdsaE37o\nZTjytkwd0+qCNevnHDMacyc2BKoBhgOPaZdm1nPwpD7e31lGW30SmzuHUFc1/MtWvi7iagGXv6Vr\nAZd02Eu7OdmI5mQjtmd2oqmy3LshW84nzhyB8SP8fToa0wAUmEocBZIVVutpR7ULyeOEEYcChS7M\nJCI6YWYrTpzVhhNnt+Hz/1WEmuqDSozQZ/mCo+bi0aWNWPyhCaFzoBerwIVd08N4ssZVjcZLna/5\n1QESqtIWyFZekkUt81HNVP50guT+lhdn3PuSZImH3kAN1FgWtudn4C+QmvzMbZyJf215BgBNPj12\nuh8uaapJYHl3HGqKJr8ljSRivN+BY+L85o/h3rcegJryLWuAVtmMba3EkZMace+W5diapS4U4hgB\nsaKjW47ElNqJeK2b9q1/N2PeEXlHiBACUzOHVak70DCYhKuzdRzqE9R6a29M4ZxjRokOaTweXB+v\nDVjbZoj4DDCsmN07wokjjgnGbSXIRCFaNZYgbKyqqgTaq+4Opmbg5gXfGHZxAAC5l06iL9sPBD//\n6AcmoK0hxayr0rHTfyulxjtfuXB2YB/TUAPkHQ/pLqcoCq6Z83m81rUC0+unlm3XVA03zbsWf1n3\nMOY1B88/aSQlNiPXgEJsA5QEJRdNU8X9VhUVcT2GrJMrkTv1O7DpJIbCiqNRlQ4nlTEtlbjinOHl\nluOZDgwZW2FU9pZpP3AsnnQBptVNxuyQMFgypkvtR6llfvyMFpxz7KhABYvrlmfX88WS21/ni+WE\nYGTFCDQmGtCSKs+h+dAxo5B5eRJeGKKqZykjAVPKjxhd3Q4vlxbkXZr1P7atEvW91YK8DcUs03pI\nmynoqg7HcyLLO0KE9zOSMQOXfmBCwPpUFAVnLRgl/q6NV+PauVehxqoWtdMAy1QPwbvUKyGARR0n\n4G8bH0d7upwggXIvwb6gcpjOdhy3f+kUhDlPYqaO044qb4RSio9Nvggv7HwFExpaA5/HTC0QTx7u\npW1qJuY2zRz2/PWJWnxy6uKyz6tSFlrrkujcUgN97Aa/R3XJjeTiIh111Th/8Ww8+vwmHD2Fkpii\nKHBcD4CCUU27n6fhoCkGiqtnY+KY6mFzH3Z3jRPaq1D7Vgp96BYKZJapBcIbANA7SGPSFSG9Cnij\nkXFVo8u2AfQ6bzjqK6GeBUPXcMmcU/DCE4/T79aswH6WqQXaKB83pTyMJHdPTJrhXRJrrCp05roD\nuQ8HGhF5R4hwCIJn/u8OYaSol3Tt8t9T7z57nz36VBzVNCvUnQr4mfEHEqX913eHay6eiQeeWheY\n+yObZuHIpvIOfg3VCZw4bQyeHqB61/EDYHGdd/wY/PTPWRQ3TII3SC3x0kXYpJrxWLVrNU4ffRLG\n1ldibFu4FT1hxL6Ve/L5U/YxPUpTVZw//Rj8YvkGuF10XsMWmJUpujiZPKqmbBscE1+c/CWMqKsu\n38awu/CWoer4ztHXI+fky/bTVQWktxVesg+jnWNw6YfKEyPlBWKlVU7eAK3+6Mx1v6sJaxF5R4jw\nHkLpy4n/fRAMbyiKMixxA8O7+A8WJnZU42sds/e8I8MHpx8JrO2GSzyMZuWP+xMzxtZhbGsVVm30\nPQSliYeXT/kICm4xkMAYhpHDtOfdEy47bRLufvRNXHRSeV7D28XMhmn45lFfxdeefx1AeF+Bs44e\nicqkhWOOaA58fvnpk/Dy6i6MaWh6R9LE1bEqhFG/rqtQMrUoLF+A6qnhYYFKSU2wKhHufeClm2Hh\nkwOFiLwjRHg/4GCw9x5Qmhl/uCFhxHHxxPMO6Hc01iSwamOv+LvU8k4YiVBteY5vXDoHb27uxbi2\nfVNIbKlL4tpLyj0Pe4vGZD14NnxYuMTQNZEhLmPhEc1YWELo+xO6poqSjPgwuvNT6yahVZuADVuK\n+MAJ4eWtnLwjt3mECBH2Cj/43AK4XnnSz26SzQ869kfM+72OxupgedecCeHW4XAY3VKB0S37ZnUf\nKBxKizZFAbhBP5znPWkkcN2xl9Me4lY4ZY6pot6RltSBW2iUIiLvCBHeA6geJpt4yqgavPRmF2aN\nrwvdfjBxqLnND0U0VvtW9e1XH79XMfxDFeYwCmXvJj555iS8sbGPlX3tObSkKsqwxA0A46vH4gfH\n/ntkeUeIEGH/4NjpLRjVVPG26qbfbZjvASI60JgyqhqTOqpx9NSm9wRxA76lezBx9NRmHD2VWsn7\nK6fz3SRuICLvCBHe01AVBR1N+67xfiDwqbMmY/32gVDVtAhBGLqGr148fKnZ4YTT53Xg2eXbUfUu\nqDjuDfzQ0qEYXBoeB3Qpd/PNN+PDH/4wLrroIrz++uuBbc8++yzOP/98fPjDH8Z///d/H8hhRIgQ\n4RDC/ClN+MjJ4/e8Y4T3FM4/fgx+eOXCQBOZQwEXn0wz6WVltsMBB8zyfv7557Fx40bcc889WLt2\nLa6//nrcc889Yvt3vvMd3HnnnWhsbMTixYvxgQ98AGPHjj1Qw4kQIUKECBHKILvQDyccsCXQkiVL\ncPLJJwMAxowZg/7+fgwNDQEANm/ejMrKSjQ3N0NVVRx33HFYsmTJgRpKhAgRIkSI8J7CAbO8u7u7\nMWWK322lpqYGXV1dSKVS6OrqQk1NTWDb5s2bd3u+6uoE9P0cI6uvP7RigYcronl854jm8J0jmsP9\ng2ge3znejTl81xLWSjV59xa9vdn9NBKK+vo0uroG9+s534+I5vGdI5rDd45oDvcPonl859jfczjc\nQuCAuc0bGhrQ3d0t/u7s7ER9fX3otp07d6KhYe/EByJEiBAhQoT3Kw4YeS9YsACPPvooAGDFihVo\naGhAKkVrTdva2jA0NIQtW7bAcRw8/vjjWLBgwYEaSoQIESJEiPCewgFzm8+aNQtTpkzBRRddBEVR\ncOONN+L+++9HOp3GKaecgptuuglf+cpXAACnn346Ro0atYczRogQIUKECBEAQCHvNBj9LmF/x2Gi\n2M7+QTSP7xzRHL5zRHO4fxDN4zvHYR/zjhAhQoQIESIcGETkHSFChAgRIhxmiMg7QoQIESJEOMwQ\nkXeECBEiRIhwmCEi7wgRIkSIEOEww2GTbR4hQoQIESJEoIgs7wgRIkSIEOEwQ0TeESJEiBAhwmGG\niLwjRIgQIUKEwwwReUeIECFChAiHGSLyjhAhQoQIEQ4zROQdIUKECBEiHGY4YF3FDmXcfPPNeO21\n16AoCq6//nocccQRB3tIhzRWr16NK664Ah//+MexePFibN++Hddccw1c10V9fT3+4z/+A6Zp4sEH\nH8RvfvMbqKqKCy+8EBdccMHBHvohg1tuuQUvvfQSHMfBZz7zGUybNi2aw71ALpfDddddh56eHhQK\nBVxxxRWYOHFiNIf7iHw+jzPPPBNXXHEF5s+fH83jXmDp0qX4whe+gHHjxgEAxo8fj09+8pPv/hyS\n9xmWLl1KPv3pTxNCCFmzZg258MILD/KIDm1kMhmyePFi8o1vfIPcfffdhBBCrrvuOvJ///d/hBBC\nfvCDH5Df/va3JJPJkEWLFpGBgQGSy+XIGWecQXp7ew/m0A8ZLFmyhHzyk58khBCya9cuctxxx0Vz\nuJd46KGHyB133EEIIWTLli1k0aJF0Ry+A/zwhz8k5557LvnTn/4UzeNe4rnnniOf//znA58djDl8\n37nNlyxZgpNPPhkAMGbMGPT392NoaOggj+rQhWma+PnPf46Ghgbx2dKlS3HSSScBAE444QQsWbIE\nr732GqZNm4Z0Oo1YLIZZs2bh5ZdfPljDPqQwd+5c/PjHPwYAVFRUIJfLRXO4lzj99NPxqU99CgCw\nfft2NDY2RnO4j1i7di3WrFmD448/HkD0e94fOBhz+L4j7+7ublRXV4u/a2pq0NXVdRBHdGhD13XE\nYrHAZ7lcDqZpAgBqa2vR1dWF7u5u1NTUiH2iefWhaRoSiQQA4L777sOxxx4bzeE+4qKLLsLVV1+N\n66+/PprDfcT3v/99XHfddeLvaB73HmvWrMFnP/tZXHzxxXjmmWcOyhy+L2PeMkikDvuOMNz8RfNa\njn/84x+477778Mtf/hKLFi0Sn0dz+Pbxhz/8AatWrcJXv/rVwPxEc/j28Oc//xkzZszAiBEjQrdH\n87hnjBw5EldeeSVOO+00bN68GZdeeilc1xXb3605fN+Rd0NDA7q7u8XfnZ2dqK+vP4gjOvyQSCSQ\nz+cRi8Wwc+dONDQ0hM7rjBkzDuIoDy089dRT+NnPfoZf/OIXSKfT0RzuJZYvX47a2lo0Nzdj0qRJ\ncF0XyWQymsO9xBNPPIHNmzfjiSeewI4dO2CaZvQs7iUaGxtx+umnAwDa29tRV1eHZcuWvetz+L5z\nmy9YsACPPvooAGDFihVoaGhAKpU6yKM6vHD00UeLOfzb3/6GY445BtOnT8eyZcswMDCATCaDl19+\nGXPmzDnIIz00MDg4iFtuuQW33347qqqqAERzuLd48cUX8ctf/hIADX1ls9loDvcBP/rRj/CnP/0J\n9957Ly644AJcccUV0TzuJR588EHceeedAICuri709PTg3HPPfdfn8H3ZVezWW2/Fiy++CEVRcOON\nN2LixIkHe0iHLJYvX47vf//72Lp1K3RdR2NjI2699VZcd911KBQKaGlpwXe/+10YhoFHHnkEd955\nJxRFweLFi3H22Wcf7OEfErjnnntw2223YdSoUeKz733ve/jGN74RzeHbRD6fx9e//nVs374d+Xwe\nV155JaZOnYprr702msN9xG233YbW1lYsXLgwmse9wNDQEK6++moMDAzAtm1ceeWVmDRp0rs+h+9L\n8o4QIUKECBEOZ7zv3OYRIkSIECHC4Y6IvCNEiBAhQoTDDBF5R4gQIUKECIcZIvKOECFChAgRDjNE\n5B0hQoQIESIcZnjfibREiHC44ZZbbsGyZctQKBSwcuVKzJw5EwBw3nnn4UMf+tDbOscdd9yB8ePH\nCz3rMHz0ox/Fr3/9a2iatj+GHcDOnTuxbt06zJ8/f7+fO0KE9yOiUrEIEQ4TbNmyBR/5yEfw5JNP\nHuyh7DUefPBBrF27Fl/60pcO9lAiRHhPILK8I0Q4jHHbbbdhy5Yt2LZtG6699lrk83nceuutME0T\n+XweN954I6ZMmYLrrrsOs2fPxvz58/Fv//ZvWLhwIV5//XVkMhncfvvtaGxsxIQJE7BixQr89Kc/\nRV9fH3bs2IGNGzfiqKOOwg033IBCoYBrr70WW7duRVNTEzRNw4IFCwI9ijOZDL7yla9gYGAAjuPg\nhBNOwJlnnokf/ehHIISgqqoKl1xyCb797W9j48aNyGQyOPPMM3H55Zfj/vvvx9///ncoioKdO3di\n9OjRuPnmm2EYxkGc4QgRDk1EMe8IEQ5zbNmyBXfddRemTp2Kvr4+3HTTTbjrrrtw6aWX4vbbby/b\nf+3atTj33HPx29/+FpMmTcLDDz9cts/KlSvxk5/8BPfddx/uv/9+9Pf348EHH4TjOPjjH/+Ib37z\nm3jmmWfKjnv22WfhOA5+97vf4Q9/+AMSiQRaW1txzjnn4Oyzz8Zll12Gu+66Cw0NDbj77rvxxz/+\nEQ899BDeeOMNAMCyZctw66234r777sO2bdsOSy9DhAjvBiLLO0KEwxzTp0+HoigAgLq6Otxyyy0o\nFAoYHBxEZWVl2f7V1dUYN24cAKClpQV9fX1l+8yePRuapkHTNFRXV6O/vx+rVq3CkUceCQCor6/H\n7Nmzy46bNWsWfvKTn+ALX/gCjjvuOFxwwQVQ1aCNsHTpUuzYsQMvvPACAKBYLGLTpk3ieN4+debM\nmVi7dq3okxwhQgQfEXlHiHCYQ3YrX3PNNfjWt76F+fPn4/HHHxfNPGSUJqSFpb2E7eN5XoCIS0kZ\noL2M//KXv+CVV17BP//5T5x33nl44IEHAvuYponPfe5zOPXUUwOf33///fA8b7fjihAhAkXkNo8Q\n4T2E7u5ujBs3Dq7r4pFHHkGxWNxv5x49ejReeeUVAEBPTw9eeun/t3eHOAoDYRTHHyGYJlwAMAjg\nAFROSC0STCWCIJCYBhwOwxEqegIkuqLBbRN0LQaBxkBZsdkaDJutmeb/05PJ517eZCbz9bYmSRLF\ncazhcKggCOQ4jm63m2q1mh6Ph6SfVv97VJ/nuXa7XdH+z+ez7ve7Xq+X0jTVYDAobX6gSmjeQIUs\nFgvNZjO1Wi3N53MFQaAoikrZezqdKo5j+b6vTqcj13XfGnq329V6vVYYhqrX6zLGqN1uy3VdrVYr\nNRoNLZdLZVkm3/f1fD7leV7xVWq/39dms9HlclGv15MxppTZgarhqRiAj1yvV6VpqvF4rDzPNZlM\ntN1ui3fn/3U4HHQ6nbTf70vZD6gymjeAjzSbTR2Px+J/4tFoVFpwA/gbmjcAAJbhwhoAAJYhvAEA\nsAzhDQCAZQhvAAAsQ3gDAGAZwhsAAMt8AxJ5C+54P8QOAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXeAVNXd//++ZdrONsqyNBUECxZQRBHUoKLoE+lP0F+i\nxMT4tRDFWBKVxG7UJPaK8mBBE40lQBAVFAQE6bAU6WWBZXvf6bec3x+3zu7M7iw7w+4Onxd/MDO3\nnXtn9rzPp5zP4RhjDARBEATRAnx7N4AgCILoHJBgEARBEAlBgkEQBEEkBAkGQRAEkRAkGARBEERC\nkGAQBEEQCUGCQRDtzLp16zBq1KiE9n399dfxxz/+sc3nIYhjgQSDaHeuvPJKnHvuuaitrY36fMKE\nCTjzzDNRXFwMAHjooYdw5plnYtu2beY+hw8fxplnnmm+nzp1Kj7//HPz/cyZMzF69GgMHToUl19+\nOe677z4AwNixYzF06FAMHToUZ511FgYPHozzzz8fQ4cOxTvvvJPK240Jx3FJ2bc15yGI1iK2dwMI\nAgD69u2LhQsX4sYbbwQA7NmzB+FwOKoD5DgOubm5ePnllzF79uyoz2Mxd+5cLFiwAB988AH69u2L\nqqoqLF26FADw5ZdfmvtNnToVEydOxP/+7/+m4tYIIm0gC4PoEEyYMAFz584138+dOxeTJk1qst+k\nSZOwe/dubNiwocVzbt++HZdeein69u0LAOjWrRumTJkSc9+WCh68/vrruOeee/DHP/4RQ4cOxfjx\n41FYWIh33nkHI0eOxBVXXIEff/zR3L+8vBx33nknhg8fjmuuuQafffaZuS0cDuOhhx7CRRddhLFj\nx0ZZTMax06dPx4gRI3DVVVfhww8/bPFeY7F//35MnToVF154IcaNG2eKJQAsX74c1113HYYOHYpR\no0bhvffeAwDU1NTgjjvuwIUXXojhw4fjpptuOqZrE+kJCQbRIRgyZAj8fj8OHDgAVVXxzTffYPz4\n8U06crfbjTvuuAMvvvhiQuecN28eZs+eje3bt0NV1Ta1cdmyZZg0aRI2bNiAQYMG4Xe/+x0YY/jh\nhx8wbdo0PPLII+a+9913H3r37o2VK1filVdewYsvvog1a9YAAF577TUUFRVhyZIlmD17NubNm2ce\nxxjDHXfcgUGDBmHlypV4//33MWfOHKxatapVbZVlGXfeeScuu+wyrF69Gn/+85/xwAMPoLCwEADw\n5z//GU899RQ2bdqEL7/8EhdffDEA4L333kPPnj2xdu1a/Pjjj7j33nvb9MyI9IIEg+gwTJgwAfPm\nzcOqVatw6qmnokePHjH3u/7661FSUoIffvih2fONHz8ejzzyCFatWoWpU6di5MiRbYpPDBs2DCNH\njgTP87j22mtRU1OD2267DYIg4Oc//zmKi4vh8/lQUlKCzZs344EHHoDD4cCZZ56JKVOmYP78+QCA\nb775BnfeeSeysrKQn5+PqVOnmtfYunUramtrceedd0IQBPTt2xdTpkzBwoULW9XWgoICBAIB3Hbb\nbRBFERdffDGuuOIK0xXndDqxb98++Hw+ZGVlYdCgQQAAURRRUVGBoqIiCIKACy644JifF5F+kGAQ\nHYbx48fjyy+/xNy5czFhwoS4+zmdTkybNg2vvPJKi66ksWPH4t1338WGDRvwxBNP4NVXX231aN2g\nW7du5mu3240uXbqY8RO32w3GGPx+PyoqKpCTkwOPx2Pu37t3b5SXlwPQXE49e/aM2mZQXFyMsrIy\nXHTRRbjoootw4YUX4u2330Z1dXWr2lpeXo5evXpFfWZvw6uvvoply5bhyiuvxNSpU1FQUAAAuPXW\nW3HyySfjlltuwdVXX90uCQBEx4UEg+gw9O7dG3369MGKFSswZsyYZvedPHkyGhoa8O233yZ0bkEQ\ncM011+CMM87A3r17k9HcuPTo0QN1dXUIBALmZyUlJabFlJeXh5KSEnObkQUGAL169ULfvn2xbt06\nrFu3DuvXr8fGjRsxc+bMVrfBfg3jOkYbzjnnHLz55ptYvXo1Ro8ejT/84Q8AgIyMDDz44IP47rvv\nMHPmTLz//vumK40gSDCIDsUzzzyDDz74AG63u9n9BEHAXXfdhVmzZsXdZ+7cuVi+fDn8fj8YY1i+\nfDn279+PwYMHJ7vZUfTs2RPnn38+XnzxRUQiEezatQuff/45xo8fDwD4n//5H7z99tuor69HaWkp\nPvroI/PYwYMHIzMzE7NmzUI4HIaiKNi7d2+TwHhLDBkyBBkZGZg1axZkWcbatWuxbNkyjB07FpIk\nYcGCBfD5fBAEAV6vF4IgANDiNIcPHwagiYcgCOY2gqC0WqLdsafFnnTSSXG3NWbs2LF455130NDQ\nEHP/zMxMzJw5EwcOHICiKOjduzcef/xxDB06NOFrtAb7eV544QU89thjuOyyy5CTk4N77rkHI0aM\nAADcddddeOyxxzB69Gjk5+dj8uTJmDNnDgCA53nMnDkTzz33HEaPHg1JktC/f3/cc889rWqLw+HA\nW2+9hccffxxvv/02evbsib///e/o168fJEnC/Pnz8fTTT0NRFPTv3x/PP/88AKCwsBBPPvkkampq\nkJOTgxtvvBEXXnhhUp4P0fnhUrmAUmlpKf70pz+hsrISgiBgypQp+PWvfx21z7p16zBt2jSzo7j6\n6qsxbdq0VDWJIAiCOEZSamEIgoCHH34YgwYNgt/vx+TJk3HJJZdgwIABUfsNGzas1T5agiAI4viS\n0hhGXl6ema7n9XoxYMAAM0uDIAiC6Fwct6B3UVERdu3aFTPgWFBQgIkTJ+K2227Dvn37jleTCIIg\niFaQ0hiGgd/vx9SpUzFt2jRcddVVTbbxPA+Px4Ply5fjmWeewaJFi1LdJIIgCKKVpNzCkGUZ06dP\nx4QJE5qIBaC5qowJTqNGjYIkSU2qljbmOGgcQRAE0YiUp9XOmDEDAwcOxM033xxze2VlJbp37w5A\nK4sAALm5uc2ek+M4VFQ0NLvPiUJeXhY9Cx16Fhb0LCzoWVjk5WW16fiUCsbGjRuxYMECnH766Zg4\ncSI4jsO9996L4uJicByHG264AYsWLcLHH38MURThdrvx0ksvpbJJBEEQxDFyXGIYqYBGDBo0erKg\nZ2FBz8KCnoVFWy0MKg1CEARBJAQJBkEQBJEQJBgEQRBEQpBgEARBEAlBgkEQBEEkBAkGQRCEDZ/P\nh7lzPz+mY//0pz/A7/clvP+7776DTz75qOUdOwgkGARBEDYaGuoxd+5nMbepqtrssX//+8vwejNT\n0awOQadcQOmR715AqDIHd1w8CdleZ3s3hyCINGLmzNdRXHwUt9xyI4YNG44RIy7Be+/NQrdu3bFv\n3x58+OGnePjhB1BRUY5IJIwpU36JceMmAgCmTBmP2bM/RCAQwAMPTMe5556H7du3IC8vH8899wKc\nzvj91d69u/H8888hHA6jT58+ePjhx5CZmYnPPvsE8+f/B6Iool+//nj88b9i8+aNePXVF/RFuzi8\n8casqDXkU0WnFIzdVfsADvhg0SDcPTm1y20SBNF+fLp0H9bvatuSCILAQVGs+ckXntkD1185MO7+\nd955NwoLD+Ddd/8JANi8eSN27tyBDz/8FD179gQAzJjxGLKyshAOh/H//t+vMWrUlcjOzgZgrbpY\nVHQETzzxLB588M949NGHsWzZUowZc23c6z799OO4774HMWTIeZg9+2289947uPvu+/DPf36Azz9f\nAFEUTXfXJ598hPvvfwjnnDMYoVCoWSFKJp3aJVUfab5IIUEQRDI466yzTbEAgE8//Rd+85tf4fbb\nf4vy8nIUFR3Wt1jC1KtXbwwYoAnTGWecidLS4rjn9/t98Pt9GDLkPADAtddeh4KCzQCAgQNPw+OP\n/xmLF38NntfWVz/33CF49dUX8fnnn6ChoR48f3y68k5pYUQOnQnnKbvAvDXt3RSCIFLI9VcObNYa\nSIRklAZxu93m682bN2LTpg1455334XQ6cffdtyMSiTQ5xj7q53kh5j524lVp+sc/XkFBwSasXLkc\n77//f/joo89w002/wciRl2H16pW4/fbf4uWX38TJJ59yjHeXOJ3SwmD+HACA5CDBIAgiuWRkZCAQ\nCMTd7vf7kJWVBafTiUOHCvHTT9tj7teaMn1ebyays7OxdWsBAGDRoq9w3nlDAQBlZaU4//wLcOed\n0+H3+xAMBnD0aBFOPXUAbrzxZpxxxiAcPlyY+A22gU5pYagBrYCWH1Xt3BKCINKN7OwcnHvuENx8\n8/+H4cNHYsSIS6K2Dx8+EvPmfYHf/OZXOPnkU3DOOefatloxDC0gnTgzZjyO559/FuFwGL1798GM\nGY9BlmU8+eQj8Pv9ABhuuOFGeL2ZmDXrLWzatAGCIKBfv1Nx8cWXtHj+ZNApq9Xe/MQi+PsthuCQ\n8dpVT7X6i0knqBKnBT0LC3oWFvQsLE7IarXvPzoGQiQHTIigLlLf3s0hCII4IeiUgsFxHFyqppSV\nwep2bg1BEMSJQacUDADwIBsAUBGgOAZBEMTxoNMKRqagZUrtrYif20wQBEEkj04rGJeeoeVmbz96\npJ1bQhAEcWLQaQVjxOn9AAaEWOKVIQmCIIhjp9MKhsAL4FQnFK752ZMEQRCtoS3lzQHg008/Rjgc\njrnt7rtvx+7du4753O1NpxUMABCYE4yPQFaaLzlMEASRKM2VN0+Ezz77GOFwKIkt6jh0ypneBg7O\nBUkMoCEgoUuWq72bQxBEGtC4vPm0adPxr399iO+//xaSJONnP7sct9xyG0KhEB599CFUVJRDVVXc\nfPOtqK6uRGVlBe6++w7k5ubilVfeinudb7/9Bh999D4A4OKLL8Gdd94NVVXx3HNPYffunQA4XHfd\neFx//S9jljhvDzq1YLh4D4JcFap9fhIMgkhD/rPvS2wu39amcwg8B0W1Clqc3+NcTB44Nu7+jcub\nr1+/BkVFhzFr1hwwxvDgg/dhy5YC1NZWo3v3PPz97y8DAAIBPzIyvPj3vz/Ga6+9rZc7j01lZSVm\nznwd7733T2RmZuHee3+PlSuXIy8vHxUV5fjgg08AwCxnHqvEeXvQqV1SHkFbMKTCR7O9CYJIDevW\nrcX69etwyy034pZbbsThw4dQVHQYp546EBs2rMPMma9jy5YCZGR49SMY7GXOY7Fr108YOnQYsrNz\nwPM8rr76WhQUbEbv3n1QUlKMl19+HmvXrjbPGavEeXvQqS2MDIcHCAM17ai4BEGkjskDxzZrDSRC\nW2tJMcYwdepvMH78pCbbZs/+CKtXr8Lbb7+Oiy66GL/5za0JnzNWGb+srCy8//7HWLt2Nf7zn0+x\ndOm3ePjhR2OWOD9ea2DY6dQWRpZTU9/qABUWIwgiOTQubz58+MVYuPC/CAaDAIDKygrU1NSgsrIS\nLpcLY8Zci1/+8ibs2bNbP96rV5eNz1lnnYMtWzajvr4OiqLgu+8W4bzzhqKurhaqqmDUqCtw6613\nYu9e7ZyxSpy3B53awsh2e4EGoC7U/JdDEASRKI3Lm0+bNh2FhYW4447fAtAE5ZFHnkJR0RG88cYr\n4HkOoujAAw88DAAYP34iHnhgOrp3z2sS9DYqa3fr1h233/573H337QCAESMuxaWX/gz79u3FM888\nAcZUcByHO+64O26J8/agU5Y3B4CKigYs3rcK8w/PxymRS/Cnaye0d5PaBSrdbEHPwoKehQU9C4sT\nsry5QY+sLgCABpl+DARBEKmmUwtG36x8AECA0VKtBEEQqaZTC0ZXTxdAFRAWKK2WIAgi1XRqweA5\nHqKcBdXZAEVV2rs5BEEQaU2nFgwAcLMccLyKcl9tezeFIAgiren0gpHt1KL+e0vL2rklBEEQ6U2n\nF4zeOV0BAHtKy9u5JQRBEOlNSgWjtLQUv/71r/Hzn/8c48aNw5w5c2Lu9/TTT2PMmDGYMGECdu7c\n2apr9M/LAwAcqa5sc3sJgiCI+KR0prcgCHj44YcxaNAg+P1+TJ48GZdccgkGDBhg7rN8+XIcPnwY\nixcvxpYtW/DYY4/h008/TfgaeZm5AID6CNWTIgiCSCUptTDy8vIwaNAgAIDX68WAAQNQXh7tOlqy\nZAkmTpwIABgyZAgaGhpQWZm4tZDt1KbIB1U/1M45aZ0gCKJTcNxiGEVFRdi1axcGDx4c9Xl5eTl6\n9uxpvs/Pz0dZWeIBbCPozYQwGgJSchpLEARBNOG4FB/0+/2YPn06ZsyYAa/XG7UtVikro0BXcxg1\nUbqqGdoxjjCYwLe5Vkpn5ES853jQs7CgZ2FBzyI5pFwwZFnG9OnTMWHCBFx11VVNtufn56O0tNR8\nX1paih49erR4XnsxMSc8CDnC2H+oBrnuTl2At9VQYTULehYW9Cws6FlYdPjigzNmzMDAgQNx8803\nx9w+evRozJs3DwBQUFCA7OxsdO/evVXX8IpecI4I9hyhyXsEQRCpIqXD8Y0bN2LBggU4/fTTMXHi\nRHAch3vvvRfFxcXgOA433HADRo0aheXLl+Pqq6+Gx+PBs88+2+rr5GXmokauxNKNhzDh0n7IcDtS\ncDcEQRAnNikVjAsuuCCheRWPPvpom66T49LMLFUIo6o+TIJBEASRAjr9TG/AypSCI4x6f6R9G0MQ\nBJGmpIVgZOlzMThHBHX+cDu3hiAIIj1JC8EwLAzOEUIdWRgEQRApIS0EIy+jGwCA9/hR5yPBIAiC\nSAVpIRi9vb3AgQOfW47qAOVbEwRBpIK0EAy36AIDA+8O4oC4or2bQxAEkZakhWAAwPCeFwAAgq7i\ndm4JQRBEepI2gvGL08YBAHjJ28KeBEEQxLGQNoKR4cgAwplgnNzeTSEIgkhL0kYwAIBXBTCeBIMg\nCCIVpJVgcMwB8ApUprZ3UwiCINKOtBIMXi+NFVFoISWCIIhkk16CwbSig2GFJu8RBEEkm7QSDEG3\nMMIK1ZMiCIJINmkmGJqFEZRC7dwSgiCI9COtBEPkNMHwR0gwCIIgkk2aCYYTABCUyCVFEASRbNJM\nMDQLI0AuKYIgiKSTVoLh5HXBkMnCIAiCSDZpJRgO3SUVIguDIAgi6aSVYLh4FwAgKJNgEARBJJv0\nEgzBAwAISMF2bglBEET6kVaC4RbcAICgTIJBEASRbNJLMETNwggq5JIiCIJINmklGB7dwggpZGEQ\nBEEkm7QSDJdDBFMEqiVFEASRAtJKMESBA5MdCKvkkiIIgkg2aSUYDpEHFAcijASDIAgi2aSVYIgC\nDyaLkFiEVt0jCIJIMmklGE6HACh6iXOavEcQBJFU0kowsjwOMFkvcS4F2rk1BEEQ6UV6CUaGA0zS\n6kn5JH87t4YgCCK9SDPBcAKyLhgRXzu3hiAIIr1IK8FwOwVwClkYBEEQqSCtBIPjOHjEDACAL0KC\nQRAEkUxSKhgzZszAyJEjMW7cuJjb161bh2HDhmHSpEmYNGkS3nzzzTZf0yt4AQANErmkCIIgkomY\nypNPnjwZU6dOxZ/+9Ke4+wwbNgwzZ85M2jWznF7UAqgPk2AQBEEkk5RaGMOGDUN2dnYqL9GEHHcW\nAKAu1HBcr0sQBJHutHsMo6CgABMnTsRtt92Gffv2tfl8mW43mCKgLkKCQRAEkUxS6pJqibPPPhvf\nf/89PB4Pli9fjt///vdYtGhRm87pdTvA6jNQI1aDMQaO45LUWoIgiBObdhUMr9drvh41ahSeeOIJ\n1NbWIjc3t8Vj8/KyYn/eNROsPANSRgMcWQxdPMfXJdYexHsWJyL0LCzoWVjQs0gOKRcMxljcbZWV\nlejevTsAYOvWrQCQkFgAQEVFbJcTUxSooQwIAHYeKcRpXU5tXYM7GXl5WXGfxYkGPQsLehYW9Cws\n2iqcKRWM+++/H2vXrkVtbS0uv/xy3H333ZAkCRzH4YYbbsCiRYvw8ccfQxRFuN1uvPTSS22+ZoZL\nBAtplktFsDLtBYMgCOJ4kVLBeOGFF5rdfuONN+LGG29M6jW9btGsJ0UFCAmCIJJHu2dJJZsMtwNg\n2m3JqtLOrSEIgkgf0k4wPG7RFAyFyQkdwxhrNtZCEARBpKFgeN0imNo6C+Optc/jlc1vp7JZBEEQ\nnZ52TatNBR6nCDBt7oWcoIVRFqhAWaAilc0iCILo9KSdhcHzHDwOLeitUAyDIAgiaaSdYABAhtsF\nAJDVxCwMgiAIomXSUjCy3JqFISVgYahMTXVzCIIg0oI0FQw3ACAsSy3uS24rgiCIxEhLwcj2aC6p\nkNSyYMiMBIMgCCIR0lIwMj2ahREhC4MgCCJpJCQYX331FXw+bQW7V155Bb/73e+wffv2lDasLeRk\naBZGRG456J1o6i1BEMSJTkKC8dZbbyEzMxNbt27FypUrMXHiRDz99NOpbtsxk5OhWxhK6ywMCoAT\nBEHEJyHBEEVtft+qVaswZcoUjBs3DuFwOKUNawtZXieYykFKIK3WHsMg9xRBEER8EhIMjuPw3//+\nFwsXLsSIESMAAFICAeX2IlMvQJhIaRC7SCgUACcIgohLQoLxl7/8Bd988w2mTJmCk046CYWFhRg+\nfHiq23bMeN0ioPIJCYA9hqGQS4ogCCIuCdWSGjp0KN58803zfb9+/fDII4+krFFtxevRLIxEBIMs\nDIIgiMRIyMJ47rnn0NDQAFmW8atf/QrnnXce5s+fn+q2HTNupwAwHmoiFoZKMQyCIIhESEgwfvzx\nR2RlZWHlypXIz8/HokWL8O6776a6bccMx3HgwENFAhYGIwuDIAgiEVo1cW/9+vW4+uqrkZ+fD47j\nUtWmpMBDAEPLMQl7gUKyMAiCIOKTkGB069YNf/nLX/DVV1/hkksugSzLUJSO3bkKnADGqVBbWEkv\n2sKgoDdBEEQ8EhKMF154AQMHDsRLL72EnJwclJaW4re//W2q29YmBE4AOBWhcPPCJlPQmyAIIiES\nEoyuXbvipptugtfrxb59+9CzZ09Mnjw51W1rEyIvgOMZGgLNTzC0i0SiS7oSBEGciCSUVrtt2zZM\nnz4dTqcTjDHIsozXXnsNZ599dqrbd8w4BAcAoMYfQn5Xb9z9yMIgCIJIjIQE469//SueeeYZc5b3\nmjVr8NRTT+GTTz5JaePaglMQAQWo9Yea3U+xTdxLJA2XIAjiRCUhl1QwGDTFAgAuvvhiBIPBlDUq\nGTj1+ld1gebbGT0Pg4LeBEEQ8UhIMDweD9asWWO+X7duHTweT8oalQzcorZMa12gJQvDFsOgUucE\nQRBxScglNWPGDNxzzz1wOvW1siUJr776akob1lY8DkMwAs3uFx3DIAuDIAgiHgkJxuDBg7F48WIc\nPHgQjDH0798fY8aMwbJly1LcvGOniycbqAXqwr5m91No4h5BEERCJCQYAOBwOHD66aeb71kLE+La\nm64ZWQCA+hYEQ6bSIARBEAlxzGt6d/TSINkuTTBCavMuKYWKDxIEQSREsxbGvn374m6TE1gvuz3J\ncmYCACJoIYZBFgZBEERCNCsYt912W9xtLpcr6Y1JJlkOTTBkNJ8lJavWyoEU9CYIgohPs4KxdOnS\n49WOpJPp1GZ3K0LzpUHCSsR8TRYGQRBEfI45htHRyXRogsGEMBRVhSSrqGloKh4h2fqMYhgEQRDx\nSVvBEHkRHBPBCRJCEQUvf7YF97+xCrW+aNE4UFpjviYLgyAIIj5pKxgAIDAR4BWEIwp2HtKEoSEg\nRe3jC1ulQ+yLKREEQRDRpFQwZsyYgZEjR2LcuHFx93n66acxZswYTJgwATt37kzq9QXOAfAqghHL\nchD4RunAvLUtKDcfICcIgjiRSalgTJ48GbNnz467ffny5Th8+DAWL16MJ598Eo899lhSry9wIjhB\nwaqtJRC6lsB5xjqE5UjUPpxoWRUkGARBEPFJqWAMGzYM2dnZcbcvWbIEEydOBAAMGTIEDQ0NqKys\nTNr1Rc4B8Aq+WXcYzoFbIORUY1/DXgDAtxuO4I7nlwG8AjXsBgAE5I5dgZcgCKI9adcYRnl5OXr2\n7Gm+z8/PR1lZWdLO7+Qd4HgVgFXGpCpUDQD4+Lu9iMgqwMuA5AIYcNRXgtpwXdKuTxAEkU60q2DE\nqkeVzJIjDl5bdc8epygLldgupoLjGZgigmMOVIdq8OdVf03a9QmCINKJhIsPpoL8/HyUlpaa70tL\nS9GjR4+Ejs3Ly2pxnwyXG5AAzmW5mioj5fDk8HCduR5SST/tQ0UAVN6Uz0TO3ZHobO1NJfQsLOhZ\nWNCzSA4pF4zmqtqOHj0a//znP/Hzn/8cBQUFyM7ORvfu3RM6b0VFQ4v78EzQ/s+w9q0JV2PJzjXg\ns6vgyq7S2qiKYLyVbpvIuTsKeXlZnaq9qYSehQU9Cwt6FhZtFc6UCsb999+PtWvXora2Fpdffjnu\nvvtuSJIEjuNwww03YNSoUVi+fDmuvvpqeDwePPvss0m9vkvQFlHiXFYBQgaGf+76PHpHRQD42HWk\n/FIAHtENnkvrKSsEQRAtklLBeOGFF1rc59FHH03Z9Z2GYDi1dFk1kBVlbRgwVYh5fGWwGk+u+Qem\nnD4el/UZEXMfgiCIE4W0Hja7BC3obcQwlOr82DvGEYxSfxkUpqDEX56S9hEEQXQm0lswRK0EO+fU\nBaO2B/p5Tmu6oyqALxze5OMGyQ8ACMvNV7wlCII4EUhrwXCLRgxDn8EtO9DV0TQLi6k8uIYe6J99\nSlSswhfRlncNKyQYbUVVGRSV1hshiM5MmguGbmHoAW0mO83MqShUHqrKIPA8VKaamV0+3cIIkWC0\nmfvfWIXpr6xs72YQBNEG2nUeRqrxOJzWG1XQXE9cDMFgAhTGIOjbVKZC4ARTMMjCaDt1/kjLOxEE\n0aFJawvD47CWkeUUTTyatTB0wTDWxfBFdAsjRgxj/9E6vDF3G8IRWkODIIgTg7QWjAyn23zNq5p4\n8FxTo4rZXFKATTBMC6Pp6PivH27Ext0VWLmtpMk2giCIdCStBcMreszXAtMD4LEsDMZDsVkYsmpY\nGC0HvRWFArkEQZwYpLVguEXLwjAEg48VtlG1x8A3dklJ2gzx5oLe8QufEARBpBdpLRgem2A4OD2e\nwZresjHDQ/F7AAAgAElEQVTTW9HDEXuO1EBSZYQULR1XVmUoKsUqCII4sTlhBEPUBSNm0FsXkcMl\nWsziixV74dfjFwaUKUUQxIlOWguGyFvuJ5cuGDFjGLpLqs6nVazNyXKgIRItGDQXgyCIE520Fgw7\nTkGzNjjEF4yIpEUkvB6hiYURK7WWIAjiROKEEQw3r1sYMQoNMt3qYLpwhCXZzJDyihkArBTbpscm\nvakEQRAdkhNGMFy6haEqMZaA1YUCTNsWliWz8ODJ2X0BANWhmqhDhG7FcA9bBL+anmuAl9UEUOtL\nvlXV3IJaBEF0bE4YwRD1dNqv1xxtutEUDO3/kCybFsXJWbEFw3HqVnA8Q6G0LUUtbl8efnsN7nt9\nVdLPS3pBEJ2XE0YwOF63LNQYt8yiLYyILCMoaym1fTJ7AQCqQ7XRh4S1SYFBRks/tgaVFIMgOi1p\nLxhy2ckAgGxeXys8xjwMQBcTm2AoqgwA6JGhHdfYwjAEI8Dqk9zi9IZcUgTReUl7wegdvgjBddeg\nb9cu+icxYhgGuphEFBmyPtvbI7qR5chsIhjGefxqHVRG5UGawy4SKukFQXRa0rq8OQA89KsLUFUf\nQqhRVVnGAK6RdjDdwpAUGbKqvRZ5EV3dXXDUVwyVqeYCS5ygWSAyIjjqK8VJWb1TfCfHj2RbAXY3\nlEqKQRCdlrS3MFxOAb27e8Hb7jS48UqENl7VdGfdwuA4hoisTeITOAFO5oXMFDToqbbaBtl8uat6\nT0ra3l4kO85gPx25pAii85L2gmHA280JxQmoMYwr3cIApyKiaIIg8gJ27NUC4Ha3FCfIYLIIDjzW\nlW5Kq44w2Sup2q2KjmxgbNxdgcNllMRAEPE4cQSDbyZ2YWAExDkGSRcMgRPBIlqA2xAMSVYAQQYL\nZyCf749ifymK/aUpaXd7kGy3kV1LO2qWlKyoeGPuNjz+3vr2bgpBdFjSPoZhIMQQjOCmK6PTbG0W\nRlh3SfEcb2ZEVYdqMe+HA/jvqoPwXKRAVQR4oQXTG5cS6cwku1O3n491UBND6aDtIoiOxIljYTSO\ncAOAHO2aMkqDgGOIyDI4cJAkBiZpa2k0RHz476pCK36hiDAeoWwrf76zeg8KKran4jaOC8mPYXR8\nl1Q6uRQJIlWcOIIRw8JwORvVlTItDIaIIkHkRS27Sq8/JTNNKIwMKaaI4Fj0sq4A8HrB/2HWtjnJ\nvoXjht0llYyOVO0EQe9kx20IIh05cQQjhoXhaSIYepYUr0JSZIi8gFBENoVE1ifzRVkYrKmFYWDM\nFu9s2AUjGa6aqLTajioYHbRdBNGROHEEI4aF4XE1CuHYLAxZVSBwAgJh2YxzGIFwu4VxqFSLXZhi\nYqM23DkLE9o1QlHa3pGyTpAlRfNDCKJlTmjBcDtjC4bgrYfMFPgDCv46ZyMYixYMiFpAHLIT/oBm\nWRgzw+2zvjutYERZGG331US5pFrRMW/YVY7f/W0pSqpSn1BAFgZBtMyJIxgxYt4ZrmiXlOrPQaYj\nE3yXMiiiD4oSXbAwohoWhiYYTHaYLimj9lRYiZjn++7QcqhMBWMMSzcVobQ6kNR7ShX2zlNOhoVx\njC6pd7/aCcaAZZuL29yGlkimhaGoKpZtPor6QKTlnQmiE3HCCEastFp3Y5eUKmJkrwut92YV29gW\nBlMcZmaVYWGEbHGLXTV7sbN6Dw6U1OOjxXvwl1lrk3ErKSdeDONQaQO+WL6/1aNxtY1ZUrES3Foi\nLDWNKTVHMgXjhy0lmLNoN96c23kz5QgiFieMYHAxg95Np6FkOb3WG0MwDFFQNaHgTJeUPehtWBjR\niw4V+0oRCGnbVCh4o2A2NpVvPfYbOQ7YO3hFsVxSywuOYuHqQyipap2ldKwuqWPtwn/YWow7X1iO\nzXsrEj4mmS6pitogAOBAMVUyJtKLE0YwYloYjbOkAHgcGdYb1TiGB2OApGdCGYLBZKdtlT5NFEK6\nYAicdu4Sf5lZH5f31mFH9W7M3v5RW28npcSzMCKyJh6hSNMAf3Mcq0vqWBXj2/VFAICVW0sSPiap\nMW/zp0ZxESK9OGEEI1bQWxCafpYheszXzL52hsrb0mptMQzd+vAFNVdUSNYEY8wpV0DkRZT4y6zl\nNljT69U0hBFppfsk1dhFQra/1q2NcOTY3T0dNbaczJnenP6F2+91b1EtZi/cYT7D9mDVthJ8vmx/\nu12f6Pyc0ILBxVgbwy4YUYstMR6SEfQ2XFK2eRiBiPaZYWFkODzIEbvgaH05Xvz3Fu043upoGWOo\nrg/h/jdW4c15HcvXbe/o7C4pST42wYhXSyoQkrBxd0XcyXxMH6G3NobBmQLdijYmUzBiXP/ZjzZh\n1bZSbNlXlbTrtJbZC3fiqzWH2u36RGIEQhLmfLML5bprsyNx4ghGgr1OhsMuGBxGnN0TYy48CVB5\nax6GKIGpHKAKtpRbXTD0oLdbcKG8SoLMJJiuCcHqaINyEAdLtMqoW/e3XycSi6gYRpSFob1ubUA5\nnkvqrXnb8cbcbVi/qxwAsPtwDZ58fz3qfNFxoFjC3hzHECNPWgxj/9E6LFwdv1OWlPa3JimFuGPz\n5epDWFZQjDf/s629m9KElAvGihUrcO211+Kaa67BO++802T73LlzMWLECEyaNAmTJk3C559/nuom\nIcMl4vSTcmP2LN6oGAaPrAwHHCIPxmwuKVECZAcAzoxzGGJiWBhu0Q2oAjieAZw+UrZZGIX1R9Bg\nS7sMyiGsOroWSowZ45v2lmHBmn1tueVWERXDUJq6pEKtzUCKCnpbr38q1Kr/Fldq8yx2FNagsLTB\nFNLGIYBV20pQVpOa1ORkdaJ//XCj+Zp10BgGTVLs2BhJMnX+jpeWndJqtaqq4qmnnsL777+PHj16\n4Be/+AVGjx6NAQMGRO133XXX4S9/+UsqmxLFq3+4DDzH4bPvm3bCjV1STgevWSeqzSXFK2BG0ULT\nwtA60bAew3AJLr04IbRSIrIzatGlN7bMxmnCRQC6AgDm7luIVcVrURWqwfgB10a16Z0d70LIqsGo\nwFPIznA1afPhsgYcrfBjxDk9W/8wYqDEmbgnHWMMI56FwXGa28b4LCJr5/WHpOgTcEBRhQ+zF+4E\nALz70JXNX9B0CSXeMaakWm2sU3aAvlpRGMSm+R5EB8HwnneAn0oTUmphbN26Faeccgr69OkDh8OB\n6667DkuWLGmy3/EuSGe6p2JYGA7eYb5mjIfLIUAUeIDxCMu64vNWQUJTMHQx8ellzr0OD5i+j2lZ\n6P93dWsl0asjWtqnxyWgMqi5pbZX7WzSJiFLG4nvKyuLeT+Pv7ces77c0bSjPUZYnIl78jHGMKLK\nm9teG9+DoUkRSXvhD+pJBbZzGKOu+NdQ8fyqt7GudJMVdNa3lQcq8fKmmagKNl6X3SIVy7J3UL1I\nyuz9YyEQknCwhFKNW4QzkiY6wq8lmpQKRllZGXr16mW+z8/PR3l5eZP9Fi9ejAkTJuCee+5BaWnq\nFiK6dvjJ+OVVp8XdfuGZPaLna6gcnKIuGCoPcCq6ZLnA8aqZHWVO3NNdSUY5kFxXDqDooqILhSEc\nkwZeBwAI6gKU5XGaIlIeqIzbvgOVzT8bo8NtK/FKg5hZUq2OYcQ+t5GIYFoY+nl9IQm+oCV+HGIn\nLdgpD1RgXVEBPtjxSZNtH+38DHtrD+CLvf+Ne/xx8+u34jLBsIxguPUpzC11NO219seTH2zAUx9s\nMOepELExfukdUC9S65JKRCGvvPJKjB07Fg6HA5988gkefPBBfPDBBy0el5eX1er2/P7686Pee3X3\njihw+Ntdl6Ffr2w4HQLcQgZCiuYr79Y1AxFJBSvlwfEq8rt5cIhXwQwxMFJleRV5eVnwq35wHIeT\n8/NNC8N0RelB7755eQAAf1hzXzkcAjiH9qwkVcL+0F5c1Oc88Hy0nleEapq97+wcD/K6eWNuKyyp\nxyNv/4iHfn0hzj61W7PPqaja+oP2ZrrNaxrfJifwLT7/3YeqwXEcTj+5C6oCVuefneMxjxV4DhIA\nt9uhfSZo9/vlj4fw5Y9W4Dgjw4luXa37inXtBsGyHhwO7TxOp4i8vCwonL4YliP+76akzpqh35rf\nViAkweMSY04MjXWuzCztee46VI1Fqw9h2i+GwCHGHreNu38+AGDBCxMSaosvEMEvH/kaYy/tj9sn\nDY66tj2dN7eLF12z3QmdM5mU12i/KyYIx/T32xaO9/XagsejeTk4jutw7U6pYPTs2RPFxVYdoLKy\nMvTo0SNqn5ycHPP19ddfj+effz6hc1dUtH3t5WBQG+EzBnTxiKir1UQii89BSAmAc4YQDkmaC0bl\nwXGAIOqdfyOXVEiSUFHRgEpfNbIdWSgr85kxDI5XwABwvHZsxMfAgYfC6YHysIQan2Wqv/jjLNx1\n3q0Y1PV0RGy1qUrrK5rct/0HVVJWDyGOu+H9/25HbUMYr/17M566dXizz6XaFliuqQmY1wyFdSuq\nPhTVjj1HarFs81H89ueDzM7vgVd/AKDFG2qqY5/P6GN9/jAqKhrQ0Cg7yiAQjKC21jrHrP9swbkD\numFAb+u3c7TassxM11lYRkVFAyKS9pyliBL3d2O/55Z+W2t+KkVOpgsOkcczH27EL0efhqsvPCnm\nvo3PtftgFQ4eqcG8lQcBAKf3ycawM7W/CZWxmNl8if7W9xdr1u2XKw/i9kmDo46zWyrl5Q1Qwslx\nXx4LtbUBVFQ4j9v18vKyktJfHC9CumtZVdWkt7utApRSl9S5556Lw4cP4+jRo4hEIli4cCFGjx4d\ntU9FhVW+YcmSJRg4cGAqmxTFWf20gPMVQ/tEfZ7j1DoizhWEUxS0CX66MDhdWmdkWg+qVXyQMYa6\ncD1yXTnarGjTwlCi/j941A+m8KaLKhRRcKAsOrW2Pqz9UHy2pV+DsERlR2E1bnluKXYVVpuf2V1S\nQTmIz/bMR0PEpzWTGXMaWk46jSo+aA96c34ArIlL6rl/bsKaHWUo2NfUnSYratxaUlYMI9ol1aQ9\nKovK1vrvqkL8dc7GqH0CsmUVMS76PEYFYaWZQEVr5mG8s2AH/vHxZmzao/12P16yF8s2H03o2K/W\nHDLFArC+j6/XHsKtf/selbq7Jl7cpzliVTMwsD/b9ophdAa2H6jCii2pL3bZHJwZw2jXZsQkpRaG\nIAh45JFHcMstt4Axhl/84hcYMGAAXn31VZx77rm44oor8OGHH2Lp0qUQRRE5OTl49tlnU9mkKAad\n0gUv/P4S5GZGj3Z6ZuZhn38XOF6By8EjIvOmMBRlaCPnxhaGwhT4JD9kpsDDe3GkrMF0WxmWhRHL\neG/hPrjPFgBBQddsF6rrw3ApQXDMiQGu83CArTNX9/NL1sjX7zqChogPWc5MfLxkLwDgw+WrAUcI\nkNxRncJXB7/DsqJVOOorwR+G3mF2ynwCQwSmMkCMACpvdtQH6gohnfEtxOL+CEeiXVqc2w+hSxlU\n9awm56ppCMfNkrJiGNp7o/RIYxSFtdjJ2Z+TKuiuD92JVhfSRLesPn7QO9EYhr0dWR4rQWLOot24\n/Pw+sQ5pFpdT+0I++16bgb1lfxVGX9A3KsgvySqcjpbTmuwDhsaps2Hbs23v9cuPpZjk8eLFT7VJ\ntj8b0rvd2mDGMIz/GcPhMh/65Hm1eGo7klLBAICf/exn+NnPfhb12fTp083X9913H+67775UNyMu\nXbKapqlOPP1qLN+5B1JJfzjO4iEKvDlBz8fpFpEhGODAmCYYxmh++x4/Nh/aAqGbkVarB70NS0MR\nwFTNwuia5UZ1fRicIIHJTuzcE4HrNGteh3FOJjnAOSSsKdmAq0+5XAsK8zL2ur+B53wguO7aqJG/\nsWTsoYYi/b0uGLYMjHjWhqKq8AxdCjXshqKeAwDYXa11aI7eBxEqvihqf9fZP4ITFBRHDgKITu39\nfN9cfW6J1pnaR/JxLQxBgpBbAaWqFwAOispa7OQCNsFo6FIAFA0BGKCoCoKKH+CA2kj8DB1FjR7R\nx3s2stx0XkpbaDwp0RBXvy3oH4ooCQmGZBOFmobo1R6jLYz2FYyOOHJujKqyFhMtUkajWmTrd5Vj\n5vyfcOXQPrhpzBnt0yadE2amd2vwONyI7DsfzJ8LABB5DpwjehKNkR2lveGhMsWsVMv02AVrlCUF\n3hb/UAWAV3WfPwNEOao2VUTVrlcf0gRDLusHMA4bywoAaB1K4zbZR+heUZuAaMRAmGlhcFh86HvM\nWPV0lLvLjk/Wrsm7QmZpEJG3OqzGabWGEMpoOtFob8Mu7Pb9BHDaeaJcUk2ypLR9nP23wTlgK4S8\nIwA0AWtRMGwuqbCnBJxX8+f7pID5B6jy4ZgrIwLRa3o3Z21INpEoScL6Jo1Fx7i0L2QXjMQypYx5\nLABQ2qiisN36SMYqiolSWh1oMgGtvQUrEaQ41u7xoHEtst1HagEA63Y2zTA93pBgxOGZ2y7GpMv6\n47STciEIPDhno1RA1TbiU3koULG7qCp6m/4/7w4AnArOFdRFhNMFQ4HLof3PcQxQHOYxRqmRyqDW\n8alBL5i/C474iiGpsjY/QrT9IYqRqFFkWLW2qUw1O2qe4zB//9eojzRgdfH6mPdeI1nxFKP4oGHp\nAEBIit2BSUwTTCvXniHMQlChgvPolpIhDkoEkZNWg8+pMMXM6PB4fd4J79XOoygsZicXDMtm525Y\nGMPyz9MeR14RGLRYjp36SHQQsSZUi/21hdFus2Y6NPszLqlsWTBueW4pNu6O/4feeIEqNY6FkQj2\nTs5+vLbNOsfxSiFmjGHGO2tw72sroz5XErTMUj0PISwp2FFYbT4P+/Xs4nu8MWuR6e+NZnUEVx4J\nRhx6ds3AuEv6g+c4OAQOnKsZwWA8GBSs31MStc2wNMT8w3CctFsbsVf1wvCztJRbjlchijAtBfsK\nfuV1flTUBrFgk7Z2BgtmQY1o5zOsBrMIIgBnv59QHCwy39sXclpyeAVUPSbCc0A3txbs31a5I+a9\n19kEw+ioa8OWOyckWokKhrABgE9uwM5DNXjqgw3aB7b28Rna8cYf5+6afVAzy+E6Y6M54jRGwcw+\nQx7QXVJNO5nfv7QCr3+h1dvx68Iw/lRtljznCAGMNRGMunC0W+qx1X/Di5vehF+2xUD0Symq0qRU\ni6SoAKdC7HkQxbVWTCTbGz/rZ86i3XG3Nb4v08I4BsGwWxGSrKLWlnUWbgcLI968oHirODYEIman\nXV4bxO/+9j1WbYsuUb+vqM7MBmsrc77Zjec/KcCan0qbtPd4WBjltUE88OYq7C2qjfrcKl4ZLWSJ\nJKykGhKMBBAEHkp1tG+eRQkGh7As4XBFbfQ22z5Cdy2LRqnqhT7dvRh0UncAgCgycE6tc2cRt+nG\nWrPzKB6cuRp8Rj2YIoCFMsxyJIYY2F1SQtcyfF8zD/uOan9MQZtgzNv/FeoytE6L5zm4BK1zO1h/\nOGo/QAtu7w/9ZL43Rlp1YWtkLuVr1XUDIQlPrPmH+Xm1VInVRwrM95zNAuIztOMNwbB3xEfFjTjS\nUGyN6owZ8oarq5kYhpGZZVgYOa5s/YLasQH9/pik3XNdIwvDiPUEFMuCUvXJbw+seBSvFcyK2l+S\nVYh99sJx8m6gj/accjKdZipvLLo1M+ehqUvKsDAsKy5Rl5TdXbZ43SHc9/oqFJZqAtkeWVLxKg/E\niv3sOVKLe15dif+sOAAAWLNd68SNcjAGz3y0sUmGXCzqfGHUNMRO0zbYul/77Rws1n4TgXB0okGq\nWbDqIKrrw5g5/6eoz824lv6TtwSj9dc4WuHD58v2J+07J8FIgByvE9LBcxAptGUBKZYYaAFs1YpV\nGBZGyAsW0YLqnD5/g6kC3E4BXpfWiQgOm2CE3QAz4h4qwCngPH6ogSyYbiwADSFdMMRGMQNexfwf\ntD84Y10OA5nXOlSO40x/v8pU7KiyRr+MMby48S1Uy5YLxReQsLNqD/bVaUFvpa4buIw6bDlUhLte\n/gE1YWt0dCS0HxsjX4PTrQm7BcRn1gK8bJbgsLuGKt3b8fGuL6yRp25lGfenKGqLo+KAHESGwwOR\n10rOc4IClVkuKTWoTfybtW0OygNNV+Lzyz4Yf6EqYwjIQURUCXtrD5hpuYDWkQi52vGcU3vGWR5H\nVPpxY5oXjNguqZYsjFjuGskmCkfKtOdbU6+10e5ikY9TDMEXjC0YscR/2wHNqv16zeGkXPve11fh\n/jdWNbuPEUMzEkKOt2Bk6ll2TZ6TEXMzBUP7P9GK23aemrMBX605hI27E199sjlIMBIgv2sGHvv1\nCPx53HXWh3pw+qW7LwWLeMA5w+Yo+srzTtb2YTxCWy+LPpkqwO0U4eK1Ea8gKmZ8hEU85nnBK4Ao\ngeMYWETrcAzrwxcOAWBaJwwgvGeotl1yIcOt/QgbWw6GEPE8h5AcAqf/+2zPfEiKBEVVsXD9XjMV\nlelVeOsCIawqXqt/xkOp1WapL9tfgHhwDv3adgsjsw6us9aYHWLjWIKTt7l09OM4dwDgFCgqMztk\nvksphPzCJtf0SwFkOrVAP8e0uJCiqKZgsGCmue+yoqYdyfd1/4F76BKAU6GqDA229hX7rJIskqxq\n7QIA2QEOgNspQpaZfm/MDPAbZGVo34nj1K1wnBztBmw82pbMVQ2tDn7TngoU7LXmuDw6ex2mv/JD\nk3uwJz3U+bRnaIjDdxssd+Xxqlbrj1P/K5aFITRKgEgWzcVBGmfpBW3ttT/LzXsrzPkxycSr/602\nFifr+7EGMEDiFgZjDDsLqxGKyKabraVabIlCgpEgp/TMQj+9pAegFSn87f+ciRyvE0q5NstX6KGN\njjJdthGlKkIN299rFoZT0Kf/Cyo4l80lZRYsVK2ihYZPX9+29WAZ+NwKCF21YoQs5IUayAQnSvC6\ntX2DSgheMQMPXHCXdgpO74Q5hpAShlzfBVJlTzRIPtSG6/HV6kOYu0brzOSyk6HWafdaFwiA5/TM\nrf2DwfzapMafiovMjlGp64bIviHmLRqussZZXHyGDypjqAnV4utCrQilGtKqAwucNafBsEw4UYLn\nwm8R5GrNUanrtAI4T9llrnpoEJACyNTXY+f0DDRZZQhKIf06lmB0cWnZb/b4i3Y9GRAjUBlQbwvy\n76zeY7tOSLMmAUCMgEErLaMyBklS4Th5FzwXLgbfpRTiSbugTXRUAV6G2L0YYs/oEbSsMESUCBwD\nCsB5a82OKmxLLFi3sxyvfmGtA19U4YM/pE0U3by3An94bSWq6kJRnZzhglIUFTUNYRSWWgJ4vGIY\ngTguqVgWRpMU1iS565uL/5hZemosC0M7rrIuiNe+2IYH3159zG2org/h9f9sQ3mj0vweV+xZDcbz\nYY0sjETXhdm6vwr/+KQgytWVLCEmwThGpk0cjMv0yT0PTbgaALS1LwBkuT3RO0vWXA+m8nA7BTh0\nwdiozrO5pBpZGKaLS/9MtzCWFhwC77UCf0xyaIFiQTLrKIXkENyiG909WoBb5YxKu/ofsSzqa3oA\nYSWMo5V+0zLQhEs7T00giCMVesC6vqs1GVGQo1xwSnUvXOwZq20zBEO3FOQqK/7DGPCZrQhgeMcI\nvb36WuiCas1X0QnxNXonZ/3oxV4HIHTT4kKSIiGiSqZggIngeAWKwkz3GwtYgmEUioyVVszxShML\nY8XR1WY6bnXIcsFxDq3NxmSqyrogxJ5aDSzXaQVw9CoE5/Fh9U+lZsaXdpNWR6ooKjaUbYHYrRTu\ns9eYHX04gUKSEUnFa19sQ70/ghVbiqMyoczzq6xJxlQy/NmL1x/B9gPNL/zVGgujsbulpa4x0Tkw\nDXHcYgDA8wycK2B20AFbuRQjHmTEkuz9bWFpfdRaNi3xr+/2YtOeCnzwTXTyQ7y4XGMLsLUxjCPl\n2mDHvjBbsgw3EoxjxO7yObVHHpyC5VLJz8mE02E9WjPrBzBdUkbgWoEEzhnSOmhVNK0IIbcSYm+t\nhARTBW3tCHswWD+nQ83U0nEVBzgOCEohHK30IygH4RHd5voeKq9nYhnrkSsOK4iuhBGKKFHBd6Md\nlfV+lNQYCxoJWhsBbU6JLhiGiDiYdi2tI2Xgs7UfrFw8AEqdJlyyKkfFBKDPPTHmsLgztG1yVU8z\n/lNcXY/NeysAm8Xi6H0QzgHbAFiikOnMwIotxZAlDuAVyKrNJSW5ENl9AQCY14o5D0VQwBhDQ0Tb\nluPMRnWoBosOfQ8AqI1YmVG8JwDnoDWQnFp5lkdmr2tyOk7P9LILPOeyRpolVQFsO2C5mzbzX6Cg\nfJs114VTwGXoqdVqdCVae1B5wY+F2HbAKBOjwtF/G/icCpTXBPHOAm2kaYxo2zoPIiwp+GTJXrz4\n6ZZmR66+QMQaVNiIlSUVq6wJn10F8DIW/FioXVeWwLm076W5GIO9w20sllHt6LYP7iErUMNrIh8M\nW22V4gh2MCzjr3M24tMYa+nEw6jj1bjN9vRi+/ca18JIUDBifSfJckOSYLSS+4ZOw+DuZ2NI3tnm\nZxzHobueqgoAORkZeHX6ZfjddYO0D6LmbAhwOQUInPUZ5wrYgujWr0LsZqXpCjxvncdmfXRvGA6A\n01JyAazacQRPfPkJwkoEHtENgRfgFJxgumWh8LovVhHNa+4trsK2I0Vw9tdcUizianQt/Yet8qb4\ncYJiWQL6viLTXW9iBJzHByGnGkpdNy1+oAuNpESQ49QzmcKZ2v2qIsJ6qrA3Wxe0iBuRQ/rzE2Ts\nKKwB74ox74FXTMEoLAri/a93aRYZr0BWmCnsTHFA1WNBxrViWxgyFJuF8asz/xcCJ2BbxU6tVlgk\nuryIkFWLqowt+sEx3B+6YHA2weAzGsx9V24rwcb9tnRovhabyreas/Yd/XbAfY42XyUsKVEj68bB\nUmPlQj6rBmLeUbjO2IgFPxaiqEJfo8V9bILhC0pRKx3a/eH7iuKnuG4J/gDPsG8BRzjKqoll4TRO\nGdUBXG0AACAASURBVC1R9sN15no4T92GuXrm1OJD38M95AfwXUpRVF/S5BwG9ooHDXql5IgiYfb2\nj7CzynIvylnac68QdkNVWVSBRimOBdMQiEB116ImmHhRQEMMGmui/bu0W2OqKRjG/CRtv+bSarcf\nrDLvO5Y4JGtOCwlGKxmQ2w+3D745yqIAgG4eSzCcghNOh2CO6KIsDKa5pK7rP8b8iBOU6DTdxqgC\nnCJvm59gCUbEGHTLxjbJHNn3d2mi5hE8UFy14Lx1OJS1WGuGKpjn++KHPVEdGot4rJnsvKKvMMgB\n4CAw2xyJRllhX63UgsOcI2LGIVRfLgDOtEIiTMKuI1r7QruG6seL5qi/78n6XIxAllXt1yitos+F\nUX1WlVoICvy6NXDoqCEO2tK4ZTU+lNbVWc9Hb0NIDmPzngrUhzXTfdzJ43CaMNw8X3lNED/sKAQA\ndPd0g4Nz4VBFDZYXFKNe1s7X3WG52VzQ3F1mMNyGZWFYLinnqdvhOtcKvBvZVgYO3mH63oXuWiE8\nMf8wnl/xEd7+cru5X60vjltEjD2qNrJyYsUwGGPYWbUHSw+vaLLtyffX4+G315hCYff1Hy5rwJqd\nxZAVFXW+6LphRdDmyAi5ZVGxhFgWRuNRcbWi/Zb4HMv6MuYNuU4rwMvbXjMXHWuMPYXYsDCe+Wou\nNpVvxZtb3zW3CbL2vQWcxXhj0cqoCgZGsLix66vSXwf3OatxOGdhzGvHwhDoxnEa+3Owx3tMC6PR\n/cQTjK37K/Hiv7fgrXnabyOWNiQrz4EEI0nYLQxXIzGxp+ACgNspINPpxbhTr7HtE7+sF1MFnHda\nd2t+Aq+YgdeI3tf07qIFcjlRMjvy+Qsi+GFrMer8WkfrPtsWuJOtWeWcIJsBdqUhFyycYbuWNlHN\neJ/tdWvioQuJ0T7thQAmi5pLqpGYGBZGfTCA0toG/Tjt56fKvFkKJcRrI3g1mNVkAp8xW1wqOg1y\neV+z7XVhv3VP9mtyKg5X1gJMqzZsuOB2HKnAa//ZhrV7tNIjny85gp8O6B06r+CNudvMLK5sZxYU\nSQAnyPh+81EE9LIpl+Zei1OyTjKvA6Dp5E6j7YIE3h2AGrTWi+dt4mK4Ag3CShgRSdFcEJJ2T0Ju\nBUqFHdhSaQW/lx9dYQbW7fEd3tYOzuU3S9JogsFQK0V3tC99ugW/+9v3eH3L/+GLfV/CF4m2vCrr\nAnCduwKf7JoHILpU+vyCDfiw5GX88V+f497XV2LpJqtqL6enR/NZtVHHxJrpbc1jYfj20DL4mR4r\nsv1d5Dpzo46xx5PsRFkYumCUBLV2uQUrAYVj1t/lT8HVUccZFkYTwQhoAwZFaJSFqBOQAiitDuDl\nz7Y0qTzcuMO3p2LbRbixBRavirNBlb6WixGzUE2LxroeBb07GD0yojOoAFvVyUZi4HZqP9QshxWI\nRTMWxjmn5GHcyH4xXVL6GkxwGX8IeufPFB4Ah/e+2hXl+zeQy0+K7pANwSg7Obo9hktK/+PPyXAB\nqhgd9LYJourPBp/hM+cqGGJiWBhLtxy2soxsM+IlFobY8wB8rBo8eM2NZVoY+ig9s0ZbA9yfY2uf\njJqAXnZEFwxr4SpN1DgmAuDQv4fW4dQFtM66rF7vcGSneQ+cIOvxnDB4JsIjujWxE2R4XCIkpnVA\nua4c3HXerdrt81qasxEEz/QPtGacCzJ4XeiU2vwm3wOXUQehizbvxUiPNmJKOV5nk98ObDXMdsmr\n4ehVCPf5S+GyDQbswuUe8gOcp23W2uVxQMg/hK/r5mBd6Sb4ghIqa4PmHAiDssbzVBwR8J4ANlZr\nMZoo102u5i6K9NkA99Al+HanlW4tqppA8hn15loqQPTI+l+7Psc3hUvMDlroWoJ5+79CNbRsMuO7\nNObG2JHi1AWzz9iuD4Qwc/1HELtrLqwcwVZpmbfNE8poiBKMQCQIX8TfpIJybSh+ActN5Vvxxx8e\nxxNz/4ut+6uwYqt2TaP/53kOBeXb8O/d88BYdLmbYEiGrKhY81NplHvqQHE99uhuP9mW1LC35gD+\ntesLKKrSRIhCakBzC9s+TkahTOA4VKs9UejltToDI2X27P5dMbBvDnqc0RubGw6a242smkynJRjN\nuaT69chFZobDCi732W8GkcMh7Ufn5LSAM++t1zp42/mYypkZXAAwKOdsbLIF2DleAThj/oXuRtM7\nJtcZG8Fk69ouhwCmCOAzfHD029Gk7XLJAAg51RC66nMXDDExr2V3ZfFR2xwn70GY5SLT6YWf8dGC\nxqngM+t0V5XDFkuR8e/lO+E8xSbM9vsSFPD6SHLMsH74oASmxcIEW0kW0Qjm6+a/IwQnMsBxnGY1\neRR4XDwa5AjAAR6HC27RBQ4c6vijcJ3lg1KjLw5Wn49Tc06x2m6Ufom4oFTnm+nQAIOYd9Rsg1qb\nBzAtxhKWFGRlOBBq7F4y53hYHQDnkMA5bEvaNrJ0hBxNELxuh/m9fLjzU8gNXyOy7zwA0Vl9ZYEK\nDMjthyUbi7CjsNoUQkCb7GkXDHsRTk6UEcksQllNAHm5HiicdhznDkRlIBmdl0/yY1WxJkKXsf+n\nP/9GI2H9O919uBZ1IT+YIkAqOg3OU3aZKdONsXf8FeEybD9guf/s7WC8BMY4qHXdIORWwu+zLKvl\ngU/x1cpaBNddA3tc0V5aRlEVCLainKuOavOVkLcfqOhhDgzNET8PzNr+IQDg2n5XRnXitb4IPl6y\nF9/bLDQAeHrOBvO1PWj+8uaZAICzup0Bvy1lPBiWsZZ9As9QCcr+oWAKoNb2SNpERLIwkkSvTEsw\njHkLToeAGTddgFN7Wu6qHrkec0SQZROMU/O7xD232+HS4iH2UiM5WkZMSP9bzuP6AbILYs9CcKIU\n1YkbqasGP+3X/fpGp+uIxO3EAd3Npb93OQXzOMOtkuG0pQ0bM9sNq8ZMCbbiL11zdNcR49Et22Va\nLwAQVANwCa6oY3i3D5zbD45X9Vnv0efjmri/LOuI4xWAieA5Dh6XA1BEy5VmCIbkjC5HwqmAIwIH\n00bITI8PFVXVotrvB1M5uByi+T0D2sREo2MNBUS49ew0TpDBGWm0iojIgcHWvBxeMcUrvPsCaAkA\nDoSVMMKSApcTUUJg3C/QNO6hfWi4xmIXRfR6RG1yKLSOn8+sg5CnB9xtAfsVR3/E90dW4p/f7sHm\nvZVR82nW7i1sNAksenTr8zE8/PYarN5eAlUfwXO8ioKqzeY+isowf//X+GjnZ+ZnZufJR1sNxiDg\nHx9vRqWvQRN3Q0SKLMuoMliF8oAW77ALRkiJFpWGUNDswFVeAhQRql9LwqhnVrzEcInxuRXgnNbz\ntE84/XL38qhzu0Xtd8tn1kHIPwSfXpGBqQyctxZlmZZwheSQaWnxORX4v6VrmoiF7SmAzy2HxJp6\nCurD9VHJD5V1ISjQ3gsDNsF1+iYA8et6tRayMJJEpiP2WtoA4BasDvW5O6zO236M8WMDADWUEeXj\ndotO8BzX1D0BmCN4t+CCw5cPKfuwlhoatM7NAtlQ6rtAyNbjA4rW0ZnFEXseMjv6Jp2u2SjtmHNP\n7YbdpdGjwAyHC8afUeM2mi4po3JvVg04XgWTNZdZbpYLDbZ5CRKLWM/COMbbAOdA3dWhOKKuY3eN\n9evRBQfqbBaP7rrjVDdEgYPLoWea6Z1uhOkjcVs8B7wCzhEGxwG8rHWuiqRtq/b74OQVPQlBvy9b\n7MCwJP535CB4jHsQ5P+/vTMPr6LK8/631rvl3pt9D1khJEAgAcIWdmQTJGkWhRe1WxRFWxRwQXrU\nntHWmcYHp/vpx8exfbrtxWec0R573ufFcXoGX0VfEW1axBZwWFQSIAkhZM9dquq8f5yqU1X3XiAo\niCT1+QdS66lTt36/81vO77AEAKLfR+tJBu9qpsrOsHZCPvbM/dEQrVnl1kujdKRDSG4znxdm3EPr\nCYJP6mT3giLFVVY2FF6SR2Lns316jS2rUmjsPonG7pMAP4+6Hy0Wxnst/w/DBLNEjnWftX0ff3EK\nsHhc32l/E5BmAVE3FFXBn/Q0ZQNjBMydJ2BPrx21pXwrxDz28T3/AABQT1SiZkQGDKupX6XfEYnI\nAE+gIIrX/u8x1FXlQOOiIKoIEnXrx/YDoHEeA9eIv4BEJYQ+ngPALP0PAH86/SamFVYj2RXEZ1+0\nQ+bNb1guPISmaAaAcmiEwD3qA1jzqvpVY+kADa5yWhur/8OF9gcWopCKPgNUEWJmE5TmQrx3oBJ1\nVTnskLZQO3r7zcFmbyix1ZVojs7XwbEwLiO3jVqDtRWr4rbHBcF1rBaGWzSPCf91KrIj49jfXsk+\n4rahj84FgYdILItBxQp8y7lMoFpiD8aIlQniGEFg3Ke2IhMen/3H55XNQGJlfoZtH2LuJeV8gQ6t\nlSkgr0sCHxNjMRWsOXrlPb0xbTfdVUYW1aIpZbhr2Sh2bcMlBU2EKPCQJUGvEkyPjyIEmXMB4G0T\nEg1hzOmCRInw7F6Gu08U4z8do8+mVRSy4CqnB71pmyVb2zneuk93BSoCzhquEd1S0HqDCB80srgM\nhUGVgtKWixJ3JbuX26uAEzSo58z3YMR2fB7pvAI+UZzLmHhpVSZfRj/F7tC/xD0zQxf41gWtzOvR\nfbGjfoAuGMbJ5sRHdg6bx0EAga4Zw+IaMCbVmQJeGHYQn4TMkf8XZ6gVEvmqki5Cxit488MT+JsX\n99K5SYrI+j6il+ePrSLASVHW7/2qPSHgz6cP4L/+3IifvXYAB0/YYz/tfZ147vVPcbYr3hrsV0K0\nbIulb901u2wWlpjRCDGtGWImtQI5bxd+9QYtxmik5Z/oakJbxKz91hEyFZoJGdBE0IHgKIzLyPis\ncZiSMyFuO88JCY6mglHk6Y/VI1mFvYhUyfzgPYbLRxMR+nQa1G5rtggVqqLAsTgGEB8TIYawAphA\nNeIVNoyPsTdo20w0ARyoHzys2T94aymUZdOG29P6mIURG7wVWLtjS4hYra04Yhan4t294EX6MYwp\nzqJ+Y0MR6rW4NJWn/SPy9HyjbDofhgS97WxCogroQlDTrS6j76Sig3pCgZBwohnn6odP8kLgBfMZ\nBMUsPBmbxaUrO7ZGivFcutDo1Vd31HoDccv9GnEKEvGYmT+CAneSns4c8qH/41n6OVTo+twSIEbg\nhh/jhRvYOQDAiQlcXEaJllilYH1mfaARbRyuX4ee09qjB2rbzNGwoZxCoEJ3Zv5UjEmnc23CSgTy\nyA/Z+ZEv9Tk4hsIQFBrEVU3XrFEdIOEETMM9Z9QzUyQ9ecFUQLTvRWaBRbWw7ZlslxMj0DQN3Rq1\n9JRmGqP6n45jaGmn76IzRli3dnfhz5+fscV8DPqVEBRVs/32OTFqm+BpxBUNSJi6SOn6NvT5jnQc\nx1fBnWwA0R1O3BeOhXENcb4qkxzHsUwpm8IAkOo2zUyfvu+Bm8ZhTd14WyE9A1HgIXNm2mZsKq/N\nOlHjLQwDluranYolefXmDo2H1y0mXLbS6zKVkUcWY6wZe1mT2O0CzyGpc7Rtn0s4v8Jgqxhqpjst\nJZd+ZC5RhiQKTFm6hlM3lhrlIQg8C9hTIURAhAibbMgEsqufCS0lIlF/uP48gr+DChNNQDDB+he8\nKwS/TGMsIi9C4kXwvk5zBr0S605T6WjW1l8iOEEDQNBJ6MhR6w3aLCoA4APt+r4AzeQCVSYun24p\nhj1A1A2tJ8DOccs8OCkMGR5oUaOWma4wdMGlfFUJF9HnlRgKw514ZUYIUXBiFEJvJpTTpbRqs369\n3sAh1nZDmRjtCGnU+gjIfrgFOsiJkAh4t+lKU9vyqJvUSEKwVihg82ki6O6L4Me/j587wgpYWtyB\nRBV0a5SwthhVEgAgrIXAwVSQysnhiJ4u0q8XxeftxxHiu6C05SB6YiQIAXojvTQOZ7lXibfc9nei\n2e4hJQRVJXHK2FYRQE5sDfZG+0BAQIjlW9Sv0xOhfcsSMAA6V0nrwtuNF67eOxAchfEtYA2OxmLU\nP/LGKIwMrxko9+gun8qiVMwdn4/a4QVx1xF5Dh7eku0SY2EE3JaYBps3kaBdFuFV4LPcR+Ph0yd+\nzR1mX6PdWgbFJQuILYUCIP6jMUqgCDwy1QpEjlWxXdaYT13WjITnWe/Rqa/V7RJkyBIPrSPTdoqq\n6BaGJACKTOMT/nZwHEE0bM+sEoJn2WS5aJhHR3fY/mECyEsNskmZPxi1xrYvYEmVLg0WgxMVljbL\n+tbmTlPs/WVZ1pfz9FDfe9TNLLTcLBe23TIWQuAcDdZG3eZgQ1DBu43yLh7WT5yg4YHVVYiQMDie\nQIIHasS0cgBTKah9SVCbS+g2MQLe3w4huQ2C6mGVio1Kxka1ZD90a1iRqFXCaeCDbbS6cVuePd4E\noF9feyQgm8ouooXNwYD+PoxFxgBAyGw076G/q7AaxpHTbQgVvQ0AiDaOgHKWWjRs5C4amXAyJE42\n+1ZXQD7Zg1HDMlkbkv0uJqi1sAuwxHi+7KJtUM9lAeAAVaJl8I0MJDEKLezBkgK9phqz0CxlhCTq\nau6N9OsWhm6hnSzV+9SqMOh5N5Wstr0rowqB2poPtbnYdq8e3RWodadAOZPH+r1d/hyvHvl3fFMc\nhfEtkJdEf8RjM0bH7TMsDLfooi4TnaJMM188yWVXJiWZMXECUMHrEUylIPEinrl7KsaU0OvwmtUl\nZZYhCR+eyASAfR/g4k2LhRAeyUm0HfWli/HszCcROlCHaONw5LoKzXMkIa4UCkAtFhvEtDACXpmu\nBWJcw+KSWla6ALnuYeZpTOjGW0cuQYZLFEAiHuYyMNpAYxg8lDN0wp9r2BEAQJeRJWkR2oK+RGxP\nN/DBwRZz3ghrn2ldTMgahxJ+PPvbGpdaVW5aaETjLMrO4l4SFHbv5CSZKYZ5tdmQ3SqIItvbxysQ\nfL0Ap0HrplaoVzLjJWYBSSPuRd/7sFwP2iI0pdZDUqBEeXYOADZXROtPQl+v/nsQo+D0kv1ZkXGI\nHKmG1pcEcATJfpkpjGx3nv5cIjghimBArwLbka6P4O3W0alOah0F5CTmumtKfSMmRsfRd6wPNIz0\nY63Pb1EYUXzRaVb/JRE3KzLJSWHw/rMQ0/T0bkWCZJTQFxQ2CVRWk1i9NQgKAl6ZlaAhETfrf06M\nsNG7UUyUKCJCagjhqAo+0AbeFQJUAZnBII2b68rKmOXv66zAoQP0fby57zhaO/qZwtB6kvX0bdou\nztcBIaUVhAATc8aA0wSmFA6dpXWsSNQFrd/IxqP7enWFYc0mg6BAwYUXkxoojsL4Fkh1p+Dv6x7D\n7aPXxu0zBIwsyCxv+4l1tSjLM2MIse4qq1Ay6Asr8ImmgJ9YnoPUgNucvGONIVhy57WuNFbKnGIq\nD06zWwrpQSqYeI6n9alCSVBOl7J2A/a0W8AapJYQ+mSGZTttgyhwtFS3Ygphq4XhcYnwy5YMNEPo\nRjzQQubzcuAgCRJy032oHp6OCaWmdURUEQLPQxYFaJ3pVGD79NURozK7QviwPf5EVBH//t4XUFqG\n2awMq8IAAAHm3wHdJQUAGZ40UxmrElyyiCdvn2S+CykCjifI8Adw45wyOjlTVyZzJmYjrIbN2JOx\ndK/SiLcaqQtGC1NhkSSbAk8TdJeUIdQs5VBO9dMRcpKWgUiEp7Emyyx6EpUBRbYIySgTRElCgM6W\nD3vAccC9KyvY+i9FQdrXdD6LgtxMe+wn1sIw4i9BVwAe68zrGBcM0QRbDEMLeaC25bPf1LmeXvzH\nXjNIThTJJuDFLEspeSLArSefcLzKhLhXS4NsJCiICqKqBjFJXx2yz2/ri96IRRgDgCIhrPUjElUh\nDTvM3lPQ6wKnSbqA1yCX6bXGoh7WFz3hPpwJtUIqOKK/Lxkk5NNTogmkAlrZluMAt0tEwO2DKNP+\n+/cj/wWoEtS2XNbHctkBCKmn0aN0mX1h6XcFA6+ueyEchfEt4ZeTErqmDJcUVRj0BWemUEG4bvRa\nLCmeb5scBMTMENcJeCX4JGvWlV3J8Kop1OIC4uGYcuw6GiFmmwmf0G8PmAvBALDXvALsEwiVeCtH\n4HlMr8qlAslou+DC/SvH4o4lleA4DgGX5XmNaxMe4QPTWbaIJEjgOA48z+He5VUYV2h1pwkQBU6P\nv3BmCitAZ3kbh/WZwt7WXk1E1OYys/cDT8y/rcqc53hz8StFgqpqyE33MSEuF1IhU5CejAW1w+Bx\ni6zv2kPnQECQ4be3CaAzigGwa/tc9P2JGY1QjOKSMZbJrz57Gaf08hj97X5EIhqdk6KnJfPufmj9\nSRg5LBmV+XROESdGzPiHYMR6aJ8E/BxdD4RwyPYnY9a4XL1iMkEwzZ75Zfw7rzYbnLsHQkYTBM2N\nTG9GwgQHI1gPjdYE41x0Do4R9F08kbphQmrETEuOuOhgQFeUvByBqCtytZNa2Sw2JqjwpOjVAfqC\nOHUmRAcEQhSn2nrpipFRN6C4zHIzYoSt+24qQgkqVOw+0MQC1NHGEeB5DjxxUYUhmgFvviPPzJQT\nFPsSBRE3tLCXPq8cYuO26CnqHvRJXmhSH4TMr6DyIag9AZCI1/atyWWf4AuiL1+rSKYVLijQuPOn\nK18KjsK4yhiL+filJDZSFwX6a6nJrMKi4nlx5/gkc2RdNzYX96+swrQxObYJdMYo2Ai4c+r54xux\nghIApo3JRllekNXI4sRInMJ49NYJmFOTh+oR6Wwbx3F214LFmkm03e+VML48A//4w5mWtrtQVZqG\nKaNpgb9kt2VGfIzbIsVFLTGZtygjAMkuUwEZLin2p8UymVJRgBEFetaZItvjFVZLyaLs3DEWn6BZ\nFYa9L41ArtaTjPJhKXHXBQCvaFhunF6sEfiwmU64Ks1Kwy/un2GLE7E26QojoK+/wid1oVdoocrC\niE/pM6e/6mpEe+QsSFTG4S/68HljB+1L0eLGCnuQnuzBD+brylGMMjcIc9vo/dCn9FOFokjweiTc\nsnAkND1773/wLr0ey2jTkwYkDby3GxwH5ChVcAkyunrtgkwLeYGoG5nJHowtobEF99h3bf2W7NXf\nn2XiY7SxHADHLEZXcjdLj40co4t7ifpvhE86B9HTD6JIaDsLLJtabCpPMQIihaD20PdoXE/K/QLH\nO7+09QH7Tej127SwG1oX/RYkuMHJYXNNmDN5UBXebm3pbZd68gDFBaL/LqXSA+AFDUTlkROh5WKM\n9VjkokO2e9sGYRZIxLRmeF8HVC5qq5D9dXEUxlVmWm4tvl+5GuWpZSjM9iMvw3fBMsaAXWE8fMtE\nVJWmQxR4uCXLaD6m2JgxpwBAvMJIkHW17vpK8DzHsrU4Vz8CMQqjOCeAtfPLael1C7bgpW0mMGdZ\n4J6ek5mi+2CtzxxTHSLda5kFH9P2FDcVUrFzXZJdZuoxUQWmhAEzPREAJpTms7LfAMeCnDzHo7LQ\ndNVZLTSPZL9Xrs+sXBuIcRdGvhoJEpUw0lVL54ggPmXZrSuMmhEZmFkyDi7ehY9a6Mxoj+iB1y3i\nb2+rxTBSYzvPUBheyW4hcqpl5r3lXXRGOi0uOIDjCHhXP3PdkKgMTSPwy0m0/Iv/HBN4xj0MIdQX\npdlkRJFoZhyA22uXQiAy+jU9vVQVMboklQn6kBKCKOuz7BUZR5o68L8/OGJruzEq9rpFdITtpdON\nZ/G4JJqRxZsTH3m9ijLpC0DrS4LqPwWS1MbaAQBpoHEtKfc4iEDb3tMfxeiSNKT5/OBFBYvqqJIy\nrG4jeQAAQoSWJmHKWBfW1HWnmNYDAJ8eT5RLP2FtiCrUqiME4LzdZu2xvjL9nvR3KfjPgfN2IuD2\n4cHV1QCA1n5zFjq9d0xsy0L0VIm+3DPdJ+UdB/GcYwkG3wRHYVxlZEHGxOxq8ByP7y8aiR//YOJF\nz0lxJ2Pl8GXYMv5u+7VkgZWdOBemPvrKIipsq4vy2XFG/MDjElGSG7C7aGIo8NOApkeSUD0iPtie\nkPNU3r11YTmCxshfz5PPTDaFtzECCql2X3a2pU6XITSeuXsq/v7OyUyhSTEKI9Vtr2wqWCwM6yz4\nJMlnsz4Ml4ZHcCPoNa+Z7DUtB3dM2u+SCebaKF6LMgcAtaUIoY/nYFp5iem600SED9WyY9J0K04U\neNx8XSUKg6Y7zbA+slK8eHjuTfh+5Wrz4npbjbXMDXjVFAzKqVJInEWBxKz+CIBNliNRF8JRFSIv\nguvIB+8KQUg5oz+XBwGfzEa0Lx9+FZwUBYnKdAY9gNqKbGQlmckNRJFQlO3HjrtnAQDeP/0R5k2l\n7qHjjf14+vd/gdKab6tHZQT9PS4RTT2nbM/F0psFDhyvlzZJp242v5EFSMzEBgiKTcCnCFkYnlwG\nTg4jTPoARURuuk/vQw8kt4KRZW57PxEeoU+m256J/V+1KAxRQW5yEE+so++1NjgbRBHB+2g8ROZd\ntMAi4aGeyQfv7oOoZ34ZvycjC83AL/tYSfqCpFzbPkPx2+ZX6Rjl/+OKnl4gXX2gOArjOwTHcXGj\n9fMxq2AaSoJFtm1uSWCZM8aodU5NPratHY/66aXmgfoo/b4VVchJ8wKqBKGtDBVCTAorgMXF12F2\nQR22zbyDZkCdh4fXVONHN9NsIbUzsWLRNIKyZOp/NjJsMlLMEdyDE36IMemVmJpbazsv22dJk9U/\nghS/C5kpXqYwYl1SAi+gIf8maCEPtO5UiJb5I2qXmYHmO4/CUImKJI+pMFycKYRjlZMk8phTMB0i\nLyLHZ0/ppXA2C+p7M0qw1KJkioL2NGlrIUt3zKhwQtY4TM6egNrsGty3YizuvGEU0j1p4L+YxALs\ngmY5R5Uw1jeV/TkqP4cJoegXMVl7UReum0DbInXnsc1EFeCWJTyxrhar6uis8rMhfSEpRbYp46DF\nFUhUEa3n+pnSA4C3mvRZ2Cy+ISP8V7N9rNSNLGBa7iRb8wxlIvI8KyjJe6k147aU5bAtiWwRxpkc\nvgAAFtxJREFU8KLAI91jWqsF6SnYciNNcy0JFiGqKdjXoseHrNcIe01LzSqg9Wu7KmgBxaxAEHkZ\n1MIszciB0mJm6vkkD1vRT23T0131+AYb+SsyIkfNWJnVcrxr7A+QKZjXY0kiSoLBGYvL2U11NdGx\nl4ijMAYRLklA9MtRiDaVob50EQBaUrksP2gTikb8QBJ55iKSzoxCTdr4uGvKgoQVw29ga4Ofj/Jh\nKSjVM7u0zgxET5aiyluHm+ePYMeoGsGkbF2p6EI74DU/wAJ/Hu6q+n6c6Wyr06WPFg0BnHoelxQA\njEorR/jATJB+P7xuy8dicc/5ZR8k0bJuQBd9zpAaZusVGGuwGyNhI25i5XtlS/DszCfhERMnEFgd\nc0umFmFpbTn7Oy9m9GhVOt6Y63Ech5srV+HWypswtiwdkyqpciFdGQgfnIwcbTRSQxVI8ZsCLyCb\nQjw3OYWlWmvdabbFqB5YMYnFc1xRM8OLKBJ8bhF+r4yZJTU2F5zEuVj2HGBXGOlJSbhuYgGrZmDF\nNlK3JB4Ygxm3LGLliGUYlzHG3GfUPhN4/GjyJtv1rKnYVrebVcBLAm+LbWX6A6yfjBU0P2r5S/w1\nwLE4H5s5DqA4RtFb331mssfWt92WQlKGK9Fg0cQy9n/N4i61Zj0mu4IYhrHmSUb/kfhBnNG3WmeG\nrSpEW/s3D3w7CmMQ4ZIFQBOhnCpLKLhMU1VflIfnMFUPLC+ZWmRzD31TlJPDUe4ej9k1+Vh/QyUC\nPhkTK7JQmVaOB8bfg++PXoX1SysvGq8xmJozEZWp5Vh3fSVumFbEtpsuqXjT3Mg6AwC/165QNo3d\niDtG3wyP6LEpU8OdUZlWztYrCPpkyIKA0P6ZSGqahYnZ1XH34jguYRbcTXOHQxJ5jCy0VyPmOR4z\n86diTsF0SDECNccikD1SYgUUiyjyIH1BjOCn4oeLpuMnG6axfUGX6U4LyH4MyzLjLJolQ84q7GVB\nBukz/67Q2y8LMlaOWMa2zxtXauu/ZIsy3bC0GqW5QXAch3UxKeW2YK3NzWO4pARIvIiiQEHcPlHg\nkO3LtGULWgcZomYpkWMZVQsCh6ClfdZvpDhQaMbXABDF7r5RWwtsbQAAn5qNGcEl7G+rRZAacGFq\neTH7Oxw2r11XUQTrEGJ4bjqeXj+ZXt+iMGLffapo/i58kg8TR9KBhW0FSgAjcvQkFMIjcthirV9g\nkbaB4lSrHURcyGUEAOHDtXopCl1hCFSQPb9lJmRJQGfv5cnVZugW8eTKbEyuNH/sxcFCFMcP0i/I\n/6pYmXA7C3rz8RaG12X+vA03zMYVVTh2shNlafkAqHKwWV+qBOHQfKy7czr6iglOn+3FzQvK8S+7\njgCKC66o74Iz92OZP7EA8yfGz8wHgFUj6hNutwpJcYCZLRuXV+EP7xzDwknDEPDJyMgwlYTVOgzI\nfvgyLLP+LQLKOodEFHmo57LAJ3XSkicWhZtvsYh8MTGboMWasQrxmswq/IvkM+s+2Xzv1oQH+n9D\n2VvbZMYwaP+nulPQHaUuKTp5sRcuWcDT62bhRx/sjrsPz3M2C8Mq4CVBQoY3jZVIJxEXinMCKM0N\n4PjpLvzo5tlY/+vfQOtJxuiSVBz+6hyWTivCud4e7NZj81ZrkOM4rJo+Gvve/QN9HsENoxwjITy8\nfBL6tG4QjYfEi5CM9FuL8pRjftMLa4vxJz1hbNPy8Th4UMNHh1sRPjgFj24owzP7fgEAaD5jqV1F\neD1BQLvwMtADxLEwBhEu+SI/CE2wuWOIvtCvrCsawz1kdWd8HQwBWT4s+SJHfnMyPelYWDgHM/On\nxu2zpqL69WcbV5aO5TNLbcfZFAYAF+eFW3QhNeDGI2vHIz8jCafO0s89mPTNA4cXQ+RFXDdsFoB4\nd9X5KMkN4MHV1XGZbACQ6gli1Yh6ZHrSURwchorCFEwbnY07llSymFehv8Am4JfPLIHSRu9dmVJp\nu55X8uDGEfVIknwoCgyz7Svw03MkXrQLe1jWvdd4WMvSrJk3nC13y3vpxDPjt2i9htXCAOyJDR7Z\nFLRBj0UhWq0XYs+ei7XCc7xm7GjNrFF4aHU11lw3An9zywRwHAe1dRhIXwCFWX688OBsFOcEkOIz\nrZzYZA2rS2ndItO11tsfZen0RsxGZoM9jrmR+hR7xV+XJGBTzQbMyp+GAn8eJlZkQuA53LGkEmmW\n2MzyGcNt5xluViNu+E1wLIxBxMUsjFnjcvH2fjPzJLZEN8dx+NnGujgBeqncNHc4GqaXXFyBXQY4\njsPS0oXn3WcQ65KyYk25BZBwzkN3H7W+aisSBbUvP8tKF2FR8bzzlsa/FFySgJlZU21Kdd0SqgQq\ni+ohSEvhkz22/hpdnIZfbVmMzvB0c20PCzPyp2J63pQ4l2JxsBBPTfsbcBwXF7BPd6fiq65GWiLe\nQpJHwuphy7Fj33PoPkWVeWYqFbb5fovCVI15SnpKttda0ZkqBlUltjZpFrcaIQRZlnO8Me0zrpfm\nTsG88YWIZerobLz/12akBszz3LKA6MlSSHnH4jKZrO3IS0nB8PwuHGnqRG9/FHmBTJzsb2QLZFnL\nAkWOjkNwxP9gYdHcuDaUJRezxJH0oAe/fGg2ezaDuqocVgZ9XFk6XMHJOID/w9xq3wRHYQwiLqYw\nblk4Emvnl6OxtQefN3YgOzU+ZnEhwXpJbfkWlMWlYLikEiHFKM7Z1Xlxx2xcUYX9R9owZVR23L4r\nAcdxl0VZANbRazwXs5is8Y9Yzhd/ssZCrFRnVmFf6ydx271uEQX+dPy07m9x5wc0iypLz57z25Yx\npuLKr7/L2QV12NW4GwVJuZB66TtU9USFm8ob8Lu3Dphr1INaGLIl1hUbjJ+ZPxUCL2B2QV3C9n9/\n0UhUD09H9XBT6QR8MpSTZfBHCzB+9riE5wF0lvniyYX42WsHMG9CATwZSfiwZR/bb/0NPnvnXEji\nfHuixkXgOA4NZdfHpc5WFKVAVZOx973ZQPT838BAcRTGIOJiCgOgftzCbD8Ks88vCAYjfu/5Pxbr\n8pX/eG9dQrfO6OI0jC5Oi9t+LTCQ38W3wdiMUZiQNQ4dZ2R8atlupJJLotnOjGTTXfS3Ux7Gm4c/\nxFu99L2kJ5vK5KfTfwwOHP5jD11kyBhoT8+bgvTp5XjtnWP44nS3vo/uTPekoa3/LOy5azQetrRk\nwXnbLwo8xpfbLUyfW8LT66cgySslVKC3j74ZRzqOISAnYWyZn8ULVS0NxYFClKdQi4rjOFw/pRA5\nad6v7facN2xm3Dae4yDKvC3V+JvgKIxBhEsWsKF+NBudOZhcyMIw1qj2uMSEyuJaxyV/N0KVPMez\ncvDaTILb/4Eu0+pOYI1a3aLpnjRMzpiKt7APxTl268WwwhItaFVRlIpHi1Jx29+/BcBUJptrNuC9\nU3sxKbsm7pyvQ1YCS92gOnMMqjPN+IVh7Qm8gAcm3GM7Nja2djkQeA6CMLBMxIFwxRXG7t278dRT\nT4EQguXLl2P9+vW2/ZFIBA8//DA+++wzpKSk4Nlnn0Vu7sACfQ7xGKl2DpQlU4vw2RftbC2PRPSF\n9bURLsEFcC0x0Mmg3yY8x+HJ2yfh4JftKMk1lcC9y8fYStwYlOUHsfnGsRienziRYiBC0fDyB10B\nXF983ddq97UGz3PQNHLxAwfIFf1CNE3DE088gZdeegmZmZlYsWIF5s6di9JSU5O+9tprCAaD+NOf\n/oQ33ngD27dvx7PPPnslm+UwhPjejBJ8b0bJBY8xXDaJYjrXMo/eOoEF67+L5Kb7WGkOA2t8IJYL\nuQQvpBR9Hgm9/dG45IahgFsW0NN/eSrVAlc4rfbAgQMoLCxEXl4eJEnC9ddfj127dtmO2bVrFxoa\nGgAACxYswJ49e65kkxwc4lg5uwzXTSjA+htGXfzga4jinACqStMvfuAgIJFLyuAnd03FxJGZmFOT\nf95jBhuP3joBs6vzUDMigxX4HJ5/iZOfEnBFLYyWlhbk5JiLwGdlZeHTTz+1HdPa2orsbJp5IggC\nAoEAOjo6kJx85XP4HRwAGrtYPW/4xQ90+M7i8+iT+hIojtL8ZGyoj1/tcjBTnBNg8Z7RxWnYfONY\nlOR8xxVGbIntgRxDCBlwuQgHBwcHAJhQnonjE7pQNybn4gcPQS5Xht8VVRjZ2dk4dcqcKNbS0oLM\nzMy4Y5qbm5GVlQVVVdHT04Ng8OKa0Fr6YKjj9IWJ0xcmQ60v7lsdXzzTYKj1xZXiisYwxowZgxMn\nTuDkyZOIRCLYuXMn5s61z16cPXs2Xn/9dQDAm2++icmTJ1/JJjk4ODg4fE04MhC/0Tdg9+7d+MlP\nfgJCCFasWIH169fj5z//OcaMGYPZs2cjEongwQcfxKFDh5CcnIwdO3YgP3/oBKccHBwcrhWuuMJw\ncHBwcBgcfPdm9Dg4ODg4fCdxFIaDg4ODw4BwFIaDg4ODw4C45hTG7t27sXDhQixYsAAvvPDC1W7O\nFWfbtm2YOnUqli5dyrZ1dnbitttuw4IFC7Bu3Tp0WxYMfvLJJzF//nwsW7YMhw4duhpNviI0Nzfj\nlltuweLFi7F06VL89re/BTA0+yISiWDlypWor6/H0qVL8Ytf0JXWmpqasGrVKixYsACbN2+Goijs\n+E2bNmH+/Pm48cYbbanugwVN09DQ0IC77roLwNDtizlz5uCGG25AfX09VqxYAeAyfyPkGkJVVTJv\n3jzS1NREIpEIueGGG8jRo0evdrOuKB999BE5ePAgWbJkCdv205/+lLzwwguEEEL+6Z/+iWzfvp0Q\nQsjbb79N7rjjDkIIIfv37ycrV6789ht8hWhtbSUHDx4khBDS09ND5s+fT44ePTok+4IQQvr6+ggh\nhCiKQlauXEn2799P7rvvPvLGG28QQgh57LHHyD//8z8TQgh5+eWXyeOPP04IIWTnzp3k/vvvvypt\nvpL8+te/Jlu2bCF33nknIYQM2b6YM2cO6ejosG27nN/INWVhDKQ21WBjwoQJCATsJZ2t9bcaGhpY\nH+zatQv19XSd6LFjx6K7uxttbW3fboOvEBkZGaioqAAA+Hw+lJaWoqWlZUj2BQB4PLQ+UCQSgaIo\n4DgOe/fuxYIFdD2HhoYG/Pd//zeAwV+vrbm5Ge+88w5WrjTXff/ggw+GZF8QQqBp9hUNL+c3ck0p\njES1qVpbW69ii64O7e3tSE+nReUyMjLQ3t4OwF6XC6D909LSclXaeCVpamrC4cOHMXbsWJw9e3ZI\n9oWmaaivr8e0adMwbdo0FBQUIBAIgNertmZnZ7PnPV+9tsHCU089hYceeoiVFDp37hyCweCQ7AuO\n47Bu3TosX74cr776KgBc1m/kmloAgDhTRi5Iov4ZbHW5ent7sXHjRmzbtg0+n++8zzfY+4Lnefzx\nj39ET08P7rnnHhw7dizuGON5Y/uCDKJ6bW+//TbS09NRUVGBvXv3AqDPF/vMQ6EvAOCVV15hSuG2\n225DcXHxZf1GrimFMZDaVEOBtLQ0tLW1IT09HWfOnEFqaioAOkJobm5mxzU3Nw+q/lEUBRs3bsSy\nZcswb948AEO3LwySkpIwceJEfPLJJ+jq6oKmaeB53va8Rl9car22a4G//OUveOutt/DOO+8gHA6j\nt7cXTz31FLq7u4dcXwDUggCA1NRUzJs3DwcOHLis38g15ZIaSG2qwUjsSGDOnDn4t3/7NwDA66+/\nzvpg7ty5+OMf/wgA2L9/PwKBADNFBwPbtm1DWVkZbr31VrZtKPZFe3s7y3QJhULYs2cPysrKMGnS\nJLz55psA7H0xZ86cQVuvbfPmzXj77bexa9cu7NixA5MmTcIzzzwzJPuiv78fvb29AIC+vj689957\nGDFixGX9Rq650iCJalMNZrZs2YK9e/eio6MD6enpuPfeezFv3jzcd999OH36NHJzc/Gzn/2MBcb/\n7u/+Du+++y48Hg+efvppjBo1OBYF2rdvH9auXYsRI0aA4zhwHIdNmzahqqoK999//5Dqi88//xxb\nt26FpmnQNA2LFy/Ghg0b0NjYiM2bN6OrqwsVFRXYvn07JEkaMvXaPvzwQ/zqV7/C888/PyT7orGx\nET/84Q/BcRxUVcXSpUuxfv16dHR0XLZv5JpTGA4ODg4OV4dryiXl4ODg4HD1cBSGg4ODg8OAcBSG\ng4ODg8OAcBSGg4ODg8OAcBSGg4ODg8OAcBSGg4ODg8OAcBSGwzXNqlWr0NDQgOuvvx6jRo1CQ0MD\nGhoasG3btku+1u233z6gctePPPII9u/f/3Wae0kcPHgQ//mf/3nF7+PgMFCceRgOg4KTJ09ixYoV\nF6w+apSKuFZ49dVXsWfPHuzYseNqN8XBAcA1VkvKweFS2LNnD7Zv345x48bh4MGDuOeee9De3o6X\nX36ZLaizdetW1NbWAgBmzpyJl156CcXFxVizZg2qq6vx8ccfo7W1FUuWLMH9998PAFizZg3uvvtu\n1NXV4cEHH0RSUhKOHTuGlpYW1NTU4OmnnwZAa/M89NBDOHfuHAoKCqCqKubMmYMbb7zR1s62tjZs\n2bIF586dAwDU1dXh9ttvx3PPPYe+vj40NDRg0qRJ2Lp1Kz7++GPs2LED/f39AICNGzdixowZOHHi\nBNasWYMlS5Zg3759iEQiePzxx1FTU/Ot9LXDEOGbLNbh4PBdoampiUyePNm27f333yeVlZXk008/\nZdusi8scPXqUzJo1i/09Y8YMcvz4cUIIIatXryZbtmwhhBDS1dVFamtrSVNTE9v37rvvEkIIeeCB\nB8jatWtJNBol4XCYLFy4kOzdu5cQQsiGDRvIL3/5S0IIIY2NjaS6upq88sorcW1/8cUXyWOPPcb+\n7urqIoQQ8q//+q9k8+bNtrbX19eTs2fPEkIIaW5uJjNmzCA9PT3kq6++IuXl5WTnzp3s2WfNmkUU\nRRl4Jzo4XATHwnAY1JSUlGD06NHs7y+//BI///nP0draCkEQ0Nraio6ODiQnJ8edu2jRIgCA3+9H\ncXExTpw4gby8vLjjrrvuOogi/ZQqKytx4sQJ1NbWYu/evXjyyScBAPn5+cySiWXcuHH4/e9/j2ee\neQYTJ05EXV1dwuP27duHpqYmrFu3jhWkFAQBjY2N8Hq98Hg8WLx4MQBgypQpEAQBX375JUpLSwfa\nXQ4OF8RRGA6DGp/PZ/t706ZNePzxxzFz5kxomoaqqiqEw+GE57pcLvZ/nuehquolHTfQdRbGjx+P\n119/He+//z7+8Ic/4MUXX8Tvfve7uOMIIRg1ahReeumluH0nTpyI26Zp2qBa68Hh6nPtRAAdHC4C\nGUD+Rk9PD6tO+sorr5xXCVwOamtrWVnpkydP4sMPP0x4XFNTE5KSkrB48WJs3boVf/3rXwHQtS6M\nMuYAUFNTg6NHj+LPf/4z23bgwAH2//7+frzxxhsA6BKlAFBYWHh5H8phSONYGA6DhoGMprdt24b1\n69cjJycHkyZNgt/vT3h+7LXOt+9Cxz366KN4+OGHsXPnTpSUlKCmpsZ2P4M9e/bgt7/9LQRBACEE\nTzzxBABg2rRp+M1vfoP6+npMnjwZW7duxXPPPYft27eju7sb0WgUBQUFeP755wEA6enpOHLkCFau\nXIlIJIIdO3ZAEISL9omDw0Bx0modHK4Q4XAYkiSB53m0tLRg5cqVePnll1FQUHDZ72VkSb333nuX\n/doODgaOheHgcIU4fvw4HnnkERBCoGkaNm3adEWUhYPDt4VjYTg4ODg4DAgn6O3g4ODgMCAcheHg\n4ODgMCAcheHg4ODgMCAcheHg4ODgMCAcheHg4ODgMCAcheHg4ODgMCD+P4xSKOOE0RxSAAAAAElF\nTkSuQmCC\n", "text/plain": [ - "" + "\u003cmatplotlib.figure.Figure at 0x7f97f1e98d90\u003e" ] }, "metadata": { "tags": [] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAFnCAYAAACPasF4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzsvXe8XVWZ///e5dTba3pCQiAJCSWE\nIJGmoSSgjsg4gmCb4Tf+dCwURUdEQXGs41gYFQvDiIyIiKIIJIAgEBJCgJBKertpt59z76m7fv9Y\nu55zboiQBCL783rllXt2WXvttfden6et55Fs27aJECFChAgRIhw1kF/vDkSIECFChAgR/jZE5B0h\nQoQIESIcZYjIO0KECBEiRDjKEJF3hAgRIkSIcJQhIu8IESJEiBDhKENE3hEiRIgQIcJRhoi8I7yp\nMW3aND796U9Xbf/iF7/ItGnTQsfdcMMNoWOWL1/OBz/4QQB2797NCSec4O3btWsXH/vYx1iwYAEL\nFizgkksu4bHHHgPgpptuYuHChSxcuJCZM2fy9re/3fudy+VC19A0jfvvv/9vvq/Vq1dz1VVXHdSx\nDzzwAF/72tde9bVcvNbz3wi46667+P73v/96dyNChFeE+np3IEKE1xsbN24kl8tRX18PCBJas2ZN\n1XErVqxg/fr1IZIeCZ/97Gd597vfzW233QbAqlWr+PCHP8zDDz/MV77yFe+4+fPn8+1vf5vTTjut\nZjvr16/n/vvv55JLLvmb7umkk07i9ttvP6hjly5dyvnnn/+qr+XitZ7/RsAHPvCB17sLESIcFCLN\nO8KbHm95y1t49NFHvd9LlizhxBNPrDruuuuu4+tf//pBtblp0yZOPvlk7/fJJ5/M4sWLGT169EH3\nq6+vj09+8pO89NJLXHHFFYCwAPz0pz9lwYIFmKbJypUrufTSS1m4cCEXX3wxS5cuBYRV4IILLgDg\n1ltv5atf/Sqf+MQnOO+883jve99LT0+Pd53ly5czffr0qmu98MIL/OM//iMXXHAB73vf++jq6gKg\nu7ubD3/4w1x88cWcf/75fO9736vZ18p7ueqqq1i4cCHz58/njjvu8PatXbuWSy+9lAULFvCBD3zA\nu85I26dNm8b+/fu9893fy5cv5/LLL+fqq6/mM5/5DAD33nsvF110ERdeeCFXXnkle/bsAcC2bb7x\njW8wf/58FixYwC9+8QtvrL74xS8CsH///pD15MknnwTAMAy++MUvsmDBAi644AI++clPVllMIkQ4\n3IjIO8KbHhdddBF//vOfvd8PPvggCxcurHmcbdssWrToFds855xz+PSnP82dd97J1q1bARg1ahSS\nJB10v9rb27nuuus45ZRT+PWvf+1tt22bxYsXoygKX/7yl7nqqqtYtGgRH/3oR7nppptqtrVo0SJu\nuOEGHnvsMdra2rjvvvsA2Lp1Kx0dHYwbNy50rVwux8c//nGuu+46Hn30UT70oQ9x9dVXA/C///u/\nzJ07l4ceeogHHniArq4uLMuq2VcXP/nJTxg/fjyLFi3il7/8Jd/97nfZt28fIISiq6++msWLF3P+\n+edzyy23HHD7gbB+/Xouv/xyvvvd79Lf389Xv/pV7rjjDh555BEmTpzIj3/8YwD+9Kc/sXr1ahYv\nXsx9993HXXfdxerVq0Ntff7zn2f69OksXryYn/3sZ3zuc59jcHCQJUuWsHv3bhYtWsQjjzzC1KlT\nWbly5Sv2LUKEQ4mIvCO86XH66aezefNm+vv7KRaLrFy5knnz5tU89oYbbuA///M/KZfLB2zzO9/5\nDldeeSUPPPAA73znO5k/fz533333Ienv2972Nu/v+++/n4suugiAOXPmeNppJU477TTGjRuHJEnM\nmDHDI85ly5bVvNcXXniBUaNGceaZZwLwzne+k127drF3717a2tpYsmQJzz//PPF4nP/6r/+is7Pz\ngH2+8cYb+dKXvgTAhAkT6OjoYPfu3Wzfvp3BwUHOPfdcQJitb7311hG3vxKSyaR3P21tbbzwwgue\nteO0007zxuepp55iwYIFxGIx6uvreeihh0LWlkKhwPLly/nIRz4CwKRJk5gzZw5PPvkkra2tbN26\nlUcffZRiscg111zD2Wef/Yp9ixDhUCLyeUd400NRFC688EIefvhhWltbOeuss1DV2p/GzJkzmTt3\nLnfccQezZ88esc1EIsFVV13FVVddxdDQEIsWLeLrX/8648ePf80TfXNzs/f3Aw88wJ133kk+n8ey\nLEYqVdDQ0OD9rSgKpmkC8Mwzz3gEFcTQ0BBdXV0hC0Q8HmdgYICPfOQjWJbFV77yFXp6erjyyiv5\n1Kc+dcA+r1mzxtO2ZVmmt7cXy7IYHBwM9U1VVVRVHXH7K6Gpqcn72zRNfvjDH/L4449jmib5fJ7J\nkycDMDg4SGNjo3dsOp0OtTM8PIxt21x++eXetkKhwBlnnMFJJ53EjTfeyK9+9Ss+//nPM3/+fG66\n6aZQexEiHG5E5B0hAnDxxRfzve99j5aWlpo+2yCuvfZaLr30UsaPH19z/8DAAC+//LKntTY2NvK+\n972Pp59+mk2bNh0yLa27u5sbb7yRe++9lxkzZrBjxw4WLFhw0OcbhsGaNWtqCiGdnZ1MmTKF3//+\n9zXP/ehHP8pHP/pRtm/fzr/+678yZ86cA17r+uuv58Mf/jDvf//7kSTJG4OWlhYymQyWZSHLMrqu\n093dPeL28ePHI8uyJ3xks9kRr/nQQw/x+OOPc9ddd9Ha2spvf/tbHnjgAe+6g4OD3rF9fX0kk0nv\nd1tbG4qicN9991FXV1fVtrs6IJPJcMMNN3D77bdz7bXXHnAMIkQ4lIjM5hEiALNnz6anp4fNmzdz\n+umnH/DYzs5OrrzyyhHNuKVSiU9/+tM8/fTT3radO3eyatWqEaPKR4KqquRyuZoa9cDAAOl0milT\npmAYBvfccw8A+Xz+oNpevXo106ZNIx6PV13r5JNPpre3l1WrVgHQ1dXF9ddfj23bfPnLX+aZZ54B\nYOLEibS3tyNJ0gH72t/fz6xZs5AkiT/84Q8Ui0UKhQLHHHMMo0eP5pFHHgHgd7/7HV/+8pdH3A7Q\n0dHBhg0bALjvvvuQ5drTWH9/P+PGjaO1tZXBwUEefvhhb2zmz5/Pgw8+iKZpFAoFrrjiCjZt2hQa\n93PPPZff/OY3ABSLRb7whS+wb98+7rvvPn70ox8BwgoyZcqUgxrvCBEOJSLyjhABkCSJCy64gLe+\n9a0jkkEQ//Iv/4Ku6zX3jR07lp/85CdeVPiFF17Itddeyxe+8IVQBPrBYM6cOfT09HD22Wd72qaL\n6dOnc84557BgwQIuu+wy5s+fzymnnOKtPX8lLF26NOTvDl4rFovxwx/+kFtuuYWLLrqIT3ziEyxc\nuBBJkrj88sv53ve+50W4z549m3nz5h2wr1dffTWf+MQneNe73kWhUOCyyy7jS1/6El1dXfzgBz/g\ntttu48ILL+TPf/4zN998M5Ik1dwOwvJx88038+53v5tUKuUt8avEO9/5TjKZDBdccAGf+cxnuOaa\na9i/fz/f/OY3ufjiiznrrLO48MILec973sN73/teTj311ND5N998MytWrGDhwoW85z3vYcKECYwZ\nM4bzzjuPdevWceGFF3LRRRexZcsW/vmf//mgxjxChEMFKarnHSFChAgRIhxdiDTvCBEiRIgQ4ShD\nRN4RIkSIECHCUYaIvCNEiBAhQoSjDBF5R4gQIUKECEcZIvKOECFChAgRjjIcNUlaenuHD2l7LS1p\nBgcLh7TNNyOicXztiMbwtSMaw0ODaBxfOw71GHZ0NNTc/qbVvFVVeb278HeBaBxfO6IxfO2IxvDQ\nIBrH144jNYZvWvKOECFChAgRjlZE5B0hQoQIESIcZYjIO0KECBEiRDjKEJF3hAgRIkSIcJQhIu8I\nESJEiBDhKENE3hEiRIgQIcJRhoi8I0SIECFChKMMEXlHiBAhQoQIRxkOK3lv2rSJ888/n7vuuqtq\n39KlS3nve9/LZZddxo9+9KPD2Y0IESJEiBDh7wqHjbwLhQK33HIL8+bNq7n/a1/7Grfeeit33303\nzzzzDFu2bDlcXYkQIUKECBH+rnDYyDsej/Pzn/+czs7Oqn1dXV00NTUxZswYZFnm3HPPZdmyZYer\nKxEivGmhGxZL1+6jWDZe76542NuXZ822/te7G0cNXtjYy879wyxduw/Lsl/v7rxq9GWKrN8x8Hp3\nA4D9AwVWbekDoKyZPPdyN7Y98tjmSzovbOw54DFHGoetMImqqqhq7eZ7e3tpbW31fre2ttLV1XXA\n9lpa0oc8Z+xICd8j/G2IxvG143CN4d2PbOTXizdw3twc11x+6mG5xt+Kf/nm4wDc/+13oSiHTn/4\ne3wP9/Tm+NEf1ni/48k4F8075rBe83CNo/vcf3XzQpobEoflGn9rX+79+jv4+d0vsmzNPmRV4aK3\nTq55/I9/8SzPv9zNdVecytvnTHjF9o/Eu3jUVBU71JVuOjoaDnmlsjcjonF87TicY7hhu9BwN+wY\neMM9p737syTjh2YK+nt9D7dWaKobt/dz2tS2w3a9IzGOXXsz6K3pw3qNg0V3zzArN/YAsGnnAKcd\n117zuA3Oc3hh/X5mTWw+YJuHegzfUFXFOjs76evr8353d3fXNK9HiBDhtcE180lIr3NPqqEZ1uvd\nhTc8SroZ+m2aR/+YvZFcOJZtY5jiG1EPYAVqrheWgsHh8hHp18HgdSHv8ePHk8vl2L17N4Zh8MQT\nT3DmmWe+Hl2JEOHvGq6LTnrjcTdGRN6viHIFeRtHsc/bRb6kv95d8GBaticQqcrIH0mLY+bP5N44\n5H3YzOZr167lW9/6Fnv27EFVVRYvXsz8+fMZP348F1xwATfffDOf+cxnALj44ouZPLm2ryFChAiv\nHW9E8tYj8n5FaHp4jEzz6CfvQun11byDQWeWZeP+UuSRddn6VAyAzAE072x5iKZE4yHp48HgsJH3\nrFmz+NWvfjXi/rlz53LPPfccrstHiPCGwf6BAo3pOOmk+Nx6MkXSCdWbEGqhe6BAQzpGOukf0z1Y\noLk+QSJWHbiZzZUxLZvWxmRou+Wazd+A7H0kzOYDQyUUWaKp/rUHSFm2TVd3jgmj6pEliZ7BAk11\nCRLx8PMoayZ9QyXGtde9pusVSjq7e3OhbYPDJbJ5jaa6uLetN1MkGVdoSMcrm6BYNtiyJ8u49rqq\ndwOEANWXLTKmrbqvA0Ml4jGF/mzJu+dK2LZNV0+Ose11ntnZtm329OUZ116H5IxTXeBdz5cM9vbl\n6WxJeedYts2W3VniMZljRjfSkynSmI6FYiJ2dQ8zpq2OmFqbZGudUwslzbdmmJZV9bdl2WzenSGV\nUEknVOJxxfuOhgo6fZkidakYqYR/naV7V/B/G+7lIye8n4s7zjng9Q8VjpqAtQgRjkaUNZMbfvYs\njXVxvv+pswD499uWIQG3//v8mufohsXNd6xg9nHtfPQfZgLQny1x48+X8455k7jk7ClV51z7388A\n8D8VbbpKhvw6cLdpmWzL7mBq85SawsOR0Lw/++OlQPW4vBosfm4X9z6xlcvPO45Tj2vn33/6LBOm\nZ7A7N3LdnH+jOdEEwDf/70V2dg/z7Y/No7059aqvd/MdK+jLlkLbNuzKcO2tS0L3c8Mf7sYup/nF\nxy6vauOexzezZNdKmuoVvnvlZVX7n1i5h3v+spmvXHU64zvqve2mZXljB3DlBcdz3pzxVeev2TbA\n9+9dxZknjuaqd5wAwOMv7uH/Ht3E+88/jtOmdfLvP32W9iZfcFi5uZdfLd7I208dxwcvnAbAqi19\n3HqfiKq//v2z+c7dK5k+sZnPXSFWSGzcNci3fr2SudM7+fgls6r6kc2V+ffbljF1XBM3fHBOjdH0\nEdT8g0vvXFJ/YVMvP7l/rbddSuZomLUSuXkaVqaTz922jHHtddzy/73FO+avu5cAsKJ7JRefeGTI\nO0qPGiHCYYRmiAlhKK8BYDj+tQMZP4tlg7Juhvxre/pymJb9N/vcfBPhkWfvezf/ie+v/CkrulfW\n3K8bZs3tbyT8cevDfGHJLWimxsrNIsh21ZY+9vYXIFair/FZ+kuDdA3v8c7Z2S0ijQcOMrhJt2qb\nkSuJuxZ2De0hPmkDieNfrLm/qzdH4riXKI15ofY1MkVsoKsnrOFXmrbXjrAuf9veLADPrNnvbXtx\nUy8AK17uYSivIaWHyE+7H7mpx2lLRG4/8aI/Zv2Be3VzAGzYlfHb3LsRKV5gxYaemv3oHiwCsGVP\ntuZ+0zIxLfG+BX3uZoC8NSe+oC9bDJ0rN/Wjy3kxxoo4d09fPnRM2RDPOqkcuSVwEXlHiHAYURlf\ndDDapghSssnGtzGsiUm1NyMmt6DPc1XvWnoLB0524pL366F5P71HJF7al+/2tlkBf+PR4PN+ZOcT\nDGnD9BbD45zJlVEa/WVcRaOaaJUDBEC52DW8m2v+egNL9jxb+wDJIjZlNXJzd2iz+1yf6FpywPZ7\nC/6qHsuuHm83IK43EyasSvIeyVRdy6Li3rdhWpiWTWzsVtHGpA0j9jN4vf4KoaW/OMCSwu+Jz3hu\nxPMrz6nEf734E7723HeBcLR7Tisi1WWIH/8Cw8ZwVV8AJFXz/pbragsHZVMck1CqXReHCxF5R4hw\nGFG5tEc/iKU+Zd1Ead9LpvU5/mfdrwF/cnU1hf35Hn625k7+47n/8jQGoCoDl6d315hkNw5sYU3f\n+oO+l1eLxri/TjVI2G/0pWLuhAygW+EI6d5MESnuE0ZBD5MfgHIQEtOjO/8KwIPbH625X2ndj9q+\nl8TxYeuFu7xpW3YnALZR7QEtaQZFxSfvWn0cibzzAQKT6rIMpNbXJH+5xj2qTuCXYdqifcVpq0Yf\na12vp6Ivq3qFCVtOjEzQwf6XtDD5dg3vYcfQLnoKfWim7l9L0fn+y98mOfNZlOZetiUfreoLgBQr\nB/7WqIWyeeSj0CPyjvC64I2UZvBwwqwgU10/OPKWG4RWtze3D/AnJ3epUG9RTMq6pYcmm0rh4EBW\n8x++9DNuW/2/r9ifkfBiz2q+8dz3KRrVpBCc6IPEFyTv16p5W7bFrSt/7hFg9X4bpW0vcsv+mvtf\nCbuGdnt/l4zw5NybKSLFAuRtVCeRMi2bIW2Y32/+MzktX7U/eI2JDeOq+g4gN4nnbNvhB1jWTWzb\nJlN2TMuKUUWufZkScr2vKeZr9NGNZu/LhImx4JiWpbosyZnL2Bd/gd3De6vOryWfuEuuTMuirJtI\nqmjLNqsDNF3BsxAwZe9zTNKphAgEfMkhb9saWRjakFvtPefKe1m+z3cZ5PScZzavJGJNyZLT86G+\nACEhDTV8zu83/5lfrL0LzXnHa1lgDhci8n6TwPUvWrZdMYGaNY97pW2vBat61/LJJz7PtuyOmvst\n2+Kl3rUU9dIBr23VEAAOVV9/uf433LT0m6/6fLcfQfI2TCtEriMJMJpmIsUFIbYmW4Cg2VycP1jy\n/YHByaaSED2zeeC3bdvkdJ9MKv3ouiGIwbKtmtqWi9vX3sXu3F5W9673zgHxXILm/JAGG+hfppzh\nzvX30F3oxbQsBofL2LZd9QxFX6rHqrfYz4bBzdy/9aGq4yzbxjBMYsesIzaxtrm2ss18SQ+ZTHcN\n++RdOSn3ZkpIcX/cCjUEGNO0+c7z/81fup7imb3Lvevphskftz7MZ578Mn0lIaTJkh+xXiwb7O4f\nRG7sQ2l0yLscDnzTdJMhLYdhi/5KEhR1v49lzWT7/iGkhE/Y+RoCxLDdR+yYtfQMD4W254o6iZlL\nSc70a04MlAZDxzyzdzm7zZeRm3tInb6IPbl92LbtRZAXk7tZMfCMr3lXQB2zleuXfImCXgwJoK5F\nJp2IYds2e/Ou8CXh2pJ0w2JgqESuqDNYyrIz/gyJ414C/JgDwzK4e+PveaFntX9fWt5/xnJ1v7Kl\noWqzeby25m1ZNn/peoqVgfaPJHlH0eZvAnQPFvjCT5/l4jMm8fLOQbbvG+J//n0+Dy7bwX1PbuPm\nf57LxFEN/PWlPdy5aCPXv382MyYJ0vjNXzbzyIouvvWxeXS8hsjZIO7fIibbv3Y9w5SmY6r2L937\nHHdv/D2N+jF0r5zOj649J7QsA2BTV4Zv/t+LfPySWcydLrLzPfp8F3c/tpkbPjgHPdVNS6KZ0XWv\nLnPfc/tFAJBhGajy3/aZvLxzkO/cvZIPLZzGceP9VIr5khEycRumTUyt1iZKuomUEGSwfVeZNW39\nXhCNaxbvCfgyQ5p3gBwf2LaY4eQQ0IYkSZR1k49/90nOOGEU557lB9Zc96On+MAFM5h/6niG8hrX\n3LqEc04eiz3xRV7u38R/nHUjsQOMgaZb/P//+SRnnTiGf3nHDD73k6VkpN0kRCBxyKToE7PFnwZv\nB6A50cSG5Z1s2JUhEVcoayafuewUZk5u5cWe1WzdrPDw091V0dvd+XDw0n/d8xLb9g3xb+85ke/+\n5iWuuGgikmKCbGFaJorsE+SqLX384Herue59JzNrShvb9w1xyy+fB+CbH5tHZ3OKVV27vON/8dBq\nxqnT/WsnXkJp9f3QtUzSmfKgR3j3P72de34Dn79iNt/69UpSpz8ROvalbfvZ2TlMc32cz922DOnY\n5SSm+wKQVKHxLVu3n98/v5LkTH9btpynLp6mpBlc/+Ol5EsGiVk+mQxr1Zp3X/ol1NR+8oqBbvhR\n0kPlAnJdmNAHy2F/76833AdAfLLQqH+y5EFSPafQ0SKeUXncc7yUAzlZ+x5iEzZj2LBpcAu5cnXf\nDNPi9kWrKDrmckm2QBZC4pduX06PE6TWcEwXeJ+5ze0PvsykUQ3stzZXxRIM6znyJdFfqYZQ8Z3f\nPUeb7FtB5MZ+5PQw2BJIdugeilr1+bWEuMOFSPN+E2CjE7X50LM72b5PfJCWbXPfk9sAPzr0waXC\nf7Z07T7v3EdWiIIxm7p8Te+1wtXmRlp77EbuZhFmuv6hamn2ryvFMfc+4ZeSve+vIjBm+cbd/PdL\nv+CW5f/5mvsa1BoPFktWi34/tGxnyOddKOkhzbsye5aLkmYguf492eLhZ3d6y1hcTb7HMZvHlXhI\nU3DJ0bAMFu34CwPNKwDxvIediPdn13fTlfMjfVEML3p2l6O1PLVqN893v0TeKJAp1Q7ScbEvI/Yv\nWbMPy7YZGCqHTI1lI0jezrNP+pO1bulelHDZuc+la/ezYWAzt6+9i8cHBUm8vCus+e2vIO91OwYp\nlk1+/dgG1NHbWbRGmFslSZivg1i0XBDzn5buAMS6eq/d/gLL973Attxmb5tml0NLBOzOTeJ/UwgE\ntczmPaVe729TEtaRxc+NUIBJ0dm0O0NXTw7dsFCawgFykmqA5L87f1ixKqQVA2RLeedehCY7arRN\nLOW/vzm9uo+u8UFp2093xo84HypWa+lBzTtoNZJi4t4yWYOd3TnvGVYimbb454unM2/mKN4+2yfI\nn6/9FXvG/L7KvVHWTZZt3hHaJsU0hgs6PYNFL9+BlvYtJA2NYoy27xtClqvzIeS0PANDzvtYg7yF\nZu5bshLTxfcjmXHn+v67nCtWZ4orRWbzCIcSlZGiUrzAQCHrmbfcCdUNsKn000LtwJRXCzenkSzV\nfv08U63j56t15ZST8CS47MM1t9nqa5N+g6biVxOIogdyJQfHslAyQj5vbQTyHtZySJK7QNsMCS8e\neRd8Yqg1BtlymKzcyF8Xe3P+RCkpJprmulWcyzb4wlqmXE3ewTEKEpebgSpE3gEBSDMsUDXkQKT2\nYDHnBWC5SMQVT4iT04JUKpPT7A1EsQ+X/D70JlcTm7iR4lh/nfJAKSx8uglz+lIr+dnqX4aEqoHC\nEHe+fA92zH+PJMW3mtiYge2Oz7aW5q0NBI4TRDE40lI/xaA3U/RiG1yhIISA1hcb5QsBVkEEBA47\nhNubKSI39DM0cTGmFCDvcjUhm5Lfn64B35ozXK6+n6DmXWt5m6aJ96w0AnnbisbZJ43lX981k7NO\nGlO1X2kJC2NlKUd8imOSdueCWJk9TuKaGcc0ITf1hPz6l54nhILebBHd9L8LN2ZgWM95Yyyp1fcg\nqToZR8gNCktSsQXbkpDrhjyXVqYQHs/GeEOkeUc4tIhVJNxPnvIUNy3/ukfq7oTvEnStmsG1siu9\nWnjBOCO8fqZDDF6QTsW1C3qRVervUNp3UyxXTxSGUjs46GAR/AC1V6F5uzm7K8k7XzIOSvMeChCv\npBj0Z/0J1jQtbNv2JlLN1MgXq33Kg+UwWelGONZhMKhNK4bXF89H3uATT7YGebtL2AAKZoA43Ykx\noKGEzeYWyROXED/Gj3LPFKsrMCVisqfpuYFKlQUt9hd88t7W7U/87uQaxEDRHw/TMulveB65foBy\n0xZW9a1DM/yJfqhUYwJWDIYdTasQ8C3bloRqJ8g774zhPV+bQS2gPTvrg0dapy8pOn2ZkhfbYBuB\n4C6XuFS/j5Ll77cKIrmKaxbvzRRDhKZaooJXvkLz3jS4FSvhH7c34z/zXMDEbmbbkGyZTDDOooal\nwRVkhgvV38yY5Hh0S/e+p2DSFu/8eFhrjU3Y4AluFB33k6qxs1tsax6TJTFNuLdkW4yHkhTj25sp\nUTQDz0lLOPeV9yPTlWrNWVI133IQ0MylvTPBjCHFyyROehqAgYL/DZzVeiGtyRaKRumIBeNG5P1m\nQ0CajKsVmrdyZDXvkczmXiCRM2lVLrfaNbybItmQ9haEJudqbg9ix9CuUDRxELkAMb0as7k7gcdU\nudpsHiDQkTSUIT3Qf8UMBVaZlk1eL2AENJ+hgJbktl+pLZtWOFguUwoICLJBWQ8njwlOpBkt7PuE\nsHBQNH1hyU0sEgzyqQxYq4zyHa6hESZiCrudSHsMYbKsDCQKmnF39PmWCKxqrTWoea8f2EivuoHE\nCc+BLO47GImdr0HekmJ4VoVcYLy1DXNRiFN0iNHVzmPHrmJDYZV/vqPlDeVqvE+2BIpBT7ZAr5sg\nJBCZHdNF/Elw3EzE3+ZQC9aQKBE6rPmad5D8k5YgviB59xT6+MHKn3r3D9CdC0SmOwKKOdSKtuUU\nVCsVGsOagVmOcDGU10LzjLZsY2voAAAgAElEQVRtFi0J0Qc3ULJWamC5gryDEfbGUKM3Bjv2i/dR\nSfnHj5VEPIIuFVBkib5MMWzCdsZzqJwj4zyDWj5vggKSs9/oHYdeSHrjLzlj1p0V42V0T6TdmEZK\nTWLaZkjjP5yIyPvvEAOlQTQzaEoNkETg5Yx55C32u2ZzzSpXSemHMsmHa3IdSZu3bLe/Yn/l8ifX\nZOx+XJU+tqLtE9NI0dI/X/Mrfvly7dz6w4Go3FdjNvfIW5FCVox8yXCehY0yagd7ctVLbwAKhk/e\nUkVErGnZVcQ8FCC/2uQttO6gmX446ANWzCrNO6g51zKbBzX3vOFf39e8S9imgm0qlMxqn3cQtZaa\nxWOyPz6KDtgVS+L00Du6dzAgyFnV01qwv7XKoxYD1oOc5vfX1WpRDE+wcTXvuvxUrFwrshX3rDWu\nEKS2+W4J28bT8mwIERtA0m5Ckm36snl6B4uOUO2/N0nTIe9gwJfzHWtbT8Z2hJu8JvrQmy2FiClh\nC7N60KKU06sF3IGCP0Yll7wHRoMZQzHTDGnDXpayWm4C1zIwVNA9rdUcGIXZN576mMidviWzXRxb\n49sXAl8wsMA/xsoJ8pcSBU/zjiXEOOp7jmVS8ngAslqWtqYkvZli2IK2XUT2DRQDgmhgjLRts5x7\nEGOcSqj+flMV30dIKLTodSL0bSNGb6ZIWhWBevkaY3M4EEWbH4XY1T3ML/68nk9eeiKdLeGi9nm9\nwJeWfoPx9WP5wunX8H+PbOIvL/oaZnACiFVq3g5Db2m+l889bdO89VLv2B/9YS3nzxnPFRccf1B9\nfOCZ7by8c5Dr3z/b+1DveXwz2ZxGLqWBAiDxg3tXsdkpSPCpfzyJyWMasQhr3pWlI7tdf6/zcXUP\nFkKBa3nTn4SeXtPFI8v3ceOHTvMi1otGkUw5S4MzET350h4ef3EPLQ0Jpk9soXNyteb98PKdrHi5\nhxs/dNorWiFcYUOpMJs//uJu9vUXkFI54pM2cHfXBo4ZfQ0dHdNC5xfMvC9WKz7hqopMMbGXb6y4\nN3R8vkLzfu7lbh7ZvAncVNWySV+2xH/+5iVQyySmr6Bo+WSlxk1PAHIzuAXJ+4k1W2nMdHHh3An8\n/qmtDA6VGTfL13SDxPenZ3aI8+MlbC2JpOrsGxxC003iMSUsSOL4CZ3+j++oZ3dvDqV1L08Ul1G2\nXQ1JRBkPFzRu+eXzDAyVuPyiseLWJBnLtugeGgScSHTJH3NbjyPFNJ7esI1ZiX5mTWmrGVQUFCCW\nb9hLYgZItoK2+VSSJz8VIkPN0kgACScVpmzF0S2DsqlVuUJsSwJLDWt5FRpfPpNAaRVBcXv6ZEa1\npukPHJM0WxlmK/Gpqyi91Iytpfz2jJj4B+zoHeCWX66gN1MiMcrC/WpiOLWoy4M8u24/v35sM23j\nstDqdlJEUvfkMlzzvb/S2ZRkz2CWeDNgim9GNtPY2GTKQ7SlWnjmZT8S37+voNbqru0W53em2wG4\nc/09nNwxq3a8i2yKNtzgMMdaUVpzJraewDZU1NE76Fk3FmjwrmFmOmlKNEFJBLt2NI9l3fYBVm7d\nBzEorT4Lu1SHjEJ/UVhrmuvj5J0xHNf/DrYMlGHKWi/4rqkuTlmvuIf++fR0OMl0VIP+vAZpMUZ9\nmRId44UrIK8XSODniT9ciDTvoxD//fs17O7Nc/+S7VX7smUhDe52tJYgcYP/UUE1eXuk5Ex++/rD\n2vdjL4i2Hti6iJuWfjNkuq3EH57ezoZdmdBktvi5Lp5d3+1V7zEsg1Vb+ymUDTI5jXXbhfZkOaTq\nkXeFGd9dJuVOYNv2DbFuh29CzZm+VnnnY2vZ11/w2ga8NchlS5DDLxdtpKsnx+qt/fz2iS0hs7mr\nzdz7xFZ27B9mYFhM/I/tepIvLf1GTbO6YYj+xlQ51Hd3PIPEuLXGWveiJTRZ24gJE51kkUooJOMK\nQx3LveMkXZBVIUBGmmFy2x/XMRQ0dct+pLrascf3IzqIxy3vOXmacUwTpk5bwlSKvLBR+JT/vHQn\nz6zd7yWPUWWVklXh/5QNpJiOrSWxTRXd0ti0W5hcS3p4vNpTbRhogM2oVnE/8amrKdhhbV+KldnX\nX2D7viGyeY2N+4VmO8FJbtJfEM9/zrQOkulAycfhZmxLwlIK/OB3Ivgp6Au1ikIjdCO1g+M1Sj8Z\nu5wS5tsa5OvmsVY1oRWu69/gCUFuxrPyunnYhhr2V1eQt2vilhQD07LpaEqScuSQs8fOo8mY6B3b\n0C76Kak6tiWDrXiad1e+i+37hsgVdU8rbUk0M8Y6EXO4ha58F89u3k6uqLN70DeB1xtCECrbBbbu\nzrJsXbfXx6ljBOm675rrLtm63/fne/0P3CMO8br7zh53BhMbxmNjM1QerhKgZEMoIak5j3uBeZKq\nYVsSdrEejDj67uORZBu5LksqoVK2xHc0a2In582ayrFNk9kwuJlp08XzH8yL91wkh5EYk5jAgN6L\nlMgzujXtPceGRJpPvWc2tiV5yk1bU9IXnB3yTlvtTFBO8PqWLfrfaaGkM7vzJE5sn0FnXTtHAhF5\nH4UYcgJC6pPVfiP7gCUvCJnN3UxIru9VqTRlSbVNzot2Pk5faaAqorkWatbudYSDsiHuo9NZu+ul\nAK2INh9Z8xb34i6BclG2AxOxc7/BW3Ozk2mmVm1Wl02yAZPycCk8ybhc/IctDzJQGqyZdcowLZAN\nCsmuqr7Hj3+exPTnvd+1AuLKtiBDu5T2+pSIKSiyhGwEAn1KQrovB8hINyykZA65MbBGOKC9h9Jo\n6qItJRYgb9MCbKRYmeZEI7aeQIqXqtJn7sntJ6HEmdQwAc0uhd6Vjsni2tZwC5gKyCb5ongPKrN8\n1cXS4n1QjFCZSxcJSbwbUkwjmw8kRTHFxDyrbTqqpNDPDuIxiX+7ZBbHT/K1HqtUJywA8ZJnBXH9\ntbGhiRyfOA2A/ny1sCNZCiCBWaE5O/uTagJFlogPC3J9dt/zvrAqmyT0Nuxio/C31iB/M9sqtErX\nv+1sb29OolllpjQdw+XT30NCrqO8+RQAzjhFVC5D1T2N2y6lMTPtKI0DyM3i25Ad8rzm1I+RUJKY\nvULI2Ws6Firn2zH6RzPJnOe0GXgXnb5ceubxpBMqaOJdcZMDlZx3Ttt6EqUX345VrAuR97hRTh4B\nh/iS8RjHNYtqeHkjXxWVPaV9lD+8DQNCaFUMMGOeddF2+iCpGsm44rXxrxefQjKhcsGkc0UDDb1M\n7KzHkp3+OO/8xLiwcClt+xjVmvb6m5QTzD6uQ5i9nW1j2tJV1gNVkZkxfpTTB92L1bDNGLppM731\nOD520j8TV0Yu9XsoEZH3UQg3pWFDuka6wVcIsAp+YO5yD9eXqCgyECAb+cDZygyrOjBDMzVRLEEO\ntx3KmuWQd8kh77FO3WOXICqXihkBn7duGV6gkrf8JlS9yaZsB5f4iD4G/es9gexfwSUvUqJA6rRH\nWbTjL962XLmCvKtyh9fI8mZaxI9byZ66p9haeDm0T2nuC/2u5VPXKGDbYDlZtSTFEOStSEiaX3fZ\ndLRGzfKfeX+5j8TMZUiq4S83CvrNAwFKdllMikrM94drugmqjiTbNMQbsEsppHiJTD6Q7U6y6Cn2\nMrZuNC1JQSZBa4LWuB3bkjB6JmBbigjGGhSknXeimG1TYczwOYK8Ee9lMmVWvXMtsrOkKFYmGxDS\nipYg77H1Yzix/QSM2BAtHWUkSfKIxb1HW0tCrOwJGG4msobisXTUif5nQwF8jvbs+DhtUw2Rr/ve\nJeQE8ZiCVaynM93OjuwuQd6ShSTbWIZ/vhCgrND5Vq4Fu9jgkYvSIN7rlqYYNjZJ1dHsFckjLtcq\nIyl6IChNwth/DBAonOFcI6UmUWQJa0jYyPP0e+MNYPaOp0FtAFsKPUM1bnrnx2Iylkve5QyWbVGS\nRTu2HgdkQdJObAKAGjP8sUMQn/usl+xZ7uVkd3FSu59tJnHcS6RmPYuk6khmjNaGROBaQEwnHlMo\nODEPKUX0rTMlNN5sOYuqytiyLtwWtqC5MbFjkWwFdew28vWbkRQD25KIqWIcG2PNSIkCclOvqG8e\n8Hm7z8H1a8tNfdhjnRUThloVVHskEJH3UYxaUeFBM26tJQth8hbHFsoOwcmSZ+6CEaIxAyjVIJ4l\ne5fzu81/Iu6UKHQTHlQm+we8oLr6VIzm+rgXqeznwq4OWOst9PmEqRiA7ZVelOoyJE56GjsogDj3\nIAX81K7mDWHylOurE9G4ZnMXlZp0Lf+pYfpJNoaMAye3qWV216WiiLB2J+eA5m1LgQxteYe8bf8e\n9pZ3ICkmetfxGN2TgLDmHXz+linGRFZNz/pSMjSSJz0FQL1aj1VOIUli+ZUXSZ7MY9kWY+pGe+lb\ng9HphprHLtWBkRBaqwQ9Q4Jsi6azpGr/MSQK47wJXU6UeEL/XxLTnwtZB1plx7edyntCK0DRsa40\nJxqZ2ijiMFIteeceAuRddDRvyRcw3ICidCzFqAZB3iUr8Jwd8rZ0550xYk6kcfC9EwlyknGFsm7S\nnmwjbxTIlYre+YYhuwMi/ne/rQpSsDVBCLGJGyFWoqlRXNclJUWWQBcEVrByoh+q7hEj+OlTvWVy\nAdO+KsvYWgoZGSvmm91BmHx1A5JyGjk9jNK2l8TMZ5AdQSKlJokpMmbJ1byz/HHrwxjNwuftWg2E\ni8dGaXfW5scCPnkH7rNetm8Fd738W4KYN2Yu7516iX8/ySFQdFRJCEiiLdcXrpGIyRSMIkkl4WXO\na3LqqWfKQyKHhaO5e5kiTJVYYTSSbLFOe1ospzNVYoo4/8KxFyFJoI7ewdi2tDf/ueMcU2XqnMC7\n2Litfl/N2EEVHDrUiMj7KEN/IUPylCeQW/bXXCccJCOj1gtVg7xdYrVtO+QTr6V5u/5qqC7WAHjL\nJJTGAZANr22fvG1PA3LN5om4Qkdziv6hEoZp+ZHyjoYeLIPZEyBeSRZtuZp3/NhVyE7mLjcgxtMw\nAm0EyTtoqQhOhi4KevgeNcMK+fprJWUIErxsB9s8sLAFIg7AUHJYpTS25ZyrGCTiCrIsYztadHnT\nqZ42plt+H12t08o3CpO1c76LEHk7yT0kxcS0bAzTIqsNeoFCti152rmUKLK/wmffnGyixZkw3XSu\nSBaWpHt+WDdCtzebc8bL0byNGLph0eFoS7HxIpuZXJ8N9bfDnoptg9wUWAoGDNvid0uiBVsT10qk\nxHlFowSWjLZtltBuXXOrI2C4qTjr4knGNAvhQx29E6VjlzceAGXNyXtQSiPJlne+uz+pCGIp6xYt\nSeH3HigP+uTtnO8SnEsGleZYs38Mck6YY+Vkgfp6cZ6vecvYegJsWJ9ZizJqJ5IEiuW7GWwthW0H\nnoOsE1fiKLLiLAGVSMuNSIkCx09o8oPLzBjDeY3ZTWcgqQbxY1cj1w1jJ7NOH5IidqMk+rJjaCeP\n7XrSfxCGK4CIMY5PWRsao+A35Uac10JKTXJK58zQNkm2ScpJLzmPa2lQO/YwNGoJBb1ISvXT5SbV\nBEklSaacJaZIQrM2VRJxcb5hWpT2jQ9dwzZjnvvw2JaJIj4hVqalIeG9h+51Fdm3HoTa0OO159rD\njIi8jzI8vnOZSBRw3Es10xAGyeBHf1hbtT84eWu2+LusmTy8fKfIchUMOlGq28+X/PaD5kkXwQpS\nUsqv4OOlHJRsQbrgVeJJxAR52zbc9sd1dGcdE6ZD8oWSwc/+tI7t+4ZYvlVIvHaAmAbcDGSBicI1\nobkfYFk3uOuRDdz57OPszPjpX3/1WKAkZg0ff7DYA8AdD7/MMxt2+PsDWt7zG3q4/+ltXoY1gBUv\nB7JG1bBkDBULWJbN/zz0Mt/9zUrW7u0CyRZBOs49SorQvFVZwpZ1UnIdVqbTm1SCfmQ3iMc2Yz75\nB4Uwl0D2noDpaeZim6abFAMa6JT6qb5Glyiw28ls5b5DxbzM48sdM6yreTt+U9fE6fZxy0AXS9fu\n8zVcI0ZXT47lS2JYxXqkej/gUJIgZTfz2TmfADOOlWsRVhGnbbmhnyFpPzNaj6cp0YBWEtey4jl+\nt/lPDJYzKHoDZt94QAqR992PbWbzfiG8NSTqGN/a4l03Ptl5FxzXgusxsYvChy4lnVgKR4BKqkkS\nMZmybnoWiKyWDWhsYdLxfMoBUhjVmgYkFIe8pXiJtMNHSVX0W5Yc06/zPONOXeykEqg1YMvCwuCQ\ntyXrYc0dyA6oSDGd3WN+i5zyg62GCjpnj52HbVbTQVJJEFcV8jlQibNreE9ovyuY6Hum+hslv5JY\ncL16LeKb0Xo8lxx7MZIk0ZRoqNqfjqX9zHqm6iXsKSf3M1jOkI6F6y00JxrZm9/PQMOLYi4zVS+W\n4p7Ht1AeaGFy9p3e8dZwi5dpsi6pYhtxJFVHVWTf8uhp3lLoHqxcE+WNp4KexDBtNnVl+PkD6w95\nIaeREJH3UQYvwtGSR9C8fXJdvbWvan9wOYcZINpHV3RhmFY4KrZm1R0/GKyW5h3URCXVr9Lk19AN\nrNX1yFtm1mThk3txU6+fwcohnadW7eXZ9d3c8svneXHnDsDRLBHLSTzLgeFrIp3pDq8PAC9s7OXJ\nrmdZXlgUShm5bmcgM1egb66/2fXLu9jTm+dXj/vJN4JLjH58/1r+9MyOUCYwSxJ/nzdnfDga10Hf\ncJ6+TJElq/exbscgK7YJ4cQq1vtai2z4ZnPZICY596m7+Zb9PnqWF1MNkH+15l3YO9Zr31svr1ue\nyVnfNY2J6SkhzXt3r/PsHRLq2qvRtdsJ7nK1UvcenWdh9ApNRx23lb+8sMeL5G9K1lPWTdZvz2L2\nj64al3pzFJObJqEbFtZwC5IEckoIdW5ynrdPOFvcS05MY7uNjSLeApDxScMn7yKPPt/FcKmIbcP0\n8e00pfzJ2DYVLjhtgvfeubGK9YrQqpV0nqb6uDdeKVVohZpmehaIgVLGF5Zcn7kTeCincsSOXYXa\nIVZttNbVMXl0g9NH8b6NHS15edBd8vVQIfx1NjaGfttaCjlRIjZlFSYaKZf8HfK2HdO7jS0sHDbU\nxVNccf5xjG2vJ2ZWL29SZMVZlSJR6vOjqMubT0HvOs57zu3pZibEnWWkqk5RFXPP5I4OLxVqXQ3N\n+x+mLOSCSW8T/ayxfGx0U6OnOYPvv3aRVivJ2zGdpzYiyRa2odJQkRBmzqQp3t9mpsPLQJlMqCTk\nJHJcFwKPa4Gq4bcHMHomYGU7aUzHMEyLp1ftZdm6/fRnj0x+84i8jzJ4ZGHEvIQQQYQCoCo0ye99\n8kzGjvJfZMM2mDymkUmjGiiUDQzDqliPWi0cZEv+MqNaPu9g6khJMTzy9uoDy7XIW+GMmaOZM80h\nXOe6biajYPUeOT0szLmOyTc4oU0d3eH9PaqCvLsHi6GsX36DZu2/HXPgLv1lnt+/MnxOILCndi7j\ngHnc6d+0Cc186rJpVUdqZjkkhA3oTgnIYr0n8UuqCNBRFAkUHQXXzxj0AYoJzrUE2IbqTToN9YHP\nXNHF0idLEe3bEpYsyLikGb7mbsQo66YnxMjJgle8xBUWTC0WIka3L+75AP/90X9gUuMElPpBhu0e\n9sbEWH78nbO5/v2zAbDyTVXjIluOS8AwPdLxVg441291TNXZrF1V79pSfWuEbz1wBQwDhRjzZo5B\nkiTU3aeKZUKKiT12LRPGiOuVyqKm9LX/cBYA8+c1M3/2uEAwWIJETMEGGmMueQ/6AW+u5u1o7kpr\nN2rbPuQ6IYTc8E9v83IPWI5PefrxKe+7cjVvL6eMHo7GnzVugvf3tz8+jwmdghzV9n0YUtkjb1fz\ndp+Vi3Qsxa1Xn8Ox45qIqTIzx4r2bEOl05jBW8ecDvhLSs0+EX9QrzZgDY5G6fdzPnz742+lNS2+\nydiY7QzYuzm+ZSpffN+5/MvFM4BqzbshVs/ExrAZ28qFBZJxLS2hnPZSxZyUrqHNB2FrqVBFwpnH\ntPD2U8czPjlZXG+oDdW5P1mSOG5MBzYW31/zQz/4L0DeDTFfwDH7x5KMKzTVJzBMS9R4l6Ct6dBU\nX3wlROR9lMElC3dyrUTIh1rxosdUORSJbdg6MVUmnVTRdKeggHJgzXsoQN7lGpp3KFtWgLx9zdvv\nk0fejmTtfaTudR3ydgPzpGQOuW4IK9vmE1egv2ogAdKYOmfpiUMm/dlSyKzuJVEIEHZQcLED2ZTu\nWH936B6DwVmVZnWlfTdKu798zG1TUYTJOwjbktCscMrUYVMEuNmlOo+0pFiZZFxBloVA4+ZxxlKw\nLRkppnnpJstWwIXg3INhB56pE8ErGEEiIdWhSeKZarqFZrmFMWLifdATyCjIyYDP2yFRvaSCGcM2\nFc9c6xKrazaPqTJtyRaQID9KVGhS+o9lcvNEjxRcK0pojB0TsW5YHmkpDYNI8aInILgTaX8mXPEL\nwFQC5K255F1EqsuKwCzbJ8J0cRJG9zEAPLN/GT3SJm98VUVmVLodWZJZ2buGPnmrFxOQiiW9dzet\nOMVB9Kz/jjvjbzlL+jwyQBBZc6LJS0lslcWzzpQy3mqKSh+xvP2tnNJxovd7Zsdx3t8xVWFWy6zQ\n8UmPvMU4G/smc0LsLG99eqXw3ZRwnoNkc4w1jytnvFcc5xatGWrj7JaFvHvM+wHoqCCphrgjPIze\nSVxOcOnUd4b2B8n7golv4wunX0MlyhtOp7R2nve7M91BIjYyTbkCigvTDs95drE+RN5u8NtFoy6l\n+OLbwYyhBoJZ61RxDz3FXuRkwUmyI85RFZn6eB0fPuFyJmbeAbZMQzqGqsjohk1vtkRrQ7KqENTh\nQkTebzDolsEL3atGXPJVMv3JtbbPO1A4voJ8Y6rirSEGQLZEBKVTYSlb0MKm3Rqad1+gwENNzTto\nNlcM8k4ke9Eh76CJ17CdJTfOB5WIK0h1Wc8n7loO3OVZSpvwVZt943yTslJtogY/o5N7P2U9LJgk\nqHP6GPQHB9ZDl/2JqXISDd5DZWrP+JS1xKesCbTpkLcsoVNhTrNUdFsLZR3TLFdzjnvFFKR4mURM\n8QOeLH+JkK3HQfXJW7c1Z3mM4pnNTcIJQkLEJTWiSQWQRIpUL3LdUB3BSyItNSKlh4jPWYSUzHkC\nUbkozKlWoQEplUNp24OccM+Pe/fdFHdIIZHHzHQwpnwasiT7BXMMv7b4jBZhnVBNJ5LesDxBQB29\nk+QpTwrLhy15/s7eTMl7Z+aOOhWAMaXT/HE2Y9iGitLc65fRDFilEjEFK+dr/xa+5qwqMnElzsJj\nzmNYy/F88REUZy11OuYHU6WoR5UUitKQJxDalkIqoYAR9zK9uVAlX5sDMHSFhBJnsJxlq5NCdErT\nJIKQyg28a8qF3u+JjX5lrrgqc+boed56cPDJ0hUQsFSmpWbzDqeNyY1+8hcARXIEViksCfnr6yVa\n9anETfE8O5rDxOmSN8C5o89hQsPY0H41UBP+H45d6AsLQVgqdqGJy6dcwTWzP8bcUbOrqskFMXfU\n7NDv9x1/ifftg1jn71aQA0g6wlZSjXvvnRog2/p4WJMXFj4xfm5g2+mjT0XVxfuSTsaIKRKGKQJn\nK8fkcCIi7zcYFu94nP9Z93/cv/Xhmvu9IDFbqql5B3OaV5KvLNuUrTC5xlWZdDIG2BhNO/xoVaiK\nNpcb+1jc/cfqvgQQIrMKn7c6ertXHxfAQiz18j5OtRSuUSyHydsNGDKHW3yTskNoqiJj4pN3a7KV\nhJIIpYN1NSZzqIVOyzFhBwQc19xp5ZrQd87wtlea+4Jthgs01Fiap7h542VPsDGHm/nQ8R/ANhVM\nWw/lHNesssiFbckhzTsekz1BwF0/DIARR1I16p01/yaaFyTkBqwFyRtVJyb5ZFkvi0lIbhhk7eAa\nj7xtM0beqaLVoDoR5bKN2tnlkVCx4JiF841IEsSPXYM6QQRTeZYRSQpN0ma2jQ4nKU88oFG17lnI\nl97yWT4y/QOUN84hVRJJRXTDCsUyiPHQUOwEsiRjWlaoZOqxzZP477d/i3GcGDonKIyBsxzPQTyu\nYA2OQt80h45Um3+Q5QsYFx9zPh+Y8T5/V66JZEz1a0obNu2pdqz4MI0Nzn2ZKumEeBZuJjcXHzxB\ntOUSgmHatCRb6C8OsGlwK82JJi8ILvhadaTamdQwgYXHnIcs++MXU2WSCVUkxnFwaufJ4hoBzTKm\nysyfcDafnfMJPjDjn0J9mtYqgs7M3rApOzPsv++9mZIXhNpWURmsMemblMc3d3IgjFQO2MVJ7TM4\nrmUKkiQRj/vvu7Z9JlY5yWzlHXz5LZ9leutxofPG1o/m+jmf8n7bxbqQ5u0+r2CKYzVQddHVvF2Y\nw63e30GN2nUDphNqiPzbm4+MyRwi8j6ieOz5Lrp6wqkphwoaDyz1g5y2O8kLdgyJZSuPrOjinsc3\ne8uhvCQNstCUlq3bz12PbOS3T2xhqKCFfd4V5Js3CuGkIgHNW27qJT55HWpnIA96KEDGJhYo4wjV\nAWuWbYfK5EmKEYo2V8eJ7E5mthUz60ySkuV9nHvl1aH2RE1rvw61p7kYcZ/YHD92XVL1lr5JG9/G\njq4yacXPmAR45KdvO9ExHVdq3k7U9daTwIxTeukcZDPBkBZ+ZkENasPgZm596o/c+9ctxBM1stsF\nNG+3kIax5zjmjj0RLAXN0vnLCr82s47uCCaSuE9bglhZWCVc4UMPrO/V40iK5UUoIxu+VcJdR+ya\n6yUTSbZIyP6k26AKv3HsmHX8pe8BhmNOX4yY9+wanWPASTiiashWjGLJoqUhUdNnbet+bEWQvO1S\n2iPvYKnaJI2MruskEVOxsh1YTlSxHtC8XcjJAoolnv/9T2/HtGxithCwOlLtSJJUpa25goxVrMPM\ntnGccoa3TxwrYWY7mBYkA0vxtFZJkpg35jRa4+K9NbonElMV7zovbupF1uqRFJN0Y9k737VqeX57\nQNtyMjNahb/YNWnbNqT0jnwAACAASURBVExrOZaSWaZgFJnaPLlm8Q5FVvjc3E/xrikLKrZLwrxs\nJLC1BAoqJ7YLAVRRwiQPMLlpkhfU6eLE9hOIbTsXfdf00PZgVbvebJGCM1e11CdCxzUHyLsj3Uot\n/MeZX+Rrb72h5r4g4oHnlwz8bfZOoLzqbYxPTmZUXW0BIRiBbpdTNc3mSlCgCZJ3haBuF/0I+CDJ\nu0pJXSoW2t4RkfffH/b05vj1Y5u56X+eC23/5cMb+MNT2/ijk6fc9dmokkJfpshv/rKZxc+JZTa6\nqfumV4e871y0kcdf3MOi5bt4fkNPyOddGdzh1om2yk6QkWx6Pu9ahelD/uBE0VtD7aLSbL5++wAl\no+RPtorOcN5J0lLWQRITsbb5VH8Nsmx6H+cwvdiWRGnVOZhZ5+OXA+StaiKBhy37EbxOn+rTMXRL\nx9bjFLJJfvC71aTUdCi5hr+EJ4ahy1X36Js7nWIMRh2y1iisCZIFWMSnP4fS0uMtWQHYYDzDw8/u\n8pa+ubBtKeTzdjNC2UYMWZbEGnDZ4PHnffK2EMk3Jo6qByTQ48LnHTCbG5r/2bpaaSJtihSwiu6T\ntvMcOjqcNe+uoBPQLprjzc44OlYBxe+jm9K0PuaTr6Tqjt88TqFk0NaYrE3eAW3ZM5sjfPluLedY\nYFJWHRLz/LPOulndsJDNMEkAyGaSkmbw4DIh7F7UcQVXTv8nprUI7TFoKhX37rgjSmm0jXOZnvTN\n6u77Z9swJu2n6cSWQxM7wPunXIm2YwZm/1hiqkzKuc79T29n5y4np3Z6nTjdUkgnVc49ZayXZAXC\na59PmSpMvP9w5jGcMdrv07njzwx0vur2PbQ42cckSfJIpLT2TK4c93FPuw1mF4yrI5ugAS47ay7Y\nMmec4I/DhXP9wLjeTNEjrvGdgqynTxTvUFOAvNuStcm7OdHkrYmvhfEd4t0MCl+1zOYH8oMDTJFP\nQ983GZBFeteKtoKat+dWIOxDNzPtmAP+OAQ17wWnC5fD204ZGyJvNxvckUBUVewIoViuvfZv/4CY\nLF3Tn0veiqzSE8gnPVzQ6Q/UL0a2KGtmyHReLBuUpaDmHSbkYUeDtMtpSJRANompCnXJcO7lifHj\n2aVtCpO/G6S07xhGG7Pon/DnqoC1fFlDUkyxbjemISkG/UMlLMumZJSRZJvpbZP5+GfO56tPbKef\nHpAt74PSEZnF7HLaXx8qW77ZPKb564fLKSRkSAhLREdTim5LCwWaqXZSRKzLplgj6yWmUCmXbVER\nyCHsKWMbSU2qZ1sOT7CoS6pCY0oBqoYkWSL5DAifbkX0uhtjEJfjDL94BvETlgc0b9kb/1s+IqKX\nZTuGpYhc4u4MLSkGthHnuHHNfPby2Vz/2FKkZE5MHE77Wjkwm7sJJGI6LQ0xioqF5Wb0slQScoJU\nyuCmj8zllj8+AAjTopsfqjXRChUp6t1o9JyjeadUn4ileAlUDb2QwrJt0kmVn3ziHfz48TgbSi+i\nNGRoUBspBsgqpHlrqZqatxfxK0tIkh+kqBsWsRqEI5kJT7iYPKaBK+efSl+fbyFprwim0ndNI3Hc\nS+h7BbknAqbYoJY3OqTNSSGTKMC4pk7MHuGLjqkyHQHTsV1hGscU39aHFkyjdetuFu0Sgkaw1vak\n0Q386NpzhGUFeNv4M2lPtYX93QcoV/Ctj83zAh49Td2Ih4g0SE6V91OJd501hVMmt4a01ffNn8q7\nz5rMt3+9kr39ec+d0tqQ4NZrziYVF8cGY0Nqrek+GHz5I3PRdCtErkGzubftAH5wgOPUuazrEgpR\nkLxdn/dImncwT4W2KRA3QVggPPeUsZw+YxTppMpTq/wA1WT8yFFqpHkfIVg1UpUCVaYxw3KLhMih\nYhCFkkFf0c/JLSt+SktX8ivr1oE1b6fghuf/U0zH5+1XParPT+WczvnO/mCqVD/pQn3MCc6p8Hkv\nyQo/va0lhd9WFVWSBoZLXtnIpkQ9qiLTXu8kvohp3sdZtovexGa7NZklV/O2QdWwveAmmTq5Ednx\ng7c3J0WQn+l/1JYernYkMi4JE2nZ4V3h47dJxhUM1zfsCACphIrlraUuh0zwthFnsiYKIXjJLZzx\nPnPs6SiWWKftEroiS2TKQ0hIjKpvce6gVhIVE0yVeFymPhUThUEUC1s2sJVgoJjTDyeoTZcLtLW4\n5nKfHBpiDWS1Idqakshp8fyntvqaVGstDckQZnt3kp7deiqzW+eIcUgNI8m2l9WsLqkSjym0SZPR\nd8yEgQl8aMpVBNXFUGCSLdf0eYeIXJFFwiBElbRa0buSmfDM+lPGNFV9R5WBQ9bgaN7ffg22YyUI\nam5BIvdWKQT6EkRdYAKPq3LIxxn0N4MTsJZUkSSJ9nTAOmGE1x2nEiqyJCFJEv90/Lt5+4Szqu53\nJKiKHCLaWvcUJKr4K5C3JElV7cnOto7mJLphsddZMphOxqhLxjyiDa7jrmXyPxioilxlNamteR+Y\nvINCyiuZzYPHnth+AnVqmium/2NVm3WBQlCSJHn9DL67ifiRo9SIvI8QauUZr7Xd07wlhd6MT475\nUrXm7aIuJV4iTTdDRSrCmrfN5sw28Zdjcg6bzZ3lN6UpXuIKKWg293Ihq6QSKkk1GdK8dctgW2GD\nc7AFZswj/L5Myat85Urnk5pEQJJclyURU9AtQ0RKuyZ3JxmDu9YbVRdm4YD/s0FpEfV3FZ2OppQg\n74DmrZXcQLhAZivHZFly5CK1bT9Kx26x3MPSPHJXFYlEXMEoOwJATAv5usHGGhjDhORkp9604Res\nUBNiQgsUtFBkiaw2RH28zsvFbLqme1dIkiyRWMJUfFOuQ85lCuiKIF+9GCAMxyeXNftobnL8pwGz\nbGO8kbxeYG+xy8vHPGOUr9U1JeqIyeIeZdstpOFkbnPMo+lEgg/NfC9WOemZ122nIlk66QpbNnax\nAXXvKbSkwlHESSXBuPix6LunoioyTfV+JLoLNaAdKrLkFXoQmncN8jYSnvm2crKH2r7HukCyjrBZ\n1m+/MR7O8hXsl/gd9h8Hr2NraYrPBXzRAZ93Q9zXhG0zTN6viFfBg0HNVKkIbHu1cO91pxO3U1dJ\nskqck9pnVvnjXytqEXWlUHWg/alEtQl+pIC1hng93z7nZs4c+5aqNmu9ZxAm/1cSKg4lIvI+QhiB\nu6vglsNUZYW+rK9590gb+O2m+/0DA8TqlgYtaWaIUIOat9zcy7J9ItLbKjnLpGJlYorsmM2dNddy\niua0Y/JSqoO9MEVO6sZ4AwOlQTQnA1le9zOvGb0TsE0VJSau35sp8v/Yu/P4qMqzf/yfs81MJpls\nkAAJ+yabICgo4i5Qt69WWxUXcKlaRVu1daFUpbUPuFT9Wbva1trqQ12hllddeLpp1YLWlcUVtAjI\nkkD2zHaW3x9nmXMmM5mQZCYZ5vP+h8xkZnLmJMx1rvu+7uuOWevL7eA9uXqMeVyl+/CrDx7Gqzv+\nbZ6npJ7Y/knrzKBmNUZxFy/ZhVRCoB0DyvxQDc0zbG533rKDrjkkbZ6rcLsrWJTvRWNwM3a173Iy\nd0kU4VckaBFX8PZUrsdR3xSBz96yUo45vxO/5IMkCOb6Z8mcKxdFc7cjuwMUAGiqfYFiPq+4OLGk\nx+nnbL3fiNaGmGiNnESCieYeVrOaFr0egZB1fK7gXR4wA+nKj58xHx8NYGBxYs4x4JfNoXMAJdoQ\nCLGg01TEzmz9PslsEqO5A5V5UWF/gNt75IiC0GGeWBAEnFJ9DtQvx6KqPODMwbqzMzkp8/YOm4uY\nV3YhYlumIbLpaMS3j4PSNNK5uEgOIgBQXtJx7tGdOaWbUxUEAQtGXYzohzM7HFcyRRZR2mFnP8HZ\nIcuI+Z2LG89FQYoe+r3NfUHiHjbvjeAdjWnmErqkQCUIAr459RKcMvLkbv+MVFLOb2e4oHFfdLmH\nsv0phs2TL9DSCaYY4QAS9RoAg/dByT1s/t6n9dANsxfuftd2llu/bEJMtTJcw0BdY9jJAPeXJ/aA\n1ttLrAIq8zXNDy8De/WtqI/sT/xQV/C1h5dlQYbeWG0uMSpuhqJ4h819QgClwSLo4SDEUKPzGu7M\ne2d9G0q1oYjpcTyx/jWs/2C3s7etumeY9fqJIri6pjBiVqFdsbWOcnRlrbmOdsBufNGyHau2/MU8\nUCt428PmghL3NOZwF0KVyFaLVCWC0lJ7eU7iP09Ts13oZm1VKCUqsdvaBGCH+SErVdShrug980nW\n/2NZMiuW7QsdqWq7N/OW4tjXHIEMa3jWmuMHzEzTzLytD3YljrgRQVxXUe4aQraXfCnDzKYgpSF7\nIwvZmUqwq5TbjVZExCYYutnD2mn5GPfDiPuwH19ik/oP8xQ0JqqI7S077SmX2JbEOmDAXPs/sMgM\n3pE2H9o3HO08xh42d9bhC4lhUfu47OBk/32bc9YdPwztAJuuGtcdJCVR8BSs+WQRgwODoe0fAqO9\nFOquMdBVn7MbXjDFvvbuzMo5Blfm7Q48/qQ51YmV46C3mFXlyRciboospXyvV0y5GPH35gGaL2Xm\nndziMxv8nmJAd/DufnAZ6JqKSHXBlC2ZsuxMz3FPz9gXAuky784Up/g7AwBZTrwWg/dByJ15P7Rq\nA155dyfuXvmOp9HK8sfeRn2zOTcc0+PY1xxFZWkAJQEFQsRd9GP9J7IztiIFUtUObCsyd/s5acDp\nAOBtB2oF4UvGLwIMEXpbGUR/GIZoNfiQzbaZPsmHoF+GVl8LQdQhVe72PB+agoaWKN57y/xD/vN7\nr+PXaz7Axm3m4+zgamgydMEMmvubo4gLVvC2Mm9REJ0Mz3OeUvTrdg9ZuzPvErnEeZ8lQSvwuTJv\nLe7q2Caaeyy7s57wl0M9VePun2suvZGgt1ZgQukkSKFGSFWJZXSxz6bCMIC4nZl7Mm+/uYey9f7E\nYDPaNHsLy0TWO7bYWspTuQcQVZSUJC5A7A+BgUHz8a/sfx5hcb815SGgusIOgmaTFA0xRIw2xHeM\nxfTBiTXqdvAGzPXtRpv5evaccNAvozpoBqq2Fsks7LOCS0u7+Tu3i3wG+BJroO3gbVfXjq01f85h\nYwc6owLuoFhZGoAAYGhVx9854B16lCUhkXlrZuadnDFqmp5YrpNuODPpQzlV4RLQ8QPXPUfaWYGX\nnbFVliay/AGl5haVMsy/02CK4J343XXN6CHm//3Dxg3M8MhERul+T6LYu5k3kH4IORvsn+WTRQyz\nKtyryjpvhuK+6PLMSSuJkbVU3+/KcSSTPXPeuQverDbPESczKWmAPPQTfLjTu7ymTdgLsbzOmeON\najFEYqq5jlY30KaJEACE6meiWbaWFllV1MGA7GzacP74ryKyx9zooXaIjMtPnAXdMPBKXQPW7/3M\nyXy11lKIZXUIi/sQ9I81s1PV3NtWFAV8e958/PKjTzF1qoh3/55ocFLiC6IZifWPdr/o3U1WW0+7\nGMfOOiUVMVWHjihEeCtSJ9YOxsdNiZaR5vPNDz17pACAOd/tt+daXQ1GrOB95IwifN76mfVzXWug\nnUYuGgTr/BiajOKAbA25CmZBmL9jsxnJNSw4o2w2Pmr+wNmJKbLhWHO/agDtLQJQYgV915y3JEWc\nJVRicTNaVfPnuzPvb596PJa/vBX75E8hKDFUlgWxwzpGe8574UmH4uebXI1rrO5XQwYU47yTxiIU\n9OGdLwdgXcM/UOoL4ZRDvo5h1SFcEDYb5OwT/us89fCRI3HWCWbryTsunYn9zebWh9VNZlCwh8JP\nnjEUf39nBzTdQHFAdoYd506agj98bG7KcsnJhyEkDMCU0WbWfszUIRhUUYTRNWaf7GWXzkSFK6hV\nlRfh9kuPwODK1FXInjlvSUQsrsEwDKfaPDljVDU9MSef5kP1vmuPxusbduGZl825fk+Rmiu4JQc0\nd5CXU2Tw9187B63huJN1L7t0JprazIs+ewcrO4ja2Zq7u9hti7xVzJnMnjIYA8sCGF2ToiNZkvsW\nH43G1ljSnHfXC9Y6M6DU3BfdMHIbvAM+GcsunYnykB+KJODLfe2oTXMRaHNfdCkpRlnSFay53X/t\nHLz18V488Tdzu9p0GXqqi4NcYPDOEbswTSzdB6m0Ac2tewAkrh63la6Fuyg3psUQi5vLqEQB2Cuo\nKJaLIDQMhVi1y1xcJOowAAQUGYJsZn2TB0zA3z7ZZ3bv8oedtZhavVWQ5jOvnu250jZhPwRBgCDH\nYaiJhgMTBtdC/FhE3JpntTPvimAJmvfHzbXWmuQUpe1vbzH/muxqcSdwxs0GNIpVze5aQlJRFAK8\nsdvJrGPbJiIweb35GnIMorVlpN6ayFxDivke3t3/Nt7d/7Z5pyvzdobQ5Rj8483vG9EihII+TB0z\nEOs273aCfak+BOMGV2P9f8zzJEuCk50FjUrobaUQi5s9xwgATc0wg7ccd4oI/ZIPoiA4PbvF4ia0\nqOZzy1xz3n6fhOpQGfaFzfqD8jLRXLalJ4bNq0PeavD4DrOJSHFAdrLYE8dNw4mY5nmcX5FQWQpU\nxkc79w0vH4RqK3sqtiqFAWDW4MPx7KsfQ9s/BJNHVngyQ3e2NW5AotBt+qihngsxURBwyPBEtfWI\nwR23dxw5OH3w6ThsbjhD58mZt98nQdONRJerNMOZpUGf50PeHdB8nmHljnP0smQeQ6oP9oqQ31lf\nDQChoA+hoLeRjP1+3BcCK+bcDkkUUaIcWMFa8rntTFmJH2VJ8/2pmrR0hyyJqAz5sa85mnYIOVvc\nf0/2KE9n5DRLwVIWrKW4QAPM3/PIFH/Hydw1BRw2Pwg5w+ZWT+WInmKHKxd7yZdfkcwPJ1GDIvoQ\njWuQkpYYybIASUlsU1jXGIERC6BdS6x7tVtzhvzmB669WUMM7TAMwwrePkSsHbxkUUaFvxx14X0Q\ny/dCHmAPi7u2WlQVZ/mUvduYMyft7GGsojUchVhsZuYhV+FOyrWg9rB7WzmiH1vLk5QoxFAj9EgR\nEHd1B/OluPp27fhlX0BIFXsgKHFIrYOh7jSDn7OUyPp9+EQ/Lp9yEcT95m5DdsEaYFZdq9aOSoYu\neLL7BmsBgFi6z5mXtzd+QDwAI+aHEGzGjlZzyL0maSlSib1LkRxDaYld7Z0YNncXOk0UToTeYI6q\ndDXzce+6lG7tbUD2I/blKECXMLC8KG27R3exXbHcvXW86aQqWItZ65cVyRu8Az4JqmZkHDYHktY4\np8mQUs2P2wVv7u1dD4Q9kuD+PZX5Qx365OdCb2XeQOJiLpeZd3d4Mu8U1eBdybzNx2U+X+6Lg1R/\nS9nC4J0jTsGatYGCs/uTLWlLQ7tHud9ndmkSJA2KYG4Dam9q4ARvSXSGtQNyAPWNYQhqAO1qO1Td\nvD+shiEKIoKKGbTsIdKI0WbuYiQYgKZ4CuiqigagOdYC//h3nPvawq41yZriFGm1WNXmRofMW0Wj\n8hnE4hZUxMd4AkiqYOLeYcoZQg81QJDjHdbRFrn28lWsYUnPPLp1DGKRtca8bZIzn+tklFa2LMIq\nHrP+I8qS4BS6tEfi0PbVmIFb9cFd6mq0l0JvLYNUXg+p2mxp65f8zsWa3h6C6I9g475NKJKLMCxU\n63kPIcVV+e/XneO2P2R8kmvNtpgYdTiQzOeiCeeiSC7C5AET0j4mZm0vW1bs82Qi7vXSgiDg4gnn\n4uyxp3d7HW86SoqlYnbzEZ8ieoJOQJGg6ZmHzYH0WZV7Pa6U4jF2Zt3Y0vlFdjr2h36uM9RUpG4U\nZ6VjX8wV+/v+fXXGezHoyox9nS8VS9aF2J2x8U22MHhnQTiqoqXduyuY0yXMyvTiRlJ3LtVbgOEE\nb8Xa9UtUAV1GLK5Bttbl2vPjih28NRkCBNQ1heEXzMBoN2Zpj4dRJAcgSaK5VCfuh2EIaFWb8ceP\nVgEAtP2DPB9WA4OuTRosEVenOMOqKPcrIiL2Bh3OnLcVOBUVMdnMumvh3bIwOXifNOxYs2DKZjVZ\nEcvMPa6T23C650EXTVqAG2dcA3XXqMTx6e75bxHlYiLrtT+c7WAfMMzXtocYJUl0/qO3RVRA9SH+\n3ymIb0/sya3IImCIiG01h6ztna38kh/2SgB7HXZYi2B8+egOGzKU+q3aASXmbCBiaHLK5TElUuLi\npegAMp+ja2bivuN+2GlbSltxkeL5MEquDp9dMxNzhx/f5Z/dZUnLxlQtfebt90nQNHPY3C4sTCdd\n5uS+v7Pg3dDazeCdIvPuK+5h855edOVL5q2kec+pMu/OCtbc2/Wm09MLou5i8M6C6x96Ddc/9Jrn\nPrt6Fk7w9gZ3Q9CgR4LmGlZRcTbZ8CsSSopkCJKOPfti5iYMgt061GroIgnOhhRtERXhqIZia3/h\npqg51xpWwwhamar5QWj2zd7eth0fNXyKQfIIaPW1GDwgEVA9OyxZhg8yX3dAqd8pShs62Oc0QnEy\nb6dtpwpVtIbsFe/8kXsI8VuHXdlh/99xQ8xWlfb/PfcmAYD3P+DQkhqMLR8FGCnmvGF1RLOqdodV\nlzgZROyzqYhvH4ehhrk8ys4AZVFwisbs4VmtvhbaPnP4fNSQUqdHtxEtcvrFA4Bf9jkdLY32xEhA\nqsy3LJAI3s7fhC55AtL/G30Kjhx8OAJS9pbq2PPcQyqDnkrsQTnaaMHdrEiWBOiG4azEUBTJO2yu\nmHPebRHVHJXqJCBJXVjDW2o1jXFn92NqzIu5IQO6N8ztVyQU+eU++2B3S3Vx0l2DrL+T0mJfhkf2\nrXS/d/v3ka63eTK7F7zYyd9YV9eJ97b+ffmUp+xCG8MwnA+WRPA2/9XQMXhDC0DdNQbV46LYGfkC\nsLbLnDV5IF54C04wcipXreCtSCIgxWFEfE5L1XJ/KRoANESbMArmnLddLFUe8mPP/nZz/tgXRZEc\nwHdnX463yxow3bUcZXz5mMTxqTK+MuJknHTUZLz7aT3iqo6nt5hrzysrJWyPxc1kU1NwzNQhEMsF\n/CeyCZKswfBFYBhCh0zbfXt8xRgIgoAbzp0Gnyxi9/52zJo4CDe/9ifnfert3jluWRKwaOL52NL4\neYcLjbISH5paDRiGGfyrS8px+lEjURr0YfaUwSgOKLj27EPx8z9thLprDIQae7g8kXnbRU32nuTj\nhpbhzGNGYV9TBDPGV+FnqzbA3GFcgN40EKK1I5tn2Nx1wTFj0FQkq7CaqMiDvsCGfebPOe+YyZ6i\no1NGmu1qX3rjC+e+AaW9u2/wLRdMx0dfNGDK6AGIxTV8/YQxB1Qo1VPupZR2sLH/litCfs8oix3I\nW9oT+5ink/yhe+uF0xGNe7OpMTVluPTUCc4GGwBw8hFD4fdJmDHeu/NWV10wd5wzrN/XejN4H35I\nFRbOH4+jJg/utdfMhuRs+vuLDkdTa+Iz1/130dn5GTE4hEtPnYBDhqcfteqrCzQG7yxStcSmCppm\nz3mbHxya4A3eEPREYxIkdtzy+yRnq0l7DbOdeTt7RUsCdCEOQwtiZ521UUdxJT4PA/sjDeZuZLrq\nZN5V5QEzeFsXEjXFQ1CsFOG4ad4sa3jpUAwOVmN3+15EP5yFY4+ag1DQh+Om1WD9B4lK7WDQgICw\ntbm9gLOPHY09cRn/ec8cNheUKBD3IVDi/aAt9lQrm+996hgzCE8YYfX/1v3QxXarUKxjRe+RVYfj\nyCGHdzj35SV+8z+rIQCCgepQGfyKhLlHJPp6H35IFYJ+Ge1R1cmU7Yst93CsnXkfOnoAJo9MVH+7\nq5zj28cDhoihlRVQRNmpcTDCJdAjRThhzHTPHL1znEWJC5KdrealwNETRnV4HODNEFJ1EOuJytIA\njp4yBIBZiX3aUSMyPKN3uTNve5jX3rSnqjzgyYrt77dFVFRXdF44l5xVpbsYOW5ajee2KAgd7jsQ\n44ZmnqLIld4M3pIo4sQZQzM/sI8lz0PbIympZJpKyPR30JWitmzo+zGdg5j7Cl/Tra+tYXNnj2Xz\nljlfavfztpc7WTtuOXt0W8HSZ+/HbDdOkTSn4GxHnVn1XVtqZtANkUa0W/PRRdY+t1X2jkuy+bo1\nJemvom8+4jpEP5wJI1zq3bQ+oCSGyJUwRH8EmpUZK7LobK0nyCoEXxRGzO/Zlxfo2s5DJa3mHLPe\n2DED6uxDKRS06wLMoFDiSz386QzJG4bntuyZ844797l55v00H+LbJmGYPsN6Qet+Q0R0w3E4b/xZ\nKX9+kc/nFA8CZuFdukpud/FVLqtac8Gdedvnede+xI5x7mFz9+890/RBbwaufCX1g6H7XMvlUHa6\nTaeyrfB+qzlkL7sCEsPmguBu2WmxN9+wMm87wxZE1QreiblQAAiIAc9rGFYWb2gydlrBe0SlOV+8\nP9Jo7kcNIGgFVLvNYeyzQzEiNAynjpyb9j0E5IDTKtL9HyIYkJ3g/Z+ItZtYuGPwhq/dXI8eD3To\nhdyV4F0ePgSRjXMQ+++UDt/r7EMped/idEt07Kvu5P9/dntUIJF5JweCVEU79lW49+VStwwFzGVP\n0Q+OcgJ4mb/jDlm2rhTP5Bv7nRquM2af50TmXeQ59+7fe6bCqT76XO1XCvECJpdD2U5ilmMcNu9F\numF45lI8mbfmLVhzb7cJwargtoKz0SHztrqLWXPefsneDMMM3ppgXQioMnZY2/UNG1AJn6hgQ/1m\nJxjYa4ZDRebws948ELfMPK/L7y8580bS7kh24xdFFlFkWM1gfFZns5i/Q1WwLMo4YegcDAqmn1eU\nZbFDoZqts0KT5Cvv9MHb/Dd52FwUEsHbnitLfs1Uy4DsD8p0u8h1PE4RRqzIXG5WuRdiJzsu2Mv4\netJoo78RBAGGYSRl3lbw3tcOvyIhFFSSNjFxZ96dz3l39fdwMGPwzi7nsz3HDp5PgT62e387rrjn\nn/j7267+1/HEsqrkgjVB6ph5G9awuW7vNuWLwO9zZ97mtZZTdWy9hu7KvJvaYigt9iHgk+GTzCD9\nft0mAMCospEA9OeBHQAAIABJREFUgPJQ9ypFk5tcOHtuW/RwCLIkmPv/Wpm37rOat8T9Kfv+njv+\nLBw39Oi0P7OzZRzJnbHcyoq9c8IBOXWBlz13XGQdWyITTPS/brcadSRn+ikzbyl1Jp+JfcEW19MX\nOdmvOXxQ560h84m9JMtd4S675rQHlgc6jES4f++ZMu+DbXqhOwrxHOTygqWvLqaZefeStz7aCwBY\n+ddPnPvcm444QyuiO/M2AAiJPautYFgSrwX8m6AM+wQ++WRnztvOvINyEIh3zLxrKspR7h/gVMi2\nurbpBIDRZWYR0qSRlTh99ghMH9e1Stprzz4UX9a3ej4EKkJ+nHroNLypbcWYkrF4+8P9MNpKofgT\nFfHmkjd7NzJft1oHdjZ3lep7t1wwHe9vrcfXjh8DUQT+ZT9WTP2zrz17Cl5Y/wVOn20VaLlesqqi\nCKOGhPD5LnP0IPkDIdV8qzNsbkVaWRJxwdxxad8DAFx51hSsa9iBrZFdnuHjZGcdMwqqpuOsY1IX\ntOWj75w/Df/3n+04+fBEEdSx02rQ0h6Hbhg4ekqiHuPCueMQ8MnY+mWip26m4D24MohTjhyOKaMq\nO33cwcyvSDhzzshO29MebIr8svmeh6R/zxfPH9/lTUk6M2N8FU6cXotjpw3p8WsdCAbvA7S18b+o\nC9fjqCHezQXswCYEWiGW1UPbMwLRlJm3GagF0XA2FnGG0q3g7YsOQoV/KBqKdwBSvMOcd5EcgBAX\nnNakLaq5DehXpo/F7JpEj+sLJ3wNL3z+NzRGm5znAeaQ8NeOTywDy+TwQ6pw+CEdA/3Xjp6Mq6uO\nwsdb67B+rbkft/sqNCgH0BSzh/SVbgZvMem22XMaSD1sPmFEhVOpfv5J4/Avc5dMKGLq4dXqiiAu\nPTWx/lpAYthbFARcd85UfPfnr6c8lmCKLlPJAf74aTU4cXpth8e5nXncGEzYfiZ+uWEfFhxydtrH\nBQMyFn7lkLTfz0dDBhTjklO869/H1pbh21/vuKzOXimwbXeLc1+mYXNBEHDeiWN74Ujz21ePHZ35\nQQeZTO/5pF6qmpclsU/+XzJ4H6AH3vkFAGDW4Bmebln2XHdgqtmcJdJa4Q3emrdgDYCZfetyIhu3\nhs1jcQ2KHgREQBOirszbqjZXJBTFi9BqBe9P2z+CKIiYPND7ITin5kjMqTkSb+95DwNTNFzpLe4P\nUPeSnqASRFMssZtXd7bLSw6YPlmCqplDy501TrAdWzsbr+5ch9HWlEEmiepz89+yksQUQ3Kmn7pg\nzTts3tWGVhWBciyddWPXHlzg3Bdt7o0/iAoJ//K7SdVVZ04ZAJKnWARRSxo2TypYgznsbcQDEKwm\nJPYcciSmQdB9ZvAWI4iq1lIxe523LCIoF6FNaoXgb8OeyJeYWDnes4mF2+GDDuvRe83Ep4hmP2rd\n8GTe7mpyo7uZd9J8kt8nOXPQXWn1eN74s/DVMachIHdvXbS3mYP3WFIOm9tz3vbwd+FNN2ad5ClY\n40cYFSYWrHWTmlRY1CGQGELSsLm9zjuRedubeiSGzc3gFo1rEKyGJHEjirC1TttemuWTRQSVIkCO\nQRpgNvaYOWh6z99UNwmC4HyIuueQPOuVXZttHIjkbPdAd0USBfGAAnfyum+3mKp5bqfaijIx523/\nfEbv3uYtWOvfG2QQZQuDdzfFda3zBwg6/vi3T7Hps30AXJm36FoTaFecJw2bb9vdgi92mvPcUSOM\ntri53tVu0qLIEkqUIATRgFS1A7IgY2rV5J6/qR6wP0QVxTtsbjNUxbOTU1clD5tne79cZ847xfda\n2uOe26kL1rpXbU5dx8ybiMG725Izb7ufucMKyA88/T6AdMPmKgZVFGFghbWHtWvplb0dZtQIO3tx\n25m3IotOYBT9EQwtHppoitJHjpo0CANKAzh8fLVzX1BJtAOVoXSrjaA7eI8cHMKF88b37EAzSZEo\nf+/iGZgwvByzJ3v34lZkEbMmVnsKopKHzZl4976JIyowqDKISSMrUFHau21iifIFL1u7SdW9WVgs\nufuVNY9tf3inK1i7Y9FMPP3uK3izHYAuosgvIRzVnK0129VE8LaboiiyiGI90XQk5Ov7db9nHjMK\nZyYtYXIPm/vl7q0td+/zfPslR2R9swdnnbfr1zRuaDluuXBGx8cKAq4+y+z89vQ/twBwVZsbicdQ\n7xo3tBx3XXVUXx8GUZ9i5t1NquEdNjdbV7rms63MuzJkZsTJvc0Bs1GLTxFRVGT9GgzRqdy2s+zW\nWBva4+3wiT5nWN0niyj3JdYvhtIUqvU1d8FadyrNgUTBmiSaLUaz3Xwh0XGte+Pe9pJBnfVqRJRF\nWc28V6xYgffffx+CIGDp0qWYOjWxdnPlypVYs2YNRFHElClT8P3vfz+bh9LrkofN46ruZNsAnK8H\nWMN6qea8BVmFJIoIBqzgrZutIOubIs6weVu8De1qGAEpALs1hSKLKJMSwbs0zaYbfc09593duWq7\nOMkO2tnecEBI7pd6gOSkJi3MvIkoG7KWeb/55pvYtm0bnnrqKSxfvhzLly93vtfa2opHHnkEK1eu\nxBNPPIGtW7fivffey9ahZEVyG8u4qnn7lVvBu9Rqv6m72qMaqnnNJPniePHzv6MdjQDMOe9ia39i\nSfdBgIDWeBva42FnO0/ALFgr8yeCd1mgf3ZOch9z8qYkXWVn3nZGm+3t9xLD5t2M3slLBhm7iSgL\nsvZJuG7dOsyda+5WNWbMGDQ1NaG11exzrSgKFEVBe3s7VFVFOBxGWVn6/Vb7Ql1jGI+t/djZDjJZ\nqszbvVOY0/LUCgLujUnsrFoYsAN/+XwtXtn5uvVY0Wk6IUkiipUgGqNNiGgRTxaryKI3ePeDOe9U\nZDExsNPtzFtK7K8N5K5Pc7eLxa0n9tU2gURUGLI2bF5fX4/JkxPLlyorK1FXV4eSkhL4/X5ce+21\nmDt3Lvx+P04//XSMGtV5v+aKiiBkuXeXCVVVpZ8rXrHyHWzZ3oiyUABXnNVxO8rikOJ5viCJiXXb\ngJN5y4qEqqoQJFkCYEAQAD3uAwLtHY+nrBhl1hy5IokYXl6DD+o+BQBUliSC9eDqEAJFiYA9fNAg\nVA3su3nvdOfRCNYC7wB6OIjSEn+n5zudygpzSkCWRef5AZ+E8cMruvV6mVx46kT86JE3cN68Q7r1\n+iWhAKqqQrj6nKn45aoNOHXO6C69TjbeS6HhOewdPI89l4tzmLNqc/cwZGtrKx5++GG89NJLKCkp\nwSWXXIKPPvoIEyZMSPv8hoaOwa4nqqpCqKtrSfv9fY1h69/2lI/b19CCOiVxf2tbzDNsPn54CB/s\nBMLhOOrqWtAeiSWK1TQZhi4msnPLVacfin+tM3+uKAoY5B+ED2AGb9lINKNoaQ5DiyZ+dVq72Ol7\nyabOzqMAH04oPh8vvl0HjDO6dYzhtqj1WnCe/7MbjoMgICvveVRVMX57y4kQRaFbr9/cHEZdXQtm\njhuIw7v4Opn+FikznsPewfPYc719DtNdCGRt2Ly6uhr19fXO7b1796KqytzcYuvWrRg2bBgqKyvh\n8/lwxBFHYNOmTdk6lG6xLzbSjdJ2GDbX9KRtPs3MW7NeJ2q0QvBHrBcXALXjdZMsys7wuiyJqA3V\nON9zL7tSJNFTCFWi9M9hcwCo8g0GNB/8Svf+1Ox13u4qc9GqPM+W3hqaL8StGIkoN7IWvOfMmYO1\na9cCADZv3ozq6mqUlJhBpra2Flu3bkUkYgazTZs2YeTIkdk6lG4xUqzTdY8edChYi2uA7B42N7Nq\nOxjvqFqDwNRXrRcSYcQ7NpdQRBmqtaRMEgUMK0kEb/dabrt/+IiQuctSd/t254IdfANK9wZ5ZDm3\nc91ERPkga8PmM2bMwOTJk7FgwQIIgoBly5Zh9erVCIVCmDdvHr7xjW9g0aJFkCQJ06dPxxFHHJH5\nRXPIvdRH1VX88aNVmO3aBlQ1OmbeYlFiqMQQzO87Veae1xZgtJVBLDYff+GEr+HThs9RVTQQmlYH\nwCxYqykZjONqZ0MWZcypmYUnsN45JgD47uGLu70eOVfsgjNfN1qjAole6WKWq8x7C+vUiCgXsjrn\nfdNNN3luu+e0FyxYgAULFmTzx/eIu8nGxvoP8cbut/HG7red76tJvc1jqg6xsinxfGgQBQGaYeCL\nvc3eFzdE6K3lQPUOAImtO4FEm1VZFCAKIs7vZH9nScxun+/eYGfe3a02l6zny3mSeff3iykiOjjk\nRzrTBxKZd+pGG8lz3jEtBiHYAr3NrArXoEIUBei6gR/8YZ33yboIvS310rhZE83+2ccfVpPy+/mm\nImQO6Q8o7V7v9UTm3b+D9xGHmPUcIwf3zzX3RHRwYW/zDARBgF/s2Jc7ntzbXG6EIBjQWiogBJuh\nG6qzx7Wn8xoAASLu/8ZXsOaLCMaVj/Z878hJgzC2tgyVKTZc+NkNxyY6teWJMbVluPvq2RhY1r3g\nbQ+79/fg/c2zJuO8ligGlhVlfjARUQ8xeKfhDJsLqbt6JQ+bq4K5lE2PBiHpkpN5R2Kad/03zD2m\ny0sCWDTp/JQ/e0CaQJevexdXl3c/oLl7m/dnkigycBNRznDYPI3EUjEBmqF3+H6HLUFFa9vOmB/Q\nRWiGBkkUsLehvUPmDZ2nvavsXuH9PfMmIsolRpE03FXDWlKWDXirzQ3DgC5Za7jjPhi6BNWIQxIF\nGAY6ZN727mCUmZ1550vBGhFRLjCKpJEp845riYC8e387oJidwIy4HzBEqIaayBalpODP4N1lSp7M\neRMR5RKjSBruOW/N6Dzz/vmfNkFQYgCAUn8I0BKZNwAIHQrWGIi6yqdIkCUBRT6WZxAR2Ri803A3\nadFTDZu75rwjMRWCEoUiKvjhJbNRO6AUcT0OwT67ycPmev9fn91fyJKI755/GM4/aWxfHwoRUb/B\ndCYDM/NOVbCWCOiabkDyx1DmC6G02I/yYDF2RXRIkpW+JxesaflZNd5XDhle0deHQETUrzDzTkN3\nNWlJOeftWuet6ToMKWoOmQPwS+YabdGa6xaS57w1XjMREVH3MXin47RHFVLPebuGzXUhBggGQtbu\nXgEreAv2RiVi0rC51rHpCxERUVcxeKdhrxRLV7AW01QnOzdEs9K8WDG37fRbu3wJaTNvDpsTEVH3\nMXhnIKYpWPt8dyN+9Zy5B7kmmpXmQTt4S1ZmbQftDpk3h82JiKj7GLy7INWcN0QNb31sbt9pWMG7\nWDaDtzNszsybiIiygME7A90wUg6bu7umGZJZvNZh2FxUARgQ/OGkF2XmTURE3cfgnYFupMm8reBt\nGAYMKXnY3NoRTFQhDdoGsbjZ05iFTVqIiKgnGLwzMAwj5Zy3ORRuQDcMCLKdeZu7StnD5pBUSKX7\nAQCXTlqQk+MlIqKDH4N3BuaweYrMGwAkFf/e+SaU2q0AgGDSnLchqBCKWmDEFVQFB+bkeImI6ODH\n4J2BYaRYKmavAZdUPPnpaufu5DlvTYpADIRhREKQRc5zExFR72DwzkDXOxasybDntL33FyctFYvI\n9eY3wqEO+38TERF1F4N3BobRcT9vUU/MaeuRIud+RTSXgNnD5mFxHwBAiIYwpHgQJMOH+M4xEFiv\nRkREPcDgnYFhGNCT5rwF3cysBUmFEU0Eb8GKyvawuV1ULmh++CQfZukLoe4cl/2DJiKigxqDdwYp\nC9bsJiuSCgjmBPjo8Hzn285SMYtgPV4wmHITEVHPMXin8NH+TyH4zMYqqQrWjLgVjK3gbRgCSvUa\n5/uKKENxFagJujeYExER9QSDd5KWWCt++t5v4J/2CgAz8/7vnibPY1S79kxSIQgGYAiQRG9WHfKF\nnK9FnbuIERFR72HwTtIWbwcAp6hM1XTsaWjzPMauX7MzbxgCxKQzGfKVOF+LOnuZExFR72HwThLT\nY57bcVV35rVtumadNsnsXW4Gb++pLHUFbwHmELoB7+sQERF1B4N3koga8dyOxXVAMAvWoh8dgaJw\nLbR6c37bnXlLSeu/Qkpi2Dz5e0RERD3B4J2kPSl4x1XNybz15gEI7T0aajQAABCUqBO8haQzWepP\nBG8hKXgn3yYiIjoQ7NmZJBz3bt8Zs4bNDQMABLRH44Dqg6EqEAJt1lruFAVrimvOW2SwJiKi3sPM\nO0lY6zhsLgg6YJinqj1ilprr4WIIgTAEUYNhCB0CtGepGDNtIiLqRQzeSbyZt4G4pjtD4wAQjpql\n5kakGIJgQPBFAUPskHlLouR8bX+L5WpERNQbGLyTeDJvwUAsrgGCDlmUMLqmFLo5fg4jXJx4nCFA\nTMquJw+YABgCYtsmdPgeERFRTzB4JwnHXcFbVJ05b1EQ4VcS2bQeDSYel2LYPOQrwYTGi6DtGclh\ncyIi6lUM3knCqmvYXNSdanMR3uAN3fV1ig5rAKwit8SwORERUW9g8E4Sdi0VEyTVWectQoLf5w7Y\n7lPXMfMG4AyxC4zeRETUixi8k3gzbw2abkBwhs1dp8u9Q1iKOW8gEbyd2M2KNSIi6gUM3kncTVoE\n0W5ibgZvn2vY3NATpy7VUjEAmDKyEgBw2NiBnvs5BU5ERD3BJi1JYpqrt7lkB28doiBBkdJn3qnm\nvOfOHIZDhldgWHVJh+8RERF1F4N3kqh7Y5KkzFv2BG9vIE+VeYuCgBGDQx3uJyIi6gkGbxfDMBDX\n4s7txLC5DkmQkoK3O1innvMmIiLKBs55u8R11bttp5TIvCVBhCy5h8q9WXiqYfNkrFcjIqLewODt\nYs93S4JZmCaIGiCqEATAJ/o9mbe7YC3dsHk6zNGJiKgnGLxdolbwDohW9zRRg+Azq89LlNABF6wl\nG1xpvu7omrLeOWAiIipInPN2iVvFakVSEG1aCyCp5sYjAEqVEGQxdcGakWadd7KTZtSiOCBj+riB\nGR9LRESUDoO3i5N5C2aGLEgaBMXMvEt9pZCTsm33110ZNpclEXMOHdJ7B0xERAWJw+YuMavS3O8M\nm6vOsHmZrzT9UrE07VGJiIiyIWPw3rp1ay6Oo1+IWcPmPqMIgJV5W8Pm5f4yyHLP5ryJiIh6Q8bg\n/e1vfxsXXHABVq1ahXA4nOnhec0eNldgBm9zztvMvCuKyrwFa8jc25yIiCgbMs55P//88/jkk0/w\n4osvYuHChZg4cSLOPfdcTJ06NRfHl1N2gxZBV2BoIgRJBXwRGLqIkFKMqBRJ/URD5LA5ERHlTJfm\nvMePH4/rr78eS5YswdatW7F48WJcdNFF+O9//5vlw8stO/OGIQG6DLG4GWKgHdq+wVCUpA5rbhw2\nJyKiHMqYee/cuRN/+tOf8Je//AVjx47F1VdfjWOPPRYbN27EzTffjGeeeSYXx5kT9py3oUkwNAmC\nYt6v7hwHRfL2Nvd0WwOYeRMRUc5kDN4LFy7E17/+dfzhD3/AoEGDnPunTp2aceh8xYoVeP/99yEI\nApYuXep5/K5du/Cd73wH8XgckyZNwp133tmDt9E77A5rhi4CmnlqDEOAEQtAlrztURXZvc5b5Jw3\nERHlTMZh8zVr1mDkyJFO4H7iiSfQ1tYGALj99tvTPu/NN9/Etm3b8NRTT2H58uVYvny55/t33303\nLr/8cjz77LOQJAlffvllT95Hr7CXihmqBEO39u6OKwAESJLgqTZP7rbGYXMiIsqVjMH7e9/7Hurr\n653bkUgEt9xyS8YXXrduHebOnQsAGDNmDJqamtDa2goA0HUdb7/9Nk466SQAwLJly1BTU9OtN9Cb\n7DlvXXNl3roMSTSryd0BO3nZGIfNiYgoVzIG78bGRixatMi5fdlll6G5uTnjC9fX16OiosK5XVlZ\nibq6OgDA/v37UVxcjLvuugsXXHAB7r///u4ce6+z57x1VYSzFEyTnEDtnvNOzrwZvImIKFcyznnH\n43Fs3boVY8aMAQBs2rQJ8Xg8w7M6MgzD8/WePXuwaNEi1NbW4qqrrsLLL7+ME044Ie3zKyqCkGXp\ngH9uZ6qqQp7bwqfmMUqiD7D28jZ0CQFFQlVVCEUlifddFFDgXMIYAgYOKO7weoWiUN93b+I57Dme\nw97B89hzuTiHGYP39773PSxevBgtLS3QNA2VlZW49957M75wdXW1Z7h97969qKqqAgBUVFSgpqYG\nw4cPBwDMnj0bn376aafBu6GhPePPPBBVVSHU1bV47mtpN39GuN2A4Lf28tYlSKKAuroWxOJa4sGu\nixEYApoa2+EvwOQ71XmkA8Nz2HM8h72D57HnevscprsQyDhsPm3aNKxduxbPP/881q5dixdffLFL\nmfecOXOwdu1aAMDmzZtRXV2NkpISAIAsyxg2bJizTnzz5s0YNWpUV99L1tjV5mocTuYNXXIqy93z\n3JJnqRg7rBERUe5kzLxbW1vx5z//GQ0NDQDMYfRVq1bhtdde6/R5M2bMwOTJk7FgwQIIgoBly5Zh\n9erVCIVCmDdvHpYuXYolS5bAMAyMHz/eKV7rS1E9BlmUoaqAPedtqDJ8VtB2B2jJ9bVhCDBARESU\nGxmD9w033ICamhq89tpr+MpXvoLXX38dP/jBD7r04jfddJPn9oQJE5yvR4wYgSeeeOLAjjbL4loc\nPlFBXNUhfjEdyvCPEd5+CJSqjgMU7gI1QTAYvImIKGcyDptHo1HceeedqK2txa233orHHnsML774\nYi6OLeeiWgw+yYeYqkNRy1C291hA9UNJUSgnJbdKNRi+iYgoNzIG73g8jvb2dui6joaGBpSXl2P7\n9u25OLaci2kx+CQz81ZkCbpuBmR3NzWbuymLJAuoCAVydpxERFTYMg6bn3XWWXj66adx7rnn4rTT\nTkNlZSVGjBiRi2PLuZgeQ7lYigZVQzCgQNV0AHDmvN3cwfvsY0alDPBERETZkDF42wVngLmka9++\nfZg4cWLWDyzXDMNATIvDJ/kQ13Qosoj2iAogc+bNGW8iIsqljOmiu7vaoEGDMGnSJCeYH0xUXYUB\nwwzeqg6fLELVzcxbSbEVqOgJ3kRERLmTMfOeOHEifvKTn2D69OlQFMW5f/bs2Vk9sFyLWq1RFVGB\nqhlQZBGaZs15KykK1kT3rmIM30RElDsZg/eHH34IAHjrrbec+wRBOOiCt92gRbY28VZkyZnzTpV5\ne4fN9RwcIRERkSlj8H788cdzcRx9zt4ONBG8RSd4y3IiUFeVB1DXGPEOmzPzJiKiHMoYvC+88MKU\nc9wrV67MygH1leTM2yeLUK1hc9k1RL78yqMQi2t4+p9bnftYsEZERLnUpQ5rtng8jvXr1yMYDGb1\noPqCvZe3CHN+293HXHb1MZcl0dkaVI8UQQyEUSQX5fBIiYio0GUM3rNmzfLcnjNnDq688sqsHVBf\nienmsLmExLC5rUM3Nfs5Hx+B4NCdOPb4g2v+n4iI+reMwTu5m9quXbvw+eefZ+2A+krMybzNU+Ju\nzCKLqZbGGTCixZD3TIFPUlJ8n4iIKDsyBu9LLrnE+VoQBJSUlOC6667L6kH1BSd4GzIA3ZN5y510\nTzv4VrwTEVF/lzF4/+Mf/4Cu6xCtoq14PO5Z732wiOnu4B3zbEYipxk2JyIi6gsZo9LatWuxePFi\n5/ZFF12El156KasH1RfsgjUY5ilxr+2WUgybc3UYERH1lYzB+9FHH8WPf/xj5/bvfvc7PProo1k9\nqL4Qt9Z5Q7fmvBV3wVr6wfGDsVUsERH1bxmDt2EYCIVCzu2SkpKDMmBFtCgAQLCCtzvzdq/ztjHx\nJiKivpJxznvKlCm44YYbMGvWLBiGgVdffRVTpkzJxbHllB287cxbUdzrvDnnTURE/UfG4H3bbbdh\nzZo12LBhAwRBwJlnnolTTjklF8eWU1HVCt6anXm7C9YOvpEGIiLKXxmDdzgchqIouP322wEATzzx\nBMLhMIqLi7N+cLlkZ96GZgZt91KxUNDX4fFVZQEAQG3VwXUeiIio/8s4Hnzrrbeivr7euR2JRHDL\nLbdk9aD6gp1566oZvH2yiOVXHolLTjkEIwaHOjz+lCOH44KTx+HKMybl9DiJiIgyBu/GxkYsWrTI\nuX3ZZZehubk5qwfVFyJaBIqoQNPM24osYsiAYhx/WG3KxyuyhHkzh6XMyomIiLIpY/COx+PYujWx\ng9bGjRsRj8ezelB9IaJFEZD8iKvWHt6ddFUjIiLqSxnnvL/3ve9h8eLFaGlpga7rqKiowL333puL\nY8upqBpFQPYjxuBNRET9XMYINW3aNKxduxarVq3CkiVLUF1djWuuuSYXx5ZTyZm3z9UelYiIqD/J\nmHm/9957WL16NV544QXouo4f/ehHmD9/fi6OLWd0Q0dUi8Ev+xHXmHkTEVH/ljZC/eY3v8Fpp52G\nG2+8EZWVlVi1ahWGDx+O008//aDbmMTuax6Q/IjHzYo1Bm8iIuqv0mbeDz74IMaOHYs77rgDRx11\nFICDt4931FrjHZADaGPmTURE/Vza4P3yyy/jT3/6E5YtWwZd13H22WcflFXmABCx1nj7JbNgTRBS\n7yRGRETUH6RNL6uqqnDVVVdh7dq1WLFiBb744gvs3LkTV199NV555ZVcHmPWOZm3VbDmk6WDdpSB\niIjyX5fGhmfOnIm7774br776Kk444QT8/Oc/z/Zx5VRYjQCAWbCm6hwyJyKifu2AolRJSQkWLFiA\np59+OlvH0ye8mbfG4E1ERP0aoxSA9ngYAFAkB9DcHkdx4OCqpiciooMLgzeAdtUM3qLuRzSmoao8\n0MdHRERElB6DN4D2eDsAIBoxT0dVeVFfHg4REVGnGLwBtFmZd6SNwZuIiPo/Bm8kMu+WVvM2gzcR\nEfVnDN5IzHk3NZnd1TjnTURE/RmDN4C2eDsUUUFLmxm8K0L+Pj4iIiKi9Bi8YQ6bFytBRGPmpiQ+\nhduBEhFR/8XgDXPYPCgXIRLX4FNEiGyNSkRE/VjBB2/d0BFWIwgqRYjFNfiZdRMRUT9X8ME7rEZg\nwECxHEQ6NoGoAAAYmElEQVSUwZuIiPIAg7ddad6so6E5Cr+PwZuIiPq3gg/eMc3co3zL9jYYADNv\nIiLq9wo+eMd1M3gbunkqGLyJiKi/Y/DWVfMLBm8iIsoTDN5W5g3dDNo+peBPCRER9XMFH6ni1pw3\nDPNUBFiwRkRE/RyDtzPnbWfeDN5ERNS/MXhzzpuIiPIMg7cz583gTURE+YHBW/MOmzN4ExFRf5fV\n4L1ixQqcf/75WLBgATZs2JDyMffffz8WLlyYzcPolDNsbhWsiSI3JSEiov4ta8H7zTffxLZt2/DU\nU09h+fLlWL58eYfHbNmyBf/5z3+ydQhdkrxUTNP0PjwaIiKizLIWvNetW4e5c+cCAMaMGYOmpia0\ntrZ6HnP33XfjxhtvzNYhdEksqcOapht9eThEREQZZS1419fXo6KiwrldWVmJuro65/bq1asxa9Ys\n1NbWZusQukR1qs3NzLu4SOnDoyEiIspMztUPMoxERtvY2IjVq1fj0UcfxZ49e7r0/IqKIGS5d4vJ\nqqpCkD63bugi5h85Al89aTwkznsfkKqqUF8fQt7jOew5nsPewfPYc7k4h1kL3tXV1aivr3du7927\nF1VVVQCA9evXY//+/bjooosQi8XwxRdfYMWKFVi6dGna12toaO/V46uqCqGurgXN7ebrGrqEuTNq\nsH9fa4Znkpt9Hqn7eA57juewd/A89lxvn8N0FwJZGzafM2cO1q5dCwDYvHkzqqurUVJSAgA45ZRT\n8MILL+Dpp5/Gz372M0yePLnTwJ1NqqvaXBILfuUcERHlgaxl3jNmzMDkyZOxYMECCIKAZcuWYfXq\n1QiFQpg3b162fuwBi7matMgSh8uJiKj/y+qc90033eS5PWHChA6PGTp0KB5//PFsHkannI1JdAmy\nxMybiIj6v4KPVqquWg1aBBaqERFRXij44B3T4xAMs4qdmTcREeWDgo9WcSt4CwJboxIRUX5g8NZU\nCKw0JyKiPFLwESuuxwFDYqU5ERHlDQZvPW4tEyv4U0FERHmioCOWYRiIaXFAl1lpTkREeaOgg7dq\naDBgwGCDFiIiyiMFHbxjWsz8QpcgcdiciIjyREFHLDt4G5rEYXMiIsobhR28rb7mhsaCNSIiyh8F\nHbHszFtn5k1ERHmkwIM3M28iIso/BR2xYnoi82a1ORER5YvCDt4sWCMiojxU4MHb3stb5FIxIiLK\nGwUdsRLrvGXOeRMRUd4o6IjlLBXTRQ6bExFR3ijs4O3qsMaCNSIiyhcM3gCgSdzPm4iI8kZBR6zE\nsLkERSnoU0FERHmkoCOWe9hcYcEaERHliYKOWFFnqZgEHzNvIiLKEwUdseJWhzWDmTcREeWRgo5Y\nMVfmrchS3x4MERFRFxV08I46c94iFLmgTwUREeWRgo5Yqq5CggRAgI/Bm4iI8kRBRyzVUCEKMgAw\n8yYiorxR0BErrschwpzrZvAmIqJ8UdARK66pruDNgjUiIsoPBR28VUOFYJingJk3ERHli4KOWKqu\nQrAybxasERFRvijoiBXXVQgG57yJiCi/FGzEMgzDzLw5bE5ERHmmYCOWqqvmF8y8iYgozxRsxIpr\nVvDW7cyb1eZERJQfCjd423t5W8PmLFgjIqJ8UbARy868DZ1z3kRElF8KNmLFrMwbmghBACRR6NsD\nIiIi6qKCDd6qlXnrugBFFiEIDN5ERJQfCjZ423t5G5oIRSrY00BERHmoYKOWXbCmaQJ8CivNiYgo\nfxRu8LaHzTWBmTcREeWVgo1acd0O3iIUpWBPAxER5aGCjVpxa85bUwXIYsGeBiIiykMFG7Xcw+ay\nzEpzIiLKH4UbvK2CNV0TITHzJiKiPFKwUcvpbW6IkCVm3kRElD8KN3jbvc11ETKrzYmIKI8UbNSy\nm7TAENkalYiI8krBBm9nP29dhMTMm4iI8kjBRq2Ys6uYBJmZNxER5ZGCDd5x97A5C9aIiCiPyNl8\n8RUrVuD999+HIAhYunQppk6d6nxv/fr1eOCBByCKIkaNGoXly5dDzOGSrYgaNb/QJBasERFRXsla\n1HrzzTexbds2PPXUU1i+fDmWL1/u+f4dd9yBhx56CE8++STa2trw6quvZutQUgqrEQCAocssWCMi\norySteC9bt06zJ07FwAwZswYNDU1obW11fn+6tWrMXjwYABAZWUlGhoasnUoKUXiZvCGJjPzJiKi\nvJK1qFVfX4+KigrndmVlJerq6pzbJSUlAIC9e/fi9ddfx/HHH5+tQ0kprEYhQLCqzZl5ExFR/sjq\nnLebYRgd7tu3bx+uvvpqLFu2zBPoU6moCEKWe2/f7XA8Ap/kRzsElJYEUFUV6rXXLjQ8dz3Hc9hz\nPIe9g+ex53JxDrMWvKurq1FfX+/c3rt3L6qqqpzbra2tuPLKK3HDDTfgmGOOyfh6DQ3tvXp8YTUC\nBQoAIBqNo66upVdfv1BUVYV47nqI57DneA57B89jz/X2OUx3IZC1YfM5c+Zg7dq1AIDNmzejurra\nGSoHgLvvvhuXXHIJjjvuuGwdQqci8QgU0QcALFgjIqK8krXMe8aMGZg8eTIWLFgAQRCwbNkyrF69\nGqFQCMcccwyee+45bNu2Dc8++ywA4IwzzsD555+frcPpIKxGUSGXAgAL1oiIKK9kdc77pptu8tye\nMGGC8/WmTZuy+aM7FddVqLoKQTffPoM3EVHfevnlv+OEE07u0mN/8pP7ce65C1BTU5vlo+q/CjJq\nRa0GLbv2xgBw2JyIqC/t2vUl/va3tV1+/PXXf7egAzeQw2rz/iSimcHb0MzqdS4VIyLqOw88cA8+\n/HAzHn30N9B1HV9+uRO7dn2JBx/8Be66607U1e1FOBzG5ZdfhTlzjsV1112F73znFvzzn39HW1sr\nvvhiG3bu3IFvf/u7mD17jvO6qqpi+fIfdHj+J598hPvvvweiKGDKlGm49trrU95n/5zRo8di1aqn\n0NjYiOnTD8eTT/4v2tvbcd11N+Ldd9/Gyy//HbquY/bsObj11u+ipaUFd955G9ra2lBSUoI77vgf\nXH75Rfj9759AMBjEhg3v4cknV2LFih93+5wVZPCOWsEb9rB5DtuyEhH1Z0//Ywv+89HeXn3NmROq\ncd5JY9N+/4ILFmL16qdx2WVX4pFHHoaqxvGLX/wWDQ37MWvWUTj11DOwc+cO3H77EsyZc6znuXv3\n7sF99z2E9ev/jT//eZUneLe0NKd8/oMP3oebb16KsWPH4Uc/ugO7d+9KeV86W7duwRNPrIbP58O7\n776NX/zitxBFEeeddxauvfabeOKJxzFr1myce+4CPPXUSrzzzls47rgT8dpr/8L8+afgtddewbx5\nX+nROS3I4G33NTc08+0z8yYi6j8mTpwMAAiFSvHhh5uxZs1qCIKI5uamDo+dOvUwAObyZHcXz86e\n/8UX2zB27DgAwO2335n2vnTGjh0Hn89crRQIBHDddVdBkiQ0NjaisbERn3zyEa644hoAwPnnXwQA\nqKmpxW9/+0vMn38K3n33bXzjG1cf+IlxKczgrSU2JQFYsEZEZDvvpLGdZsm5oChmD46//vUlNDc3\n4+c//y2am5txxRULOzxWkhLNu5KbgaV7fqpNsFLdJwiJxE5V1Q7Ht3v3Ljz11Er87ncrEQwGsXDh\nedZrSTAM3fNaY8eOw759+/Dhh5sxatQY+P3+zk9CBgUZtSL2piR25s2CNSKiPiOKIjRN63B/Y2Mj\nhgypgSiKeOWVfyAejx/Q66Z7/siRo7B5s7ni6a677sR///t5yvuKi4uxb5/ZbGzjxvdTvn5FRQWC\nwSA+/vgj7N69G/F4HBMnTsLbb/8HAPDcc6vw4ot/AQCcdNI8PPDAPZg375QDeh+pFGTwthlx88qH\nmTcRUd8ZMWIUPv74Izz00P2e+0844ST8+9+v4vrrr0FRURGqq6vx6KO/6fLrpnv+9dffhJ/97P/D\nNdd8A6FQKUaOHJXyvjPPPAf3338vbr75egwcWNXh9ceNG4+ioiCuueZy/P3v/4ezzjoHP/zhD3Hu\nuRdg06YNuO66q/Dvf7+G448/EQBw8snzsHfvXhx++MyenTAAgpGq6Xg/1Jvt5uJaHNf89hnojdWA\nIeKWC6ZjwojOe6tTamyn2HM8hz3Hc9g7eB57rrNz+Pzza7B79y584xvfPKDXS6Ug57wVSYHeMNi5\nzcybiIiy6Z57/gdffrkTd911X6+8XkEG72SsNiciomy69dbbevX1CjLl1HXvTAEL1oiIKJ8UZPCO\nxr1VjRw2JyKifFKQUSvWIXgz8yYiovxRkME7OfOW2B6ViIjySEFGrWjc2/mGmTcRUd96+eW/H/Bz\n3nvvHTQ07M/C0fR/hRm8Y0mZN+e8iYj6zIFuCWp7/vk1BRu8C3KpWMdhc2beRER9xb0l6PnnX4gV\nK36IlpYWaJqGG264GWPHjsP//u/v8cor/4Qoipgz51hMnDgJr776Mj7//DP8z//ci8GDzd4dfbEN\n6OWXX+VsAxqLReD3F2VlG1A3Bm+w2pyIyLZ6y1/w7t6Nvfqa06sPxTljz0j7ffeWoL///W9x5JFH\n4//9v6/i888/w09+ch8efPAXePLJ/8Vzz70ESZLw3HOrMHPmURg7djy+851bnMAN9M02oOeff6Gz\nDejixVfiZz/7VVa2AXVj8AabtBAR9RcbN25AY2MD1q59AQAQjZobSZ1wwsm44YbFmDfvFMyfn35j\nj77YBrS5uTkn24C6FWTwrgz54ZNF6IYBVTMgCgzeREQAcM7YMzrNkrNNUWTceOPNmDJlquf+m276\nHrZt+y/+8Y+/4lvf+iZ+/es/pHz+wbwNqOfYe+2V8sghwyvw1IrT8fBNJ+DXN5/Q14dDRFTQ3FuC\nTpo0Bf/618sAgM8//wxPPvm/aG1txaOP/gYjRozEZZddiVCoDO3tbSm3Ej2YtwH1nLNefbU8Iksi\nBEHgfDcRUR9zbwn69a+fj507t2Px4itwzz3/g8MOm4GSkhI0NjbgyisX4dvfvhqTJ09BaWkZDjts\nBm677VZ89tlW57X6YhvQ+++/x9kGdOHChVnbBtStILcEBbj1XW/heew5nsOe4znsHTyPPZd8Druz\nDWjy66VSkHPeRERE2dbb24C6MXgTERFlQW9vA+rGCV8iIqI8w+BNRESUZxi8iYiI8gyDNxERUZ5h\n8CYiIsozDN5ERER5hsGbiIgozzB4ExER5Zm8aY9KREREJmbeREREeYbBm4iIKM8weBMREeUZBm8i\nIqI8w+BNRESUZxi8iYiI8kxB7ue9YsUKvP/++xAEAUuXLsXUqVP7+pD6tU8++QSLFy/GpZdeiosv\nvhi7du3CLbfcAk3TUFVVhR//+Mfw+XxYs2YN/vCHP0AURZx33nk499xz+/rQ+417770Xb7/9NlRV\nxTe/+U0ceuihPIcHIBwOY8mSJdi3bx+i0SgWL16MCRMm8Bx2UyQSwRlnnIHFixdj9uzZPI8H4I03\n3sD111+PcePGAQDGjx+PK664Ivfn0Cgwb7zxhnHVVVcZhmEYW7ZsMc4777w+PqL+ra2tzbj44ouN\n2267zXj88ccNwzCMJUuWGC+88IJhGIZx//33GytXrjTa2tqM+fPnG83NzUY4HDZOP/10o6GhoS8P\nvd9Yt26dccUVVxiGYRj79+83jj/+eJ7DA/T8888bv/71rw3DMIwdO3YY8+fP5znsgQceeMA455xz\njFWrVvE8HqD169cb3/rWtzz39cU5LLhh83Xr1mHu3LkAgDFjxqCpqQmtra19fFT9l8/nw29+8xtU\nV1c7973xxhs4+eSTAQAnnngi1q1bh/fffx+HHnooQqEQAoEAZsyYgXfeeaevDrtfmTlzJn7yk58A\nAEpLSxEOh3kOD9Bpp52GK6+8EgCwa9cuDBo0iOewm7Zu3YotW7bghBNOAMD/z72hL85hwQXv+vp6\nVFRUOLcrKytRV1fXh0fUv8myjEAg4LkvHA7D5/MBAAYMGIC6ujrU19ejsrLSeQzPa4IkSQgGgwCA\nZ599FscddxzPYTctWLAAN910E5YuXcpz2E333HMPlixZ4tzmeTxwW7ZswdVXX40LLrgAr7/+ep+c\nw4Kc83Yz2B22R9KdP57Xjv72t7/h2Wefxe9+9zvMnz/fuZ/nsOuefPJJfPjhh7j55ps954fnsGue\ne+45HHbYYRg2bFjK7/M8ZjZy5Ehcd911OPXUU7F9+3YsWrQImqY538/VOSy44F1dXY36+nrn9t69\ne1FVVdWHR5R/gsEgIpEIAoEA9uzZg+rq6pTn9bDDDuvDo+xfXn31VfzqV7/Cb3/7W4RCIZ7DA7Rp\n0yYMGDAAQ4YMwcSJE6FpGoqLi3kOD9DLL7+M7du34+WXX8bu3bvh8/n4t3iABg0ahNNOOw0AMHz4\ncAwcOBAbN27M+TksuGHzOXPmYO3atQCAzZs3o7q6GiUlJX18VPnl6KOPds7h//3f/+HYY4/FtGnT\nsHHjRjQ3N6OtrQ3vvPMOjjjiiD4+0v6hpaUF9957Lx5++GGUl5cD4Dk8UG+99RZ+97vfATCnvtrb\n23kOu+HBBx/EqlWr8PTTT+Pcc8/F4sWLeR4P0Jo1a/DII48AAOrq6rBv3z6cc845OT+HBbmr2H33\n3Ye33noLgiBg2bJlmDBhQl8fUr+1adMm3HPPPdi5cydkWcagQYNw3333YcmSJYhGo6ipqcFdd90F\nRVHw0ksv4ZFHHoEgCLj44otx5pln9vXh9wtPPfUUfvrTn2LUqFHOfXfffTduu+02nsMuikQi+P73\nv49du3YhEonguuuuw5QpU3DrrbfyHHbTT3/6U9TW1uKYY47heTwAra2tuOmmm9Dc3Ix4PI7rrrsO\nEydOzPk5LMjgTURElM8KbticiIgo3zF4ExER5RkGbyIiojzD4E1ERJRnGLyJiIjyTME1aSHKN/fe\ney82btyIaDSKDz74ANOnTwcAfO1rX8NXv/rVLr3Gr3/9a4wfP97pZ53KwoUL8fvf/x6SJPXGYXvs\n2bMHn332GWbPnt3rr01UiLhUjChP7NixAxdeeCH+9a9/9fWhHLA1a9Zg69atuPHGG/v6UIgOCsy8\nifLYT3/6U+zYsQNffvklbr31VkQiEdx3333w+XyIRCJYtmwZJk+ejCVLluDwww/H7Nmzcc011+CY\nY47Bhg0b0NbWhocffhiDBg3CIYccgs2bN+OXv/wlGhsbsXv3bmzbtg1HHnkkbr/9dkSjUdx6663Y\nuXMnBg8eDEmSMGfOHM8exW1tbfjud7+L5uZmqKqKE088EWeccQYefPBBGIaB8vJyXHTRRbjzzjux\nbds2tLW14YwzzsDll1+O1atX469//SsEQcCePXswevRorFixAoqi9OEZJuqfOOdNlOd27NiBxx57\nDFOmTEFjYyN+8IMf4LHHHsOiRYvw8MMPd3j81q1bcc4552DlypWYOHEiXnzxxQ6P+eCDD/DQQw/h\n2WefxerVq9HU1IQ1a9ZAVVU888wzuOOOO/D66693eN6///1vqKqKP/7xj3jyyScRDAZRW1uLs88+\nG2eeeSYuu+wyPPbYY6iursbjjz+OZ555Bs8//zw++ugjAMDGjRv///bu2CW1MIzj+NcONQQRQi3W\nYnBsjDoSBFKNOVaEo0M4REO4HGyrKQin5ob+gDBaoiVyECEipakhWkKkQKFoiERPd5DOzYxLlysX\njvw+4+F5X97tx/PyHh7S6TSHh4eUy2VP3jKI/A/qvEU8bmJiAp/PB8DQ0BC7u7u8vb3x8vLC4OBg\nW73f78c0TQACgQBPT09tNZZlYRgGhmHg9/t5fn7m5uaG6elpAIaHh7Esq23d1NQUe3t7bGxsMDc3\nx8rKCj09rT3CxcUFDw8PXF5eAlCr1bi/v3fXf4xPnZyc5O7uzp2TLCK/KbxFPO7ztbJt22xvbzMz\nM8P5+bk7zOOzrw/Svnv28l2N4zgtQfw1lKE5y/j4+JhiscjZ2RnLy8scHR211PT19bG+vs7CwkLL\n90wmg+M4fzyXiDTp2lyki1QqFUzTpNFocHp6Sq1W69jeY2NjFItFAKrVKldXV201uVyObDaLZVnY\ntk1/fz/VahWfz0e9XgeaXf3HVb3jOOzs7Ljd//X1Na+vr7y/v1MoFBgfH+/Y+UW6iTpvkS6SSCSI\nx+MEAgFWV1exbZuDg4OO7L20tEQ2myUWizE6Oko4HG7r0IPBIKlUiv39fQzDIBKJMDIyQjgcJplM\n0tvby9raGre3t8RiMRqNBvPz8+6o1FAoxObmJqVSCdM0iUQiHTm7SLfRr2Ii8iOPj48UCgWi0SiO\n47C4uMjW1pb73/m/ymQy5PN50ul0R/YT6WbqvEXkRwYGBjg5OXHnE8/OznYsuEXk76jzFhER8Rg9\nWBMREfEYhbeIiIjHKLxFREQ8RuEtIiLiMQpvERERj1F4i4iIeMwvRph4T/csGFUAAAAASUVORK5C\nYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXeAHMWZ/v/pNGlnc5S0ymmFUE6WEAgQ2UJkGxtjsMEG\nbDD+YnNwZ3PnH+fD2GcwnDFHMBmcwETLIiMJ5YByzqvd1eY0eTr9/uie7p7dlRACHQ7z/LM73dVV\n1dXd71NvqLcE0zRNcsghhxxyyOFjIH7eHcghhxxyyOHvAznCyCGHHHLI4ZiQI4wccsghhxyOCTnC\nyCGHHHLI4ZiQI4wccsghhxyOCTnCyCGHHHLI4ZiQI4wccviUWL16NXPmzDmmsg899BC33377Ce5R\nDjmcGOQII4dPjTPPPJNx48bR2dmZdfyiiy6ipqaGhoYGAO68805qamrYvHmzU6a2tpaamhrn99VX\nX81LL73k/H7kkUeYO3cukydP5vTTT+e2224DYN68eUyePJnJkydz0kknMX78eCZNmsTkyZN57LHH\nTuTt9glBEE5I2Rxy+FuC/Hl3IId/DFRXV7NgwQKuuuoqAHbt2kUqlcoSjoIgUFRUxAMPPMATTzyR\ndbwvvPLKK7zxxhs888wzVFdX09bWxvvvvw/AX/7yF6fc1VdfzcUXX8xll112Im7tnwa6riNJ0ufd\njRz+hpHTMHL4THDRRRfxyiuvOL9feeUVLrnkkl7lLrnkEnbu3MnatWs/ts4tW7Ywe/ZsqqurASgt\nLeWKK67os+zHJSx46KGHuPXWW7n99tuZPHky8+fP58CBAzz22GPMmjWLM844g+XLlzvlm5ubuemm\nm5gxYwbnnnsuL774onMulUpx5513Mn36dObNm5elMWWu/d73vsfMmTM566yzeO655z72XgG6u7u5\n8cYbmTlzJjNmzODGG2+kqanJOd/V1cW//uu/cuqppzJjxgxuvvlm59y7777LxRdfzJQpUzjnnHNY\nunQpYGl/K1asyBqHjEmsvr6empoaXnrpJc444wyuvfZaAG699VZmz57NtGnTuPrqq9mzZ0/Wvd97\n772ceeaZTJ06lauuuopUKsUNN9zACy+8kHU/8+fP57333jume8/h7wM5wsjhM8GECROIxWLs27cP\nwzB48803mT9/fi9BHggEuPHGG7n//vuPqc5XX32VJ554gi1btmAYxqfq46JFi7jkkktYu3YtY8aM\n4brrrsM0TT788EO+853vcNdddzllb7vtNvr378/SpUt58MEHuf/++1m5ciUAv/71r6mrq+O9997j\niSee4NVXX3WuM02TG2+8kTFjxrB06VKefvppnn32WZYtW/ax/TMMg8suu4zFixfzwQcfEAgEuPvu\nu53zt99+O6lUioULF7J8+XJHwG/atIk777yTO+64g3Xr1vH8888zYMCAI7bTU6Nbu3YtCxcudLS+\nOXPm8M4777B8+XJOOukkfvjDHzpl7733XrZt28Yf//hH1qxZw+23344oilx88cW89tprTrkdO3bQ\n3Nx8zL6dHP4+kCOMHD4zXHTRRbz66qssW7aMYcOGUVFR0We5L33pSxw+fJgPP/zwqPXNnz+fu+66\ni2XLlnH11Vcza9asT+WfmDp1KrNmzUIURc477zw6Ojr49re/jSRJXHDBBTQ0NBCNRjl8+DDr16/n\nhz/8IYqiUFNTwxVXXOEIxDfffJObbrqJ/Px8Kisrufrqq502Nm3aRGdnJzfddBOSJFFdXc0VV1zB\nggULPrZ/RUVFnH322fh8PkKhEDfccIOjiTU3N7N06VLuvvtuwuEwkiQxdepUAF566SUuv/xyZs6c\nCUBFRQVDhw49pjERBIFbbrmFQCCAz+cD4NJLLyUYDKIoCt/97nfZsWMH0WgU0zR5+eWX+fGPf0x5\neTmCIDBx4kQURWHu3LkcPHiQ2tpaAF577TUuuOACZDln9f5HQu5p5vCZYf78+Xzta1+jrq6Oiy66\n6IjlfD4f3/nOd3jwwQe57777jlrnvHnzmDdvHrqu8+677/KDH/yAsWPHcsopp3zi/pWWljr/BwIB\niouLndl2IBDANE1isRgtLS0UFhYSDAad8v3792fr1q2AJbyrqqqyzmXQ0NBAU1MT06dPByyNwzAM\npk2b9rH9SyaT3HPPPSxdupTu7m5M0yQej2OaJo2NjRQWFhIOh3td19jY+Klm8t57MQyD+++/n7fe\neouOjg4EQUAQBDo6Okin06TTaQYOHNirDp/Px/nnn8/rr7/Od7/7XRYsWMCvf/3r4+5TDn+byGkY\nOXxm6N+/PwMGDGDJkiWcc845Ry176aWXEolEeOedd46pbkmSOPfccxk9ejS7d+/+LLp7RFRUVNDV\n1UU8HneOHT582NGYysvLOXz4sHMuEwUG0K9fP6qrq1m9ejWrV69mzZo1rFu3jkceeeRj233yySc5\ncOAAL730EmvXrnV8AqZp0q9fP7q6uohGo72uq6qq4tChQ33WGQqFSCaTzu+WlpZeZbwmqjfeeIMP\nPviAZ555hrVr1/L+++87ZsXi4mL8fr+jRfTExRdfzOuvv86KFSsIBoNMmDDhY+85h78v5Agjh88U\n99xzD8888wyBQOCo5SRJ4uabb+bxxx8/YplXXnmFxYsXE4vFME2TxYsXs3fvXsaPH/9ZdzsLVVVV\nTJo0ifvvv590Os2OHTt46aWXmD9/PgDnn38+jz76KN3d3TQ2NvL88887144fP55wOMzjjz9OKpVC\n13V2797dyzHeF2KxGIFAgHA4TGdnZ9YMvby8nNNOO42f/OQndHd3o2maY666/PLLefnll1m5ciWm\nadLU1MS+ffsAqKmpYcGCBWiaxubNm3nrrbey2uzpY4rFYvh8PgoKCojH49x3330OoQiCwKWXXsq9\n995Lc3MzhmGwYcMGVFUFYOLEiQiCwL333ntUDTOHv1/kCCOHTw3vDHXgwIGMHTu2z3M9MW/ePCoq\nKnqF3mYQDod55JFHOPPMM5k2bRr33XcfP/nJT5g8efIR2/808NZz3333UVdXx6mnnsr3vvc9br31\nVsdHcPPNN9O/f3/mzp3L9ddfz8UXX+xcJ4oijzzyCDt27GDu3LnMmjWLu+66q0/NoCeuueYaEokE\nM2bM4Morr+xlZvrFL36BLMucf/75nHLKKTz77LOARVL33HMP99xzD1OmTOHrX/+6owHdeuut1NbW\nMn36dH7zm99w4YUXHvGewdIS+vXrx2mnnca8efOYNGlS1vk77riDUaNGcfnllzNjxgzuu+++LNK5\n+OKL2b17t0OuOfxjQTiRGyj927/9G4sWLaK0tJQ33nijzzI//elPWbJkCcFgkHvvvZcxY8acqO7k\nkEMOJxivvvoqL774Yq8Q2xz+MXBCNYxLL700a4FWTyxevJja2lrefvtt7r77bv7jP/7jRHYnhxxy\nOIFIJBL8/ve/58tf/vLn3ZUcThBOKGFMnTqVgoKCI55/7733HHV+woQJRCIRWltbT2SXcsghhxOA\npUuXMmvWLMrLy5k3b97n3Z0cThA+17DanuGJlZWVNDU1UVZW9jn2KocccvikmD17NuvXr/+8u5HD\nCcbn6vTuy32SS8yWQw455PC3ic9Vw6isrKSxsdH53djYeMTVwV6Yppkjlj6w+1AHtz2whLOmDeLW\nKycdsdyFP7BWLL/+y/m5cQSu+veFdMfSnDdzCN+9vO+1A5kxe+JHZ1NREvq/7N5x48ofLSAYUHjq\nrqOvifm/RmYsM3juJ+dRlO//VHXe98I6Fn1UR2VJiN/+6Ow+2/vpjbOYMLL8U7Xz94LMPT/6r3N5\n8vWtrNrayOCqfB66/cxPVe8JJ4yjBWHNnTuXF154gQsuuIANGzZQUFBwTOYoQRBoaYl8lt38u0V5\neb4zFoebrL+xeOqYxudwYzeK/I8TWe0di08Cv2KNQWdX4mOv37anBWFoyXH17/8S5eX5pFQD01T/\n5r+Vg3UdqGV5n6qOZMpaC2IYxhHvt+sYnu8/Glpbo6TTGgC6/ulyscEJJowf/OAHrFq1is7OTk4/\n/XRuueUWVFVFEAS+/OUvM2fOHBYvXszZZ59NMBjkZz/72Ynszj88kpkXwzi2SGlVM/6hCON4EfBZ\nn0EipX1s2cb2OGP/DgjDNE003cAwzL95jTyaUP9vGjpxKwj+ZqHrn+09n1DC+Lg8QQD//u//fiK7\n8HeFV5bsQ5FF5s0aclzXJ9M6AFqPl0TTDR59fSunnNyPiSNdDU7tMeMwTJPH39jG5FHlTKv5eNPg\nkbB6exMbdrdy/YUnIX4Ggmr7wQ4+3NjANy4Y84kJ7u3Vtai6wRdnDgFg4cqDxFMal80Z7pQJ+qw9\nIHYe6uRXf9rI9fPGkB/y9VlfY1u8z+PHCtM0eXrhDkZWFzF7fL9jvm7jnlbW7mzmG+ePQRSPPqYv\nLdpLSZGVB8swTVTNwKdIJFIaD7+yme64ypTR5Zw7bRCPvLaFc6cPomZwcVYde+q7+MvyA5w7fRDv\nravj+nljHGI9XpimydKGlQihbsy4Gz0ZswnDNE1eeGcXowYWMX1MZda1ndEUzyzcwVfPHkV5UZBn\n39zBkH4FnDahP0fC+l0trNja2Ov42wc/YEndCn404zaCspWRIJnWeGLBdubNHMLgqnyn7CtL9iHL\nIudOG8hv/7KNc6YPQhDg7dWH+OYXxyAKAk/+dTtnTBrAqIFFR+zLB+vr2X6gHd0wue6LJxHwSTy5\ncAszavpRVhTkpUV7+cYFYwgHlaOOoWGaPPfWTsYMLu41Ru3dSZ57aydfO2e0c+wXv19Pfsiq87Og\njlzywb8hvLH8AMBxE0bKJoyequfO2k7W7Wxh3c4WnrzTtWFqWna5uuYoq7Y1sWpbE9PuPH5b5yOv\nWUn65s8eStVnYO//799b0TcnDyth1snHLmQB/vC+tZdDhjBeXLQXIIswAn7rM0imdTbva2PBioNc\nOXdkVj2CYE1Qu2Kp47qHDJJpnQ83HebDTYc/EWE8+NImAM6ZNoiBFb0TEHrx15UHs9tUdXyKxKHm\nKFsPdADQHUtTkh9g4942Nu5ty3ovAH7+wkfohsmmvW0ArNhSzBmTq4+5v31hS9t2/rDzFQKjAyTW\nn+4cz2gY0YTK+x/V8/5H9VnCcEf7bl5eu5Y9eytpj2zmrmumsmhDA2xoOCJh7O7Yy0MLt2AmXOGv\n2xrGa3sXAlAfPcyIIiur73vr6li3s4Wt+9t5+DZ3hX3mmxxVXcjanS0E/DLLNh3GBEYMKCQUkFm9\nby8fRRfxmytvQBazReob+94iko7y7lsuIb+5+iCD+iusDz7HmlWDKY9PpaE1RuHivXz9vBqOhuaO\nBIs3NLB4Q0MvwvjtX7axo7YT5QN3/5JoQiWaUBH8cUw+/beYs0f8A8HRMHqYpOJHMLX01DAy139W\n+KyTCLR19y2sNUPrs62jte89F/Rnf+R9jYMsiUc890lgfMoxMY7R3OhFMq2zs30Pf2n4MwhW/6MJ\nFekomkrGrCnkdYKkEgpkz3y70xHePvgBunHs47Hy8DrrHymddTyaUDFMgx1te+hrHvzrDY9TL68H\nOUVje5y0mmnT4L9W3c/Le/6SVd4wDR5Y/yiBccuy6lM1g7Tumr/SejrrnFjYgjFkFRtatvTqQyRu\nXdfUHndqTKQ1UqqO/+RlyFUH2dDcO1/YmwfeY1nDqqx+mCbsi1i5vuSqg6iabtenO/3f3bG3z/e3\nuaNvDVc3dGqL30Cu3oUiZYt1sbCFwIQlJPx1fV77SZAjjL8RaJ+BQyp5BOdW0kMY3pdQ7aFhpNTP\nmjA+m3qCfstk1BntTRgN0UZuXfRvvFu7uNe5tOreX8/xTXvuvafZrOe4eMt8WsI4VptyR7KTrW07\nevUn84w/CZIpjf/Z8Bh7YzsR861913XDxMRECHXjG7GepfUre10nFjcSGLsSZeBOeloWf7n2N7y2\ndyEfNW/qdd2KhjV9Po+D3XZG3VS2hhRNqLy463We3v00UqmV+TcSt4R5d9p1Uguyagl9ezzEgg4a\nYo28V7skS1uOqwn3HvLbnf9VzeBQpN75HVOzha/cfy9SUSsv7sqO4gKIJlUEX5zD7VGHaFXNQNMM\nBNF+pj0GyUtOSO5zkyWRjnRH1m9w39E/7XqNB9Y/yrrmjb364TWJxpJu/S2JNkx/DKX/Pgrzss2p\nUqG1GDoldvWq75MiRxh/IzheQaRqOnsbupw6hGCEpsKlJDUrpfXuuk7W7Gy2CitJNh5yzRVb9reh\n6Qa76zqtvReSfQuj7liaw22xY+pPbVME5DRCsBvdMDnUHKUj0lvQ17VE+WhXizO7OhqKwn7kfnvZ\nl9rE4bYYLZ2uQFh52MrY+vq+N3td53Wm9iTDlGe89R47+Xn71BlN0dQedz7mhJo9O/44dMfTNLS6\nY9fXxKC2KdKLDH+8/B4e3vgknamurPtNqTrN8RY6U9kf//aWvexv7OxTgzkcdVOaC7Lb/2hCQ648\niFTSxO93vuwcb+uy3h2l2kojLxa0O+S7pXU7d3z4/9GWtASxZljvzKHmKA2tMZYcWM/zO17klT0L\n2LinlbSqs35XC3/e9i4dKYusDDF7DOtaoyypt7bHFQKWQFy5rQnTNNnZ7ppXBMW6LqNhSCWufyLz\n/bR0Jlm5y33HMwQJ1thn+g0QVd3nEjHakeyykXSUVdsbWbuj2Tm/uHYVgYlLSBbudrSvnYc62RV3\ntZGE5qaRB9hS56ac9457Q2uMpqSVHNLURYcwDhyO0NAa48N6a0vdPa3ZGoGmG/z10EL8J60ATJra\nrfdibeP6LHI50Nht/aOk8I34CKnEakslu3/Hg5wP428ExzNzBPjv59exYvNhbr9yIsm0hlxeRzJY\nx57O/QwMDONnz3/klA1OWsTjexYB5wHw4gd7WbDccgJ/4/yarFl3Bg3RRn76wROk9kzgkVvOR5Gl\nI/ZFNwx+8tQafKM2IxW1cChaw2//ZM0Ye9rI//2J1QB8/bzRnD7xyNuJAsgSyAP20KqH+NHjpVn1\ntdtCqMTf2+HoJYxkSifPY1ZJpjUK7JmYN6pMHrCLneHFrG5UmV41mcde30pDWxzdMJEqD9A+eAeH\nIoMZmJ9tO09oSVJ6iiJ/YdbxH/5mGZpu8ugPT0eRxSzCeLd2MZXBCn711OGse1IN913oSnXT2elu\n5BRPqfxy3aP0z6vi+5NvBGB72y4e2vxbzHSA6092d//LoDHu7gsu+JKOP+ZQpA653J1xx9UEISXI\n7f+7HDAQApZANVNB0jaJvrxnQZagTWgJuqIp7n56Dbphogzajmwnb3jwlbV8oWYgK3cdJDhpkdsH\nOTsqavPBRoKZva1MS3j+/t3dHPQvYX2LZ5ZtC92uWBowkIotwhBMKWvC9YclW/DXZNpyiVjVDKIe\noo157mOT9h7YCoJu6jz6149AzaToN2kOWxqYWNgKh4cBsC+6C7/hrm6PezSWw20xHv7ravwZ/7Os\ngt2VNTua8U9oQrSXnkiS1XBbd5IfP7GcoL3X1vvrGri8xnAIZdX2w6jFe61ZvpKmuSNOVbnCU9t+\nnzWeO2o7AAGl/16kEpf0eo778SCnYXzO2Nd1gOe2/4l46thnrpqh8dy2P7Gv6wArNlvCpqUraWkY\n9kee0JJEssIVvTNPj/qe0kDUeLnlCbZE1/Zq65FNTyHkdSH320c0cXRSy3y0UpE1o13ZvKJXmc5U\nF49vfg4hYKX7jnn6uKN9Ny9sf7GXXTxhRhBEE1Po3X5H0iKM4kBvwuiMuTPzbEI2Wd+yyTGneM1E\nctVBNCnBxpatmKbJwaYI3bE0CAa+wZaJ6EB3tlO5NdHGv3z4E/5r1f29+q77u5BKGhytRbXbEnwJ\nXtmzgEc2P9Wr33s79zv/d6W6s3xQjYkmYmqcxpgrCFoSrXadSZ7Y9Thi2DV3AFnaiOBLUhS2JNVH\nvJJVzjv7FgJxx8IiyKqjYfQPV2VdE1XjtEdSrs/D7wpNQVGpa445moFzXNJBcN9B7+wbyX0fssgC\nMIwonQdWEImriMXNCIodXSXoJNLudVnt2f/Xr36SSDRKV6o7q+8ZJOjEiBWgHh5i1eFzZ+OZb6on\npEIrIEA9NArINnF1x9IIfvf9y+qToCP4rHOCZCDJ7jsj5nVnXdNqa3sHuw+xtMPdbEzwWd93T7Oa\n1THrfTH17MndsEFH36PmWJAjjM8Z9617mJWH17KtfYdz7OOcxR81b2Jl41ruW/ewcywvoPQgjESW\nDd/7ASBlCzUx3ElajLBLX96rrbakJXxM1Z8l3PtCMmU77aJWyGRnuqNXmUc2PsWGls3I/Syh6J3d\n/3rD4yw/vIZaj50ZIClYH5Ep9m6/w+6fX8peKdyZ6uKJgw+gDLYitrwzUKmsntfrXubJLVYKbs0x\nSZkgWuW60910x9Ik7HvyCkJnKmpjY8tWDNMgriWIqO6+F2lVJ3DycnwjNtEcs4SLrhuI4Q4CE10b\nf0Z4uPW5Zo7udMTtu6RSn7DGLaJGHRu5LGY7pMViV6MQi5pZ3f2+p62ko1n1RGui3Xaqm/hGeyYP\nctrRMCQhW2TE1FiWJpcxKYFFBLGkmkUIercdLeSx6XuFqSBpDKrsOwrM0KN0HlxBJJ7GN3gbpiFg\nJKwFfwnNHXdBSXn+t+oeMP2byEowizAyGoZmaJiCjqkpmGlLqGa+F8GXIDB+qVufl0jsdvRua11O\nTHPvfWf3dnxDtmeNhXeMvO4OFY/PJeya0JDTNLZbdf732oeoVbdl9SMaV4lrvQnDuX+jhzVA+vQa\nRs4k9TeCtMdubpgm0lHWL/S0lYJlDoqnUwglCadMWnDr9M6SBEnD1F0hk0UmR4IufewCq2TGTyBb\nwiClZ9vldUPnUNQyUZmqJeD7cgKrRnY7qhSxZjaiYc1MbbOFZmh02U5Rb8QLQF2kAR0NufIQWtNg\nkmndIWKxwBLeJiYJLen2QUk5H3JXqtv5WCF7jBJq9nh5TRGdqS7HLNXs8T3s7NjDkNIqVN1ALGrO\nul4saEdvtcxyHzVvYkm9q5l1pSOItnYUnPIe2zy32ZnqpCJUTlLLJhzRQ27+UR9lnUNJUZTn46BH\ny0zvOxnfsC00x1qJhVWEvG5Ev0cwejSMuN3WndO+z71rHiCmxok6zl0zezYuq0SjKkKBdX6wOpO9\nyUNQ0IEgq5iaDyGvC/+Y1VltDanKp7ap94ZTzWtXoMbbePBn38McahKuGknzijUopSbJg1sYdOoP\nqV/zDIZQj2molM0cSPFJFqHse+9nbJ/4bbbVr2bfcxvJG1TE/ob1bB24ih/ffbfVgC5jpizzX6xp\nOy2rX8IUUygb0wy+fCxSyI9pxmnc+CeSnXWIqxNUnjEIfypE9+42/vT4U7znf5WioiLU+WEaP9iP\n5JMoP2UQgpzmwOL7GTD9GyhVm9jx65WEhxQTr+vGd9ZYmjavJdl5COQIReNLqDpjKIKSYv2GjTx6\n3wvsbtmHKIsMu2Yi+5/fSMXkcqLJycRUld2/XUf1haMJ2kQrKGnMZLYJSjDFLBPc8SJHGJ8Cj72+\nleqKMBd8YfDHll27o5n31tXx/740AZ/S2w+wfHMTYAmaX/xuPd+/YgKvr9iNIMCX5libSv158V40\n3aB0RG/BrWoGcbPLEXhr9zTQb6ibF0kIel4WMVvD8M4KAXx2qoy31rqmEUSjF2F8uLGBrQfa+fb8\nsTzxl+2I9uRTsGePqulKt39/YhVCqAsyoeO2SSKjYUTSroCIqjF+984u8kMKF8wcjOGLuqqwpIHm\n44/v70aVXVNLysgmjPakO1MT/AmSac1pSwy5kTc/enoRna3WjNtLCm2JLnYeyq7D6V86gW4YPPDi\nJmaMqaRRd/vRmewCe01aoydQYE/3Ps5ltkVOerZGkJkRNnXEeXbVe5AHp/afyYcNK+hOdRNIZZtw\nMuhIdlERKidhE7PW2h9faTOGp6+mJiPYBB4S84kF4hT4fM4MX2+vwIhZ791bm7YxavoUZzast1eC\npCEVtpHUUqRUnX2NbQiKyLLVUZIb57DCFFlt2DNpSSO1cY6nbQXDMKCznOSGOew2/OjmaLSG4Zj2\n7NdMB0husK6RShoRC1sZXFUAG9090zOonD2BZFOCC274Aet5lc51PhLNbQz68jTMhjkYUeh/6lxC\n47dgqDq7H11HwUjL1yQIAjvTFjGl2xMMu2ISgeIyzPc6efbFV2EwmLqMmbSEbv6IPIK+m5FKGoh0\nvUnj2+30O30cTWtWIPoGMmTObfgnLEZPpkltTlP3wQ5GXDeFSmEW3z3lbH665efW/dn3KfhS9t8E\nUkE7qbYEA848jeoL06jpA5TVnIevsgNl2Eb2PrmZxKgkgao2nv3lvdzz01/wh8hLpFIJ9JbhlExp\noWvPFnYWnsaqvRswdYNgZRhTlxAkHd+odSTXneU8Y/XQSIoGtmWZ4I4XOcI4TpimycptTbCt6ZgI\n4+FXLTPDtgMdWautM2iLxsgQxu66LhZtqGeJ8RSmIfIl7sU0TRassGznlw/pHVmk6gYJwRVcta0d\nxPpZAr60wE+XR033mgPAQLRnhULm5bZZ509LNxPIbJ8t6r0I46mFlhlt3qwh2StqbdVX8xBGXUsM\nqbKOjDFEsEkrE6FUH3UFRDQd4911ll3+zMnV2dqRqGHi463Vh5BKDuMbYR1P6T0JwzWHCZJGMq1b\nhCEYCEGXnLrT3UAZYKIM3OVWIBi8unwXYAn3LA0jneJwa5yt+9vZur+dorFNYKdC6vD4Cxq6XZ9A\nl+2c13TDGZ9Qqpq4v85x5m7Z105STyIBs8pO48OGFXSlIwiq3uOZ2fdo15mJiNOaBpNXHEf3R7F8\nVgJmOoAgW/dbKJYTV/bhl9IOqZu6jJkIY2oKCV8zb66qdWamelcpYoF1D3E1wZZ97cTVJAISopKZ\nHRhoug6IzkTE1BS7jky4qfXXMHCseYKoW85eUUQ3PSHO4U5KigVkScA0rUhVpWkcauVmRJsIu9Jd\n4ANT9REor8BXFCDVbOeSalzHzqUbQVNIR1Ko3d3IFbVWqpRoAUIgSaikADlcjq5ECBcNYMee/UiD\nsTSMZAgVe2coAAAgAElEQVQjGUIz66lb9Ti61g5SAtlfweDScvbu66BqwjzARPSlkM0COjtqCQ8p\nwlcUoN3YwMrdE62bMUFrGoSpg1RxyHr37O/QVxBEjp2BEVuBmNdN5PA6utd/CIKK2mmS6ugCOQ+l\nSCQiFqGhQrICvWUoRSfvo+n9j6gfGKH90DpKJvWjguHUbqsiMG4ZgqRbwQ32M9ZaqvEPitOuu+/j\n8SLnwzhO9Ey/cawwj7RAX8wWCM2dthAXrY/Jigyx0BrrnUBN1QxU0bXPIqnUtViC4sJThhIMecxT\nGR+GqBOY/D5SiW3zNi3C0DQDwzSz7bWSSxh7Ovez6NAypLI6lOEbaenwmEQE3YlL1/EIDUCyTUHW\nj8yqdOt8c9wN/Yx6/AD3b3jIcS5a11njJPgS+Ea4TtGeJikvYZAhDN1AGbTD0sJMwa7Hnvn540i2\ncDRVW+OwP24hrwtlwF6nuriazHqKpmchmtdG3p5w+xDXrdmdphvOhyx1DrTqtwV0bVPENhdKiIYP\nRZTpSnWRTGsIHvtzRdCacOzptBZ/ZQgDTYZ0yNIoMuYIu63/N/kmCgQr3Uta6XDMhugKgyrzMbqL\nEf0JyyeUaUtXQLPGIqHHiSbSlilJV/jSmSMITFxMYOJifKOsSKHAycvwjV6LGIoQmLgYuf9+EHQC\n45ZaPht/FLGgzbnu9usHMe2sJue3MmgngqTzSsPvqShREATQO8vpPjgAvbMcMa8bQdSJaDYpaz5E\nyeqff9RHjCnvJNVcx8hvT2XIWTcQrCjC0HR8Q2zbv01oAbEAIxFGEE1SZoqUaj3nYZWlgIDRVUb9\nwu0U10xi+BWXUX3haEzN4LSRYyw3V7iLqgofCAZjq/sxwJM4URAN3k09ab0XyUICYhCtfgSCaODz\nPBdBCFptRYtIdyTorHuHEdePY+T1c8grr0HrtFaoC5LGa13/a9Wn+UD1kxcIkz+8hGjzRiJ12yke\nV4nZWo2ZyEdtsFauC/6EO8nQFQrkEnxS376rT4IcYRwn+lrcdSS0JToITPwAsbAl67jXuS30cEQf\naM+2cze1xwETqayO2m7LKWwa7uPTdANN8URYSBqHbDuwTxadWSzgkJPgSzjmCsg4lU10w6S1K5nt\n2/BoGL/66H95cfdr+IZtQS49TG17b4FuNYBr/pJTiIWtGCnbqZjRMOwxaEm4dURSrkbRmMg2TWSE\nrVdLKPIX9iKMTMy/dY1KMq2hGabjP0jvH2t1K2Bf57H3aq22GcMmE9leTJZBQku6z05Oo/pcYsi0\na5om27QPneMZ56Smm05biYjlx8kQxsHGiDV+ukxKMygJFNOe7LSc3vZzUvQwd07/PmXBUtY1bSCu\nJhyflqnL6HEr/YMYtLQMQUkR0ssYUTSUAsEimqjQ7BCQqcvkh3yOWapN2YNUbI2RqfkcX1Pc7Ka2\nvc0aE122khp2WvZFQUkC1n2Zmg9Ts7UyOY3cf58zjqbmw+ioRK21Yk3rY41Zi+wyaEk24Q+a9jWW\nEURrGYDokzH0JHHDevam6sfUPUYSqRMpKCPKIsm2CPGG7KALQdIsTSqlOSlDkkYcVbfGNuy3828l\n8zBSOr5CGUFWad/QCKbA2LIa8keU0LlvFSm/pVH7NR9SuJrI7iRp22el2d+JEixGSDRhJMLEGyJE\nOzsQ82yys18fMxVET+mIPgnRL5FuDBJr2YnePJBgXn/UaJp4g/Vd63ER0zQZVjCUkmkVtGx/hdCA\nAqSgQn2T6tQHOBqGaYhgikzNP527Z97Za6w/KXImqeNEz7QaALXddeyvTzBr5HAUWcQwTdbvamWX\nsRTBl8I3YgO6foZTPsuM4o0aCUY4rLaR8XRs3NtMR0RFzG/HN2wLhzPWJU8UxKY9bZgFUTAES1BL\nGrV1liaiyFLWLNgRut64bDUAStIS8IZMXXO0F2HEEmqfC+0aOj3CWe5hOpE0MGTE/A4E0URrHYA4\nYK9DJJpmsGxLA+vbXHNQdyoKFPRpt3c0DJsAvzr6Mt7Zv4yo1sq2/W3UHe6iJXWY/V3uoilkjb31\n3fQvy0OQVYxYAUbMcjTkF+qk6tyxUA+NtGZyeKJNMuTWVerY8zNrVvwnrcAUVcsMI2mOZlPb1kZM\nsEjQSAVI+ZNEEylWbWtyxj/SKRMwcQiktjlKoFrDTPtZta2J4tJimuIttGh1jqCRuwdxoD7G2KKx\nLD68hHV1u0lmggt0mURHPkopSOWHEIuaEUSTZERm8YZ65GQZpilRl96NMtg1H+UXKZjddhK+4u3O\ne2dqCoYdAXQgtpe60IcgWOGaq3c0kd49Cd+kd0FWEfwJS3NTfZjpgKUlFbZhJj35izQFENC7SlGA\nDc2b2W1rST1hZhb3ZQhB9SGHFEJVFax78vcU1BTiZwiioZA+WINv8A4aBx4kujRG08OrkcVWAkWD\n7f6bCIL1jI2khKabTnRVVypCLKVRSMhJRGimglSeMYT6hS+jbA4QGlhAyhQp8hcy8NRx7PvrWjb+\n+SEQBUZdWU1Mq6by5MvZ/7s/ASZyno/h10wkXF1Dqm45+xc8R94IEX9pyF33YgduGKkgwUFhgv3C\n7HxoNUqgjGDxUECkIDKRwZcdpv4v2zE0A8HMY8DkGkYVj2Br/21IAZGSSVY+MkO1EwymLcKQ++2z\nfJP2+IV8PvJ9R89BdizIEcZxomfiPt3Q+fna/wFg+66vc+NFJ7N002GeXriD/hNbwQcYEgnPegBv\nDLXgIQwrB46LB1/5iEElJb3CYREM8kM+IvE0O+vbCFRGMOMFVtiepNFt57/xKyKm5PVhZMI0M06x\nUYihbqTSRkfAb2/Zh1LtWWVraxjrdmVrSQDN3RHIiJketnZB0jBVV8CbSVt9t4Xwxr1trGhfhNLf\nDQWNZKI5PGYYPVKElN/pjpMtZNvadRpbU0gFGnc89CEgIA/cgdLPQK0bgVK9B0HS2LCnlQ17mwlO\n0zE0BdNelCVlTFKOTV9xZtWCkrQCADIhlgemwIS3SeopVDsiTAzY8fSyiqiFaEtYZq3/+v1ylLGg\nNVdbZORP8r2H3wfVj2+MhmkIFuHrisfcZIKkYep5vLu2Dt+QBFIFNBa/j8+ORu3qMvn579ZTPcKA\nEvjj7j9jyvYs3RTRuotQALnM1czSCZln3twJgDK8nEhpI2JGjusy4aAbTpoFTcFI52OqCnKFu+pY\nDEb57RvbMUxLQImhCIEJS6wuaAoYMnpbP+sa2+8gdw90TJ5mMoxihhyyEBD4cuVNPL/5dacdof9u\n0HA0iAyJV593CmJBO4KskvxoNGd8fQ5rDlkmp6SQoPqrJxEQQ3SsPM1qt3wHcIARX/4acngHkhxm\nyJzbMFWLgMMnjySUCgHbCSlBQMNMhigcX05hjbvZ0ujpVwKQ5ytm0KUnOcdPH3cm+Wsklqo6pefN\n6hGZlMcPf/Rz/ufljwhOfTdraIecfROyJqLb79qgS6w6UzunYHRZ7QbS+YxWvkno238EQG0Yilbn\n46Sykfxpg/Xe5o8occcd8JsWKYghWwuzAz0CviMvuP0kyJmkjhM9NYxme/EUwOrtlkq/z07Z0RW3\nPhrTELPWA8Q0T+RSH07NDARRo7Y548j0HJd0CsLWiyLmdSGIJka0yI6McV9cRRYxPITRU8MwNcX5\nMDPn6hPZi9MyUVKxPhbvtXt8KkLPWO8+2hJN2THBdcfSjo9CShTjl3xE7Ygp12nXH1oH91lfdwQw\n7HmPTUKZ64yovZgv0yfJ7QOaYglYKZFdRpO5cJplMhF8KQZV5CMoKUxdol9xIaYukVCTpDSjlwYk\nqnl0pSOkdRVDsAlS87k+EZs0BVm1Z36C7SBOO/0XBNOZFRqpIL1gR1jV19rCV/aadATQ/Jjp7DUp\nGQIEMDqzd5zLmKRMtTdhmLZGYESzU5+bhuSkIDHV7Igv2bTqccYea11Ov9gpngpERhinuj8xmTVm\nEN+ddhWTS6xlzo3avqz7dbQ+XxLBH8eI53PV2aO5ft4Yrj9nalYf8hU3Q22GCH1DtyJIumvCsutD\ndgMA8ny2hpHuPe43zLOiP4JiftbxylAFXzt7FP9x7TR8PVwERrSIQRVh9/304AeXTWf2uH4Y8QKM\nuFun939ZEqgqDZHaPg0z7XdCrzd+uJbdj6+l6qxhDM0fglQ3BQyZorCP//z66VntZL6THGF8zujp\nw2iI9g4DzJCDKGUMltmE0Z3yOK+dUNc+nOJStiD0oiDfeoQZk4UeKbY+fiXtCEFJBiTVTQbo+BWs\n8yWhsPsh2W2kNDvssqvUmg3bGkYqbc+OPfDaoTMLpRw7dg+NwNR8Vqih2Nu0JR6YSZG/iC579ud1\nzJaGM05AeywywlfzuStae2hOVeHSrD44JKpbgtBn5qEKcfucq2GcNtZyHKKkGFQVRvAnMdMBqsvD\nln9BT6NqRtaCO61xMELamrZ/cOhDN4eRprjCTk4jD9yBGIw54ZboEoIvbZmQQvYCxcysug/CyNj0\nzXQAI9l3umovQQCOoAHQu0t7FBYI+qS+NQy7jxnzHcDs/jMYFHfNqpnV1hmU5RXafXClp5EMUxTO\nlqaFRjUPnH4PY0tr+GrNZUiiyPjhpVQXZffPebaagmlaa1YEAcx4mJrBxUiiyPQRgykLlCCLMpMq\nxnN6v9Pd6+PZAj4gWfdZErLfJ1l1vpOwL5R131747b1A8noQRnmwFJ8iMbgqH9MT7ZXeOx40n5OO\nRu8qRcKtt6a6wtJeDYnUllMY4B9ivZeeZyeJApUlIYxIKckNZzghv+ef/0V+/L8/Y9bsU7l50nVU\nYKXqLykIUBLOo+DwaaS2zbBuJWb199PuZZJBziR1nPDmBDJNk4Zo741aMoudRMkua4hZSe/aeoR+\nWoV7C1JB0qwgyZ4mKSAUykT72I7PVBCjuwQpvwMxvx2js5LX619yzgmBBMqAvWjNAx1tYES/MtbU\ntme1ldRS4AO1biT+0WudKKmkpiKIJnpnGXpbf3zDN3kcxiZyv/2YJugd5cjlDS7ZZcpoCromZt2n\noKQQ1TxSSYHSQDFN8WYQNQTbOW9qClXFhXR67tNLQJkP3Aq59Ttj6RdDCAh9ajkAQTGPLpoAN9QV\nXabAlw+m1a+64GKEtIqp+qgsCUKHhGqmSau6syq3Mj2eA7X9CA6tBaxEiMpAPP2zSF0qbUSusHwr\n7sI46/n5hm51H6pNGHpXH9sVO2s4BFJbZiGGO/HXrHVs8pBNGKmdUzBTHmJRA4wqqGFXtxUSbSbz\nrDDqPoRkpm+GZ0+Jr9RcxsPbNwMtWWOZQUAMuved6U86QFFpNokZpokiynxnwjezjs+pnoUoiAzM\nH8Db+5ew0dGIBNAUh6CMeIEVzAGIgsh/zPwXTNNEEiUi8TTPYq3OHhQeTO0u3VnAWJIXphsYUFrI\nLl1CUNKIpjVpKA4UAJZ14Cv9b+Cpv+5AGbKNEZXucwhIQSezzpSKCUiiO26Z8GC1YRh6mxU4kVnT\nlN45lSvPG41QVks0HUUUxKy8bFcN/xq/fnkzCVxLgCSJR9xPZu6g05g76DT7/oWsv/5UBUY0yuDO\neezYZ1kxPqudNXMaxnHCq2E89PJmlh305sI3eWt1LRv2WGaqhO7amJdsbOAHv1nGDx9exs5GTwqM\noxBG5pwv4LaZSbFQG7TTPkiuQDZsQZMJE21OWmTmnWlKth0YYOzAKlcjsDWE1qgdhaTLljARdeJJ\njZitTZi67NEi3BBOMRTB6C7FiJTY5/oQ1rpkCUw5TSaSRzaDpFSdPMme+fkTjoYRkP2UBSqsuPxQ\nt3WNrWHEYgJkZqGiq2GYhoBf8hOQ/S5ZeUgLICSFARPfiI1O/itTU5BECUH3I/iTHEpbfhwjUoxf\nkZBQMFDpSHXgG24984BZCAjEunrvlmZqiiOwM2QBONpeJmIo65oMKegKqe3TUBuGec65c7zivDyM\n7jLmyNeS2jrTLWObpExDdOzhXnxp6JepqL2MxPrTMZNhR/CqDUPRGgdz57Tvc8fU7znlja5S9O5i\nrj3pq9b9emar6r5xaI2DLU3UhALJ3r5W9RKGv1fK7SPt6xGQA5w9+HRqSkZy3divY6ZDVBRnk5Bp\nWuG2smffB1EQHeHtTTJZXZ6P0VnhaGahgFWmqiRkmQP9CcTCVoRUPpVhV7vpX1COmQ6S3jWF2cXn\nO8eDinsf3zz5qqy+OyHzpquBC4Jgj69AMmUwp3oWXxx2DuDuJW/1J8+a3HggScIxbUCW2aUv8zez\njsqnF4Lmd+r6LJDTMI4TXsLYcOgAgWKPM1jS+OP7vdMyI2pZi98+OlBL5vtyBGsfZqfMMcVnkAZS\n26chV1o+hiitWAI0EyapYCZsf4Q/AZjEtThiogitYTi+/vsxRZ3+VQqqItEFDC4rwUzYaQWCEaev\nAKP6l9AgKhiiRgroiEUhz1LfC0tLaAIKC0UKggaGqNEMmKmQa4bx234a2TaJ6bITIRKc/L6V7E0A\nn2l9GPFuxel7hoiCcoCg7MfszkMq6CAw5V3XBxLVPFFNacyEPV6GjF+WCEgBYpLttLZJZkBxMeWF\n5ZRVRjncsMddh4IrkAv9BXTq1jM1Yvmoh0YhjxbxC3kkpC7WR62QWb2rlCADgE6MVO+Pe9zgKjZu\nVjENwVmfotYPR++01kQU0p90ohgj6AkB9chSI1JKuTIQv5yiRavP0h5GDypi5dYmGpu1LDt55h6O\ntD+SLIkEfYqTjfULY6vYUduBblQwqDLfycT7pTMMJ6tvLDmKaVVDgGx7uJkOotZamQgQNQomlAP1\nWRqGqAcJBrJFzbFsJBXyy5wzbSBD+xVwsDHCYqx3yegu4dSThvQyczntiQLzZg2hrDDA5FHlmILA\nTsqI0UhC7GDK6GmcMq4fi1f5EPyWGfCU6slZRBgOevrr6eqXJp3G3sV7OXf4ab3a/ebYq3h++4uc\nVDwJIRxkUKU1+fnxNVN5fel+5kzMznA8fngZm/a2MbgqH38fPgZZFKksDvGFkyqtRcJHwNfOGYUg\nwFVnj7Lv3zpumCY3XXwym/a0UlHUhz/sOJAjjOOE1+mdlTAMPLmaTKSyekQ79YaXDARfAqmkCcGU\n0ZN+O1bd7B0JBc7MOUMY86aP5C9rTDd1sZy2BbLghiEaohUWKqvopo6oBQCBkeo57PIvJL9fC/u7\nLRNKcSjsONsy2TIzff32vAk8vHk1TVFLW+mIRyAPJg/vx1nDJnL3ync4eVQeqxvfdrprpv0Y3SWY\nhkigooVo/UhL+NtOVG+4rtLvAAB+wRK2sYgCPpswbNJSBL/luI8VIgZjDlmYhkB7JIUpZaKabHVe\n0jB1GZ8iEZQDCJLtRLcjdvoVFfLtOeN4v7Z3csSM9jG8tD/rmi3C0BqHgO5DlkTyKSNBA43GXkxV\nIb1zCsnBlmTuy+cwdXg1GzfWYkRKHOe+1jDcIc1pNRXUh8vYF3P74kSSYZkZ7vnWDDRjCk+/s5kV\nmvuuDanMZ+XWJg4191jImVmUeAT7gSQKWcJRkUW+deHYXuXOmzGoz+sD/iM4UL3OXc//kh7o5XQ1\njmEZkyAIzla5M06qpHHdaHZ27USrH8k3bh5z1GsvPc3Vyu78+jSW7/LxwPpHOGfw6cz4wjgg25x2\n/qjZ+D0+hnDQoyF5CCPPH+Cn59zUZ5tTKicwpXJCr+PV5WG+c8m4XscHVoT5169NOeI9SJKAKAp8\ne/5Y5k6p5r+eW9dnuZKCALdcNt75ndEwTNN6v6bVVByxjU+KnEnqOOENq80IIjFtxzlnVvKWHsY3\nzLPdo3fXrSorT5NsBDET1voAb8RGFjL1KVabhUHLFKE1WpFD/lEfWZu/2AIZBGQziOCP4x+zyuqj\nZs0mFXsmnyELsGbw6ApGKmjnWXKJK98fxC/50AXLfNSVtMgvpAQJyZaAbI67EWJg29AN2TJlKBGU\nYZusML+MIOuR7hogKFpj195qvZK+wdtRBlob+CiCD0US0Vtck1p6/1jSuyfRGU31zjBqL37zySJB\nOWiNn5J0MuTmS5Y5r6iPlOgZQd4vz90vORMlZJgmJZnNHsBe7CaSyKQf13rPeDOOVL3Ds/+y6X52\niizik+2oqGSIacp8x/4N1kxXEAQUScHXY0/monzL1JPZutbnmDhs34PZt1SWJfFTRc18UgeqpId6\nXXM8W9VeUHUuh1/194raOhaMLB7G7JbxTCw52TmWeV/KxGqK/IX4PTnegh5S/LTb6h4vvCY3fx/5\n546ETD6549nO9+PwT0UYa3c0H/POcQBbD7Q75ZfVr+LB9Y+x6NAyHlz/GCnVE29tE0Y6Ypt1MkK/\n15oETyimLTTDTV/AsGeUYjCWlSJEtGc8gqSBnKbbb4UaFoes8hlBKYatqCKvfdtPHoKiWnWCE32h\nkD0LvnnC9c6MxIgUIShppPI6pIJ2ezcw2UkpoAzZSjRtEUbYZxGGX/JxwEM+Vr9sG3pmEZG9JsBU\nA4693IuS5BiqJWvG2NSH5q0IPkvDiJSSPlhDet/J6C0DMboqSKV1d92Eo6VpmJpHwxCsPToE0UBr\nGkSRYtmqi/wFWe3oHRVkhG2VhzAymkNbV5Iqv0tahm3Gc/ercG1AWtMgpGgV+b48u+5K+xpXewCL\nMBz7sikwrGBYFqH0lagyg4BPptJj4y7Jt94Hvd0itUtGfLHP6yRJ+FSE8UmEF1ihtgGlp4bxyYWZ\npBl07l7/8QWPgJdf+hOplKvdavUj0CNFTAudC1imrAwET7boz5IvdP3Yd9b07rkuHmX/9Z5wNYwc\nYRw3uqIpHn51Cz96fNUxlVc1nfv+sMEp/7udf2ZXxx5e3P0auzr20Kl6NqXxxzFNHD+AQxR9PS8x\n21fR1OheJwSijrlletVkTi+4xDouq/g9+xP0K7EEXc9QSO8aiJ6z0YyJQBEUfPb+CacNmMWY0lFO\nkYxQcyN2rBfvi0MtJ51cUYdUbS0AG5jfD0mUmFHVW6XOxPR7yUnvLkatHU1Bno/UzsluWU1mgDrN\nEaz0CAkFCAhhZJto9KYh6K3V2e1lNAwlZa9lsO7Xp9gaBm5Kc729Etk28hb63N3xUtunkd7t9qsq\nz1Lj8+QQX5w5BIAR1YUUhcKOZpcJLvBubas1DsJI5KEeHENB8yn4MpEwqp/k5lmkdkzP6rsiiRT5\n8537KAhlayleQpgwIjtqKuCTGFLlRjBl9pEw4wXcPv7HzB14GtPHWPcxdog7K5dFkZKCQK/6jxVe\nshk+wHoX+9v5lE4aUuwIuglcSGr3RBRJ7mXGOnlYySdu9+knH0GNt1G37H94+GFrkezvfvcc3/rW\n17n22q/y5JOPAZBMJvmXf/k+3/jGV7nmmitZuHAhL730B1pbW7jllhu59VbLpKS39yO9/QsUBQp4\n+unf8q1vXcOBxffTtOnPAEwaWUY61sbzj/yEa6/9KtdddzUNDVagygsvPMM111zJN77xVR599DcA\n3HLLDezcaUWfdXV1csUV8wFYuPAv3HXXndxxx//jtttuIZFIcOut3+G6667mmmu+wtKl1t4oM0+u\npLtuHQcW/4qDSx5g6YLHiMfjXHHFRYSD1viVF4hcccX8oxKP6DFJfdb4p/FhJNVjZ3bg43eX09zw\nN9GfsNIhZGyits9h6thiNtmLuX1qCWmlHUHWkIUApcUSHYaAronki4VowMk1QbbssGZANcUj6U5b\nAk0IRrN24qosCvOf189gb9ce/njQTcAnKCrhoEI0oeLvQRhCOjOzFcj35dOWbGdcmWsHlkQBvbOc\nfMqJ2CGTGfIaXjSEfKmQiN6F6E+iNQxl9BlW7Pe4spOy9m8A+NcvzaIsXMi6Vh8v77MIRj00GjNW\nREF/H60NFWiNg5GrDmJqlvbgmizcmZR6aBR6exWhYWEUj3r+X9+awdqdLbyyxNK4fvSVmfxqxyLL\nxODJwqrIEtjpHjIRY0a8wJnRF/o9C6aS2TP/fnmVfHvcNQzKH0CRv5DZ4/tRWRwimlBRa2vQmgc6\ncfEZk9SVZ47gD+5eRUii4BAdgJnI1mgAZFnk4hEXsHhjA2r9CAKTJX51y2zASo8+sMJN5zBxRBnX\nnl/D03aW4IBP4tLThjFpZBmSKFKc73cWjQZ9fgRB4LovnsQVp49g+ZbDbD1g+UkkSeC8GYMYNbDo\nmKJweiLgk5EH7kAqaSSWH6BqqDWrrTIMXm1bReUpJqYJO9M6yiCVuLiLx3Z/gH+CvTo56OPt6Cre\n7rFf16SKcVw6Yt4R273pplvYu28vj//2BYJ+mTVrVlJXV8vjjz+LaZrcccdtbNy4gc7OdsrKyvnF\nLx6wxiIoMHWqyR//+Ht+/etHKSjIfg6yJHDZZV/m2muvJxJP88tf3M3y5Uu56eJZfLTgl3ztm9cx\ne/YcVFXFMAxWrlzO0qVLePzxZ/H5fEQivZOBWnDf5a1bN/Pss38kHA5jGAY/+9kvCYVCdHV1csMN\n32D27DlMGGDyx9YVjD3rFmKqzIzRhYRCISZPnsKm9av4+Y0z+fD9vzLo9LlI0pG1vIwyciI0jH8a\nwhCOsiFRX/i4zYISzqY1JigppEQxWo/V0uEwELdCJwdXF3KQ1Yj57ZSZZdZCsbi12tcvBtEAjZSb\nUVP2kySIqUvZ2VptDCjLo6L4JA7qk1m2qd6J9Mns+RASLPu8IsrcOP4b/O/WRkBFECxhmNbTjCwe\n7tTnUyQSKZPx+kWYFftY3vZ+VnsFvgIiiS7MtB+haYwznhUhd9Z77uAzKQ4UMaLKmtUWxz0fph1m\nmZlBqw3DEQJx1EOjUEb3bVM3U0HMVAhZErPiyPuV5hFQ3FTNAyvCBHblEVNSrjlQl/ErIrKc0T7S\nVuJDXXFsw7Loef378D9MKHcdwZXFlmDND1p+ogxZgDXmfkVi3PBS/uCJjpMlIYvo+oIii4SUIOoB\nq62AX3JCUHuGogIMKHeJLeCT8CkSowdZ2oNXQGSISpFFSgsDTsglWEQmCAKjBvbhwzkGeLUFr9lE\nsl7OvssAACAASURBVO9VEAQrd1MmlTnZ35/8KUI8JVEg6Lee2+rVq1izZjXf/OZVmKZJIpGkrq6W\n8eMn8pvfPMgjjzzEzJmzOeusU0kkbN9cH2q/KAqsW7ea3/3uOVKpJJFIhFEjRzJx4mS6OtuYPdva\nr0NRrDFcu3Y1X/zihfjspd35+fm96uyJadNmEA5b74xhGDz66ENs2LAeURRobW2ho6OdDRvWcdbc\ns9mWCoGaxh+w3rl58y7id797jtmz5/D22wu4444fH7Utx8T88cP5ifFPQxifFD0JwycqpD07waV0\n2xYqWInNArKPZIYwZBVREBDkTNK6MvoNHsRBczVSSSO0jUJHdXwOQTFIDEgZSQT7W/RLfuv1TuYh\n2NpFdbg/pw1w4+0VSeG2U77FB6/+GZ+S4ltf+CK/XW9F0fQXxnDR5AmUBIooCRQjmNZaDEEQuOak\nK1ENFcUjMH2ySCJl7fw3INhjNTBWfDxYkSU+z4KjYr8rdMaW1jC8aIjzO19xhWrGz+Bsh6r5SO+y\nzFmyLGaFFRrRAsRwt+MjUGSxVyoW78zdp0j4hSBxOe5ESpmqgiKJ+GSP2c7ug1fI5UtFdGtdWX6D\noyEv2HutBVhC0DsuVjtiVj/7Qk9C+Tj/gN/TRk9HcpZQFrPr9fb7k06eevVBkdAO1aAdquE/7zzz\niOUWbajn2Td30q8qn3/56iS+c7+Vb+rWG77gEPCngWmaXH31tcyff0mvc0888TwrVizj0UcfYteu\nzVxxxdVHrEfXNO6//xc8+eTzlJWV8+STj5FOW0EeR2oXeo+hJEnOam/rehfBoGuefeedN+ns7OSp\np15AFC0TUyqVdgg/U3OG/8eNm0Bj48/ZsOEjDMNg6NBhHA0nUsP4p/Fh6H1klz0avPtX17VEScaz\nBUU0kw7DdmIHFZ8zSxWVNGWFAYdUTE2hUClCNoIIgTiabpA0kgiGveLYFmppM+msp/BLfnyymJX+\n4bwhczllwIzenTVk0tu/wJTKCc5MMi/gY0TRUEoC1uwzbM/sg36JkBKksIfDt7TQ9jvIEiXB3qYT\nE3vmbkieaByyVro6foi+ftuJ5zIC0jtTVmSRoEf4pXZOJbVthpOCWpFEZ9V8xmneU9AGpCCC5K6+\nNhNhEARnbMFNV+FdxPSV6utJrj271/0eCflHIAxJElGU7D7JkpBFTn2h5wrcjyvvHfujOa5TPUyw\n4SP0+0Qi0z9ZErOIsCeZHStCoRDxuJuwc8aML7BgweskEta3aM3UO2htbcXv93POOefxla98jW3b\nttnX5xGL9Q56EdEQBCgoKCQej7No0XtO+YqKSj78cBH8/+3deXxU1f038M+9d2Yy2ReyEjBCEAWM\nAsomNMgiQcKSFKIsVm1Q3BGiCNIifUqr/YHlKTwqlmKlVV7Sal36M6htQUULYl0ALaKCYkggC4Ts\nyyz3PH/cmTuZbDMJmSQz+bxfr76aO3MzOXNk7ne+59zzPQCsVisaGxswdqz2d50T6FVV2he6pKRk\nHD+u/a133/1Xi7/jVFNTg+joGMiyjM8++wTFxdpNIddcMxbvvvsv2B03ljTUu9qakTELv/jFz5CZ\nOddjPzm/HIQEdf1/8z6TYXT0rozqJgHjnY8L9LBvLUqFMfkkyqqrAARr+0xD+7Y/69qh2Ft3CCkD\nTZgWNxifWBxbVzrWBBgagmBVamG122GxW2CUItEAINhkhFkxo1GthxxdqxW6C43HgCtM2HcmAWeh\nZQfNL/JOt994hT7xuiLnKrzz8WnccO1At3Puy74Sez76AZlt7A54X3Ya3vjwe/w4fTAMBoGYE8lI\n6+cakokLjcF31d9DrQ9zK2kAuLKv5uWTw4zux4C2O194iBGzr7sUj2w76Og7GUMHRmHssHiYTQom\npfXHZ9+U4e2PtbuvDAYZE0Yk4lRxNW64Vpvwbn6hHRAdjeLSH/RbcZ3lLNwDhpZhNL1gpQ2Kw/Uj\nB8JkkPGP/zQpid6GfpFmTB2djJIL9YgMNeHAl45V9HbV7ds/oF38w4KNuHHcJahtsKG8ugH9Isw4\nXlDh2N/EFfhWLRyJ4wUViPOwwKpp37dW7mHdbdfi0LESDLs0BufPu/YM6cqAMbh/BDLGDsS1l7d/\nf78zAzIoknv208kyFRERkUhLuxq33bYQ48Zdh3vvXY5Tp07h7rt/CkALKOvWbUBh4Wk8/fQWyLIE\ng8GIX/96AwBg7twsPPzwcsTGxmHLlm149JbROPhlMa4dMRBz5mTj1ltvRlJSfwwb5vp3//Of/x9s\n2vQ4duz4PYxGIzZs+A3GjZuAEye+wdKlt8JkMmL8+IlYtuxeLFq0BOvWPYp33nkL11wzps33MWPG\nTKxenYc777wVQ4ZcjpQUrXbZoEGDceutufjt//t/UIUEUTAE98y/1vE7N2LHjmcxffoMj/20cNpl\nMBpkZE1qPxPpjD4TMOwdDBhNh6TCgo1AowrVUacJySfRqDrKYjvmHAySAbPHXIG97wNh4SrGDkvA\nB582aIvpVAVGgwwDTIChEjZo6WqwYkY1AHOQAaHGYJxvOA85SCuJ7RwCmjUyDc99qU1sR5paHytN\nv9p1335yXBhyM1suakrqF4qlmcNbPO4UHR6E22+8Qj/eMOVBt+dzhs7FocNVsBamICjW/QP/iwmr\nUdFYqd+R5KSViwaCJDOcMz4hZgNuv9G9fUaDjBCzAXfPc90jP2RApB4wjI45jFszXGU0mo+qhAe5\nByfn4rembWotw5BlCbdmXI7PvynzKmBIkoRbZriX8zjwZTEaLPYWGYZzTD9nyhC3x785XYHf7NJq\nGzkvnsMujcGwSz3fOdS0nERrQ0uDkiIwKCmixW2YXRkwZEnCzVMv83hecJMMo6mLmcN47LENbsc5\nOQuRk7PQ7bH+/ZMxdux4/TguLhxlZdWYP/9mzJ9/s/74ZQOicNkAbUj1jjvuxh133N3i7w0YMBBb\ntmxr8fiSJbdhyZLb3B675JJL8ac/vaQfO1/vxhtn48YbXZP5kZFRePbZP7b6/mbOzMS/votCeVUj\nJqa51vwcOfI5rr9+GkJDPe9pERUW1O5n/WIwYLShpt41BilLEiRZdZQB1z54+hyGI8MwyAaYFCOC\nDWZUWbS7Jupt9XoZa5NRhkEKgiQJ2GXt22WoY4evIKOCEGOIXozQVurKDgaEuYJBRBsZRncINgQj\nvOpK1Kv1LdYGRAZFtJr9yJKMX123Fke/rcCfoN3RpLRykfNmYri5pkUcASDM6Brisl+Id5UfaSXD\naHUMv5PXMOeF2K4K/XZGp7aGl5rOJ3h67821ty6jPT0xJBXUZsDoMyPhF8dxyfrd7zbho48O4skn\nt/Rse9CXAkaTPbj3HzmDlIRwpCS2/MYuhMAbJ97BB7YPIIeNhloTjeo6K2CyA6pZ2zcZTSe9HUNS\njrUNEaYIVFq0Mc16W4Ne9MxkUGCStAuW3VAHBUBEkHaRM5sUfW2EWh8CUedaHxAb7PrW2XSSuic4\n747pSOXLaHMUgg1NbkFu5SLqaYiitQtM8zH60CYBo+l6ioSQJsX3HHNMVbUtV5o3L/zmrbYmwYG2\nA0bTi3dHq4h6muNoS2cDzcVoOiTV1MVkGH2B89+i84q1YsWqnmtMM30m1NubFK/Z+dZxvPHh962e\n91Hxp/jn6X2QjFZHZVSgqs4CSCqEKusZhr4VqbPOk+NiHmkKR621DjbVhjpbvV6O2qBISIrSAkGQ\no8rpgOh+UGQJ/WND9Q2Y1NpITBntWk0sSzIWXp6Nm4e2vBOkuzk3u4+NbGXvhHa43XrZygWvMxmG\ncyhh2mhtTsOstFzwd/nAKIQYQ/QsbdwQbf5mUFIrmVonr2FtTYK3J7RJIb6Oftt2ZkfhIZ3LGDob\ncDojMtQEk9G1SND576Z5JkbuenP39J0Mo9mQVFVdy2+ZAHDgzMf6zwYDYAdQWdsAqZ+AJBRXcT/F\nimnXDMC732jrAYyKI8NwLAYrrTsHi90CYXUsvpMkJEVG4kgFYIMFI+PSMOuydMxcriA4yIC3/q39\nK8m6ZjRmDnatvgaAHzW5lbYn/XTWMMwcl4Kkfh27JdKt5EJrAcPTraetPJ+SGI7fPTCp1QvnM3np\nsFhVRDjWMeRdcy++vXASI/pdgYUTbK0Oz3T2M9pehtHWIGjTINGZfQqeWpEOo6HjLX4mL73TmVRn\nBAcZsOme6/Ry448vG49Gq/2ib+vtK3qohFW7+mzAaG1hnipUFNWc0Y8jwhSUAaiu14afTIoBjfq2\nmlaMuiwW755wZhiOW9kM2sX0bK1294yzrpIkAcFG1wRs7ojF2i2pjv8Cd111Gw6e+Q+mX3pdr/1A\nGRTZbeWxtzxlGJ6+ZbeVgUQ0Wdg2Kj4NpxtO49p+Wplqc5M1b0GKCVc6VrV39Vh+cHs1mbz4wHcm\nYISYO/ex7apd1zoivEmpE4Mic/7CC66Pf++LGH0nYNjdO7+2WcAQQuBc3Xk02i0wS2FoEDVw7pVS\n3dAIAwBFcmyPaTVBMjZiQHyYPodhUrTnzAYtQJyt1Uo06HWOmq0JaLp+AQAujbgEl0a0Xk7a3zXN\nMFobjvC0uYs3m78YZAPuGrMEZWVtlWloX2djdHsXYW8+7ryAUlt6X7joSwGjWQH+2gYb7KoKxXFP\n/rbXv8TnZV/ANASIkhJQLGrgqAQAq2prGTDMtQgJUiDJzoDhvgjvrVPawh1nwDAbFZQ3qT/VlzS9\nM6q1DKM3jGl3tAKrU2hwOwHDizGFjlQhpb6hX4QZZRUNCA9ufYOonuTzgLF//348/vjjEEJg/vz5\nWLZsmdvzZ8+exerVq1FdXQ1VVZGXl4fJkyd3eTtau622tsGm1zb65OsyKLGOrVCFNrlrNDqW6jdZ\nawEAsJkgSY6tVx0BI8jgWEltcJ8QXnx9Gi6cicDll0QBF5IAoNUKr4HMLcNoLWB4uGh2xxDd0IFR\nmH1dCkYPbbmlaXuS+oUi+0eD9HpOD908Er/9y2GPv7d68Sh8W1jZar0o6tvumD0c//jPacydeGlP\nN6UFnwYMVVWxYcMG7Ny5E/Hx8ViwYAGmTZuG1FRX0btt27Zh1qxZWLhwIU6ePIk777wT+/bta+dV\nO6fVgFFvRUSIybUK3BEYZKF9iA3O4W5HUHBmGMFKCCwAqi01gKT9jp5hKO4BY2hiIpKHaIHiipjL\nsHrMciSHJnXZ+/IHngKGp3jQHd/BJUnCj9NTPZ/YijkTB+k/jxgUg4HxYThdWtPupOXll0TrQYao\nqZgIs77TYG/j0wHUo0ePIiUlBcnJyTAajcjMzMTevXvdzpEkCTU1WgmDqqoqJCQktPZSF635HAbg\nmvgur3Ls1OYIDJLzVliDM5Boj+sbGtm1eYoaa22TDEP7HXOTDCM18lL0D3Wt1gSAS8IHtJi/CHSe\n5jA8DUn1ghGrDvGz5hJ5zacZRklJCZKSXN+mExIS8MUXX7idc//99yM3NxcvvPACGhoa8Pzzz/uk\nLc3nMABXwCi+4Cho5pjA/vpULUypgKw4Aogji5BaCRh6kHEU12saMG4amtVr73jqTp7u/fc8h+Fn\nfehnzSXylk8DhjeTfvn5+Zg/fz5uv/12HD58GKtWrUJ+fr7H34uL81yDvqngkJYLuyRFQVxcOGzf\nOfZWcFz8nWXHTWatrr/z8eEpcTh9BLhueAr2lR2BMFmR0j8UZwAMHhCLuLhw1BtdpcFTkhIQE9yx\ndnZGR/uiu9VYXcG6aVunXjsQ+z45jauuSEBkWMv/PjdOuBRvHTyFMWlJ6BfZflG+1l6/pxgdE+hG\nk9Kj7ekNfdFbsC+6hk8DRmJiIs6cca1rKCkpQXy8e4XLV155Bc899xwAYOTIkWhsbER5eTliYtov\nxNbR2ycrK+tbPHa2rBplZdU478wwZOdeDY7yHxYLQs1G1Dke7xcWjGcfmoTvq7/HvjKguPw8RgyK\nxJkCQG1UUVZWjfoGV8mKxiqBsprO3ebpLWdhtd6sssJVkrppW5dMG4KbJg+Gpd6CsvqWCylzJg/G\nvOtSoFpsXr3H3tIXNpv276Wx0bt2+0Jv6YvegH3hcrGB06dzGGlpaSgoKEBRUREsFgvy8/Mxbdo0\nt3P69++PAwe0vRpPnjwJi8XiMVh0RmuT3s4hKYujLpHkGJISqgFCAHZhQ2iwUb9LyigbYTIqCHPs\n81BtrYXVsamSwbFwz9xk0tvQw7Wfeou27oKSJMljjaOeqIFERK3z6RVNURSsW7cOubm5EEJgwYIF\nSE1NxdatW5GWloYpU6Zg9erV+PnPf46dO3dClmX8z//8T5e2obSiHmFmY6tzGM7FexbHN0LnXVJQ\nZUBVYBM2hAUbcM6qfft1VkR17vNQY6lBiFFb2e2sJeVcuEcufW2tQfMd04gChc+/AqenpyM9Pd3t\nseXLl+s/p6am4qWXXmr+a13is2/K8NSrXyA4yICMMQNbPF9Tr627cGYYzklvqDIgZNiEHZEhJkj1\n2oK7CMd+FGGOIFFjrdVrSDkDhizJkCAhOaxv3TrbntZKmgeyhJgQnCqu7nCRRqLeLqDHTM5VarfL\n1jfaWpTDBoAaRwFC5/af+qS3UABVhk21Ys7ES1H+5ccoBRDpKCyoyApCDSGottSgzlYPo2xw26ti\n65QnfPiu/E9fyzCW3DAUybGhmOqopEsUKAI6YDTdx7uhlYBRGvI5vjwXAoujUrnzFlmoMoQqw6ra\ncGliBGKLJZSWa3tdOIWZQlFcp9WLuixqsNteFbLE+kBNdWdJ7d4gLFjbgpYo0AT0lc3WZKK7+Q5t\nMDagMeobbDv6vGsOo+mQlKrApmqRpNJShSDF5DY/Ue7YHQ8ALo/unasye4u+lmEQBaqADhhNM4wD\nXxa7PRfUpISPPofhzDCENofhvAOqylKtz184OSe+r44dgWmXuM/RkLu+lmEQBarAHpJqZx/vsHAJ\nztUBFpsKJeYslIhyCFUCIOlDUnbVjhpLLeIj3YvS3X3V7ThR+T0mJ/fe/St6C/YPUWAI6IBhs7e8\nldYpOFi4AobVDtOQIwAASXYEGVWGKlRcaKyAgEC0OdLt9weE98eA8P6+aHbAYYZBFBgCfEiq7QzD\nFOya09DnMJpy1IYqrdP22o4OiuraxvUhnMMgCgyBnWG0MSQlRxfjbJhrzwJLK3dQCUd5kMJqrbRJ\njJkBo7MYMIgCQ4BnGK0PSRmTvnc7Pnu+rsU5wqIVuztZqZ0bzYDRab1hRz0iuniBHTBayTDWLBmN\n/v3Cmj3qfp7JKGP2tcMAACcqTgHgkNTFemB+GtbfPqanm0FEFyGwh6RayTAGxIUhvCwIxU2315bd\nh6TSBvVDapwROAM02LXV4lHNJr2pY0Zd1rGtT4mo9wn4DMOQ9B2MqYfhzCIUWYLavCpcs4DRaLMj\nxuzaPlOChBCDd/sxEBEFqoAOGFa7DcaB38DQrxgwaHWjFEVCjbXG7TypWcCwWlW3gBFiCGa5DyLq\n8wL6KlituDZvks11SEkIhyJLqLI020xFsQNCm5iV6qKxaPplWikQx94WIUZmF0REAT2H0ShX6T9L\nQXVY/9MxsNitqLc1wCAZYLUCksGmD0n1D03E6uuX6xsfRZjC0FDfoO95QUTUlwV0hmEXrm0/JbN2\n62xFYyUAYHjUlbCVpGjPKTZAEgg3hbntkucsNhiscF8DIqLADhiSVf9ZCnIGjAoAQIw5ErBrq7kl\ng3ObVfeES5G051Vw6zQiosAOGHAFDNlchxMV3+O7ygIAQGxIDITq2C/aETCMzQOGrD1vd5Q5JyLq\nywJ6DkN1ZBjCrkAOq8T//Wyb/lxCWAx+lDYQh6q/ajPDMEjasV20XcSQiKivCOgMwxkw1NqIFs9F\nm6NwSZxjMZ7SesBw1o+KbLYXBhFRXxTgGYYNQgCiLhyIuOD2XHRQJIIUbRclSR+SMrqdM/+yOTAb\nzMhImdo9DSYi6sUCNmD885PTsAkLJFWB2hCqPy5Bwk1D58FsMMNs1AKEpGhzFAbHnIWT2WDG/Mvm\ndF+jiYh6sYAdknrpX98Cih2SasSg8MH64z8fl4f0AdcBAMwGxz6tbWQYRETkErABA3BkDnYFa3Mm\n64/FmGP0n4ONQY7zHHMYknuGQURELgE7JAUAUGxQHftaPDZ+FSobK2FSXFlEkJ5haENSzDCIiNoW\nwAFDhSSrUG1a1pAQEoeEEPcS284AoWcYSgB3BxHRRQrcISnHRDbUtoOAHjAcGYZz3QUREbUUsAFD\nMtcDAISl7TpQpmYZRfOV3kRE5BKwAUMO1kqYq3VtL7prPmfRfOEeERG5BG7ACNEChqhvvn+3iyIp\naLr5HjMMIqK2BdwV8u8ffo+TZ6ogBWu76rWXYUiSBKiKtoESmGEQEbUn4K6Qr3/4PQAgaLgNQpVw\n95yr2/8FVWbAICLyQsAOSUFSIUPB2GEJ7Z6mlzgH12EQEbUncAOGrEISXqzcVl1d0LyWFBERuQRu\nwJBUSN68PWYYRERe8XnA2L9/P2bOnImMjAxs37691XP27NmDzMxMzJkzBw8//HCX/F3JywxDbrJY\nj3MYRERt8+kVUlVVbNiwATt37kR8fDwWLFiAadOmITU1VT/nhx9+wI4dO/CXv/wFYWFhKC8v75o/\n7mWGkZoYjZNV2l4ZvK2WiKhtPs0wjh49ipSUFCQnJ8NoNCIzMxN79+51O+evf/0rFi9ejLAwbb1E\nTExMay/VcbI26e2Js2ItwAyDiKg9Pg0YJSUlSEpK0o8TEhJQWlrqds6pU6fw/fffY9GiRVi4cCE+\n+OCDrvnjkgrJi4BhNrgCBjMMIqK2ebxClpSUICGh/VtT2yKaLqNug91uR0FBAXbt2oUzZ85gyZIl\nyM/P1zOOzhGQZAFZ9RwwghRmGERE3vB4hZw/fz5GjRqFxYsXY8KECR168cTERJw5c0Y/LikpQXx8\nvNs5CQkJGDVqFGRZxoABAzBo0CCcOnUKV155ZbuvHRfX9gpuSFqgMshK++cBiC5yPZ8YH6Wt/vYz\nnt5jX8K+cGFfuLAvuobHgLFv3z7s2bMHv/vd77BhwwYsWbIE8+bN8yoDSEtLQ0FBAYqKihAXF4f8\n/Hxs3rzZ7Zzp06cjPz8fWVlZKC8vxw8//ICBAwd6fO2ysuq2n5RU7f+F3P55AFSLK0CcO1fj8e/2\nNnFx4R7fY1/BvnBhX7iwL1wuNnB6DBgmkwlZWVnIysrCZ599hry8PPz2t79FdnY27r33XvTr16/N\n31UUBevWrUNubi6EEFiwYAFSU1OxdetWpKWlYcqUKfjRj36Ef//738jMzISiKHjkkUcQGRnZ6Tck\nSYCQtYCheDPp3WRIioiI2ubVoH1RURF2796NN998ExMmTEBOTg4++ugjLF26FK+//nq7v5ueno70\n9HS3x5YvX+52vGbNGqxZs6aDTW9JCKFVn3VkGLIXGyIFGRgwiIi84fGKevfdd+Obb77BwoUL8eqr\nryI6OhoAMHr0aOzZs8fnDewIu6rNXUgdyDDMzDCIiLziMWDMmzcPM2bMgKK0vPi++eabPmlUZ9nt\njruy9AyjY3dJERFR2zyuw4iMjERdXZ1+XFVVhYMHD/q0UZ3lzDDgyDAMXgQMs6HtLVyJiMjFY8DY\nuHGj2x1RYWFh2Lhxo08b1Vl21XF3lCPDULyYw+CQFBGRdzwGDCGE29oEWZZht9t92qjOcs1haO1T\nOCRFRNRlPAaM0NBQHDlyRD8+cuQIQkJCfNqoztLnMJyT3l4EDO6BQUTkHY9jNqtWrcJ9992HIUOG\nAABOnDiBp556yucN6wzXkJQWOIQX5c0jTOEwK0EYkzjal00jIvJ7HgPGqFGjkJ+fj8OHD0MIgVGj\nRl3Uwjpf0ie99ZXenst8KLKCJ9N/6ZclQYiIupNXC/ciIyMxefJkX7flojVfh+HVFq0AgwURkRc8\nBozjx49j/fr1OH78OCwWi/74V1995dOGdYbdLgDFCslUrz2gBu4OtERE3c1jwPjFL36BFStW4Ikn\nnsCOHTuwa9cuhIaGdkfbOsyuCgSNOAjZrK0bEQwYRERdxuMV1WKxYMKECRBCID4+HitXruy6TY66\nmF1V9WABAAaVi/KIiLqKx4Ahy9opkZGROH78OC5cuICioiKfN6wzVNV9wyazLbaHWkJEFHg8Dkll\nZmbiwoULWLZsGRYtWgRVVVtUm+0tbM0CRlRocA+1hIgo8LQbMFRVxYQJExAdHY309HR8/PHHaGxs\nvMjtU33HbhcQqgxJVjFWmY/Z4y/t6SYREQWMdoekZFnGz372M/3YaDT22mABAFa7DZKsop+cjNsm\nj0NwEPfoJiLqKh7nMFJTU1FYWNgdbbloFrt2269RMvVwS4iIAo/Hr+Dl5eWYO3currnmGrcaUlu2\nbPFpwzqjwd4IADDKDBhERF3Nq0nvzMzM7mjLRWu0OTIM2djDLSEiCjweA0Z2dnZ3tKNLNNgbAAAm\nmSXLiYi6mseAsXz58lZrLfXGISmLygyDiMhXPAaMKVOm6D83NjbinXfeQWpqqk8b1VnOgBHEDIOI\nqMt1eEjqxz/+Me655x6fNehiOO+SMimc9CYi6modrs4nSVKvvc3Womp3SXGfbiKirtehOQwhBL7+\n+mtMmDDB5w3rjEbHbbVmA4sOEhF1tQ7NYSiKgtzcXIwcOdKnjeqsRlXbByPM2Dv3HCci8mcBdVtt\no3AGjN5bvoSIyF95nMNYtGgRKisr9eOKigosWbLEp43qrEahrcMIN/XODZ6IiPyZx4BRV1eHyMhI\n/TgqKgo1NTU+bVRnWVEPoUoIMXIOg4ioq3kMGKqqoq7OtYtdbW0t7Ha7TxvVWVbRANhMMBqUnm4K\nEVHA8TiHMXv2bOTm5mLRokUAgJdeeglz5871ecM6wyo1QNiCoCgtV6YTEdHF8Rgw7rrrLsTHx2Pf\nvn0QQmDhwoXIysrqjrZ1iF21Q5WsELZwGJUOLy8hIiIPvNphKDs7u9ffLVVcVwoAEFYTFAYMF7dU\nkAAAFDhJREFUIqIu5/HK+sADD6CiokI/vnDhAh588EGfNqoz3jq1FwBgP5/EDIOIyAc8XllPnz6N\nqKgo/Tg6OhoFBQU+bVRnlNaVQVKNUCviOYdBROQDHgOG3W53uyvKarXCYrH4tFGdUW9rgKwaIUGC\nIjNgEBF1NY8BY9KkSVi5ciU++eQTfPLJJ8jLy0N6errXf2D//v2YOXMmMjIysH379jbPe/vtt3HF\nFVfgv//9r9ev3VS9rQGSaoSiyK3u30FERBfH46R3Xl4efv/73+M3v/kNAK221Lhx47x6cVVVsWHD\nBuzcuRPx8fFYsGABpk2b1mI/jdraWrz44oudrlElhECDrQGKGgIDh6OIiHzCY4ZhNBpx//334+mn\nn8YNN9yAv//971i7dq1XL3706FGkpKQgOTkZRqMRmZmZ2Lt3b4vztmzZgjvvvBNGY+d2ymu0WyAg\nALsBBk54ExH5RLsZhs1mw759+/C3v/0Nhw8fhs1mw3PPPed1JlBSUoKkpCT9OCEhAV988YXbOV99\n9RWKi4sxefJk7NixoxNvwbWXN+xGZhhERD7S5tfxJ554Atdffz12796N2bNn4/3330dkZGSHho2E\nEB6ff/zxx7FmzRqvf6c1DTYtYAhmGEREPtNmhvHSSy9h1KhRWLZsGcaPHw8AHZ5MTkxMxJkzZ/Tj\nkpISxMfH68e1tbU4ceIEfvKTn0AIgXPnzuHee+/Ftm3bMGLEiHZfOy4uXP/5glSm/WA3IMhkcHuu\nL+hr77c97AsX9oUL+6JrtBkwPvzwQ/zv//4vNm7ciMrKSmRlZXW46GBaWhoKCgpQVFSEuLg45Ofn\nY/PmzfrzYWFhOHjwoH78k5/8BI8++iiGDx/u8bXLyqr1n8+eLwcA2K0KpGbPBbq4uPA+9X7bw75w\nYV+4sC9cLjZwtjl+ExERgSVLluDVV1/F008/jcrKSjQ0NGDJkiXYvXu3Vy+uKArWrVuH3NxczJ49\nG5mZmUhNTcXWrVvx7rvvtjhfkqTODUk5tmZVbQrnMIiIfEQSHbhCW61W/POf/8Rrr72GP/zhD75s\nl0dlZdUoqCrEy9/+HcNiLkP+9/+E5WQaBpmHY+1PrunRtnUnfntyYV+4sC9c2BcuF5theFV80Mlo\nNGLWrFmYNWvWRf3RrrL7m9fwQ9VpfFd5CoA26W2x9c69OoiI/J1f31IUFRTp/oDdgPOVDT3TGCKi\nAOfXASM2OMbtWNgNqG2w9VBriIgCm18HjBbsnVspTkREnvl1wLCr7vMVwm5AanJED7WGiCiwdWjS\nu7exqe7DT4umXIGJIwb0UGuIiAKbX2cYNuGeYQzpH4MQs1/HQCKiXsuvA0bzISmzSemhlhARBT6/\nDhjlNfVux2YTswsiIl/x64BRXe++5oIZBhGR7/h1wLA3m8MIMjJgEBH5SkAFDFlm4UEiIl/x64Ch\nwhUwbGXJPdgSIqLA59ezxKpQIVQZDZ9NBwSzCyIiX/LrgGEXNkDI2v+IiMin/PpKq0JlsCAi6iZ+\nfbUVsEOofv0WiIj8hl9fbbUMg3MXRETdwa/nMATsgDBg2ugBGDs8vqebQ0QU0Pw8YKiAKuOmqakw\nGrhoj4jIl/x6SEpI2qS3ovj12yAi8gt+faUVjrukZInzGEREvua3AUMVKiAJSP77FoiI/IrfXm2d\nu+1JXIdBRNQt/PZqa3NsniSBk91ERN3BbwOGs1Kt7L9vgYjIr/jt1VYfkmKGQUTULfw4YDgzDAYM\nIqLu4LcBwy60DINDUkRE3cNvr7ZWZ4YhMcMgIuoOfhswnHMYCgMGEVG38NuAYbE7h6QYMIiIuoP/\nBgybFQCgyAwYRETdwW8DRqMzYDDDICLqFn4ZMKpqLbDYtCEpAzMMIqJu4Zf7YSx57C0kDq4AYjnp\nTUTUXXyeYezfvx8zZ85ERkYGtm/f3uL5nTt3IjMzE/PmzcNPf/pTnD171qvXLausAwAYZL+MeURE\nfsenAUNVVWzYsAHPPfcc3nzzTeTn5+PkyZNu5wwfPhyvvvoq3njjDcyYMQMbN2707sUlFQADBhFR\nd/FpwDh69ChSUlKQnJwMo9GIzMxM7N271+2csWPHIigoCAAwcuRIlJSUePfishYwjAoDBhFRd/Bp\nwCgpKUFSUpJ+nJCQgNLS0jbPf+WVV5Cenu7di+sZBucwiIi6g0+/ngshvD73jTfewH//+1+88MIL\nXp0vSdprh4cEIy4uvFPtCxR9/f03xb5wYV+4sC+6hk8DRmJiIs6cOaMfl5SUID4+vsV5Bw4cwPbt\n2/Hiiy/CaDR69+KOISnVBpSVVXdJe/1RXFx4n37/TbEvXNgXLuwLl4sNnD4dkkpLS0NBQQGKiopg\nsViQn5+PadOmuZ1z7NgxrF+/Htu2bUN0dLT3L+4YkjJxDoOIqFv49GqrKArWrVuH3NxcCCGwYMEC\npKamYuvWrUhLS8OUKVOwadMm1NfX48EHH4QQAv3798czzzzj+cUl56Q35zCIiLqDz7+ep6ent5jI\nXr58uf7z888/36nXlWRnhuHlEBYREV0UvywNAgBwTHqbDAwYRETdwS8DRnR4EOcwiIi6mV8GDINB\n1u+SCvL2rioiIroo/hkwZFnPMIKYYRARdQv/DBgGSV+4Z2KGQUTULfwyYCiya0jKzElvIqJu4ZcB\nw6BI+pCUmRkGEVG38NOA0WQOw8A5DCKi7uCXAUNRZEiyCqHKMBm40puIqDv4ZcAwKjKg2AC7AqPB\nL98CEZHf8curraJIkIyNENYgBgwiom7il1dbWRGQDDYIaxAUxS/fAhGR3/HPq63SCAAQVhNkSerh\nxhAR9Q1+GTBUgzNgBPVwS4iI+g6/DBh2uR4AINkZMIiIuotfBgxVbgAAGEVwD7eEiKjv8MuAcVoc\nAQAEqRE93BIior7DLwNGg1QJ27n+CFHjeropRER9hl8GDABQq2IQEsSyIERE3cV/A0ZdOIIZMIiI\nuo1/BgwBiPowmE2sI0VE1F38MmAEl6cBQkFYMEubExF1F78MGOvmLsaVg2Iwd+Kgnm4KEVGf4ZeT\nAEMGRCHv5pE93Qwioj7FLzMMIiLqfgwYRETkFQYMIiLyCgMGERF5hQGDiIi8woBBREReYcAgIiKv\nMGAQEZFXGDCIiMgrDBhEROQVBgwiIvKKzwPG/v37MXPmTGRkZGD79u0tnrdYLFi5ciVmzJiBm2++\nGWfOnPF1k4iIqBN8GjBUVcWGDRvw3HPP4c0330R+fj5Onjzpds4rr7yCyMhI/OMf/8Btt92GTZs2\n+bJJRETUST4NGEePHkVKSgqSk5NhNBqRmZmJvXv3up2zd+9eZGdnAwAyMjJw8OBBXzaJiIg6yacB\no6SkBElJSfpxQkICSktL3c4pLS1FYmIiAEBRFERERKCiosKXzSIiok7wacAQQnT4HCEEJEnyVZOI\niKiTfLqBUmJiotskdklJCeLj41ucU1xcjISEBNjtdtTU1CAyMtLja8fFhXd5e/0V+8KFfeHCvnBh\nX3QNn2YYaWlpKCgoQFFRESwWC/Lz8zFt2jS3c6ZMmYLXXnsNAPD2229j/PjxvmwSERF1kiS8GTe6\nCPv378evf/1rCCGwYMECLFu2DFu3bkVaWhqmTJkCi8WCVatW4auvvkJUVBQ2b96MAQMG+LJJRETU\nCT4PGEREFBi40puIiLzCgEFERF5hwCAiIq/4XcDwVJsq0KxduxbXXXcd5syZoz9WWVmJ3NxcZGRk\nYOnSpaiurtaf+9WvfoUZM2Zg3rx5+Oqrr3qiyT5RXFyMW2+9FbNmzcKcOXPw5z//GUDf7AuLxYKc\nnBxkZWVhzpw5eOqppwAAhYWFuOmmm5CRkYG8vDzYbDb9/ECv16aqKrKzs3H33XcD6Lt9MXXqVMyd\nOxdZWVlYsGABgC7+jAg/YrfbxfTp00VhYaGwWCxi7ty54sSJEz3dLJ/6z3/+I44dOyZmz56tP7Zx\n40axfft2IYQQv//978WmTZuEEEK899574s477xRCCHH48GGRk5PT/Q32kdLSUnHs2DEhhBA1NTVi\nxowZ4sSJE32yL4QQoq6uTgghhM1mEzk5OeLw4cPiwQcfFHv27BFCCPHYY4+Jl156SQghxK5du8T6\n9euFEELk5+eLFStW9Eibfen5558XDz30kLjrrruEEKLP9sXUqVNFRUWF22Nd+RnxqwzDm9pUgeba\na69FRESE22NN629lZ2frfbB3715kZWUBAK6++mpUV1fj3Llz3dtgH4mLi8OwYcMAAKGhoUhNTUVJ\nSUmf7AsACA4OBqB9Y7bZbJAkCYcOHUJGRgYArS/+9a9/AQj8em3FxcV4//33kZOToz/20Ucf9cm+\nEEJAVVW3x7ryM+JXAcOb2lR9QXl5OWJjYwFoF9Ly8nIA7nW5AK1/SkpKeqSNvlRYWIjjx4/j6quv\nxvnz5/tkX6iqiqysLEycOBETJ07EwIEDERERAVnWPtKJiYn6+w30em2PP/44HnnkEb2k0IULFxAZ\nGdkn+0KSJCxduhTz58/Hyy+/DABd+hnxaWmQria4ZKRdrfVPoNXlqq2txfLly7F27VqEhoa2+f4C\nvS9kWcbrr7+Ompoa3HfffS22DQBc77d5X4gAqtf23nvvITY2FsOGDcOhQ4cAaO+v+XvuC30BALt3\n79aDQm5uLgYNGtSlnxG/Chje1KbqC/r164dz584hNjYWZWVliImJAaB9QyguLtbPKy4uDqj+sdls\nWL58OebNm4fp06cD6Lt94RQWFoYxY8bgyJEjqKqqgqqqkGXZ7f06+6Kj9dr8wWeffYZ9+/bh/fff\nR2NjI2pra/H444+jurq6z/UFoGUQABATE4Pp06fj6NGjXfoZ8ashKW9qUwWi5t8Epk6dildffRUA\n8Nprr+l9MG3aNLz++usAgMOHDyMiIkJPRQPB2rVrMWTIENx22236Y32xL8rLy/U7XRoaGnDw4EEM\nGTIE48aNw9tvvw3AvS+mTp0asPXa8vLy8N5772Hv3r3YvHkzxo0bhyeffLJP9kV9fT1qa2sBAHV1\ndfjwww8xdOjQLv2M+F1pkNZqUwWyhx56CIcOHUJFRQViY2PxwAMPYPr06XjwwQdx9uxZ9O/fH1u2\nbNEnxn/5y1/igw8+QHBwMJ544gmMGDGih99B1/j0009xyy23YOjQoZAkCZIkYeXKlbjqqquwYsWK\nPtUXX3/9NdasWQNVVaGqKmbNmoV77rkHp0+fRl5eHqqqqjBs2DBs2rQJRqOxz9Rr+/jjj/HHP/4R\nzz77bJ/si9OnT+P++++HJEmw2+2YM2cOli1bhoqKii77jPhdwCAiop7hV0NSRETUcxgwiIjIKwwY\nRETkFQYMIiLyCgMGERF5hQGDiIi8woBBfu2mm25CdnY2MjMzMWLECGRnZyM7Oxtr167t8Gvdcccd\nXpW7fvTRR3H48OHONLdDjh07hnfeecfnf4fIW1yHQQGhqKgICxYsaLf6qLNUhL94+eWXcfDgQWze\nvLmnm0IEwM9qSRF1xMGDB7Fp0yaMHDkSx44dw3333Yfy8nLs2rVL31BnzZo1GDt2LABg8uTJ2Llz\nJwYNGoTFixdj1KhR+Pzzz1FaWorZs2djxYoVAIDFixfj3nvvxaRJk7Bq1SqEhYXh5MmTKCkpwejR\no/HEE08A0GrzPPLII7hw4QIGDhwIu92OqVOn4uabb3Zr57lz5/DQQw/hwoULAIBJkybhjjvuwDPP\nPIO6ujpkZ2dj3LhxWLNmDT7//HNs3rwZ9fX1AIDly5cjPT0dBQUFWLx4MWbPno1PP/0UFosF69ev\nx+jRo7ulr6mPuJjNOoh6i8LCQjF+/Hi3xw4cOCCGDx8uvvjiC/2xppvLnDhxQlx//fX6cXp6uvju\nu++EEEIsWrRIPPTQQ0IIIaqqqsTYsWNFYWGh/twHH3wghBDi4YcfFrfccouwWq2isbFRzJw5Uxw6\ndEgIIcQ999wj/vCHPwghhDh9+rQYNWqU2L17d4u279ixQzz22GP6cVVVlRBCiL/+9a8iLy/Pre1Z\nWVni/PnzQgghiouLRXp6uqipqRE//PCDuPzyy0V+fr7+3q+//nphs9m870QiD5hhUEAbPHgwrrzy\nSv341KlT2Lp1K0pLS6EoCkpLS1FRUYGoqKgWv3vjjTcCAMLDwzFo0CAUFBQgOTm5xXk33HADDAbt\nozR8+HAUFBRg7NixOHToEH71q18BAAYMGKBnMs2NHDkSL774Ip588kmMGTMGkyZNavW8Tz/9FIWF\nhVi6dKlekFJRFJw+fRohISEIDg7GrFmzAAATJkyAoig4deoUUlNTve0uonYxYFBACw0NdTteuXIl\n1q9fj8mTJ0NVVVx11VVobGxs9XeDgoL0n2VZht1u79B53u6zcM011+C1117DgQMH8Le//Q07duzA\nCy+80OI8IQRGjBiBnTt3tniuoKCgxWOqqgbUXg/U8/xnBpDIA+HF/Rs1NTV6ddLdu3e3GQS6wtix\nY/Wy0kVFRfj4449bPa+wsBBhYWGYNWsW1qxZgy+//BKAtteFs4w5AIwePRonTpzAJ598oj929OhR\n/ef6+nrs2bMHgLZFKQCkpKR07ZuiPo0ZBgUMb75Nr127FsuWLUNSUhLGjRuH8PDwVn+/+Wu19Vx7\n561btw6rV69Gfn4+Bg8ejNGjR7v9PaeDBw/iz3/+MxRFgRACGzZsAABMnDgRf/rTn5CVlYXx48dj\nzZo1eOaZZ7Bp0yZUV1fDarVi4MCBePbZZwEAsbGx+Pbbb5GTkwOLxYLNmzdDURSPfULkLd5WS+Qj\njY2NMBqNkGUZJSUlyMnJwa5duzBw4MAu/1vOu6Q+/PDDLn9tIidmGEQ+8t133+HRRx+FEAKqqmLl\nypU+CRZE3YUZBhEReYWT3kRE5BUGDCIi8goDBhEReYUBg4iIvMKAQUREXmHAICIir/x/apbYj523\no60AAAAASUVORK5CYII=\n", "text/plain": [ - "" + "\u003cmatplotlib.figure.Figure at 0x7f97f1330850\u003e" ] }, "metadata": { "tags": [] - } + }, + "output_type": "display_data" } + ], + "source": [ + "def plot(train, test, label):\n", + " plt.title('MNIST model %s' % label)\n", + " plt.plot(train, label='train %s' % label)\n", + " plt.plot(test, label='test %s' % label)\n", + " plt.legend()\n", + " plt.xlabel('Training step')\n", + " plt.ylabel(label.capitalize())\n", + " plt.show()\n", + " \n", + "\n", + "with tf.Graph().as_default():\n", + " hp = tf.contrib.training.HParams(\n", + " learning_rate=0.05,\n", + " max_steps=tf.constant(500),\n", + " )\n", + " train_ds = setup_mnist_data(True, hp, 50)\n", + " test_ds = setup_mnist_data(False, hp, 1000)\n", + " tf_train = autograph.to_graph(train)\n", + " all_losses = tf_train(train_ds, test_ds, hp)\n", + "\n", + " with tf.Session() as sess:\n", + " sess.run(tf.global_variables_initializer())\n", + " (train_losses, test_losses, train_accuracies,\n", + " test_accuracies) = sess.run(all_losses)\n", + " \n", + " plot(train_losses, test_losses, 'loss')\n", + " plot(train_accuracies, test_accuracies, 'accuracy')" ] }, { + "cell_type": "markdown", "metadata": { - "id": "HNqUFL4deCsL", - "colab_type": "text" + "colab_type": "text", + "id": "HNqUFL4deCsL" }, - "cell_type": "markdown", "source": [ "# 4. Case study: building an RNN\n" ] }, { + "cell_type": "markdown", "metadata": { - "id": "YkC1k4HEQ7rw", - "colab_type": "text" + "colab_type": "text", + "id": "YkC1k4HEQ7rw" }, - "cell_type": "markdown", "source": [ "In this exercise we build and train a model similar to the RNNColorbot model that was used in the main Eager notebook. The model is adapted for converting and training in graph mode." ] }, { + "cell_type": "markdown", "metadata": { - "id": "7nkPDl5CTCNb", - "colab_type": "text" + "colab_type": "text", + "id": "7nkPDl5CTCNb" }, - "cell_type": "markdown", "source": [ "To get started, we load the colorbot dataset. The code is identical to that used in the other exercise and its details are unimportant." ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "A0uREmVXCQEw", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "A0uREmVXCQEw" }, - "cell_type": "code", + "outputs": [], "source": [ "def parse(line):\n", " \"\"\"Parses a line from the colors dataset.\n", @@ -1137,7 +1034,7 @@ " A tuple of three tensors (rgb, chars, length), of shapes: (batch_size, 3),\n", " (batch_size, max_sequence_length, 256) and respectively (batch_size).\n", " \"\"\"\n", - " items = tf.string_split([line], \",\").values\n", + " items = tf.string_split(tf.expand_dims(line, 0), \",\").values\n", " rgb = tf.string_to_number(items[1:], out_type=tf.float32) / 255.0\n", " color_name = items[0]\n", " chars = tf.one_hot(tf.decode_raw(color_name, tf.uint8), depth=256)\n", @@ -1169,23 +1066,21 @@ " dataset = dataset.repeat()\n", " if training:\n", " dataset = dataset.shuffle(buffer_size=3000)\n", - " dataset = dataset.padded_batch(batch_size, padded_shapes=([None], [None, None], []))\n", + " dataset = dataset.padded_batch(batch_size, padded_shapes=((None,), (None, None), ()))\n", " return dataset\n", "\n", "\n", "train_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/train.csv\"\n", "test_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/test.csv\"\n", "data_dir = \"tmp/rnn/data\"" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "waZ89t3DTUla", - "colab_type": "text" + "colab_type": "text", + "id": "waZ89t3DTUla" }, - "cell_type": "markdown", "source": [ "Next, we set up the RNNColobot model, which is very similar to the one we used in the main exercise.\n", "\n", @@ -1193,17 +1088,19 @@ ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "9v8AJouiC44V", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "9v8AJouiC44V" }, - "cell_type": "code", + "outputs": [], "source": [ "def model_components():\n", " lower_cell = tf.contrib.rnn.LSTMBlockCell(256)\n", @@ -1227,12 +1124,13 @@ " Returns:\n", " A Tensor of shape (max_sequence_length, batch_size, output_size).\n", " \"\"\"\n", - " hidden_outputs = []\n", - " autograph.utils.set_element_type(hidden_outputs, tf.float32)\n", + " hidden_outputs = tf.TensorArray(tf.float32, size=0, dynamic_size=True)\n", " state, output = cell.zero_state(batch_size, tf.float32)\n", + " initial_state_shape = state.shape\n", + " initial_output_shape = output.shape\n", " n = tf.shape(chars)[0]\n", " i = 0\n", - " while i < n:\n", + " while i \u003c n:\n", " ch = chars[i]\n", " cell_output, (state, output) = cell.call(ch, (state, output))\n", " hidden_outputs.append(cell_output)\n", @@ -1261,50 +1159,51 @@ " A Tensor of shape (batch_size, 3) - the model predictions.\n", " \"\"\"\n", " (chars, length) = inputs\n", - " chars_time_major = tf.transpose(chars, [1, 0, 2])\n", + " chars_time_major = tf.transpose(chars, (1, 0, 2))\n", " chars_time_major.set_shape((None, batch_size, 256))\n", "\n", " hidden_outputs = rnn_layer(chars_time_major, lower_cell, batch_size, training)\n", " final_outputs = rnn_layer(hidden_outputs, upper_cell, batch_size, training)\n", "\n", " # Grab just the end-of-sequence from each output.\n", - " indices = tf.stack([length - 1, range(batch_size)], axis=1)\n", + " indices = tf.stack((length - 1, range(batch_size)), axis=1)\n", " sequence_ends = tf.gather_nd(final_outputs, indices)\n", + " sequence_ends.set_shape((batch_size, 128))\n", " return relu_layer(sequence_ends)\n", "\n", "def loss_fn(labels, predictions):\n", " return tf.reduce_mean((predictions - labels) ** 2)" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "JjK4gXFvFsf4", - "colab_type": "text" + "colab_type": "text", + "id": "JjK4gXFvFsf4" }, - "cell_type": "markdown", "source": [ "The train and test functions are also similar to the ones used in the Eager notebook. Since the network requires a fixed batch size, we'll train in a single shot, rather than by epoch." ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "ZWQMExk0S6X6", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "ZWQMExk0S6X6" }, - "cell_type": "code", + "outputs": [], "source": [ "def train(optimizer, train_data, lower_cell, upper_cell, relu_layer, batch_size, num_steps):\n", " iterator = train_data.make_one_shot_iterator()\n", " step = 0\n", - " while step < num_steps:\n", + " while step \u003c num_steps:\n", " labels, chars, sequence_length = iterator.get_next()\n", " predictions = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, batch_size, training=True)\n", " loss = loss_fn(labels, predictions)\n", @@ -1319,7 +1218,7 @@ " total_loss = 0.0\n", " iterator = eval_data.make_one_shot_iterator()\n", " step = 0\n", - " while step < num_steps:\n", + " while step \u003c num_steps:\n", " labels, chars, sequence_length = iterator.get_next()\n", " predictions = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, batch_size, training=False)\n", " total_loss += loss_fn(labels, predictions)\n", @@ -1340,16 +1239,14 @@ " # Here, we create a no_op that will drive the execution of all other code in\n", " # this function. Autograph will add the necessary control dependencies.\n", " return tf.no_op()" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "iopcs5hXG2od", - "colab_type": "text" + "colab_type": "text", + "id": "iopcs5hXG2od" }, - "cell_type": "markdown", "source": [ "Finally, we add code to run inference on a single input, which we'll read from the input.\n", "\n", @@ -1357,17 +1254,19 @@ ] }, { + "cell_type": "code", + "execution_count": 0, "metadata": { - "id": "DyU0wnnAFEYj", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } - } + }, + "colab_type": "code", + "id": "DyU0wnnAFEYj" }, - "cell_type": "code", + "outputs": [], "source": [ "@autograph.do_not_convert(run_as=autograph.RunMode.PY_FUNC)\n", "def draw_prediction(color_name, pred):\n", @@ -1389,16 +1288,14 @@ " draw_prediction(color_name, pred)\n", " # Create an op that will drive the entire function.\n", " return tf.no_op()" - ], - "execution_count": 0, - "outputs": [] + ] }, { + "cell_type": "markdown", "metadata": { - "id": "Nt0Kv5OCHip0", - "colab_type": "text" + "colab_type": "text", + "id": "Nt0Kv5OCHip0" }, - "cell_type": "markdown", "source": [ "Finally, we put everything together.\n", "\n", @@ -1406,218 +1303,132 @@ ] }, { + "cell_type": "code", + "execution_count": 22, "metadata": { - "id": "-GmWa0GtYWdh", - "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, - "output_extras": [ - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {}, - {} - ], - "base_uri": "https://localhost:8080/", - "height": 668 + "height": 415 }, - "outputId": "61f4af1d-c81e-44db-9079-1a7b8ed8ce58", + "colab_type": "code", "executionInfo": { + "elapsed": 15536, "status": "ok", - "timestamp": 1522345877153, - "user_tz": 240, - "elapsed": 75500, + "timestamp": 1531750946373, "user": { - "displayName": "Dan Moldovan", - "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg", - "userId": "112023154726779574577" - } - } + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "-GmWa0GtYWdh", + "outputId": "2e7a9856-9809-43a3-8b43-3c8514ea43e9" }, - "cell_type": "code", - "source": [ - "def run_input_loop(sess, inference_ops, color_name_placeholder):\n", - " \"\"\"Helper function that reads from input and calls the inference ops in a loop.\"\"\"\n", - "\n", - " tb = widgets.TabBar([\"RNN Colorbot\"])\n", - " while True:\n", - " with tb.output_to(0):\n", - " try:\n", - " color_name = six.moves.input(\"Give me a color name (or press 'enter' to exit): \")\n", - " except (EOFError, KeyboardInterrupt):\n", - " break\n", - " if not color_name:\n", - " break\n", - " with tb.output_to(0):\n", - " tb.clear_tab()\n", - " sess.run(inference_ops, {color_name_placeholder: color_name})\n", - " plt.show()\n", - "\n", - "with tf.Graph().as_default():\n", - " # Read the data.\n", - " batch_size = 64\n", - " train_data = load_dataset(data_dir, train_url, batch_size)\n", - " eval_data = load_dataset(data_dir, test_url, 50, training=False)\n", - " \n", - " # Create the model components.\n", - " lower_cell, upper_cell, relu_layer = model_components()\n", - " # Create the helper placeholder for inference.\n", - " color_name_placeholder = tf.placeholder(tf.string, shape=())\n", - " \n", - " # Compile the train / test code.\n", - " tf_train_model = autograph.to_graph(train_model)\n", - " train_model_ops = tf_train_model(\n", - " train_data, eval_data, batch_size, lower_cell, upper_cell, relu_layer, train_steps=100)\n", - " \n", - " # Compile the inference code.\n", - " tf_inference = autograph.to_graph(inference)\n", - " inference_ops = tf_inference(color_name_placeholder, lower_cell, upper_cell, relu_layer)\n", - " \n", - " with tf.Session() as sess:\n", - " sess.run(tf.global_variables_initializer())\n", - " \n", - " # Run training and testing.\n", - " sess.run(train_model_ops)\n", - " \n", - " # Run the inference loop.\n", - " run_input_loop(sess, inference_ops, color_name_placeholder)" - ], - "execution_count": 0, "outputs": [ { + "name": "stdout", "output_type": "stream", "text": [ - "('Successfully downloaded', 'train.csv', 28010L, 'bytes.')\n", - "('Successfully downloaded', 'test.csv', 2414L, 'bytes.')\n", - "Step 0 train loss 0.37890616\n", - "Step 10 train loss 0.18515904\n", - "Step 20 train loss 0.0892782\n", - "Step 30 train loss 0.07883155\n", - "Step 40 train loss 0.08585831\n", - "Step 50 train loss 0.09302989\n", - "Step 60 train loss 0.089012615\n", - "Step 70 train loss 0.07275697\n", - "Step 80 train loss 0.06644974\n", - "Step 90 train loss 0.0854013\n", - "Test loss 0.13216865Colorbot is ready to generate colors!\n", - "\n", + "Test loss 0.138294\n", + "Colorbot is ready to generate colors!\n", "\n", "\n" - ], - "name": "stdout" + ] }, { - "output_type": "display_data", "data": { - "text/plain": [ - "" - ], "text/html": [ - "" + "\u003clink rel=stylesheet type=text/css href='/nbextensions/google.colab/tabbar.css'\u003e\u003c/link\u003e" + ], + "text/plain": [ + "\u003cIPython.core.display.HTML at 0x7f97ee42bb90\u003e" ] }, "metadata": { "tags": [ "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { - "text/plain": [ - "" - ], "text/html": [ - "" + "\u003cscript src='/nbextensions/google.colab/tabbar_main.min.js'\u003e\u003c/script\u003e" + ], + "text/plain": [ + "\u003cIPython.core.display.HTML at 0x7f97ee42be10\u003e" ] }, "metadata": { "tags": [ "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { - "text/plain": [ - "" - ], "text/html": [ - "

    " + "\u003cdiv id=\"id1\"\u003e\u003c/div\u003e" + ], + "text/plain": [ + "\u003cIPython.core.display.HTML at 0x7f97ee42bd90\u003e" ] }, "metadata": { "tags": [ "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"b102d936-3379-11e8-ac70-0242ac110002\"] = colab_lib.createTabBar({\"contentBorder\": [\"0px\"], \"borderColor\": [\"#a7a7a7\"], \"tabNames\": [\"RNN Colorbot\"], \"initialSelection\": 0, \"location\": \"top\", \"contentHeight\": [\"initial\"], \"elementId\": \"id1\"});\n", - "//# sourceURL=js_e223a56194" + "window[\"a6045494-8903-11e8-99f9-c8d3ffb5fbe0\"] = colab_lib.createTabBar({\"location\": \"top\", \"borderColor\": [\"#a7a7a7\"], \"initialSelection\": 0, \"elementId\": \"id1\", \"contentHeight\": [\"initial\"], \"contentBorder\": [\"0px\"], \"tabNames\": [\"RNN Colorbot\"]});\n", + "//# sourceURL=js_02f896cbda" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2ab810\u003e" ] }, "metadata": { "tags": [ "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"b103532a-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_b8c6a821fb" + "window[\"a6045495-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_7e8f9f77a0" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2ab710\u003e" ] }, "metadata": { "tags": [ "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"b105b28c-3379-11e8-ac70-0242ac110002\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_44805e254b" + "window[\"a6045496-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_5531553c2f" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2ab6d0\u003e" ] }, "metadata": { @@ -1625,17 +1436,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"b106197a-3379-11e8-ac70-0242ac110002\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_a63d3c6c47" + "window[\"a6045497-8903-11e8-99f9-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_d1f809ec17" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2ab990\u003e" ] }, "metadata": { @@ -1643,17 +1454,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"b1069f44-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"b106197a-3379-11e8-ac70-0242ac110002\"]);\n", - "//# sourceURL=js_7e203b8bce" + "window[\"a6045498-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a6045497-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_3a3123cadb" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2aba50\u003e" ] }, "metadata": { @@ -1661,17 +1472,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"b1070f38-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_d53293d4a7" + "window[\"a6045499-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_1a0e1f7d6f" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2ab890\u003e" ] }, "metadata": { @@ -1679,17 +1490,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c6d90d5c-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"b105b28c-3379-11e8-ac70-0242ac110002\"]);\n", - "//# sourceURL=js_3000dc2c05" + "window[\"a8e54762-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a6045496-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_6213539615" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2abad0\u003e" ] }, "metadata": { @@ -1697,17 +1508,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c6da872c-3379-11e8-ac70-0242ac110002\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_4136f669a3" + "window[\"a8e54763-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_0bd7f95c6e" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2ab950\u003e" ] }, "metadata": { @@ -1715,17 +1526,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c6dac868-3379-11e8-ac70-0242ac110002\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_2f70dd9aee" + "window[\"a8e54764-8903-11e8-99f9-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_215f004f6b" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2abb10\u003e" ] }, "metadata": { @@ -1733,17 +1544,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c6db07d8-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c6dac868-3379-11e8-ac70-0242ac110002\"]);\n", - "//# sourceURL=js_7226726048" + "window[\"a8e54765-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a8e54764-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_a06186c8ad" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2aba90\u003e" ] }, "metadata": { @@ -1751,17 +1562,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c6dcc6fe-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_72e7709865" + "window[\"a8e54766-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_383fbaae67" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ee2abc50\u003e" ] }, "metadata": { @@ -1769,14 +1580,14 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAAFZCAYAAADHDNdrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAB9JJREFUeJzt3E1Lle0ax+HTF4jeEAyMBhE0DawI\nwsCH0AIlaGBWNJBo0CDoA0TQhmDXuKAGDioiCA2KlEAlnl05FD9Co8BeaGCQoBDa2jPZsXt4Bvu/\n0+o4Rmvd1zW4rsmP84bFamo0Go0C4H/WvNYHAPhVCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKDy\nUxgeHq5Dhw7V4OBgPXz4sHp7e+vWrVt15cqVOnnyZN2/f78ajUbdvn27+vr6qqenp65du1YrKytV\nVfXhw4e6cOFC9fX1VV9fX01PT1dV1dzcXHV3d9eDBw/q+PHj9ccff9TExMRaXpWfWOtaHwD+zuvX\nr+vOnTs1MTFRbW1tdf78+dW16enpGh8fr/b29hobG6upqal6/Phxbdy4sS5evFgjIyM1NDRUly5d\nqv3799fw8HC9efOmTp8+XVNTU1VV9enTp2pubq5nz57V5ORk3bhxo44dO7ZW1+UnZkJl3Zudna2D\nBw9WR0dHbdiwoQYHB1fX9u7dW+3t7VVV9fLlyxocHKytW7dWa2trnTp1qp4/f16Li4s1MzNT586d\nq6qqXbt21YEDB1an1OXl5Tpx4kRVVe3Zs6fevXv3Yy/IL8OEyrr3+fPnamtrW/2+ffv21c//+Xxh\nYaHu3r1bjx49qqqqlZWVam9vr4WFhWo0GnXmzJnVvYuLi9XV1VVVVS0tLbVp06aqqmpubq6vX7/+\nX+/Dr0tQWfe2bNlSi4uLq98/fvz43X0dHR3V29tbQ0ND3zxfXl6ulpaWevLkSW3evPmbtbm5ufyB\n+W155Wfd6+zsrJmZmZqfn68vX77U2NjYd/cdOXKkxsfHa2lpqaqqRkdH6+nTp9Xa2lqHDx+u0dHR\nqqpaWlqqy5cv1/v373/YHfg9CCrrXmdnZw0MDNTAwECdPXu2enp6vrvv6NGj1dPTUwMDA9Xf318v\nXryo7u7uqqq6evVqzc7OVn9/fw0MDNTOnTtrx44dP/Ia/Aaa/B8qP4NGo1FNTU1VVfXq1au6efPm\nX06qsFZMqKx78/Pz1dXVVW/fvq1Go1GTk5O1b9++tT4W/BcTKj+FkZGRunfvXjU1NdXu3bvr+vXr\ntW3btrU+FnxDUAFCvPIDhAgqQMi6+WH/kX8eXesjAPytf/3jz79cM6EChAgqQIigAoQIKkCIoAKE\nCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQI\nKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgq\nQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpA\niKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCI\noAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIig\nAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAC\nhAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKE\nCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQI\nKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgq\nQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpA\niKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCI\noAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIig\nAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAC\nhAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKE\nCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQI\nKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgq\nQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpA\niKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkBI\nU6PRaKz1IQB+BSZUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBB\nBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAkH8D1Aj8lNhhe7QAAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAAENCAYAAAD60Fs2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACL9JREFUeJzt3F+IlXUex/Gv2ziiBRGVOQaFd2JBzOg5aiH+IZGoJgmM\n/uhVGIlgFE0QEYHdFQaRGBJ10VX0D5TAi8jKomGmILsYjEAkmBwbRIxKGDV/e7G7w8ouux9jd911\nX6+rcx6e85zveS7e/J7zb0ZrrRVA4A8XewDgf4dgADHBAGKCAcQEA4gJBhATDC6Kp59+urrdbt13\n3301OjpaK1euvNgjERCMS9yaNWtqeHj4Yo9xnq+++qqGh4frs88+q7fffruqqmbMmHGRpyIhGPxH\n/fbbb/XDDz/U9ddfX7NmzbrY43CBBOMS9tRTT9XExERt2bKlBgYG6vXXX69vvvmm7r///up0OrV+\n/foaHR2d3n/Tpk318ssv1wMPPFADAwP18MMP18mTJ6uq6vTp0zU0NFRLly6tTqdTGzZsqBMnTlRV\n1eTkZG3ZsqWWLl1a69atq3feeWf6mDt37qxt27bV0NBQLVmypN5777169tln6+DBgzUwMFA7d+78\nm7kPHz5cmzZtqk6nU3fffXft37+/qqrGx8er0+lM7/fMM8/UrbfeOn1/aGio3nzzzX/tSeR8jUva\n6tWr2/DwcGuttWPHjrVut9sOHDjQWmvtiy++aN1ut504caK11trGjRvb2rVr2/fff9+mpqbaxo0b\n244dO1prrb311lvt0UcfbVNTU+3cuXNtbGys/fLLL6211h566KG2ffv2dvr06Xbo0KG2bNmy6ed8\n5ZVX2k033dQ++uij1lprU1NT7f33328PPvjg9IwjIyNt5cqVrbXWzpw509auXdt2797dzpw504aH\nh1t/f387cuTI9OsZGxtrrbW2bt26dvvtt7fDhw+31lpbtWpVO3To0L/rVNJas8L4P9D+/HOhvXv3\n1qpVq2rFihVVVbV8+fK6+eab69NPP53e9957760bbrihent764477qhDhw5VVVVPT0+dPHmyjhw5\nUjNmzKhFixbV5ZdfXseOHauvv/66nnzyyZo5c2YtXLiwNmzYUHv27Jk+Zn9/f61Zs6aqqnp7e//h\nrAcPHqxTp07VI488Uj09PbVs2bJavXp1ffDBB1VVtWTJkhodHa3jx49XVdW6devqyy+/rPHx8fr1\n119r4cKF/6Kzxt/Tc7EH4D/n6NGjtW/fvvr444+r6k8hOXv2bC1fvnx6n2uuuWb69uzZs+vUqVNV\nVXXPPffUsWPH6oknnqiff/65BgcH6/HHH6/Jycm68sora/bs2dOPmz9/fo2NjU3fnzdvXjzj5ORk\n9fX1nbdt/vz5NTk5WVVVnU6n9u/fX9ddd111u93qdru1Z8+e6u3trcWLF1/A2eD3EIxL3F9/+tDX\n11fr16+v7du3X/Bxenp6auvWrbV169Y6evRobd68uRYsWFC33XZb/fTTT3Xq1KmaM2dOVVVNTEzU\n3Llz/+4M/8zcuXNrYmLivG1Hjx6tBQsWVFVVt9utF198sfr6+qrT6dTAwEA999xz1dvbW91u94Jf\nFxfGJckl7tprr63x8fGqqhocHKz9+/fX559/XufOnaupqakaHR2tH3/88Z8eZ2RkpL777rs6d+5c\nzZkzp3p6euqyyy6refPmVX9/f7300kt1+vTp+vbbb+vdd9+twcHB3zXvLbfcUnPmzKnXXnutzp49\nWyMjI/XJJ5/UnXfeWVVVN954Y82aNav27t1bnU6nrrjiirr66qvrww8/PO8NUf49BOMSt3nz5tq1\na1d1u93at29f7dq1q3bv3l3Lly+v1atX1xtvvDH9Hsc/WgkcP368tm3bVosXL6677rqrli5dOh2F\nHTt21Pj4eK1YsaK2bdtWjz322HmXORdi5syZ9eqrr9aBAwdq2bJl9fzzz9cLL7wwvcKo+tMq46qr\nrpq+1PlLKBYtWvS7npPcjNb8gQ6QscIAYoIBxAQDiAkGEPuv/R7GxN7+iz0C/F/rG/z6b7ZZYQAx\nwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQE\nA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMM\nICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCA\nmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBi\nggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJ\nBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYY\nQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAA\nMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHE\nBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhAT\nDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiAkGEBMMICYYQEww\ngJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOICQYQEwwgJhhATDCAmGAAMcEA\nYoIBxAQDiAkGEBMMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBiggHEBAOI\nCQYQEwwgNqO11i72EMD/BisMICYYQEwwgJhgADHBAGKCAcQEA4gJBhATDCAmGEBMMICYYAAxwQBi\nggHEBAOICQYQEwwgJhhATDCAmGAAMcEAYoIBxAQDiP0RoqNMBlokHDIAAAAASUVORK5CYII=\n", "text/plain": [ - "" + "\u003cmatplotlib.figure.Figure at 0x7f97ee42bb90\u003e" ] }, "metadata": { @@ -1785,17 +1596,17 @@ "outputarea_id1", "user_output" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c70592aa-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c6da872c-3379-11e8-ac70-0242ac110002\"]);\n", - "//# sourceURL=js_25c3aaf79a" + "window[\"a8e54767-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a8e54763-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_28bd08ac10" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ea9efc10\u003e" ] }, "metadata": { @@ -1803,17 +1614,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c70842c0-3379-11e8-ac70-0242ac110002\"] = google.colab.output.getActiveOutputArea();\n", - "//# sourceURL=js_984c56b816" + "window[\"a8e54768-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.getActiveOutputArea();\n", + "//# sourceURL=js_ae2887f57d" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ea9efb50\u003e" ] }, "metadata": { @@ -1821,17 +1632,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c708dec4-3379-11e8-ac70-0242ac110002\"] = document.querySelector(\"#id1_content_0\");\n", - "//# sourceURL=js_e0451a1217" + "window[\"a8e54769-8903-11e8-99f9-c8d3ffb5fbe0\"] = document.querySelector(\"#id1_content_0\");\n", + "//# sourceURL=js_608805a786" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ea9ef710\u003e" ] }, "metadata": { @@ -1839,17 +1650,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c7092726-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c708dec4-3379-11e8-ac70-0242ac110002\"]);\n", - "//# sourceURL=js_7aa23d7385" + "window[\"a8e5476a-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a8e54769-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_3d87cf7d0f" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ea9efa90\u003e" ] }, "metadata": { @@ -1857,17 +1668,17 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c7099044-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n", - "//# sourceURL=js_5722756ddb" + "window[\"a8e5476b-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"id1\"].setSelectedTabIndex(0);\n", + "//# sourceURL=js_5e91101199" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ea9efa50\u003e" ] }, "metadata": { @@ -1875,24 +1686,149 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" }, { - "output_type": "stream", - "text": [ - "Give me a color name (or press 'enter' to exit): \n" - ], - "name": "stdout" + "data": { + "text/html": [ + "\u003cdiv class=id_45185901 style=\"margin-right:10px; display:flex;align-items:center;\"\u003e\u003cspan style=\"margin-right: 3px;\"\u003e\u003c/span\u003e\u003c/div\u003e" + ], + "text/plain": [ + "\u003cIPython.core.display.HTML at 0x7f97ee42bd90\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"a8e5476c-8903-11e8-99f9-c8d3ffb5fbe0\"] = jQuery(\".id_45185901 span\");\n", + "//# sourceURL=js_f43052a94e" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f97ea9ef750\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"a8e5476d-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"a8e5476c-8903-11e8-99f9-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", + "//# sourceURL=js_bfc0fb76ce" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f97ea9efb10\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"a9e9b8b0-8903-11e8-99f9-c8d3ffb5fbe0\"] = jQuery(\".id_45185901 input\");\n", + "//# sourceURL=js_7f167283fa" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f97ea9ef610\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"a9e9b8b1-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"a9e9b8b0-8903-11e8-99f9-c8d3ffb5fbe0\"].remove();\n", + "//# sourceURL=js_016ae4bf21" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f97ea9ef250\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"a9e9b8b2-8903-11e8-99f9-c8d3ffb5fbe0\"] = jQuery(\".id_45185901 span\");\n", + "//# sourceURL=js_e666f179bc" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f97ea9ef550\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" }, { - "output_type": "display_data", "data": { "application/javascript": [ - "window[\"c7baac12-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c70842c0-3379-11e8-ac70-0242ac110002\"]);\n", - "//# sourceURL=js_cdd622e58f" + "window[\"a9e9b8b3-8903-11e8-99f9-c8d3ffb5fbe0\"] = window[\"a9e9b8b2-8903-11e8-99f9-c8d3ffb5fbe0\"].text(\"Give me a color name (or press 'enter' to exit): \");\n", + "//# sourceURL=js_cbb9d14aec" ], "text/plain": [ - "" + "\u003cIPython.core.display.Javascript at 0x7f97ea9ef1d0\u003e" + ] + }, + "metadata": { + "tags": [ + "id1_content_0", + "outputarea_id1", + "user_output" + ] + }, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": [ + "window[\"a9e9b8b4-8903-11e8-99f9-c8d3ffb5fbe0\"] = google.colab.output.setActiveOutputArea(window[\"a8e54768-8903-11e8-99f9-c8d3ffb5fbe0\"]);\n", + "//# sourceURL=js_2967a79665" + ], + "text/plain": [ + "\u003cIPython.core.display.Javascript at 0x7f97ea9ef1d0\u003e" ] }, "metadata": { @@ -1900,21 +1836,102 @@ "id1_content_0", "outputarea_id1" ] - } + }, + "output_type": "display_data" } + ], + "source": [ + "def run_input_loop(sess, inference_ops, color_name_placeholder):\n", + " \"\"\"Helper function that reads from input and calls the inference ops in a loop.\"\"\"\n", + "\n", + " tb = widgets.TabBar([\"RNN Colorbot\"])\n", + " while True:\n", + " with tb.output_to(0):\n", + " try:\n", + " color_name = six.moves.input(\"Give me a color name (or press 'enter' to exit): \")\n", + " except (EOFError, KeyboardInterrupt):\n", + " break\n", + " if not color_name:\n", + " break\n", + " with tb.output_to(0):\n", + " tb.clear_tab()\n", + " sess.run(inference_ops, {color_name_placeholder: color_name})\n", + " plt.show()\n", + "\n", + "with tf.Graph().as_default():\n", + " # Read the data.\n", + " batch_size = 64\n", + " train_data = load_dataset(data_dir, train_url, batch_size)\n", + " eval_data = load_dataset(data_dir, test_url, 50, training=False)\n", + " \n", + " # Create the model components.\n", + " lower_cell, upper_cell, relu_layer = model_components()\n", + " # Create the helper placeholder for inference.\n", + " color_name_placeholder = tf.placeholder(tf.string, shape=())\n", + " \n", + " # Compile the train / test code.\n", + " tf_train_model = autograph.to_graph(train_model)\n", + " train_model_ops = tf_train_model(\n", + " train_data, eval_data, batch_size, lower_cell, upper_cell, relu_layer, train_steps=100)\n", + " \n", + " # Compile the inference code.\n", + " tf_inference = autograph.to_graph(inference)\n", + " inference_ops = tf_inference(color_name_placeholder, lower_cell, upper_cell, relu_layer)\n", + " \n", + " with tf.Session() as sess:\n", + " sess.run(tf.global_variables_initializer())\n", + " \n", + " # Run training and testing.\n", + " sess.run(train_model_ops)\n", + " \n", + " # Run the inference loop.\n", + " run_input_loop(sess, inference_ops, color_name_placeholder)" ] }, { + "cell_type": "markdown", "metadata": { - "id": "AHJ2c47U-A5W", - "colab_type": "text" + "colab_type": "text", + "id": "AHJ2c47U-A5W" }, - "cell_type": "markdown", "source": [ "# Where do we go next?\n", "\n", - "Autograph is available in tensorflow.contrib, but it's still in its early stages. We're excited about the possibilities it brings — write your machine learning code in the flexible Eager style, but still enjoy all the benefits that come with running in graph mode. A beta version will be available soon -- stay tuned!" + "AutoGraph is still in its early stages, but is available in [tensorflow.contrib](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/autograph). We're excited about the possibilities it brings. New versions will be available soon — stay tuned!" ] } - ] + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "default_view": {}, + "last_runtime": { + "build_target": "", + "kind": "local" + }, + "name": "Dev Summit 2018 - Autograph", + "provenance": [ + { + "file_id": "1wCZUh73zTNs1jzzYjqoxMIdaBWCdKJ2K", + "timestamp": 1522238054357 + }, + { + "file_id": "1_HpC-RrmIv4lNaqeoslUeWaX8zH5IXaJ", + "timestamp": 1521743157199 + }, + { + "file_id": "1mjO2fQ2F9hxpAzw2mnrrUkcgfb7xSGW-", + "timestamp": 1520522344607 + } + ], + "version": "0.3.2", + "views": {} + }, + "kernelspec": { + "display_name": "Python 2", + "name": "python2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } -- cgit v1.2.3 From d27953bb69ba44431b85fdf7ac43ed83c4422e40 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 08:45:18 -0700 Subject: Fix bug in masked_autoregressive_default_template where custom name was not creating custom variable scopes. PiperOrigin-RevId: 204747987 --- .../distributions/python/ops/bijectors/masked_autoregressive.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py index b8f2a4b2c7..296e66f2b2 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py @@ -514,9 +514,8 @@ def masked_autoregressive_default_template( Masked Autoencoder for Distribution Estimation. In _International Conference on Machine Learning_, 2015. https://arxiv.org/abs/1502.03509 """ - - with ops.name_scope(name, "masked_autoregressive_default_template", - values=[log_scale_min_clip, log_scale_max_clip]): + name = name or "masked_autoregressive_default_template" + with ops.name_scope(name, values=[log_scale_min_clip, log_scale_max_clip]): def _fn(x): """MADE parameterized via `masked_autoregressive_default_template`.""" # TODO(b/67594795): Better support of dynamic shape. @@ -552,8 +551,7 @@ def masked_autoregressive_default_template( else _clip_by_value_preserve_grad) log_scale = which_clip(log_scale, log_scale_min_clip, log_scale_max_clip) return shift, log_scale - return template_ops.make_template( - "masked_autoregressive_default_template", _fn) + return template_ops.make_template(name, _fn) @deprecation.deprecated( -- cgit v1.2.3 From 70b89c7eb28f3a2e87168a55d0f2c3f46f1e8add Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Mon, 16 Jul 2018 08:50:29 -0700 Subject: Bring the workshop notebook to date. PiperOrigin-RevId: 204748706 --- .../autograph/examples/notebooks/workshop.ipynb | 140 ++++++++++++--------- 1 file changed, 79 insertions(+), 61 deletions(-) diff --git a/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb b/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb index 4643656ff4..e7dfb13e15 100644 --- a/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb +++ b/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb @@ -11,11 +11,11 @@ } }, "colab_type": "code", - "id": "yFn4b8J0CKQV" + "id": "u3B7Uh50lozN" }, "outputs": [], "source": [ - "pip install -U tf-nightly" + "!pip install -U -q tf-nightly" ] }, { @@ -33,10 +33,10 @@ }, "outputs": [], "source": [ - "import os\n", - "import matplotlib.pyplot as plt\n", "import tensorflow as tf\n", - "from tensorflow.contrib import autograph" + "from tensorflow.contrib import autograph\n", + "\n", + "import matplotlib.pyplot as plt" ] }, { @@ -80,7 +80,7 @@ "# ...into graph-building functions like this:\n", "def tf_g(x):\n", " with tf.name_scope('g'):\n", - " \n", + "\n", " def if_true():\n", " with tf.name_scope('if_true'):\n", " x_1, = x,\n", @@ -94,7 +94,7 @@ " return x_1,\n", "\n", " x = autograph_utils.run_cond(tf.greater(x, 0), if_true, if_false)\n", - " return x\n" + " return x" ] }, { @@ -119,14 +119,14 @@ "# Generate a graph-version of g and call it:\n", "tf_g = autograph.to_graph(g)\n", "\n", - "with tf.Graph().as_default(): \n", + "with tf.Graph().as_default():\n", " # The result works like a regular op: takes tensors in, returns tensors.\n", " # You can inspect the graph using tf.get_default_graph().as_graph_def()\n", " g_ops = tf_g(tf.constant(9.0))\n", " with tf.Session() as sess:\n", " print('Autograph value: %2.2f\\n' % sess.run(g_ops))\n", - " \n", - " \n", + "\n", + "\n", "# You can view, debug and tweak the generated code:\n", "print(autograph.to_code(g))" ] @@ -173,10 +173,10 @@ "print('Original value: %d' % f([10,12,15,20]))\n", "\n", "tf_f = autograph.to_graph(f)\n", - "with tf.Graph().as_default(): \n", + "with tf.Graph().as_default():\n", " with tf.Session():\n", " print('Graph value: %d\\n\\n' % tf_f(tf.constant([10,12,15,20])).eval())\n", - " \n", + "\n", "print(autograph.to_code(f))" ] }, @@ -212,7 +212,7 @@ " return x * x\n", "\n", "tf_f = autograph.to_graph(f)\n", - "with tf.Graph().as_default(): \n", + "with tf.Graph().as_default():\n", " with tf.Session():\n", " try:\n", " print(tf_f(tf.constant(0)).eval())\n", @@ -251,7 +251,7 @@ " n += 1\n", " print(n)\n", " return n\n", - " \n", + "\n", "tf_f = autograph.to_graph(f)\n", "with tf.Graph().as_default():\n", " with tf.Session():\n", @@ -265,7 +265,7 @@ "id": "NqF0GT-VCVFh" }, "source": [ - "Appending to lists in loops also works (we create a `TensorArray` for you behind the scenes)" + "Appending to lists in loops also works (we create a tensor list ops behind the scenes)" ] }, { @@ -286,15 +286,15 @@ "def f(n):\n", " z = []\n", " # We ask you to tell us the element dtype of the list\n", - " z = autograph.utils.set_element_type(z, tf.int32)\n", + " autograph.set_element_type(z, tf.int32)\n", " for i in range(n):\n", " z.append(i)\n", " # when you're done with the list, stack it\n", " # (this is just like np.stack)\n", - " return autograph.stack(z) \n", + " return autograph.stack(z)\n", "\n", "tf_f = autograph.to_graph(f)\n", - "with tf.Graph().as_default(): \n", + "with tf.Graph().as_default():\n", " with tf.Session():\n", " print(tf_f(tf.constant(3)).eval())\n", "\n", @@ -345,7 +345,7 @@ "source": [ "tf_g = autograph.to_graph(fizzbuzz)\n", "\n", - "with tf.Graph().as_default(): \n", + "with tf.Graph().as_default():\n", " # The result works like a regular op: takes tensors in, returns tensors.\n", " # You can inspect the graph using tf.get_default_graph().as_graph_def()\n", " g_ops = tf_g(tf.constant(15))\n", @@ -402,7 +402,7 @@ " return x\n", "\n", "\n", - "with tf.Graph().as_default(): \n", + "with tf.Graph().as_default():\n", " with tf.Session() as sess:\n", " print(sess.run(square_log(tf.constant(4))))" ] @@ -414,7 +414,7 @@ "id": "_R-Q7BbxmkBF" }, "source": [ - "#### Now some exercises. Convert the TensorFlow code into AutoGraph'd Python code." + "#### Convert the TensorFlow code into Python code for AutoGraph" ] }, { @@ -457,8 +457,10 @@ "source": [ "@autograph.convert()\n", "def square_if_positive(x):\n", - " ... # \u003c\u003c\u003c fill it in!\n", - " \n", + "\n", + " pass # TODO: fill it in!\n", + "\n", + "\n", "with tf.Session() as sess:\n", " print(sess.run(square_if_positive(tf.constant(4))))" ] @@ -535,7 +537,7 @@ " x = tf.cond(tf.greater(x, 0), if_positive, lambda: x)\n", " return x\n", "\n", - "with tf.Graph().as_default(): \n", + "with tf.Graph().as_default():\n", " with tf.Session() as sess:\n", " print(sess.run(nearest_odd_square(tf.constant(4))))" ] @@ -557,8 +559,10 @@ "source": [ "@autograph.convert()\n", "def nearest_odd_square(x):\n", - " ... # \u003c\u003c\u003c fill it in!\n", - " \n", + "\n", + " pass # TODO: fill it in!\n", + "\n", + "\n", "with tf.Session() as sess:\n", " print(sess.run(nearest_odd_square(tf.constant(4))))" ] @@ -596,7 +600,7 @@ " x = x + 1\n", " return x\n", "\n", - "with tf.Graph().as_default(): \n", + "with tf.Graph().as_default():\n", " with tf.Session() as sess:\n", " print(sess.run(nearest_odd_square(tf.constant(4))))" ] @@ -630,8 +634,8 @@ "def square_until_stop(x, y):\n", " x = tf.while_loop(lambda x: tf.less(x, y), lambda x: x * x, [x])\n", " return x\n", - " \n", - "with tf.Graph().as_default(): \n", + "\n", + "with tf.Graph().as_default():\n", " with tf.Session() as sess:\n", " print(sess.run(square_until_stop(tf.constant(4), tf.constant(100))))" ] @@ -653,9 +657,11 @@ "source": [ "@autograph.convert()\n", "def square_until_stop(x, y):\n", - " ... # fill it in!\n", - " \n", - "with tf.Graph().as_default(): \n", + "\n", + " pass # TODO: fill it in!\n", + "\n", + "\n", + "with tf.Graph().as_default():\n", " with tf.Session() as sess:\n", " print(sess.run(square_until_stop(tf.constant(4), tf.constant(100))))" ] @@ -690,8 +696,8 @@ " while x \u003c y:\n", " x = x * x\n", " return x\n", - " \n", - "with tf.Graph().as_default(): \n", + "\n", + "with tf.Graph().as_default():\n", " with tf.Session() as sess:\n", " print(sess.run(square_until_stop(tf.constant(4), tf.constant(100))))" ] @@ -725,7 +731,7 @@ "def argwhere_cumsum(x, threshold):\n", " current_sum = 0.0\n", " idx = 0\n", - " \n", + "\n", " for i in range(len(x)):\n", " idx = i\n", " if current_sum \u003e= threshold:\n", @@ -733,10 +739,10 @@ " current_sum += x[i]\n", " return idx\n", "\n", - "N = 10\n", - "with tf.Graph().as_default(): \n", + "n = 10\n", + "with tf.Graph().as_default():\n", " with tf.Session() as sess:\n", - " idx = argwhere_cumsum(tf.ones(N), tf.constant(float(N/2)))\n", + " idx = argwhere_cumsum(tf.ones(n), tf.constant(float(n / 2)))\n", " print(sess.run(idx))" ] }, @@ -757,12 +763,14 @@ "source": [ "@autograph.convert()\n", "def argwhere_cumsum(x, threshold):\n", - " ...\n", "\n", - "N = 10\n", - "with tf.Graph().as_default(): \n", + " pass # TODO: fill it in!\n", + "\n", + "\n", + "n = 10\n", + "with tf.Graph().as_default():\n", " with tf.Session() as sess:\n", - " idx = argwhere_cumsum(tf.ones(N), tf.constant(float(N/2)))\n", + " idx = argwhere_cumsum(tf.ones(n), tf.constant(float(n / 2)))\n", " print(sess.run(idx))" ] }, @@ -802,10 +810,10 @@ " current_sum += x[i]\n", " return idx\n", "\n", - "N = 10\n", + "n = 10\n", "with tf.Graph().as_default(): \n", " with tf.Session() as sess:\n", - " idx = argwhere_cumsum(tf.ones(N), tf.constant(float(N/2)))\n", + " idx = argwhere_cumsum(tf.ones(n), tf.constant(float(n / 2)))\n", " print(sess.run(idx))" ] }, @@ -998,43 +1006,50 @@ "def train(train_ds, test_ds, hp):\n", " m = mlp_model((28 * 28,))\n", " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", - " \n", + "\n", " # We'd like to save our losses to a list. In order for AutoGraph\n", " # to convert these lists into their graph equivalent,\n", " # we need to specify the element type of the lists.\n", " train_losses = []\n", - " train_losses = autograph.utils.set_element_type(train_losses, tf.float32)\n", " test_losses = []\n", - " test_losses = autograph.utils.set_element_type(test_losses, tf.float32)\n", " train_accuracies = []\n", - " train_accuracies = autograph.utils.set_element_type(train_accuracies, tf.float32)\n", " test_accuracies = []\n", - " test_accuracies = autograph.utils.set_element_type(test_accuracies, tf.float32)\n", - " \n", + " autograph.set_element_type(train_losses, tf.float32)\n", + " autograph.set_element_type(test_losses, tf.float32)\n", + " autograph.set_element_type(train_accuracies, tf.float32)\n", + " autograph.set_element_type(test_accuracies, tf.float32)\n", + "\n", " # This entire training loop will be run in-graph.\n", " i = tf.constant(0)\n", " while i \u003c hp.max_steps:\n", " train_x, train_y = get_next_batch(train_ds)\n", " test_x, test_y = get_next_batch(test_ds)\n", - " # add get next\n", + "\n", " step_train_loss, step_train_accuracy = fit(m, train_x, train_y, opt)\n", " step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n", + "\n", " if i % (hp.max_steps // 10) == 0:\n", " print('Step', i, 'train loss:', step_train_loss, 'test loss:',\n", " step_test_loss, 'train accuracy:', step_train_accuracy,\n", " 'test accuracy:', step_test_accuracy)\n", + "\n", " train_losses.append(step_train_loss)\n", " test_losses.append(step_test_loss)\n", " train_accuracies.append(step_train_accuracy)\n", " test_accuracies.append(step_test_accuracy)\n", + "\n", " i += 1\n", - " \n", - " # We've recorded our loss values and accuracies \n", + "\n", + " # We've recorded our loss values and accuracies\n", " # to a list in a graph with AutoGraph's help.\n", - " # In order to return the values as a Tensor, \n", + " # In order to return the values as a Tensor,\n", " # we need to stack them before returning them.\n", - " return (autograph.stack(train_losses), autograph.stack(test_losses), autograph.stack(train_accuracies),\n", - " autograph.stack(test_accuracies))" + " return (\n", + " autograph.stack(train_losses),\n", + " autograph.stack(test_losses),\n", + " autograph.stack(train_accuracies),\n", + " autograph.stack(test_accuracies),\n", + " )" ] }, { @@ -1060,14 +1075,17 @@ " train_ds = setup_mnist_data(True, hp, 50)\n", " test_ds = setup_mnist_data(False, hp, 1000)\n", " tf_train = autograph.to_graph(train)\n", - " (train_losses, test_losses, train_accuracies,\n", - " test_accuracies) = tf_train(train_ds, test_ds, hp)\n", + " loss_tensors = tf_train(train_ds, test_ds, hp)\n", "\n", " with tf.Session() as sess:\n", " sess.run(tf.global_variables_initializer())\n", - " (train_losses, test_losses, train_accuracies,\n", - " test_accuracies) = sess.run([train_losses, test_losses, train_accuracies,\n", - " test_accuracies])\n", + " (\n", + " train_losses,\n", + " test_losses,\n", + " train_accuracies,\n", + " test_accuracies\n", + " ) = sess.run(loss_tensors)\n", + "\n", " plt.title('MNIST train/test losses')\n", " plt.plot(train_losses, label='train loss')\n", " plt.plot(test_losses, label='test loss')\n", -- cgit v1.2.3 From cde36bc1667d80c9569bfa09b1cb6e71a77700b9 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Mon, 16 Jul 2018 08:58:38 -0700 Subject: Fix reaching_definitions to correctly mark the definition of modified symbols in the statement that replaces them, e.g. a = a. PiperOrigin-RevId: 204749753 --- .../pyct/static_analysis/reaching_definitions.py | 69 +++++++++++++++------- .../static_analysis/reaching_definitions_test.py | 66 +++++++++++++++++---- 2 files changed, 102 insertions(+), 33 deletions(-) diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py b/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py index 4ea7fd93cd..9a84f1231c 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions.py @@ -112,7 +112,6 @@ class Analyzer(cfg.GraphVisitor): def __init__(self, graph, definition_factory): self._definition_factory = definition_factory super(Analyzer, self).__init__(graph) - self.defs_by_ast_node = {} # This allows communicating that nodes have extra reaching definitions, # e.g. those that a function closes over. self.extra_in = {} @@ -160,13 +159,12 @@ class Analyzer(cfg.GraphVisitor): self.in_[node] = defs_in self.out[node] = defs_out - self.defs_by_ast_node[node.ast_node] = defs_out.value # TODO(mdan): Move this to the superclass? return prev_defs_out != defs_out -class WholeTreeAnalyzer(transformer.Base): +class TreeAnnotator(transformer.Base): """AST visitor that annotates each symbol name with its reaching definitions. Simultaneously, the visitor runs the dataflow analysis on each function node, @@ -179,12 +177,11 @@ class WholeTreeAnalyzer(transformer.Base): """ def __init__(self, source_info, graphs, definition_factory): - super(WholeTreeAnalyzer, self).__init__(source_info) - self.stmt_reaching_defs_info = None + super(TreeAnnotator, self).__init__(source_info) + self.definition_factory = definition_factory self.graphs = graphs self.current_analyzer = None - self.definition_factory = definition_factory - self.current_stmt_defs = None + self.current_cfg_node = None def visit_FunctionDef(self, node): parent_analyzer = self.current_analyzer @@ -209,7 +206,11 @@ class WholeTreeAnalyzer(transformer.Base): # Recursively process any remaining subfunctions. self.current_analyzer = analyzer - node = self.generic_visit(node) + # Note: not visiting name, decorator_list and returns because they don't + # apply to this anlysis. + # TODO(mdan): Should we still process the function name? + node.args = self.visit(node.args) + node.body = self.visit_block(node.body) self.current_analyzer = parent_analyzer return node @@ -226,11 +227,19 @@ class WholeTreeAnalyzer(transformer.Base): # definitions. return node + analyzer = self.current_analyzer + cfg_node = self.current_cfg_node + + assert cfg_node is not None, 'name node outside of any statement?' + qn = anno.getanno(node, anno.Basic.QN) - assert self.current_stmt_defs is not None, ( - 'name node outside of any statement?') - anno.setanno(node, anno.Static.DEFINITIONS, - tuple(self.current_stmt_defs.get(qn, ()))) + if isinstance(node.ctx, gast.Load): + anno.setanno(node, anno.Static.DEFINITIONS, + tuple(analyzer.in_[cfg_node].value.get(qn, ()))) + else: + anno.setanno(node, anno.Static.DEFINITIONS, + tuple(analyzer.out[cfg_node].value.get(qn, ()))) + return node def _aggregate_predecessors_defined_in(self, node): @@ -239,23 +248,41 @@ class WholeTreeAnalyzer(transformer.Base): for p in preds: node_defined_in |= set(self.current_analyzer.out[p].value.keys()) anno.setanno(node, anno.Static.DEFINED_VARS_IN, frozenset(node_defined_in)) - node = self.generic_visit(node) - return node def visit_If(self, node): - return self._aggregate_predecessors_defined_in(node) + self._aggregate_predecessors_defined_in(node) + return self.generic_visit(node) def visit_For(self, node): - return self._aggregate_predecessors_defined_in(node) + self._aggregate_predecessors_defined_in(node) + + # Manually accounting for the shortcoming described in + # cfg.AstToCfg.visit_For. + parent = self.current_cfg_node + self.current_cfg_node = self.current_analyzer.graph.index[node.iter] + node.target = self.visit(node.target) + self.current_cfg_node = parent + + node.iter = self.visit(node.iter) + node.body = self.visit_block(node.body) + node.orelse = self.visit_block(node.orelse) + + return node def visit_While(self, node): - return self._aggregate_predecessors_defined_in(node) + self._aggregate_predecessors_defined_in(node) + return self.generic_visit(node) def visit(self, node): + parent = self.current_cfg_node + if (self.current_analyzer is not None and - node in self.current_analyzer.defs_by_ast_node): - self.current_stmt_defs = self.current_analyzer.defs_by_ast_node[node] - return super(WholeTreeAnalyzer, self).visit(node) + node in self.current_analyzer.graph.index): + self.current_cfg_node = self.current_analyzer.graph.index[node] + node = super(TreeAnnotator, self).visit(node) + + self.current_cfg_node = parent + return node def resolve(node, source_info, graphs, definition_factory): @@ -269,6 +296,6 @@ def resolve(node, source_info, graphs, definition_factory): Returns: ast.AST """ - visitor = WholeTreeAnalyzer(source_info, graphs, definition_factory) + visitor = TreeAnnotator(source_info, graphs, definition_factory) node = visitor.visit(node) return node diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions_test.py index 0410bb2a35..243fe804b2 100644 --- a/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions_test.py +++ b/tensorflow/contrib/autograph/pyct/static_analysis/reaching_definitions_test.py @@ -61,6 +61,20 @@ class DefinitionInfoTest(test.TestCase): expected = (expected,) self.assertSetEqual(defined_in_str, set(expected)) + def assertSameDef(self, first, second): + self.assertHasDefs(first, 1) + self.assertHasDefs(second, 1) + self.assertIs( + anno.getanno(first, anno.Static.DEFINITIONS)[0], + anno.getanno(second, anno.Static.DEFINITIONS)[0]) + + def assertNotSameDef(self, first, second): + self.assertHasDefs(first, 1) + self.assertHasDefs(second, 1) + self.assertIsNot( + anno.getanno(first, anno.Static.DEFINITIONS)[0], + anno.getanno(second, anno.Static.DEFINITIONS)[0]) + def test_conditional(self): def test_fn(a, b): @@ -93,10 +107,10 @@ class DefinitionInfoTest(test.TestCase): self.assertHasDefs(fn_body[0].value.args[0], 1) self.assertHasDefs(fn_body[1].body[0].targets[0], 1) - self.assertHasDefs(fn_body[1].body[0].value, 1) self.assertHasDefs(fn_body[1].body[1].targets[0], 1) self.assertHasDefs(fn_body[1].body[1].value, 1) # The loop does have an invariant test, but the CFG doesn't know that. + self.assertHasDefs(fn_body[1].body[0].value, 2) self.assertHasDefs(fn_body[2].value, 2) def test_while_else(self): @@ -171,10 +185,7 @@ class DefinitionInfoTest(test.TestCase): self.assertHasDefs(fn_body[2].value, 2) inner_fn_body = fn_body[1].body[1].body - self.assertHasDefs(inner_fn_body[0].value, 1) - self.assertTrue( - anno.getanno(inner_fn_body[0].value, anno.Static.DEFINITIONS)[0] is - anno.getanno(def_of_a_in_if, anno.Static.DEFINITIONS)[0]) + self.assertSameDef(inner_fn_body[0].value, def_of_a_in_if) def test_nested_functions_isolation(self): @@ -191,17 +202,12 @@ class DefinitionInfoTest(test.TestCase): node = self._parse_and_analyze(test_fn) fn_body = node.body[0].body - self.assertHasDefs(fn_body[3].value, 1) - self.assertHasDefs(fn_body[1].body[1].value, 1) - parent_return = fn_body[3] child_return = fn_body[1].body[1] # The assignment `a = 1` makes `a` local to `child`. - self.assertFalse( - anno.getanno(parent_return.value, anno.Static.DEFINITIONS)[0] is - anno.getanno(child_return.value, anno.Static.DEFINITIONS)[0]) + self.assertNotSameDef(parent_return.value, child_return.value) - def test_debug(self): + def test_function_call_in_with(self): def foo(_): pass @@ -216,6 +222,42 @@ class DefinitionInfoTest(test.TestCase): self.assertHasDefs(fn_body[0].items[0].context_expr.func, 0) self.assertHasDefs(fn_body[0].items[0].context_expr.args[0], 1) + def test_mutation_subscript(self): + + def test_fn(a): + l = [] + l[0] = a + return l + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + creation = fn_body[0].targets[0] + mutation = fn_body[1].targets[0].value + use = fn_body[2].value + self.assertSameDef(creation, mutation) + self.assertSameDef(creation, use) + + def test_replacement(self): + + def foo(a): + return a + + def test_fn(a): + a = foo(a) + return a + + node = self._parse_and_analyze(test_fn) + fn_body = node.body[0].body + + param = node.body[0].args.args[0] + source = fn_body[0].value.args[0] + target = fn_body[0].targets[0] + retval = fn_body[1].value + self.assertSameDef(param, source) + self.assertNotSameDef(source, target) + self.assertSameDef(target, retval) + if __name__ == '__main__': test.main() -- cgit v1.2.3 From 3ab5ef05eed4645b2f412522451be7554ef9df8c Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Mon, 16 Jul 2018 09:33:25 -0700 Subject: [tf.data] Add handling for gzip & zlib compression types in CsvDataset and make_csv_dataset Closes #19954. PiperOrigin-RevId: 204755011 --- tensorflow/contrib/data/kernels/BUILD | 1 + tensorflow/contrib/data/kernels/csv_dataset_op.cc | 59 +- tensorflow/contrib/data/ops/dataset_ops.cc | 12 +- .../python/kernel_tests/csv_dataset_op_test.py | 143 ++-- .../python/kernel_tests/reader_dataset_ops_test.py | 875 ++++++++++++--------- tensorflow/contrib/data/python/ops/readers.py | 17 +- 6 files changed, 653 insertions(+), 454 deletions(-) diff --git a/tensorflow/contrib/data/kernels/BUILD b/tensorflow/contrib/data/kernels/BUILD index 7b69e10441..c2c04ac7b3 100644 --- a/tensorflow/contrib/data/kernels/BUILD +++ b/tensorflow/contrib/data/kernels/BUILD @@ -34,6 +34,7 @@ cc_library( srcs = ["csv_dataset_op.cc"], deps = [ "//tensorflow/core:framework_headers_lib", + "//tensorflow/core:lib_internal", "//third_party/eigen3", "@protobuf_archive//:protobuf_headers", ], diff --git a/tensorflow/contrib/data/kernels/csv_dataset_op.cc b/tensorflow/contrib/data/kernels/csv_dataset_op.cc index 4657807785..7a13b92005 100644 --- a/tensorflow/contrib/data/kernels/csv_dataset_op.cc +++ b/tensorflow/contrib/data/kernels/csv_dataset_op.cc @@ -18,7 +18,10 @@ limitations under the License. #include "tensorflow/core/framework/dataset.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/lib/io/inputstream_interface.h" #include "tensorflow/core/lib/io/random_inputstream.h" +#include "tensorflow/core/lib/io/zlib_compression_options.h" +#include "tensorflow/core/lib/io/zlib_inputstream.h" namespace tensorflow { namespace { @@ -37,6 +40,10 @@ class CSVDatasetOp : public DatasetOpKernel { ctx, filenames_tensor->dims() <= 1, errors::InvalidArgument("`filenames` must be a scalar or a vector.")); + string compression_type; + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "compression_type", + &compression_type)); + OpInputList record_defaults_list; OP_REQUIRES_OK(ctx, ctx->input_list("record_defaults", &record_defaults_list)); @@ -86,6 +93,19 @@ class CSVDatasetOp : public DatasetOpKernel { filenames.push_back(filenames_tensor->flat()(i)); } + io::ZlibCompressionOptions zlib_compression_options = + io::ZlibCompressionOptions::DEFAULT(); + if (compression_type == "ZLIB") { + zlib_compression_options = io::ZlibCompressionOptions::DEFAULT(); + } else if (compression_type == "GZIP") { + zlib_compression_options = io::ZlibCompressionOptions::GZIP(); + } else { + OP_REQUIRES(ctx, compression_type.empty(), + errors::InvalidArgument("Unsupported compression_type: ", + compression_type, ".")); + } + zlib_compression_options.input_buffer_size = buffer_size; + std::vector select_cols; select_cols.reserve(select_cols_tensor->NumElements()); for (int i = 0; i < select_cols_tensor->NumElements(); ++i) { @@ -103,31 +123,34 @@ class CSVDatasetOp : public DatasetOpKernel { ctx, select_cols.empty() || select_cols.front() >= 0, errors::InvalidArgument("select_cols should be non-negative indices")); - *output = new Dataset(ctx, std::move(filenames), header, buffer_size, - output_types_, output_shapes_, - std::move(record_defaults), std::move(select_cols), - use_quote_delim, delim[0], std::move(na_value)); + *output = new Dataset( + ctx, std::move(filenames), header, std::move(compression_type), + zlib_compression_options, output_types_, output_shapes_, + std::move(record_defaults), std::move(select_cols), use_quote_delim, + delim[0], std::move(na_value)); } private: class Dataset : public GraphDatasetBase { public: Dataset(OpKernelContext* ctx, std::vector filenames, bool header, - int64 buffer_size, const DataTypeVector& output_types, + string compression_type, io::ZlibCompressionOptions options, + const DataTypeVector& output_types, const std::vector& output_shapes, std::vector record_defaults, std::vector select_cols, bool use_quote_delim, char delim, string na_value) : GraphDatasetBase(ctx), filenames_(std::move(filenames)), header_(header), - buffer_size_(buffer_size), out_type_(output_types), output_shapes_(output_shapes), record_defaults_(std::move(record_defaults)), select_cols_(std::move(select_cols)), use_quote_delim_(use_quote_delim), delim_(delim), - na_value_(std::move(na_value)) {} + na_value_(std::move(na_value)), + use_compression_(!compression_type.empty()), + options_(options) {} std::unique_ptr MakeIteratorInternal( const string& prefix) const override { @@ -510,7 +533,8 @@ class CSVDatasetOp : public DatasetOpKernel { Status FillBuffer(string* result) EXCLUSIVE_LOCKS_REQUIRED(mu_) { result->clear(); - Status s = input_stream_->ReadNBytes(dataset()->buffer_size_, result); + Status s = input_stream_->ReadNBytes( + dataset()->options_.input_buffer_size, result); if (errors::IsOutOfRange(s) && !result->empty()) { // Ignore OutOfRange error when ReadNBytes read < N bytes. @@ -675,8 +699,17 @@ class CSVDatasetOp : public DatasetOpKernel { // Actually move on to next file. TF_RETURN_IF_ERROR(env->NewRandomAccessFile( dataset()->filenames_[current_file_index_], &file_)); - input_stream_.reset( - new io::RandomAccessInputStream(file_.get(), false)); + random_access_input_stream_ = + std::make_shared(file_.get(), false); + + if (dataset()->use_compression_) { + input_stream_ = std::make_shared( + random_access_input_stream_.get(), + dataset()->options_.input_buffer_size, + dataset()->options_.input_buffer_size, dataset()->options_); + } else { + input_stream_ = random_access_input_stream_; + } buffer_.clear(); pos_ = 0; if (dataset()->header_) { @@ -704,8 +737,9 @@ class CSVDatasetOp : public DatasetOpKernel { string buffer_ GUARDED_BY(mu_); // Maintain our own buffer size_t pos_ GUARDED_BY( mu_); // Index into the buffer must be maintained between iters - std::unique_ptr input_stream_ + std::shared_ptr random_access_input_stream_ GUARDED_BY(mu_); + std::shared_ptr input_stream_ GUARDED_BY(mu_); size_t current_file_index_ GUARDED_BY(mu_) = 0; std::unique_ptr file_ GUARDED_BY(mu_); // must outlive input_stream_ @@ -713,7 +747,6 @@ class CSVDatasetOp : public DatasetOpKernel { const std::vector filenames_; const bool header_; - const int64 buffer_size_; const DataTypeVector out_type_; const std::vector output_shapes_; const std::vector record_defaults_; @@ -721,6 +754,8 @@ class CSVDatasetOp : public DatasetOpKernel { const bool use_quote_delim_; const char delim_; const string na_value_; + const bool use_compression_; + const io::ZlibCompressionOptions options_; }; // class Dataset DataTypeVector output_types_; diff --git a/tensorflow/contrib/data/ops/dataset_ops.cc b/tensorflow/contrib/data/ops/dataset_ops.cc index 8413fcaf87..a623c27ff8 100644 --- a/tensorflow/contrib/data/ops/dataset_ops.cc +++ b/tensorflow/contrib/data/ops/dataset_ops.cc @@ -36,6 +36,7 @@ data_input_datasets: `N` datasets with the same type that will be interleaved REGISTER_OP("CSVDataset") .Input("filenames: string") + .Input("compression_type: string") .Input("buffer_size: int64") .Input("header: bool") .Input("field_delim: string") @@ -52,17 +53,18 @@ REGISTER_OP("CSVDataset") shape_inference::ShapeHandle unused; // `filenames` must be a scalar or a vector. TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(0), 1, &unused)); - // `buffer_size`, `header`, `field_delim`, `use_quote_delim`, - // `na_value` must be scalars + // `compression_type`, `buffer_size`, `header`, `field_delim`, + // `use_quote_delim`, `na_value` must be scalars TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused)); TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused)); TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(6), 0, &unused)); // `select_cols` must be a vector - TF_RETURN_IF_ERROR(c->WithRank(c->input(6), 1, &unused)); - // `record_defaults` must be a list of scalars...? - for (size_t i = 7; i < c->num_inputs(); ++i) { + TF_RETURN_IF_ERROR(c->WithRank(c->input(7), 1, &unused)); + // `record_defaults` must be lists of scalars + for (size_t i = 8; i < c->num_inputs(); ++i) { TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 1, &unused)); } return shape_inference::ScalarShape(c); diff --git a/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py index df115175f5..2a0e64caeb 100644 --- a/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py @@ -18,10 +18,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import gzip import os import string import tempfile import time +import zlib import numpy as np @@ -62,18 +64,29 @@ class CsvDatasetOpTest(test.TestCase): op2 = sess.run(next2) self.assertAllEqual(op1, op2) - def setup_files(self, inputs, linebreak='\n'): + def _setup_files(self, inputs, linebreak='\n', compression_type=None): filenames = [] for i, ip in enumerate(inputs): fn = os.path.join(self.get_temp_dir(), 'temp_%d.csv' % i) - with open(fn, 'wb') as f: - f.write(linebreak.join(ip).encode('utf-8')) + contents = linebreak.join(ip).encode('utf-8') + if compression_type is None: + with open(fn, 'wb') as f: + f.write(contents) + elif compression_type == 'GZIP': + with gzip.GzipFile(fn, 'wb') as f: + f.write(contents) + elif compression_type == 'ZLIB': + contents = zlib.compress(contents) + with open(fn, 'wb') as f: + f.write(contents) + else: + raise ValueError('Unsupported compression_type', compression_type) filenames.append(fn) return filenames def _make_test_datasets(self, inputs, **kwargs): # Test by comparing its output to what we could get with map->decode_csv - filenames = self.setup_files(inputs) + filenames = self._setup_files(inputs) dataset_expected = core_readers.TextLineDataset(filenames) dataset_expected = dataset_expected.map( lambda l: parsing_ops.decode_csv(l, **kwargs)) @@ -112,15 +125,18 @@ class CsvDatasetOpTest(test.TestCase): except errors.OutOfRangeError: break - def _test_dataset(self, - inputs, - expected_output=None, - expected_err_re=None, - linebreak='\n', - **kwargs): + def _test_dataset( + self, + inputs, + expected_output=None, + expected_err_re=None, + linebreak='\n', + compression_type=None, # Used for both setup and parsing + **kwargs): """Checks that elements produced by CsvDataset match expected output.""" # Convert str type because py3 tf strings are bytestrings - filenames = self.setup_files(inputs, linebreak) + filenames = self._setup_files(inputs, linebreak, compression_type) + kwargs['compression_type'] = compression_type with ops.Graph().as_default() as g: with self.test_session(graph=g) as sess: dataset = readers.CsvDataset(filenames, **kwargs) @@ -174,7 +190,7 @@ class CsvDatasetOpTest(test.TestCase): def testCsvDataset_ignoreErrWithUnescapedQuotes(self): record_defaults = [['']] * 3 inputs = [['1,"2"3",4', '1,"2"3",4",5,5', 'a,b,"c"d"', 'e,f,g']] - filenames = self.setup_files(inputs) + filenames = self._setup_files(inputs) with ops.Graph().as_default() as g: with self.test_session(graph=g) as sess: dataset = readers.CsvDataset(filenames, record_defaults=record_defaults) @@ -184,7 +200,7 @@ class CsvDatasetOpTest(test.TestCase): def testCsvDataset_ignoreErrWithUnquotedQuotes(self): record_defaults = [['']] * 3 inputs = [['1,2"3,4', 'a,b,c"d', '9,8"7,6,5', 'e,f,g']] - filenames = self.setup_files(inputs) + filenames = self._setup_files(inputs) with ops.Graph().as_default() as g: with self.test_session(graph=g) as sess: dataset = readers.CsvDataset(filenames, record_defaults=record_defaults) @@ -355,7 +371,7 @@ class CsvDatasetOpTest(test.TestCase): '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19', '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19' ]] - file_path = self.setup_files(data) + file_path = self._setup_files(data) with ops.Graph().as_default() as g: ds = readers.make_csv_dataset( @@ -432,14 +448,29 @@ class CsvDatasetOpTest(test.TestCase): record_defaults=record_defaults, buffer_size=0) - def testCsvDataset_withBufferSize(self): + def _test_dataset_on_buffer_sizes(self, + inputs, + expected, + linebreak, + record_defaults, + compression_type=None, + num_sizes_to_test=20): + # Testing reading with a range of buffer sizes that should all work. + for i in list(range(1, 1 + num_sizes_to_test)) + [None]: + self._test_dataset( + inputs, + expected, + linebreak=linebreak, + compression_type=compression_type, + record_defaults=record_defaults, + buffer_size=i) + + def testCsvDataset_withLF(self): record_defaults = [['NA']] * 3 inputs = [['abc,def,ghi', '0,1,2', ',,']] expected = [['abc', 'def', 'ghi'], ['0', '1', '2'], ['NA', 'NA', 'NA']] - for i in range(20): - # Test a range of buffer sizes that should all work - self._test_dataset( - inputs, expected, record_defaults=record_defaults, buffer_size=i + 1) + self._test_dataset_on_buffer_sizes( + inputs, expected, linebreak='\n', record_defaults=record_defaults) def testCsvDataset_withCR(self): # Test that when the line separator is '\r', parsing works with all buffer @@ -447,14 +478,8 @@ class CsvDatasetOpTest(test.TestCase): record_defaults = [['NA']] * 3 inputs = [['abc,def,ghi', '0,1,2', ',,']] expected = [['abc', 'def', 'ghi'], ['0', '1', '2'], ['NA', 'NA', 'NA']] - for i in range(20): - # Test a range of buffer sizes that should all work - self._test_dataset( - inputs, - expected, - linebreak='\r', - record_defaults=record_defaults, - buffer_size=i + 1) + self._test_dataset_on_buffer_sizes( + inputs, expected, linebreak='\r', record_defaults=record_defaults) def testCsvDataset_withCRLF(self): # Test that when the line separator is '\r\n', parsing works with all buffer @@ -462,29 +487,15 @@ class CsvDatasetOpTest(test.TestCase): record_defaults = [['NA']] * 3 inputs = [['abc,def,ghi', '0,1,2', ',,']] expected = [['abc', 'def', 'ghi'], ['0', '1', '2'], ['NA', 'NA', 'NA']] - for i in range(20): - # Test a range of buffer sizes that should all work - self._test_dataset( - inputs, - expected, - linebreak='\r\n', - record_defaults=record_defaults, - buffer_size=i + 1) + self._test_dataset_on_buffer_sizes( + inputs, expected, linebreak='\r\n', record_defaults=record_defaults) def testCsvDataset_withBufferSizeAndQuoted(self): record_defaults = [['NA']] * 3 inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']] expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'], ['NA', 'NA', 'NA']] - for i in range(20): - # Test a range of buffer sizes that should all work - self._test_dataset( - inputs, - expected, - linebreak='\n', - record_defaults=record_defaults, - buffer_size=i + 1) - self._test_dataset( + self._test_dataset_on_buffer_sizes( inputs, expected, linebreak='\n', record_defaults=record_defaults) def testCsvDataset_withCRAndQuoted(self): @@ -494,15 +505,7 @@ class CsvDatasetOpTest(test.TestCase): inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']] expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'], ['NA', 'NA', 'NA']] - for i in range(20): - # Test a range of buffer sizes that should all work - self._test_dataset( - inputs, - expected, - linebreak='\r', - record_defaults=record_defaults, - buffer_size=i + 1) - self._test_dataset( + self._test_dataset_on_buffer_sizes( inputs, expected, linebreak='\r', record_defaults=record_defaults) def testCsvDataset_withCRLFAndQuoted(self): @@ -512,17 +515,33 @@ class CsvDatasetOpTest(test.TestCase): inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']] expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'], ['NA', 'NA', 'NA']] - for i in range(20): - # Test a range of buffer sizes that should all work - self._test_dataset( - inputs, - expected, - linebreak='\r\n', - record_defaults=record_defaults, - buffer_size=i + 1) - self._test_dataset( + self._test_dataset_on_buffer_sizes( inputs, expected, linebreak='\r\n', record_defaults=record_defaults) + def testCsvDataset_withGzipCompressionType(self): + record_defaults = [['NA']] * 3 + inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']] + expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'], + ['NA', 'NA', 'NA']] + self._test_dataset_on_buffer_sizes( + inputs, + expected, + linebreak='\r\n', + compression_type='GZIP', + record_defaults=record_defaults) + + def testCsvDataset_withZlibCompressionType(self): + record_defaults = [['NA']] * 3 + inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']] + expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'], + ['NA', 'NA', 'NA']] + self._test_dataset_on_buffer_sizes( + inputs, + expected, + linebreak='\r\n', + compression_type='ZLIB', + record_defaults=record_defaults) + class CsvDatasetBenchmark(test.Benchmark): """Benchmarks for the various ways of creating a dataset from CSV files. diff --git a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py index 9df403ef50..851a33dfc8 100644 --- a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py @@ -17,13 +17,16 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import gzip import os +import zlib import numpy as np from tensorflow.contrib.data.python.kernel_tests import reader_dataset_ops_test_base from tensorflow.contrib.data.python.ops import readers from tensorflow.python.data.ops import readers as core_readers +from tensorflow.python.data.util import nest from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors @@ -182,264 +185,363 @@ class ReadBatchFeaturesTest( class MakeCsvDatasetTest(test.TestCase): - COLUMN_TYPES = [ - dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64, dtypes.string - ] - COLUMNS = ["col%d" % i for i in range(len(COLUMN_TYPES))] - DEFAULT_VALS = [[], [], [], [], ["NULL"]] - DEFAULTS = [ - constant_op.constant([], dtype=dtypes.int32), - constant_op.constant([], dtype=dtypes.int64), - constant_op.constant([], dtype=dtypes.float32), - constant_op.constant([], dtype=dtypes.float64), - constant_op.constant(["NULL"], dtype=dtypes.string) - ] - LABEL = COLUMNS[0] - - def setUp(self): - super(MakeCsvDatasetTest, self).setUp() - self._num_files = 2 - self._num_records = 11 - self._test_filenames = self._create_files() - - def _csv_values(self, fileno, recordno): - return [ - fileno, - recordno, - fileno * recordno * 0.5, - fileno * recordno + 0.5, - "record %d" % recordno if recordno % 2 == 1 else "", - ] + def _make_csv_dataset(self, filenames, batch_size, num_epochs=1, **kwargs): + return readers.make_csv_dataset( + filenames, batch_size=batch_size, num_epochs=num_epochs, **kwargs) - def _write_file(self, filename, rows): - for i in range(len(rows)): - if isinstance(rows[i], list): - rows[i] = ",".join(str(v) if v is not None else "" for v in rows[i]) - fn = os.path.join(self.get_temp_dir(), filename) - f = open(fn, "w") - f.write("\n".join(rows)) - f.close() - return fn - - def _create_file(self, fileno, header=True): - rows = [] - if header: - rows.append(self.COLUMNS) - for recno in range(self._num_records): - rows.append(self._csv_values(fileno, recno)) - return self._write_file("csv_file%d.csv" % fileno, rows) - - def _create_files(self): + def _setup_files(self, inputs, linebreak="\n", compression_type=None): filenames = [] - for i in range(self._num_files): - filenames.append(self._create_file(i)) + for i, ip in enumerate(inputs): + fn = os.path.join(self.get_temp_dir(), "temp_%d.csv" % i) + contents = linebreak.join(ip).encode("utf-8") + if compression_type is None: + with open(fn, "wb") as f: + f.write(contents) + elif compression_type == "GZIP": + with gzip.GzipFile(fn, "wb") as f: + f.write(contents) + elif compression_type == "ZLIB": + contents = zlib.compress(contents) + with open(fn, "wb") as f: + f.write(contents) + else: + raise ValueError("Unsupported compression_type", compression_type) + filenames.append(fn) return filenames - def _make_csv_dataset( - self, - filenames, - defaults, - column_names=COLUMNS, - label_name=LABEL, - select_cols=None, - batch_size=1, - num_epochs=1, - shuffle=False, - shuffle_seed=None, - header=True, - na_value="", - ): - return readers.make_csv_dataset( - filenames, - batch_size=batch_size, - column_names=column_names, - column_defaults=defaults, - label_name=label_name, - num_epochs=num_epochs, - shuffle=shuffle, - shuffle_seed=shuffle_seed, - header=header, - na_value=na_value, - select_columns=select_cols, - ) - - def _next_actual_batch(self, file_indices, batch_size, num_epochs, defaults): - features = {col: list() for col in self.COLUMNS} + def _next_expected_batch(self, expected_output, expected_keys, batch_size, + num_epochs): + features = {k: [] for k in expected_keys} for _ in range(num_epochs): - for i in file_indices: - for j in range(self._num_records): - values = self._csv_values(i, j) - for n, v in enumerate(values): - if v == "": # pylint: disable=g-explicit-bool-comparison - values[n] = defaults[n][0] - values[-1] = values[-1].encode("utf-8") - - # Regroup lists by column instead of row - for n, col in enumerate(self.COLUMNS): - features[col].append(values[n]) - if len(list(features.values())[0]) == batch_size: - yield features - features = {col: list() for col in self.COLUMNS} - - def _run_actual_batch(self, outputs, sess): - features, labels = sess.run(outputs) - batch = [features[k] for k in self.COLUMNS if k != self.LABEL] - batch.append(labels) - return batch - - def _verify_records( + for values in expected_output: + for n, key in enumerate(expected_keys): + features[key].append(values[n]) + if len(features[expected_keys[0]]) == batch_size: + yield features + features = {k: [] for k in expected_keys} + if features[expected_keys[0]]: # Leftover from the last batch + yield features + + def _verify_output( self, sess, dataset, - file_indices, - defaults=tuple(DEFAULT_VALS), - label_name=LABEL, - batch_size=1, - num_epochs=1, + batch_size, + num_epochs, + label_name, + expected_output, + expected_keys, ): - iterator = dataset.make_one_shot_iterator() - get_next = iterator.get_next() + nxt = dataset.make_one_shot_iterator().get_next() - for expected_features in self._next_actual_batch(file_indices, batch_size, - num_epochs, defaults): - actual_features = sess.run(get_next) + for expected_features in self._next_expected_batch( + expected_output, + expected_keys, + batch_size, + num_epochs, + ): + actual_features = sess.run(nxt) if label_name is not None: expected_labels = expected_features.pop(label_name) - # Compare labels self.assertAllEqual(expected_labels, actual_features[1]) - actual_features = actual_features[0] # Extract features dict from tuple + actual_features = actual_features[0] for k in expected_features.keys(): # Compare features self.assertAllEqual(expected_features[k], actual_features[k]) with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testMakeCSVDataset(self): - defaults = self.DEFAULTS - - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - # Basic test: read from file 0. - dataset = self._make_csv_dataset(self._test_filenames[0], defaults) - self._verify_records(sess, dataset, [0]) - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - # Basic test: read from file 1. - dataset = self._make_csv_dataset(self._test_filenames[1], defaults) - self._verify_records(sess, dataset, [1]) + sess.run(nxt) + + def _test_dataset(self, + inputs, + expected_output, + expected_keys, + batch_size=1, + num_epochs=1, + label_name=None, + **kwargs): + """Checks that elements produced by CsvDataset match expected output.""" + # Convert str type because py3 tf strings are bytestrings + filenames = self._setup_files( + inputs, compression_type=kwargs.get("compression_type", None)) with ops.Graph().as_default() as g: with self.test_session(graph=g) as sess: - # Read from both files. - dataset = self._make_csv_dataset(self._test_filenames, defaults) - self._verify_records(sess, dataset, range(self._num_files)) - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - # Read from both files. Exercise the `batch` and `num_epochs` parameters - # of make_csv_dataset and make sure they work. dataset = self._make_csv_dataset( - self._test_filenames, defaults, batch_size=2, num_epochs=10) - self._verify_records( - sess, dataset, range(self._num_files), batch_size=2, num_epochs=10) + filenames, + batch_size=batch_size, + num_epochs=num_epochs, + label_name=label_name, + **kwargs) + self._verify_output(sess, dataset, batch_size, num_epochs, label_name, + expected_output, expected_keys) + + def testMakeCSVDataset(self): + """Tests making a CSV dataset with keys and defaults provided.""" + record_defaults = [ + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.int64), + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.float64), + constant_op.constant([], dtypes.string) + ] + + column_names = ["col%d" % i for i in range(5)] + inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [ + ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19" + ]] + expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"], + [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]] + label = "col0" + + self._test_dataset( + inputs, + expected_output=expected_output, + expected_keys=column_names, + column_names=column_names, + label_name=label, + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + column_defaults=record_defaults, + ) + + def testMakeCSVDataset_withBatchSizeAndEpochs(self): + """Tests making a CSV dataset with keys and defaults provided.""" + record_defaults = [ + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.int64), + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.float64), + constant_op.constant([], dtypes.string) + ] + + column_names = ["col%d" % i for i in range(5)] + inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [ + ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19" + ]] + expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"], + [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]] + label = "col0" + + self._test_dataset( + inputs, + expected_output=expected_output, + expected_keys=column_names, + column_names=column_names, + label_name=label, + batch_size=3, + num_epochs=10, + shuffle=False, + header=True, + column_defaults=record_defaults, + ) - def testMakeCSVDataset_withBadColumns(self): + def testMakeCSVDataset_withCompressionType(self): + """Tests `compression_type` argument.""" + record_defaults = [ + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.int64), + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.float64), + constant_op.constant([], dtypes.string) + ] + + column_names = ["col%d" % i for i in range(5)] + inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [ + ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19" + ]] + expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"], + [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]] + label = "col0" + + for compression_type in ("GZIP", "ZLIB"): + self._test_dataset( + inputs, + expected_output=expected_output, + expected_keys=column_names, + column_names=column_names, + label_name=label, + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + column_defaults=record_defaults, + compression_type=compression_type, + ) + + def testMakeCSVDataset_withBadInputs(self): """Tests that exception is raised when input is malformed. """ - dupe_columns = self.COLUMNS[:-1] + self.COLUMNS[:1] - defaults = self.DEFAULTS + record_defaults = [ + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.int64), + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.float64), + constant_op.constant([], dtypes.string) + ] + + column_names = ["col%d" % i for i in range(5)] + inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [ + ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19" + ]] + filenames = self._setup_files(inputs) # Duplicate column names with self.assertRaises(ValueError): self._make_csv_dataset( - self._test_filenames, defaults, column_names=dupe_columns) + filenames, + batch_size=1, + column_defaults=record_defaults, + label_name="col0", + column_names=column_names * 2) # Label key not one of column names with self.assertRaises(ValueError): self._make_csv_dataset( - self._test_filenames, defaults, label_name="not_a_real_label") + filenames, + batch_size=1, + column_defaults=record_defaults, + label_name="not_a_real_label", + column_names=column_names) def testMakeCSVDataset_withNoLabel(self): - """Tests that CSV datasets can be created when no label is specified. - """ - defaults = self.DEFAULTS - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - # Read from both files. Make sure this works with no label key supplied. - dataset = self._make_csv_dataset( - self._test_filenames, - defaults, - batch_size=2, - num_epochs=10, - label_name=None) - self._verify_records( - sess, - dataset, - range(self._num_files), - batch_size=2, - num_epochs=10, - label_name=None) + """Tests making a CSV dataset with no label provided.""" + record_defaults = [ + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.int64), + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.float64), + constant_op.constant([], dtypes.string) + ] + + column_names = ["col%d" % i for i in range(5)] + inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [ + ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19" + ]] + expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"], + [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]] + + self._test_dataset( + inputs, + expected_output=expected_output, + expected_keys=column_names, + column_names=column_names, + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + column_defaults=record_defaults, + ) def testMakeCSVDataset_withNoHeader(self): """Tests that datasets can be created from CSV files with no header line. """ - defaults = self.DEFAULTS - file_without_header = self._create_file( - len(self._test_filenames), header=False) - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - dataset = self._make_csv_dataset( - file_without_header, - defaults, - batch_size=2, - num_epochs=10, - header=False, - ) - self._verify_records( - sess, - dataset, - [len(self._test_filenames)], - batch_size=2, - num_epochs=10, - ) + record_defaults = [ + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.int64), + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.float64), + constant_op.constant([], dtypes.string) + ] + + column_names = ["col%d" % i for i in range(5)] + inputs = [["0,1,2,3,4", "5,6,7,8,9"], ["10,11,12,13,14", "15,16,17,18,19"]] + expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"], + [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]] + label = "col0" + + self._test_dataset( + inputs, + expected_output=expected_output, + expected_keys=column_names, + column_names=column_names, + label_name=label, + batch_size=1, + num_epochs=1, + shuffle=False, + header=False, + column_defaults=record_defaults, + ) def testMakeCSVDataset_withTypes(self): """Tests that defaults can be a dtype instead of a Tensor for required vals. """ - defaults = [d for d in self.COLUMN_TYPES[:-1]] - defaults.append(constant_op.constant(["NULL"], dtype=dtypes.string)) - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - dataset = self._make_csv_dataset(self._test_filenames, defaults) - self._verify_records(sess, dataset, range(self._num_files)) + record_defaults = [ + dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64, + dtypes.string + ] + + column_names = ["col%d" % i for i in range(5)] + inputs = [[",".join(x[0] for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], + [ + ",".join(x[0] for x in column_names), "10,11,12,13,14", + "15,16,17,18,19" + ]] + expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"], + [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]] + label = "col0" + + self._test_dataset( + inputs, + expected_output=expected_output, + expected_keys=column_names, + column_names=column_names, + label_name=label, + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + column_defaults=record_defaults, + ) def testMakeCSVDataset_withNoColNames(self): """Tests that datasets can be created when column names are not specified. In that case, we should infer the column names from the header lines. """ - defaults = self.DEFAULTS - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - # Read from both files. Exercise the `batch` and `num_epochs` parameters - # of make_csv_dataset and make sure they work. - dataset = self._make_csv_dataset( - self._test_filenames, - defaults, - column_names=None, - batch_size=2, - num_epochs=10) - self._verify_records( - sess, dataset, range(self._num_files), batch_size=2, num_epochs=10) + record_defaults = [ + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.int64), + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.float64), + constant_op.constant([], dtypes.string) + ] + + column_names = ["col%d" % i for i in range(5)] + inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [ + ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19" + ]] + expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"], + [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]] + label = "col0" + + self._test_dataset( + inputs, + expected_output=expected_output, + expected_keys=column_names, + label_name=label, + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + column_defaults=record_defaults, + ) def testMakeCSVDataset_withTypeInferenceMismatch(self): # Test that error is thrown when num fields doesn't match columns + column_names = ["col%d" % i for i in range(5)] + inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [ + ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19" + ]] + filenames = self._setup_files(inputs) with self.assertRaises(ValueError): self._make_csv_dataset( - self._test_filenames, - column_names=self.COLUMNS + ["extra_name"], - defaults=None, + filenames, + column_names=column_names + ["extra_name"], + column_defaults=None, batch_size=2, num_epochs=10) @@ -448,197 +550,215 @@ class MakeCsvDatasetTest(test.TestCase): In that case, we should infer the types from the first N records. """ - # Test that it works with standard test files (with header, etc) - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - dataset = self._make_csv_dataset( - self._test_filenames, defaults=None, batch_size=2, num_epochs=10) - self._verify_records( - sess, - dataset, - range(self._num_files), - batch_size=2, - num_epochs=10, - defaults=[[], [], [], [], [""]]) - - def testMakeCSVDataset_withTypeInferenceTricky(self): - # Test on a deliberately tricky file (type changes as we read more rows, and - # there are null values) - fn = os.path.join(self.get_temp_dir(), "file.csv") - expected_dtypes = [ - dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float32, - dtypes.string, dtypes.string - ] - col_names = ["col%d" % i for i in range(len(expected_dtypes))] - rows = [[None, None, None, "NAN", "", - "a"], [1, 2**31 + 1, 2**64, 123, "NAN", ""], - ['"123"', 2, 2**64, 123.4, "NAN", '"cd,efg"']] - expected = [[0, 0, 0, 0, "", "a"], [1, 2**31 + 1, 2**64, 123, "", ""], - [123, 2, 2**64, 123.4, "", "cd,efg"]] - for row in expected: - row[-1] = row[-1].encode("utf-8") # py3 expects byte strings - row[-2] = row[-2].encode("utf-8") # py3 expects byte strings - self._write_file("file.csv", [col_names] + rows) + column_names = ["col%d" % i for i in range(5)] + str_int32_max = str(2**33) + inputs = [[ + ",".join(x for x in column_names), + "0,%s,2.0,3e50,rabbit" % str_int32_max + ]] + expected_output = [[0, 2**33, 2.0, 3e50, b"rabbit"]] + label = "col0" - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - dataset = self._make_csv_dataset( - fn, - defaults=None, - column_names=None, - label_name=None, - na_value="NAN", - ) - features = dataset.make_one_shot_iterator().get_next() - # Check that types match - for i in range(len(expected_dtypes)): - print(features["col%d" % i].dtype, expected_dtypes[i]) - assert features["col%d" % i].dtype == expected_dtypes[i] - for i in range(len(rows)): - assert sess.run(features) == dict(zip(col_names, expected[i])) - - def testMakeCSVDataset_withTypeInferenceAllTypes(self): - # Test that we make the correct inference for all types with fallthrough - fn = os.path.join(self.get_temp_dir(), "file.csv") - expected_dtypes = [ - dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64, - dtypes.string, dtypes.string + self._test_dataset( + inputs, + expected_output=expected_output, + expected_keys=column_names, + column_names=column_names, + label_name=label, + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + ) + + def testMakeCSVDataset_withTypeInferenceFallthrough(self): + """Tests that datasets can be created when no defaults are specified. + + Tests on a deliberately tricky file. + """ + column_names = ["col%d" % i for i in range(5)] + str_int32_max = str(2**33) + inputs = [[ + ",".join(x for x in column_names), + ",,,,", + "0,0,0.0,0.0,0.0", + "0,%s,2.0,3e50,rabbit" % str_int32_max, + ",,,,", + ]] + expected_output = [[0, 0, 0, 0, b""], [0, 0, 0, 0, b"0.0"], + [0, 2**33, 2.0, 3e50, b"rabbit"], [0, 0, 0, 0, b""]] + label = "col0" + + self._test_dataset( + inputs, + expected_output=expected_output, + expected_keys=column_names, + column_names=column_names, + label_name=label, + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + ) + + def testMakeCSVDataset_withSelectCols(self): + record_defaults = [ + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.int64), + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.float64), + constant_op.constant([], dtypes.string) ] - col_names = ["col%d" % i for i in range(len(expected_dtypes))] - rows = [[1, 2**31 + 1, 1.0, 4e40, "abc", ""]] - expected = [[ - 1, 2**31 + 1, 1.0, 4e40, "abc".encode("utf-8"), "".encode("utf-8") + column_names = ["col%d" % i for i in range(5)] + str_int32_max = str(2**33) + inputs = [[ + ",".join(x for x in column_names), + "0,%s,2.0,3e50,rabbit" % str_int32_max ]] - self._write_file("file.csv", [col_names] + rows) + expected_output = [[0, 2**33, 2.0, 3e50, b"rabbit"]] - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - dataset = self._make_csv_dataset( - fn, - defaults=None, - column_names=None, - label_name=None, - na_value="NAN", - ) - features = dataset.make_one_shot_iterator().get_next() - # Check that types match - for i in range(len(expected_dtypes)): - self.assertAllEqual(features["col%d" % i].dtype, expected_dtypes[i]) - for i in range(len(rows)): - self.assertAllEqual( - sess.run(features), dict(zip(col_names, expected[i]))) + select_cols = [1, 3, 4] + self._test_dataset( + inputs, + expected_output=[[x[i] for i in select_cols] for x in expected_output], + expected_keys=[column_names[i] for i in select_cols], + column_names=column_names, + column_defaults=[record_defaults[i] for i in select_cols], + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + select_columns=select_cols, + ) + + # Can still do inference without provided defaults + self._test_dataset( + inputs, + expected_output=[[x[i] for i in select_cols] for x in expected_output], + expected_keys=[column_names[i] for i in select_cols], + column_names=column_names, + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + select_columns=select_cols, + ) + + # Can still do column name inference + self._test_dataset( + inputs, + expected_output=[[x[i] for i in select_cols] for x in expected_output], + expected_keys=[column_names[i] for i in select_cols], + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + select_columns=select_cols, + ) + + # Can specify column names instead of indices + self._test_dataset( + inputs, + expected_output=[[x[i] for i in select_cols] for x in expected_output], + expected_keys=[column_names[i] for i in select_cols], + column_names=column_names, + batch_size=1, + num_epochs=1, + shuffle=False, + header=True, + select_columns=[column_names[i] for i in select_cols], + ) def testMakeCSVDataset_withSelectColsError(self): - data = [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]] - col_names = ["col%d" % i for i in range(5)] - fn = self._write_file("file.csv", [col_names] + data) + record_defaults = [ + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.int64), + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.float64), + constant_op.constant([], dtypes.string) + ] + column_names = ["col%d" % i for i in range(5)] + str_int32_max = str(2**33) + inputs = [[ + ",".join(x for x in column_names), + "0,%s,2.0,3e50,rabbit" % str_int32_max + ]] + + select_cols = [1, 3, 4] + filenames = self._setup_files(inputs) + with self.assertRaises(ValueError): # Mismatch in number of defaults and number of columns selected, # should raise an error self._make_csv_dataset( - fn, - defaults=[[0]] * 5, - column_names=col_names, - label_name=None, - select_cols=[1, 3]) + filenames, + batch_size=1, + column_defaults=record_defaults, + column_names=column_names, + select_columns=select_cols) + with self.assertRaises(ValueError): # Invalid column name should raise an error self._make_csv_dataset( - fn, - defaults=[[0]], - column_names=col_names, + filenames, + batch_size=1, + column_defaults=[[0]], + column_names=column_names, label_name=None, - select_cols=["invalid_col_name"]) - - def testMakeCSVDataset_withSelectCols(self): - data = [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]] - col_names = ["col%d" % i for i in range(5)] - fn = self._write_file("file.csv", [col_names] + data) - # If select_cols is specified, should only yield a subset of columns - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - dataset = self._make_csv_dataset( - fn, - defaults=[[0], [0]], - column_names=col_names, - label_name=None, - select_cols=[1, 3]) - expected = [[1, 3], [6, 8]] - features = dataset.make_one_shot_iterator().get_next() - for i in range(len(data)): - self.assertAllEqual( - sess.run(features), - dict(zip([col_names[1], col_names[3]], expected[i]))) - # Can still do default inference with select_cols - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - dataset = self._make_csv_dataset( - fn, - defaults=None, - column_names=col_names, - label_name=None, - select_cols=[1, 3]) - expected = [[1, 3], [6, 8]] - features = dataset.make_one_shot_iterator().get_next() - for i in range(len(data)): - self.assertAllEqual( - sess.run(features), - dict(zip([col_names[1], col_names[3]], expected[i]))) - # Can still do column name inference - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - dataset = self._make_csv_dataset( - fn, - defaults=None, - column_names=None, - label_name=None, - select_cols=[1, 3]) - expected = [[1, 3], [6, 8]] - features = dataset.make_one_shot_iterator().get_next() - for i in range(len(data)): - self.assertAllEqual( - sess.run(features), - dict(zip([col_names[1], col_names[3]], expected[i]))) - # Can specify column names instead of indices - with ops.Graph().as_default() as g: - with self.test_session(graph=g) as sess: - dataset = self._make_csv_dataset( - fn, - defaults=None, - column_names=None, - label_name=None, - select_cols=[col_names[1], col_names[3]]) - expected = [[1, 3], [6, 8]] - features = dataset.make_one_shot_iterator().get_next() - for i in range(len(data)): - self.assertAllEqual( - sess.run(features), - dict(zip([col_names[1], col_names[3]], expected[i]))) + select_columns=["invalid_col_name"]) def testMakeCSVDataset_withShuffle(self): - total_records = self._num_files * self._num_records - defaults = self.DEFAULTS + record_defaults = [ + constant_op.constant([], dtypes.int32), + constant_op.constant([], dtypes.int64), + constant_op.constant([], dtypes.float32), + constant_op.constant([], dtypes.float64), + constant_op.constant([], dtypes.string) + ] + + def str_series(st): + return ",".join(str(i) for i in range(st, st + 5)) + + column_names = ["col%d" % i for i in range(5)] + inputs = [ + [",".join(x for x in column_names) + ] + [str_series(5 * i) for i in range(15)], + [",".join(x for x in column_names)] + + [str_series(5 * i) for i in range(15, 20)], + ] + + filenames = self._setup_files(inputs) + + total_records = 20 for batch_size in [1, 2]: with ops.Graph().as_default() as g: with self.test_session(graph=g) as sess: # Test that shuffling with the same seed produces the same result dataset1 = self._make_csv_dataset( - self._test_filenames, - defaults, + filenames, + column_defaults=record_defaults, + column_names=column_names, batch_size=batch_size, + header=True, shuffle=True, - shuffle_seed=5) + shuffle_seed=5, + num_epochs=2, + ) dataset2 = self._make_csv_dataset( - self._test_filenames, - defaults, + filenames, + column_defaults=record_defaults, + column_names=column_names, batch_size=batch_size, + header=True, shuffle=True, - shuffle_seed=5) + shuffle_seed=5, + num_epochs=2, + ) outputs1 = dataset1.make_one_shot_iterator().get_next() outputs2 = dataset2.make_one_shot_iterator().get_next() for _ in range(total_records // batch_size): - batch1 = self._run_actual_batch(outputs1, sess) - batch2 = self._run_actual_batch(outputs2, sess) + batch1 = nest.flatten(sess.run(outputs1)) + batch2 = nest.flatten(sess.run(outputs2)) for i in range(len(batch1)): self.assertAllEqual(batch1[i], batch2[i]) @@ -646,23 +766,31 @@ class MakeCsvDatasetTest(test.TestCase): with self.test_session(graph=g) as sess: # Test that shuffling with a different seed produces different results dataset1 = self._make_csv_dataset( - self._test_filenames, - defaults, + filenames, + column_defaults=record_defaults, + column_names=column_names, batch_size=batch_size, + header=True, shuffle=True, - shuffle_seed=5) + shuffle_seed=5, + num_epochs=2, + ) dataset2 = self._make_csv_dataset( - self._test_filenames, - defaults, + filenames, + column_defaults=record_defaults, + column_names=column_names, batch_size=batch_size, + header=True, shuffle=True, - shuffle_seed=6) + shuffle_seed=6, + num_epochs=2, + ) outputs1 = dataset1.make_one_shot_iterator().get_next() outputs2 = dataset2.make_one_shot_iterator().get_next() all_equal = False for _ in range(total_records // batch_size): - batch1 = self._run_actual_batch(outputs1, sess) - batch2 = self._run_actual_batch(outputs2, sess) + batch1 = nest.flatten(sess.run(outputs1)) + batch2 = nest.flatten(sess.run(outputs2)) for i in range(len(batch1)): all_equal = all_equal and np.array_equal(batch1[i], batch2[i]) self.assertFalse(all_equal) @@ -874,6 +1002,5 @@ class MakeTFRecordDatasetTest( self._shuffle_test(batch_size, num_epochs, num_parallel_reads, seed=21345) - if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py index 9373e37f5f..f018dd02e6 100644 --- a/tensorflow/contrib/data/python/ops/readers.py +++ b/tensorflow/contrib/data/python/ops/readers.py @@ -326,6 +326,7 @@ def make_csv_dataset( num_parallel_parser_calls=2, sloppy=False, num_rows_for_inference=100, + compression_type=None, ): """Reads CSV files into a dataset. @@ -399,6 +400,8 @@ def make_csv_dataset( num_rows_for_inference: Number of rows of a file to use for type inference if record_defaults is not provided. If None, reads all the rows of all the files. Defaults to 100. + compression_type: (Optional.) A `tf.string` scalar evaluating to one of + `""` (no compression), `"ZLIB"`, or `"GZIP"`. Defaults to no compression. Returns: A dataset, where each element is a (features, labels) tuple that corresponds @@ -461,7 +464,9 @@ def make_csv_dataset( use_quote_delim=use_quote_delim, na_value=na_value, select_cols=select_columns, - header=header) + header=header, + compression_type=compression_type, + ) def map_fn(*columns): """Organizes columns into a features dictionary. @@ -505,6 +510,7 @@ class CsvDataset(dataset_ops.Dataset): def __init__(self, filenames, record_defaults, + compression_type=None, buffer_size=None, header=False, field_delim=",", @@ -562,6 +568,9 @@ class CsvDataset(dataset_ops.Dataset): both this and `select_columns` are specified, these must have the same lengths, and `column_defaults` is assumed to be sorted in order of increasing column index. + compression_type: (Optional.) A `tf.string` scalar evaluating to one of + `""` (no compression), `"ZLIB"`, or `"GZIP"`. Defaults to no + compression. buffer_size: (Optional.) A `tf.int64` scalar denoting the number of bytes to buffer while reading files. Defaults to 4MB. header: (Optional.) A `tf.bool` scalar indicating whether the CSV file(s) @@ -581,6 +590,11 @@ class CsvDataset(dataset_ops.Dataset): super(CsvDataset, self).__init__() self._filenames = ops.convert_to_tensor( filenames, dtype=dtypes.string, name="filenames") + self._compression_type = convert.optional_param_to_tensor( + "compression_type", + compression_type, + argument_default="", + argument_dtype=dtypes.string) record_defaults = [ constant_op.constant([], dtype=x) if x in _ACCEPTABLE_CSV_TYPES else x for x in record_defaults @@ -621,6 +635,7 @@ class CsvDataset(dataset_ops.Dataset): use_quote_delim=self._use_quote_delim, na_value=self._na_value, select_cols=self._select_cols, + compression_type=self._compression_type, ) @property -- cgit v1.2.3 From 74fce066580ca286b2c776a64ab624f12a473b28 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 09:37:57 -0700 Subject: BEGIN_PUBLIC Add initial support for interpolating filename and line number in error messages returned from C++. END_PUBLIC RELEASE_NOTES: n/a Automated rollback of commit 8e7d3dc7326bb81ef55175c48f51436408219c4a PiperOrigin-RevId: 204755755 --- tensorflow/python/BUILD | 5 +- tensorflow/python/framework/error_interpolation.py | 82 +++++++++++++++- .../python/framework/error_interpolation_test.py | 104 +++++++++++++++++++-- tensorflow/python/util/tf_stack.py | 6 ++ 4 files changed, 182 insertions(+), 15 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 924db54cbc..2fba3c2acb 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -705,7 +705,9 @@ py_library( "framework/error_interpolation.py", ], srcs_version = "PY2AND3", - deps = [], + deps = [ + ":util", + ], ) py_library( @@ -1040,6 +1042,7 @@ py_test( srcs_version = "PY2AND3", deps = [ ":client_testlib", + ":constant_op", ":error_interpolation", ], ) diff --git a/tensorflow/python/framework/error_interpolation.py b/tensorflow/python/framework/error_interpolation.py index 9ccae76147..519e0fda0a 100644 --- a/tensorflow/python/framework/error_interpolation.py +++ b/tensorflow/python/framework/error_interpolation.py @@ -29,6 +29,9 @@ import string import six +from tensorflow.python.util import tf_stack + + _NAME_REGEX = r"[A-Za-z0-9.][A-Za-z0-9_.\-/]*?" _FORMAT_REGEX = r"[A-Za-z0-9_.\-/${}:]+" _TAG_REGEX = r"\^\^({name}):({name}):({fmt})\^\^".format( @@ -38,6 +41,8 @@ _INTERPOLATION_PATTERN = re.compile(_INTERPOLATION_REGEX) _ParseTag = collections.namedtuple("_ParseTag", ["type", "name", "format"]) +_BAD_FILE_SUBSTRINGS = ["tensorflow/python", " + file: Replaced with the filename in which the node was defined. + line: Replaced by the line number at which the node was defined. + Args: message: String to parse @@ -72,9 +81,47 @@ def _parse_message(message): return seps, tags -# TODO(jtkeeling): Modify to actually interpolate format strings rather than -# echoing them. -def interpolate(error_message): +def _get_field_dict_from_traceback(tf_traceback, frame_index): + """Convert traceback elements into interpolation dictionary and return.""" + frame = tf_traceback[frame_index] + return { + "file": frame[tf_stack.TB_FILENAME], + "line": frame[tf_stack.TB_LINENO], + } + + +def _find_index_of_defining_frame_for_op(op): + """Return index in op._traceback with first 'useful' frame. + + This method reads through the stack stored in op._traceback looking for the + innermost frame which (hopefully) belongs to the caller. It accomplishes this + by rejecting frames whose filename appears to come from TensorFlow (see + error_interpolation._BAD_FILE_SUBSTRINGS for the list of rejected substrings). + + Args: + op: the Operation object for which we would like to find the defining + location. + + Returns: + Integer index into op._traceback where the first non-TF file was found + (innermost to outermost), or 0 (for the outermost stack frame) if all files + came from TensorFlow. + """ + # pylint: disable=protected-access + # Index 0 of tf_traceback is the outermost frame. + tf_traceback = tf_stack.convert_stack(op._traceback) + size = len(tf_traceback) + # pylint: enable=protected-access + filenames = [frame[tf_stack.TB_FILENAME] for frame in tf_traceback] + # We process the filenames from the innermost frame to outermost. + for idx, filename in enumerate(reversed(filenames)): + contains_bad_substrings = [ss in filename for ss in _BAD_FILE_SUBSTRINGS] + if not any(contains_bad_substrings): + return size - idx - 1 + return 0 + + +def interpolate(error_message, graph): """Interpolates an error message. The error message can contain tags of the form ^^type:name:format^^ which will @@ -82,11 +129,38 @@ def interpolate(error_message): Args: error_message: A string to interpolate. + graph: ops.Graph object containing all nodes referenced in the error + message. Returns: The string with tags of the form ^^type:name:format^^ interpolated. """ seps, tags = _parse_message(error_message) - subs = [string.Template(tag.format).safe_substitute({}) for tag in tags] + + node_name_to_substitution_dict = {} + for name in [t.name for t in tags]: + try: + op = graph.get_operation_by_name(name) + except KeyError: + op = None + + if op: + frame_index = _find_index_of_defining_frame_for_op(op) + # pylint: disable=protected-access + field_dict = _get_field_dict_from_traceback(op._traceback, frame_index) + # pylint: enable=protected-access + else: + field_dict = { + "file": "", + "line": "", + "func": "", + "code": None, + } + node_name_to_substitution_dict[name] = field_dict + + subs = [ + string.Template(tag.format).safe_substitute( + node_name_to_substitution_dict[tag.name]) for tag in tags + ] return "".join( itertools.chain(*six.moves.zip_longest(seps, subs, fillvalue=""))) diff --git a/tensorflow/python/framework/error_interpolation_test.py b/tensorflow/python/framework/error_interpolation_test.py index ad448deb62..6d19f75586 100644 --- a/tensorflow/python/framework/error_interpolation_test.py +++ b/tensorflow/python/framework/error_interpolation_test.py @@ -18,31 +18,115 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import constant_op from tensorflow.python.framework import error_interpolation from tensorflow.python.platform import test +from tensorflow.python.util import tf_stack + + +def _make_frame_with_filename(op, idx, filename): + """Return a copy of an existing stack frame with a new filename.""" + stack_frame = list(op._traceback[idx]) + stack_frame[tf_stack.TB_FILENAME] = filename + return tuple(stack_frame) + + +def _modify_op_stack_with_filenames(op, num_user_frames, user_filename, + num_inner_tf_frames): + """Replace op._traceback with a new traceback using special filenames.""" + tf_filename = "%d" + error_interpolation._BAD_FILE_SUBSTRINGS[0] + user_filename = "%d/my_favorite_file.py" + + num_requested_frames = num_user_frames + num_inner_tf_frames + num_actual_frames = len(op._traceback) + num_outer_frames = num_actual_frames - num_requested_frames + assert num_requested_frames <= num_actual_frames, "Too few real frames." + + # The op's traceback has outermost frame at index 0. + stack = [] + for idx in range(0, num_outer_frames): + stack.append(op._traceback[idx]) + for idx in range(len(stack), len(stack)+num_user_frames): + stack.append(_make_frame_with_filename(op, idx, user_filename % idx)) + for idx in range(len(stack), len(stack)+num_inner_tf_frames): + stack.append(_make_frame_with_filename(op, idx, tf_filename % idx)) + op._traceback = stack class InterpolateTest(test.TestCase): + def setUp(self): + # Add nodes to the graph for retrieval by name later. + constant_op.constant(1, name="One") + constant_op.constant(2, name="Two") + three = constant_op.constant(3, name="Three") + self.graph = three.graph + + # Change the list of bad file substrings so that constant_op.py is chosen + # as the defining stack frame for constant_op.constant ops. + self.old_bad_strings = error_interpolation._BAD_FILE_SUBSTRINGS + error_interpolation._BAD_FILE_SUBSTRINGS = ["/ops.py", "/util"] + + def tearDown(self): + error_interpolation._BAD_FILE_SUBSTRINGS = self.old_bad_strings + + def testFindIndexOfDefiningFrameForOp(self): + local_op = constant_op.constant(42).op + user_filename = "hope.py" + _modify_op_stack_with_filenames(local_op, + num_user_frames=3, + user_filename=user_filename, + num_inner_tf_frames=5) + idx = error_interpolation._find_index_of_defining_frame_for_op(local_op) + # Expected frame is 6th from the end because there are 5 inner frames witih + # TF filenames. + expected_frame = len(local_op._traceback) - 6 + self.assertEqual(expected_frame, idx) + + def testFindIndexOfDefiningFrameForOpReturnsZeroOnError(self): + local_op = constant_op.constant(43).op + # Truncate stack to known length. + local_op._traceback = local_op._traceback[:7] + # Ensure all frames look like TF frames. + _modify_op_stack_with_filenames(local_op, + num_user_frames=0, + user_filename="user_file.py", + num_inner_tf_frames=7) + idx = error_interpolation._find_index_of_defining_frame_for_op(local_op) + self.assertEqual(0, idx) + def testNothingToDo(self): normal_string = "This is just a normal string" - interpolated_string = error_interpolation.interpolate(normal_string) + interpolated_string = error_interpolation.interpolate(normal_string, + self.graph) self.assertEqual(interpolated_string, normal_string) def testOneTag(self): - one_tag_string = "^^node:Foo:${file}^^" - interpolated_string = error_interpolation.interpolate(one_tag_string) - self.assertEqual(interpolated_string, "${file}") + one_tag_string = "^^node:Two:${file}^^" + interpolated_string = error_interpolation.interpolate(one_tag_string, + self.graph) + self.assertTrue(interpolated_string.endswith("op.py"), + "interpolated_string '%s' did not end with op.py" + % interpolated_string) + + def testOneTagWithAFakeNameResultsInPlaceholders(self): + one_tag_string = "^^node:MinusOne:${file}^^" + interpolated_string = error_interpolation.interpolate(one_tag_string, + self.graph) + self.assertEqual(interpolated_string, "") def testTwoTagsNoSeps(self): - two_tags_no_seps = "^^node:Foo:${file}^^^^node:Bar:${line}^^" - interpolated_string = error_interpolation.interpolate(two_tags_no_seps) - self.assertEqual(interpolated_string, "${file}${line}") + two_tags_no_seps = "^^node:One:${file}^^^^node:Three:${line}^^" + interpolated_string = error_interpolation.interpolate(two_tags_no_seps, + self.graph) + self.assertRegexpMatches(interpolated_string, "op.py[0-9]+") def testTwoTagsWithSeps(self): - two_tags_with_seps = "123^^node:Foo:${file}^^456^^node:Bar:${line}^^789" - interpolated_string = error_interpolation.interpolate(two_tags_with_seps) - self.assertEqual(interpolated_string, "123${file}456${line}789") + two_tags_with_seps = ";;;^^node:Two:${file}^^,,,^^node:Three:${line}^^;;;" + interpolated_string = error_interpolation.interpolate(two_tags_with_seps, + self.graph) + expected_regex = "^;;;.*op.py,,,[0-9]*;;;$" + self.assertRegexpMatches(interpolated_string, expected_regex) if __name__ == "__main__": diff --git a/tensorflow/python/util/tf_stack.py b/tensorflow/python/util/tf_stack.py index dacc1ce83e..fe4f4a63eb 100644 --- a/tensorflow/python/util/tf_stack.py +++ b/tensorflow/python/util/tf_stack.py @@ -21,6 +21,12 @@ from __future__ import print_function import linecache import sys +# Names for indices into TF traceback tuples. +TB_FILENAME = 0 +TB_LINENO = 1 +TB_FUNCNAME = 2 +TB_CODEDICT = 3 # Dictionary of Python interpreter state. + def extract_stack(extract_frame_info_fn=None): """A lightweight, extensible re-implementation of traceback.extract_stack. -- cgit v1.2.3 From b70a39b4e67eeb2b3c2a635f1ef6d4e1c8b0f82a Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Mon, 16 Jul 2018 09:48:27 -0700 Subject: Two speed test notebooks: MNIST training loop and Collatz. PiperOrigin-RevId: 204757439 --- .../notebooks/ag_vs_eager_collatz_speed_test.ipynb | 299 ++++++++++ .../notebooks/ag_vs_eager_mnist_speed_test.ipynb | 652 +++++++++++++++++++++ .../autograph_vs_eager_mnist_benchmark.ipynb | 577 ------------------ 3 files changed, 951 insertions(+), 577 deletions(-) create mode 100644 tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_collatz_speed_test.ipynb create mode 100644 tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_mnist_speed_test.ipynb delete mode 100644 tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb diff --git a/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_collatz_speed_test.ipynb b/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_collatz_speed_test.ipynb new file mode 100644 index 0000000000..c10a5741f6 --- /dev/null +++ b/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_collatz_speed_test.ipynb @@ -0,0 +1,299 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "aQkTGc-d8I1k" + }, + "source": [ + "This notebook runs a basic speed test for a simple algorithm that implements the process described in Collatz Conjecture.\n", + "\n", + "https://en.wikipedia.org/wiki/Collatz_conjecture" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "x5ChBlH09jk_" + }, + "source": [ + "### Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "X-QAUpWdPxUh" + }, + "outputs": [], + "source": [ + "!pip install -U -q tf-nightly" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "wiKQu3w05eCa" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "from matplotlib import pyplot as plt\n", + "import tensorflow as tf\n", + "from tensorflow.contrib import autograph as ag\n", + "from tensorflow.python.eager import context" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "_cRFTcwT9mnn" + }, + "source": [ + "### Plotting helpers" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "ww7rc0GQ9pMu" + }, + "outputs": [], + "source": [ + "def plot_results(counts, times, title):\n", + " plt.plot(counts, np.array(times) * 1000., 'o')\n", + " plt.ylabel('Time (milliseconds)')\n", + " plt.xlabel('Collatz counter')\n", + " plt.title(title)\n", + " plt.ylim(0, 30)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "ESZGw9s9-Y5_" + }, + "source": [ + "### Collatz function definition" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "qeunWm9m-dT7" + }, + "outputs": [], + "source": [ + "def collatz(a):\n", + " count = 0\n", + " while a \u003e 1.1:\n", + " if a % 2 \u003c 0.1:\n", + " a //= 2\n", + " else:\n", + " a = 3 * a + 1\n", + " count += 1\n", + " return count\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "nnFmPDvScsDo" + }, + "source": [ + "# AutoGraph" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 301 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 9153, + "status": "ok", + "timestamp": 1531757473651, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "6fU4vlxYcsDe", + "outputId": "11b50f28-aced-4506-a743-4b749e9645c3" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEcCAYAAAAydkhNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtcVGXCB/DfGRBUQA0ZURQvyIspm1reQkxNSPICgoqW\npWZu1vbmjZJV3Jc+axappVLu7guV25rU5g3wlq3iBd1wXHSN3hXy9ZaCgoOIIKAzMOf9g5dZkTkz\nB5i7v+9fzJlzzjzPHD2/Oc/znOcIoiiKICIiMkBh6wIQEZH9YkgQEZEkhgQREUliSBARkSSGBBER\nSWJIEBGRJIYEkQNYsWIFkpOTbV0MegQxJMipzJ49G8OHD4dWq5W9zeOPP45r164163O2bt2KqKgo\nDB48GKNGjcKcOXOwf//+5haXyO4xJMhpFBUV4fTp0xAEAYcPH5a9nSAIzfqc9957D1999RVWrFiB\nU6dO4fjx41iyZAmOHz8uuQ3vWSVHxZAgp5GRkYHBgwdj6tSpSE9P1y+fPXs2duzYoX+dnp6OWbNm\nAQBefvlliKKIqKgoPPXUU/juu+8AANu2bcP48eMxYsQIvPnmm7h58yYA4PLly/jmm2+wYcMGhISE\nwM3NDYIg4KmnnkJSUlKjz9ywYQNefPFFDB48GIWFhdi1axcmTpyIp556Cs899xy+/fZb/fqnTp3C\nmDFjkJKSgqeffhphYWHYs2dPo/rduXMHr7/+Op566inMnDmz2Vc/RC3BkCCnkZmZiaioKEyePBkn\nTpxAWVmZ5LoNVw9bt24FAOzevRtnzpzBhAkTkJOTg/Xr1+OTTz7BiRMn4Ofnh7i4OACASqVCt27d\nMGDAAJPl2bNnD1avXo0zZ86gW7du6Ny5M1JTU3HmzBkkJSUhKSkJ+fn5+vVLS0tRXl6O48eP48MP\nP0RiYiKuXLmif3/fvn1YuHAhcnNz4e/vj40bN7bkayJqFoYEOYXc3Fxcv34dEyZMQHBwMHr27Nnk\nl7hce/fuxfTp0/H444+jTZs2iIuLw9mzZ3H9+nXcvn0bSqWy0fpjxozBsGHDMHDgQNy4cUO/PCYm\nBn379oVCoYCrqyvGjBmDHj16AACGDh2K0NBQ5Obm6tcXBAFLlixBmzZtMGzYMIwZM0Z/ZQMA48eP\nx69+9SsoFApERkY2ChgiS2FIkFPIzMzEqFGj0LFjRwDApEmTkJGR0aJ93bx5E35+fvrX7du3R6dO\nnVBSUoJOnTrpm54aHDt2DCdPnoRWq23U99C1a9cm682cORMjRozAsGHDkJ2djdu3b+vf79ChA9zd\n3fWv/fz8Gn2Wj4+P/u927dqhurq6RfUjag5XWxeAqLXu37+P7777DjqdDqNGjQIAaDQaVFZWoqCg\nAO3bt8e9e/f066vVaqP769KlC65fv65/XV1djfLycvj6+qJTp05YvXo1/vWvfyE4OLjRdg93Tj/Y\nIa7RaLB48WKsW7cOYWFhUCgU+M///M9G21RUVODevXto27YtAODGjRsICgpq5rdBZF68kiCHd/Dg\nQbi4uOC7775DZmYmMjMz8d1332Ho0KHIzMxE//798be//Q337t3DL7/8gp07dzba3sfHp1En8OTJ\nk7Fr1y4UFBRAo9Fg/fr1GDRoEPz8/NCnTx/MnDkTcXFx+OGHH3D//n3odDqcOXPG6CgprVYLrVaL\nxx57DAqFAseOHcPf//73RuuIoohPPvkEWq0Wubm5OHr0KCZMmGDeL4uomXglQQ4vIyMD06ZNg6+v\nb6Pls2bNwvvvv4+9e/fip59+QmhoKPr164fIyEjk5OTo11u4cCHi4+Oh0WiwatUqPP/881i8eDEW\nLlyIiooKPPnkk1i/fr1+/cTERGzduhVJSUm4du0avLy80Lt3b2zcuFHfTPVwYHh4eGDlypVYvHgx\ntFotnn32WYSFhTVaR6lUomPHjnjmmWfQvn17rFq1Cr179zbzt0XUPIIlHzqk0Wjw0ksvQavVoq6u\nDhEREXjrrbdQWFiIuLg43LlzB8HBwVi7di1cXZlX9Og6deoU4uPjcfToUVsXhagRizY3ubm5YcuW\nLcjIyEBGRgays7Px448/4qOPPsK8efPw/fffw8vLq9EYdiIish8W75No164dgPqritraWgiCAJVK\nhYiICAD1wwQPHjxo6WIQEVELWDwkdDodoqOjERoaitDQUPj7+6NDhw5QKOo/umvXrk2GFBI9aoYP\nH86mJrJLFg8JhUKhb2rKy8vDxYsXm6zT3LlziIjIOqw2BNbT0xPDhg3Djz/+iIqKCuh0OgBAcXEx\nunTpYnJ7TpBGRGR9Fh1SVFZWhjZt2sDLywv37t1DTk4OFixYgBEjRuDAgQOYOHEi0tPTmwwFNEQQ\nBKjVlZYsrk0plV6sn4Ny5roBrJ+jUyq9WrW9RUNCrVZj+fLl0Ol00Ol0mDhxIsaMGYOAgADExcUh\nOTkZ/fv3x/Tp0y1ZDCIiaiGL3idhbs6e9qyfY3LmugGsn6Nr7ZUEp+UgIiJJDAkiIpLEkCAiIkkM\nCSIiksSQICIiSQwJIiKSxJAgIiJJDAkiIpLEkCAiIkkMCSIiksSQICIiSQwJIiKSxJAgIiJJDAki\nIpLEkCAiIkkMCSIiksSQICIiSQwJIiKSxJAgIiJJDAkiIpLEkCAiIkkMCSIiksSQICIiSQwJIiKS\nxJAgIiJJDAkiIpLEkCAiIkkMCSIikuRqyZ0XFxcjPj4epaWlcHFxwYwZMzB79mxs2rQJ27ZtQ+fO\nnQEAS5cuxejRoy1ZFCIiagGLhoSLiwtWrFiB/v37o6qqClOnTsXIkSMBAPPmzcO8efMs+fFERNRK\nFg0JpVIJpVIJAPDw8EDfvn1x8+ZNAIAoipb8aCIiMgOr9UkUFhaioKAAAwcOBACkpaVhypQpWLly\nJSorK61VDCIiagarhERVVRUWLVqEhIQEeHh4YNasWTh06BAyMzPh4+ODpKQkaxSDiIiaSRAt3O5T\nW1uL119/HaNHj8bcuXObvF9UVIQ33ngDe/bssWQxiIioBSzaJwEACQkJCAwMbBQQarVa31dx8OBB\nBAUFydqXWu28zVJKpRfr56CcuW4A6+folEqvVm1v0ZA4ffo09uzZg6CgIERHR0MQBCxduhR79+5F\nfn4+FAoFunfvjlWrVlmyGERE1EIWDYkhQ4YgPz+/yXLeE0FE5Bh4xzUREUliSBARkSSGBBERSWJI\nEBGRJIYEERFJYkgQEZEkhgQREUliSBARkSSGBBERSWJIEBGRJIYEERFJYkgQEZEkhgQREUliSBAR\nkSSGBBERSWJIEBGRJIYEERFJYkgQEZEk2Y8vvXfvHtRqNdzd3dGlSxdLlomIiOyE0ZDQ6XTIyMjA\n9u3bUVBQAE9PT2g0Gri6uiI8PByvvPIK+vTpY62yEhGRlRkNiRdffBGDBw/GihUrEBwcDBcXFwDA\nrVu3cPz4cSQmJuKFF17ApEmTrFJYIiKyLkEURVHqzbKyMnh7exvdgZx1zEWtrrTK59iCUunF+jko\nZ64bwPo5OqXSq1XbG+24NnTyv3XrFs6ePWt0HSIicg6yRjfNmjULlZWVqKioQHR0NFauXIk1a9ZY\numxERGRjskKiuroaXl5eOHLkCCIjI7Fnzx6cOHHC0mUjIiIbkxUSGo0GAKBSqTBy5EgoFAp9JzYR\nETkvWSExfPhwREREIDc3F8OHD0dFRQUUCt6HR0Tk7GTdTPfuu++ioKAA/v7+cHNzw927d7F69WpL\nl42IiGzMaEhcuHBB/3ebNm1QXFysf+3m5ma5UhERkV0wGhILFiyAIAgQRRE3btyAp6cnAODu3bvo\n1q0bDh8+bHTnxcXFiI+PR2lpKVxcXBAbG4s5c+bgzp07WLp0KYqKitCjRw9s3LgRXl6tG8tLRETm\nZzQkGkJg9erVGDJkCCZMmAAAOHDgAM6dO2dy5y4uLlixYgX69++PqqoqTJ06FaGhodi1axdCQkLw\n2muvITU1FSkpKXjnnXfMUB0iIjInWb3PeXl5+oAAgOeffx4nT540uZ1SqUT//v0BAB4eHujbty9K\nSkqQlZWFmJgYAEBMTAwOHTrUkrITEZGFyQqJmpoa5Obm6l/n5uaipqamWR9UWFiIgoICDBo0CLdu\n3YKPjw+A+iC5fft2s/ZFRETWIXt0U1xcHNq1awcAuH//Pj7++GPZH1JVVYVFixYhISEBHh4eEASh\nRYVt7Rwk9o71c1zOXDeA9XuUyQqJoUOH4tChQ7h8+TJEUURAQIDs0U21tbVYtGgRpkyZgvDwcABA\n586dUVpaCh8fH6jVatnzPzn7JFysn2Ny5roBrJ+js+gEfw+qq6uDm5sbXF1dcfXq1UbDY41JSEhA\nYGAg5s6dq182btw47Nq1CwCQnp6OsLCwZhabiIisQdaVRFpaGj766CN06tRJ31QkCAKysrKMbnf6\n9Gns2bMHQUFBiI6OhiAIWLp0KV577TUsWbIEO3fuhJ+fH5KTk1tfEyIiMjtZIbF582bs3bsX3bt3\nb9bOhwwZgvz8fIPvffnll83aFxERWZ+s5ialUtnsgCAiIscn60pi5MiRWLt2LSZNmgR3d3f98sDA\nQIsVjIiIbE9WSGRkZACov9O6gZw+CSIicmyyQsLUHE1EROScZIUEUD8jrEqlAgA8/fTT6Nu3r8UK\nRURE9kFWx3VGRgZeeeUV5OfnIz8/H/PmzcPu3bstXTYiIrIx2UNg09PToVQqAQBqtRrz589HVFSU\nRQtHRES2JfuO64aAePhvIiJyXrJComfPnvjkk09QUlKCmzdvYtOmTfD397d02YiIyMZkhcTvf/97\nXL58GVFRUYiKisKlS5ewatUqS5eNiIhsTFafROfOnbFhwwZLl4WIiOyMrCuJ1NRUlJeX61/fvn0b\nn3/+ucUKRURE9kFWSOzbtw+dOnXSv37sscewd+9eixWKiIjsg6yQEEWxybK6ujqzF4aIiOyLrJDo\n3bs3/vznP0MUReh0OmzevBk9e/a0dNmIiMjGZIXEypUrceTIEQwcOBCDBw/GsWPHkJiYaOmyERGR\njcka3eTr64stW7aguroaANC+fXuLFoqIiOyD7D6J7du3449//CPat2+PwsJCnDlzxtJlIyIiG5MV\nEklJSTh58iQOHToEAPDw8MAHH3xg0YIREZHtyQoJlUqFjz76CG3btgVQPwT2/v37Fi0YERHZnqyQ\ncHd3hyAI+tc6nc5iBSIiIvshq+M6KCgIu3fvhiiKKCwsRGpqKoYMGWLpshERkY3JupJYvnw5Tp06\nBbVajdjYWNTV1WHZsmWWLhsREdmYrCsJT09PrF692tJlISIiOyPrSmL//v24e/cuACA5ORnz58/H\n//zP/1i0YEREZHuyQuJPf/oTPD09kZeXhxMnTiA6OppXFkREjwBZIeHqWt8q9fe//x2xsbGIjIzk\nEFgiokeArJAQBAG7d+/Gvn37EBISAgDQarUWLRgREdmerJD43e9+hwMHDiA2Nhb+/v64cuUKRowY\nYXK7hIQEjBw5EpGRkfplmzZtwujRoxETE4OYmBhkZ2e3vPRERGRRgmjoYRFmkpubCw8PD8THx2PP\nnj0A6kPCw8MD8+bNa/b+1OpKcxfRbiiVXqyfg3LmugGsn6NTKr1atb3RIbB/+ctfMHfuXKxdu9bg\n+/Hx8UZ3PnToUBQVFTVZbsFcIiIiMzIaEu7u7gDMPzV4WloaMjMz8atf/QrLly+Hl1frko6IiCzD\nos1NAFBUVIQ33nhD39xUVlaGxx57DIIgYMOGDVCr1ZxRlojIThm9kkhLSzO68UsvvdTsD/T29tb/\nPWPGDLzxxhuyt3X2dkPWzzE5c90A1s/RWbRPwhx3VT98oaJWq6FUKgEABw8eRFBQUKs/g4iILMNo\nSCQlJbVq52+//TZUKhXKy8sxduxYLFy4ECqVCvn5+VAoFOjevTtWrVrVqs8gIiLLMRoSx44dM7rx\nmDFjjL7/8ccfN1k2bdo0GcUiIiJ7YDQkPv/8c8n3BEEwGRJEROTYjIbEV199Za1yEBGRHTIaEteu\nXYO/vz8uXLhg8P3AwECLFIqIiOyD0ZBYvXo1UlJSsGDBgibvCYKArKwsixWMiIhsz2hIpKSkAAAO\nHz5slcIQEZF9kfX4UgCoqalBcXEx6urq9MvY3ERE5NxkhcSWLVuwYcMGdOzYEQpF/ezibG4iInJ+\nskLiL3/5Cw4cOABfX19Ll4eIiOyIrIcOde3alQFBRPQIknUlsXDhQqxcuRJjxozRTx8OmL7jmoiI\nHJuskDhy5AiOHDmCK1euNOqTYEgQETk3WSFx8OBBHD58GG3btrV0eYiIyI7I6pPw9/eHq6vs0bJE\nROQkZJ35e/Xqhblz5yI8PBxubm765S156BARETkOWSGh1WrRs2dPnD9/3tLlISIiOyIrJFr78CEi\nInJMRvskTD2+VKPR4OLFi2YtEBER2Q+TE/zV1NRg8uTJGDRoEHx8fHD//n1cvnwZx48fx7Fjx7B8\n+XL07dvXWuUlIiIrMhoSn376KfLy8vDtt9/iD3/4A4qLi9GuXTsEBQUhPDwcaWlp8PT0tFZZiYjI\nykz2SQwcOBADBw60RlmIiMjOyLpPgoiIHk0MCSIiksSQICIiSQwJIiKSJCskbt26hXfeeUc/DUdB\nQQG++eYbixaMiIhsT1ZI/O53v8OQIUNQUVEBAAgICMDXX39t0YIREZHtyQqJkpISvPjii3BxcQEA\nuLm56Z8rQUREzkvWmf7hacIrKiogiqJFCkRERPZDVkiMHz8eiYmJqKqqwq5du/Dqq69i2rRpJrdL\nSEjAyJEjERkZqV92584dvPrqq4iIiMD8+fNRWVnZ8tITEZFFyQqJX//61xg6dCiCg4Nx7NgxzJ49\nG3PnzjW53dSpU/HFF180WpaamoqQkBB8//33GDFiBFJSUlpWciIisjjZj5uLiopCVFRUs3Y+dOhQ\nFBUVNVqWlZWFrVu3AgBiYmIwe/ZsvPPOO83aLxERWYeskLh16xa2bt2Kq1evora2Vr88OTm52R9Y\nVlYGHx8fAIBSqcTt27ebvQ8iIrIOWSHx5ptvYsCAAQgJCdGPcLIFpdLLZp9tDayf43LmugGs36NM\nVkjU1NTg3XffNcsHdu7cGaWlpfDx8YFarYa3t7fsbdVq5+3kViq9WD8H5cx1A1g/R9faAJTVcT1o\n0CD8/PPPLfqAh4fKjhs3Drt27QIApKenIywsrEX7JSIiy5N1JfHCCy/g5ZdfRteuXeHu7q5fvmPH\nDqPbvf3221CpVCgvL8fYsWOxcOFCLFiwAIsXL8bOnTvh5+fXon4NIiKyDlkhsWzZMrzxxhsYMGBA\ns/okPv74Y4PLv/zyS9n7ICIi25EVEu7u7pg/f76ly0JERHZGVp/EM888g+zsbEuXhYiI7IysK4lt\n27YhNTUVHh4ecHNzgyiKEAQBOTk5li4fERHZkKyQ2Llzp6XLQUREdkhWSHTv3t3S5SAiIjtkNCSW\nLVuGdevWYdq0aRAEocn7pobAEhGRYzMaEg0zvf72t7+1SmGIiMi+GA2Jr7/+Gh988AGGDx9urfIQ\nEZEdMToENj8/31rlICIiO8QHVRMRkSSjzU3nz59HSEhIk+W8T4KI6NFgNCR69+6N1NRUa5WFiIjs\njNGQcHNz4z0SRESPMKN9Em3atLFWOYiIyA4ZDYlt27ZZqxxERGSHOLqJiIgkMSSIiEgSQ4KIiCQx\nJIiISBJDgoiIJDEkiIhIEkOCiIgkMSSIiEgSQ4KIiCTJesY1EdGjTnWuBPtyruB6aTX8fNpjUkhv\njBjga+tiWRxDgojIBNW5EqTs/pf+daG6Sv/a2YOCIUFEVuHIv8T35VyRWP6Lw9ShpRgSRGRxjv5L\n/HpptcHlN25VWbkk1mezkBg3bhw8PT2hUCjg6uqKHTt22KooRGRhjv5L3M+nPQrVTQOhW2cPG5TG\numwWEoIg4KuvvkLHjh1tVQQishJH/yU+KaR3oyuhfy/v1ei1IzepSbFZSIiiCJ1OZ6uPJ3I69nyC\nsuYvcUt8Dw3b78v5BTduVaFbZw9MCunVaL+O3qQmxaZXEvPnz4cgCJg5cyZmzJhhq6IQOTx7P0HJ\n/SXeWpb8HkYM8DW6D0dvUpNis5D461//CqVSibKyMsybNw8BAQEYOnSorYpDZHXm/MVr7ycoOb/E\nzcGW34OjN6lJsVlIKJVKAIC3tzeee+45/PTTTyZDQqn0skbRbIb1c1zNrVv2PwsN/uLt0KEtRj/Z\no9F627P+F1dLKtHT1wuxYf/R6P0G129Jn6DM8b2bYx+Tx3hh8pjAZm0jt/4NWvo9mKN+Pbt64cqN\niibL/X29HPrfvk1CoqamBjqdDh4eHqiursaJEyfw1ltvmdxOra60QulsQ6n0Yv0cVEvq9s33BRLL\nf0b/HvWDOR5uOrlyowLrtp5GRcW9Jr+K/TpLt/m39nu31bFrTv0btOR7MFf9Iob5G2xSixjmb9N/\n+60NKJuERGlpKd566y0IgoC6ujpERkZi1KhRtigKkU3IaZpoTtOJsTZ/e+7QNqYlTUfW6vswxFpN\natZmk5Dw9/dHZmamLT6ayCKaeyI2NdpHda7E4PuA4TZuqRMUALvu0DbGVJAa+85tdaI21bntiHjH\nNVErSfUvANInYlO//A2916Cjp5vB5YZOUIlfqAyuay8d2sYYC1JTo5jsvW6OhFOF0yNLda4EiV+o\n8Os1R5D4hQqqcyUt2s/2rP81uHxfzi+S24wY4IvXo4LRQ+kJF4WAHkpPvB4VjBEDfCWbWRqUVdyX\nXVZHHnEzKaS3xPJeRpuiyLx4JUGPJHOOp79aYrhT0tSJWOoXr9SJ/UFyrwQceToJY01Hn+05Z3Ab\nRwg/R8OQoEeSOcfT9/Q1PPSxpSdiqRP7gwrVd/HrNUdM9n/YsiPXHKSC1JHDz9GwuYkeSVK/1otK\n7za7CSo27D8MLm/piViqmeVhOlHUXwFJldNYs5YjM9YURebFKwl6JEn9EhVF6JfLbYIa/WQPVFTc\nM9uImobtth+5gLLK+7K2MXYF5IwdubYexfQoYUhQqznSOPyGshaVym+7bugMNVZHS5yI5QYE0LQt\n3pGOSUs5Y/jZI4YEtZjqXEmTX7v2NA7/4RNlv56PIet0YZP1BAF4zNNd8qRcVHrX6vcaSPWZtHFR\nQFvXdPbkB9vi7X2yP3IsDAlqEVNj+c05Dr8lv4oNnSilOoMf83JHWYX0r3ZXheETsyXvNZDqM6mV\nmF7/wbZ4e5/sjxwLQ4JaxNRYfnMNRZTzq9hQiJgq34OMBQQgfWK25HBLqT6T7j6e/3+fgHRbvCPf\nG0H2hyHhpCzdJm1qLL+5hiJuP3LB8PKjFzBigK9kiAiCWT4er0cFY1/OFbMPtzR1fIwNXTXVFs/h\noWRODAknZI02aVNj+Zs7FFHqpCnVT9Dw61/qikGqicgQby/D/RE9lJ7678uc9xrIOT6tGb3j6PdG\nkH1hSDgha7RJS52IvDu4I3ZsYLM+x9hJ09R2zW27DxvSAz9fLTc6CV6DhvfMPdxS7vFp6egdDg8l\nc2JIOCFrtEmb80Rk7KQp9SsfqD+xu0g0K8lpuzf0eVLrmnO4pbWOD0OBzIEh4YRMzZ7Zmr4KS/R1\nSJ00C9V3YaproU40vFxO2/2DrHlSZZ8BORKGhBOSagrq17NTq/oqpJqFUnf/C92VHvrASDt4Htln\ni6CtE+EiAO3atkH1/Vr4dTYcKsb6NyQyoIk2LgroRNEhmlbYZ0COhCHhhKSaglrTV6E6V4LN+wzP\nvCni34Fx4qcb+NflMv17dSJwt0YLQDqUpE6azaETRXwW/2yr9mEt7DMgR8KQcFKGmk9aOr2yqRvn\nHvRgQEgx1EELGO48lsvRmmrYZ0COgiHhhKT6DVraFt6cG9PkkHr8ptT9CA28vdwBwfDNb2yqIbIM\nThXuZBp+9Reqq5pMJd3S6ZXlPASnOaRCydQU2bHPBuKjN0OdcuprInvFKwknY6zfYdX84fq/m9MW\nLuchOA2C+3ibbHKSCqUH2+qLSu/CVaFAnU4Hv/8fzvrgjWYMBSLrYEg4GVNj8FtygpXqWDZ0Y9q/\nRzddh7ZOBxdBQLu2rqi5XysrlBgARPaFIWFhDz6/QCEIqNPVD+r09nJH7LPNuzNZDkuMwW/uaJyX\nngvCS88FNVqmVHpBrTb8LGgisl8MiYcYegbBz1dvm7x5zFBnMdB4xE6d+O9R/2WV9y0yx7+lxuDz\nFz7Ro4kh8QBTzyCQGucvdZOZt5e7yc809xz/HINPRObEkHiA3KGeD5/YpbaT8/hJS8zxz1/9RGQu\nHAL7ALlDPR8+sbdmiKij3QRGRI8WhsQD/Hzay1rv4RO71HZympt4ExgR2TObhUR2djaef/55RERE\nIDU1tUX7UJ0rQeIXKvx6zREkfqGC6lxJq8pk6mauf6/X66HXhreLfTZQf+OXQgBcFP+e09Tby503\ngRGR3bNJn4ROp8N7772HL7/8El26dMH06dMRFhaGvn37Sm4zZdnuRrOIWuLpa4Y6ffv17GTwXgBT\n2z188xcRkSOySUjk5eWhV69e6N69OwBg0qRJyMrKMhoSOp3YKAgs9fS11jwNjGFARM7GJs1NJSUl\n6Natm/61r68vbt68KXv7fTm/WOXpXkREjzqbhIQoyn2UjGE3blVJdhZztBARkfnYpLmpa9euuH79\nuv51SUkJunTpInt7f18vxIb9B9ZtPd3kvRcj+kGp9DJLOa3NUcstlzPXz5nrBrB+jzJBbO3P+hao\nq6vD888/jy+//BJKpRKxsbFYv3690T4JIiKyPptcSbi4uOC//uu/8Oqrr0IURUyfPp0BQURkh2xy\nJUFERI6Bd1wTEZEkhgQREUliSBARkSS7DwlzzPFkb8aNG4eoqChER0dj+vTpAIA7d+7g1VdfRURE\nBObPn4/KSsd5iltCQgJGjhyJyMhI/TJj9Vm9ejXGjx+PKVOmID8/3xZFbhZD9du0aRNGjx6NmJgY\nxMTEIDs7W/9eSkoKxo8fjwkTJuDEiRO2KLJsxcXFmDNnDiZOnIjIyEhs2bIFgPMcv4fr99VXXwFw\nnuOn0WiMkXxQAAAKiUlEQVQQGxuL6OhoREZGYtOmTQCAwsJCzJgxAxEREYiLi0Ntba1+/aVLl2L8\n+PGYOXNmo1sRJIl2rK6uTgwPDxcLCwtFjUYjRkVFiRcuXLB1sVpt3LhxYnl5eaNla9euFVNTU0VR\nFMWUlBRx3bp1tihai/zjH/8Qz507J06ePFm/TKo+R48eFV977TVRFEXx7NmzYmxsrPUL3EyG6vfp\np5+KmzdvbrLuhQsXxClTpoharVa8du2aGB4eLup0OmsWt1lu3rwpnjt3ThRFUbx79644fvx48cKF\nC05z/KTq5yzHTxRFsbq6WhRFUaytrRVjY2PFs2fPiosXLxb3798viqIoJiYmit98840oiqKYlpYm\nvvvuu6IoiuK+ffvEJUuWmNy/XV9JPDjHU5s2bfRzPDk6URSh0+kaLcvKykJMTAwAICYmBocOHbJF\n0Vpk6NCh6NChQ6NlD9en4bhlZWUhOjoaADBo0CBUVlaitLTUugVuJkP1AwzPHJCVlYWJEyfC1dUV\nPXr0QK9evZCXl2eNYraIUqlE//79AQAeHh7o27cvSkpKnOb4GapfwxRAznD8AKBdu3YA6q8Samtr\nIQgCVCoVIiIiADQ+nzx4XCMiIpCTk2Ny/3YdEq2d48leCYKA+fPnY9q0adi+fTsA4NatW/Dx8QFQ\n/w/79u3btixiq5WVlTWqT1lZGQDg5s2b6Nq1q349X19flJS0bop3W0lLS8OUKVOwcuVKfXOMoX+z\njlK/wsJCFBQUYNCgQU3+PTrD8Wuo38CBAwE4z/HT6XSIjo5GaGgoQkND4e/vjw4dOkChqD+9d+3a\nVV+HB4+fi4sLOnTogPLycqP7t+uQMJT0zuCvf/0rdu3ahc8++wxpaWnIzc2FIAimN3QCho6pI9Z9\n1qxZOHToEDIzM+Hj44MPP/wQgOPWr6qqCosWLUJCQgI8PDwky+ws9XOm46dQKJCRkYHs7Gzk5eXh\n4sWLTdZpqMPD9RNF0WT97DokWjvHk71SKpUAAG9vb4SHhyMvLw+dO3fWX7ar1Wp4e3vbsoitJlUf\nX19fFBcX69crLi52yGPq7e2t/881Y8YMfZNE165dcePGDf16jlC/2tpaLFq0CFOmTEF4eDgA5zp+\nhurnTMevgaenJ4YNG4Yff/wRFRUV+ibtB+vw4PGrq6vD3bt30bFjR6P7teuQeOKJJ3D16lUUFRVB\no9Fg3759CAsLs3WxWqWmpgZVVfXTmVdXV+PEiRMICgrCuHHjsGvXLgBAenq6w9Xz4V8oUvUJCwtD\nRkYGAODs2bPo0KGDvlnDnj1cP7Varf/74MGDCAoKAlBf7/3790Oj0eDatWu4evWqvnnDXiUkJCAw\nMBBz587VL3Om42eofs5y/MrKyvRNZffu3UNOTg4CAwMxYsQIHDhwAEDj4zdu3Dikp6cDAA4cOICn\nn37a5GfY/bQc2dnZeP/99/VzPC1YsMDWRWqVa9eu4a233oIgCKirq0NkZCQWLFiA8vJyLFmyBDdu\n3ICfnx+Sk5MNdpbao7fffhsqlQrl5eXw8fHBwoULER4ejsWLFxusz6pVq3D8+HG0a9cOSUlJCA4O\ntnENjDNUP5VKhfz8fCgUCnTv3h2rVq3SnyxTUlKwY8cOuLq6YuXKlRg1apSNayDt9OnTePnllxEU\nFARBECAIApYuXYqBAwdK/nt0pOMnVb+9e/c6xfH7+eefsXz5cuh0Ouh0OkycOBG/+c1vcO3aNcTF\nxaGiogL9+/fHunXr0KZNG2g0Gixbtgz5+fno1KkT1q9fjx49ehj9DLsPCSIish27bm4iIiLbYkgQ\nEZEkhgQREUliSBARkSSGBBERSWJIEBGRJIYE2b3a2lokJycjIiICkZGRmDRpEtasWYO6ujqj261Y\nsQJpaWkA6qeGXrt2rcnPOnToEH766SezlNsSioqKsG3bNlsXgx4hDAmye8uXL8fFixeRkZGBPXv2\nYPfu3QgICIBGozH7Z2VlZdn1rJ+FhYX49ttvW7StqVAlMsTV1gUgMuaXX35BVlaW/g5foH72ytjY\nWAD1M2CuW7dO/3CYUaNGIT4+3uikZefPn8fvf/971NTUQKPRYMaMGZgzZw5OnDiBw4cPIycnBzt2\n7MArr7yCwsJCHDx4EIIgQKPR4NKlS/jHP/4BT0/PRvv85z//iXXr1qGqqgqCICA+Ph4jR45EXl4e\nPvjgA9TU1KBdu3ZYuXIlnnjiCZw6dQpr1qzBzp07AaDR61OnTuGDDz7AwIEDcfbsWSgUCqxfvx4B\nAQF47733UFRUhJiYGPTs2RPJycm4dOkSkpKSUF5eDq1Wizlz5mDq1KkAgMcffxzLli3D0aNHMWzY\nMCxatMjsx4icnFmeekFkIfv37xejo6Ml3//666/FefPmibW1taJWqxXnzp2rf8DK8uXLxa1bt4qi\nWP+QoDVr1oiiKIpVVVWiRqPR/z1x4kTx4sWLTbZ52LJly8QPP/ywyfLy8nIxNDRUPHv2rCiKoqjT\n6cSKigpRo9GIY8eOFXNyckRRFMUffvhBHDt2rKjVakWVSiVOmzZNv48HX6tUKjE4OFjMz88XRVEU\n//SnP4nvvPNOk/VEsf5BMzExMeKlS5dEUax/sE5ERIT+db9+/cTPP/9c8vsjMoVXEmTXRBOzxuTk\n5CAmJgYuLi4AgKlTp+LQoUN44YUXJLepqanBu+++i4KCAigUCqjVahQUFCAgIEBym40bN6Kmpga/\n/e1vm7x39uxZBAYGYtCgQQDqp2X28vLC+fPn4ebmpp9ELSQkBG5ubrh8+bLJevfp0wePP/44gPqH\n+xw9etTgeleuXMGlS5cQFxen/660Wi0uXryIPn36AID+IUFELcGQILsWHByMK1euoLKyEl5eXk3e\nFw3Mh29qfvz169dDqVRi7dq1+gdAGevf2LlzJ06ePKl//rOhMshd3lBeFxeXRk8nvH//fqP13N3d\n9X+7uLjon1FsaH/e3t76mT0fJggC2rdvb/A9IjnYcU12rVevXhg3bhwSExP1U6zX1dVhy5YtqKmp\nwciRI5Geno7a2lpotVpkZGQgNDTU6D4rKyvRrVs3CIKA8+fPIzc3V/+eh4cH7t69q3/9ww8/4LPP\nPsMf//hHuLm5Gdzfk08+iQsXLuDHH38EUN9PUlFRgYCAAGi1Wpw6dQoAcPLkSdTW1qJ3797o0aMH\nCgsLUVlZCVEUsW/fPlnfh6enp35qaKD+iqNt27bIzMzUL7t06ZL+uzJ1JUZkCq8kyO6tWbMGn376\nKaZOnQo3NzeIoojRo0fDzc0NM2fOxNWrV/XP7X3mmWf0ndpSfvOb3yA+Ph67d+9Gz549MWzYMP17\nU6ZMwYoVK3DgwAG88sor2LlzJ2pqajB//nz9VUBaWlqjX+cdO3bEpk2bkJSUhOrqari4uCA+Ph4h\nISH45JNPsHr1an3H9aeffgpXV1f4+vpi3rx5iImJgb+/P5544glcuHDB5HfRr18/9OnTB5GRkQgI\nCEBycjL++7//G++//z42b96Muro6+Pj4YOPGjQDs/6lqZP84VTgREUlicxMREUliSBARkSSGBBER\nSWJIEBGRJIYEERFJYkgQEZEkhgQREUliSBARkaT/AzLfG+oMx+5pAAAAAElFTkSuQmCC\n", + "text/plain": [ + "\u003cmatplotlib.figure.Figure at 0x7fc3b259add0\u003e" + ] + }, + "metadata": { + "tags": [] + }, + "output_type": "display_data" + } + ], + "source": [ + "counts = []\n", + "times = []\n", + "for n in np.logspace(0, 7, 50):\n", + "\n", + " with tf.Graph().as_default():\n", + " tf_collatz = ag.to_graph(collatz)\n", + " count = tf_collatz(tf.constant(n, dtype=tf.float32))\n", + " with tf.Session() as sess:\n", + " count_value = sess.run(count)\n", + "\n", + " res = %timeit -n10 -r1 -o -q sess.run(count)\n", + " counts.append(count_value)\n", + " times.append(res.best)\n", + " \n", + "plot_results(counts, times, 'AutoGraph')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "RRENYzLRF_f3" + }, + "source": [ + "# Eager" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 301 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 5003, + "status": "ok", + "timestamp": 1531757478713, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "dhDf8LLdF_f-", + "outputId": "3de0a5a5-7a11-4b41-8ab0-e4e21ce8d59b" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEcCAYAAAAydkhNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtYVWW+B/Dv2hshBdSQHaighhwas7Qeb6GFDjIyI3LZ\nGphdJLLMzqSlKaPQsTPm5KhZkZ7moKOnManGK17wsUfIS87QNj2jnEnIg5cQEtyAyDWBvdf5g4d9\nBPbaLGCvfeP7+QvW2mvt38tGvq71vut9BVEURRAREZmhsncBRETkuBgSREQkiSFBRESSGBJERCSJ\nIUFERJIYEkREJIkhQUREktzsXQCRvYWHh6OiogJqtRqiKEIQBMyePRtvv/22vUsjsjuGBBGA9PR0\nPPHEE3Z5b4PBALVabZf3JuoMbzcRATA38cCNGzeQmJiISZMmITQ0FMuXL0dtba1p//fffw+tVotx\n48bhjTfewNKlS5GWlmbaf+LECcTFxWHChAmYN28efvjhB9O+8PBwbNu2DTExMXj88cdhNBqVbSBR\nNzEkiCSIoohFixbhb3/7G44ePYqysjJs3rwZANDU1ITFixdjzpw5OHv2LGbNmoXjx4+bjv3++++R\nmpqKd999F2fPnsXcuXPx2muvoampyfSao0ePYtu2bTh37hxUKv5TJMfE30wiAL/97W8xceJETJgw\nARMnTsSePXswbNgwhIaGws3NDffffz8SExPx3XffAQAuXLgAg8GA559/Hmq1Gr/61a8wZswY0/n2\n7NmDZ555Bo8++igEQUBcXBzc3d1x8eJF02vmz58PPz8/uLu727y9RHKxT4IIwCeffNKhT6KyshJr\n167FuXPnUF9fD4PBgIEDBwIA9Ho9/Pz82rx+8ODBpq9/+uknHDx4ELt27QLQclXS3NyMW7dumV7j\n7++vVHOIrIYhQQTzfRKbNm2CIAg4cuQI+vfvj+zsbKxduxYAoNFoUFZW1ub1N2/exLBhwwC0BMCi\nRYvw6quvKl88kYJ4u4lIQl1dHTw9PeHl5YWysjJs377dtO+xxx6DWq1GRkYGDAYDsrOzkZeXZ9qf\nkJCAL7/80rStvr4ep06dQn19vc3bQdQTvJIgAvDaa69BpVKZnpOYMmUK3nzzTaxYsQLjx4/H8OHD\nERsbi08//RQA0KdPH2zevBmpqanYtGkTwsLCEB4ebupfeOSRR/Duu+9izZo1KCoqgoeHB8aNG4cJ\nEyYAAARBsFdTibpEUHLRocbGRjz33HNoamqCwWBAZGQkXn/9dRQXF2PZsmW4c+cORo8ejQ0bNsDN\njXlFzi0hIQHz5s2DVqu1dylEVqPo7SZ3d3fs3LkTmZmZyMzMxOnTp3Hx4kW8//77SEpKwldffQVv\nb2/s3btXyTKIFPHdd9+hvLwcBoMBBw4cwOXLl/HUU0/Zuywiq1K8T6Jv374AWq4qmpubIQgCdDod\nIiMjAQBarbbN+HIiZ3Ht2jXExsZi/Pjx+PTTT/Hxxx/D19fX3mURWZXi93iMRiNmz56NoqIiPPfc\ncwgMDET//v1NDw/5+/u3GRZI5CwSEhKQkJBg7zKIFKX4lYRKpTLdasrLy8OVK1c6vIadeEREjslm\nQ2C9vLwwYcIEXLx4EdXV1aa5akpLS/HAAw90eryC/etERCRB0dtNlZWV6NOnD7y9vfHzzz8jNzcX\nCxcuxKRJk3Ds2DHMnDkTBw4cwPTp0zs9lyAI0OtrlCzXrjQab7bPSbly2wC2z9lpNN49Ol7RkNDr\n9Vi5ciWMRiOMRiNmzpyJqVOnIigoCMuWLUNaWhpGjRqFp59+WskyiIiomxR9TsLaXD3t2T7n5Mpt\nA9g+Z9fTKwlOy0FERJIYEkREJIkhQUREkhgSREQkiSFBRESSGBJERCSJIUFERJIYEkREJIkhQURE\nkhgSREQkiSFBRESSGBJERCSJIUFERJIYEkREJIkhQUREkhgSREQkiSFBRESSGBJERCSJIUFERJIY\nEkREJIkhQUREkhgSREQkiSFBRESSGBJERCSJIUFERJIYEkREJIkhQUREkhgSREQkyU3Jk5eWliI5\nORnl5eVQq9VISEjACy+8gC1btmD37t0YNGgQAGDp0qUICwtTshQiIuoGRUNCrVZj1apVGDVqFOrq\n6jB79mxMnjwZAJCUlISkpCQl356IiHpI0ZDQaDTQaDQAAE9PT4wcORK3bt0CAIiiqORbExGRFdis\nT6K4uBgFBQUYM2YMACAjIwOxsbFITU1FTU2NrcogIqIusElI1NXVYcmSJUhJSYGnpyeeffZZZGdn\n4+DBg/D19cW6detsUQYREXWRICp836e5uRmvvvoqwsLCkJiY2GF/SUkJFi1ahMOHDytZBhERdYOi\nfRIAkJKSguDg4DYBodfrTX0Vx48fR0hIiKxz6fWue1tKo/Fm+5yUK7cNYPucnUbj3aPjFQ2J8+fP\n4/DhwwgJCUFcXBwEQcDSpUtx5MgR5OfnQ6VSYejQoVizZo2SZRARUTcpGhLjxo1Dfn5+h+18JoKI\nyDnwiWsiIpLEkCAiIkkMCSIiksSQICIiSQwJIiKSxJAgIiJJDAkiIpLEkCAiIkkMCSIiksSQICIi\nSQwJIiKSxJAgIiJJDAkiIpLEkCAiIkkMCSIiksSQICIiSQwJIiKSxJAgIiJJspcv/fnnn6HX6+Hh\n4YEHHnhAyZqIiMhBWAwJo9GIzMxM7NmzBwUFBfDy8kJjYyPc3NwQERGBF198EQ8++KCtaiUiIhuz\nGBLz5s3DY489hlWrVmH06NFQq9UAgIqKCnzzzTdYvXo1nnnmGURFRdmkWCIisi1BFEVRamdlZSV8\nfHwsnkDOa6xFr6+xyfvYg0bjzfY5KVduG8D2OTuNxrtHx1vsuDb3x7+iogIXLlyw+BoiInINskY3\nPfvss6ipqUF1dTXi4uKQmpqK9evXK10bERHZmayQqK+vh7e3N06cOIHo6GgcPnwYZ86cUbo2IiKy\nM1kh0djYCADQ6XSYPHkyVCqVqRObiIhcl6yQmDhxIiIjI3Hu3DlMnDgR1dXVUKn4HB4RkauT9TDd\nO++8g4KCAgQGBsLd3R21tbVYu3at0rUREZGdWQyJwsJC09d9+vRBaWmp6Xt3d3flqiIiIodgMSQW\nLlwIQRAgiiJu3rwJLy8vAEBtbS0GDx6Mr7/+2uLJS0tLkZycjPLycqjVasTHx2P+/Pm4c+cOli5d\nipKSEgQEBOCjjz6Ct3fPxvISEZH1WQyJ1hBYu3Ytxo0bh9/85jcAgGPHjuHSpUudnlytVmPVqlUY\nNWoU6urqMHv2bEyZMgX79+9HaGgoXnnlFWzduhXp6elYvny5FZpDRETWJKv3OS8vzxQQAPDrX/8a\n3377bafHaTQajBo1CgDg6emJkSNHoqysDDk5OdBqtQAArVaL7Ozs7tROREQKkxUSDQ0NOHfunOn7\nc+fOoaGhoUtvVFxcjIKCAowdOxYVFRXw9fUF0BIkt2/f7tK5iIjINmSPblq2bBn69u0LALh79y42\nbdok+03q6uqwZMkSpKSkwNPTE4IgdKvYns5B4ujYPuflym0D2D65Tv+jGHty/hdFZTUY5ueN+On/\ngrDHA6xybnuRFRLjx49HdnY2rl27BlEUERQUJHt0U3NzM5YsWYLY2FhEREQAAAYNGoTy8nL4+vpC\nr9fLnv/J1SfhYvuckyu3DWD75NJdKkP6oe9N31+/WY2Nu86juvpnTHrYr8fn7y5FJ/i7l8FggLu7\nO9zc3FBUVNRmeKwlKSkpCA4ORmJiomlbeHg49u/fDwA4cOAApk+f3sWyiYgcS1budYntP9q0DmuT\ndSWRkZGB999/HwMHDjTdKhIEATk5ORaPO3/+PA4fPoyQkBDExcVBEAQsXboUr7zyCt58803s27cP\nQ4YMQVpaWs9bQkRkRz+V15vdfrOizsaVWJeskNixYweOHDmCoUOHdunk48aNQ35+vtl9n376aZfO\nRUTkyIb49kOxvmMgDB7kaYdqrEfW7SaNRtPlgCAi6k2iQkdIbB9u20KsTNaVxOTJk7FhwwZERUXB\nw8PDtD04OFixwoiIHInuUhmycq/jp/J6DPHth6jQEW06pFu/zsr9ETcr6jB4kCeiQofbtdPaGmSF\nRGZmJoCWJ61byemTICJyBe1HLhXr60zftw8KZw+F9mSFRGdzNBERuTJLI5dcLRTakxUSQMuMsDqd\nDgDwxBNPYOTIkYoVRUTUXZ3dFuoOVx25JIesjuvMzEy8+OKLyM/PR35+PpKSknDo0CGlayMi6pLW\n20LF+joYRdF0W0h3qaxH5x3i28/sdmcfuSSH7CGwBw4cgEajAQDo9XosWLAAMTExihZHRNQVSt0W\nigod0aZP4v+3O/fIJTlk325qDYj2XxMROQqlbgu56sglOWSFxLBhw/Dxxx9j7ty5EAQBu3fvRmBg\noNK1ERF1iZIPtLniyCU5ZPVJ/P73v8e1a9cQExODmJgYXL16FWvWrFG6NiKiLnHVB9rsSdaVxKBB\ng/Dhhx8qXQsRUY/05ttCSpEVElu3bkVCQgIGDhwIALh9+zb27duHl19+WdHiiIi6qrfeFlKKrNtN\nWVlZpoAAgPvvvx9HjhxRrCgiInIMskJCFMUO2wwGg9WLISIixyIrJEaMGIH/+q//giiKMBqN2LFj\nB4YNG6Z0bUREZGeyQiI1NRUnTpzAmDFj8Nhjj+HUqVNYvXq10rUREZGdyeq49vPzw86dO1Ff3/Kg\nSr9+5h9RJyIi1yK7T2LPnj345JNP0K9fPxQXF+O///u/la6NiIjsTFZIrFu3Dt9++y2ys7MBAJ6e\nnnjvvfcULYyIiOxP1u0mnU6HzMxMaLVaAC1DYO/evatoYUREligxJTh1JCskPDw8IAiC6Xuj0ahY\nQUREnZG7Uhz1nKzbTSEhITh06BBEUURxcTH+/d//HePGjVO6NiIisyxNCU7WJSskVq5cibNnz0Kv\n1yM+Ph4GgwErVqxQujYiIrN680pxtibrdpOXlxfWrl2rdC1ERLIoOSU4tSXrSuLo0aOora0FAKSl\npWHBggX45z//qWhhRES6S2VYvV2Hl9efwOrtOtMypJwS3HZkhcSf/vQneHl5IS8vD2fOnEFcXByv\nLIhIUZbWq570sB9ejRmNAI0X1CoBARovvBozmp3WCpB1u8nNreVlf/vb3xAfH4/o6Gjs2LFD0cKI\nqHfrbL1qTgluG7KuJARBwKFDh5CVlYXQ0FAAQFNTk6KFEVHvxs5pxyArJN5++20cO3YM8fHxCAwM\nxPXr1zFp0qROj0tJScHkyZMRHR1t2rZlyxaEhYVBq9VCq9Xi9OnT3a+eiFzWEF/zc8Sxc9q2BNHc\nYhFWcu7cOXh6eiI5ORmHDx8G0BISnp6eSEpK6vL59Poaa5foMDQab7bPSbly2wD7ta/9A3OtrN33\n0Bs+v56w2Cfxl7/8BYmJidiwYYPZ/cnJyRZPPn78eJSUlHTYrmAuEZGD6uo0Glyv2jFYDAkPDw8A\n1p8aPCMjAwcPHsQjjzyClStXwtu7Z0lHRI6tu9NosHPa/hS93QQAJSUlWLRokel2U2VlJe6//34I\ngoAPP/wQer2eM8oSubjF75/A9ZvVHbaPGNwfm5f/0g4VkVwWryQyMjIsHvzcc891+Q19fHxMXyck\nJGDRokWyj3X1+4Zsn3Ny5bYB1mlfUan542+U1dj9Z9cbPr+esBgS1niquv2Fil6vh0ajAQAcP34c\nISEhPX4PInJsnEbDeVkMiXXr1vXo5G+99RZ0Oh2qqqowbdo0LF68GDqdDvn5+VCpVBg6dCjWrFnT\no/cgIscXFTrC7EglTqPh+CyGxKlTpywePHXqVIv7N23a1GHbnDlzZJRFRK6EI5Wcl8WQ+POf/yy5\nTxCETkOCiKgVRyo5J4sh8dlnn9mqDiIickAWQ+LGjRsIDAxEYWGh2f3BwcGKFEVERI7BYkisXbsW\n6enpWLhwYYd9giAgJydHscKIiMj+LIZEeno6AODrr7+2STFERORYZK0nAQANDQ0oLS2FwWAwbePt\nJiLH1NV5koikyAqJnTt34sMPP8SAAQOgUrXMLs7bTUSOqbvzJBGZIysk/vKXv+DYsWPw8+MvGJGj\n62xFN6KukLXokL+/PwOCyElIrehWrK/F6u066C6V2bgicmayriQWL16M1NRUTJ061TR9OND5E9dE\nJM3a/Qat5zNamNiZt56oq2SFxIkTJ3DixAlcv369TZ8EQ4Koe6zdbyC1ipsU3noiuWSFxPHjx/H1\n11/jvvvuU7oeol6hJ/0G5q5ApM4n5WZFxxlZicyRFRKBgYFwc5M9WpaIOiHVb9DZH2+pKxBB6Nr7\nc4pukkvWX/7hw4cjMTERERERcHd3N23vzqJDRNT5+gpS/RVSVwxuKhWaDMYO2328PVBZc7fDdk7R\nTXLJCommpiYMGzYMly9fVroeol7B0voKlvorpK5Amo0dAwIA4n/Z8sArp+im7pIVEj1dfIiI2rK0\nvsLq7Tqzx6Qf+h591AKMho77hvp6ISp0uGQYMBSouzpdvvSRRx6R3N/Y2IgbN25g5MiRVi+MyNVJ\nra8gdbUAAE0G88NbWwOBYUDW1ukEfw0NDZg1axbGjh0LX19f3L17F9euXcM333yDU6dOYeXKlQwJ\nIiuS6q+4Vx+1CkZR5O0jUpzFkNi8eTPy8vLw17/+Ff/xH/+B0tJS9O3bFyEhIYiIiEBGRga8vLxs\nVStRryDVX3EvoyhiW/IvbVQR9Wad9kmMGTMGY8aMsUUtRISW21B7ThSaHZXUikNYyVZkzd1ERLbV\nOipJCoewkq3wCTkiO+hs3qZJD/uhsOQOcs4Xdzh2+rgA9kGQzTAkiGxM7rxNz/0qBMFDB/AZB7Ir\nhgSRjXVl3iYOayV7k9UnUVFRgeXLl5um4SgoKMAXX3yhaGFErqq78zYR2YOskHj77bcxbtw4VFdX\nAwCCgoLw+eefK1oYkasa4tvP7HaOWCJHJCskysrKMG/ePKjVagCAu7u7aV0JIuqaqNAREts5Yokc\nj6w+ifbThFdXV0O0sPoVUW/TlVXmLM3bRORoZIXEjBkzsHr1atTV1WH//v34/PPPMWfOnE6PS0lJ\nwcmTJzFo0CAcPnwYAHDnzh0sXboUJSUlCAgIwEcffQRvb++etYLIjk7/o7jLq8yxQ5qchax7Ri+/\n/DLGjx+P0aNH49SpU3jhhReQmJjY6XGzZ8/G9u3b22zbunUrQkND8dVXX2HSpElIT0/vXuVEDmJP\nzv+a3Z6V+6ONKyGyPtlDYGNiYhATE9Olk48fPx4lJSVttuXk5GDXrl0AAK1WixdeeAHLly/v0nmJ\nHElRWY3Z7RytRK5AVkhUVFRg165dKCoqQnNzs2l7Wlpal9+wsrISvr6+AACNRoPbt293+RxE1tCV\nfgRLrx3m543rN6s7HMPRSuQKZIXEv/7rv+Lhhx9GaGioaYSTPWg0rt13wfbZjlQ/wtbD32O4f3/E\nT/8XhD0eYPG1/fvfh7DHAxA//V+wcdf5Du8xL/Ihh2pzT7hKO6S4evt6QlZINDQ04J133rHKGw4a\nNAjl5eXw9fWFXq+Hj4+P7GP1evOX9a5Ao/Fm+2zoi68KzG4XReD6zWps3HUe2w/+E/G/DJZ8QvqL\nr37AqIABCHs8ANXVP3cYrTQqYIBDtbm7HO2zs7be0L6ekBUSY8eOxQ8//ICHHnqoy2/QfqhseHg4\n9u/fj4ULF+LAgQOYPn16l89J1FOWVn9rVVlzF+mHvocgmN9/b58DRyuRq5IVEs888wyef/55+Pv7\nw8PDw7R97969Fo976623oNPpUFVVhWnTpmHx4sVYuHAh3njjDezbtw9DhgzpVr8GUU/JWf2tlZtK\nhSaDscN29jlQbyArJFasWIFFixbh4Ycf7lKfxKZNm8xu//TTT2Wfg8iaWjugS8rljzxqNnYMCIBP\nSFPvICskPDw8sGDBAqVrIeoWuaOU2k/RLddQXy9EhQ7nE9LUK8kKiaeeegqnT59GWFiY0vUQdYnc\ntRkA6Sm6AzQtIbDnZCEqqzsuGdoaCAwF6o1khcTu3buxdetWeHp6wt3dHaIoQhAE5ObmKl0fkUWW\n1mZo3d96hSF1i+lmRZ0pBFquSnjFQNRKVkjs27dP6TqIukVqlFJJeW2HKwwp93ZA84qBqC1ZITF0\n6FCl6yDqFqlRSlIjksxhBzSRNIshsWLFCmzcuBFz5syBYGaweGdDYImUcG9H9UAvd7OvkRqRJAgt\nHdG8nUQkj8WQaJ3p9Xe/+51NiiEyp30oVNb8f+dy69c+3h64U9do+sOflXvd7BXGUF8vrFkw0UaV\nEzk/iyHx+eef47333sPEifxHRfbRfvTSvQFxr3739cH7v53SZpu54a68tUTUNRZDIj8/31Z1EJm1\n50ShrNe1n5abq78RWYfs9SSIlNT+gbiHht2PH4puS145tGduigyOVCLqOYshcfnyZYSGhnbYzuck\nyJrMPRAnd16lVryNRKQMiyExYsQIbN261Va1UC8l9UCcHH3UKrwUNYpXDEQKsRgS7u7ufEaCFCdn\n2m4pDAgiZaks7ezTp4+t6qBebIhvP9mv7aNWQSW0zLf0asxoBgSRwixeSezevdtWdVAvFhU6Qtbs\nrAwFItvj6Cayu9Y//FKzsPr090D8tGAGBJEdMCTI6syt7zBrquV1djkLK5FjEsT2i1A7MFdfrNzZ\n2mcuDADzTzq3zJnkKbkgkDNzxs+uK9g+56bRWP4PWmd4JUHdIrXYj4+3h9nXi6LlBYGIyDFZHN1E\nJEXq2QY5T0i3LghERI6PIUHd0pNnG9rPs0REjou3m0iW9v0P/e5zQ21DU7fOZW6eJSJyTAwJ6pS5\n/oee4DxLRM6DIUGd6sncSq1UAjDE14tDWomcDEOCOtWd/of2K8XNmhrs0sMMiVwVQ4I6NcS3n+xb\nTHw6msi1MCSoU3LmVgrQ8FYSkStiSFCnTHMrnSg0+xwEJ94jcl12C4nw8HB4eXlBpVLBzc0Ne/fu\ntVcpJAPnViLqnewWEoIg4LPPPsOAAQPsVUKvZG6+pa78kee60US9i91CQhRFGI1Ge719ryQ13xLA\nuZSIyDy7XkksWLAAgiBg7ty5SEhIsFcpLkfqakHqeYes3B8ZEkRklt1C4ssvv4RGo0FlZSWSkpIQ\nFBSE8ePH26sclyF1tVBYckfyeQfOpUREUhxiPYktW7bA09MTSUlJ9i7F6S1+/wSu36w2u893YF+U\nVzV02D5icH9sXv5LpUsjIidklyuJhoYGGI1GeHp6or6+HmfOnMHrr7/e6XGu/MSutRY+KSqVPodU\nH1DkhEDFf7auvLCLK7cNYPucnVMuOlReXo7XX38dgiDAYDAgOjoaTz75pD1KcTmWno6+U9uIV2NG\ncwgrEclml5AIDAzEwYMH7fHWLs/S09GDB3lyCCsRdQkXHXIxkx72w/RxAWb3cYpuIuoqTsvhgp77\nVQiChw7gbSUi6jGGhIvibSUisgbebiIiIkkMCSIiksSQICIiSeyTsKGezsBKRGRrDAkbyTh+GTnn\ni03fcwZWInIGDAmF6S6VSa7oBnAGViJybAwJBbWfkdUczsBKRI6MIWEl5vobpNZvuNfgQZ5Kl0ZE\n1G0MCSuQWsNBEDo/llNlEJEj4xBYK5C6YnBTWf7xTh8XwP4IInJovJKwAqkV35ol1m/w8fZA/C+D\nGRBE5PAYElYgtYbDUF8vRIUO50R7ROS0GBKdkPMAnNQaDq2BwFAgImfFkLBAqkMaaPsAXOvXvGIg\nIlfDkGjn3isHtUS/s7kH4HjFQESuiCFxj/ZXDkaD+dfxATgi6i04BPYech5+A/gAHBH1HgyJe0gN\nZW2PD8ARUW/B2033kBrK2ketglEU2SFNRL0OQ+IeUkNZX4oaxWAgol6JIXEPDmUlImqLIdEOh7IS\nEf0/dlwTEZEkp76S4JrRRETKctqQkDtlBhERdZ/dbjedPn0av/71rxEZGYmtW7d2+XipB9+ycn/s\nWWFERGRil5AwGo149913sX37dhw5cgRZWVm4cuVKl84h9eAbp8wgIrIeu4REXl4ehg8fjqFDh6JP\nnz6IiopCTk6OxWNiVxzC6u066C6VAWh58M0cTplBRGQ9dgmJsrIyDB482PS9n58fbt26ZfEYo1E0\n9TvoLpUhKnSE2ddxygwiIuuxS0iIotij41un6n41ZjQCNF5QqwQEaLzwasxodloTEVmRXUY3+fv7\n46effjJ9X1ZWhgceeED28Tcr6qDReGPWVG/MmhqsRIl2odF427sERbly+1y5bQDb15vZJSQeffRR\nFBUVoaSkBBqNBllZWfjggw8sHnN4U6yNqiMiolZ2CQm1Wo1/+7d/w0svvQRRFPH0009j5MiR9iiF\niIgsEMSedhAQEZHL4txNREQkiSFBRESSGBJERCTJ4UOip3M8OaLw8HDExMQgLi4OTz/9NADgzp07\neOmllxAZGYkFCxagpqbGzlXKl5KSgsmTJyM6Otq0zVJ71q5dixkzZiA2Nhb5+fn2KLlLzLVvy5Yt\nCAsLg1arhVarxenTp0370tPTMWPGDPzmN7/BmTNn7FGybKWlpZg/fz5mzpyJ6Oho7Ny5E4DrfH7t\n2/fZZ58BcJ3Pr7GxEfHx8YiLi0N0dDS2bNkCACguLkZCQgIiIyOxbNkyNDc3m16/dOlSzJgxA3Pn\nzm3zKIIk0YEZDAYxIiJCLC4uFhsbG8WYmBixsLDQ3mX1WHh4uFhVVdVm24YNG8StW7eKoiiK6enp\n4saNG+1RWrd899134qVLl8RZs2aZtkm15+TJk+Irr7wiiqIoXrhwQYyPj7d9wV1krn2bN28Wd+zY\n0eG1hYWFYmxsrNjU1CTeuHFDjIiIEI1Goy3L7ZJbt26Jly5dEkVRFGtra8UZM2aIhYWFLvP5SbXP\nVT4/URTF+vp6URRFsbm5WYyPjxcvXLggvvHGG+LRo0dFURTF1atXi1988YUoiqKYkZEhvvPOO6Io\nimJWVpb45ptvdnp+h76S6M4cT85AFEUYjcY223JycqDVagEAWq0W2dnZ9iitW8aPH4/+/fu32da+\nPa2fW04mDIcbAAAJLUlEQVRODuLi4gAAY8eORU1NDcrLy21bcBeZax9gfuaAnJwczJw5E25ubggI\nCMDw4cORl5dnizK7RaPRYNSoUQAAT09PjBw5EmVlZS7z+ZlrX+sUQK7w+QFA3759AbRcJTQ3N0MQ\nBOh0OkRGRgJo+/fk3s81MjISubm5nZ7foUOiO3M8OQNBELBgwQLMmTMHe/bsAQBUVFTA19cXQMsv\n9u3bt+1ZYo9VVla2aU9lZSUA4NatW/D39ze9zs/PD2VlZXapsacyMjIQGxuL1NRU0+0Yc7+zztK+\n4uJiFBQUYOzYsR1+H13h82tt35gxYwC4zudnNBoRFxeHKVOmYMqUKQgMDET//v2hUrX8eff39ze1\n4d7PT61Wo3///qiqqrJ4focOCXNJ7wq+/PJL7N+/H9u2bUNGRgbOnTsHQRDsXZZNmPtMnbHtzz77\nLLKzs3Hw4EH4+vrij3/8IwDnbV9dXR2WLFmClJQUeHp6StbsKu1zpc9PpVIhMzMTp0+fRl5entll\nF1rb0L59oih22j6HDomezvHkqDQaDQDAx8cHERERyMvLw6BBg0yX7Xq9Hj4+PvYsscek2uPn54fS\n0lLT60pLS53yM/Xx8TH940pISDDdkvD398fNmzdNr3OG9jU3N2PJkiWIjY1FREQEANf6/My1z5U+\nv1ZeXl6YMGECLl68iOrqatMt7XvbcO/nZzAYUFtbiwEDBlg8r0OHxL1zPDU2NiIrKwvTp0+3d1k9\n0tDQgLq6loWR6uvrcebMGYSEhCA8PBz79+8HABw4cMDp2tn+fyhS7Zk+fToyMzMBABcuXED//v1N\ntzUcWfv26fV609fHjx9HSEgIgJZ2Hz16FI2Njbhx4waKiopMtzccVUpKCoKDg5GYmGja5kqfn7n2\nucrnV1lZabpV9vPPPyM3NxfBwcGYNGkSjh07BqDt5xceHo4DBw4AAI4dO4Ynnnii0/dw+Gk5Tp8+\njT/84Q+mOZ4WLlxo75J65MaNG3j99dchCAIMBgOio6OxcOFCVFVV4c0338TNmzcxZMgQpKWlme0s\ndURvvfUWdDodqqqq4Ovri8WLFyMiIgJvvPGG2fasWbMG33zzDfr27Yt169Zh9OjRdm6BZebap9Pp\nkJ+fD5VKhaFDh2LNmjWmP5bp6enYu3cv3NzckJqaiieffNLOLZB2/vx5PP/88wgJCYEgCBAEAUuX\nLsWYMWMkfx+d6fOTat+RI0dc4vP74YcfsHLlShiNRhiNRsycOROvvfYabty4gWXLlqG6uhqjRo3C\nxo0b0adPHzQ2NmLFihXIz8/HwIED8cEHHyAgIMDiezh8SBARkf049O0mIiKyL4YEERFJYkgQEZEk\nhgQREUliSBARkSSGBBERSWJIkMNrbm5GWloaIiMjER0djaioKKxfvx4Gg8HicatWrUJGRgaAlqmh\nN2zY0Ol7ZWdn43/+53+sUrcSSkpKsHv3bnuXQb0IQ4Ic3sqVK3HlyhVkZmbi8OHDOHToEIKCgtDY\n2Gj198rJyXHoWT+Li4vx17/+tVvHdhaqROa42bsAIkt+/PFH5OTkmJ7wBVpmr4yPjwfQMgPmxo0b\nTYvDPPnkk0hOTrY4adnly5fx+9//Hg0NDWhsbERCQgLmz5+PM2fO4Ouvv0Zubi727t2LF198EcXF\nxTh+/DgEQUBjYyOuXr2K7777Dl5eXm3O+Y9//AMbN25EXV0dBEFAcnIyJk+ejLy8PLz33ntoaGhA\n3759kZqaikcffRRnz57F+vXrsW/fPgBo8/3Zs2fx3nvvYcyYMbhw4QJUKhU++OADBAUF4d1330VJ\nSQm0Wi2GDRuGtLQ0XL16FevWrUNVVRWampowf/58zJ49GwDwi1/8AitWrMDJkycxYcIELFmyxOqf\nEbk4q6x6QaSQo0ePinFxcZL7P//8czEpKUlsbm4Wm5qaxMTERNMCKytXrhR37dolimLLIkHr168X\nRVEU6+rqxMbGRtPXM2fOFK9cudLhmPZWrFgh/vGPf+ywvaqqSpwyZYp44cIFURRF0Wg0itXV1WJj\nY6M4bdo0MTc3VxRFUfz73/8uTps2TWxqahJ1Op04Z84c0znu/V6n04mjR48W8/PzRVEUxT/96U/i\n8uXLO7xOFFsWmtFqteLVq1dFUWxZWCcyMtL0/UMPPST++c9/lvz5EXWGVxLk0MROZo3Jzc2FVquF\nWq0GAMyePRvZ2dl45plnJI9paGjAO++8g4KCAqhUKuj1ehQUFCAoKEjymI8++ggNDQ343e9+12Hf\nhQsXEBwcjLFjxwJomZbZ29sbly9fhru7u2kStdDQULi7u+PatWudtvvBBx/EL37xCwAti/ucPHnS\n7OuuX7+Oq1evYtmyZaafVVNTE65cuYIHH3wQAEyLBBF1B0OCHNro0aNx/fp11NTUwNvbu8N+0cx8\n+J3Nj//BBx9Ao9Fgw4YNpgWgLPVv7Nu3D99++61p/WdzNcjd3lqvWq1uszrh3bt327zOw8PD9LVa\nrTatUWzufD4+PqaZPdsTBAH9+vUzu49IDnZck0MbPnw4wsPDsXr1atMU6waDATt37kRDQwMmT56M\nAwcOoLm5GU1NTcjMzMSUKVMsnrOmpgaDBw+GIAi4fPkyzp07Z9rn6emJ2tpa0/d///vfsW3bNnzy\nySdwd3c3e77HH38chYWFuHjxIoCWfpLq6moEBQWhqakJZ8+eBQB8++23aG5uxogRIxAQEIDi4mLU\n1NRAFEVkZWXJ+nl4eXmZpoYGWq447rvvPhw8eNC07erVq6afVWdXYkSd4ZUEObz169dj8+bNmD17\nNtzd3SGKIsLCwuDu7o65c+eiqKjItG7vU089ZerUlvLaa68hOTkZhw4dwrBhwzBhwgTTvtjYWKxa\ntQrHjh3Diy++iH379qGhoQELFiwwXQVkZGS0+d/5gAEDsGXLFqxbtw719fVQq9VITk5GaGgoPv74\nY6xdu9bUcb1582a4ubnBz88PSUlJ0Gq1CAwMxKOPPorCwsJOfxYPPfQQHnzwQURHRyMoKAhpaWn4\nz//8T/zhD3/Ajh07YDAY4Ovri48++giA46+qRo6PU4UTEZEk3m4iIiJJDAkiIpLEkCAiIkkMCSIi\nksSQICIiSQwJIiKSxJAgIiJJDAkiIpL0f3zF2/hGE4QYAAAAAElFTkSuQmCC\n", + "text/plain": [ + "\u003cmatplotlib.figure.Figure at 0x7fc3af690a50\u003e" + ] + }, + "metadata": { + "tags": [] + }, + "output_type": "display_data" + } + ], + "source": [ + "with context.eager_mode():\n", + "\n", + " counts = []\n", + " times = [] \n", + " for n in np.logspace(0, 7, 50):\n", + "\n", + " n_tensor = tf.constant(n, dtype=tf.float32)\n", + " count = collatz(n_tensor)\n", + "\n", + " res = %timeit -n10 -r1 -o -q collatz(n_tensor)\n", + " times.append(res.best)\n", + " counts.append(count)\n", + " \n", + "plot_results(counts, times, 'Eager')\n" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [ + "x5ChBlH09jk_", + "_cRFTcwT9mnn" + ], + "default_view": {}, + "last_runtime": { + "build_target": "", + "kind": "local" + }, + "name": "Autograph vs. Eager Collatz speed test", + "provenance": [ + { + "file_id": "0B8bm7KvwJklpMUQtbnVpYkdJUjRtOTRyWVVfSEhpRl9HYm5n", + "timestamp": 1531512047714 + } + ], + "version": "0.3.2", + "views": {} + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_mnist_speed_test.ipynb b/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_mnist_speed_test.ipynb new file mode 100644 index 0000000000..952ec091fb --- /dev/null +++ b/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_mnist_speed_test.ipynb @@ -0,0 +1,652 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "etTmZVFN8fYO" + }, + "source": [ + "This notebook runs a basic speed test for a short training loop of a neural network training on the MNIST dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "eqOvRhOz8SWs" + }, + "source": [ + "### Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "nHY0tntRizGb" + }, + "outputs": [], + "source": [ + "!pip install -U -q tf-nightly" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "Pa2qpEmoVOGe" + }, + "outputs": [], + "source": [ + "import gzip\n", + "import os\n", + "import shutil\n", + "import time\n", + "\n", + "import numpy as np\n", + "import six\n", + "from six.moves import urllib\n", + "import tensorflow as tf\n", + "\n", + "from tensorflow.contrib import autograph as ag\n", + "from tensorflow.contrib.eager.python import tfe\n", + "from tensorflow.python.eager import context\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "PZWxEJFM9A7b" + }, + "source": [ + "### Testing boilerplate" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "kfZk9EFZ5TeQ" + }, + "outputs": [], + "source": [ + "# Test-only parameters. Test checks successful completion not correctness. \n", + "burn_ins = 1\n", + "trials = 1\n", + "max_steps = 2\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "k0GKbZBJ9Gt9" + }, + "source": [ + "### Speed test configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "gWXV8WHn43iZ" + }, + "outputs": [], + "source": [ + "#@test {\"skip\": true} \n", + "burn_ins = 3\n", + "trials = 10\n", + "max_steps = 500\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "kZV_3pGy8033" + }, + "source": [ + "### Data source setup" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "YfnHJbBOBKae" + }, + "outputs": [], + "source": [ + "def download(directory, filename):\n", + " filepath = os.path.join(directory, filename)\n", + " if tf.gfile.Exists(filepath):\n", + " return filepath\n", + " if not tf.gfile.Exists(directory):\n", + " tf.gfile.MakeDirs(directory)\n", + " url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'\n", + " zipped_filepath = filepath + '.gz'\n", + " print('Downloading %s to %s' % (url, zipped_filepath))\n", + " urllib.request.urlretrieve(url, zipped_filepath)\n", + " with gzip.open(zipped_filepath, 'rb') as f_in, open(filepath, 'wb') as f_out:\n", + " shutil.copyfileobj(f_in, f_out)\n", + " os.remove(zipped_filepath)\n", + " return filepath\n", + "\n", + "\n", + "def dataset(directory, images_file, labels_file):\n", + " images_file = download(directory, images_file)\n", + " labels_file = download(directory, labels_file)\n", + "\n", + " def decode_image(image):\n", + " # Normalize from [0, 255] to [0.0, 1.0]\n", + " image = tf.decode_raw(image, tf.uint8)\n", + " image = tf.cast(image, tf.float32)\n", + " image = tf.reshape(image, [784])\n", + " return image / 255.0\n", + "\n", + " def decode_label(label):\n", + " label = tf.decode_raw(label, tf.uint8)\n", + " label = tf.reshape(label, [])\n", + " return tf.to_int32(label)\n", + "\n", + " images = tf.data.FixedLengthRecordDataset(\n", + " images_file, 28 * 28, header_bytes=16).map(decode_image)\n", + " labels = tf.data.FixedLengthRecordDataset(\n", + " labels_file, 1, header_bytes=8).map(decode_label)\n", + " return tf.data.Dataset.zip((images, labels))\n", + "\n", + "\n", + "def mnist_train(directory):\n", + " return dataset(directory, 'train-images-idx3-ubyte',\n", + " 'train-labels-idx1-ubyte')\n", + "\n", + "def mnist_test(directory):\n", + " return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')\n", + "\n", + "def setup_mnist_data(is_training, hp, batch_size):\n", + " if is_training:\n", + " ds = mnist_train('/tmp/autograph_mnist_data')\n", + " ds = ds.cache()\n", + " ds = ds.shuffle(batch_size * 10)\n", + " else:\n", + " ds = mnist_test('/tmp/autograph_mnist_data')\n", + " ds = ds.cache()\n", + " ds = ds.repeat()\n", + " ds = ds.batch(batch_size)\n", + " return ds\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "qzkZyZcS9THu" + }, + "source": [ + "### Keras model definition" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "x_MU13boiok2" + }, + "outputs": [], + "source": [ + "def mlp_model(input_shape):\n", + " model = tf.keras.Sequential((\n", + " tf.keras.layers.Dense(100, activation='relu', input_shape=input_shape),\n", + " tf.keras.layers.Dense(100, activation='relu'),\n", + " tf.keras.layers.Dense(10, activation='softmax')))\n", + " model.build()\n", + " return model\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "DXt4GoTxtvn2" + }, + "source": [ + "# AutoGraph" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "W51sfbONiz_5" + }, + "outputs": [], + "source": [ + "def predict(m, x, y):\n", + " y_p = m(x)\n", + " losses = tf.keras.losses.categorical_crossentropy(y, y_p)\n", + " l = tf.reduce_mean(losses)\n", + " accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n", + " accuracy = tf.reduce_mean(accuracies)\n", + " return l, accuracy\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "CsAD0ajbi9iZ" + }, + "outputs": [], + "source": [ + "def fit(m, x, y, opt):\n", + " l, accuracy = predict(m, x, y)\n", + " opt.minimize(l)\n", + " return l, accuracy\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "RVw57HdTjPzi" + }, + "outputs": [], + "source": [ + "def get_next_batch(ds):\n", + " itr = ds.make_one_shot_iterator()\n", + " image, label = itr.get_next()\n", + " x = tf.to_float(tf.reshape(image, (-1, 28 * 28)))\n", + " y = tf.one_hot(tf.squeeze(label), 10)\n", + " return x, y\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "UUI0566FjZPx" + }, + "outputs": [], + "source": [ + "def train(train_ds, test_ds, hp):\n", + " m = mlp_model((28 * 28,))\n", + " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", + "\n", + " train_losses = []\n", + " test_losses = []\n", + " train_accuracies = []\n", + " test_accuracies = []\n", + " ag.set_element_type(train_losses, tf.float32)\n", + " ag.set_element_type(test_losses, tf.float32)\n", + " ag.set_element_type(train_accuracies, tf.float32)\n", + " ag.set_element_type(test_accuracies, tf.float32)\n", + "\n", + " i = tf.constant(0)\n", + " while i \u003c hp.max_steps:\n", + " train_x, train_y = get_next_batch(train_ds)\n", + " test_x, test_y = get_next_batch(test_ds)\n", + " step_train_loss, step_train_accuracy = fit(m, train_x, train_y, opt)\n", + " step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n", + "\n", + " train_losses.append(step_train_loss)\n", + " test_losses.append(step_test_loss)\n", + " train_accuracies.append(step_train_accuracy)\n", + " test_accuracies.append(step_test_accuracy)\n", + "\n", + " i += 1\n", + " return (ag.stack(train_losses), ag.stack(test_losses),\n", + " ag.stack(train_accuracies), ag.stack(test_accuracies))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 215 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 12156, + "status": "ok", + "timestamp": 1531752050611, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "K1m8TwOKjdNd", + "outputId": "bd5746f2-bf91-44aa-9eff-38eb11ced33f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "('Duration:', 0.6226680278778076)\n", + "('Duration:', 0.6082069873809814)\n", + "('Duration:', 0.6223258972167969)\n", + "('Duration:', 0.6176440715789795)\n", + "('Duration:', 0.6309840679168701)\n", + "('Duration:', 0.6180410385131836)\n", + "('Duration:', 0.6219630241394043)\n", + "('Duration:', 0.6183009147644043)\n", + "('Duration:', 0.6176400184631348)\n", + "('Duration:', 0.6476900577545166)\n", + "('Mean duration:', 0.62254641056060789, '+/-', 0.0099792188690656976)\n" + ] + } + ], + "source": [ + "#@test {\"timeout\": 90}\n", + "with tf.Graph().as_default():\n", + " hp = tf.contrib.training.HParams(\n", + " learning_rate=0.05,\n", + " max_steps=max_steps,\n", + " )\n", + " train_ds = setup_mnist_data(True, hp, 500)\n", + " test_ds = setup_mnist_data(False, hp, 100)\n", + " tf_train = ag.to_graph(train)\n", + " losses = tf_train(train_ds, test_ds, hp)\n", + "\n", + " with tf.Session() as sess:\n", + " durations = []\n", + " for t in range(burn_ins + trials):\n", + " sess.run(tf.global_variables_initializer())\n", + "\n", + " start = time.time()\n", + " (train_losses, test_losses, train_accuracies,\n", + " test_accuracies) = sess.run(losses)\n", + "\n", + " if t \u003c burn_ins:\n", + " continue\n", + "\n", + " duration = time.time() - start\n", + " durations.append(duration)\n", + " print('Duration:', duration)\n", + "\n", + " print('Mean duration:', np.mean(durations), '+/-', np.std(durations))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "A06kdgtZtlce" + }, + "source": [ + "# Eager" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "hBKOKGrWty4e" + }, + "outputs": [], + "source": [ + "def predict(m, x, y):\n", + " y_p = m(x)\n", + " losses = tf.keras.losses.categorical_crossentropy(tf.cast(y, tf.float32), y_p)\n", + " l = tf.reduce_mean(losses)\n", + " accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n", + " accuracy = tf.reduce_mean(accuracies)\n", + " return l, accuracy\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "HCgTZ0MTt6vt" + }, + "outputs": [], + "source": [ + "def train(ds, hp):\n", + " m = mlp_model((28 * 28,))\n", + " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", + "\n", + " train_losses = []\n", + " test_losses = []\n", + " train_accuracies = []\n", + " test_accuracies = []\n", + "\n", + " i = 0\n", + " train_test_itr = tfe.Iterator(ds)\n", + " for (train_x, train_y), (test_x, test_y) in train_test_itr:\n", + " train_x = tf.to_float(tf.reshape(train_x, (-1, 28 * 28)))\n", + " train_y = tf.one_hot(tf.squeeze(train_y), 10)\n", + " test_x = tf.to_float(tf.reshape(test_x, (-1, 28 * 28)))\n", + " test_y = tf.one_hot(tf.squeeze(test_y), 10)\n", + "\n", + " if i \u003e hp.max_steps:\n", + " break\n", + "\n", + " with tf.GradientTape() as tape:\n", + " step_train_loss, step_train_accuracy = predict(m, train_x, train_y)\n", + " grad = tape.gradient(step_train_loss, m.variables)\n", + " opt.apply_gradients(zip(grad, m.variables))\n", + " step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n", + "\n", + " train_losses.append(step_train_loss)\n", + " test_losses.append(step_test_loss)\n", + " train_accuracies.append(step_train_accuracy)\n", + " test_accuracies.append(step_test_accuracy)\n", + "\n", + " i += 1\n", + " return train_losses, test_losses, train_accuracies, test_accuracies\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 215 + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 52499, + "status": "ok", + "timestamp": 1531752103279, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "plv_yrn_t8Dy", + "outputId": "55d5ab3d-252d-48ba-8fb4-20ec3c3e6d00" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "('Duration:', 3.9973549842834473)\n", + "('Duration:', 4.018772125244141)\n", + "('Duration:', 3.9740989208221436)\n", + "('Duration:', 3.9922947883605957)\n", + "('Duration:', 3.9795801639556885)\n", + "('Duration:', 3.966722011566162)\n", + "('Duration:', 3.986541986465454)\n", + "('Duration:', 3.992305040359497)\n", + "('Duration:', 4.012261867523193)\n", + "('Duration:', 4.004716157913208)\n", + "('Mean duration:', 3.9924648046493529, '+/-', 0.015681688635624851)\n" + ] + } + ], + "source": [ + "#@test {\"timeout\": 90}\n", + "with context.eager_mode():\n", + " durations = []\n", + " for t in range(burn_ins + trials):\n", + " hp = tf.contrib.training.HParams(\n", + " learning_rate=0.05,\n", + " max_steps=max_steps,\n", + " )\n", + " train_ds = setup_mnist_data(True, hp, 500)\n", + " test_ds = setup_mnist_data(False, hp, 100)\n", + " ds = tf.data.Dataset.zip((train_ds, test_ds))\n", + " start = time.time()\n", + " (train_losses, test_losses, train_accuracies,\n", + " test_accuracies) = train(ds, hp)\n", + " \n", + " train_losses[-1].numpy()\n", + " test_losses[-1].numpy()\n", + " train_accuracies[-1].numpy()\n", + " test_accuracies[-1].numpy()\n", + "\n", + " if t \u003c burn_ins:\n", + " continue\n", + "\n", + " duration = time.time() - start\n", + " durations.append(duration)\n", + " print('Duration:', duration)\n", + "\n", + " print('Mean duration:', np.mean(durations), '+/-', np.std(durations))\n" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [ + "eqOvRhOz8SWs", + "PZWxEJFM9A7b", + "kZV_3pGy8033" + ], + "default_view": {}, + "name": "Autograph vs. Eager MNIST speed test", + "provenance": [ + { + "file_id": "1tAQW5tHUgAc8M4-iwwJm6Xs6dV9nEqtD", + "timestamp": 1530297010607 + }, + { + "file_id": "18dCjshrmHiPTIe1CNsL8tnpdGkuXgpM9", + "timestamp": 1530289467317 + }, + { + "file_id": "1DcfimonWU11tmyivKBGVrbpAl3BIOaRG", + "timestamp": 1522272821237 + }, + { + "file_id": "1wCZUh73zTNs1jzzYjqoxMIdaBWCdKJ2K", + "timestamp": 1522238054357 + }, + { + "file_id": "1_HpC-RrmIv4lNaqeoslUeWaX8zH5IXaJ", + "timestamp": 1521743157199 + }, + { + "file_id": "1mjO2fQ2F9hxpAzw2mnrrUkcgfb7xSGW-", + "timestamp": 1520522344607 + } + ], + "version": "0.3.2", + "views": {} + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb b/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb deleted file mode 100644 index 18eb84cca9..0000000000 --- a/tensorflow/contrib/autograph/examples/notebooks/autograph_vs_eager_mnist_benchmark.ipynb +++ /dev/null @@ -1,577 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "Pa2qpEmoVOGe" - }, - "outputs": [], - "source": [ - "import os\n", - "import time\n", - "\n", - "import numpy as np\n", - "import six\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.contrib import autograph\n", - "from tensorflow.contrib.eager.python import tfe\n", - "from tensorflow.python.eager import context\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "YfnHJbBOBKae" - }, - "outputs": [], - "source": [ - "import gzip\n", - "import shutil\n", - "\n", - "from six.moves import urllib\n", - "\n", - "\n", - "def download(directory, filename):\n", - " filepath = os.path.join(directory, filename)\n", - " if tf.gfile.Exists(filepath):\n", - " return filepath\n", - " if not tf.gfile.Exists(directory):\n", - " tf.gfile.MakeDirs(directory)\n", - " url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'\n", - " zipped_filepath = filepath + '.gz'\n", - " print('Downloading %s to %s' % (url, zipped_filepath))\n", - " urllib.request.urlretrieve(url, zipped_filepath)\n", - " with gzip.open(zipped_filepath, 'rb') as f_in, open(filepath, 'wb') as f_out:\n", - " shutil.copyfileobj(f_in, f_out)\n", - " os.remove(zipped_filepath)\n", - " return filepath\n", - "\n", - "\n", - "def dataset(directory, images_file, labels_file):\n", - " images_file = download(directory, images_file)\n", - " labels_file = download(directory, labels_file)\n", - "\n", - " def decode_image(image):\n", - " # Normalize from [0, 255] to [0.0, 1.0]\n", - " image = tf.decode_raw(image, tf.uint8)\n", - " image = tf.cast(image, tf.float32)\n", - " image = tf.reshape(image, [784])\n", - " return image / 255.0\n", - "\n", - " def decode_label(label):\n", - " label = tf.decode_raw(label, tf.uint8)\n", - " label = tf.reshape(label, [])\n", - " return tf.to_int32(label)\n", - "\n", - " images = tf.data.FixedLengthRecordDataset(\n", - " images_file, 28 * 28, header_bytes=16).map(decode_image)\n", - " labels = tf.data.FixedLengthRecordDataset(\n", - " labels_file, 1, header_bytes=8).map(decode_label)\n", - " return tf.data.Dataset.zip((images, labels))\n", - "\n", - "\n", - "def mnist_train(directory):\n", - " return dataset(directory, 'train-images-idx3-ubyte',\n", - " 'train-labels-idx1-ubyte')\n", - "\n", - "def mnist_test(directory):\n", - " return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')\n", - "\n", - "def setup_mnist_data(is_training, hp, batch_size):\n", - " if is_training:\n", - " ds = mnist_train('/tmp/autograph_mnist_data')\n", - " ds = ds.cache()\n", - " ds = ds.shuffle(batch_size * 10)\n", - " else:\n", - " ds = mnist_test('/tmp/autograph_mnist_data')\n", - " ds = ds.cache()\n", - " ds = ds.repeat()\n", - " ds = ds.batch(batch_size)\n", - " return ds\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "x_MU13boiok2" - }, - "outputs": [], - "source": [ - "def mlp_model(input_shape):\n", - " model = tf.keras.Sequential((\n", - " tf.keras.layers.Dense(100, activation='relu', input_shape=input_shape),\n", - " tf.keras.layers.Dense(100, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')))\n", - " model.build()\n", - " return model\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "kfZk9EFZ5TeQ" - }, - "outputs": [], - "source": [ - "# Test-only parameters. Test checks successful completion not correctness. \n", - "burn_ins = 1\n", - "trials = 1\n", - "max_steps = 2\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "gWXV8WHn43iZ" - }, - "outputs": [], - "source": [ - "#@test {\"skip\": true} \n", - "burn_ins = 3\n", - "trials = 10\n", - "max_steps = 500\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DXt4GoTxtvn2" - }, - "source": [ - "# Autograph" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "W51sfbONiz_5" - }, - "outputs": [], - "source": [ - "def predict(m, x, y):\n", - " y_p = m(x)\n", - " losses = tf.keras.losses.categorical_crossentropy(y, y_p)\n", - " l = tf.reduce_mean(losses)\n", - " accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n", - " accuracy = tf.reduce_mean(accuracies)\n", - " return l, accuracy\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "CsAD0ajbi9iZ" - }, - "outputs": [], - "source": [ - "def fit(m, x, y, opt):\n", - " l, accuracy = predict(m, x, y)\n", - " opt.minimize(l)\n", - " return l, accuracy\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "RVw57HdTjPzi" - }, - "outputs": [], - "source": [ - "def get_next_batch(ds):\n", - " itr = ds.make_one_shot_iterator()\n", - " image, label = itr.get_next()\n", - " x = tf.to_float(tf.reshape(image, (-1, 28 * 28)))\n", - " y = tf.one_hot(tf.squeeze(label), 10)\n", - " return x, y\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "UUI0566FjZPx" - }, - "outputs": [], - "source": [ - "def train(train_ds, test_ds, hp):\n", - " m = mlp_model((28 * 28,))\n", - " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", - "\n", - " train_losses = []\n", - " test_losses = []\n", - " train_accuracies = []\n", - " test_accuracies = []\n", - " autograph.set_element_type(train_losses, tf.float32)\n", - " autograph.set_element_type(test_losses, tf.float32)\n", - " autograph.set_element_type(train_accuracies, tf.float32)\n", - " autograph.set_element_type(test_accuracies, tf.float32)\n", - "\n", - " i = tf.constant(0)\n", - " while i \u003c hp.max_steps:\n", - " train_x, train_y = get_next_batch(train_ds)\n", - " test_x, test_y = get_next_batch(test_ds)\n", - " step_train_loss, step_train_accuracy = fit(m, train_x, train_y, opt)\n", - " step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n", - "\n", - " train_losses.append(step_train_loss)\n", - " test_losses.append(step_test_loss)\n", - " train_accuracies.append(step_train_accuracy)\n", - " test_accuracies.append(step_test_accuracy)\n", - "\n", - " i += 1\n", - " return (autograph.stack(train_losses), autograph.stack(test_losses),\n", - " autograph.stack(train_accuracies), autograph.stack(test_accuracies))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 220 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 12896, - "status": "ok", - "timestamp": 1531534784996, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "K1m8TwOKjdNd", - "outputId": "2ee3ff78-9aae-4fac-a1fd-32bf3b2f18f4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "('Duration:', 0.7540969848632812)\n", - "('Duration:', 0.7829370498657227)\n", - "('Duration:', 0.7111489772796631)\n", - "('Duration:', 0.6126768589019775)\n", - "('Duration:', 0.6143529415130615)\n", - "('Duration:', 0.6174650192260742)\n", - "('Duration:', 0.6425611972808838)\n", - "('Duration:', 0.6188449859619141)\n", - "('Duration:', 0.6388339996337891)\n", - "('Duration:', 0.6235959529876709)\n", - "('Mean duration:', 0.66165139675140383, '+/-', 0.060382254849383483)\n" - ] - } - ], - "source": [ - "#@test {\"timeout\": 90}\n", - "with tf.Graph().as_default():\n", - " hp = tf.contrib.training.HParams(\n", - " learning_rate=0.05,\n", - " max_steps=max_steps,\n", - " )\n", - " train_ds = setup_mnist_data(True, hp, 500)\n", - " test_ds = setup_mnist_data(False, hp, 100)\n", - " tf_train = autograph.to_graph(train)\n", - " losses = tf_train(train_ds, test_ds, hp)\n", - "\n", - " with tf.Session() as sess:\n", - " durations = []\n", - " for t in range(burn_ins + trials):\n", - " sess.run(tf.global_variables_initializer())\n", - "\n", - " start = time.time()\n", - " (train_losses, test_losses, train_accuracies,\n", - " test_accuracies) = sess.run(losses)\n", - "\n", - " if t \u003c burn_ins:\n", - " continue\n", - "\n", - " duration = time.time() - start\n", - " durations.append(duration)\n", - " print('Duration:', duration)\n", - "\n", - " print('Mean duration:', np.mean(durations), '+/-', np.std(durations))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "A06kdgtZtlce" - }, - "source": [ - "# Eager" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "hBKOKGrWty4e" - }, - "outputs": [], - "source": [ - "def predict(m, x, y):\n", - " y_p = m(x)\n", - " losses = tf.keras.losses.categorical_crossentropy(tf.cast(y, tf.float32), y_p)\n", - " l = tf.reduce_mean(losses)\n", - " accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n", - " accuracy = tf.reduce_mean(accuracies)\n", - " return l, accuracy\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - } - }, - "colab_type": "code", - "id": "HCgTZ0MTt6vt" - }, - "outputs": [], - "source": [ - "def train(ds, hp):\n", - " m = mlp_model((28 * 28,))\n", - " opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n", - "\n", - " train_losses = []\n", - " test_losses = []\n", - " train_accuracies = []\n", - " test_accuracies = []\n", - "\n", - " i = 0\n", - " train_test_itr = tfe.Iterator(ds)\n", - " for (train_x, train_y), (test_x, test_y) in train_test_itr:\n", - " train_x = tf.to_float(tf.reshape(train_x, (-1, 28 * 28)))\n", - " train_y = tf.one_hot(tf.squeeze(train_y), 10)\n", - " test_x = tf.to_float(tf.reshape(test_x, (-1, 28 * 28)))\n", - " test_y = tf.one_hot(tf.squeeze(test_y), 10)\n", - "\n", - " if i \u003e hp.max_steps:\n", - " break\n", - "\n", - " with tf.GradientTape() as tape:\n", - " step_train_loss, step_train_accuracy = predict(m, train_x, train_y)\n", - " grad = tape.gradient(step_train_loss, m.variables)\n", - " opt.apply_gradients(zip(grad, m.variables))\n", - " step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n", - "\n", - " train_losses.append(step_train_loss)\n", - " test_losses.append(step_test_loss)\n", - " train_accuracies.append(step_train_accuracy)\n", - " test_accuracies.append(step_test_accuracy)\n", - "\n", - " i += 1\n", - " return train_losses, test_losses, train_accuracies, test_accuracies\n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "height": 220 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 53945, - "status": "ok", - "timestamp": 1531534839296, - "user": { - "displayName": "", - "photoUrl": "", - "userId": "" - }, - "user_tz": 240 - }, - "id": "plv_yrn_t8Dy", - "outputId": "93f2f468-7191-430c-88d2-948b4ce1ea06" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "('Duration:', 4.146992206573486)\n", - "('Duration:', 4.107615947723389)\n", - "('Duration:', 4.07602596282959)\n", - "('Duration:', 4.113464832305908)\n", - "('Duration:', 4.100026845932007)\n", - "('Duration:', 4.145462989807129)\n", - "('Duration:', 4.11216402053833)\n", - "('Duration:', 4.094243049621582)\n", - "('Duration:', 4.095034837722778)\n", - "('Duration:', 4.11162805557251)\n", - "('Mean duration:', 4.1102658748626713, '+/-', 0.020919605607527668)\n" - ] - } - ], - "source": [ - "#@test {\"timeout\": 90}\n", - "with context.eager_mode():\n", - " durations = []\n", - " for t in range(burn_ins + trials):\n", - " hp = tf.contrib.training.HParams(\n", - " learning_rate=0.05,\n", - " max_steps=max_steps,\n", - " )\n", - " train_ds = setup_mnist_data(True, hp, 500)\n", - " test_ds = setup_mnist_data(False, hp, 100)\n", - " ds = tf.data.Dataset.zip((train_ds, test_ds))\n", - " start = time.time()\n", - " (train_losses, test_losses, train_accuracies,\n", - " test_accuracies) = train(ds, hp)\n", - " \n", - " train_losses[-1].numpy()\n", - " test_losses[-1].numpy()\n", - " train_accuracies[-1].numpy()\n", - " test_accuracies[-1].numpy()\n", - "\n", - " if t \u003c burn_ins:\n", - " continue\n", - "\n", - " duration = time.time() - start\n", - " durations.append(duration)\n", - " print('Duration:', duration)\n", - "\n", - " print('Mean duration:', np.mean(durations), '+/-', np.std(durations))\n" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "default_view": {}, - "last_runtime": { - "build_target": "", - "kind": "local" - }, - "name": "Autograph vs. Eager MNIST benchmark", - "provenance": [ - { - "file_id": "1tAQW5tHUgAc8M4-iwwJm6Xs6dV9nEqtD", - "timestamp": 1530297010607 - }, - { - "file_id": "18dCjshrmHiPTIe1CNsL8tnpdGkuXgpM9", - "timestamp": 1530289467317 - }, - { - "file_id": "1DcfimonWU11tmyivKBGVrbpAl3BIOaRG", - "timestamp": 1522272821237 - }, - { - "file_id": "1wCZUh73zTNs1jzzYjqoxMIdaBWCdKJ2K", - "timestamp": 1522238054357 - }, - { - "file_id": "1_HpC-RrmIv4lNaqeoslUeWaX8zH5IXaJ", - "timestamp": 1521743157199 - }, - { - "file_id": "1mjO2fQ2F9hxpAzw2mnrrUkcgfb7xSGW-", - "timestamp": 1520522344607 - } - ], - "version": "0.3.2", - "views": {} - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} -- cgit v1.2.3 From caae486f919069268ca7ad9dd98a6e53efede4bb Mon Sep 17 00:00:00 2001 From: Andrew Selle Date: Mon, 16 Jul 2018 10:13:10 -0700 Subject: Describe that the semantics of gfile are different. Fixes #19296. --- tensorflow/python/platform/gfile.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/platform/gfile.py b/tensorflow/python/platform/gfile.py index fd697d70bf..cfc50e8223 100644 --- a/tensorflow/python/platform/gfile.py +++ b/tensorflow/python/platform/gfile.py @@ -38,7 +38,14 @@ from tensorflow.python.util.tf_export import tf_export @tf_export('gfile.GFile', 'gfile.Open') class GFile(_FileIO): - """File I/O wrappers without thread locking.""" + """File I/O wrappers without thread locking. + + Note, that this is somewhat like builtin Python file I/O, but + there are semantic differences to make it more efficient for + some backing filesystems. For example, a write mode file will + not be opened until the first write call (to minimize RPC + invocations in network filesystems). + """ def __init__(self, name, mode='r'): super(GFile, self).__init__(name=name, mode=mode) @@ -46,7 +53,14 @@ class GFile(_FileIO): @tf_export('gfile.FastGFile') class FastGFile(_FileIO): - """File I/O wrappers without thread locking.""" + """File I/O wrappers without thread locking. + + Note, that this is somewhat like builtin Python file I/O, but + there are semantic differences to make it more efficient for + some backing filesystems. For example, a write mode file will + not be opened until the first write call (to minimize RPC + invocations in network filesystems). + """ def __init__(self, name, mode='r'): super(FastGFile, self).__init__(name=name, mode=mode) -- cgit v1.2.3 From fd04b76337f9f65c5a2ce35a4f7336a25511435b Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Mon, 16 Jul 2018 10:15:40 -0700 Subject: Give EagerTensor a fully qualified name so __module__ doesn't generate an error This code still differs between py2 and py3 (__module__ returns "__builtin__" in py2, and the correct value in py3) - but its strictly better than before since earlier it would differ between py2 and py3 and generate an error in py3. We don't seem to correctly initialize the tp_dict in py2, so even when passing the correct, fully qualified name, we get back "__builtin__". Fixes #20701 PiperOrigin-RevId: 204762170 --- tensorflow/python/eager/pywrap_tensor.cc | 32 ++++++++++++++++++++++++++++---- tensorflow/python/eager/tensor_test.py | 2 +- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc index ea604647fa..cefd5b1206 100644 --- a/tensorflow/python/eager/pywrap_tensor.cc +++ b/tensorflow/python/eager/pywrap_tensor.cc @@ -620,10 +620,6 @@ static PyType_Slot EagerTensor_Type_slots[] = { {Py_tp_init, reinterpret_cast(EagerTensor_init)}, {0, nullptr}, }; - -PyType_Spec EagerTensor_Type_spec = {"EagerTensor", sizeof(EagerTensor), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE, - EagerTensor_Type_slots}; #else // TODO(agarwal): support active_trace. static PyTypeObject _EagerTensorType = { @@ -754,6 +750,34 @@ PyObject* TFE_Py_InitEagerTensor(PyObject* base_class) { #if PY_MAJOR_VERSION >= 3 PyObject* bases = PyTuple_New(1); PyTuple_SET_ITEM(bases, 0, base_class); + + tensorflow::Safe_PyObjectPtr base_class_module( + PyObject_GetAttrString(base_class, "__module__")); + const char* module = nullptr; + if (PyErr_Occurred()) { + PyErr_Clear(); + module = "__builtin__"; + } else { + module = PyBytes_AsString(base_class_module.get()); + if (module == nullptr) { + PyErr_Clear(); + module = PyUnicode_AsUTF8(base_class_module.get()); + if (module == nullptr) { + PyErr_Clear(); + module = "__builtin__"; + } + } + } + + // NOTE: The c_str from this string needs to outlast the function, hence is + // static. + static tensorflow::string fully_qualified_name = + tensorflow::strings::StrCat(module, ".EagerTensor"); + + static PyType_Spec EagerTensor_Type_spec = { + fully_qualified_name.c_str(), sizeof(EagerTensor), 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE, EagerTensor_Type_slots}; + EagerTensorType = reinterpret_cast( PyType_FromSpecWithBases(&EagerTensor_Type_spec, bases)); if (PyErr_Occurred()) { diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py index 626a4eb1ee..871136e2c8 100644 --- a/tensorflow/python/eager/tensor_test.py +++ b/tensorflow/python/eager/tensor_test.py @@ -278,7 +278,7 @@ class TFETensorUtilTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp( TypeError, - r"tensors argument must be a list or a tuple. Got \"EagerTensor\""): + r"tensors argument must be a list or a tuple. Got.*EagerTensor"): pywrap_tensorflow.TFE_Py_TensorShapeSlice(t1, -2) def testNegativeSliceDim(self): -- cgit v1.2.3 From c067242a12b8a0cc0cc9996e2a3e1eed7de4f53b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 10:23:41 -0700 Subject: Upgrade Bazel to 0.14.1 PiperOrigin-RevId: 204763605 --- tensorflow/tools/ci_build/install/install_bazel.sh | 2 +- tensorflow/tools/ci_build/install/install_bazel_from_source.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/ci_build/install/install_bazel.sh b/tensorflow/tools/ci_build/install/install_bazel.sh index 3e27a94cf2..adbff8f6ef 100755 --- a/tensorflow/tools/ci_build/install/install_bazel.sh +++ b/tensorflow/tools/ci_build/install/install_bazel.sh @@ -15,7 +15,7 @@ # ============================================================================== # Select bazel version. -BAZEL_VERSION="0.11.0" +BAZEL_VERSION="0.14.1" set +e local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}') diff --git a/tensorflow/tools/ci_build/install/install_bazel_from_source.sh b/tensorflow/tools/ci_build/install/install_bazel_from_source.sh index ddad00c5f0..9d24b3e421 100755 --- a/tensorflow/tools/ci_build/install/install_bazel_from_source.sh +++ b/tensorflow/tools/ci_build/install/install_bazel_from_source.sh @@ -18,7 +18,7 @@ # It will compile bazel from source and install it in /usr/local/bin # Select bazel version. -BAZEL_VERSION="0.11.0" +BAZEL_VERSION="0.14.1" set +e local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}') -- cgit v1.2.3 From fea79c7c22cc70e910c4f4dbd72d2f28491ba6ff Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Thu, 12 Jul 2018 22:09:41 +0000 Subject: [XLA:GPU] rename GpuCompiler to NVPTXCompiler To cope with upcoming introduction of AMDGPU support in XLA, rename GpuCompiler and associated gpu_backend_lib module to NVPTXCompiler and nvptx_backend_lib. --- tensorflow/compiler/xla/service/gpu/BUILD | 4 +- .../compiler/xla/service/gpu/gpu_compiler.cc | 802 --------------------- tensorflow/compiler/xla/service/gpu/gpu_compiler.h | 155 ---- .../xla/service/gpu/gpu_transfer_manager.cc | 2 +- .../xla/service/gpu/llvm_gpu_backend/BUILD | 4 +- .../gpu/llvm_gpu_backend/gpu_backend_lib.cc | 506 ------------- .../service/gpu/llvm_gpu_backend/gpu_backend_lib.h | 47 -- .../gpu/llvm_gpu_backend/nvptx_backend_lib.cc | 506 +++++++++++++ .../gpu/llvm_gpu_backend/nvptx_backend_lib.h | 47 ++ .../compiler/xla/service/gpu/nvptx_compiler.cc | 802 +++++++++++++++++++++ .../compiler/xla/service/gpu/nvptx_compiler.h | 155 ++++ 11 files changed, 1515 insertions(+), 1515 deletions(-) delete mode 100644 tensorflow/compiler/xla/service/gpu/gpu_compiler.cc delete mode 100644 tensorflow/compiler/xla/service/gpu/gpu_compiler.h delete mode 100644 tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc delete mode 100644 tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.h create mode 100644 tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc create mode 100644 tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.h create mode 100644 tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc create mode 100644 tensorflow/compiler/xla/service/gpu/nvptx_compiler.h diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 59172e53d3..ceb3b5b5df 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -561,8 +561,8 @@ cc_library( cc_library( name = "gpu_compiler", - srcs = ["gpu_compiler.cc"], - hdrs = ["gpu_compiler.h"], + srcs = ["nvptx_compiler.cc"], + hdrs = ["nvptx_compiler.h"], deps = [ ":cudnn_convolution_algorithm_picker", ":cudnn_convolution_rewriter", diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc deleted file mode 100644 index 5e5d893582..0000000000 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ /dev/null @@ -1,802 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/xla/service/gpu/gpu_compiler.h" - -#include -#include -#include -#include // NOLINT(build/c++11): only using std::call_once, not mutex. -#include - -#include "llvm/IR/DiagnosticInfo.h" -#include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Verifier.h" -#include "tensorflow/compiler/xla/protobuf_util.h" -#include "tensorflow/compiler/xla/ptr_util.h" -#include "tensorflow/compiler/xla/service/algebraic_simplifier.h" -#include "tensorflow/compiler/xla/service/batchnorm_expander.h" -#include "tensorflow/compiler/xla/service/buffer_assignment.h" -#include "tensorflow/compiler/xla/service/buffer_liveness.h" -#include "tensorflow/compiler/xla/service/call_inliner.h" -#include "tensorflow/compiler/xla/service/conditional_simplifier.h" -#include "tensorflow/compiler/xla/service/dot_decomposer.h" -#include "tensorflow/compiler/xla/service/flatten_call_graph.h" -#include "tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.h" -#include "tensorflow/compiler/xla/service/gpu/cudnn_convolution_algorithm_picker.h" -#include "tensorflow/compiler/xla/service/gpu/cudnn_convolution_rewriter.h" -#include "tensorflow/compiler/xla/service/gpu/fusion_merger.h" -#include "tensorflow/compiler/xla/service/gpu/gpu_constants.h" -#include "tensorflow/compiler/xla/service/gpu/gpu_copy_insertion.h" -#include "tensorflow/compiler/xla/service/gpu/gpu_executable.h" -#include "tensorflow/compiler/xla/service/gpu/gpu_hlo_support_checker.h" -#include "tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.h" -#include "tensorflow/compiler/xla/service/gpu/hlo_schedule.h" -#include "tensorflow/compiler/xla/service/gpu/instruction_fusion.h" -#include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h" -#include "tensorflow/compiler/xla/service/gpu/ir_emitter_context.h" -#include "tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h" -#include "tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.h" -#include "tensorflow/compiler/xla/service/gpu/multi_output_fusion.h" -#include "tensorflow/compiler/xla/service/gpu/pad_insertion.h" -#include "tensorflow/compiler/xla/service/gpu/partition_assignment.h" -#include "tensorflow/compiler/xla/service/gpu/stream_assignment.h" -#include "tensorflow/compiler/xla/service/gpu/thunk_schedule.h" -#include "tensorflow/compiler/xla/service/hlo.pb.h" -#include "tensorflow/compiler/xla/service/hlo_computation.h" -#include "tensorflow/compiler/xla/service/hlo_constant_folding.h" -#include "tensorflow/compiler/xla/service/hlo_cse.h" -#include "tensorflow/compiler/xla/service/hlo_dce.h" -#include "tensorflow/compiler/xla/service/hlo_element_type_converter.h" -#include "tensorflow/compiler/xla/service/hlo_instruction.h" -#include "tensorflow/compiler/xla/service/hlo_pass_fix.h" -#include "tensorflow/compiler/xla/service/hlo_pass_pipeline.h" -#include "tensorflow/compiler/xla/service/hlo_proto_util.h" -#include "tensorflow/compiler/xla/service/hlo_subcomputation_unification.h" -#include "tensorflow/compiler/xla/service/hlo_verifier.h" -#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h" -#include "tensorflow/compiler/xla/service/reduce_precision_insertion.h" -#include "tensorflow/compiler/xla/service/reshape_mover.h" -#include "tensorflow/compiler/xla/service/transpose_folding.h" -#include "tensorflow/compiler/xla/service/tuple_simplifier.h" -#include "tensorflow/compiler/xla/service/while_loop_constant_sinking.h" -#include "tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h" -#include "tensorflow/compiler/xla/service/while_loop_simplifier.h" -#include "tensorflow/compiler/xla/service/zero_sized_hlo_elimination.h" -#include "tensorflow/compiler/xla/status_macros.h" -#include "tensorflow/compiler/xla/types.h" -#include "tensorflow/compiler/xla/util.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/gtl/cleanup.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/cuda_libdevice_path.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/regexp.h" -#include "tensorflow/core/platform/stream_executor_no_cuda.h" -#include "tensorflow/core/platform/subprocess.h" -#include "tensorflow/core/platform/tracing.h" -#include "tensorflow/stream_executor/cuda/cuda_diagnostics.h" - -namespace xla { -namespace gpu { - -/* static */ const char* GpuCompiler::kTargetTriple = "nvptx64-nvidia-cuda"; -/* static */ const char* GpuCompiler::kDataLayout = - "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"; - -namespace { - -namespace tracing = tensorflow::tracing; - -// Returns the directory containing nvvm libdevice files. config_cuda_data_dir -// should be equal to config().debug_options().xla_gpu_cuda_data_dir() of the -// HloModule being compiled. -string GetLibdeviceDir(const string& config_cuda_data_dir) { - std::vector potential_libdevice_dirs; - if (!config_cuda_data_dir.empty()) { - potential_libdevice_dirs.push_back(config_cuda_data_dir); - } - potential_libdevice_dirs.push_back(tensorflow::LibdeviceRoot()); - - // Tries all potential libdevice directories in the order they are inserted. - // Returns the first directory that exists in the file system. - for (const string& potential_libdevice_dir : potential_libdevice_dirs) { - if (tensorflow::Env::Default()->IsDirectory(potential_libdevice_dir).ok()) { - VLOG(2) << "Found libdevice dir " << potential_libdevice_dir; - return potential_libdevice_dir; - } - VLOG(2) << "Unable to find potential libdevice dir " - << potential_libdevice_dir; - } - - // Last resort: maybe in the current folder. - return "."; -} - -// Runs optimization passes on the given HLO module. -Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec, - DeviceMemoryAllocator* device_allocator) { - { - HloPassPipeline pipeline("optimization"); - pipeline.AddInvariantChecker(); - pipeline.AddPass(); - ReducePrecisionInsertion::AddPasses( - &pipeline, hlo_module->config().debug_options(), - ReducePrecisionInsertion::PassTiming::BEFORE_OPTIMIZATION); - - // TODO(b/64094172): make Call work on GPU instead of inlining. - pipeline.AddPass(); - // Convert BF16 operations to F32 operations so that the GPU backend can - // support BF16 operations without directly implementing a BF16 lowering for - // most ops. - pipeline.AddPass(BF16, F32); - pipeline.AddPass(); - - { - auto& pass = - pipeline.AddPass>("simplification"); - pass.AddInvariantChecker(); - - // If cudnn batchnorms are enabled, rewrite batchnorm HLOs to cudnn calls - // where possible. Not every batchnorm op can be implemented as a call to - // cudnn, so decompose any remaining batchnorm ops into a soup of HLOs. - if (hlo_module->config().debug_options().xla_gpu_use_cudnn_batchnorm()) { - pass.AddPass(); - } - pass.AddPass( - /*rewrite_training_op=*/true, - /*rewrite_inference_op=*/true, - /*rewrite_grad_op=*/true); - - // BatchNormExpander can create zero-sized ops, so zero-sized HLO - // elimination has to come after that pass. - pipeline.AddPass(); - - pass.AddPass( - /*is_layout_sensitive=*/false, - [](const Shape&, const Shape&) { return false; }); - pass.AddPass(); - pass.AddPass(); - pass.AddPass(); - pass.AddPass(); - pass.AddPass(); - pass.AddPass(); - pass.AddPass(); - } - - pipeline.AddPass( - [](const HloInstruction& dot, - const TransposeFolding::OperandIndices& candidate_operands) { - return ImplementedAsGemm(dot) ? candidate_operands - : TransposeFolding::OperandIndices{}; - }, - TransposeFolding::NeverFoldTranspose); - pipeline.AddPass(/*is_layout_sensitive=*/false); - pipeline.AddPass(); - TF_RETURN_IF_ERROR(pipeline.Run(hlo_module).status()); - } - - { - // Convert convolutions into CustomCalls to cudnn, then canonicalize them - // (PadInsertion). - HloPassPipeline pipeline("conv_canonicalization"); - pipeline.AddInvariantChecker(); - pipeline.AddPass(); - pipeline.AddPass(); - TF_RETURN_IF_ERROR(pipeline.Run(hlo_module).status()); - } - - { - HloPassPipeline pipeline("layout_assignment"); - pipeline.AddPass( - hlo_module->mutable_entry_computation_layout(), stream_exec); - - // The LayoutAssignment pass may leave behind kCopy instructions which are - // duplicate or NOPs, so remove them with algebraic simplification and CSE. - pipeline.AddPass>( - /*is_layout_sensitive=*/true, - /*valid_bitcast_callback=*/[](const Shape&, const Shape&) { - return true; - }); - - // Choose the fastest algorithm for each conv. - // - // We pick the algorithm before fusion so we can generate better HLO. After - // CudnnConvolutionRewriter, our convolutions are CustomCalls which return a - // tuple (conv_result, scratch_memory), and the each conv uses 0 bytes of - // scratch: - // - // customcall = (f32[...], f32[0]) - // return gte(customcall, 0) - // - // The algorithm picker then chooses the best algorithm, and potentially - // increases the scratch space. It replaces customcall with new_tuple, - // giving us the following: - // - // new_customcall = (f32[...], f32[N]) - // new_tuple = tuple(gte(new_customcall, 0), constant f32[0]) - // return gte(new_tuple, 0) - // - // The new tuple and gte instructions then be simplified away, because - // nobody is expected to use the scratch value. - // - // However, if we were to run CudnnConvolutionAlgorithmPicker after fusion - // the gte(customcall, 0) would probably already be into a fusion node. We - // can't simplify across HloComputation boundaries, so in this case we - // wouldn't be able to simplify away the new_tuple bits. - pipeline.AddPass(stream_exec, - device_allocator); - // Clean up new_tuple described above. - pipeline.AddPass(); - - pipeline.AddPass(/*is_layout_sensitive=*/true); - TF_RETURN_IF_ERROR(pipeline.Run(hlo_module).status()); - } - - { - HloPassFix fusion("fusion"); - fusion.AddInvariantChecker(); - fusion.AddPass(/*may_duplicate=*/false); - fusion.AddPass(/*may_duplicate=*/true); - fusion.AddPass(); - fusion.AddPass(); - fusion.AddPass(/*is_layout_sensitive=*/true, - /*only_fusion_computations=*/true); - TF_RETURN_IF_ERROR(fusion.Run(hlo_module).status()); - - HloPassPipeline reduce_pipeline("reduce-precision"); - reduce_pipeline.AddInvariantChecker(); - ReducePrecisionInsertion::AddPasses( - &reduce_pipeline, hlo_module->config().debug_options(), - ReducePrecisionInsertion::PassTiming::AFTER_FUSION); - StatusOr reduce_result = reduce_pipeline.Run(hlo_module); - TF_RETURN_IF_ERROR(reduce_result.status()); - - if (reduce_result.ValueOrDie()) { - // Do another fusion pass, with the expectation that we may be able to - // fuse the new ReducePrecision operations. - TF_RETURN_IF_ERROR(fusion.Run(hlo_module).status()); - } - } - - { - // Do an aggressive LICM pass over while loops. In particular, this hoists - // constants that were sunk by WhileLoopConstantSinking. Leaving them in - // the while loop may result in unnecessary copies. - HloPassPipeline pipeline("while-loop-licm"); - pipeline.AddPass(true); - TF_RETURN_IF_ERROR(pipeline.Run(hlo_module).status()); - } - return Status::OK(); -} - -// Modifies the given HLO module so that it will be accepted by IrEmitter. -// Unlike optimization passes, the passes are necessary for correctness. -Status PrepareHloModuleForIrEmitting(HloModule* hlo_module) { - // In some cases, we have to place the result of an instruction in a temporary - // buffer. For instance, the buffer that holds an external parameter is - // assumed immutable at this point, and should not be reused for output - // (b/27180329). Therefore, in that case, we set the output to be a copy of - // the parameter. - HloPassPipeline pipeline("GPU-ir-emit-prepare"); - pipeline.AddInvariantChecker(); - - // Copy insertion should be performed immediately before IR emission to avoid - // inserting unnecessary copies (later pass adds an instruction which - // materializes the value) or missing a necessary copy (later pass removes an - // instruction which materializes a value). DCE must be run immediately before - // (and sometime after) copy insertion, to avoid dead code from interfering - // with the rewrites. - pipeline.AddPass(); - pipeline.AddPass(); - pipeline.AddPass(); - return pipeline.Run(hlo_module).status(); -} - -// Prints a warning if the ptxas at ptxas_path has known bugs. -// -// Only prints a warning the first time it's called for a particular value of -// ptxas_path. -void WarnIfBadPtxasVersion(const string& ptxas_path) { - static tensorflow::mutex mu(tensorflow::LINKER_INITIALIZED); - static std::unordered_set* seen_ptxas_paths GUARDED_BY(mu) = - new std::unordered_set(); - - tensorflow::mutex_lock lock(mu); - if (!seen_ptxas_paths->insert(ptxas_path).second) { - // Already checked this ptx binary, nothing to do. - return; - } - - tensorflow::SubProcess ptxas; - ptxas.SetProgram(ptxas_path, {ptxas_path, "--version"}); - ptxas.SetChannelAction(tensorflow::CHAN_STDOUT, tensorflow::ACTION_PIPE); - if (!ptxas.Start()) { - LOG(WARNING) << "Couldn't invoke " << ptxas_path << " --version"; - return; - } - - string out; - int exit_code = ptxas.Communicate(/*stdin_input=*/nullptr, &out, - /*stderr_output=*/nullptr); - if (exit_code != 0) { - LOG(WARNING) << "Running " << ptxas_path << " --version returned " - << exit_code; - return; - } - - int64 vmaj, vmin, vdot; - string vmaj_str, vmin_str, vdot_str; - if (!RE2::PartialMatch(out, R"(\bV(\d+)\.(\d+)\.(\d+)\b)", &vmaj_str, - &vmin_str, &vdot_str) || - !tensorflow::strings::safe_strto64(vmaj_str, &vmaj) || - !tensorflow::strings::safe_strto64(vmin_str, &vmin) || - !tensorflow::strings::safe_strto64(vdot_str, &vdot)) { - LOG(WARNING) << "Couldn't parse ptxas version in output of " << ptxas_path - << " --version:\n" - << out; - return; - } - - // We need ptxas >= 9.0 as a hard requirement, because we compile targeting - // PTX 6.0. An older ptxas will just fail to compile any of our code. - // - // ptxas 9.0 before 9.0.276 and ptxas 9.1 before 9.1.121 miscompile some - // address calculations with large offsets (e.g. "load ptr + large_constant"), - // b/70245379. - // - // ptxas 9.1.121 miscompiles some large multioutput fusions, again in a way - // that appears related to address calculations. ptxas 9.2.88 appears to - // work, as far as we can tell. - if (vmaj < 9) { - LOG(ERROR) - << "You are using ptxas 8.x, but XLA requires ptxas 9.x (and strongly " - "prefers >= 9.2.88). Compilation of XLA kernels below will likely " - "fail.\n\nYou do not need to update CUDA; cherry-picking the ptxas " - "binary is sufficient."; - } else if ((vmaj < 9 || vmin < 2 || vdot < 88)) { - LOG(WARNING) - << "*** WARNING *** You are using ptxas " << vmaj << "." << vmin << "." - << vdot - << ", which older than 9.2.88. ptxas 9.x before 9.2.88 is known to " - "miscompile XLA code, leading to incorrect results or " - "invalid-address errors.\n\nYou do not need to update to CUDA " - "9.2.88; cherry-picking the ptxas binary is sufficient."; - } -} - -// Prints a warning if the ptx->sass JIT in the driver has known bugs. -// -// Using such a driver only a problem if we fail to use ptxas to compile our ptx -// and have to use the driver instead, so you should only call this function if -// we're going to use the driver JIT. -// -// Only prints a warning the first time it's called. -void WarnIfBadDriverJITVersion() { - static std::once_flag run_once; - std::call_once(run_once, [] { - auto version_or_status = se::cuda::Diagnostician::FindKernelDriverVersion(); - if (!version_or_status.ok()) { - LOG(WARNING) << "Couldn't read CUDA driver version."; - return; - } - se::cuda::DriverVersion version = version_or_status.ValueOrDie(); - - // The following versions of the driver JIT miscompile some address - // calculations with large offsets (e.g. "load ptr + large_constant"), - // b/70245379: - // - // - 384.x before 384.108 - // - 387.x before 387.40 - // - 390.x before 390.10. - // - // TODO(jlebar): This list does not cover the address-calculation bug we've - // observed in ptxas 9.1.121. Need to get a new safe range from nvidia - // corresponding to ptxas >= 9.2.88. - auto vmaj = std::get<0>(version); - auto vmin = std::get<1>(version); - if ((vmaj == 384 && vmin < 108) || // - (vmaj == 387 && vmin < 40) || // - (vmaj == 390 && vmin < 10)) { - LOG(WARNING) - << "*** WARNING *** Invoking the PTX->SASS JIT from driver version " - << se::cuda::DriverVersionToString(version) - << ", which is in range [384.0.0, 384.108.0) + [387.0.0, 387.40.0) + " - "[390.0.0, 390.10.0). These versions are known to miscompile XLA " - "code, leading to incorrect results or invalid-address errors."; - } - }); -} - -// Compiles the given PTX string using ptxas and returns the resulting machine -// code (i.e. a cubin) as a byte array. -StatusOr> CompilePtx(const string& ptx, int cc_major, - int cc_minor) { - tracing::ScopedActivity activity("Compile PTX", /*is_expensive=*/true); - const string ptxas_path = - tensorflow::io::JoinPath(tensorflow::CudaRoot(), "bin", "ptxas"); - VLOG(2) << "Using ptxas at " << ptxas_path; - auto env = tensorflow::Env::Default(); - TF_RETURN_IF_ERROR(env->FileExists(ptxas_path)); - - WarnIfBadPtxasVersion(ptxas_path); - - // Write ptx into a temporary file. - string ptx_path; - if (!env->LocalTempFilename(&ptx_path)) { - return InternalError("couldn't get temp PTX file name"); - } - auto ptx_cleaner = tensorflow::gtl::MakeCleanup([&ptx_path] { - TF_CHECK_OK(tensorflow::Env::Default()->DeleteFile(ptx_path)); - }); - - TF_RETURN_IF_ERROR(tensorflow::WriteStringToFile(env, ptx_path, ptx)); - VLOG(2) << "ptx written to: " << ptx_path; - - // Invoke ptxas and collect its output. - string cubin_path; - if (!env->LocalTempFilename(&cubin_path)) { - return InternalError("couldn't get temp CUBIN file name"); - } - auto cubin_cleaner = tensorflow::gtl::MakeCleanup([&cubin_path] { - // CUBIN file may never be created, so the failure to delete it should not - // produce TF error. - tensorflow::Env::Default()->DeleteFile(cubin_path).IgnoreError(); - }); - tensorflow::SubProcess ptxas_info_dumper; - std::vector ptxas_args = { - ptxas_path, ptx_path, "-o", cubin_path, - tensorflow::strings::StrCat("-arch=sm_", cc_major, cc_minor)}; - if (VLOG_IS_ON(2)) { - ptxas_args.push_back("-v"); - } - ptxas_info_dumper.SetProgram(ptxas_path, ptxas_args); - ptxas_info_dumper.SetChannelAction(tensorflow::CHAN_STDERR, - tensorflow::ACTION_PIPE); - if (!ptxas_info_dumper.Start()) { - return InternalError("Failed to launch ptxas"); - } - string stderr_output; - int exit_status = ptxas_info_dumper.Communicate( - /*stdin_input=*/nullptr, /*stdout_output=*/nullptr, &stderr_output); - XLA_LOG_LINES(tensorflow::INFO, stderr_output); - if (exit_status != 0) { - return InternalError("ptxas exited with non-zero error code %d", - exit_status); - } - - // Read in the result of compilation and return it as a byte vector. - string cubin; - TF_RETURN_IF_ERROR(tensorflow::ReadFileToString(tensorflow::Env::Default(), - cubin_path, &cubin)); - std::vector cubin_vector(cubin.begin(), cubin.end()); - return cubin_vector; -} - -} // namespace - -GpuCompiler::GpuCompiler() - : pointer_size_(llvm::DataLayout(kDataLayout) - .getPointerSize(0 /* default address space */)) {} - -StatusOr> GpuCompiler::RunHloPasses( - std::unique_ptr module, se::StreamExecutor* stream_exec, - DeviceMemoryAllocator* device_allocator) { - XLA_SCOPED_LOGGING_TIMER("GpuCompiler::RunHloPasses"); - tracing::ScopedActivity activity("HLO Transforms", module->name(), - /*is_expensive=*/true); - TF_RETURN_IF_ERROR( - OptimizeHloModule(module.get(), stream_exec, device_allocator)); - return std::move(module); -} - -StatusOr> GpuCompiler::RunBackend( - std::unique_ptr module, se::StreamExecutor* stream_exec, - DeviceMemoryAllocator* device_allocator) { - XLA_SCOPED_LOGGING_TIMER("GpuCompiler::RunBackend"); - - TF_RET_CHECK(stream_exec != nullptr); - - TF_RETURN_IF_ERROR(PrepareHloModuleForIrEmitting(module.get())); - - llvm::LLVMContext llvm_context; - std::string buffer; - llvm::raw_string_ostream error(buffer); - llvm::DiagnosticPrinterRawOStream printer(error); - auto DiagnosticHandler = [](const llvm::DiagnosticInfo& diag_info, - void* Context) { - auto printer = static_cast(Context); - diag_info.print(*printer); - }; - llvm_context.setDiagnosticHandlerCallBack(DiagnosticHandler, &printer); - - llvm::Module llvm_module(module->name().c_str(), llvm_context); - // Set the target triple and the data layout. - llvm_module.setTargetTriple(kTargetTriple); - llvm_module.setDataLayout(kDataLayout); - - // Determine the HLO schedule, which is an ordering of HLO instructions. This - // is used by buffer assignment to enable buffer reuse, and the same ordering - // must also be used to determine the thunk launch schedule. - std::unique_ptr stream_assignment = AssignStreams(*module); - TF_ASSIGN_OR_RETURN( - std::unique_ptr hlo_schedule, - HloSchedule::Build(*module, *stream_assignment, pointer_size_)); - - // Run buffer analysis on the HLO graph. This analysis figures out which - // temporary buffers are required to run the computation. - TF_ASSIGN_OR_RETURN( - std::unique_ptr buffer_assignment, - BufferAssigner::Run(module.get(), hlo_schedule->ConsumeHloOrdering(), - BufferSizeBytesFunction(), - /*color_alignment=*/[](LogicalBuffer::Color) { - return kCudaMallocAlignBytes; - })); - // BufferAssignment::Stats::ToString() and BufferAssignment::ToString() - // include headers, so no need for us to print them ourselves. - XLA_VLOG_LINES(1, buffer_assignment->GetStats().ToString()); - XLA_VLOG_LINES(2, buffer_assignment->ToString()); - XLA_VLOG_LINES(2, module->ToString()); - const string xla_dump_optimized_hlo_proto_to = - module->config().debug_options().xla_dump_optimized_hlo_proto_to(); - if (!xla_dump_optimized_hlo_proto_to.empty()) { - HloProto proto = MakeHloProto(*module, *buffer_assignment); - TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory( - proto, xla_dump_optimized_hlo_proto_to, module->name())); - } - - IrEmitterContext ir_emitter_context(module.get(), buffer_assignment.get(), - &stream_exec->GetDeviceDescription(), - &llvm_module); - - HloComputation* entry_computation = module->entry_computation(); - IrEmitterUnnested ir_emitter(module->config(), entry_computation, - &ir_emitter_context); - { - XLA_SCOPED_LOGGING_TIMER("GpuCompiler::RunBackend - IR emission"); - TF_RETURN_IF_ERROR(entry_computation->Accept(&ir_emitter)); - } - - if (user_pre_optimization_hook_) { - TF_CHECK_OK(user_pre_optimization_hook_(llvm_module)); - } - string ir_module_string_before_opt; - const bool embed_ir_in_executable = - module->config().debug_options().xla_embed_ir_in_executable(); - if (VLOG_IS_ON(2) || embed_ir_in_executable) { - ir_module_string_before_opt = llvm_ir::DumpModuleToString(llvm_module); - VLOG(2) << "LLVM module before optimizations:"; - XLA_VLOG_LINES(2, ir_module_string_before_opt); - } - - const string& ir_dump_directory = - module->config().debug_options().xla_dump_ir_to(); - - if (!ir_dump_directory.empty()) { - TF_RETURN_IF_ERROR(llvm_ir::DumpIRToDirectory( - /*directory_name=*/ir_dump_directory, - /*hlo_module_name=*/module->name(), llvm_module, - /*optimized=*/false)); - } - - { - XLA_SCOPED_LOGGING_TIMER("GpuCompiler::RunBackend - Running LLVM verifier"); - - std::string err; - llvm::raw_string_ostream err_stream(err); - - // verifyModule() returns true if the module is broken. - TF_RET_CHECK(!llvm::verifyModule(llvm_module, &err_stream)) - << "Invalid LLVM IR before optimizations:\n" - << err_stream.str() - << "\nThis probably indicates a bug in the HLO -> LLVM IR lowering. " - "Rerun with --xla_dump_ir_to to get the IR. "; - } - - string libdevice_dir; - { - tensorflow::mutex_lock lock(mutex_); - - // Find the directory containing libdevice. To avoid searching for it every - // time, we have a one-element cache, keyed on the module's config's - // cuda_data_dir. - const auto& config_cuda_data_dir = - module->config().debug_options().xla_gpu_cuda_data_dir(); - if (cached_libdevice_dir_.empty() || - cached_cuda_data_dir_ != config_cuda_data_dir) { - cached_cuda_data_dir_ = config_cuda_data_dir; - cached_libdevice_dir_ = GetLibdeviceDir(config_cuda_data_dir); - } - libdevice_dir = cached_libdevice_dir_; - } - int cc_major, cc_minor; - if (!stream_exec->GetDeviceDescription().cuda_compute_capability(&cc_major, - &cc_minor)) { - LOG(WARNING) - << "Couldn't get compute capability for device; assuming sm_20."; - cc_major = 2; - cc_minor = 0; - } - - string ptx; - { - XLA_SCOPED_LOGGING_TIMER("GpuCompiler::RunBackend - CompileToPtx"); - TF_ASSIGN_OR_RETURN(ptx, CompileToPtx(&llvm_module, {cc_major, cc_minor}, - module->config(), libdevice_dir)); - } - - if (!ir_dump_directory.empty()) { - TF_RETURN_IF_ERROR(llvm_ir::DumpIRToDirectory( - /*directory_name=*/ir_dump_directory, - /*hlo_module_name=*/module->name(), llvm_module, - /*optimized=*/true)); - } - - if (user_post_optimization_hook_) { - TF_CHECK_OK(user_post_optimization_hook_(llvm_module)); - } - VLOG(2) << "LLVM module after optimizations:"; - XLA_VLOG_LINES(2, llvm_ir::DumpModuleToString(llvm_module)); - VLOG(2) << "PTX:"; - XLA_VLOG_LINES(2, ptx); - - // Write PTX to IR dump directory, if IR dumping was requested. - if (!ir_dump_directory.empty()) { - const string ptx_outfile = tensorflow::io::JoinPath( - ir_dump_directory, tensorflow::strings::StrCat(module->name(), ".ptx")); - auto status = [&] { - auto* env = tensorflow::Env::Default(); - TF_RETURN_IF_ERROR(env->RecursivelyCreateDir(ir_dump_directory)); - TF_RETURN_IF_ERROR(tensorflow::WriteStringToFile(env, ptx_outfile, ptx)); - return Status::OK(); - }(); - if (!status.ok()) { - LOG(WARNING) << "Couldn't dump PTX for module " << module->name() - << " to " << ptx_outfile << ": " << status; - } - } - - const std::vector cubin = - CompilePtxOrGetCachedResult(ptx, cc_major, cc_minor); - - auto thunk_schedule = MakeUnique( - ir_emitter.ConsumeThunkSequence(), std::move(stream_assignment), - hlo_schedule->ThunkLaunchOrder()); - VLOG(2) << "Printing the thunk schedule..."; - XLA_VLOG_LINES(2, thunk_schedule->ToString()); - - std::unique_ptr profile_index_map; - std::unique_ptr profile_printer; - - if (module->config().hlo_profiling_enabled()) { - HloCostAnalysis cost_analysis(ShapeSizeBytesFunction()); - cost_analysis.set_bytes_per_second( - stream_exec->GetDeviceDescription().memory_bandwidth()); - TF_RETURN_IF_ERROR(module->entry_computation()->Accept(&cost_analysis)); - profile_index_map = MakeUnique(*module); - profile_printer = - CreateHloProfilePrinterData(*profile_index_map, cost_analysis); - } - - auto* gpu_executable = new GpuExecutable( - ptx, cubin, {cc_major, cc_minor}, std::move(thunk_schedule), - std::move(module), std::move(buffer_assignment), - std::move(profile_printer), std::move(profile_index_map)); - if (embed_ir_in_executable) { - DCHECK_NE("", ir_module_string_before_opt); - gpu_executable->set_ir_module_string(ir_module_string_before_opt); - } - return std::unique_ptr(gpu_executable); -} - -std::vector GpuCompiler::CompilePtxOrGetCachedResult(const string& ptx, - int cc_major, - int cc_minor) { - XLA_SCOPED_LOGGING_TIMER("GpuCompiler::CompilePtxOrGetCachedResult"); - tracing::ScopedActivity activity("PTX->CUBIN", /*is_expensive=*/true); - bool inserted; - decltype(compilation_cache_.begin()) iter; - // Pointers into compilation_cache_ where the ptx and (optional) cubin are - // stored. - const string* cache_ptx = nullptr; - CompilationCacheValue* cache_value = nullptr; - - { - tensorflow::mutex_lock lock(mutex_); - std::tie(iter, inserted) = compilation_cache_.emplace( - std::piecewise_construct, - std::forward_as_tuple(ptx, cc_major, cc_minor), - std::forward_as_tuple()); - cache_ptx = &iter->first.ptx; - cache_value = &iter->second; - } - - // Compile the ptx if it wasn't in the cache before we called this function. - // Other threads asking for the same compilation key will block on - // cache_value->mutex_ until compilation is done. - { - tensorflow::mutex_lock lock(cache_value->mutex_); - if (inserted) { - CHECK(!cache_value->compilation_done); - if (!ptx.empty()) { - StatusOr> maybe_cubin = - CompilePtx(*cache_ptx, cc_major, cc_minor); - if (maybe_cubin.ok()) { - cache_value->cubin_data = std::move(maybe_cubin).ValueOrDie(); - VLOG(2) << "Compiled PTX size:" << ptx.size() - << " CUBIN size: " << cache_value->cubin_data.size(); - } else { - bool log_warning = true; - if (maybe_cubin.status().code() == - tensorflow::error::Code::NOT_FOUND) { - // Missing ptxas is expected in some environments where CUDA SDK - // binaries are not available. We don't want to spam logs with - // identical warnings in this case. - - // TODO(zhengxq): we should implement a LOG_FIRST_N and LOG_EVERY_N - // for more general usage. - static std::atomic warning_done(false); - log_warning = !warning_done.exchange(true); - } - if (log_warning) { - LOG(WARNING) - << "Failed to compile ptx to cubin. Will attempt to let " - "GPU driver compile the ptx. " - << maybe_cubin.status(); - } - - // We're going to use the driver to JIT our PTX->SASS, so warn if - // the JIT in the driver has known bugs. - WarnIfBadDriverJITVersion(); - } - } - cache_value->compilation_done = true; - cache_value->compilation_done_cv_.notify_all(); - } else { - while (!cache_value->compilation_done) { - cache_value->compilation_done_cv_.wait(lock); - } - } - } - - CHECK(cache_value != nullptr); - CHECK(cache_value->compilation_done); - return cache_value->cubin_data; -} - -StatusOr>> -GpuCompiler::CompileAheadOfTime(std::vector> module, - const AotCompilationOptions& options) { - return Unimplemented("not yet implemented: GpuCompiler::CompileAheadOfTime"); -} - -se::Platform::Id GpuCompiler::PlatformId() const { - return se::cuda::kCudaPlatformId; -} - -} // namespace gpu -} // namespace xla - -static bool InitModule() { - xla::Compiler::RegisterCompilerFactory( - stream_executor::cuda::kCudaPlatformId, - []() { return xla::MakeUnique(); }); - return true; -} -static bool module_initialized = InitModule(); diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.h b/tensorflow/compiler/xla/service/gpu/gpu_compiler.h deleted file mode 100644 index f3b02ae5d8..0000000000 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.h +++ /dev/null @@ -1,155 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_GPU_COMPILER_H_ -#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_GPU_COMPILER_H_ - -#include -#include -#include - -#include "tensorflow/compiler/xla/service/executable.h" -#include "tensorflow/compiler/xla/service/hlo_module.h" -#include "tensorflow/compiler/xla/service/llvm_compiler.h" -#include "tensorflow/compiler/xla/statusor.h" -#include "tensorflow/compiler/xla/types.h" -#include "tensorflow/core/lib/gtl/array_slice.h" -#include "tensorflow/core/lib/gtl/optional.h" -#include "tensorflow/core/lib/hash/hash.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/stream_executor_no_cuda.h" -#include "tensorflow/core/platform/thread_annotations.h" - -namespace xla { -namespace gpu { - -// The GPU compiler generates efficient GPU executables. -class GpuCompiler : public LLVMCompiler { - public: - GpuCompiler(); - ~GpuCompiler() override {} - - // Bring in - // StatusOr>> Compile( - // std::vector> modules, - // std::vector> - // stream_execs) - using LLVMCompiler::Compile; - - StatusOr> RunHloPasses( - std::unique_ptr module, se::StreamExecutor* stream_exec, - DeviceMemoryAllocator* device_allocator) override; - - StatusOr> RunBackend( - std::unique_ptr module, se::StreamExecutor* stream_exec, - DeviceMemoryAllocator* device_allocator) override; - - StatusOr>> - CompileAheadOfTime(std::vector> module, - AotCompilationOptions const& options) override; - - se::Platform::Id PlatformId() const override; - - HloCostAnalysis::ShapeSizeFunction ShapeSizeBytesFunction() const override { - // Capture just the pointer size, not the entire GpuCompiler object. - int64 pointer_size = pointer_size_; - return [pointer_size](const Shape& shape) { - return ShapeUtil::ByteSizeOf(shape, pointer_size); - }; - } - - // The triple that represents our target. - static const char* kTargetTriple; - - // The data layout of the emitted module. Copied from computeDataLayout in - // NVPTXTargetMachine.cpp. - static const char* kDataLayout; - - private: - // The size in bytes of a pointer. Used by ShapeSizeBytesFunction. - const int64 pointer_size_; - - tensorflow::mutex mutex_; - - // When compiling an HLO module, we need to find a path to the nvvm libdevice - // files. We search in the module's config.debug_options().cuda_data_dir() - // and in tensorflow::LibdeviceRoot(), the latter of which is a constant. - // - // We cache the cuda_data_dir() and the result of our search, so that if the - // next module we have to compile has the same cuda_data_dir(), we can skip - // the search. - string cached_cuda_data_dir_ GUARDED_BY(mutex_); - string cached_libdevice_dir_ GUARDED_BY(mutex_); - - // Tries to compile the given ptx string to cubin. Returns a vector with the - // compiled cubin. If compilation was unsuccessful, returns an empty vector. - std::vector CompilePtxOrGetCachedResult(const string& ptx, - int cc_major, int cc_minor); - - // The compilation_cache_ map is a cache from {ptx string, cc_major, cc_minor} - // -> cubin so we don't recompile the same ptx twice. This is important for - // some interactive workflows. (We also cache at the HLO level, but sometimes - // we can't realize that two modules are the same until we lower to ptx.) - // - // Compilation of distinct PTX happens in parallel. If more than one thread - // attempts to compile the same PTX, the fist thread to obtain - // cache_value_->mutex_ performs the compilation. The rest wait() on - // cache_value_->compilation_done_cv_ until the compilation is done. - // - // If compiling the ptx fails, we return an empty cubin, cross our fingers, - // and leave compilation up to the driver. - struct CompilationCacheKey { - CompilationCacheKey(std::string ptx, int cc_major, int cc_minor) - : ptx(std::move(ptx)), cc_major(cc_major), cc_minor(cc_minor) {} - string ptx; - int cc_major; - int cc_minor; - }; - struct CompilationCacheHash { - size_t operator()(const CompilationCacheKey& key) const { - return tensorflow::Hash64Combine( - tensorflow::Hash64Combine(tensorflow::Hash64(key.ptx), key.cc_major), - key.cc_minor); - } - }; - struct CompilationCacheEq { - size_t operator()(const CompilationCacheKey& a, - const CompilationCacheKey& b) const { - return a.cc_major == b.cc_major && a.cc_minor == b.cc_minor && - a.ptx == b.ptx; - } - }; - struct CompilationCacheValue { - bool compilation_done = false; - std::vector cubin_data; - // mutex and condition variable to serialize compilation completing. - tensorflow::mutex mutex_; - tensorflow::condition_variable compilation_done_cv_; - }; - - // Don't even think about switching this to FlatMap; iterator stability is - // critical here. - std::unordered_map - compilation_cache_ GUARDED_BY(mutex_); - - TF_DISALLOW_COPY_AND_ASSIGN(GpuCompiler); -}; - -} // namespace gpu -} // namespace xla - -#endif // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_GPU_COMPILER_H_ diff --git a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc index 6c23228976..d34e5f18b2 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc @@ -22,7 +22,7 @@ limitations under the License. #include "llvm/IR/DataLayout.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/literal_util.h" -#include "tensorflow/compiler/xla/service/gpu/gpu_compiler.h" +#include "tensorflow/compiler/xla/service/gpu/nvptx_compiler.h" #include "tensorflow/compiler/xla/service/gpu/outfeed_manager.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/status_macros.h" diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD index da31c65b7e..eb93efc560 100644 --- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD +++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD @@ -17,12 +17,12 @@ cc_library( name = "llvm_gpu_backend", srcs = [ "dump_ir_pass.cc", - "gpu_backend_lib.cc", + "nvptx_backend_lib.cc", "utils.cc", ], hdrs = [ "dump_ir_pass.h", - "gpu_backend_lib.h", + "nvptx_backend_lib.h", "utils.h", ], deps = [ diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc deleted file mode 100644 index 2b0d6924a2..0000000000 --- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc +++ /dev/null @@ -1,506 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.h" - -#include -#include -#include -#include - -#include "tensorflow/compiler/xla/ptr_util.h" -#include "tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/dump_ir_pass.h" -#include "tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/utils.h" -#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h" -#include "tensorflow/compiler/xla/status_macros.h" -#include "tensorflow/compiler/xla/util.h" - -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringMap.h" -#include "llvm/ADT/StringSet.h" -#include "llvm/Analysis/TargetLibraryInfo.h" -#include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/Bitcode/BitcodeReader.h" -#include "llvm/Bitcode/BitcodeWriter.h" -#include "llvm/CodeGen/CommandFlags.inc" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/LegacyPassManager.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Verifier.h" -#include "llvm/Linker/Linker.h" -#include "llvm/PassRegistry.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/FormattedStream.h" -#include "llvm/Support/TargetRegistry.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/ToolOutputFile.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Transforms/IPO.h" -#include "llvm/Transforms/IPO/AlwaysInliner.h" -#include "llvm/Transforms/IPO/Internalize.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" -#include "llvm/Transforms/Scalar.h" -#include "tensorflow/compiler/xla/types.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/lib/strings/stringprintf.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/tracing.h" - -namespace xla { -namespace gpu { -namespace { - -// Default inline threshold value to use in llvm. -const int kDefaultInlineThreshold = 1100; - -// Gets the libdevice filename for a particular compute capability. When -// presented with a GPU we don't recognize, we just return the libdevice from -// compute_20. -static string GetLibdeviceFilename(const string& libdevice_dir_path, - std::pair compute_capability) { - // Since CUDA 9.0, all GPU versions are included in a single file - const char* unified_libdevice_filename = "libdevice.10.bc"; - std::vector unified_libdevice_files; - const Status status = tensorflow::Env::Default()->GetMatchingPaths( - tensorflow::io::JoinPath(libdevice_dir_path, unified_libdevice_filename), - &unified_libdevice_files); - if (status.ok() && unified_libdevice_files.size() == 1) { - return unified_libdevice_filename; - } - // There are only four libdevice files: compute_{20,30,35,50}. Each GPU - // version gets mapped to one of these. Note in particular that sm_60 and - // sm_61 map to libdevice.compute_30. - static auto* m = new std::map, int>({{{2, 0}, 20}, - {{2, 1}, 20}, - {{3, 0}, 30}, - {{3, 2}, 30}, - {{3, 5}, 35}, - {{3, 7}, 35}, - {{5, 0}, 50}, - {{5, 2}, 50}, - {{5, 3}, 50}, - {{6, 0}, 30}, - {{6, 1}, 30}, - {{6, 2}, 30}}); - int libdevice_version = 20; - auto it = m->find(compute_capability); - if (it != m->end()) { - libdevice_version = it->second; - } else { - LOG(WARNING) << "Unknown compute capability (" << compute_capability.first - << ", " << compute_capability.second << ") ." - << "Defaulting to libdevice for compute_" << libdevice_version; - } - return tensorflow::strings::StrCat("libdevice.compute_", libdevice_version, - ".10.bc"); -} - -// Gets the GPU name as it's known to LLVM for a given compute capability. If -// we see an unrecognized compute capability, we return "sm_30". -static string GetSmName(std::pair compute_capability) { - static auto* m = new std::map, int>({{{2, 0}, 20}, - {{2, 1}, 21}, - {{3, 0}, 30}, - {{3, 2}, 32}, - {{3, 5}, 35}, - {{3, 7}, 37}, - {{5, 0}, 50}, - {{5, 2}, 52}, - {{5, 3}, 53}, - {{6, 0}, 60}, - {{6, 1}, 61}, - {{6, 2}, 62}, - // TODO: Change this to 70 once LLVM NVPTX supports it - {{7, 0}, 60}}); - int sm_version = 30; - auto it = m->find(compute_capability); - if (it != m->end()) { - sm_version = it->second; - } else { - LOG(WARNING) << "Unknown compute capability (" << compute_capability.first - << ", " << compute_capability.second << ") ." - << "Defaulting to telling LLVM that we're compiling for sm_" - << sm_version; - } - return tensorflow::strings::StrCat("sm_", sm_version); -} - -// Convenience function for producing a name of a temporary compilation product -// from the input filename. -string MakeNameForTempProduct(const std::string& input_filename, - tensorflow::StringPiece extension) { - return ReplaceFilenameExtension( - tensorflow::io::Basename(llvm_ir::AsString(input_filename)), extension); -} - -// Initializes LLVM passes. Uses the PassRegistry mechanism. -void InitializePasses(llvm::PassRegistry* pass_registry) { - llvm::initializeCore(*pass_registry); - llvm::initializeCodeGen(*pass_registry); - llvm::initializeScalarOpts(*pass_registry); - llvm::initializeObjCARCOpts(*pass_registry); - llvm::initializeVectorization(*pass_registry); - llvm::initializeIPO(*pass_registry); - llvm::initializeAnalysis(*pass_registry); - llvm::initializeTransformUtils(*pass_registry); - llvm::initializeInstCombine(*pass_registry); - llvm::initializeInstrumentation(*pass_registry); - llvm::initializeTarget(*pass_registry); - llvm::initializeCodeGenPreparePass(*pass_registry); -} - -// Returns the TargetMachine, given a triple. -std::unique_ptr GetTargetMachine( - llvm::Triple triple, tensorflow::StringPiece cpu_name, - const HloModuleConfig& hlo_module_config) { - std::string error; - const llvm::Target* target = TargetRegistry::lookupTarget("", triple, error); - if (target == nullptr) { - LOG(FATAL) << "Unable to find Target for triple '" << triple.str() << "'" - << " -- " << error; - return nullptr; - } - - TargetOptions target_options = InitTargetOptionsFromCodeGenFlags(); - llvm_ir::SetTargetOptions( - /*fast_math_enabled=*/hlo_module_config.debug_options() - .xla_enable_fast_math(), - &target_options); - - // Enable FMA synthesis. - target_options.AllowFPOpFusion = FPOpFusion::Fast; - - // Set the verbose assembly options. - target_options.MCOptions.AsmVerbose = false; - - // The selection of codegen optimization level is copied from function - // GetCodeGenOptLevel in //third_party/llvm/llvm/tools/opt/opt.cpp. - CodeGenOpt::Level codegen_opt_level; - switch (hlo_module_config.debug_options().xla_backend_optimization_level()) { - case 1: - codegen_opt_level = CodeGenOpt::Less; - break; - case 2: - codegen_opt_level = CodeGenOpt::Default; - break; - case 3: - codegen_opt_level = CodeGenOpt::Aggressive; - break; - default: - codegen_opt_level = CodeGenOpt::None; - } - return WrapUnique(target->createTargetMachine( - triple.str(), llvm_ir::AsStringRef(cpu_name), "+ptx60", target_options, - Optional(RelocModel), Optional(CMModel), - codegen_opt_level)); -} - -// Adds the standard LLVM optimization passes, based on the speed optimization -// level (opt_level) and size optimization level (size_level). Both module -// and function-level passes are added, so two pass managers are passed in and -// modified by this function. -void AddOptimizationPasses(unsigned opt_level, unsigned size_level, - llvm::TargetMachine* target_machine, - llvm::legacy::PassManagerBase* module_passes, - llvm::legacy::FunctionPassManager* function_passes) { - PassManagerBuilder builder; - builder.OptLevel = opt_level; - builder.SizeLevel = size_level; - - if (opt_level > 1) { - builder.Inliner = llvm::createFunctionInliningPass(kDefaultInlineThreshold); - } else { - // Only inline functions marked with "alwaysinline". - builder.Inliner = llvm::createAlwaysInlinerLegacyPass(); - } - - builder.DisableUnitAtATime = false; - builder.DisableUnrollLoops = opt_level == 0; - builder.LoopVectorize = opt_level > 0; - builder.SLPVectorize = opt_level > 1 && size_level < 2; - - // NVPTX's early-as-possible passes include NVVM reflect. - target_machine->adjustPassManager(builder); - - builder.populateFunctionPassManager(*function_passes); - builder.populateModulePassManager(*module_passes); -} - -// Emits the given module to a bit code file. -void EmitBitcodeToFile(const Module& module, tensorflow::StringPiece filename) { - std::error_code error_code; - llvm::ToolOutputFile outfile(filename.ToString().c_str(), error_code, - llvm::sys::fs::F_None); - if (error_code) { - LOG(FATAL) << "opening bitcode file for writing: " << error_code.message(); - } - - llvm::WriteBitcodeToFile(module, outfile.os()); - outfile.keep(); -} - -// Emits the given module to PTX. target_machine is an initialized TargetMachine -// for the NVPTX target. -string EmitModuleToPTX(Module* module, llvm::TargetMachine* target_machine) { - std::string ptx; // need a std::string instead of a ::string. - { - llvm::raw_string_ostream stream(ptx); - llvm::buffer_ostream pstream(stream); - // The extension is stripped by IrDumpingPassManager, so we need to - // get creative to add a suffix. - string module_id(llvm_ir::AsString(module->getModuleIdentifier())); - IrDumpingPassManager codegen_passes( - ReplaceFilenameExtension(tensorflow::io::Basename(module_id), - "-nvptx.dummy"), - "", false); - codegen_passes.add(new llvm::TargetLibraryInfoWrapperPass( - llvm::Triple(module->getTargetTriple()))); - - target_machine->addPassesToEmitFile(codegen_passes, pstream, nullptr, - llvm::TargetMachine::CGFT_AssemblyFile); - codegen_passes.run(*module); - } - - return ptx; -} - -// LLVM has an extensive flags mechanism of its own, which is only accessible -// through the command line. Internal libraries within LLVM register parsers for -// flags, with no other way to configure them except pass these flags. -// To do this programmatically, we invoke ParseCommandLineOptions manually with -// a "fake argv". -// Note: setting flags with this method is stateful, since flags are just -// static globals within LLVM libraries. -void FeedLLVMWithFlags(const std::vector& cl_opts) { - std::vector fake_argv = {""}; - for (const string& cl_opt : cl_opts) { - fake_argv.push_back(cl_opt.c_str()); - } - llvm::cl::ParseCommandLineOptions(fake_argv.size(), &fake_argv[0]); -} - -// Returns whether the module could use any libdevice functions. This function -// may have false positives -- the module might not use libdevice even if this -// function returns true. -bool CouldNeedLibdevice(const llvm::Module& module) { - for (const llvm::Function& function : module.functions()) { - // This is a conservative approximation -- not all such functions are in - // libdevice. - if (!function.isIntrinsic() && function.isDeclaration()) { - return true; - } - } - return false; -} - -// Links libdevice into the given module if the module needs libdevice. -Status LinkLibdeviceIfNecessary(llvm::Module* module, - std::pair compute_capability, - const string& libdevice_dir_path) { - if (!CouldNeedLibdevice(*module)) { - return Status::OK(); - } - - llvm::Linker linker(*module); - string libdevice_path = tensorflow::io::JoinPath( - libdevice_dir_path, GetLibdeviceFilename(libdevice_dir_path, - compute_capability)); - TF_RETURN_IF_ERROR(tensorflow::Env::Default()->FileExists(libdevice_path)); - VLOG(1) << "Linking with libdevice from: " << libdevice_path; - std::unique_ptr libdevice_module = - LoadIRModule(libdevice_path, &module->getContext()); - if (linker.linkInModule( - std::move(libdevice_module), llvm::Linker::Flags::LinkOnlyNeeded, - [](Module& M, const StringSet<>& GVS) { - internalizeModule(M, [&M, &GVS](const GlobalValue& GV) { - return !GV.hasName() || (GVS.count(GV.getName()) == 0); - }); - })) { - return tensorflow::errors::Internal(tensorflow::strings::StrCat( - "Error linking libdevice from ", libdevice_path)); - } - return Status::OK(); -} - -StatusOr CompileModuleToPtx(llvm::Module* module, - std::pair compute_capability, - const HloModuleConfig& hlo_module_config, - const string& libdevice_dir_path) { - // If the module has no functions or globals, there's nothing to compile. Just - // return an empty string. - if (module->empty() && module->global_empty()) { - VLOG(2) << "Module '" << llvm_ir::AsString(module->getName()) - << "' is empty. Skipping compilation."; - return string(); - } - // Link the input module with libdevice, to pull in implementations of some - // builtins. - TF_RETURN_IF_ERROR( - LinkLibdeviceIfNecessary(module, compute_capability, libdevice_dir_path)); - - // Set the flush-denormals-to-zero flag on the module so the NVVM reflect pass - // can access it. - module->addModuleFlag(llvm::Module::Override, "nvvm-reflect-ftz", - hlo_module_config.debug_options().xla_gpu_ftz()); - - // If ftz is enabled, set it as an attribute on every function in the module. - if (hlo_module_config.debug_options().xla_gpu_ftz()) { - for (llvm::Function& fn : *module) { - fn.addFnAttr("nvptx-f32ftz", "true"); - } - } - - IrDumpingPassManager module_passes(module->getModuleIdentifier(), "", false); - - // Add an appropriate TargetLibraryInfo pass for the module's triple. - llvm::TargetLibraryInfoWrapperPass* tliwp = - new llvm::TargetLibraryInfoWrapperPass( - llvm::Triple(module->getTargetTriple())); - module_passes.add(tliwp); - - // Try to fetch the target triple from the module. If not present, set a - // default target triple. - llvm::Triple target_triple = llvm::Triple(module->getTargetTriple()); - if (target_triple.getArch() == llvm::Triple::UnknownArch) { - LOG(WARNING) << "target triple not found in the module"; - target_triple = llvm::Triple("nvptx64-unknown-unknown"); - } - - // Figure out the exact name of the processor as known to the NVPTX backend - // from the gpu_architecture flag. - std::unique_ptr target_machine = GetTargetMachine( - target_triple, GetSmName(compute_capability), hlo_module_config); - module_passes.add(llvm::createTargetTransformInfoWrapperPass( - target_machine->getTargetIRAnalysis())); - - // The LLVM IR verifier performs sanity checking on the IR. This helps - // discover problems and report them in a meaningful manner, rather than let - // later passes report obscure assertions because of unfulfilled invariants. - module_passes.add(llvm::createVerifierPass()); - - // Create the function-level pass manager. It needs data layout information - // too. - llvm::legacy::FunctionPassManager function_passes(module); - - int32 opt_level = - hlo_module_config.debug_options().xla_backend_optimization_level(); - - CHECK_GE(opt_level, 2) - << "The XLA GPU backend doesn't support unoptimized code generation"; - - AddOptimizationPasses(opt_level, - /*size_level=*/0, target_machine.get(), &module_passes, - &function_passes); - - // Loop unrolling exposes more opportunities for SROA. Therefore, we run SROA - // again after the standard optimization passes [http://b/13329423]. - // TODO(jingyue): SROA may further expose more optimization opportunities such - // as more precise alias analysis and more function inlining (SROA may change - // the inlining cost of a function). For now, running SROA already emits good - // enough code for the evaluated benchmarks. We may want to run more - // optimizations later. - if (opt_level > 0) { - // LLVM's optimizer turns on SROA when the optimization level is greater - // than 0. We mimic this behavior here. - module_passes.add(llvm::createSROAPass()); - } - - // Verify that the module is well formed after optimizations ran. - module_passes.add(llvm::createVerifierPass()); - - // Done populating the pass managers. Now run them. - - function_passes.doInitialization(); - for (auto func = module->begin(); func != module->end(); ++func) { - function_passes.run(*func); - } - function_passes.doFinalization(); - module_passes.run(*module); - - // Finally, produce PTX. - return EmitModuleToPTX(module, target_machine.get()); -} - -// One-time module initializer. -// Must be called only once -- DO NOT CALL DIRECTLY. -void GPUBackendInit(const HloModuleConfig& hlo_module_config) { - // Feed all customized flags here, so we can override them with llvm_cl_opts - // without redeploy the compiler for development purpose. - - // This flag tunes a threshold in branch folding. The default threshold, which - // is one, is not suitable for CUDA programs where branches are more expensive - // than for CPU programs. Setting the threshold to 2 improves the latency of - // TwoDPatchDotProductKernel_IND_3_ND_48 by over 5%, and does not affect the - // latency of other benchmarks so far. - // - // I also tried setting this threshold to other values: - // * 3-6 gives similar results as 2; - // * >6 start hurting the performance of at least dot product kernels. - // - // TODO(jingyue): The current threshold only considers the numbr of IR - // instructions which do not accurately reflect the true cost. We need a - // better cost model. - FeedLLVMWithFlags({"-bonus-inst-threshold=2"}); - // TODO(b/22073864): Increase limit when scan memory dependency. - // This helps to reduce more redundant load instructions. - // - // The specific value is currently large enough for s3d in shoc benchmark, - // which contains a lot of load instructions and many arithmetic instructions - // between those loads. - FeedLLVMWithFlags({"-memdep-block-scan-limit=500"}); - - llvm_ir::InitializeLLVMCommandLineOptions(hlo_module_config); - - // Initialize the NVPTX target; it's the only target we link with, so call its - // specific initialization functions instead of the catch-all InitializeAll*. - LLVMInitializeNVPTXTarget(); - LLVMInitializeNVPTXTargetInfo(); - LLVMInitializeNVPTXTargetMC(); - LLVMInitializeNVPTXAsmPrinter(); - - // Initialize the LLVM optimization passes. - llvm::PassRegistry* registry = llvm::PassRegistry::getPassRegistry(); - InitializePasses(registry); -} - -} // namespace - -StatusOr CompileToPtx(llvm::Module* module, - std::pair compute_capability, - const HloModuleConfig& hlo_module_config, - const string& libdevice_dir_path) { - static std::once_flag backend_init_flag; - std::call_once(backend_init_flag, GPUBackendInit, hlo_module_config); - - string ptx; - { - tensorflow::tracing::ScopedActivity activity( - "Compiling IR", llvm_ir::AsString(module->getName()), - /*is_expensive=*/true); - XLA_SCOPED_LOGGING_TIMER("Compile module " + - llvm_ir::AsString(module->getName())); - TF_ASSIGN_OR_RETURN( - ptx, CompileModuleToPtx(module, compute_capability, hlo_module_config, - libdevice_dir_path)); - } - return ptx; -} - -} // namespace gpu -} // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.h b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.h deleted file mode 100644 index 0a345191d3..0000000000 --- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.h +++ /dev/null @@ -1,47 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// LLVM-based compiler backend. -#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_LLVM_GPU_BACKEND_GPU_BACKEND_LIB_H_ -#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_LLVM_GPU_BACKEND_GPU_BACKEND_LIB_H_ - -#include -#include - -#include "llvm/IR/Module.h" -#include "tensorflow/compiler/xla/service/hlo_module_config.h" -#include "tensorflow/compiler/xla/statusor.h" -#include "tensorflow/compiler/xla/types.h" -#include "tensorflow/core/lib/core/stringpiece.h" - -namespace xla { -namespace gpu { - -// Compiles the argument module and returns it. libdevice_dir_path is the parent -// directory of the libdevice bitcode libraries. The contents of the module may -// be changed. -// -// The Compile.* interfaces each create their own llvm::LLVMContext objects for -// thread safety, but note that LLVM's multithreaded support is very -// preliminary; multithreaded use is not recommended at this time. -StatusOr CompileToPtx(llvm::Module* module, - std::pair compute_capability, - const HloModuleConfig& hlo_module_config, - const string& libdevice_dir_path); - -} // namespace gpu -} // namespace xla - -#endif // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_LLVM_GPU_BACKEND_GPU_BACKEND_LIB_H_ diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc new file mode 100644 index 0000000000..b178bc4d11 --- /dev/null +++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.cc @@ -0,0 +1,506 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.h" + +#include +#include +#include +#include + +#include "tensorflow/compiler/xla/ptr_util.h" +#include "tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/dump_ir_pass.h" +#include "tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/utils.h" +#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/util.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/CodeGen/CommandFlags.inc" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Linker/Linker.h" +#include "llvm/PassRegistry.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/ToolOutputFile.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/IPO.h" +#include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/Transforms/IPO/Internalize.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#include "llvm/Transforms/Scalar.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/lib/strings/stringprintf.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/tracing.h" + +namespace xla { +namespace gpu { +namespace { + +// Default inline threshold value to use in llvm. +const int kDefaultInlineThreshold = 1100; + +// Gets the libdevice filename for a particular compute capability. When +// presented with a GPU we don't recognize, we just return the libdevice from +// compute_20. +static string GetLibdeviceFilename(const string& libdevice_dir_path, + std::pair compute_capability) { + // Since CUDA 9.0, all GPU versions are included in a single file + const char* unified_libdevice_filename = "libdevice.10.bc"; + std::vector unified_libdevice_files; + const Status status = tensorflow::Env::Default()->GetMatchingPaths( + tensorflow::io::JoinPath(libdevice_dir_path, unified_libdevice_filename), + &unified_libdevice_files); + if (status.ok() && unified_libdevice_files.size() == 1) { + return unified_libdevice_filename; + } + // There are only four libdevice files: compute_{20,30,35,50}. Each GPU + // version gets mapped to one of these. Note in particular that sm_60 and + // sm_61 map to libdevice.compute_30. + static auto* m = new std::map, int>({{{2, 0}, 20}, + {{2, 1}, 20}, + {{3, 0}, 30}, + {{3, 2}, 30}, + {{3, 5}, 35}, + {{3, 7}, 35}, + {{5, 0}, 50}, + {{5, 2}, 50}, + {{5, 3}, 50}, + {{6, 0}, 30}, + {{6, 1}, 30}, + {{6, 2}, 30}}); + int libdevice_version = 20; + auto it = m->find(compute_capability); + if (it != m->end()) { + libdevice_version = it->second; + } else { + LOG(WARNING) << "Unknown compute capability (" << compute_capability.first + << ", " << compute_capability.second << ") ." + << "Defaulting to libdevice for compute_" << libdevice_version; + } + return tensorflow::strings::StrCat("libdevice.compute_", libdevice_version, + ".10.bc"); +} + +// Gets the GPU name as it's known to LLVM for a given compute capability. If +// we see an unrecognized compute capability, we return "sm_30". +static string GetSmName(std::pair compute_capability) { + static auto* m = new std::map, int>({{{2, 0}, 20}, + {{2, 1}, 21}, + {{3, 0}, 30}, + {{3, 2}, 32}, + {{3, 5}, 35}, + {{3, 7}, 37}, + {{5, 0}, 50}, + {{5, 2}, 52}, + {{5, 3}, 53}, + {{6, 0}, 60}, + {{6, 1}, 61}, + {{6, 2}, 62}, + // TODO: Change this to 70 once LLVM NVPTX supports it + {{7, 0}, 60}}); + int sm_version = 30; + auto it = m->find(compute_capability); + if (it != m->end()) { + sm_version = it->second; + } else { + LOG(WARNING) << "Unknown compute capability (" << compute_capability.first + << ", " << compute_capability.second << ") ." + << "Defaulting to telling LLVM that we're compiling for sm_" + << sm_version; + } + return tensorflow::strings::StrCat("sm_", sm_version); +} + +// Convenience function for producing a name of a temporary compilation product +// from the input filename. +string MakeNameForTempProduct(const std::string& input_filename, + tensorflow::StringPiece extension) { + return ReplaceFilenameExtension( + tensorflow::io::Basename(llvm_ir::AsString(input_filename)), extension); +} + +// Initializes LLVM passes. Uses the PassRegistry mechanism. +void InitializePasses(llvm::PassRegistry* pass_registry) { + llvm::initializeCore(*pass_registry); + llvm::initializeCodeGen(*pass_registry); + llvm::initializeScalarOpts(*pass_registry); + llvm::initializeObjCARCOpts(*pass_registry); + llvm::initializeVectorization(*pass_registry); + llvm::initializeIPO(*pass_registry); + llvm::initializeAnalysis(*pass_registry); + llvm::initializeTransformUtils(*pass_registry); + llvm::initializeInstCombine(*pass_registry); + llvm::initializeInstrumentation(*pass_registry); + llvm::initializeTarget(*pass_registry); + llvm::initializeCodeGenPreparePass(*pass_registry); +} + +// Returns the TargetMachine, given a triple. +std::unique_ptr GetTargetMachine( + llvm::Triple triple, tensorflow::StringPiece cpu_name, + const HloModuleConfig& hlo_module_config) { + std::string error; + const llvm::Target* target = TargetRegistry::lookupTarget("", triple, error); + if (target == nullptr) { + LOG(FATAL) << "Unable to find Target for triple '" << triple.str() << "'" + << " -- " << error; + return nullptr; + } + + TargetOptions target_options = InitTargetOptionsFromCodeGenFlags(); + llvm_ir::SetTargetOptions( + /*fast_math_enabled=*/hlo_module_config.debug_options() + .xla_enable_fast_math(), + &target_options); + + // Enable FMA synthesis. + target_options.AllowFPOpFusion = FPOpFusion::Fast; + + // Set the verbose assembly options. + target_options.MCOptions.AsmVerbose = false; + + // The selection of codegen optimization level is copied from function + // GetCodeGenOptLevel in //third_party/llvm/llvm/tools/opt/opt.cpp. + CodeGenOpt::Level codegen_opt_level; + switch (hlo_module_config.debug_options().xla_backend_optimization_level()) { + case 1: + codegen_opt_level = CodeGenOpt::Less; + break; + case 2: + codegen_opt_level = CodeGenOpt::Default; + break; + case 3: + codegen_opt_level = CodeGenOpt::Aggressive; + break; + default: + codegen_opt_level = CodeGenOpt::None; + } + return WrapUnique(target->createTargetMachine( + triple.str(), llvm_ir::AsStringRef(cpu_name), "+ptx60", target_options, + Optional(RelocModel), Optional(CMModel), + codegen_opt_level)); +} + +// Adds the standard LLVM optimization passes, based on the speed optimization +// level (opt_level) and size optimization level (size_level). Both module +// and function-level passes are added, so two pass managers are passed in and +// modified by this function. +void AddOptimizationPasses(unsigned opt_level, unsigned size_level, + llvm::TargetMachine* target_machine, + llvm::legacy::PassManagerBase* module_passes, + llvm::legacy::FunctionPassManager* function_passes) { + PassManagerBuilder builder; + builder.OptLevel = opt_level; + builder.SizeLevel = size_level; + + if (opt_level > 1) { + builder.Inliner = llvm::createFunctionInliningPass(kDefaultInlineThreshold); + } else { + // Only inline functions marked with "alwaysinline". + builder.Inliner = llvm::createAlwaysInlinerLegacyPass(); + } + + builder.DisableUnitAtATime = false; + builder.DisableUnrollLoops = opt_level == 0; + builder.LoopVectorize = opt_level > 0; + builder.SLPVectorize = opt_level > 1 && size_level < 2; + + // NVPTX's early-as-possible passes include NVVM reflect. + target_machine->adjustPassManager(builder); + + builder.populateFunctionPassManager(*function_passes); + builder.populateModulePassManager(*module_passes); +} + +// Emits the given module to a bit code file. +void EmitBitcodeToFile(const Module& module, tensorflow::StringPiece filename) { + std::error_code error_code; + llvm::ToolOutputFile outfile(filename.ToString().c_str(), error_code, + llvm::sys::fs::F_None); + if (error_code) { + LOG(FATAL) << "opening bitcode file for writing: " << error_code.message(); + } + + llvm::WriteBitcodeToFile(module, outfile.os()); + outfile.keep(); +} + +// Emits the given module to PTX. target_machine is an initialized TargetMachine +// for the NVPTX target. +string EmitModuleToPTX(Module* module, llvm::TargetMachine* target_machine) { + std::string ptx; // need a std::string instead of a ::string. + { + llvm::raw_string_ostream stream(ptx); + llvm::buffer_ostream pstream(stream); + // The extension is stripped by IrDumpingPassManager, so we need to + // get creative to add a suffix. + string module_id(llvm_ir::AsString(module->getModuleIdentifier())); + IrDumpingPassManager codegen_passes( + ReplaceFilenameExtension(tensorflow::io::Basename(module_id), + "-nvptx.dummy"), + "", false); + codegen_passes.add(new llvm::TargetLibraryInfoWrapperPass( + llvm::Triple(module->getTargetTriple()))); + + target_machine->addPassesToEmitFile(codegen_passes, pstream, nullptr, + llvm::TargetMachine::CGFT_AssemblyFile); + codegen_passes.run(*module); + } + + return ptx; +} + +// LLVM has an extensive flags mechanism of its own, which is only accessible +// through the command line. Internal libraries within LLVM register parsers for +// flags, with no other way to configure them except pass these flags. +// To do this programmatically, we invoke ParseCommandLineOptions manually with +// a "fake argv". +// Note: setting flags with this method is stateful, since flags are just +// static globals within LLVM libraries. +void FeedLLVMWithFlags(const std::vector& cl_opts) { + std::vector fake_argv = {""}; + for (const string& cl_opt : cl_opts) { + fake_argv.push_back(cl_opt.c_str()); + } + llvm::cl::ParseCommandLineOptions(fake_argv.size(), &fake_argv[0]); +} + +// Returns whether the module could use any libdevice functions. This function +// may have false positives -- the module might not use libdevice even if this +// function returns true. +bool CouldNeedLibdevice(const llvm::Module& module) { + for (const llvm::Function& function : module.functions()) { + // This is a conservative approximation -- not all such functions are in + // libdevice. + if (!function.isIntrinsic() && function.isDeclaration()) { + return true; + } + } + return false; +} + +// Links libdevice into the given module if the module needs libdevice. +Status LinkLibdeviceIfNecessary(llvm::Module* module, + std::pair compute_capability, + const string& libdevice_dir_path) { + if (!CouldNeedLibdevice(*module)) { + return Status::OK(); + } + + llvm::Linker linker(*module); + string libdevice_path = tensorflow::io::JoinPath( + libdevice_dir_path, GetLibdeviceFilename(libdevice_dir_path, + compute_capability)); + TF_RETURN_IF_ERROR(tensorflow::Env::Default()->FileExists(libdevice_path)); + VLOG(1) << "Linking with libdevice from: " << libdevice_path; + std::unique_ptr libdevice_module = + LoadIRModule(libdevice_path, &module->getContext()); + if (linker.linkInModule( + std::move(libdevice_module), llvm::Linker::Flags::LinkOnlyNeeded, + [](Module& M, const StringSet<>& GVS) { + internalizeModule(M, [&M, &GVS](const GlobalValue& GV) { + return !GV.hasName() || (GVS.count(GV.getName()) == 0); + }); + })) { + return tensorflow::errors::Internal(tensorflow::strings::StrCat( + "Error linking libdevice from ", libdevice_path)); + } + return Status::OK(); +} + +StatusOr CompileModuleToPtx(llvm::Module* module, + std::pair compute_capability, + const HloModuleConfig& hlo_module_config, + const string& libdevice_dir_path) { + // If the module has no functions or globals, there's nothing to compile. Just + // return an empty string. + if (module->empty() && module->global_empty()) { + VLOG(2) << "Module '" << llvm_ir::AsString(module->getName()) + << "' is empty. Skipping compilation."; + return string(); + } + // Link the input module with libdevice, to pull in implementations of some + // builtins. + TF_RETURN_IF_ERROR( + LinkLibdeviceIfNecessary(module, compute_capability, libdevice_dir_path)); + + // Set the flush-denormals-to-zero flag on the module so the NVVM reflect pass + // can access it. + module->addModuleFlag(llvm::Module::Override, "nvvm-reflect-ftz", + hlo_module_config.debug_options().xla_gpu_ftz()); + + // If ftz is enabled, set it as an attribute on every function in the module. + if (hlo_module_config.debug_options().xla_gpu_ftz()) { + for (llvm::Function& fn : *module) { + fn.addFnAttr("nvptx-f32ftz", "true"); + } + } + + IrDumpingPassManager module_passes(module->getModuleIdentifier(), "", false); + + // Add an appropriate TargetLibraryInfo pass for the module's triple. + llvm::TargetLibraryInfoWrapperPass* tliwp = + new llvm::TargetLibraryInfoWrapperPass( + llvm::Triple(module->getTargetTriple())); + module_passes.add(tliwp); + + // Try to fetch the target triple from the module. If not present, set a + // default target triple. + llvm::Triple target_triple = llvm::Triple(module->getTargetTriple()); + if (target_triple.getArch() == llvm::Triple::UnknownArch) { + LOG(WARNING) << "target triple not found in the module"; + target_triple = llvm::Triple("nvptx64-unknown-unknown"); + } + + // Figure out the exact name of the processor as known to the NVPTX backend + // from the gpu_architecture flag. + std::unique_ptr target_machine = GetTargetMachine( + target_triple, GetSmName(compute_capability), hlo_module_config); + module_passes.add(llvm::createTargetTransformInfoWrapperPass( + target_machine->getTargetIRAnalysis())); + + // The LLVM IR verifier performs sanity checking on the IR. This helps + // discover problems and report them in a meaningful manner, rather than let + // later passes report obscure assertions because of unfulfilled invariants. + module_passes.add(llvm::createVerifierPass()); + + // Create the function-level pass manager. It needs data layout information + // too. + llvm::legacy::FunctionPassManager function_passes(module); + + int32 opt_level = + hlo_module_config.debug_options().xla_backend_optimization_level(); + + CHECK_GE(opt_level, 2) + << "The XLA GPU backend doesn't support unoptimized code generation"; + + AddOptimizationPasses(opt_level, + /*size_level=*/0, target_machine.get(), &module_passes, + &function_passes); + + // Loop unrolling exposes more opportunities for SROA. Therefore, we run SROA + // again after the standard optimization passes [http://b/13329423]. + // TODO(jingyue): SROA may further expose more optimization opportunities such + // as more precise alias analysis and more function inlining (SROA may change + // the inlining cost of a function). For now, running SROA already emits good + // enough code for the evaluated benchmarks. We may want to run more + // optimizations later. + if (opt_level > 0) { + // LLVM's optimizer turns on SROA when the optimization level is greater + // than 0. We mimic this behavior here. + module_passes.add(llvm::createSROAPass()); + } + + // Verify that the module is well formed after optimizations ran. + module_passes.add(llvm::createVerifierPass()); + + // Done populating the pass managers. Now run them. + + function_passes.doInitialization(); + for (auto func = module->begin(); func != module->end(); ++func) { + function_passes.run(*func); + } + function_passes.doFinalization(); + module_passes.run(*module); + + // Finally, produce PTX. + return EmitModuleToPTX(module, target_machine.get()); +} + +// One-time module initializer. +// Must be called only once -- DO NOT CALL DIRECTLY. +void GPUBackendInit(const HloModuleConfig& hlo_module_config) { + // Feed all customized flags here, so we can override them with llvm_cl_opts + // without redeploy the compiler for development purpose. + + // This flag tunes a threshold in branch folding. The default threshold, which + // is one, is not suitable for CUDA programs where branches are more expensive + // than for CPU programs. Setting the threshold to 2 improves the latency of + // TwoDPatchDotProductKernel_IND_3_ND_48 by over 5%, and does not affect the + // latency of other benchmarks so far. + // + // I also tried setting this threshold to other values: + // * 3-6 gives similar results as 2; + // * >6 start hurting the performance of at least dot product kernels. + // + // TODO(jingyue): The current threshold only considers the numbr of IR + // instructions which do not accurately reflect the true cost. We need a + // better cost model. + FeedLLVMWithFlags({"-bonus-inst-threshold=2"}); + // TODO(b/22073864): Increase limit when scan memory dependency. + // This helps to reduce more redundant load instructions. + // + // The specific value is currently large enough for s3d in shoc benchmark, + // which contains a lot of load instructions and many arithmetic instructions + // between those loads. + FeedLLVMWithFlags({"-memdep-block-scan-limit=500"}); + + llvm_ir::InitializeLLVMCommandLineOptions(hlo_module_config); + + // Initialize the NVPTX target; it's the only target we link with, so call its + // specific initialization functions instead of the catch-all InitializeAll*. + LLVMInitializeNVPTXTarget(); + LLVMInitializeNVPTXTargetInfo(); + LLVMInitializeNVPTXTargetMC(); + LLVMInitializeNVPTXAsmPrinter(); + + // Initialize the LLVM optimization passes. + llvm::PassRegistry* registry = llvm::PassRegistry::getPassRegistry(); + InitializePasses(registry); +} + +} // namespace + +StatusOr CompileToPtx(llvm::Module* module, + std::pair compute_capability, + const HloModuleConfig& hlo_module_config, + const string& libdevice_dir_path) { + static std::once_flag backend_init_flag; + std::call_once(backend_init_flag, GPUBackendInit, hlo_module_config); + + string ptx; + { + tensorflow::tracing::ScopedActivity activity( + "Compiling IR", llvm_ir::AsString(module->getName()), + /*is_expensive=*/true); + XLA_SCOPED_LOGGING_TIMER("Compile module " + + llvm_ir::AsString(module->getName())); + TF_ASSIGN_OR_RETURN( + ptx, CompileModuleToPtx(module, compute_capability, hlo_module_config, + libdevice_dir_path)); + } + return ptx; +} + +} // namespace gpu +} // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.h b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.h new file mode 100644 index 0000000000..54e0e140de --- /dev/null +++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.h @@ -0,0 +1,47 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// LLVM-based compiler backend. +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_LLVM_GPU_BACKEND_NVPTX_BACKEND_LIB_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_LLVM_GPU_BACKEND_NVPTX_BACKEND_LIB_H_ + +#include +#include + +#include "llvm/IR/Module.h" +#include "tensorflow/compiler/xla/service/hlo_module_config.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/lib/core/stringpiece.h" + +namespace xla { +namespace gpu { + +// Compiles the argument module and returns it. libdevice_dir_path is the parent +// directory of the libdevice bitcode libraries. The contents of the module may +// be changed. +// +// The Compile.* interfaces each create their own llvm::LLVMContext objects for +// thread safety, but note that LLVM's multithreaded support is very +// preliminary; multithreaded use is not recommended at this time. +StatusOr CompileToPtx(llvm::Module* module, + std::pair compute_capability, + const HloModuleConfig& hlo_module_config, + const string& libdevice_dir_path); + +} // namespace gpu +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_LLVM_GPU_BACKEND_NVPTX_BACKEND_LIB_H_ diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc new file mode 100644 index 0000000000..0845ab462d --- /dev/null +++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc @@ -0,0 +1,802 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/gpu/nvptx_compiler.h" + +#include +#include +#include +#include // NOLINT(build/c++11): only using std::call_once, not mutex. +#include + +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Verifier.h" +#include "tensorflow/compiler/xla/protobuf_util.h" +#include "tensorflow/compiler/xla/ptr_util.h" +#include "tensorflow/compiler/xla/service/algebraic_simplifier.h" +#include "tensorflow/compiler/xla/service/batchnorm_expander.h" +#include "tensorflow/compiler/xla/service/buffer_assignment.h" +#include "tensorflow/compiler/xla/service/buffer_liveness.h" +#include "tensorflow/compiler/xla/service/call_inliner.h" +#include "tensorflow/compiler/xla/service/conditional_simplifier.h" +#include "tensorflow/compiler/xla/service/dot_decomposer.h" +#include "tensorflow/compiler/xla/service/flatten_call_graph.h" +#include "tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.h" +#include "tensorflow/compiler/xla/service/gpu/cudnn_convolution_algorithm_picker.h" +#include "tensorflow/compiler/xla/service/gpu/cudnn_convolution_rewriter.h" +#include "tensorflow/compiler/xla/service/gpu/fusion_merger.h" +#include "tensorflow/compiler/xla/service/gpu/gpu_constants.h" +#include "tensorflow/compiler/xla/service/gpu/gpu_copy_insertion.h" +#include "tensorflow/compiler/xla/service/gpu/gpu_executable.h" +#include "tensorflow/compiler/xla/service/gpu/gpu_hlo_support_checker.h" +#include "tensorflow/compiler/xla/service/gpu/gpu_layout_assignment.h" +#include "tensorflow/compiler/xla/service/gpu/hlo_schedule.h" +#include "tensorflow/compiler/xla/service/gpu/instruction_fusion.h" +#include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h" +#include "tensorflow/compiler/xla/service/gpu/ir_emitter_context.h" +#include "tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h" +#include "tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.h" +#include "tensorflow/compiler/xla/service/gpu/multi_output_fusion.h" +#include "tensorflow/compiler/xla/service/gpu/pad_insertion.h" +#include "tensorflow/compiler/xla/service/gpu/partition_assignment.h" +#include "tensorflow/compiler/xla/service/gpu/stream_assignment.h" +#include "tensorflow/compiler/xla/service/gpu/thunk_schedule.h" +#include "tensorflow/compiler/xla/service/hlo.pb.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_constant_folding.h" +#include "tensorflow/compiler/xla/service/hlo_cse.h" +#include "tensorflow/compiler/xla/service/hlo_dce.h" +#include "tensorflow/compiler/xla/service/hlo_element_type_converter.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_pass_fix.h" +#include "tensorflow/compiler/xla/service/hlo_pass_pipeline.h" +#include "tensorflow/compiler/xla/service/hlo_proto_util.h" +#include "tensorflow/compiler/xla/service/hlo_subcomputation_unification.h" +#include "tensorflow/compiler/xla/service/hlo_verifier.h" +#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h" +#include "tensorflow/compiler/xla/service/reduce_precision_insertion.h" +#include "tensorflow/compiler/xla/service/reshape_mover.h" +#include "tensorflow/compiler/xla/service/transpose_folding.h" +#include "tensorflow/compiler/xla/service/tuple_simplifier.h" +#include "tensorflow/compiler/xla/service/while_loop_constant_sinking.h" +#include "tensorflow/compiler/xla/service/while_loop_invariant_code_motion.h" +#include "tensorflow/compiler/xla/service/while_loop_simplifier.h" +#include "tensorflow/compiler/xla/service/zero_sized_hlo_elimination.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/gtl/cleanup.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/cuda_libdevice_path.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/regexp.h" +#include "tensorflow/core/platform/stream_executor_no_cuda.h" +#include "tensorflow/core/platform/subprocess.h" +#include "tensorflow/core/platform/tracing.h" +#include "tensorflow/stream_executor/cuda/cuda_diagnostics.h" + +namespace xla { +namespace gpu { + +/* static */ const char* NVPTXCompiler::kTargetTriple = "nvptx64-nvidia-cuda"; +/* static */ const char* NVPTXCompiler::kDataLayout = + "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"; + +namespace { + +namespace tracing = tensorflow::tracing; + +// Returns the directory containing nvvm libdevice files. config_cuda_data_dir +// should be equal to config().debug_options().xla_gpu_cuda_data_dir() of the +// HloModule being compiled. +string GetLibdeviceDir(const string& config_cuda_data_dir) { + std::vector potential_libdevice_dirs; + if (!config_cuda_data_dir.empty()) { + potential_libdevice_dirs.push_back(config_cuda_data_dir); + } + potential_libdevice_dirs.push_back(tensorflow::LibdeviceRoot()); + + // Tries all potential libdevice directories in the order they are inserted. + // Returns the first directory that exists in the file system. + for (const string& potential_libdevice_dir : potential_libdevice_dirs) { + if (tensorflow::Env::Default()->IsDirectory(potential_libdevice_dir).ok()) { + VLOG(2) << "Found libdevice dir " << potential_libdevice_dir; + return potential_libdevice_dir; + } + VLOG(2) << "Unable to find potential libdevice dir " + << potential_libdevice_dir; + } + + // Last resort: maybe in the current folder. + return "."; +} + +// Runs optimization passes on the given HLO module. +Status OptimizeHloModule(HloModule* hlo_module, se::StreamExecutor* stream_exec, + DeviceMemoryAllocator* device_allocator) { + { + HloPassPipeline pipeline("optimization"); + pipeline.AddInvariantChecker(); + pipeline.AddPass(); + ReducePrecisionInsertion::AddPasses( + &pipeline, hlo_module->config().debug_options(), + ReducePrecisionInsertion::PassTiming::BEFORE_OPTIMIZATION); + + // TODO(b/64094172): make Call work on GPU instead of inlining. + pipeline.AddPass(); + // Convert BF16 operations to F32 operations so that the GPU backend can + // support BF16 operations without directly implementing a BF16 lowering for + // most ops. + pipeline.AddPass(BF16, F32); + pipeline.AddPass(); + + { + auto& pass = + pipeline.AddPass>("simplification"); + pass.AddInvariantChecker(); + + // If cudnn batchnorms are enabled, rewrite batchnorm HLOs to cudnn calls + // where possible. Not every batchnorm op can be implemented as a call to + // cudnn, so decompose any remaining batchnorm ops into a soup of HLOs. + if (hlo_module->config().debug_options().xla_gpu_use_cudnn_batchnorm()) { + pass.AddPass(); + } + pass.AddPass( + /*rewrite_training_op=*/true, + /*rewrite_inference_op=*/true, + /*rewrite_grad_op=*/true); + + // BatchNormExpander can create zero-sized ops, so zero-sized HLO + // elimination has to come after that pass. + pipeline.AddPass(); + + pass.AddPass( + /*is_layout_sensitive=*/false, + [](const Shape&, const Shape&) { return false; }); + pass.AddPass(); + pass.AddPass(); + pass.AddPass(); + pass.AddPass(); + pass.AddPass(); + pass.AddPass(); + pass.AddPass(); + } + + pipeline.AddPass( + [](const HloInstruction& dot, + const TransposeFolding::OperandIndices& candidate_operands) { + return ImplementedAsGemm(dot) ? candidate_operands + : TransposeFolding::OperandIndices{}; + }, + TransposeFolding::NeverFoldTranspose); + pipeline.AddPass(/*is_layout_sensitive=*/false); + pipeline.AddPass(); + TF_RETURN_IF_ERROR(pipeline.Run(hlo_module).status()); + } + + { + // Convert convolutions into CustomCalls to cudnn, then canonicalize them + // (PadInsertion). + HloPassPipeline pipeline("conv_canonicalization"); + pipeline.AddInvariantChecker(); + pipeline.AddPass(); + pipeline.AddPass(); + TF_RETURN_IF_ERROR(pipeline.Run(hlo_module).status()); + } + + { + HloPassPipeline pipeline("layout_assignment"); + pipeline.AddPass( + hlo_module->mutable_entry_computation_layout(), stream_exec); + + // The LayoutAssignment pass may leave behind kCopy instructions which are + // duplicate or NOPs, so remove them with algebraic simplification and CSE. + pipeline.AddPass>( + /*is_layout_sensitive=*/true, + /*valid_bitcast_callback=*/[](const Shape&, const Shape&) { + return true; + }); + + // Choose the fastest algorithm for each conv. + // + // We pick the algorithm before fusion so we can generate better HLO. After + // CudnnConvolutionRewriter, our convolutions are CustomCalls which return a + // tuple (conv_result, scratch_memory), and the each conv uses 0 bytes of + // scratch: + // + // customcall = (f32[...], f32[0]) + // return gte(customcall, 0) + // + // The algorithm picker then chooses the best algorithm, and potentially + // increases the scratch space. It replaces customcall with new_tuple, + // giving us the following: + // + // new_customcall = (f32[...], f32[N]) + // new_tuple = tuple(gte(new_customcall, 0), constant f32[0]) + // return gte(new_tuple, 0) + // + // The new tuple and gte instructions then be simplified away, because + // nobody is expected to use the scratch value. + // + // However, if we were to run CudnnConvolutionAlgorithmPicker after fusion + // the gte(customcall, 0) would probably already be into a fusion node. We + // can't simplify across HloComputation boundaries, so in this case we + // wouldn't be able to simplify away the new_tuple bits. + pipeline.AddPass(stream_exec, + device_allocator); + // Clean up new_tuple described above. + pipeline.AddPass(); + + pipeline.AddPass(/*is_layout_sensitive=*/true); + TF_RETURN_IF_ERROR(pipeline.Run(hlo_module).status()); + } + + { + HloPassFix fusion("fusion"); + fusion.AddInvariantChecker(); + fusion.AddPass(/*may_duplicate=*/false); + fusion.AddPass(/*may_duplicate=*/true); + fusion.AddPass(); + fusion.AddPass(); + fusion.AddPass(/*is_layout_sensitive=*/true, + /*only_fusion_computations=*/true); + TF_RETURN_IF_ERROR(fusion.Run(hlo_module).status()); + + HloPassPipeline reduce_pipeline("reduce-precision"); + reduce_pipeline.AddInvariantChecker(); + ReducePrecisionInsertion::AddPasses( + &reduce_pipeline, hlo_module->config().debug_options(), + ReducePrecisionInsertion::PassTiming::AFTER_FUSION); + StatusOr reduce_result = reduce_pipeline.Run(hlo_module); + TF_RETURN_IF_ERROR(reduce_result.status()); + + if (reduce_result.ValueOrDie()) { + // Do another fusion pass, with the expectation that we may be able to + // fuse the new ReducePrecision operations. + TF_RETURN_IF_ERROR(fusion.Run(hlo_module).status()); + } + } + + { + // Do an aggressive LICM pass over while loops. In particular, this hoists + // constants that were sunk by WhileLoopConstantSinking. Leaving them in + // the while loop may result in unnecessary copies. + HloPassPipeline pipeline("while-loop-licm"); + pipeline.AddPass(true); + TF_RETURN_IF_ERROR(pipeline.Run(hlo_module).status()); + } + return Status::OK(); +} + +// Modifies the given HLO module so that it will be accepted by IrEmitter. +// Unlike optimization passes, the passes are necessary for correctness. +Status PrepareHloModuleForIrEmitting(HloModule* hlo_module) { + // In some cases, we have to place the result of an instruction in a temporary + // buffer. For instance, the buffer that holds an external parameter is + // assumed immutable at this point, and should not be reused for output + // (b/27180329). Therefore, in that case, we set the output to be a copy of + // the parameter. + HloPassPipeline pipeline("GPU-ir-emit-prepare"); + pipeline.AddInvariantChecker(); + + // Copy insertion should be performed immediately before IR emission to avoid + // inserting unnecessary copies (later pass adds an instruction which + // materializes the value) or missing a necessary copy (later pass removes an + // instruction which materializes a value). DCE must be run immediately before + // (and sometime after) copy insertion, to avoid dead code from interfering + // with the rewrites. + pipeline.AddPass(); + pipeline.AddPass(); + pipeline.AddPass(); + return pipeline.Run(hlo_module).status(); +} + +// Prints a warning if the ptxas at ptxas_path has known bugs. +// +// Only prints a warning the first time it's called for a particular value of +// ptxas_path. +void WarnIfBadPtxasVersion(const string& ptxas_path) { + static tensorflow::mutex mu(tensorflow::LINKER_INITIALIZED); + static std::unordered_set* seen_ptxas_paths GUARDED_BY(mu) = + new std::unordered_set(); + + tensorflow::mutex_lock lock(mu); + if (!seen_ptxas_paths->insert(ptxas_path).second) { + // Already checked this ptx binary, nothing to do. + return; + } + + tensorflow::SubProcess ptxas; + ptxas.SetProgram(ptxas_path, {ptxas_path, "--version"}); + ptxas.SetChannelAction(tensorflow::CHAN_STDOUT, tensorflow::ACTION_PIPE); + if (!ptxas.Start()) { + LOG(WARNING) << "Couldn't invoke " << ptxas_path << " --version"; + return; + } + + string out; + int exit_code = ptxas.Communicate(/*stdin_input=*/nullptr, &out, + /*stderr_output=*/nullptr); + if (exit_code != 0) { + LOG(WARNING) << "Running " << ptxas_path << " --version returned " + << exit_code; + return; + } + + int64 vmaj, vmin, vdot; + string vmaj_str, vmin_str, vdot_str; + if (!RE2::PartialMatch(out, R"(\bV(\d+)\.(\d+)\.(\d+)\b)", &vmaj_str, + &vmin_str, &vdot_str) || + !tensorflow::strings::safe_strto64(vmaj_str, &vmaj) || + !tensorflow::strings::safe_strto64(vmin_str, &vmin) || + !tensorflow::strings::safe_strto64(vdot_str, &vdot)) { + LOG(WARNING) << "Couldn't parse ptxas version in output of " << ptxas_path + << " --version:\n" + << out; + return; + } + + // We need ptxas >= 9.0 as a hard requirement, because we compile targeting + // PTX 6.0. An older ptxas will just fail to compile any of our code. + // + // ptxas 9.0 before 9.0.276 and ptxas 9.1 before 9.1.121 miscompile some + // address calculations with large offsets (e.g. "load ptr + large_constant"), + // b/70245379. + // + // ptxas 9.1.121 miscompiles some large multioutput fusions, again in a way + // that appears related to address calculations. ptxas 9.2.88 appears to + // work, as far as we can tell. + if (vmaj < 9) { + LOG(ERROR) + << "You are using ptxas 8.x, but XLA requires ptxas 9.x (and strongly " + "prefers >= 9.2.88). Compilation of XLA kernels below will likely " + "fail.\n\nYou do not need to update CUDA; cherry-picking the ptxas " + "binary is sufficient."; + } else if ((vmaj < 9 || vmin < 2 || vdot < 88)) { + LOG(WARNING) + << "*** WARNING *** You are using ptxas " << vmaj << "." << vmin << "." + << vdot + << ", which older than 9.2.88. ptxas 9.x before 9.2.88 is known to " + "miscompile XLA code, leading to incorrect results or " + "invalid-address errors.\n\nYou do not need to update to CUDA " + "9.2.88; cherry-picking the ptxas binary is sufficient."; + } +} + +// Prints a warning if the ptx->sass JIT in the driver has known bugs. +// +// Using such a driver only a problem if we fail to use ptxas to compile our ptx +// and have to use the driver instead, so you should only call this function if +// we're going to use the driver JIT. +// +// Only prints a warning the first time it's called. +void WarnIfBadDriverJITVersion() { + static std::once_flag run_once; + std::call_once(run_once, [] { + auto version_or_status = se::cuda::Diagnostician::FindKernelDriverVersion(); + if (!version_or_status.ok()) { + LOG(WARNING) << "Couldn't read CUDA driver version."; + return; + } + se::cuda::DriverVersion version = version_or_status.ValueOrDie(); + + // The following versions of the driver JIT miscompile some address + // calculations with large offsets (e.g. "load ptr + large_constant"), + // b/70245379: + // + // - 384.x before 384.108 + // - 387.x before 387.40 + // - 390.x before 390.10. + // + // TODO(jlebar): This list does not cover the address-calculation bug we've + // observed in ptxas 9.1.121. Need to get a new safe range from nvidia + // corresponding to ptxas >= 9.2.88. + auto vmaj = std::get<0>(version); + auto vmin = std::get<1>(version); + if ((vmaj == 384 && vmin < 108) || // + (vmaj == 387 && vmin < 40) || // + (vmaj == 390 && vmin < 10)) { + LOG(WARNING) + << "*** WARNING *** Invoking the PTX->SASS JIT from driver version " + << se::cuda::DriverVersionToString(version) + << ", which is in range [384.0.0, 384.108.0) + [387.0.0, 387.40.0) + " + "[390.0.0, 390.10.0). These versions are known to miscompile XLA " + "code, leading to incorrect results or invalid-address errors."; + } + }); +} + +// Compiles the given PTX string using ptxas and returns the resulting machine +// code (i.e. a cubin) as a byte array. +StatusOr> CompilePtx(const string& ptx, int cc_major, + int cc_minor) { + tracing::ScopedActivity activity("Compile PTX", /*is_expensive=*/true); + const string ptxas_path = + tensorflow::io::JoinPath(tensorflow::CudaRoot(), "bin", "ptxas"); + VLOG(2) << "Using ptxas at " << ptxas_path; + auto env = tensorflow::Env::Default(); + TF_RETURN_IF_ERROR(env->FileExists(ptxas_path)); + + WarnIfBadPtxasVersion(ptxas_path); + + // Write ptx into a temporary file. + string ptx_path; + if (!env->LocalTempFilename(&ptx_path)) { + return InternalError("couldn't get temp PTX file name"); + } + auto ptx_cleaner = tensorflow::gtl::MakeCleanup([&ptx_path] { + TF_CHECK_OK(tensorflow::Env::Default()->DeleteFile(ptx_path)); + }); + + TF_RETURN_IF_ERROR(tensorflow::WriteStringToFile(env, ptx_path, ptx)); + VLOG(2) << "ptx written to: " << ptx_path; + + // Invoke ptxas and collect its output. + string cubin_path; + if (!env->LocalTempFilename(&cubin_path)) { + return InternalError("couldn't get temp CUBIN file name"); + } + auto cubin_cleaner = tensorflow::gtl::MakeCleanup([&cubin_path] { + // CUBIN file may never be created, so the failure to delete it should not + // produce TF error. + tensorflow::Env::Default()->DeleteFile(cubin_path).IgnoreError(); + }); + tensorflow::SubProcess ptxas_info_dumper; + std::vector ptxas_args = { + ptxas_path, ptx_path, "-o", cubin_path, + tensorflow::strings::StrCat("-arch=sm_", cc_major, cc_minor)}; + if (VLOG_IS_ON(2)) { + ptxas_args.push_back("-v"); + } + ptxas_info_dumper.SetProgram(ptxas_path, ptxas_args); + ptxas_info_dumper.SetChannelAction(tensorflow::CHAN_STDERR, + tensorflow::ACTION_PIPE); + if (!ptxas_info_dumper.Start()) { + return InternalError("Failed to launch ptxas"); + } + string stderr_output; + int exit_status = ptxas_info_dumper.Communicate( + /*stdin_input=*/nullptr, /*stdout_output=*/nullptr, &stderr_output); + XLA_LOG_LINES(tensorflow::INFO, stderr_output); + if (exit_status != 0) { + return InternalError("ptxas exited with non-zero error code %d", + exit_status); + } + + // Read in the result of compilation and return it as a byte vector. + string cubin; + TF_RETURN_IF_ERROR(tensorflow::ReadFileToString(tensorflow::Env::Default(), + cubin_path, &cubin)); + std::vector cubin_vector(cubin.begin(), cubin.end()); + return cubin_vector; +} + +} // namespace + +NVPTXCompiler::NVPTXCompiler() + : pointer_size_(llvm::DataLayout(kDataLayout) + .getPointerSize(0 /* default address space */)) {} + +StatusOr> NVPTXCompiler::RunHloPasses( + std::unique_ptr module, se::StreamExecutor* stream_exec, + DeviceMemoryAllocator* device_allocator) { + XLA_SCOPED_LOGGING_TIMER("NVPTXCompiler::RunHloPasses"); + tracing::ScopedActivity activity("HLO Transforms", module->name(), + /*is_expensive=*/true); + TF_RETURN_IF_ERROR( + OptimizeHloModule(module.get(), stream_exec, device_allocator)); + return std::move(module); +} + +StatusOr> NVPTXCompiler::RunBackend( + std::unique_ptr module, se::StreamExecutor* stream_exec, + DeviceMemoryAllocator* device_allocator) { + XLA_SCOPED_LOGGING_TIMER("NVPTXCompiler::RunBackend"); + + TF_RET_CHECK(stream_exec != nullptr); + + TF_RETURN_IF_ERROR(PrepareHloModuleForIrEmitting(module.get())); + + llvm::LLVMContext llvm_context; + std::string buffer; + llvm::raw_string_ostream error(buffer); + llvm::DiagnosticPrinterRawOStream printer(error); + auto DiagnosticHandler = [](const llvm::DiagnosticInfo& diag_info, + void* Context) { + auto printer = static_cast(Context); + diag_info.print(*printer); + }; + llvm_context.setDiagnosticHandlerCallBack(DiagnosticHandler, &printer); + + llvm::Module llvm_module(module->name().c_str(), llvm_context); + // Set the target triple and the data layout. + llvm_module.setTargetTriple(kTargetTriple); + llvm_module.setDataLayout(kDataLayout); + + // Determine the HLO schedule, which is an ordering of HLO instructions. This + // is used by buffer assignment to enable buffer reuse, and the same ordering + // must also be used to determine the thunk launch schedule. + std::unique_ptr stream_assignment = AssignStreams(*module); + TF_ASSIGN_OR_RETURN( + std::unique_ptr hlo_schedule, + HloSchedule::Build(*module, *stream_assignment, pointer_size_)); + + // Run buffer analysis on the HLO graph. This analysis figures out which + // temporary buffers are required to run the computation. + TF_ASSIGN_OR_RETURN( + std::unique_ptr buffer_assignment, + BufferAssigner::Run(module.get(), hlo_schedule->ConsumeHloOrdering(), + BufferSizeBytesFunction(), + /*color_alignment=*/[](LogicalBuffer::Color) { + return kCudaMallocAlignBytes; + })); + // BufferAssignment::Stats::ToString() and BufferAssignment::ToString() + // include headers, so no need for us to print them ourselves. + XLA_VLOG_LINES(1, buffer_assignment->GetStats().ToString()); + XLA_VLOG_LINES(2, buffer_assignment->ToString()); + XLA_VLOG_LINES(2, module->ToString()); + const string xla_dump_optimized_hlo_proto_to = + module->config().debug_options().xla_dump_optimized_hlo_proto_to(); + if (!xla_dump_optimized_hlo_proto_to.empty()) { + HloProto proto = MakeHloProto(*module, *buffer_assignment); + TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory( + proto, xla_dump_optimized_hlo_proto_to, module->name())); + } + + IrEmitterContext ir_emitter_context(module.get(), buffer_assignment.get(), + &stream_exec->GetDeviceDescription(), + &llvm_module); + + HloComputation* entry_computation = module->entry_computation(); + IrEmitterUnnested ir_emitter(module->config(), entry_computation, + &ir_emitter_context); + { + XLA_SCOPED_LOGGING_TIMER("NVPTXCompiler::RunBackend - IR emission"); + TF_RETURN_IF_ERROR(entry_computation->Accept(&ir_emitter)); + } + + if (user_pre_optimization_hook_) { + TF_CHECK_OK(user_pre_optimization_hook_(llvm_module)); + } + string ir_module_string_before_opt; + const bool embed_ir_in_executable = + module->config().debug_options().xla_embed_ir_in_executable(); + if (VLOG_IS_ON(2) || embed_ir_in_executable) { + ir_module_string_before_opt = llvm_ir::DumpModuleToString(llvm_module); + VLOG(2) << "LLVM module before optimizations:"; + XLA_VLOG_LINES(2, ir_module_string_before_opt); + } + + const string& ir_dump_directory = + module->config().debug_options().xla_dump_ir_to(); + + if (!ir_dump_directory.empty()) { + TF_RETURN_IF_ERROR(llvm_ir::DumpIRToDirectory( + /*directory_name=*/ir_dump_directory, + /*hlo_module_name=*/module->name(), llvm_module, + /*optimized=*/false)); + } + + { + XLA_SCOPED_LOGGING_TIMER("NVPTXCompiler::RunBackend - Running LLVM verifier"); + + std::string err; + llvm::raw_string_ostream err_stream(err); + + // verifyModule() returns true if the module is broken. + TF_RET_CHECK(!llvm::verifyModule(llvm_module, &err_stream)) + << "Invalid LLVM IR before optimizations:\n" + << err_stream.str() + << "\nThis probably indicates a bug in the HLO -> LLVM IR lowering. " + "Rerun with --xla_dump_ir_to to get the IR. "; + } + + string libdevice_dir; + { + tensorflow::mutex_lock lock(mutex_); + + // Find the directory containing libdevice. To avoid searching for it every + // time, we have a one-element cache, keyed on the module's config's + // cuda_data_dir. + const auto& config_cuda_data_dir = + module->config().debug_options().xla_gpu_cuda_data_dir(); + if (cached_libdevice_dir_.empty() || + cached_cuda_data_dir_ != config_cuda_data_dir) { + cached_cuda_data_dir_ = config_cuda_data_dir; + cached_libdevice_dir_ = GetLibdeviceDir(config_cuda_data_dir); + } + libdevice_dir = cached_libdevice_dir_; + } + int cc_major, cc_minor; + if (!stream_exec->GetDeviceDescription().cuda_compute_capability(&cc_major, + &cc_minor)) { + LOG(WARNING) + << "Couldn't get compute capability for device; assuming sm_20."; + cc_major = 2; + cc_minor = 0; + } + + string ptx; + { + XLA_SCOPED_LOGGING_TIMER("NVPTXCompiler::RunBackend - CompileToPtx"); + TF_ASSIGN_OR_RETURN(ptx, CompileToPtx(&llvm_module, {cc_major, cc_minor}, + module->config(), libdevice_dir)); + } + + if (!ir_dump_directory.empty()) { + TF_RETURN_IF_ERROR(llvm_ir::DumpIRToDirectory( + /*directory_name=*/ir_dump_directory, + /*hlo_module_name=*/module->name(), llvm_module, + /*optimized=*/true)); + } + + if (user_post_optimization_hook_) { + TF_CHECK_OK(user_post_optimization_hook_(llvm_module)); + } + VLOG(2) << "LLVM module after optimizations:"; + XLA_VLOG_LINES(2, llvm_ir::DumpModuleToString(llvm_module)); + VLOG(2) << "PTX:"; + XLA_VLOG_LINES(2, ptx); + + // Write PTX to IR dump directory, if IR dumping was requested. + if (!ir_dump_directory.empty()) { + const string ptx_outfile = tensorflow::io::JoinPath( + ir_dump_directory, tensorflow::strings::StrCat(module->name(), ".ptx")); + auto status = [&] { + auto* env = tensorflow::Env::Default(); + TF_RETURN_IF_ERROR(env->RecursivelyCreateDir(ir_dump_directory)); + TF_RETURN_IF_ERROR(tensorflow::WriteStringToFile(env, ptx_outfile, ptx)); + return Status::OK(); + }(); + if (!status.ok()) { + LOG(WARNING) << "Couldn't dump PTX for module " << module->name() + << " to " << ptx_outfile << ": " << status; + } + } + + const std::vector cubin = + CompilePtxOrGetCachedResult(ptx, cc_major, cc_minor); + + auto thunk_schedule = MakeUnique( + ir_emitter.ConsumeThunkSequence(), std::move(stream_assignment), + hlo_schedule->ThunkLaunchOrder()); + VLOG(2) << "Printing the thunk schedule..."; + XLA_VLOG_LINES(2, thunk_schedule->ToString()); + + std::unique_ptr profile_index_map; + std::unique_ptr profile_printer; + + if (module->config().hlo_profiling_enabled()) { + HloCostAnalysis cost_analysis(ShapeSizeBytesFunction()); + cost_analysis.set_bytes_per_second( + stream_exec->GetDeviceDescription().memory_bandwidth()); + TF_RETURN_IF_ERROR(module->entry_computation()->Accept(&cost_analysis)); + profile_index_map = MakeUnique(*module); + profile_printer = + CreateHloProfilePrinterData(*profile_index_map, cost_analysis); + } + + auto* gpu_executable = new GpuExecutable( + ptx, cubin, {cc_major, cc_minor}, std::move(thunk_schedule), + std::move(module), std::move(buffer_assignment), + std::move(profile_printer), std::move(profile_index_map)); + if (embed_ir_in_executable) { + DCHECK_NE("", ir_module_string_before_opt); + gpu_executable->set_ir_module_string(ir_module_string_before_opt); + } + return std::unique_ptr(gpu_executable); +} + +std::vector NVPTXCompiler::CompilePtxOrGetCachedResult(const string& ptx, + int cc_major, + int cc_minor) { + XLA_SCOPED_LOGGING_TIMER("NVPTXCompiler::CompilePtxOrGetCachedResult"); + tracing::ScopedActivity activity("PTX->CUBIN", /*is_expensive=*/true); + bool inserted; + decltype(compilation_cache_.begin()) iter; + // Pointers into compilation_cache_ where the ptx and (optional) cubin are + // stored. + const string* cache_ptx = nullptr; + CompilationCacheValue* cache_value = nullptr; + + { + tensorflow::mutex_lock lock(mutex_); + std::tie(iter, inserted) = compilation_cache_.emplace( + std::piecewise_construct, + std::forward_as_tuple(ptx, cc_major, cc_minor), + std::forward_as_tuple()); + cache_ptx = &iter->first.ptx; + cache_value = &iter->second; + } + + // Compile the ptx if it wasn't in the cache before we called this function. + // Other threads asking for the same compilation key will block on + // cache_value->mutex_ until compilation is done. + { + tensorflow::mutex_lock lock(cache_value->mutex_); + if (inserted) { + CHECK(!cache_value->compilation_done); + if (!ptx.empty()) { + StatusOr> maybe_cubin = + CompilePtx(*cache_ptx, cc_major, cc_minor); + if (maybe_cubin.ok()) { + cache_value->cubin_data = std::move(maybe_cubin).ValueOrDie(); + VLOG(2) << "Compiled PTX size:" << ptx.size() + << " CUBIN size: " << cache_value->cubin_data.size(); + } else { + bool log_warning = true; + if (maybe_cubin.status().code() == + tensorflow::error::Code::NOT_FOUND) { + // Missing ptxas is expected in some environments where CUDA SDK + // binaries are not available. We don't want to spam logs with + // identical warnings in this case. + + // TODO(zhengxq): we should implement a LOG_FIRST_N and LOG_EVERY_N + // for more general usage. + static std::atomic warning_done(false); + log_warning = !warning_done.exchange(true); + } + if (log_warning) { + LOG(WARNING) + << "Failed to compile ptx to cubin. Will attempt to let " + "GPU driver compile the ptx. " + << maybe_cubin.status(); + } + + // We're going to use the driver to JIT our PTX->SASS, so warn if + // the JIT in the driver has known bugs. + WarnIfBadDriverJITVersion(); + } + } + cache_value->compilation_done = true; + cache_value->compilation_done_cv_.notify_all(); + } else { + while (!cache_value->compilation_done) { + cache_value->compilation_done_cv_.wait(lock); + } + } + } + + CHECK(cache_value != nullptr); + CHECK(cache_value->compilation_done); + return cache_value->cubin_data; +} + +StatusOr>> +NVPTXCompiler::CompileAheadOfTime(std::vector> module, + const AotCompilationOptions& options) { + return Unimplemented("not yet implemented: NVPTXCompiler::CompileAheadOfTime"); +} + +se::Platform::Id NVPTXCompiler::PlatformId() const { + return se::cuda::kCudaPlatformId; +} + +} // namespace gpu +} // namespace xla + +static bool InitModule() { + xla::Compiler::RegisterCompilerFactory( + stream_executor::cuda::kCudaPlatformId, + []() { return xla::MakeUnique(); }); + return true; +} +static bool module_initialized = InitModule(); diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.h b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.h new file mode 100644 index 0000000000..d4d2909f1b --- /dev/null +++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.h @@ -0,0 +1,155 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_NVPTX_COMPILER_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_NVPTX_COMPILER_H_ + +#include +#include +#include + +#include "tensorflow/compiler/xla/service/executable.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/llvm_compiler.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/lib/gtl/optional.h" +#include "tensorflow/core/lib/hash/hash.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/stream_executor_no_cuda.h" +#include "tensorflow/core/platform/thread_annotations.h" + +namespace xla { +namespace gpu { + +// The GPU compiler generates efficient GPU executables. +class NVPTXCompiler : public LLVMCompiler { + public: + NVPTXCompiler(); + ~NVPTXCompiler() override {} + + // Bring in + // StatusOr>> Compile( + // std::vector> modules, + // std::vector> + // stream_execs) + using LLVMCompiler::Compile; + + StatusOr> RunHloPasses( + std::unique_ptr module, se::StreamExecutor* stream_exec, + DeviceMemoryAllocator* device_allocator) override; + + StatusOr> RunBackend( + std::unique_ptr module, se::StreamExecutor* stream_exec, + DeviceMemoryAllocator* device_allocator) override; + + StatusOr>> + CompileAheadOfTime(std::vector> module, + AotCompilationOptions const& options) override; + + se::Platform::Id PlatformId() const override; + + HloCostAnalysis::ShapeSizeFunction ShapeSizeBytesFunction() const override { + // Capture just the pointer size, not the entire NVPTXCompiler object. + int64 pointer_size = pointer_size_; + return [pointer_size](const Shape& shape) { + return ShapeUtil::ByteSizeOf(shape, pointer_size); + }; + } + + // The triple that represents our target. + static const char* kTargetTriple; + + // The data layout of the emitted module. Copied from computeDataLayout in + // NVPTXTargetMachine.cpp. + static const char* kDataLayout; + + private: + // The size in bytes of a pointer. Used by ShapeSizeBytesFunction. + const int64 pointer_size_; + + tensorflow::mutex mutex_; + + // When compiling an HLO module, we need to find a path to the nvvm libdevice + // files. We search in the module's config.debug_options().cuda_data_dir() + // and in tensorflow::LibdeviceRoot(), the latter of which is a constant. + // + // We cache the cuda_data_dir() and the result of our search, so that if the + // next module we have to compile has the same cuda_data_dir(), we can skip + // the search. + string cached_cuda_data_dir_ GUARDED_BY(mutex_); + string cached_libdevice_dir_ GUARDED_BY(mutex_); + + // Tries to compile the given ptx string to cubin. Returns a vector with the + // compiled cubin. If compilation was unsuccessful, returns an empty vector. + std::vector CompilePtxOrGetCachedResult(const string& ptx, + int cc_major, int cc_minor); + + // The compilation_cache_ map is a cache from {ptx string, cc_major, cc_minor} + // -> cubin so we don't recompile the same ptx twice. This is important for + // some interactive workflows. (We also cache at the HLO level, but sometimes + // we can't realize that two modules are the same until we lower to ptx.) + // + // Compilation of distinct PTX happens in parallel. If more than one thread + // attempts to compile the same PTX, the fist thread to obtain + // cache_value_->mutex_ performs the compilation. The rest wait() on + // cache_value_->compilation_done_cv_ until the compilation is done. + // + // If compiling the ptx fails, we return an empty cubin, cross our fingers, + // and leave compilation up to the driver. + struct CompilationCacheKey { + CompilationCacheKey(std::string ptx, int cc_major, int cc_minor) + : ptx(std::move(ptx)), cc_major(cc_major), cc_minor(cc_minor) {} + string ptx; + int cc_major; + int cc_minor; + }; + struct CompilationCacheHash { + size_t operator()(const CompilationCacheKey& key) const { + return tensorflow::Hash64Combine( + tensorflow::Hash64Combine(tensorflow::Hash64(key.ptx), key.cc_major), + key.cc_minor); + } + }; + struct CompilationCacheEq { + size_t operator()(const CompilationCacheKey& a, + const CompilationCacheKey& b) const { + return a.cc_major == b.cc_major && a.cc_minor == b.cc_minor && + a.ptx == b.ptx; + } + }; + struct CompilationCacheValue { + bool compilation_done = false; + std::vector cubin_data; + // mutex and condition variable to serialize compilation completing. + tensorflow::mutex mutex_; + tensorflow::condition_variable compilation_done_cv_; + }; + + // Don't even think about switching this to FlatMap; iterator stability is + // critical here. + std::unordered_map + compilation_cache_ GUARDED_BY(mutex_); + + TF_DISALLOW_COPY_AND_ASSIGN(NVPTXCompiler); +}; + +} // namespace gpu +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_NVPTX_COMPILER_H_ -- cgit v1.2.3 From 0e14f11516d5aa4350f3f101ca6e60a3c9db9744 Mon Sep 17 00:00:00 2001 From: James Keeling Date: Mon, 16 Jul 2018 11:01:38 -0700 Subject: Add additional Python info into two error messages. If the experimental option is enabled, this will insert Python format information into two error messages in placer.cc. The error messages are emitted when Placer fails to assign a device for an operation. The additional info will cause the Python layer to output where the node was defined in Python. PiperOrigin-RevId: 204770562 --- tensorflow/core/BUILD | 1 + tensorflow/core/common_runtime/placer.cc | 30 +++++++++++++----- tensorflow/core/common_runtime/placer.h | 1 + tensorflow/core/common_runtime/placer_test.cc | 44 +++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 8a43220ec5..514713bb96 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -846,6 +846,7 @@ tf_cuda_library( "util/sparse/sparse_tensor.h", "util/stat_summarizer.h", "util/stat_summarizer_options.h", + "util/status_util.h", "util/stream_executor_util.h", "util/strided_slice_op.h", "util/tensor_format.h", diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc index 0be44662dd..6781c87f6c 100644 --- a/tensorflow/core/common_runtime/placer.cc +++ b/tensorflow/core/common_runtime/placer.cc @@ -30,6 +30,7 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/util/status_util.h" namespace tensorflow { @@ -822,10 +823,10 @@ Status Placer::Run() { std::vector* devices; Status status = colocation_graph.GetDevicesForNode(node, &devices); if (!status.ok()) { - return AttachDef( - errors::InvalidArgument("Cannot assign a device for operation '", - node->name(), "': ", status.error_message()), - *node); + return AttachDef(errors::InvalidArgument( + "Cannot assign a device for operation ", + RichNodeName(node), ": ", status.error_message()), + *node); } // Returns the first device in sorted devices list so we will always @@ -869,10 +870,10 @@ Status Placer::Run() { std::vector* devices; Status status = colocation_graph.GetDevicesForNode(node, &devices); if (!status.ok()) { - return AttachDef( - errors::InvalidArgument("Cannot assign a device for operation '", - node->name(), "': ", status.error_message()), - *node); + return AttachDef(errors::InvalidArgument( + "Cannot assign a device for operation ", + RichNodeName(node), ": ", status.error_message()), + *node); } int assigned_device = -1; @@ -943,4 +944,17 @@ bool Placer::ClientHandlesErrorFormatting() const { options_->config.experimental().client_handles_error_formatting(); } +// Returns the node name in single quotes. If the client handles formatted +// errors, appends a formatting tag which the client will reformat into, for +// example, " (defined at filename:123)". +string Placer::RichNodeName(const Node* node) const { + string quoted_name = strings::StrCat("'", node->name(), "'"); + if (ClientHandlesErrorFormatting()) { + string file_and_line = error_format_tag(*node, "${file}:${line}"); + return strings::StrCat(quoted_name, " (defined at ", file_and_line, ")"); + } else { + return quoted_name; + } +} + } // namespace tensorflow diff --git a/tensorflow/core/common_runtime/placer.h b/tensorflow/core/common_runtime/placer.h index 1f8b450103..fce87269c5 100644 --- a/tensorflow/core/common_runtime/placer.h +++ b/tensorflow/core/common_runtime/placer.h @@ -88,6 +88,7 @@ class Placer { void AssignAndLog(int assigned_device, Node* node) const; void LogDeviceAssignment(const Node* node) const; bool ClientHandlesErrorFormatting() const; + string RichNodeName(const Node* node) const; Graph* const graph_; // Not owned. const DeviceSet* const devices_; // Not owned. diff --git a/tensorflow/core/common_runtime/placer_test.cc b/tensorflow/core/common_runtime/placer_test.cc index 07a7724f16..cede899842 100644 --- a/tensorflow/core/common_runtime/placer_test.cc +++ b/tensorflow/core/common_runtime/placer_test.cc @@ -1142,6 +1142,50 @@ TEST_F(PlacerTest, TestNonexistentGpuNoAllowSoftPlacement) { EXPECT_TRUE(str_util::StrContains(s.error_message(), "/device:fakegpu:11")); } +// Test that the "Cannot assign a device" error message contains a format tag +// when requested. +TEST_F(PlacerTest, TestNonexistentGpuNoAllowSoftPlacementFormatTag) { + Graph g(OpRegistry::Global()); + { // Scope for temporary variables used to construct g. + GraphDefBuilder b(GraphDefBuilder::kFailImmediately); + ops::SourceOp("TestDevice", + b.opts().WithName("in").WithDevice("/device:fakegpu:11")); + TF_EXPECT_OK(BuildGraph(b, &g)); + } + + SessionOptions options; + options.config.mutable_experimental()->set_client_handles_error_formatting( + true); + Status s = Place(&g, &options); + EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); + EXPECT_TRUE( + str_util::StrContains(s.error_message(), + "Cannot assign a device for operation 'in'" + " (defined at ^^node:in:${file}:${line}^^)")); +} + +// Test that the "Cannot assign a device" error message does not contain a +// format tag when not it shouldn't +TEST_F(PlacerTest, TestNonexistentGpuNoAllowSoftPlacementNoFormatTag) { + Graph g(OpRegistry::Global()); + { // Scope for temporary variables used to construct g. + GraphDefBuilder b(GraphDefBuilder::kFailImmediately); + ops::SourceOp("TestDevice", + b.opts().WithName("in").WithDevice("/device:fakegpu:11")); + TF_EXPECT_OK(BuildGraph(b, &g)); + } + + SessionOptions options; + options.config.mutable_experimental()->set_client_handles_error_formatting( + false); + Status s = Place(&g, &options); + EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); + EXPECT_TRUE(str_util::StrContains( + s.error_message(), "Cannot assign a device for operation 'in'")); + EXPECT_FALSE(str_util::StrContains( + s.error_message(), "'in' (defined at ^^node:in:${file}:${line}^^)")); +} + // Test that placement fails when a node requests an explicit device that is not // supported by the registered kernels if allow_soft_placement is no set. TEST_F(PlacerTest, TestUnsupportedDeviceNoAllowSoftPlacement) { -- cgit v1.2.3 From d1e7fe7e95b5f80830e9692ba2e7df0cb3d6373d Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Mon, 16 Jul 2018 11:03:17 -0700 Subject: [tf.data] Generalization of `tf.contrib.data.sliding_window_batch`. Prior to this CL, the signature of the method was `sliding_window_batch(window_size, stride=1)`. This CL changes the signature to `sliding_window_batch(window_size, stride=None, window_shift=None, window_stride=1)`, where the `window_shift` argument acts as the original `stride` argument (determining the shift between consecutive windows), while the `window_stride` argument determines the stride of the input elements in the window. For example, if `a = { [1], [2], [3], [4], [5], [6] }` is a dataset, then: ``` a.apply(sliding_window_batch(window_size=3)) == { [[1], [2], [3]], [[2], [3], [4]], [[3], [4], [5]], [[4], [5], [6]] } a.apply(sliding_window_batch(window_size=3, window_shift=2)) == { [[1], [2], [3]], [[3], [4], [5]] } a.apply(sliding_window_batch(window_size=3, window_stride=2)) == { [[1], [3], [5]], [[2], [4], [6]] } ``` PiperOrigin-RevId: 204770909 --- tensorflow/contrib/data/python/kernel_tests/BUILD | 1 + .../python/kernel_tests/slide_dataset_op_test.py | 255 +++++++++++++-------- tensorflow/contrib/data/python/ops/sliding.py | 69 ++++-- .../api_def/base_api/api_def_SlideDataset.pbtxt | 9 +- tensorflow/core/kernels/data/slide_dataset_op.cc | 158 +++++++------ tensorflow/core/ops/compat/ops_history.v1.pbtxt | 6 +- tensorflow/core/ops/dataset_ops.cc | 7 +- 7 files changed, 308 insertions(+), 197 deletions(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index 9a454efc4c..18457320b9 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -380,6 +380,7 @@ py_test( "//tensorflow/python:sparse_tensor", "//tensorflow/python/data/ops:dataset_ops", "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", ], ) diff --git a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py index 5590a4bf78..8b2f846494 100644 --- a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from absl.testing import parameterized import numpy as np from tensorflow.contrib.data.python.ops import sliding @@ -29,28 +30,45 @@ from tensorflow.python.ops import math_ops from tensorflow.python.platform import test -class SlideDatasetTest(test.TestCase): - - def testSlideDataset(self): - """Test an dataset that maps a TF function across its input elements.""" +class SlideDatasetTest(test.TestCase, parameterized.TestCase): + + @parameterized.parameters( + (20, 14, 7, 1), + (20, 17, 9, 1), + (20, 14, 14, 1), + (20, 10, 14, 1), + (20, 14, 19, 1), + (20, 4, 1, 2), + (20, 2, 1, 6), + (20, 4, 7, 2), + (20, 2, 7, 6), + (1, 10, 4, 1), + (0, 10, 4, 1), + ) + def testSlideDataset(self, count, window_size, window_shift, window_stride): + """Tests a dataset that slides a window its input elements.""" components = (np.arange(7), np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], np.array(37.0) * np.arange(7)) - count = array_ops.placeholder(dtypes.int64, shape=[]) - window_size = array_ops.placeholder(dtypes.int64, shape=[]) - stride = array_ops.placeholder(dtypes.int64, shape=[]) + count_t = array_ops.placeholder(dtypes.int64, shape=[]) + window_size_t = array_ops.placeholder(dtypes.int64, shape=[]) + window_shift_t = array_ops.placeholder(dtypes.int64, shape=[]) + window_stride_t = array_ops.placeholder(dtypes.int64, shape=[]) def _map_fn(x, y, z): return math_ops.square(x), math_ops.square(y), math_ops.square(z) # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> - # RepeatDataset(count) -> _SlideDataset(window_size, stride). - iterator = (dataset_ops.Dataset.from_tensor_slices(components) - .map(_map_fn) - .repeat(count) - .apply(sliding.sliding_window_batch(window_size, stride)) - .make_initializable_iterator()) + # RepeatDataset(count) -> + # _SlideDataset(window_size, window_shift, window_stride). + iterator = ( + dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) + .repeat(count).apply( + sliding.sliding_window_batch( + window_size=window_size_t, + window_shift=window_shift_t, + window_stride=window_stride_t)).make_initializable_iterator()) init_op = iterator.initializer get_next = iterator.get_next() @@ -58,90 +76,126 @@ class SlideDatasetTest(test.TestCase): [t.shape.as_list() for t in get_next]) with self.test_session() as sess: - # stride < window_size. - # Slide over a finite input, where the window_size divides the - # total number of elements. - sess.run(init_op, feed_dict={count: 20, window_size: 14, stride: 7}) - # Same formula with convolution layer. - num_batches = (20 * 7 - 14) // 7 + 1 - for i in range(num_batches): - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range(14): - self.assertAllEqual(component[(i*7 + j) % 7]**2, - result_component[j]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - # Slide over a finite input, where the window_size does not - # divide the total number of elements. - sess.run(init_op, feed_dict={count: 20, window_size: 17, stride: 9}) - num_batches = (20 * 7 - 17) // 9 + 1 + sess.run( + init_op, + feed_dict={ + count_t: count, + window_size_t: window_size, + window_shift_t: window_shift, + window_stride_t: window_stride + }) + num_batches = (count * 7 - ( + (window_size - 1) * window_stride + 1)) // window_shift + 1 for i in range(num_batches): result = sess.run(get_next) for component, result_component in zip(components, result): - for j in range(17): - self.assertAllEqual(component[(i*9 + j) % 7]**2, - result_component[j]) + for j in range(window_size): + self.assertAllEqual( + component[(i * window_shift + j * window_stride) % 7]**2, + result_component[j]) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) - # stride == window_size. - sess.run(init_op, feed_dict={count: 20, window_size: 14, stride: 14}) - num_batches = 20 * 7 // 14 - for i in range(num_batches): - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range(14): - self.assertAllEqual(component[(i*14 + j) % 7]**2, - result_component[j]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + @parameterized.parameters( + (20, 14, 7, 1), + (20, 17, 9, 1), + (20, 14, 14, 1), + (20, 10, 14, 1), + (20, 14, 19, 1), + (20, 4, 1, 2), + (20, 2, 1, 6), + (20, 4, 7, 2), + (20, 2, 7, 6), + (1, 10, 4, 1), + (0, 10, 4, 1), + ) + def testSlideDatasetDeprecated(self, count, window_size, stride, + window_stride): + """Tests a dataset that slides a window its input elements.""" + components = (np.arange(7), + np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis], + np.array(37.0) * np.arange(7)) - # stride > window_size. - sess.run(init_op, feed_dict={count: 20, window_size: 10, stride: 14}) - num_batches = 20 * 7 // 14 - for i in range(num_batches): - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range(10): - self.assertAllEqual(component[(i*14 + j) % 7]**2, - result_component[j]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - # Drop the last batch which is smaller than window_size. - sess.run(init_op, feed_dict={count: 20, window_size: 14, stride: 19}) - num_batches = (20 * 7 - 7) // 19 # = 19 * 7 // 19 - for i in range(num_batches): - result = sess.run(get_next) - for component, result_component in zip(components, result): - for j in range(14): - self.assertAllEqual(component[(i*19 + j) % 7]**2, - result_component[j]) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + count_t = array_ops.placeholder(dtypes.int64, shape=[]) + window_size_t = array_ops.placeholder(dtypes.int64, shape=[]) + stride_t = array_ops.placeholder(dtypes.int64, shape=[]) + window_stride_t = array_ops.placeholder(dtypes.int64, shape=[]) - # Slide over a finite input, which is less than window_size, - # should fail straight away. - sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 4}) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + def _map_fn(x, y, z): + return math_ops.square(x), math_ops.square(y), math_ops.square(z) - sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 8}) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) + # The pipeline is TensorSliceDataset -> MapDataset(square_3) -> + # RepeatDataset(count) -> _SlideDataset(window_size, stride, window_stride). + iterator = ( + dataset_ops.Dataset.from_tensor_slices(components).map(_map_fn) + .repeat(count).apply( + sliding.sliding_window_batch( + window_size=window_size_t, + stride=stride_t, + window_stride=window_stride_t)).make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() - # Slide over an empty input should fail straight away. - sess.run(init_op, feed_dict={count: 0, window_size: 8, stride: 4}) + self.assertEqual([[None] + list(c.shape[1:]) for c in components], + [t.shape.as_list() for t in get_next]) + + with self.test_session() as sess: + sess.run( + init_op, + feed_dict={ + count_t: count, + window_size_t: window_size, + stride_t: stride, + window_stride_t: window_stride + }) + num_batches = (count * 7 - ( + (window_size - 1) * window_stride + 1)) // stride + 1 + for i in range(num_batches): + result = sess.run(get_next) + for component, result_component in zip(components, result): + for j in range(window_size): + self.assertAllEqual( + component[(i * stride + j * window_stride) % 7]**2, + result_component[j]) with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) - # Empty window_size should be an initialization time error. - with self.assertRaises(errors.InvalidArgumentError): - sess.run(init_op, feed_dict={count: 14, window_size: 0, stride: 0}) + @parameterized.parameters( + (14, 0, 3, 1), + (14, 3, 0, 1), + (14, 3, 3, 0), + ) + def testSlideDatasetInvalid(self, count, window_size, window_shift, + window_stride): + count_t = array_ops.placeholder(dtypes.int64, shape=[]) + window_size_t = array_ops.placeholder(dtypes.int64, shape=[]) + window_shift_t = array_ops.placeholder(dtypes.int64, shape=[]) + window_stride_t = array_ops.placeholder(dtypes.int64, shape=[]) + + iterator = ( + dataset_ops.Dataset.range(10).map(lambda x: x).repeat(count_t).apply( + sliding.sliding_window_batch( + window_size=window_size_t, + window_shift=window_shift_t, + window_stride=window_stride_t)).make_initializable_iterator()) + init_op = iterator.initializer - # Invalid stride should be an initialization time error. + with self.test_session() as sess: with self.assertRaises(errors.InvalidArgumentError): - sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 0}) + sess.run( + init_op, + feed_dict={ + count_t: count, + window_size_t: window_size, + window_shift_t: window_shift, + window_stride_t: window_stride + }) + + def testSlideDatasetValueError(self): + with self.assertRaises(ValueError): + dataset_ops.Dataset.range(10).map(lambda x: x).apply( + sliding.sliding_window_batch( + window_size=1, stride=1, window_shift=1, window_stride=1)) def assertSparseValuesEqual(self, a, b): self.assertAllEqual(a.indices, b.indices) @@ -155,7 +209,8 @@ class SlideDatasetTest(test.TestCase): indices=[[0]], values=(i * [1]), dense_shape=[1]) iterator = dataset_ops.Dataset.range(10).map(_sparse).apply( - sliding.sliding_window_batch(5, 3)).make_initializable_iterator() + sliding.sliding_window_batch( + window_size=5, window_shift=3)).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() @@ -183,7 +238,8 @@ class SlideDatasetTest(test.TestCase): dense_shape=[i]) iterator = dataset_ops.Dataset.range(10).map(_sparse).apply( - sliding.sliding_window_batch(5, 3)).make_initializable_iterator() + sliding.sliding_window_batch( + window_size=5, window_shift=3)).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() @@ -213,11 +269,11 @@ class SlideDatasetTest(test.TestCase): return sparse_tensor.SparseTensorValue( indices=[[0]], values=(i * [1]), dense_shape=[1]) - iterator = (dataset_ops.Dataset.range(10) - .map(_sparse) - .apply(sliding.sliding_window_batch(4, 2)) - .apply(sliding.sliding_window_batch(3, 1)) - .make_initializable_iterator()) + iterator = ( + dataset_ops.Dataset.range(10).map(_sparse).apply( + sliding.sliding_window_batch(window_size=4, window_shift=2)).apply( + sliding.sliding_window_batch(window_size=3, window_shift=1)) + .make_initializable_iterator()) init_op = iterator.initializer get_next = iterator.get_next() @@ -226,9 +282,9 @@ class SlideDatasetTest(test.TestCase): # Slide: 1st batch. actual = sess.run(get_next) expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], - [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], - [2, 0, 0], [2, 1, 0], [2, 2, 0], [2, 3, 0]], + indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0], + [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0], + [2, 2, 0], [2, 3, 0]], values=[0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7], dense_shape=[3, 4, 1]) self.assertTrue(sparse_tensor.is_sparse(actual)) @@ -236,9 +292,9 @@ class SlideDatasetTest(test.TestCase): # Slide: 2nd batch. actual = sess.run(get_next) expected = sparse_tensor.SparseTensorValue( - indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], - [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0], - [2, 0, 0], [2, 1, 0], [2, 2, 0], [2, 3, 0]], + indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0], [1, 0, 0], + [1, 1, 0], [1, 2, 0], [1, 3, 0], [2, 0, 0], [2, 1, 0], + [2, 2, 0], [2, 3, 0]], values=[2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9], dense_shape=[3, 4, 1]) self.assertTrue(sparse_tensor.is_sparse(actual)) @@ -253,10 +309,11 @@ class SlideDatasetTest(test.TestCase): yield [4.0, 5.0, 6.0] yield [7.0, 8.0, 9.0, 10.0] - iterator = (dataset_ops.Dataset.from_generator(generator, dtypes.float32, - output_shapes=[None]) - .apply(sliding.sliding_window_batch(3, 1)) - .make_initializable_iterator()) + iterator = ( + dataset_ops.Dataset.from_generator( + generator, dtypes.float32, output_shapes=[None]).apply( + sliding.sliding_window_batch(window_size=3, window_shift=1)) + .make_initializable_iterator()) next_element = iterator.get_next() with self.test_session() as sess: diff --git a/tensorflow/contrib/data/python/ops/sliding.py b/tensorflow/contrib/data/python/ops/sliding.py index 3f3c5ca17c..e9dd74530a 100644 --- a/tensorflow/contrib/data/python/ops/sliding.py +++ b/tensorflow/contrib/data/python/ops/sliding.py @@ -23,25 +23,29 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.util import deprecation class _SlideDataset(dataset_ops.Dataset): """A `Dataset` that passes a sliding window over its input.""" - def __init__(self, input_dataset, window_size, stride=1): + def __init__(self, input_dataset, window_size, window_shift, window_stride): """See `sliding_window_batch` for details.""" super(_SlideDataset, self).__init__() self._input_dataset = input_dataset self._window_size = ops.convert_to_tensor( - window_size, dtype=dtypes.int64, name="window_size") - self._stride = ops.convert_to_tensor( - stride, dtype=dtypes.int64, name="stride") + window_size, dtype=dtypes.int64, name="window_stride") + self._window_stride = ops.convert_to_tensor( + window_stride, dtype=dtypes.int64, name="window_stride") + self._window_shift = ops.convert_to_tensor( + window_shift, dtype=dtypes.int64, name="window_shift") def _as_variant_tensor(self): return gen_dataset_ops.slide_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access window_size=self._window_size, - stride=self._stride, + window_shift=self._window_shift, + window_stride=self._window_stride, **dataset_ops.flat_structure(self)) @property @@ -61,38 +65,63 @@ class _SlideDataset(dataset_ops.Dataset): return self._input_dataset.output_types -def sliding_window_batch(window_size, stride=1): - """A sliding window with size of `window_size` and step of `stride`. +@deprecation.deprecated_args( + None, "stride is deprecated, use window_shift instead", "stride") +def sliding_window_batch(window_size, + stride=None, + window_shift=None, + window_stride=1): + """A sliding window over a dataset. - This transformation passes a sliding window over this dataset. The - window size is `window_size` and step size is `stride`. If the left - elements cannot fill up the sliding window, this transformation will - drop the final smaller element. For example: + This transformation passes a sliding window over this dataset. The window size + is `window_size`, the stride of the input elements is `window_stride`, and the + shift between consecutive windows is `window_shift`. If the remaining elements + cannot fill up the sliding window, this transformation will drop the final + smaller element. For example: ```python # NOTE: The following examples use `{ ... }` to represent the # contents of a dataset. a = { [1], [2], [3], [4], [5], [6] } - a.apply(tf.contrib.data.sliding_window_batch(window_size=3, stride=2)) == - { - [[1], [2], [3]], - [[3], [4], [5]], - } + a.apply(sliding_window_batch(window_size=3)) == + { [[1], [2], [3]], [[2], [3], [4]], [[3], [4], [5]], [[4], [5], [6]] } + + a.apply(sliding_window_batch(window_size=3, window_shift=2)) == + { [[1], [2], [3]], [[3], [4], [5]] } + + a.apply(sliding_window_batch(window_size=3, window_stride=2)) == + { [[1], [3], [5]], [[2], [4], [6]] } ``` Args: window_size: A `tf.int64` scalar `tf.Tensor`, representing the number of - elements in the sliding window. + elements in the sliding window. It must be positive. stride: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the - steps moving the sliding window forward for one iteration. The default - is `1`. It must be positive. + forward shift of the sliding window in each iteration. The default is `1`. + It must be positive. Deprecated alias for `window_shift`. + window_shift: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the + forward shift of the sliding window in each iteration. The default is `1`. + It must be positive. + window_stride: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the + stride of the input elements in the sliding window. The default is `1`. + It must be positive. Returns: A `Dataset` transformation function, which can be passed to @{tf.data.Dataset.apply}. + + Raises: + ValueError: if invalid arguments are provided. """ + if stride is None and window_shift is None: + window_shift = 1 + elif stride is not None and window_shift is None: + window_shift = stride + elif stride is not None and window_shift is not None: + raise ValueError("Cannot specify both `stride` and `window_shift`") + def _apply_fn(dataset): - return _SlideDataset(dataset, window_size, stride) + return _SlideDataset(dataset, window_size, window_shift, window_stride) return _apply_fn diff --git a/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt index c80ee77f73..ddde3ee5b4 100644 --- a/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt @@ -8,10 +8,17 @@ sliding window. END } in_arg { - name: "stride" + name: "window_shift" description: < +#include + #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/kernels/data/dataset.h" @@ -33,36 +37,40 @@ class SlideDatasetOp : public UnaryDatasetOpKernel { void MakeDataset(OpKernelContext* ctx, DatasetBase* input, DatasetBase** output) override { int64 window_size = 0; - int64 stride = 0; OP_REQUIRES_OK( ctx, ParseScalarArgument(ctx, "window_size", &window_size)); - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "stride", &stride)); OP_REQUIRES( ctx, window_size > 0, errors::InvalidArgument("Window size must be greater than zero.")); - OP_REQUIRES(ctx, stride > 0, - errors::InvalidArgument("Stride must be greater than zero.")); - if (stride == window_size) { - LOG(WARNING) << "stride: " << stride + int64 window_shift = 0; + OP_REQUIRES_OK( + ctx, ParseScalarArgument(ctx, "window_shift", &window_shift)); + OP_REQUIRES( + ctx, window_shift > 0, + errors::InvalidArgument("Window shift must be greater than zero.")); + int64 window_stride = 0; + OP_REQUIRES_OK( + ctx, ParseScalarArgument(ctx, "window_stride", &window_stride)); + OP_REQUIRES( + ctx, window_stride > 0, + errors::InvalidArgument("window_stride must be greater than zero.")); + if (window_size == window_shift && window_stride == 1) { + LOG(WARNING) << "window_shift: " << window_shift << " is equal to window_size: " << window_size - << ", to use `batch` instead."; - } else if (stride > window_size) { - LOG(WARNING) << "stride: " << stride - << " is greater than window_size: " << window_size - << ", you will lose some data."; + << " and window_stride is 1, use `batch` instead."; } - - *output = new Dataset(ctx, window_size, stride, input); + *output = new Dataset(ctx, window_size, window_shift, window_stride, input); } private: class Dataset : public GraphDatasetBase { public: - Dataset(OpKernelContext* ctx, int64 window_size, int64 stride, - const DatasetBase* input) + Dataset(OpKernelContext* ctx, int64 window_size, int64 window_shift, + int64 window_stride, const DatasetBase* input) : GraphDatasetBase(ctx), window_size_(window_size), - stride_(stride), + window_shift_(window_shift), + window_stride_(window_stride), input_(input) { input_->Ref(); @@ -91,8 +99,8 @@ class SlideDatasetOp : public UnaryDatasetOpKernel { } string DebugString() const override { - return strings::StrCat("SlideDatasetOp(", window_size_, ", ", stride_, - ")::Dataset"); + return strings::StrCat("SlideDatasetOp(", window_size_, ", ", + window_shift_, ", ", window_stride_, ")::Dataset"); } protected: @@ -101,16 +109,18 @@ class SlideDatasetOp : public UnaryDatasetOpKernel { Node* input_graph_node = nullptr; TF_RETURN_IF_ERROR(b->AddParentDataset(ctx, input_, &input_graph_node)); Node* window_size = nullptr; - Node* stride = nullptr; + Node* window_shift = nullptr; + Node* window_stride = nullptr; TF_RETURN_IF_ERROR(b->AddScalar(window_size_, &window_size)); - TF_RETURN_IF_ERROR(b->AddScalar(stride_, &stride)); - TF_RETURN_IF_ERROR( - b->AddDataset(this, {input_graph_node, window_size, stride}, output)); + TF_RETURN_IF_ERROR(b->AddScalar(window_shift_, &window_shift)); + TF_RETURN_IF_ERROR(b->AddScalar(window_stride_, &window_stride)); + TF_RETURN_IF_ERROR(b->AddDataset( + this, {input_graph_node, window_size, window_shift, window_stride}, + output)); return Status::OK(); } private: - class Iterator : public DatasetIterator { public: explicit Iterator(const Params& params) @@ -124,7 +134,8 @@ class SlideDatasetOp : public UnaryDatasetOpKernel { std::vector* out_tensors, bool* end_of_sequence) override { const int64 window_size = dataset()->window_size_; - const int64 stride = dataset()->stride_; + const int64 window_shift = dataset()->window_shift_; + const int64 window_stride = dataset()->window_stride_; std::vector> batch_elements; { mutex_lock l(mu_); @@ -133,55 +144,51 @@ class SlideDatasetOp : public UnaryDatasetOpKernel { return Status::OK(); } batch_elements.reserve(window_size); - // Use cache if stride < window_size. - if (stride < window_size) { - const bool first_call = cache_.empty(); - if (first_call) { - cache_.reserve(window_size); - } else { - // Reuse cache in the previous iteration. - cache_.swap(batch_elements); - } - } - // Fill up with new elements. + + // Fill up buffer. + size_t target_size = TargetBufferSize(window_size, window_stride); *end_of_sequence = false; - for (size_t i = batch_elements.size(); i < window_size && !*end_of_sequence; - ++i) { - std::vector batch_element_tuple; - TF_RETURN_IF_ERROR(input_impl_->GetNext(ctx, &batch_element_tuple, - end_of_sequence)); + for (size_t i = buffer_.size(); i < target_size && !*end_of_sequence; + ++i) { + std::vector element; + TF_RETURN_IF_ERROR( + input_impl_->GetNext(ctx, &element, end_of_sequence)); if (!*end_of_sequence) { - batch_elements.push_back(std::move(batch_element_tuple)); + buffer_.push_back(std::move(element)); } else { input_impl_.reset(); } } - // Drop the final smaller blocks. - if (batch_elements.size() < window_size) { + + // Drop the final smaller batch. + if (buffer_.size() < target_size) { DCHECK(*end_of_sequence); return Status::OK(); } - if (stride < window_size) { - // Cache the data used for the next iteration. - for (size_t i = stride; i < window_size; ++i) { - cache_.emplace_back(batch_elements[i]); - } - } else if (stride > window_size) { - // Drop the data before the next iteration. - std::vector batch_element_tuple; - for (size_t i = window_size; i < stride && !*end_of_sequence; ++i) { - TF_RETURN_IF_ERROR(input_impl_->GetNext(ctx, &batch_element_tuple, - end_of_sequence)); - if (*end_of_sequence) { + for (size_t i = 0; i < window_size; ++i) { + batch_elements.emplace_back(buffer_[window_stride * i]); + } + + // Drop the data before the next iteration. + if (window_shift >= buffer_.size()) { + for (size_t i = buffer_.size(); i < window_shift; ++i) { + bool end_of_input; + std::vector element; + TF_RETURN_IF_ERROR( + input_impl_->GetNext(ctx, &element, &end_of_input)); + if (end_of_input) { input_impl_.reset(); + break; } } + buffer_.clear(); + } else { + buffer_.erase(buffer_.begin(), buffer_.begin() + window_shift); } } // Construct output tensors. - // Those codes below are copied from batch_dataset_op.cc. const size_t num_tuple_components = batch_elements[0].size(); const int64 num_batch_elements = batch_elements.size(); for (size_t component_index = 0; component_index < num_tuple_components; @@ -223,15 +230,15 @@ class SlideDatasetOp : public UnaryDatasetOpKernel { } else { TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_)); } - // Save cache. - TF_RETURN_IF_ERROR( - writer->WriteScalar(strings::StrCat("cache_size"), cache_.size())); - for (int64 i = 0; i < cache_.size(); i++) { + // Save buffer. + TF_RETURN_IF_ERROR(writer->WriteScalar(strings::StrCat("buffer_size"), + buffer_.size())); + for (int64 i = 0; i < buffer_.size(); i++) { TF_RETURN_IF_ERROR(writer->WriteScalar( - strings::StrCat("cache[", i, "]_size"), cache_[i].size())); - for (int64 j = 0; j < cache_[i].size(); j++) { + strings::StrCat("buffer[", i, "]_size"), buffer_[i].size())); + for (int64 j = 0; j < buffer_[i].size(); j++) { TF_RETURN_IF_ERROR(writer->WriteTensor( - strings::StrCat("cache[", i, "][", j, "]"), cache_[i][j])); + strings::StrCat("buffer[", i, "][", j, "]"), buffer_[i][j])); } } return Status::OK(); @@ -245,32 +252,37 @@ class SlideDatasetOp : public UnaryDatasetOpKernel { } else { input_impl_.reset(); } - // Restore cache. - int64 cache_size; + // Restore buffer. + int64 buffer_size; TF_RETURN_IF_ERROR( - reader->ReadScalar(strings::StrCat("cache_size"), &cache_size)); - cache_.resize(cache_size); - for (int64 i = 0; i < cache_size; i++) { + reader->ReadScalar(strings::StrCat("buffer_size"), &buffer_size)); + buffer_.resize(buffer_size); + for (int64 i = 0; i < buffer_size; i++) { int64 vector_size; TF_RETURN_IF_ERROR(reader->ReadScalar( - strings::StrCat("cache[", i, "]_size"), &vector_size)); - cache_[i].resize(vector_size); + strings::StrCat("buffer[", i, "]_size"), &vector_size)); + buffer_[i].resize(vector_size); for (int64 j = 0; j < vector_size; j++) { TF_RETURN_IF_ERROR(reader->ReadTensor( - strings::StrCat("cache[", i, "][", j, "]"), &cache_[i][j])); + strings::StrCat("buffer[", i, "][", j, "]"), &buffer_[i][j])); } } return Status::OK(); } private: + size_t TargetBufferSize(int64 window_size, int64 window_stride) { + return (window_size - 1) * window_stride + 1; + } + mutex mu_; - std::vector> cache_ GUARDED_BY(mu_); + std::deque> buffer_ GUARDED_BY(mu_); std::unique_ptr input_impl_ GUARDED_BY(mu_); }; const int64 window_size_; - const int64 stride_; + const int64 window_shift_; + const int64 window_stride_; const DatasetBase* const input_; std::vector output_shapes_; }; diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index be72ee8066..d94fa2cad7 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -58171,7 +58171,11 @@ op { type: DT_INT64 } input_arg { - name: "stride" + name: "window_shift" + type: DT_INT64 + } + input_arg { + name: "window_stride" type: DT_INT64 } output_arg { diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index c8bc11155a..8c83a09597 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -404,19 +404,20 @@ REGISTER_OP("BatchDatasetV2") return shape_inference::ScalarShape(c); }); -// TODO(mrry): move SlideDataset to contrib in the future. REGISTER_OP("SlideDataset") .Input("input_dataset: variant") .Input("window_size: int64") - .Input("stride: int64") + .Input("window_shift: int64") + .Input("window_stride: int64") .Output("handle: variant") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") .SetShapeFn([](shape_inference::InferenceContext* c) { shape_inference::ShapeHandle unused; - // window_size and stride should be scalars. + // window_size, window_shift, and window_stride should be scalars. TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused)); return shape_inference::ScalarShape(c); }); -- cgit v1.2.3 From 17bbfe25d0225f7d693384d4e0dcaa5f49a8c697 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 16 Jul 2018 11:04:35 -0700 Subject: Also clear FunctionLibraryRuntime when clearing tfe Context caches. Fixes flakiness in random seed test. PiperOrigin-RevId: 204771286 --- tensorflow/core/common_runtime/eager/context.cc | 7 ++++++- tensorflow/core/common_runtime/eager/context.h | 7 +++---- tensorflow/python/eager/function_test.py | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc index 70208fb6d1..074c311c27 100644 --- a/tensorflow/core/common_runtime/eager/context.cc +++ b/tensorflow/core/common_runtime/eager/context.cc @@ -34,7 +34,8 @@ EagerContext::EagerContext(const SessionOptions& opts, local_device_manager_.get(), opts.env, TF_GRAPH_DEF_VERSION, &func_lib_def_, {}, thread_pool_.get())), log_device_placement_(opts.config.log_device_placement()), - async_default_(async) { + async_default_(async), + env_(opts.env) { InitDeviceMapAndAsync(); } @@ -57,6 +58,7 @@ EagerContext::EagerContext( log_device_placement_(opts.config.log_device_placement()), async_default_(async), remote_device_manager_(std::move(remote_device_manager)), + env_(opts.env), server_(std::move(server)), remote_eager_workers_(std::move(remote_eager_workers)), remote_contexts_(remote_contexts) { @@ -109,6 +111,9 @@ Status EagerContext::SetAsyncForThread(bool async) { void EagerContext::ClearCaches() { mutex_lock ml(cache_mu_); gtl::STLDeleteValues(&kernel_cache_); + pflr_.reset(new ProcessFunctionLibraryRuntime( + local_device_manager_.get(), env_, TF_GRAPH_DEF_VERSION, &func_lib_def_, + {}, thread_pool_.get())); } void EagerContext::SetThreadLocalDevicePlacementPolicy( diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h index 864f514a19..458557c5b9 100644 --- a/tensorflow/core/common_runtime/eager/context.h +++ b/tensorflow/core/common_runtime/eager/context.h @@ -209,10 +209,7 @@ class EagerContext { std::unique_ptr thread_pool_; - // One FunctionLibraryRuntime per device. - // func_libs[i] is the FunctionLibraryRuntime corresponding to - // session->devices[i]. - const std::unique_ptr pflr_; + std::unique_ptr pflr_; mutex cache_mu_; std::unordered_map kernel_cache_ @@ -235,6 +232,8 @@ class EagerContext { const std::unique_ptr remote_device_manager_; + tensorflow::Env* const env_; + // The server_ is not const since we release it when the context is destroyed. // Therefore the server_ object is not marked as const (even though it should // be). diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index cdd9fe1760..13c4ee7f15 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -138,7 +138,7 @@ class FunctionTest(test.TestCase): out = sq_op(t) self.assertAllEqual(out, math_ops.matmul(t, t).numpy()) - def disabled_testRandomSeed(self): + def testRandomSeed(self): @function.defun def f(): -- cgit v1.2.3 From 374c5c6739f9487bf6006d046a3b5ad90c4f2d37 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 11:17:45 -0700 Subject: Update ops-related pbtxt files. PiperOrigin-RevId: 204773978 --- tensorflow/core/ops/ops.pbtxt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 76572061a4..4f24ab480f 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -27578,7 +27578,11 @@ op { type: DT_INT64 } input_arg { - name: "stride" + name: "window_shift" + type: DT_INT64 + } + input_arg { + name: "window_stride" type: DT_INT64 } output_arg { -- cgit v1.2.3 From c654856e386cbb1d379055a7ab0b4e30f3bb4ec2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 11:30:37 -0700 Subject: nit: fix wording PiperOrigin-RevId: 204776258 --- tensorflow/python/keras/layers/convolutional_recurrent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/keras/layers/convolutional_recurrent.py b/tensorflow/python/keras/layers/convolutional_recurrent.py index 84d794cada..e61dd3043d 100644 --- a/tensorflow/python/keras/layers/convolutional_recurrent.py +++ b/tensorflow/python/keras/layers/convolutional_recurrent.py @@ -788,7 +788,7 @@ class ConvLSTM2D(ConvRNN2D): Arguments: filters: Integer, the dimensionality of the output space - (i.e. the number output of filters in the convolution). + (i.e. the number of output filters in the convolution). kernel_size: An integer or tuple/list of n integers, specifying the dimensions of the convolution window. strides: An integer or tuple/list of n integers, -- cgit v1.2.3 From 7c505ddfe6cf32f82a1ea27f61c5f4fe93409098 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Mon, 16 Jul 2018 11:54:09 -0700 Subject: Add support for possible extra : in input names --- tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc index 5bb0ffc797..4957daae10 100644 --- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc +++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/core/grappler/clusters/cluster.h" #include "tensorflow/core/grappler/grappler_item.h" #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h" +#include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/logging.h" @@ -236,7 +237,16 @@ tensorflow::Status TRTOptimizationPass::Optimize( std::vector nodes_to_preserve; for (const auto& n : item.NodesToPreserve()) { auto tokens = str_util::Split(n, ":"); - nodes_to_preserve.push_back(tokens.at(0)); + string s=tokens.at(0); + for(int i=1;i 1 && !strings::safe_strto32(tokens.back(),&dumm_port)){ + StrAppend(&s,":",tokens.back()); + } + nodes_to_preserve.push_back(s); } cp.input_graph_def = &item.graph; cp.output_names = &nodes_to_preserve; -- cgit v1.2.3 From beb290b6b6bd38869cd1b0494ebb13c659bf4d30 Mon Sep 17 00:00:00 2001 From: Rasmus Munk Larsen Date: Mon, 16 Jul 2018 11:57:22 -0700 Subject: Update ctc_loss_util.h --- tensorflow/core/util/ctc/ctc_loss_util.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/util/ctc/ctc_loss_util.h b/tensorflow/core/util/ctc/ctc_loss_util.h index 3c77396881..50f8f49f1c 100644 --- a/tensorflow/core/util/ctc/ctc_loss_util.h +++ b/tensorflow/core/util/ctc/ctc_loss_util.h @@ -33,8 +33,8 @@ inline float LogSumExp(float log_prob_1, float log_prob_2) { // blowing up. if (log_prob_1 == kLogZero) { return log_prob_2; - } else if (log_prob_2 == kLogZero){ - return log_prob_1; + } else if (log_prob_2 == kLogZero) { + return log_prob_1; } else { return (log_prob_1 > log_prob_2) ? log_prob_1 + log1pf(expf(log_prob_2 - log_prob_1)) -- cgit v1.2.3 From 814f9ccd9b34a828f93d33eee6265e0cac07095a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 12:06:37 -0700 Subject: Use the safe sparse tensor API that returns errors rather than crashing in all TensorFlow core kernels. PiperOrigin-RevId: 204782675 --- .../boosted_trees/lib/utils/batch_features.cc | 17 ++++++--- .../lib/utils/examples_iterable_test.cc | 24 ++++++++---- tensorflow/core/kernels/ctc_loss_op.cc | 6 ++- .../kernels/data/sparse_tensor_slice_dataset_op.cc | 10 +++-- .../core/kernels/deserialize_sparse_string_op.cc | 5 ++- tensorflow/core/kernels/edit_distance_op.cc | 13 +++++-- tensorflow/core/kernels/reshape_util.cc | 1 - tensorflow/core/kernels/sdca_internal.cc | 1 + tensorflow/core/kernels/sdca_internal.h | 2 - tensorflow/core/kernels/serialize_sparse_op.cc | 6 ++- tensorflow/core/kernels/set_kernels.cc | 44 ++++++++++++---------- tensorflow/core/kernels/sparse_concat_op.cc | 9 +++-- tensorflow/core/kernels/sparse_reduce_op.cc | 12 ++++-- tensorflow/core/kernels/sparse_reorder_op.cc | 13 +++++-- tensorflow/core/kernels/sparse_slice_grad_op.cc | 1 - tensorflow/core/kernels/sparse_slice_op.cc | 7 +++- tensorflow/core/kernels/sparse_softmax_op.cc | 7 +++- tensorflow/core/kernels/sparse_split_op.cc | 14 +++++-- tensorflow/core/kernels/sparse_tensors_map_ops.cc | 36 ++++++++++++------ tensorflow/core/kernels/sparse_to_dense_op.cc | 6 ++- 20 files changed, 150 insertions(+), 84 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc index 35b059f349..4fab2b0b7d 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc @@ -16,6 +16,7 @@ #include "tensorflow/contrib/boosted_trees/lib/utils/batch_features.h" #include "tensorflow/contrib/boosted_trees/lib/utils/macros.h" #include "tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h" +#include "tensorflow/core/lib/core/errors.h" namespace tensorflow { namespace boosted_trees { @@ -96,9 +97,11 @@ Status BatchFeatures::Initialize( "Sparse float feature shape incompatible with batch size.")); auto tensor_shape = TensorShape({shape_flat(0), shape_flat(1)}); auto order_dims = sparse::SparseTensor::VarDimArray({0, 1}); - sparse_float_feature_columns_.emplace_back(sparse_float_feature_indices, - sparse_float_feature_values, - tensor_shape, order_dims); + sparse::SparseTensor sparse_tensor; + TF_RETURN_IF_ERROR(sparse::SparseTensor::Create( + sparse_float_feature_indices, sparse_float_feature_values, tensor_shape, + order_dims, &sparse_tensor)); + sparse_float_feature_columns_.push_back(std::move(sparse_tensor)); } // Read sparse int features. @@ -136,9 +139,11 @@ Status BatchFeatures::Initialize( "Sparse int feature shape incompatible with batch size.")); auto tensor_shape = TensorShape({shape_flat(0), shape_flat(1)}); auto order_dims = sparse::SparseTensor::VarDimArray({0, 1}); - sparse_int_feature_columns_.emplace_back(sparse_int_feature_indices, - sparse_int_feature_values, - tensor_shape, order_dims); + sparse::SparseTensor sparse_tensor; + TF_RETURN_IF_ERROR(sparse::SparseTensor::Create( + sparse_int_feature_indices, sparse_int_feature_values, tensor_shape, + order_dims, &sparse_tensor)); + sparse_int_feature_columns_.push_back(std::move(sparse_tensor)); } return Status::OK(); } diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc index d8a6088648..30c37435fe 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc @@ -43,27 +43,35 @@ TEST_F(ExamplesIterableTest, Iterate) { test::AsTensor({0, 0, 2, 0, 3, 0, 4, 0}, {4, 2}); auto sparse_float_values1 = test::AsTensor({-3.0f, 0.0f, 5.0f, 0.0f}); auto sparse_float_shape1 = TensorShape({8, 1}); - sparse::SparseTensor sparse_float_tensor1( - sparse_float_indices1, sparse_float_values1, sparse_float_shape1); + sparse::SparseTensor sparse_float_tensor1; + TF_ASSERT_OK( + sparse::SparseTensor::Create(sparse_float_indices1, sparse_float_values1, + sparse_float_shape1, &sparse_float_tensor1)); auto sparse_float_indices2 = test::AsTensor( {0, 1, 1, 0, 2, 1, 3, 0, 4, 1, 5, 0, 5, 1, 7, 0}, {8, 2}); auto sparse_float_values2 = test::AsTensor({1.f, 4.0f, 3.f, 7.0f, 4.3f, 9.0f, 0.8f, -4.0f}); auto sparse_float_shape2 = TensorShape({8, 2}); - sparse::SparseTensor sparse_float_tensor2( - sparse_float_indices2, sparse_float_values2, sparse_float_shape2); + sparse::SparseTensor sparse_float_tensor2; + TF_ASSERT_OK( + sparse::SparseTensor::Create(sparse_float_indices2, sparse_float_values2, + sparse_float_shape2, &sparse_float_tensor2)); auto sparse_int_indices1 = test::AsTensor({0, 0, 0, 1, 1, 0, 3, 0, 3, 1, 7, 0}, {6, 2}); auto sparse_int_values1 = test::AsTensor({1, 8, 0, 2, 0, 5}); auto sparse_int_shape1 = TensorShape({8, 2}); - sparse::SparseTensor sparse_int_tensor1( - sparse_int_indices1, sparse_int_values1, sparse_int_shape1); + sparse::SparseTensor sparse_int_tensor1; + TF_ASSERT_OK( + sparse::SparseTensor::Create(sparse_int_indices1, sparse_int_values1, + sparse_int_shape1, &sparse_int_tensor1)); auto sparse_int_indices2 = test::AsTensor({1, 0, 2, 0, 3, 0, 4, 0}, {4, 2}); auto sparse_int_values2 = test::AsTensor({7, 13, 4, 0}); auto sparse_int_shape2 = TensorShape({8, 1}); - sparse::SparseTensor sparse_int_tensor2( - sparse_int_indices2, sparse_int_values2, sparse_int_shape2); + sparse::SparseTensor sparse_int_tensor2; + TF_ASSERT_OK( + sparse::SparseTensor::Create(sparse_int_indices2, sparse_int_values2, + sparse_int_shape2, &sparse_int_tensor2)); auto validate_example_features = [](int64 example_idx, const Example& example) { diff --git a/tensorflow/core/kernels/ctc_loss_op.cc b/tensorflow/core/kernels/ctc_loss_op.cc index b38d838bf1..fb375ee4b3 100644 --- a/tensorflow/core/kernels/ctc_loss_op.cc +++ b/tensorflow/core/kernels/ctc_loss_op.cc @@ -100,8 +100,10 @@ class CTCLossOp : public OpKernel { TensorShape labels_shape({batch_size, max_label_len}); std::vector order{0, 1}; - sparse::SparseTensor labels_sp(*labels_indices, *labels_values, - labels_shape, order); + sparse::SparseTensor labels_sp; + OP_REQUIRES_OK( + ctx, sparse::SparseTensor::Create(*labels_indices, *labels_values, + labels_shape, order, &labels_sp)); Status labels_sp_valid = labels_sp.IndicesValid(); OP_REQUIRES(ctx, labels_sp_valid.ok(), diff --git a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc index 2604822cc9..b5dff48d2d 100644 --- a/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc +++ b/tensorflow/core/kernels/data/sparse_tensor_slice_dataset_op.cc @@ -252,10 +252,12 @@ class SparseTensorSliceDatasetOp : public DatasetOpKernel { previous_batch_index = next_batch_index; } gtl::InlinedVector std_order(dense_shape->NumElements(), 0); - sparse::SparseTensor sparse_tensor( - *indices, *values, TensorShape(dense_shape->vec()), std_order); - - *output = new Dataset(ctx, sparse_tensor); + sparse::SparseTensor tensor; + OP_REQUIRES_OK( + ctx, sparse::SparseTensor::Create( + *indices, *values, TensorShape(dense_shape->vec()), + std_order, &tensor)); + *output = new Dataset(ctx, std::move(tensor)); } private: diff --git a/tensorflow/core/kernels/deserialize_sparse_string_op.cc b/tensorflow/core/kernels/deserialize_sparse_string_op.cc index 6fb07c11e9..2c13f24ad6 100644 --- a/tensorflow/core/kernels/deserialize_sparse_string_op.cc +++ b/tensorflow/core/kernels/deserialize_sparse_string_op.cc @@ -165,7 +165,10 @@ class DeserializeSparseOp : public OpKernel { std::vector tensors; tensors.reserve(num_sparse_tensors); for (int i = 0; i < num_sparse_tensors; ++i) { - tensors.emplace_back(indices[i], values[i], shape, std_order); + SparseTensor tensor; + OP_REQUIRES_OK(context, SparseTensor::Create(indices[i], values[i], shape, + std_order, &tensor)); + tensors.push_back(std::move(tensor)); } gtl::optional maybe_output; diff --git a/tensorflow/core/kernels/edit_distance_op.cc b/tensorflow/core/kernels/edit_distance_op.cc index 20d857c721..4aecdc9e41 100644 --- a/tensorflow/core/kernels/edit_distance_op.cc +++ b/tensorflow/core/kernels/edit_distance_op.cc @@ -133,10 +133,15 @@ class EditDistanceOp : public OpKernel { std::vector sorted_order(truth_st_shape.dims()); std::iota(sorted_order.begin(), sorted_order.end(), 0); - sparse::SparseTensor hypothesis(*hypothesis_indices, *hypothesis_values, - hypothesis_st_shape, sorted_order); - sparse::SparseTensor truth(*truth_indices, *truth_values, truth_st_shape, - sorted_order); + sparse::SparseTensor hypothesis; + OP_REQUIRES_OK(ctx, sparse::SparseTensor::Create( + *hypothesis_indices, *hypothesis_values, + hypothesis_st_shape, sorted_order, &hypothesis)); + + sparse::SparseTensor truth; + OP_REQUIRES_OK(ctx, sparse::SparseTensor::Create( + *truth_indices, *truth_values, truth_st_shape, + sorted_order, &truth)); // Group dims 0, 1, ..., RANK - 1. The very last dim is assumed // to store the variable length sequences. diff --git a/tensorflow/core/kernels/reshape_util.cc b/tensorflow/core/kernels/reshape_util.cc index ac301f3342..50fdc17916 100644 --- a/tensorflow/core/kernels/reshape_util.cc +++ b/tensorflow/core/kernels/reshape_util.cc @@ -28,7 +28,6 @@ limitations under the License. #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" -#include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/sdca_internal.cc b/tensorflow/core/kernels/sdca_internal.cc index 3e16ba8d04..1c071d3d41 100644 --- a/tensorflow/core/kernels/sdca_internal.cc +++ b/tensorflow/core/kernels/sdca_internal.cc @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/core/kernels/sdca_internal.h" #include +#include #include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" diff --git a/tensorflow/core/kernels/sdca_internal.h b/tensorflow/core/kernels/sdca_internal.h index 897c488702..1eff4b15fa 100644 --- a/tensorflow/core/kernels/sdca_internal.h +++ b/tensorflow/core/kernels/sdca_internal.h @@ -43,8 +43,6 @@ limitations under the License. #include "tensorflow/core/lib/random/distribution_sampler.h" #include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/util/guarded_philox_random.h" -#include "tensorflow/core/util/sparse/group_iterator.h" -#include "tensorflow/core/util/sparse/sparse_tensor.h" #include "tensorflow/core/util/work_sharder.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/serialize_sparse_op.cc b/tensorflow/core/kernels/serialize_sparse_op.cc index 852cef29c7..577e327809 100644 --- a/tensorflow/core/kernels/serialize_sparse_op.cc +++ b/tensorflow/core/kernels/serialize_sparse_op.cc @@ -190,8 +190,10 @@ class SerializeManySparseOp : public SerializeManySparseOpBase { TensorShape tensor_input_shape(input_shape->vec()); gtl::InlinedVector std_order(rank); std::iota(std_order.begin(), std_order.end(), 0); - SparseTensor input_st(*input_indices, *input_values, tensor_input_shape, - std_order); + SparseTensor input_st; + OP_REQUIRES_OK(context, SparseTensor::Create(*input_indices, *input_values, + tensor_input_shape, std_order, + &input_st)); auto input_shape_t = input_shape->vec(); const int64 N = input_shape_t(0); diff --git a/tensorflow/core/kernels/set_kernels.cc b/tensorflow/core/kernels/set_kernels.cc index e836c764ac..f893d4e945 100644 --- a/tensorflow/core/kernels/set_kernels.cc +++ b/tensorflow/core/kernels/set_kernels.cc @@ -63,9 +63,9 @@ Status GroupShape(const VarDimArray& input_shape, ShapeArray* grouped_shape) { // Build `SparseTensor` from indices, values, and shape in inputs // [base_index, base_index + 3), and validate its rank and indices. -sparse::SparseTensor SparseTensorFromContext(OpKernelContext* ctx, - const int32 base_index, - bool validate_indices) { +Status SparseTensorFromContext(OpKernelContext* ctx, const int32 base_index, + bool validate_indices, + sparse::SparseTensor* tensor) { // Assume row-major order. const TensorShape shape = TensorShape(ctx->input(base_index + 2).vec()); @@ -73,13 +73,8 @@ sparse::SparseTensor SparseTensorFromContext(OpKernelContext* ctx, std::vector order(shape.dims()); std::iota(order.begin(), order.end(), 0); - const sparse::SparseTensor st(ctx->input(base_index), - ctx->input(base_index + 1), shape, order); - if (validate_indices) { - Status s = st.IndicesValid(); - if (!s.ok()) ctx->SetStatus(s); - } - return st; + return sparse::SparseTensor::Create( + ctx->input(base_index), ctx->input(base_index + 1), shape, order, tensor); } // TODO(ptucker): CheckGroup is just a sanity check on the result of @@ -253,11 +248,13 @@ class SetSizeOp : public OpKernel { template void SetSizeOp::Compute(OpKernelContext* ctx) { - const sparse::SparseTensor set_st = - SparseTensorFromContext(ctx, 0, validate_indices_); + sparse::SparseTensor set_st; + OP_REQUIRES_OK(ctx, + SparseTensorFromContext(ctx, 0, validate_indices_, &set_st)); + OP_REQUIRES_OK(ctx, set_st.IndicesValid()); - // Output shape is same as input except for last dimension, which reduces to - // the set size of values along that dimension. + // Output shape is same as input except for last dimension, which reduces + // to the set size of values along that dimension. ShapeArray output_shape; OP_REQUIRES_OK(ctx, GroupShape(set_st.shape(), &output_shape)); const auto output_strides = Strides(output_shape); @@ -484,8 +481,10 @@ void SetOperationOp::ComputeDenseToDense(OpKernelContext* ctx) const { template void SetOperationOp::ComputeDenseToSparse(OpKernelContext* ctx) const { const Tensor& set1_t = ctx->input(0); - const sparse::SparseTensor set2_st = - SparseTensorFromContext(ctx, 1, validate_indices_); + sparse::SparseTensor set2_st; + OP_REQUIRES_OK(ctx, + SparseTensorFromContext(ctx, 1, validate_indices_, &set2_st)); + OP_REQUIRES_OK(ctx, set2_st.IndicesValid()); // The following should stay in sync with `_dense_to_sparse_shape` shape // assertions in python/ops/set_ops.py, and `SetShapeFn` for // `DenseToSparseSetOperation` in ops/set_ops.cc. @@ -597,10 +596,15 @@ const std::vector GROUP_ITER_END; // with the same first n-1 dimensions in set1 and set2. template void SetOperationOp::ComputeSparseToSparse(OpKernelContext* ctx) const { - const sparse::SparseTensor set1_st = - SparseTensorFromContext(ctx, 0, validate_indices_); - const sparse::SparseTensor set2_st = - SparseTensorFromContext(ctx, 3, validate_indices_); + sparse::SparseTensor set1_st; + OP_REQUIRES_OK(ctx, + SparseTensorFromContext(ctx, 0, validate_indices_, &set1_st)); + OP_REQUIRES_OK(ctx, set1_st.IndicesValid()); + + sparse::SparseTensor set2_st; + OP_REQUIRES_OK(ctx, + SparseTensorFromContext(ctx, 3, validate_indices_, &set2_st)); + // The following should stay in sync with `_sparse_to_sparse_shape` shape // assertions in python/ops/set_ops.py, and `SetShapeFn` for // `SparseToSparseSetOperation` in ops/set_ops.cc. diff --git a/tensorflow/core/kernels/sparse_concat_op.cc b/tensorflow/core/kernels/sparse_concat_op.cc index f813794374..3b2a0cb0f3 100644 --- a/tensorflow/core/kernels/sparse_concat_op.cc +++ b/tensorflow/core/kernels/sparse_concat_op.cc @@ -124,9 +124,12 @@ class SparseConcatOp : public OpKernel { std::vector sp_inputs; for (int i = 0; i < N; ++i) { const TensorShape current_shape(shapes[i].vec()); - sp_inputs.emplace_back(tensor::DeepCopy(inds[i]), - tensor::DeepCopy(vals[i]), current_shape, - std_order); + sparse::SparseTensor tensor; + OP_REQUIRES_OK(context, + sparse::SparseTensor::Create( + tensor::DeepCopy(inds[i]), tensor::DeepCopy(vals[i]), + current_shape, std_order, &tensor)); + sp_inputs.push_back(std::move(tensor)); sp_inputs[i].Reorder(concat_order); } diff --git a/tensorflow/core/kernels/sparse_reduce_op.cc b/tensorflow/core/kernels/sparse_reduce_op.cc index 9e60791f97..a465564739 100644 --- a/tensorflow/core/kernels/sparse_reduce_op.cc +++ b/tensorflow/core/kernels/sparse_reduce_op.cc @@ -172,8 +172,10 @@ class SparseReduceOp : public OpKernel { // making deep copies here. Remove this if/when we change Reorder()'s // semantics. const auto shape_vec = shape_t->vec(); - SparseTensor sp(tensor::DeepCopy(*indices_t), tensor::DeepCopy(*values_t), - TensorShape(shape_vec)); + SparseTensor sp; + OP_REQUIRES_OK(ctx, SparseTensor::Create( + tensor::DeepCopy(*indices_t), tensor::DeepCopy(*values_t), + TensorShape(shape_vec), &sp)); ReduceDetails reduction = SparseTensorReduceHelper( sp, reduction_axes_t->flat(), keep_dims_); @@ -260,8 +262,10 @@ class SparseReduceSparseOp : public OpKernel { OP_REQUIRES_OK(ctx, ValidateInputs(shape_t, reduction_axes_t)); - SparseTensor sp(tensor::DeepCopy(*indices_t), tensor::DeepCopy(*values_t), - TensorShape(shape_t->vec())); + SparseTensor sp; + OP_REQUIRES_OK(ctx, SparseTensor::Create(tensor::DeepCopy(*indices_t), + tensor::DeepCopy(*values_t), + TensorShape(shape_t->vec()), &sp)); ReduceDetails reduction = SparseTensorReduceHelper( sp, reduction_axes_t->flat(), keep_dims_); diff --git a/tensorflow/core/kernels/sparse_reorder_op.cc b/tensorflow/core/kernels/sparse_reorder_op.cc index d1373fe0ef..6f9065827f 100644 --- a/tensorflow/core/kernels/sparse_reorder_op.cc +++ b/tensorflow/core/kernels/sparse_reorder_op.cc @@ -60,16 +60,21 @@ class SparseReorderOp : public OpKernel { std::iota(std_order.begin(), std_order.end(), 0); // Check if the sparse tensor is already ordered correctly - sparse::SparseTensor input_sp(input_ind, input_val, input_shape, std_order); + sparse::SparseTensor input_sp; + OP_REQUIRES_OK( + context, sparse::SparseTensor::Create(input_ind, input_val, input_shape, + std_order, &input_sp)); if (input_sp.IndicesValid().ok()) { context->set_output(0, input_sp.indices()); context->set_output(1, input_sp.values()); } else { // Deep-copy the input Tensors, then reorder in-place - sparse::SparseTensor reordered_sp(tensor::DeepCopy(input_ind), - tensor::DeepCopy(input_val), - input_shape); + sparse::SparseTensor reordered_sp; + OP_REQUIRES_OK(context, + sparse::SparseTensor::Create(tensor::DeepCopy(input_ind), + tensor::DeepCopy(input_val), + input_shape, &reordered_sp)); reordered_sp.Reorder(std_order); context->set_output(0, reordered_sp.indices()); context->set_output(1, reordered_sp.values()); diff --git a/tensorflow/core/kernels/sparse_slice_grad_op.cc b/tensorflow/core/kernels/sparse_slice_grad_op.cc index 90a39ed818..f92b6414ff 100644 --- a/tensorflow/core/kernels/sparse_slice_grad_op.cc +++ b/tensorflow/core/kernels/sparse_slice_grad_op.cc @@ -18,7 +18,6 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/util/sparse/sparse_tensor.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse_slice_op.cc b/tensorflow/core/kernels/sparse_slice_op.cc index 10dc208ab6..6aaf4fd88f 100644 --- a/tensorflow/core/kernels/sparse_slice_op.cc +++ b/tensorflow/core/kernels/sparse_slice_op.cc @@ -66,8 +66,11 @@ class SparseSliceOp : public OpKernel { "Expected size to be a vector of length ", input_dims, " but got length ", input_size.NumElements())); - sparse::SparseTensor sparse_tensor(input_indices, input_values, - TensorShape(input_shape.vec())); + sparse::SparseTensor sparse_tensor; + OP_REQUIRES_OK(context, + sparse::SparseTensor::Create( + input_indices, input_values, + TensorShape(input_shape.vec()), &sparse_tensor)); const gtl::ArraySlice start(input_start.flat().data(), input_dims); diff --git a/tensorflow/core/kernels/sparse_softmax_op.cc b/tensorflow/core/kernels/sparse_softmax_op.cc index 444a5f657a..dc3119bba4 100644 --- a/tensorflow/core/kernels/sparse_softmax_op.cc +++ b/tensorflow/core/kernels/sparse_softmax_op.cc @@ -69,8 +69,11 @@ class SparseSoftmaxOp : public OpKernel { const int nnz = static_cast(indices_t->dim_size(0)); const int rank = static_cast(indices_t->dim_size(1)); - SparseTensor st(tensor::DeepCopy(*indices_t), tensor::DeepCopy(*values_t), - TensorShape(shape_t->flat())); + SparseTensor st; + OP_REQUIRES_OK( + context, SparseTensor::Create( + tensor::DeepCopy(*indices_t), tensor::DeepCopy(*values_t), + TensorShape(shape_t->flat()), &st)); Tensor *output_values = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape({nnz}), diff --git a/tensorflow/core/kernels/sparse_split_op.cc b/tensorflow/core/kernels/sparse_split_op.cc index 67dcf05a6c..3d02be47cb 100644 --- a/tensorflow/core/kernels/sparse_split_op.cc +++ b/tensorflow/core/kernels/sparse_split_op.cc @@ -63,10 +63,16 @@ class SparseSplitOp : public OpKernel { input_shape.vec()(split_dim), "), got ", num_split_)); - sparse::SparseTensor sparse_tensor(input_indices, input_values, - TensorShape(input_shape.vec())); - const std::vector outputs = - sparse::SparseTensor::Split(sparse_tensor, split_dim, num_split_); + sparse::SparseTensor sparse_tensor; + OP_REQUIRES_OK(context, + sparse::SparseTensor::Create( + input_indices, input_values, + TensorShape(input_shape.vec()), &sparse_tensor)); + + std::vector outputs; + OP_REQUIRES_OK(context, + sparse::SparseTensor::Split(sparse_tensor, split_dim, + num_split_, &outputs)); for (int slice_index = 0; slice_index < num_split_; ++slice_index) { context->set_output(slice_index, outputs[slice_index].indices()); diff --git a/tensorflow/core/kernels/sparse_tensors_map_ops.cc b/tensorflow/core/kernels/sparse_tensors_map_ops.cc index 2aadd92475..74fa3a15f0 100644 --- a/tensorflow/core/kernels/sparse_tensors_map_ops.cc +++ b/tensorflow/core/kernels/sparse_tensors_map_ops.cc @@ -93,8 +93,9 @@ class SparseTensorsMap : public ResourceBase { const Tensor* ix = sp_iter->second.indices.AccessTensor(ctx); const Tensor* values = sp_iter->second.values.AccessTensor(ctx); const auto& shape = sp_iter->second.shape; - sparse_tensors->emplace_back(*ix, *values, shape); - + SparseTensor tensor; + TF_RETURN_IF_ERROR(SparseTensor::Create(*ix, *values, shape, &tensor)); + sparse_tensors->push_back(std::move(tensor)); sp_tensors_.erase(sp_iter); } } @@ -195,7 +196,9 @@ class AddSparseToTensorsMapOp : public SparseTensorAccessingOp { TensorShapeUtils::MakeShape(input_shape->vec().data(), input_shape->NumElements(), &input_shape_object)); - SparseTensor st(*input_indices, *input_values, input_shape_object); + SparseTensor st; + OP_REQUIRES_OK(context, SparseTensor::Create(*input_indices, *input_values, + input_shape_object, &st)); int64 handle; OP_REQUIRES_OK(context, map->AddSparseTensor(context, st, &handle)); @@ -253,8 +256,10 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { TensorShape tensor_input_shape(input_shape->vec()); gtl::InlinedVector std_order(rank); std::iota(std_order.begin(), std_order.end(), 0); - SparseTensor input_st(*input_indices, *input_values, tensor_input_shape, - std_order); + SparseTensor input_st; + OP_REQUIRES_OK(context, SparseTensor::Create(*input_indices, *input_values, + tensor_input_shape, std_order, + &input_st)); auto input_shape_t = input_shape->vec(); const int64 N = input_shape_t(0); @@ -300,7 +305,10 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { output_values_t(i) = values(i); } - SparseTensor st_i(output_indices, output_values, output_shape); + SparseTensor st_i; + OP_REQUIRES_OK(context, + SparseTensor::Create(output_indices, output_values, + output_shape, &st_i)); int64 handle; OP_REQUIRES_OK(context, map->AddSparseTensor(context, st_i, &handle)); sparse_handles_t(b) = handle; @@ -311,7 +319,9 @@ class AddManySparseToTensorsMapOp : public SparseTensorAccessingOp { if (visited.size() < N) { Tensor empty_indices(DT_INT64, {0, rank - 1}); Tensor empty_values(DataTypeToEnum::value, {0}); - SparseTensor empty_st(empty_indices, empty_values, output_shape); + SparseTensor empty_st; + OP_REQUIRES_OK(context, SparseTensor::Create(empty_indices, empty_values, + output_shape, &empty_st)); for (int64 b = 0; b < N; ++b) { // We skipped this batch entry. @@ -466,13 +476,15 @@ class TakeManySparseFromTensorsMapOp : public SparseTensorAccessingOp { std::vector tensors_to_concat; tensors_to_concat.reserve(N); for (int i = 0; i < N; ++i) { - tensors_to_concat.emplace_back(std::move(indices_to_concat[i]), - std::move(values_to_concat[i]), - preconcat_shape, std_order); + SparseTensor tensor; + OP_REQUIRES_OK(context, + SparseTensor::Create(std::move(indices_to_concat[i]), + std::move(values_to_concat[i]), + preconcat_shape, std_order, &tensor)); + tensors_to_concat.push_back(std::move(tensor)); } - SparseTensor output(SparseTensor::Concat(tensors_to_concat)); - + auto output = SparseTensor::Concat(tensors_to_concat); Tensor final_output_shape(DT_INT64, TensorShape({output.dims()})); std::copy_n(output.shape().data(), output.dims(), diff --git a/tensorflow/core/kernels/sparse_to_dense_op.cc b/tensorflow/core/kernels/sparse_to_dense_op.cc index ba3da21a43..f79a4d0494 100644 --- a/tensorflow/core/kernels/sparse_to_dense_op.cc +++ b/tensorflow/core/kernels/sparse_to_dense_op.cc @@ -119,8 +119,10 @@ class SparseToDense : public OpKernel { // Assume SparseTensor is lexicographically sorted. gtl::InlinedVector order(output->shape().dims()); std::iota(order.begin(), order.end(), 0); - sparse::SparseTensor st(indices_shaped, sparse_values_b, output->shape(), - order); + sparse::SparseTensor st; + OP_REQUIRES_OK(c, + sparse::SparseTensor::Create(indices_shaped, sparse_values_b, + output->shape(), order, &st)); if (validate_indices_) { OP_REQUIRES_OK(c, st.IndicesValid()); -- cgit v1.2.3 From 602b03d5fb521aef6561a5c131075d915907357e Mon Sep 17 00:00:00 2001 From: Paul Woitaschek Date: Mon, 16 Jul 2018 21:37:42 +0200 Subject: Update AndroidManifest.xml --- tensorflow/contrib/lite/java/AndroidManifest.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/java/AndroidManifest.xml b/tensorflow/contrib/lite/java/AndroidManifest.xml index f954bba739..c3849e6868 100644 --- a/tensorflow/contrib/lite/java/AndroidManifest.xml +++ b/tensorflow/contrib/lite/java/AndroidManifest.xml @@ -4,7 +4,7 @@ + android:targetSdkVersion="19" /> -- cgit v1.2.3 From 24764abb514c2664d8219700f9da7a6c3a8c6146 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Mon, 16 Jul 2018 12:44:41 -0700 Subject: Clang format, updated comment --- .../contrib/tensorrt/convert/trt_optimization_pass.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc index 4957daae10..b39c8bdd64 100644 --- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc +++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc @@ -237,14 +237,16 @@ tensorflow::Status TRTOptimizationPass::Optimize( std::vector nodes_to_preserve; for (const auto& n : item.NodesToPreserve()) { auto tokens = str_util::Split(n, ":"); - string s=tokens.at(0); - for(int i=1;i 1 && !strings::safe_strto32(tokens.back(),&dumm_port)){ - StrAppend(&s,":",tokens.back()); + int dumm_port = -1; + // If the last token is not an integer, it must be part of the name. + // Otherwise it is port number. + if (tokens.size() > 1 && + !strings::safe_strto32(tokens.back(), &dumm_port)) { + StrAppend(&s, ":", tokens.back()); } nodes_to_preserve.push_back(s); } -- cgit v1.2.3 From bf0f48d9bddc8f1f646190b1469d85cd1908ef6a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 12:53:05 -0700 Subject: [TF:XLA] Improve conditional_test by using parameters instead of constants. Before this change the conditional simplifier would fold away almost all conditional expressions. PiperOrigin-RevId: 204790426 --- tensorflow/compiler/xla/tests/conditional_test.cc | 104 +++++++++++++--------- 1 file changed, 63 insertions(+), 41 deletions(-) diff --git a/tensorflow/compiler/xla/tests/conditional_test.cc b/tensorflow/compiler/xla/tests/conditional_test.cc index 35f1400fb2..369663de15 100644 --- a/tensorflow/compiler/xla/tests/conditional_test.cc +++ b/tensorflow/compiler/xla/tests/conditional_test.cc @@ -172,88 +172,95 @@ class ConditionalOpTest : public ClientLibraryTestBase { // Test true and false computations that do not take any parameters. XLA_TEST_F(ConditionalOpTest, Parameters0) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, true); + XlaOp pred; + auto pred_arg = CreateR0Parameter(true, 0, "pred", &builder, &pred); auto operands = Tuple(&builder, {}); auto true_computation = CreateR0ConstantComputation(56.0f); auto false_computation = CreateR0ConstantComputation(12.0f); Conditional(pred, operands, true_computation, operands, false_computation); - ComputeAndCompareR0(&builder, 56.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 56.0f, {pred_arg.get()}, error_spec_); } // Test true and false computations that take in 1 parameter. XLA_TEST_F(ConditionalOpTest, Parameters1) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand1 = ConstantR0(&builder, 56.0f); auto operand2 = ConstantR0(&builder, 12.0f); auto identity = CreateR0IdentityComputation(); Conditional(pred, operand1, identity, operand2, identity); - ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 12.0f, {pred_arg.get()}, error_spec_); } // Test conditional with two different computations in the true and false cases // that take in different arguments. XLA_TEST_F(ConditionalOpTest, DiffComputationsDiffArgs) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand1 = ConstantR0(&builder, 56.4f); auto operand2 = ConstantR0(&builder, 12.6f); Conditional(pred, operand1, CreateR0CeilComputation(), operand2, CreateR0FloorComputation()); - ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 12.0f, {pred_arg.get()}, error_spec_); } // Test conditional with two different computations in the true and false cases // that take in the same arguments. XLA_TEST_F(ConditionalOpTest, DiffComputationsSameArg) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand = ConstantR0(&builder, 12.6f); Conditional(pred, operand, CreateR0CeilComputation(), operand, CreateR0FloorComputation()); - ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 12.0f, {pred_arg.get()}, error_spec_); } // Test conditional with the same computation in the true and false cases but // take in different arguments. XLA_TEST_F(ConditionalOpTest, SameComputationDiffArgs) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand1 = ConstantR0(&builder, 56.4f); auto operand2 = ConstantR0(&builder, 12.6f); auto floor = CreateR0FloorComputation(); Conditional(pred, operand1, floor, operand2, floor); - ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 12.0f, {pred_arg.get()}, error_spec_); } // Test conditional with the same computation in the true and false cases that // take in the same arguments. XLA_TEST_F(ConditionalOpTest, SameComputationSameArg) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand = ConstantR0(&builder, 12.6f); auto floor = CreateR0FloorComputation(); Conditional(pred, operand, floor, operand, floor); - ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 12.0f, {pred_arg.get()}, error_spec_); } // Test conditional with different instances of the same computation in the true // and false cases. XLA_TEST_F(ConditionalOpTest, SameComputationDiffInstances) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand1 = ConstantR0(&builder, 56.4f); auto operand2 = ConstantR0(&builder, 12.6f); Conditional(pred, operand1, CreateR0FloorComputation(), operand2, CreateR0FloorComputation()); - ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 12.0f, {pred_arg.get()}, error_spec_); } // Test the case when a call invokes a computation that contains a conditional. @@ -268,75 +275,83 @@ XLA_TEST_F(ConditionalOpTest, ConditionalWithCall) { auto inner_builder_result = inner_builder.Build(); XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand1 = ConstantR0(&builder, 56.4f); auto operand2 = ConstantR0(&builder, 12.6f); Call(&builder, inner_builder_result.ConsumeValueOrDie(), {pred, operand1, operand2}); - ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 12.0f, {pred_arg.get()}, error_spec_); } // Test true and false computations that take in 2 parameters and predicate is // true. XLA_TEST_F(ConditionalOpTest, Parameters2TrueBranch) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, true); + XlaOp pred; + auto pred_arg = CreateR0Parameter(true, 0, "pred", &builder, &pred); auto operand1 = ConstantR0(&builder, 56.0f); auto operand2 = ConstantR0(&builder, 12.0f); auto operands = Tuple(&builder, {operand1, operand2}); Conditional(pred, operands, CreateR0TupleAddComputation(), operands, CreateR0TupleSubComputation()); - ComputeAndCompareR0(&builder, 68.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 68.0f, {pred_arg.get()}, error_spec_); } // Test true and false computations that take in 2 parameters and predicate is // false. XLA_TEST_F(ConditionalOpTest, Parameters2FalseBranch) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand1 = ConstantR0(&builder, 56.0f); auto operand2 = ConstantR0(&builder, 12.0f); auto operands = Tuple(&builder, {operand1, operand2}); Conditional(pred, operands, CreateR0TupleAddComputation(), operands, CreateR0TupleSubComputation()); - ComputeAndCompareR0(&builder, 44.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 44.0f, {pred_arg.get()}, error_spec_); } // Test true and false computations that take in 2 array parameters and // predicate is true. XLA_TEST_F(ConditionalOpTest, Parameters2ArrayTrueBranch) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, true); + XlaOp pred; + auto pred_arg = CreateR0Parameter(true, 0, "pred", &builder, &pred); auto operand1 = ConstantR1(&builder, {24.0f, 56.0f}); auto operand2 = ConstantR1(&builder, {10.0f, 11.0f}); auto operands = Tuple(&builder, {operand1, operand2}); Conditional(pred, operands, CreateR1TupleAddComputation(), operands, CreateR1TupleSubComputation()); - ComputeAndCompareR1(&builder, {34.0f, 67.0f}, {}, error_spec_); + ComputeAndCompareR1(&builder, {34.0f, 67.0f}, {pred_arg.get()}, + error_spec_); } // Test true and false computations that take in 2 array parameters and // predicate is false. XLA_TEST_F(ConditionalOpTest, Parameters2ArrayFalseBranch) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand1 = ConstantR1(&builder, {24.0f, 56.0f}); auto operand2 = ConstantR1(&builder, {10.0f, 11.0f}); auto operands = Tuple(&builder, {operand1, operand2}); Conditional(pred, operands, CreateR1TupleAddComputation(), operands, CreateR1TupleSubComputation()); - ComputeAndCompareR1(&builder, {14.0f, 45.0f}, {}, error_spec_); + ComputeAndCompareR1(&builder, {14.0f, 45.0f}, {pred_arg.get()}, + error_spec_); } // Test true and false computations that return a tuple of scalars. XLA_TEST_F(ConditionalOpTest, ReturnTupleOfScalars) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operands = Tuple(&builder, {ConstantR0(&builder, 12.2f), ConstantR0(&builder, 25.6f)}); Conditional(pred, operands, CreateR0TupleCeilComputation(), operands, @@ -346,13 +361,14 @@ XLA_TEST_F(ConditionalOpTest, ReturnTupleOfScalars) { &builder, *LiteralUtil::MakeTuple({LiteralUtil::CreateR0(12.0f).get(), LiteralUtil::CreateR0(25.0f).get()}), - {}, error_spec_); + {pred_arg.get()}, error_spec_); } // Test true and false computations that return a tuple of arrays. XLA_TEST_F(ConditionalOpTest, ReturnTupleOfArrays) { XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, true); + XlaOp pred; + auto pred_arg = CreateR0Parameter(true, 0, "pred", &builder, &pred); auto operands = Tuple(&builder, {ConstantR1(&builder, {12.2f, 15.8f}), ConstantR1(&builder, {25.6f, 29.2f})}); @@ -364,7 +380,7 @@ XLA_TEST_F(ConditionalOpTest, ReturnTupleOfArrays) { *LiteralUtil::MakeTuple( {LiteralUtil::CreateR1({13.0f, 16.0f}).get(), LiteralUtil::CreateR1({26.0f, 30.0f}).get()}), - {}, error_spec_); + {pred_arg.get()}, error_spec_); } // Test true and false computations that return a tuple of a predicate, a @@ -393,7 +409,8 @@ XLA_TEST_F(ConditionalOpTest, ReturnTupleofPredicateScalarArray) { EXPECT_IS_OK(false_builder_result.status()); XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, true); + XlaOp pred; + auto pred_arg = CreateR0Parameter(true, 0, "pred", &builder, &pred); auto operands = Tuple(&builder, {}); Conditional(pred, operands, true_builder_result.ConsumeValueOrDie(), operands, false_builder_result.ConsumeValueOrDie()); @@ -404,7 +421,7 @@ XLA_TEST_F(ConditionalOpTest, ReturnTupleofPredicateScalarArray) { {LiteralUtil::CreateR0(true).get(), LiteralUtil::CreateR0(12.2f).get(), LiteralUtil::CreateR1({12.8f, 14.6f}).get()}), - {}, error_spec_); + {pred_arg.get()}, error_spec_); } // Test true and false computations that return a nested tuple. @@ -438,7 +455,8 @@ XLA_TEST_F(ConditionalOpTest, ReturnNestedTuple) { EXPECT_IS_OK(false_builder_result.status()); XlaBuilder builder(TestName()); - auto pred = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operands = Tuple(&builder, {}); Conditional(pred, operands, true_builder_result.ConsumeValueOrDie(), operands, false_builder_result.ConsumeValueOrDie()); @@ -454,7 +472,7 @@ XLA_TEST_F(ConditionalOpTest, ReturnNestedTuple) { {LiteralUtil::CreateR1({62.1f, 67.4f}).get(), LiteralUtil::CreateR0(9.3f).get()}) .get()}), - {}, error_spec_); + {pred_arg.get()}, error_spec_); } // Test conditional that takes in scalar operands in the form of external @@ -515,8 +533,9 @@ XLA_TEST_F(ConditionalOpTest, NestedConditionals) { EXPECT_IS_OK(inner_builder_result.status()); XlaBuilder builder(TestName()); - auto pred1 = ConstantR0(&builder, true); - auto pred2 = ConstantR0(&builder, false); + XlaOp pred1, pred2; + auto pred1_arg = CreateR0Parameter(true, 0, "pred1", &builder, &pred1); + auto pred2_arg = CreateR0Parameter(false, 1, "pred2", &builder, &pred2); auto operand1 = ConstantR0(&builder, 1.1f); auto operand2 = ConstantR0(&builder, 12.2f); auto operand3 = ConstantR0(&builder, 43.3f); @@ -524,7 +543,8 @@ XLA_TEST_F(ConditionalOpTest, NestedConditionals) { Conditional(pred1, tuple_operand, inner_builder_result.ConsumeValueOrDie(), operand3, CreateR0IdentityComputation()); - ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 12.0f, + {pred1_arg.get(), pred2_arg.get()}, error_spec_); } XLA_TEST_F(ConditionalOpTest, ConditionalInNestedComputation) { @@ -543,13 +563,14 @@ XLA_TEST_F(ConditionalOpTest, ConditionalInNestedComputation) { EXPECT_IS_OK(inner_builder_result.status()); XlaBuilder builder(TestName()); - auto pred2 = ConstantR0(&builder, false); + XlaOp pred; + auto pred_arg = CreateR0Parameter(false, 0, "pred", &builder, &pred); auto operand1 = ConstantR0(&builder, 1.1f); auto operand2 = ConstantR0(&builder, 12.2f); - auto tuple_operand = Tuple(&builder, {pred2, operand1, operand2}); + auto tuple_operand = Tuple(&builder, {pred, operand1, operand2}); Call(&builder, inner_builder_result.ConsumeValueOrDie(), {tuple_operand}); - ComputeAndCompareR0(&builder, 12.0f, {}, error_spec_); + ComputeAndCompareR0(&builder, 12.0f, {pred_arg.get()}, error_spec_); } // Test a mismatch in the shape of the true operand and true computation. @@ -604,8 +625,9 @@ XLA_TEST_F(ConditionalOpTest, SwappedInputsInSequentialConditionals) { auto test_swap = [&](float a, float b) { XlaBuilder builder(TestName()); - auto x = ConstantR0(&builder, a); - auto y = ConstantR0(&builder, b); + XlaOp x, y; + auto x_arg = CreateR0Parameter(a, 0, "x", &builder, &x); + auto y_arg = CreateR0Parameter(b, 1, "y", &builder, &y); auto tuple_operand = Tuple(&builder, {x, y}); Call(&builder, main, {tuple_operand}); @@ -613,7 +635,7 @@ XLA_TEST_F(ConditionalOpTest, SwappedInputsInSequentialConditionals) { &builder, *LiteralUtil::MakeTuple({LiteralUtil::CreateR0(a).get(), LiteralUtil::CreateR0(b).get()}), - {}, error_spec_); + {x_arg.get(), y_arg.get()}, error_spec_); }; test_swap(3.11f, 9.4f); -- cgit v1.2.3 From 094474edae3438fb06f52ca1ab3a5083d3c0160f Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 16 Jul 2018 18:50:43 +0000 Subject: Improvement of shape function in HistogramFixedWidth In the HistogramFixedWidth op, there are restrictions over the shape of range_value and nbins. The range_value should be a vector of 2 elements and nbins should be a scalar. This fix adds the restriction to shape function of HistogramFixedWidth. Signed-off-by: Yong Tang --- tensorflow/core/ops/math_ops.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index c229bd5a41..542bdad914 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -1380,6 +1380,14 @@ REGISTER_OP("HistogramFixedWidth") .Attr("T: {int32, int64, float32, float64}") .Attr("dtype: {int32, int64} = DT_INT32") .SetShapeFn([](InferenceContext* c) { + // value_range should be a vector. + ShapeHandle value_range_shape; + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &value_range_shape)); + // value_range should have two elements. + DimensionHandle unused; + TF_RETURN_IF_ERROR(c->WithValue(c->Dim(value_range_shape, 0), 2, &unused)); + + // If nbins is available, set the shape from nbins. const Tensor* nbins_input = c->input_tensor(2); if (nbins_input != nullptr) { int64 nbins; -- cgit v1.2.3 From 767c0eb9f011529c2e2bf43d82d98e15d276861e Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 16 Jul 2018 18:52:20 +0000 Subject: Add test cases for nbins/range_value for shape function of HistogramFixedWidth Signed-off-by: Yong Tang --- tensorflow/python/ops/histogram_ops_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py index a226ac81bb..7c1fda8e4f 100644 --- a/tensorflow/python/ops/histogram_ops_test.py +++ b/tensorflow/python/ops/histogram_ops_test.py @@ -84,6 +84,18 @@ class HistogramFixedWidthTest(test.TestCase): def setUp(self): self.rng = np.random.RandomState(0) + def test_with_invalid_value_range(self): + values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] + with self.assertRaisesRegexp(ValueError, "Shape must be rank 1 but is rank 0"): + histogram_ops.histogram_fixed_width(values, 1.0) + with self.assertRaisesRegexp(ValueError, "Dimension must be 2 but is 3"): + histogram_ops.histogram_fixed_width(values, [1.0, 2.0, 3.0]) + + def test_with_invalid_nbins(self): + values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] + with self.assertRaisesRegexp(ValueError, "Input must be scalar but has rank 1"): + histogram_ops.histogram_fixed_width(values, [1.0, 5.0], nbins=[1, 2]) + def test_empty_input_gives_all_zero_counts(self): # Bins will be: # (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf) -- cgit v1.2.3 From d27f39654665473878e8a5876e02350d90c664f6 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 16 Jul 2018 20:00:05 +0000 Subject: Pylint and clang-format fix Signed-off-by: Yong Tang --- tensorflow/core/ops/math_ops.cc | 3 ++- tensorflow/python/ops/histogram_ops_test.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 542bdad914..589b542b43 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -1385,7 +1385,8 @@ REGISTER_OP("HistogramFixedWidth") TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &value_range_shape)); // value_range should have two elements. DimensionHandle unused; - TF_RETURN_IF_ERROR(c->WithValue(c->Dim(value_range_shape, 0), 2, &unused)); + TF_RETURN_IF_ERROR( + c->WithValue(c->Dim(value_range_shape, 0), 2, &unused)); // If nbins is available, set the shape from nbins. const Tensor* nbins_input = c->input_tensor(2); diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py index 7c1fda8e4f..0f8fe32745 100644 --- a/tensorflow/python/ops/histogram_ops_test.py +++ b/tensorflow/python/ops/histogram_ops_test.py @@ -86,14 +86,16 @@ class HistogramFixedWidthTest(test.TestCase): def test_with_invalid_value_range(self): values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] - with self.assertRaisesRegexp(ValueError, "Shape must be rank 1 but is rank 0"): + with self.assertRaisesRegexp( + ValueError, "Shape must be rank 1 but is rank 0"): histogram_ops.histogram_fixed_width(values, 1.0) with self.assertRaisesRegexp(ValueError, "Dimension must be 2 but is 3"): histogram_ops.histogram_fixed_width(values, [1.0, 2.0, 3.0]) def test_with_invalid_nbins(self): values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] - with self.assertRaisesRegexp(ValueError, "Input must be scalar but has rank 1"): + with self.assertRaisesRegexp( + ValueError, "Input must be scalar but has rank 1"): histogram_ops.histogram_fixed_width(values, [1.0, 5.0], nbins=[1, 2]) def test_empty_input_gives_all_zero_counts(self): -- cgit v1.2.3 From b54ded72a83f5df267380d64811b70b20afcbc4a Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 16 Jul 2018 20:09:46 +0000 Subject: Add additional check that nbins has to be > 0 Signed-off-by: Yong Tang --- tensorflow/core/ops/math_ops.cc | 4 ++++ tensorflow/python/ops/histogram_ops_test.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 589b542b43..cd5e0e81c3 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -1393,6 +1393,10 @@ REGISTER_OP("HistogramFixedWidth") if (nbins_input != nullptr) { int64 nbins; TF_RETURN_IF_ERROR(c->GetScalarFromTensor(nbins_input, &nbins)); + // nbins has to be positive. + if (nbins <= 0) { + return errors::InvalidArgument("Requires nbins > 0: ", nbins); + } c->set_output(0, c->Vector(nbins)); } else { c->set_output(0, c->UnknownShapeOfRank(1)); diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py index 0f8fe32745..06a0d9ac69 100644 --- a/tensorflow/python/ops/histogram_ops_test.py +++ b/tensorflow/python/ops/histogram_ops_test.py @@ -97,6 +97,9 @@ class HistogramFixedWidthTest(test.TestCase): with self.assertRaisesRegexp( ValueError, "Input must be scalar but has rank 1"): histogram_ops.histogram_fixed_width(values, [1.0, 5.0], nbins=[1, 2]) + with self.assertRaisesRegexp( + ValueError, "Requires nbins > 0"): + histogram_ops.histogram_fixed_width(values, [1.0, 5.0], nbins=-5) def test_empty_input_gives_all_zero_counts(self): # Bins will be: -- cgit v1.2.3 From e224e2d59ba48890f4b8ff0431ba06b25bf91c72 Mon Sep 17 00:00:00 2001 From: Billy Lamberta Date: Mon, 16 Jul 2018 13:08:04 -0700 Subject: Add note about colab notebooks on install page. Remove some gerunds. PiperOrigin-RevId: 204792915 --- tensorflow/docs_src/install/index.md | 31 ++++++++++++++----------- tensorflow/docs_src/install/install_c.md | 2 +- tensorflow/docs_src/install/install_go.md | 2 +- tensorflow/docs_src/install/install_java.md | 2 +- tensorflow/docs_src/install/install_linux.md | 2 +- tensorflow/docs_src/install/install_mac.md | 2 +- tensorflow/docs_src/install/install_raspbian.md | 2 +- tensorflow/docs_src/install/install_sources.md | 2 +- tensorflow/docs_src/install/install_windows.md | 2 +- tensorflow/docs_src/install/migration.md | 3 +-- 10 files changed, 26 insertions(+), 24 deletions(-) diff --git a/tensorflow/docs_src/install/index.md b/tensorflow/docs_src/install/index.md index c2e5a991d4..55481cc400 100644 --- a/tensorflow/docs_src/install/index.md +++ b/tensorflow/docs_src/install/index.md @@ -1,36 +1,39 @@ -# Installing TensorFlow +# Install TensorFlow -We've built and tested TensorFlow on the following 64-bit laptop/desktop -operating systems: +Note: Run the [TensorFlow tutorials](../tutorials) in a pre-configured +[Colab notebook environment](https://colab.research.google.com/notebooks/welcome.ipynb){: .external}, +without installation. + +TensorFlow is built and tested on the following 64-bit operating systems: * macOS 10.12.6 (Sierra) or later. * Ubuntu 16.04 or later * Windows 7 or later. * Raspbian 9.0 or later. -Although you might be able to install TensorFlow on other laptop or desktop -systems, we only support (and only fix issues in) the preceding configurations. +While TensorFlow may work on other systems, we only support—and fix issues in—the +systems listed above. The following guides explain how to install a version of TensorFlow that enables you to write applications in Python: - * @{$install_linux$Installing TensorFlow on Ubuntu} - * @{$install_mac$Installing TensorFlow on macOS} - * @{$install_windows$Installing TensorFlow on Windows} - * @{$install_raspbian$Installing TensorFlow on a Raspberry Pi} - * @{$install_sources$Installing TensorFlow from Sources} + * @{$install_linux$Install TensorFlow on Ubuntu} + * @{$install_mac$Install TensorFlow on macOS} + * @{$install_windows$Install TensorFlow on Windows} + * @{$install_raspbian$Install TensorFlow on a Raspberry Pi} + * @{$install_sources$Install TensorFlow from source code} Many aspects of the Python TensorFlow API changed from version 0.n to 1.0. The following guide explains how to migrate older TensorFlow applications to Version 1.0: - * @{$migration$Transitioning to TensorFlow 1.0} + * @{$migration$Transition to TensorFlow 1.0} The following guides explain how to install TensorFlow libraries for use in other programming languages. These APIs are aimed at deploying TensorFlow models in applications and are not as extensive as the Python APIs. - * @{$install_java$Installing TensorFlow for Java} - * @{$install_c$Installing TensorFlow for C} - * @{$install_go$Installing TensorFlow for Go} + * @{$install_java$Install TensorFlow for Java} + * @{$install_c$Install TensorFlow for C} + * @{$install_go$Install TensorFlow for Go} diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md index 2901848745..4e1c32f972 100644 --- a/tensorflow/docs_src/install/install_c.md +++ b/tensorflow/docs_src/install/install_c.md @@ -1,4 +1,4 @@ -# Installing TensorFlow for C +# Install TensorFlow for C TensorFlow provides a C API defined in [`c_api.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/c/c_api.h), diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index 2c126df5aa..162a820f22 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -1,4 +1,4 @@ -# Installing TensorFlow for Go +# Install TensorFlow for Go TensorFlow provides APIs for use in Go programs. These APIs are particularly well-suited to loading models created in Python and executing them within diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index 692dfc9cef..c196bb9b31 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -1,4 +1,4 @@ -# Installing TensorFlow for Java +# Install TensorFlow for Java TensorFlow provides APIs for use in Java programs. These APIs are particularly well-suited to loading models created in Python and executing them within a diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 541a55e184..8ff8fa6def 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -1,4 +1,4 @@ -# Installing TensorFlow on Ubuntu +# Install TensorFlow on Ubuntu This guide explains how to install TensorFlow on Ubuntu Linux. While these instructions may work on other Linux variants, they are tested and supported with diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index c6f0c17924..3372e9e1e0 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -1,4 +1,4 @@ -# Installing TensorFlow on macOS +# Install TensorFlow on macOS This guide explains how to install TensorFlow on macOS. Although these instructions might also work on other macOS variants, we have only diff --git a/tensorflow/docs_src/install/install_raspbian.md b/tensorflow/docs_src/install/install_raspbian.md index 46c4944ca7..58a5285c78 100644 --- a/tensorflow/docs_src/install/install_raspbian.md +++ b/tensorflow/docs_src/install/install_raspbian.md @@ -1,4 +1,4 @@ -# Installing TensorFlow on Raspbian +# Install TensorFlow on Raspbian This guide explains how to install TensorFlow on a Raspberry Pi running Raspbian. Although these instructions might also work on other Pi variants, we diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index 5caf36eed1..edaa855aa2 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -1,4 +1,4 @@ -# Installing TensorFlow from Sources +# Install TensorFlow from Sources This guide explains how to build TensorFlow sources into a TensorFlow binary and how to install that TensorFlow binary. Note that we provide diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md index 7b7b17ce81..e9061bf3c1 100644 --- a/tensorflow/docs_src/install/install_windows.md +++ b/tensorflow/docs_src/install/install_windows.md @@ -1,4 +1,4 @@ -# Installing TensorFlow on Windows +# Install TensorFlow on Windows This guide explains how to install TensorFlow on Windows. Although these instructions might also work on other Windows variants, we have only diff --git a/tensorflow/docs_src/install/migration.md b/tensorflow/docs_src/install/migration.md index d6c31f96bd..19315ace2d 100644 --- a/tensorflow/docs_src/install/migration.md +++ b/tensorflow/docs_src/install/migration.md @@ -1,5 +1,4 @@ - -# Transitioning to TensorFlow 1.0 +# Transition to TensorFlow 1.0 The APIs in TensorFlow 1.0 have changed in ways that are not all backwards -- cgit v1.2.3 From 053443589925adea4e24aeb589f668df5bbc823d Mon Sep 17 00:00:00 2001 From: Nathan Luehr Date: Thu, 12 Jul 2018 11:50:16 -0500 Subject: Reformat with proper code style. --- tensorflow/core/kernels/crop_and_resize_op.cc | 55 ++- .../core/kernels/crop_resize_bilinear_core.h | 483 +++++++++++---------- tensorflow/core/kernels/resize_bilinear_op.cc | 20 +- 3 files changed, 299 insertions(+), 259 deletions(-) diff --git a/tensorflow/core/kernels/crop_and_resize_op.cc b/tensorflow/core/kernels/crop_and_resize_op.cc index de8274db89..17d7785f47 100644 --- a/tensorflow/core/kernels/crop_and_resize_op.cc +++ b/tensorflow/core/kernels/crop_and_resize_op.cc @@ -22,18 +22,18 @@ limitations under the License. #include #include -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/bounds_check.h" +#include "tensorflow/core/kernels/crop_resize_bilinear_core.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/work_sharder.h" -#include "tensorflow/core/kernels/crop_resize_bilinear_core.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #if GOOGLE_CUDA #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h" @@ -43,6 +43,10 @@ limitations under the License. using stream_executor::cuda::ScopedActivateExecutorContext; #endif // GOOGLE_CUDA +using ::tensorflow::internal::CachedInterpolation; +using ::tensorflow::internal::compute_interpolation_weights; +using ::tensorflow::internal::crop_resize_single_image; + namespace tensorflow { namespace { @@ -250,27 +254,32 @@ struct CropAndResize { continue; } if (method_name == "bilinear") { - CachedInterpolation *interp_x=0l, *interp_y=0l; - int min_ix, max_ix, min_iy, max_iy; - compute_interpolation_weights(crop_width,image_width,x1,x2,min_ix,max_ix,interp_x); - compute_interpolation_weights(crop_height,image_height,y1,y2,min_iy,max_iy,interp_y); - - // multiply by depth to avoid multiplication in resize_single_image. - for (int i = min_ix; i <= max_ix; ++i) { - interp_x[i-min_ix].lower *= depth; - interp_x[i-min_ix].upper *= depth; - } - - crop_resize_single_image( - image.data() + (int64)b_in * (int64)image_height * (int64)image_width * (int64)depth, - image_height,image_width,crop_height,crop_width,depth, - min_ix,max_ix,interp_x, - min_iy,max_iy,interp_y, - extrapolation_value,false,false, - crops.data() + (int64)b * (int64)crop_height * (int64)crop_width * (int64)depth); - - delete [] interp_y; - delete [] interp_x; + CachedInterpolation *interp_x = 0l, *interp_y = 0l; + int min_ix, max_ix, min_iy, max_iy; + compute_interpolation_weights(crop_width, image_width, x1, x2, + min_ix, max_ix, interp_x); + compute_interpolation_weights(crop_height, image_height, y1, y2, + min_iy, max_iy, interp_y); + + // multiply by depth to avoid multiplication in resize_single_image. + for (int i = min_ix; i <= max_ix; ++i) { + interp_x[i - min_ix].lower *= depth; + interp_x[i - min_ix].upper *= depth; + } + + crop_resize_single_image( + image.data() + + (int64)b_in * (int64)image_height * (int64)image_width * + (int64)depth, + image_height, image_width, crop_height, crop_width, depth, + min_ix, max_ix, interp_x, min_iy, max_iy, interp_y, + extrapolation_value, false, false, + crops.data() + + (int64)b * (int64)crop_height * (int64)crop_width * + (int64)depth); + + delete[] interp_y; + delete[] interp_x; } else { // method == "nearest" for (int x = 0; x < crop_width; ++x) { const float in_x = (crop_width > 1) diff --git a/tensorflow/core/kernels/crop_resize_bilinear_core.h b/tensorflow/core/kernels/crop_resize_bilinear_core.h index 5f707c6296..3e319d753a 100644 --- a/tensorflow/core/kernels/crop_resize_bilinear_core.h +++ b/tensorflow/core/kernels/crop_resize_bilinear_core.h @@ -17,7 +17,7 @@ limitations under the License. #define TENSORFLOW_CORE_KERNELS_CROP_RESIZE_BILINEAR_CORE_H_ namespace tensorflow { -namespace { +namespace internal { // Compute the interpolation indices only once. struct CachedInterpolation { int lower; // Lower source index used in the interpolation @@ -28,47 +28,45 @@ struct CachedInterpolation { }; inline bool compute_single_interpolation_weight( - const int in_size, - const float out2in_scale, - const float out2in_start, - const bool clip, - const int i, - int& lower, - int& upper, - float& lerp) { + const int in_size, const float out2in_scale, const float out2in_start, + const bool clip, const int i, int* lower, int* upper, float* lerp) { const float in = i * out2in_scale + out2in_start; - lower = (int)floor(in); - upper = (int)ceil(in); - lerp = (float)(in - (float)lower); + *lower = (int)floor(in); + *upper = (int)ceil(in); + *lerp = (float)(in - (float)*lower); if (clip) { - if (lower < 0) lower = 0; - else if (lower >= in_size) lower = in_size - 1; - if (upper < 0) upper = 0; - else if (upper >= in_size) upper = in_size - 1; + if (*lower < 0) + *lower = 0; + else if (*lower >= in_size) + *lower = in_size - 1; + if (*upper < 0) + *upper = 0; + else if (*upper >= in_size) + *upper = in_size - 1; return true; } else { - return (lower >= 0 && upper < in_size) ? true : false; + return (*lower >= 0 && *upper < in_size) ? true : false; } } /** - * Compute interpolation values for output indexes in range [out_start,out_start+out_size-1]. - * Returns true if all output indexes have lower and upper (input) indexes within range [0,in_size-1]. + * Compute interpolation values for output indexes in range + * [out_start,out_start+out_size-1]. + * Returns true if all output indexes have lower and upper (input) indexes + * within range [0,in_size-1]. */ -inline bool compute_interpolation_weights( - const int min_i, - const int max_i, - const int in_size, - const float out2in_scale, - const float out2in_start, - const bool clip, - CachedInterpolation* interpolation) { +inline bool compute_interpolation_weights(const int min_i, const int max_i, + const int in_size, + const float out2in_scale, + const float out2in_start, + const bool clip, + CachedInterpolation* interpolation) { bool rval = true; int num_i = max_i - min_i + 1; - for (int i = 0; i < num_i; ++i) { + for (int i = 0; i < num_i; ++i) { if (!compute_single_interpolation_weight( - in_size,out2in_scale,out2in_start,clip, - i+min_i, - interpolation[i].lower,interpolation[i].upper,interpolation[i].lerp)) { + in_size, out2in_scale, out2in_start, clip, i + min_i, + &interpolation[i].lower, &interpolation[i].upper, + &interpolation[i].lerp)) { rval = false; } } @@ -77,37 +75,38 @@ inline bool compute_interpolation_weights( /** * Compatibility method for resize_bilinear_op.cc */ -inline void compute_interpolation_weights( - const int out_size, - const int in_size, - const float out2in_scale, - CachedInterpolation* interpolation) { +inline void compute_interpolation_weights(const int out_size, const int in_size, + const float out2in_scale, + CachedInterpolation* interpolation) { interpolation[out_size].lower = 0; interpolation[out_size].upper = 0; const bool clip = true; - if (!compute_interpolation_weights(0,out_size-1,in_size,out2in_scale,0.0f,clip,interpolation)) { + if (!compute_interpolation_weights(0, out_size - 1, in_size, out2in_scale, + 0.0f, clip, interpolation)) { // Should never happen, check for it anyway - printf("Warning! Interpolation values have lower,upper indexes outside of range [0,in_size-1]\n"); + printf( + "Warning! Interpolation values have lower,upper indexes outside of " + "range [0,in_size-1]\n"); } } /** - * Compute minimum and maximum (output) i where both lower and upper (input) is in range [0,in_size-1] - * If no values of i satisfy condition, min_i = in_size, max_i = -1 and method returns false. + * Compute minimum and maximum (output) i where both lower and upper (input) is + * in range [0,in_size-1] + * If no values of i satisfy condition, min_i = in_size, max_i = -1 and method + * returns false. * Returns true if min_i >= max_i. */ -inline bool compute_minmax_indexes( - const int out_size, - const int in_size, - const float out2in_scale, - const float out2in_start, - int& min_i, - int& max_i) { +inline bool compute_minmax_indexes(const int out_size, const int in_size, + const float out2in_scale, + const float out2in_start, int& min_i, + int& max_i) { min_i = out_size; max_i = -1; int lower, upper; float lerp; - for (int i = 0; i < out_size; ++i) { - if (compute_single_interpolation_weight(in_size,out2in_scale,out2in_start,false,i,lower,upper,lerp)) { + for (int i = 0; i < out_size; ++i) { + if (compute_single_interpolation_weight(in_size, out2in_scale, out2in_start, + false, i, &lower, &upper, &lerp)) { if (i < min_i) min_i = i; if (i > max_i) max_i = i; } @@ -120,21 +119,29 @@ inline bool compute_minmax_indexes( * Returns true if at least one point requires interpolation, false otherwise. */ inline bool compute_interpolation_weights( - const int out_size, - const int in_size, - const float x1, // lower bounding box, crop region starts at in_size*x1 - const float x2, // upper bounding box, crop region ends at in_size*x2 - int& min_i, - int& max_i, - CachedInterpolation*& interpolation) { - float out2in_start = out_size > 1 ? (float)(in_size-1) * (float)x1 : (float)(in_size-1) * (float)(x1 + x2) / 2.0f; - float out2in_scale = out_size > 1 ? (float)(x2-x1) * (float)(in_size-1) / (float)(out_size-1) : 0.0f; - if (compute_minmax_indexes(out_size,in_size,out2in_scale,out2in_start,min_i,max_i)) { - interpolation = new CachedInterpolation[max_i-min_i+1]; - bool all_inputs_ok = compute_interpolation_weights(min_i,max_i,in_size,out2in_scale,out2in_start,false,interpolation); + const int out_size, const int in_size, + const float x1, // lower bounding box, crop region starts at in_size*x1 + const float x2, // upper bounding box, crop region ends at in_size*x2 + int& min_i, int& max_i, CachedInterpolation*& interpolation) { + float out2in_start = out_size > 1 + ? (float)(in_size - 1) * (float)x1 + : (float)(in_size - 1) * (float)(x1 + x2) / 2.0f; + float out2in_scale = + out_size > 1 + ? (float)(x2 - x1) * (float)(in_size - 1) / (float)(out_size - 1) + : 0.0f; + if (compute_minmax_indexes(out_size, in_size, out2in_scale, out2in_start, + min_i, max_i)) { + interpolation = new CachedInterpolation[max_i - min_i + 1]; + bool all_inputs_ok = + compute_interpolation_weights(min_i, max_i, in_size, out2in_scale, + out2in_start, false, interpolation); if (!all_inputs_ok) { - // should never happen, purpose of compute_minmax_indexes is to ensure that all inputs are ok. - printf("Error! compute_interpolation_weights returned input indexes outside valid range - SEGV will likely ensue.\n"); + // should never happen, purpose of compute_minmax_indexes is to ensure + // that all inputs are ok. + printf( + "Error! compute_interpolation_weights returned input indexes outside " + "valid range - SEGV will likely ensue.\n"); } return true; } else { @@ -143,9 +150,17 @@ inline bool compute_interpolation_weights( } } -template U cast_to(float v, float min_val, float max_val, U u_min_val, U u_max_val); -template -inline U cast_to(float v, float min_val, float max_val, U u_min_val, U u_max_val) { +/** + * Cast float v to type U with range clamping. + * + * If vmax_val, + * return value is clamped to u_max_val. + */ +template +U cast_to(float v, float min_val, float max_val, U u_min_val, U u_max_val); +template +inline U cast_to(float v, float min_val, float max_val, U u_min_val, + U u_max_val) { if (v < min_val) return u_min_val; else if (v > max_val) @@ -153,14 +168,15 @@ inline U cast_to(float v, float min_val, float max_val, U u_min_val, U u_max_val else return static_cast(v); } -template<> -inline float cast_to(float v, float min_val, float max_val, float u_min_val, float u_max_val) { +template <> +inline float cast_to(float v, float min_val, float max_val, + float u_min_val, float u_max_val) { return v; } inline float compute_lerp(const float top_left, const float top_right, - const float bottom_left, const float bottom_right, - const float x_lerp, const float y_lerp) { + const float bottom_left, const float bottom_right, + const float x_lerp, const float y_lerp) { const float top = top_left + (top_right - top_left) * x_lerp; const float bottom = bottom_left + (bottom_right - bottom_left) * x_lerp; return top + (bottom - top) * y_lerp; @@ -173,75 +189,71 @@ inline float compute_lerp(const float top_left, const float top_right, * Optionally flips horizontal and/or vertical axis. */ template -void crop_resize_single_image( - const T* image, - const int64 in_height, const int64 in_width, - const int64 out_height, const int64 out_width, - const int channels, - const int min_ix, const int max_ix, - const CachedInterpolation* xs, - const int min_iy, const int max_iy, - const CachedInterpolation* ys, - const float extrapolated_value, - const bool flip_x, - const bool flip_y, - U* output) TF_ATTRIBUTE_NOINLINE; +void crop_resize_single_image(const T* image, const int64 in_height, + const int64 in_width, const int64 out_height, + const int64 out_width, const int channels, + const int min_ix, const int max_ix, + const CachedInterpolation* xs, const int min_iy, + const int max_iy, const CachedInterpolation* ys, + const float extrapolated_value, const bool flip_x, + const bool flip_y, + U* output) TF_ATTRIBUTE_NOINLINE; template -void crop_resize_single_image( - const T* image, - const int64 in_height, const int64 in_width, - const int64 out_height, const int64 out_width, - const int channels, - const int min_ix, const int max_ix, - const CachedInterpolation* xs, - const int min_iy, const int max_iy, - const CachedInterpolation* ys, - const float extrapolated_value, - const bool flip_x, - const bool flip_y, - U* output) { +void crop_resize_single_image(const T* image, const int64 in_height, + const int64 in_width, const int64 out_height, + const int64 out_width, const int channels, + const int min_ix, const int max_ix, + const CachedInterpolation* xs, const int min_iy, + const int max_iy, const CachedInterpolation* ys, + const float extrapolated_value, const bool flip_x, + const bool flip_y, U* output) { const int64 in_row_size = in_width * channels; const int64 out_row_size = out_width * channels; U u_min_val = std::numeric_limits::min(); U u_max_val = std::numeric_limits::max(); float min_val = static_cast(u_min_val); float max_val = static_cast(u_max_val); - U uEx = cast_to(extrapolated_value,min_val,max_val,u_min_val,u_max_val); + U uEx = + cast_to(extrapolated_value, min_val, max_val, u_min_val, u_max_val); // low y extrapolation zone if (min_iy > 0) { - U* p = flip_y ? output + out_row_size * (out_height - min_iy) : output; + U* p = flip_y ? output + out_row_size * (out_height - min_iy) : output; int64 nn = out_row_size * (int64)min_iy; - for (int64 i = 0; i < nn; ++i) p[i] = uEx; + for (int64 i = 0; i < nn; ++i) p[i] = uEx; } // high y extrapolation zone - if (max_iy < out_height-1) { + if (max_iy < out_height - 1) { U* p = flip_y ? output : output + out_row_size * (max_iy + 1); int64 nn = out_row_size * (int64)(out_height - 1 - max_iy); - for (int64 i = 0; i < nn; ++i) p[i] = uEx; + for (int64 i = 0; i < nn; ++i) p[i] = uEx; } // low x extrapolation zone if (min_ix > 0) { - for (int iy = min_iy; iy <= max_iy; ++iy) { - int xx0 = flip_x ? (out_width-min_ix)*channels : 0; - int nxx = min_ix*channels; - U* p = output + xx0 + out_row_size * (int64)(flip_y ? out_height - 1 - iy : iy); - for (int ix = 0; ix < nxx; ++ix) { + for (int iy = min_iy; iy <= max_iy; ++iy) { + int xx0 = flip_x ? (out_width - min_ix) * channels : 0; + int nxx = min_ix * channels; + U* p = output + xx0 + + out_row_size * (int64)(flip_y ? out_height - 1 - iy : iy); + for (int ix = 0; ix < nxx; ++ix) { p[ix] = uEx; } } } // high x extrapolation zone - if (max_ix < out_width-1) { - for (int iy = min_iy; iy <= max_iy; ++iy) { - int xx0 = flip_x ? 0 : (max_ix+1)*channels; - int nxx = (out_width-1-max_ix)*channels; - U* p = output + xx0 + out_row_size * (int64)(flip_y ? out_height - 1 - iy : iy); - for (int ix = 0; ix < nxx; ++ix) { + if (max_ix < out_width - 1) { + for (int iy = min_iy; iy <= max_iy; ++iy) { + int xx0 = flip_x ? 0 : (max_ix + 1) * channels; + int nxx = (out_width - 1 - max_ix) * channels; + U* p = output + xx0 + + out_row_size * (int64)(flip_y ? out_height - 1 - iy : iy); + for (int ix = 0; ix < nxx; ++ix) { p[ix] = uEx; } } } - U* output_y_ptr = output + out_row_size * (int64)(flip_y ? out_height - 1 - min_iy : min_iy); + U* output_y_ptr = + output + + out_row_size * (int64)(flip_y ? out_height - 1 - min_iy : min_iy); // interpolation zone if (channels == 1) { for (int y = min_iy; y <= max_iy; ++y) { @@ -253,21 +265,24 @@ void crop_resize_single_image( const int x1 = flip_x ? out_width - 1 - min_ix : max_ix; for (int x = x0; x <= x1; ++x) { const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix; - const int64 xs_lower = xs[ix].lower; - const int64 xs_upper = xs[ix].upper; - const float xs_lerp = xs[ix].lerp; + const int64 xs_lower = xs[ix].lower; + const int64 xs_upper = xs[ix].upper; + const float xs_lerp = xs[ix].lerp; - // Read channel 0. - const float top_left0(ys_input_lower_ptr[xs_lower]); - const float top_right0(ys_input_lower_ptr[xs_upper]); - const float bottom_left0(ys_input_upper_ptr[xs_lower]); - const float bottom_right0(ys_input_upper_ptr[xs_upper]); + // Read channel 0. + const float top_left0(ys_input_lower_ptr[xs_lower]); + const float top_right0(ys_input_lower_ptr[xs_upper]); + const float bottom_left0(ys_input_upper_ptr[xs_lower]); + const float bottom_right0(ys_input_upper_ptr[xs_upper]); - // Compute output. - float result0 = compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0,xs_lerp, ys_lerp); - output_y_ptr[x] = cast_to(result0,min_val,max_val,u_min_val,u_max_val); + // Compute output. + float result0 = compute_lerp(top_left0, top_right0, bottom_left0, + bottom_right0, xs_lerp, ys_lerp); + output_y_ptr[x] = + cast_to(result0, min_val, max_val, u_min_val, u_max_val); } - output_y_ptr = flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; + output_y_ptr = + flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; } } else if (channels == 2) { for (int y = min_iy; y <= max_iy; ++y) { @@ -279,31 +294,34 @@ void crop_resize_single_image( const int x1 = flip_x ? out_width - 1 - min_ix : max_ix; for (int x = x0; x <= x1; ++x) { const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix; - const int64 xs_lower = xs[ix].lower; - const int64 xs_upper = xs[ix].upper; - const float xs_lerp = xs[ix].lerp; + const int64 xs_lower = xs[ix].lower; + const int64 xs_upper = xs[ix].upper; + const float xs_lerp = xs[ix].lerp; - // Read channel 0. - const float top_left0(ys_input_lower_ptr[xs_lower + 0]); - const float top_right0(ys_input_lower_ptr[xs_upper + 0]); - const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]); - const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]); + // Read channel 0. + const float top_left0(ys_input_lower_ptr[xs_lower + 0]); + const float top_right0(ys_input_lower_ptr[xs_upper + 0]); + const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]); + const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]); - // Read channel 1. - const float top_left1(ys_input_lower_ptr[xs_lower + 1]); - const float top_right1(ys_input_lower_ptr[xs_upper + 1]); - const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]); - const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]); + // Read channel 1. + const float top_left1(ys_input_lower_ptr[xs_lower + 1]); + const float top_right1(ys_input_lower_ptr[xs_upper + 1]); + const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]); + const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]); - // Compute output. - float result0 = compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0, - xs_lerp, ys_lerp); - float result1 = compute_lerp(top_left1, top_right1, bottom_left1, bottom_right1, - xs_lerp, ys_lerp); - output_y_ptr[x*2 + 0] = cast_to(result0,min_val,max_val,u_min_val,u_max_val); - output_y_ptr[x*2 + 1] = cast_to(result1,min_val,max_val,u_min_val,u_max_val); + // Compute output. + float result0 = compute_lerp(top_left0, top_right0, bottom_left0, + bottom_right0, xs_lerp, ys_lerp); + float result1 = compute_lerp(top_left1, top_right1, bottom_left1, + bottom_right1, xs_lerp, ys_lerp); + output_y_ptr[x * 2 + 0] = + cast_to(result0, min_val, max_val, u_min_val, u_max_val); + output_y_ptr[x * 2 + 1] = + cast_to(result1, min_val, max_val, u_min_val, u_max_val); } - output_y_ptr = flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; + output_y_ptr = + flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; } } else if (channels == 3) { for (int y = min_iy; y <= max_iy; ++y) { @@ -315,40 +333,44 @@ void crop_resize_single_image( const int x1 = flip_x ? out_width - 1 - min_ix : max_ix; for (int x = x0; x <= x1; ++x) { const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix; - const int64 xs_lower = xs[ix].lower; - const int64 xs_upper = xs[ix].upper; - const float xs_lerp = xs[ix].lerp; + const int64 xs_lower = xs[ix].lower; + const int64 xs_upper = xs[ix].upper; + const float xs_lerp = xs[ix].lerp; - // Read channel 0. - const float top_left0(ys_input_lower_ptr[xs_lower + 0]); - const float top_right0(ys_input_lower_ptr[xs_upper + 0]); - const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]); - const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]); + // Read channel 0. + const float top_left0(ys_input_lower_ptr[xs_lower + 0]); + const float top_right0(ys_input_lower_ptr[xs_upper + 0]); + const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]); + const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]); - // Read channel 1. - const float top_left1(ys_input_lower_ptr[xs_lower + 1]); - const float top_right1(ys_input_lower_ptr[xs_upper + 1]); - const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]); - const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]); + // Read channel 1. + const float top_left1(ys_input_lower_ptr[xs_lower + 1]); + const float top_right1(ys_input_lower_ptr[xs_upper + 1]); + const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]); + const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]); - // Read channel 2. - const float top_left2(ys_input_lower_ptr[xs_lower + 2]); - const float top_right2(ys_input_lower_ptr[xs_upper + 2]); - const float bottom_left2(ys_input_upper_ptr[xs_lower + 2]); - const float bottom_right2(ys_input_upper_ptr[xs_upper + 2]); + // Read channel 2. + const float top_left2(ys_input_lower_ptr[xs_lower + 2]); + const float top_right2(ys_input_lower_ptr[xs_upper + 2]); + const float bottom_left2(ys_input_upper_ptr[xs_lower + 2]); + const float bottom_right2(ys_input_upper_ptr[xs_upper + 2]); - // Compute output. - float result0 = compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0, - xs_lerp, ys_lerp); - float result1 = compute_lerp(top_left1, top_right1, bottom_left1, bottom_right1, - xs_lerp, ys_lerp); - float result2 = compute_lerp(top_left2, top_right2, bottom_left2, bottom_right2, - xs_lerp, ys_lerp); - output_y_ptr[x*3 + 0] = cast_to(result0,min_val,max_val,u_min_val,u_max_val); - output_y_ptr[x*3 + 1] = cast_to(result1,min_val,max_val,u_min_val,u_max_val); - output_y_ptr[x*3 + 2] = cast_to(result2,min_val,max_val,u_min_val,u_max_val); + // Compute output. + float result0 = compute_lerp(top_left0, top_right0, bottom_left0, + bottom_right0, xs_lerp, ys_lerp); + float result1 = compute_lerp(top_left1, top_right1, bottom_left1, + bottom_right1, xs_lerp, ys_lerp); + float result2 = compute_lerp(top_left2, top_right2, bottom_left2, + bottom_right2, xs_lerp, ys_lerp); + output_y_ptr[x * 3 + 0] = + cast_to(result0, min_val, max_val, u_min_val, u_max_val); + output_y_ptr[x * 3 + 1] = + cast_to(result1, min_val, max_val, u_min_val, u_max_val); + output_y_ptr[x * 3 + 2] = + cast_to(result2, min_val, max_val, u_min_val, u_max_val); } - output_y_ptr = flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; + output_y_ptr = + flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; } } else if (channels == 4) { for (int y = min_iy; y <= max_iy; ++y) { @@ -360,49 +382,54 @@ void crop_resize_single_image( const int x1 = flip_x ? out_width - 1 - min_ix : max_ix; for (int x = x0; x <= x1; ++x) { const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix; - const int64 xs_lower = xs[ix].lower; - const int64 xs_upper = xs[ix].upper; - const float xs_lerp = xs[ix].lerp; + const int64 xs_lower = xs[ix].lower; + const int64 xs_upper = xs[ix].upper; + const float xs_lerp = xs[ix].lerp; - // Read channel 0. - const float top_left0(ys_input_lower_ptr[xs_lower + 0]); - const float top_right0(ys_input_lower_ptr[xs_upper + 0]); - const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]); - const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]); + // Read channel 0. + const float top_left0(ys_input_lower_ptr[xs_lower + 0]); + const float top_right0(ys_input_lower_ptr[xs_upper + 0]); + const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]); + const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]); - // Read channel 1. - const float top_left1(ys_input_lower_ptr[xs_lower + 1]); - const float top_right1(ys_input_lower_ptr[xs_upper + 1]); - const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]); - const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]); + // Read channel 1. + const float top_left1(ys_input_lower_ptr[xs_lower + 1]); + const float top_right1(ys_input_lower_ptr[xs_upper + 1]); + const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]); + const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]); - // Read channel 2. - const float top_left2(ys_input_lower_ptr[xs_lower + 2]); - const float top_right2(ys_input_lower_ptr[xs_upper + 2]); - const float bottom_left2(ys_input_upper_ptr[xs_lower + 2]); - const float bottom_right2(ys_input_upper_ptr[xs_upper + 2]); + // Read channel 2. + const float top_left2(ys_input_lower_ptr[xs_lower + 2]); + const float top_right2(ys_input_lower_ptr[xs_upper + 2]); + const float bottom_left2(ys_input_upper_ptr[xs_lower + 2]); + const float bottom_right2(ys_input_upper_ptr[xs_upper + 2]); - // Read channel 3. - const float top_left3(ys_input_lower_ptr[xs_lower + 3]); - const float top_right3(ys_input_lower_ptr[xs_upper + 3]); - const float bottom_left3(ys_input_upper_ptr[xs_lower + 3]); - const float bottom_right3(ys_input_upper_ptr[xs_upper + 3]); + // Read channel 3. + const float top_left3(ys_input_lower_ptr[xs_lower + 3]); + const float top_right3(ys_input_lower_ptr[xs_upper + 3]); + const float bottom_left3(ys_input_upper_ptr[xs_lower + 3]); + const float bottom_right3(ys_input_upper_ptr[xs_upper + 3]); - // Compute output. - float result0 = compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0, - xs_lerp, ys_lerp); - float result1 = compute_lerp(top_left1, top_right1, bottom_left1, bottom_right1, - xs_lerp, ys_lerp); - float result2 = compute_lerp(top_left2, top_right2, bottom_left2, bottom_right2, - xs_lerp, ys_lerp); - float result3 = compute_lerp(top_left3, top_right3, bottom_left3, bottom_right3, - xs_lerp, ys_lerp); - output_y_ptr[x*4 + 0] = cast_to(result0,min_val,max_val,u_min_val,u_max_val); - output_y_ptr[x*4 + 1] = cast_to(result1,min_val,max_val,u_min_val,u_max_val); - output_y_ptr[x*4 + 2] = cast_to(result2,min_val,max_val,u_min_val,u_max_val); - output_y_ptr[x*4 + 3] = cast_to(result3,min_val,max_val,u_min_val,u_max_val); + // Compute output. + float result0 = compute_lerp(top_left0, top_right0, bottom_left0, + bottom_right0, xs_lerp, ys_lerp); + float result1 = compute_lerp(top_left1, top_right1, bottom_left1, + bottom_right1, xs_lerp, ys_lerp); + float result2 = compute_lerp(top_left2, top_right2, bottom_left2, + bottom_right2, xs_lerp, ys_lerp); + float result3 = compute_lerp(top_left3, top_right3, bottom_left3, + bottom_right3, xs_lerp, ys_lerp); + output_y_ptr[x * 4 + 0] = + cast_to(result0, min_val, max_val, u_min_val, u_max_val); + output_y_ptr[x * 4 + 1] = + cast_to(result1, min_val, max_val, u_min_val, u_max_val); + output_y_ptr[x * 4 + 2] = + cast_to(result2, min_val, max_val, u_min_val, u_max_val); + output_y_ptr[x * 4 + 3] = + cast_to(result3, min_val, max_val, u_min_val, u_max_val); } - output_y_ptr = flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; + output_y_ptr = + flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; } } else { for (int y = min_iy; y <= max_iy; ++y) { @@ -414,23 +441,25 @@ void crop_resize_single_image( const int x1 = flip_x ? out_width - 1 - min_ix : max_ix; for (int x = x0; x <= x1; ++x) { const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix; - const int64 xs_lower = xs[ix].lower; - const int64 xs_upper = xs[ix].upper; - const float xs_lerp = xs[ix].lerp; - for (int ichan = 0; ichan < channels; ++ichan) { - const float top_left0(ys_input_lower_ptr[xs_lower + ichan]); - const float top_right0(ys_input_lower_ptr[xs_upper + ichan]); - const float bottom_left0(ys_input_upper_ptr[xs_lower + ichan]); - const float bottom_right0(ys_input_upper_ptr[xs_upper + ichan]); - float result0 = compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0, - xs_lerp, ys_lerp); - output_y_ptr[x*channels + ichan] = cast_to(result0,min_val,max_val,u_min_val,u_max_val); - } + const int64 xs_lower = xs[ix].lower; + const int64 xs_upper = xs[ix].upper; + const float xs_lerp = xs[ix].lerp; + for (int ichan = 0; ichan < channels; ++ichan) { + const float top_left0(ys_input_lower_ptr[xs_lower + ichan]); + const float top_right0(ys_input_lower_ptr[xs_upper + ichan]); + const float bottom_left0(ys_input_upper_ptr[xs_lower + ichan]); + const float bottom_right0(ys_input_upper_ptr[xs_upper + ichan]); + float result0 = compute_lerp(top_left0, top_right0, bottom_left0, + bottom_right0, xs_lerp, ys_lerp); + output_y_ptr[x * channels + ichan] = + cast_to(result0, min_val, max_val, u_min_val, u_max_val); + } } - output_y_ptr = flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; + output_y_ptr = + flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size; } } } -} // namespace +} // namespace internal } // namespace tensorflow -#endif +#endif // TENSORFLOW_CORE_KERNELS_CROP_RESIZE_BILINEAR_CORE_H_ diff --git a/tensorflow/core/kernels/resize_bilinear_op.cc b/tensorflow/core/kernels/resize_bilinear_op.cc index 3f1589dcee..fc66bf3355 100644 --- a/tensorflow/core/kernels/resize_bilinear_op.cc +++ b/tensorflow/core/kernels/resize_bilinear_op.cc @@ -19,16 +19,20 @@ limitations under the License. #include "tensorflow/core/kernels/resize_bilinear_op.h" #include -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/crop_resize_bilinear_core.h" #include "tensorflow/core/kernels/image_resizer_state.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/kernels/crop_resize_bilinear_core.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + +using ::tensorflow::internal::CachedInterpolation; +using ::tensorflow::internal::compute_interpolation_weights; +using ::tensorflow::internal::crop_resize_single_image; namespace tensorflow { @@ -105,13 +109,11 @@ struct ResizeBilinear { } for (int b = 0; b < batch_size; ++b) { - crop_resize_single_image( - images.data() + (int64)b * in_batch_num_values, - in_height,in_width,out_height,out_width,channels, - 0,out_width-1,xs.data(), - 0,out_height-1,ys.data(), - 0.0f,false,false, - output.data() + (int64)b * out_batch_num_values); + crop_resize_single_image(images.data() + (int64)b * in_batch_num_values, + in_height, in_width, out_height, out_width, + channels, 0, out_width - 1, xs.data(), 0, + out_height - 1, ys.data(), 0.0f, false, false, + output.data() + (int64)b * out_batch_num_values); } } }; -- cgit v1.2.3 From 0c432e8e2eafc4320b55d0f083806982b0ccc218 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 16 Jul 2018 13:17:52 -0700 Subject: Add autograph.ipynb to guide PiperOrigin-RevId: 204794405 --- tensorflow/docs_src/guide/autograph.md | 3 +++ tensorflow/docs_src/guide/leftnav_files | 1 + 2 files changed, 4 insertions(+) create mode 100644 tensorflow/docs_src/guide/autograph.md diff --git a/tensorflow/docs_src/guide/autograph.md b/tensorflow/docs_src/guide/autograph.md new file mode 100644 index 0000000000..823e1c6d6b --- /dev/null +++ b/tensorflow/docs_src/guide/autograph.md @@ -0,0 +1,3 @@ +# AutoGraph: Easy control flow for graphs + +[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/guide/autograph.ipynb) diff --git a/tensorflow/docs_src/guide/leftnav_files b/tensorflow/docs_src/guide/leftnav_files index b3324278c1..c4e235b41a 100644 --- a/tensorflow/docs_src/guide/leftnav_files +++ b/tensorflow/docs_src/guide/leftnav_files @@ -23,6 +23,7 @@ tensors.md variables.md graphs.md saved_model.md +autograph.md : Control flow ### ML Concepts embedding.md -- cgit v1.2.3 From bf87fa55a3280961bec8f31256016d362faf7c30 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Mon, 16 Jul 2018 13:29:47 -0700 Subject: [XLA] Simplify A*0 and 0*A to 0 for integral types. PiperOrigin-RevId: 204797049 --- .../compiler/xla/service/algebraic_simplifier.cc | 13 +++++++++++++ .../xla/service/algebraic_simplifier_test.cc | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index af7728da54..2205a7ec18 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -1156,6 +1156,19 @@ Status AlgebraicSimplifierVisitor::HandleMultiply(HloInstruction* multiply) { return Status::OK(); } + // 0*A => 0. Only applies for integral types for correct NaN-handling. + if (IsAll(lhs, 0) && + primitive_util::IsIntegralType(multiply->shape().element_type()) && + ReplaceInstructionIfSameShape(multiply, lhs)) { + return Status::OK(); + } + // A*0 => 0 + if (IsAll(rhs, 0) && + primitive_util::IsIntegralType(multiply->shape().element_type()) && + ReplaceInstructionIfSameShape(multiply, rhs)) { + return Status::OK(); + } + // exp(A) * exp(B) => exp(A+B) if (Match(multiply, m::Multiply(m::Exp(m::Op(&lhs)), m::Exp(m::Op(&rhs))))) { auto add = computation_->AddInstruction(HloInstruction::CreateBinary( diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc index 92bbcbd740..3f0f2afadd 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc @@ -74,6 +74,26 @@ TEST_F(AlgebraicSimplifierTest, AddZero) { EXPECT_EQ(root, param0); } +// Test that A * 0 is simplified to 0 +TEST_F(AlgebraicSimplifierTest, MulZero) { + Shape r0s32 = ShapeUtil::MakeShape(S32, {}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r0s32, "param0")); + HloInstruction* zero = builder.AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::CreateR0(0))); + builder.AddInstruction( + HloInstruction::CreateBinary(r0s32, HloOpcode::kMultiply, param0, zero)); + + auto computation = module().AddEntryComputation(builder.Build()); + HloInstruction* root = computation->root_instruction(); + EXPECT_EQ(root->opcode(), HloOpcode::kMultiply); + AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, + non_bitcasting_callback()); + ASSERT_TRUE(simplifier.Run(&module()).ValueOrDie()); + EXPECT_EQ(computation->root_instruction(), zero); +} + // Test that Reduce(Reduce(A)) -> Reduce(A) TEST_F(AlgebraicSimplifierTest, TwoReducesToOne) { HloComputation::Builder builder(TestName()); -- cgit v1.2.3 From f5a7bea2b78c8a8b4b76060978369c3436b60b55 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 13:40:34 -0700 Subject: When fitting in EagerMode, convert all user input to EagerIterators PiperOrigin-RevId: 204799111 --- tensorflow/python/keras/BUILD | 13 + tensorflow/python/keras/engine/training_eager.py | 473 ++++----------------- tensorflow/python/keras/engine/training_utils.py | 138 +++++- .../python/keras/engine/training_utils_test.py | 150 +++++++ 4 files changed, 378 insertions(+), 396 deletions(-) create mode 100644 tensorflow/python/keras/engine/training_utils_test.py diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index 4056818a95..01f1184766 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -792,6 +792,19 @@ py_test( ], ) +py_test( + name = "training_utils_test", + size = "medium", + srcs = ["engine/training_utils_test.py"], + srcs_version = "PY2AND3", + tags = ["notsan"], + deps = [ + ":keras", + "//tensorflow/python:client_testlib", + "//third_party/py/numpy", + ], +) + py_test( name = "model_subclassing_test", size = "medium", diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py index c78684c9f4..397de42985 100644 --- a/tensorflow/python/keras/engine/training_eager.py +++ b/tensorflow/python/keras/engine/training_eager.py @@ -34,7 +34,6 @@ from tensorflow.python.keras import losses from tensorflow.python.keras import metrics as metrics_module from tensorflow.python.keras.engine import training_utils from tensorflow.python.keras.utils import generic_utils -from tensorflow.python.ops import array_ops from tensorflow.python.platform import tf_logging as logging @@ -194,7 +193,8 @@ def iterator_fit_loop(model, callbacks=None, callback_metrics=None, validation_steps=None, - do_validation=False): + do_validation=False, + batch_size=None): """Fit function for eager execution when input is given as dataset iterator. Updates the given epoch logs. @@ -224,16 +224,23 @@ def iterator_fit_loop(model, validation_steps: Number of steps to run validation for (only if doing validation from data tensors). Ignored with default value of `None`. do_validation: Boolean value indicating whether we should do validation. + batch_size: int, val_inputs and val_targets will be evaled batch by + batch with size batch_size if they are array. Raises: ValueError: In case of mismatch between given number of inputs and expectations of the model. """ assert isinstance(inputs, iterator_ops.EagerIterator) + + # make sure either x,y or x,y,sample_weights is provided + if (not isinstance(inputs.output_shapes, (list, tuple)) or + len(inputs.output_shapes) not in (2, 3)): + raise ValueError('Please provide either inputs and targets' + 'or inputs, targets, and sample_weights') + for step_index in range(steps_per_epoch): - batch_logs = {} - batch_logs['batch'] = step_index - batch_logs['size'] = 1 + batch_logs = {'batch': step_index, 'size': 1} callbacks.on_batch_begin(step_index, batch_logs) # Get data from the iterator. @@ -247,19 +254,21 @@ def iterator_fit_loop(model, 'batches (in this case, %d batches).' % steps_per_epoch * epochs) break - if not isinstance(next_element, (list, tuple)) or len(next_element) != 2: - raise ValueError('Please provide data as a list or tuple of 2 elements ' - ' - input and target pair. Received %s' % next_element) - x, y = next_element + if len(inputs.output_shapes) == 2: + x, y = next_element + sample_weights = None + else: + x, y, sample_weights = next_element # Validate and standardize data. x, y, sample_weights = model._standardize_user_data( - x, y, class_weight=class_weight) + x, y, sample_weight=sample_weights, class_weight=class_weight) x = training_utils.cast_if_floating_dtype(x) y = training_utils.cast_if_floating_dtype(y) if sample_weights: sample_weights = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) + training_utils.cast_if_floating_dtype( + ops.convert_to_tensor(val, dtype=backend.floatx())) if val is not None else None for val in sample_weights ] @@ -307,122 +316,8 @@ def iterator_fit_loop(model, val_targets, sample_weights=val_sample_weights, steps=validation_steps, - verbose=0) - if not isinstance(val_outs, list): - val_outs = [val_outs] - # Same labels assumed. - for l, o in zip(out_labels, val_outs): - epoch_logs['val_' + l] = o - - -def batch_fit_loop(model, - inputs, - targets, - epoch_logs, - index_array, - out_labels, - callback_model, - batch_size, - sample_weights=None, - val_inputs=None, - val_targets=None, - val_sample_weights=None, - callbacks=None, - shuffle=True, - num_train_samples=None, - do_validation=False): - """Fit function for eager execution when input is given as arrays or tensors. - - Updates the given epoch logs. - - Arguments: - model: Instance of the `Model`. - inputs: List of input arrays. - targets: List of target arrays. - epoch_logs: Dictionary of logs from every epoch. - index_array: Index array generated from number of training samples. - out_labels: Output labels generated from model metric names. - callback_model: Instance of `Model` to callback. - batch_size: Integer batch size or None if unknown. - sample_weights: Optional list of sample weight arrays. - val_inputs: Input data for validation. - val_targets: Target data for validation. - val_sample_weights: Sample weight data for validation. - callbacks: List of callbacks to be called during training. - shuffle: Whether to shuffle the data at the beginning of each epoch. - num_train_samples: Integer number of training samples. - do_validation: Boolean value indicating whether we should do validation. - """ - # TODO(psv): Create a dataset iterator instead of manually creating batches - # here and in batch_test_loop, batch_predict_loop. - if shuffle == 'batch': - index_array = model._batch_shuffle(index_array, batch_size) - elif shuffle: - np.random.shuffle(index_array) - - batches = generic_utils.make_batches(num_train_samples, batch_size) - - for batch_index, (batch_start, batch_end) in enumerate(batches): - batch_ids = index_array[batch_start:batch_end] - inputs_batch = slice_arrays(inputs, batch_ids, contiguous=not shuffle) - targets_batch = slice_arrays(targets, batch_ids, contiguous=not shuffle) - if sample_weights: - sample_weights_batch = slice_arrays( - sample_weights, batch_ids, contiguous=not shuffle) - else: - sample_weights_batch = None - batch_logs = {} - batch_logs['batch'] = batch_index - batch_logs['size'] = len(batch_ids) - - callbacks.on_batch_begin(batch_index, batch_logs) - - inputs_batch = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) - for val in inputs_batch - ] - targets_batch = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) - for val in targets_batch - ] - if sample_weights: - sample_weights_batch = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) - if val is not None else None for val in sample_weights_batch - ] - - outs, loss, loss_metrics = _process_single_batch( - model, - inputs_batch, - targets_batch, - sample_weights=sample_weights_batch, - training=True) - - if not isinstance(outs, list): - outs = [outs] - - for l, o in zip(out_labels, outs): - batch_logs[l] = o - # Required for eager execution - metrics_results = _eager_metrics_fn(model, outs, targets_batch) - batch_logs['loss'] = tensor_util.constant_value(backend.mean(loss)) - - for k, v in zip(model.metrics_names, - [backend.mean(loss)] + loss_metrics + metrics_results): - batch_logs[k] = tensor_util.constant_value(v) - callbacks.on_batch_end(batch_index, batch_logs) - if callback_model.stop_training: - break - - if batch_index == len(batches) - 1: # Last batch. - if do_validation: - val_outs = test_loop( - model, - val_inputs, - val_targets, - sample_weights=val_sample_weights, - batch_size=batch_size, - verbose=0) + verbose=0, + batch_size=batch_size) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. @@ -451,6 +346,11 @@ def iterator_test_loop(model, inputs, steps, verbose=0): expectations of the model. """ assert isinstance(inputs, iterator_ops.EagerIterator) + # make sure either x,y or x,y,sample_weights is provided + if (not isinstance(inputs.output_shapes, (list, tuple)) or + len(inputs.output_shapes) < 2 or len(inputs.output_shapes) > 3): + raise ValueError('Please provide either inputs and targets' + 'or inputs, targets, and sample_weights') outs = [] num_samples = 0 if verbose == 1: @@ -466,10 +366,11 @@ def iterator_test_loop(model, inputs, steps, verbose=0): '(in this case, %d batches).', steps) break - if not isinstance(next_element, (list, tuple)) or len(next_element) != 2: - raise ValueError('Please provide data as a list or tuple of 2 elements ' - ' - input and target pair. Received %s' % next_element) - x, y = next_element + if len(inputs.output_shapes) == 2: + x, y = next_element + sample_weights = None + else: + x, y, sample_weights = next_element # Validate and standardize data. x, y, sample_weights = model._standardize_user_data(x, y) @@ -512,94 +413,6 @@ def iterator_test_loop(model, inputs, steps, verbose=0): return outs -def batch_test_loop(model, - inputs, - targets, - batch_size, - sample_weights=None, - verbose=0): - """Test function for eager execution when input is given as arrays or tensors. - - Arguments: - model: Model instance that is being evaluated in Eager mode. - inputs: List of input arrays. - targets: List of target arrays. - batch_size: Integer batch size. - sample_weights: Optional list of sample weight arrays. - verbose: Verbosity mode. - - Returns: - Scalar loss (if the model has a single output and no metrics) - or list of scalars (if the model has multiple outputs - and/or metrics). The attribute `model.metrics_names` will give you - the display labels for the scalar outputs. - """ - outs = [] - feed_data = inputs + targets - if sample_weights: - feed_data += sample_weights - num_samples = training_utils.check_num_samples( - feed_data, batch_size=batch_size) - if verbose == 1: - progbar = generic_utils.Progbar(target=num_samples) - batches = generic_utils.make_batches(num_samples, batch_size) - index_array = np.arange(num_samples) - for batch_index, (batch_start, batch_end) in enumerate(batches): - batch_ids = index_array[batch_start:batch_end] - inputs_batch = slice_arrays(inputs, batch_ids) - targets_batch = slice_arrays(targets, batch_ids) - if sample_weights: - sample_weights_batch = slice_arrays(sample_weights, batch_ids) - else: - sample_weights_batch = None - - inputs_batch = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) - for val in inputs_batch - ] - targets_batch = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) - for val in targets_batch - ] - if sample_weights: - sample_weights_batch = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) - if val is not None else None for val in sample_weights_batch - ] - - loss_outs, loss, loss_metrics = _model_loss( - model, - inputs_batch, - targets_batch, - sample_weights=sample_weights_batch, - training=False) - metrics_results = _eager_metrics_fn(model, loss_outs, targets_batch) - batch_outs = [] - for _, v in zip(model.metrics_names, - [backend.mean(loss)] + loss_metrics + metrics_results): - batch_outs.append(tensor_util.constant_value(v)) - - if isinstance(batch_outs, list): - if batch_index == 0: - for _ in enumerate(batch_outs): - outs.append(0.) - for i, batch_out in enumerate(batch_outs): - outs[i] += batch_out * len(batch_ids) - else: - if batch_index == 0: - outs.append(0.) - outs[0] += batch_outs * len(batch_ids) - - if verbose == 1: - progbar.update(batch_end) - - for i in range(len(outs)): - outs[i] /= num_samples - if len(outs) == 1: - return outs[0] - return outs - - def iterator_predict_loop(model, inputs, steps, verbose=0): """Predict function for eager execution when input is dataset iterator. @@ -619,6 +432,12 @@ def iterator_predict_loop(model, inputs, steps, verbose=0): expectations of the model. """ assert isinstance(inputs, iterator_ops.EagerIterator) + if not isinstance(inputs.output_shapes, + (list, tuple)) or len(inputs.output_shapes) > 2: + raise ValueError( + 'Please provide data as a list or tuple of 1 or 2 elements ' + ' - input or input and target pair. Received %s. We do not use the ' + '`target` value here.' % inputs.output_shapes) outs = [] if verbose == 1: progbar = generic_utils.Progbar(target=steps) @@ -634,12 +453,8 @@ def iterator_predict_loop(model, inputs, steps, verbose=0): 'batches (in this case, %d batches).', steps) break - if not isinstance(next_element, (list, tuple)) or len(next_element) != 2: - raise ValueError( - 'Please provide data as a list or tuple of 2 elements ' - ' - input and target pair. Received %s. We do not use the ' - '`target` value here.' % next_element) - x, _ = next_element + # expects a tuple, where first element of tuple represents inputs + x = next_element[0] # Validate and standardize data. x, _, _ = model._standardize_user_data(x) @@ -670,99 +485,6 @@ def iterator_predict_loop(model, inputs, steps, verbose=0): return outs -def batch_predict_loop(model, inputs, batch_size, verbose=0): - """Predict function for eager execution when input is arrays or tensors. - - Arguments: - model: Instance of `Model`. - inputs: List of input arrays. - batch_size: Integer batch size. - verbose: Verbosity mode. - - Returns: - Array of predictions (if the model has a single output) - or list of arrays of predictions (if the model has multiple outputs). - """ - outs = [] - num_samples = training_utils.check_num_samples(inputs, batch_size) - if verbose == 1: - progbar = generic_utils.Progbar(target=num_samples) - batches = generic_utils.make_batches(num_samples, batch_size) - index_array = np.arange(num_samples) - for batch_index, (batch_start, batch_end) in enumerate(batches): - batch_ids = index_array[batch_start:batch_end] - inputs_batch = slice_arrays(inputs, batch_ids) - - inputs_batch = [ - ops.convert_to_tensor(val, dtype=backend.floatx()) - for val in inputs_batch - ] - - if len(inputs_batch) == 1: - if model._expects_training_arg: - batch_outs = model.call(inputs_batch[0], training=False) - else: - batch_outs = model.call(inputs_batch[0]) - else: - if model._expects_training_arg: - batch_outs = model.call(inputs_batch, training=False) - else: - batch_outs = model.call(inputs_batch) - - if not isinstance(batch_outs, list): - batch_outs = [batch_outs] - if batch_index == 0: - # Pre-allocate the results arrays. - for batch_out in batch_outs: - dims = batch_out.shape[1:].dims - dims_list = [d.value for d in dims] - shape = (num_samples,) + tuple(dims_list) - outs.append(np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype)) - for i, batch_out in enumerate(batch_outs): - outs[i][batch_start:batch_end] = batch_out - if verbose == 1: - progbar.update(batch_end) - - if len(outs) == 1: - return outs[0] - return outs - - -def slice_arrays(arrays, indices, contiguous=True): - """Slices batches out of provided arrays (workaround for eager tensors). - - Unfortunately eager tensors don't have the same slicing behavior as - Numpy arrays (they follow the same slicing behavior as symbolic TF tensors), - hence we cannot use `generic_utils.slice_arrays` directly - and we have to implement this workaround based on `concat`. This has a - performance cost. - - Arguments: - arrays: Single array or list of arrays. - indices: List of indices in the array that should be included in the output - batch. - contiguous: Boolean flag indicating whether the indices are contiguous. - - Returns: - Slice of data (either single array or list of arrays). - """ - if any(tensor_util.is_tensor(x) for x in arrays): - converted_to_list = False - if not isinstance(arrays, list): - converted_to_list = True - arrays = [arrays] - if not contiguous: - entries = [[x[i:i + 1] for i in indices] for x in arrays] - slices = [array_ops.concat(x, axis=0) for x in entries] - else: - slices = [x[indices[0]:indices[-1] + 1] for x in arrays] - if converted_to_list: - slices = slices[0] - return slices - else: - return generic_utils.slice_arrays(arrays, indices) - - def _process_single_batch(model, inputs, targets, @@ -935,19 +657,24 @@ def fit_loop(model, Raises: ValueError: In case of invalid argument values. """ + # Convert training inputs to an EagerIterator + inputs, steps_per_epoch = training_utils.convert_to_iterator( + x=inputs, + y=targets, + sample_weights=sample_weights, + batch_size=batch_size, + steps_per_epoch=steps_per_epoch, + epochs=epochs, + shuffle=shuffle) # Required for eager execution with backend.learning_phase_scope(1): do_validation = False if val_inputs: do_validation = True - if (steps_per_epoch is None and verbose and inputs and - hasattr(inputs[0], 'shape') and hasattr(val_inputs[0], 'shape')): - print('Train on %d samples, validate on %d samples' % - (inputs[0].shape[0], val_inputs[0].shape[0])) num_train_samples = None out_labels = None - if steps_per_epoch is None or model._is_compiled: + if model._is_compiled: out_labels = model.metrics_names if do_validation: callback_metrics = copy.copy(out_labels) + [ @@ -956,28 +683,10 @@ def fit_loop(model, else: callback_metrics = copy.copy(out_labels) - if steps_per_epoch is None: - if sample_weights: - feed_data = inputs + targets + sample_weights - else: - feed_data = inputs + targets - num_train_samples = training_utils.check_num_samples( - feed_data, - batch_size=batch_size, - steps=steps_per_epoch, - steps_name='steps_per_epoch') - - if num_train_samples is not None: - index_array = np.arange(num_train_samples) - model.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [model.history] if verbose: - if steps_per_epoch is not None: - count_mode = 'steps' - else: - count_mode = 'samples' - callbacks += [cbks.ProgbarLogger(count_mode)] + callbacks += [cbks.ProgbarLogger('steps')] callbacks = cbks.CallbackList(callbacks) # it's possible to callback a different model than self @@ -1019,43 +728,24 @@ def fit_loop(model, for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) epoch_logs = {} - - if steps_per_epoch is not None: - iterator_fit_loop( - model, - inputs, - class_weight, - steps_per_epoch=steps_per_epoch, - callback_model=callback_model, - out_labels=out_labels, - epoch_logs=epoch_logs, - val_inputs=val_inputs, - val_targets=val_targets, - val_sample_weights=val_sample_weights, - epochs=epochs, - verbose=verbose, - callbacks=callbacks, - callback_metrics=callback_metrics, - validation_steps=validation_steps, - do_validation=do_validation) - else: - batch_fit_loop( - model, - inputs, - targets, - epoch_logs=epoch_logs, - index_array=index_array, - out_labels=out_labels, - callback_model=callback_model, - batch_size=batch_size, - sample_weights=sample_weights, - val_inputs=val_inputs, - val_targets=val_targets, - val_sample_weights=val_sample_weights, - callbacks=callbacks, - shuffle=shuffle, - num_train_samples=num_train_samples, - do_validation=do_validation) + iterator_fit_loop( + model, + inputs, + class_weight, + steps_per_epoch=steps_per_epoch, + callback_model=callback_model, + out_labels=out_labels, + epoch_logs=epoch_logs, + val_inputs=val_inputs, + val_targets=val_targets, + val_sample_weights=val_sample_weights, + epochs=epochs, + verbose=verbose, + callbacks=callbacks, + callback_metrics=callback_metrics, + validation_steps=validation_steps, + do_validation=do_validation, + batch_size=batch_size) callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break @@ -1087,17 +777,14 @@ def test_loop(model, inputs, targets, and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ + inputs, steps = training_utils.convert_to_iterator( + x=inputs, + y=targets, + sample_weights=sample_weights, + batch_size=batch_size, + steps_per_epoch=steps) with backend.learning_phase_scope(0): - if steps is not None: - return iterator_test_loop(model, inputs, steps, verbose=verbose) - else: - return batch_test_loop( - model, - inputs, - targets, - batch_size=batch_size, - sample_weights=sample_weights, - verbose=verbose) + return iterator_test_loop(model, inputs, steps, verbose=verbose) def predict_loop(model, inputs, @@ -1121,8 +808,6 @@ def predict_loop(model, inputs, (if the model has multiple outputs). """ with backend.learning_phase_scope(0): - if steps is not None: - return iterator_predict_loop(model, inputs, steps, verbose=verbose) - else: - return batch_predict_loop( - model, inputs, batch_size=batch_size, verbose=verbose) + inputs, steps = training_utils.convert_to_iterator( + x=inputs, batch_size=batch_size, steps_per_epoch=steps) + return iterator_predict_loop(model, inputs, steps, verbose=verbose) diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py index 728a2b493b..dbbc87daf9 100644 --- a/tensorflow/python/keras/engine/training_utils.py +++ b/tensorflow/python/keras/engine/training_utils.py @@ -19,9 +19,11 @@ from __future__ import division from __future__ import print_function import copy +import math import numpy as np +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import iterator_ops from tensorflow.python.eager import context from tensorflow.python.framework import tensor_util @@ -31,6 +33,135 @@ from tensorflow.python.keras import metrics as metrics_module from tensorflow.python.ops import math_ops +def _map_nested(data, func): + """Maps each nested element using func.""" + if isinstance(data, list): + return [_map_nested(nested_data, func) for nested_data in data] + elif isinstance(data, tuple): + return tuple(_map_nested(nested_data, func) for nested_data in data) + elif isinstance(data, dict): + return { + k: _map_nested(nested_data, func) for k, nested_data in data.items() + } + else: + return func(data) + + +def _nested_all(data, cond_func): + """Checks if all elements in a nested structure satisfy cond_func.""" + if isinstance(data, (tuple, list)): + return all([_nested_all(nested_data, cond_func) for nested_data in data]) + elif isinstance(data, dict): + return all( + [_nested_all(nested_data, cond_func) for nested_data in data.values()]) + else: + return cond_func(data) + + +def _nested_any(data, cond_func): + """Checks if any nested_elements in a nested structure satisfy cond_func.""" + if isinstance(data, (tuple, list)): + return any([_nested_any(nested_data, cond_func) for nested_data in data]) + elif isinstance(data, dict): + return any( + [_nested_any(nested_data, cond_func) for nested_data in data.values()]) + else: + return cond_func(data) + + +def _convert_lists_to_tuples(data): + """Converts all lists to tuples, since Datasets expect tuples.""" + if isinstance(data, (tuple, list)): + return tuple(_convert_lists_to_tuples(nested_data) for nested_data in data) + elif isinstance(data, dict): + return { + k: _convert_lists_to_tuples(nested_data) + for k, nested_data in data.items() + } + else: + return data + + +def _get_batch_axis_size(data): + """Returns batch axis shape for nested data.""" + if isinstance(data, (tuple, list)): + return _get_batch_axis_size(data[0]) + elif isinstance(data, dict): + return _get_batch_axis_size(list(data.values())) + else: + return int(data.shape[0]) + + +def convert_to_iterator(x=None, + y=None, + sample_weights=None, + batch_size=None, + steps_per_epoch=None, + epochs=1, + shuffle=False): + """Converts NumPy arrays or EagerTensors to an EagerIterator. + + Combines all provided data into a single EagerIterator. + + Arguments: + x: NumPy array or EagerTensor, or list of Numpy arrays or EagerTensors + representing inputs to a model. + y: Optional. NumPy array or EagerTensor, or list of Numpy arrays or + EagerTensors representing targets of a model. + sample_weights: Optional NumPy array or EagerTensor representing sample + weights. + batch_size: Used to batch data and calculate how many steps EagerIterator + should take per epoch. + steps_per_epoch: If provided, how many steps EagerIterator should take per + epoch. + epochs: Epochs to repeat iterator for. + shuffle: Whether to shuffle data after each epoch. + + Raises: + ValueError: if steps_per_epoch cannot be calculated from the data + provided. + + Returns: + (Iterator, steps_per_epoch). + + """ + if isinstance(x, iterator_ops.EagerIterator): + return x, steps_per_epoch + + if not _nested_any(sample_weights, lambda x: x is None): + data = (x, y, sample_weights) + elif not _nested_any(y, lambda x: x is None): + data = (x, y) + else: + # always wrap in a tuple, so we know y, sample_weights weren't set + # even when x has multiple elements + data = (x,) + + data = _convert_lists_to_tuples(data) + if steps_per_epoch is None and batch_size is not None: + num_samples = _get_batch_axis_size(data) + steps_per_epoch = int(math.ceil(num_samples / batch_size)) + + if steps_per_epoch is None: + raise ValueError('Could not determine steps_per_epoch.' + 'Please provide either batch_size or' + 'steps_per_epoch.') + + # TODO(omalleyt) for NumPy arrays in graph mode + # placeholder ops should be used + # this is only ideal for eager mode + dataset = dataset_ops.Dataset.from_tensor_slices(data) + + if batch_size is not None: + dataset = dataset.batch(batch_size) + if shuffle: + dataset = dataset.shuffle(buffer_size=10000) + dataset = dataset.repeat(epochs) + iterator = dataset.make_one_shot_iterator() + + return iterator, steps_per_epoch + + def check_num_samples(ins, batch_size=None, steps=None, @@ -128,8 +259,8 @@ def standardize_input_data(data, except KeyError as e: raise ValueError('No data provided for "' + e.args[0] + '". Need data ' 'for each key in: ' + str(names)) - elif isinstance(data, list): - if isinstance(data[0], list): + elif isinstance(data, (list, tuple)): + if isinstance(data[0], (list, tuple)): data = [np.asarray(d) for d in data] elif len(names) == 1 and isinstance(data[0], (float, int)): data = [np.asarray(data)] @@ -482,6 +613,9 @@ def standardize_weights(y, Raises: ValueError: In case of invalid user-provided arguments. """ + # Iterator may return sample_weight as 1-tuple + if isinstance(sample_weight, tuple): + sample_weight = sample_weight[0] if sample_weight_mode is not None: if sample_weight_mode != 'temporal': raise ValueError('"sample_weight_mode ' diff --git a/tensorflow/python/keras/engine/training_utils_test.py b/tensorflow/python/keras/engine/training_utils_test.py new file mode 100644 index 0000000000..297a1ae494 --- /dev/null +++ b/tensorflow/python/keras/engine/training_utils_test.py @@ -0,0 +1,150 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for training utility functions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.keras.engine import training_utils +from tensorflow.python.platform import test + + +class TrainingUtilTest(test.TestCase): + + @test_util.run_in_graph_and_eager_modes + def test_convert_to_iterator_single_numpy(self): + batch_size = 2 + a = np.ones([10, 10]) + iterator, steps_per_epoch = training_utils.convert_to_iterator( + x=a, batch_size=batch_size) + self.assertEquals(steps_per_epoch, 5) + + expected_batch = a[:batch_size, :] + actual_batch, = iterator.get_next() + self.assertAllEqual(expected_batch, actual_batch) + + @test_util.run_in_graph_and_eager_modes + def test_convert_to_iterator_single_tensor(self): + batch_size = 2 + a = ops.convert_to_tensor(np.ones([10, 10])) + iterator, steps_per_epoch = training_utils.convert_to_iterator( + x=a, batch_size=batch_size) + self.assertEquals(steps_per_epoch, 5) + + expected_batch = a[:batch_size, :] + actual_batch, = iterator.get_next() + self.assertAllEqual(expected_batch, actual_batch) + + @test_util.run_in_graph_and_eager_modes + def test_convert_to_iterator_y(self): + batch_size = 2 + a = np.ones([10, 100]) + b = np.ones([10, 10]) + iterator, steps_per_epoch = training_utils.convert_to_iterator( + x=a, y=b, batch_size=batch_size) + self.assertEquals(steps_per_epoch, 5) + + expected_x = a[:batch_size, :] + expected_y = b[:batch_size, :] + actual_x, actual_y = iterator.get_next() + self.assertAllEqual(expected_x, actual_x) + self.assertAllEqual(expected_y, actual_y) + + @test_util.run_in_graph_and_eager_modes + def test_convert_to_iterator_sample_weights(self): + batch_size = 2 + a = ops.convert_to_tensor(np.ones([10, 100])) + b = ops.convert_to_tensor(np.ones([10, 10])) + sw = ops.convert_to_tensor(np.ones([10])) + iterator, steps_per_epoch = training_utils.convert_to_iterator( + x=a, y=b, sample_weights=sw, batch_size=batch_size) + self.assertEquals(steps_per_epoch, 5) + + expected_x = a[:batch_size, :] + expected_y = b[:batch_size, :] + expected_sw = sw[:batch_size] + actual_x, actual_y, actual_sw = iterator.get_next() + self.assertAllEqual(expected_x, actual_x) + self.assertAllEqual(expected_y, actual_y) + self.assertAllEqual(expected_sw, actual_sw) + + @test_util.run_in_graph_and_eager_modes + def test_convert_to_iterator_nested(self): + batch_size = 2 + x = {'1': np.ones([10, 100]), '2': [np.zeros([10, 10]), np.ones([10, 20])]} + iterator, steps_per_epoch = training_utils.convert_to_iterator( + x=x, batch_size=batch_size) + self.assertEquals(steps_per_epoch, 5) + + expected_x1 = x['1'][:batch_size, :] + expected_x2_0 = x['2'][0][:batch_size, :] + expected_x2_1 = x['2'][1][:batch_size, :] + + actual_x, = iterator.get_next() + actual_x1 = actual_x['1'][:batch_size, :] + actual_x2_0 = actual_x['2'][0][:batch_size, :] + actual_x2_1 = actual_x['2'][1][:batch_size, :] + + self.assertAllEqual(expected_x1, actual_x1) + self.assertAllEqual(expected_x2_0, actual_x2_0) + self.assertAllEqual(expected_x2_1, actual_x2_1) + + @test_util.run_in_graph_and_eager_modes + def test_convert_to_iterator_epochs(self): + batch_size = 2 + a = np.ones([10, 10]) + iterator, steps_per_epoch = training_utils.convert_to_iterator( + x=a, batch_size=batch_size, epochs=2) + self.assertEquals(steps_per_epoch, 5) + + expected_batch = a[:batch_size, :] + # loop through one whole epoch + for _ in range(6): + actual_batch, = iterator.get_next() + self.assertAllEqual(expected_batch, actual_batch) + + @test_util.run_in_graph_and_eager_modes + def test_convert_to_iterator_insufficient_info(self): + # with batch_size and steps_per_epoch not set + with self.assertRaises(ValueError): + a = np.ones([10, 10]) + _ = training_utils.convert_to_iterator(x=a) + + def test_nested_all(self): + nested_data = {'a': True, 'b': [True, True, (False, True)]} + all_true = training_utils._nested_all(nested_data, lambda x: x) + self.assertEquals(all_true, False) + + nested_data = {'a': True, 'b': [True, True, (True, True)]} + all_true = training_utils._nested_all(nested_data, lambda x: x) + self.assertEquals(all_true, True) + + def test_nested_any(self): + nested_data = [False, {'a': False, 'b': (False, True)}] + any_true = training_utils._nested_any(nested_data, lambda x: x) + self.assertEquals(any_true, True) + + nested_data = [False, {'a': False, 'b': (False, False)}] + any_true = training_utils._nested_any(nested_data, lambda x: x) + self.assertEquals(any_true, False) + + +if __name__ == '__main__': + test.main() -- cgit v1.2.3 From 4a24f07a2c4d1f6bd9df5b7432506d1742e81da2 Mon Sep 17 00:00:00 2001 From: Rasmus Munk Larsen Date: Mon, 16 Jul 2018 13:45:50 -0700 Subject: Update trt_optimization_pass.cc --- tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc index b39c8bdd64..687e1eb7c2 100644 --- a/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc +++ b/tensorflow/contrib/tensorrt/convert/trt_optimization_pass.cc @@ -245,7 +245,7 @@ tensorflow::Status TRTOptimizationPass::Optimize( // If the last token is not an integer, it must be part of the name. // Otherwise it is port number. if (tokens.size() > 1 && - !strings::safe_strto32(tokens.back(), &dumm_port)) { + !strings::safe_strto32(tokens.back(), &dumm_port)) { StrAppend(&s, ":", tokens.back()); } nodes_to_preserve.push_back(s); -- cgit v1.2.3 From 1d38580d6db3dc48b916453c2ce6cce691a00fe6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 13:59:13 -0700 Subject: Fix androidx espresso-core gradle dependency PiperOrigin-RevId: 204802505 --- tensorflow/contrib/lite/examples/android/app/build.gradle | 2 +- tensorflow/contrib/lite/java/demo/app/build.gradle | 2 +- tensorflow/contrib/lite/java/ovic/demo/app/build.gradle | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/lite/examples/android/app/build.gradle b/tensorflow/contrib/lite/examples/android/app/build.gradle index 1ffb9dd377..eb7fd705e1 100644 --- a/tensorflow/contrib/lite/examples/android/app/build.gradle +++ b/tensorflow/contrib/lite/examples/android/app/build.gradle @@ -51,7 +51,7 @@ apply from: "download-models.gradle" dependencies { compile fileTree(dir: 'libs', include: ['*.jar']) - androidTestCompile('com.androidx.test.espresso:espresso-core:2.2.2', { + androidTestCompile('androidx.test.espresso:espresso-core:3.1.0-alpha3', { exclude group: 'com.android.support', module: 'support-annotations' }) compile 'org.tensorflow:tensorflow-lite:0.0.0-nightly' diff --git a/tensorflow/contrib/lite/java/demo/app/build.gradle b/tensorflow/contrib/lite/java/demo/app/build.gradle index 49868c5a75..92f04c651c 100644 --- a/tensorflow/contrib/lite/java/demo/app/build.gradle +++ b/tensorflow/contrib/lite/java/demo/app/build.gradle @@ -44,7 +44,7 @@ repositories { dependencies { compile fileTree(dir: 'libs', include: ['*.jar']) - androidTestCompile('com.androidx.test.espresso:espresso-core:2.2.2', { + androidTestCompile('androidx.test.espresso:espresso-core:3.1.0-alpha3', { exclude group: 'com.android.support', module: 'support-annotations' }) compile 'com.android.support:appcompat-v7:25.2.0' diff --git a/tensorflow/contrib/lite/java/ovic/demo/app/build.gradle b/tensorflow/contrib/lite/java/ovic/demo/app/build.gradle index 3f32d62e5c..2a08608bbb 100644 --- a/tensorflow/contrib/lite/java/ovic/demo/app/build.gradle +++ b/tensorflow/contrib/lite/java/ovic/demo/app/build.gradle @@ -43,7 +43,7 @@ repositories { dependencies { compile fileTree(dir: 'libs', include: ['*.jar']) - androidTestCompile('com.androidx.test.espresso:espresso-core:2.2.2', { + androidTestCompile('androidx.test.espresso:espresso-core:3.1.0-alpha3', { exclude group: 'com.android.support', module: 'support-annotations' }) compile 'com.android.support:appcompat-v7:25.2.0' -- cgit v1.2.3 From 646c179b2ca8e3366d2b0d27f5f14839aa49e658 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 14:02:31 -0700 Subject: Re-enable some flaky tests that were affected by low precision of fused winograd implementation of convolution backprop. PiperOrigin-RevId: 204803121 --- tensorflow/python/ops/parallel_for/BUILD | 1 - tensorflow/python/ops/parallel_for/gradients_test.py | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/parallel_for/BUILD b/tensorflow/python/ops/parallel_for/BUILD index 065c2caedc..6c804a50e7 100644 --- a/tensorflow/python/ops/parallel_for/BUILD +++ b/tensorflow/python/ops/parallel_for/BUILD @@ -125,5 +125,4 @@ cuda_py_test( "//tensorflow/python:random_ops", "//tensorflow/python/ops/losses", ], - tags = ["no_gpu"], # TODO(b/80127739): test is flaky ) diff --git a/tensorflow/python/ops/parallel_for/gradients_test.py b/tensorflow/python/ops/parallel_for/gradients_test.py index 310a2154f7..3a6d9149ad 100644 --- a/tensorflow/python/ops/parallel_for/gradients_test.py +++ b/tensorflow/python/ops/parallel_for/gradients_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function import functools +import os import time import numpy as np @@ -444,6 +445,10 @@ class GradientsTest(test.TestCase): self.run_and_assert_equal(pfor_outputs, while_outputs) def test_mnist_per_eg_grad(self): + # It looks like CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED + # configuration of Winograd can cause low precision output resulting in + # tests failing. So we disable that here. + os.environ["TF_ENABLE_WINOGRAD_NONFUSED"] = "0" data_format = ("channels_first" if test.is_gpu_available() else "channels_last") # Note that we we are setting training=False here so that dropout produces @@ -451,8 +456,13 @@ class GradientsTest(test.TestCase): pfor_outputs, while_outputs = create_mnist_per_eg_grad( 4, data_format, training=False) self.run_and_assert_equal(pfor_outputs, while_outputs, rtol=1e-3) + os.environ.pop("TF_ENABLE_WINOGRAD_NONFUSED", None) def test_mnist_per_eg_jacobian(self): + # It looks like CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED + # configuration of Winograd can cause low precision output resulting in + # tests failing. So we disable that here. + os.environ["TF_ENABLE_WINOGRAD_NONFUSED"] = "0" data_format = ("channels_first" if test.is_gpu_available() else "channels_last") # Note that we we are setting training=False here so that dropout produces @@ -460,6 +470,7 @@ class GradientsTest(test.TestCase): pfor_outputs, while_outputs = create_mnist_per_eg_jacobian( 2, data_format, training=False) self.run_and_assert_equal(pfor_outputs, while_outputs, rtol=1e-3) + os.environ.pop("TF_ENABLE_WINOGRAD_NONFUSED", None) def test_fc_jacobian(self): jacobians, per_eg_jacobians_pfor, per_eg_jacobians_while = ( -- cgit v1.2.3 From 6e97fb388ad00df87beb58ebc5a1b02bd6a5dff0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 14:07:29 -0700 Subject: Disable more TF_NEED_* on Windows by default. This simplifies ./configure process on Windows, we can remove them when we actually support the corresponding feature on Windows. PiperOrigin-RevId: 204804112 --- configure.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.py b/configure.py index df6259778e..cd9d5d32a0 100644 --- a/configure.py +++ b/configure.py @@ -1451,6 +1451,11 @@ def main(): # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on # Windows. environ_cp['TF_DOWNLOAD_CLANG'] = '0' + environ_cp['TF_ENABLE_XLA'] = '0' + environ_cp['TF_NEED_GDR'] = '0' + environ_cp['TF_NEED_VERBS'] = '0' + environ_cp['TF_NEED_MPI'] = '0' + environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0' if is_macos(): environ_cp['TF_NEED_JEMALLOC'] = '0' -- cgit v1.2.3 From c1322043a853601ec9561157b23a5c86cdadc689 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 14:11:24 -0700 Subject: Runtime improvements to triangular solve. PiperOrigin-RevId: 204804841 --- tensorflow/compiler/tf2xla/lib/BUILD | 1 + tensorflow/compiler/tf2xla/lib/triangular_solve.cc | 245 +++++++++++---------- tensorflow/compiler/tf2xla/lib/triangular_solve.h | 2 +- tensorflow/compiler/xla/client/lib/BUILD | 3 + tensorflow/compiler/xla/client/lib/numeric.cc | 31 ++- tensorflow/compiler/xla/client/lib/numeric.h | 3 + 6 files changed, 168 insertions(+), 117 deletions(-) diff --git a/tensorflow/compiler/tf2xla/lib/BUILD b/tensorflow/compiler/tf2xla/lib/BUILD index becc8b84fe..30039e256a 100644 --- a/tensorflow/compiler/tf2xla/lib/BUILD +++ b/tensorflow/compiler/tf2xla/lib/BUILD @@ -120,6 +120,7 @@ cc_library( "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla/client/lib:constants", + "//tensorflow/compiler/xla/client/lib:numeric", "//tensorflow/compiler/xla/client/xla_client:xla_builder", "//tensorflow/compiler/xla/client/xla_client:xla_computation", "//tensorflow/core:lib", diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc index ce0f28db8f..e405f8dfaa 100644 --- a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc +++ b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/lib/batch_dot.h" #include "tensorflow/compiler/tf2xla/lib/util.h" #include "tensorflow/compiler/xla/client/lib/constants.h" +#include "tensorflow/compiler/xla/client/lib/numeric.h" #include "tensorflow/compiler/xla/client/xla_client/xla_builder.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/shape_util.h" @@ -150,40 +151,38 @@ xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side, int64 k = std::min(block_size, n - i); // output[..., :, i:i+k] = triangular_solve( - // a[..., i:i+k, i:i+k], b[..., :, i:i+k], ..., block_size=1) + // a[..., i:i+k, i:i+k], + // b[..., :, i:i+k] - np.matmul(output[..., :, :i], + // a[..., :i, i:i+k]), + // ..., block_size=1) auto a_slice = SliceInMinorDims(a, {i, i}, {i + k, i + k}); auto b_slice = SliceInMinorDims(b, {0, i}, {m, i + k}); + + // Note that we multiply with the full output, since this is faster + // than slicing, and output[..., :, i:] = 0 + xla::XlaOp a_prev; + if (lower) { + a_prev = SliceInMinorDims(a, {i, 0}, {i + k, n}); + } else { + a_prev = SliceInMinorDims(a, {0, i}, {n, i + k}); + } + auto prev_contribution = BatchDot(output, a_prev, + /*transpose_x=*/false, + /*transpose_y=*/transpose_a, + /*conjugate_x=*/false, + /*conjugate_y=*/conjugate_a); + auto to_solve = b_slice - prev_contribution; + xla::XlaOp update; if (k > 1) { TF_ASSIGN_OR_RETURN(xla::XlaComputation * solve, get_base_triangular_solve(k)); - update = xla::Call(builder, *solve, {a_slice, b_slice}); + update = xla::Call(builder, *solve, {a_slice, to_solve}); } else { auto a_slice_conj = MaybeConjugate(a_slice, conjugate_a); - update = b_slice / a_slice_conj; + update = to_solve / a_slice_conj; } output = UpdateSliceInMinorDims(output, update, {0, i}); - - // if i + k < a.shape[-1]: - // a_slice_2 = a[..., i+k:, i:i+k] if lower else a[..., i:i+k, i+k:] - // a_slice_2 = T(a_slice_2) if transpose_a else a_slice_2 - // b[..., :, i+k:] -= np.matmul(output[..., :, i:i+k], a_slice_2) - if (i + k < n) { - xla::XlaOp a_slice_2; - if (lower) { - a_slice_2 = SliceInMinorDims(a, {i + k, i}, {n, i + k}); - } else { - a_slice_2 = SliceInMinorDims(a, {i, i + k}, {i + k, n}); - } - - auto b_update = BatchDot(update, a_slice_2, - /*transpose_x=*/false, - /*transpose_y=*/transpose_a, - /*conjugate_x=*/false, - /*conjugate_y=*/conjugate_a); - auto b_slice_2 = SliceInMinorDims(b, {0, i + k}, {m, n}); - b = UpdateSliceInMinorDims(b, b_slice_2 - b_update, {0, i + k}); - } } } else if (left_side && lower != transpose_a) { @@ -192,40 +191,36 @@ xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side, int64 k = std::min(block_size, m - i); // output[..., i:i+k, :] = triangular_solve( - // a[..., i:i+k, i:i+k], b[..., i:i+k, :], ..., block_size=1) + // a[..., i:i+k, i:i+k], + // b[..., i:i+k, :] - np.matmul(a[..., i:i+k, :i], + // output[..., :i, :]), + // ..., block_size=1) auto a_slice = SliceInMinorDims(a, {i, i}, {i + k, i + k}); auto b_slice = SliceInMinorDims(b, {i, 0}, {i + k, n}); + + xla::XlaOp a_prev; + if (lower) { + a_prev = SliceInMinorDims(a, {i, 0}, {i + k, m}); + } else { + a_prev = SliceInMinorDims(a, {0, i}, {m, i + k}); + } + auto prev_contribution = BatchDot(a_prev, output, + /*transpose_x=*/transpose_a, + /*transpose_y=*/false, + /*conjugate_x=*/conjugate_a, + /*conjugate_y=*/false); + auto to_solve = b_slice - prev_contribution; + xla::XlaOp update; if (k > 1) { TF_ASSIGN_OR_RETURN(xla::XlaComputation * solve, get_base_triangular_solve(k)); - update = xla::Call(builder, *solve, {a_slice, b_slice}); + update = xla::Call(builder, *solve, {a_slice, to_solve}); } else { auto a_slice_conj = MaybeConjugate(a_slice, conjugate_a); - update = b_slice / a_slice_conj; + update = to_solve / a_slice_conj; } output = UpdateSliceInMinorDims(output, update, {i, 0}); - - // if i + k < a.shape[-1]: - // a_slice_2 = a[..., i+k:, i:i+k] if lower else a[..., i:i+k, i+k:] - // a_slice_2 = T(a_slice_2) if transpose_a else a_slice_2 - // b[..., i+k:, :] -= np.matmul(a_slice_2, output[..., i:i+k, :]) - if (i + k < m) { - xla::XlaOp a_slice_2; - if (lower) { - a_slice_2 = SliceInMinorDims(a, {i + k, i}, {m, i + k}); - } else { - a_slice_2 = SliceInMinorDims(a, {i, i + k}, {i + k, m}); - } - - auto b_update = BatchDot(a_slice_2, update, - /*transpose_x=*/transpose_a, - /*transpose_y=*/false, - /*conjugate_x=*/conjugate_a, - /*conjugate_y=*/false); - auto b_slice_2 = SliceInMinorDims(b, {i + k, 0}, {m, n}); - b = UpdateSliceInMinorDims(b, b_slice_2 - b_update, {i + k, 0}); - } } } else if (!left_side && lower != transpose_a) { // for i in reversed(range(0, a.shape[-1], block_size)): @@ -234,41 +229,37 @@ xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side, for (int64 i = last_blk_ix; i >= 0; i -= block_size) { int64 k = std::min(block_size, n - i); - // output[..., :, i:i+k] triangular_solve( - // a[..., i:i+k, i:i+k], b[..., :, i:i+k], ..., block_size=1) + // output[..., :, i:i+k] = triangular_solve( + // a[..., i:i+k, i:i+k], + // b[..., :, i:i+k] - np.matmul(output[..., :, :i], + // a[..., :i, i:i+k]),\ + // ..., block_size=1) auto a_slice = SliceInMinorDims(a, {i, i}, {i + k, i + k}); auto b_slice = SliceInMinorDims(b, {0, i}, {m, i + k}); + + xla::XlaOp a_prev; + if (lower) { + a_prev = SliceInMinorDims(a, {0, i}, {n, i + k}); + } else { + a_prev = SliceInMinorDims(a, {i, 0}, {i + k, n}); + } + auto prev_contribution = BatchDot(output, a_prev, + /*transpose_x=*/false, + /*transpose_y=*/transpose_a, + /*conjugate_x=*/false, + /*conjugate_y=*/conjugate_a); + auto to_solve = b_slice - prev_contribution; + xla::XlaOp update; if (k > 1) { TF_ASSIGN_OR_RETURN(xla::XlaComputation * solve, get_base_triangular_solve(k)); - update = xla::Call(builder, *solve, {a_slice, b_slice}); + update = xla::Call(builder, *solve, {a_slice, to_solve}); } else { auto a_slice_conj = MaybeConjugate(a_slice, conjugate_a); - update = b_slice / a_slice_conj; + update = to_solve / a_slice_conj; } output = UpdateSliceInMinorDims(output, update, {0, i}); - - // if i - k >= 0: - // a_slice_2 = a[..., i:i+k, :i] if lower else a[..., :i, i:i+k] - // a_slice_2 = T(a_slice_2) if transpose_a else a_slice_2 - // b[..., :, :i] -= np.matmul(out[..., :, i:i+k], a_slice_2) - if (i - k >= 0) { - xla::XlaOp a_slice_2; - if (lower) { - a_slice_2 = SliceInMinorDims(a, {i, 0}, {i + k, i}); - } else { - a_slice_2 = SliceInMinorDims(a, {0, i}, {i, i + k}); - } - - auto b_update = BatchDot(update, a_slice_2, - /*transpose_x=*/false, - /*transpose_y=*/transpose_a, - /*conjugate_x=*/false, - /*conjugate_y=*/conjugate_a); - auto b_slice_2 = SliceInMinorDims(b, {0, 0}, {m, i}); - b = UpdateSliceInMinorDims(b, b_slice_2 - b_update, {0, 0}); - } } } else { // left_side && lower == transpose_a // for i in reversed(range(0, a.shape[-1], block_size)): @@ -277,41 +268,37 @@ xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side, for (int64 i = last_blk_ix; i >= 0; i -= block_size) { int64 k = std::min(block_size, m - i); - // output[..., i:i+k, :] triangular_solve( - // a[..., i:i+k, i:i+k], b[..., i:i+k, :], ..., block_size=1) + // output[..., i:i+k, :] = triangular_solve( + // a[..., i:i+k, i:i+k], + // b[..., i:i+k, :] - np.matmul(a[..., i:i+k, :i], + // output[..., :i, :]), + // ..., block_size=1) auto a_slice = SliceInMinorDims(a, {i, i}, {i + k, i + k}); auto b_slice = SliceInMinorDims(b, {i, 0}, {i + k, n}); + + xla::XlaOp a_prev; + if (lower) { + a_prev = SliceInMinorDims(a, {0, i}, {m, i + k}); + } else { + a_prev = SliceInMinorDims(a, {i, 0}, {i + k, m}); + } + auto prev_contribution = BatchDot(a_prev, output, + /*transpose_x=*/transpose_a, + /*transpose_y=*/false, + /*conjugate_x=*/conjugate_a, + /*conjugate_y=*/false); + auto to_solve = b_slice - prev_contribution; + xla::XlaOp update; if (k > 1) { TF_ASSIGN_OR_RETURN(xla::XlaComputation * solve, get_base_triangular_solve(k)); - update = xla::Call(builder, *solve, {a_slice, b_slice}); + update = xla::Call(builder, *solve, {a_slice, to_solve}); } else { auto a_slice_conj = MaybeConjugate(a_slice, conjugate_a); - update = b_slice / a_slice_conj; + update = to_solve / a_slice_conj; } output = UpdateSliceInMinorDims(output, update, {i, 0}); - - // if i - k >= 0: - // a_slice_2 = a[..., i:i+k, :i] if lower else a[..., :i, i:i+k] - // a_slice_2 = T(a_slice_2) if transpose_a else a_slice_2 - // b[..., :i, :] -= np.matmul(a_slice_2, out[..., i:i+k, :]) - if (i - k >= 0) { - xla::XlaOp a_slice_2; - if (lower) { - a_slice_2 = SliceInMinorDims(a, {i, 0}, {i + k, i}); - } else { - a_slice_2 = SliceInMinorDims(a, {0, i}, {i, i + k}); - } - - auto b_update = BatchDot(a_slice_2, update, - /*transpose_x=*/transpose_a, - /*transpose_y=*/false, - /*conjugate_x=*/conjugate_a, - /*conjugate_y=*/false); - auto b_slice_2 = SliceInMinorDims(b, {0, 0}, {i, n}); - b = UpdateSliceInMinorDims(b, b_slice_2 - b_update, {0, 0}); - } } } @@ -330,9 +317,24 @@ xla::XlaOp TriangularSolveLeftLooking(xla::XlaOp a, xla::XlaOp b, const int64 ndims = xla::ShapeUtil::Rank(a_shape); std::vector batch_dimensions; + int64 num_batches = 1; for (int i = 0; i < ndims - 2; ++i) { int64 a_size = a_shape.dimensions(i); batch_dimensions.push_back(a_size); + num_batches = num_batches * a_size; + } + + // Rescale the input to be unit triangular + auto diag = Diagonal(a); + xla::XlaOp scaled_a; + std::vector broadcast_dimensions(ndims - 1); + std::iota(broadcast_dimensions.begin(), broadcast_dimensions.end(), 0); + if (transpose_a) { + scaled_a = Div(a, diag, broadcast_dimensions); + } else { + // Broadcast over the rows + broadcast_dimensions[ndims - 2] = ndims - 1; + scaled_a = Div(a, diag, broadcast_dimensions); } // The main computation is performed in a While loop. @@ -346,7 +348,7 @@ xla::XlaOp TriangularSolveLeftLooking(xla::XlaOp a, xla::XlaOp b, xla::XlaOp output = xla::ZerosLike(b); { auto i = transpose_a ? m - 1 : 0; - auto a_slice = SliceInMinorDims(a, {i, i}, {i + 1, i + 1}); + auto a_slice = SliceInMinorDims(scaled_a, {i, i}, {i + 1, i + 1}); auto b_slice = SliceInMinorDims(b, {i, 0}, {i + 1, n}); auto a_slice_conj = MaybeConjugate(a_slice, conjugate_a); auto update = b_slice / a_slice_conj; @@ -369,7 +371,7 @@ xla::XlaOp TriangularSolveLeftLooking(xla::XlaOp a, xla::XlaOp b, b_shape}; xla::Shape tuple_shape = xla::ShapeUtil::MakeTupleShape(tuple_shapes); auto init_i = xla::ConstantR0(builder, transpose_a ? m - 2 : 1); - auto init = xla::Tuple(builder, {init_i, output, a, b}); + auto init = xla::Tuple(builder, {init_i, output, scaled_a, b}); // Construct the loop condition function, // def cond_fun(loop_carry): @@ -445,11 +447,8 @@ xla::XlaOp TriangularSolveLeftLooking(xla::XlaOp a, xla::XlaOp b, DynamicSliceInMinorDims(body_b, {i, zero}, {1, n}); auto result_row = result_row_slice - b_update; - // body_out[..., i:i+1, :] = result_row / a[..., i:i+1, i:i+1] - auto a_elt = DynamicSliceInMinorDims(body_a, {i, i}, {1, 1}); - auto a_elt_conj = MaybeConjugate(a_elt, conjugate_a); - auto div_result = xla::Div(result_row, a_elt_conj); - body_out = DynamicUpdateSliceInMinorDims(body_out, div_result, {i, zero}); + // body_out[..., i:i+1, :] = result_row + body_out = DynamicUpdateSliceInMinorDims(body_out, result_row, {i, zero}); // if transpose_a: // return (i - 1, body_out, a, b) @@ -464,7 +463,11 @@ xla::XlaOp TriangularSolveLeftLooking(xla::XlaOp a, xla::XlaOp b, // Construct the While loop and return the result, // return while_loop(cond_fun, body_fun, init)[1] auto triangular_solve_left_looking_while = xla::While(cond, body, init); - return xla::GetTupleElement(triangular_solve_left_looking_while, 1); + output = xla::GetTupleElement(triangular_solve_left_looking_while, 1); + auto scaling = MaybeConjugate(diag, conjugate_a); + // Broadcast over the columns + broadcast_dimensions[ndims - 2] = ndims - 2; + return Div(output, scaling, broadcast_dimensions); }); } @@ -479,9 +482,24 @@ xla::XlaOp TriangularSolveRightLooking(xla::XlaOp a, xla::XlaOp b, const int64 ndims = xla::ShapeUtil::Rank(a_shape); std::vector batch_dimensions; + int64 num_batches = 1; for (int i = 0; i < ndims - 2; ++i) { int64 a_size = a_shape.dimensions(i); batch_dimensions.push_back(a_size); + num_batches = num_batches * a_size; + } + + // Rescale the input to be unit triangular + auto diag = Diagonal(a); + xla::XlaOp scaled_a; + std::vector broadcast_dimensions(ndims - 1); + std::iota(broadcast_dimensions.begin(), broadcast_dimensions.end(), 0); + if (transpose_a) { + // Broadcast over the rows + broadcast_dimensions[ndims - 2] = ndims - 1; + scaled_a = Div(a, diag, broadcast_dimensions); + } else { + scaled_a = Div(a, diag, broadcast_dimensions); } // The main computation is performed in a While loop. @@ -503,7 +521,7 @@ xla::XlaOp TriangularSolveRightLooking(xla::XlaOp a, xla::XlaOp b, b_shape}; xla::Shape tuple_shape = xla::ShapeUtil::MakeTupleShape(tuple_shapes); auto init_i = xla::ConstantR0(builder, transpose_a ? 0 : n - 1); - auto init = xla::Tuple(builder, {init_i, output, a, b}); + auto init = xla::Tuple(builder, {init_i, output, scaled_a, b}); // Construct the loop condition function, // def cond_fun(loop_carry): @@ -568,11 +586,8 @@ xla::XlaOp TriangularSolveRightLooking(xla::XlaOp a, xla::XlaOp b, /*conjugate_x=*/false, /*conjugate_y=*/conjugate_a); - // body_out[..., :, i:i+1] = b_update / a[..., i:i+1, i:i+1] - auto a_ii = DynamicSliceInMinorDims(body_a, {i, i}, {1, 1}); - auto a_ii_conj = MaybeConjugate(a_ii, conjugate_a); - body_out = DynamicUpdateSliceInMinorDims(body_out, b_update / a_ii_conj, - {zero, i}); + // body_out[..., :, i:i+1] = b_update + body_out = DynamicUpdateSliceInMinorDims(body_out, b_update, {zero, i}); // if transpose_a: // return (i + 1, body_out, a, b) @@ -587,7 +602,11 @@ xla::XlaOp TriangularSolveRightLooking(xla::XlaOp a, xla::XlaOp b, // Construct the While loop and return the result, // return while_loop(cond_fun, body_fun, init)[1] auto triangular_solve_left_looking_while = xla::While(cond, body, init); - return xla::GetTupleElement(triangular_solve_left_looking_while, 1); + output = xla::GetTupleElement(triangular_solve_left_looking_while, 1); + auto scaling = MaybeConjugate(diag, conjugate_a); + // Broadcast over the rows + broadcast_dimensions[ndims - 2] = ndims - 1; + return Div(output, scaling, broadcast_dimensions); }); } diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.h b/tensorflow/compiler/tf2xla/lib/triangular_solve.h index 80c2bc4c9c..7eb9238014 100644 --- a/tensorflow/compiler/tf2xla/lib/triangular_solve.h +++ b/tensorflow/compiler/tf2xla/lib/triangular_solve.h @@ -59,7 +59,7 @@ namespace tensorflow { // blocking is used. xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side, bool lower, bool transpose_a, bool conjugate_a, - int64 block_size = 256); + int64 block_size = 128); xla::XlaOp TriangularSolveLeftLooking(xla::XlaOp a, xla::XlaOp b, bool transpose_a, bool conjugate_a); diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD index ece5a885b5..77ba474cf6 100644 --- a/tensorflow/compiler/xla/client/lib/BUILD +++ b/tensorflow/compiler/xla/client/lib/BUILD @@ -97,9 +97,12 @@ cc_library( srcs = ["numeric.cc"], hdrs = ["numeric.h"], deps = [ + ":arithmetic", + ":constants", "//tensorflow/compiler/xla:types", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client/xla_client:xla_builder", + "//tensorflow/core:lib", ], ) diff --git a/tensorflow/compiler/xla/client/lib/numeric.cc b/tensorflow/compiler/xla/client/lib/numeric.cc index fd4e8fc390..cdbeb189f4 100644 --- a/tensorflow/compiler/xla/client/lib/numeric.cc +++ b/tensorflow/compiler/xla/client/lib/numeric.cc @@ -13,11 +13,14 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/compiler/xla/client/lib/numeric.h" - #include #include +#include "tensorflow/compiler/xla/client/lib/arithmetic.h" +#include "tensorflow/compiler/xla/client/lib/constants.h" +#include "tensorflow/compiler/xla/client/lib/numeric.h" +#include "tensorflow/core/lib/gtl/array_slice.h" + namespace xla { namespace { @@ -28,7 +31,7 @@ XlaOp MakeIota(XlaBuilder* builder, int64 size) { for (int64 i = 0; i < size; ++i) { values[i] = static_cast(i); } - return xla::ConstantR1(builder, values); + return ConstantR1(builder, values); } } // namespace @@ -76,4 +79,26 @@ XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m, return ConvertElementType(indicator, type); } +XlaOp Diagonal(XlaOp x) { + XlaBuilder* builder = x.builder(); + return builder->ReportErrorOrReturn([&]() -> StatusOr { + TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x)); + const int64 n_dims = ShapeUtil::Rank(shape); + TF_RET_CHECK(n_dims >= 2); + const int64 n = shape.dimensions(n_dims - 1); + const int64 m = shape.dimensions(n_dims - 2); + tensorflow::gtl::ArraySlice major_dims( + AsInt64Slice(shape.dimensions()), /*pos=*/0, /*len=*/n_dims - 2); + auto a = Iota(builder, U32, n); + auto b = Iota(builder, U32, m); + auto indicator = Eq(a, Broadcast(b, {n}), /*broadcast_dimensions=*/{0}); + auto mask = Broadcast(indicator, major_dims); + XlaComputation add = + CreateScalarAddComputation(shape.element_type(), builder); + auto diag = Reduce(Select(mask, x, Zeros(builder, shape)), ScalarLike(x, 0), + add, {n_dims - 1}); + return diag; + }); +} + } // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/numeric.h b/tensorflow/compiler/xla/client/lib/numeric.h index 79707007b2..3ec084636b 100644 --- a/tensorflow/compiler/xla/client/lib/numeric.h +++ b/tensorflow/compiler/xla/client/lib/numeric.h @@ -29,6 +29,9 @@ XlaOp Iota(XlaBuilder* builder, PrimitiveType type, int64 size); // else. XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m, int64 n); +// Get the diagonals of the last two dimensions. +XlaOp Diagonal(XlaOp x); + } // namespace xla #endif // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_NUMERIC_H_ -- cgit v1.2.3 From 33af29b33f14cc74725ff081d50e6e59247ef546 Mon Sep 17 00:00:00 2001 From: Jeremy Lau Date: Mon, 16 Jul 2018 14:19:00 -0700 Subject: Automated rollback of commit 590af170ca85a4921db0c28e4fa2785462bdcebd PiperOrigin-RevId: 204806075 --- tensorflow/contrib/tpu/python/tpu/tpu_context.py | 10 +- tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 101 +++------------------ 2 files changed, 20 insertions(+), 91 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py index e54395f05d..211c59cb90 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py @@ -234,7 +234,7 @@ class _InternalTPUContext(object): def mode(self): return self._assert_mode() - def master_address(self): + def _get_master_address(self): mode = self._assert_mode() config = self._config master = ( @@ -244,7 +244,7 @@ class _InternalTPUContext(object): def _get_tpu_system_metadata(self): """Gets the (maybe cached) TPU system metadata.""" - master = self.master_address() + master = self._get_master_address() tpu_system_metadata = self._lazy_tpu_system_metadata_dict.get(master) if tpu_system_metadata is not None: return tpu_system_metadata @@ -261,7 +261,7 @@ class _InternalTPUContext(object): def _get_device_assignment(self): """Gets the (maybe cached) TPU device assignment.""" - master = self.master_address() + master = self._get_master_address() device_assignment = self._lazy_device_assignment_dict.get(master) if device_assignment is not None: return device_assignment @@ -589,7 +589,7 @@ class _InternalTPUContext(object): 'model-parallelism, the total number of TPU cores should be ' 'num_cores_per_replica * num_replicas. Please set it ' 'accordingly or leave it as `None`'.format( - self.master_address(), num_replicas, + self._get_master_address(), num_replicas, user_provided_num_replicas)) raise ValueError(message) @@ -644,7 +644,7 @@ class _OneCoreTPUContext(_InternalTPUContext): def _get_tpu_system_metadata(self): """Gets the (maybe cached) TPU system metadata.""" - master = self.master_address() + master = self._get_master_address() tpu_system_metadata = self._lazy_tpu_system_metadata_dict.get(master) if tpu_system_metadata is not None: return tpu_system_metadata diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index be6a5dc57d..718ea630a8 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -43,7 +43,6 @@ from tensorflow.contrib.training.python.training import hparam from tensorflow.core.framework import variable_pb2 from tensorflow.core.framework.summary_pb2 import Summary from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session as session_lib from tensorflow.python.data.ops import dataset_ops from tensorflow.python.estimator import estimator as estimator_lib from tensorflow.python.estimator import model_fn as model_fn_lib @@ -68,7 +67,6 @@ from tensorflow.python.saved_model import tag_constants from tensorflow.python.summary import summary from tensorflow.python.training import basic_session_run_hooks from tensorflow.python.training import evaluation -from tensorflow.python.training import monitored_session from tensorflow.python.training import session_run_hook from tensorflow.python.training import training from tensorflow.python.training import training_util @@ -384,14 +382,7 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook): def begin(self): logging.info('TPU job name %s', self._master_job) self._iterations_per_loop_var = _create_or_get_iterations_per_loop() - self._init_ops = [] - # For distributed sessions, we can't run initialize_system in a separate - # graph here because 'begin' is only invoked when the MonitoredSession is - # created. We need to reinitialize the system every time MonitoredSession - # creates an underlying tf.Session, so we initialize from Scaffold.finalize. - # See _get_and_wrap_scaffold for more details. - if self._master_job is None: - self._init_ops.append(tpu.initialize_system(job=self._master_job)) + self._init_ops = [tpu.initialize_system(job=self._master_job)] self._finalize_ops = [tpu.shutdown_system(job=self._master_job)] summary_writer_init_ops = contrib_summary.summary_writer_initializer_op() @@ -493,7 +484,7 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook): return _OpQueueContext(name=name, target=target, args=args) def after_create_session(self, session, coord): - logging.info('Running init_ops') + logging.info('Init TPU system') session.run(self._init_ops, options=config_pb2.RunOptions(timeout_in_ms=5 * 60 * 1000)) @@ -2709,7 +2700,7 @@ def _eval_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn): outputs_from_all_shards=False, device_assignment=ctx.device_assignment) - scaffold = _get_and_wrap_scaffold(captured_scaffold_fn, ctx) + scaffold = _get_scaffold(captured_scaffold_fn) return loss, host_calls, scaffold @@ -2732,7 +2723,7 @@ def _train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn): outputs_from_all_shards=False, device_assignment=ctx.device_assignment) - scaffold = _get_and_wrap_scaffold(captured_scaffold_fn, ctx) + scaffold = _get_scaffold(captured_scaffold_fn) return loss, host_call, scaffold @@ -2760,7 +2751,7 @@ def _predict_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn): num_shards=num_cores, outputs_from_all_shards=False) - scaffold = _get_and_wrap_scaffold(captured_scaffold_fn, ctx) + scaffold = _get_scaffold(captured_scaffold_fn) return dummy_predict_op, host_calls, scaffold @@ -2850,20 +2841,8 @@ class _CapturedObject(object): return self._object -def _get_and_wrap_scaffold(captured_scaffold_fn, ctx): - """Retrieves the Scaffold from `captured_scaffold_fn`. - - Also wraps the scaffold's finalize method to initialize the TPU after the - graph is finalized. - - Args: - captured_scaffold_fn: a `_CapturedObject` containing a scaffold_fn. - ctx: A `_InternalTPUContext` instance used to initialize the TPU. - - Returns: - The Scaffold produced by captured_scaffold_fn, wrapped to initialize the TPU - after the graph is finalized. - """ +def _get_scaffold(captured_scaffold_fn): + """Retrieves the Scaffold from `captured_scaffold_fn`.""" with _CapturingContext(message='Inside scaffold_fn'): scaffold_fn = captured_scaffold_fn.get() if scaffold_fn: @@ -2874,64 +2853,14 @@ def _get_and_wrap_scaffold(captured_scaffold_fn, ctx): else: scaffold = None - if scaffold is None: - # When master_address is None, we are using DirectSession, so we can't - # invoke initialize_system from finalize. See comments below. - if ctx.master_address() is None: - return scaffold - scaffold = monitored_session.Scaffold() - - wrapped_finalize = scaffold.finalize - - def _finalize(): - """Invoke wrapped_finalize and initialize the TPU.""" - with _CapturingContext('Inside Scaffold.finalize'): - wrapped_finalize() - # Run tpu.initialize_system in its own graph after finalizing the main graph - # for distributed sessions. This is necessary because the TPU must be - # initialized before the TPU graph rewrite pass runs. We can't put the - # initialization op in the main graph because the main graph also contains - # replicate ops created by tpu.shard. If we tried to run initialization from - # the main graph, the TPU graph rewrite pass would rewrite the replicate ops - # before actually evaluating the initialization ops. - # - # For distributed sessions, the master may independently restart. After a - # master restarts, the rewrite pass runs again when any op in the main graph - # runs, so we must reinitialize the system every time the main graph is - # finalized. - # - # Special case: When master_address is unset, we're using DirectSession. - # DirectSession resets device state between sessions, and uses - # place_pruned_graph. Initialization currently passes state to replication - # through the TPU_SYSTEM resource manager. Under DirectSession, this - # resource manager gets reset when init_session is closed, so DirectSession - # can't initialize here, and must instead initialize from the main graph's - # init_ops. This is possible with DirectSession because it uses - # place_pruned_graph, which removes unreferenced ops before invoking the - # rewrite pass. This makes it possible to run init_ops from the main graph, - # which contains both tpu.initialize_system and tpu.shard ops, without first - # triggering the TPU graph rewrite. We can't do this for distributed - # sessions because they don't support place_pruned_graph. - # - # TODO(b/110943344) Clean this up as part of the initialize_system dataflow - # cleanup. It should be possible to remove the special case for - # DirectSession and the other call to initialize_system from - # _obtain_topology, when topology info is always explicitly passed from - # tpu.initialize_system to tpu.shard, though this requires editing or - # rebuilding the main graph each time the master restarts. - if ctx.master_address() is None: - return - with ops.Graph().as_default(): - logging.info('Init TPU system master_address %s', ctx.master_address()) - with session_lib.Session( - ctx.master_address(), - config=ctx.config.session_config) as init_session: - run_options = config_pb2.RunOptions(timeout_in_ms=5 * 60 * 1000) - init_session.run( - tpu.initialize_system(job=ctx.master_job), options=run_options) - logging.info('TPU system initialized') - - scaffold.finalize = _finalize + if scaffold: + wrapped_finalize = scaffold.finalize + + def _finalize(): + with _CapturingContext('Inside Scaffold.finalize'): + wrapped_finalize() + + scaffold.finalize = _finalize return scaffold -- cgit v1.2.3 From f8f0d7f000349ab573f0d912c37ebc3675cc6154 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 14:38:44 -0700 Subject: Refactoring some of the boosted trees code for growing ensemble. PiperOrigin-RevId: 204809484 --- .../python/estimator/canned/boosted_trees.py | 439 +++++++++++++-------- 1 file changed, 269 insertions(+), 170 deletions(-) diff --git a/tensorflow/python/estimator/canned/boosted_trees.py b/tensorflow/python/estimator/canned/boosted_trees.py index 3c832c7569..3292e2724d 100644 --- a/tensorflow/python/estimator/canned/boosted_trees.py +++ b/tensorflow/python/estimator/canned/boosted_trees.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import abc import collections import functools @@ -384,6 +385,249 @@ class _StopAtAttemptsHook(session_run_hook.SessionRunHook): run_context.request_stop() +def _get_max_splits(tree_hparams): + """Calculates the max possible number of splits based on tree params.""" + # maximum number of splits possible in the whole tree =2^(D-1)-1 + max_splits = (1 << tree_hparams.max_depth) - 1 + return max_splits + + +class _EnsembleGrower(object): + """Abstract base class for different types of ensemble growers. + + Use it to receive training ops for growing and centering bias, depending + on the implementation (for example, in memory or accumulator-based + distributed): + grower = ...create subclass grower(tree_ensemble, tree_hparams) + grow_op = grower.grow_tree(stats_summaries_list, feature_ids_list, + last_layer_nodes_range) + training_ops.append(grow_op) + """ + + def __init__(self, tree_ensemble, tree_hparams): + """Initializes a grower object. + + Args: + tree_ensemble: A TreeEnsemble variable. + tree_hparams: TODO. collections.namedtuple for hyper parameters. + """ + self._tree_ensemble = tree_ensemble + self._tree_hparams = tree_hparams + + @abc.abstractmethod + def center_bias(self, center_bias_var, gradients, hessians): + """Centers bias, if ready, based on statistics. + + Args: + center_bias_var: A variable that will be updated when bias centering + finished. + gradients: A rank 2 tensor of gradients. + hessians: A rank 2 tensor of hessians. + + Returns: + An operation for centering bias. + """ + + @abc.abstractmethod + def grow_tree(self, stats_summaries_list, feature_ids_list, + last_layer_nodes_range): + """Grows a tree, if ready, based on provided statistics. + + Args: + stats_summaries_list: List of stats summary tensors, representing sums of + gradients and hessians for each feature bucket. + feature_ids_list: a list of lists of feature ids for each bucket size. + last_layer_nodes_range: A tensor representing ids of the nodes in the + current layer, to be split. + + Returns: + An op for growing a tree. + """ + + # ============= Helper methods =========== + + def _center_bias_fn(self, center_bias_var, mean_gradients, mean_hessians): + """Updates the ensembles and cache (if needed) with logits prior.""" + continue_centering = boosted_trees_ops.center_bias( + self._tree_ensemble.resource_handle, + mean_gradients=mean_gradients, + mean_hessians=mean_hessians, + l1=self._tree_hparams.l1, + l2=self._tree_hparams.l2) + return center_bias_var.assign(continue_centering) + + def _grow_tree_from_stats_summaries(self, stats_summaries_list, + feature_ids_list, last_layer_nodes_range): + """Updates ensemble based on the best gains from stats summaries.""" + node_ids_per_feature = [] + gains_list = [] + thresholds_list = [] + left_node_contribs_list = [] + right_node_contribs_list = [] + all_feature_ids = [] + assert len(stats_summaries_list) == len(feature_ids_list) + + max_splits = _get_max_splits(self._tree_hparams) + + for i, feature_ids in enumerate(feature_ids_list): + (numeric_node_ids_per_feature, numeric_gains_list, + numeric_thresholds_list, numeric_left_node_contribs_list, + numeric_right_node_contribs_list) = ( + boosted_trees_ops.calculate_best_gains_per_feature( + node_id_range=last_layer_nodes_range, + stats_summary_list=stats_summaries_list[i], + l1=self._tree_hparams.l1, + l2=self._tree_hparams.l2, + tree_complexity=self._tree_hparams.tree_complexity, + min_node_weight=self._tree_hparams.min_node_weight, + max_splits=max_splits)) + + all_feature_ids += feature_ids + node_ids_per_feature += numeric_node_ids_per_feature + gains_list += numeric_gains_list + thresholds_list += numeric_thresholds_list + left_node_contribs_list += numeric_left_node_contribs_list + right_node_contribs_list += numeric_right_node_contribs_list + + grow_op = boosted_trees_ops.update_ensemble( + # Confirm if local_tree_ensemble or tree_ensemble should be used. + self._tree_ensemble.resource_handle, + feature_ids=all_feature_ids, + node_ids=node_ids_per_feature, + gains=gains_list, + thresholds=thresholds_list, + left_node_contribs=left_node_contribs_list, + right_node_contribs=right_node_contribs_list, + learning_rate=self._tree_hparams.learning_rate, + max_depth=self._tree_hparams.max_depth, + pruning_mode=boosted_trees_ops.PruningMode.NO_PRUNING) + return grow_op + + +class _InMemoryEnsembleGrower(_EnsembleGrower): + """A base class for ensemble growers.""" + + def __init__(self, tree_ensemble, tree_hparams): + + super(_InMemoryEnsembleGrower, self).__init__( + tree_ensemble=tree_ensemble, tree_hparams=tree_hparams) + + def center_bias(self, center_bias_var, gradients, hessians): + # For in memory, we already have a full batch of gradients and hessians, + # so just take a mean and proceed with centering. + mean_gradients = array_ops.expand_dims( + math_ops.reduce_mean(gradients, 0), 0) + mean_heassians = array_ops.expand_dims(math_ops.reduce_mean(hessians, 0), 0) + return self._center_bias_fn(center_bias_var, mean_gradients, mean_heassians) + + def grow_tree(self, stats_summaries_list, feature_ids_list, + last_layer_nodes_range): + # For in memory, we already have full data in one batch, so we can grow the + # tree immediately. + return self._grow_tree_from_stats_summaries( + stats_summaries_list, feature_ids_list, last_layer_nodes_range) + + +class _AccumulatorEnsembleGrower(_EnsembleGrower): + """A base class for ensemble growers.""" + + def __init__(self, tree_ensemble, tree_hparams, stamp_token, + n_batches_per_layer, bucket_size_list, is_chief): + super(_AccumulatorEnsembleGrower, self).__init__( + tree_ensemble=tree_ensemble, tree_hparams=tree_hparams) + self._stamp_token = stamp_token + self._n_batches_per_layer = n_batches_per_layer + self._bucket_size_list = bucket_size_list + self._is_chief = is_chief + + def center_bias(self, center_bias_var, gradients, hessians): + # For not in memory situation, we need to accumulate enough of batches first + # before proceeding with centering bias. + + # Create an accumulator. + bias_dependencies = [] + bias_accumulator = data_flow_ops.ConditionalAccumulator( + dtype=dtypes.float32, + # The stats consist of grads and hessians means only. + # TODO(nponomareva): this will change for a multiclass + shape=[2, 1], + shared_name='bias_accumulator') + + grads_and_hess = array_ops.stack([gradients, hessians], axis=0) + grads_and_hess = math_ops.reduce_mean(grads_and_hess, axis=1) + + apply_grad = bias_accumulator.apply_grad(grads_and_hess, self._stamp_token) + bias_dependencies.append(apply_grad) + + # Center bias if enough batches were processed. + with ops.control_dependencies(bias_dependencies): + if not self._is_chief: + return control_flow_ops.no_op() + + def center_bias_from_accumulator(): + accumulated = array_ops.unstack(bias_accumulator.take_grad(1), axis=0) + return self._center_bias_fn(center_bias_var, + array_ops.expand_dims(accumulated[0], 0), + array_ops.expand_dims(accumulated[1], 0)) + + center_bias_op = control_flow_ops.cond( + math_ops.greater_equal(bias_accumulator.num_accumulated(), + self._n_batches_per_layer), + center_bias_from_accumulator, + control_flow_ops.no_op, + name='wait_until_n_batches_for_bias_accumulated') + return center_bias_op + + def grow_tree(self, stats_summaries_list, feature_ids_list, + last_layer_nodes_range): + # For not in memory situation, we need to accumulate enough of batches first + # before proceeding with building a tree layer. + max_splits = _get_max_splits(self._tree_hparams) + + # Prepare accumulators. + accumulators = [] + dependencies = [] + for i, feature_ids in enumerate(feature_ids_list): + stats_summaries = stats_summaries_list[i] + accumulator = data_flow_ops.ConditionalAccumulator( + dtype=dtypes.float32, + # The stats consist of grads and hessians (the last dimension). + shape=[len(feature_ids), max_splits, self._bucket_size_list[i], 2], + shared_name='numeric_stats_summary_accumulator_' + str(i)) + accumulators.append(accumulator) + + apply_grad = accumulator.apply_grad( + array_ops.stack(stats_summaries, axis=0), self._stamp_token) + dependencies.append(apply_grad) + + # Grow the tree if enough batches is accumulated. + with ops.control_dependencies(dependencies): + if not self._is_chief: + return control_flow_ops.no_op() + + min_accumulated = math_ops.reduce_min( + array_ops.stack([acc.num_accumulated() for acc in accumulators])) + + def grow_tree_from_accumulated_summaries_fn(): + """Updates tree with the best layer from accumulated summaries.""" + # Take out the accumulated summaries from the accumulator and grow. + stats_summaries_list = [] + stats_summaries_list = [ + array_ops.unstack(accumulator.take_grad(1), axis=0) + for accumulator in accumulators + ] + grow_op = self._grow_tree_from_stats_summaries( + stats_summaries_list, feature_ids_list, last_layer_nodes_range) + return grow_op + + grow_model = control_flow_ops.cond( + math_ops.greater_equal(min_accumulated, self._n_batches_per_layer), + grow_tree_from_accumulated_summaries_fn, + control_flow_ops.no_op, + name='wait_until_n_batches_accumulated') + return grow_model + + def _bt_model_fn( features, labels, @@ -441,11 +685,6 @@ def _bt_model_fn( raise ValueError('train_in_memory is supported only for ' 'non-distributed training.') worker_device = control_flow_ops.no_op().device - # maximum number of splits possible in the whole tree =2^(D-1)-1 - # TODO(youngheek): perhaps storage could be optimized by storing stats with - # the dimension max_splits_per_layer, instead of max_splits (for the entire - # tree). - max_splits = (1 << tree_hparams.max_depth) - 1 train_op = [] with ops.name_scope(name) as name: # Prepare. @@ -543,6 +782,11 @@ def _bt_model_fn( hessians = gradients_impl.gradients( gradients, logits, name='Hessians')[0] + # TODO(youngheek): perhaps storage could be optimized by storing stats + # with the dimension max_splits_per_layer, instead of max_splits (for the + # entire tree). + max_splits = _get_max_splits(tree_hparams) + stats_summaries_list = [] for i, feature_ids in enumerate(feature_ids_list): num_buckets = bucket_size_list[i] @@ -559,173 +803,28 @@ def _bt_model_fn( ] stats_summaries_list.append(summaries) - # ========= Helper methods for both in and not in memory. ============== - def grow_tree_from_stats_summaries(stats_summaries_list, - feature_ids_list): - """Updates ensemble based on the best gains from stats summaries.""" - node_ids_per_feature = [] - gains_list = [] - thresholds_list = [] - left_node_contribs_list = [] - right_node_contribs_list = [] - all_feature_ids = [] - - assert len(stats_summaries_list) == len(feature_ids_list) - - for i, feature_ids in enumerate(feature_ids_list): - (numeric_node_ids_per_feature, numeric_gains_list, - numeric_thresholds_list, numeric_left_node_contribs_list, - numeric_right_node_contribs_list) = ( - boosted_trees_ops.calculate_best_gains_per_feature( - node_id_range=last_layer_nodes_range, - stats_summary_list=stats_summaries_list[i], - l1=tree_hparams.l1, - l2=tree_hparams.l2, - tree_complexity=tree_hparams.tree_complexity, - min_node_weight=tree_hparams.min_node_weight, - max_splits=max_splits)) - - all_feature_ids += feature_ids - node_ids_per_feature += numeric_node_ids_per_feature - gains_list += numeric_gains_list - thresholds_list += numeric_thresholds_list - left_node_contribs_list += numeric_left_node_contribs_list - right_node_contribs_list += numeric_right_node_contribs_list - - grow_op = boosted_trees_ops.update_ensemble( - # Confirm if local_tree_ensemble or tree_ensemble should be used. - tree_ensemble.resource_handle, - feature_ids=all_feature_ids, - node_ids=node_ids_per_feature, - gains=gains_list, - thresholds=thresholds_list, - left_node_contribs=left_node_contribs_list, - right_node_contribs=right_node_contribs_list, - learning_rate=tree_hparams.learning_rate, - max_depth=tree_hparams.max_depth, - pruning_mode=boosted_trees_ops.PruningMode.NO_PRUNING) - return grow_op - - def _center_bias_fn(mean_gradients, mean_hessians): - """Updates the ensembles and cache (if needed) with logits prior.""" - continue_centering = boosted_trees_ops.center_bias( - tree_ensemble.resource_handle, - mean_gradients=mean_gradients, - mean_hessians=mean_hessians, - l1=tree_hparams.l1, - l2=tree_hparams.l2 - ) - return center_bias_var.assign(continue_centering) - - # ========= End of helper methods. ============== - if train_in_memory and is_single_machine: - train_op.append(distribute_lib.increment_var(global_step)) - - mean_gradients = array_ops.expand_dims( - math_ops.reduce_mean(gradients, 0), 0) - mean_heassians = array_ops.expand_dims( - math_ops.reduce_mean(hessians, 0), 0) - - train_op.append( - control_flow_ops.cond( - center_bias_var, - lambda: _center_bias_fn(mean_gradients, mean_heassians), - functools.partial(grow_tree_from_stats_summaries, - stats_summaries_list, feature_ids_list))) + grower = _InMemoryEnsembleGrower(tree_ensemble, tree_hparams) else: - - def center_bias_not_in_mem(): - """Accumulates the data and updates the logits bias, when ready.""" - bias_dependencies = [] - - bias_accumulator = data_flow_ops.ConditionalAccumulator( - dtype=dtypes.float32, - # The stats consist of grads and hessians means only. - # TODO(nponomareva): this will change for a multiclass - shape=[2, 1], - shared_name='bias_accumulator') - - grads_and_hess = array_ops.stack([gradients, hessians], axis=0) - grads_and_hess = math_ops.reduce_mean(grads_and_hess, axis=1) - - apply_grad = bias_accumulator.apply_grad(grads_and_hess, stamp_token) - bias_dependencies.append(apply_grad) - - def center_bias_from_accumulator(): - accumulated = array_ops.unstack( - bias_accumulator.take_grad(1), axis=0) - return _center_bias_fn( - array_ops.expand_dims(accumulated[0], 0), - array_ops.expand_dims(accumulated[1], 0)) - - with ops.control_dependencies(bias_dependencies): - if config.is_chief: - center_bias_op = control_flow_ops.cond( - math_ops.greater_equal(bias_accumulator.num_accumulated(), - n_batches_per_layer), - center_bias_from_accumulator, - control_flow_ops.no_op, - name='wait_until_n_batches_for_bias_accumulated') - - return center_bias_op - else: - return control_flow_ops.no_op() - - def grow_not_in_mem(): - """Accumulates the data and grows a layer when ready.""" - - accumulators = [] - dependencies = [] - for i, feature_ids in enumerate(feature_ids_list): - stats_summaries = stats_summaries_list[i] - accumulator = data_flow_ops.ConditionalAccumulator( - dtype=dtypes.float32, - # The stats consist of grads and hessians (the last dimension). - shape=[len(feature_ids), max_splits, bucket_size_list[i], 2], - shared_name='numeric_stats_summary_accumulator_' + str(i)) - accumulators.append(accumulator) - - apply_grad = accumulator.apply_grad( - array_ops.stack(stats_summaries, axis=0), stamp_token) - dependencies.append(apply_grad) - - def grow_tree_from_accumulated_summaries_fn(): - """Updates tree with the best layer from accumulated summaries.""" - # Take out the accumulated summaries from the accumulator and grow. - stats_summaries_list = [] - - stats_summaries_list = [ - array_ops.unstack(accumulator.take_grad(1), axis=0) - for accumulator in accumulators - ] - - grow_op = grow_tree_from_stats_summaries(stats_summaries_list, - feature_ids_list) - return grow_op - - with ops.control_dependencies(dependencies): - if config.is_chief: - min_accumulated = math_ops.reduce_min( - array_ops.stack( - [acc.num_accumulated() for acc in accumulators])) - - grow_model = control_flow_ops.cond( - math_ops.greater_equal(min_accumulated, n_batches_per_layer), - grow_tree_from_accumulated_summaries_fn, - control_flow_ops.no_op, - name='wait_until_n_batches_accumulated') - - return grow_model - else: - return control_flow_ops.no_op() - - update_model = control_flow_ops.cond( - center_bias_var, center_bias_not_in_mem, grow_not_in_mem) - train_op.append(update_model) - with ops.control_dependencies([update_model]): - increment_global = distribute_lib.increment_var(global_step) - train_op.append(increment_global) + grower = _AccumulatorEnsembleGrower(tree_ensemble, tree_hparams, + stamp_token, n_batches_per_layer, + bucket_size_list, config.is_chief) + + update_model = control_flow_ops.cond( + center_bias_var, + functools.partial( + grower.center_bias, + center_bias_var, + gradients, + hessians, + ), + functools.partial(grower.grow_tree, stats_summaries_list, + feature_ids_list, last_layer_nodes_range)) + train_op.append(update_model) + + with ops.control_dependencies([update_model]): + increment_global = distribute_lib.increment_var(global_step) + train_op.append(increment_global) return control_flow_ops.group(train_op, name='train_op') -- cgit v1.2.3 From bd6c04a86cd77d1b969d88fd243c80b7780b6db3 Mon Sep 17 00:00:00 2001 From: gracehoney <31743510+aaroey@users.noreply.github.com> Date: Mon, 16 Jul 2018 14:48:20 -0700 Subject: Refactor tf_trt_integration_test so we can extend it to other graphs. --- tensorflow/contrib/tensorrt/BUILD | 28 +- tensorflow/contrib/tensorrt/test/base_test.py | 125 ++++++++ tensorflow/contrib/tensorrt/test/base_unit_test.py | 118 ------- tensorflow/contrib/tensorrt/test/run_test.py | 184 ----------- .../tensorrt/test/tf_trt_integration_test.py | 347 --------------------- .../tensorrt/test/tf_trt_integration_test_base.py | 293 +++++++++++++++++ tensorflow/contrib/tensorrt/test/unit_tests.py | 67 ---- tensorflow/contrib/tensorrt/test/utilities.py | 30 -- 8 files changed, 433 insertions(+), 759 deletions(-) create mode 100644 tensorflow/contrib/tensorrt/test/base_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/base_unit_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/run_test.py delete mode 100644 tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py create mode 100644 tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py delete mode 100644 tensorflow/contrib/tensorrt/test/unit_tests.py delete mode 100644 tensorflow/contrib/tensorrt/test/utilities.py diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD index d957ca0861..7aed241fd0 100644 --- a/tensorflow/contrib/tensorrt/BUILD +++ b/tensorflow/contrib/tensorrt/BUILD @@ -11,7 +11,6 @@ exports_files(["LICENSE"]) load( "//tensorflow:tensorflow.bzl", - "py_test", "tf_cc_test", "tf_copts", "tf_cuda_library", @@ -20,6 +19,7 @@ load( "tf_gen_op_libs", "tf_gen_op_wrapper_py", ) +load("//tensorflow:tensorflow.bzl", "cuda_py_tests") load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test") load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc") @@ -33,7 +33,6 @@ tf_cuda_cc_test( size = "small", srcs = ["tensorrt_test.cc"], tags = [ - "manual", "notap", ], deps = [ @@ -311,7 +310,6 @@ tf_cuda_cc_test( size = "small", srcs = ["plugin/trt_plugin_factory_test.cc"], tags = [ - "manual", "notap", ], deps = [ @@ -325,15 +323,9 @@ tf_cuda_cc_test( ]), ) -py_test( - name = "tf_trt_integration_test", - srcs = ["test/tf_trt_integration_test.py"], - main = "test/tf_trt_integration_test.py", - srcs_version = "PY2AND3", - tags = [ - "manual", - "notap", - ], +py_library( + name = "tf_trt_integration_test_base", + srcs = ["test/tf_trt_integration_test_base.py"], deps = [ ":init_py", "//tensorflow/python:client_testlib", @@ -341,6 +333,17 @@ py_test( ], ) +cuda_py_tests( + name = "tf_trt_integration_test", + srcs = ["test/base_test.py"], + additional_deps = [ + ":tf_trt_integration_test_base", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_test_lib", + ], + prefix = "integration_test", +) + py_test( name = "converter_unit_tests", srcs = [ @@ -362,7 +365,6 @@ py_test( main = "test/unit_tests.py", srcs_version = "PY2AND3", tags = [ - "manual", "notap", ], deps = [ diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py new file mode 100644 index 0000000000..4b9e6d668f --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/base_test.py @@ -0,0 +1,125 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Basic tests for TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_ops +from tensorflow.python.platform import test +from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test + + +# TODO(aaroey): test graph with different dtypes. +def _GetSingleEngineGraphDef(dtype=dtypes.float32): + """Create a graph containing single segment.""" + input_dims = [100, 24, 24, 2] + g = ops.Graph() + with g.as_default(): + inp = array_ops.placeholder( + dtype=dtype, shape=[None] + input_dims[1:], name=trt_test.INPUT_NAME) + with g.device("/GPU:0"): + conv_filter = constant_op.constant( + [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]], + name="weights", + dtype=dtype) + conv = nn.conv2d( + input=inp, + filter=conv_filter, + strides=[1, 2, 2, 1], + padding="SAME", + name="conv") + bias = constant_op.constant( + [4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtype) + added = nn.bias_add(conv, bias, name="bias_add") + relu = nn.relu(added, "relu") + identity = array_ops.identity(relu, "identity") + pool = nn_ops.max_pool( + identity, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool") + array_ops.squeeze(pool, name=trt_test.OUTPUT_NAME) + return trt_test.TfTrtIntegrationTestParams( + graph_name="SimpleSingleEngine", + gdef=g.as_graph_def(), + input_dims=input_dims, + num_expected_engines=1, + expected_output_dims=(100, 6, 6, 6), + allclose_atol=1.e-03, + allclose_rtol=1.e-03) + + +# TODO(aaroey): test graph with different dtypes. +def _GetMultiEngineGraphDef(dtype=dtypes.float32): + """Create a graph containing multiple segment.""" + input_dims = [100, 24, 24, 2] + g = ops.Graph() + with g.as_default(): + inp = array_ops.placeholder( + dtype=dtype, shape=[None] + input_dims[1:], name=trt_test.INPUT_NAME) + with g.device("/GPU:0"): + conv_filter = constant_op.constant( + [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]], + name="weights", + dtype=dtype) + conv = nn.conv2d( + input=inp, + filter=conv_filter, + strides=[1, 2, 2, 1], + padding="SAME", + name="conv") + c1 = constant_op.constant( + np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype) + p = conv * c1 + c2 = constant_op.constant( + np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype) + q = conv / c2 + + edge = trt_test.TRT_INCOMPATIBLE_OP(q) + edge /= edge + r = edge + edge + + p -= edge + q *= edge + s = p + q + s -= r + array_ops.squeeze(s, name=trt_test.OUTPUT_NAME) + return trt_test.TfTrtIntegrationTestParams( + graph_name="SimpleMultipleEngines", + gdef=g.as_graph_def(), + input_dims=input_dims, + num_expected_engines=2, + expected_output_dims=(100, 12, 12, 6), + allclose_atol=1.e-03, + allclose_rtol=1.e-03) + + +class BaseTest(trt_test.TfTrtIntegrationTestBase): + """Class to test Tensorflow-TensorRT integration.""" + pass + + +if __name__ == "__main__": + # TODO(aaroey): add a large complex graph to test. + trt_test.AddTests(BaseTest, + [_GetSingleEngineGraphDef(), + _GetMultiEngineGraphDef()]) + test.main() diff --git a/tensorflow/contrib/tensorrt/test/base_unit_test.py b/tensorflow/contrib/tensorrt/test/base_unit_test.py deleted file mode 100644 index 8a6c648ab6..0000000000 --- a/tensorflow/contrib/tensorrt/test/base_unit_test.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Base class to facilitate development of integration tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np - - -class BaseUnitTest(object): - """Base class for unit tests in TF-TRT""" - - def __init__(self, log_file='log.txt'): - self.static_mode_list = {} - self.dynamic_mode_list = {} - self.dummy_input = None - self.get_network = None - self.expect_nb_nodes = None - self.test_name = None - self.log_file = log_file - self.ckpt = None - self.allclose_rtol = 0.01 - self.allclose_atol = 0.01 - self.allclose_equal_nan = True - # saves out graphdef - self.debug = False - # require node count check fail leads to test failure - self.check_node_count = False - - def run(self, run_test_context): - run_test_context.run_test(self.get_network, self.static_mode_list, - self.dynamic_mode_list, self.dummy_input, - self.ckpt) - return self.log_result(run_test_context) - - def log_result(self, run_test_result): - log = open(self.log_file, 'a') - log.write(("================= model: %s\n") % (self.test_name)) - - if self.debug: - open(self.test_name + "_native.pb", - 'wb').write(run_test_result.native_network.SerializeToString()) - all_success = True - if len(run_test_result.tftrt_conversion_flag) != 0: - log.write(" -- static_mode\n") - for static_mode in run_test_result.tftrt_conversion_flag: - if self.debug: - open(self.test_name + "_" + static_mode + ".pb", - 'wb').write(run_test_result.tftrt[static_mode].SerializeToString()) - log.write(" ----\n") - log.write((" mode: [%s]\n") % (static_mode)) - if run_test_result.tftrt_conversion_flag[static_mode]: - if run_test_result.tftrt_nb_nodes[static_mode] != self.expect_nb_nodes: - log.write( - ("[WARNING]: converted node number does not match (%d,%d,%d)!!!\n" - ) % (run_test_result.tftrt_nb_nodes[static_mode], - self.expect_nb_nodes, run_test_result.native_nb_nodes)) - if self.check_node_count: - all_success = False - - if np.array_equal(run_test_result.tftrt_result[static_mode], - run_test_result.native_result): - log.write(" output: equal\n") - elif np.allclose( - run_test_result.tftrt_result[static_mode], - run_test_result.native_result, - atol=self.allclose_atol, - rtol=self.allclose_rtol, - equal_nan=self.allclose_equal_nan): - log.write(" output: allclose\n") - else: - diff = run_test_result.tftrt_result[static_mode] - run_test_result.native_result - log.write("[ERROR]: output does not match!!!\n") - log.write("max diff: " + str(np.max(diff))) - log.write("\ntftrt:\n") - log.write(str(run_test_result.tftrt_result[static_mode])) - log.write("\nnative:\n") - log.write(str(run_test_result.native_result)) - log.write("\ndiff:\n") - log.write(str(diff)) - all_success = False - else: - log.write("[ERROR]: conversion failed!!!\n") - all_success = False - - if len(run_test_result.tftrt_dynamic_conversion_flag) != 0: - log.write(" -- dynamic_mode\n") - for dynamic_mode in run_test_result.tftrt_dynamic_conversion_flag: - log.write("\n ----\n") - log.write((" mode: [%s]\n") % (dynamic_mode)) - if run_test_result.tftrt_dynamic_conversion_flag[dynamic_mode]: - if np.array_equal(run_test_result.tftrt_dynamic_result[dynamic_mode], - run_test_result.native_result): - log.write(" output: equal\n") - elif np.allclose(run_test_result.tftrt_dynamic_result[dynamic_mode], - run_test_result.native_result): - log.write(" output: allclose\n") - else: - log.write("[ERROR]: output does not match!!!\n") - all_success = False - else: - log.write("[ERROR]: conversion failed!!!\n") - all_success = False - return all_success diff --git a/tensorflow/contrib/tensorrt/test/run_test.py b/tensorflow/contrib/tensorrt/test/run_test.py deleted file mode 100644 index 4d109cc378..0000000000 --- a/tensorflow/contrib/tensorrt/test/run_test.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""script to convert and execute TF-TensorRT graph.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 -from tensorflow.core.protobuf import rewriter_config_pb2 -from tensorflow.python.client import session -from tensorflow.python.framework import importer -from tensorflow.python.framework import ops -from tensorflow.python.training import training -from tensorflow.contrib.tensorrt.test.utilities import get_all_variables - -OUTPUT_NODE = "output" -INPUT_NODE = "input" -CALIB_COUNT = 5 # calibration iteration - - -class RunTest: - """base class to run TR-TRT conversion and execution""" - - def __init__(self): - self.clean() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.clean() - - def clean(self): - self.tftrt = {} - self.tftrt_conversion_flag = {} - self.tftrt_nb_nodes = {} - self.tftrt_result = {} - self.tftrt_dynamic_conversion_flag = {} - self.tftrt_dynamic_result = {} - self.check_file = None - self.native_network = None - - def run_test(self, - network, - static_mode_list, - dynamic_mode_list, - dummy_input, - file_name=None): - self.native_network = network() - success = True - initialization = False - if file_name != None: - initialization = True - self.check_file = file_name - self.native_result, self.native_nb_nodes = self.execute_graph( - self.native_network, dummy_input, initialization) - for mode in static_mode_list: - try: - self.run_static_convert_network(mode, dummy_input, initialization) - self.tftrt_conversion_flag[mode] = True - except Exception as inst: - self.tftrt_conversion_flag[mode] = False - success = False - for mode in dynamic_mode_list: - try: - self.run_dynamic_convert_network(mode, dummy_input, initialization) - self.tftrt_dynamic_conversion_flag[mode] = True - except Exception as inst: - self.tftrt_dynamic_conversion_flag[mode] = False - success = False - return success - - def run_dynamic_convert_network(self, mode, dummy_input, initialization=True): - inp_dims = dummy_input.shape - if mode == "FP32" or mode == "FP16": - opt_config = rewriter_config_pb2.RewriterConfig() - opt_config.optimizers.extend(["constfold", "layout"]) - custom_op = opt_config.custom_optimizers.add() - custom_op.name = "TensorRTOptimizer" - custom_op.parameter_map["minimum_segment_size"].i = 3 - custom_op.parameter_map["precision_mode"].s = mode - custom_op.parameter_map["max_batch_size"].i = inp_dims[0] - custom_op.parameter_map["max_workspace_size_bytes"].i = 1 << 25 - print(custom_op) - gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50) - graph_options = config_pb2.GraphOptions(rewrite_options=opt_config) - sessconfig = config_pb2.ConfigProto( - gpu_options=gpu_options, graph_options=graph_options) - print(sessconfig) - g = ops.Graph() - ops.reset_default_graph() - with g.as_default(): - inp, out = importer.import_graph_def( - graph_def=self.native_network, return_elements=["input", "output"]) - inp = inp.outputs[0] - out = out.outputs[0] - with session.Session(config=sessconfig, graph=g) as sess: - if (initialization): - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - saver.restore(sess, self.check_file) - self.tftrt_dynamic_result[mode] = sess.run(out, {inp: dummy_input}) - else: - raise Exception("dynamic op mode: " + mode + " not supported") - - def run_static_convert_network(self, mode, dummy_input, initialization=True): - inp_dims = dummy_input.shape - if mode == "FP32" or mode == "FP16" or mode == "INT8": - trt_graph = trt.create_inference_graph( - input_graph_def=self.native_network, - outputs=[OUTPUT_NODE], - max_batch_size=inp_dims[0], - max_workspace_size_bytes=1 << 25, - precision_mode=mode, # TRT Engine precision "FP32","FP16" or "INT8" - minimum_segment_size=2 # minimum number of nodes in an engine - ) - if mode == "INT8": - _ = self.execute_calibration(trt_graph, dummy_input, initialization) - trt_graph = trt.calib_graph_to_infer_graph(trt_graph) - trt_result, nb_nodes = self.execute_graph(trt_graph, dummy_input, - initialization) - self.tftrt[mode] = trt_graph - self.tftrt_nb_nodes[mode] = nb_nodes - self.tftrt_result[mode] = trt_result - else: - raise Exception("mode: " + mode + " not supported") - - def execute_graph(self, gdef, dummy_input, initialization=True): - """Run given graphdef once.""" - gpu_options = config_pb2.GPUOptions() - sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options) - ops.reset_default_graph() - g = ops.Graph() - nb_nodes = 0 - with g.as_default(): - inp, out = importer.import_graph_def( - graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") - nb_nodes = len(g.get_operations()) - inp = inp.outputs[0] - out = out.outputs[0] - with session.Session(config=sessconfig, graph=g) as sess: - if (initialization): - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - saver.restore(sess, self.check_file) - val = sess.run(out, {inp: dummy_input}) - return val, nb_nodes - - # Use real data that is representative of the inference dataset - # for calibration. For this test script it is random data. - def execute_calibration(self, gdef, dummy_input, initialization=True): - """Run given calibration graph multiple times.""" - gpu_options = config_pb2.GPUOptions() - ops.reset_default_graph() - g = ops.Graph() - with g.as_default(): - inp, out = importer.import_graph_def( - graph_def=gdef, return_elements=[INPUT_NODE, OUTPUT_NODE], name="") - inp = inp.outputs[0] - out = out.outputs[0] - with session.Session( - config=config_pb2.ConfigProto(gpu_options=gpu_options), - graph=g) as sess: - if (initialization): - names_var_list = get_all_variables(sess) - saver = training.Saver(names_var_list) - saver.restore(sess, self.check_file) - for _ in range(CALIB_COUNT): - val = sess.run(out, {inp: dummy_input}) - return val diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py deleted file mode 100644 index d9c41f90d0..0000000000 --- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py +++ /dev/null @@ -1,347 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Script to test TF-TensorRT integration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from collections import namedtuple -import itertools -import warnings -import numpy as np -import six - -from tensorflow.contrib import tensorrt as trt -from tensorflow.core.protobuf import config_pb2 -from tensorflow.core.protobuf import rewriter_config_pb2 -from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import importer -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import nn -from tensorflow.python.ops import nn_ops -from tensorflow.python.platform import test - -INPUT_NAME = "input" -OUTPUT_NAME = "output" -INPUT_DIMS = [100, 24, 24, 2] -MODE_FP32 = "FP32" -MODE_FP16 = "FP16" -MODE_INT8 = "INT8" - -if six.PY2: - to_bytes = lambda s: s - to_string = lambda s: s -else: - to_bytes = lambda s: s.encode("utf-8", errors="surrogateescape") - to_string = lambda s: s.decode("utf-8") - - -# TODO(aaroey): test graph with different dtypes. -def GetSingleEngineGraphDef(dtype=dtypes.float32): - """Create a graph containing single segment.""" - g = ops.Graph() - with g.as_default(): - inp = array_ops.placeholder( - dtype=dtype, shape=[None] + INPUT_DIMS[1:], name=INPUT_NAME) - with g.device("/GPU:0"): - conv_filter = constant_op.constant( - [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]], - name="weights", - dtype=dtype) - conv = nn.conv2d( - input=inp, - filter=conv_filter, - strides=[1, 2, 2, 1], - padding="SAME", - name="conv") - bias = constant_op.constant( - [4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtype) - added = nn.bias_add(conv, bias, name="bias_add") - relu = nn.relu(added, "relu") - identity = array_ops.identity(relu, "identity") - pool = nn_ops.max_pool( - identity, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool") - array_ops.squeeze(pool, name=OUTPUT_NAME) - return g.as_graph_def() - - -# TODO(aaroey): test graph with different dtypes. -def GetMultiEngineGraphDef(dtype=dtypes.float32): - """Create a graph containing multiple segment.""" - g = ops.Graph() - with g.as_default(): - inp = array_ops.placeholder( - dtype=dtype, shape=[None] + INPUT_DIMS[1:], name=INPUT_NAME) - with g.device("/GPU:0"): - conv_filter = constant_op.constant( - [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]], - name="weights", - dtype=dtype) - conv = nn.conv2d( - input=inp, - filter=conv_filter, - strides=[1, 2, 2, 1], - padding="SAME", - name="conv") - c1 = constant_op.constant( - np.random.randn(INPUT_DIMS[0], 12, 12, 6), dtype=dtype) - p = conv * c1 - c2 = constant_op.constant( - np.random.randn(INPUT_DIMS[0], 12, 12, 6), dtype=dtype) - q = conv / c2 - - edge = math_ops.sin(q) - edge /= edge - r = edge + edge - - p -= edge - q *= edge - s = p + q - s -= r - array_ops.squeeze(s, name=OUTPUT_NAME) - return g.as_graph_def() - - -TestGraph = namedtuple("TestGraph", - ["gdef", "num_expected_engines", "expected_output_dims"]) - -TEST_GRAPHS = { - "SingleEngineGraph": - TestGraph( - gdef=GetSingleEngineGraphDef(), - num_expected_engines=1, - expected_output_dims=(100, 6, 6, 6)), - "MultiEngineGraph": - TestGraph( - gdef=GetMultiEngineGraphDef(), - num_expected_engines=2, - expected_output_dims=(100, 12, 12, 6)), - # TODO(aaroey): add a large complex graph to test. -} - - -class TfTrtIntegrationTest(test_util.TensorFlowTestCase): - """Class to test Tensorflow-TensorRT integration.""" - - def setUp(self): - """Setup method.""" - super(TfTrtIntegrationTest, self).setUp() - warnings.simplefilter("always") - self._input = np.random.random_sample(INPUT_DIMS) - - def _GetConfigProto(self, - use_optimizer, - precision_mode=None, - is_dynamic_op=None): - if use_optimizer: - rewriter_cfg = rewriter_config_pb2.RewriterConfig() - rewriter_cfg.optimizers.extend(["constfold", "layout"]) - custom_op = rewriter_cfg.custom_optimizers.add() - custom_op.name = "TensorRTOptimizer" - custom_op.parameter_map["minimum_segment_size"].i = 3 - custom_op.parameter_map["max_batch_size"].i = self._input.shape[0] - custom_op.parameter_map["is_dynamic_op"].b = is_dynamic_op - custom_op.parameter_map["max_workspace_size_bytes"].i = 1 << 25 - custom_op.parameter_map["precision_mode"].s = to_bytes(precision_mode) - graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_cfg) - else: - graph_options = config_pb2.GraphOptions() - - gpu_options = config_pb2.GPUOptions() - if trt.trt_convert.get_linked_tensorrt_version()[0] == 3: - gpu_options.per_process_gpu_memory_fraction = 0.50 - - config = config_pb2.ConfigProto( - gpu_options=gpu_options, graph_options=graph_options) - return config - - def _RunGraph(self, graph_key, gdef, input_data, config, num_runs=2): - """Run given graphdef multiple times.""" - g = ops.Graph() - with g.as_default(): - inp, out = importer.import_graph_def( - graph_def=gdef, return_elements=[INPUT_NAME, OUTPUT_NAME], name="") - inp = inp.outputs[0] - out = out.outputs[0] - with self.test_session( - graph=g, config=config, use_gpu=True, force_gpu=True) as sess: - val = None - # Defaults to 2 runs to verify result across multiple runs is same. - for _ in range(num_runs): - new_val = sess.run(out, {inp: input_data}) - self.assertEquals(TEST_GRAPHS[graph_key].expected_output_dims, - new_val.shape) - if val is not None: - self.assertAllEqual(new_val, val) - val = new_val - return val - - # Use real data that is representative of the inference dataset - # for calibration. For this test script it is random data. - def _RunCalibration(self, graph_key, gdef, input_data, config): - """Run calibration on given graph.""" - return self._RunGraph(graph_key, gdef, input_data, config, 30) - - def _GetTrtGraph(self, gdef, precision_mode, is_dynamic_op): - """Return trt converted graph.""" - return trt.create_inference_graph( - input_graph_def=gdef, - outputs=[OUTPUT_NAME], - max_batch_size=self._input.shape[0], - max_workspace_size_bytes=1 << 25, - precision_mode=precision_mode, - minimum_segment_size=2, - is_dynamic_op=is_dynamic_op) - - def _VerifyGraphDef(self, - graph_key, - gdef, - precision_mode=None, - is_calibrated=None, - dynamic_engine=None): - num_engines = 0 - for n in gdef.node: - if n.op == "TRTEngineOp": - num_engines += 1 - self.assertNotEqual("", n.attr["serialized_segment"].s) - self.assertNotEqual("", n.attr["segment_funcdef_name"].s) - self.assertEquals(n.attr["precision_mode"].s, precision_mode) - self.assertEquals(n.attr["static_engine"].b, not dynamic_engine) - if precision_mode == MODE_INT8 and is_calibrated: - self.assertNotEqual("", n.attr["calibration_data"].s) - else: - self.assertEquals("", n.attr["calibration_data"].s) - if precision_mode is None: - self.assertEquals(num_engines, 0) - else: - self.assertEquals(num_engines, - TEST_GRAPHS[graph_key].num_expected_engines) - - def _RunTest(self, graph_key, use_optimizer, precision_mode, - dynamic_infer_engine, dynamic_calib_engine): - assert precision_mode in [MODE_FP32, MODE_FP16, MODE_INT8] - input_gdef = TEST_GRAPHS[graph_key].gdef - self._VerifyGraphDef(graph_key, input_gdef) - - # Get reference result without running trt. - config_no_trt = self._GetConfigProto(False) - print("Running original graph w/o trt, config:\n%s" % str(config_no_trt)) - ref_result = self._RunGraph(graph_key, input_gdef, self._input, - config_no_trt) - - # Run calibration if necessary. - if precision_mode == MODE_INT8: - - calib_config = self._GetConfigProto(use_optimizer, precision_mode, - dynamic_calib_engine) - print("Running calibration graph, config:\n%s" % str(calib_config)) - if use_optimizer: - self.assertTrue(False) - # TODO(aaroey): uncomment this and get infer_gdef when this mode is - # supported. - # result = self._RunCalibration(graph_key, input_gdef, self._input, - # calib_config) - else: - calib_gdef = self._GetTrtGraph(input_gdef, precision_mode, - dynamic_calib_engine) - self._VerifyGraphDef(graph_key, calib_gdef, precision_mode, False, - dynamic_calib_engine) - result = self._RunCalibration(graph_key, calib_gdef, self._input, - calib_config) - infer_gdef = trt.calib_graph_to_infer_graph(calib_gdef) - self._VerifyGraphDef(graph_key, infer_gdef, precision_mode, True, - dynamic_calib_engine) - self.assertAllClose(ref_result, result, rtol=1.e-03) - else: - infer_gdef = input_gdef - - # Run inference. - infer_config = self._GetConfigProto(use_optimizer, precision_mode, - dynamic_infer_engine) - print("Running final inference graph, config:\n%s" % str(infer_config)) - if use_optimizer: - result = self._RunGraph(graph_key, infer_gdef, self._input, infer_config) - else: - trt_infer_gdef = self._GetTrtGraph(infer_gdef, precision_mode, - dynamic_infer_engine) - self._VerifyGraphDef(graph_key, trt_infer_gdef, precision_mode, True, - dynamic_infer_engine) - result = self._RunGraph(graph_key, trt_infer_gdef, self._input, - infer_config) - self.assertAllClose(ref_result, result, rtol=1.e-03) - - def testIdempotence(self): - # Test that applying tensorrt optimizer or offline conversion tools multiple - # times to the same graph will result in same graph. - # TODO(aaroey): implement this. - pass - - -def GetTests(): - - def _GetTest(g, u, p, i, c): - - def _Test(self): - print("Running test with parameters: graph_key=%s, use_optimizer=%s, " - "precision_mode=%s, dynamic_infer_engine=%s, " - "dynamic_calib_engine=%s" % (g, u, p, i, c)) - self._RunTest(g, u, p, i, c) - - return _Test - - use_optimizer_options = [False, True] - precision_mode_options = [MODE_FP32, MODE_FP16, MODE_INT8] - dynamic_infer_engine_options = [False, True] - dynamic_calib_engine_options = [False, True] - for (graph_key, use_optimizer, precision_mode, - dynamic_infer_engine, dynamic_calib_engine) in itertools.product( - TEST_GRAPHS, use_optimizer_options, precision_mode_options, - dynamic_infer_engine_options, dynamic_calib_engine_options): - if precision_mode == MODE_INT8: - if not dynamic_calib_engine and dynamic_infer_engine: - # TODO(aaroey): test this case, the conversion from static calibration - # engine to dynamic inference engine should be a noop. - continue - if use_optimizer: - # TODO(aaroey): if use_optimizer is True we need to get the inference - # graphdef using custom python wrapper class, which is not currently - # supported yet. - continue - if not dynamic_calib_engine: - # TODO(aaroey): construction of static calibration engine is not - # supported yet. - continue - if dynamic_calib_engine and not dynamic_infer_engine: - # TODO(aaroey): construction of static inference engine using dynamic - # calibration engine is not supported yet. - continue - else: # In non int8 mode. - if dynamic_calib_engine: - # dynamic_calib_engine doesn't affect non-int8 modes, so just let - # related tests run once on dynamic_calib_engine=False. - continue - yield _GetTest(graph_key, use_optimizer, precision_mode, - dynamic_infer_engine, dynamic_calib_engine) - - -if __name__ == "__main__": - for index, t in enumerate(GetTests()): - setattr(TfTrtIntegrationTest, "testTfTRT_" + str(index), t) - test.main() diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py new file mode 100644 index 0000000000..980cc87366 --- /dev/null +++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py @@ -0,0 +1,293 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities to test TF-TensorRT integration.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from collections import namedtuple +import itertools +import warnings +import numpy as np +import re +import six + +from tensorflow.contrib import tensorrt as trt +from tensorflow.core.protobuf import config_pb2 +from tensorflow.core.protobuf import rewriter_config_pb2 +from tensorflow.python.framework import importer +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test +from tensorflow.python.platform import tf_logging as logging + +TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [ + "graph_name", "gdef", "input_dims", "num_expected_engines", + "expected_output_dims", "allclose_atol", "allclose_rtol" +]) + +INPUT_NAME = "input" +OUTPUT_NAME = "output" +TRT_INCOMPATIBLE_OP = math_ops.sin +PRECISION_MODES = ["FP32", "FP16", "INT8"] + + +def IsQuantizationMode(mode): + return mode == "INT8" + + +class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): + """Class to test Tensorflow-TensorRT integration.""" + + def _ToBytes(self, s): + if six.PY2: + return s + else: + return s.encode("utf-8") + + def _ToString(self, s): + if six.PY2: + return s + else: + return s.decode("utf-8") + + def setUp(self): + """Setup method.""" + super(TfTrtIntegrationTestBase, self).setUp() + warnings.simplefilter("always") + + def _GetConfigProto(self, + params, + use_optimizer, + precision_mode=None, + is_dynamic_op=None): + """Get config proto based on specific settings.""" + if use_optimizer: + rewriter_cfg = rewriter_config_pb2.RewriterConfig() + rewriter_cfg.optimizers.extend(["constfold", "layout"]) + custom_op = rewriter_cfg.custom_optimizers.add() + custom_op.name = "TensorRTOptimizer" + custom_op.parameter_map["minimum_segment_size"].i = 3 + custom_op.parameter_map["max_batch_size"].i = params.input_dims[0] + custom_op.parameter_map["is_dynamic_op"].b = is_dynamic_op + custom_op.parameter_map["max_workspace_size_bytes"].i = 1 << 25 + custom_op.parameter_map["precision_mode"].s = self._ToBytes( + precision_mode) + graph_options = config_pb2.GraphOptions(rewrite_options=rewriter_cfg) + else: + graph_options = config_pb2.GraphOptions() + + gpu_options = config_pb2.GPUOptions() + if trt.trt_convert.get_linked_tensorrt_version()[0] == 3: + gpu_options.per_process_gpu_memory_fraction = 0.50 + + config = config_pb2.ConfigProto( + gpu_options=gpu_options, graph_options=graph_options) + return config + + def _RunGraph(self, params, gdef, input_data, config, num_runs=2): + """Run given graphdef multiple times.""" + g = ops.Graph() + with g.as_default(): + inp, out = importer.import_graph_def( + graph_def=gdef, return_elements=[INPUT_NAME, OUTPUT_NAME], name="") + inp = inp.outputs[0] + out = out.outputs[0] + with self.test_session( + graph=g, config=config, use_gpu=True, force_gpu=True) as sess: + val = None + # Defaults to 2 runs to verify result across multiple runs is same. + for _ in range(num_runs): + new_val = sess.run(out, {inp: input_data}) + self.assertEquals(params.expected_output_dims, new_val.shape) + if val is not None: + self.assertAllEqual(new_val, val) + val = new_val + return val + + # Use real data that is representative of the inference dataset + # for calibration. For this test script it is random data. + def _RunCalibration(self, params, gdef, input_data, config): + """Run calibration on given graph.""" + return self._RunGraph(params, gdef, input_data, config, 30) + + def _GetTrtGraphDef(self, params, gdef, precision_mode, is_dynamic_op): + """Return trt converted graphdef.""" + return trt.create_inference_graph( + input_graph_def=gdef, + outputs=[OUTPUT_NAME], + max_batch_size=params.input_dims[0], + max_workspace_size_bytes=1 << 25, + precision_mode=precision_mode, + minimum_segment_size=2, + is_dynamic_op=is_dynamic_op) + + def _VerifyGraphDef(self, + params, + gdef, + precision_mode=None, + is_calibrated=None, + dynamic_engine=None): + num_engines = 0 + for n in gdef.node: + if n.op == "TRTEngineOp": + num_engines += 1 + self.assertNotEqual("", n.attr["serialized_segment"].s) + self.assertNotEqual("", n.attr["segment_funcdef_name"].s) + self.assertEquals(n.attr["precision_mode"].s, precision_mode) + self.assertEquals(n.attr["static_engine"].b, not dynamic_engine) + if IsQuantizationMode(precision_mode) and is_calibrated: + self.assertNotEqual("", n.attr["calibration_data"].s) + else: + self.assertEquals("", n.attr["calibration_data"].s) + if precision_mode is None: + self.assertEquals(num_engines, 0) + else: + self.assertEquals(num_engines, params.num_expected_engines) + + def _RunTest(self, params, use_optimizer, precision_mode, + dynamic_infer_engine, dynamic_calib_engine): + assert precision_mode in PRECISION_MODES + inp = np.random.random_sample(params.input_dims) + input_gdef = params.gdef + self._VerifyGraphDef(params, input_gdef) + + # Get reference result without running trt. + config_no_trt = self._GetConfigProto(params, False) + logging.info("Running original graph w/o trt, config:\n%s", + str(config_no_trt)) + ref_result = self._RunGraph(params, input_gdef, inp, config_no_trt) + + # Run calibration if necessary. + if IsQuantizationMode(precision_mode): + + calib_config = self._GetConfigProto(params, use_optimizer, precision_mode, + dynamic_calib_engine) + logging.info("Running calibration graph, config:\n%s", str(calib_config)) + if use_optimizer: + self.assertTrue(False) + # TODO(aaroey): uncomment this and get infer_gdef when this mode is + # supported. + # result = self._RunCalibration(params, input_gdef, inp, calib_config) + else: + calib_gdef = self._GetTrtGraphDef(params, input_gdef, precision_mode, + dynamic_calib_engine) + self._VerifyGraphDef(params, calib_gdef, precision_mode, False, + dynamic_calib_engine) + result = self._RunCalibration(params, calib_gdef, inp, calib_config) + infer_gdef = trt.calib_graph_to_infer_graph(calib_gdef) + self._VerifyGraphDef(params, infer_gdef, precision_mode, True, + dynamic_calib_engine) + + self.assertAllClose( + ref_result, + result, + atol=params.allclose_atol, + rtol=params.allclose_rtol) + else: + infer_gdef = input_gdef + + # Run inference. + infer_config = self._GetConfigProto(params, use_optimizer, precision_mode, + dynamic_infer_engine) + logging.info("Running final inference graph, config:\n%s", + str(infer_config)) + if use_optimizer: + result = self._RunGraph(params, infer_gdef, inp, infer_config) + else: + trt_infer_gdef = self._GetTrtGraphDef(params, infer_gdef, precision_mode, + dynamic_infer_engine) + self._VerifyGraphDef(params, trt_infer_gdef, precision_mode, True, + dynamic_infer_engine) + result = self._RunGraph(params, trt_infer_gdef, inp, infer_config) + + self.assertAllClose( + ref_result, + result, + atol=params.allclose_atol, + rtol=params.allclose_rtol) + + def testIdempotence(self): + # Test that applying tensorrt optimizer or offline conversion tools multiple + # times to the same graph will result in same graph. + # TODO(aaroey): implement this. + pass + + +def AddTests(test_class, params_list): + + def _GetTest(params, use_optimizer, precision_mode, dynamic_infer_engine, + dynamic_calib_engine): + + def _Test(self): + logging.info( + "Running test with parameters: graph_name=%s, " + "use_optimizer=%s, precision_mode=%s, " + "dynamic_infer_engine=%s, dynamic_calib_engine=%s", params.graph_name, + use_optimizer, precision_mode, dynamic_infer_engine, + dynamic_calib_engine) + self._RunTest(params, use_optimizer, precision_mode, dynamic_infer_engine, + dynamic_calib_engine) + + return _Test + + use_optimizer_options = [False, True] + dynamic_infer_engine_options = [False, True] + dynamic_calib_engine_options = [False, True] + for (params, use_optimizer, precision_mode, + dynamic_infer_engine, dynamic_calib_engine) in itertools.product( + params_list, use_optimizer_options, PRECISION_MODES, + dynamic_infer_engine_options, dynamic_calib_engine_options): + if IsQuantizationMode(precision_mode): + if not dynamic_calib_engine and dynamic_infer_engine: + # TODO(aaroey): test this case, the conversion from static calibration + # engine to dynamic inference engine should be a noop. + continue + if use_optimizer: + # TODO(aaroey): if use_optimizer is True we need to get the inference + # graphdef using custom python wrapper class, which is not currently + # supported yet. + continue + if not dynamic_calib_engine: + # TODO(aaroey): construction of static calibration engine is not + # supported yet. + continue + if dynamic_calib_engine and not dynamic_infer_engine: + # TODO(aaroey): construction of static inference engine using dynamic + # calibration engine is not supported yet. + continue + else: # In non int8 mode. + if dynamic_calib_engine: + # dynamic_calib_engine doesn't affect non-int8 modes, so just let + # related tests run once on dynamic_calib_engine=False. + continue + + conversion = "OptimizerConversion" if use_optimizer else "ToolConversion" + infer_engine_type = ("DynamicInferEngine" + if dynamic_infer_engine else "StaticInferEngine") + calib_engine_type = "" + if precision_mode == "INT8": + calib_engine_type = ("DynamicCalibEngine" + if dynamic_calib_engine else "StaticCalibEngine") + test_name = "%s_%s_%s_%s%s" % (re.sub( + "[^a-zA-Z0-9]+", "", params.graph_name), conversion, precision_mode, + infer_engine_type, ("_" + calib_engine_type) + if len(calib_engine_type) else "") + setattr( + test_class, "testTfTRT_" + test_name, + _GetTest(params, use_optimizer, precision_mode, dynamic_infer_engine, + dynamic_calib_engine)) diff --git a/tensorflow/contrib/tensorrt/test/unit_tests.py b/tensorflow/contrib/tensorrt/test/unit_tests.py deleted file mode 100644 index ac6e3b13ee..0000000000 --- a/tensorflow/contrib/tensorrt/test/unit_tests.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Script to execute and log all integration tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.contrib.tensorrt.test.batch_matmul_test import BatchMatMulTest -from tensorflow.contrib.tensorrt.test.biasadd_matmul_test import BiasaddMatMulTest -from tensorflow.contrib.tensorrt.test.binary_tensor_weight_broadcast_test import BinaryTensorWeightBroadcastTest -from tensorflow.contrib.tensorrt.test.concatenation_test import ConcatenationTest -from tensorflow.contrib.tensorrt.test.multi_connection_neighbor_engine_test import MultiConnectionNeighborEngineTest -from tensorflow.contrib.tensorrt.test.neighboring_engine_test import NeighboringEngineTest -from tensorflow.contrib.tensorrt.test.unary_test import UnaryTest -from tensorflow.contrib.tensorrt.test.vgg_block_nchw_test import VGGBlockNCHWTest -from tensorflow.contrib.tensorrt.test.vgg_block_test import VGGBlockTest -from tensorflow.contrib.tensorrt.test.const_broadcast_test import ConstBroadcastTest - -from tensorflow.contrib.tensorrt.test.run_test import RunTest - -tests = 0 -passed_test = 0 - -failed_list = [] -test_list = [] - -test_list.append(BatchMatMulTest()) -test_list.append(BiasaddMatMulTest()) -test_list.append(BinaryTensorWeightBroadcastTest()) -test_list.append(ConcatenationTest()) -test_list.append(NeighboringEngineTest()) -test_list.append(UnaryTest()) -test_list.append(VGGBlockNCHWTest()) -test_list.append(VGGBlockTest()) -test_list.append(MultiConnectionNeighborEngineTest()) -test_list.append(ConstBroadcastTest()) - -for test in test_list: - test.debug = True - test.check_node_count = False - with RunTest() as context: - tests += 1 - if test.run(context): - passed_test += 1 - else: - failed_list.append(test.test_name) - print("Failed test: %s\n", test.test_name) - -if passed_test == tests: - print("Passed\n") -else: - print(("%d out of %d passed\n -- failed list:") % (passed_test, tests)) - for test in failed_list: - print(" - " + test) diff --git a/tensorflow/contrib/tensorrt/test/utilities.py b/tensorflow/contrib/tensorrt/test/utilities.py deleted file mode 100644 index 0ea5f5b883..0000000000 --- a/tensorflow/contrib/tensorrt/test/utilities.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities script for TF-TensorRT integration tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.ops import variables - - -def get_all_variables(sess): - var_names = sess.run(variables.report_uninitialized_variables()) - names_var_list = {} - for name in var_names: - names_var_list[name] = sess.graph.get_tensor_by_name(name + ":0") - print(var_names) - return names_var_list -- cgit v1.2.3 From 4e652775bb5bd315d709394316112f3ac06b4ab1 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 16 Jul 2018 14:47:45 -0700 Subject: Respects multiple threads in eager function execution. Closes #19945. To use multiple threads, pass the desired thread number to the ConfigProto when calling tf.enable_eager_execution. The default is still to use no threads as this is faster for many models. PiperOrigin-RevId: 204810971 --- tensorflow/core/common_runtime/eager/context.cc | 7 +++++++ tensorflow/core/common_runtime/eager/context.h | 4 ++++ tensorflow/core/common_runtime/eager/execute.cc | 3 ++- tensorflow/core/common_runtime/eager/kernel_and_device.cc | 14 ++++++++++---- tensorflow/core/common_runtime/eager/kernel_and_device.h | 3 +++ .../core/common_runtime/eager/kernel_and_device_test.cc | 8 ++++---- 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc index 074c311c27..aaca633cc5 100644 --- a/tensorflow/core/common_runtime/eager/context.cc +++ b/tensorflow/core/common_runtime/eager/context.cc @@ -37,6 +37,13 @@ EagerContext::EagerContext(const SessionOptions& opts, async_default_(async), env_(opts.env) { InitDeviceMapAndAsync(); + if (opts.config.inter_op_parallelism_threads() > 0) { + runner_ = [this](std::function closure) { + this->thread_pool_->Schedule(closure); + }; + } else { + runner_ = [](std::function closure) { closure(); }; + } } #ifndef __ANDROID__ diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h index 458557c5b9..6825c39ef3 100644 --- a/tensorflow/core/common_runtime/eager/context.h +++ b/tensorflow/core/common_runtime/eager/context.h @@ -105,6 +105,8 @@ class EagerContext { EagerExecutor* Executor() { return &executor_; } + std::function)>* runner() { return &runner_; } + // Sets whether this thread should run in synchronous or asynchronous mode. Status SetAsyncForThread(bool async); @@ -211,6 +213,8 @@ class EagerContext { std::unique_ptr pflr_; + std::function)> runner_; + mutex cache_mu_; std::unordered_map kernel_cache_ GUARDED_BY(cache_mu_); diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc index 7a2b477845..5ea814ed4e 100644 --- a/tensorflow/core/common_runtime/eager/execute.cc +++ b/tensorflow/core/common_runtime/eager/execute.cc @@ -512,7 +512,8 @@ Status EagerLocalExecute(EagerOperation* op, // See WARNING comment in Execute (before kernel->Run) - would be nice to // rework to avoid this subtlety. tf_shared_lock l(*ctx->FunctionsMu()); - status = KernelAndDevice::Init(ndef, ctx->func_lib(device), kernel); + status = KernelAndDevice::Init(ndef, ctx->func_lib(device), ctx->runner(), + kernel); if (!status.ok()) { delete kernel; return status; diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc index b410ea175b..dae5d1983f 100644 --- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc +++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc @@ -41,17 +41,22 @@ Status KernelAndDevice::InitOp(Device* device, const NodeDef& ndef, out->device_ = device; out->kernel_.reset(k); out->flib_ = nullptr; + out->runner_ = nullptr; + out->default_runner_ = [](std::function f) { f(); }; return s; } // static Status KernelAndDevice::Init(const NodeDef& ndef, FunctionLibraryRuntime* flib, + std::function)>* runner, KernelAndDevice* out) { OpKernel* k = nullptr; Status s = flib->CreateKernel(ndef, &k); out->device_ = flib->device(); out->kernel_.reset(k); out->flib_ = flib; + out->runner_ = runner; + out->default_runner_ = [](std::function f) { f(); }; return s; } @@ -83,10 +88,11 @@ Status KernelAndDevice::Run(std::vector* input_tensors, if (stats != nullptr) { params.track_allocations = true; } - // TODO(apassos): use a thread pool. - std::function)> runner = - [](std::function f) { f(); }; - params.runner = &runner; + if (runner_ == nullptr) { + params.runner = &default_runner_; + } else { + params.runner = runner_; + } ScopedStepContainer step_container(0, [this](const string& name) { device_->resource_manager()->Cleanup(name).IgnoreError(); diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.h b/tensorflow/core/common_runtime/eager/kernel_and_device.h index c41a0972b1..c0b676b285 100644 --- a/tensorflow/core/common_runtime/eager/kernel_and_device.h +++ b/tensorflow/core/common_runtime/eager/kernel_and_device.h @@ -57,6 +57,7 @@ class KernelAndDevice { // the FunctionLibraryRuntime is pushed on to the caller (see locking in // c_api.cc). static Status Init(const NodeDef& ndef, FunctionLibraryRuntime* flib, + std::function)>* runner, KernelAndDevice* out); // TODO(ashankar): Remove this static Status InitOp(Device* device, const NodeDef& ndef, @@ -88,6 +89,8 @@ class KernelAndDevice { checkpoint::TensorSliceReaderCacheWrapper slice_reader_cache_; Rendezvous* rendez_; DataTypeVector output_dtypes_; + std::function)>* runner_; + std::function)> default_runner_; }; } // namespace tensorflow diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc index b4349e1dee..6abe98f53c 100644 --- a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc +++ b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc @@ -107,8 +107,8 @@ void BM_KernelAndDeviceInit(int iters) { KernelAndDevice k(nullptr); tensorflow::testing::StartTiming(); for (int i = 0; i < iters; ++i) { - TF_CHECK_OK( - KernelAndDevice::Init(ndef, env.function_library_runtime(), &k)); + TF_CHECK_OK(KernelAndDevice::Init(ndef, env.function_library_runtime(), + nullptr, &k)); } } BENCHMARK(BM_KernelAndDeviceInit); @@ -128,8 +128,8 @@ void BM_KernelAndDeviceRun(int iters) { .BuildNodeDef()); TestEnv env; KernelAndDevice kernel(nullptr); - TF_CHECK_OK( - KernelAndDevice::Init(ndef, env.function_library_runtime(), &kernel)); + TF_CHECK_OK(KernelAndDevice::Init(ndef, env.function_library_runtime(), + nullptr, &kernel)); tensorflow::testing::StartTiming(); for (int i = 0; i < iters; ++i) { TF_CHECK_OK(kernel.Run(&inputs, &outputs, nullptr)); -- cgit v1.2.3 From a968fd3993f0cc0a54431a3f897a1a8ff4e717e3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 14:52:24 -0700 Subject: Fix model_analyzer_test_gpu PiperOrigin-RevId: 204811714 --- tensorflow/python/profiler/BUILD | 1 - tensorflow/python/profiler/model_analyzer_test.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/python/profiler/BUILD b/tensorflow/python/profiler/BUILD index 52f6f248a3..0654104a34 100644 --- a/tensorflow/python/profiler/BUILD +++ b/tensorflow/python/profiler/BUILD @@ -58,7 +58,6 @@ cuda_py_test( "//tensorflow/python:variables", ], tags = [ - "no_gpu", "no_pip", "oss_serial", ], diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py index f9891f3b1e..c0e16ca536 100644 --- a/tensorflow/python/profiler/model_analyzer_test.py +++ b/tensorflow/python/profiler/model_analyzer_test.py @@ -106,7 +106,7 @@ class PrintModelAnalysisTest(test.TestCase): # Make sure time is profiled. gap = 1 if test.is_gpu_available() else 2 for i in range(3, 6, gap): - mat = re.search('(.*)[um]s/(.*)[um]s', metrics[i]) + mat = re.search('(.*)(?:us|ms|sec)/(.*)(?:us|ms|sec)', metrics[i]) self.assertGreater(float(mat.group(1)), 0.0) self.assertGreater(float(mat.group(2)), 0.0) # Make sure device is profiled. -- cgit v1.2.3 From 653b290777cff0d46a669bef0e67c995c762d99d Mon Sep 17 00:00:00 2001 From: Jon Perl Date: Mon, 16 Jul 2018 18:17:10 -0400 Subject: Make InMemoryEvaluatorHook test agnostic to different platforms keys() order --- tensorflow/contrib/estimator/python/estimator/hooks_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/estimator/python/estimator/hooks_test.py b/tensorflow/contrib/estimator/python/estimator/hooks_test.py index e094dade6a..ee88d5ecf5 100644 --- a/tensorflow/contrib/estimator/python/estimator/hooks_test.py +++ b/tensorflow/contrib/estimator/python/estimator/hooks_test.py @@ -111,7 +111,7 @@ class InMemoryEvaluatorHookTest(test.TestCase): self.assertEqual(4.5, step_keyword_to_value[8]['mean_of_features']) # end self.assertEqual(4.5, step_keyword_to_value[10]['mean_of_features']) - self.assertEqual([0, 4, 8, 10], list(step_keyword_to_value.keys())) + self.assertEqual(set([0, 4, 8, 10]), set(step_keyword_to_value.keys())) def test_uses_latest_variable_value(self): -- cgit v1.2.3 From 3618796b3bee7bd0eb06425d6a069d28b95e6f42 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 15:14:39 -0700 Subject: Implement lgamma for XLA Add support for Real and Imag for real floating point types. Compute the Lgamma function using Lanczos' approximation from "A Precision Approximation of the Gamma Function". SIAM Journal on Numerical Analysis series B. Vol. 1: lgamma(z + 1) = (log(2) + log(pi)) / 2 + (z + 1/2) * log(t(z)) - t(z) + A(z) t(z) = z + kLanczosGamma + 1/2 A(z) = kBaseLanczosCoeff + sigma(k = 1, n, kLanczosCoefficients[i] / (z + k)) PiperOrigin-RevId: 204815805 --- tensorflow/compiler/tests/unary_ops_test.py | 24 ++++++++ tensorflow/compiler/tf2xla/kernels/unary_ops.cc | 23 ++++++++ tensorflow/compiler/xla/client/lib/math.cc | 68 ++++++++++++++++++++++ tensorflow/compiler/xla/client/lib/math.h | 3 + tensorflow/compiler/xla/client/lib/math_test.cc | 23 ++++++++ .../compiler/xla/service/elemental_ir_emitter.cc | 4 ++ .../xla/service/hlo_evaluator_typed_visitor.h | 16 +++++ tensorflow/compiler/xla/service/shape_inference.cc | 11 ++-- 8 files changed, 168 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py index 6a7011aea6..0419419ea5 100644 --- a/tensorflow/compiler/tests/unary_ops_test.py +++ b/tensorflow/compiler/tests/unary_ops_test.py @@ -382,6 +382,30 @@ class UnaryOpsTest(xla_test.XLATestCase): expected=np.array( [[True, False, True], [False, True, True]], dtype=np.bool)) + self._assertOpOutputMatchesExpected( + math_ops.lgamma, + np.array( + [[1, 2, 3], [4, 5, 6], [1 / 2, 3 / 2, 5 / 2], + [-3 / 2, -7 / 2, -11 / 2]], + dtype=dtype), + expected=np.array( + [ + [0, 0, np.log(2.0)], + [np.log(6.0), np.log(24.0), + np.log(120)], + [ + np.log(np.pi) / 2, + np.log(np.pi) / 2 - np.log(2), + np.log(np.pi) / 2 - np.log(4) + np.log(3) + ], + [ + np.log(np.pi) / 2 - np.log(3) + np.log(4), + np.log(np.pi) / 2 - np.log(105) + np.log(16), + np.log(np.pi) / 2 - np.log(10395) + np.log(64), + ], + ], + dtype=dtype)) + def quantize_and_dequantize_v2(x): return array_ops.quantize_and_dequantize_v2( x, -127, 127, signed_input=True, num_bits=8) diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc index 116a020437..76ab8b4c00 100644 --- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc @@ -189,5 +189,28 @@ class ErfcOp : public XlaOpKernel { }; REGISTER_XLA_OP(Name("Erfc"), ErfcOp); +class LgammaOp : public XlaOpKernel { + public: + explicit LgammaOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} + // Calculate lgamma using the Lanczos approximation + // (https://en.wikipedia.org/wiki/Lanczos_approximation). + void Compile(XlaOpKernelContext* ctx) override { + xla::XlaOp input = ctx->Input(0); + xla::PrimitiveType input_type = ctx->input_xla_type(0); + + if (input_type == xla::F16 || input_type == xla::BF16) { + // The approximation works better with at least 32-bits of accuracy. + xla::XlaOp input_f32 = xla::ConvertElementType(input, xla::F32); + xla::XlaOp result_f32 = xla::Lgamma(input_f32); + xla::XlaOp result_x16 = xla::ConvertElementType(result_f32, input_type); + ctx->SetOutput(0, result_x16); + } else { + xla::XlaOp result = xla::Lgamma(input); + ctx->SetOutput(0, result); + } + } +}; // namespace +REGISTER_XLA_OP(Name("Lgamma"), LgammaOp); + } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc index 5587559040..fdc7057de3 100644 --- a/tensorflow/compiler/xla/client/lib/math.cc +++ b/tensorflow/compiler/xla/client/lib/math.cc @@ -149,4 +149,72 @@ XlaOp ErfInv(XlaOp x) { }); } +namespace { +// Coefficients for the Lanczos approximation of the gamma function. The +// coefficients are uniquely determined by the choice of g and n (kLanczosGamma +// and kLanczosCoefficients.size() + 1). The coefficients below correspond to +// [7, 9]. [5, 7], [7, 9], [9, 10], and [607/128.0, 15] were evaluated and [7, +// 9] seemed to be the least sensitive to the quality of the log function. In +// particular, [5, 7] is the only choice where -1.5e-5 <= lgamma(2) <= 1.5e-5 +// for a particularly inaccurate log function. +static constexpr double kLanczosGamma = 7; // aka g +static constexpr double kBaseLanczosCoeff = 0.99999999999980993227684700473478; +static constexpr std::array kLanczosCoefficients = { + 676.520368121885098567009190444019, -1259.13921672240287047156078755283, + 771.3234287776530788486528258894, -176.61502916214059906584551354, + 12.507343278686904814458936853, -0.13857109526572011689554707, + 9.984369578019570859563e-6, 1.50563273514931155834e-7}; +} // namespace + +// Compute the Lgamma function using Lanczos' approximation from "A Precision +// Approximation of the Gamma Function". SIAM Journal on Numerical Analysis +// series B. Vol. 1: +// lgamma(z + 1) = (log(2) + log(pi)) / 2 + (z + 1/2) * log(t(z)) - t(z) + A(z) +// t(z) = z + kLanczosGamma + 1/2 +// A(z) = kBaseLanczosCoeff + sigma(k = 1, n, kLanczosCoefficients[i] / (z + k)) +xla::XlaOp Lgamma(xla::XlaOp input) { + xla::XlaOp one_half = xla::ScalarLike(input, 0.5); + xla::XlaOp one = xla::ScalarLike(input, 1); + + xla::XlaOp pi = xla::ScalarLike(input, M_PI); + xla::XlaOp log_pi = xla::ScalarLike(input, std::log(M_PI)); + xla::XlaOp log_sqrt_two_pi = + xla::ScalarLike(input, (std::log(2) + std::log(M_PI)) / 2); + + xla::XlaOp lanczos_gamma_plus_one_half = + xla::ScalarLike(input, kLanczosGamma + 0.5); + xla::XlaOp log_lanczos_gamma_plus_one_half = + xla::ScalarLike(input, std::log(kLanczosGamma + 0.5)); + + xla::XlaOp base_lanczos_coeff = xla::ScalarLike(input, kBaseLanczosCoeff); + + // If the input is less than 0.5 use Gauss's reflection formula: + // gamma(x) = pi / sin(pi * x) * gamma(1 - x) + xla::XlaOp need_to_reflect = xla::Lt(xla::Real(input), one_half); + xla::XlaOp z = xla::Select(need_to_reflect, -input, input - one); + + xla::XlaOp x = base_lanczos_coeff; + for (int i = 0; i < kLanczosCoefficients.size(); ++i) { + xla::XlaOp lanczos_coefficient = + xla::ScalarLike(input, kLanczosCoefficients[i]); + xla::XlaOp index = xla::ScalarLike(input, i); + x = x + lanczos_coefficient / (z + index + one); + } + + // To improve accuracy on platforms with less-precise log implementations, + // compute log(lanczos_gamma_plus_one_half) at compile time and use log1p on + // the device. + // log(t) = log(kLanczosGamma + 0.5 + z) + // = log(kLanczosGamma + 0.5) + log1p(z / (kLanczosGamma + 0.5)) + xla::XlaOp t = lanczos_gamma_plus_one_half + z; + xla::XlaOp log_t = log_lanczos_gamma_plus_one_half + + xla::Log1p(z / lanczos_gamma_plus_one_half); + + xla::XlaOp log_y = log_sqrt_two_pi + (z + one_half) * log_t - t + xla::Log(x); + + xla::XlaOp reflection = log_pi - xla::Log(xla::Sin(pi * input)) - log_y; + xla::XlaOp result = xla::Select(need_to_reflect, reflection, log_y); + return result; +} + } // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/math.h b/tensorflow/compiler/xla/client/lib/math.h index e7c8b50273..c89c351cfc 100644 --- a/tensorflow/compiler/xla/client/lib/math.h +++ b/tensorflow/compiler/xla/client/lib/math.h @@ -46,6 +46,9 @@ XlaOp Erf(XlaOp x); // Computes an approximation of the inverse of the error function. XlaOp ErfInv(XlaOp x); +// Computes an approximation of the lgamma function. +XlaOp Lgamma(XlaOp input); + } // namespace xla #endif // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATH_H_ diff --git a/tensorflow/compiler/xla/client/lib/math_test.cc b/tensorflow/compiler/xla/client/lib/math_test.cc index 068cd2e586..86e195a8c6 100644 --- a/tensorflow/compiler/xla/client/lib/math_test.cc +++ b/tensorflow/compiler/xla/client/lib/math_test.cc @@ -82,5 +82,28 @@ XLA_TEST_F(MathTest, SqrtSixValues) { std::vector expected = {4, 1, 32, 0.4, 0.4472, 111.1080}; ComputeAndCompareR1(&builder, expected, {}, error_spec_); } + +XLA_TEST_F(MathTest, Lgamma) { + XlaBuilder builder(TestName()); + auto x = ConstantR1(&builder, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 0.5, 1.5, + 2.5, -1.5, -3.5, -5.5}); + Lgamma(x); + + std::vector expected = { + 0, + 0, + static_cast(std::log(2)), + static_cast(std::log(6)), + static_cast(std::log(24)), + static_cast(std::log(120)), + static_cast(std::log(M_PI) / 2), + static_cast(std::log(M_PI) / 2 - std::log(2)), + static_cast(std::log(M_PI) / 2 - std::log(4) + std::log(3)), + static_cast(std::log(M_PI) / 2 - std::log(3) + std::log(4)), + static_cast(std::log(M_PI) / 2 - std::log(105) + std::log(16)), + static_cast(std::log(M_PI) / 2 - std::log(10395) + std::log(64))}; + error_spec_ = ErrorSpec{0.001}; + ComputeAndCompareR1(&builder, expected, {}, error_spec_); +} } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc index 004a80d19d..c51632597a 100644 --- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc @@ -468,6 +468,10 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp( } case HloOpcode::kNegate: return ir_builder_->CreateFNeg(operand_value); + case HloOpcode::kReal: + return operand_value; + case HloOpcode::kImag: + return llvm::ConstantFP::get(operand_value->getType(), 0.0); default: return Unimplemented("unary floating-point op '%s'", HloOpcodeString(op->opcode()).c_str()); diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h index 2ae5f8bf36..e1924a0f8e 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h +++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h @@ -301,6 +301,14 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault { return HandleFloor(floor); } + Status HandleImag(HloInstruction* imag) override { + TF_ASSIGN_OR_RETURN(parent_->evaluated_[imag], + ElementWiseUnaryOp(imag, [](ElementwiseT elem_operand) { + return std::imag(elem_operand); + })); + return Status::OK(); + } + Status HandleLog(HloInstruction* log) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[log], ElementWiseUnaryOp(log, [](ElementwiseT elem_operand) { @@ -604,6 +612,14 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } + Status HandleReal(HloInstruction* real) override { + TF_ASSIGN_OR_RETURN(parent_->evaluated_[real], + ElementWiseUnaryOp(real, [](ElementwiseT elem_operand) { + return std::real(elem_operand); + })); + return Status::OK(); + } + template < typename NativeT, typename std::enable_if::value>::type* = nullptr> diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc index 70edf7883f..214146cf68 100644 --- a/tensorflow/compiler/xla/service/shape_inference.cc +++ b/tensorflow/compiler/xla/service/shape_inference.cc @@ -222,13 +222,16 @@ StatusOr InferWindowOutputShape(const Shape& base_shape, return shape; case HloOpcode::kReal: case HloOpcode::kImag: - if (!ShapeUtil::ElementIsComplex(shape)) { + if (ShapeUtil::ElementIsComplex(shape)) { + return ShapeUtil::ComplexComponentShape(shape); + } else if (ShapeUtil::ElementIsFloating(shape)) { + return shape; + } else { return InvalidArgument( - "Expected element type in shape to be complex for real/imag " - "operation; got %s.", + "Expected element type in shape to be floating or complex for " + "real/imag operation; got %s.", PrimitiveType_Name(shape.element_type()).c_str()); } - return ShapeUtil::ChangeElementType(shape, F32); case HloOpcode::kAbs: if (ShapeUtil::ElementIsComplex(shape)) { return ShapeUtil::ChangeElementType( -- cgit v1.2.3 From ab2e484be44c96d97b6da0b4aea63427234008c9 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 16 Jul 2018 22:24:17 +0000 Subject: Add unit test in math_opt_test.cc for shape function of HistogramFixedWidth Signed-off-by: Yong Tang --- tensorflow/core/ops/math_ops_test.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tensorflow/core/ops/math_ops_test.cc b/tensorflow/core/ops/math_ops_test.cc index 8f974d5367..da8eae1a2c 100644 --- a/tensorflow/core/ops/math_ops_test.cc +++ b/tensorflow/core/ops/math_ops_test.cc @@ -528,4 +528,15 @@ TEST(MathOpsTest, Cross_ShapeFn) { INFER_OK(op, "[?];[?]", "in0"); INFER_OK(op, "[1,?,3];[?,?,?]", "in0"); } + +TEST(MathOpsTest, HistogramFixedWidth_ShapeFn) { + ShapeInferenceTestOp op("HistogramFixedWidth"); + + INFER_ERROR("Shape must be rank 1 but is rank 0", op, "[];[];[]"); + INFER_ERROR("Dimension must be 2 but is 3", op, "[];[3];[]"); + + INFER_OK(op, "?;?;?", "[?]"); + INFER_OK(op, "[?];[2];[]", "[?]"); + INFER_OK(op, "[?];[2];?", "[?]"); +} } // end namespace tensorflow -- cgit v1.2.3 From dee9561680141ff916f3f487e212b3106da23a2f Mon Sep 17 00:00:00 2001 From: Toby Boyd Date: Mon, 16 Jul 2018 15:20:45 -0700 Subject: Add NCCL 2.x to install_sources.md PiperOrigin-RevId: 204816895 --- tensorflow/docs_src/install/install_sources.md | 323 ++++++++++++------------- 1 file changed, 161 insertions(+), 162 deletions(-) diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index edaa855aa2..502f4de7a6 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -1,28 +1,27 @@ # Install TensorFlow from Sources -This guide explains how to build TensorFlow sources into a TensorFlow -binary and how to install that TensorFlow binary. Note that we provide -well-tested, pre-built TensorFlow binaries for Ubuntu, macOS, and Windows -systems. In addition, there are pre-built TensorFlow -[docker images](https://hub.docker.com/r/tensorflow/tensorflow/). -So, don't build a TensorFlow binary yourself unless you are very -comfortable building complex packages from source and dealing with -the inevitable aftermath should things not go exactly as documented. - -If the last paragraph didn't scare you off, welcome. This guide explains -how to build TensorFlow on 64-bit desktops and laptops running either of -the following operating systems: +This guide explains how to build TensorFlow sources into a TensorFlow binary and +how to install that TensorFlow binary. Note that we provide well-tested, +pre-built TensorFlow binaries for Ubuntu, macOS, and Windows systems. In +addition, there are pre-built TensorFlow +[docker images](https://hub.docker.com/r/tensorflow/tensorflow/). So, don't +build a TensorFlow binary yourself unless you are very comfortable building +complex packages from source and dealing with the inevitable aftermath should +things not go exactly as documented. + +If the last paragraph didn't scare you off, welcome. This guide explains how to +build TensorFlow on 64-bit desktops and laptops running either of the following +operating systems: * Ubuntu * macOS X -Note: Some users have successfully built and installed TensorFlow from -sources on non-supported systems. Please remember that we do not fix -issues stemming from these attempts. +Note: Some users have successfully built and installed TensorFlow from sources +on non-supported systems. Please remember that we do not fix issues stemming +from these attempts. -We **do not support** building TensorFlow on Windows. That said, if you'd -like to try to build TensorFlow on Windows anyway, use either of the -following: +We **do not support** building TensorFlow on Windows. That said, if you'd like +to try to build TensorFlow on Windows anyway, use either of the following: * [Bazel on Windows](https://bazel.build/versions/master/docs/windows.html) * [TensorFlow CMake build](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/cmake) @@ -32,38 +31,33 @@ instructions. Older CPUs may not be able to execute these binaries. ## Determine which TensorFlow to install -You must choose one of the following types of TensorFlow to build and -install: - -* **TensorFlow with CPU support only**. If your system does not have a - NVIDIA® GPU, build and install this version. Note that this version of - TensorFlow is typically easier to build and install, so even if you - have an NVIDIA GPU, we recommend building and installing this version - first. -* **TensorFlow with GPU support**. TensorFlow programs typically run - significantly faster on a GPU than on a CPU. Therefore, if your system - has a NVIDIA GPU and you need to run performance-critical applications, - you should ultimately build and install this version. - Beyond the NVIDIA GPU itself, your system must also fulfill the NVIDIA - software requirements described in one of the following documents: +You must choose one of the following types of TensorFlow to build and install: - * @{$install_linux#NVIDIARequirements$Installing TensorFlow on Ubuntu} - * @{$install_mac#NVIDIARequirements$Installing TensorFlow on macOS} +* **TensorFlow with CPU support only**. If your system does not have a NVIDIA® + GPU, build and install this version. Note that this version of TensorFlow is + typically easier to build and install, so even if you have an NVIDIA GPU, we + recommend building and installing this version first. +* **TensorFlow with GPU support**. TensorFlow programs typically run + significantly faster on a GPU than on a CPU. Therefore, if your system has a + NVIDIA GPU and you need to run performance-critical applications, you should + ultimately build and install this version. Beyond the NVIDIA GPU itself, + your system must also fulfill the NVIDIA software requirements described in + one of the following documents: + * @ {$install_linux#NVIDIARequirements$Installing TensorFlow on Ubuntu} + * @ {$install_mac#NVIDIARequirements$Installing TensorFlow on macOS} ## Clone the TensorFlow repository -Start the process of building TensorFlow by cloning a TensorFlow -repository. +Start the process of building TensorFlow by cloning a TensorFlow repository. To clone **the latest** TensorFlow repository, issue the following command: 
    $ git clone https://github.com/tensorflow/tensorflow
    -The preceding git clone command creates a subdirectory -named `tensorflow`. After cloning, you may optionally build a -**specific branch** (such as a release branch) by invoking the -following commands: +The preceding git clone command creates a subdirectory named +`tensorflow`. After cloning, you may optionally build a **specific branch** +(such as a release branch) by invoking the following commands: 
     $ cd tensorflow
@@ -75,38 +69,34 @@ issue the following command:
 
 
    $ git checkout r1.0
    
 
-Next, you must prepare your environment for
-[Linux](#PrepareLinux)
-or
+Next, you must prepare your environment for [Linux](#PrepareLinux) or
 [macOS](#PrepareMac)
 
-
 
-## Prepare environment for Linux
 
-Before building TensorFlow on Linux, install the following build
-tools on your system:
+## Prepare environment for Linux
 
-  * bazel
-  * TensorFlow Python dependencies
-  * optionally, NVIDIA packages to support TensorFlow for GPU.
+Before building TensorFlow on Linux, install the following build tools on your
+system:
 
+*   bazel
+*   TensorFlow Python dependencies
+*   optionally, NVIDIA packages to support TensorFlow for GPU.
 
 ### Install Bazel
 
 If bazel is not installed on your system, install it now by following
 [these directions](https://bazel.build/versions/master/docs/install.html).
 
-
 ### Install TensorFlow Python dependencies
 
 To install TensorFlow, you must install the following packages:
 
-  * `numpy`, which is a numerical processing package that TensorFlow requires.
-  * `dev`, which enables adding extensions to Python.
-  * `pip`, which enables you to install and manage certain Python packages.
-  * `wheel`, which enables you to manage Python compressed packages in
-    the wheel (.whl) format.
+*   `numpy`, which is a numerical processing package that TensorFlow requires.
+*   `dev`, which enables adding extensions to Python.
+*   `pip`, which enables you to install and manage certain Python packages.
+*   `wheel`, which enables you to manage Python compressed packages in the wheel
+    (.whl) format.
 
 To install these packages for Python 2.7, issue the following command:
 
@@ -120,68 +110,70 @@ To install these packages for Python 3.n, issue the following command:
 $ sudo apt-get install python3-numpy python3-dev python3-pip python3-wheel
    - ### Optional: install TensorFlow for GPU prerequisites If you are building TensorFlow without GPU support, skip this section. -The following NVIDIA hardware must be installed on your system: - - * GPU card with CUDA Compute Capability 3.0 or higher. See - [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) - for a list of supported GPU cards. - -The following NVIDIA software must be installed on your system: - - * [CUDA Toolkit](http://nvidia.com/cuda) (>= 8.0). We recommend version 9.0. - For details, see - [NVIDIA's documentation](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/). - Ensure that you append the relevant CUDA pathnames to the - `LD_LIBRARY_PATH` environment variable as described in the - NVIDIA documentation. - * [GPU drivers](http://nvidia.com/driver) supporting your version of the CUDA - Toolkit. - * [cuDNN SDK](http://developer.nvidia.com/cudnn) (>= 6.0). We recommend version 7.0. For details, see - [NVIDIA's documentation](http://docs.nvidia.com/deeplearning/sdk/cudnn-install/). - * [CUPTI](http://docs.nvidia.com/cuda/cupti/) ships with the CUDA Toolkit, but - you also need to append its path to the `LD_LIBRARY_PATH` environment - variable: +The following NVIDIA® hardware must be installed on your system: + +* GPU card with CUDA Compute Capability 3.5 or higher. See + [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for a list of + supported GPU cards. + +The following NVIDIA® software must be installed on your system: - 
     $ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64
    +* [GPU drivers](http://nvidia.com/driver). CUDA 9.0 requires 384.x or higher. +* [CUDA Toolkit](http://nvidia.com/cuda) (>= 8.0). We recommend version 9.0. +* [cuDNN SDK](http://developer.nvidia.com/cudnn) (>= 6.0). We recommend + version 7.1.x. +* [CUPTI](http://docs.nvidia.com/cuda/cupti/) ships with the CUDA Toolkit, but + you also need to append its path to the `LD_LIBRARY_PATH` environment + variable: `export + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64` +* *OPTIONAL*: [NCCL 2.2](https://developer.nvidia.com/nccl) to use TensorFlow + with multiple GPUs. +* *OPTIONAL*: + [TensorRT](http://docs.nvidia.com/deeplearning/sdk/tensorrt-install-guide/index.html) + which can improve latency and throughput for inference for some models. + +While it is possible to install the NVIDIA libraries via `apt-get` from the +NVIDIA repository, the libraries and headers are installed in locations that +make it difficult to configure and debug build issues. Downloading and +installing the libraries manually or using docker +([latest-devel-gpu](https://hub.docker.com/r/tensorflow/tensorflow/tags/)) is +recommended. ### Next After preparing the environment, you must now [configure the installation](#ConfigureInstallation). - + ## Prepare environment for macOS Before building TensorFlow, you must install the following on your system: - * bazel - * TensorFlow Python dependencies. - * optionally, NVIDIA packages to support TensorFlow for GPU. - +* bazel +* TensorFlow Python dependencies. +* optionally, NVIDIA packages to support TensorFlow for GPU. ### Install bazel If bazel is not installed on your system, install it now by following [these directions](https://bazel.build/versions/master/docs/install.html#mac-os-x). - ### Install python dependencies To build TensorFlow, you must install the following packages: - * six - * numpy, which is a numerical processing package that TensorFlow requires. - * wheel, which enables you to manage Python compressed packages - in the wheel (.whl) format. +* six +* numpy, which is a numerical processing package that TensorFlow requires. +* wheel, which enables you to manage Python compressed packages in the wheel + (.whl) format. -You may install the python dependencies using pip. If you don't have pip -on your machine, we recommend using homebrew to install Python and pip as +You may install the python dependencies using pip. If you don't have pip on your +machine, we recommend using homebrew to install Python and pip as [documented here](http://docs.python-guide.org/en/latest/starting/install/osx/). If you follow these instructions, you will not need to disable SIP. @@ -192,22 +184,23 @@ After installing pip, invoke the following commands: Note: These are just the minimum requirements to _build_ tensorflow. Installing the pip package will download additional packages required to _run_ it. If you plan on executing tasks directly with `bazel` , without the pip installation, -you may need to install additional python packages. For example, you should -`pip install mock enum34` before running TensorFlow's tests with bazel. +you may need to install additional python packages. For example, you should `pip +install mock enum34` before running TensorFlow's tests with bazel. + ## Configure the installation -The root of the source tree contains a bash script named -configure. This script asks you to identify the pathname of all -relevant TensorFlow dependencies and specify other build configuration options -such as compiler flags. You must run this script *prior* to -creating the pip package and installing TensorFlow. +The root of the source tree contains a bash script named configure. +This script asks you to identify the pathname of all relevant TensorFlow +dependencies and specify other build configuration options such as compiler +flags. You must run this script *prior* to creating the pip package and +installing TensorFlow. -If you wish to build TensorFlow with GPU, `configure` will ask -you to specify the version numbers of CUDA and cuDNN. If several -versions of CUDA or cuDNN are installed on your system, explicitly select -the desired version instead of relying on the default. +If you wish to build TensorFlow with GPU, `configure` will ask you to specify +the version numbers of CUDA and cuDNN. If several versions of CUDA or cuDNN are +installed on your system, explicitly select the desired version instead of +relying on the default. One of the questions that `configure` will ask is as follows: @@ -215,17 +208,17 @@ One of the questions that `configure` will ask is as follows: Please specify optimization flags to use during compilation when bazel option "--config=opt" is specified [Default is -march=native] -This question refers to a later phase in which you'll use bazel to [build the -pip package](#build-the-pip-package) or the [C/Java libraries](#BuildCorJava). -We recommend accepting the default (`-march=native`), which will optimize the -generated code for your local machine's CPU type. However, if you are building -TensorFlow on one CPU type but will run TensorFlow on a different CPU type, then -consider specifying a more specific optimization -flag as described in [the gcc -documentation](https://gcc.gnu.org/onlinedocs/gcc-4.5.3/gcc/i386-and-x86_002d64-Options.html). +This question refers to a later phase in which you'll use bazel to +[build the pip package](#build-the-pip-package) or the +[C/Java libraries](#BuildCorJava). We recommend accepting the default +(`-march=native`), which will optimize the generated code for your local +machine's CPU type. However, if you are building TensorFlow on one CPU type but +will run TensorFlow on a different CPU type, then consider specifying a more +specific optimization flag as described in +[the gcc documentation](https://gcc.gnu.org/onlinedocs/gcc-4.5.3/gcc/i386-and-x86_002d64-Options.html). -Here is an example execution of the `configure` script. Note that your -own input will likely differ from our sample input: +Here is an example execution of the `configure` script. Note that your own input +will likely differ from our sample input: 
     $ cd tensorflow  # cd to the top-level directory created
@@ -262,26 +255,26 @@ Please specify the location where cuDNN 7 library is installed. Refer to README.
 Please specify a list of comma-separated CUDA compute capabilities you want to build with.
 You can find the compute capability of your device at: https://developer.nvidia.com/cuda-gpus.
 Please note that each additional compute capability significantly increases your build time and binary size.
-[Default is: "3.5,7.0"]: 6.0,7.0
+
 Do you wish to build TensorFlow with MPI support? [y/N]
 MPI support will not be enabled for TensorFlow
 Configuration finished
    -If you told `configure` to build for GPU support, then `configure` -will create a canonical set of symbolic links to the CUDA libraries -on your system. Therefore, every time you change the CUDA library paths, -you must rerun the `configure` script before re-invoking -the bazel build command. +[Default is: "3.5,7.0"]: 6.0,7.0 -Note the following: +If you told `configure` to build for GPU support, then `configure` will create a +canonical set of symbolic links to the CUDA libraries on your system. Therefore, +every time you change the CUDA library paths, you must rerun the `configure` +script before re-invoking the bazel build command. - * Although it is possible to build both CUDA and non-CUDA configs - under the same source tree, we recommend running `bazel clean` when - switching between these two configurations in the same source tree. - * If you don't run the `configure` script *before* running the - `bazel build` command, the `bazel build` command will fail. +Note the following: +* Although it is possible to build both CUDA and non-CUDA configs under the + same source tree, we recommend running `bazel clean` when switching between + these two configurations in the same source tree. +* If you don't run the `configure` script *before* running the `bazel build` + command, the `bazel build` command will fail. ## Build the pip package @@ -297,7 +290,8 @@ To build a pip package for TensorFlow with CPU-only support: $ bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package -To build a pip package for TensorFlow with CPU-only support for the Intel® MKL-DNN: +To build a pip package for TensorFlow with CPU-only support for the Intel® +MKL-DNN: 
     $ bazel build --config=mkl --config=opt //tensorflow/tools/pip_package:build_pip_package
@@ -311,32 +305,30 @@ To build a pip package for TensorFlow with GPU support:
 $ bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package
    -**NOTE on gcc 5 or later:** the binary pip packages available on the -TensorFlow website are built with gcc 4, which uses the older ABI. To -make your build compatible with the older ABI, you need to add -`--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"` to your `bazel build` command. -ABI compatibility allows custom ops built against the TensorFlow pip package -to continue to work against your built package. +**NOTE on gcc 5 or later:** the binary pip packages available on the TensorFlow +website are built with gcc 4, which uses the older ABI. To make your build +compatible with the older ABI, you need to add +`--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"` to your `bazel build` command. ABI +compatibility allows custom ops built against the TensorFlow pip package to +continue to work against your built package. -Tip: By default, building TensorFlow from sources consumes -a lot of RAM. If RAM is an issue on your system, you may limit RAM usage -by specifying --local_resources 2048,.5,1.0 while -invoking `bazel`. +Tip: By default, building TensorFlow from sources consumes a lot of RAM. +If RAM is an issue on your system, you may limit RAM usage by specifying +--local_resources 2048,.5,1.0 while invoking `bazel`. -The bazel build command builds a script named -`build_pip_package`. Running this script as follows will build -a `.whl` file within the `/tmp/tensorflow_pkg` directory: +The bazel build command builds a script named `build_pip_package`. +Running this script as follows will build a `.whl` file within the +`/tmp/tensorflow_pkg` directory: 
     $ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
    - ## Install the pip package -Invoke `pip install` to install that pip package. -The filename of the `.whl` file depends on your platform. -For example, the following command will install the pip package +Invoke `pip install` to install that pip package. The filename of the `.whl` +file depends on your platform. For example, the following command will install +the pip package for TensorFlow 1.9.0rc0 on Linux: @@ -374,26 +366,29 @@ TensorFlow programs: To learn more, see the [TensorFlow tutorials](../tutorials/). -If the system outputs an error message instead of a greeting, see [Common -installation problems](#common_installation_problems). +If the system outputs an error message instead of a greeting, see +[Common installation problems](#common_installation_problems). ## Common build and installation problems The build and installation problems you encounter typically depend on the -operating system. See the "Common installation problems" section -of one of the following guides: - - * @{$install_linux#common_installation_problems$Installing TensorFlow on Linux} - * @{$install_mac#common_installation_problems$Installing TensorFlow on Mac OS} - * @{$install_windows#common_installation_problems$Installing TensorFlow on Windows} - -Beyond the errors documented in those two guides, the following table -notes additional errors specific to building TensorFlow. Note that we -are relying on Stack Overflow as the repository for build and installation -problems. If you encounter an error message not listed in the preceding -two guides or in the following table, search for it on Stack Overflow. If -Stack Overflow doesn't show the error message, ask a new question on -Stack Overflow and specify the `tensorflow` tag. +operating system. See the "Common installation problems" section of one of the +following guides: + +* @ + {$install_linux#common_installation_problems$Installing TensorFlow on Linux} +* @ + {$install_mac#common_installation_problems$Installing TensorFlow on Mac OS} +* @ + {$install_windows#common_installation_problems$Installing TensorFlow on Windows} + +Beyond the errors documented in those two guides, the following table notes +additional errors specific to building TensorFlow. Note that we are relying on +Stack Overflow as the repository for build and installation problems. If you +encounter an error message not listed in the preceding two guides or in the +following table, search for it on Stack Overflow. If Stack Overflow doesn't show +the error message, ask a new question on Stack Overflow and specify the +`tensorflow` tag. @@ -440,6 +435,7 @@ Stack Overflow and specify the `tensorflow` tag.
    Stack Overflow Link Error Message
    ## Tested source configurations + **Linux** @@ -508,6 +504,7 @@ Stack Overflow and specify the `tensorflow` tag.
    Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
    + ## Build the C or Java libraries The instructions above are tailored to building the TensorFlow Python packages. @@ -516,10 +513,12 @@ If you're interested in building the libraries for the TensorFlow C API, do the following: 1. Follow the steps up to [Configure the installation](#ConfigureInstallation) -2. Build the C libraries following instructions in the [README](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/lib_package/README.md). +2. Build the C libraries following instructions in the + [README](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/lib_package/README.md). -If you're interested inv building the libraries for the TensorFlow Java API, -do the following: +If you're interested inv building the libraries for the TensorFlow Java API, do +the following: 1. Follow the steps up to [Configure the installation](#ConfigureInstallation) -2. Build the Java library following instructions in the [README](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/lib_package/README.md). +2. Build the Java library following instructions in the + [README](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/lib_package/README.md). -- cgit v1.2.3 From 700e045dec81eb84f778b14a6e57abb3fa65ca70 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 16 Jul 2018 22:27:40 +0000 Subject: Add additional check so that nbins is scalar, in case nbins is not known (but shape known) Signed-off-by: Yong Tang --- tensorflow/core/ops/math_ops.cc | 3 +++ tensorflow/core/ops/math_ops_test.cc | 4 ++++ tensorflow/python/ops/histogram_ops_test.py | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index cd5e0e81c3..4b0591c6e8 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -1387,6 +1387,9 @@ REGISTER_OP("HistogramFixedWidth") DimensionHandle unused; TF_RETURN_IF_ERROR( c->WithValue(c->Dim(value_range_shape, 0), 2, &unused)); + // nbins should be a scalar. + ShapeHandle nbins_shape; + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &nbins_shape)); // If nbins is available, set the shape from nbins. const Tensor* nbins_input = c->input_tensor(2); diff --git a/tensorflow/core/ops/math_ops_test.cc b/tensorflow/core/ops/math_ops_test.cc index da8eae1a2c..25dc033065 100644 --- a/tensorflow/core/ops/math_ops_test.cc +++ b/tensorflow/core/ops/math_ops_test.cc @@ -532,8 +532,12 @@ TEST(MathOpsTest, Cross_ShapeFn) { TEST(MathOpsTest, HistogramFixedWidth_ShapeFn) { ShapeInferenceTestOp op("HistogramFixedWidth"); + // value_range should be vector. INFER_ERROR("Shape must be rank 1 but is rank 0", op, "[];[];[]"); + // value_range should have 2 elements. INFER_ERROR("Dimension must be 2 but is 3", op, "[];[3];[]"); + // nbins should be scalar. + INFER_ERROR("Shape must be rank 0 but is rank 1", op, "[];[2];[2]"); INFER_OK(op, "?;?;?", "[?]"); INFER_OK(op, "[?];[2];[]", "[?]"); diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py index 06a0d9ac69..2e57ae8a2d 100644 --- a/tensorflow/python/ops/histogram_ops_test.py +++ b/tensorflow/python/ops/histogram_ops_test.py @@ -95,7 +95,7 @@ class HistogramFixedWidthTest(test.TestCase): def test_with_invalid_nbins(self): values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] with self.assertRaisesRegexp( - ValueError, "Input must be scalar but has rank 1"): + ValueError, "Shape must be rank 0 but is rank 1"): histogram_ops.histogram_fixed_width(values, [1.0, 5.0], nbins=[1, 2]) with self.assertRaisesRegexp( ValueError, "Requires nbins > 0"): -- cgit v1.2.3 From b027ac978f6ed03b634fde2a5ee3fa20d766921e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 16 Jul 2018 15:36:38 -0700 Subject: This CL fixes a bug preventing Eager tapes from working remotely. Previously, tapes would attempt to access remote TensorHandle tensors directly (unsupported remotely) to get their shapes, causing an error. They now access the remote shape via a new TensorHandle `Shape` method, which can unify local and remote TensorHandle shape accesses. This CL also adds some tests to ensure taping during remote execution works. PiperOrigin-RevId: 204819435 --- tensorflow/core/common_runtime/eager/tensor_handle.cc | 13 +++++++++++++ tensorflow/core/common_runtime/eager/tensor_handle.h | 2 ++ tensorflow/python/eager/pywrap_tfe_src.cc | 8 ++++---- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc index f9b9abcc99..85b0b79bce 100644 --- a/tensorflow/core/common_runtime/eager/tensor_handle.cc +++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc @@ -109,6 +109,19 @@ Status TensorHandle::TensorAndDevice(const tensorflow::Tensor** tensor, return Status::OK(); } +Status TensorHandle::Shape(tensorflow::TensorShape* shape) { + if (IsRemote()) { + TF_RETURN_IF_ERROR(WaitForNode(remote_shape_node_id_, false)); + CHECK(remote_shape_ != nullptr); + *shape = *(remote_shape_.get()); + } else { + TF_RETURN_IF_ERROR(WaitReady()); + DCHECK(IsReady()); + *shape = tensor_.shape(); + } + return Status::OK(); +} + Status TensorHandle::NumDims(int* num_dims) { if (IsRemote()) { TF_RETURN_IF_ERROR(WaitForNode(remote_shape_node_id_, false)); diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h index 46bc94f875..5580d37234 100644 --- a/tensorflow/core/common_runtime/eager/tensor_handle.h +++ b/tensorflow/core/common_runtime/eager/tensor_handle.h @@ -109,6 +109,8 @@ class TensorHandle : public core::RefCounted { tensorflow::Device** device, tensorflow::Device** op_device); + Status Shape(tensorflow::TensorShape* shape); + Status NumDims(int* num_dims); Status Dim(int dim_index, int64* dim); diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index ec7e2371e9..4d28e98961 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -1173,14 +1173,14 @@ static tensorflow::eager::TapeTensor TapeTensorFromTensor(PyObject* tensor) { if (EagerTensor_CheckExact(tensor)) { TFE_TensorHandle* t = EagerTensor_Handle(tensor); tensorflow::int64 id = EagerTensor_id(tensor); - const tensorflow::Tensor* tensor = nullptr; - const tensorflow::Status status = t->handle->Tensor(&tensor); + tensorflow::TensorShape tensor_shape; + const tensorflow::Status status = t->handle->Shape(&tensor_shape); + if (MaybeRaiseExceptionFromStatus(status, nullptr)) { return tensorflow::eager::TapeTensor{id, t->handle->dtype, tensorflow::TensorShape({})}; } else { - return tensorflow::eager::TapeTensor{id, t->handle->dtype, - tensor->shape()}; + return tensorflow::eager::TapeTensor{id, t->handle->dtype, tensor_shape}; } } tensorflow::int64 id = FastTensorId(tensor); -- cgit v1.2.3 From f4c88d2a1abd043db592ca680bbdda91fa5145a9 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Mon, 16 Jul 2018 15:46:51 -0700 Subject: Co-allocate call operands with callee parameters Right now we give these parameters an additional allocation and then don't use it. This is interesting for the CPU backend because we use call instructions to represent fork-join parallelism (i.e. a specially annotated kCall instruction tells the CPU IR emitter to shard the called computation across CPU threads). Moreover, I need this for a principled fix to b/111116907. PiperOrigin-RevId: 204820965 --- .../compiler/xla/service/buffer_assignment.cc | 19 +++++- .../compiler/xla/service/buffer_assignment_test.cc | 70 +++++++++++++++++++--- tensorflow/compiler/xla/service/hlo_computation.cc | 9 +++ tensorflow/compiler/xla/service/hlo_computation.h | 4 ++ 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc index afe4b2e142..783e3f7e73 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment.cc +++ b/tensorflow/compiler/xla/service/buffer_assignment.cc @@ -1444,8 +1444,23 @@ void BufferAssigner::BuildColocatedBufferSets( }); } else if (opcode == HloOpcode::kCall) { const HloInstruction* call_hlo = instruction; - const HloInstruction* root_hlo = - call_hlo->to_apply()->root_instruction(); + const HloComputation* callee = call_hlo->to_apply(); + const HloInstruction* root_hlo = callee->root_instruction(); + for (int64 i = 0; i < call_hlo->operand_count(); i++) { + const HloInstruction* call_param = callee->parameter_instruction(i); + const HloInstruction* call_operand = call_hlo->operand(i); + ShapeUtil::ForEachSubshape( + call_operand->shape(), + [&](const Shape& /*subshape*/, const ShapeIndex& index) { + std::vector colocated_set; + AddBufferToColocatedSet(call_param, index, points_to_analysis, + &colocated_set); + AddBufferToColocatedSet(call_operand, index, points_to_analysis, + &colocated_set); + AddSetToColocatedBufferSets(colocated_set, + colocated_buffer_sets); + }); + } ShapeUtil::ForEachSubshape( call_hlo->shape(), [this, call_hlo, root_hlo, &points_to_analysis, diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc index 125ade2a11..bfd20921e2 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc +++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc @@ -1094,7 +1094,7 @@ TEST_F(BufferAssignmentTest, EmbeddedComputationBuffers) { // Allocations for the call computation should not be thread-local. auto& call_param_alloc = GetTopLevelAllocation(*assignment, call_param); - EXPECT_FALSE(call_param_alloc.is_entry_computation_parameter()); + EXPECT_TRUE(call_param_alloc.is_entry_computation_parameter()); EXPECT_FALSE(call_param_alloc.maybe_live_out()); EXPECT_FALSE(call_param_alloc.is_thread_local()); @@ -1253,16 +1253,18 @@ TEST_F(BufferAssignmentTest, TupleCallAsOutput) { auto assignment = RunBufferAssignment(module.get()); - EXPECT_EQ(3, assignment->Allocations().size()); + EXPECT_EQ(2, assignment->Allocations().size()); // Buffers for call are colocated with the sub-computation. EXPECT_EQ(GetAllocation(*assignment, call, /*index=*/{}), GetAllocation(*assignment, sub_tuple, /*index=*/{})); EXPECT_EQ(GetAllocation(*assignment, call, /*index=*/{0}), GetAllocation(*assignment, sub_param, /*index=*/{})); - // The parameter isn't aliased with anything. + + // The parameter isn't aliased with the result tuple, but it is aliased with + // the call operand. EXPECT_NE(GetTopLevelAllocation(*assignment, param), GetTopLevelAllocation(*assignment, sub_tuple)); - EXPECT_NE(GetTopLevelAllocation(*assignment, param), + EXPECT_EQ(GetTopLevelAllocation(*assignment, param), GetTopLevelAllocation(*assignment, sub_param)); } @@ -1326,13 +1328,15 @@ TEST_F(BufferAssignmentTest, TupleChainedCallAsOutput) { GetAllocation(*assignment, c_call, /*index=*/{0})); EXPECT_EQ(GetAllocation(*assignment, c_call, /*index=*/{0}), GetAllocation(*assignment, d_param, /*index=*/{0})); - // The parameters aren't aliased with anything. + EXPECT_TRUE(BuffersDistinct({a_param}, {b_param}, *assignment)); EXPECT_TRUE(BuffersDistinct({a_param}, {c_param}, *assignment)); EXPECT_TRUE(BuffersDistinct({a_param}, {d_param}, *assignment)); - EXPECT_TRUE(BuffersDistinct({b_param}, {c_param}, *assignment)); - EXPECT_TRUE(BuffersDistinct({b_param}, {d_param}, *assignment)); - EXPECT_TRUE(BuffersDistinct({c_param}, {d_param}, *assignment)); + + EXPECT_EQ(GetAllocation(*assignment, b_param, /*index=*/{0}), + GetAllocation(*assignment, c_param, /*index=*/{0})); + EXPECT_EQ(GetAllocation(*assignment, c_param, /*index=*/{0}), + GetAllocation(*assignment, d_param, /*index=*/{0})); } TEST_F(BufferAssignmentTest, BitcastAsOutput) { @@ -2031,6 +2035,56 @@ TEST_F(BufferAssignmentTest, TwoCalls) { EXPECT_TRUE(BuffersDistinct({call1}, {call2}, *assignment)); } +TEST_F(BufferAssignmentTest, CallParamCoAllocation) { + const char* hlo_text = R"( +HloModule CallParamCoAllocation + +Callee { + param0 = (f32[100],(f32[200],f32[300])) parameter(0) + param1 = s32[20] parameter(1) + ROOT constant = f32[] constant(1) +} + +ENTRY Main { + entry_param0 = f32[100] parameter(0) + entry_param1 = s32[20] parameter(1) + custom_call = (f32[200],f32[300]) custom-call(), custom_call_target="call-target" + call_op0 = (f32[100],(f32[200],f32[300])) tuple(entry_param0, custom_call) + ROOT call_result = f32[] call(call_op0, entry_param1), to_apply=Callee +} +)"; + + TF_ASSERT_OK_AND_ASSIGN( + std::unique_ptr module, + HloRunner::CreateModuleFromString( + hlo_text, legacy_flags::GetDebugOptionsFromFlags())); + + auto buffers = RunBufferAssignment(module.get()); + + HloComputation* main = module->entry_computation(); + HloComputation* callee = module->GetComputationWithName("Callee"); + EXPECT_NE(callee, nullptr); + + HloInstruction* param0 = callee->parameter_instruction(0); + HloInstruction* param1 = callee->parameter_instruction(1); + + HloInstruction* entry_param0 = main->parameter_instruction(0); + HloInstruction* entry_param1 = main->parameter_instruction(1); + HloInstruction* custom_call = main->GetInstructionWithName("custom_call"); + + EXPECT_EQ(GetAllocation(*buffers, entry_param0, {}), + GetAllocation(*buffers, param0, {0})); + EXPECT_EQ(GetAllocation(*buffers, entry_param1, {}), + GetAllocation(*buffers, param1, {})); + + EXPECT_EQ(GetAllocation(*buffers, custom_call, {}), + GetAllocation(*buffers, param0, {1})); + EXPECT_EQ(GetAllocation(*buffers, custom_call, {0}), + GetAllocation(*buffers, param0, {1, 0})); + EXPECT_EQ(GetAllocation(*buffers, custom_call, {1}), + GetAllocation(*buffers, param0, {1, 1})); +} + static bool IsPostOrderTraversal( const std::vector& sequence) { tensorflow::gtl::FlatSet seen_so_far; diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc index 166a83fade..441288da1a 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.cc +++ b/tensorflow/compiler/xla/service/hlo_computation.cc @@ -898,4 +898,13 @@ void HloComputation::UniquifyName(NameUniquer* name_uniquer) { name_ = name_uniquer->GetUniqueName(name_); } +HloInstruction* HloComputation::GetInstructionWithName( + tensorflow::StringPiece name) { + auto instructions_in_computation = instructions(); + auto it = c_find_if(instructions_in_computation, [&](HloInstruction* instr) { + return instr->name() == name; + }); + return it == instructions_in_computation.end() ? nullptr : *it; +} + } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h index abc1da4da3..49ed65910f 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.h +++ b/tensorflow/compiler/xla/service/hlo_computation.h @@ -365,6 +365,10 @@ class HloComputation { unique_id_ = id; } + // Returns the instruction in this computation that has name `name`. Returns + // null if there is no such computation. + HloInstruction* GetInstructionWithName(tensorflow::StringPiece name); + int64 unique_id() const { return unique_id_; } private: -- cgit v1.2.3 From 973d80e8ed664e881e5a15903690fd767bb53b22 Mon Sep 17 00:00:00 2001 From: Toby Boyd Date: Mon, 16 Jul 2018 16:02:26 -0700 Subject: apt-get install NVIDIA libs PiperOrigin-RevId: 204823373 --- tensorflow/docs_src/install/install_linux.md | 402 ++++++++++++--------------- 1 file changed, 183 insertions(+), 219 deletions(-) diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 8ff8fa6def..7534d0fac1 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -1,38 +1,38 @@ # Install TensorFlow on Ubuntu This guide explains how to install TensorFlow on Ubuntu Linux. While these -instructions may work on other Linux variants, they are tested and supported with -the following system requirements: - -* 64-bit desktops or laptops -* Ubuntu 16.04 or higher +instructions may work on other Linux variants, they are tested and supported +with the following system requirements: +* 64-bit desktops or laptops +* Ubuntu 16.04 or higher ## Choose which TensorFlow to install The following TensorFlow variants are available for installation: -* __TensorFlow with CPU support only__. If your system does not have a - NVIDIA® GPU, you must install this version. This version of TensorFlow is - usually easier to install, so even if you have an NVIDIA GPU, we recommend - installing this version first. -* __TensorFlow with GPU support__. TensorFlow programs usually run much faster on - a GPU instead of a CPU. If you run performance-critical applications and your - system has an NVIDIA® GPU that meets the prerequisites, you should install - this version. See [TensorFlow GPU support](#NVIDIARequirements) for details. - +* __TensorFlow with CPU support only__. If your system does not have a + NVIDIA® GPU, you must install this version. This version of TensorFlow + is usually easier to install, so even if you have an NVIDIA GPU, we + recommend installing this version first. +* __TensorFlow with GPU support__. TensorFlow programs usually run much faster + on a GPU instead of a CPU. If you run performance-critical applications and + your system has an NVIDIA® GPU that meets the prerequisites, you should + install this version. See [TensorFlow GPU support](#NVIDIARequirements) for + details. ## How to install TensorFlow There are a few options to install TensorFlow on your machine: -* [Use pip in a virtual environment](#InstallingVirtualenv) *(recommended)* -* [Use pip in your system environment](#InstallingNativePip) -* [Configure a Docker container](#InstallingDocker) -* [Use pip in Anaconda](#InstallingAnaconda) -* [Install TensorFlow from source](/install/install_sources) +* [Use pip in a virtual environment](#InstallingVirtualenv) *(recommended)* +* [Use pip in your system environment](#InstallingNativePip) +* [Configure a Docker container](#InstallingDocker) +* [Use pip in Anaconda](#InstallingAnaconda) +* [Install TensorFlow from source](/install/install_sources) + ### Use `pip` in a virtual environment Key Point: Using a virtual environment is the recommended install method. @@ -41,8 +41,8 @@ The [Virtualenv](https://virtualenv.pypa.io/en/stable/) tool creates virtual Python environments that are isolated from other Python development on the same machine. In this scenario, you install TensorFlow and its dependencies within a virtual environment that is available when *activated*. Virtualenv provides a -reliable way to install and run TensorFlow while avoiding conflicts with the rest -of the system. +reliable way to install and run TensorFlow while avoiding conflicts with the +rest of the system. ##### 1. Install Python, `pip`, and `virtualenv`. @@ -62,7 +62,7 @@ To install these packages on Ubuntu: We *recommend* using `pip` version 8.1 or higher. If using a release before -version 8.1, upgrade `pip`: +version 8.1, upgrade `pip`: 
       sudo pip install -U pip
@@ -112,10 +112,10 @@ affecting packages outside the `virtualenv`.
 
 Choose one of the available TensorFlow packages for installation:
 
-* `tensorflow` —Current release for CPU
-* `tensorflow-gpu` —Current release with GPU support
-* `tf-nightly` —Nightly build for CPU
-* `tf-nightly-gpu` —Nightly build with GPU support
+*   `tensorflow` —Current release for CPU
+*   `tensorflow-gpu` —Current release with GPU support
+*   `tf-nightly` —Nightly build for CPU
+*   `tf-nightly-gpu` —Nightly build with GPU support
 
 Within an active Virtualenv environment, use `pip` to install the package:
 
@@ -160,14 +160,14 @@ To uninstall TensorFlow, remove the Virtualenv directory you created in step 2:
   rm -r ~/tensorflow/venv
    - + ### Use `pip` in your system environment Use `pip` to install the TensorFlow package directly on your system without using a container or virtual environment for isolation. This method is -recommended for system administrators that want a TensorFlow installation that is -available to everyone on a multi-user system. +recommended for system administrators that want a TensorFlow installation that +is available to everyone on a multi-user system. Since a system install is not isolated, it could interfere with other Python-based installations. But if you understand `pip` and your Python @@ -195,7 +195,7 @@ To install these packages on Ubuntu: We *recommend* using `pip` version 8.1 or higher. If using a release before -version 8.1, upgrade `pip`: +version 8.1, upgrade `pip`: 
       sudo pip install -U pip
@@ -212,10 +212,10 @@ installed, use `easy_install` to install `pip`:
 
 Choose one of the available TensorFlow packages for installation:
 
-* `tensorflow` —Current release for CPU
-* `tensorflow-gpu` —Current release with GPU support
-* `tf-nightly` —Nightly build for CPU
-* `tf-nightly-gpu` —Nightly build with GPU support
+*   `tensorflow` —Current release for CPU
+*   `tensorflow-gpu` —Current release with GPU support
+*   `tf-nightly` —Nightly build for CPU
+*   `tf-nightly-gpu` —Nightly build with GPU support
 
 And use `pip` to install the package for Python 2 or 3:
 
@@ -260,37 +260,36 @@ To uninstall TensorFlow on your system, use one of following commands:
    + ### Configure a Docker container -Docker completely isolates the TensorFlow installation -from pre-existing packages on your machine. The Docker container contains -TensorFlow and all its dependencies. Note that the Docker image can be quite -large (hundreds of MBs). You might choose the Docker installation if you are -incorporating TensorFlow into a larger application architecture that already -uses Docker. +Docker completely isolates the TensorFlow installation from pre-existing +packages on your machine. The Docker container contains TensorFlow and all its +dependencies. Note that the Docker image can be quite large (hundreds of MBs). +You might choose the Docker installation if you are incorporating TensorFlow +into a larger application architecture that already uses Docker. Take the following steps to install TensorFlow through Docker: - 1. Install Docker on your machine as described in the - [Docker documentation](http://docs.docker.com/engine/installation/). - 2. Optionally, create a Linux group called docker to allow - launching containers without sudo as described in the - [Docker documentation](https://docs.docker.com/engine/installation/linux/linux-postinstall/). - (If you don't do this step, you'll have to use sudo each time - you invoke Docker.) - 3. To install a version of TensorFlow that supports GPUs, you must first - install [nvidia-docker](https://github.com/NVIDIA/nvidia-docker), which - is stored in github. - 4. Launch a Docker container that contains one of the - [TensorFlow binary images](https://hub.docker.com/r/tensorflow/tensorflow/tags/). +1. Install Docker on your machine as described in the + [Docker documentation](http://docs.docker.com/engine/installation/). +2. Optionally, create a Linux group called docker to allow + launching containers without sudo as described in the + [Docker documentation](https://docs.docker.com/engine/installation/linux/linux-postinstall/). + (If you don't do this step, you'll have to use sudo each time you invoke + Docker.) +3. To install a version of TensorFlow that supports GPUs, you must first + install [nvidia-docker](https://github.com/NVIDIA/nvidia-docker), which is + stored in github. +4. Launch a Docker container that contains one of the + [TensorFlow binary images](https://hub.docker.com/r/tensorflow/tensorflow/tags/). The remainder of this section explains how to launch a Docker container. - #### CPU-only -To launch a Docker container with CPU-only support (that is, without -GPU support), enter a command of the following format: +To launch a Docker container with CPU-only support (that is, without GPU +support), enter a command of the following format: 
     $ docker run -it -p hostPort:containerPort TensorFlowCPUImage
@@ -298,29 +297,31 @@ $ docker run -it -p hostPort:containerPort TensorFlowCPUImage
 
 where:
 
-  * -p hostPort:containerPort is optional.
-    If you plan to run TensorFlow programs from the shell, omit this option.
-    If you plan to run TensorFlow programs as Jupyter notebooks, set both
-    hostPort and containerPort
-    to 8888.  If you'd like to run TensorBoard inside the container,
-    add a second `-p` flag, setting both hostPort and containerPort
-    to 6006.
-  * TensorFlowCPUImage is required. It identifies the Docker
+*   -p hostPort:containerPort is optional. If you plan to run
+    TensorFlow programs from the shell, omit this option. If you plan to run
+    TensorFlow programs as Jupyter notebooks, set both hostPort
+    and containerPort to 8888. If you'd like to run
+    TensorBoard inside the container, add a second `-p` flag, setting both
+    hostPort and containerPort to 6006.
+*   TensorFlowCPUImage is required. It identifies the Docker
     container. Specify one of the following values:
-    * tensorflow/tensorflow, which is the TensorFlow CPU binary image.
-    * tensorflow/tensorflow:latest-devel, which is the latest
-      TensorFlow CPU Binary image plus source code.
-    * tensorflow/tensorflow:version, which is the
-      specified version (for example, 1.1.0rc1) of TensorFlow CPU binary image.
-    * tensorflow/tensorflow:version-devel, which is
-      the specified version (for example, 1.1.0rc1) of the TensorFlow GPU
-      binary image plus source code.
+
+    *   tensorflow/tensorflow, which is the TensorFlow CPU binary
+        image.
+    *   tensorflow/tensorflow:latest-devel, which is the latest
+        TensorFlow CPU Binary image plus source code.
+    *   tensorflow/tensorflow:version, which is the specified
+        version (for example, 1.1.0rc1) of TensorFlow CPU binary image.
+    *   tensorflow/tensorflow:version-devel, which is the
+        specified version (for example, 1.1.0rc1) of the TensorFlow GPU binary
+        image plus source code.
 
     TensorFlow images are available at
     [dockerhub](https://hub.docker.com/r/tensorflow/tensorflow/).
 
-For example, the following command launches the latest TensorFlow CPU binary image
-in a Docker container from which you can run TensorFlow programs in a shell:
+For example, the following command launches the latest TensorFlow CPU binary
+image in a Docker container from which you can run TensorFlow programs in a
+shell:
 
 
     $ docker run -it tensorflow/tensorflow bash
@@ -336,10 +337,11 @@ $ docker run -it -p 8888:8888 tensorflow/tensorflow
 
 Docker will download the TensorFlow binary image the first time you launch it.
 
-
 #### GPU support
 
-To launch a Docker container with NVidia GPU support, enter a command of the following format (this [does not require any local CUDA installation](https://github.com/nvidia/nvidia-docker/wiki/CUDA#requirements)):
+To launch a Docker container with NVidia GPU support, enter a command of the
+following format (this
+[does not require any local CUDA installation](https://github.com/nvidia/nvidia-docker/wiki/CUDA#requirements)):
 
 
     $ nvidia-docker run -it -p hostPort:containerPort TensorFlowGPUImage
@@ -347,34 +349,34 @@ $ nvidia-docker run -it -p hostPort:containerPort TensorFlowGPUImage-p hostPort:containerPort is optional. If you plan
-    to run TensorFlow programs from the shell, omit this option. If you plan
-    to run TensorFlow programs as Jupyter notebooks, set both
-    hostPort and containerPort to `8888`.
-  * TensorFlowGPUImage specifies the Docker container. You must
-    specify one of the following values:
-    * tensorflow/tensorflow:latest-gpu, which is the latest
-      TensorFlow GPU binary image.
-    * tensorflow/tensorflow:latest-devel-gpu, which is
-      the latest TensorFlow GPU Binary image plus source code.
-    * tensorflow/tensorflow:version-gpu, which is the
-      specified version (for example, 0.12.1) of the TensorFlow GPU
-      binary image.
-    * tensorflow/tensorflow:version-devel-gpu, which is
-      the specified version (for example, 0.12.1) of the TensorFlow GPU
-      binary image plus source code.
-
-We recommend installing one of the `latest` versions. For example, the
-following command launches the latest TensorFlow GPU binary image in a
-Docker container from which you can run TensorFlow programs in a shell:
+*   -p hostPort:containerPort is optional. If you plan to run
+    TensorFlow programs from the shell, omit this option. If you plan to run
+    TensorFlow programs as Jupyter notebooks, set both hostPort
+    and containerPort to `8888`.
+*   TensorFlowGPUImage specifies the Docker container. You must specify
+    one of the following values:
+    *   tensorflow/tensorflow:latest-gpu, which is the latest
+        TensorFlow GPU binary image.
+    *   tensorflow/tensorflow:latest-devel-gpu, which is the latest
+        TensorFlow GPU Binary image plus source code.
+    *   tensorflow/tensorflow:version-gpu, which is the
+        specified version (for example, 0.12.1) of the TensorFlow GPU binary
+        image.
+    *   tensorflow/tensorflow:version-devel-gpu, which is the
+        specified version (for example, 0.12.1) of the TensorFlow GPU binary
+        image plus source code.
+
+We recommend installing one of the `latest` versions. For example, the following
+command launches the latest TensorFlow GPU binary image in a Docker container
+from which you can run TensorFlow programs in a shell:
 
 
     $ nvidia-docker run -it tensorflow/tensorflow:latest-gpu bash
 
    
 
-The following command also launches the latest TensorFlow GPU binary image
-in a Docker container. In this Docker container, you can run TensorFlow
-programs in a Jupyter notebook:
+The following command also launches the latest TensorFlow GPU binary image in a
+Docker container. In this Docker container, you can run TensorFlow programs in a
+Jupyter notebook:
 
 
     $ nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:latest-gpu
@@ -390,14 +392,12 @@ Docker will download the TensorFlow binary image the first time you launch it.
 For more details see the
 [TensorFlow docker readme](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/docker).
 
-
 #### Next Steps
 
-You should now
-[validate your installation](#ValidateYourInstallation).
-
+You should now [validate your installation](#ValidateYourInstallation).
 
 
+
 ### Use `pip` in Anaconda
 
 Anaconda provides the `conda` utility to create a virtual environment. However,
@@ -410,61 +410,59 @@ not tested on new TensorFlow releases.
 
 Take the following steps to install TensorFlow in an Anaconda environment:
 
-  1. Follow the instructions on the
-     [Anaconda download site](https://www.continuum.io/downloads)
-     to download and install Anaconda.
+1.  Follow the instructions on the
+    [Anaconda download site](https://www.continuum.io/downloads) to download and
+    install Anaconda.
 
-  2. Create a conda environment named tensorflow to run a version
-     of Python by invoking the following command:
+2.  Create a conda environment named tensorflow to run a version of
+    Python by invoking the following command:
 
      
    $ conda create -n tensorflow pip python=2.7 # or python=3.3, etc.
    
 
-  3. Activate the conda environment by issuing the following command:
+3.  Activate the conda environment by issuing the following command:
 
      
    $ source activate tensorflow
      (tensorflow)$  # Your prompt should change 
    
 
-  4. Issue a command of the following format to install
-     TensorFlow inside your conda environment:
+4.  Issue a command of the following format to install TensorFlow inside your
+    conda environment:
 
      
    (tensorflow)$ pip install --ignore-installed --upgrade tfBinaryURL
    
 
-     where tfBinaryURL is the
-     [URL of the TensorFlow Python package](#the_url_of_the_tensorflow_python_package).
-     For example, the following command installs the CPU-only version of
-     TensorFlow for Python 3.4:
+    where tfBinaryURL is the
+    [URL of the TensorFlow Python package](#the_url_of_the_tensorflow_python_package).
+    For example, the following command installs the CPU-only version of
+    TensorFlow for Python 3.4:
 
      
          (tensorflow)$ pip install --ignore-installed --upgrade \
      https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
    
 
 
+
 ## Validate your installation
 
 To validate your TensorFlow installation, do the following:
 
-  1. Ensure that your environment is prepared to run TensorFlow programs.
-  2. Run a short TensorFlow program.
-
+1.  Ensure that your environment is prepared to run TensorFlow programs.
+2.  Run a short TensorFlow program.
 
 ### Prepare your environment
 
-If you installed on native pip, Virtualenv, or Anaconda, then
-do the following:
+If you installed on native pip, Virtualenv, or Anaconda, then do the following:
 
-  1. Start a terminal.
-  2. If you installed with Virtualenv or Anaconda, activate your container.
-  3. If you installed TensorFlow source code, navigate to any
-     directory *except* one containing TensorFlow source code.
+1.  Start a terminal.
+2.  If you installed with Virtualenv or Anaconda, activate your container.
+3.  If you installed TensorFlow source code, navigate to any directory *except*
+    one containing TensorFlow source code.
 
-If you installed through Docker, start a Docker container
-from which you can run bash. For example:
+If you installed through Docker, start a Docker container from which you can run
+bash. For example:
 
 
     $ docker run -it tensorflow/tensorflow bash
 
    
 
-
 ### Run a short TensorFlow program
 
 Invoke python from your shell as follows:
@@ -486,96 +484,71 @@ TensorFlow programs:
 
 
    Hello, TensorFlow!
    
 
-If the system outputs an error message instead of a greeting, see [Common
-installation problems](#common_installation_problems).
+If the system outputs an error message instead of a greeting, see
+[Common installation problems](#common_installation_problems).
 
 To learn more, see the [TensorFlow tutorials](../tutorials/).
 
 
-## TensorFlow GPU support
-
-To install TensorFlow with GPU support, configure the following NVIDIA® software
-on your system:
-
-* [CUDA Toolkit 9.0](http://nvidia.com/cuda). For details, see
-  [NVIDIA's documentation](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/).
-  Append the relevant CUDA pathnames to the `LD_LIBRARY_PATH` environmental
-  variable as described in the NVIDIA documentation.
-* [cuDNN SDK v7](http://developer.nvidia.com/cudnn). For details, see
-  [NVIDIA's documentation](http://docs.nvidia.com/deeplearning/sdk/cudnn-install/).
-  Create the `CUDA_HOME` environment variable as described in the NVIDIA
-  documentation.
-* A GPU card with CUDA Compute Capability 3.0 or higher for building TensorFlow
-  from source. To use the TensorFlow binaries, version 3.5 or higher is required.
-  See the [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for a
-  list of supported GPU cards.
-* [GPU drivers](http://nvidia.com/drivers) that support your version of the CUDA
-  Toolkit.
-* NCCL 2.2 to use TensorFlow with multiple GPUs. For details, see [NVIDIA's
-  documentation](https://developer.nvidia.com/nccl).
-* The `libcupti-dev` library is the NVIDIA CUDA Profile Tools Interface. This
-  library provides advanced profiling support. To install this library,
-  use the following command for CUDA Toolkit >= 8.0:
-
-
    -  sudo apt-get install cuda-command-line-tools
-
    
-
-Add this path to the `LD_LIBRARY_PATH` environmental variable:
-
-
    -  export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}/usr/local/cuda/extras/CUPTI/lib64
-
    
-
-* *OPTIONAL*:  For optimized performance during inference, install
-  *NVIDIA TensorRT 3.0*. To install the minimal amount of TensorRT
-  runtime components required to use with the pre-built `tensorflow-gpu` package:
 
-
    -  wget https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1404/x86_64/nvinfer-runtime-trt-repo-ubuntu1404-3.0.4-ga-cuda9.0_1.0-1_amd64.deb
-  sudo dpkg -i nvinfer-runtime-trt-repo-ubuntu1404-3.0.4-ga-cuda9.0_1.0-1_amd64.deb
-  sudo apt-get update
-  sudo apt-get install -y --allow-downgrades libnvinfer-dev libcudnn7-dev=7.0.5.15-1+cuda9.0 libcudnn7=7.0.5.15-1+cuda9.0
-
    
-
-Note: For compatibility with the pre-built `tensorflow-gpu` package, use the
-Ubuntu *14.04* package of TensorRT (shown above). Use this even when installing
-on an Ubuntu 16.04 system.
-
-To build the TensorFlow-TensorRT integration module from source instead of using
-the pre-built binaries, see the
-[module documentation](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/tensorrt#using-tensorrt-in-tensorflow).
-For detailed TensorRT installation instructions, see
-[NVIDIA's TensorRT documentation](http://docs.nvidia.com/deeplearning/sdk/tensorrt-install-guide/index.html).
-
-To avoid cuDNN version conflicts during later system upgrades, hold the cuDNN
-version at 7.0.5:
-
-
    -  sudo apt-mark hold libcudnn7 libcudnn7-dev
-
    
-
-To allow upgrades, remove the this hold:
-
-
    -  sudo apt-mark unhold libcudnn7 libcudnn7-dev
-
    
-
-If you have an earlier version of the preceding packages, upgrade to the
-specified versions. If upgrading is not possible, you can still run TensorFlow
-with GPU support by @{$install_sources}.
+## TensorFlow GPU support
 
+Note: Due to the number of libraries required, using [Docker](#InstallingDocker)
+is recommended over installing directly on the host system.
+
+The following NVIDIA® hardware must be installed on your system:
+
+*   GPU card with CUDA Compute Capability 3.5 or higher. See
+    [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for a list of
+    supported GPU cards.
+
+The following NVIDIA® software must be installed on your system:
+
+*   [GPU drivers](http://nvidia.com/driver). CUDA 9.0 requires 384.x or higher.
+*   [CUDA Toolkit 9.0](http://nvidia.com/cuda).
+*   [cuDNN SDK](http://developer.nvidia.com/cudnn) (>= 7.0). Version 7.1 is
+    recommended.
+*   [CUPTI](http://docs.nvidia.com/cuda/cupti/) ships with the CUDA Toolkit, but
+    you also need to append its path to the `LD_LIBRARY_PATH` environment
+    variable: `export
+    LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64`
+*   *OPTIONAL*: [NCCL 2.2](https://developer.nvidia.com/nccl) to use TensorFlow
+    with multiple GPUs.
+*   *OPTIONAL*:
+    [TensorRT](http://docs.nvidia.com/deeplearning/sdk/tensorrt-install-guide/index.html)
+    which can improve latency and throughput for inference for some models.
+
+To use a GPU with CUDA Compute Capability 3.0, or different versions of the
+preceding NVIDIA libraries see
+@{$install_sources$installing TensorFlow from Sources}. If using Ubuntu 16.04
+and possibly other Debian based linux distros, `apt-get` can be used with the
+NVIDIA repository to simplify installation.
+
+```bash
+# Adds NVIDIA package repository.
+sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub
+wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-repo-ubuntu1604_9.1.85-1_amd64.deb
+wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb
+sudo dpkg -i cuda-repo-ubuntu1604_9.1.85-1_amd64.deb
+sudo dpkg -i nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb
+sudo apt-get update
+# Includes optional NCCL 2.x.
+sudo apt-get install cuda9.0 cuda-cublas-9-0 cuda-cufft-9-0 cuda-curand-9-0 \
+  cuda-cusolver-9-0 cuda-cusparse-9-0 libcudnn7=7.1.4.18-1+cuda9.0 \
+   libnccl2=2.2.13-1+cuda9.0 cuda-command-line-tools-9-0
+# Optionally install TensorRT runtime, must be done after above cuda install.
+sudo apt-get update
+sudo apt-get install libnvinfer4=4.1.2-1+cuda9.0
+```
 
 ## Common installation problems
 
 We are relying on Stack Overflow to document TensorFlow installation problems
-and their remedies.  The following table contains links to Stack Overflow
-answers for some common installation problems.
-If you encounter an error message or other
-installation problem not listed in the following table, search for it
-on Stack Overflow.  If Stack Overflow doesn't show the error message,
-ask a new question about it on Stack Overflow and specify
-the `tensorflow` tag.
+and their remedies. The following table contains links to Stack Overflow answers
+for some common installation problems. If you encounter an error message or
+other installation problem not listed in the following table, search for it on
+Stack Overflow. If Stack Overflow doesn't show the error message, ask a new
+question about it on Stack Overflow and specify the `tensorflow` tag.
 
 
@@ -659,20 +632,19 @@ the `tensorflow` tag.
 
 
    
 
     Link to GitHub or Stack Overflow Error Message 
    
 
-
 
+
 ## The URL of the TensorFlow Python package
 
 A few installation mechanisms require the URL of the TensorFlow Python package.
 The value you specify depends on three factors:
 
-  * operating system
-  * Python version
-  * CPU only vs. GPU support
+*   operating system
+*   Python version
+*   CPU only vs. GPU support
 
 This section documents the relevant values for Linux installations.
 
-
 ### Python 2.7
 
 CPU only:
@@ -681,7 +653,6 @@ CPU only:
 https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp27-none-linux_x86_64.whl
 
    
 
-
 GPU support:
 
 
    @@ -691,7 +662,6 @@ https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp27
 Note that GPU support requires the NVIDIA hardware and software described in
 [NVIDIA requirements to run TensorFlow with GPU support](#NVIDIARequirements).
 
-
 ### Python 3.4
 
 CPU only:
@@ -700,7 +670,6 @@ CPU only:
 https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp34-cp34m-linux_x86_64.whl
 
    
 
-
 GPU support:
 
 
    @@ -710,7 +679,6 @@ https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp34
 Note that GPU support requires the NVIDIA hardware and software described in
 [NVIDIA requirements to run TensorFlow with GPU support](#NVIDIARequirements).
 
-
 ### Python 3.5
 
 CPU only:
@@ -719,14 +687,12 @@ CPU only:
 https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp35-cp35m-linux_x86_64.whl
 
    
 
-
 GPU support:
 
 
     https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp35-cp35m-linux_x86_64.whl
 
    
 
-
 Note that GPU support requires the NVIDIA hardware and software described in
 [NVIDIA requirements to run TensorFlow with GPU support](#NVIDIARequirements).
 
@@ -738,13 +704,11 @@ CPU only:
 https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.9.0rc0-cp36-cp36m-linux_x86_64.whl
 
    
 
-
 GPU support:
 
 
     https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.9.0rc0-cp36-cp36m-linux_x86_64.whl
 
    
 
-
 Note that GPU support requires the NVIDIA hardware and software described in
 [NVIDIA requirements to run TensorFlow with GPU support](#NVIDIARequirements).
-- 
cgit v1.2.3


From 67e0f5d68729d508469a2c811a5b021c17942a7f Mon Sep 17 00:00:00 2001
From: Akshay Modi 
Date: Mon, 16 Jul 2018 16:14:29 -0700
Subject: Add a method to check if a tensor handle is on the host cpu.

PiperOrigin-RevId: 204825266
---
 tensorflow/c/eager/c_api.cc                          | 12 ++++++------
 tensorflow/core/common_runtime/eager/tensor_handle.h |  6 ++++++
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 82ca2be2cf..6c510536d6 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -664,17 +664,17 @@ TFE_TensorHandle* TFE_NewTensorHandle(const tensorflow::Tensor& t) {
 
 const tensorflow::Tensor* TFE_TensorHandleUnderlyingTensorInHostMemory(
     TFE_TensorHandle* h, TF_Status* status) {
-  tensorflow::Device* d = nullptr;
-  tensorflow::Device* op_device = nullptr;
-  const tensorflow::Tensor* t = nullptr;
-  status->status = h->handle->TensorAndDevice(&t, &d, &op_device);
-  if (!status->status.ok()) return nullptr;
-  if (d != nullptr) {
+  if (!h->handle->OnHostCPU()) {
     status->status = tensorflow::errors::FailedPrecondition(
         "TFE_TensorHandle is placed in device (not host) memory. Cannot return "
         "a tensorflow::Tensor");
     return nullptr;
   }
+  tensorflow::Device* d = nullptr;
+  tensorflow::Device* op_device = nullptr;
+  const tensorflow::Tensor* t = nullptr;
+  status->status = h->handle->TensorAndDevice(&t, &d, &op_device);
+  if (!status->status.ok()) return nullptr;
   return t;
 }
 
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index 5580d37234..1bc9c6531a 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -140,6 +140,12 @@ class TensorHandle : public core::RefCounted {
     remote_shape_ = std::move(remote_shape);
   }
 
+  bool OnHostCPU() {
+    mutex_lock ml(ctx_mutex_);
+    return device_ == nullptr ||
+           (ctx_ == nullptr || ctx_->HostCPU() == device_);
+  }
+
  private:
   // If the contents of the Tensor pointed to by this handle is yet to be
   // computed by a EagerNode, this function will block till that compuatation is
-- 
cgit v1.2.3


From 0ba51c741981c4f264dc06356a44b89ab9dbacd1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 16 Jul 2018 16:14:38 -0700
Subject: Automated rollback of commit 3ab5ef05eed4645b2f412522451be7554ef9df8c

PiperOrigin-RevId: 204825285
---
 tensorflow/contrib/data/kernels/BUILD              |   1 -
 tensorflow/contrib/data/kernels/csv_dataset_op.cc  |  59 +-
 tensorflow/contrib/data/ops/dataset_ops.cc         |  12 +-
 .../python/kernel_tests/csv_dataset_op_test.py     | 143 ++--
 .../python/kernel_tests/reader_dataset_ops_test.py | 875 +++++++++------------
 tensorflow/contrib/data/python/ops/readers.py      |  17 +-
 6 files changed, 454 insertions(+), 653 deletions(-)

diff --git a/tensorflow/contrib/data/kernels/BUILD b/tensorflow/contrib/data/kernels/BUILD
index c2c04ac7b3..7b69e10441 100644
--- a/tensorflow/contrib/data/kernels/BUILD
+++ b/tensorflow/contrib/data/kernels/BUILD
@@ -34,7 +34,6 @@ cc_library(
     srcs = ["csv_dataset_op.cc"],
     deps = [
         "//tensorflow/core:framework_headers_lib",
-        "//tensorflow/core:lib_internal",
         "//third_party/eigen3",
         "@protobuf_archive//:protobuf_headers",
     ],
diff --git a/tensorflow/contrib/data/kernels/csv_dataset_op.cc b/tensorflow/contrib/data/kernels/csv_dataset_op.cc
index 7a13b92005..4657807785 100644
--- a/tensorflow/contrib/data/kernels/csv_dataset_op.cc
+++ b/tensorflow/contrib/data/kernels/csv_dataset_op.cc
@@ -18,10 +18,7 @@ limitations under the License.
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/shape_inference.h"
-#include "tensorflow/core/lib/io/inputstream_interface.h"
 #include "tensorflow/core/lib/io/random_inputstream.h"
-#include "tensorflow/core/lib/io/zlib_compression_options.h"
-#include "tensorflow/core/lib/io/zlib_inputstream.h"
 
 namespace tensorflow {
 namespace {
@@ -40,10 +37,6 @@ class CSVDatasetOp : public DatasetOpKernel {
         ctx, filenames_tensor->dims() <= 1,
         errors::InvalidArgument("`filenames` must be a scalar or a vector."));
 
-    string compression_type;
-    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "compression_type",
-                                                    &compression_type));
-
     OpInputList record_defaults_list;
     OP_REQUIRES_OK(ctx,
                    ctx->input_list("record_defaults", &record_defaults_list));
@@ -93,19 +86,6 @@ class CSVDatasetOp : public DatasetOpKernel {
       filenames.push_back(filenames_tensor->flat()(i));
     }
 
-    io::ZlibCompressionOptions zlib_compression_options =
-        io::ZlibCompressionOptions::DEFAULT();
-    if (compression_type == "ZLIB") {
-      zlib_compression_options = io::ZlibCompressionOptions::DEFAULT();
-    } else if (compression_type == "GZIP") {
-      zlib_compression_options = io::ZlibCompressionOptions::GZIP();
-    } else {
-      OP_REQUIRES(ctx, compression_type.empty(),
-                  errors::InvalidArgument("Unsupported compression_type: ",
-                                          compression_type, "."));
-    }
-    zlib_compression_options.input_buffer_size = buffer_size;
-
     std::vector select_cols;
     select_cols.reserve(select_cols_tensor->NumElements());
     for (int i = 0; i < select_cols_tensor->NumElements(); ++i) {
@@ -123,34 +103,31 @@ class CSVDatasetOp : public DatasetOpKernel {
         ctx, select_cols.empty() || select_cols.front() >= 0,
         errors::InvalidArgument("select_cols should be non-negative indices"));
 
-    *output = new Dataset(
-        ctx, std::move(filenames), header, std::move(compression_type),
-        zlib_compression_options, output_types_, output_shapes_,
-        std::move(record_defaults), std::move(select_cols), use_quote_delim,
-        delim[0], std::move(na_value));
+    *output = new Dataset(ctx, std::move(filenames), header, buffer_size,
+                          output_types_, output_shapes_,
+                          std::move(record_defaults), std::move(select_cols),
+                          use_quote_delim, delim[0], std::move(na_value));
   }
 
  private:
   class Dataset : public GraphDatasetBase {
    public:
     Dataset(OpKernelContext* ctx, std::vector filenames, bool header,
-            string compression_type, io::ZlibCompressionOptions options,
-            const DataTypeVector& output_types,
+            int64 buffer_size, const DataTypeVector& output_types,
             const std::vector& output_shapes,
             std::vector record_defaults, std::vector select_cols,
             bool use_quote_delim, char delim, string na_value)
         : GraphDatasetBase(ctx),
           filenames_(std::move(filenames)),
           header_(header),
+          buffer_size_(buffer_size),
           out_type_(output_types),
           output_shapes_(output_shapes),
           record_defaults_(std::move(record_defaults)),
           select_cols_(std::move(select_cols)),
           use_quote_delim_(use_quote_delim),
           delim_(delim),
-          na_value_(std::move(na_value)),
-          use_compression_(!compression_type.empty()),
-          options_(options) {}
+          na_value_(std::move(na_value)) {}
 
     std::unique_ptr MakeIteratorInternal(
         const string& prefix) const override {
@@ -533,8 +510,7 @@ class CSVDatasetOp : public DatasetOpKernel {
 
       Status FillBuffer(string* result) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
         result->clear();
-        Status s = input_stream_->ReadNBytes(
-            dataset()->options_.input_buffer_size, result);
+        Status s = input_stream_->ReadNBytes(dataset()->buffer_size_, result);
 
         if (errors::IsOutOfRange(s) && !result->empty()) {
           // Ignore OutOfRange error when ReadNBytes read < N bytes.
@@ -699,17 +675,8 @@ class CSVDatasetOp : public DatasetOpKernel {
         // Actually move on to next file.
         TF_RETURN_IF_ERROR(env->NewRandomAccessFile(
             dataset()->filenames_[current_file_index_], &file_));
-        random_access_input_stream_ =
-            std::make_shared(file_.get(), false);
-
-        if (dataset()->use_compression_) {
-          input_stream_ = std::make_shared(
-              random_access_input_stream_.get(),
-              dataset()->options_.input_buffer_size,
-              dataset()->options_.input_buffer_size, dataset()->options_);
-        } else {
-          input_stream_ = random_access_input_stream_;
-        }
+        input_stream_.reset(
+            new io::RandomAccessInputStream(file_.get(), false));
         buffer_.clear();
         pos_ = 0;
         if (dataset()->header_) {
@@ -737,9 +704,8 @@ class CSVDatasetOp : public DatasetOpKernel {
       string buffer_ GUARDED_BY(mu_);  // Maintain our own buffer
       size_t pos_ GUARDED_BY(
           mu_);  // Index into the buffer must be maintained between iters
-      std::shared_ptr random_access_input_stream_
+      std::unique_ptr input_stream_
           GUARDED_BY(mu_);
-      std::shared_ptr input_stream_ GUARDED_BY(mu_);
       size_t current_file_index_ GUARDED_BY(mu_) = 0;
       std::unique_ptr file_
           GUARDED_BY(mu_);  // must outlive input_stream_
@@ -747,6 +713,7 @@ class CSVDatasetOp : public DatasetOpKernel {
 
     const std::vector filenames_;
     const bool header_;
+    const int64 buffer_size_;
     const DataTypeVector out_type_;
     const std::vector output_shapes_;
     const std::vector record_defaults_;
@@ -754,8 +721,6 @@ class CSVDatasetOp : public DatasetOpKernel {
     const bool use_quote_delim_;
     const char delim_;
     const string na_value_;
-    const bool use_compression_;
-    const io::ZlibCompressionOptions options_;
   };  // class Dataset
 
   DataTypeVector output_types_;
diff --git a/tensorflow/contrib/data/ops/dataset_ops.cc b/tensorflow/contrib/data/ops/dataset_ops.cc
index a623c27ff8..8413fcaf87 100644
--- a/tensorflow/contrib/data/ops/dataset_ops.cc
+++ b/tensorflow/contrib/data/ops/dataset_ops.cc
@@ -36,7 +36,6 @@ data_input_datasets: `N` datasets with the same type that will be interleaved
 
 REGISTER_OP("CSVDataset")
     .Input("filenames: string")
-    .Input("compression_type: string")
     .Input("buffer_size: int64")
     .Input("header: bool")
     .Input("field_delim: string")
@@ -53,18 +52,17 @@ REGISTER_OP("CSVDataset")
       shape_inference::ShapeHandle unused;
       // `filenames` must be a scalar or a vector.
       TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(0), 1, &unused));
-      // `compression_type`, `buffer_size`, `header`, `field_delim`,
-      // `use_quote_delim`, `na_value` must be scalars
+      // `buffer_size`, `header`, `field_delim`, `use_quote_delim`,
+      // `na_value` must be scalars
       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused));
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(6), 0, &unused));
       // `select_cols` must be a vector
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(7), 1, &unused));
-      // `record_defaults` must be lists of scalars
-      for (size_t i = 8; i < c->num_inputs(); ++i) {
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(6), 1, &unused));
+      // `record_defaults` must be a list of scalars...?
+      for (size_t i = 7; i < c->num_inputs(); ++i) {
         TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 1, &unused));
       }
       return shape_inference::ScalarShape(c);
diff --git a/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py
index 2a0e64caeb..df115175f5 100644
--- a/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py
@@ -18,12 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import gzip
 import os
 import string
 import tempfile
 import time
-import zlib
 
 import numpy as np
 
@@ -64,29 +62,18 @@ class CsvDatasetOpTest(test.TestCase):
         op2 = sess.run(next2)
         self.assertAllEqual(op1, op2)
 
-  def _setup_files(self, inputs, linebreak='\n', compression_type=None):
+  def setup_files(self, inputs, linebreak='\n'):
     filenames = []
     for i, ip in enumerate(inputs):
       fn = os.path.join(self.get_temp_dir(), 'temp_%d.csv' % i)
-      contents = linebreak.join(ip).encode('utf-8')
-      if compression_type is None:
-        with open(fn, 'wb') as f:
-          f.write(contents)
-      elif compression_type == 'GZIP':
-        with gzip.GzipFile(fn, 'wb') as f:
-          f.write(contents)
-      elif compression_type == 'ZLIB':
-        contents = zlib.compress(contents)
-        with open(fn, 'wb') as f:
-          f.write(contents)
-      else:
-        raise ValueError('Unsupported compression_type', compression_type)
+      with open(fn, 'wb') as f:
+        f.write(linebreak.join(ip).encode('utf-8'))
       filenames.append(fn)
     return filenames
 
   def _make_test_datasets(self, inputs, **kwargs):
     # Test by comparing its output to what we could get with map->decode_csv
-    filenames = self._setup_files(inputs)
+    filenames = self.setup_files(inputs)
     dataset_expected = core_readers.TextLineDataset(filenames)
     dataset_expected = dataset_expected.map(
         lambda l: parsing_ops.decode_csv(l, **kwargs))
@@ -125,18 +112,15 @@ class CsvDatasetOpTest(test.TestCase):
           except errors.OutOfRangeError:
             break
 
-  def _test_dataset(
-      self,
-      inputs,
-      expected_output=None,
-      expected_err_re=None,
-      linebreak='\n',
-      compression_type=None,  # Used for both setup and parsing
-      **kwargs):
+  def _test_dataset(self,
+                    inputs,
+                    expected_output=None,
+                    expected_err_re=None,
+                    linebreak='\n',
+                    **kwargs):
     """Checks that elements produced by CsvDataset match expected output."""
     # Convert str type because py3 tf strings are bytestrings
-    filenames = self._setup_files(inputs, linebreak, compression_type)
-    kwargs['compression_type'] = compression_type
+    filenames = self.setup_files(inputs, linebreak)
     with ops.Graph().as_default() as g:
       with self.test_session(graph=g) as sess:
         dataset = readers.CsvDataset(filenames, **kwargs)
@@ -190,7 +174,7 @@ class CsvDatasetOpTest(test.TestCase):
   def testCsvDataset_ignoreErrWithUnescapedQuotes(self):
     record_defaults = [['']] * 3
     inputs = [['1,"2"3",4', '1,"2"3",4",5,5', 'a,b,"c"d"', 'e,f,g']]
-    filenames = self._setup_files(inputs)
+    filenames = self.setup_files(inputs)
     with ops.Graph().as_default() as g:
       with self.test_session(graph=g) as sess:
         dataset = readers.CsvDataset(filenames, record_defaults=record_defaults)
@@ -200,7 +184,7 @@ class CsvDatasetOpTest(test.TestCase):
   def testCsvDataset_ignoreErrWithUnquotedQuotes(self):
     record_defaults = [['']] * 3
     inputs = [['1,2"3,4', 'a,b,c"d', '9,8"7,6,5', 'e,f,g']]
-    filenames = self._setup_files(inputs)
+    filenames = self.setup_files(inputs)
     with ops.Graph().as_default() as g:
       with self.test_session(graph=g) as sess:
         dataset = readers.CsvDataset(filenames, record_defaults=record_defaults)
@@ -371,7 +355,7 @@ class CsvDatasetOpTest(test.TestCase):
         '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19',
         '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19'
     ]]
-    file_path = self._setup_files(data)
+    file_path = self.setup_files(data)
 
     with ops.Graph().as_default() as g:
       ds = readers.make_csv_dataset(
@@ -448,29 +432,14 @@ class CsvDatasetOpTest(test.TestCase):
         record_defaults=record_defaults,
         buffer_size=0)
 
-  def _test_dataset_on_buffer_sizes(self,
-                                    inputs,
-                                    expected,
-                                    linebreak,
-                                    record_defaults,
-                                    compression_type=None,
-                                    num_sizes_to_test=20):
-    # Testing reading with a range of buffer sizes that should all work.
-    for i in list(range(1, 1 + num_sizes_to_test)) + [None]:
-      self._test_dataset(
-          inputs,
-          expected,
-          linebreak=linebreak,
-          compression_type=compression_type,
-          record_defaults=record_defaults,
-          buffer_size=i)
-
-  def testCsvDataset_withLF(self):
+  def testCsvDataset_withBufferSize(self):
     record_defaults = [['NA']] * 3
     inputs = [['abc,def,ghi', '0,1,2', ',,']]
     expected = [['abc', 'def', 'ghi'], ['0', '1', '2'], ['NA', 'NA', 'NA']]
-    self._test_dataset_on_buffer_sizes(
-        inputs, expected, linebreak='\n', record_defaults=record_defaults)
+    for i in range(20):
+      # Test a range of buffer sizes that should all work
+      self._test_dataset(
+          inputs, expected, record_defaults=record_defaults, buffer_size=i + 1)
 
   def testCsvDataset_withCR(self):
     # Test that when the line separator is '\r', parsing works with all buffer
@@ -478,8 +447,14 @@ class CsvDatasetOpTest(test.TestCase):
     record_defaults = [['NA']] * 3
     inputs = [['abc,def,ghi', '0,1,2', ',,']]
     expected = [['abc', 'def', 'ghi'], ['0', '1', '2'], ['NA', 'NA', 'NA']]
-    self._test_dataset_on_buffer_sizes(
-        inputs, expected, linebreak='\r', record_defaults=record_defaults)
+    for i in range(20):
+      # Test a range of buffer sizes that should all work
+      self._test_dataset(
+          inputs,
+          expected,
+          linebreak='\r',
+          record_defaults=record_defaults,
+          buffer_size=i + 1)
 
   def testCsvDataset_withCRLF(self):
     # Test that when the line separator is '\r\n', parsing works with all buffer
@@ -487,15 +462,29 @@ class CsvDatasetOpTest(test.TestCase):
     record_defaults = [['NA']] * 3
     inputs = [['abc,def,ghi', '0,1,2', ',,']]
     expected = [['abc', 'def', 'ghi'], ['0', '1', '2'], ['NA', 'NA', 'NA']]
-    self._test_dataset_on_buffer_sizes(
-        inputs, expected, linebreak='\r\n', record_defaults=record_defaults)
+    for i in range(20):
+      # Test a range of buffer sizes that should all work
+      self._test_dataset(
+          inputs,
+          expected,
+          linebreak='\r\n',
+          record_defaults=record_defaults,
+          buffer_size=i + 1)
 
   def testCsvDataset_withBufferSizeAndQuoted(self):
     record_defaults = [['NA']] * 3
     inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']]
     expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'],
                 ['NA', 'NA', 'NA']]
-    self._test_dataset_on_buffer_sizes(
+    for i in range(20):
+      # Test a range of buffer sizes that should all work
+      self._test_dataset(
+          inputs,
+          expected,
+          linebreak='\n',
+          record_defaults=record_defaults,
+          buffer_size=i + 1)
+    self._test_dataset(
         inputs, expected, linebreak='\n', record_defaults=record_defaults)
 
   def testCsvDataset_withCRAndQuoted(self):
@@ -505,7 +494,15 @@ class CsvDatasetOpTest(test.TestCase):
     inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']]
     expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'],
                 ['NA', 'NA', 'NA']]
-    self._test_dataset_on_buffer_sizes(
+    for i in range(20):
+      # Test a range of buffer sizes that should all work
+      self._test_dataset(
+          inputs,
+          expected,
+          linebreak='\r',
+          record_defaults=record_defaults,
+          buffer_size=i + 1)
+    self._test_dataset(
         inputs, expected, linebreak='\r', record_defaults=record_defaults)
 
   def testCsvDataset_withCRLFAndQuoted(self):
@@ -515,33 +512,17 @@ class CsvDatasetOpTest(test.TestCase):
     inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']]
     expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'],
                 ['NA', 'NA', 'NA']]
-    self._test_dataset_on_buffer_sizes(
+    for i in range(20):
+      # Test a range of buffer sizes that should all work
+      self._test_dataset(
+          inputs,
+          expected,
+          linebreak='\r\n',
+          record_defaults=record_defaults,
+          buffer_size=i + 1)
+    self._test_dataset(
         inputs, expected, linebreak='\r\n', record_defaults=record_defaults)
 
-  def testCsvDataset_withGzipCompressionType(self):
-    record_defaults = [['NA']] * 3
-    inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']]
-    expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'],
-                ['NA', 'NA', 'NA']]
-    self._test_dataset_on_buffer_sizes(
-        inputs,
-        expected,
-        linebreak='\r\n',
-        compression_type='GZIP',
-        record_defaults=record_defaults)
-
-  def testCsvDataset_withZlibCompressionType(self):
-    record_defaults = [['NA']] * 3
-    inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']]
-    expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'],
-                ['NA', 'NA', 'NA']]
-    self._test_dataset_on_buffer_sizes(
-        inputs,
-        expected,
-        linebreak='\r\n',
-        compression_type='ZLIB',
-        record_defaults=record_defaults)
-
 
 class CsvDatasetBenchmark(test.Benchmark):
   """Benchmarks for the various ways of creating a dataset from CSV files.
diff --git a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
index 851a33dfc8..9df403ef50 100644
--- a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
@@ -17,16 +17,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import gzip
 import os
-import zlib
 
 import numpy as np
 
 from tensorflow.contrib.data.python.kernel_tests import reader_dataset_ops_test_base
 from tensorflow.contrib.data.python.ops import readers
 from tensorflow.python.data.ops import readers as core_readers
-from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -185,363 +182,264 @@ class ReadBatchFeaturesTest(
 
 class MakeCsvDatasetTest(test.TestCase):
 
-  def _make_csv_dataset(self, filenames, batch_size, num_epochs=1, **kwargs):
-    return readers.make_csv_dataset(
-        filenames, batch_size=batch_size, num_epochs=num_epochs, **kwargs)
+  COLUMN_TYPES = [
+      dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64, dtypes.string
+  ]
+  COLUMNS = ["col%d" % i for i in range(len(COLUMN_TYPES))]
+  DEFAULT_VALS = [[], [], [], [], ["NULL"]]
+  DEFAULTS = [
+      constant_op.constant([], dtype=dtypes.int32),
+      constant_op.constant([], dtype=dtypes.int64),
+      constant_op.constant([], dtype=dtypes.float32),
+      constant_op.constant([], dtype=dtypes.float64),
+      constant_op.constant(["NULL"], dtype=dtypes.string)
+  ]
+  LABEL = COLUMNS[0]
+
+  def setUp(self):
+    super(MakeCsvDatasetTest, self).setUp()
+    self._num_files = 2
+    self._num_records = 11
+    self._test_filenames = self._create_files()
+
+  def _csv_values(self, fileno, recordno):
+    return [
+        fileno,
+        recordno,
+        fileno * recordno * 0.5,
+        fileno * recordno + 0.5,
+        "record %d" % recordno if recordno % 2 == 1 else "",
+    ]
 
-  def _setup_files(self, inputs, linebreak="\n", compression_type=None):
+  def _write_file(self, filename, rows):
+    for i in range(len(rows)):
+      if isinstance(rows[i], list):
+        rows[i] = ",".join(str(v) if v is not None else "" for v in rows[i])
+    fn = os.path.join(self.get_temp_dir(), filename)
+    f = open(fn, "w")
+    f.write("\n".join(rows))
+    f.close()
+    return fn
+
+  def _create_file(self, fileno, header=True):
+    rows = []
+    if header:
+      rows.append(self.COLUMNS)
+    for recno in range(self._num_records):
+      rows.append(self._csv_values(fileno, recno))
+    return self._write_file("csv_file%d.csv" % fileno, rows)
+
+  def _create_files(self):
     filenames = []
-    for i, ip in enumerate(inputs):
-      fn = os.path.join(self.get_temp_dir(), "temp_%d.csv" % i)
-      contents = linebreak.join(ip).encode("utf-8")
-      if compression_type is None:
-        with open(fn, "wb") as f:
-          f.write(contents)
-      elif compression_type == "GZIP":
-        with gzip.GzipFile(fn, "wb") as f:
-          f.write(contents)
-      elif compression_type == "ZLIB":
-        contents = zlib.compress(contents)
-        with open(fn, "wb") as f:
-          f.write(contents)
-      else:
-        raise ValueError("Unsupported compression_type", compression_type)
-      filenames.append(fn)
+    for i in range(self._num_files):
+      filenames.append(self._create_file(i))
     return filenames
 
-  def _next_expected_batch(self, expected_output, expected_keys, batch_size,
-                           num_epochs):
-    features = {k: [] for k in expected_keys}
+  def _make_csv_dataset(
+      self,
+      filenames,
+      defaults,
+      column_names=COLUMNS,
+      label_name=LABEL,
+      select_cols=None,
+      batch_size=1,
+      num_epochs=1,
+      shuffle=False,
+      shuffle_seed=None,
+      header=True,
+      na_value="",
+  ):
+    return readers.make_csv_dataset(
+        filenames,
+        batch_size=batch_size,
+        column_names=column_names,
+        column_defaults=defaults,
+        label_name=label_name,
+        num_epochs=num_epochs,
+        shuffle=shuffle,
+        shuffle_seed=shuffle_seed,
+        header=header,
+        na_value=na_value,
+        select_columns=select_cols,
+    )
+
+  def _next_actual_batch(self, file_indices, batch_size, num_epochs, defaults):
+    features = {col: list() for col in self.COLUMNS}
     for _ in range(num_epochs):
-      for values in expected_output:
-        for n, key in enumerate(expected_keys):
-          features[key].append(values[n])
-        if len(features[expected_keys[0]]) == batch_size:
-          yield features
-          features = {k: [] for k in expected_keys}
-    if features[expected_keys[0]]:  # Leftover from the last batch
-      yield features
-
-  def _verify_output(
+      for i in file_indices:
+        for j in range(self._num_records):
+          values = self._csv_values(i, j)
+          for n, v in enumerate(values):
+            if v == "":  # pylint: disable=g-explicit-bool-comparison
+              values[n] = defaults[n][0]
+          values[-1] = values[-1].encode("utf-8")
+
+          # Regroup lists by column instead of row
+          for n, col in enumerate(self.COLUMNS):
+            features[col].append(values[n])
+          if len(list(features.values())[0]) == batch_size:
+            yield features
+            features = {col: list() for col in self.COLUMNS}
+
+  def _run_actual_batch(self, outputs, sess):
+    features, labels = sess.run(outputs)
+    batch = [features[k] for k in self.COLUMNS if k != self.LABEL]
+    batch.append(labels)
+    return batch
+
+  def _verify_records(
       self,
       sess,
       dataset,
-      batch_size,
-      num_epochs,
-      label_name,
-      expected_output,
-      expected_keys,
+      file_indices,
+      defaults=tuple(DEFAULT_VALS),
+      label_name=LABEL,
+      batch_size=1,
+      num_epochs=1,
   ):
-    nxt = dataset.make_one_shot_iterator().get_next()
+    iterator = dataset.make_one_shot_iterator()
+    get_next = iterator.get_next()
 
-    for expected_features in self._next_expected_batch(
-        expected_output,
-        expected_keys,
-        batch_size,
-        num_epochs,
-    ):
-      actual_features = sess.run(nxt)
+    for expected_features in self._next_actual_batch(file_indices, batch_size,
+                                                     num_epochs, defaults):
+      actual_features = sess.run(get_next)
 
       if label_name is not None:
         expected_labels = expected_features.pop(label_name)
+        # Compare labels
         self.assertAllEqual(expected_labels, actual_features[1])
-        actual_features = actual_features[0]
+        actual_features = actual_features[0]  # Extract features dict from tuple
 
       for k in expected_features.keys():
         # Compare features
         self.assertAllEqual(expected_features[k], actual_features[k])
 
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(nxt)
-
-  def _test_dataset(self,
-                    inputs,
-                    expected_output,
-                    expected_keys,
-                    batch_size=1,
-                    num_epochs=1,
-                    label_name=None,
-                    **kwargs):
-    """Checks that elements produced by CsvDataset match expected output."""
-    # Convert str type because py3 tf strings are bytestrings
-    filenames = self._setup_files(
-        inputs, compression_type=kwargs.get("compression_type", None))
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        dataset = self._make_csv_dataset(
-            filenames,
-            batch_size=batch_size,
-            num_epochs=num_epochs,
-            label_name=label_name,
-            **kwargs)
-        self._verify_output(sess, dataset, batch_size, num_epochs, label_name,
-                            expected_output, expected_keys)
+      sess.run(get_next)
 
   def testMakeCSVDataset(self):
-    """Tests making a CSV dataset with keys and defaults provided."""
-    record_defaults = [
-        constant_op.constant([], dtypes.int32),
-        constant_op.constant([], dtypes.int64),
-        constant_op.constant([], dtypes.float32),
-        constant_op.constant([], dtypes.float64),
-        constant_op.constant([], dtypes.string)
-    ]
-
-    column_names = ["col%d" % i for i in range(5)]
-    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
-        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
-    ]]
-    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
-                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
-    label = "col0"
-
-    self._test_dataset(
-        inputs,
-        expected_output=expected_output,
-        expected_keys=column_names,
-        column_names=column_names,
-        label_name=label,
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=True,
-        column_defaults=record_defaults,
-    )
-
-  def testMakeCSVDataset_withBatchSizeAndEpochs(self):
-    """Tests making a CSV dataset with keys and defaults provided."""
-    record_defaults = [
-        constant_op.constant([], dtypes.int32),
-        constant_op.constant([], dtypes.int64),
-        constant_op.constant([], dtypes.float32),
-        constant_op.constant([], dtypes.float64),
-        constant_op.constant([], dtypes.string)
-    ]
-
-    column_names = ["col%d" % i for i in range(5)]
-    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
-        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
-    ]]
-    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
-                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
-    label = "col0"
-
-    self._test_dataset(
-        inputs,
-        expected_output=expected_output,
-        expected_keys=column_names,
-        column_names=column_names,
-        label_name=label,
-        batch_size=3,
-        num_epochs=10,
-        shuffle=False,
-        header=True,
-        column_defaults=record_defaults,
-    )
+    defaults = self.DEFAULTS
 
-  def testMakeCSVDataset_withCompressionType(self):
-    """Tests `compression_type` argument."""
-    record_defaults = [
-        constant_op.constant([], dtypes.int32),
-        constant_op.constant([], dtypes.int64),
-        constant_op.constant([], dtypes.float32),
-        constant_op.constant([], dtypes.float64),
-        constant_op.constant([], dtypes.string)
-    ]
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        # Basic test: read from file 0.
+        dataset = self._make_csv_dataset(self._test_filenames[0], defaults)
+        self._verify_records(sess, dataset, [0])
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        # Basic test: read from file 1.
+        dataset = self._make_csv_dataset(self._test_filenames[1], defaults)
+        self._verify_records(sess, dataset, [1])
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        # Read from both files.
+        dataset = self._make_csv_dataset(self._test_filenames, defaults)
+        self._verify_records(sess, dataset, range(self._num_files))
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        # Read from both files. Exercise the `batch` and `num_epochs` parameters
+        # of make_csv_dataset and make sure they work.
+        dataset = self._make_csv_dataset(
+            self._test_filenames, defaults, batch_size=2, num_epochs=10)
+        self._verify_records(
+            sess, dataset, range(self._num_files), batch_size=2, num_epochs=10)
 
-    column_names = ["col%d" % i for i in range(5)]
-    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
-        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
-    ]]
-    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
-                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
-    label = "col0"
-
-    for compression_type in ("GZIP", "ZLIB"):
-      self._test_dataset(
-          inputs,
-          expected_output=expected_output,
-          expected_keys=column_names,
-          column_names=column_names,
-          label_name=label,
-          batch_size=1,
-          num_epochs=1,
-          shuffle=False,
-          header=True,
-          column_defaults=record_defaults,
-          compression_type=compression_type,
-      )
-
-  def testMakeCSVDataset_withBadInputs(self):
+  def testMakeCSVDataset_withBadColumns(self):
     """Tests that exception is raised when input is malformed.
     """
-    record_defaults = [
-        constant_op.constant([], dtypes.int32),
-        constant_op.constant([], dtypes.int64),
-        constant_op.constant([], dtypes.float32),
-        constant_op.constant([], dtypes.float64),
-        constant_op.constant([], dtypes.string)
-    ]
-
-    column_names = ["col%d" % i for i in range(5)]
-    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
-        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
-    ]]
-    filenames = self._setup_files(inputs)
+    dupe_columns = self.COLUMNS[:-1] + self.COLUMNS[:1]
+    defaults = self.DEFAULTS
 
     # Duplicate column names
     with self.assertRaises(ValueError):
       self._make_csv_dataset(
-          filenames,
-          batch_size=1,
-          column_defaults=record_defaults,
-          label_name="col0",
-          column_names=column_names * 2)
+          self._test_filenames, defaults, column_names=dupe_columns)
 
     # Label key not one of column names
     with self.assertRaises(ValueError):
       self._make_csv_dataset(
-          filenames,
-          batch_size=1,
-          column_defaults=record_defaults,
-          label_name="not_a_real_label",
-          column_names=column_names)
+          self._test_filenames, defaults, label_name="not_a_real_label")
 
   def testMakeCSVDataset_withNoLabel(self):
-    """Tests making a CSV dataset with no label provided."""
-    record_defaults = [
-        constant_op.constant([], dtypes.int32),
-        constant_op.constant([], dtypes.int64),
-        constant_op.constant([], dtypes.float32),
-        constant_op.constant([], dtypes.float64),
-        constant_op.constant([], dtypes.string)
-    ]
-
-    column_names = ["col%d" % i for i in range(5)]
-    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
-        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
-    ]]
-    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
-                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
-
-    self._test_dataset(
-        inputs,
-        expected_output=expected_output,
-        expected_keys=column_names,
-        column_names=column_names,
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=True,
-        column_defaults=record_defaults,
-    )
+    """Tests that CSV datasets can be created when no label is specified.
+    """
+    defaults = self.DEFAULTS
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        # Read from both files. Make sure this works with no label key supplied.
+        dataset = self._make_csv_dataset(
+            self._test_filenames,
+            defaults,
+            batch_size=2,
+            num_epochs=10,
+            label_name=None)
+        self._verify_records(
+            sess,
+            dataset,
+            range(self._num_files),
+            batch_size=2,
+            num_epochs=10,
+            label_name=None)
 
   def testMakeCSVDataset_withNoHeader(self):
     """Tests that datasets can be created from CSV files with no header line.
     """
-    record_defaults = [
-        constant_op.constant([], dtypes.int32),
-        constant_op.constant([], dtypes.int64),
-        constant_op.constant([], dtypes.float32),
-        constant_op.constant([], dtypes.float64),
-        constant_op.constant([], dtypes.string)
-    ]
-
-    column_names = ["col%d" % i for i in range(5)]
-    inputs = [["0,1,2,3,4", "5,6,7,8,9"], ["10,11,12,13,14", "15,16,17,18,19"]]
-    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
-                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
-    label = "col0"
-
-    self._test_dataset(
-        inputs,
-        expected_output=expected_output,
-        expected_keys=column_names,
-        column_names=column_names,
-        label_name=label,
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=False,
-        column_defaults=record_defaults,
-    )
+    defaults = self.DEFAULTS
+    file_without_header = self._create_file(
+        len(self._test_filenames), header=False)
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            file_without_header,
+            defaults,
+            batch_size=2,
+            num_epochs=10,
+            header=False,
+        )
+        self._verify_records(
+            sess,
+            dataset,
+            [len(self._test_filenames)],
+            batch_size=2,
+            num_epochs=10,
+        )
 
   def testMakeCSVDataset_withTypes(self):
     """Tests that defaults can be a dtype instead of a Tensor for required vals.
     """
-    record_defaults = [
-        dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64,
-        dtypes.string
-    ]
-
-    column_names = ["col%d" % i for i in range(5)]
-    inputs = [[",".join(x[0] for x in column_names), "0,1,2,3,4", "5,6,7,8,9"],
-              [
-                  ",".join(x[0] for x in column_names), "10,11,12,13,14",
-                  "15,16,17,18,19"
-              ]]
-    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
-                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
-    label = "col0"
-
-    self._test_dataset(
-        inputs,
-        expected_output=expected_output,
-        expected_keys=column_names,
-        column_names=column_names,
-        label_name=label,
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=True,
-        column_defaults=record_defaults,
-    )
+    defaults = [d for d in self.COLUMN_TYPES[:-1]]
+    defaults.append(constant_op.constant(["NULL"], dtype=dtypes.string))
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(self._test_filenames, defaults)
+        self._verify_records(sess, dataset, range(self._num_files))
 
   def testMakeCSVDataset_withNoColNames(self):
     """Tests that datasets can be created when column names are not specified.
 
     In that case, we should infer the column names from the header lines.
     """
-    record_defaults = [
-        constant_op.constant([], dtypes.int32),
-        constant_op.constant([], dtypes.int64),
-        constant_op.constant([], dtypes.float32),
-        constant_op.constant([], dtypes.float64),
-        constant_op.constant([], dtypes.string)
-    ]
-
-    column_names = ["col%d" % i for i in range(5)]
-    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
-        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
-    ]]
-    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
-                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
-    label = "col0"
-
-    self._test_dataset(
-        inputs,
-        expected_output=expected_output,
-        expected_keys=column_names,
-        label_name=label,
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=True,
-        column_defaults=record_defaults,
-    )
+    defaults = self.DEFAULTS
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        # Read from both files. Exercise the `batch` and `num_epochs` parameters
+        # of make_csv_dataset and make sure they work.
+        dataset = self._make_csv_dataset(
+            self._test_filenames,
+            defaults,
+            column_names=None,
+            batch_size=2,
+            num_epochs=10)
+        self._verify_records(
+            sess, dataset, range(self._num_files), batch_size=2, num_epochs=10)
 
   def testMakeCSVDataset_withTypeInferenceMismatch(self):
     # Test that error is thrown when num fields doesn't match columns
-    column_names = ["col%d" % i for i in range(5)]
-    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
-        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
-    ]]
-    filenames = self._setup_files(inputs)
     with self.assertRaises(ValueError):
       self._make_csv_dataset(
-          filenames,
-          column_names=column_names + ["extra_name"],
-          column_defaults=None,
+          self._test_filenames,
+          column_names=self.COLUMNS + ["extra_name"],
+          defaults=None,
           batch_size=2,
           num_epochs=10)
 
@@ -550,215 +448,197 @@ class MakeCsvDatasetTest(test.TestCase):
 
     In that case, we should infer the types from the first N records.
     """
-    column_names = ["col%d" % i for i in range(5)]
-    str_int32_max = str(2**33)
-    inputs = [[
-        ",".join(x for x in column_names),
-        "0,%s,2.0,3e50,rabbit" % str_int32_max
-    ]]
-    expected_output = [[0, 2**33, 2.0, 3e50, b"rabbit"]]
-    label = "col0"
-
-    self._test_dataset(
-        inputs,
-        expected_output=expected_output,
-        expected_keys=column_names,
-        column_names=column_names,
-        label_name=label,
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=True,
-    )
-
-  def testMakeCSVDataset_withTypeInferenceFallthrough(self):
-    """Tests that datasets can be created when no defaults are specified.
-
-    Tests on a deliberately tricky file.
-    """
-    column_names = ["col%d" % i for i in range(5)]
-    str_int32_max = str(2**33)
-    inputs = [[
-        ",".join(x for x in column_names),
-        ",,,,",
-        "0,0,0.0,0.0,0.0",
-        "0,%s,2.0,3e50,rabbit" % str_int32_max,
-        ",,,,",
-    ]]
-    expected_output = [[0, 0, 0, 0, b""], [0, 0, 0, 0, b"0.0"],
-                       [0, 2**33, 2.0, 3e50, b"rabbit"], [0, 0, 0, 0, b""]]
-    label = "col0"
-
-    self._test_dataset(
-        inputs,
-        expected_output=expected_output,
-        expected_keys=column_names,
-        column_names=column_names,
-        label_name=label,
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=True,
-    )
-
-  def testMakeCSVDataset_withSelectCols(self):
-    record_defaults = [
-        constant_op.constant([], dtypes.int32),
-        constant_op.constant([], dtypes.int64),
-        constant_op.constant([], dtypes.float32),
-        constant_op.constant([], dtypes.float64),
-        constant_op.constant([], dtypes.string)
+    # Test that it works with standard test files (with header, etc)
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            self._test_filenames, defaults=None, batch_size=2, num_epochs=10)
+        self._verify_records(
+            sess,
+            dataset,
+            range(self._num_files),
+            batch_size=2,
+            num_epochs=10,
+            defaults=[[], [], [], [], [""]])
+
+  def testMakeCSVDataset_withTypeInferenceTricky(self):
+    # Test on a deliberately tricky file (type changes as we read more rows, and
+    # there are null values)
+    fn = os.path.join(self.get_temp_dir(), "file.csv")
+    expected_dtypes = [
+        dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float32,
+        dtypes.string, dtypes.string
     ]
-    column_names = ["col%d" % i for i in range(5)]
-    str_int32_max = str(2**33)
-    inputs = [[
-        ",".join(x for x in column_names),
-        "0,%s,2.0,3e50,rabbit" % str_int32_max
-    ]]
-    expected_output = [[0, 2**33, 2.0, 3e50, b"rabbit"]]
-
-    select_cols = [1, 3, 4]
-    self._test_dataset(
-        inputs,
-        expected_output=[[x[i] for i in select_cols] for x in expected_output],
-        expected_keys=[column_names[i] for i in select_cols],
-        column_names=column_names,
-        column_defaults=[record_defaults[i] for i in select_cols],
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=True,
-        select_columns=select_cols,
-    )
-
-    # Can still do inference without provided defaults
-    self._test_dataset(
-        inputs,
-        expected_output=[[x[i] for i in select_cols] for x in expected_output],
-        expected_keys=[column_names[i] for i in select_cols],
-        column_names=column_names,
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=True,
-        select_columns=select_cols,
-    )
-
-    # Can still do column name inference
-    self._test_dataset(
-        inputs,
-        expected_output=[[x[i] for i in select_cols] for x in expected_output],
-        expected_keys=[column_names[i] for i in select_cols],
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=True,
-        select_columns=select_cols,
-    )
-
-    # Can specify column names instead of indices
-    self._test_dataset(
-        inputs,
-        expected_output=[[x[i] for i in select_cols] for x in expected_output],
-        expected_keys=[column_names[i] for i in select_cols],
-        column_names=column_names,
-        batch_size=1,
-        num_epochs=1,
-        shuffle=False,
-        header=True,
-        select_columns=[column_names[i] for i in select_cols],
-    )
+    col_names = ["col%d" % i for i in range(len(expected_dtypes))]
+    rows = [[None, None, None, "NAN", "",
+             "a"], [1, 2**31 + 1, 2**64, 123, "NAN", ""],
+            ['"123"', 2, 2**64, 123.4, "NAN", '"cd,efg"']]
+    expected = [[0, 0, 0, 0, "", "a"], [1, 2**31 + 1, 2**64, 123, "", ""],
+                [123, 2, 2**64, 123.4, "", "cd,efg"]]
+    for row in expected:
+      row[-1] = row[-1].encode("utf-8")  # py3 expects byte strings
+      row[-2] = row[-2].encode("utf-8")  # py3 expects byte strings
+    self._write_file("file.csv", [col_names] + rows)
 
-  def testMakeCSVDataset_withSelectColsError(self):
-    record_defaults = [
-        constant_op.constant([], dtypes.int32),
-        constant_op.constant([], dtypes.int64),
-        constant_op.constant([], dtypes.float32),
-        constant_op.constant([], dtypes.float64),
-        constant_op.constant([], dtypes.string)
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            fn,
+            defaults=None,
+            column_names=None,
+            label_name=None,
+            na_value="NAN",
+        )
+        features = dataset.make_one_shot_iterator().get_next()
+        # Check that types match
+        for i in range(len(expected_dtypes)):
+          print(features["col%d" % i].dtype, expected_dtypes[i])
+          assert features["col%d" % i].dtype == expected_dtypes[i]
+        for i in range(len(rows)):
+          assert sess.run(features) == dict(zip(col_names, expected[i]))
+
+  def testMakeCSVDataset_withTypeInferenceAllTypes(self):
+    # Test that we make the correct inference for all types with fallthrough
+    fn = os.path.join(self.get_temp_dir(), "file.csv")
+    expected_dtypes = [
+        dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64,
+        dtypes.string, dtypes.string
     ]
-    column_names = ["col%d" % i for i in range(5)]
-    str_int32_max = str(2**33)
-    inputs = [[
-        ",".join(x for x in column_names),
-        "0,%s,2.0,3e50,rabbit" % str_int32_max
+    col_names = ["col%d" % i for i in range(len(expected_dtypes))]
+    rows = [[1, 2**31 + 1, 1.0, 4e40, "abc", ""]]
+    expected = [[
+        1, 2**31 + 1, 1.0, 4e40, "abc".encode("utf-8"), "".encode("utf-8")
     ]]
+    self._write_file("file.csv", [col_names] + rows)
 
-    select_cols = [1, 3, 4]
-    filenames = self._setup_files(inputs)
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            fn,
+            defaults=None,
+            column_names=None,
+            label_name=None,
+            na_value="NAN",
+        )
+        features = dataset.make_one_shot_iterator().get_next()
+        # Check that types match
+        for i in range(len(expected_dtypes)):
+          self.assertAllEqual(features["col%d" % i].dtype, expected_dtypes[i])
+        for i in range(len(rows)):
+          self.assertAllEqual(
+              sess.run(features), dict(zip(col_names, expected[i])))
 
+  def testMakeCSVDataset_withSelectColsError(self):
+    data = [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
+    col_names = ["col%d" % i for i in range(5)]
+    fn = self._write_file("file.csv", [col_names] + data)
     with self.assertRaises(ValueError):
       # Mismatch in number of defaults and number of columns selected,
       # should raise an error
       self._make_csv_dataset(
-          filenames,
-          batch_size=1,
-          column_defaults=record_defaults,
-          column_names=column_names,
-          select_columns=select_cols)
-
+          fn,
+          defaults=[[0]] * 5,
+          column_names=col_names,
+          label_name=None,
+          select_cols=[1, 3])
     with self.assertRaises(ValueError):
       # Invalid column name should raise an error
       self._make_csv_dataset(
-          filenames,
-          batch_size=1,
-          column_defaults=[[0]],
-          column_names=column_names,
+          fn,
+          defaults=[[0]],
+          column_names=col_names,
           label_name=None,
-          select_columns=["invalid_col_name"])
-
-  def testMakeCSVDataset_withShuffle(self):
-    record_defaults = [
-        constant_op.constant([], dtypes.int32),
-        constant_op.constant([], dtypes.int64),
-        constant_op.constant([], dtypes.float32),
-        constant_op.constant([], dtypes.float64),
-        constant_op.constant([], dtypes.string)
-    ]
-
-    def str_series(st):
-      return ",".join(str(i) for i in range(st, st + 5))
+          select_cols=["invalid_col_name"])
 
-    column_names = ["col%d" % i for i in range(5)]
-    inputs = [
-        [",".join(x for x in column_names)
-        ] + [str_series(5 * i) for i in range(15)],
-        [",".join(x for x in column_names)] +
-        [str_series(5 * i) for i in range(15, 20)],
-    ]
-
-    filenames = self._setup_files(inputs)
+  def testMakeCSVDataset_withSelectCols(self):
+    data = [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
+    col_names = ["col%d" % i for i in range(5)]
+    fn = self._write_file("file.csv", [col_names] + data)
+    # If select_cols is specified, should only yield a subset of columns
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            fn,
+            defaults=[[0], [0]],
+            column_names=col_names,
+            label_name=None,
+            select_cols=[1, 3])
+        expected = [[1, 3], [6, 8]]
+        features = dataset.make_one_shot_iterator().get_next()
+        for i in range(len(data)):
+          self.assertAllEqual(
+              sess.run(features),
+              dict(zip([col_names[1], col_names[3]], expected[i])))
+    # Can still do default inference with select_cols
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            fn,
+            defaults=None,
+            column_names=col_names,
+            label_name=None,
+            select_cols=[1, 3])
+        expected = [[1, 3], [6, 8]]
+        features = dataset.make_one_shot_iterator().get_next()
+        for i in range(len(data)):
+          self.assertAllEqual(
+              sess.run(features),
+              dict(zip([col_names[1], col_names[3]], expected[i])))
+    # Can still do column name inference
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            fn,
+            defaults=None,
+            column_names=None,
+            label_name=None,
+            select_cols=[1, 3])
+        expected = [[1, 3], [6, 8]]
+        features = dataset.make_one_shot_iterator().get_next()
+        for i in range(len(data)):
+          self.assertAllEqual(
+              sess.run(features),
+              dict(zip([col_names[1], col_names[3]], expected[i])))
+    # Can specify column names instead of indices
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            fn,
+            defaults=None,
+            column_names=None,
+            label_name=None,
+            select_cols=[col_names[1], col_names[3]])
+        expected = [[1, 3], [6, 8]]
+        features = dataset.make_one_shot_iterator().get_next()
+        for i in range(len(data)):
+          self.assertAllEqual(
+              sess.run(features),
+              dict(zip([col_names[1], col_names[3]], expected[i])))
 
-    total_records = 20
+  def testMakeCSVDataset_withShuffle(self):
+    total_records = self._num_files * self._num_records
+    defaults = self.DEFAULTS
     for batch_size in [1, 2]:
       with ops.Graph().as_default() as g:
         with self.test_session(graph=g) as sess:
           # Test that shuffling with the same seed produces the same result
           dataset1 = self._make_csv_dataset(
-              filenames,
-              column_defaults=record_defaults,
-              column_names=column_names,
+              self._test_filenames,
+              defaults,
               batch_size=batch_size,
-              header=True,
               shuffle=True,
-              shuffle_seed=5,
-              num_epochs=2,
-          )
+              shuffle_seed=5)
           dataset2 = self._make_csv_dataset(
-              filenames,
-              column_defaults=record_defaults,
-              column_names=column_names,
+              self._test_filenames,
+              defaults,
               batch_size=batch_size,
-              header=True,
               shuffle=True,
-              shuffle_seed=5,
-              num_epochs=2,
-          )
+              shuffle_seed=5)
           outputs1 = dataset1.make_one_shot_iterator().get_next()
           outputs2 = dataset2.make_one_shot_iterator().get_next()
           for _ in range(total_records // batch_size):
-            batch1 = nest.flatten(sess.run(outputs1))
-            batch2 = nest.flatten(sess.run(outputs2))
+            batch1 = self._run_actual_batch(outputs1, sess)
+            batch2 = self._run_actual_batch(outputs2, sess)
             for i in range(len(batch1)):
               self.assertAllEqual(batch1[i], batch2[i])
 
@@ -766,31 +646,23 @@ class MakeCsvDatasetTest(test.TestCase):
         with self.test_session(graph=g) as sess:
           # Test that shuffling with a different seed produces different results
           dataset1 = self._make_csv_dataset(
-              filenames,
-              column_defaults=record_defaults,
-              column_names=column_names,
+              self._test_filenames,
+              defaults,
               batch_size=batch_size,
-              header=True,
               shuffle=True,
-              shuffle_seed=5,
-              num_epochs=2,
-          )
+              shuffle_seed=5)
           dataset2 = self._make_csv_dataset(
-              filenames,
-              column_defaults=record_defaults,
-              column_names=column_names,
+              self._test_filenames,
+              defaults,
               batch_size=batch_size,
-              header=True,
               shuffle=True,
-              shuffle_seed=6,
-              num_epochs=2,
-          )
+              shuffle_seed=6)
           outputs1 = dataset1.make_one_shot_iterator().get_next()
           outputs2 = dataset2.make_one_shot_iterator().get_next()
           all_equal = False
           for _ in range(total_records // batch_size):
-            batch1 = nest.flatten(sess.run(outputs1))
-            batch2 = nest.flatten(sess.run(outputs2))
+            batch1 = self._run_actual_batch(outputs1, sess)
+            batch2 = self._run_actual_batch(outputs2, sess)
             for i in range(len(batch1)):
               all_equal = all_equal and np.array_equal(batch1[i], batch2[i])
           self.assertFalse(all_equal)
@@ -1002,5 +874,6 @@ class MakeTFRecordDatasetTest(
           self._shuffle_test(batch_size, num_epochs, num_parallel_reads,
                              seed=21345)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py
index f018dd02e6..9373e37f5f 100644
--- a/tensorflow/contrib/data/python/ops/readers.py
+++ b/tensorflow/contrib/data/python/ops/readers.py
@@ -326,7 +326,6 @@ def make_csv_dataset(
     num_parallel_parser_calls=2,
     sloppy=False,
     num_rows_for_inference=100,
-    compression_type=None,
 ):
   """Reads CSV files into a dataset.
 
@@ -400,8 +399,6 @@ def make_csv_dataset(
     num_rows_for_inference: Number of rows of a file to use for type inference
       if record_defaults is not provided. If None, reads all the rows of all
       the files. Defaults to 100.
-    compression_type: (Optional.) A `tf.string` scalar evaluating to one of
-      `""` (no compression), `"ZLIB"`, or `"GZIP"`. Defaults to no compression.
 
   Returns:
     A dataset, where each element is a (features, labels) tuple that corresponds
@@ -464,9 +461,7 @@ def make_csv_dataset(
         use_quote_delim=use_quote_delim,
         na_value=na_value,
         select_cols=select_columns,
-        header=header,
-        compression_type=compression_type,
-    )
+        header=header)
 
   def map_fn(*columns):
     """Organizes columns into a features dictionary.
@@ -510,7 +505,6 @@ class CsvDataset(dataset_ops.Dataset):
   def __init__(self,
                filenames,
                record_defaults,
-               compression_type=None,
                buffer_size=None,
                header=False,
                field_delim=",",
@@ -568,9 +562,6 @@ class CsvDataset(dataset_ops.Dataset):
         both this and `select_columns` are specified, these must have the same
         lengths, and `column_defaults` is assumed to be sorted in order of
         increasing column index.
-      compression_type: (Optional.) A `tf.string` scalar evaluating to one of
-        `""` (no compression), `"ZLIB"`, or `"GZIP"`. Defaults to no
-        compression.
       buffer_size: (Optional.) A `tf.int64` scalar denoting the number of bytes
         to buffer while reading files. Defaults to 4MB.
       header: (Optional.) A `tf.bool` scalar indicating whether the CSV file(s)
@@ -590,11 +581,6 @@ class CsvDataset(dataset_ops.Dataset):
     super(CsvDataset, self).__init__()
     self._filenames = ops.convert_to_tensor(
         filenames, dtype=dtypes.string, name="filenames")
-    self._compression_type = convert.optional_param_to_tensor(
-        "compression_type",
-        compression_type,
-        argument_default="",
-        argument_dtype=dtypes.string)
     record_defaults = [
         constant_op.constant([], dtype=x) if x in _ACCEPTABLE_CSV_TYPES else x
         for x in record_defaults
@@ -635,7 +621,6 @@ class CsvDataset(dataset_ops.Dataset):
         use_quote_delim=self._use_quote_delim,
         na_value=self._na_value,
         select_cols=self._select_cols,
-        compression_type=self._compression_type,
     )
 
   @property
-- 
cgit v1.2.3


From eedee8236e7693f921723ad942baef7b61b3ceda Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 16 Jul 2018 16:21:39 -0700
Subject: Disable rewriting errors for classes due to source map overwriting.

PiperOrigin-RevId: 204826362
---
 tensorflow/contrib/autograph/impl/conversion.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/autograph/impl/conversion.py b/tensorflow/contrib/autograph/impl/conversion.py
index 1c768b659f..7bd0ba3f2d 100644
--- a/tensorflow/contrib/autograph/impl/conversion.py
+++ b/tensorflow/contrib/autograph/impl/conversion.py
@@ -158,7 +158,8 @@ def class_to_graph(c, program_ctx):
         program_ctx=program_ctx,
         arg_values={},
         arg_types={'self': (c.__name__, c)},
-        owner_type=c)
+        owner_type=c,
+        rewrite_errors=False)
     if class_namespace is None:
       class_namespace = namespace
     else:
@@ -242,7 +243,12 @@ def _add_self_references(namespace, autograph_module):
   _add_reserved_symbol(namespace, 'ag__', ag_internal)
 
 
-def function_to_graph(f, program_ctx, arg_values, arg_types, owner_type=None):
+def function_to_graph(f,
+                      program_ctx,
+                      arg_values,
+                      arg_types,
+                      owner_type=None,
+                      rewrite_errors=True):
   """Specialization of `entity_to_graph` for callable functions."""
 
   node, source = parser.parse_entity(f)
@@ -260,7 +266,7 @@ def function_to_graph(f, program_ctx, arg_values, arg_types, owner_type=None):
       arg_types=arg_types,
       owner_type=owner_type)
   context = converter.EntityContext(namer, entity_info, program_ctx)
-  node = node_to_graph(node, context)
+  node = node_to_graph(node, context, rewrite_errors=rewrite_errors)
 
   # TODO(mdan): This somewhat duplicates the call rename logic in call_treest.py
   new_name, did_rename = namer.compiled_function_name(f.__name__, f, owner_type)
@@ -276,12 +282,13 @@ def function_to_graph(f, program_ctx, arg_values, arg_types, owner_type=None):
   return node, new_name, namespace
 
 
-def node_to_graph(node, context):
+def node_to_graph(node, context, rewrite_errors=True):
   """Convert Python code to equivalent TF graph mode code.
 
   Args:
     node: AST, the code to convert.
     context: converter.EntityContext
+    rewrite_errors: Boolean, whether or not to rewrite the error traceback.
 
   Returns:
     A tuple (node, deps):
@@ -316,5 +323,6 @@ def node_to_graph(node, context):
   node = converter.apply_(node, context, logical_expressions)
   node = converter.apply_(node, context, side_effect_guards)
   node = converter.apply_(node, context, name_scopes)
-  node = converter.apply_(node, context, error_handlers)
+  if rewrite_errors:
+    node = converter.apply_(node, context, error_handlers)
   return node
-- 
cgit v1.2.3


From 8e27c65acffaa6a033faea0084669f86b012dcd1 Mon Sep 17 00:00:00 2001
From: Youlong Cheng 
Date: Mon, 16 Jul 2018 17:02:57 -0700
Subject: PUBLIC: Support replicated variable backed by tpu_replicated_input.
 RELNOTES: n/a

PiperOrigin-RevId: 204832490
---
 tensorflow/contrib/tpu/python/tpu/tpu.py | 35 ++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index 6a64893d9a..7216626a58 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -151,6 +151,41 @@ class TPUReplicateContext(control_flow_ops.XLAControlFlowContext):
     self._name = name
     self._unsupported_ops = []
     self._pivot = pivot
+    self._replicated_vars = {}
+
+  def get_replicated_var_handle(self, var):
+    """Returns a variable handle for replicated TPU variable 'var'.
+
+    This is an method used by an experimental replicated variable
+    implementation and is not intended as a public API.
+
+    Args:
+      var: The replicated TPU variable.
+
+    Returns:
+      The handle of the TPU replicated input node.
+    """
+    handle = self._replicated_vars.get(var)
+    if handle is not None:
+      return handle
+
+    # Builds a TPUReplicatedInput node for the variable, if one does not already
+    # exist. The TPUReplicatedInput node must belong to the enclosing
+    # control-flow scope of the TPUReplicateContext.
+    # TODO(phawkins): consider changing the contract of the TPU encapsulation
+    # so the TPUReplicatedInput nodes go inside the TPUReplicateContext scope
+    # instead.
+
+    # pylint: disable=protected-access
+    graph = ops.get_default_graph()
+    saved_context = graph._get_control_flow_context()
+    graph._set_control_flow_context(self.outer_context)
+    handle = tpu_ops.tpu_replicated_input(
+        [v.handle for v in var._vars], name=var.name + "/handle")
+    graph._set_control_flow_context(saved_context)
+    # pylint: enable=protected-access
+    self._replicated_vars[var] = handle
+    return handle
 
   def report_unsupported_operations(self):
     if self._unsupported_ops:
-- 
cgit v1.2.3


From e4c0dbcab8b404949a67c282fe7cae89c5b4ad87 Mon Sep 17 00:00:00 2001
From: Toby Boyd 
Date: Mon, 16 Jul 2018 17:04:48 -0700
Subject: internal change

PiperOrigin-RevId: 204832902
---
 configure.py                        | 4 +---
 third_party/nccl/nccl_configure.bzl | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/configure.py b/configure.py
index cd9d5d32a0..10387493b0 100644
--- a/configure.py
+++ b/configure.py
@@ -1138,9 +1138,7 @@ def set_tf_nccl_install_path(environ_cp):
 
     nccl_lib_path = os.path.join(nccl_install_path, nccl_lib_path)
     nccl_hdr_path = os.path.join(nccl_install_path, 'include/nccl.h')
-    nccl_license_path = os.path.join(nccl_install_path, 'NCCL-SLA.txt')
-    if os.path.exists(nccl_lib_path) and os.path.exists(
-        nccl_hdr_path) and os.path.exists(nccl_license_path):
+    if os.path.exists(nccl_lib_path) and os.path.exists(nccl_hdr_path):
       # Set NCCL_INSTALL_PATH
       environ_cp['NCCL_INSTALL_PATH'] = nccl_install_path
       write_action_env_to_bazelrc('NCCL_INSTALL_PATH', nccl_install_path)
diff --git a/third_party/nccl/nccl_configure.bzl b/third_party/nccl/nccl_configure.bzl
index 9dfcb18369..5d1ebf0686 100644
--- a/third_party/nccl/nccl_configure.bzl
+++ b/third_party/nccl/nccl_configure.bzl
@@ -47,10 +47,10 @@ alias(
 )
 """
 
+# Local build results in dynamic link and the license should not be included.
 _NCCL_LOCAL_BUILD_TEMPLATE = """
 filegroup(
   name = "LICENSE",
-  data = ["nccl/NCCL-SLA.txt"],
   visibility = ["//visibility:public"],
 )
 
-- 
cgit v1.2.3


From 6bfa38ef2963f0062fbe12d532ab188c7d5ea8dd Mon Sep 17 00:00:00 2001
From: Pavithra Vijay 
Date: Mon, 16 Jul 2018 17:10:04 -0700
Subject: Fix checkpointable dependencies in recurrent layers. 1. Add
 dependency between RNN layer and RNN Cell. 2. Add dependency between
 Bidirectional layer and the inner forward and backward layers.

PiperOrigin-RevId: 204833646
---
 tensorflow/python/keras/layers/recurrent.py      |  3 +++
 tensorflow/python/keras/layers/recurrent_test.py | 18 ++++++++++++++++++
 tensorflow/python/keras/layers/wrappers.py       |  7 +++++--
 tensorflow/python/keras/layers/wrappers_test.py  |  8 ++++++++
 4 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/layers/recurrent.py b/tensorflow/python/keras/layers/recurrent.py
index 61775da47b..534c0eca08 100644
--- a/tensorflow/python/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/layers/recurrent.py
@@ -37,6 +37,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training.checkpointable import base as checkpointable
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -403,6 +404,8 @@ class RNN(Layer):
                        'one integer per RNN state).')
     super(RNN, self).__init__(**kwargs)
     self.cell = cell
+    if isinstance(cell, checkpointable.CheckpointableBase):
+      self._track_checkpointable(self.cell, name='cell')
     self.return_sequences = return_sequences
     self.return_state = return_state
     self.go_backwards = go_backwards
diff --git a/tensorflow/python/keras/layers/recurrent_test.py b/tensorflow/python/keras/layers/recurrent_test.py
index 802374d2d2..fefb92826b 100644
--- a/tensorflow/python/keras/layers/recurrent_test.py
+++ b/tensorflow/python/keras/layers/recurrent_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.platform import test
+from tensorflow.python.training.checkpointable import util as checkpointable_util
 
 
 class RNNTest(test.TestCase):
@@ -556,5 +557,22 @@ class RNNTest(test.TestCase):
         [tuple(o.as_list()) for o in output_shape],
         expected_output_shape)
 
+  def test_checkpointable_dependencies(self):
+    rnn = keras.layers.SimpleRNN
+    with self.test_session():
+      x = np.random.random((2, 2, 2))
+      y = np.random.random((2, 2))
+      model = keras.models.Sequential()
+      model.add(rnn(2))
+      model.compile(optimizer='rmsprop', loss='mse')
+      model.fit(x, y, epochs=1, batch_size=1)
+
+      # check whether the model variables are present in the
+      # checkpointable list of objects
+      checkpointed_objects = set(checkpointable_util.list_objects(model))
+      for v in model.variables:
+        self.assertIn(v, checkpointed_objects)
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/keras/layers/wrappers.py b/tensorflow/python/keras/layers/wrappers.py
index f651e03874..f0c1e76156 100644
--- a/tensorflow/python/keras/layers/wrappers.py
+++ b/tensorflow/python/keras/layers/wrappers.py
@@ -47,7 +47,6 @@ class Wrapper(Layer):
   def __init__(self, layer, **kwargs):
     assert isinstance(layer, Layer)
     self.layer = layer
-    self._track_checkpointable(layer, name='layer')
     # Tracks mapping of Wrapper inputs to inner layer inputs. Useful when
     # the inner layer has update ops that depend on its inputs (as opposed
     # to the inputs to the Wrapper layer).
@@ -168,6 +167,7 @@ class TimeDistributed(Wrapper):
           '`Layer` instance. You passed: {input}'.format(input=layer))
     super(TimeDistributed, self).__init__(layer, **kwargs)
     self.supports_masking = True
+    self._track_checkpointable(layer, name='layer')
 
   def _get_shape_tuple(self, init_tuple, tensor, start_idx, int_shape=None):
     """Finds non-specific dimensions in the static shapes.
@@ -417,6 +417,8 @@ class Bidirectional(Wrapper):
     self._num_constants = None
     super(Bidirectional, self).__init__(layer, **kwargs)
     self.input_spec = layer.input_spec
+    self._track_checkpointable(self.forward_layer, name='forward_layer')
+    self._track_checkpointable(self.backward_layer, name='backward_layer')
 
   @property
   def trainable(self):
@@ -526,7 +528,8 @@ class Bidirectional(Wrapper):
     else:
       return super(Bidirectional, self).__call__(inputs, **kwargs)
 
-  def call(self, inputs,
+  def call(self,
+           inputs,
            training=None,
            mask=None,
            initial_state=None,
diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py
index 3f268acf5c..0cd774ef0f 100644
--- a/tensorflow/python/keras/layers/wrappers_test.py
+++ b/tensorflow/python/keras/layers/wrappers_test.py
@@ -87,6 +87,8 @@ class TimeDistributedTest(test.TestCase):
     # test config
     model.get_config()
 
+    # check whether the model variables are present in the
+    # checkpointable list of objects
     checkpointed_objects = set(checkpointable_util.list_objects(model))
     for v in model.variables:
       self.assertIn(v, checkpointed_objects)
@@ -278,6 +280,12 @@ class BidirectionalTest(test.TestCase):
         model.compile(optimizer=RMSPropOptimizer(0.01), loss='mse')
         model.fit(x, y, epochs=1, batch_size=1)
 
+        # check whether the model variables are present in the
+        # checkpointable list of objects
+        checkpointed_objects = set(checkpointable_util.list_objects(model))
+        for v in model.variables:
+          self.assertIn(v, checkpointed_objects)
+
         # test compute output shape
         ref_shape = model.layers[-1].output.get_shape()
         shape = model.layers[-1].compute_output_shape(
-- 
cgit v1.2.3


From 545458669207ab1ffbbb531c40f709a22130043f Mon Sep 17 00:00:00 2001
From: Clayne Robison 
Date: Mon, 16 Jul 2018 17:14:26 -0700
Subject: Adding support for MKL builds with AVX2. MKL-DNN ignores the compiler
 switches and takes code paths based on the platform detected at runtime. This
 commit will add support for avx2 instructions to the rest of TensorFlow.

---
 .../ci_build/linux/mkl/build-dev-container.sh      | 25 ++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh b/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh
index ad22ebe4eb..1d9c832d66 100755
--- a/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh
+++ b/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh
@@ -34,6 +34,9 @@ echo "TF_DOCKER_BUILD_DEVEL_BRANCH=${TF_DOCKER_BUILD_DEVEL_BRANCH}"
 echo "TF_DOCKER_BUILD_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME}"
 echo "TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION}"
 
+# Build containers for AVX
+#"TF_BAZEL_BUILD_OPTIONS": "'{}'" (default build option= avx)
+
 # build the python 2 container and whl
 TF_DOCKER_BUILD_TYPE="MKL" \
   TF_DOCKER_BUILD_IS_DEVEL="YES" \
@@ -49,5 +52,27 @@ TF_DOCKER_BUILD_TYPE="MKL" \
   TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
   TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \
   TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3" \
+  ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh
+
+# Build containers for AVX2
+TF_BAZEL_BUILD_OPTIONS="--config=mkl --copt=-mavx2 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0"
+
+# build the python 2 container and whl
+TF_DOCKER_BUILD_TYPE="MKL" \
+  TF_DOCKER_BUILD_IS_DEVEL="YES" \
+  TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \
+  TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
+  TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}-avx2" \
+  TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \
   ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh 
 
+# build the python 3 container and whl
+TF_DOCKER_BUILD_TYPE="MKL" \
+  TF_DOCKER_BUILD_IS_DEVEL="YES" \
+  TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \
+  TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
+  TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}-avx2" \
+  TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3" \
+  TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \
+  ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh
+
-- 
cgit v1.2.3


From 2c442d26f36a0f167685fd31b9ecdb4e290c2b29 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 16 Jul 2018 17:13:42 -0700
Subject: Implement digamma for XLA

Compute the Lgamma function using Lanczos' approximation from "A Precision Approximation of the Gamma Function". SIAM Journal on Numerical Analysis series B. Vol. 1:
digamma(z + 1) = log(t(z)) + A'(z) / A(z) - kLanczosGamma / t(z)
t(z) = z + kLanczosGamma + 1/2
A(z) = kBaseLanczosCoeff + sigma(k = 1, n, kLanczosCoefficients[i] / (z + k))
A'(z) = sigma(k = 1, n, kLanczosCoefficients[i] / (z + k) / (z + k))

PiperOrigin-RevId: 204834091
---
 tensorflow/compiler/tests/unary_ops_test.py     | 32 +++++++++++++++
 tensorflow/compiler/tf2xla/kernels/unary_ops.cc | 23 +++++++++++
 tensorflow/compiler/xla/client/lib/math.cc      | 52 +++++++++++++++++++++++++
 tensorflow/compiler/xla/client/lib/math.h       |  3 ++
 tensorflow/compiler/xla/client/lib/math_test.cc | 31 +++++++++++++++
 5 files changed, 141 insertions(+)

diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index 0419419ea5..5f25ff9002 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -406,6 +406,38 @@ class UnaryOpsTest(xla_test.XLATestCase):
               ],
               dtype=dtype))
 
+      self._assertOpOutputMatchesExpected(
+          math_ops.digamma,
+          np.array(
+              [[1.0, 0.5, 1 / 3.0], [0.25, 1 / 6.0, 0.125], [2.0, 3.0, 4.0],
+               [6.0, 8.0, 9.0]],
+              dtype=dtype),
+          expected=np.array(
+              [
+                  [
+                      -np.euler_gamma, -2 * np.log(2) - np.euler_gamma,
+                      -np.pi / 2 / np.sqrt(3) - 3 * np.log(3) / 2 -
+                      np.euler_gamma
+                  ],
+                  [
+                      -np.pi / 2 - 3 * np.log(2) - np.euler_gamma,
+                      -np.pi * np.sqrt(3) / 2 - 2 * np.log(2) -
+                      3 * np.log(3) / 2 - np.euler_gamma,
+                      -np.pi / 2 - 4 * np.log(2) -
+                      (np.pi + np.log(2 + np.sqrt(2)) - np.log(2 - np.sqrt(2)))
+                      / np.sqrt(2) - np.euler_gamma
+                  ],
+                  [
+                      1 - np.euler_gamma, 1.5 - np.euler_gamma,
+                      11 / 6.0 - np.euler_gamma
+                  ],
+                  [
+                      137 / 60.0 - np.euler_gamma, 363 / 140.0 - np.euler_gamma,
+                      761 / 280.0 - np.euler_gamma
+                  ],
+              ],
+              dtype=dtype))
+
       def quantize_and_dequantize_v2(x):
         return array_ops.quantize_and_dequantize_v2(
             x, -127, 127, signed_input=True, num_bits=8)
diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
index 76ab8b4c00..4bb31f4117 100644
--- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
@@ -212,5 +212,28 @@ class LgammaOp : public XlaOpKernel {
 };  // namespace
 REGISTER_XLA_OP(Name("Lgamma"), LgammaOp);
 
+class DigammaOp : public XlaOpKernel {
+ public:
+  explicit DigammaOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
+  // Calculate lgamma using the Lanczos approximation
+  // (https://en.wikipedia.org/wiki/Lanczos_approximation).
+  void Compile(XlaOpKernelContext* ctx) override {
+    xla::XlaOp input = ctx->Input(0);
+    xla::PrimitiveType input_type = ctx->input_xla_type(0);
+
+    if (input_type == xla::F16 || input_type == xla::BF16) {
+      // The approximation works better with at least 32-bits of accuracy.
+      xla::XlaOp input_f32 = xla::ConvertElementType(input, xla::F32);
+      xla::XlaOp result_f32 = xla::Digamma(input_f32);
+      xla::XlaOp result_x16 = xla::ConvertElementType(result_f32, input_type);
+      ctx->SetOutput(0, result_x16);
+    } else {
+      xla::XlaOp result = xla::Digamma(input);
+      ctx->SetOutput(0, result);
+    }
+  }
+};  // namespace
+REGISTER_XLA_OP(Name("Digamma"), DigammaOp);
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index fdc7057de3..2a7ac1d716 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -217,4 +217,56 @@ xla::XlaOp Lgamma(xla::XlaOp input) {
   return result;
 }
 
+// Compute the Digamma function using Lanczos' approximation from "A Precision
+// Approximation of the Gamma Function". SIAM Journal on Numerical Analysis
+// series B. Vol. 1:
+// digamma(z + 1) = log(t(z)) + A'(z) / A(z) - kLanczosGamma / t(z)
+// t(z) = z + kLanczosGamma + 1/2
+// A(z) = kBaseLanczosCoeff + sigma(k = 1, n, kLanczosCoefficients[i] / (z + k))
+// A'(z) = sigma(k = 1, n, kLanczosCoefficients[i] / (z + k) / (z + k))
+xla::XlaOp Digamma(xla::XlaOp input) {
+  xla::XlaOp zero = xla::ScalarLike(input, 0);
+  xla::XlaOp one_half = xla::ScalarLike(input, 0.5);
+  xla::XlaOp one = xla::ScalarLike(input, 1);
+
+  xla::XlaOp pi = xla::ScalarLike(input, M_PI);
+
+  xla::XlaOp lanczos_gamma = xla::ScalarLike(input, kLanczosGamma);
+  xla::XlaOp lanczos_gamma_plus_one_half =
+      xla::ScalarLike(input, kLanczosGamma + 0.5);
+  xla::XlaOp log_lanczos_gamma_plus_one_half =
+      xla::ScalarLike(input, std::log(kLanczosGamma + 0.5));
+
+  xla::XlaOp base_lanczos_coeff = xla::ScalarLike(input, kBaseLanczosCoeff);
+
+  // If the input is less than 0.5 use Gauss's reflection formula:
+  // digamma(x) = digamma(1 - x) - pi * cot(pi * x)
+  xla::XlaOp need_to_reflect = xla::Lt(xla::Real(input), one_half);
+  xla::XlaOp z = xla::Select(need_to_reflect, -input, input - one);
+
+  xla::XlaOp num = zero;
+  xla::XlaOp denom = base_lanczos_coeff;
+  for (int i = 0; i < kLanczosCoefficients.size(); ++i) {
+    xla::XlaOp lanczos_coefficient =
+        xla::ScalarLike(input, kLanczosCoefficients[i]);
+    xla::XlaOp index = xla::ScalarLike(input, i);
+    num = num - lanczos_coefficient / ((z + index + one) * (z + index + one));
+    denom = denom + lanczos_coefficient / (z + index + one);
+  }
+
+  // To improve accuracy on platforms with less-precise log implementations,
+  // compute log(lanczos_gamma_plus_one_half) at compile time and use log1p on
+  // the device.
+  // log(t) = log(kLanczosGamma + 0.5 + z)
+  //        = log(kLanczosGamma + 0.5) + log1p(z / (kLanczosGamma + 0.5))
+  xla::XlaOp t = lanczos_gamma_plus_one_half + z;
+  xla::XlaOp log_t = log_lanczos_gamma_plus_one_half +
+                     xla::Log1p(z / lanczos_gamma_plus_one_half);
+
+  xla::XlaOp y = log_t + num / denom - lanczos_gamma / t;
+  xla::XlaOp reflection = y - pi * xla::Cos(pi * input) / xla::Sin(pi * input);
+  xla::XlaOp result = xla::Select(need_to_reflect, reflection, y);
+  return result;
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/math.h b/tensorflow/compiler/xla/client/lib/math.h
index c89c351cfc..e4c79b5f52 100644
--- a/tensorflow/compiler/xla/client/lib/math.h
+++ b/tensorflow/compiler/xla/client/lib/math.h
@@ -49,6 +49,9 @@ XlaOp ErfInv(XlaOp x);
 // Computes an approximation of the lgamma function.
 XlaOp Lgamma(XlaOp input);
 
+// Computes an approximation of the digamma function.
+XlaOp Digamma(XlaOp input);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATH_H_
diff --git a/tensorflow/compiler/xla/client/lib/math_test.cc b/tensorflow/compiler/xla/client/lib/math_test.cc
index 86e195a8c6..1df287d7db 100644
--- a/tensorflow/compiler/xla/client/lib/math_test.cc
+++ b/tensorflow/compiler/xla/client/lib/math_test.cc
@@ -105,5 +105,36 @@ XLA_TEST_F(MathTest, Lgamma) {
   error_spec_ = ErrorSpec{0.001};
   ComputeAndCompareR1(&builder, expected, {}, error_spec_);
 }
+
+XLA_TEST_F(MathTest, Digamma) {
+  XlaBuilder builder(TestName());
+  auto x = ConstantR1(&builder, {1.0, 0.5, 1 / 3.0, 0.25, 1 / 6.0, 0.125,
+                                        2.0, 3.0, 4.0, 6.0, 8.0, 9.0});
+  Digamma(x);
+
+  constexpr double euler_mascheroni =
+      0.57721566490153286060651209008240243104215933593992;
+  std::vector expected = {
+      static_cast(-euler_mascheroni),
+      static_cast(-2 * std::log(2) - euler_mascheroni),
+      static_cast(-M_PI / 2 / std::sqrt(3) - 3 * std::log(3) / 2 -
+                         euler_mascheroni),
+      static_cast(-M_PI / 2 - 3 * std::log(2) - euler_mascheroni),
+      static_cast(-M_PI * std::sqrt(3) / 2 - 2 * std::log(2) -
+                         3 * std::log(3) / 2 - euler_mascheroni),
+      static_cast(
+          -M_PI / 2 - 4 * std::log(2) -
+          (M_PI + std::log(2 + std::sqrt(2)) - std::log(2 - std::sqrt(2))) /
+              std::sqrt(2) -
+          euler_mascheroni),
+      static_cast(1 - euler_mascheroni),
+      static_cast(1.5 - euler_mascheroni),
+      static_cast(11 / 6.0 - euler_mascheroni),
+      static_cast(137 / 60.0 - euler_mascheroni),
+      static_cast(363 / 140.0 - euler_mascheroni),
+      static_cast(761 / 280.0 - euler_mascheroni)};
+  ComputeAndCompareR1(&builder, expected, {}, error_spec_);
+}
+
 }  // namespace
 }  // namespace xla
-- 
cgit v1.2.3


From ccfc6845916e24635ba1600cae222041c14c14fe Mon Sep 17 00:00:00 2001
From: Jie 
Date: Mon, 16 Jul 2018 15:23:41 -0700
Subject: [tftrt]   Added alignment in trt_allocator   Since TensorFlow
 gpu_bfc_allocator does not abide to the alignment requested.

---
 .../contrib/tensorrt/convert/convert_graph.cc      |  2 +-
 .../contrib/tensorrt/kernels/trt_engine_op.h       |  2 +-
 .../contrib/tensorrt/resources/trt_allocator.cc    | 24 +++++++++++++++++++++-
 .../contrib/tensorrt/resources/trt_allocator.h     | 15 ++++++++++++--
 .../contrib/tensorrt/resources/trt_resources.h     |  2 +-
 5 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 63d8eec7db..bcc867efea 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -829,7 +829,7 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
     // The allocator is used to build the engine. The build and the built engine
     // will be destroyed after we get the serialized engine string, so it's fine
     // to use unique_ptr here.
-    std::unique_ptr alloc;
+    std::unique_ptr alloc;
     auto device_alloc = GetDeviceAndAllocator(params, engine);
     int cuda_device_id = 0;
     if (device_alloc.first >= 0) {
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
index 6fe318be6a..9265250605 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.h
@@ -81,7 +81,7 @@ class TRTEngineOp : public AsyncOpKernel {
   std::vector output_nodes_;
 
   // keep device allocator for TRT.
-  std::unique_ptr allocator_;
+  std::unique_ptr allocator_;
 
   // serialized protobuf segment or trt engine depending on static_engine_ flag.
   string serialized_segment_;
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
index 9f115990c3..e2bc5a61d0 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
@@ -37,8 +37,22 @@ void TRTCudaAllocator::free(void* memory) { cudaFree(memory); }
 
 void* TRTDeviceAllocator::allocate(uint64_t size, uint64_t alignment,
                                    uint32_t flags) {
+  // WAR for allocator alignment requirement
+  alignment = 512;
   assert((alignment & (alignment - 1)) == 0);  // zero or a power of 2.
-  void* mem = allocator_->AllocateRaw(alignment, size);
+  void* mem = allocator_->AllocateRaw(alignment, size + alignment);
+
+  CHECK(mem_pool.count(mem) == 0);
+  mem_pool.insert(mem);
+  CHECK(mem);
+  void* alloc_mem = mem;
+  uint64_t total_size = size + alignment;
+  std::align(alignment, size, mem, total_size);
+  CHECK(mem);
+  if (mem != alloc_mem) {
+    CHECK(mem_map.count(mem) == 0);
+    mem_map[mem] = alloc_mem;
+  }
   VLOG(2) << "Allocated " << size << " bytes with alignment " << alignment
           << " @ " << mem;
   return mem;
@@ -51,6 +65,14 @@ TRTDeviceAllocator::TRTDeviceAllocator(tensorflow::Allocator* allocator)
 
 void TRTDeviceAllocator::free(void* memory) {
   VLOG(2) << "Deallocating @ " << memory;
+  // allocated memory adjusted for alignment, restore the original pointer
+  if (mem_map.count(memory) != 0) {
+    auto alloc_mem = mem_map[memory];
+    mem_map.erase(memory);
+    memory = alloc_mem;
+  }
+  CHECK(mem_pool.count(memory) != 0);
+  mem_pool.erase(memory);
   allocator_->DeallocateRaw(memory);
 }
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.h b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
index c5d2cec730..9ec0b3c4ff 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
@@ -37,7 +37,14 @@ class IGpuAllocator {
 namespace tensorflow {
 namespace tensorrt {
 
-class TRTCudaAllocator : public nvinfer1::IGpuAllocator {
+class TRTBaseAllocator : public nvinfer1::IGpuAllocator {
+  // Base allocator class so we can have a virtual destructor;
+ public:
+  // python wrapper seems to be not happy with an pure virtual destructor;
+  virtual ~TRTBaseAllocator() = default;
+};
+
+class TRTCudaAllocator : public TRTBaseAllocator {
   // Allocator implementation that is using cuda allocator instead of device
   // allocator in case we can't get device allocator from TF.
  public:
@@ -47,7 +54,7 @@ class TRTCudaAllocator : public nvinfer1::IGpuAllocator {
   void free(void* memory) override;
 };
 
-class TRTDeviceAllocator : public nvinfer1::IGpuAllocator {
+class TRTDeviceAllocator : public TRTBaseAllocator {
   // Allocator implementation wrapping TF device allocators.
  public:
   TRTDeviceAllocator(tensorflow::Allocator* allocator);
@@ -59,6 +66,10 @@ class TRTDeviceAllocator : public nvinfer1::IGpuAllocator {
 
  private:
   tensorflow::Allocator* allocator_;
+
+  // supporting alignment from allocation request requires a map to free;
+  std::unordered_map mem_map;
+  std::set mem_pool;
 };
 
 }  // namespace tensorrt
diff --git a/tensorflow/contrib/tensorrt/resources/trt_resources.h b/tensorflow/contrib/tensorrt/resources/trt_resources.h
index b7d5ffd674..d7d56cb95e 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_resources.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_resources.h
@@ -64,7 +64,7 @@ class TRTCalibrationResource : public tensorflow::ResourceBase {
   std::unique_ptr calibrator_;
   TrtUniquePtrType builder_;
   TrtUniquePtrType engine_;
-  std::unique_ptr allocator_;
+  std::unique_ptr allocator_;
   tensorflow::tensorrt::Logger logger_;
   // TODO(sami): Use threadpool threads!
   std::unique_ptr thr_;
-- 
cgit v1.2.3


From d8f3425e5b054dff01b5ece80e8c8a101c4ed816 Mon Sep 17 00:00:00 2001
From: Anna R 
Date: Mon, 16 Jul 2018 18:11:37 -0700
Subject: Handle deprecated fields in api_def.proto. Also update how canonical
 endpoint name is set in doc_generator_visitor.py.

PiperOrigin-RevId: 204841165
---
 tensorflow/python/BUILD                            |  3 ++
 tensorflow/python/framework/python_op_gen.cc       |  1 +
 .../python/framework/python_op_gen_internal.cc     | 25 +++++++++++++
 tensorflow/python/tools/api/generator/api_gen.bzl  |  1 +
 tensorflow/python/util/deprecation.py              | 34 ++++++++++++++++++
 tensorflow/python/util/deprecation_test.py         | 22 ++++++++++++
 tensorflow/python/util/tf_export.py                | 42 ++++++++++++++++++++++
 tensorflow/tools/docs/doc_generator_visitor.py     | 15 +++++---
 8 files changed, 138 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 2fba3c2acb..51e6d5aabf 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3337,6 +3337,9 @@ py_library(
         ],
     ),
     srcs_version = "PY2AND3",
+    visibility = visibility + [
+        "//tensorflow:__pkg__",
+    ],
     deps = [
         "//third_party/py/numpy",
         "@org_python_pypi_backports_weakref",
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index ec3748b40e..76d4c2017c 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -943,6 +943,7 @@ from tensorflow.python.framework import common_shapes as _common_shapes
 from tensorflow.python.framework import op_def_registry as _op_def_registry
 from tensorflow.python.framework import ops as _ops
 from tensorflow.python.framework import op_def_library as _op_def_library
+from tensorflow.python.util.deprecation import deprecated_endpoints
 from tensorflow.python.util.tf_export import tf_export
 
 )");
diff --git a/tensorflow/python/framework/python_op_gen_internal.cc b/tensorflow/python/framework/python_op_gen_internal.cc
index 940bffb906..031b4a384e 100644
--- a/tensorflow/python/framework/python_op_gen_internal.cc
+++ b/tensorflow/python/framework/python_op_gen_internal.cc
@@ -588,10 +588,12 @@ void GenPythonOp::AddExport() {
     return;
   }
 
+  // Add @tf_export decorator.
   strings::StrAppend(&result_, "@tf_export(");
 
   // Add all endpoint names to tf_export.
   bool first_endpoint = true;
+  std::vector deprecated_endpoints;
   for (const auto& endpoint : api_def_.endpoint()) {
     if (!first_endpoint) {
       strings::StrAppend(&result_, ", ");
@@ -601,9 +603,32 @@ void GenPythonOp::AddExport() {
     string endpoint_name;
     python_op_gen_internal::GenerateLowerCaseOpName(endpoint.name(),
                                                     &endpoint_name);
+    if (endpoint.deprecated()) {
+      deprecated_endpoints.push_back(endpoint_name);
+    }
     strings::StrAppend(&result_, "'", endpoint_name, "'");
   }
   strings::StrAppend(&result_, ")\n");
+
+  // If all endpoints are deprecated, add @deprecated decorator.
+  if (!api_def_.deprecation_message().empty()) {
+    const string instructions = api_def_.deprecation_message();
+    strings::StrAppend(&result_, "@deprecated(None, '", instructions, "')\n");
+  }
+  // Add @deprecated_endpoints decorator.
+  if (!deprecated_endpoints.empty()) {
+    strings::StrAppend(&result_, "@deprecated_endpoints(");
+    bool first_endpoint = true;
+    for (auto& endpoint_name : deprecated_endpoints) {
+      if (first_endpoint) {
+        first_endpoint = false;
+      } else {
+        strings::StrAppend(&result_, ", ");
+      }
+      strings::StrAppend(&result_, "'", endpoint_name, "'");
+    }
+    strings::StrAppend(&result_, ")\n");
+  }
 }
 
 void GenPythonOp::AddDefLine(const string& function_name,
diff --git a/tensorflow/python/tools/api/generator/api_gen.bzl b/tensorflow/python/tools/api/generator/api_gen.bzl
index 2a32e8a893..00e1c4e199 100644
--- a/tensorflow/python/tools/api/generator/api_gen.bzl
+++ b/tensorflow/python/tools/api/generator/api_gen.bzl
@@ -150,6 +150,7 @@ def gen_api_init_files(
         visibility = ["//visibility:public"],
         deps = [
             package_dep,
+            "//tensorflow/python:util",
             "//tensorflow/python/tools/api/generator:doc_srcs",
         ],
     )
diff --git a/tensorflow/python/util/deprecation.py b/tensorflow/python/util/deprecation.py
index c8ed2b715d..9e2202eaf8 100644
--- a/tensorflow/python/util/deprecation.py
+++ b/tensorflow/python/util/deprecation.py
@@ -37,6 +37,11 @@ _PRINT_DEPRECATION_WARNINGS = True
 _PRINTED_WARNING = {}
 
 
+class DeprecatedNamesAlreadySet(Exception):
+  """Raised when setting deprecated names multiple times for the same symbol."""
+  pass
+
+
 def _add_deprecated_function_notice_to_docstring(doc, date, instructions):
   """Adds a deprecation notice to a docstring for deprecated functions."""
   main_text = ['THIS FUNCTION IS DEPRECATED. It will be removed %s.' %
@@ -219,6 +224,35 @@ def deprecated_alias(deprecated_name, name, func_or_class, warn_once=True):
             func_or_class.__doc__, None, 'Please use %s instead.' % name))
 
 
+def deprecated_endpoints(*args):
+  """Decorator for marking endpoints deprecated.
+
+  This decorator does not print deprecation messages.
+  TODO(annarev): eventually start printing deprecation warnings when
+  @deprecation_endpoints decorator is added.
+
+  Args:
+    *args: Deprecated endpoint names.
+
+  Returns:
+    A function that takes symbol as an argument and adds
+    _tf_deprecated_api_names to that symbol.
+    _tf_deprecated_api_names would be set to a list of deprecated
+    endpoint names for the symbol.
+  """
+  def deprecated_wrapper(func):
+    # pylint: disable=protected-access
+    if '_tf_deprecated_api_names' in func.__dict__:
+      raise DeprecatedNamesAlreadySet(
+          'Cannot set deprecated names for %s to %s. '
+          'Deprecated names are already set to %s.' % (
+              func.__name__, str(args), str(func._tf_deprecated_api_names)))
+    func._tf_deprecated_api_names = args
+    # pylint: disable=protected-access
+    return func
+  return deprecated_wrapper
+
+
 def deprecated(date, instructions, warn_once=True):
   """Decorator for marking functions or methods deprecated.
 
diff --git a/tensorflow/python/util/deprecation_test.py b/tensorflow/python/util/deprecation_test.py
index 1ea695e4d6..90c73a0a58 100644
--- a/tensorflow/python/util/deprecation_test.py
+++ b/tensorflow/python/util/deprecation_test.py
@@ -935,5 +935,27 @@ class DeprecationArgumentsTest(test.TestCase):
     self.assertEqual(new_docs, new_docs_ref)
 
 
+class DeprecatedEndpointsTest(test.TestCase):
+
+  def testSingleDeprecatedEndpoint(self):
+    @deprecation.deprecated_endpoints("foo1")
+    def foo():
+      pass
+    self.assertEqual(("foo1",), foo._tf_deprecated_api_names)
+
+  def testMultipleDeprecatedEndpoint(self):
+    @deprecation.deprecated_endpoints("foo1", "foo2")
+    def foo():
+      pass
+    self.assertEqual(("foo1", "foo2"), foo._tf_deprecated_api_names)
+
+  def testCannotSetDeprecatedEndpointsTwice(self):
+    with self.assertRaises(deprecation.DeprecatedNamesAlreadySet):
+      @deprecation.deprecated_endpoints("foo1")
+      @deprecation.deprecated_endpoints("foo2")
+      def foo():  # pylint: disable=unused-variable
+        pass
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/util/tf_export.py b/tensorflow/python/util/tf_export.py
index c362d588ab..274f32c21f 100644
--- a/tensorflow/python/util/tf_export.py
+++ b/tensorflow/python/util/tf_export.py
@@ -78,6 +78,48 @@ class SymbolAlreadyExposedError(Exception):
   pass
 
 
+def get_canonical_name_for_symbol(symbol, api_name=TENSORFLOW_API_NAME):
+  """Get canonical name for the API symbol.
+
+  Canonical name is the first non-deprecated endpoint name.
+
+  Args:
+    symbol: API function or class.
+    api_name: API name (tensorflow or estimator).
+
+  Returns:
+    Canonical name for the API symbol (for e.g. initializers.zeros) if
+    canonical name could be determined. Otherwise, returns None.
+  """
+  if not hasattr(symbol, '__dict__'):
+    return None
+  api_names_attr = API_ATTRS[api_name].names
+  _, undecorated_symbol = tf_decorator.unwrap(symbol)
+  if api_names_attr not in undecorated_symbol.__dict__:
+    return None
+  api_names = getattr(undecorated_symbol, api_names_attr)
+  # TODO(annarev): may be add a separate deprecated attribute
+  # for estimator names.
+  deprecated_api_names = undecorated_symbol.__dict__.get(
+      '_tf_deprecated_api_names', [])
+  return get_canonical_name(api_names, deprecated_api_names)
+
+
+def get_canonical_name(api_names, deprecated_api_names):
+  """Get first non-deprecated endpoint name.
+
+  Args:
+    api_names: API names iterable.
+    deprecated_api_names: Deprecated API names iterable.
+  Returns:
+    Canonical name if there is at least one non-deprecated endpoint.
+    Otherwise returns None.
+  """
+  return next(
+      (name for name in api_names if name not in deprecated_api_names),
+      None)
+
+
 class api_export(object):  # pylint: disable=invalid-name
   """Provides ways to export symbols to the TensorFlow API."""
 
diff --git a/tensorflow/tools/docs/doc_generator_visitor.py b/tensorflow/tools/docs/doc_generator_visitor.py
index 259a4694fd..c090dbd8da 100644
--- a/tensorflow/tools/docs/doc_generator_visitor.py
+++ b/tensorflow/tools/docs/doc_generator_visitor.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import six
 
+from tensorflow.python.util import tf_export
 from tensorflow.python.util import tf_inspect
 
 
@@ -201,7 +202,6 @@ class DocGeneratorVisitor(object):
             raw_duplicates[master_name] = [master_name, full_name]
         else:
           reverse_index[object_id] = full_name
-
     # Decide on master names, rewire duplicates and make a duplicate_of map
     # mapping all non-master duplicates to the master name. The master symbol
     # does not have an entry in this map.
@@ -211,10 +211,15 @@ class DocGeneratorVisitor(object):
     duplicates = {}
     for names in raw_duplicates.values():
       names = sorted(names)
-
-      # Choose the lexicographically first name with the minimum number of
-      # submodules. This will prefer highest level namespace for any symbol.
-      master_name = min(names, key=lambda name: name.count('.'))
+      master_name = (
+          tf_export.get_canonical_name_for_symbol(self._index[names[0]])
+          if names else None)
+      if master_name:
+        master_name = 'tf.%s' % master_name
+      else:
+        # Choose the lexicographically first name with the minimum number of
+        # submodules. This will prefer highest level namespace for any symbol.
+        master_name = min(names, key=lambda name: name.count('.'))
 
       duplicates[master_name] = names
       for name in names:
-- 
cgit v1.2.3


From 1a2af489b1087eb22ec76863867e4e397e453e34 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 16 Jul 2018 19:09:52 -0700
Subject: Support reduce_max and reduce_prod

PiperOrigin-RevId: 204846139
---
 tensorflow/contrib/lite/build_def.bzl              |   2 +
 tensorflow/contrib/lite/builtin_ops.h              |   2 +
 .../kernels/internal/reference/reference_ops.h     |  99 +++++-
 tensorflow/contrib/lite/kernels/reduce.cc          | 111 +++++++
 tensorflow/contrib/lite/kernels/reduce_test.cc     | 353 ++++++++++++++++++---
 tensorflow/contrib/lite/kernels/register.cc        |   4 +
 tensorflow/contrib/lite/model.cc                   |   2 +
 tensorflow/contrib/lite/nnapi_delegate.cc          |   2 +
 tensorflow/contrib/lite/schema/schema.fbs          |   4 +-
 tensorflow/contrib/lite/schema/schema_generated.h  |  12 +-
 .../contrib/lite/testing/generate_examples.py      |  12 +-
 tensorflow/contrib/lite/toco/BUILD                 |   2 +-
 tensorflow/contrib/lite/toco/export_tensorflow.cc  |  23 +-
 .../graph_transformations/graph_transformations.h  |   2 +-
 .../graph_transformations/propagate_fixed_sizes.cc |   7 +-
 .../resolve_constant_unary.cc                      |   4 +-
 .../resolve_mean_attributes.cc                     |  45 ---
 .../resolve_reduce_attributes.cc                   |  58 ++++
 tensorflow/contrib/lite/toco/import_tensorflow.cc  |  45 +--
 tensorflow/contrib/lite/toco/model.h               |  31 +-
 tensorflow/contrib/lite/toco/tflite/operator.cc    |  42 +++
 tensorflow/contrib/lite/toco/toco_tooling.cc       |   2 +-
 tensorflow/contrib/lite/toco/tooling_util.cc       |   3 +-
 23 files changed, 694 insertions(+), 173 deletions(-)
 delete mode 100644 tensorflow/contrib/lite/toco/graph_transformations/resolve_mean_attributes.cc
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/resolve_reduce_attributes.cc

diff --git a/tensorflow/contrib/lite/build_def.bzl b/tensorflow/contrib/lite/build_def.bzl
index b735d08b4b..bed862454e 100644
--- a/tensorflow/contrib/lite/build_def.bzl
+++ b/tensorflow/contrib/lite/build_def.bzl
@@ -234,6 +234,8 @@ def generated_test_models():
         "padv2",
         "prelu",
         "pow",
+        "reduce_max",
+        "reduce_prod",
         "relu",
         "relu1",
         "relu6",
diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h
index 6bde5d2e6d..4c7b27c4e0 100644
--- a/tensorflow/contrib/lite/builtin_ops.h
+++ b/tensorflow/contrib/lite/builtin_ops.h
@@ -106,6 +106,8 @@ typedef enum {
   kTfLiteBuiltinPow = 78,
   kTfLiteBuiltinArgMin = 79,
   kTfLiteBuiltinFakeQuant = 80,
+  kTfLiteBuiltinReduceProd = 81,
+  kTfLiteBuiltinReduceMax = 82,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index 080b4e2d03..6fabb9c268 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -3468,7 +3468,8 @@ inline bool Reduce(const In* input_data, const int* input_dims,
                    const int* output_dims, const int input_num_dims,
                    const int output_num_dims, const int* axis,
                    const int num_axis, int* input_iter,
-                   Out reducer(Out current, const In in), Out* output_data) {
+                   Out reducer(const Out current, const In in),
+                   Out* output_data) {
   // Reset input iterator.
   TFLITE_DCHECK(input_num_dims > 0);
   for (int idx = 0; idx < input_num_dims; ++idx) {
@@ -3486,11 +3487,12 @@ inline bool Reduce(const In* input_data, const int* input_dims,
   return true;
 }
 
-inline bool ResolveAxis(const int num_dims, const int* axis, const int num_axis,
-                        int* out_axis, int* out_num_axis) {
+inline bool ResolveAxis(const int num_dims, const int* axis,
+                        const int64_t num_axis, int* out_axis,
+                        int* out_num_axis) {
   *out_num_axis = 0;  // Just in case.
   // o(n^2) is fine since out_num_axis should be really small, mostly <= 4
-  for (int idx = 0; idx < num_axis; ++idx) {
+  for (int64_t idx = 0; idx < num_axis; ++idx) {
     // Handle negative index.
     int current = axis[idx] < 0 ? (axis[idx] + num_dims) : axis[idx];
     TFLITE_DCHECK(current >= 0 && current < num_dims);
@@ -3516,7 +3518,7 @@ inline bool ReduceSumImpl(const In* input_data, const int* input_dims,
                           const int output_num_dims, const int* axis,
                           const int num_axis, int* input_iter,
                           Out* output_data) {
-  auto reducer = [](Out current, const In in) -> Out {
+  auto reducer = [](const Out current, const In in) -> Out {
     const Out actual_in = static_cast(in);
     return current + actual_in;
   };
@@ -3525,6 +3527,24 @@ inline bool ReduceSumImpl(const In* input_data, const int* input_dims,
                          output_data);
 }
 
+template 
+inline bool InitTensorDataForReduce(const int* dims, const int num_dims,
+                                    const T init_value, T* data) {
+  size_t num_elements = 1;
+  for (int idx = 0; idx < num_dims; ++idx) {
+    size_t current = static_cast(dims[idx]);
+    // Overflow prevention.
+    if (num_elements > std::numeric_limits::max() / current) {
+      return false;
+    }
+    num_elements *= current;
+  }
+  for (size_t idx = 0; idx < num_elements; ++idx) {
+    data[idx] = init_value;
+  }
+  return true;
+}
+
 // Computes the sum of elements across dimensions given in axis.
 template 
 inline bool Sum(const T* input_data, const int* input_dims,
@@ -3533,17 +3553,9 @@ inline bool Sum(const T* input_data, const int* input_dims,
                 const int* axis, const int num_axis_dimensions, bool keep_dims,
                 int* temp_index, int* resolved_axis) {
   // Reset output data.
-  size_t num_outputs = 1;
-  for (int idx = 0; idx < output_num_dims; ++idx) {
-    size_t current = static_cast(output_dims[idx]);
-    // Overflow prevention.
-    if (num_outputs > std::numeric_limits::max() / current) {
-      return false;
-    }
-    num_outputs *= current;
-  }
-  for (size_t idx = 0; idx < num_outputs; ++idx) {
-    output_data[idx] = T();
+  if (!InitTensorDataForReduce(output_dims, output_num_dims, static_cast(0),
+                               output_data)) {
+    return false;
   }
 
   // Resolve axis.
@@ -3558,6 +3570,61 @@ inline bool Sum(const T* input_data, const int* input_dims,
                              num_resolved_axis, temp_index, output_data);
 }
 
+// Computes the max of elements across dimensions given in axis.
+template 
+inline bool ReduceMax(const T* input_data, const int* input_dims,
+                      const int input_num_dims, T* output_data,
+                      const int* output_dims, const int output_num_dims,
+                      const int* axis, const int64_t num_axis_dimensions,
+                      bool keep_dims, int* temp_index, int* resolved_axis) {
+  T init_value = std::numeric_limits::lowest();
+  // Reset output data.
+  if (!InitTensorDataForReduce(output_dims, output_num_dims, init_value,
+                               output_data)) {
+    return false;
+  }
+
+  // Resolve axis.
+  int num_resolved_axis = 0;
+  if (!ResolveAxis(input_num_dims, axis, num_axis_dimensions, resolved_axis,
+                   &num_resolved_axis)) {
+    return false;
+  }
+
+  auto reducer = [](const T current, const T in) -> T {
+    return (in > current) ? in : current;
+  };
+  return Reduce(input_data, input_dims, output_dims, input_num_dims,
+                      output_num_dims, resolved_axis, num_resolved_axis,
+                      temp_index, reducer, output_data);
+}
+
+// Computes the prod of elements across dimensions given in axis.
+template 
+inline bool ReduceProd(const T* input_data, const int* input_dims,
+                       const int input_num_dims, T* output_data,
+                       const int* output_dims, const int output_num_dims,
+                       const int* axis, const int64_t num_axis_dimensions,
+                       bool keep_dims, int* temp_index, int* resolved_axis) {
+  // Reset output data.
+  if (!InitTensorDataForReduce(output_dims, output_num_dims, static_cast(1),
+                               output_data)) {
+    return false;
+  }
+
+  // Resolve axis.
+  int num_resolved_axis = 0;
+  if (!ResolveAxis(input_num_dims, axis, num_axis_dimensions, resolved_axis,
+                   &num_resolved_axis)) {
+    return false;
+  }
+
+  auto reducer = [](const T current, const T in) -> T { return in * current; };
+  return Reduce(input_data, input_dims, output_dims, input_num_dims,
+                      output_num_dims, resolved_axis, num_resolved_axis,
+                      temp_index, reducer, output_data);
+}
+
 // Computes the mean of elements across dimensions given in axis.
 // It does so in two stages, first calculates the sum of elements along the axis
 // then divides it by the number of element in axis.
diff --git a/tensorflow/contrib/lite/kernels/reduce.cc b/tensorflow/contrib/lite/kernels/reduce.cc
index 31c331a8c6..52e4084ff8 100644
--- a/tensorflow/contrib/lite/kernels/reduce.cc
+++ b/tensorflow/contrib/lite/kernels/reduce.cc
@@ -315,6 +315,99 @@ TfLiteStatus EvalSum(TfLiteContext* context, TfLiteNode* node) {
   return kTfLiteOk;
 }
 
+template 
+TfLiteStatus EvalProd(TfLiteContext* context, TfLiteNode* node) {
+  OpContext op_context(context, node);
+  int64_t num_axis = NumElements(op_context.axis);
+  TfLiteTensor* temp_index = GetTemporary(context, node, /*index=*/0);
+  TfLiteTensor* resolved_axis = GetTemporary(context, node, /*index=*/1);
+  // Resize the output tensor if the output tensor is dynamic.
+  if (IsDynamicTensor(op_context.output)) {
+    TF_LITE_ENSURE_OK(context,
+                      ResizeTempAxis(context, &op_context, resolved_axis));
+    TF_LITE_ENSURE_OK(context, ResizeOutputTensor(context, &op_context));
+  }
+
+#define TF_LITE_PROD(kernel_type, data_type)                        \
+  kernel_type::ReduceProd<>(                                        \
+      GetTensorData(op_context.input),                   \
+      op_context.input->dims->data, op_context.input->dims->size,   \
+      GetTensorData(op_context.output),                  \
+      op_context.output->dims->data, op_context.output->dims->size, \
+      GetTensorData(op_context.axis), num_axis,                \
+      op_context.params->keep_dims, GetTensorData(temp_index), \
+      GetTensorData(resolved_axis))
+
+  if (kernel_type == kReference) {
+    switch (op_context.input->type) {
+      case kTfLiteFloat32:
+        TF_LITE_ENSURE(context, TF_LITE_PROD(reference_ops, float));
+        break;
+      case kTfLiteInt32:
+        TF_LITE_ENSURE(context, TF_LITE_PROD(reference_ops, int));
+        break;
+      case kTfLiteInt64:
+        TF_LITE_ENSURE(context, TF_LITE_PROD(reference_ops, int64_t));
+        break;
+      case kTfLiteUInt8:
+        // TODO(wangtz): uint8 reduce_prod is not yet supported.
+      default:
+        return kTfLiteError;
+    }
+  }
+#undef TF_LITE_PROD
+  return kTfLiteOk;
+}
+
+template 
+TfLiteStatus EvalMax(TfLiteContext* context, TfLiteNode* node) {
+  OpContext op_context(context, node);
+  int64_t num_axis = NumElements(op_context.axis);
+  TfLiteTensor* temp_index = GetTemporary(context, node, /*index=*/0);
+  TfLiteTensor* resolved_axis = GetTemporary(context, node, /*index=*/1);
+  // Resize the output tensor if the output tensor is dynamic.
+  if (IsDynamicTensor(op_context.output)) {
+    TF_LITE_ENSURE_OK(context,
+                      ResizeTempAxis(context, &op_context, resolved_axis));
+    TF_LITE_ENSURE_OK(context, ResizeOutputTensor(context, &op_context));
+  }
+
+#define TF_LITE_MAX(kernel_type, data_type)                         \
+  kernel_type::ReduceMax<>(                                         \
+      GetTensorData(op_context.input),                   \
+      op_context.input->dims->data, op_context.input->dims->size,   \
+      GetTensorData(op_context.output),                  \
+      op_context.output->dims->data, op_context.output->dims->size, \
+      GetTensorData(op_context.axis), num_axis,                \
+      op_context.params->keep_dims, GetTensorData(temp_index), \
+      GetTensorData(resolved_axis))
+
+  if (kernel_type == kReference) {
+    switch (op_context.input->type) {
+      case kTfLiteFloat32:
+        TF_LITE_ENSURE(context, TF_LITE_MAX(reference_ops, float));
+        break;
+      case kTfLiteInt32:
+        TF_LITE_ENSURE(context, TF_LITE_MAX(reference_ops, int));
+        break;
+      case kTfLiteInt64:
+        TF_LITE_ENSURE(context, TF_LITE_MAX(reference_ops, int64_t));
+        break;
+      case kTfLiteUInt8:
+        TF_LITE_ENSURE_EQ(context, op_context.input->params.scale,
+                          op_context.output->params.scale);
+        TF_LITE_ENSURE_EQ(context, op_context.input->params.zero_point,
+                          op_context.output->params.zero_point);
+        TF_LITE_ENSURE(context, TF_LITE_MAX(reference_ops, uint8_t));
+        break;
+      default:
+        return kTfLiteError;
+    }
+  }
+#undef TF_LITE_MAX
+  return kTfLiteOk;
+}
+
 }  // namespace reduce
 
 TfLiteRegistration* Register_MEAN_REF() {
@@ -331,9 +424,27 @@ TfLiteRegistration* Register_SUM_REF() {
   return &r;
 }
 
+TfLiteRegistration* Register_REDUCE_PROD_REF() {
+  static TfLiteRegistration r = {reduce::Init, reduce::Free,
+                                 reduce::PrepareSimple,
+                                 reduce::EvalProd};
+  return &r;
+}
+
+TfLiteRegistration* Register_REDUCE_MAX_REF() {
+  static TfLiteRegistration r = {reduce::Init, reduce::Free,
+                                 reduce::PrepareSimple,
+                                 reduce::EvalMax};
+  return &r;
+}
+
 // TODO(kanlig): add optimized implementation of Mean.
 TfLiteRegistration* Register_MEAN() { return Register_MEAN_REF(); }
 TfLiteRegistration* Register_SUM() { return Register_SUM_REF(); }
+TfLiteRegistration* Register_REDUCE_PROD() {
+  return Register_REDUCE_PROD_REF();
+}
+TfLiteRegistration* Register_REDUCE_MAX() { return Register_REDUCE_MAX_REF(); }
 
 }  // namespace builtin
 }  // namespace ops
diff --git a/tensorflow/contrib/lite/kernels/reduce_test.cc b/tensorflow/contrib/lite/kernels/reduce_test.cc
index 9e946822c6..7d28931ecd 100644
--- a/tensorflow/contrib/lite/kernels/reduce_test.cc
+++ b/tensorflow/contrib/lite/kernels/reduce_test.cc
@@ -25,10 +25,10 @@ using ::testing::ElementsAreArray;
 
 class BaseOpModel : public SingleOpModel {
  public:
-  void SetAxis(std::initializer_list data) { PopulateTensor(axis_, data); }
+  void SetAxis(const std::vector& data) { PopulateTensor(axis_, data); }
 
   template 
-  void SetInput(std::initializer_list data) {
+  void SetInput(std::vector data) {
     PopulateTensor(input_, data);
   }
 
@@ -110,14 +110,72 @@ class SumOpDynamicModel : public BaseOpModel {
   }
 };
 
+// Model for the tests case where axis is a const tensor.
+class ProdOpConstModel : public BaseOpModel {
+ public:
+  ProdOpConstModel(const TensorData& input, const TensorData& output,
+                   std::initializer_list axis_shape,
+                   std::initializer_list axis, bool keep_dims) {
+    input_ = AddInput(input);
+    axis_ = AddConstInput(TensorType_INT32, axis, axis_shape);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_REDUCE_PROD, BuiltinOptions_ReducerOptions,
+                 CreateReducerOptions(builder_, keep_dims).Union());
+    BuildInterpreter({GetShape(input_)});
+  }
+};
+
+// Model for the tests case where axis is a dynamic tensor.
+class ProdOpDynamicModel : public BaseOpModel {
+ public:
+  ProdOpDynamicModel(const TensorData& input, const TensorData& output,
+                     const TensorData& axis, bool keep_dims) {
+    input_ = AddInput(input);
+    axis_ = AddInput(axis);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_REDUCE_PROD, BuiltinOptions_ReducerOptions,
+                 CreateReducerOptions(builder_, keep_dims).Union());
+    BuildInterpreter({GetShape(input_)});
+  }
+};
+
+// Model for the tests case where axis is a const tensor.
+class MaxOpConstModel : public BaseOpModel {
+ public:
+  MaxOpConstModel(const TensorData& input, const TensorData& output,
+                  std::initializer_list axis_shape,
+                  std::initializer_list axis, bool keep_dims) {
+    input_ = AddInput(input);
+    axis_ = AddConstInput(TensorType_INT32, axis, axis_shape);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_REDUCE_MAX, BuiltinOptions_ReducerOptions,
+                 CreateReducerOptions(builder_, keep_dims).Union());
+    BuildInterpreter({GetShape(input_)});
+  }
+};
+
+// Model for the tests case where axis is a dynamic tensor.
+class MaxOpDynamicModel : public BaseOpModel {
+ public:
+  MaxOpDynamicModel(const TensorData& input, const TensorData& output,
+                    const TensorData& axis, bool keep_dims) {
+    input_ = AddInput(input);
+    axis_ = AddInput(axis);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_REDUCE_MAX, BuiltinOptions_ReducerOptions,
+                 CreateReducerOptions(builder_, keep_dims).Union());
+    BuildInterpreter({GetShape(input_)});
+  }
+};
+
 // for quantized Add, the error shouldn't exceed step
 float GetTolerance(int min, int max) { return (max - min) / 255.0; }
 
 // Tests for reduce_mean
 TEST(ConstFloatMeanOpTest, NotKeepDims) {
-  std::initializer_list data = {
-      1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,  9.0,  10.0, 11.0, 12.0,
-      13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
   MeanOpConstModel m({TensorType_FLOAT32, {4, 3, 2}}, {TensorType_FLOAT32, {2}},
                      {4}, {1, 0, -3, -3}, false);
   m.SetInput(data);
@@ -127,9 +185,9 @@ TEST(ConstFloatMeanOpTest, NotKeepDims) {
 }
 
 TEST(ConstFloatMeanOpTest, KeepDims) {
-  std::initializer_list data = {
-      1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,  9.0,  10.0, 11.0, 12.0,
-      13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
   MeanOpConstModel m({TensorType_FLOAT32, {4, 3, 2}}, {TensorType_FLOAT32, {3}},
                      {2}, {0, 2}, true);
   m.SetInput(data);
@@ -140,13 +198,13 @@ TEST(ConstFloatMeanOpTest, KeepDims) {
 }
 
 TEST(DynamicFloatMeanOpTest, NotKeepDims) {
-  std::initializer_list data = {
-      1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,  9.0,  10.0, 11.0, 12.0,
-      13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
   MeanOpDynamicModel m({TensorType_FLOAT32, {4, 3, 2}},
                        {TensorType_FLOAT32, {2}}, {TensorType_INT32, {4}},
                        false);
-  std::initializer_list axis = {1, 0, -3, -3};
+  std::vector axis = {1, 0, -3, -3};
   m.SetAxis(axis);
   m.SetInput(data);
   m.Invoke();
@@ -155,13 +213,13 @@ TEST(DynamicFloatMeanOpTest, NotKeepDims) {
 }
 
 TEST(DynamicFloatMeanOpTest, KeepDims) {
-  std::initializer_list data = {
-      1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,  9.0,  10.0, 11.0, 12.0,
-      13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
   MeanOpDynamicModel m({TensorType_FLOAT32, {4, 3, 2}},
                        {TensorType_FLOAT32, {3}}, {TensorType_INT32, {2}},
                        true);
-  std::initializer_list axis = {0, 2};
+  std::vector axis = {0, 2};
   m.SetAxis(axis);
   m.SetInput(data);
   m.Invoke();
@@ -171,10 +229,10 @@ TEST(DynamicFloatMeanOpTest, KeepDims) {
 }
 
 TEST(DynamicFloatMeanOpTest, Scale) {
-  std::initializer_list data = {9.527};
+  std::vector data = {9.527};
   MeanOpDynamicModel m({TensorType_FLOAT32, {1}}, {TensorType_FLOAT32, {1}},
                        {TensorType_INT32, {1}}, true);
-  std::initializer_list axis = {0};
+  std::vector axis = {0};
   m.SetAxis(axis);
   m.SetInput(data);
   m.Invoke();
@@ -185,7 +243,7 @@ TEST(DynamicFloatMeanOpTest, Scale) {
 
 TEST(ConstUint8MeanOpTest, NotKeepDims) {
   float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
-  std::initializer_list data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  std::vector data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
   MeanOpConstModel m({TensorType_UINT8, {1, 3, 2}, -1.0, 1.0},
                      {TensorType_UINT8, {2}, -1.0, 1.0}, {1}, {1}, false);
   m.QuantizeAndPopulate(m.Input(), data);
@@ -197,7 +255,7 @@ TEST(ConstUint8MeanOpTest, NotKeepDims) {
 
 TEST(ConstUint8MeanOpTest, KeepDims) {
   float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
-  std::initializer_list data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  std::vector data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
   MeanOpConstModel m({TensorType_UINT8, {3, 2}, -1.0, 1.0},
                      {TensorType_UINT8, {3}, -1.0, 1.0}, {1}, {1}, true);
   m.QuantizeAndPopulate(m.Input(), data);
@@ -210,11 +268,11 @@ TEST(ConstUint8MeanOpTest, KeepDims) {
 
 TEST(DynamicUint8MeanOpTest, NotKeepDims) {
   float kQuantizedTolerance = GetTolerance(-5.0, 2.0);
-  std::initializer_list data = {1.3, -4.8, -3.6, 0.24};
+  std::vector data = {1.3, -4.8, -3.6, 0.24};
   MeanOpDynamicModel m({TensorType_UINT8, {2, 2}, -5.0, 2.0},
                        {TensorType_UINT8, {2}, -5.0, 2.0},
                        {TensorType_INT32, {1}}, false);
-  std::initializer_list axis = {1};
+  std::vector axis = {1};
   m.SetAxis(axis);
   m.QuantizeAndPopulate(m.Input(), data);
   m.Invoke();
@@ -226,11 +284,11 @@ TEST(DynamicUint8MeanOpTest, NotKeepDims) {
 
 TEST(DynamicUint8MeanOpTest, KeepDims) {
   float kQuantizedTolerance = GetTolerance(-10.0, 12.0);
-  std::initializer_list data = {11.14, -0.14, 7.423, 0.879};
+  std::vector data = {11.14, -0.14, 7.423, 0.879};
   MeanOpDynamicModel m({TensorType_UINT8, {2, 2}, -10.0, 12.0},
                        {TensorType_UINT8, {2}, -10.0, 12.0},
                        {TensorType_INT32, {1}}, true);
-  std::initializer_list axis = {0};
+  std::vector axis = {0};
   m.SetAxis(axis);
   m.QuantizeAndPopulate(m.Input(), data);
   m.Invoke();
@@ -243,9 +301,9 @@ TEST(DynamicUint8MeanOpTest, KeepDims) {
 // Tests for reduce_sum
 
 TEST(ConstFloatSumOpTest, NotKeepDims) {
-  std::initializer_list data = {
-      1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,  9.0,  10.0, 11.0, 12.0,
-      13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
   SumOpConstModel m({TensorType_FLOAT32, {4, 3, 2}}, {TensorType_FLOAT32, {2}},
                     {4}, {1, 0, -3, -3}, false);
   m.SetInput(data);
@@ -256,9 +314,9 @@ TEST(ConstFloatSumOpTest, NotKeepDims) {
 }
 
 TEST(ConstFloatSumOpTest, KeepDims) {
-  std::initializer_list data = {
-      1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,  9.0,  10.0, 11.0, 12.0,
-      13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
   SumOpConstModel m({TensorType_FLOAT32, {4, 3, 2}}, {TensorType_FLOAT32, {3}},
                     {2}, {0, 2}, true);
   m.SetInput(data);
@@ -269,13 +327,13 @@ TEST(ConstFloatSumOpTest, KeepDims) {
 }
 
 TEST(DynamicFloatSumOpTest, NotKeepDims) {
-  std::initializer_list data = {
-      1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,  9.0,  10.0, 11.0, 12.0,
-      13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
   SumOpDynamicModel m({TensorType_FLOAT32, {4, 3, 2}},
                       {TensorType_FLOAT32, {2}}, {TensorType_INT32, {4}},
                       false);
-  std::initializer_list axis = {1, 0, -3, -3};
+  std::vector axis = {1, 0, -3, -3};
   m.SetAxis(axis);
   m.SetInput(data);
   m.Invoke();
@@ -285,12 +343,12 @@ TEST(DynamicFloatSumOpTest, NotKeepDims) {
 }
 
 TEST(DynamicFloatSumOpTest, KeepDims) {
-  std::initializer_list data = {
-      1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,  9.0,  10.0, 11.0, 12.0,
-      13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
   SumOpDynamicModel m({TensorType_FLOAT32, {4, 3, 2}},
                       {TensorType_FLOAT32, {3}}, {TensorType_INT32, {2}}, true);
-  std::initializer_list axis = {0, 2};
+  std::vector axis = {0, 2};
   m.SetAxis(axis);
   m.SetInput(data);
   m.Invoke();
@@ -300,10 +358,10 @@ TEST(DynamicFloatSumOpTest, KeepDims) {
 }
 
 TEST(DynamicFloatSumOpTest, Scale) {
-  std::initializer_list data = {9.527};
+  std::vector data = {9.527};
   SumOpDynamicModel m({TensorType_FLOAT32, {1}}, {TensorType_FLOAT32, {1}},
                       {TensorType_INT32, {1}}, true);
-  std::initializer_list axis = {0};
+  std::vector axis = {0};
   m.SetAxis(axis);
   m.SetInput(data);
   m.Invoke();
@@ -313,7 +371,7 @@ TEST(DynamicFloatSumOpTest, Scale) {
 
 TEST(ConstUint8SumOpTest, NotKeepDims) {
   float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
-  std::initializer_list data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  std::vector data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
   SumOpConstModel m({TensorType_UINT8, {1, 3, 2}, -1.0, 1.0},
                     {TensorType_UINT8, {2}, -1.0, 1.0}, {1}, {1}, false);
   m.QuantizeAndPopulate(m.Input(), data);
@@ -326,7 +384,7 @@ TEST(ConstUint8SumOpTest, NotKeepDims) {
 
 TEST(ConstUint8SumOpTest, KeepDims) {
   float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
-  std::initializer_list data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  std::vector data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
   SumOpConstModel m({TensorType_UINT8, {3, 2}, -1.0, 1.0},
                     {TensorType_UINT8, {3}, -1.0, 1.0}, {1}, {1}, true);
   m.QuantizeAndPopulate(m.Input(), data);
@@ -339,11 +397,11 @@ TEST(ConstUint8SumOpTest, KeepDims) {
 
 TEST(DynamicUint8SumOpTest, NotKeepDims) {
   float kQuantizedTolerance = GetTolerance(-5.0, 2.0);
-  std::initializer_list data = {1.3, -4.8, -3.6, 0.24};
+  std::vector data = {1.3, -4.8, -3.6, 0.24};
   SumOpDynamicModel m({TensorType_UINT8, {2, 2}, -5.0, 2.0},
                       {TensorType_UINT8, {2}, -5.0, 2.0},
                       {TensorType_INT32, {1}}, false);
-  std::initializer_list axis = {1};
+  std::vector axis = {1};
   m.SetAxis(axis);
   m.QuantizeAndPopulate(m.Input(), data);
   m.Invoke();
@@ -355,11 +413,11 @@ TEST(DynamicUint8SumOpTest, NotKeepDims) {
 
 TEST(DynamicUint8SumOpTest, KeepDims) {
   float kQuantizedTolerance = GetTolerance(-10.0, 12.0);
-  std::initializer_list data = {11.14, -0.14, 7.423, 0.879};
+  std::vector data = {11.14, -0.14, 7.423, 0.879};
   SumOpDynamicModel m({TensorType_UINT8, {2, 2}, -10.0, 12.0},
                       {TensorType_UINT8, {2}, -10.0, 12.0},
                       {TensorType_INT32, {1}}, true);
-  std::initializer_list axis = {0};
+  std::vector axis = {0};
   m.SetAxis(axis);
   m.QuantizeAndPopulate(m.Input(), data);
   m.Invoke();
@@ -369,6 +427,209 @@ TEST(DynamicUint8SumOpTest, KeepDims) {
       ElementsAreArray(ArrayFloatNear({6.47059, 10.698}, kQuantizedTolerance)));
 }
 
+// Tests for reduce_prod
+
+TEST(ConstFloatProdOpTest, NotKeepDims) {
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  ProdOpConstModel m({TensorType_FLOAT32, {4, 3, 2}}, {TensorType_FLOAT32, {2}},
+                     {4}, {1, 0, -3, -3}, false);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
+  EXPECT_THAT(
+      m.GetOutput(),
+      ElementsAreArray(ArrayFloatNear({3.162341376e+11, 1.9619905536e+12})));
+}
+
+TEST(ConstFloatProdOpTest, KeepDims) {
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  ProdOpConstModel m({TensorType_FLOAT32, {4, 3, 2}}, {TensorType_FLOAT32, {3}},
+                     {2}, {0, 2}, true);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 3, 1}));
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear({7.74592e+06, 1.197504e+08, 6.6889152e+08})));
+}
+
+TEST(DynamicFloatProdOpTest, NotKeepDims) {
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  ProdOpDynamicModel m({TensorType_FLOAT32, {4, 3, 2}},
+                       {TensorType_FLOAT32, {2}}, {TensorType_INT32, {4}},
+                       false);
+  std::vector axis = {1, 0, -3, -3};
+  m.SetAxis(axis);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
+  EXPECT_THAT(
+      m.GetOutput(),
+      ElementsAreArray(ArrayFloatNear({3.16234143225e+11, 1.9619905536e+12})));
+}
+
+TEST(DynamicFloatProdOpTest, KeepDims) {
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  ProdOpDynamicModel m({TensorType_FLOAT32, {4, 3, 2}},
+                       {TensorType_FLOAT32, {3}}, {TensorType_INT32, {2}},
+                       true);
+  std::vector axis = {0, 2};
+  m.SetAxis(axis);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 3, 1}));
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear({7.74592e+06, 1.197504e+08, 6.6889152e+08})));
+}
+
+TEST(DynamicFloatProdOpTest, Scale) {
+  std::vector data = {9.527};
+  ProdOpDynamicModel m({TensorType_FLOAT32, {1}}, {TensorType_FLOAT32, {1}},
+                       {TensorType_INT32, {1}}, true);
+  std::vector axis = {0};
+  m.SetAxis(axis);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1}));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear({9.527})));
+}
+
+// Tests for reduce_max
+
+TEST(ConstFloatMaxOpTest, NotKeepDims) {
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  MaxOpConstModel m({TensorType_FLOAT32, {4, 3, 2}}, {TensorType_FLOAT32, {2}},
+                    {4}, {1, 0, -3, -3}, false);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear({23, 24})));
+}
+
+TEST(ConstFloatMaxOpTest, KeepDims) {
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  MaxOpConstModel m({TensorType_FLOAT32, {4, 3, 2}}, {TensorType_FLOAT32, {3}},
+                    {2}, {0, 2}, true);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 3, 1}));
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(ArrayFloatNear({20, 22, 24})));
+}
+
+TEST(DynamicFloatMaxOpTest, NotKeepDims) {
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  MaxOpDynamicModel m({TensorType_FLOAT32, {4, 3, 2}},
+                      {TensorType_FLOAT32, {2}}, {TensorType_INT32, {4}},
+                      false);
+  std::vector axis = {1, 0, -3, -3};
+  m.SetAxis(axis);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear({23, 24})));
+}
+
+TEST(DynamicFloatMaxOpTest, KeepDims) {
+  std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
+                             9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
+                             17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0};
+  MaxOpDynamicModel m({TensorType_FLOAT32, {4, 3, 2}},
+                      {TensorType_FLOAT32, {3}}, {TensorType_INT32, {2}}, true);
+  std::vector axis = {0, 2};
+  m.SetAxis(axis);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 3, 1}));
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(ArrayFloatNear({20, 22, 24})));
+}
+
+TEST(DynamicFloatMaxOpTest, Scale) {
+  std::vector data = {9.527};
+  MaxOpDynamicModel m({TensorType_FLOAT32, {1}}, {TensorType_FLOAT32, {1}},
+                      {TensorType_INT32, {1}}, true);
+  std::vector axis = {0};
+  m.SetAxis(axis);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1}));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear({9.527})));
+}
+
+TEST(ConstUint8MaxOpTest, NotKeepDims) {
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::vector data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  MaxOpConstModel m({TensorType_UINT8, {1, 3, 2}, -1.0, 1.0},
+                    {TensorType_UINT8, {2}, -1.0, 1.0}, {1}, {1}, false);
+  m.QuantizeAndPopulate(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear({0.501961, 0.603922}, kQuantizedTolerance)));
+}
+
+TEST(ConstUint8MaxOpTest, KeepDims) {
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::vector data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  MaxOpConstModel m({TensorType_UINT8, {3, 2}, -1.0, 1.0},
+                    {TensorType_UINT8, {3}, -1.0, 1.0}, {1}, {1}, true);
+  m.QuantizeAndPopulate(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1}));
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear({0.4, 0.4, 0.603922}, kQuantizedTolerance)));
+}
+
+TEST(DynamicUint8MaxOpTest, NotKeepDims) {
+  float kQuantizedTolerance = GetTolerance(-5.0, 2.0);
+  std::vector data = {1.3, -4.8, -3.6, 0.24};
+  MaxOpDynamicModel m({TensorType_UINT8, {2, 2}, -5.0, 2.0},
+                      {TensorType_UINT8, {2}, -5.0, 2.0},
+                      {TensorType_INT32, {1}}, false);
+  std::vector axis = {1};
+  m.SetAxis(axis);
+  m.QuantizeAndPopulate(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear({1.2902, 0.247059}, kQuantizedTolerance)));
+}
+
+TEST(DynamicUint8MaxOpTest, KeepDims) {
+  float kQuantizedTolerance = GetTolerance(-10.0, 12.0);
+  std::vector data = {11.14, -0.14, 7.423, 0.879};
+  MaxOpDynamicModel m({TensorType_UINT8, {2, 2}, -10.0, 12.0},
+                      {TensorType_UINT8, {2}, -10.0, 12.0},
+                      {TensorType_INT32, {1}}, true);
+  std::vector axis = {0};
+  m.SetAxis(axis);
+  m.QuantizeAndPopulate(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear({11.1294, 0.862745}, kQuantizedTolerance)));
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc
index 22a507e6a4..f0f2757277 100644
--- a/tensorflow/contrib/lite/kernels/register.cc
+++ b/tensorflow/contrib/lite/kernels/register.cc
@@ -91,6 +91,8 @@ TfLiteRegistration* Register_FLOOR();
 TfLiteRegistration* Register_TILE();
 TfLiteRegistration* Register_NEG();
 TfLiteRegistration* Register_SUM();
+TfLiteRegistration* Register_REDUCE_PROD();
+TfLiteRegistration* Register_REDUCE_MAX();
 TfLiteRegistration* Register_SELECT();
 TfLiteRegistration* Register_SLICE();
 TfLiteRegistration* Register_SIN();
@@ -182,6 +184,8 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_TRANSPOSE_CONV, Register_TRANSPOSE_CONV());
   AddBuiltin(BuiltinOperator_TILE, Register_TILE());
   AddBuiltin(BuiltinOperator_SUM, Register_SUM());
+  AddBuiltin(BuiltinOperator_REDUCE_PROD, Register_REDUCE_PROD());
+  AddBuiltin(BuiltinOperator_REDUCE_MAX, Register_REDUCE_MAX());
   AddBuiltin(BuiltinOperator_EXPAND_DIMS, Register_EXPAND_DIMS());
   AddBuiltin(BuiltinOperator_SPARSE_TO_DENSE, Register_SPARSE_TO_DENSE());
   AddBuiltin(BuiltinOperator_EQUAL, Register_EQUAL());
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index 71e38c3f13..6c1ba3694a 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -616,6 +616,8 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
       break;
     }
     case BuiltinOperator_MEAN:
+    case BuiltinOperator_REDUCE_MAX:
+    case BuiltinOperator_REDUCE_PROD:
     case BuiltinOperator_SUM: {
       auto* params = MallocPOD();
       if (auto* schema_params = op->builtin_options_as_ReducerOptions()) {
diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc
index cc668485a4..5950840e8a 100644
--- a/tensorflow/contrib/lite/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/nnapi_delegate.cc
@@ -607,6 +607,8 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_EQUAL:
       case tflite::BuiltinOperator_NOT_EQUAL:
       case tflite::BuiltinOperator_SUM:
+      case tflite::BuiltinOperator_REDUCE_MAX:
+      case tflite::BuiltinOperator_REDUCE_PROD:
       case tflite::BuiltinOperator_SQRT:
       case tflite::BuiltinOperator_RSQRT:
       case tflite::BuiltinOperator_SHAPE:
diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs
index 64830b1dc3..6c3189a884 100644
--- a/tensorflow/contrib/lite/schema/schema.fbs
+++ b/tensorflow/contrib/lite/schema/schema.fbs
@@ -155,13 +155,15 @@ enum BuiltinOperator : byte {
   EQUAL = 71,
   NOT_EQUAL = 72,
   LOG = 73,
-  SUM=74,
+  SUM = 74,
   SQRT = 75,
   RSQRT = 76,
   SHAPE = 77,
   POW = 78,
   ARG_MIN = 79,
   FAKE_QUANT = 80,
+  REDUCE_PROD = 81,
+  REDUCE_MAX = 82,
 }
 
 // Options for the builtin operators.
diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h
index c0b57039cb..8052404319 100755
--- a/tensorflow/contrib/lite/schema/schema_generated.h
+++ b/tensorflow/contrib/lite/schema/schema_generated.h
@@ -351,11 +351,13 @@ enum BuiltinOperator {
   BuiltinOperator_POW = 78,
   BuiltinOperator_ARG_MIN = 79,
   BuiltinOperator_FAKE_QUANT = 80,
+  BuiltinOperator_REDUCE_PROD = 81,
+  BuiltinOperator_REDUCE_MAX = 82,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_FAKE_QUANT
+  BuiltinOperator_MAX = BuiltinOperator_REDUCE_MAX
 };
 
-inline BuiltinOperator (&EnumValuesBuiltinOperator())[80] {
+inline BuiltinOperator (&EnumValuesBuiltinOperator())[82] {
   static BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -436,7 +438,9 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[80] {
     BuiltinOperator_SHAPE,
     BuiltinOperator_POW,
     BuiltinOperator_ARG_MIN,
-    BuiltinOperator_FAKE_QUANT
+    BuiltinOperator_FAKE_QUANT,
+    BuiltinOperator_REDUCE_PROD,
+    BuiltinOperator_REDUCE_MAX
   };
   return values;
 }
@@ -524,6 +528,8 @@ inline const char **EnumNamesBuiltinOperator() {
     "POW",
     "ARG_MIN",
     "FAKE_QUANT",
+    "REDUCE_PROD",
+    "REDUCE_MAX",
     nullptr
   };
   return names;
diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index 1093bd2cbe..32d04c0717 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -879,16 +879,24 @@ def make_reduce_tests(reduce_op):
 
 def make_mean_tests(zip_path):
   """Make a set of tests to do mean."""
-
   return make_reduce_tests(tf.reduce_mean)(zip_path)
 
 
 def make_sum_tests(zip_path):
   """Make a set of tests to do sum."""
-
   return make_reduce_tests(tf.reduce_sum)(zip_path)
 
 
+def make_reduce_prod_tests(zip_path):
+  """Make a set of tests to do prod."""
+  return make_reduce_tests(tf.reduce_prod)(zip_path)
+
+
+def make_reduce_max_tests(zip_path):
+  """Make a set of tests to do max."""
+  return make_reduce_tests(tf.reduce_max)(zip_path)
+
+
 def make_exp_tests(zip_path):
   """Make a set of tests to do exp."""
 
diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index 2c469c0e75..bbce93f61a 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -246,10 +246,10 @@ cc_library(
         "graph_transformations/resolve_constant_transpose.cc",
         "graph_transformations/resolve_constant_unary.cc",
         "graph_transformations/resolve_fake_quant_args_from_vars.cc",
-        "graph_transformations/resolve_mean_attributes.cc",
         "graph_transformations/resolve_multiply_by_zero.cc",
         "graph_transformations/resolve_pad_attributes.cc",
         "graph_transformations/resolve_padv2_attributes.cc",
+        "graph_transformations/resolve_reduce_attributes.cc",
         "graph_transformations/resolve_reorder_axes.cc",
         "graph_transformations/resolve_reshape_attributes.cc",
         "graph_transformations/resolve_slice_attributes.cc",
diff --git a/tensorflow/contrib/lite/toco/export_tensorflow.cc b/tensorflow/contrib/lite/toco/export_tensorflow.cc
index bf9a51a525..17375d19be 100644
--- a/tensorflow/contrib/lite/toco/export_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/export_tensorflow.cc
@@ -1623,10 +1623,11 @@ void ConvertSliceOperator(const Model& model, const SliceOperator& src_op,
   CreateSliceInput(src_op.inputs[2], src_op.size, tensorflow_graph);
 }
 
-void ConvertMeanOperator(const Model& model, const MeanOperator& src_op,
-                         GraphDef* tensorflow_graph) {
+template 
+void ConvertReduceOperator(const Model& model, const T& src_op,
+                           GraphDef* tensorflow_graph, const string& op_name) {
   tensorflow::NodeDef* new_op = tensorflow_graph->add_node();
-  new_op->set_op("Mean");
+  new_op->set_op(op_name);
   new_op->set_name(src_op.outputs[0]);
   CHECK_EQ(src_op.inputs.size(), 2);
   *new_op->add_input() = src_op.inputs[0];
@@ -1961,8 +1962,20 @@ void ConvertOperator(const Model& model, const Operator& src_op,
         model, static_cast(src_op),
         tensorflow_graph);
   } else if (src_op.type == OperatorType::kMean) {
-    ConvertMeanOperator(model, static_cast(src_op),
-                        tensorflow_graph);
+    ConvertReduceOperator(model, static_cast(src_op),
+                          tensorflow_graph, "Mean");
+  } else if (src_op.type == OperatorType::kSum) {
+    ConvertReduceOperator(model,
+                          static_cast(src_op),
+                          tensorflow_graph, "Sum");
+  } else if (src_op.type == OperatorType::kReduceProd) {
+    ConvertReduceOperator(model,
+                          static_cast(src_op),
+                          tensorflow_graph, "Prod");
+  } else if (src_op.type == OperatorType::kReduceMax) {
+    ConvertReduceOperator(model,
+                          static_cast(src_op),
+                          tensorflow_graph, "Max");
   } else if (src_op.type == OperatorType::kSub) {
     ConvertSubOperator(model, static_cast(src_op),
                        tensorflow_graph);
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
index 7cc9bb75d7..8db7df5c0e 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
@@ -180,7 +180,7 @@ DECLARE_GRAPH_TRANSFORMATION(ResolvePadAttributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolvePadV2Attributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveStridedSliceAttributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveSliceAttributes)
-DECLARE_GRAPH_TRANSFORMATION(ResolveMeanAttributes)
+DECLARE_GRAPH_TRANSFORMATION(ResolveReduceAttributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveTransposeAttributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantRandomUniform)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantRange)
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 4f95c57451..f422e3a9c7 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -524,10 +524,12 @@ bool KeepDims(const Operator& op) {
   switch (op.type) {
     case OperatorType::kMin:  //  Reduction Min
       return static_cast(op).keep_dims;
-    case OperatorType::kMax:  //  Reduction Max
+    case OperatorType::kReduceMax:  //  Reduction Max
       return static_cast(op).keep_dims;
     case OperatorType::kSum:
       return static_cast(op).keep_dims;
+    case OperatorType::kReduceProd:
+      return static_cast(op).keep_dims;
     case OperatorType::kMean:
       return static_cast(op).keep_dims;
     default:
@@ -1606,8 +1608,9 @@ bool PropagateFixedSizes::Run(Model* model, std::size_t op_index) {
       ProcessL2PoolOperator(model, static_cast(op));
       break;
     case OperatorType::kMin:  //  Reduction Min
-    case OperatorType::kMax:  //  Reduction Max
+    case OperatorType::kReduceMax:  //  Reduction Max
     case OperatorType::kSum:
+    case OperatorType::kReduceProd:
     case OperatorType::kMean:
       ProcessTensorFlowReductionOperator(model, op);
       break;
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc
index f89ef85fdb..51099cf74a 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc
@@ -58,7 +58,7 @@ bool ResolveConstantUnaryOperator::Run(Model* model, std::size_t op_index) {
     case OperatorType::kSquare:
     case OperatorType::kSum:
     case OperatorType::kMin:  //  Reduction Min
-    case OperatorType::kMax:  //  Reduction Max
+    case OperatorType::kReduceMax:  //  Reduction Max
     case OperatorType::kReshape:
     case OperatorType::kRelu6:
     case OperatorType::kRelu1:
@@ -207,7 +207,7 @@ bool ResolveConstantUnaryOperator::Run(Model* model, std::size_t op_index) {
       min = std::min(min, (*input_float_data)[i]);
     }
     output_float_data[0] = min;
-  } else if (unary_op->type == OperatorType::kMax) {
+  } else if (unary_op->type == OperatorType::kReduceMax) {
     // At the moment only full reduction across all dimensions is supported.
     // TODO(starka): Output should not be padded.
     for (int i = 0; i < output_dims_count; i++) {
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_mean_attributes.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_mean_attributes.cc
deleted file mode 100644
index 013b50ac9b..0000000000
--- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_mean_attributes.cc
+++ /dev/null
@@ -1,45 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include 
-#include 
-#include 
-#include 
-
-#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
-#include "tensorflow/contrib/lite/toco/model.h"
-#include "tensorflow/contrib/lite/toco/tooling_util.h"
-#include "tensorflow/core/platform/logging.h"
-
-namespace toco {
-
-bool ResolveMeanAttributes::Run(Model* model, std::size_t op_index) {
-  auto* mean_op = model->operators[op_index].get();
-  if (mean_op->type != OperatorType::kMean) return false;
-  auto* op = static_cast(mean_op);
-
-  if (!op->axis.empty()) {
-    // Attributes already resolved
-    return false;
-  }
-  if (op->inputs.size() != 2) return false;
-  if (!IsConstantParameterArray(*model, op->inputs[1])) return false;
-
-  const auto& indices_array = model->GetArray(op->inputs[1]);
-  if (!indices_array.has_shape()) return false;
-  op->axis = indices_array.GetBuffer().data;
-  return true;
-}
-
-}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_reduce_attributes.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_reduce_attributes.cc
new file mode 100644
index 0000000000..5f8a06ba92
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_reduce_attributes.cc
@@ -0,0 +1,58 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include 
+#include 
+#include 
+#include 
+
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace toco {
+
+template 
+bool ResolveAttributes(Model* model, T* op) {
+  if (!op->axis.empty()) {
+    // Attributes already resolved
+    return false;
+  }
+  if (op->inputs.size() != 2) return false;
+  if (!IsConstantParameterArray(*model, op->inputs[1])) return false;
+
+  const Array& indices_array = model->GetArray(op->inputs[1]);
+  if (!indices_array.has_shape()) return false;
+  op->axis = indices_array.GetBuffer().data;
+  return true;
+}
+
+bool ResolveReduceAttributes::Run(Model* model, std::size_t op_index) {
+  Operator* op = model->operators[op_index].get();
+  switch (op->type) {
+    case OperatorType::kMean:
+      return ResolveAttributes(model, static_cast(op));
+    case OperatorType::kSum:
+      return ResolveAttributes(model, static_cast(op));
+    case OperatorType::kReduceProd:
+      return ResolveAttributes(model, static_cast(op));
+    case OperatorType::kReduceMax:
+      return ResolveAttributes(model, static_cast(op));
+    default:
+      return false;
+  }
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc
index ab3762e7ea..2ffab49e7a 100644
--- a/tensorflow/contrib/lite/toco/import_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc
@@ -805,22 +805,6 @@ tensorflow::Status ConvertSqueezeOperator(
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status ConvertSumOperator(
-    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
-    Model* model) {
-  CHECK_EQ(node.op(), "Sum");
-  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 2));
-  auto* op = new TensorFlowSumOperator;
-  op->inputs.push_back(node.input(0));
-  op->inputs.push_back(node.input(1));
-  op->outputs.push_back(node.name());
-  model->operators.emplace_back(op);
-  if (HasAttr(node, "keep_dims")) {
-    op->keep_dims = GetBoolAttr(node, "keep_dims");
-  }
-  return tensorflow::Status::OK();
-}
-
 tensorflow::Status ConvertSplitOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
@@ -1058,22 +1042,6 @@ tensorflow::Status ConvertSimpleOperator(
   return ConvertSimpleOperator(node, tf_import_flags, model);
 }
 
-tensorflow::Status ConvertMaxOperator(
-    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
-    Model* model) {
-  CHECK_EQ(node.op(), "Max");
-  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 2));
-  auto* op = new TensorFlowMaxOperator;
-  op->inputs.push_back(node.input(0));
-  op->inputs.push_back(node.input(1));
-  op->outputs.push_back(node.name());
-  model->operators.emplace_back(op);
-  if (HasAttr(node, "keep_dims")) {
-    op->keep_dims = GetBoolAttr(node, "keep_dims");
-  }
-  return tensorflow::Status::OK();
-}
-
 tensorflow::Status ConvertMinOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
@@ -1412,12 +1380,12 @@ tensorflow::Status ConvertBatchToSpaceNDOperator(
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status ConvertMeanOperator(
+template 
+tensorflow::Status ConvertReduceOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
-  CHECK_EQ(node.op(), "Mean");
   TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 2));
-  auto* op = new MeanOperator;
+  auto* op = new T;
   op->inputs.push_back(node.input(0));
   op->inputs.push_back(node.input(1));
   op->outputs.push_back(node.name());
@@ -1893,10 +1861,10 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Log", ConvertSimpleOperator},
       {"LogSoftmax", ConvertSimpleOperator},
       {"MatMul", ConvertMatMulOperator},
-      {"Max", ConvertMaxOperator},
+      {"Max", ConvertReduceOperator},
       {"MaxPool", ConvertMaxPoolOperator},
       {"Maximum", ConvertSimpleOperator},
-      {"Mean", ConvertMeanOperator},
+      {"Mean", ConvertReduceOperator},
       {"Merge", ConvertSimpleOperator},
       {"Min", ConvertMinOperator},
       {"Minimum", ConvertSimpleOperator},
@@ -1912,6 +1880,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Placeholder", ConvertPlaceholderOperator},
       {"PlaceholderWithDefault", ConvertIdentityOperator},
       {"Pow", ConvertSimpleOperator},
+      {"Prod", ConvertReduceOperator},
       {"RandomUniform", ConvertRandomUniform},
       {"Range", ConvertRangeOperator},
       {"Rank", ConvertSimpleOperator},
@@ -1938,7 +1907,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"StopGradient", ConvertIdentityOperator},
       {"StridedSlice", ConvertStridedSliceOperator},
       {"Sub", ConvertSimpleOperator},
-      {"Sum", ConvertSumOperator},
+      {"Sum", ConvertReduceOperator},
       {"Svdf", ConvertSvdfOperator},
       {"Switch", ConvertSwitchOperator},
       {"Tanh", ConvertSimpleOperator},
diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h
index d06a30b638..37f4188cf7 100644
--- a/tensorflow/contrib/lite/toco/model.h
+++ b/tensorflow/contrib/lite/toco/model.h
@@ -85,6 +85,7 @@ enum class OperatorType : uint8 {
   kBatchToSpaceND,
   kPad,
   kPadV2,
+  kReduceProd,  // Reduction product
   kStridedSlice,
   kSlice,
   kSqueeze,
@@ -106,10 +107,10 @@ enum class OperatorType : uint8 {
   kIdentity,
   kLess,
   kLessEqual,
-  kMax,      //  Reduction Max
-  kMaximum,  //  Element-wise Maximum
-  kMin,      //  Reduction Min
-  kMinimum,  //  Element-wise Minimum
+  kReduceMax,  //  Reduction Max
+  kMaximum,    //  Element-wise Maximum
+  kMin,        //  Reduction Min
+  kMinimum,    //  Element-wise Minimum
   kMatMul,
   kMerge,
   kNeg,
@@ -1229,6 +1230,19 @@ struct SubOperator : Operator {
 // TensorFlow equivalent: Sum
 struct TensorFlowSumOperator : Operator {
   TensorFlowSumOperator() : Operator(OperatorType::kSum) {}
+  std::vector axis;
+  bool keep_dims = false;
+};
+
+// Prod reduction: computes the product of all of entries across the axes.
+//
+// Inputs:
+//   inputs[0]: required: the input array
+//
+// TensorFlow equivalent: Prod
+struct TensorFlowProdOperator : Operator {
+  TensorFlowProdOperator() : Operator(OperatorType::kReduceProd) {}
+  std::vector axis;
   bool keep_dims = false;
 };
 
@@ -1388,16 +1402,15 @@ struct TensorFlowNotEqualOperator : Operator {
   TensorFlowNotEqualOperator() : Operator(OperatorType::kNotEqual) {}
 };
 
-// Global max reduction: computes the max of all of entries in the input array.
-// Thus the output is "0-dimensional": it consists of a single scalar value.
+// Max reduction: computes the max of all of entries across the axes.
 //
 // Inputs:
 //   inputs[0]: required: the input array
 //
-// TensorFlow equivalent: Max --- except that we only support the special case
-// of global reduction across all dimensions.
+// TensorFlow equivalent: Max
 struct TensorFlowMaxOperator : Operator {
-  TensorFlowMaxOperator() : Operator(OperatorType::kMax) {}
+  TensorFlowMaxOperator() : Operator(OperatorType::kReduceMax) {}
+  std::vector axis;
   bool keep_dims = false;
 };
 
diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc
index a791e60f91..68d13586f1 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator.cc
@@ -767,6 +767,44 @@ class Sum
   int GetVersion(const Operator& op) const override { return 1; }
 };
 
+class ReduceMax
+    : public BuiltinOperator {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateReducerOptions(*builder, op.keep_dims);
+  }
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    op->keep_dims = options.keep_dims();
+  }
+
+  int GetVersion(const Operator& op) const override { return 1; }
+};
+
+class ReduceProd
+    : public BuiltinOperator {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateReducerOptions(*builder, op.keep_dims);
+  }
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    op->keep_dims = options.keep_dims();
+  }
+
+  int GetVersion(const Operator& op) const override { return 1; }
+};
+
 class ResizeBilinear
     : public BuiltinOperator> BuildOperatorList() {
   ops.emplace_back(
       new Mean(::tflite::BuiltinOperator_MEAN, OperatorType::kMean));
   ops.emplace_back(new Sum(::tflite::BuiltinOperator_SUM, OperatorType::kSum));
+  ops.emplace_back(new ReduceProd(::tflite::BuiltinOperator_REDUCE_PROD,
+                                  OperatorType::kReduceProd));
+  ops.emplace_back(new ReduceMax(::tflite::BuiltinOperator_REDUCE_MAX,
+                                 OperatorType::kReduceMax));
   ops.emplace_back(new ResizeBilinear(::tflite::BuiltinOperator_RESIZE_BILINEAR,
                                       OperatorType::kResizeBilinear));
   ops.emplace_back(
diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc
index a4dc1bbe93..7a0d9608cc 100644
--- a/tensorflow/contrib/lite/toco/toco_tooling.cc
+++ b/tensorflow/contrib/lite/toco/toco_tooling.cc
@@ -113,7 +113,7 @@ void MakeGeneralGraphTransformationsSet(
   transformations->Add(new ResolvePadV2Attributes);
   transformations->Add(new ResolveStridedSliceAttributes);
   transformations->Add(new ResolveSliceAttributes);
-  transformations->Add(new ResolveMeanAttributes);
+  transformations->Add(new ResolveReduceAttributes);
   transformations->Add(new ResolveConstantShapeOrRank);
   transformations->Add(new MakeInitialDequantizeOperator);
   transformations->Add(new UnpartitionEmbeddingLookup);
diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index 4ec74e351f..45cd10ec7b 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -350,7 +350,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(Less)
     HANDLE_OPERATORTYPENAME_CASE(LessEqual)
     HANDLE_OPERATORTYPENAME_CASE(MatMul)
-    HANDLE_OPERATORTYPENAME_CASE(Max)      //  Reduction Max
+    HANDLE_OPERATORTYPENAME_CASE(ReduceMax)  //  Reduction Max
     HANDLE_OPERATORTYPENAME_CASE(Maximum)  //  Element-wise Maximum
     HANDLE_OPERATORTYPENAME_CASE(Merge)
     HANDLE_OPERATORTYPENAME_CASE(Min)      //  Reduction Min
@@ -385,6 +385,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(SpaceToBatchND)
     HANDLE_OPERATORTYPENAME_CASE(BatchToSpaceND)
     HANDLE_OPERATORTYPENAME_CASE(Mean)
+    HANDLE_OPERATORTYPENAME_CASE(ReduceProd)
     HANDLE_OPERATORTYPENAME_CASE(Svdf)
     HANDLE_OPERATORTYPENAME_CASE(ArgMax)
     HANDLE_OPERATORTYPENAME_CASE(ArgMin)
-- 
cgit v1.2.3


From 6b7198b495ae7a4acd9604dbeda41e7855f97bdd Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Mon, 16 Jul 2018 20:00:48 -0700
Subject: Refactor the test base class so each subclass can define its own
 graph.

---
 tensorflow/contrib/tensorrt/test/base_test.py      | 163 ++++++++++-----------
 .../tensorrt/test/tf_trt_integration_test_base.py  |  73 +++++----
 2 files changed, 126 insertions(+), 110 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py
index 4b9e6d668f..f057e377cb 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/contrib/tensorrt/test/base_test.py
@@ -30,96 +30,93 @@ from tensorflow.python.platform import test
 from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 
 
-# TODO(aaroey): test graph with different dtypes.
-def _GetSingleEngineGraphDef(dtype=dtypes.float32):
-  """Create a graph containing single segment."""
-  input_dims = [100, 24, 24, 2]
-  g = ops.Graph()
-  with g.as_default():
-    inp = array_ops.placeholder(
-        dtype=dtype, shape=[None] + input_dims[1:], name=trt_test.INPUT_NAME)
-    with g.device("/GPU:0"):
-      conv_filter = constant_op.constant(
-          [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
-          name="weights",
-          dtype=dtype)
-      conv = nn.conv2d(
-          input=inp,
-          filter=conv_filter,
-          strides=[1, 2, 2, 1],
-          padding="SAME",
-          name="conv")
-      bias = constant_op.constant(
-          [4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtype)
-      added = nn.bias_add(conv, bias, name="bias_add")
-      relu = nn.relu(added, "relu")
-      identity = array_ops.identity(relu, "identity")
-      pool = nn_ops.max_pool(
-          identity, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool")
-    array_ops.squeeze(pool, name=trt_test.OUTPUT_NAME)
-  return trt_test.TfTrtIntegrationTestParams(
-      graph_name="SimpleSingleEngine",
-      gdef=g.as_graph_def(),
-      input_dims=input_dims,
-      num_expected_engines=1,
-      expected_output_dims=(100, 6, 6, 6),
-      allclose_atol=1.e-03,
-      allclose_rtol=1.e-03)
+class SimpleSingleEngineGraphDefTest(trt_test.TfTrtIntegrationTestBase):
 
+  def GetParams(self):
+    """Create a graph containing single segment."""
+    # TODO(aaroey): test graph with different dtypes.
+    dtype = dtypes.float32
+    input_dims = [100, 24, 24, 2]
+    g = ops.Graph()
+    with g.as_default():
+      inp = array_ops.placeholder(
+          dtype=dtype, shape=[None] + input_dims[1:], name=self.input_name)
+      with g.device("/GPU:0"):
+        conv_filter = constant_op.constant(
+            [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
+            name="weights",
+            dtype=dtype)
+        conv = nn.conv2d(
+            input=inp,
+            filter=conv_filter,
+            strides=[1, 2, 2, 1],
+            padding="SAME",
+            name="conv")
+        bias = constant_op.constant(
+            [4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtype)
+        added = nn.bias_add(conv, bias, name="bias_add")
+        relu = nn.relu(added, "relu")
+        identity = array_ops.identity(relu, "identity")
+        pool = nn_ops.max_pool(
+            identity, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool")
+      array_ops.squeeze(pool, name=self.output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_dims=input_dims,
+        num_expected_engines=1,
+        expected_output_dims=(100, 6, 6, 6),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
 
-# TODO(aaroey): test graph with different dtypes.
-def _GetMultiEngineGraphDef(dtype=dtypes.float32):
-  """Create a graph containing multiple segment."""
-  input_dims = [100, 24, 24, 2]
-  g = ops.Graph()
-  with g.as_default():
-    inp = array_ops.placeholder(
-        dtype=dtype, shape=[None] + input_dims[1:], name=trt_test.INPUT_NAME)
-    with g.device("/GPU:0"):
-      conv_filter = constant_op.constant(
-          [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
-          name="weights",
-          dtype=dtype)
-      conv = nn.conv2d(
-          input=inp,
-          filter=conv_filter,
-          strides=[1, 2, 2, 1],
-          padding="SAME",
-          name="conv")
-      c1 = constant_op.constant(
-          np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype)
-      p = conv * c1
-      c2 = constant_op.constant(
-          np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype)
-      q = conv / c2
 
-      edge = trt_test.TRT_INCOMPATIBLE_OP(q)
-      edge /= edge
-      r = edge + edge
+class SimpleMultiEngineGraphDefTest(trt_test.TfTrtIntegrationTestBase):
 
-      p -= edge
-      q *= edge
-      s = p + q
-      s -= r
-    array_ops.squeeze(s, name=trt_test.OUTPUT_NAME)
-  return trt_test.TfTrtIntegrationTestParams(
-      graph_name="SimpleMultipleEngines",
-      gdef=g.as_graph_def(),
-      input_dims=input_dims,
-      num_expected_engines=2,
-      expected_output_dims=(100, 12, 12, 6),
-      allclose_atol=1.e-03,
-      allclose_rtol=1.e-03)
+  def GetParams(self):
+    """Create a graph containing multiple segment."""
+    # TODO(aaroey): test graph with different dtypes.
+    dtype = dtypes.float32
+    input_dims = [100, 24, 24, 2]
+    g = ops.Graph()
+    with g.as_default():
+      inp = array_ops.placeholder(
+          dtype=dtype, shape=[None] + input_dims[1:], name=self.input_name)
+      with g.device("/GPU:0"):
+        conv_filter = constant_op.constant(
+            [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
+            name="weights",
+            dtype=dtype)
+        conv = nn.conv2d(
+            input=inp,
+            filter=conv_filter,
+            strides=[1, 2, 2, 1],
+            padding="SAME",
+            name="conv")
+        c1 = constant_op.constant(
+            np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype)
+        p = conv * c1
+        c2 = constant_op.constant(
+            np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype)
+        q = conv / c2
 
+        edge = self.trt_incompatible_op(q)
+        edge /= edge
+        r = edge + edge
 
-class BaseTest(trt_test.TfTrtIntegrationTestBase):
-  """Class to test Tensorflow-TensorRT integration."""
-  pass
+        p -= edge
+        q *= edge
+        s = p + q
+        s -= r
+      array_ops.squeeze(s, name=self.output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_dims=input_dims,
+        num_expected_engines=2,
+        expected_output_dims=(100, 12, 12, 6),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
 
 
+# TODO(aaroey): add a large complex graph to test.
+
 if __name__ == "__main__":
-  # TODO(aaroey): add a large complex graph to test.
-  trt_test.AddTests(BaseTest,
-                    [_GetSingleEngineGraphDef(),
-                     _GetMultiEngineGraphDef()])
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 980cc87366..b1dc7b649f 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -22,7 +22,6 @@ from collections import namedtuple
 import itertools
 import warnings
 import numpy as np
-import re
 import six
 
 from tensorflow.contrib import tensorrt as trt
@@ -36,23 +35,36 @@ from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 
 TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [
-    "graph_name", "gdef", "input_dims", "num_expected_engines",
-    "expected_output_dims", "allclose_atol", "allclose_rtol"
+    "gdef", "input_dims", "num_expected_engines", "expected_output_dims",
+    "allclose_atol", "allclose_rtol"
 ])
 
-INPUT_NAME = "input"
-OUTPUT_NAME = "output"
-TRT_INCOMPATIBLE_OP = math_ops.sin
 PRECISION_MODES = ["FP32", "FP16", "INT8"]
 
 
-def IsQuantizationMode(mode):
+def _IsQuantizationMode(mode):
   return mode == "INT8"
 
 
 class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   """Class to test Tensorflow-TensorRT integration."""
 
+  @property
+  def input_name(self):
+    return "input"
+
+  @property
+  def output_name(self):
+    return "output"
+
+  @property
+  def trt_incompatible_op(self):
+    return math_ops.sin
+
+  @property
+  def precision_modes(self):
+    return ["FP32", "FP16", "INT8"]
+
   def _ToBytes(self, s):
     if six.PY2:
       return s
@@ -70,6 +82,10 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     super(TfTrtIntegrationTestBase, self).setUp()
     warnings.simplefilter("always")
 
+  def GetParams(self):
+    """Return a TfTrtIntegrationTestParams for test, implemented by subclass."""
+    raise NotImplementedError()
+
   def _GetConfigProto(self,
                       params,
                       use_optimizer,
@@ -104,7 +120,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     g = ops.Graph()
     with g.as_default():
       inp, out = importer.import_graph_def(
-          graph_def=gdef, return_elements=[INPUT_NAME, OUTPUT_NAME], name="")
+          graph_def=gdef,
+          return_elements=[self.input_name, self.output_name],
+          name="")
       inp = inp.outputs[0]
       out = out.outputs[0]
     with self.test_session(
@@ -129,7 +147,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     """Return trt converted graphdef."""
     return trt.create_inference_graph(
         input_graph_def=gdef,
-        outputs=[OUTPUT_NAME],
+        outputs=[self.output_name],
         max_batch_size=params.input_dims[0],
         max_workspace_size_bytes=1 << 25,
         precision_mode=precision_mode,
@@ -150,7 +168,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         self.assertNotEqual("", n.attr["segment_funcdef_name"].s)
         self.assertEquals(n.attr["precision_mode"].s, precision_mode)
         self.assertEquals(n.attr["static_engine"].b, not dynamic_engine)
-        if IsQuantizationMode(precision_mode) and is_calibrated:
+        if _IsQuantizationMode(precision_mode) and is_calibrated:
           self.assertNotEqual("", n.attr["calibration_data"].s)
         else:
           self.assertEquals("", n.attr["calibration_data"].s)
@@ -173,7 +191,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     ref_result = self._RunGraph(params, input_gdef, inp, config_no_trt)
 
     # Run calibration if necessary.
-    if IsQuantizationMode(precision_mode):
+    if _IsQuantizationMode(precision_mode):
 
       calib_config = self._GetConfigProto(params, use_optimizer, precision_mode,
                                           dynamic_calib_engine)
@@ -228,18 +246,17 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     pass
 
 
-def AddTests(test_class, params_list):
+def _AddTests(test_class):
 
-  def _GetTest(params, use_optimizer, precision_mode, dynamic_infer_engine,
+  def _GetTest(use_optimizer, precision_mode, dynamic_infer_engine,
                dynamic_calib_engine):
 
     def _Test(self):
+      params = self.GetParams()
       logging.info(
-          "Running test with parameters: graph_name=%s, "
-          "use_optimizer=%s, precision_mode=%s, "
-          "dynamic_infer_engine=%s, dynamic_calib_engine=%s", params.graph_name,
-          use_optimizer, precision_mode, dynamic_infer_engine,
-          dynamic_calib_engine)
+          "Running test with parameters: use_optimizer=%s, precision_mode=%s, "
+          "dynamic_infer_engine=%s, dynamic_calib_engine=%s", use_optimizer,
+          precision_mode, dynamic_infer_engine, dynamic_calib_engine)
       self._RunTest(params, use_optimizer, precision_mode, dynamic_infer_engine,
                     dynamic_calib_engine)
 
@@ -248,11 +265,11 @@ def AddTests(test_class, params_list):
   use_optimizer_options = [False, True]
   dynamic_infer_engine_options = [False, True]
   dynamic_calib_engine_options = [False, True]
-  for (params, use_optimizer, precision_mode,
+  for (use_optimizer, precision_mode,
        dynamic_infer_engine, dynamic_calib_engine) in itertools.product(
-           params_list, use_optimizer_options, PRECISION_MODES,
-           dynamic_infer_engine_options, dynamic_calib_engine_options):
-    if IsQuantizationMode(precision_mode):
+           use_optimizer_options, PRECISION_MODES, dynamic_infer_engine_options,
+           dynamic_calib_engine_options):
+    if _IsQuantizationMode(precision_mode):
       if not dynamic_calib_engine and dynamic_infer_engine:
         # TODO(aaroey): test this case, the conversion from static calibration
         # engine to dynamic inference engine should be a noop.
@@ -283,11 +300,13 @@ def AddTests(test_class, params_list):
     if precision_mode == "INT8":
       calib_engine_type = ("DynamicCalibEngine"
                            if dynamic_calib_engine else "StaticCalibEngine")
-    test_name = "%s_%s_%s_%s%s" % (re.sub(
-        "[^a-zA-Z0-9]+", "", params.graph_name), conversion, precision_mode,
-                                   infer_engine_type, ("_" + calib_engine_type)
-                                   if len(calib_engine_type) else "")
+    test_name = "%s_%s_%s%s" % (conversion, precision_mode, infer_engine_type,
+                                ("_" + calib_engine_type)
+                                if len(calib_engine_type) else "")
     setattr(
         test_class, "testTfTRT_" + test_name,
-        _GetTest(params, use_optimizer, precision_mode, dynamic_infer_engine,
+        _GetTest(use_optimizer, precision_mode, dynamic_infer_engine,
                  dynamic_calib_engine))
+
+
+_AddTests(TfTrtIntegrationTestBase)
-- 
cgit v1.2.3


From c091185930eee5a3f87cfbe5e367a3cfc8b717e8 Mon Sep 17 00:00:00 2001
From: Guangda Lai 
Date: Mon, 16 Jul 2018 22:56:37 -0700
Subject: Fix various building/testing issues and enable basic TensorRT tests,
 details are: 1. Add IsGoogleTensorRTEnabled() method and
 is_tensorrt_enabled() python wrapper to guard trt python tests. 2. Fix nvcc
 build problems and add corresponding TODOs in some c++ code. 3. Fix various
 kokoro test config problems (e.g. fix oss build dependencies, add nomac tags
 for some tests, etc)

PiperOrigin-RevId: 204862004
---
 tensorflow/contrib/BUILD                           |  6 ++--
 tensorflow/contrib/tensorrt/BUILD                  | 32 ++++++++++----------
 .../contrib/tensorrt/convert/convert_graph.cc      |  8 +++--
 .../contrib/tensorrt/convert/convert_nodes.cc      |  9 ++----
 tensorflow/contrib/tensorrt/convert/utils.cc       | 35 ++++++++++++++++++++++
 tensorflow/contrib/tensorrt/convert/utils.h        |  2 ++
 tensorflow/contrib/tensorrt/python/__init__.py     |  1 +
 tensorflow/contrib/tensorrt/python/trt_convert.py  |  1 +
 .../contrib/tensorrt/resources/trt_allocator.h     |  3 ++
 .../tensorrt/test/tf_trt_integration_test.py       |  5 ++--
 tensorflow/contrib/tensorrt/trt_conversion.i       | 12 ++++++--
 tensorflow/tools/pip_package/BUILD                 |  5 +---
 12 files changed, 81 insertions(+), 38 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/convert/utils.cc

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 60be9db263..1322056d80 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -7,7 +7,6 @@ package(default_visibility = ["//tensorflow:__subpackages__"])
 
 load("//third_party/mpi:mpi.bzl", "if_mpi")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
-load("@local_config_tensorrt//:build_defs.bzl", "if_tensorrt")
 load("//tensorflow:tensorflow.bzl", "if_not_windows")
 load("//tensorflow:tensorflow.bzl", "if_not_windows_cuda")
 
@@ -103,6 +102,7 @@ py_library(
         "//tensorflow/contrib/summary:summary",
         "//tensorflow/contrib/tensor_forest:init_py",
         "//tensorflow/contrib/tensorboard",
+        "//tensorflow/contrib/tensorrt:init_py",
         "//tensorflow/contrib/testing:testing_py",
         "//tensorflow/contrib/text:text_py",
         "//tensorflow/contrib/tfprof",
@@ -113,9 +113,7 @@ py_library(
         "//tensorflow/contrib/util:util_py",
         "//tensorflow/python:util",
         "//tensorflow/python/estimator:estimator_py",
-    ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_tensorrt([
-        "//tensorflow/contrib/tensorrt:init_py",
-    ]) + select({
+    ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + select({
         "//tensorflow:with_kafka_support_windows_override": [],
         "//tensorflow:with_kafka_support": [
             "//tensorflow/contrib/kafka",
diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index adda0b758b..cb2daa7b12 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -11,7 +11,7 @@ exports_files(["LICENSE"])
 
 load(
     "//tensorflow:tensorflow.bzl",
-    "py_test",
+    "cuda_py_test",
     "tf_cc_test",
     "tf_copts",
     "tf_cuda_library",
@@ -32,10 +32,7 @@ tf_cuda_cc_test(
     name = "tensorrt_test_cc",
     size = "small",
     srcs = ["tensorrt_test.cc"],
-    tags = [
-        "manual",
-        "notap",
-    ],
+    tags = ["no_windows"],
     deps = [
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
@@ -185,6 +182,9 @@ tf_py_wrap_cc(
     name = "wrap_conversion",
     srcs = ["trt_conversion.i"],
     copts = tf_copts(),
+    swig_includes = [
+        "//tensorflow/python:platform/base.i",
+    ],
     deps = [
         ":trt_conversion",
         ":trt_engine_op_kernel",
@@ -275,6 +275,7 @@ tf_cc_test(
     name = "segment_test",
     size = "small",
     srcs = ["segment/segment_test.cc"],
+    tags = ["no_windows"],
     deps = [
         ":segment",
         "//tensorflow/c:c_api",
@@ -310,10 +311,6 @@ tf_cuda_cc_test(
     name = "trt_plugin_factory_test",
     size = "small",
     srcs = ["plugin/trt_plugin_factory_test.cc"],
-    tags = [
-        "manual",
-        "notap",
-    ],
     deps = [
         ":trt_plugins",
         "//tensorflow/core:lib",
@@ -325,23 +322,24 @@ tf_cuda_cc_test(
     ]),
 )
 
-py_test(
+cuda_py_test(
     name = "tf_trt_integration_test",
     srcs = ["test/tf_trt_integration_test.py"],
-    main = "test/tf_trt_integration_test.py",
-    srcs_version = "PY2AND3",
-    tags = [
-        "manual",
-        "notap",
-    ],
-    deps = [
+    additional_deps = [
         ":init_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_test_lib",
     ],
+    main = "test/tf_trt_integration_test.py",
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
 )
 
 cc_library(
     name = "utils",
+    srcs = ["convert/utils.cc"],
     hdrs = ["convert/utils.h"],
+    copts = tf_copts(),
 )
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 63d8eec7db..089b03dcb5 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -624,7 +624,9 @@ tensorflow::Status RegisterSegmentFunctionToFunctionLibrary(
         edge->src()->output_type(edge->src_output()));
     VLOG(1) << " input " << nout.node << ":" << nout.index
             << " dtype=" << tensorflow::DataTypeString(nout.data_type);
-    node_builder.Input({nout});
+    // nvcc complains that Input() is
+    // ambiguous, so do not use Input({nout}).
+    node_builder.Input(nout);
     TF_RETURN_IF_ERROR(node_builder.Attr("T", node->output_type(0))
                            .Attr("index", i)
                            .Finalize(&nd));
@@ -829,7 +831,9 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
     // The allocator is used to build the engine. The build and the built engine
     // will be destroyed after we get the serialized engine string, so it's fine
     // to use unique_ptr here.
-    std::unique_ptr alloc;
+    // TODO(aaroey): nvinfer1::IGpuAllocator doesn't have a virtual destructor
+    // and destructing the unique_ptr will result in segfault, fix it.
+    std::unique_ptr alloc;
     auto device_alloc = GetDeviceAndAllocator(params, engine);
     int cuda_device_id = 0;
     if (device_alloc.first >= 0) {
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 0ee708bc1c..65fef27533 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -630,6 +630,7 @@ class Converter {
     const string& op = node_def.op();
     std::vector outputs;
     if (PluginFactoryTensorRT::GetInstance()->IsPlugin(op)) {
+      // TODO(aaroey): plugin_converter_ is not set, fix it.
       TF_RETURN_IF_ERROR(plugin_converter_(*this, node_def, inputs, &outputs));
     } else {
       if (!op_registry_.count(op)) {
@@ -1756,7 +1757,7 @@ tensorflow::Status ConvertBinary(Converter& ctx,
   } else {
 #else
   }
-  if (inputs.at(0).is_tensor() && inputs.at(1).is_tensor() || !status.ok()) {
+  if ((inputs.at(0).is_tensor() && inputs.at(1).is_tensor()) || !status.ok()) {
 #endif
     status = BinaryTensorOpTensor(ctx, node_def, inputs.at(0), inputs.at(1),
                                   outputs);
@@ -2371,10 +2372,7 @@ tensorflow::Status ConvertMatMul(Converter& ctx,
                                                node_def.name());
   }
 
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
-
   TFAttrs attrs(node_def);
-
   // TODO(jie): INT32 should be converted?
   tensorflow::DataType tf_dtype = attrs.get("T");
   if (tf_dtype != tensorflow::DataType::DT_FLOAT &&
@@ -2383,12 +2381,9 @@ tensorflow::Status ConvertMatMul(Converter& ctx,
         "data type is not supported, for node " + node_def.name() + " got " +
         tensorflow::DataTypeString(tf_dtype));
   }
-
   bool transpose_a = attrs.get("transpose_a");
   bool transpose_b = attrs.get("transpose_b");
 
-  nvinfer1::ITensor* output_tensor;
-
   // FullyConnected:
   if (transpose_a) {
     return tensorflow::errors::Internal(
diff --git a/tensorflow/contrib/tensorrt/convert/utils.cc b/tensorflow/contrib/tensorrt/convert/utils.cc
new file mode 100644
index 0000000000..24591cf84b
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/convert/utils.cc
@@ -0,0 +1,35 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/tensorrt/convert/utils.h"
+
+namespace tensorflow {
+namespace tensorrt {
+
+bool IsGoogleTensorRTEnabled() {
+  // TODO(laigd): consider also checking if tensorrt shared libraries are
+  // accessible. We can then direct users to this function to make sure they can
+  // safely write code that uses tensorrt conditionally. E.g. if it does not
+  // check for for tensorrt, and user mistakenly uses tensorrt, they will just
+  // crash and burn.
+#ifdef GOOGLE_TENSORRT
+  return true;
+#else
+  return false;
+#endif
+}
+
+}  // namespace tensorrt
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/tensorrt/convert/utils.h b/tensorflow/contrib/tensorrt/convert/utils.h
index f601c06701..8b5f4d614a 100644
--- a/tensorflow/contrib/tensorrt/convert/utils.h
+++ b/tensorflow/contrib/tensorrt/convert/utils.h
@@ -31,6 +31,8 @@ struct TrtDestroyer {
 template 
 using TrtUniquePtrType = std::unique_ptr>;
 
+bool IsGoogleTensorRTEnabled();
+
 }  // namespace tensorrt
 }  // namespace tensorflow
 
diff --git a/tensorflow/contrib/tensorrt/python/__init__.py b/tensorflow/contrib/tensorrt/python/__init__.py
index 0b2321b5fc..fe4fa166a1 100644
--- a/tensorflow/contrib/tensorrt/python/__init__.py
+++ b/tensorflow/contrib/tensorrt/python/__init__.py
@@ -22,4 +22,5 @@ from __future__ import print_function
 from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
 from tensorflow.contrib.tensorrt.python.trt_convert import calib_graph_to_infer_graph
 from tensorflow.contrib.tensorrt.python.trt_convert import create_inference_graph
+from tensorflow.contrib.tensorrt.python.trt_convert import is_tensorrt_enabled
 # pylint: enable=unused-import,line-too-long
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 79f512dbcf..2b67931661 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -23,6 +23,7 @@ import six as _six
 from tensorflow.contrib.tensorrt.wrap_conversion import calib_convert
 from tensorflow.contrib.tensorrt.wrap_conversion import get_linked_tensorrt_version
 from tensorflow.contrib.tensorrt.wrap_conversion import get_loaded_tensorrt_version
+from tensorflow.contrib.tensorrt.wrap_conversion import is_tensorrt_enabled
 from tensorflow.contrib.tensorrt.wrap_conversion import trt_convert
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.h b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
index c5d2cec730..97ac82ca5d 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
@@ -51,6 +51,9 @@ class TRTDeviceAllocator : public nvinfer1::IGpuAllocator {
   // Allocator implementation wrapping TF device allocators.
  public:
   TRTDeviceAllocator(tensorflow::Allocator* allocator);
+
+  // TODO(aaroey): base class doesn't have a virtual destructor, work with
+  // Nvidia to fix it.
   virtual ~TRTDeviceAllocator() {
     VLOG(1) << "Destroying allocator attached to " << allocator_->Name();
   }
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py
index 3c68c6e4e9..7c3ef498c9 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py
@@ -347,6 +347,7 @@ def GetTests():
 
 
 if __name__ == "__main__":
-  for index, t in enumerate(GetTests()):
-    setattr(TfTrtIntegrationTest, "testTfTRT_" + str(index), t)
+  if trt.is_tensorrt_enabled():
+    for index, t in enumerate(GetTests()):
+      setattr(TfTrtIntegrationTest, "testTfTRT_" + str(index), t)
   test.main()
diff --git a/tensorflow/contrib/tensorrt/trt_conversion.i b/tensorflow/contrib/tensorrt/trt_conversion.i
index d6628cd1eb..422740fdf6 100644
--- a/tensorflow/contrib/tensorrt/trt_conversion.i
+++ b/tensorflow/contrib/tensorrt/trt_conversion.i
@@ -100,6 +100,7 @@ _LIST_OUTPUT_TYPEMAP(int, PyLong_FromLong);
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/util/stat_summarizer.h"
 #include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
+#include "tensorflow/contrib/tensorrt/convert/utils.h"
 %}
 
 %ignoreall
@@ -108,6 +109,7 @@ _LIST_OUTPUT_TYPEMAP(int, PyLong_FromLong);
 %unignore calib_convert;
 %unignore get_linked_tensorrt_version;
 %unignore get_loaded_tensorrt_version;
+%unignore is_tensorrt_enabled;
 
 %{
 
@@ -140,7 +142,7 @@ std::pair trt_convert(
     return std::pair{out_status, ""};
   }
 
-  if(precision_mode < 0 || precision_mode > 2){
+  if (precision_mode < 0 || precision_mode > 2) {
     out_status = "InvalidArgument;Invalid precision_mode";
     return std::pair{out_status, ""};
   }
@@ -232,7 +234,8 @@ version_struct get_linked_tensorrt_version() {
 #endif  // GOOGLE_CUDA && GOOGLE_TENSORRT
   return s;
 }
-version_struct get_loaded_tensorrt_version(){
+
+version_struct get_loaded_tensorrt_version() {
   // Return the version from the loaded library.
   version_struct s;
 #if GOOGLE_CUDA && GOOGLE_TENSORRT
@@ -244,6 +247,10 @@ version_struct get_loaded_tensorrt_version(){
   return s;
 }
 
+bool is_tensorrt_enabled() {
+  return tensorflow::tensorrt::IsGoogleTensorRTEnabled();
+}
+
 %}
 
 std::pair calib_convert(string graph_def_string, bool is_dyn_op);
@@ -258,5 +265,6 @@ std::pair trt_convert(string graph_def_string,
                                       std::vector cached_engine_batches);
 version_struct get_linked_tensorrt_version();
 version_struct get_loaded_tensorrt_version();
+bool is_tensorrt_enabled();
 
 %unignoreall
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 6d876b786a..e661fb1adc 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -11,7 +11,6 @@ load(
 )
 load("//third_party/mkl:build_defs.bzl", "if_mkl")
 load("//tensorflow:tensorflow.bzl", "if_cuda")
-load("@local_config_tensorrt//:build_defs.bzl", "if_tensorrt")
 load("@local_config_syslibs//:build_defs.bzl", "if_not_system_lib")
 load("//tensorflow/core:platform/default/build_config_root.bzl", "tf_additional_license_deps")
 
@@ -190,9 +189,7 @@ sh_binary(
             "//tensorflow/contrib/lite/python:tflite_convert",
             "//tensorflow/contrib/lite/toco/python:toco_from_protos",
         ],
-    }) + if_mkl(["//third_party/mkl:intel_binary_blob"]) + if_tensorrt([
-        "//tensorflow/contrib/tensorrt:init_py",
-    ]),
+    }) + if_mkl(["//third_party/mkl:intel_binary_blob"]),
 )
 
 # A genrule for generating a marker file for the pip package on Windows
-- 
cgit v1.2.3


From 75783525c9061569c81045085800988a95615290 Mon Sep 17 00:00:00 2001
From: Koan-Sin Tan 
Date: Tue, 17 Jul 2018 17:13:11 +0800
Subject: make TFLite kernel tests work again

pow_test.cc doesn't build with
```
bazel test -c opt //tensorflow/contrib/lite/kernels:all
```

s/int32/int32_t/ to make it build and run
---
 tensorflow/contrib/lite/kernels/pow_test.cc | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/pow_test.cc b/tensorflow/contrib/lite/kernels/pow_test.cc
index 474d323bc3..74b3aef5bd 100644
--- a/tensorflow/contrib/lite/kernels/pow_test.cc
+++ b/tensorflow/contrib/lite/kernels/pow_test.cc
@@ -50,22 +50,22 @@ class PowOpModel : public SingleOpModel {
 };
 
 TEST(PowOpModel, Simple) {
-  PowOpModel model({TensorType_INT32, {1, 2, 2, 1}},
-                          {TensorType_INT32, {1, 2, 2, 1}},
-                          {TensorType_INT32, {}});
-  model.PopulateTensor(model.input1(), {12, 2, 7, 8});
-  model.PopulateTensor(model.input2(), {1, 2, 3, 1});
+  PowOpModel model({TensorType_INT32, {1, 2, 2, 1}},
+                            {TensorType_INT32, {1, 2, 2, 1}},
+                            {TensorType_INT32, {}});
+  model.PopulateTensor(model.input1(), {12, 2, 7, 8});
+  model.PopulateTensor(model.input2(), {1, 2, 3, 1});
   model.Invoke();
   EXPECT_THAT(model.GetOutputShape(), ElementsAre(1, 2, 2, 1));
   EXPECT_THAT(model.GetOutput(), ElementsAre(12, 4, 343, 8));
 }
 
 TEST(PowOpModel, NegativeAndZeroValue) {
-  PowOpModel model({TensorType_INT32, {1, 2, 2, 1}},
-                          {TensorType_INT32, {1, 2, 2, 1}},
-                          {TensorType_INT32, {}});
-  model.PopulateTensor(model.input1(), {0, 2, -7, 8});
-  model.PopulateTensor(model.input2(), {1, 2, 3, 0});
+  PowOpModel model({TensorType_INT32, {1, 2, 2, 1}},
+                            {TensorType_INT32, {1, 2, 2, 1}},
+                            {TensorType_INT32, {}});
+  model.PopulateTensor(model.input1(), {0, 2, -7, 8});
+  model.PopulateTensor(model.input2(), {1, 2, 3, 0});
   model.Invoke();
   EXPECT_THAT(model.GetOutputShape(), ElementsAre(1, 2, 2, 1));
   EXPECT_THAT(model.GetOutput(), ElementsAre(0, 4, -343, 1));
@@ -98,10 +98,10 @@ TEST(PowOpModel, NegativeFloatTest) {
 }
 
 TEST(PowOpModel, BroadcastTest) {
-  PowOpModel model({TensorType_INT32, {1, 2, 2, 1}},
-                          {TensorType_INT32, {1}}, {TensorType_INT32, {}});
-  model.PopulateTensor(model.input1(), {12, 2, 7, 8});
-  model.PopulateTensor(model.input2(), {4});
+  PowOpModel model({TensorType_INT32, {1, 2, 2, 1}},
+                            {TensorType_INT32, {1}}, {TensorType_INT32, {}});
+  model.PopulateTensor(model.input1(), {12, 2, 7, 8});
+  model.PopulateTensor(model.input2(), {4});
   model.Invoke();
   EXPECT_THAT(model.GetOutputShape(), ElementsAre(1, 2, 2, 1));
   EXPECT_THAT(model.GetOutput(), ElementsAre(20736, 16, 2401, 4096));
-- 
cgit v1.2.3


From 5601154e98e693d217b12808ddb60871772cea9c Mon Sep 17 00:00:00 2001
From: Pratik Kalshetti 
Date: Tue, 17 Jul 2018 16:24:55 +0530
Subject: correct url

correct broken url for 1) Eager Execution
---
 tensorflow/docs_src/tutorials/eager/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/tutorials/eager/index.md b/tensorflow/docs_src/tutorials/eager/index.md
index 5445e0c343..a13b396094 100644
--- a/tensorflow/docs_src/tutorials/eager/index.md
+++ b/tensorflow/docs_src/tutorials/eager/index.md
@@ -5,7 +5,7 @@ operations. Write custom layers, forward passes, and training loops with
 auto differentiation. Start with these notebooks, then read the
 [eager execution guide](../../guide/eager).
 
-1. [Eager execution](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/eager_intro.ipynb){:.external}
+1. [Eager execution](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/eager_basics.ipynb){:.external}
 2. [Automatic differentiation and gradient tape](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/automatic_differentiation.ipynb){:.external}
 3. [Custom training: basics](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb){:.external}
 4. [Custom layers](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/notebooks/custom_layers.ipynb){:.external}
-- 
cgit v1.2.3


From 2bdc5f12a1e66fd851e2621889c743c3b8da65e5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 05:09:36 -0700
Subject: Fix file patterns checked during stack traversal to be
 os-independent.

PiperOrigin-RevId: 204895256
---
 tensorflow/python/framework/error_interpolation.py      |  6 +++++-
 tensorflow/python/framework/error_interpolation_test.py | 17 +++++++++++------
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/framework/error_interpolation.py b/tensorflow/python/framework/error_interpolation.py
index 519e0fda0a..72d5dc99a8 100644
--- a/tensorflow/python/framework/error_interpolation.py
+++ b/tensorflow/python/framework/error_interpolation.py
@@ -24,6 +24,7 @@ from __future__ import print_function
 
 import collections
 import itertools
+import os
 import re
 import string
 
@@ -41,7 +42,10 @@ _INTERPOLATION_PATTERN = re.compile(_INTERPOLATION_REGEX)
 
 _ParseTag = collections.namedtuple("_ParseTag", ["type", "name", "format"])
 
-_BAD_FILE_SUBSTRINGS = ["tensorflow/python", "
Date: Tue, 17 Jul 2018 07:47:27 -0700
Subject: Makes Variable an abstract base class with a factory-constructing
 metaclass.

The metaclass, when asked to make instances of Variable, will loop over a
scope of creator functions calling them, before finally bottoming out on
a function which constructs a subclass of Variable, which goes through
the normal route.

This allows us to split Variable from RefVariable and to, in a follow-up CL,
add a use_resource argument to the Variable constructor so it can also return
ResourceVariable.

This is the minimal change to get things working; it does not expose the
creator stack or add the use_resource argument.

PiperOrigin-RevId: 204910138
---
 .../contrib/framework/python/ops/variables_test.py |  10 +-
 tensorflow/python/framework/ops.py                 |  10 +-
 tensorflow/python/framework/subscribe.py           |   2 +-
 tensorflow/python/ops/resource_variable_ops.py     |   3 +-
 tensorflow/python/ops/variables.py                 | 826 ++++++++++++++++-----
 tensorflow/tools/api/golden/tensorflow.pbtxt       |   2 +-
 6 files changed, 670 insertions(+), 183 deletions(-)

diff --git a/tensorflow/contrib/framework/python/ops/variables_test.py b/tensorflow/contrib/framework/python/ops/variables_test.py
index 7e0c7dbec1..3c44630a51 100644
--- a/tensorflow/contrib/framework/python/ops/variables_test.py
+++ b/tensorflow/contrib/framework/python/ops/variables_test.py
@@ -106,8 +106,9 @@ class LocalVariableTest(test.TestCase):
   def testResourceVariable(self):
     a = variables_lib2.local_variable(0)
     b = variables_lib2.local_variable(0, use_resource=True)
-    self.assertEqual(type(a), variables_lib.Variable)
-    self.assertEqual(type(b), resource_variable_ops.ResourceVariable)
+    self.assertTrue(isinstance(a, variables_lib.Variable))
+    self.assertFalse(isinstance(a, resource_variable_ops.ResourceVariable))
+    self.assertTrue(isinstance(b, resource_variable_ops.ResourceVariable))
 
 
 class GlobalVariableTest(test.TestCase):
@@ -176,8 +177,9 @@ class GlobalVariableTest(test.TestCase):
   def testResourceVariable(self):
     a = variables_lib2.global_variable(0)
     b = variables_lib2.global_variable(0, use_resource=True)
-    self.assertEqual(type(a), variables_lib.Variable)
-    self.assertEqual(type(b), resource_variable_ops.ResourceVariable)
+    self.assertTrue(isinstance(a, variables_lib.Variable))
+    self.assertFalse(isinstance(a, resource_variable_ops.ResourceVariable))
+    self.assertTrue(isinstance(b, resource_variable_ops.ResourceVariable))
 
 
 class GlobalStepTest(test.TestCase):
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index c4f58f0847..a3b56b0f63 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -2772,23 +2772,23 @@ class Graph(object):
     # This step makes a copy of the existing stack, and it also initializes
     # self._thread_local._variable_creator_stack if it doesn't exist yet.
     old = list(self._variable_creator_stack)
-    self._thread_local._variable_creator_stack.append(creator)
+    self._thread_local._variable_creator_stack.append(creator)  # pylint: disable=protected-access
     try:
       yield
     finally:
-      self._thread_local._variable_creator_stack = old
+      self._thread_local._variable_creator_stack = old  # pylint: disable=protected-access
 
   # Note: this method is private because the API of tf.Graph() is public and
   # frozen, and this functionality is still not ready for public visibility.
   @property
   def _variable_creator_stack(self):
     if not hasattr(self._thread_local, "_variable_creator_stack"):
-      self._thread_local._variable_creator_stack = []
-    return list(self._thread_local._variable_creator_stack)
+      self._thread_local._variable_creator_stack = []  # pylint: disable=protected-access
+    return list(self._thread_local._variable_creator_stack)  # pylint: disable=protected-access
 
   @_variable_creator_stack.setter
   def _variable_creator_stack(self, variable_creator_stack):
-    self._thread_local._variable_creator_stack = variable_creator_stack
+    self._thread_local._variable_creator_stack = variable_creator_stack  # pylint: disable=protected-access
 
   def _check_not_finalized(self):
     """Check if the graph is finalized.
diff --git a/tensorflow/python/framework/subscribe.py b/tensorflow/python/framework/subscribe.py
index 7797d991da..cee7398974 100644
--- a/tensorflow/python/framework/subscribe.py
+++ b/tensorflow/python/framework/subscribe.py
@@ -47,7 +47,7 @@ def _recursive_apply(tensors, apply_fn):
   tensors_type = type(tensors)
   if tensors_type is ops.Tensor:
     return apply_fn(tensors)
-  elif tensors_type is variables.Variable:
+  elif isinstance(tensors, variables.Variable):
     return apply_fn(tensors.value())
   elif isinstance(tensors, (list, tuple)):
     tensors = [_recursive_apply(t, apply_fn) for t in tensors]
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 70a89e5ebb..1f56ad25bf 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -181,7 +181,8 @@ def shape_safe_assign_variable_handle(handle, shape, value, name=None):
                                                       name=name)
 
 
-class ResourceVariable(variables.Variable):
+# TODO(apassos) make this be variables.Variable
+class ResourceVariable(variables.RefVariable):
   """Variable based on resource handles.
 
   See the @{$variables$Variables How To} for a high level overview.
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index d3b8da6d2a..87e0de197c 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -17,6 +17,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import six
+
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.framework import variable_pb2
 from tensorflow.python.eager import context
@@ -36,8 +38,32 @@ from tensorflow.python.util.deprecation import deprecated
 from tensorflow.python.util.tf_export import tf_export
 
 
+def _default_variable_creator(_, *args, **kwds):
+  return RefVariable(*args, **kwds)
+
+
+def _make_getter(captured_getter, captured_previous):
+  """To avoid capturing loop variables."""
+  def getter(*args, **kwargs):
+    return captured_getter(captured_previous, *args, **kwargs)
+  return getter
+
+
+class VariableMetaclass(type):
+  """Metaclass to allow construction of tf.Variable to be overridden."""
+
+  def __call__(cls, *args, **kwargs):
+    if cls is Variable:
+      previous_getter = lambda *a, **k: _default_variable_creator(None, *a, **k)
+      # TODO(apassos) use a stack of getters here
+      return previous_getter(*args, **kwargs)
+    else:
+      return super(VariableMetaclass, cls).__call__(*args, **kwargs)
+
+
 @tf_export("Variable")
-class Variable(checkpointable.CheckpointableBase):
+class Variable(six.with_metaclass(VariableMetaclass,
+                                  checkpointable.CheckpointableBase)):
   """See the @{$variables$Variables How To} for a high level overview.
 
   A variable maintains state in the graph across calls to `run()`. You add a
@@ -57,104 +83,652 @@ class Variable(checkpointable.CheckpointableBase):
   overloaded for the `Tensor` class are carried over to variables, so you can
   also add nodes to the graph by just doing arithmetic on variables.
 
-  ```python
-  import tensorflow as tf
+  ```python
+  import tensorflow as tf
+
+  # Create a variable.
+  w = tf.Variable(, name=)
+
+  # Use the variable in the graph like any Tensor.
+  y = tf.matmul(w, ...another variable or tensor...)
+
+  # The overloaded operators are available too.
+  z = tf.sigmoid(w + y)
+
+  # Assign a new value to the variable with `assign()` or a related method.
+  w.assign(w + 1.0)
+  w.assign_add(1.0)
+  ```
+
+  When you launch the graph, variables have to be explicitly initialized before
+  you can run Ops that use their value. You can initialize a variable by
+  running its *initializer op*, restoring the variable from a save file, or
+  simply running an `assign` Op that assigns a value to the variable. In fact,
+  the variable *initializer op* is just an `assign` Op that assigns the
+  variable's initial value to the variable itself.
+
+  ```python
+  # Launch the graph in a session.
+  with tf.Session() as sess:
+      # Run the variable initializer.
+      sess.run(w.initializer)
+      # ...you now can run ops that use the value of 'w'...
+  ```
+
+  The most common initialization pattern is to use the convenience function
+  `global_variables_initializer()` to add an Op to the graph that initializes
+  all the variables. You then run that Op after launching the graph.
+
+  ```python
+  # Add an Op to initialize global variables.
+  init_op = tf.global_variables_initializer()
+
+  # Launch the graph in a session.
+  with tf.Session() as sess:
+      # Run the Op that initializes global variables.
+      sess.run(init_op)
+      # ...you can now run any Op that uses variable values...
+  ```
+
+  If you need to create a variable with an initial value dependent on another
+  variable, use the other variable's `initialized_value()`. This ensures that
+  variables are initialized in the right order.
+
+  All variables are automatically collected in the graph where they are
+  created. By default, the constructor adds the new variable to the graph
+  collection `GraphKeys.GLOBAL_VARIABLES`. The convenience function
+  `global_variables()` returns the contents of that collection.
+
+  When building a machine learning model it is often convenient to distinguish
+  between variables holding the trainable model parameters and other variables
+  such as a `global step` variable used to count training steps. To make this
+  easier, the variable constructor supports a `trainable=` parameter. If
+  `True`, the new variable is also added to the graph collection
+  `GraphKeys.TRAINABLE_VARIABLES`. The convenience function
+  `trainable_variables()` returns the contents of this collection. The
+  various `Optimizer` classes use this collection as the default list of
+  variables to optimize.
+
+  WARNING: tf.Variable objects have a non-intuitive memory model. A Variable is
+  represented internally as a mutable Tensor which can non-deterministically
+  alias other Tensors in a graph. The set of operations which consume a Variable
+  and can lead to aliasing is undetermined and can change across TensorFlow
+  versions. Avoid writing code which relies on the value of a Variable either
+  changing or not changing as other operations happen. For example, using
+  Variable objects or simple functions thereof as predicates in a `tf.cond` is
+  dangerous and error-prone:
+
+  ```
+  v = tf.Variable(True)
+  tf.cond(v, lambda: v.assign(False), my_false_fn)  # Note: this is broken.
+  ```
+
+  Here replacing tf.Variable with tf.contrib.eager.Variable will fix any
+  nondeterminism issues.
+
+  To use the replacement for variables which does
+  not have these issues:
+
+  * Replace `tf.Variable` with `tf.contrib.eager.Variable`;
+  * Call `tf.get_variable_scope().set_use_resource(True)` inside a
+    `tf.variable_scope` before the `tf.get_variable()` call.
+
+  @compatibility(eager)
+  `tf.Variable` is not compatible with eager execution.  Use
+  `tf.contrib.eager.Variable` instead which is compatible with both eager
+  execution and graph construction.  See [the TensorFlow Eager Execution
+  guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
+  for details on how variables work in eager execution.
+  @end_compatibility
+  """
+
+  def __init__(self,
+               initial_value=None,
+               trainable=True,
+               collections=None,
+               validate_shape=True,
+               caching_device=None,
+               name=None,
+               variable_def=None,
+               dtype=None,
+               expected_shape=None,
+               import_scope=None,
+               constraint=None):
+    """Creates a new variable with value `initial_value`.
+
+    The new variable is added to the graph collections listed in `collections`,
+    which defaults to `[GraphKeys.GLOBAL_VARIABLES]`.
+
+    If `trainable` is `True` the variable is also added to the graph collection
+    `GraphKeys.TRAINABLE_VARIABLES`.
+
+    This constructor creates both a `variable` Op and an `assign` Op to set the
+    variable to its initial value.
+
+    Args:
+      initial_value: A `Tensor`, or Python object convertible to a `Tensor`,
+        which is the initial value for the Variable. The initial value must have
+        a shape specified unless `validate_shape` is set to False. Can also be a
+        callable with no argument that returns the initial value when called. In
+        that case, `dtype` must be specified. (Note that initializer functions
+        from init_ops.py must first be bound to a shape before being used here.)
+      trainable: If `True`, the default, also adds the variable to the graph
+        collection `GraphKeys.TRAINABLE_VARIABLES`. This collection is used as
+        the default list of variables to use by the `Optimizer` classes.
+      collections: List of graph collections keys. The new variable is added to
+        these collections. Defaults to `[GraphKeys.GLOBAL_VARIABLES]`.
+      validate_shape: If `False`, allows the variable to be initialized with a
+        value of unknown shape. If `True`, the default, the shape of
+        `initial_value` must be known.
+      caching_device: Optional device string describing where the Variable
+        should be cached for reading.  Defaults to the Variable's device.
+        If not `None`, caches on another device.  Typical use is to cache
+        on the device where the Ops using the Variable reside, to deduplicate
+        copying through `Switch` and other conditional statements.
+      name: Optional name for the variable. Defaults to `'Variable'` and gets
+        uniquified automatically.
+      variable_def: `VariableDef` protocol buffer. If not `None`, recreates
+        the Variable object with its contents, referencing the variable's nodes
+        in the graph, which must already exist. The graph is not changed.
+        `variable_def` and the other arguments are mutually exclusive.
+      dtype: If set, initial_value will be converted to the given type.
+        If `None`, either the datatype will be kept (if `initial_value` is
+        a Tensor), or `convert_to_tensor` will decide.
+      expected_shape: A TensorShape. If set, initial_value is expected
+        to have this shape.
+      import_scope: Optional `string`. Name scope to add to the
+        `Variable.` Only used when initializing from protocol buffer.
+      constraint: An optional projection function to be applied to the variable
+        after being updated by an `Optimizer` (e.g. used to implement norm
+        constraints or value constraints for layer weights). The function must
+        take as input the unprojected Tensor representing the value of the
+        variable and return the Tensor for the projected value
+        (which must have the same shape). Constraints are not safe to
+        use when doing asynchronous distributed training.
+
+    Raises:
+      ValueError: If both `variable_def` and initial_value are specified.
+      ValueError: If the initial value is not specified, or does not have a
+        shape and `validate_shape` is `True`.
+      RuntimeError: If eager execution is enabled.
+
+    @compatibility(eager)
+    `tf.Variable` is not compatible with eager execution.  Use
+    `tfe.Variable` instead which is compatible with both eager execution
+    and graph construction.  See [the TensorFlow Eager Execution
+    guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
+    for details on how variables work in eager execution.
+    @end_compatibility
+    """
+    raise NotImplementedError
+
+  def __repr__(self):
+    raise NotImplementedError
+
+  def value(self):
+    """Returns the last snapshot of this variable.
+
+    You usually do not need to call this method as all ops that need the value
+    of the variable call it automatically through a `convert_to_tensor()` call.
+
+    Returns a `Tensor` which holds the value of the variable.  You can not
+    assign a new value to this tensor as it is not a reference to the variable.
+
+    To avoid copies, if the consumer of the returned value is on the same device
+    as the variable, this actually returns the live value of the variable, not
+    a copy.  Updates to the variable are seen by the consumer.  If the consumer
+    is on a different device it will get a copy of the variable.
+
+    Returns:
+      A `Tensor` containing the value of the variable.
+    """
+    raise NotImplementedError
+
+  def read_value(self):
+    """Returns the value of this variable, read in the current context.
+
+    Can be different from value() if it's on another device, with control
+    dependencies, etc.
+
+    Returns:
+      A `Tensor` containing the value of the variable.
+    """
+    raise NotImplementedError
+
+  def set_shape(self, shape):
+    """Overrides the shape for this variable.
+
+    Args:
+      shape: the `TensorShape` representing the overridden shape.
+    """
+    raise NotImplementedError
+
+  @property
+  def trainable(self):
+    raise NotImplementedError
+
+  def eval(self, session=None):
+    """In a session, computes and returns the value of this variable.
+
+    This is not a graph construction method, it does not add ops to the graph.
+
+    This convenience method requires a session where the graph
+    containing this variable has been launched. If no session is
+    passed, the default session is used.  See @{tf.Session} for more
+    information on launching a graph and on sessions.
+
+    ```python
+    v = tf.Variable([1, 2])
+    init = tf.global_variables_initializer()
+
+    with tf.Session() as sess:
+        sess.run(init)
+        # Usage passing the session explicitly.
+        print(v.eval(sess))
+        # Usage with the default session.  The 'with' block
+        # above makes 'sess' the default session.
+        print(v.eval())
+    ```
+
+    Args:
+      session: The session to use to evaluate this variable. If
+        none, the default session is used.
+
+    Returns:
+      A numpy `ndarray` with a copy of the value of this variable.
+    """
+    raise NotImplementedError
+
+  def initialized_value(self):
+    """Returns the value of the initialized variable.
+
+    You should use this instead of the variable itself to initialize another
+    variable with a value that depends on the value of this variable.
+
+    ```python
+    # Initialize 'v' with a random tensor.
+    v = tf.Variable(tf.truncated_normal([10, 40]))
+    # Use `initialized_value` to guarantee that `v` has been
+    # initialized before its value is used to initialize `w`.
+    # The random values are picked only once.
+    w = tf.Variable(v.initialized_value() * 2.0)
+    ```
+
+    Returns:
+      A `Tensor` holding the value of this variable after its initializer
+      has run.
+    """
+    raise NotImplementedError
+
+  @property
+  def initial_value(self):
+    """Returns the Tensor used as the initial value for the variable.
+
+    Note that this is different from `initialized_value()` which runs
+    the op that initializes the variable before returning its value.
+    This method returns the tensor that is used by the op that initializes
+    the variable.
+
+    Returns:
+      A `Tensor`.
+    """
+    raise NotImplementedError
+
+  @property
+  def constraint(self):
+    """Returns the constraint function associated with this variable.
+
+    Returns:
+      The constraint function that was passed to the variable constructor.
+      Can be `None` if no constraint was passed.
+    """
+    raise NotImplementedError
+
+  def assign(self, value, use_locking=False):
+    """Assigns a new value to the variable.
+
+    This is essentially a shortcut for `assign(self, value)`.
+
+    Args:
+      value: A `Tensor`. The new value for this variable.
+      use_locking: If `True`, use locking during the assignment.
+
+    Returns:
+      A `Tensor` that will hold the new value of this variable after
+      the assignment has completed.
+    """
+    raise NotImplementedError
+
+  def assign_add(self, delta, use_locking=False):
+    """Adds a value to this variable.
+
+     This is essentially a shortcut for `assign_add(self, delta)`.
+
+    Args:
+      delta: A `Tensor`. The value to add to this variable.
+      use_locking: If `True`, use locking during the operation.
+
+    Returns:
+      A `Tensor` that will hold the new value of this variable after
+      the addition has completed.
+    """
+    raise NotImplementedError
+
+  def assign_sub(self, delta, use_locking=False):
+    """Subtracts a value from this variable.
+
+    This is essentially a shortcut for `assign_sub(self, delta)`.
+
+    Args:
+      delta: A `Tensor`. The value to subtract from this variable.
+      use_locking: If `True`, use locking during the operation.
+
+    Returns:
+      A `Tensor` that will hold the new value of this variable after
+      the subtraction has completed.
+    """
+    raise NotImplementedError
+
+  def scatter_sub(self, sparse_delta, use_locking=False):
+    """Subtracts `IndexedSlices` from this variable.
+
+    This is essentially a shortcut for `scatter_sub(self, sparse_delta.indices,
+    sparse_delta.values)`.
+
+    Args:
+      sparse_delta: `IndexedSlices` to be subtracted from this variable.
+      use_locking: If `True`, use locking during the operation.
+
+    Returns:
+      A `Tensor` that will hold the new value of this variable after
+      the scattered subtraction has completed.
+
+    Raises:
+      ValueError: if `sparse_delta` is not an `IndexedSlices`.
+    """
+    raise NotImplementedError
+
+  def count_up_to(self, limit):
+    """Increments this variable until it reaches `limit`.
+
+    When that Op is run it tries to increment the variable by `1`. If
+    incrementing the variable would bring it above `limit` then the Op raises
+    the exception `OutOfRangeError`.
+
+    If no error is raised, the Op outputs the value of the variable before
+    the increment.
+
+    This is essentially a shortcut for `count_up_to(self, limit)`.
+
+    Args:
+      limit: value at which incrementing the variable raises an error.
+
+    Returns:
+      A `Tensor` that will hold the variable value before the increment. If no
+      other Op modifies this variable, the values produced will all be
+      distinct.
+    """
+    raise NotImplementedError
+
+  def load(self, value, session=None):
+    """Load new value into this variable.
+
+    Writes new value to variable's memory. Doesn't add ops to the graph.
+
+    This convenience method requires a session where the graph
+    containing this variable has been launched. If no session is
+    passed, the default session is used.  See @{tf.Session} for more
+    information on launching a graph and on sessions.
+
+    ```python
+    v = tf.Variable([1, 2])
+    init = tf.global_variables_initializer()
+
+    with tf.Session() as sess:
+        sess.run(init)
+        # Usage passing the session explicitly.
+        v.load([2, 3], sess)
+        print(v.eval(sess)) # prints [2 3]
+        # Usage with the default session.  The 'with' block
+        # above makes 'sess' the default session.
+        v.load([3, 4], sess)
+        print(v.eval()) # prints [3 4]
+    ```
+
+    Args:
+        value: New variable value
+        session: The session to use to evaluate this variable. If
+          none, the default session is used.
+
+    Raises:
+        ValueError: Session is not passed and no default session
+    """
+    raise NotImplementedError
+
+  # Conversion to tensor.
+  @staticmethod
+  def _TensorConversionFunction(v, dtype=None, name=None, as_ref=False):  # pylint: disable=invalid-name
+    """Utility function for converting a Variable to a Tensor."""
+    _ = name
+    if dtype and not dtype.is_compatible_with(v.dtype):
+      raise ValueError(
+          "Incompatible type conversion requested to type '%s' for variable "
+          "of type '%s'" % (dtype.name, v.dtype.name))
+    if as_ref:
+      return v._ref()  # pylint: disable=protected-access
+    else:
+      return v.value()
+
+  @staticmethod
+  def _OverloadAllOperators():  # pylint: disable=invalid-name
+    """Register overloads for all operators."""
+    for operator in ops.Tensor.OVERLOADABLE_OPERATORS:
+      Variable._OverloadOperator(operator)
+    # For slicing, bind getitem differently than a tensor (use SliceHelperVar
+    # instead)
+    # pylint: disable=protected-access
+    setattr(Variable, "__getitem__", array_ops._SliceHelperVar)
+
+  @staticmethod
+  def _OverloadOperator(operator):  # pylint: disable=invalid-name
+    """Defer an operator overload to `ops.Tensor`.
+
+    We pull the operator out of ops.Tensor dynamically to avoid ordering issues.
+
+    Args:
+      operator: string. The operator name.
+    """
+
+    def _run_op(a, *args):
+      # pylint: disable=protected-access
+      return getattr(ops.Tensor, operator)(a._AsTensor(), *args)
+    # Propagate __doc__ to wrapper
+    try:
+      _run_op.__doc__ = getattr(ops.Tensor, operator).__doc__
+    except AttributeError:
+      pass
+
+    setattr(Variable, operator, _run_op)
+
+  # NOTE(mrry): This enables the Variable's overloaded "right" binary
+  # operators to run when the left operand is an ndarray, because it
+  # accords the Variable class higher priority than an ndarray, or a
+  # numpy matrix.
+  # TODO(mrry): Convert this to using numpy's __numpy_ufunc__
+  # mechanism, which allows more control over how Variables interact
+  # with ndarrays.
+  __array_priority__ = 100
+
+  @property
+  def name(self):
+    """The name of this variable."""
+    raise NotImplementedError
+
+  @property
+  def initializer(self):
+    """The initializer operation for this variable."""
+    raise NotImplementedError
+
+  @property
+  def device(self):
+    """The device of this variable."""
+    raise NotImplementedError
+
+  @property
+  def dtype(self):
+    """The `DType` of this variable."""
+    raise NotImplementedError
+
+  @property
+  def op(self):
+    """The `Operation` of this variable."""
+    raise NotImplementedError
+
+  @property
+  def graph(self):
+    """The `Graph` of this variable."""
+    raise NotImplementedError
+
+  @property
+  def shape(self):
+    """The `TensorShape` of this variable.
+
+    Returns:
+      A `TensorShape`.
+    """
+    raise NotImplementedError
+
+  def get_shape(self):
+    """Alias of Variable.shape."""
+    raise NotImplementedError
+
+  def to_proto(self, export_scope=None):
+    """Converts a `Variable` to a `VariableDef` protocol buffer.
+
+    Args:
+      export_scope: Optional `string`. Name scope to remove.
+
+    Returns:
+      A `VariableDef` protocol buffer, or `None` if the `Variable` is not
+      in the specified name scope.
+    """
+    raise NotImplementedError
+
+  @staticmethod
+  def from_proto(variable_def, import_scope=None):
+    """Returns a `Variable` object created from `variable_def`."""
+    return Variable(variable_def=variable_def,
+                    import_scope=import_scope)
+
+  class SaveSliceInfo(object):
+    """Information on how to save this Variable as a slice.
 
-  # Create a variable.
-  w = tf.Variable(, name=)
+    Provides internal support for saving variables as slices of a larger
+    variable.  This API is not public and is subject to change.
 
-  # Use the variable in the graph like any Tensor.
-  y = tf.matmul(w, ...another variable or tensor...)
+    Available properties:
 
-  # The overloaded operators are available too.
-  z = tf.sigmoid(w + y)
+    * full_name
+    * full_shape
+    * var_offset
+    * var_shape
+    """
 
-  # Assign a new value to the variable with `assign()` or a related method.
-  w.assign(w + 1.0)
-  w.assign_add(1.0)
-  ```
+    def __init__(self,
+                 full_name=None,
+                 full_shape=None,
+                 var_offset=None,
+                 var_shape=None,
+                 save_slice_info_def=None,
+                 import_scope=None):
+      """Create a `SaveSliceInfo`.
 
-  When you launch the graph, variables have to be explicitly initialized before
-  you can run Ops that use their value. You can initialize a variable by
-  running its *initializer op*, restoring the variable from a save file, or
-  simply running an `assign` Op that assigns a value to the variable. In fact,
-  the variable *initializer op* is just an `assign` Op that assigns the
-  variable's initial value to the variable itself.
+      Args:
+        full_name: Name of the full variable of which this `Variable` is a
+            slice.
+        full_shape: Shape of the full variable, as a list of int.
+        var_offset: Offset of this `Variable` into the full variable, as a
+            list of int.
+        var_shape: Shape of this `Variable`, as a list of int.
+        save_slice_info_def: `SaveSliceInfoDef` protocol buffer. If not `None`,
+          recreates the SaveSliceInfo object its contents.
+          `save_slice_info_def` and other arguments are mutually
+          exclusive.
+        import_scope: Optional `string`. Name scope to add. Only used
+          when initializing from protocol buffer.
+      """
+      if save_slice_info_def:
+        assert isinstance(save_slice_info_def, variable_pb2.SaveSliceInfoDef)
+        self.full_name = ops.prepend_name_scope(
+            save_slice_info_def.full_name, import_scope=import_scope)
+        self.full_shape = [i for i in save_slice_info_def.full_shape]
+        self.var_offset = [i for i in save_slice_info_def.var_offset]
+        self.var_shape = [i for i in save_slice_info_def.var_shape]
+      else:
+        self.full_name = full_name
+        self.full_shape = full_shape
+        self.var_offset = var_offset
+        self.var_shape = var_shape
 
-  ```python
-  # Launch the graph in a session.
-  with tf.Session() as sess:
-      # Run the variable initializer.
-      sess.run(w.initializer)
-      # ...you now can run ops that use the value of 'w'...
-  ```
+    @property
+    def spec(self):
+      """Computes the spec string used for saving."""
+      full_shape_str = " ".join(["%d" % d for d in self.full_shape]) + " "
+      sl_spec = ":".join([
+          "%d,%d" % (o, s) for o, s in zip(self.var_offset, self.var_shape)
+      ])
+      return full_shape_str + sl_spec
 
-  The most common initialization pattern is to use the convenience function
-  `global_variables_initializer()` to add an Op to the graph that initializes
-  all the variables. You then run that Op after launching the graph.
+    def to_proto(self, export_scope=None):
+      """Returns a SaveSliceInfoDef() proto.
 
-  ```python
-  # Add an Op to initialize global variables.
-  init_op = tf.global_variables_initializer()
+      Args:
+        export_scope: Optional `string`. Name scope to remove.
 
-  # Launch the graph in a session.
-  with tf.Session() as sess:
-      # Run the Op that initializes global variables.
-      sess.run(init_op)
-      # ...you can now run any Op that uses variable values...
-  ```
+      Returns:
+        A `SaveSliceInfoDef` protocol buffer, or None if the `Variable` is not
+        in the specified name scope.
+      """
+      if (export_scope is None or
+          self.full_name.startswith(export_scope)):
+        save_slice_info_def = variable_pb2.SaveSliceInfoDef()
+        save_slice_info_def.full_name = ops.strip_name_scope(
+            self.full_name, export_scope)
+        for i in self.full_shape:
+          save_slice_info_def.full_shape.append(i)
+        for i in self.var_offset:
+          save_slice_info_def.var_offset.append(i)
+        for i in self.var_shape:
+          save_slice_info_def.var_shape.append(i)
+        return save_slice_info_def
+      else:
+        return None
 
-  If you need to create a variable with an initial value dependent on another
-  variable, use the other variable's `initialized_value()`. This ensures that
-  variables are initialized in the right order.
+  def __iadd__(self, other):
+    raise NotImplementedError
 
-  All variables are automatically collected in the graph where they are
-  created. By default, the constructor adds the new variable to the graph
-  collection `GraphKeys.GLOBAL_VARIABLES`. The convenience function
-  `global_variables()` returns the contents of that collection.
+  def __isub__(self, other):
+    raise NotImplementedError
 
-  When building a machine learning model it is often convenient to distinguish
-  between variables holding the trainable model parameters and other variables
-  such as a `global step` variable used to count training steps. To make this
-  easier, the variable constructor supports a `trainable=` parameter. If
-  `True`, the new variable is also added to the graph collection
-  `GraphKeys.TRAINABLE_VARIABLES`. The convenience function
-  `trainable_variables()` returns the contents of this collection. The
-  various `Optimizer` classes use this collection as the default list of
-  variables to optimize.
+  def __imul__(self, other):
+    raise NotImplementedError
 
-  WARNING: tf.Variable objects have a non-intuitive memory model. A Variable is
-  represented internally as a mutable Tensor which can non-deterministically
-  alias other Tensors in a graph. The set of operations which consume a Variable
-  and can lead to aliasing is undetermined and can change across TensorFlow
-  versions. Avoid writing code which relies on the value of a Variable either
-  changing or not changing as other operations happen. For example, using
-  Variable objects or simple functions thereof as predicates in a `tf.cond` is
-  dangerous and error-prone:
+  def __idiv__(self, other):
+    raise NotImplementedError
 
-  ```
-  v = tf.Variable(True)
-  tf.cond(v, lambda: v.assign(False), my_false_fn)  # Note: this is broken.
-  ```
+  def __itruediv__(self, other):
+    raise NotImplementedError
 
-  Here replacing tf.Variable with tf.contrib.eager.Variable will fix any
-  nondeterminism issues.
+  def __irealdiv__(self, other):
+    raise NotImplementedError
 
-  To use the replacement for variables which does
-  not have these issues:
+  def __ipow__(self, other):
+    raise NotImplementedError
 
-  * Replace `tf.Variable` with `tf.contrib.eager.Variable`;
-  * Call `tf.get_variable_scope().set_use_resource(True)` inside a
-    `tf.variable_scope` before the `tf.get_variable()` call.
 
-  @compatibility(eager)
-  `tf.Variable` is not compatible with eager execution.  Use
-  `tf.contrib.eager.Variable` instead which is compatible with both eager
-  execution and graph construction.  See [the TensorFlow Eager Execution
-  guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
-  for details on how variables work in eager execution.
-  @end_compatibility
-  """
+# TODO(apassos): do not repeat all comments here
+class RefVariable(Variable):
+  """Ref-based implementation of variables."""
 
   def __init__(self,
                initial_value=None,
@@ -1068,12 +1642,6 @@ class Variable(checkpointable.CheckpointableBase):
     else:
       return None
 
-  @staticmethod
-  def from_proto(variable_def, import_scope=None):
-    """Returns a `Variable` object created from `variable_def`."""
-    return Variable(variable_def=variable_def,
-                    import_scope=import_scope)
-
   def __iadd__(self, other):
     logging.log_first_n(
         logging.WARN,
@@ -1130,90 +1698,6 @@ class Variable(checkpointable.CheckpointableBase):
         " if you want a new python Tensor object.", 1)
     return self ** other
 
-  class SaveSliceInfo(object):
-    """Information on how to save this Variable as a slice.
-
-    Provides internal support for saving variables as slices of a larger
-    variable.  This API is not public and is subject to change.
-
-    Available properties:
-
-    * full_name
-    * full_shape
-    * var_offset
-    * var_shape
-    """
-
-    def __init__(self,
-                 full_name=None,
-                 full_shape=None,
-                 var_offset=None,
-                 var_shape=None,
-                 save_slice_info_def=None,
-                 import_scope=None):
-      """Create a `SaveSliceInfo`.
-
-      Args:
-        full_name: Name of the full variable of which this `Variable` is a
-            slice.
-        full_shape: Shape of the full variable, as a list of int.
-        var_offset: Offset of this `Variable` into the full variable, as a
-            list of int.
-        var_shape: Shape of this `Variable`, as a list of int.
-        save_slice_info_def: `SaveSliceInfoDef` protocol buffer. If not `None`,
-          recreates the SaveSliceInfo object its contents.
-          `save_slice_info_def` and other arguments are mutually
-          exclusive.
-        import_scope: Optional `string`. Name scope to add. Only used
-          when initializing from protocol buffer.
-      """
-      if save_slice_info_def:
-        assert isinstance(save_slice_info_def, variable_pb2.SaveSliceInfoDef)
-        self.full_name = ops.prepend_name_scope(
-            save_slice_info_def.full_name, import_scope=import_scope)
-        self.full_shape = [i for i in save_slice_info_def.full_shape]
-        self.var_offset = [i for i in save_slice_info_def.var_offset]
-        self.var_shape = [i for i in save_slice_info_def.var_shape]
-      else:
-        self.full_name = full_name
-        self.full_shape = full_shape
-        self.var_offset = var_offset
-        self.var_shape = var_shape
-
-    @property
-    def spec(self):
-      """Computes the spec string used for saving."""
-      full_shape_str = " ".join(["%d" % d for d in self.full_shape]) + " "
-      sl_spec = ":".join([
-          "%d,%d" % (o, s) for o, s in zip(self.var_offset, self.var_shape)
-      ])
-      return full_shape_str + sl_spec
-
-    def to_proto(self, export_scope=None):
-      """Returns a SaveSliceInfoDef() proto.
-
-      Args:
-        export_scope: Optional `string`. Name scope to remove.
-
-      Returns:
-        A `SaveSliceInfoDef` protocol buffer, or None if the `Variable` is not
-        in the specified name scope.
-      """
-      if (export_scope is None or
-          self.full_name.startswith(export_scope)):
-        save_slice_info_def = variable_pb2.SaveSliceInfoDef()
-        save_slice_info_def.full_name = ops.strip_name_scope(
-            self.full_name, export_scope)
-        for i in self.full_shape:
-          save_slice_info_def.full_shape.append(i)
-        for i in self.var_offset:
-          save_slice_info_def.var_offset.append(i)
-        for i in self.var_shape:
-          save_slice_info_def.var_shape.append(i)
-        return save_slice_info_def
-      else:
-        return None
-
   def _set_save_slice_info(self, save_slice_info):
     """Sets the slice info for this `Variable`.
 
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index 4f90743fec..5eb42b4db3 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -258,7 +258,7 @@ tf_module {
   }
   member {
     name: "Variable"
-    mtype: ""
+    mtype: ""
   }
   member {
     name: "VariableAggregation"
-- 
cgit v1.2.3


From 09d35773d849b7f6aed72adbb3bf7eb79648a49a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 07:49:24 -0700
Subject: Fix a bug in implementation of the len built in. Scalars were getting
 the wrong error.

PiperOrigin-RevId: 204910333
---
 tensorflow/contrib/autograph/utils/builtins.py      | 2 +-
 tensorflow/contrib/autograph/utils/builtins_test.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/autograph/utils/builtins.py b/tensorflow/contrib/autograph/utils/builtins.py
index 998087e056..71079cfdc0 100644
--- a/tensorflow/contrib/autograph/utils/builtins.py
+++ b/tensorflow/contrib/autograph/utils/builtins.py
@@ -52,7 +52,7 @@ def dynamic_len(list_or_tensor):
   """Implementation of len using dynamic dispatch."""
   if tensor_util.is_tensor(list_or_tensor):
     shape = list_or_tensor.shape
-    if not shape:
+    if not shape.ndims:
       raise ValueError(
           'len requires non-zero rank for tensor "%s"' % list_or_tensor)
     return array_ops.shape(list_or_tensor)[0]
diff --git a/tensorflow/contrib/autograph/utils/builtins_test.py b/tensorflow/contrib/autograph/utils/builtins_test.py
index 0c2312178a..b4821f36fc 100644
--- a/tensorflow/contrib/autograph/utils/builtins_test.py
+++ b/tensorflow/contrib/autograph/utils/builtins_test.py
@@ -33,7 +33,8 @@ class BuiltinsTest(test.TestCase):
   def test_dynamic_len_tf_scalar(self):
     a = constant_op.constant(1)
 
-    with self.assertRaises(ValueError):
+    with self.assertRaisesRegexp(ValueError,
+                                 'len requires non-zero rank for tensor.*'):
       with self.test_session() as sess:
         sess.run(builtins.dynamic_builtin(len, a))
 
-- 
cgit v1.2.3


From e02fbb25784498b44e73d9370da65a3f23f6de15 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Tue, 17 Jul 2018 08:02:04 -0700
Subject: Fix review comments and formatting issues.

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc |  4 ++--
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc | 17 +++++++++--------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 3b42a5ee96..8a0e4caa9c 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -49,9 +49,9 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
-#include "tensorflow/core/protobuf/config.pb.h"  // NOLINT
+#include "tensorflow/core/protobuf/config.pb.h"             // NOLINT
 #include "tensorflow/core/protobuf/device_properties.pb.h"  // NOLINT
-#include "tensorflow/core/protobuf/rewriter_config.pb.h"  // NOLINT
+#include "tensorflow/core/protobuf/rewriter_config.pb.h"    // NOLINT
 #include "tensorflow/core/util/device_name_utils.h"
 
 #if GOOGLE_CUDA
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index e4ffc230e4..4dee51e1e8 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -125,8 +125,8 @@ void GetInputProperties(const grappler::GraphProperties& graph_properties,
 
 void GetOutputProperties(const grappler::GraphProperties& graph_properties,
                          const Node* outside_node, const int in_port,
-                        PartialTensorShape* shape,
-                        tensorflow::DataType* dtype) {
+                         PartialTensorShape* shape,
+                         tensorflow::DataType* dtype) {
   if (graph_properties.HasInputProperties(outside_node->name())) {
     auto input_params =
         graph_properties.GetInputProperties(outside_node->name());
@@ -141,10 +141,11 @@ void GetOutputProperties(const grappler::GraphProperties& graph_properties,
 tensorflow::Status ValidateInputProperties(const PartialTensorShape& shape,
                                            const tensorflow::DataType dtype,
                                            nvinfer1::DataType* trt_dtype) {
+  // TODO(aaroey): some of these checks also apply to IsTensorRTCandidate(), so
+  // put them there instead.
   TF_RETURN_IF_ERROR(ConvertDType(dtype, trt_dtype));
   if (shape.dims() < 0) {
-    return tensorflow::errors::InvalidArgument(
-        "Input tensor rank is unknown.");
+    return tensorflow::errors::InvalidArgument("Input tensor rank is unknown.");
   }
   if (shape.dims() > 8) {
     return tensorflow::errors::OutOfRange(
@@ -153,7 +154,7 @@ tensorflow::Status ValidateInputProperties(const PartialTensorShape& shape,
   for (int d = 1; d < shape.dims(); ++d) {
     if (shape.dim_size(d) < 0) {
       return tensorflow::errors::InvalidArgument(
-          "Input tensor has a unknow non-batch dimemension at dim ", d);
+          "Input tensor has a unknown non-batch dimemension at dim ", d);
     }
   }
   return Status::OK();
@@ -2703,9 +2704,9 @@ tensorflow::Status ConvertGraphDefToEngine(
       auto status = ValidateInputProperties(
           shape, node_def.attr().at("dtype").type(), &dtype);
       if (!status.ok()) {
-        const string error_message = StrCat(
-            "Validation failed for ", node_name, " and input slot ",
-            slot_number, ": ", status.error_message());
+        const string error_message =
+            StrCat("Validation failed for ", node_name, " and input slot ",
+                   slot_number, ": ", status.error_message());
         LOG(WARNING) << error_message;
         return Status(status.code(), error_message);
       }
-- 
cgit v1.2.3


From 17b9b3f957654178b33c2b13a83b5f6b0690f16d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 08:08:14 -0700
Subject: A map from TF Lite tensor indices to TensorFlow tensors.

PiperOrigin-RevId: 204912661
---
 tensorflow/contrib/lite/delegates/eager/BUILD      |  31 ++++
 .../contrib/lite/delegates/eager/buffer_map.cc     | 105 +++++++++++++
 .../contrib/lite/delegates/eager/buffer_map.h      |  59 +++++++
 .../lite/delegates/eager/buffer_map_test.cc        | 172 +++++++++++++++++++++
 tensorflow/contrib/lite/delegates/eager/util.cc    |  23 +++
 tensorflow/contrib/lite/delegates/eager/util.h     |   5 +
 .../contrib/lite/delegates/eager/util_test.cc      |  17 +-
 tensorflow/contrib/lite/util.h                     |   7 +-
 8 files changed, 415 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/contrib/lite/delegates/eager/buffer_map.cc
 create mode 100644 tensorflow/contrib/lite/delegates/eager/buffer_map.h
 create mode 100644 tensorflow/contrib/lite/delegates/eager/buffer_map_test.cc

diff --git a/tensorflow/contrib/lite/delegates/eager/BUILD b/tensorflow/contrib/lite/delegates/eager/BUILD
index 066b106215..270d83d188 100644
--- a/tensorflow/contrib/lite/delegates/eager/BUILD
+++ b/tensorflow/contrib/lite/delegates/eager/BUILD
@@ -7,11 +7,42 @@ package(default_visibility = [
 
 licenses(["notice"])  # Apache 2.0
 
+cc_library(
+    name = "buffer_map",
+    srcs = ["buffer_map.cc"],
+    hdrs = ["buffer_map.h"],
+    deps = [
+        ":util",
+        "//tensorflow/c:c_api_internal",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite:kernel_api",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
+
+cc_test(
+    name = "buffer_map_test",
+    size = "small",
+    srcs = ["buffer_map_test.cc"],
+    tags = [
+        "tflite_not_portable",
+    ],
+    deps = [
+        ":buffer_map",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite:util",
+        "//tensorflow/contrib/lite/testing:util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 cc_library(
     name = "util",
     srcs = ["util.cc"],
     hdrs = ["util.h"],
     deps = [
+        "//tensorflow/c:c_api_internal",
         "//tensorflow/contrib/lite:framework",
         "//tensorflow/contrib/lite:kernel_api",
         "//tensorflow/core:framework",
diff --git a/tensorflow/contrib/lite/delegates/eager/buffer_map.cc b/tensorflow/contrib/lite/delegates/eager/buffer_map.cc
new file mode 100644
index 0000000000..e4a780b735
--- /dev/null
+++ b/tensorflow/contrib/lite/delegates/eager/buffer_map.cc
@@ -0,0 +1,105 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/contrib/lite/delegates/eager/buffer_map.h"
+
+#include "tensorflow/c/c_api_internal.h"
+#include "tensorflow/contrib/lite/delegates/eager/util.h"
+#include "tensorflow/core/framework/allocation_description.pb.h"
+#include "tensorflow/core/framework/log_memory.h"
+
+namespace tflite {
+namespace {
+// A tensor buffer that is allocated, deallocated and populated by TF Lite.
+class TfLiteTensorBuffer : public tensorflow::TensorBuffer {
+ public:
+  explicit TfLiteTensorBuffer(const TfLiteTensor* tensor) {
+    len_ = tensor->bytes;
+    // TODO(ahentz): if we can guarantee that TF Lite allocated tensors with
+    // the same alignment as TensorFlow (EIGEN_MAX_ALIGN_BYTES), then we can
+    // potentially eliminate the copy below.
+    data_ =
+        tensorflow::cpu_allocator()->AllocateRaw(EIGEN_MAX_ALIGN_BYTES, len_);
+    if (data_ != nullptr) {
+      if (tensorflow::LogMemory::IsEnabled()) {
+        tensorflow::LogMemory::RecordRawAllocation(
+            "TfLiteTensorBuffer_New",
+            tensorflow::LogMemory::EXTERNAL_TENSOR_ALLOCATION_STEP_ID, len_,
+            data_, tensorflow::cpu_allocator());
+      }
+      std::memcpy(data_, tensor->data.raw, tensor->bytes);
+    }
+  }
+
+  ~TfLiteTensorBuffer() override {
+    if (tensorflow::LogMemory::IsEnabled() && data_ != nullptr) {
+      tensorflow::LogMemory::RecordRawDeallocation(
+          "TfLiteTensorBuffer_Delete",
+          tensorflow::LogMemory::EXTERNAL_TENSOR_ALLOCATION_STEP_ID, data_,
+          tensorflow::cpu_allocator(), false);
+    }
+    tensorflow::cpu_allocator()->DeallocateRaw(data_);
+  }
+
+  void* data() const override { return data_; }
+  size_t size() const override { return len_; }
+
+  TensorBuffer* root_buffer() override { return this; }
+  void FillAllocationDescription(
+      tensorflow::AllocationDescription* proto) const override {
+    tensorflow::int64 rb = size();
+    proto->set_requested_bytes(rb);
+    proto->set_allocator_name(tensorflow::cpu_allocator()->Name());
+  }
+
+  // Prevents input forwarding from mutating this buffer.
+  bool OwnsMemory() const override { return false; }
+
+ private:
+  void* data_;
+  size_t len_;
+};
+}  // namespace
+
+BufferMap::BufferMap() {}
+
+BufferMap::~BufferMap() {}
+
+bool BufferMap::HasTensor(int tensor_index) const {
+  return id_to_tensor_.count(tensor_index) != 0;
+}
+
+tensorflow::Tensor BufferMap::GetTensor(int tensor_index) const {
+  return id_to_tensor_.at(tensor_index);
+}
+
+void BufferMap::SetFromTfLite(int tensor_index, const TfLiteTensor* tensor) {
+  tensorflow::TensorShape shape;
+  int num_dims = tensor->dims->size;
+  for (int i = 0; i < num_dims; ++i) {
+    shape.AddDim(tensor->dims->data[i]);
+  }
+  auto* buf = new TfLiteTensorBuffer(tensor);
+  tensorflow::Tensor t = tensorflow::TensorCApi::MakeTensor(
+      GetTensorFlowDataType(tensor->type), shape, buf);
+  buf->Unref();
+
+  SetFromTensorFlow(tensor_index, std::move(t));
+}
+
+void BufferMap::SetFromTensorFlow(int tensor_index, tensorflow::Tensor tensor) {
+  id_to_tensor_[tensor_index] = std::move(tensor);
+}
+
+}  // namespace tflite
diff --git a/tensorflow/contrib/lite/delegates/eager/buffer_map.h b/tensorflow/contrib/lite/delegates/eager/buffer_map.h
new file mode 100644
index 0000000000..922f67f574
--- /dev/null
+++ b/tensorflow/contrib/lite/delegates/eager/buffer_map.h
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_BUFFER_MAP_H_
+#define TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_BUFFER_MAP_H_
+
+#include 
+
+#include "tensorflow/contrib/lite/context.h"
+#include "tensorflow/core/framework/tensor.h"
+
+namespace tflite {
+
+// Maps a TF Lite tensor index into a TensorFlow tensor.
+//
+// The TF Lite interpreter assigns integer indices to each of its tensors, but
+// the Eager delegate deals in terms of TensorFlow tensors. This class maps
+// from indices to tensors and allows the creation of new tensors to be
+// associated with a given index.
+class BufferMap {
+ public:
+  BufferMap();
+  ~BufferMap();
+
+  // Returns true if the given 'tensor_index' has a corresponding
+  // tensorflow::Tensor.
+  bool HasTensor(int tensor_index) const;
+
+  // Returns the tensorflow::Tensor associated with the given 'tensor_index'.
+  // Precondition: HasTensor() is true.
+  tensorflow::Tensor GetTensor(int tensor_index) const;
+
+  // Associates the given tensorflow::Tensor with the given 'tensor_index'.
+  // Note that tensorflow Tensors share data buffers, so this method is only a
+  // shallow copy.
+  void SetFromTensorFlow(int tensor_index, tensorflow::Tensor tensor);
+
+  // Same as above but creates a new tensorflow::Tensor with a copy of the
+  // given TfLiteTensor's data.
+  void SetFromTfLite(int tensor_index, const TfLiteTensor* tensor);
+
+ private:
+  std::map id_to_tensor_;
+};
+
+}  // namespace tflite
+
+#endif  // TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_BUFFER_MAP_H_
diff --git a/tensorflow/contrib/lite/delegates/eager/buffer_map_test.cc b/tensorflow/contrib/lite/delegates/eager/buffer_map_test.cc
new file mode 100644
index 0000000000..c447eeaa05
--- /dev/null
+++ b/tensorflow/contrib/lite/delegates/eager/buffer_map_test.cc
@@ -0,0 +1,172 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/contrib/lite/delegates/eager/buffer_map.h"
+
+#include 
+#include 
+#include "tensorflow/contrib/lite/interpreter.h"
+#include "tensorflow/contrib/lite/testing/util.h"
+#include "tensorflow/contrib/lite/util.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAre;
+
+// A bit of RAII to simplify handling of TfLiteTensors in the tests.
+using UniqueTfLiteTensor =
+    std::unique_ptr>;
+
+template 
+UniqueTfLiteTensor MakeLiteTensor(const std::vector& shape,
+                                  const std::vector& data) {
+  auto tensor = UniqueTfLiteTensor(new TfLiteTensor, [](TfLiteTensor* t) {
+    TfLiteTensorDataFree(t);
+    TfLiteIntArrayFree(t->dims);
+    delete t;
+  });
+  tensor->allocation_type = kTfLiteDynamic;
+  tensor->type = typeToTfLiteType();
+  tensor->dims = ConvertVectorToTfLiteIntArray(shape);
+  tensor->data.raw = nullptr;
+  TfLiteTensorRealloc(data.size() * sizeof(T), tensor.get());
+  memcpy(tensor->data.raw, data.data(), data.size() * sizeof(T));
+  return tensor;
+}
+
+template 
+tensorflow::Tensor MakeTensor(const std::vector& shape,
+                              const std::vector& data) {
+  BufferMap buffer_map;  // BufferMap is the easiest way to build the tensor.
+  UniqueTfLiteTensor t1 = MakeLiteTensor(shape, data);
+  buffer_map.SetFromTfLite(0, t1.get());
+  return buffer_map.GetTensor(0);
+}
+
+std::vector GetTensorShape(const tensorflow::Tensor& t) {
+  std::vector shape(t.dims());
+  for (int i = 0; i < t.dims(); ++i) {
+    shape[i] = t.dim_size(i);
+  }
+  return shape;
+}
+
+template 
+std::vector GetTensorData(const tensorflow::Tensor& t) {
+  const T* data = t.flat().data();
+  return std::vector(data, data + t.NumElements());
+}
+
+TEST(BufferMapTest, EmptyBuffer) {
+  BufferMap buffer_map;
+  EXPECT_FALSE(buffer_map.HasTensor(0));
+}
+
+TEST(BufferMapTest, SetFromTfLite) {
+  BufferMap buffer_map;
+
+  UniqueTfLiteTensor t =
+      MakeLiteTensor({1, 2, 1, 3}, {0, 0, 0, 0.123f, 0, 0});
+  buffer_map.SetFromTfLite(0, t.get());
+  ASSERT_TRUE(buffer_map.HasTensor(0));
+
+  EXPECT_THAT(GetTensorData(buffer_map.GetTensor(0)),
+              ElementsAre(0, 0, 0, 0.123f, 0, 0));
+
+  // Also check details of the tensor.
+  tensorflow::Tensor out_tensor = buffer_map.GetTensor(0);
+  ASSERT_EQ(out_tensor.dtype(), tensorflow::DT_FLOAT);
+  ASSERT_EQ(out_tensor.NumElements(), 6);
+  ASSERT_THAT(GetTensorShape(out_tensor), ElementsAre(1, 2, 1, 3));
+}
+
+TEST(BufferMapTest, SetFromTfLiteTwice) {
+  UniqueTfLiteTensor t1 =
+      MakeLiteTensor({1, 2, 1, 3}, {0, 0, 0, 0.123f, 0, 0});
+  UniqueTfLiteTensor t2 =
+      MakeLiteTensor({1, 2, 4}, {0, 0, 0, 3, 0, 0, 1, 2});
+
+  BufferMap buffer_map;
+  buffer_map.SetFromTfLite(0, t1.get());
+  buffer_map.SetFromTfLite(0, t2.get());
+
+  EXPECT_THAT(GetTensorData(buffer_map.GetTensor(0)),
+              ElementsAre(0, 0, 0, 3, 0, 0, 1, 2));
+}
+
+TEST(BufferMapTest, SetFromTensorFlow) {
+  tensorflow::Tensor t1 =
+      MakeTensor({1, 2, 1, 3}, {0, 0, 0, 0.123f, 0, 0});
+
+  BufferMap buffer_map;
+  buffer_map.SetFromTensorFlow(0, t1);
+
+  EXPECT_THAT(GetTensorData(buffer_map.GetTensor(0)),
+              ElementsAre(0, 0, 0, 0.123f, 0, 0));
+
+  // Also check details of the tensor.
+  tensorflow::Tensor out_tensor = buffer_map.GetTensor(0);
+  ASSERT_EQ(out_tensor.dtype(), tensorflow::DT_FLOAT);
+  ASSERT_EQ(out_tensor.NumElements(), 6);
+  ASSERT_THAT(GetTensorShape(out_tensor), ElementsAre(1, 2, 1, 3));
+}
+
+TEST(BufferMapTest, SetFromTensorFlowTwice) {
+  tensorflow::Tensor t1 =
+      MakeTensor({1, 2, 1, 3}, {0, 0, 0, 0.123f, 0, 0});
+  tensorflow::Tensor t2 = MakeTensor({1, 2, 4}, {0, 0, 0, 3, 0, 0, 1, 2});
+  BufferMap buffer_map;
+  buffer_map.SetFromTensorFlow(0, t1);
+  buffer_map.SetFromTensorFlow(0, t2);
+
+  EXPECT_THAT(GetTensorData(buffer_map.GetTensor(0)),
+              ElementsAre(0, 0, 0, 3, 0, 0, 1, 2));
+}
+
+TEST(BufferMapTest, TfLiteOverwritesTensorFlow) {
+  tensorflow::Tensor t1 =
+      MakeTensor({1, 2, 1, 3}, {0, 0, 0, 0.123f, 0, 0});
+  UniqueTfLiteTensor t2 =
+      MakeLiteTensor({1, 2, 4}, {0, 0, 0, 3, 0, 0, 1, 2});
+
+  BufferMap buffer_map;
+  buffer_map.SetFromTensorFlow(0, t1);
+  buffer_map.SetFromTfLite(0, t2.get());
+
+  EXPECT_THAT(GetTensorData(buffer_map.GetTensor(0)),
+              ElementsAre(0, 0, 0, 3, 0, 0, 1, 2));
+}
+
+TEST(BufferMapTest, TensorFlowOverwritesTfLite) {
+  tensorflow::Tensor t1 =
+      MakeTensor({1, 2, 1, 3}, {0, 0, 0, 0.123f, 0, 0});
+  UniqueTfLiteTensor t2 =
+      MakeLiteTensor({1, 2, 4}, {0, 0, 0, 3, 0, 0, 1, 2});
+  BufferMap buffer_map;
+  buffer_map.SetFromTfLite(0, t2.get());
+  buffer_map.SetFromTensorFlow(0, t1);
+
+  EXPECT_THAT(GetTensorData(buffer_map.GetTensor(0)),
+              ElementsAre(0, 0, 0, 0.123f, 0, 0));
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/delegates/eager/util.cc b/tensorflow/contrib/lite/delegates/eager/util.cc
index 04a852e515..e1879bdaff 100644
--- a/tensorflow/contrib/lite/delegates/eager/util.cc
+++ b/tensorflow/contrib/lite/delegates/eager/util.cc
@@ -44,4 +44,27 @@ TfLiteStatus CopyShape(TfLiteContext* context, const tensorflow::Tensor& src,
   return context->ResizeTensor(context, tensor, shape);
 }
 
+TF_DataType GetTensorFlowDataType(TfLiteType type) {
+  switch (type) {
+    case kTfLiteNoType:
+      return TF_FLOAT;
+    case kTfLiteFloat32:
+      return TF_FLOAT;
+    case kTfLiteInt16:
+      return TF_INT16;
+    case kTfLiteInt32:
+      return TF_INT32;
+    case kTfLiteUInt8:
+      return TF_UINT8;
+    case kTfLiteInt64:
+      return TF_INT64;
+    case kTfLiteComplex64:
+      return TF_COMPLEX64;
+    case kTfLiteString:
+      return TF_STRING;
+    case kTfLiteBool:
+      return TF_BOOL;
+  }
+}
+
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/delegates/eager/util.h b/tensorflow/contrib/lite/delegates/eager/util.h
index 2696ca8d0d..12b33b9b49 100644
--- a/tensorflow/contrib/lite/delegates/eager/util.h
+++ b/tensorflow/contrib/lite/delegates/eager/util.h
@@ -15,6 +15,7 @@ limitations under the License.
 #ifndef TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_UTIL_H_
 #define TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_UTIL_H_
 
+#include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/contrib/lite/context.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -30,6 +31,10 @@ TfLiteStatus ConvertStatus(TfLiteContext* context,
 // error and returns kTfLiteError if the shape can't be converted.
 TfLiteStatus CopyShape(TfLiteContext* context, const tensorflow::Tensor& src,
                        TfLiteTensor* tensor);
+
+// Returns the TF C API Data type that corresponds to the given TfLiteType.
+TF_DataType GetTensorFlowDataType(TfLiteType type);
+
 }  // namespace tflite
 
 #endif  // TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_UTIL_H_
diff --git a/tensorflow/contrib/lite/delegates/eager/util_test.cc b/tensorflow/contrib/lite/delegates/eager/util_test.cc
index 563f82dec3..53ed4db972 100644
--- a/tensorflow/contrib/lite/delegates/eager/util_test.cc
+++ b/tensorflow/contrib/lite/delegates/eager/util_test.cc
@@ -23,6 +23,8 @@ limitations under the License.
 namespace tflite {
 namespace {
 
+using tensorflow::DT_FLOAT;
+using tensorflow::Tensor;
 using ::testing::ElementsAre;
 
 struct TestContext : public TfLiteContext {
@@ -72,9 +74,6 @@ TEST(UtilTest, CopyShape) {
   context.ReportError = ReportError;
   context.ResizeTensor = ResizeTensor;
 
-  using tensorflow::DT_FLOAT;
-  using tensorflow::Tensor;
-
   TfLiteTensor dst;
 
   EXPECT_EQ(CopyShape(&context, Tensor(), &dst), kTfLiteOk);
@@ -90,6 +89,18 @@ TEST(UtilTest, CopyShape) {
             "TF Lite");
 }
 
+TEST(UtilTest, TypeConversions) {
+  EXPECT_EQ(TF_FLOAT, GetTensorFlowDataType(kTfLiteNoType));
+  EXPECT_EQ(TF_FLOAT, GetTensorFlowDataType(kTfLiteFloat32));
+  EXPECT_EQ(TF_INT16, GetTensorFlowDataType(kTfLiteInt16));
+  EXPECT_EQ(TF_INT32, GetTensorFlowDataType(kTfLiteInt32));
+  EXPECT_EQ(TF_UINT8, GetTensorFlowDataType(kTfLiteUInt8));
+  EXPECT_EQ(TF_INT64, GetTensorFlowDataType(kTfLiteInt64));
+  EXPECT_EQ(TF_COMPLEX64, GetTensorFlowDataType(kTfLiteComplex64));
+  EXPECT_EQ(TF_STRING, GetTensorFlowDataType(kTfLiteString));
+  EXPECT_EQ(TF_BOOL, GetTensorFlowDataType(kTfLiteBool));
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/contrib/lite/util.h b/tensorflow/contrib/lite/util.h
index 89d9b4f5cf..3c4801183b 100644
--- a/tensorflow/contrib/lite/util.h
+++ b/tensorflow/contrib/lite/util.h
@@ -26,12 +26,17 @@ limitations under the License.
 
 namespace tflite {
 
-// Converts a `std::vector` to a `TfLiteIntArray`.
+// Converts a `std::vector` to a `TfLiteIntArray`. The caller takes ownership
+// of the returned pointer.
 TfLiteIntArray* ConvertVectorToTfLiteIntArray(const std::vector& input);
 
+// Converts an array (of the given size) to a `TfLiteIntArray`. The caller
+// takes ownership of the returned pointer, and must make sure 'dims' has at
+// least 'rank' elemnts.
 TfLiteIntArray* ConvertArrayToTfLiteIntArray(const int rank, const int* dims);
 
 // Checks whether a `TfLiteIntArray` and an int array have matching elements.
+// The caller must guarantee that 'b' has at least 'b_size' elements.
 bool EqualArrayAndTfLiteIntArray(const TfLiteIntArray* a, const int b_size,
                                  const int* b);
 
-- 
cgit v1.2.3


From c0ff0cccd4d6f770f03756279dd39ff43d4a7bca Mon Sep 17 00:00:00 2001
From: Shashi Shekhar 
Date: Tue, 17 Jul 2018 09:00:24 -0700
Subject: Fix default SDK path for Mac.

PiperOrigin-RevId: 204919096
---
 configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index eaff83d2cc..c482628ec8 100644
--- a/configure.py
+++ b/configure.py
@@ -680,7 +680,7 @@ def create_android_sdk_rule(environ_cp):
   if is_windows() or is_cygwin():
     default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA'])
   elif is_macos():
-    default_sdk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
+    default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME']
   else:
     default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME']
 
-- 
cgit v1.2.3


From e209107920f8d6fe63e2f410d55be40a4c7a2d94 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 09:16:33 -0700
Subject: Quick clarification about the stack nature of the default graph
 context.

PiperOrigin-RevId: 204921538
---
 tensorflow/python/framework/ops.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index a3b56b0f63..ea7a9986fe 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -3558,9 +3558,13 @@ class Graph(object):
     This method should be used if you want to create multiple graphs
     in the same process. For convenience, a global default graph is
     provided, and all ops will be added to this graph if you do not
-    create a new graph explicitly. Use this method with the `with` keyword
-    to specify that ops created within the scope of a block should be
-    added to this graph.
+    create a new graph explicitly.
+
+    Use this method with the `with` keyword to specify that ops created within
+    the scope of a block should be added to this graph. In this case, once
+    the scope of the `with` is exited, the previous default graph is set again
+    as default. There is a stack, so it's ok to have multiple nested levels
+    of `as_default` calls.
 
     The default graph is a property of the current thread. If you
     create a new thread, and wish to use the default graph in that
-- 
cgit v1.2.3


From d61653a5276ecb7a20a7b07a4c702f3c34635062 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 09:20:16 -0700
Subject: Support for pseudo half precision in depthwise convolution kernels.
 Returning Status as promised a long time ago in CL 186037306.

PiperOrigin-RevId: 204922021
---
 tensorflow/core/kernels/BUILD                      |  23 +
 .../core/kernels/depthwise_conv_op_gpu.cu.cc       | 486 ++++++++++++---------
 tensorflow/core/kernels/depthwise_conv_ops_test.cc | 114 +++++
 3 files changed, 420 insertions(+), 203 deletions(-)
 create mode 100644 tensorflow/core/kernels/depthwise_conv_ops_test.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 7599cf7db2..1e889b1ea8 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -1105,6 +1105,29 @@ tf_cc_test(
     ],
 )
 
+tf_cuda_cc_test(
+    name = "depthwise_conv_ops_test",
+    size = "small",
+    srcs = ["depthwise_conv_ops_test.cc"],
+    tags = ["requires-gpu-sm35"],
+    deps = [
+        ":conv_ops",
+        ":image",
+        ":ops_testutil",
+        ":ops_util",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:tensorflow",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cc_test(
     name = "decode_wav_op_test",
     size = "small",
diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
index 5390222b3a..5472a192d9 100644
--- a/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
@@ -165,15 +165,18 @@ __global__ void __launch_bounds__(1024, 2)
 // one each in the lower and upper half of a tile.
 // Backprop input direction is the same as forward direction with the filter
 // rotated by 180°.
+// T is the tensors' data type. S is the math type the kernel uses. This is the
+// same as T for all cases but pseudo half (which has T=Eigen::half, S=float).
 template 
+          bool kKnownEvenHeight, typename S>
 __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNHWCSmall(
     const DepthwiseArgs args, const T* input, const T* filter, T* output) {
   assert(CanLaunchDepthwiseConv2dGPUSmall(args));
   // Holds block plus halo and filter data for blockDim.x depths.
-  extern __shared__ __align__(sizeof(T)) unsigned char shared_memory[];
-  T* const shared_data = reinterpret_cast(shared_memory);
+  extern __shared__ __align__(8) unsigned char shared_memory[];
+  static_assert(sizeof(S) <= 8, "Insufficient alignement detected");
+  S* const shared_data = reinterpret_cast(shared_memory);
 
   const int num_batches = args.batch;
   const int in_height = args.in_rows;
@@ -219,7 +222,7 @@ __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNHWCSmall(
 
   // Initialize tile, in particular the padding.
   for (int i = thread_idx; i < tile_size; i += block_size) {
-    shared_data[i] = T(0);
+    shared_data[i] = S();
   }
   __syncthreads();
 
@@ -254,14 +257,15 @@ __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNHWCSmall(
 
     if (channel_in_range) {
       const T* const in_ptr = inout_offset + input;
-      T* const tile_ptr = tile_idx + shared_data;
-      tile_ptr[0] = ldg(in_ptr);
+      S* const tile_ptr = tile_idx + shared_data;
+      tile_ptr[0] = static_cast(ldg(in_ptr));
       if (!skip_second) {
-        tile_ptr[tile_offset] = ldg(tensor_offset + in_ptr);
+        tile_ptr[tile_offset] = static_cast(ldg(tensor_offset + in_ptr));
       }
 
       if (filter_write_offset != 0) {
-        shared_data[filter_write_offset] = ldg(filter_offset + filter);
+        shared_data[filter_write_offset] =
+            static_cast(ldg(filter_offset + filter));
       }
     }
 
@@ -269,17 +273,17 @@ __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNHWCSmall(
     __syncthreads();
 
     if (channel_in_range) {
-      T sum1 = static_cast(0);
-      T sum2 = static_cast(0);
+      S sum1 = S();
+      S sum2 = S();
       int shared_offset = data_idx;
-      const T* filter_ptr = filter_read_offset + shared_data;
+      const S* filter_ptr = filter_read_offset + shared_data;
       UNROLL for (int r = 0; r < filter_height; ++r) {
         UNROLL for (int c = 0; c < filter_width; ++c) {
           if (kDirection == DIRECTION_BACKWARD) {
             filter_ptr -= kBlockDepth;
           }
-          const T filter_value = *filter_ptr;
-          const T* const tile_ptr = shared_offset + shared_data;
+          const S filter_value = *filter_ptr;
+          const S* const tile_ptr = shared_offset + shared_data;
           sum1 += filter_value * tile_ptr[0];
           sum2 += filter_value * tile_ptr[tile_offset];
           shared_offset += kBlockDepth;
@@ -290,9 +294,9 @@ __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNHWCSmall(
         shared_offset += in_increment;
       }
       T* const out_ptr = inout_offset + output;
-      out_ptr[0] = sum1;
+      out_ptr[0] = static_cast(sum1);
       if (!skip_second) {
-        out_ptr[tensor_offset] = sum2;
+        out_ptr[tensor_offset] = static_cast(sum2);
       }
     }
 
@@ -445,15 +449,18 @@ __global__ void __launch_bounds__(1024, 2)
 // one each in the lower and upper half of a tile.
 // Backprop input direction is the same as forward direction with the filter
 // rotated by 180°.
+// T is the tensors' data type. S is the math type the kernel uses. This is the
+// same as T for all cases but pseudo half (which has T=Eigen::half, S=float).
 template 
+          bool kKnownEvenHeight, typename S>
 __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNCHWSmall(
     const DepthwiseArgs args, const T* input, const T* filter, T* output) {
   assert(CanLaunchDepthwiseConv2dGPUSmall(args));
   // Holds block plus halo and filter data for blockDim.z depths.
-  extern __shared__ __align__(sizeof(T)) unsigned char shared_memory[];
-  T* const shared_data = reinterpret_cast(shared_memory);
+  extern __shared__ __align__(8) unsigned char shared_memory[];
+  static_assert(sizeof(S) <= 8, "Insufficient alignement detected");
+  S* const shared_data = reinterpret_cast(shared_memory);
 
   const int num_batches = args.batch;
   const int in_height = args.in_rows;
@@ -498,7 +505,7 @@ __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNCHWSmall(
 
   // Initialize tile, in particular the padding.
   for (int i = thread_idx; i < tile_size; i += block_size) {
-    shared_data[i] = T(0);
+    shared_data[i] = S();
   }
   __syncthreads();
 
@@ -534,34 +541,35 @@ __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNCHWSmall(
 
     if (channel_in_range) {
       const T* const in_ptr = inout_offset + input;
-      T* const tile_ptr = tile_idx + shared_data;
-      tile_ptr[0] = ldg(in_ptr);
+      S* const tile_ptr = tile_idx + shared_data;
+      tile_ptr[0] = static_cast(ldg(in_ptr));
       if (!skip_second) {
-        tile_ptr[tile_offset] = ldg(block_pixels + in_ptr);
+        tile_ptr[tile_offset] = static_cast(ldg(block_pixels + in_ptr));
       }
     }
 
     if (filter_write_offset != 0) {
       const int filter_offset =
           filter_idx + (channel + filter_channel) % in_depth;
-      shared_data[filter_write_offset] = ldg(filter_offset + filter);
+      shared_data[filter_write_offset] =
+          static_cast(ldg(filter_offset + filter));
     }
 
     // Note: the condition to reach this is uniform across the entire block.
     __syncthreads();
 
     if (channel_in_range) {
-      T sum1 = static_cast(0);
-      T sum2 = static_cast(0);
+      S sum1 = S();
+      S sum2 = S();
       int shared_offset = data_idx;
-      const T* filter_ptr = filter_read_offset + shared_data;
+      const S* filter_ptr = filter_read_offset + shared_data;
       UNROLL for (int r = 0; r < filter_height; ++r) {
         UNROLL for (int c = 0; c < filter_width; ++c) {
           if (kDirection == DIRECTION_BACKWARD) {
             filter_ptr -= kBlockDepth;
           }
-          const T filter_value = *filter_ptr;
-          const T* const tile_ptr = shared_offset + shared_data;
+          const S filter_value = *filter_ptr;
+          const S* const tile_ptr = shared_offset + shared_data;
           sum1 += filter_value * tile_ptr[0];
           sum2 += filter_value * tile_ptr[tile_offset];
           ++shared_offset;
@@ -572,9 +580,9 @@ __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNCHWSmall(
         shared_offset += in_increment;
       }
       T* const out_ptr = inout_offset + output;
-      out_ptr[0] = sum1;
+      out_ptr[0] = static_cast(sum1);
       if (!skip_second) {
-        out_ptr[block_pixels] = sum2;
+        out_ptr[block_pixels] = static_cast(sum2);
       }
     }
 
@@ -585,11 +593,11 @@ __global__ __launch_bounds__(1024, 2) void DepthwiseConv2dGPUKernelNCHWSmall(
 
 template 
-void LaunchDepthwiseConv2dGPUSmall(const GpuDevice& device,
-                                   const DepthwiseArgs& args, const T* input,
-                                   const T* filter, T* output,
-                                   TensorFormat data_format) {
+          bool kKnownEvenHeight, typename S>
+Status LaunchDepthwiseConv2dGPUSmall(OpKernelContext* ctx,
+                                     const DepthwiseArgs& args, const T* input,
+                                     const T* filter, T* output,
+                                     TensorFormat data_format) {
   const int block_height = (args.in_rows + 1) / 2;
   dim3 block_dim;
   int block_count;
@@ -602,7 +610,7 @@ void LaunchDepthwiseConv2dGPUSmall(const GpuDevice& device,
       kernel =
           DepthwiseConv2dGPUKernelNHWCSmall;
+                                            kKnownEvenHeight, S>;
       break;
     case FORMAT_NCHW:
       block_dim = dim3(args.in_cols, block_height, kBlockDepth);
@@ -611,73 +619,126 @@ void LaunchDepthwiseConv2dGPUSmall(const GpuDevice& device,
       kernel =
           DepthwiseConv2dGPUKernelNCHWSmall;
+                                            kKnownEvenHeight, S>;
       break;
     default:
-      LOG(ERROR) << "FORMAT_" << ToString(data_format) << " is not supported";
-      return;
+      return errors::InvalidArgument("FORMAT_", ToString(data_format),
+                                     " is not supported");
   }
   const int tile_width = args.in_cols + args.filter_cols - 1;
   const int tile_height = block_height * 2 + args.filter_rows - 1;
   const int tile_pixels = tile_height * tile_width;
   const int filter_pixels = args.filter_rows * args.filter_cols;
   const int shared_memory_size =
-      kBlockDepth * (tile_pixels + filter_pixels) * sizeof(T);
+      kBlockDepth * (tile_pixels + filter_pixels) * sizeof(S);
   const int num_outputs = args.out_rows * args.out_cols * block_count;
+  auto device = ctx->eigen_gpu_device();
   CudaLaunchConfig config = GetCudaLaunchConfigFixedBlockSize(
       num_outputs, device, kernel, shared_memory_size,
       block_dim.x * block_dim.y * block_dim.z);
   kernel<<>>(args, input, filter, output);
+  return Status::OK();
+}
+
+namespace {
+// Returns whether the context's GPU supports efficient fp16 math.
+bool HasFastHalfMath(OpKernelContext* ctx) {
+  int major, minor;
+  ctx->op_device_context()
+      ->stream()
+      ->parent()
+      ->GetDeviceDescription()
+      .cuda_compute_capability(&major, &minor);
+  auto cuda_arch = major * 100 + minor * 10;
+  // GPUs before sm_53 don't support fp16 math, and sm_61's fp16 math is slow.
+  return cuda_arch >= 530 && cuda_arch != 610;
+}
+
+namespace detail {
+template 
+struct PseudoHalfType {
+  using Type = T;
+};
+template <>
+struct PseudoHalfType {
+  using Type = float;
+};
+}  // namespace detail
+
+// Maps to float if T is __half, and to T otherwise.
+template 
+using PseudoHalfType = typename detail::PseudoHalfType::Type;
+}  // namespace
+
+template 
+Status LaunchDepthwiseConv2dGPUSmall(OpKernelContext* ctx,
+                                     const DepthwiseArgs& args, const T* input,
+                                     const T* filter, T* output,
+                                     TensorFormat data_format) {
+#if !defined __CUDA_ARCH__ || __CUDA_ARCH__ >= 530
+  if (HasFastHalfMath(ctx)) {
+    return LaunchDepthwiseConv2dGPUSmall(
+        ctx, args, input, filter, output, data_format);
+  }
+#endif
+  return LaunchDepthwiseConv2dGPUSmall>(
+      ctx, args, input, filter, output, data_format);
 }
 
 template 
-void LaunchDepthwiseConv2dGPUSmall(const GpuDevice& device,
-                                   const DepthwiseArgs& args, const T* input,
-                                   const T* filter, T* output,
-                                   TensorFormat data_format) {
+Status LaunchDepthwiseConv2dGPUSmall(OpKernelContext* ctx,
+                                     const DepthwiseArgs& args, const T* input,
+                                     const T* filter, T* output,
+                                     TensorFormat data_format) {
   if (args.in_rows & 1) {
-    LaunchDepthwiseConv2dGPUSmall(
-        device, args, input, filter, output, data_format);
+    return LaunchDepthwiseConv2dGPUSmall(ctx, args, input, filter,
+                                                output, data_format);
   } else {
-    LaunchDepthwiseConv2dGPUSmall(
-        device, args, input, filter, output, data_format);
+    return LaunchDepthwiseConv2dGPUSmall(
+        ctx, args, input, filter, output, data_format);
   }
 }
 
 template 
-void LaunchDepthwiseConv2dGPUSmall(const GpuDevice& device,
-                                   const DepthwiseArgs& args, const T* input,
-                                   const T* filter, T* output,
-                                   TensorFormat data_format) {
+Status LaunchDepthwiseConv2dGPUSmall(OpKernelContext* ctx,
+                                     const DepthwiseArgs& args, const T* input,
+                                     const T* filter, T* output,
+                                     TensorFormat data_format) {
   // Maximize (power of two) kBlockDepth while keeping a block within 1024
   // threads (2 pixels per thread).
   const int block_pixels = (args.in_rows + 1) / 2 * args.in_cols;
   if (block_pixels > 256) {
-    LaunchDepthwiseConv2dGPUSmall(
-        device, args, input, filter, output, data_format);
+    return LaunchDepthwiseConv2dGPUSmall(
+        ctx, args, input, filter, output, data_format);
   } else if (block_pixels > 128) {
-    LaunchDepthwiseConv2dGPUSmall(
-        device, args, input, filter, output, data_format);
+    return LaunchDepthwiseConv2dGPUSmall(
+        ctx, args, input, filter, output, data_format);
   } else {
-    LaunchDepthwiseConv2dGPUSmall(
-        device, args, input, filter, output, data_format);
+    return LaunchDepthwiseConv2dGPUSmall(
+        ctx, args, input, filter, output, data_format);
   }
 }
 
 template 
-void LaunchDepthwiseConv2dGPU(const GpuDevice& device,
-                              const DepthwiseArgs& args, const T* input,
-                              const T* filter, T* output,
-                              TensorFormat data_format) {
+Status LaunchDepthwiseConv2dGPU(OpKernelContext* ctx, const DepthwiseArgs& args,
+                                const T* input, const T* filter, T* output,
+                                TensorFormat data_format) {
   void (*kernel)(const DepthwiseArgs, const T*, const T*, T*, int);
   switch (data_format) {
     case FORMAT_NHWC:
@@ -691,11 +752,12 @@ void LaunchDepthwiseConv2dGPU(const GpuDevice& device,
                                        kKnownDepthMultiplier>;
       break;
     default:
-      LOG(ERROR) << "FORMAT_" << ToString(data_format) << " is not supported";
-      return;
+      return errors::InvalidArgument("FORMAT_", ToString(data_format),
+                                     " is not supported");
   }
   const int num_outputs =
       args.batch * args.out_rows * args.out_cols * args.out_depth;
+  auto device = ctx->eigen_gpu_device();
   CudaLaunchConfig config =
       GetCudaLaunchConfig(num_outputs, device, kernel, 0, 0);
   // The compile-time constant version runs faster with a single block.
@@ -706,26 +768,27 @@ void LaunchDepthwiseConv2dGPU(const GpuDevice& device,
   kernel<<>>(args, input, filter,
                                                           output, num_outputs);
+  return Status::OK();
 }
 
 template 
-void LaunchDepthwiseConv2dGPU(const GpuDevice& device,
-                              const DepthwiseArgs& args, const T* input,
-                              const T* filter, T* output,
-                              TensorFormat data_format) {
+Status LaunchDepthwiseConv2dGPU(OpKernelContext* ctx, const DepthwiseArgs& args,
+                                const T* input, const T* filter, T* output,
+                                TensorFormat data_format) {
   if (args.depth_multiplier == 1) {
     if (CanLaunchDepthwiseConv2dGPUSmall(args)) {
-      LaunchDepthwiseConv2dGPUSmall(
-          device, args, input, filter, output, data_format);
-      return;
+      return LaunchDepthwiseConv2dGPUSmall<
+          T, DIRECTION_FORWARD, kKnownFilterWidth, kKnownFilterHeight>(
+          ctx, args, input, filter, output, data_format);
     }
 
-    LaunchDepthwiseConv2dGPU(
-        device, args, input, filter, output, data_format);
+    return LaunchDepthwiseConv2dGPU(ctx, args, input, filter, output,
+                                       data_format);
   } else {
-    LaunchDepthwiseConv2dGPU(
-        device, args, input, filter, output, data_format);
+    return LaunchDepthwiseConv2dGPU(ctx, args, input, filter, output,
+                                        data_format);
   }
 }
 
@@ -736,18 +799,13 @@ void LaunchDepthwiseConvOp::operator()(OpKernelContext* ctx,
                                                      const T* input,
                                                      const T* filter, T* output,
                                                      TensorFormat data_format) {
-  const GpuDevice& device = ctx->eigen_device();
   if (args.filter_rows == 3 && args.filter_cols == 3) {
-    LaunchDepthwiseConv2dGPU(device, args, input, filter, output,
-                                      data_format);
+    OP_REQUIRES_OK(ctx, LaunchDepthwiseConv2dGPU(
+                            ctx, args, input, filter, output, data_format));
   } else {
-    LaunchDepthwiseConv2dGPU(device, args, input, filter, output,
-                                        data_format);
+    OP_REQUIRES_OK(ctx, LaunchDepthwiseConv2dGPU(
+                            ctx, args, input, filter, output, data_format));
   }
-  auto stream = ctx->op_device_context()->stream();
-  OP_REQUIRES(ctx, stream->ok(),
-              errors::Internal(
-                  "Launch of gpu kernel for DepthwiseConv2dGPULaunch failed"));
 }
 
 template struct LaunchDepthwiseConvOp;
@@ -904,11 +962,11 @@ __global__ void __launch_bounds__(640, 2)
 
 template 
-void LaunchDepthwiseConv2dBackpropInputGPU(const GpuDevice& device,
-                                           const DepthwiseArgs& args,
-                                           const T* out_backprop,
-                                           const T* filter, T* in_backprop,
-                                           TensorFormat data_format) {
+Status LaunchDepthwiseConv2dBackpropInputGPU(OpKernelContext* ctx,
+                                             const DepthwiseArgs& args,
+                                             const T* out_backprop,
+                                             const T* filter, T* in_backprop,
+                                             TensorFormat data_format) {
   void (*kernel)(const DepthwiseArgs, const T*, const T*, T*, int);
   switch (data_format) {
     case FORMAT_NHWC:
@@ -920,38 +978,39 @@ void LaunchDepthwiseConv2dBackpropInputGPU(const GpuDevice& device,
           T, kKnownFilterWidth, kKnownFilterHeight, kKnownDepthMultiplier>;
       break;
     default:
-      LOG(ERROR) << "FORMAT_" << ToString(data_format) << " is not supported";
-      return;
+      return errors::InvalidArgument("FORMAT_", ToString(data_format),
+                                     " is not supported");
   }
   const int num_in_backprop =
       args.batch * args.in_rows * args.in_cols * args.in_depth;
+  auto device = ctx->eigen_gpu_device();
   CudaLaunchConfig config =
       GetCudaLaunchConfig(num_in_backprop, device, kernel, 0, 0);
   kernel<<>>(
       args, out_backprop, filter, in_backprop, num_in_backprop);
+  return Status::OK();
 }
 
 template 
-void LaunchDepthwiseConv2dBackpropInputGPU(const GpuDevice& device,
-                                           const DepthwiseArgs& args,
-                                           const T* out_backprop,
-                                           const T* filter, T* in_backprop,
-                                           TensorFormat data_format) {
+Status LaunchDepthwiseConv2dBackpropInputGPU(OpKernelContext* ctx,
+                                             const DepthwiseArgs& args,
+                                             const T* out_backprop,
+                                             const T* filter, T* in_backprop,
+                                             TensorFormat data_format) {
   if (args.depth_multiplier == 1) {
     if (CanLaunchDepthwiseConv2dGPUSmall(args)) {
-      LaunchDepthwiseConv2dGPUSmall(
-          device, args, out_backprop, filter, in_backprop, data_format);
-      return;
+      return LaunchDepthwiseConv2dGPUSmall<
+          T, DIRECTION_BACKWARD, kKnownFilterWidth, kKnownFilterHeight>(
+          ctx, args, out_backprop, filter, in_backprop, data_format);
     }
 
-    LaunchDepthwiseConv2dBackpropInputGPU(
-        device, args, out_backprop, filter, in_backprop, data_format);
+    return LaunchDepthwiseConv2dBackpropInputGPU(
+        ctx, args, out_backprop, filter, in_backprop, data_format);
   } else {
-    LaunchDepthwiseConv2dBackpropInputGPU(
-        device, args, out_backprop, filter, in_backprop, data_format);
+    return LaunchDepthwiseConv2dBackpropInputGPU(
+        ctx, args, out_backprop, filter, in_backprop, data_format);
   }
 }
 
@@ -960,19 +1019,15 @@ template 
 void LaunchDepthwiseConvBackpropInputOp::operator()(
     OpKernelContext* ctx, const DepthwiseArgs& args, const T* out_backprop,
     const T* filter, T* in_backprop, TensorFormat data_format) {
-  const GpuDevice& device = ctx->eigen_device();
   if (args.filter_rows == 3 && args.filter_cols == 3) {
-    LaunchDepthwiseConv2dBackpropInputGPU(
-        device, args, out_backprop, filter, in_backprop, data_format);
+    OP_REQUIRES_OK(
+        ctx, LaunchDepthwiseConv2dBackpropInputGPU(
+                 ctx, args, out_backprop, filter, in_backprop, data_format));
   } else {
-    LaunchDepthwiseConv2dBackpropInputGPU(
-        device, args, out_backprop, filter, in_backprop, data_format);
+    OP_REQUIRES_OK(
+        ctx, LaunchDepthwiseConv2dBackpropInputGPU(
+                 ctx, args, out_backprop, filter, in_backprop, data_format));
   }
-  auto stream = ctx->op_device_context()->stream();
-  OP_REQUIRES(ctx, stream->ok(),
-              errors::Internal("Launch of gpu kernel for "
-                               "DepthwiseConv2dBackpropInp"
-                               "utGPULaunch failed"));
 }
 
 template struct LaunchDepthwiseConvBackpropInputOp;
@@ -1111,15 +1166,18 @@ __device__ __forceinline__ T WarpSumReduce(T val) {
 // up in global memory using atomics.
 // Requirements: threads per block must be multiple of 32 and <= launch_bounds,
 // kAccumPixels * 64 >= args.in_rows * args.in_cols * kBlockDepth.
+// T is the tensors' data type. S is the math type the kernel uses. This is the
+// same as T for all cases but pseudo half (which has T=Eigen::half, S=float).
 template 
+          int kBlockDepth, int kAccumPixels, typename S>
 __global__
 __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNHWCSmall(
     const DepthwiseArgs args, const T* output, const T* input, T* filter) {
   assert(CanLaunchDepthwiseConv2dBackpropFilterGPUSmall(args, blockDim.z));
   // Holds block plus halo and filter data for blockDim.x depths.
-  extern __shared__ __align__(sizeof(T)) unsigned char shared_memory[];
-  T* const shared_data = reinterpret_cast(shared_memory);
+  extern __shared__ __align__(8) unsigned char shared_memory[];
+  static_assert(sizeof(S) <= 8, "Insufficient alignement detected");
+  S* const shared_data = reinterpret_cast(shared_memory);
 
   const int num_batches = args.batch;
   const int in_height = args.in_rows;
@@ -1169,7 +1227,7 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNHWCSmall(
 
   // Initialize tile, in particular the padding and accumulator.
   for (int i = thread_idx; i < tile_size + accum_size; i += block_size) {
-    shared_data[i] = T(0);
+    shared_data[i] = S();
   }
   __syncthreads();
 
@@ -1203,10 +1261,10 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNHWCSmall(
 
     if (channel_in_range) {
       const T* const in_ptr = inout_offset + input;
-      T* const tile_ptr = tile_idx + shared_data;
-      tile_ptr[0] = ldg(in_ptr);
+      S* const tile_ptr = tile_idx + shared_data;
+      tile_ptr[0] = static_cast(ldg(in_ptr));
       if (!skip_second) {
-        tile_ptr[tile_offset] = ldg(tensor_offset + in_ptr);
+        tile_ptr[tile_offset] = static_cast(ldg(tensor_offset + in_ptr));
       }
     }
 
@@ -1216,14 +1274,15 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNHWCSmall(
 
     if (channel_in_range) {
       const T* const out_ptr = inout_offset + output;
-      const T out1 = ldg(out_ptr);
-      const T out2 = skip_second ? T(0) : ldg(tensor_offset + out_ptr);
+      const S out1 = static_cast(ldg(out_ptr));
+      const S out2 =
+          skip_second ? S() : static_cast(ldg(tensor_offset + out_ptr));
       int shared_offset = data_idx;
-      T* accum_ptr = accum_offset + shared_data;
+      S* accum_ptr = accum_offset + shared_data;
       UNROLL for (int r = 0; r < filter_height; ++r) {
         UNROLL for (int c = 0; c < filter_width; ++c) {
-          const T* const tile_ptr = shared_offset + shared_data;
-          T val = out1 * tile_ptr[0] + out2 * tile_ptr[tile_offset];
+          const S* const tile_ptr = shared_offset + shared_data;
+          S val = out1 * tile_ptr[0] + out2 * tile_ptr[tile_offset];
           // Warp-accumulate pixels of the same depth and write to accumulator.
           for (int delta = 16; delta >= kBlockDepth; delta /= 2) {
             val += CudaShuffleXorSync(active_threads, val, delta);
@@ -1241,18 +1300,18 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNHWCSmall(
     // Note: the condition to reach this is uniform across the entire block.
     __syncthreads();
 
-    const T* const accum_data = tile_size + shared_data;
+    const S* const accum_data = tile_size + shared_data;
     for (int i = thread_idx; i < accum_size; i += block_size) {
       const int filter_idx = i / kAccumPixels;
       const int filter_pix = filter_idx / kBlockDepth;
       const int filter_channel = filter_idx % kBlockDepth + start_channel;
       const int filter_offset = filter_pix * in_depth + filter_channel;
       if (filter_channel < in_depth) {
-        T val = accum_data[i];
+        S val = accum_data[i];
         // Warp-accumulate the pixels of the same depth from the accumulator.
         val = WarpSumReduce(val);
         if (!(thread_idx & kAccumPixels - 1)) {
-          CudaAtomicAdd(filter_offset + filter, val);
+          CudaAtomicAdd(filter_offset + filter, static_cast(val));
         }
       }
     }
@@ -1382,14 +1441,15 @@ __global__ void __launch_bounds__(640, 2)
 // Requirements: threads per block must be multiple of 32 and <= launch_bounds,
 // kAccumPixels * 64 >= args.in_rows * args.in_cols * kBlockDepth.
 template 
+          int kBlockDepth, int kAccumPixels, typename S>
 __global__
 __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNCHWSmall(
     const DepthwiseArgs args, const T* output, const T* input, T* filter) {
   assert(CanLaunchDepthwiseConv2dBackpropFilterGPUSmall(args, blockDim.x));
   // Holds block plus halo and filter data for blockDim.z depths.
-  extern __shared__ __align__(sizeof(T)) unsigned char shared_memory[];
-  T* const shared_data = reinterpret_cast(shared_memory);
+  extern __shared__ __align__(8) unsigned char shared_memory[];
+  static_assert(sizeof(S) <= 8, "Insufficient alignement detected");
+  S* const shared_data = reinterpret_cast(shared_memory);
 
   const int num_batches = args.batch;
   const int in_height = args.in_rows;
@@ -1438,7 +1498,7 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNCHWSmall(
 
   // Initialize tile, in particular the padding and accumulator.
   for (int i = thread_idx; i < tile_size + accum_size; i += block_size) {
-    shared_data[i] = T(0);
+    shared_data[i] = S();
   }
   __syncthreads();
 
@@ -1468,10 +1528,10 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNCHWSmall(
 
     if (channel_in_range) {
       const T* const in_ptr = inout_offset + input;
-      T* const tile_ptr = tile_idx + shared_data;
-      tile_ptr[0] = ldg(in_ptr);
+      S* const tile_ptr = tile_idx + shared_data;
+      tile_ptr[0] = static_cast(ldg(in_ptr));
       if (!skip_second) {
-        tile_ptr[tile_offset] = ldg(block_pixels + in_ptr);
+        tile_ptr[tile_offset] = static_cast(ldg(block_pixels + in_ptr));
       }
     }
 
@@ -1481,14 +1541,15 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNCHWSmall(
 
     if (channel_in_range) {
       const T* const out_ptr = inout_offset + output;
-      const T out1 = ldg(out_ptr);
-      const T out2 = skip_second ? T(0) : ldg(block_pixels + out_ptr);
+      const S out1 = static_cast(ldg(out_ptr));
+      const S out2 =
+          skip_second ? S() : static_cast(ldg(block_pixels + out_ptr));
       int shared_offset = data_idx;
-      T* accum_ptr = accum_offset + shared_data;
+      S* accum_ptr = accum_offset + shared_data;
       UNROLL for (int r = 0; r < filter_height; ++r) {
         UNROLL for (int c = 0; c < filter_width; ++c) {
-          const T* const tile_ptr = shared_offset + shared_data;
-          T val = out1 * tile_ptr[0] + out2 * tile_ptr[tile_offset];
+          const S* const tile_ptr = shared_offset + shared_data;
+          S val = out1 * tile_ptr[0] + out2 * tile_ptr[tile_offset];
           // Warp-accumulate pixels of the same depth and write to accumulator.
           for (int delta = 16 / kBlockDepth; delta > 0; delta /= 2) {
             val += CudaShuffleXorSync(active_threads, val, delta);
@@ -1506,7 +1567,7 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNCHWSmall(
     // Note: the condition to reach this is uniform across the entire block.
     __syncthreads();
 
-    const T* const accum_data = tile_size + shared_data;
+    const S* const accum_data = tile_size + shared_data;
     for (int i = thread_idx; i < accum_size; i += block_size) {
       const int filter_idx = i / kAccumPixels;
       const int filter_pix = filter_idx / kBlockDepth;
@@ -1514,11 +1575,11 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNCHWSmall(
           (channel + filter_idx % kBlockDepth) % in_depth;
       const int filter_offset = filter_pix * in_depth + filter_channel;
       if (filter_channel < in_depth) {
-        T val = accum_data[i];
+        S val = accum_data[i];
         // Warp-accumulate pixels of the same depth from the accumulator.
         val = WarpSumReduce(val);
         if (!(thread_idx & kAccumPixels - 1)) {
-          CudaAtomicAdd(filter_offset + filter, val);
+          CudaAtomicAdd(filter_offset + filter, static_cast(val));
         }
       }
     }
@@ -1526,19 +1587,20 @@ __launch_bounds__(1024, 2) void DepthwiseConv2dBackpropFilterGPUKernelNCHWSmall(
 }
 
 template 
-bool TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
-    const GpuDevice& device, const DepthwiseArgs& args, const int block_height,
+          int kBlockDepth, int kAccumPixels, typename S>
+Status TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
+    OpKernelContext* ctx, const DepthwiseArgs& args, const int block_height,
     const T* out_backprop, const T* input, T* filter_backprop,
     TensorFormat data_format) {
+  auto device = ctx->eigen_gpu_device();
   const int tile_width = args.in_cols + args.filter_cols - 1;
   const int tile_height = block_height * 2 + args.filter_rows - 1;
   const int tile_pixels = tile_height * tile_width;
   const int filter_pixels = args.filter_rows * args.filter_cols;
   const int shared_memory_size =
-      kBlockDepth * (tile_pixels + filter_pixels * kAccumPixels) * sizeof(T);
+      kBlockDepth * (tile_pixels + filter_pixels * kAccumPixels) * sizeof(S);
   if (shared_memory_size > device.sharedMemPerBlock()) {
-    return false;
+    return errors::FailedPrecondition("Not enough shared memory");
   }
 
   dim3 block_dim;
@@ -1550,18 +1612,20 @@ bool TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
       block_count =
           args.batch * DivUp(args.out_depth, kBlockDepth) * kBlockDepth;
       kernel = DepthwiseConv2dBackpropFilterGPUKernelNHWCSmall<
-          T, kKnownFilterWidth, kKnownFilterHeight, kBlockDepth, kAccumPixels>;
+          T, kKnownFilterWidth, kKnownFilterHeight, kBlockDepth, kAccumPixels,
+          S>;
       break;
     case FORMAT_NCHW:
       block_dim = dim3(args.in_cols, block_height, kBlockDepth);
       block_count =
           DivUp(args.batch * args.out_depth, kBlockDepth) * kBlockDepth;
       kernel = DepthwiseConv2dBackpropFilterGPUKernelNCHWSmall<
-          T, kKnownFilterWidth, kKnownFilterHeight, kBlockDepth, kAccumPixels>;
+          T, kKnownFilterWidth, kKnownFilterHeight, kBlockDepth, kAccumPixels,
+          S>;
       break;
     default:
-      LOG(ERROR) << "FORMAT_" << ToString(data_format) << " is not supported";
-      return false;
+      return errors::InvalidArgument("FORMAT_", ToString(data_format),
+                                     " is not supported");
   }
   const int num_out_backprop = args.out_rows * args.out_cols * block_count;
   CudaLaunchConfig config = GetCudaLaunchConfigFixedBlockSize(
@@ -1569,13 +1633,33 @@ bool TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
       block_dim.x * block_dim.y * block_dim.z);
   kernel<<>>(args, out_backprop, input, filter_backprop);
-  return true;
+  return Status::OK();
+}
+
+template 
+Status TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
+    OpKernelContext* ctx, const DepthwiseArgs& args, const int block_height,
+    const T* out_backprop, const T* input, T* filter_backprop,
+    TensorFormat data_format) {
+#if !defined __CUDA_ARCH__ || __CUDA_ARCH__ >= 530
+  if (HasFastHalfMath(ctx)) {
+    return TryLaunchDepthwiseConv2dBackpropFilterGPUSmall<
+        T, kKnownFilterWidth, kKnownFilterHeight, kBlockDepth, kAccumPixels, T>(
+        ctx, args, block_height, out_backprop, input, filter_backprop,
+        data_format);
+  }
+#endif
+  return TryLaunchDepthwiseConv2dBackpropFilterGPUSmall<
+      T, kKnownFilterWidth, kKnownFilterHeight, kBlockDepth, kAccumPixels,
+      PseudoHalfType>(ctx, args, block_height, out_backprop, input,
+                         filter_backprop, data_format);
 }
 
 template 
-bool TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
-    const GpuDevice& device, const DepthwiseArgs& args, const int block_height,
+Status TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
+    OpKernelContext* ctx, const DepthwiseArgs& args, const int block_height,
     const T* out_backprop, const T* input, T* filter_backprop,
     TensorFormat data_format) {
   // Minimize (power of two) kAccumPixels, while satisfying
@@ -1584,24 +1668,24 @@ bool TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
   if (block_pixels > 512) {
     return TryLaunchDepthwiseConv2dBackpropFilterGPUSmall<
         T, kKnownFilterWidth, kKnownFilterHeight, kBlockDepth, 32>(
-        device, args, block_height, out_backprop, input, filter_backprop,
+        ctx, args, block_height, out_backprop, input, filter_backprop,
         data_format);
   } else if (block_pixels > 256) {
     return TryLaunchDepthwiseConv2dBackpropFilterGPUSmall<
         T, kKnownFilterWidth, kKnownFilterHeight, kBlockDepth, 16>(
-        device, args, block_height, out_backprop, input, filter_backprop,
+        ctx, args, block_height, out_backprop, input, filter_backprop,
         data_format);
   } else {
     return TryLaunchDepthwiseConv2dBackpropFilterGPUSmall<
         T, kKnownFilterWidth, kKnownFilterHeight, kBlockDepth, 8>(
-        device, args, block_height, out_backprop, input, filter_backprop,
+        ctx, args, block_height, out_backprop, input, filter_backprop,
         data_format);
   }
 }
 
 template 
-bool TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
-    const GpuDevice& device, const DepthwiseArgs& args, const T* out_backprop,
+Status TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
+    OpKernelContext* ctx, const DepthwiseArgs& args, const T* out_backprop,
     const T* input, T* filter_backprop, TensorFormat data_format) {
   // Maximize (power of two) kBlockDepth while keeping a block within 1024
   // threads (2 pixels per thread).
@@ -1621,37 +1705,35 @@ bool TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
   }
 
   if (!CanLaunchDepthwiseConv2dBackpropFilterGPUSmall(args, block_height)) {
-    return false;
+    return errors::FailedPrecondition("Cannot launch this configuration");
   }
 
   switch (block_depth) {
     case 8:
       return TryLaunchDepthwiseConv2dBackpropFilterGPUSmall<
           T, kKnownFilterWidth, kKnownFilterHeight, 8>(
-          device, args, block_height, out_backprop, input, filter_backprop,
+          ctx, args, block_height, out_backprop, input, filter_backprop,
           data_format);
     case 4:
       return TryLaunchDepthwiseConv2dBackpropFilterGPUSmall<
           T, kKnownFilterWidth, kKnownFilterHeight, 4>(
-          device, args, block_height, out_backprop, input, filter_backprop,
+          ctx, args, block_height, out_backprop, input, filter_backprop,
           data_format);
     case 2:
       return TryLaunchDepthwiseConv2dBackpropFilterGPUSmall<
           T, kKnownFilterWidth, kKnownFilterHeight, 2>(
-          device, args, block_height, out_backprop, input, filter_backprop,
+          ctx, args, block_height, out_backprop, input, filter_backprop,
           data_format);
     default:
-      return false;
+      return errors::InvalidArgument("Unexpected block depth");
   }
 }
 
 template 
-void LaunchDepthwiseConv2dBackpropFilterGPU(const GpuDevice& device,
-                                            const DepthwiseArgs& args,
-                                            const T* out_backprop,
-                                            const T* input, T* filter_backprop,
-                                            TensorFormat data_format) {
+Status LaunchDepthwiseConv2dBackpropFilterGPU(
+    OpKernelContext* ctx, const DepthwiseArgs& args, const T* out_backprop,
+    const T* input, T* filter_backprop, TensorFormat data_format) {
   void (*kernel)(const DepthwiseArgs, const T*, const T*, T*, int);
   switch (data_format) {
     case FORMAT_NHWC:
@@ -1663,37 +1745,38 @@ void LaunchDepthwiseConv2dBackpropFilterGPU(const GpuDevice& device,
           T, kKnownFilterWidth, kKnownFilterHeight, kKnownDepthMultiplier>;
       break;
     default:
-      LOG(ERROR) << "FORMAT_" << ToString(data_format) << " is not supported";
-      return;
+      return errors::InvalidArgument("FORMAT_", ToString(data_format),
+                                     " is not supported");
   }
   const int num_out_backprop =
       args.batch * args.out_rows * args.out_cols * args.out_depth;
+  auto device = ctx->eigen_gpu_device();
   CudaLaunchConfig config =
       GetCudaLaunchConfig(num_out_backprop, device, kernel, 0, 0);
   kernel<<>>(
       args, out_backprop, input, filter_backprop, num_out_backprop);
+  return Status::OK();
 }
 
 template 
-void LaunchDepthwiseConv2dBackpropFilterGPU(const GpuDevice& device,
-                                            const DepthwiseArgs& args,
-                                            const T* out_backprop,
-                                            const T* input, T* filter_backprop,
-                                            TensorFormat data_format) {
+Status LaunchDepthwiseConv2dBackpropFilterGPU(
+    OpKernelContext* ctx, const DepthwiseArgs& args, const T* out_backprop,
+    const T* input, T* filter_backprop, TensorFormat data_format) {
   if (args.depth_multiplier == 1) {
     if (TryLaunchDepthwiseConv2dBackpropFilterGPUSmall(
-            device, args, out_backprop, input, filter_backprop, data_format)) {
-      return;
+            ctx, args, out_backprop, input, filter_backprop, data_format)
+            .ok()) {
+      return Status::OK();
     }
 
-    LaunchDepthwiseConv2dBackpropFilterGPU(
-        device, args, out_backprop, input, filter_backprop, data_format);
+    return LaunchDepthwiseConv2dBackpropFilterGPU(
+        ctx, args, out_backprop, input, filter_backprop, data_format);
   } else {
-    LaunchDepthwiseConv2dBackpropFilterGPU(
-        device, args, out_backprop, input, filter_backprop, data_format);
+    return LaunchDepthwiseConv2dBackpropFilterGPU(
+        ctx, args, out_backprop, input, filter_backprop, data_format);
   }
 }
 
@@ -1702,7 +1785,6 @@ template 
 void LaunchDepthwiseConvBackpropFilterOp::operator()(
     OpKernelContext* ctx, const DepthwiseArgs& args, const T* out_backprop,
     const T* input, T* filter_backprop, TensorFormat data_format) {
-  const GpuDevice& device = ctx->eigen_device();
   auto stream = ctx->op_device_context()->stream();
 
   // Initialize the results to 0.
@@ -1712,16 +1794,14 @@ void LaunchDepthwiseConvBackpropFilterOp::operator()(
   stream->ThenMemset32(&filter_bp_ptr, 0, num_filter_backprop * sizeof(T));
 
   if (args.filter_rows == 3 && args.filter_cols == 3) {
-    LaunchDepthwiseConv2dBackpropFilterGPU(
-        device, args, out_backprop, input, filter_backprop, data_format);
+    OP_REQUIRES_OK(
+        ctx, LaunchDepthwiseConv2dBackpropFilterGPU(
+                 ctx, args, out_backprop, input, filter_backprop, data_format));
   } else {
-    LaunchDepthwiseConv2dBackpropFilterGPU(
-        device, args, out_backprop, input, filter_backprop, data_format);
+    OP_REQUIRES_OK(
+        ctx, LaunchDepthwiseConv2dBackpropFilterGPU(
+                 ctx, args, out_backprop, input, filter_backprop, data_format));
   }
-  OP_REQUIRES(ctx, stream->ok(),
-              errors::Internal("Launch of gpu kernel for "
-                               "DepthwiseConv2dBackpropFil"
-                               "terGPULaunch failed"));
 }
 
 template struct LaunchDepthwiseConvBackpropFilterOp;
diff --git a/tensorflow/core/kernels/depthwise_conv_ops_test.cc b/tensorflow/core/kernels/depthwise_conv_ops_test.cc
new file mode 100644
index 0000000000..87bb68a43b
--- /dev/null
+++ b/tensorflow/core/kernels/depthwise_conv_ops_test.cc
@@ -0,0 +1,114 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/cc/ops/const_op.h"
+#include "tensorflow/cc/ops/image_ops.h"
+#include "tensorflow/cc/ops/nn_ops.h"
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/conv_ops_gpu.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/public/session.h"
+
+namespace tensorflow {
+namespace {
+class DepthwiseConvOpTest : public OpsTestBase {
+ protected:
+  enum class Device { CPU, GPU };
+
+  template 
+  void Run(Device device) {
+    if (device == Device::GPU) {
+      SetDevice(DEVICE_GPU,
+                std::unique_ptr(DeviceFactory::NewDevice(
+                    "GPU", {}, "/job:a/replica:0/task:0")));
+    }
+    DataType dtype = DataTypeToEnum::value;
+    TF_EXPECT_OK(NodeDefBuilder("depthwise_conv2d", "DepthwiseConv2dNative")
+                     .Input(FakeInput(dtype))
+                     .Input(FakeInput(dtype))
+                     .Attr("T", dtype)
+                     .Attr("strides", {1, 1, 1, 1})
+                     .Attr("padding", "SAME")
+                     .Finalize(node_def()));
+    TF_EXPECT_OK(InitOp());
+    const int depth = 2;
+    const int image_width = 2;
+    const int image_height = 3;
+    const int batch_count = 1;
+    // The image matrix is ('first/second' channel):
+    // | 1/2  |  3/4  |
+    // | 5/6  |  7/8  |
+    // | 9/10 | 11/12 |
+    Tensor image(dtype, {batch_count, image_height, image_width, depth});
+    test::FillValues(&image, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
+
+    // The filter matrix is:
+    // | 1/2 |  7/8  | 13/14 |
+    // | 3/4 |  9/10 | 15/16 |
+    // | 5/6 | 11/12 | 17/18 |
+    const int filter_size = 3;
+    const int filter_count = 1;
+    Tensor filter(dtype, {filter_size, filter_size, depth, filter_count});
+    test::FillValues(&filter, {1, 2, 7, 8, 13, 14, 3, 4, 9, 10, 15, 16, 5, 6,
+                                  11, 12, 17, 18});
+
+    AddInputFromArray(image.shape(), image.flat());
+    AddInputFromArray(filter.shape(), filter.flat());
+    TF_ASSERT_OK(RunOpKernel());
+
+    // We're sliding two 3x3 filters across the 3x2 image, with accesses outside
+    // the input set to zero because we're using the 'SAME' padding mode.
+    // This means we should end up with this matrix:
+    // | 105/150 | 183/95  |
+    // | 235/312 | 357/178 |
+    // | 187/234 | 261/121 |
+    Tensor expected(dtype, image.shape());
+    test::FillValues(&expected, {228, 300, 132, 180, 482, 596, 266, 344, 372,
+                                    452, 180, 236});
+    const Tensor& output = *GetOutput(0);
+    // TODO(csigg): This should happen as part of GetOutput.
+    TF_EXPECT_OK(device_->Sync());
+    test::ExpectTensorNear(expected, output, 1e-5);
+  }
+};
+
+TEST_F(DepthwiseConvOpTest, DepthwiseConvFloatCpu) { Run(Device::CPU); }
+TEST_F(DepthwiseConvOpTest, DepthwiseConvDoubleCpu) {
+  Run(Device::CPU);
+}
+TEST_F(DepthwiseConvOpTest, DepthwiseConvHalfCpu) {
+  Run(Device::CPU);
+}
+
+#ifdef GOOGLE_CUDA
+TEST_F(DepthwiseConvOpTest, DepthwiseConvFloatGpu) { Run(Device::GPU); }
+TEST_F(DepthwiseConvOpTest, DepthwiseConvDoubleGpu) {
+  Run(Device::GPU);
+}
+TEST_F(DepthwiseConvOpTest, DepthwiseConvHalfGpu) {
+  Run(Device::GPU);
+}
+#endif
+
+}  // namespace
+}  // namespace tensorflow
-- 
cgit v1.2.3


From fc26a5829c668ea49187a989e6b9657b6b8b1f02 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 09:22:19 -0700
Subject: Benchmark for densenet model under eager and graph execution.

PiperOrigin-RevId: 204922359
---
 .../contrib/eager/python/examples/densenet/BUILD   |  19 ++
 .../eager/python/examples/densenet/densenet.py     |  48 ++--
 .../examples/densenet/densenet_graph_test.py       | 149 +++++++++++++
 .../python/examples/densenet/densenet_test.py      | 241 ++++++++++++++++++++-
 4 files changed, 437 insertions(+), 20 deletions(-)
 create mode 100644 tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py

diff --git a/tensorflow/contrib/eager/python/examples/densenet/BUILD b/tensorflow/contrib/eager/python/examples/densenet/BUILD
index de2a817d17..2dc196f550 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/densenet/BUILD
@@ -16,6 +16,7 @@ py_binary(
 
 cuda_py_test(
     name = "densenet_test",
+    size = "large",
     srcs = ["densenet_test.py"],
     additional_deps = [
         ":densenet",
@@ -27,3 +28,21 @@ cuda_py_test(
         "optonly",
     ],
 )
+
+cuda_py_test(
+    name = "densenet_graph_test",
+    size = "large",
+    srcs = ["densenet_graph_test.py"],
+    additional_deps = [
+        ":densenet",
+        "//third_party/py/numpy",
+        "//tensorflow:tensorflow_py",
+    ],
+    tags = [
+        "no_pip",
+        "noasan",
+        "nomsan",
+        "notsan",
+        "optonly",
+    ],
+)
diff --git a/tensorflow/contrib/eager/python/examples/densenet/densenet.py b/tensorflow/contrib/eager/python/examples/densenet/densenet.py
index 3a2b2de250..6de4e69400 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/densenet.py
+++ b/tensorflow/contrib/eager/python/examples/densenet/densenet.py
@@ -32,24 +32,28 @@ class ConvBlock(tf.keras.Model):
 
   Arguments:
     num_filters: number of filters passed to a convolutional layer.
+    data_format: "channels_first" or "channels_last"
     bottleneck: if True, then a 1x1 Conv is performed followed by 3x3 Conv.
     weight_decay: weight decay
     dropout_rate: dropout rate.
   """
 
-  def __init__(self, num_filters, bottleneck, weight_decay=1e-4,
+  def __init__(self, num_filters, data_format, bottleneck, weight_decay=1e-4,
                dropout_rate=0):
     super(ConvBlock, self).__init__()
     self.bottleneck = bottleneck
+
+    axis = -1 if data_format == "channels_last" else 1
     inter_filter = num_filters * 4
     # don't forget to set use_bias=False when using batchnorm
     self.conv2 = tf.keras.layers.Conv2D(num_filters,
                                         (3, 3),
                                         padding="same",
                                         use_bias=False,
+                                        data_format=data_format,
                                         kernel_initializer="he_normal",
                                         kernel_regularizer=l2(weight_decay))
-    self.batchnorm1 = tf.keras.layers.BatchNormalization()
+    self.batchnorm1 = tf.keras.layers.BatchNormalization(axis=axis)
     self.dropout = tf.keras.layers.Dropout(dropout_rate)
 
     if self.bottleneck:
@@ -57,9 +61,10 @@ class ConvBlock(tf.keras.Model):
                                           (1, 1),
                                           padding="same",
                                           use_bias=False,
+                                          data_format=data_format,
                                           kernel_initializer="he_normal",
                                           kernel_regularizer=l2(weight_decay))
-      self.batchnorm2 = tf.keras.layers.BatchNormalization()
+      self.batchnorm2 = tf.keras.layers.BatchNormalization(axis=axis)
 
   def call(self, x, training=True):
     output = self.batchnorm1(x, training=training)
@@ -79,20 +84,25 @@ class TransitionBlock(tf.keras.Model):
 
   Arguments:
     num_filters: number of filters passed to a convolutional layer.
+    data_format: "channels_first" or "channels_last"
     weight_decay: weight decay
     dropout_rate: dropout rate.
   """
 
-  def __init__(self, num_filters, weight_decay=1e-4, dropout_rate=0):
+  def __init__(self, num_filters, data_format,
+               weight_decay=1e-4, dropout_rate=0):
     super(TransitionBlock, self).__init__()
-    self.batchnorm = tf.keras.layers.BatchNormalization()
+    axis = -1 if data_format == "channels_last" else 1
+
+    self.batchnorm = tf.keras.layers.BatchNormalization(axis=axis)
     self.conv = tf.keras.layers.Conv2D(num_filters,
                                        (1, 1),
                                        padding="same",
                                        use_bias=False,
+                                       data_format=data_format,
                                        kernel_initializer="he_normal",
                                        kernel_regularizer=l2(weight_decay))
-    self.avg_pool = tf.keras.layers.AveragePooling2D()
+    self.avg_pool = tf.keras.layers.AveragePooling2D(data_format=data_format)
 
   def call(self, x, training=True):
     output = self.batchnorm(x, training=training)
@@ -108,19 +118,22 @@ class DenseBlock(tf.keras.Model):
   Arguments:
     num_layers: Number of layers in each block.
     growth_rate: number of filters to add per conv block.
+    data_format: "channels_first" or "channels_last"
     bottleneck: boolean, that decides which part of ConvBlock to call.
     weight_decay: weight decay
     dropout_rate: dropout rate.
   """
 
-  def __init__(self, num_layers, growth_rate, bottleneck,
+  def __init__(self, num_layers, growth_rate, data_format, bottleneck,
                weight_decay=1e-4, dropout_rate=0):
     super(DenseBlock, self).__init__()
     self.num_layers = num_layers
+    self.axis = -1 if data_format == "channels_last" else 1
 
     self.blocks = []
     for _ in range(int(self.num_layers)):
       self.blocks.append(ConvBlock(growth_rate,
+                                   data_format,
                                    bottleneck,
                                    weight_decay,
                                    dropout_rate))
@@ -128,7 +141,7 @@ class DenseBlock(tf.keras.Model):
   def call(self, x, training=True):
     for i in range(int(self.num_layers)):
       output = self.blocks[i](x, training=training)
-      x = tf.concat([x, output], axis=-1)
+      x = tf.concat([x, output], axis=self.axis)
 
     return x
 
@@ -146,6 +159,7 @@ class DenseNet(tf.keras.Model):
                               If positive integer, then the it is used as the
                                 number of layers per block.
                               If list or tuple, then this list is used directly.
+    data_format: "channels_first" or "channels_last"
     bottleneck: boolean, to decide which part of conv block to call.
     compression: reducing the number of inputs(filters) to the transition block.
     weight_decay: weight decay
@@ -157,7 +171,7 @@ class DenseNet(tf.keras.Model):
   """
 
   def __init__(self, depth_of_model, growth_rate, num_of_blocks,
-               output_classes, num_layers_in_each_block,
+               output_classes, num_layers_in_each_block, data_format,
                bottleneck=True, compression=0.5, weight_decay=1e-4,
                dropout_rate=0, pool_initial=False, include_top=True):
     super(DenseNet, self).__init__()
@@ -166,6 +180,7 @@ class DenseNet(tf.keras.Model):
     self.num_of_blocks = num_of_blocks
     self.output_classes = output_classes
     self.num_layers_in_each_block = num_layers_in_each_block
+    self.data_format = data_format
     self.bottleneck = bottleneck
     self.compression = compression
     self.weight_decay = weight_decay
@@ -193,6 +208,8 @@ class DenseNet(tf.keras.Model):
         self.num_layers_in_each_block = [
             self.num_layers_in_each_block] * self.num_of_blocks
 
+    axis = -1 if self.data_format == "channels_last" else 1
+
     # setting the filters and stride of the initial covn layer.
     if self.pool_initial:
       init_filters = (7, 7)
@@ -209,20 +226,23 @@ class DenseNet(tf.keras.Model):
                                         strides=stride,
                                         padding="same",
                                         use_bias=False,
+                                        data_format=self.data_format,
                                         kernel_initializer="he_normal",
                                         kernel_regularizer=l2(
                                             self.weight_decay))
     if self.pool_initial:
       self.pool1 = tf.keras.layers.MaxPooling2D(pool_size=(3, 3),
                                                 strides=(2, 2),
-                                                padding="same")
-      self.batchnorm1 = tf.keras.layers.BatchNormalization()
+                                                padding="same",
+                                                data_format=self.data_format)
+      self.batchnorm1 = tf.keras.layers.BatchNormalization(axis=axis)
 
-    self.batchnorm2 = tf.keras.layers.BatchNormalization()
+    self.batchnorm2 = tf.keras.layers.BatchNormalization(axis=axis)
 
     # last pooling and fc layer
     if self.include_top:
-      self.last_pool = tf.keras.layers.GlobalAveragePooling2D()
+      self.last_pool = tf.keras.layers.GlobalAveragePooling2D(
+          data_format=self.data_format)
       self.classifier = tf.keras.layers.Dense(self.output_classes)
 
     # calculating the number of filters after each block
@@ -241,12 +261,14 @@ class DenseNet(tf.keras.Model):
     for i in range(self.num_of_blocks):
       self.dense_blocks.append(DenseBlock(self.num_layers_in_each_block[i],
                                           self.growth_rate,
+                                          self.data_format,
                                           self.bottleneck,
                                           self.weight_decay,
                                           self.dropout_rate))
       if i+1 < self.num_of_blocks:
         self.transition_blocks.append(
             TransitionBlock(num_filters_after_each_block[i+1],
+                            self.data_format,
                             self.weight_decay,
                             self.dropout_rate))
 
diff --git a/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py b/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py
new file mode 100644
index 0000000000..bd0057fb1a
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/densenet/densenet_graph_test.py
@@ -0,0 +1,149 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests and Benchmarks for Densenet model under graph execution."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.contrib.eager.python.examples.densenet import densenet
+
+
+def data_format():
+  return 'channels_first' if tf.test.is_gpu_available() else 'channels_last'
+
+
+def image_shape(batch_size):
+  if data_format() == 'channels_first':
+    return [batch_size, 3, 224, 224]
+  return [batch_size, 224, 224, 3]
+
+
+def random_batch(batch_size):
+  images = np.random.rand(*image_shape(batch_size)).astype(np.float32)
+  num_classes = 1000
+  labels = np.random.randint(
+      low=0, high=num_classes, size=[batch_size]).astype(np.int32)
+  one_hot = np.zeros((batch_size, num_classes)).astype(np.float32)
+  one_hot[np.arange(batch_size), labels] = 1.
+  return images, one_hot
+
+
+class DensenetGraphTest(tf.test.TestCase):
+
+  def testApply(self):
+    depth = 7
+    growth_rate = 2
+    num_blocks = 3
+    output_classes = 10
+    num_layers_in_each_block = -1
+    batch_size = 1
+    with tf.Graph().as_default():
+      images = tf.placeholder(tf.float32, image_shape(None))
+      model = densenet.DenseNet(depth, growth_rate, num_blocks,
+                                output_classes, num_layers_in_each_block,
+                                data_format(), bottleneck=True, compression=0.5,
+                                weight_decay=1e-4, dropout_rate=0,
+                                pool_initial=False, include_top=True)
+      predictions = model(images, training=False)
+
+      init = tf.global_variables_initializer()
+
+      with tf.Session() as sess:
+        sess.run(init)
+        np_images, _ = random_batch(batch_size)
+        out = sess.run(predictions, feed_dict={images: np_images})
+        self.assertAllEqual([batch_size, output_classes], out.shape)
+
+
+class DensenetBenchmark(tf.test.Benchmark):
+
+  def __init__(self):
+    self.depth = 121
+    self.growth_rate = 32
+    self.num_blocks = 4
+    self.output_classes = 1000
+    self.num_layers_in_each_block = [6, 12, 24, 16]
+
+  def _report(self, label, start, num_iters, batch_size):
+    avg_time = (time.time() - start) / num_iters
+    dev = 'gpu' if tf.test.is_gpu_available() else 'cpu'
+    name = 'graph_%s_%s_batch_%d_%s' % (label, dev, batch_size, data_format())
+    extras = {'examples_per_sec': batch_size / avg_time}
+    self.report_benchmark(
+        iters=num_iters, wall_time=avg_time, name=name, extras=extras)
+
+  def benchmark_graph_apply(self):
+    with tf.Graph().as_default():
+      images = tf.placeholder(tf.float32, image_shape(None))
+      model = densenet.DenseNet(self.depth, self.growth_rate, self.num_blocks,
+                                self.output_classes,
+                                self.num_layers_in_each_block, data_format(),
+                                bottleneck=True, compression=0.5,
+                                weight_decay=1e-4, dropout_rate=0,
+                                pool_initial=True, include_top=True)
+      predictions = model(images, training=False)
+
+      init = tf.global_variables_initializer()
+
+      batch_size = 64
+      with tf.Session() as sess:
+        sess.run(init)
+        np_images, _ = random_batch(batch_size)
+        num_burn, num_iters = (3, 30)
+        for _ in range(num_burn):
+          sess.run(predictions, feed_dict={images: np_images})
+        start = time.time()
+        for _ in range(num_iters):
+          sess.run(predictions, feed_dict={images: np_images})
+        self._report('apply', start, num_iters, batch_size)
+
+  def benchmark_graph_train(self):
+    for batch_size in [16, 32, 64]:
+      with tf.Graph().as_default():
+        np_images, np_labels = random_batch(batch_size)
+        dataset = tf.data.Dataset.from_tensors((np_images, np_labels)).repeat()
+        (images, labels) = dataset.make_one_shot_iterator().get_next()
+
+        model = densenet.DenseNet(self.depth, self.growth_rate, self.num_blocks,
+                                  self.output_classes,
+                                  self.num_layers_in_each_block, data_format(),
+                                  bottleneck=True, compression=0.5,
+                                  weight_decay=1e-4, dropout_rate=0,
+                                  pool_initial=True, include_top=True)
+        logits = model(images, training=True)
+        loss = tf.losses.softmax_cross_entropy(
+            logits=logits, onehot_labels=labels)
+        optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)
+        train_op = optimizer.minimize(loss)
+
+        init = tf.global_variables_initializer()
+        with tf.Session() as sess:
+          sess.run(init)
+          (num_burn, num_iters) = (5, 10)
+          for _ in range(num_burn):
+            sess.run(train_op)
+          start = time.time()
+          for _ in range(num_iters):
+            sess.run(train_op)
+          self._report('train', start, num_iters, batch_size)
+
+
+if __name__ == '__main__':
+  tf.test.main()
diff --git a/tensorflow/contrib/eager/python/examples/densenet/densenet_test.py b/tensorflow/contrib/eager/python/examples/densenet/densenet_test.py
index 56d3362f3b..4f19711fb8 100644
--- a/tensorflow/contrib/eager/python/examples/densenet/densenet_test.py
+++ b/tensorflow/contrib/eager/python/examples/densenet/densenet_test.py
@@ -12,14 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for various Densenet architectures."""
+"""Tests and Benchmarks for Densenet model."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import gc
+import time
 import tensorflow as tf
+import tensorflow.contrib.eager as tfe
+
 from tensorflow.contrib.eager.python.examples.densenet import densenet
+from tensorflow.python.client import device_lib
 
 
 class DensenetTest(tf.test.TestCase):
@@ -31,14 +36,19 @@ class DensenetTest(tf.test.TestCase):
     output_classes = 10
     num_layers_in_each_block = -1
     batch_size = 1
+    data_format = ('channels_first') if tf.test.is_gpu_available() else (
+        'channels_last')
 
     model = densenet.DenseNet(depth, growth_rate, num_blocks,
                               output_classes, num_layers_in_each_block,
-                              bottleneck=True, compression=0.5,
+                              data_format, bottleneck=True, compression=0.5,
                               weight_decay=1e-4, dropout_rate=0,
                               pool_initial=False, include_top=True)
 
-    rand_input = tf.random_uniform((batch_size, 32, 32, 3))
+    if data_format == 'channels_last':
+      rand_input = tf.random_uniform((batch_size, 32, 32, 3))
+    else:
+      rand_input = tf.random_uniform((batch_size, 3, 32, 32))
     output_shape = model(rand_input).shape
     self.assertEqual(output_shape, (batch_size, output_classes))
 
@@ -49,14 +59,19 @@ class DensenetTest(tf.test.TestCase):
     output_classes = 10
     num_layers_in_each_block = -1
     batch_size = 1
+    data_format = ('channels_first') if tf.test.is_gpu_available() else (
+        'channels_last')
 
     model = densenet.DenseNet(depth, growth_rate, num_blocks,
                               output_classes, num_layers_in_each_block,
-                              bottleneck=False, compression=0.5,
+                              data_format, bottleneck=False, compression=0.5,
                               weight_decay=1e-4, dropout_rate=0,
                               pool_initial=False, include_top=True)
 
-    rand_input = tf.random_uniform((batch_size, 32, 32, 3))
+    if data_format == 'channels_last':
+      rand_input = tf.random_uniform((batch_size, 32, 32, 3))
+    else:
+      rand_input = tf.random_uniform((batch_size, 3, 32, 32))
     output_shape = model(rand_input).shape
     self.assertEqual(output_shape, (batch_size, output_classes))
 
@@ -67,17 +82,229 @@ class DensenetTest(tf.test.TestCase):
     output_classes = 10
     num_layers_in_each_block = [1, 2, 2, 1]
     batch_size = 1
+    data_format = ('channels_first') if tf.test.is_gpu_available() else (
+        'channels_last')
 
     model = densenet.DenseNet(depth, growth_rate, num_blocks,
                               output_classes, num_layers_in_each_block,
-                              bottleneck=True, compression=0.5,
+                              data_format, bottleneck=True, compression=0.5,
                               weight_decay=1e-4, dropout_rate=0,
                               pool_initial=True, include_top=True)
 
-    rand_input = tf.random_uniform((batch_size, 32, 32, 3))
+    if data_format == 'channels_last':
+      rand_input = tf.random_uniform((batch_size, 32, 32, 3))
+    else:
+      rand_input = tf.random_uniform((batch_size, 3, 32, 32))
     output_shape = model(rand_input).shape
     self.assertEqual(output_shape, (batch_size, output_classes))
 
+
+def compute_gradients(model, images, labels):
+  with tf.GradientTape() as tape:
+    logits = model(images, training=True)
+    loss = tf.losses.softmax_cross_entropy(
+        logits=logits, onehot_labels=labels)
+    tf.contrib.summary.scalar(name='loss', tensor=loss)
+  return tape.gradient(loss, model.variables)
+
+
+def apply_gradients(model, optimizer, gradients):
+  optimizer.apply_gradients(zip(gradients, model.variables))
+
+
+def device_and_data_format():
+  return ('/gpu:0',
+          'channels_first') if tf.test.is_gpu_available() else ('/cpu:0',
+                                                                'channels_last')
+
+
+def random_batch(batch_size, data_format):
+  shape = (3, 224, 224) if data_format == 'channels_first' else (224, 224, 3)
+  shape = (batch_size,) + shape
+
+  num_classes = 1000
+  images = tf.random_uniform(shape)
+  labels = tf.random_uniform(
+      [batch_size], minval=0, maxval=num_classes, dtype=tf.int32)
+  one_hot = tf.one_hot(labels, num_classes)
+
+  return images, one_hot
+
+
+class MockIterator(object):
+
+  def __init__(self, tensors):
+    self._tensors = [tf.identity(x) for x in tensors]
+
+  def next(self):
+    return self._tensors
+
+
+class DensenetBenchmark(tf.test.Benchmark):
+
+  def __init__(self):
+    self.depth = 121
+    self.growth_rate = 32
+    self.num_blocks = 4
+    self.output_classes = 1000
+    self.num_layers_in_each_block = [6, 12, 24, 16]
+
+  def _train_batch_sizes(self):
+    """Choose batch sizes based on GPU capability."""
+    for device in device_lib.list_local_devices():
+      if tf.DeviceSpec.from_string(device.name).device_type == 'GPU':
+        if 'K20' in device.physical_device_desc:
+          return (16,)
+        if 'P100' in device.physical_device_desc:
+          return (16, 32, 64)
+
+      if tf.DeviceSpec.from_string(device.name).device_type == 'TPU':
+        return (32,)
+    return (16, 32)
+
+  def _report(self, label, start, num_iters, device, batch_size, data_format):
+    avg_time = (time.time() - start) / num_iters
+    dev = tf.DeviceSpec.from_string(device).device_type.lower()
+    name = '%s_%s_batch_%d_%s' % (label, dev, batch_size, data_format)
+    extras = {'examples_per_sec': batch_size / avg_time}
+    self.report_benchmark(
+        iters=num_iters, wall_time=avg_time, name=name, extras=extras)
+
+  def _force_device_sync(self):
+    # If this function is called in the context of a non-CPU device
+    # (e.g., inside a 'with tf.device("/gpu:0")' block)
+    # then this will force a copy from CPU->NON_CPU_DEVICE->CPU,
+    # which forces a sync. This is a roundabout way, yes.
+    tf.constant(1.).cpu()
+
+  def _benchmark_eager_apply(self, label, device_and_format, defun=False,
+                             execution_mode=None, compiled=False):
+    with tfe.execution_mode(execution_mode):
+      device, data_format = device_and_format
+      model = densenet.DenseNet(self.depth, self.growth_rate, self.num_blocks,
+                                self.output_classes,
+                                self.num_layers_in_each_block, data_format,
+                                bottleneck=True, compression=0.5,
+                                weight_decay=1e-4, dropout_rate=0,
+                                pool_initial=True, include_top=True)
+      if defun:
+        model.call = tfe.defun(model.call, compiled=compiled)
+      batch_size = 64
+      num_burn = 5
+      num_iters = 30
+      with tf.device(device):
+        images, _ = random_batch(batch_size, data_format)
+        for _ in xrange(num_burn):
+          model(images, training=False).cpu()
+        if execution_mode:
+          tfe.async_wait()
+        gc.collect()
+        start = time.time()
+        for _ in xrange(num_iters):
+          model(images, training=False).cpu()
+        if execution_mode:
+          tfe.async_wait()
+        self._report(label, start, num_iters, device, batch_size, data_format)
+
+  def benchmark_eager_apply_sync(self):
+    self._benchmark_eager_apply('eager_apply', device_and_data_format(),
+                                defun=False)
+
+  def benchmark_eager_apply_async(self):
+    self._benchmark_eager_apply(
+        'eager_apply_async', device_and_data_format(), defun=False,
+        execution_mode=tfe.ASYNC)
+
+  def benchmark_eager_apply_with_defun(self):
+    self._benchmark_eager_apply('eager_apply_with_defun',
+                                device_and_data_format(), defun=True)
+
+  def _benchmark_eager_train(self,
+                             label,
+                             make_iterator,
+                             device_and_format,
+                             defun=False,
+                             execution_mode=None,
+                             compiled=False):
+    with tfe.execution_mode(execution_mode):
+      device, data_format = device_and_format
+      for batch_size in self._train_batch_sizes():
+        (images, labels) = random_batch(batch_size, data_format)
+        model = densenet.DenseNet(self.depth, self.growth_rate, self.num_blocks,
+                                  self.output_classes,
+                                  self.num_layers_in_each_block, data_format,
+                                  bottleneck=True, compression=0.5,
+                                  weight_decay=1e-4, dropout_rate=0,
+                                  pool_initial=True, include_top=True)
+        optimizer = tf.train.GradientDescentOptimizer(0.1)
+        apply_grads = apply_gradients
+        if defun:
+          model.call = tfe.defun(model.call, compiled=compiled)
+          apply_grads = tfe.defun(apply_gradients, compiled=compiled)
+
+        num_burn = 3
+        num_iters = 10
+        with tf.device(device):
+          iterator = make_iterator((images, labels))
+          for _ in xrange(num_burn):
+            (images, labels) = iterator.next()
+            apply_grads(model, optimizer,
+                        compute_gradients(model, images, labels))
+          if execution_mode:
+            tfe.async_wait()
+          self._force_device_sync()
+          gc.collect()
+
+          start = time.time()
+          for _ in xrange(num_iters):
+            (images, labels) = iterator.next()
+            apply_grads(model, optimizer,
+                        compute_gradients(model, images, labels))
+          if execution_mode:
+            tfe.async_wait()
+          self._force_device_sync()
+          self._report(label, start, num_iters, device, batch_size, data_format)
+
+  def benchmark_eager_train_sync(self):
+    self._benchmark_eager_train('eager_train', MockIterator,
+                                device_and_data_format(), defun=False)
+
+  def benchmark_eager_train_async(self):
+    self._benchmark_eager_train(
+        'eager_train_async',
+        MockIterator,
+        device_and_data_format(),
+        defun=False,
+        execution_mode=tfe.ASYNC)
+
+  def benchmark_eager_train_with_defun(self):
+    self._benchmark_eager_train(
+        'eager_train_with_defun', MockIterator,
+        device_and_data_format(), defun=True)
+
+  def benchmark_eager_train_datasets(self):
+
+    def make_iterator(tensors):
+      with tf.device('/device:CPU:0'):
+        ds = tf.data.Dataset.from_tensors(tensors).repeat()
+      return tfe.Iterator(ds)
+
+    self._benchmark_eager_train(
+        'eager_train_dataset', make_iterator,
+        device_and_data_format(), defun=False)
+
+  def benchmark_eager_train_datasets_with_defun(self):
+
+    def make_iterator(tensors):
+      with tf.device('/device:CPU:0'):
+        ds = tf.data.Dataset.from_tensors(tensors).repeat()
+      return tfe.Iterator(ds)
+
+    self._benchmark_eager_train(
+        'eager_train_dataset_with_defun', make_iterator,
+        device_and_data_format(), defun=True)
+
+
 if __name__ == '__main__':
   tf.enable_eager_execution()
   tf.test.main()
-- 
cgit v1.2.3


From 97b1ef3ee8432d9a3bf664d367377028e95e0e1f Mon Sep 17 00:00:00 2001
From: Wesley Qian 
Date: Tue, 17 Jul 2018 09:30:05 -0700
Subject: Add StarGAN model for TFGAN.

- Defined namedtuple for StarGAN model.
- Function for StarGAN model creation.
- Test for StarGAN model creation.
- Fix small lint issue in train.py.

PiperOrigin-RevId: 204923505
---
 tensorflow/contrib/gan/BUILD                 |   5 +-
 tensorflow/contrib/gan/python/namedtuples.py |  50 +++++-
 tensorflow/contrib/gan/python/train.py       | 182 +++++++++++++++----
 tensorflow/contrib/gan/python/train_test.py  | 259 ++++++++++++++++++++++-----
 4 files changed, 412 insertions(+), 84 deletions(-)

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 10a8796bcb..c8c2af49d4 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -42,8 +42,10 @@ py_library(
         "//tensorflow/contrib/training:training_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:init_ops",
+        "//tensorflow/python:random_ops",
         "//tensorflow/python:training",
         "//tensorflow/python:training_util",
         "//tensorflow/python:variable_scope",
@@ -58,17 +60,18 @@ py_test(
     srcs_version = "PY2AND3",
     tags = ["notsan"],
     deps = [
-        ":features",
         ":namedtuples",
         ":random_tensor_pool",
         ":train",
         "//tensorflow/contrib/framework:framework_py",
+        "//tensorflow/contrib/layers:layers_py",
         "//tensorflow/contrib/slim:learning",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python:random_ops",
         "//tensorflow/python:random_seed",
         "//tensorflow/python:training",
diff --git a/tensorflow/contrib/gan/python/namedtuples.py b/tensorflow/contrib/gan/python/namedtuples.py
index 25cfeafeec..a462b68e28 100644
--- a/tensorflow/contrib/gan/python/namedtuples.py
+++ b/tensorflow/contrib/gan/python/namedtuples.py
@@ -25,12 +25,12 @@ from __future__ import print_function
 
 import collections
 
-
 __all__ = [
     'GANModel',
     'InfoGANModel',
     'ACGANModel',
     'CycleGANModel',
+    'StarGANModel',
     'GANLoss',
     'CycleGANLoss',
     'GANTrainOps',
@@ -136,6 +136,54 @@ class CycleGANModel(
   """
 
 
+class StarGANModel(
+    collections.namedtuple('StarGANModel', (
+        'input_data',
+        'input_data_domain_label',
+        'generated_data',
+        'generated_data_domain_target',
+        'reconstructed_data',
+        'discriminator_input_data_source_predication',
+        'discriminator_generated_data_source_predication',
+        'discriminator_input_data_domain_predication',
+        'discriminator_generated_data_domain_predication',
+        'generator_variables',
+        'generator_scope',
+        'generator_fn',
+        'discriminator_variables',
+        'discriminator_scope',
+        'discriminator_fn',
+    ))):
+  """A StarGANModel contains all the pieces needed for StarGAN training.
+
+  Args:
+    input_data: The real images that need to be transferred by the generator.
+    input_data_domain_label: The real domain labels associated with the real
+      images.
+    generated_data: The generated images produced by the generator. It has the
+      same shape as the input_data.
+    generated_data_domain_target: The target domain that the generated images
+      belong to. It has the same shape as the input_data_domain_label.
+    reconstructed_data: The reconstructed images produced by the G(enerator).
+      reconstructed_data = G(G(input_data, generated_data_domain_target),
+      input_data_domain_label).
+    discriminator_input_data_source: The discriminator's output for predicting
+      the source (real/generated) of input_data.
+    discriminator_generated_data_source: The discriminator's output for
+      predicting the source (real/generated) of  generated_data.
+    discriminator_input_data_domain_predication: The discriminator's output for
+      predicting the domain_label for the input_data.
+    discriminator_generated_data_domain_predication: The discriminatorr's output
+      for predicting the domain_target for the generated_data.
+    generator_variables: A list of all generator variables.
+    generator_scope: Variable scope all generator variables live in.
+    generator_fn: The generator function.
+    discriminator_variables: A list of all discriminator variables.
+    discriminator_scope: Variable scope all discriminator variables live in.
+    discriminator_fn: The discriminator function.
+  """
+
+
 class GANLoss(
     collections.namedtuple('GANLoss', (
         'generator_loss',
diff --git a/tensorflow/contrib/gan/python/train.py b/tensorflow/contrib/gan/python/train.py
index 6fa43059f3..49d9327333 100644
--- a/tensorflow/contrib/gan/python/train.py
+++ b/tensorflow/contrib/gan/python/train.py
@@ -36,10 +36,12 @@ from tensorflow.contrib.gan.python import losses as tfgan_losses
 from tensorflow.contrib.gan.python import namedtuples
 from tensorflow.contrib.slim.python.slim import learning as slim_learning
 from tensorflow.contrib.training.python.training import training
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.distributions import distribution as ds
 from tensorflow.python.ops.losses import losses
@@ -47,12 +49,12 @@ from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import sync_replicas_optimizer
 from tensorflow.python.training import training_util
 
-
 __all__ = [
     'gan_model',
     'infogan_model',
     'acgan_model',
     'cyclegan_model',
+    'stargan_model',
     'gan_loss',
     'cyclegan_loss',
     'gan_train_ops',
@@ -123,16 +125,9 @@ def gan_model(
   discriminator_variables = variables_lib.get_trainable_variables(dis_scope)
 
   return namedtuples.GANModel(
-      generator_inputs,
-      generated_data,
-      generator_variables,
-      gen_scope,
-      generator_fn,
-      real_data,
-      discriminator_real_outputs,
-      discriminator_gen_outputs,
-      discriminator_variables,
-      dis_scope,
+      generator_inputs, generated_data, generator_variables, gen_scope,
+      generator_fn, real_data, discriminator_real_outputs,
+      discriminator_gen_outputs, discriminator_variables, dis_scope,
       discriminator_fn)
 
 
@@ -201,8 +196,7 @@ def infogan_model(
 
   # Get model-specific variables.
   generator_variables = variables_lib.get_trainable_variables(gen_scope)
-  discriminator_variables = variables_lib.get_trainable_variables(
-      disc_scope)
+  discriminator_variables = variables_lib.get_trainable_variables(disc_scope)
 
   return namedtuples.InfoGANModel(
       generator_inputs,
@@ -279,12 +273,12 @@ def acgan_model(
     generator_inputs = _convert_tensor_or_l_or_d(generator_inputs)
     generated_data = generator_fn(generator_inputs)
   with variable_scope.variable_scope(discriminator_scope) as dis_scope:
-    with ops.name_scope(dis_scope.name+'/generated/'):
+    with ops.name_scope(dis_scope.name + '/generated/'):
       (discriminator_gen_outputs, discriminator_gen_classification_logits
       ) = _validate_acgan_discriminator_outputs(
           discriminator_fn(generated_data, generator_inputs))
   with variable_scope.variable_scope(dis_scope, reuse=True):
-    with ops.name_scope(dis_scope.name+'/real/'):
+    with ops.name_scope(dis_scope.name + '/real/'):
       real_data = ops.convert_to_tensor(real_data)
       (discriminator_real_outputs, discriminator_real_classification_logits
       ) = _validate_acgan_discriminator_outputs(
@@ -297,8 +291,7 @@ def acgan_model(
 
   # Get model-specific variables.
   generator_variables = variables_lib.get_trainable_variables(gen_scope)
-  discriminator_variables = variables_lib.get_trainable_variables(
-      dis_scope)
+  discriminator_variables = variables_lib.get_trainable_variables(dis_scope)
 
   return namedtuples.ACGANModel(
       generator_inputs, generated_data, generator_variables, gen_scope,
@@ -379,6 +372,108 @@ def cyclegan_model(
                                    reconstructed_y)
 
 
+def stargan_model(generator_fn,
+                  discriminator_fn,
+                  input_data,
+                  input_data_domain_label,
+                  generator_scope='Generator',
+                  discriminator_scope='Discriminator'):
+  """Returns a StarGAN model outputs and variables.
+
+  See https://arxiv.org/abs/1711.09020 for more details.
+
+  Args:
+    generator_fn: A python lambda that takes `inputs` and `targets` as inputs
+      and returns 'generated_data' as the transformed version of `input` based
+      on the `target`. `input` has shape (n, h, w, c), `targets` has shape (n,
+      num_domains), and `generated_data` has the same shape as `input`.
+    discriminator_fn: A python lambda that takes `inputs` and `num_domains` as
+      inputs and returns a tuple (`source_prediction`, `domain_prediction`).
+      `source_prediction` represents the source(real/generated) prediction by
+      the discriminator, and `domain_prediction` represents the domain
+      prediction/classification by the discriminator. `source_prediction` has
+      shape (n) and `domain_prediction` has shape (n, num_domains).
+    input_data: Tensor or a list of tensor of shape (n, h, w, c) representing
+      the real input images.
+    input_data_domain_label: Tensor or a list of tensor of shape (batch_size,
+      num_domains) representing the domain label associated with the real
+      images.
+    generator_scope: Optional generator variable scope. Useful if you want to
+      reuse a subgraph that has already been created.
+    discriminator_scope: Optional discriminator variable scope. Useful if you
+      want to reuse a subgraph that has already been created.
+
+  Returns:
+    StarGANModel nametuple return the tensor that are needed to compute the
+    loss.
+
+  Raises:
+    ValueError: If the shape of `input_data_domain_label` is not rank 2 or fully
+    defined in every dimensions.
+  """
+
+  # Convert to tensor.
+  input_data = _convert_tensor_or_l_or_d(input_data)
+  input_data_domain_label = _convert_tensor_or_l_or_d(input_data_domain_label)
+
+  # Convert list of tensor to a single tensor if applicable.
+  if isinstance(input_data, (list, tuple)):
+    input_data = array_ops.concat(
+        [ops.convert_to_tensor(x) for x in input_data], 0)
+  if isinstance(input_data_domain_label, (list, tuple)):
+    input_data_domain_label = array_ops.concat(
+        [ops.convert_to_tensor(x) for x in input_data_domain_label], 0)
+
+  # Get batch_size, num_domains from the labels.
+  input_data_domain_label.shape.assert_has_rank(2)
+  input_data_domain_label.shape.assert_is_fully_defined()
+  batch_size, num_domains = input_data_domain_label.shape.as_list()
+
+  # Transform input_data to random target domains.
+  with variable_scope.variable_scope(generator_scope) as generator_scope:
+    generated_data_domain_target = _generate_stargan_random_domain_target(
+        batch_size, num_domains)
+    generated_data = generator_fn(input_data, generated_data_domain_target)
+
+  # Transform generated_data back to the original input_data domain.
+  with variable_scope.variable_scope(generator_scope, reuse=True):
+    reconstructed_data = generator_fn(generated_data, input_data_domain_label)
+
+  # Predict source and domain for the generated_data using the discriminator.
+  with variable_scope.variable_scope(
+      discriminator_scope) as discriminator_scope:
+    disc_gen_data_source_pred, disc_gen_data_domain_pred = discriminator_fn(
+        generated_data, num_domains)
+
+  # Predict source and domain for the input_data using the discriminator.
+  with variable_scope.variable_scope(discriminator_scope, reuse=True):
+    disc_input_data_source_pred, disc_input_data_domain_pred = discriminator_fn(
+        input_data, num_domains)
+
+  # Collect trainable variables from the neural networks.
+  generator_variables = variables_lib.get_trainable_variables(generator_scope)
+  discriminator_variables = variables_lib.get_trainable_variables(
+      discriminator_scope)
+
+  # Create the StarGANModel namedtuple.
+  return namedtuples.StarGANModel(
+      input_data=input_data,
+      input_data_domain_label=input_data_domain_label,
+      generated_data=generated_data,
+      generated_data_domain_target=generated_data_domain_target,
+      reconstructed_data=reconstructed_data,
+      discriminator_input_data_source_predication=disc_input_data_source_pred,
+      discriminator_generated_data_source_predication=disc_gen_data_source_pred,
+      discriminator_input_data_domain_predication=disc_input_data_domain_pred,
+      discriminator_generated_data_domain_predication=disc_gen_data_domain_pred,
+      generator_variables=generator_variables,
+      generator_scope=generator_scope,
+      generator_fn=generator_fn,
+      discriminator_variables=discriminator_variables,
+      discriminator_scope=discriminator_scope,
+      discriminator_fn=discriminator_fn)
+
+
 def _validate_aux_loss_weight(aux_loss_weight, name='aux_loss_weight'):
   if isinstance(aux_loss_weight, ops.Tensor):
     aux_loss_weight.shape.assert_is_compatible_with([])
@@ -512,8 +607,8 @@ def gan_loss(
       `model` isn't an `InfoGANModel`.
   """
   # Validate arguments.
-  gradient_penalty_weight = _validate_aux_loss_weight(gradient_penalty_weight,
-                                                      'gradient_penalty_weight')
+  gradient_penalty_weight = _validate_aux_loss_weight(
+      gradient_penalty_weight, 'gradient_penalty_weight')
   mutual_information_penalty_weight = _validate_aux_loss_weight(
       mutual_information_penalty_weight, 'infogan_weight')
   aux_cond_generator_weight = _validate_aux_loss_weight(
@@ -631,8 +726,8 @@ def cyclegan_loss(
         generator_loss_fn=generator_loss_fn,
         discriminator_loss_fn=discriminator_loss_fn,
         **kwargs)
-    return partial_loss._replace(
-        generator_loss=partial_loss.generator_loss + aux_loss)
+    return partial_loss._replace(generator_loss=partial_loss.generator_loss +
+                                 aux_loss)
 
   with ops.name_scope('cyclegan_loss_x2y'):
     loss_x2y = _partial_loss(model.model_x2y)
@@ -822,12 +917,14 @@ def get_sequential_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)):
   Returns:
     A function that takes a GANTrainOps tuple and returns a list of hooks.
   """
+
   def get_hooks(train_ops):
     generator_hook = RunTrainOpsHook(train_ops.generator_train_op,
                                      train_steps.generator_train_steps)
     discriminator_hook = RunTrainOpsHook(train_ops.discriminator_train_op,
                                          train_steps.discriminator_train_steps)
     return [generator_hook, discriminator_hook]
+
   return get_hooks
 
 
@@ -881,23 +978,23 @@ def get_joint_train_hooks(train_steps=namedtuples.GANTrainSteps(1, 1)):
     d_hook = RunTrainOpsHook(d_op, num_d_steps)
 
     return [joint_hook, g_hook, d_hook]
+
   return get_hooks
 
 
 # TODO(joelshor): This function currently returns the global step. Find a
 # good way for it to return the generator, discriminator, and final losses.
-def gan_train(
-    train_ops,
-    logdir,
-    get_hooks_fn=get_sequential_train_hooks(),
-    master='',
-    is_chief=True,
-    scaffold=None,
-    hooks=None,
-    chief_only_hooks=None,
-    save_checkpoint_secs=600,
-    save_summaries_steps=100,
-    config=None):
+def gan_train(train_ops,
+              logdir,
+              get_hooks_fn=get_sequential_train_hooks(),
+              master='',
+              is_chief=True,
+              scaffold=None,
+              hooks=None,
+              chief_only_hooks=None,
+              save_checkpoint_secs=600,
+              save_summaries_steps=100,
+              config=None):
   """A wrapper around `contrib.training.train` that uses GAN hooks.
 
   Args:
@@ -943,8 +1040,7 @@ def gan_train(
       config=config)
 
 
-def get_sequential_train_steps(
-    train_steps=namedtuples.GANTrainSteps(1, 1)):
+def get_sequential_train_steps(train_steps=namedtuples.GANTrainSteps(1, 1)):
   """Returns a thin wrapper around slim.learning.train_step, for GANs.
 
   This function is to provide support for the Supervisor. For new code, please
@@ -1042,3 +1138,19 @@ def _validate_acgan_discriminator_outputs(discriminator_output):
         'A discriminator function for ACGAN must output a tuple '
         'consisting of (discrimination logits, classification logits).')
   return a, b
+
+
+def _generate_stargan_random_domain_target(batch_size, num_domains):
+  """Generate random domain label.
+
+  Args:
+    batch_size: (int) Number of random domain label.
+    num_domains: (int) Number of domains representing with the label.
+
+  Returns:
+    Tensor of shape (batch_size, num_domains) representing random label.
+  """
+  domain_idx = random_ops.random_uniform(
+      [batch_size], minval=0, maxval=num_domains, dtype=dtypes.int32)
+
+  return array_ops.one_hot(domain_idx, num_domains)
diff --git a/tensorflow/contrib/gan/python/train_test.py b/tensorflow/contrib/gan/python/train_test.py
index 3ebbe55d05..06681eaf83 100644
--- a/tensorflow/contrib/gan/python/train_test.py
+++ b/tensorflow/contrib/gan/python/train_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.contrib import layers
 from tensorflow.contrib.framework.python.ops import variables as variables_lib
 from tensorflow.contrib.gan.python import namedtuples
 from tensorflow.contrib.gan.python import train
@@ -30,6 +31,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
@@ -84,19 +86,47 @@ class InfoGANDiscriminator(object):
 
 
 def acgan_discriminator_model(inputs, _, num_classes=10):
-  return (discriminator_model(inputs, _), array_ops.one_hot(
-      # TODO(haeusser): infer batch size from input
-      random_ops.random_uniform([3], maxval=num_classes, dtype=dtypes.int32),
-      num_classes))
+  return (
+      discriminator_model(inputs, _),
+      array_ops.one_hot(
+          # TODO(haeusser): infer batch size from input
+          random_ops.random_uniform(
+              [3], maxval=num_classes, dtype=dtypes.int32),
+          num_classes))
 
 
 class ACGANDiscriminator(object):
 
   def __call__(self, inputs, _, num_classes=10):
-    return (discriminator_model(inputs, _), array_ops.one_hot(
-        # TODO(haeusser): infer batch size from input
-        random_ops.random_uniform([3], maxval=num_classes, dtype=dtypes.int32),
-        num_classes))
+    return (
+        discriminator_model(inputs, _),
+        array_ops.one_hot(
+            # TODO(haeusser): infer batch size from input
+            random_ops.random_uniform(
+                [3], maxval=num_classes, dtype=dtypes.int32),
+            num_classes))
+
+
+def stargan_generator_model(inputs, _):
+  """Dummy generator for StarGAN."""
+
+  return variable_scope.get_variable('dummy_g', initializer=0.5) * inputs
+
+
+def stargan_discriminator_model(inputs, num_domains):
+  """Differentiable dummy discriminator for StarGAN."""
+
+  hidden = layers.flatten(inputs)
+
+  output_src = math_ops.reduce_mean(hidden, axis=1)
+
+  output_cls = layers.fully_connected(
+      inputs=hidden,
+      num_outputs=num_domains,
+      activation_fn=None,
+      normalizer_fn=None,
+      biases_initializer=None)
+  return output_src, output_cls
 
 
 def get_gan_model():
@@ -122,8 +152,7 @@ def get_gan_model():
 def get_callable_gan_model():
   ganmodel = get_gan_model()
   return ganmodel._replace(
-      generator_fn=Generator(),
-      discriminator_fn=Discriminator())
+      generator_fn=Generator(), discriminator_fn=Discriminator())
 
 
 def create_gan_model():
@@ -283,15 +312,15 @@ class GANModelTest(test.TestCase):
     self._test_output_type_helper(get_infogan_model, namedtuples.InfoGANModel)
 
   def test_output_type_callable_infogan(self):
-    self._test_output_type_helper(
-        get_callable_infogan_model, namedtuples.InfoGANModel)
+    self._test_output_type_helper(get_callable_infogan_model,
+                                  namedtuples.InfoGANModel)
 
   def test_output_type_acgan(self):
     self._test_output_type_helper(get_acgan_model, namedtuples.ACGANModel)
 
   def test_output_type_callable_acgan(self):
-    self._test_output_type_helper(
-        get_callable_acgan_model, namedtuples.ACGANModel)
+    self._test_output_type_helper(get_callable_acgan_model,
+                                  namedtuples.ACGANModel)
 
   def test_output_type_cyclegan(self):
     self._test_output_type_helper(get_cyclegan_model, namedtuples.CycleGANModel)
@@ -301,10 +330,13 @@ class GANModelTest(test.TestCase):
                                   namedtuples.CycleGANModel)
 
   def test_no_shape_check(self):
+
     def dummy_generator_model(_):
       return (None, None)
+
     def dummy_discriminator_model(data, conditioning):  # pylint: disable=unused-argument
       return 1
+
     with self.assertRaisesRegexp(AttributeError, 'object has no attribute'):
       train.gan_model(
           dummy_generator_model,
@@ -320,6 +352,138 @@ class GANModelTest(test.TestCase):
         check_shapes=False)
 
 
+class StarGANModelTest(test.TestCase):
+  """Tests for `stargan_model`."""
+
+  @staticmethod
+  def create_input_and_label_tensor(batch_size, img_size, c_size, num_domains):
+
+    input_tensor_list = []
+    label_tensor_list = []
+    for _ in range(num_domains):
+      input_tensor_list.append(
+          random_ops.random_uniform((batch_size, img_size, img_size, c_size)))
+      domain_idx = random_ops.random_uniform(
+          [batch_size], minval=0, maxval=num_domains, dtype=dtypes.int32)
+      label_tensor_list.append(array_ops.one_hot(domain_idx, num_domains))
+    return input_tensor_list, label_tensor_list
+
+  def test_generate_stargan_random_domain_target(self):
+
+    batch_size = 8
+    domain_numbers = 3
+
+    target_tensor = train._generate_stargan_random_domain_target(
+        batch_size, domain_numbers)
+
+    with self.test_session() as sess:
+      targets = sess.run(target_tensor)
+      self.assertTupleEqual((batch_size, domain_numbers), targets.shape)
+      for target in targets:
+        self.assertEqual(1, np.sum(target))
+        self.assertEqual(1, np.max(target))
+
+  def test_stargan_model_output_type(self):
+
+    batch_size = 2
+    img_size = 16
+    c_size = 3
+    num_domains = 5
+
+    input_tensor, label_tensor = StarGANModelTest.create_input_and_label_tensor(
+        batch_size, img_size, c_size, num_domains)
+    model = train.stargan_model(
+        generator_fn=stargan_generator_model,
+        discriminator_fn=stargan_discriminator_model,
+        input_data=input_tensor,
+        input_data_domain_label=label_tensor)
+
+    self.assertIsInstance(model, namedtuples.StarGANModel)
+    self.assertTrue(isinstance(model.discriminator_variables, list))
+    self.assertTrue(isinstance(model.generator_variables, list))
+    self.assertIsInstance(model.discriminator_scope,
+                          variable_scope.VariableScope)
+    self.assertTrue(model.generator_scope, variable_scope.VariableScope)
+    self.assertTrue(callable(model.discriminator_fn))
+    self.assertTrue(callable(model.generator_fn))
+
+  def test_stargan_model_generator_output(self):
+
+    batch_size = 2
+    img_size = 16
+    c_size = 3
+    num_domains = 5
+
+    input_tensor, label_tensor = StarGANModelTest.create_input_and_label_tensor(
+        batch_size, img_size, c_size, num_domains)
+    model = train.stargan_model(
+        generator_fn=stargan_generator_model,
+        discriminator_fn=stargan_discriminator_model,
+        input_data=input_tensor,
+        input_data_domain_label=label_tensor)
+
+    with self.test_session(use_gpu=True) as sess:
+
+      sess.run(variables.global_variables_initializer())
+
+      input_data, generated_data, reconstructed_data = sess.run(
+          [model.input_data, model.generated_data, model.reconstructed_data])
+      self.assertTupleEqual(
+          (batch_size * num_domains, img_size, img_size, c_size),
+          input_data.shape)
+      self.assertTupleEqual(
+          (batch_size * num_domains, img_size, img_size, c_size),
+          generated_data.shape)
+      self.assertTupleEqual(
+          (batch_size * num_domains, img_size, img_size, c_size),
+          reconstructed_data.shape)
+
+  def test_stargan_model_discriminator_output(self):
+
+    batch_size = 2
+    img_size = 16
+    c_size = 3
+    num_domains = 5
+
+    input_tensor, label_tensor = StarGANModelTest.create_input_and_label_tensor(
+        batch_size, img_size, c_size, num_domains)
+    model = train.stargan_model(
+        generator_fn=stargan_generator_model,
+        discriminator_fn=stargan_discriminator_model,
+        input_data=input_tensor,
+        input_data_domain_label=label_tensor)
+
+    with self.test_session(use_gpu=True) as sess:
+
+      sess.run(variables.global_variables_initializer())
+
+      disc_input_data_source_pred, disc_gen_data_source_pred = sess.run([
+          model.discriminator_input_data_source_predication,
+          model.discriminator_generated_data_source_predication
+      ])
+      self.assertEqual(1, len(disc_input_data_source_pred.shape))
+      self.assertEqual(batch_size * num_domains,
+                       disc_input_data_source_pred.shape[0])
+      self.assertEqual(1, len(disc_gen_data_source_pred.shape))
+      self.assertEqual(batch_size * num_domains,
+                       disc_gen_data_source_pred.shape[0])
+
+      input_label, disc_input_label, gen_label, disc_gen_label = sess.run([
+          model.input_data_domain_label,
+          model.discriminator_input_data_domain_predication,
+          model.generated_data_domain_target,
+          model.discriminator_generated_data_domain_predication
+      ])
+      self.assertTupleEqual((batch_size * num_domains, num_domains),
+                            input_label.shape)
+      self.assertTupleEqual((batch_size * num_domains, num_domains),
+                            disc_input_label.shape)
+      self.assertTupleEqual((batch_size * num_domains, num_domains),
+                            gen_label.shape)
+      self.assertTupleEqual((batch_size * num_domains, num_domains),
+                            disc_gen_label.shape)
+
+
 class GANLossTest(test.TestCase):
   """Tests for `gan_loss`."""
 
@@ -362,9 +526,10 @@ class GANLossTest(test.TestCase):
   def _test_grad_penalty_helper(self, create_gan_model_fn, one_sided=False):
     model = create_gan_model_fn()
     loss = train.gan_loss(model)
-    loss_gp = train.gan_loss(model,
-                             gradient_penalty_weight=1.0,
-                             gradient_penalty_one_sided=one_sided)
+    loss_gp = train.gan_loss(
+        model,
+        gradient_penalty_weight=1.0,
+        gradient_penalty_one_sided=one_sided)
     self.assertTrue(isinstance(loss_gp, namedtuples.GANLoss))
 
     # Check values.
@@ -417,8 +582,9 @@ class GANLossTest(test.TestCase):
 
   # Test mutual information penalty option.
   def _test_mutual_info_penalty_helper(self, create_gan_model_fn):
-    train.gan_loss(create_gan_model_fn(),
-                   mutual_information_penalty_weight=constant_op.constant(1.0))
+    train.gan_loss(
+        create_gan_model_fn(),
+        mutual_information_penalty_weight=constant_op.constant(1.0))
 
   def test_mutual_info_penalty_infogan(self):
     self._test_mutual_info_penalty_helper(get_infogan_model)
@@ -435,11 +601,11 @@ class GANLossTest(test.TestCase):
       no_reg_loss_dis_np = no_reg_loss.discriminator_loss.eval()
 
     with ops.name_scope(get_gan_model_fn().generator_scope.name):
-      ops.add_to_collection(
-          ops.GraphKeys.REGULARIZATION_LOSSES, constant_op.constant(3.0))
+      ops.add_to_collection(ops.GraphKeys.REGULARIZATION_LOSSES,
+                            constant_op.constant(3.0))
     with ops.name_scope(get_gan_model_fn().discriminator_scope.name):
-      ops.add_to_collection(
-          ops.GraphKeys.REGULARIZATION_LOSSES, constant_op.constant(2.0))
+      ops.add_to_collection(ops.GraphKeys.REGULARIZATION_LOSSES,
+                            constant_op.constant(2.0))
 
     # Check that losses now include the correct regularization values.
     reg_loss = train.gan_loss(get_gan_model_fn())
@@ -481,14 +647,14 @@ class GANLossTest(test.TestCase):
     # Check values.
     with self.test_session(use_gpu=True) as sess:
       variables.global_variables_initializer().run()
-      loss_gen_np, loss_ac_gen_gen_np, loss_ac_dis_gen_np = sess.run(
-          [loss.generator_loss,
-           loss_ac_gen.generator_loss,
-           loss_ac_dis.generator_loss])
-      loss_dis_np, loss_ac_gen_dis_np, loss_ac_dis_dis_np = sess.run(
-          [loss.discriminator_loss,
-           loss_ac_gen.discriminator_loss,
-           loss_ac_dis.discriminator_loss])
+      loss_gen_np, loss_ac_gen_gen_np, loss_ac_dis_gen_np = sess.run([
+          loss.generator_loss, loss_ac_gen.generator_loss,
+          loss_ac_dis.generator_loss
+      ])
+      loss_dis_np, loss_ac_gen_dis_np, loss_ac_dis_dis_np = sess.run([
+          loss.discriminator_loss, loss_ac_gen.discriminator_loss,
+          loss_ac_dis.discriminator_loss
+      ])
 
     self.assertTrue(loss_gen_np < loss_dis_np)
     self.assertTrue(np.isscalar(loss_ac_gen_gen_np))
@@ -707,8 +873,11 @@ class GANTrainOpsTest(test.TestCase):
 
     # Add an update op outside the generator and discriminator scopes.
     if provide_update_ops:
-      kwargs = {'update_ops':
-                [constant_op.constant(1.0), gen_update_op, dis_update_op]}
+      kwargs = {
+          'update_ops': [
+              constant_op.constant(1.0), gen_update_op, dis_update_op
+          ]
+      }
     else:
       ops.add_to_collection(ops.GraphKeys.UPDATE_OPS, constant_op.constant(1.0))
       kwargs = {}
@@ -717,8 +886,8 @@ class GANTrainOpsTest(test.TestCase):
     d_opt = gradient_descent.GradientDescentOptimizer(1.0)
 
     with self.assertRaisesRegexp(ValueError, 'There are unused update ops:'):
-      train.gan_train_ops(model, loss, g_opt, d_opt,
-                          check_for_unused_update_ops=True, **kwargs)
+      train.gan_train_ops(
+          model, loss, g_opt, d_opt, check_for_unused_update_ops=True, **kwargs)
     train_ops = train.gan_train_ops(
         model, loss, g_opt, d_opt, check_for_unused_update_ops=False, **kwargs)
 
@@ -771,8 +940,9 @@ class GANTrainOpsTest(test.TestCase):
   def test_unused_update_ops_callable_acgan_provideupdates(self):
     self._test_unused_update_ops(create_callable_acgan_model, True)
 
-  def _test_sync_replicas_helper(
-      self, create_gan_model_fn, create_global_step=False):
+  def _test_sync_replicas_helper(self,
+                                 create_gan_model_fn,
+                                 create_global_step=False):
     model = create_gan_model_fn()
     loss = train.gan_loss(model)
     num_trainable_vars = len(variables_lib.get_trainable_variables())
@@ -785,10 +955,7 @@ class GANTrainOpsTest(test.TestCase):
     g_opt = get_sync_optimizer()
     d_opt = get_sync_optimizer()
     train_ops = train.gan_train_ops(
-        model,
-        loss,
-        generator_optimizer=g_opt,
-        discriminator_optimizer=d_opt)
+        model, loss, generator_optimizer=g_opt, discriminator_optimizer=d_opt)
     self.assertTrue(isinstance(train_ops, namedtuples.GANTrainOps))
     # No new trainable variables should have been added.
     self.assertEqual(num_trainable_vars,
@@ -860,8 +1027,8 @@ class GANTrainTest(test.TestCase):
     # joint training.
     train_ops = namedtuples.GANTrainOps(
         generator_train_op=step.assign_add(generator_add, use_locking=True),
-        discriminator_train_op=step.assign_add(discriminator_add,
-                                               use_locking=True),
+        discriminator_train_op=step.assign_add(
+            discriminator_add, use_locking=True),
         global_step_inc_op=step.assign_add(1))
     return train_ops
 
@@ -903,8 +1070,7 @@ class GANTrainTest(test.TestCase):
   def _test_multiple_steps_helper(self, get_hooks_fn_fn):
     train_ops = self._gan_train_ops(generator_add=10, discriminator_add=100)
     train_steps = namedtuples.GANTrainSteps(
-        generator_train_steps=3,
-        discriminator_train_steps=4)
+        generator_train_steps=3, discriminator_train_steps=4)
     final_step = train.gan_train(
         train_ops,
         get_hooks_fn=get_hooks_fn_fn(train_steps),
@@ -927,8 +1093,7 @@ class GANTrainTest(test.TestCase):
         discriminator_train_op=constant_op.constant(2.0),
         global_step_inc_op=step.assign_add(1))
     train_steps = namedtuples.GANTrainSteps(
-        generator_train_steps=3,
-        discriminator_train_steps=4)
+        generator_train_steps=3, discriminator_train_steps=4)
 
     final_loss = slim_learning.train(
         train_op=train_ops,
-- 
cgit v1.2.3


From 6971bc84c3e743e3fb0ab768226e52c3816a989f Mon Sep 17 00:00:00 2001
From: Asim Shankar 
Date: Tue, 17 Jul 2018 09:57:10 -0700
Subject: Make forward compatibility utilities visible as
 tf.compat.forward_compatible.

PiperOrigin-RevId: 204927574
---
 tensorflow/python/compat/BUILD                      | 1 +
 tensorflow/python/compat/compat.py                  | 7 +++++++
 tensorflow/tools/api/golden/tensorflow.compat.pbtxt | 8 ++++++++
 3 files changed, 16 insertions(+)

diff --git a/tensorflow/python/compat/BUILD b/tensorflow/python/compat/BUILD
index 58ceafca06..e0a1c8e057 100644
--- a/tensorflow/python/compat/BUILD
+++ b/tensorflow/python/compat/BUILD
@@ -9,6 +9,7 @@ py_library(
     srcs = ["compat.py"],
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:internal"],
+    deps = ["//tensorflow/python:util"],
 )
 
 tf_py_test(
diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py
index 68a6421c2c..247ea7349d 100644
--- a/tensorflow/python/compat/compat.py
+++ b/tensorflow/python/compat/compat.py
@@ -24,13 +24,17 @@ from __future__ import print_function
 
 import datetime
 from tensorflow.python.util import tf_contextlib
+from tensorflow.python.util.tf_export import tf_export
 
 _FORWARD_COMPATIBILITY_HORIZON = datetime.date(2018, 8, 1)
 
 
+@tf_export("compat.forward_compatible")
 def forward_compatible(year, month, day):
   """Return true if the forward compatibility window has expired.
 
+  See @{$guide/version_compat#backward_and_partial_forward_compatibility}.
+
   Forward-compatibility refers to scenarios where the producer of a TensorFlow
   model (a GraphDef or SavedModel) is compiled against a version of the
   TensorFlow library newer than what the consumer was compiled against. The
@@ -82,10 +86,13 @@ def forward_compatible(year, month, day):
   return _FORWARD_COMPATIBILITY_HORIZON > datetime.date(year, month, day)
 
 
+@tf_export("compat.forward_compatibility_horizon")
 @tf_contextlib.contextmanager
 def forward_compatibility_horizon(year, month, day):
   """Context manager for testing forward compatibility of generated graphs.
 
+  See @{$guide/version_compat#backward_and_partial_forward_compatibility}.
+
   To ensure forward compatibility of generated graphs (see `forward_compatible`)
   with older binaries, new features can be gated with:
 
diff --git a/tensorflow/tools/api/golden/tensorflow.compat.pbtxt b/tensorflow/tools/api/golden/tensorflow.compat.pbtxt
index bab480ff9b..f1d760603e 100644
--- a/tensorflow/tools/api/golden/tensorflow.compat.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.compat.pbtxt
@@ -32,6 +32,14 @@ tf_module {
     name: "as_text"
     argspec: "args=[\'bytes_or_text\', \'encoding\'], varargs=None, keywords=None, defaults=[\'utf-8\'], "
   }
+  member_method {
+    name: "forward_compatibility_horizon"
+    argspec: "args=[\'year\', \'month\', \'day\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "forward_compatible"
+    argspec: "args=[\'year\', \'month\', \'day\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "path_to_str"
     argspec: "args=[\'path\'], varargs=None, keywords=None, defaults=None"
-- 
cgit v1.2.3


From e6d3562263d4e54764e7321655ea56dddfb1dfd1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 10:27:24 -0700
Subject: More info on op prepare/eval failures

PiperOrigin-RevId: 204932609
---
 tensorflow/contrib/lite/interpreter.cc | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/lite/interpreter.cc b/tensorflow/contrib/lite/interpreter.cc
index d103786694..26fecceab0 100644
--- a/tensorflow/contrib/lite/interpreter.cc
+++ b/tensorflow/contrib/lite/interpreter.cc
@@ -40,6 +40,16 @@ class NNAPIDelegate {};
 
 namespace {
 
+TfLiteStatus ReportOpError(TfLiteContext* context, const TfLiteNode& node,
+                           const TfLiteRegistration& registration,
+                           int node_index, const char* message) {
+  context->ReportError(context, "Node number %d (%s) %s.\n", node_index,
+                       EnumNameBuiltinOperator(static_cast(
+                           registration.builtin_code)),
+                       message);
+  return kTfLiteError;
+}
+
 // Stub method which returns kTfLiteError when the function is forbidden.
 // We're registrating this function to several different function to save
 // compiled binary size. Please note the restrictions:
@@ -572,9 +582,8 @@ TfLiteStatus Interpreter::PrepareOpsStartingAt(
         nodes_and_registration_[node_index].second;
     EnsureTensorsVectorCapacity();
     if (OpPrepare(registration, &node) == kTfLiteError) {
-      context_.ReportError(&context_, "Node %d failed to prepare.\n",
-                           node_index);
-      return kTfLiteError;
+      return ReportOpError(&context_, node, registration, node_index,
+                           "failed to prepare");
     }
 
     *last_execution_plan_index_prepared = execution_plan_index;
@@ -674,9 +683,8 @@ TfLiteStatus Interpreter::Invoke() {
     EnsureTensorsVectorCapacity();
     tensor_resized_since_op_invoke_ = false;
     if (OpInvoke(registration, &node) == kTfLiteError) {
-      context_.ReportError(&context_, "Node %d failed to invoke.\n",
-                           node_index);
-      status = kTfLiteError;
+      status = ReportOpError(&context_, node, registration, node_index,
+                             "failed to invoke");
     }
 
     // Force execution prep for downstream ops if the latest op triggered the
-- 
cgit v1.2.3


From 4eaf597cbaf1ef1f2a216a1a83289d06419f87fd Mon Sep 17 00:00:00 2001
From: Peter Hawkins 
Date: Tue, 17 Jul 2018 11:27:23 -0700
Subject: [TF:XLA] Add a 2000x2000 test case to Cholesky and QR decomposition
 tests.

PiperOrigin-RevId: 204943689
---
 tensorflow/compiler/tests/cholesky_op_test.py | 8 ++------
 tensorflow/compiler/tests/qr_op_test.py       | 5 ++++-
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/tests/cholesky_op_test.py b/tensorflow/compiler/tests/cholesky_op_test.py
index d2867278af..ed532db0ee 100644
--- a/tensorflow/compiler/tests/cholesky_op_test.py
+++ b/tensorflow/compiler/tests/cholesky_op_test.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import unittest
-
 import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
@@ -103,9 +101,8 @@ class CholeskyOpTest(xla_test.XLATestCase):
       with self.assertRaises(ValueError):
         linalg_ops.cholesky(tensor3)
 
-  @unittest.skip("Test is slow")
-  def testLarge(self):
-    n = 200
+  def testLarge2000x2000(self):
+    n = 2000
     shape = (n, n)
     data = np.ones(shape).astype(np.float32) / (2.0 * n) + np.diag(
         np.ones(n).astype(np.float32))
@@ -128,6 +125,5 @@ class CholeskyOpTest(xla_test.XLATestCase):
       matrix = np.dot(np.dot(w, np.diag(v)), w.T).astype(dtype)
       self._verifyCholesky(matrix, atol=1e-4)
 
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/compiler/tests/qr_op_test.py b/tensorflow/compiler/tests/qr_op_test.py
index 93752a21db..1b969ee2b3 100644
--- a/tensorflow/compiler/tests/qr_op_test.py
+++ b/tensorflow/compiler/tests/qr_op_test.py
@@ -57,7 +57,7 @@ class QrOpTest(xla_test.XLATestCase, parameterized.TestCase):
   def CheckApproximation(self, a, q, r):
     # Tests that a ~= q*r.
     precision = self.AdjustedNorm(a - np.matmul(q, r))
-    self.assertTrue(np.all(precision < 5.0))
+    self.assertTrue(np.all(precision < 10.0))
 
   def CheckUnitary(self, x):
     # Tests that x[...,:,:]^H * x[...,:,:] is close to the identity.
@@ -107,6 +107,9 @@ class QrOpTest(xla_test.XLATestCase, parameterized.TestCase):
       for batch_dims in [(), (3,)] + [(3, 2)] * (max(rows, cols) < 10):
         self._test(dtype, batch_dims + (rows, cols), full_matrices)
 
+  def testLarge2000x2000(self):
+    self._test(np.float32, (2000, 2000), full_matrices=True)
+
 
 if __name__ == "__main__":
   test.main()
-- 
cgit v1.2.3


From 9d126b26ff219d9258a78832ead0bb272b898bf0 Mon Sep 17 00:00:00 2001
From: Priya Gupta 
Date: Tue, 17 Jul 2018 11:52:47 -0700
Subject: Update distribution strategy readme to reflect metrics support during
 training.

PiperOrigin-RevId: 204948972
---
 tensorflow/contrib/distribute/README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/distribute/README.md b/tensorflow/contrib/distribute/README.md
index 44a4481021..2f5dd10550 100644
--- a/tensorflow/contrib/distribute/README.md
+++ b/tensorflow/contrib/distribute/README.md
@@ -116,8 +116,6 @@ in the input function gives a solid boost in performance. When using
 ## Caveats
 This feature is in early stages and there are a lot of improvements forthcoming:
 
-* Metrics are not yet supported during distributed training. They are still
-supported during the evaluation.
 * Summaries are only computed in the first tower in `MirroredStrategy`.
 * Evaluation is not yet distributed.
 * Eager support is in the works; performance can be more challenging with eager
-- 
cgit v1.2.3


From e94b49718f800276d2c045349db4480bbe12dd6b Mon Sep 17 00:00:00 2001
From: Paul Woitaschek 
Date: Tue, 17 Jul 2018 21:03:38 +0200
Subject: Update tflite_convert.py

---
 tensorflow/contrib/lite/python/tflite_convert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/python/tflite_convert.py b/tensorflow/contrib/lite/python/tflite_convert.py
index 9bd1f4f76e..d17482e601 100644
--- a/tensorflow/contrib/lite/python/tflite_convert.py
+++ b/tensorflow/contrib/lite/python/tflite_convert.py
@@ -257,7 +257,7 @@ def run_main(_):
   parser.add_argument(
       "--input_arrays",
       type=str,
-      help="Names of the output arrays, comma-separated.")
+      help="Names of the input arrays, comma-separated.")
   parser.add_argument(
       "--input_shapes",
       type=str,
-- 
cgit v1.2.3


From 9b12cb84292d23522c1c3f75700d97d9f9af8abd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 12:04:04 -0700
Subject: toco thinks Stack is the same as Pack.

Stack is not Pack.  tf.stack() yields Pack ops. Pack ops stack tensors.  Stack ops manipulate the TF runtime stack.

This cl unaliases "Stack" and "Pack" ops in toco, and renames most things that refer to "Stack" ops to "Pack" to be consistent across the codebase.

In summary:

Stack is whack.  'Stack' should be 'Pack'.  Hack 'Stack's into 'Pack's like a maniac.  This keeps 'Stack's from wracking runtime graphs.

(We apologize for the fault in the change description.  Those responsible have been...er...sacked).

PiperOrigin-RevId: 204951155
---
 tensorflow/contrib/lite/toco/BUILD                 |   4 +-
 tensorflow/contrib/lite/toco/export_tensorflow.cc  |  24 +++--
 .../convert_trivial_pack_to_reshape.cc             |  81 ++++++++++++++
 .../convert_trivial_stack_to_reshape.cc            |  81 --------------
 .../graph_transformations/graph_transformations.h  |   4 +-
 .../graph_transformations/propagate_fixed_sizes.cc |  26 ++---
 .../graph_transformations/resolve_constant_pack.cc | 117 +++++++++++++++++++++
 .../resolve_constant_stack.cc                      | 117 ---------------------
 .../graph_transformations/unroll_batch_matmul.cc   |  16 +--
 tensorflow/contrib/lite/toco/import_tensorflow.cc  |  17 +--
 tensorflow/contrib/lite/toco/model.h               |   9 +-
 tensorflow/contrib/lite/toco/toco_tooling.cc       |   4 +-
 tensorflow/contrib/lite/toco/tooling_util.cc       |   2 +-
 13 files changed, 254 insertions(+), 248 deletions(-)
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/convert_trivial_pack_to_reshape.cc
 delete mode 100644 tensorflow/contrib/lite/toco/graph_transformations/convert_trivial_stack_to_reshape.cc
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_pack.cc
 delete mode 100644 tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_stack.cc

diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index bbce93f61a..2252fe175a 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -176,7 +176,7 @@ cc_library(
         "graph_transformations/convert_reorder_axes.cc",
         "graph_transformations/convert_squeeze_to_reshape.cc",
         "graph_transformations/convert_trivial_addn_to_add.cc",
-        "graph_transformations/convert_trivial_stack_to_reshape.cc",
+        "graph_transformations/convert_trivial_pack_to_reshape.cc",
         "graph_transformations/convert_trivial_tile_to_concat.cc",
         "graph_transformations/convert_trivial_transpose_to_reshape.cc",
         "graph_transformations/create_im2col_arrays.cc",
@@ -236,12 +236,12 @@ cc_library(
         "graph_transformations/resolve_constant_fake_quant.cc",
         "graph_transformations/resolve_constant_fill.cc",
         "graph_transformations/resolve_constant_gather.cc",
+        "graph_transformations/resolve_constant_pack.cc",
         "graph_transformations/resolve_constant_random_uniform.cc",
         "graph_transformations/resolve_constant_range.cc",
         "graph_transformations/resolve_constant_reshape.cc",
         "graph_transformations/resolve_constant_shape_or_rank.cc",
         "graph_transformations/resolve_constant_slice.cc",
-        "graph_transformations/resolve_constant_stack.cc",
         "graph_transformations/resolve_constant_strided_slice.cc",
         "graph_transformations/resolve_constant_transpose.cc",
         "graph_transformations/resolve_constant_unary.cc",
diff --git a/tensorflow/contrib/lite/toco/export_tensorflow.cc b/tensorflow/contrib/lite/toco/export_tensorflow.cc
index 17375d19be..4508aa6632 100644
--- a/tensorflow/contrib/lite/toco/export_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/export_tensorflow.cc
@@ -1207,17 +1207,19 @@ void ConvertRangeOperator(const Model& model, const RangeOperator& src_op,
       GetTensorFlowDataType(src_op.dtype));
 }
 
-void ConvertStackOperator(const Model& model, const StackOperator& src_op,
-                          GraphDef* tensorflow_graph) {
-  tensorflow::NodeDef* stack_op = tensorflow_graph->add_node();
-  stack_op->set_op("Stack");
-  stack_op->set_name(src_op.outputs[0]);
+void ConvertPackOperator(const Model& model, const PackOperator& src_op,
+                         GraphDef* tensorflow_graph) {
+  tensorflow::NodeDef* pack_op = tensorflow_graph->add_node();
+  pack_op->set_op("Pack");
+  pack_op->set_name(src_op.outputs[0]);
   for (const auto& input : src_op.inputs) {
-    *stack_op->add_input() = input;
+    *pack_op->add_input() = input;
   }
-  (*stack_op->mutable_attr())["elem_type"].set_type(
+  (*pack_op->mutable_attr())["elem_type"].set_type(
       GetTensorFlowDataType(model, src_op.outputs[0]));
-  (*stack_op->mutable_attr())["axis"].set_i(src_op.axis);
+  (*pack_op->mutable_attr())["axis"].set_i(src_op.axis);
+  (*pack_op->mutable_attr())["N"].set_i(src_op.inputs.size());
+  (*pack_op->mutable_attr())["T"].set_type(GetTensorFlowDataType(src_op.dtype));
 }
 
 void ConvertFillOperator(const Model& model, const FillOperator& src_op,
@@ -2015,9 +2017,9 @@ void ConvertOperator(const Model& model, const Operator& src_op,
   } else if (src_op.type == OperatorType::kRange) {
     ConvertRangeOperator(model, static_cast(src_op),
                          tensorflow_graph);
-  } else if (src_op.type == OperatorType::kStack) {
-    ConvertStackOperator(model, static_cast(src_op),
-                         tensorflow_graph);
+  } else if (src_op.type == OperatorType::kPack) {
+    ConvertPackOperator(model, static_cast(src_op),
+                        tensorflow_graph);
   } else if (src_op.type == OperatorType::kFill) {
     ConvertFillOperator(model, static_cast(src_op),
                         tensorflow_graph);
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/convert_trivial_pack_to_reshape.cc b/tensorflow/contrib/lite/toco/graph_transformations/convert_trivial_pack_to_reshape.cc
new file mode 100644
index 0000000000..75113a2a8c
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/convert_trivial_pack_to_reshape.cc
@@ -0,0 +1,81 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include 
+#include 
+#include 
+#include 
+
+#include "absl/strings/str_cat.h"
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace toco {
+
+bool ConvertTrivialPackToReshape::Run(Model* model, std::size_t op_index) {
+  auto pack_it = model->operators.begin() + op_index;
+  if (pack_it->get()->type != OperatorType::kPack) {
+    return false;
+  }
+  auto* pack_op = static_cast(pack_it->get());
+  if (pack_op->inputs.size() > 1) {
+    // Not trivial.
+    return false;
+  }
+  CHECK_EQ(pack_op->outputs.size(), 1);
+
+  const auto& input_array = model->GetArray(pack_op->inputs[0]);
+  if (!input_array.has_shape()) {
+    // Yield until input dims have been resolved.
+    return false;
+  }
+  if (input_array.shape().dimensions_count() == 0) {
+    // Input array cannot be 0-D.
+    // (Unsure if this is TF behavior, but was required to get a test to pass.)
+    return false;
+  }
+
+  AddMessageF("Converting trivial %s to a reshape", LogName(*pack_op));
+
+  // Note that we could convert to ExpandDims but toco prefers reshapes.
+  auto* reshape_op = new TensorFlowReshapeOperator;
+  reshape_op->inputs = {pack_op->inputs[0]};
+  reshape_op->outputs = pack_op->outputs;
+
+  // Create shape param.
+  string shape_array_name =
+      AvailableArrayName(*model, pack_op->outputs[0] + "_shape");
+  Array& shape_array = model->GetOrCreateArray(shape_array_name);
+  *(shape_array.mutable_shape()->mutable_dims()) = {
+      1 + input_array.shape().dimensions_count()};
+  reshape_op->inputs.push_back(shape_array_name);
+  shape_array.data_type = ArrayDataType::kInt32;
+  auto& shape_buffer = shape_array.GetMutableBuffer();
+  shape_buffer.data.push_back(1);
+  for (int dim : input_array.shape().dims()) {
+    shape_buffer.data.push_back(dim);
+  }
+
+  // Replace the operator in the graph.
+  const auto reshape_it = model->operators.emplace(pack_it, reshape_op);
+  pack_it = reshape_it + 1;
+  CHECK_EQ(pack_it->get(), pack_op);
+  model->operators.erase(pack_it);
+
+  return true;
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/convert_trivial_stack_to_reshape.cc b/tensorflow/contrib/lite/toco/graph_transformations/convert_trivial_stack_to_reshape.cc
deleted file mode 100644
index 0615b5e6c6..0000000000
--- a/tensorflow/contrib/lite/toco/graph_transformations/convert_trivial_stack_to_reshape.cc
+++ /dev/null
@@ -1,81 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include 
-#include 
-#include 
-#include 
-
-#include "absl/strings/str_cat.h"
-#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
-#include "tensorflow/contrib/lite/toco/model.h"
-#include "tensorflow/contrib/lite/toco/tooling_util.h"
-#include "tensorflow/core/platform/logging.h"
-
-namespace toco {
-
-bool ConvertTrivialStackToReshape::Run(Model* model, std::size_t op_index) {
-  auto stack_it = model->operators.begin() + op_index;
-  if (stack_it->get()->type != OperatorType::kStack) {
-    return false;
-  }
-  auto* stack_op = static_cast(stack_it->get());
-  if (stack_op->inputs.size() > 1) {
-    // Not trivial.
-    return false;
-  }
-  CHECK_EQ(stack_op->outputs.size(), 1);
-
-  const auto& input_array = model->GetArray(stack_op->inputs[0]);
-  if (!input_array.has_shape()) {
-    // Yield until input dims have been resolved.
-    return false;
-  }
-  if (input_array.shape().dimensions_count() == 0) {
-    // Input array cannot be 0-D.
-    // (Unsure if this is TF behavior, but was required to get a test to pass.)
-    return false;
-  }
-
-  AddMessageF("Converting trivial %s to a reshape", LogName(*stack_op));
-
-  // Note that we could convert to ExpandDims but toco prefers reshapes.
-  auto* reshape_op = new TensorFlowReshapeOperator;
-  reshape_op->inputs = {stack_op->inputs[0]};
-  reshape_op->outputs = stack_op->outputs;
-
-  // Create shape param.
-  string shape_array_name =
-      AvailableArrayName(*model, stack_op->outputs[0] + "_shape");
-  Array& shape_array = model->GetOrCreateArray(shape_array_name);
-  *(shape_array.mutable_shape()->mutable_dims()) = {
-      1 + input_array.shape().dimensions_count()};
-  reshape_op->inputs.push_back(shape_array_name);
-  shape_array.data_type = ArrayDataType::kInt32;
-  auto& shape_buffer = shape_array.GetMutableBuffer();
-  shape_buffer.data.push_back(1);
-  for (int dim : input_array.shape().dims()) {
-    shape_buffer.data.push_back(dim);
-  }
-
-  // Replace the operator in the graph.
-  const auto reshape_it = model->operators.emplace(stack_it, reshape_op);
-  stack_it = reshape_it + 1;
-  CHECK_EQ(stack_it->get(), stack_op);
-  model->operators.erase(stack_it);
-
-  return true;
-}
-
-}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
index 8db7df5c0e..5cee08fd4c 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
@@ -116,7 +116,7 @@ DECLARE_GRAPH_TRANSFORMATION(ConvertExpandDimsToReshape)
 DECLARE_GRAPH_TRANSFORMATION(ConvertPureConvToDepthwise)
 DECLARE_GRAPH_TRANSFORMATION(ConvertSqueezeToReshape)
 DECLARE_GRAPH_TRANSFORMATION(ConvertTrivialAddNToAdd)
-DECLARE_GRAPH_TRANSFORMATION(ConvertTrivialStackToReshape)
+DECLARE_GRAPH_TRANSFORMATION(ConvertTrivialPackToReshape)
 DECLARE_GRAPH_TRANSFORMATION(ConvertTrivialTileToConcat)
 DECLARE_GRAPH_TRANSFORMATION(ConvertTrivialTransposeToReshape)
 DECLARE_GRAPH_TRANSFORMATION(ConvertReorderAxes)
@@ -182,11 +182,11 @@ DECLARE_GRAPH_TRANSFORMATION(ResolveStridedSliceAttributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveSliceAttributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveReduceAttributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveTransposeAttributes)
+DECLARE_GRAPH_TRANSFORMATION(ResolveConstantPack)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantRandomUniform)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantRange)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantShapeOrRank)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantSlice)
-DECLARE_GRAPH_TRANSFORMATION(ResolveConstantStack)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantStridedSlice)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantFill)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantGather)
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index f422e3a9c7..5e2ba0eca7 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1192,7 +1192,7 @@ void ProcessShapeOperator(Model* model, TensorFlowShapeOperator* op) {
   output_shape->ReplaceDims({input_array.shape().dimensions_count()});
 }
 
-void ProcessStackOperator(Model* model, StackOperator* op) {
+void ProcessPackOperator(Model* model, PackOperator* op) {
   CHECK_GE(op->inputs.size(), 1);
   CHECK_EQ(op->outputs.size(), 1);
   auto& output_array = model->GetArray(op->outputs[0]);
@@ -1201,7 +1201,7 @@ void ProcessStackOperator(Model* model, StackOperator* op) {
     return;
   }
 
-  std::unique_ptr stacked_shape;
+  std::unique_ptr packed_shape;
   for (const auto& input : op->inputs) {
     const auto& input_array = model->GetArray(input);
     if (!input_array.has_shape()) {
@@ -1210,23 +1210,23 @@ void ProcessStackOperator(Model* model, StackOperator* op) {
     }
 
     Shape shape = input_array.shape();
-    if (!stacked_shape) {
-      stacked_shape.reset(new Shape(shape));
+    if (!packed_shape) {
+      packed_shape.reset(new Shape(shape));
     } else {
-      CHECK(*stacked_shape == shape) << "All input arrays to Stack operators "
-                                        "must have the same shape. Input \""
-                                     << input << "\" is different.";
+      CHECK(*packed_shape == shape) << "All input arrays to Pack operators "
+                                       "must have the same shape. Input \""
+                                    << input << "\" is different.";
     }
   }
 
   int axis = op->axis;
   if (axis < 0) {
     // Handle negative axis
-    axis += stacked_shape->dims().size() + 1;
+    axis += packed_shape->dims().size() + 1;
   }
-  stacked_shape->mutable_dims()->insert(
-      stacked_shape->mutable_dims()->begin() + axis, op->inputs.size());
-  output_array.copy_shape(*stacked_shape);
+  packed_shape->mutable_dims()->insert(
+      packed_shape->mutable_dims()->begin() + axis, op->inputs.size());
+  output_array.copy_shape(*packed_shape);
 }
 
 void ProcessStridedSliceOperator(Model* model, StridedSliceOperator* op) {
@@ -1659,8 +1659,8 @@ bool PropagateFixedSizes::Run(Model* model, std::size_t op_index) {
     case OperatorType::kShape:
       ProcessShapeOperator(model, static_cast(op));
       break;
-    case OperatorType::kStack:
-      ProcessStackOperator(model, static_cast(op));
+    case OperatorType::kPack:
+      ProcessPackOperator(model, static_cast(op));
       break;
     case OperatorType::kReorderAxes:
       ProcessReorderAxesOperator(model, static_cast(op));
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_pack.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_pack.cc
new file mode 100644
index 0000000000..e86616574d
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_pack.cc
@@ -0,0 +1,117 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include 
+
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace toco {
+
+namespace {
+
+template 
+void Pack(Model* model, PackOperator const& op) {
+  auto& output_array = model->GetArray(op.outputs[0]);
+  CHECK(output_array.data_type == Type);
+
+  // Create a buffer for the output array
+  std::vector>& output_data =
+      output_array.GetMutableBuffer().data;
+  output_data.resize(RequiredBufferSizeForShape(output_array.shape()));
+
+  // Pack inputs into buffer
+  CHECK_EQ(op.axis, 0) << "Packing only supported along first axis";
+  int dst_offset = 0;
+  for (int i = 0; i < op.inputs.size(); i++) {
+    // Append array data to output for each input array
+    const auto& input_array = model->GetArray(op.inputs[i]);
+    int input_size = RequiredBufferSizeForShape(input_array.shape());
+    memcpy(&output_data[dst_offset], &input_array.GetBuffer().data[0],
+           input_size * ElementSize(Type));
+    dst_offset += input_size;
+  }
+  CHECK_EQ(dst_offset, output_data.size());
+}
+
+}  // namespace
+
+bool ResolveConstantPack::Run(Model* model, std::size_t op_index) {
+  auto it = model->operators.begin() + op_index;
+  const auto* base_op = it->get();
+  if (base_op->type != OperatorType::kPack) {
+    return false;
+  }
+  const auto* op = static_cast(base_op);
+
+  CHECK_GE(op->inputs.size(), 1);
+  CHECK_EQ(op->outputs.size(), 1);
+  auto& output_array = model->GetArray(op->outputs[0]);
+  if (output_array.data_type == ArrayDataType::kNone) {
+    // Yield until the output type has been set by PropagateArrayDataTypes
+    return false;
+  }
+
+  if (!output_array.has_shape()) {
+    // Yield until the output shape has been set by PropagateFixedShapes
+    return false;
+  }
+
+  for (const auto& input : op->inputs) {
+    if (!IsConstantParameterArray(*model, input)) {
+      // Yield if any input is mutable
+      return false;
+    }
+  }
+
+  int axis = op->axis;
+  if (axis < 0) {
+    // Handle negative axis
+    axis += model->GetArray(op->inputs[0]).shape().dims().size();
+  }
+  CHECK_EQ(axis, 0) << "Packing only supported along 0th axis";
+
+  CHECK(!output_array.buffer);
+  switch (output_array.data_type) {
+    case ArrayDataType::kFloat:
+      Pack(model, *op);
+      break;
+    case ArrayDataType::kUint8:
+      Pack(model, *op);
+      break;
+    case ArrayDataType::kInt32:
+      Pack(model, *op);
+      break;
+    case ArrayDataType::kInt64:
+      Pack(model, *op);
+      break;
+    default:
+      LOG(FATAL) << "Unsupported data type given to Pack op with output \""
+                 << op->outputs[0] << "\"";
+      break;
+  }
+
+  // Erase input arrays if no longer used
+  for (const auto& input : op->inputs) {
+    toco::DeleteArrayIfUsedOnce(input, model);
+  }
+
+  // Erase the operator
+  model->operators.erase(it);
+  return true;
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_stack.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_stack.cc
deleted file mode 100644
index a4d5f1923a..0000000000
--- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_stack.cc
+++ /dev/null
@@ -1,117 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include 
-
-#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
-#include "tensorflow/contrib/lite/toco/model.h"
-#include "tensorflow/contrib/lite/toco/tooling_util.h"
-#include "tensorflow/core/platform/logging.h"
-
-namespace toco {
-
-namespace {
-
-template 
-void Stack(Model* model, StackOperator const& op) {
-  auto& output_array = model->GetArray(op.outputs[0]);
-  CHECK(output_array.data_type == Type);
-
-  // Create a buffer for the output array
-  std::vector>& output_data =
-      output_array.GetMutableBuffer().data;
-  output_data.resize(RequiredBufferSizeForShape(output_array.shape()));
-
-  // Stack inputs into buffer
-  CHECK_EQ(op.axis, 0) << "Stacking only supported along first axis";
-  int dst_offset = 0;
-  for (int i = 0; i < op.inputs.size(); i++) {
-    // Append array data to output for each input array
-    const auto& input_array = model->GetArray(op.inputs[i]);
-    int input_size = RequiredBufferSizeForShape(input_array.shape());
-    memcpy(&output_data[dst_offset], &input_array.GetBuffer().data[0],
-           input_size * ElementSize(Type));
-    dst_offset += input_size;
-  }
-  CHECK_EQ(dst_offset, output_data.size());
-}
-
-}  // namespace
-
-bool ResolveConstantStack::Run(Model* model, std::size_t op_index) {
-  auto it = model->operators.begin() + op_index;
-  const auto* base_op = it->get();
-  if (base_op->type != OperatorType::kStack) {
-    return false;
-  }
-  const auto* op = static_cast(base_op);
-
-  CHECK_GE(op->inputs.size(), 1);
-  CHECK_EQ(op->outputs.size(), 1);
-  auto& output_array = model->GetArray(op->outputs[0]);
-  if (output_array.data_type == ArrayDataType::kNone) {
-    // Yield until the output type has been set by PropagateArrayDataTypes
-    return false;
-  }
-
-  if (!output_array.has_shape()) {
-    // Yield until the output shape has been set by PropagateFixedShapes
-    return false;
-  }
-
-  for (const auto& input : op->inputs) {
-    if (!IsConstantParameterArray(*model, input)) {
-      // Yield if any input is mutable
-      return false;
-    }
-  }
-
-  int axis = op->axis;
-  if (axis < 0) {
-    // Handle negative axis
-    axis += model->GetArray(op->inputs[0]).shape().dims().size();
-  }
-  CHECK_EQ(axis, 0) << "Stacking only supported along 0th axis";
-
-  CHECK(!output_array.buffer);
-  switch (output_array.data_type) {
-    case ArrayDataType::kFloat:
-      Stack(model, *op);
-      break;
-    case ArrayDataType::kUint8:
-      Stack(model, *op);
-      break;
-    case ArrayDataType::kInt32:
-      Stack(model, *op);
-      break;
-    case ArrayDataType::kInt64:
-      Stack(model, *op);
-      break;
-    default:
-      LOG(FATAL) << "Unsupported data type given to Stack op with output \""
-                 << op->outputs[0] << "\"";
-      break;
-  }
-
-  // Erase input arrays if no longer used
-  for (const auto& input : op->inputs) {
-    toco::DeleteArrayIfUsedOnce(input, model);
-  }
-
-  // Erase the operator
-  model->operators.erase(it);
-  return true;
-}
-
-}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/unroll_batch_matmul.cc b/tensorflow/contrib/lite/toco/graph_transformations/unroll_batch_matmul.cc
index da81ea2ff3..5f0cece67a 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/unroll_batch_matmul.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/unroll_batch_matmul.cc
@@ -76,7 +76,7 @@ bool UnrollBatchMatMul::Run(Model* model, std::size_t op_index) {
   AddMessageF("Unrolling BatchMatMul %s %d times", LogName(*batch_op),
               batch_count);
   auto tail_it = batch_op_it;
-  std::vector stack_inputs;
+  std::vector pack_inputs;
   for (int batch = 0; batch < batch_count; ++batch) {
     std::string batch_name =
         std::string(batch_op->outputs[0]) + "_b" + std::to_string(batch);
@@ -146,15 +146,15 @@ bool UnrollBatchMatMul::Run(Model* model, std::size_t op_index) {
     tail_it = model->operators.emplace(tail_it, matmul_op) + 1;
 
     // Add to stack.
-    stack_inputs.push_back(matmul_op->outputs[0]);
+    pack_inputs.push_back(matmul_op->outputs[0]);
   }
 
-  // The stack that will join all the individual matmul results together.
-  auto* stack_op = new StackOperator;
-  stack_op->inputs = stack_inputs;
-  stack_op->outputs = {batch_op->outputs[0]};
-  stack_op->axis = 0;
-  model->operators.emplace(tail_it, stack_op);
+  // The pack that will join all the individual matmul results together.
+  auto* pack_op = new PackOperator;
+  pack_op->inputs = pack_inputs;
+  pack_op->outputs = {batch_op->outputs[0]};
+  pack_op->axis = 0;
+  model->operators.emplace(tail_it, pack_op);
 
   // Remove the old batch matmul now that we've unrolled.
   batch_op_it = model->operators.begin();
diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc
index 2ffab49e7a..576eb71534 100644
--- a/tensorflow/contrib/lite/toco/import_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc
@@ -1518,11 +1518,15 @@ tensorflow::Status ConvertRangeOperator(
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status ConvertStackOperator(
+// Note that it's easy to confuse/conflate "Stack" and "Pack" operators, but
+// they aren't the same thing.  tf.stack results in a "Pack" operator.  "Stack"
+// operators also exist, but involve manipulating the TF runtime stack, and are
+// not directly related to tf.stack() usage.
+tensorflow::Status ConvertPackOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
-  CHECK((node.op() == "Stack") || (node.op() == "Pack"));
-  auto* op = new StackOperator;
+  CHECK_EQ(node.op(), "Pack");
+  auto op = absl::make_unique();
   const int num_inputs = GetInputsCount(node, tf_import_flags);
   QCHECK_GE(num_inputs, 1)
       << node.op()
@@ -1532,10 +1536,10 @@ tensorflow::Status ConvertStackOperator(
   for (int i = 0; i < num_inputs; ++i) {
     op->inputs.push_back(node.input(i));
   }
-  // Both "Stack" and "Pack" have the "axis" attribute.
   op->axis = HasAttr(node, "axis") ? GetIntAttr(node, "axis") : 0;
+  op->dtype = ConvertDataType(toco::GetDataTypeAttr(node, "T"));
   op->outputs.push_back(node.name());
-  model->operators.emplace_back(op);
+  model->operators.emplace_back(std::move(op));
   return tensorflow::Status::OK();
 }
 
@@ -1873,7 +1877,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"NextIteration", ConvertOperatorSpecialCasedAsRNNBackEdge},
       {"NoOp", ConvertNoOpOperator},
       {"NotEqual", ConvertSimpleOperator},
-      {"Pack", ConvertStackOperator},
+      {"Pack", ConvertPackOperator},
       {"Pad", ConvertSimpleOperator},
       {"PadV2", ConvertSimpleOperator},
       {"ParallelDynamicStitch", ConvertDynamicStitchOperator},
@@ -1903,7 +1907,6 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Sqrt", ConvertSimpleOperator},
       {"Square", ConvertSimpleOperator},
       {"Squeeze", ConvertSqueezeOperator},
-      {"Stack", ConvertStackOperator},
       {"StopGradient", ConvertIdentityOperator},
       {"StridedSlice", ConvertStridedSliceOperator},
       {"Sub", ConvertSimpleOperator},
diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h
index 37f4188cf7..8fff68cf47 100644
--- a/tensorflow/contrib/lite/toco/model.h
+++ b/tensorflow/contrib/lite/toco/model.h
@@ -81,7 +81,7 @@ enum class OperatorType : uint8 {
   kResizeBilinear,
   kSin,
   kSpaceToBatchND,
-  kStack,
+  kPack,
   kBatchToSpaceND,
   kPad,
   kPadV2,
@@ -1157,10 +1157,11 @@ struct TensorFlowRsqrtOperator : Operator {
 // Inputs: this operator accepts any number >= 1 of inputs.
 //   inputs[i]: the i-th array to merge.
 //
-// TensorFlow equivalent: Stack or Pack
-struct StackOperator : Operator {
-  StackOperator() : Operator(OperatorType::kStack) {}
+// TensorFlow equivalent: Pack
+struct PackOperator : Operator {
+  PackOperator() : Operator(OperatorType::kPack) {}
   int axis = 0;
+  ArrayDataType dtype = ArrayDataType::kNone;
 };
 
 // Shape operator. Extracts the shape of the tensor.
diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc
index 7a0d9608cc..d8964ebc13 100644
--- a/tensorflow/contrib/lite/toco/toco_tooling.cc
+++ b/tensorflow/contrib/lite/toco/toco_tooling.cc
@@ -55,7 +55,7 @@ void MakeGeneralGraphTransformationsSet(
   transformations->Add(new ConvertExpandDimsToReshape);
   transformations->Add(new ConvertSqueezeToReshape);
   transformations->Add(new ConvertTrivialAddNToAdd);
-  transformations->Add(new ConvertTrivialStackToReshape);
+  transformations->Add(new ConvertTrivialPackToReshape);
   transformations->Add(new ConvertTrivialTileToConcat);
   transformations->Add(new ConvertTrivialTransposeToReshape);
   transformations->Add(new ConvertReorderAxes);
@@ -86,11 +86,11 @@ void MakeGeneralGraphTransformationsSet(
   transformations->Add(new ResolveConstantBinaryOperator);
   transformations->Add(new ResolveConstantFill);
   transformations->Add(new ResolveConstantGather);
+  transformations->Add(new ResolveConstantPack);
   transformations->Add(new ResolveConstantRandomUniform);
   transformations->Add(new ResolveConstantRange);
   transformations->Add(new ResolveConstantReshape);
   transformations->Add(new ResolveConstantSlice);
-  transformations->Add(new ResolveConstantStack);
   transformations->Add(new ResolveConstantStridedSlice);
   transformations->Add(new ResolveConstantTranspose);
   transformations->Add(new ResolveConstantUnaryOperator);
diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index 45cd10ec7b..4305727c8c 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -356,10 +356,10 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(Min)      //  Reduction Min
     HANDLE_OPERATORTYPENAME_CASE(Minimum)  //  Element-wise Minimum
     HANDLE_OPERATORTYPENAME_CASE(Neg)
+    HANDLE_OPERATORTYPENAME_CASE(Pack)
     HANDLE_OPERATORTYPENAME_CASE(Pad)
     HANDLE_OPERATORTYPENAME_CASE(PadV2)
     HANDLE_OPERATORTYPENAME_CASE(StridedSlice)
-    HANDLE_OPERATORTYPENAME_CASE(Stack)
     HANDLE_OPERATORTYPENAME_CASE(Range)
     HANDLE_OPERATORTYPENAME_CASE(Rank)
     HANDLE_OPERATORTYPENAME_CASE(Reshape)
-- 
cgit v1.2.3


From bc13af4b9f2a76b6424af63da7f88aac4602c0cb Mon Sep 17 00:00:00 2001
From: Yifei Feng 
Date: Tue, 17 Jul 2018 12:20:20 -0700
Subject: Fix sanity failure from cl/204944404

PiperOrigin-RevId: 204953899
---
 tensorflow/docs_src/performance/xla/developing_new_backend.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/performance/xla/developing_new_backend.md b/tensorflow/docs_src/performance/xla/developing_new_backend.md
index 74ea15bb2b..840f6983c2 100644
--- a/tensorflow/docs_src/performance/xla/developing_new_backend.md
+++ b/tensorflow/docs_src/performance/xla/developing_new_backend.md
@@ -44,7 +44,7 @@ It is possible to model a new
 implementation on the existing [`xla::CPUCompiler`]
 (https://www.tensorflow.org/code/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc)
 and [`xla::GPUCompiler`]
-(https://www.tensorflow.org/code/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc)
+(https://www.tensorflow.org/code/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc)
 classes, since these already emit LLVM IR. Depending on the nature of the
 hardware, it is possible that many of the LLVM IR generation aspects will have
 to be changed, but a lot of code can be shared with the existing backends.
-- 
cgit v1.2.3


From fb42ab1be6e88977091de28b6e14f4ebf40080ef Mon Sep 17 00:00:00 2001
From: Michael Kuperstein 
Date: Tue, 17 Jul 2018 12:45:11 -0700
Subject: [XLA] Implement sorting second dimension of a rank-2 in the evaluator

PiperOrigin-RevId: 204957859
---
 tensorflow/compiler/xla/service/hlo_evaluator.cc   | 116 +++++++++++++++------
 .../xla/service/hlo_evaluator_typed_visitor.h      |  51 ++++++---
 2 files changed, 122 insertions(+), 45 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index dfdfeb49a2..51353eea6e 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1122,45 +1122,90 @@ Status HloEvaluator::HandleWhile(HloInstruction* while_hlo) {
 // hoops to make this work.
 namespace {
 template 
-std::unique_ptr EvaluateSortInternal(HloInstruction* sort,
-                                              const Literal& keys_literal,
-                                              const Literal& values_literal) {
-  CHECK_EQ(sort->operand_count(), 2);
+StatusOr> EvaluateSortInternal(
+    HloInstruction* sort, const Literal& keys_literal,
+    const Literal& values_literal) {
+  auto rank = ShapeUtil::Rank(keys_literal.shape());
+  TF_RET_CHECK(
+      ShapeUtil::SameDimensions(keys_literal.shape(), values_literal.shape()))
+      << "Sort keys and values must have the same dimensions";
+  TF_RET_CHECK(rank > 0 && rank <= 2)
+      << "Sort is only supported for rank-1 and rank-2 shapes, rank is: "
+      << rank;
+  TF_RET_CHECK(sort->operand_count() == 2) << "Expected key-value sort";
   // We need to sort and array of keys and an array of values, where the
   // sorted order of the values is determined by the keys. The simplest(?)
   // way to do this is to go to an array-of-pairs representation, sort the
   // array using the keys, and then go back to pair-of-arrays.
   VLOG(3) << "HandleSort keys_literal: " << keys_literal.ToString();
   VLOG(3) << "HandleSort values_literal: " << values_literal.ToString();
-  const auto& keys_data = keys_literal.data();
-  const auto& values_data = values_literal.data();
-  using kv_pair = std::pair;
-  std::vector key_value_vector;
-  CHECK_EQ(keys_data.size(), values_data.size());
-  key_value_vector.reserve(keys_data.size());
-  for (int i = 0; i < keys_data.size(); ++i) {
-    key_value_vector.push_back(std::make_pair(keys_data[i], values_data[i]));
-  }
-  std::sort(key_value_vector.begin(), key_value_vector.end(),
-            [](const kv_pair& a, const kv_pair& b) {
-              return SafeLess(a.first, b.first);
-            });
-  std::vector result_keys;
-  std::vector result_values;
-  for (const auto& key_value : key_value_vector) {
-    result_keys.push_back(key_value.first);
-    result_values.push_back(key_value.second);
+
+  auto sort_r1 = [](const Literal& keys_literal,
+                    const Literal& values_literal) {
+    const auto& keys_data = keys_literal.data();
+    const auto& values_data = values_literal.data();
+
+    using kv_pair = std::pair;
+    std::vector key_value_vector;
+    CHECK_EQ(keys_data.size(), values_data.size());
+    key_value_vector.reserve(keys_data.size());
+    for (int i = 0; i < keys_data.size(); ++i) {
+      key_value_vector.push_back(std::make_pair(keys_data[i], values_data[i]));
+    }
+    std::sort(key_value_vector.begin(), key_value_vector.end(),
+              [](const kv_pair& a, const kv_pair& b) {
+                return SafeLess(a.first, b.first);
+              });
+    std::vector result_keys;
+    std::vector result_values;
+    for (const auto& key_value : key_value_vector) {
+      result_keys.push_back(key_value.first);
+      result_values.push_back(key_value.second);
+    }
+    auto result_keys_literal = MakeUnique(keys_literal.shape());
+    result_keys_literal->PopulateR1(
+        tensorflow::gtl::ArraySlice(result_keys));
+    auto result_values_literal = MakeUnique(values_literal.shape());
+    result_values_literal->PopulateR1(
+        tensorflow::gtl::ArraySlice(result_values));
+    return std::make_pair(std::move(result_keys_literal),
+                          std::move(result_values_literal));
+  };
+
+  std::unique_ptr result_tuple;
+  if (rank == 1) {
+    auto result_pair = sort_r1(keys_literal, values_literal);
+    result_tuple = LiteralUtil::MakeTuple(
+        {result_pair.first.get(), result_pair.second.get()});
+  } else {
+    // For R2 sort, the desired semantics are to sort each matrix row
+    // independently.
+    auto keys_result_literal = MakeUnique(keys_literal.shape());
+    auto values_result_literal = MakeUnique(values_literal.shape());
+    int64 r1_length = keys_literal.shape().dimensions(1);
+    for (int64 row = 0; row < keys_literal.shape().dimensions(0); ++row) {
+      TF_ASSIGN_OR_RETURN(auto keys_r1_slice,
+                          keys_literal.Slice({row, 0}, {row + 1, r1_length})
+                              ->Reshape({r1_length}));
+      TF_ASSIGN_OR_RETURN(auto values_r1_slice,
+                          values_literal.Slice({row, 0}, {row + 1, r1_length})
+                              ->Reshape({r1_length}));
+      auto r1_result_pair = sort_r1(*keys_r1_slice, *values_r1_slice);
+      TF_ASSIGN_OR_RETURN(auto sorted_keys,
+                          r1_result_pair.first->Reshape({1, r1_length}));
+      TF_ASSIGN_OR_RETURN(auto sorted_values,
+                          r1_result_pair.second->Reshape({1, r1_length}));
+      TF_RETURN_IF_ERROR(keys_result_literal->CopySliceFrom(
+          *sorted_keys, {0, 0}, {row, 0}, {1, r1_length}));
+      TF_RETURN_IF_ERROR(values_result_literal->CopySliceFrom(
+          *sorted_values, {0, 0}, {row, 0}, {1, r1_length}));
+    }
+    result_tuple = LiteralUtil::MakeTuple(
+        {keys_result_literal.get(), values_result_literal.get()});
   }
-  auto result_keys_literal = MakeUnique(sort->operand(0)->shape());
-  result_keys_literal->PopulateR1(
-      tensorflow::gtl::ArraySlice(result_keys));
-  auto result_values_literal = MakeUnique(sort->operand(1)->shape());
-  result_values_literal->PopulateR1(
-      tensorflow::gtl::ArraySlice(result_values));
-  auto result_tuple = LiteralUtil::MakeTuple(
-      {result_keys_literal.get(), result_values_literal.get()});
+
   VLOG(3) << "HandleSort result_tuple: " << result_tuple->ToString();
-  return result_tuple;
+  return std::move(result_tuple);
 }
 
 template 
@@ -1204,6 +1249,15 @@ StatusOr> EvaluateSort(HloInstruction* sort,
 }  // namespace
 
 Status HloEvaluator::HandleSort(HloInstruction* sort) {
+  const int64 sort_dim = sort->dimensions(0);
+  const int64 rank = ShapeUtil::Rank(sort->operand(0)->shape());
+  if (sort_dim != rank - 1) {
+    return Unimplemented(
+        "Trying to support along dimension %lld, which is not the last "
+        "dimension",
+        sort_dim);
+  }
+
   if (!ShapeUtil::IsTuple(sort->shape())) {
     return DefaultAction(sort);
   } else {
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index e1924a0f8e..c0a8ea8bcb 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1415,25 +1415,48 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
                 !std::is_same::value>::type* = nullptr>
   Status HandleSort(HloInstruction* sort) {
     auto keys = sort->operand(0);
-    TF_RET_CHECK(ShapeUtil::Rank(keys->shape()) == 1)
-        << "Sort is only supported for R1 shapes";
+    auto rank = ShapeUtil::Rank(keys->shape());
+    TF_RET_CHECK(rank > 0 && rank <= 2)
+        << "Sort is only supported for R1 and R2 shapes";
     TF_RET_CHECK(sort->operand_count() == 1)
         << "Typed visitor does not support key-value sort";
 
     const Literal& keys_literal = parent_->GetEvaluatedLiteralFor(keys);
-    VLOG(3) << "HandleSort keys_literal: " << keys_literal.ToString();
-    const auto& keys_data = keys_literal.data();
 
-    std::vector result_data(keys_data.begin(), keys_data.end());
-    std::sort(result_data.begin(), result_data.end(),
-              [](const ReturnT& a, const ReturnT& b) {
-                return SafeLess(a, b);
-              });
-    auto result_literal = MakeUnique(sort->shape());
-    result_literal->PopulateR1(
-        tensorflow::gtl::ArraySlice(result_data));
-    VLOG(3) << "HandleSort result_literal: " << result_literal->ToString();
-    parent_->evaluated_[sort] = std::move(result_literal);
+    auto sort_r1 = [this](const Literal& keys_literal) {
+      VLOG(3) << "HandleSort keys_literal: " << keys_literal.ToString();
+      const auto& keys_data = keys_literal.data();
+
+      std::vector result_data(keys_data.begin(), keys_data.end());
+      std::sort(result_data.begin(), result_data.end(),
+                [](const ReturnT& a, const ReturnT& b) {
+                  return SafeLess(a, b);
+                });
+      auto result_literal = MakeUnique(keys_literal.shape());
+      result_literal->PopulateR1(
+          tensorflow::gtl::ArraySlice(result_data));
+      VLOG(3) << "HandleSort result_literal: " << result_literal->ToString();
+      return result_literal;
+    };
+
+    if (rank == 1) {
+      parent_->evaluated_[sort] = std::move(sort_r1(keys_literal));
+    } else {
+      // For R2 sort, the desired semantics are to sort each matrix row
+      // independently.
+      auto result_literal = MakeUnique(keys_literal.shape());
+      int64 r1_length = keys->shape().dimensions(1);
+      for (int64 row = 0; row < keys->shape().dimensions(0); ++row) {
+        TF_ASSIGN_OR_RETURN(auto r1_slice,
+                            keys_literal.Slice({row, 0}, {row + 1, r1_length})
+                                ->Reshape({r1_length}));
+        auto r1_result = sort_r1(*r1_slice);
+        TF_ASSIGN_OR_RETURN(r1_result, r1_result->Reshape({1, r1_length}));
+        TF_RETURN_IF_ERROR(result_literal->CopySliceFrom(
+            *r1_result, {0, 0}, {row, 0}, {1, r1_length}));
+      }
+      parent_->evaluated_[sort] = std::move(result_literal);
+    }
     return Status::OK();
   }
 
-- 
cgit v1.2.3


From 82fa2f491e809c753c04587f14d671367946b121 Mon Sep 17 00:00:00 2001
From: Asim Shankar 
Date: Tue, 17 Jul 2018 13:18:55 -0700
Subject: [Java]: Remove some dead code.

PiperOrigin-RevId: 204962948
---
 .../src/main/java/org/tensorflow/types/TFBool.java | 30 -------------
 .../main/java/org/tensorflow/types/TFDouble.java   | 30 -------------
 .../main/java/org/tensorflow/types/TFFloat.java    | 30 -------------
 .../main/java/org/tensorflow/types/TFInt32.java    | 30 -------------
 .../main/java/org/tensorflow/types/TFInt64.java    | 30 -------------
 .../main/java/org/tensorflow/types/TFString.java   | 27 -----------
 .../src/main/java/org/tensorflow/types/TFType.java | 20 ---------
 .../main/java/org/tensorflow/types/TFUInt8.java    | 30 -------------
 .../src/main/java/org/tensorflow/types/Types.java  | 52 ----------------------
 9 files changed, 279 deletions(-)
 delete mode 100644 tensorflow/java/src/main/java/org/tensorflow/types/TFBool.java
 delete mode 100644 tensorflow/java/src/main/java/org/tensorflow/types/TFDouble.java
 delete mode 100644 tensorflow/java/src/main/java/org/tensorflow/types/TFFloat.java
 delete mode 100644 tensorflow/java/src/main/java/org/tensorflow/types/TFInt32.java
 delete mode 100644 tensorflow/java/src/main/java/org/tensorflow/types/TFInt64.java
 delete mode 100644 tensorflow/java/src/main/java/org/tensorflow/types/TFString.java
 delete mode 100644 tensorflow/java/src/main/java/org/tensorflow/types/TFType.java
 delete mode 100644 tensorflow/java/src/main/java/org/tensorflow/types/TFUInt8.java
 delete mode 100644 tensorflow/java/src/main/java/org/tensorflow/types/Types.java

diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/TFBool.java b/tensorflow/java/src/main/java/org/tensorflow/types/TFBool.java
deleted file mode 100644
index ab34f6aa12..0000000000
--- a/tensorflow/java/src/main/java/org/tensorflow/types/TFBool.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// GENERATED FILE. To update, edit tftypes.pl instead.
-
-package org.tensorflow.types;
-
-import org.tensorflow.DataType;
-
-/** Represents a boolean. */
-public class TFBool implements TFType {
-  private TFBool() {}
-  static {
-    Types.typeCodes.put(TFBool.class, DataType.BOOL);
-  }
-  static {
-    Types.scalars.put(TFBool.class, false);
-  }
-}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/TFDouble.java b/tensorflow/java/src/main/java/org/tensorflow/types/TFDouble.java
deleted file mode 100644
index 49e5d9f2f3..0000000000
--- a/tensorflow/java/src/main/java/org/tensorflow/types/TFDouble.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// GENERATED FILE. To update, edit tftypes.pl instead.
-
-package org.tensorflow.types;
-
-import org.tensorflow.DataType;
-
-/** Represents a 64-bit double precision floating point number. */
-public class TFDouble implements TFType {
-  private TFDouble() {}
-  static {
-    Types.typeCodes.put(TFDouble.class, DataType.DOUBLE);
-  }
-  static {
-    Types.scalars.put(TFDouble.class, 0.0);
-  }
-}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/TFFloat.java b/tensorflow/java/src/main/java/org/tensorflow/types/TFFloat.java
deleted file mode 100644
index 8426ee41f0..0000000000
--- a/tensorflow/java/src/main/java/org/tensorflow/types/TFFloat.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// GENERATED FILE. To update, edit tftypes.pl instead.
-
-package org.tensorflow.types;
-
-import org.tensorflow.DataType;
-
-/** Represents a 32-bit single precision floating point number. */
-public class TFFloat implements TFType {
-  private TFFloat() {}
-  static {
-    Types.typeCodes.put(TFFloat.class, DataType.FLOAT);
-  }
-  static {
-    Types.scalars.put(TFFloat.class, 0f);
-  }
-}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/TFInt32.java b/tensorflow/java/src/main/java/org/tensorflow/types/TFInt32.java
deleted file mode 100644
index 3947b6ad09..0000000000
--- a/tensorflow/java/src/main/java/org/tensorflow/types/TFInt32.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// GENERATED FILE. To update, edit tftypes.pl instead.
-
-package org.tensorflow.types;
-
-import org.tensorflow.DataType;
-
-/** Represents a 32-bit signed integer. */
-public class TFInt32 implements TFType {
-  private TFInt32() {}
-  static {
-    Types.typeCodes.put(TFInt32.class, DataType.INT32);
-  }
-  static {
-    Types.scalars.put(TFInt32.class, 0);
-  }
-}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/TFInt64.java b/tensorflow/java/src/main/java/org/tensorflow/types/TFInt64.java
deleted file mode 100644
index ccdded8693..0000000000
--- a/tensorflow/java/src/main/java/org/tensorflow/types/TFInt64.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// GENERATED FILE. To update, edit tftypes.pl instead.
-
-package org.tensorflow.types;
-
-import org.tensorflow.DataType;
-
-/** Represents a 64-bit signed integer. */
-public class TFInt64 implements TFType {
-  private TFInt64() {}
-  static {
-    Types.typeCodes.put(TFInt64.class, DataType.INT64);
-  }
-  static {
-    Types.scalars.put(TFInt64.class, 0L);
-  }
-}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/TFString.java b/tensorflow/java/src/main/java/org/tensorflow/types/TFString.java
deleted file mode 100644
index e7327e8c57..0000000000
--- a/tensorflow/java/src/main/java/org/tensorflow/types/TFString.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// GENERATED FILE. To update, edit tftypes.pl instead.
-
-package org.tensorflow.types;
-
-import org.tensorflow.DataType;
-
-/** Represents an arbitrary sequence of bytes. */
-public class TFString implements TFType {
-  private TFString() {}
-  static {
-    Types.typeCodes.put(TFString.class, DataType.STRING);
-  }
-}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/TFType.java b/tensorflow/java/src/main/java/org/tensorflow/types/TFType.java
deleted file mode 100644
index 562953ac9d..0000000000
--- a/tensorflow/java/src/main/java/org/tensorflow/types/TFType.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-package org.tensorflow.types;
-
-/**
- * A marker interface for classes representing TensorFlow types.
- */
-public interface TFType {}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/TFUInt8.java b/tensorflow/java/src/main/java/org/tensorflow/types/TFUInt8.java
deleted file mode 100644
index d7305ca5a8..0000000000
--- a/tensorflow/java/src/main/java/org/tensorflow/types/TFUInt8.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// GENERATED FILE. To update, edit tftypes.pl instead.
-
-package org.tensorflow.types;
-
-import org.tensorflow.DataType;
-
-/** Represents an 8-bit unsigned integer. */
-public class TFUInt8 implements TFType {
-  private TFUInt8() {}
-  static {
-    Types.typeCodes.put(TFUInt8.class, DataType.UINT8);
-  }
-  static {
-    Types.scalars.put(TFUInt8.class, (byte)0);
-  }
-}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/Types.java b/tensorflow/java/src/main/java/org/tensorflow/types/Types.java
deleted file mode 100644
index 976cd9fd34..0000000000
--- a/tensorflow/java/src/main/java/org/tensorflow/types/Types.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-package org.tensorflow.types;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.tensorflow.DataType;
-
-/**
- * Utility class for managing the representation of TensorFlow types as Java
- * types. For each TensorFlow type (e.g., int32), there is a corresponding Java
- * type (e.g., TFInt32) that represents it at compile time and a corresponding
- * class object (e.g., TFInt32.class) that represents it at run time. There is
- * also an enumeration value in DataType that can be used to represent the
- * type, though that should rarely be required.
- */
-public class Types {
-
-  private Types() {} // not instantiable
-
-  static final Map, DataType> typeCodes = new HashMap<>();
-
-  /** Returns the DataType value corresponding to a TensorFlow type class. */
-  public static DataType dataType(Class c) {
-    DataType dtype = typeCodes.get(c);
-    if (dtype == null) {
-      throw new IllegalArgumentException("" + c + " is not a TensorFlow type.");
-    }
-    return dtype;
-  }
-
-  static final Map, Object> scalars = new HashMap<>();
-
-  /** Returns the zero value of type described by {@code c}, or null if
-   *  the type (e.g., string) is not numeric and therefore has no zero value.
-   */
-  public static Object zeroValue(Class c) {
-    return scalars.get(c);
-  }
-}
-- 
cgit v1.2.3


From 176ae62c10832459eba5809cc9dab35fd78f67a8 Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Tue, 17 Jul 2018 13:31:01 -0700
Subject: Automated rollback of commit 17bbfe25d0225f7d693384d4e0dcaa5f49a8c697

PiperOrigin-RevId: 204964956
---
 tensorflow/core/common_runtime/eager/context.cc | 7 +------
 tensorflow/core/common_runtime/eager/context.h  | 7 ++++---
 tensorflow/python/eager/function_test.py        | 2 +-
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index aaca633cc5..1c5e9a2a31 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -34,8 +34,7 @@ EagerContext::EagerContext(const SessionOptions& opts,
           local_device_manager_.get(), opts.env, TF_GRAPH_DEF_VERSION,
           &func_lib_def_, {}, thread_pool_.get())),
       log_device_placement_(opts.config.log_device_placement()),
-      async_default_(async),
-      env_(opts.env) {
+      async_default_(async) {
   InitDeviceMapAndAsync();
   if (opts.config.inter_op_parallelism_threads() > 0) {
     runner_ = [this](std::function closure) {
@@ -65,7 +64,6 @@ EagerContext::EagerContext(
       log_device_placement_(opts.config.log_device_placement()),
       async_default_(async),
       remote_device_manager_(std::move(remote_device_manager)),
-      env_(opts.env),
       server_(std::move(server)),
       remote_eager_workers_(std::move(remote_eager_workers)),
       remote_contexts_(remote_contexts) {
@@ -118,9 +116,6 @@ Status EagerContext::SetAsyncForThread(bool async) {
 void EagerContext::ClearCaches() {
   mutex_lock ml(cache_mu_);
   gtl::STLDeleteValues(&kernel_cache_);
-  pflr_.reset(new ProcessFunctionLibraryRuntime(
-      local_device_manager_.get(), env_, TF_GRAPH_DEF_VERSION, &func_lib_def_,
-      {}, thread_pool_.get()));
 }
 
 void EagerContext::SetThreadLocalDevicePlacementPolicy(
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index 6825c39ef3..d0563280bf 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -211,7 +211,10 @@ class EagerContext {
 
   std::unique_ptr thread_pool_;
 
-  std::unique_ptr pflr_;
+  // One FunctionLibraryRuntime per device.
+  // func_libs[i] is the FunctionLibraryRuntime corresponding to
+  // session->devices[i].
+  const std::unique_ptr pflr_;
 
   std::function)> runner_;
 
@@ -236,8 +239,6 @@ class EagerContext {
 
   const std::unique_ptr remote_device_manager_;
 
-  tensorflow::Env* const env_;
-
   // The server_ is not const since we release it when the context is destroyed.
   // Therefore the server_ object is not marked as const (even though it should
   // be).
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 13c4ee7f15..cdd9fe1760 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -138,7 +138,7 @@ class FunctionTest(test.TestCase):
     out = sq_op(t)
     self.assertAllEqual(out, math_ops.matmul(t, t).numpy())
 
-  def testRandomSeed(self):
+  def disabled_testRandomSeed(self):
 
     @function.defun
     def f():
-- 
cgit v1.2.3


From 32ebe93cdf487e7275b8a9c74f61959683ff0977 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 13:45:50 -0700
Subject: Go: Update generated wrapper functions for TensorFlow ops.
 PiperOrigin-RevId: 204967456

---
 tensorflow/go/op/wrappers.go | 1300 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 1168 insertions(+), 132 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index f49e1cecaf..18d7425323 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -327,12 +327,12 @@ func FakeQuantWithMinMaxArgs(scope *Scope, inputs tf.Output, optional ...FakeQua
 	return op.Output(0)
 }
 
-// Scatter `updates` into a new (initially zero) tensor according to `indices`.
+// Scatter `updates` into a new tensor according to `indices`.
 //
-// Creates a new tensor by applying sparse `updates` to individual
-// values or slices within a zero tensor of the given `shape` according to
-// indices.  This operator is the inverse of the @{tf.gather_nd} operator which
-// extracts values or slices from a given tensor.
+// Creates a new tensor by applying sparse `updates` to individual values or
+// slices within a tensor (initially zero for numeric, empty for string) of
+// the given `shape` according to indices.  This operator is the inverse of the
+// @{tf.gather_nd} operator which extracts values or slices from a given tensor.
 //
 // **WARNING**: The order in which updates are applied is nondeterministic, so the
 // output will be nondeterministic if `indices` contains duplicates.
@@ -430,7 +430,8 @@ type QuantizeAndDequantizeV2Attr func(optionalAttr)
 
 // QuantizeAndDequantizeV2SignedInput sets the optional signed_input attribute to value.
 //
-// value: If the quantization is signed or unsigned.
+// value: Whether the quantization is signed or unsigned. (actually this parameter should
+// have been called `signed_output`)
 // If not specified, defaults to true
 func QuantizeAndDequantizeV2SignedInput(value bool) QuantizeAndDequantizeV2Attr {
 	return func(m optionalAttr) {
@@ -450,7 +451,7 @@ func QuantizeAndDequantizeV2NumBits(value int64) QuantizeAndDequantizeV2Attr {
 
 // QuantizeAndDequantizeV2RangeGiven sets the optional range_given attribute to value.
 //
-// value: If the range is given or should be computed from the tensor.
+// value: Whether the range is given or should be determined from the `input` tensor.
 // If not specified, defaults to false
 func QuantizeAndDequantizeV2RangeGiven(value bool) QuantizeAndDequantizeV2Attr {
 	return func(m optionalAttr) {
@@ -461,61 +462,64 @@ func QuantizeAndDequantizeV2RangeGiven(value bool) QuantizeAndDequantizeV2Attr {
 // Quantizes then dequantizes a tensor.
 //
 // This op simulates the precision loss from the quantized forward pass by:
+//
 // 1. Quantizing the tensor to fixed point numbers, which should match the target
 //    quantization method when it is used in inference.
 // 2. Dequantizing it back to floating point numbers for the following ops, most
 //    likely matmul.
 //
-// There are different ways to quantize. This version does not use the full range
-// of the output type, choosing to elide the lowest possible value for symmetry
-// (e.g., output range is -127 to 127, not -128 to 127 for signed 8 bit
-// quantization), so that 0.0 maps to 0.
-//
-// To perform this op, we first find the range of values in our tensor. The range
-// we use is always centered on 0, so we find m such that
-//
-// 1. m = max(abs(input_min), abs(input_max)) if range_given is true,
-// 2. m = max(abs(min_elem(input)), abs(max_elem(input))) otherwise.
-//
-// Our input tensor range is then [-m, m].
+// There are different ways to quantize. This version uses only scaling, so 0.0
+// maps to 0.
 //
-// Next, we choose our fixed-point quantization buckets, [min_fixed, max_fixed].
-// If signed_input is true, this is
+// From the specified 'num_bits' in the quantized output type, it determines
+// minimum and maximum representable quantized values.
 //
-//   [min_fixed, max_fixed ] =
-//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1].
+// e.g.
 //
-// Otherwise, if signed_input is false, the fixed-point range is
+// *   [-128, 127] for signed, num_bits = 8, or
+// *   [0, 255] for unsigned, num_bits = 8.
 //
-//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1].
+// If range_given == False, the initial input_min, input_max will be determined
+// automatically as the minimum and maximum values in the input tensor, otherwise
+// the specified values of input_min, input_max are used.
 //
-// From this we compute our scaling factor, s:
+// Note: If the input_min, input_max are specified, they do not need to equal the
+// actual minimum and maximum values in the tensor. e.g. in some cases it may be
+// beneficial to specify these values such that the low probability extremes of the
+// input distribution are clipped.
 //
-//   s = (max_fixed - min_fixed) / (2 * m).
+// This op determines the maximum scale_factor that would map the initial
+// [input_min, input_max] range to a range that lies within the representable
+// quantized range.
 //
-// Now we can quantize and dequantize the elements of our tensor.  An element e
-// is transformed into e':
+// It determines the scale from one of input_min and input_max, then updates the
+// other one to maximize the respresentable range.
 //
-//   e' = (e * s).round_to_nearest() / s.
+// e.g.
 //
-// Note that we have a different number of buckets in the signed vs. unsigned
-// cases.  For example, if num_bits == 8, we get 254 buckets in the signed case
-// vs. 255 in the unsigned case.
+// *   if the output is signed, num_bits = 8, [input_min, input_max] = [-10.0,
+//     5.0]: it would use a scale_factor of -128 / -10.0 = 12.8 In this case, it
+//     would update input_max to be 127 / 12.8 = 9.921875
+// *   if the output is signed, num_bits = 8, [input_min, input_max] = [-10.0,
+//     10.0]: it would use a scale_factor of 127 / 10.0 = 12.7 In this case, it
+//     would update input_min to be 128.0 / 12.7 = -10.07874
+// *   if the output is unsigned, input_min is forced to be 0, and only the
+//     specified input_max is used.
 //
-// For example, suppose num_bits = 8 and m = 1.  Then
+// After determining the scale_factor and updating the input range, it applies the
+// following to each value in the 'input' tensor.
 //
-//   [min_fixed, max_fixed] = [-127, 127], and
-//   s = (127 + 127) / 2 = 127.
+// output = round(clamp(value, input_min, input_max) * scale_factor) / scale_factor.
 //
-// Given the vector {-1, -0.5, 0, 0.3}, this is quantized to
-// {-127, -63, 0, 38}, and dequantized to {-1, -63.0/127, 0, 38.0/127}.
 //
 // Arguments:
 //	input: Tensor to quantize and then dequantize.
-//	input_min: If range_given, this is the min of the range, otherwise this input
-// will be ignored.
-//	input_max: If range_given, this is the max of the range, otherwise this input
-// will be ignored.
+//	input_min: If `range_given == True`, this specifies the minimum input value that needs to
+// be represented, otherwise it is determined from the min value of the `input`
+// tensor.
+//	input_max: If `range_given == True`, this specifies the maximum input value that needs to
+// be represented, otherwise it is determined from the max value of the `input`
+// tensor.
 func QuantizeAndDequantizeV2(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, optional ...QuantizeAndDequantizeV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -2249,7 +2253,7 @@ func CheckNumerics(scope *Scope, tensor tf.Output, message string) (output tf.Ou
 // (K-1)-dimensional tensor of indices into `params`, where each element defines a
 // slice of `params`:
 //
-//     output[i_0, ..., i_{K-2}] = params[indices[i0, ..., i_{K-2}]]
+//     output[\\(i_0, ..., i_{K-2}\\)] = params[indices[\\(i_0, ..., i_{K-2}\\)]]
 //
 // Whereas in @{tf.gather} `indices` defines slices into the first
 // dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the
@@ -3015,6 +3019,45 @@ func Concat(scope *Scope, concat_dim tf.Output, values []tf.Output) (output tf.O
 	return op.Output(0)
 }
 
+// Broadcast an array for a compatible shape.
+//
+// Broadcasting is the process of making arrays to have compatible shapes
+// for arithmetic operations. Two shapes are compatible if for each
+// dimension pair they are either equal or one of them is one. When trying
+// to broadcast a Tensor to a shape, it starts with the trailing dimensions,
+// and works its way forward.
+//
+// For example,
+// ```
+// >>> x = tf.constant([1, 2, 3])
+// >>> y = tf.broadcast_to(x, [3, 3])
+// >>> sess.run(y)
+// array([[1, 2, 3],
+//        [1, 2, 3],
+//        [1, 2, 3]], dtype=int32)
+// ```
+// In the above example, the input Tensor with the shape of `[1, 3]`
+// is broadcasted to output Tensor with shape of `[3, 3]`.
+//
+// Arguments:
+//	input: A Tensor to broadcast.
+//	shape: An 1-D `int` Tensor. The shape of the desired output.
+//
+// Returns A Tensor.
+func BroadcastTo(scope *Scope, input tf.Output, shape tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BroadcastTo",
+		Input: []tf.Input{
+			input, shape,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Converts a flat index or array of flat indices into a tuple of
 //
 // coordinate arrays.
@@ -3506,7 +3549,7 @@ func Relu6(scope *Scope, features tf.Output) (activations tf.Output) {
 // segments.
 //
 // Computes a tensor such that
-// `(output[i] = sum_{j...} data[j...]` where the sum is over tuples `j...` such
+// \\(output[i] = sum_{j...} data[j...]\\) where the sum is over tuples `j...` such
 // that `segment_ids[j...] == i`.  Unlike `SegmentSum`, `segment_ids`
 // need not be sorted and need not cover all values in the full
 // range of valid values.
@@ -3875,11 +3918,13 @@ func Atan2(scope *Scope, y tf.Output, x tf.Output) (z tf.Output) {
 //
 //	window_size: A scalar representing the number of elements in the
 // sliding window.
-//	stride: A scalar representing the steps moving the sliding window
-// forward in one iteration. It must be in `[1, window_size)`.
+//	window_shift: A scalar representing the steps moving the sliding window
+// forward in one iteration. It must be positive.
+//	window_stride: A scalar representing the stride of the input elements of the sliding window.
+// It must be positive.
 //
 //
-func SlideDataset(scope *Scope, input_dataset tf.Output, window_size tf.Output, stride tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+func SlideDataset(scope *Scope, input_dataset tf.Output, window_size tf.Output, window_shift tf.Output, window_stride tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -3887,7 +3932,7 @@ func SlideDataset(scope *Scope, input_dataset tf.Output, window_size tf.Output,
 	opspec := tf.OpSpec{
 		Type: "SlideDataset",
 		Input: []tf.Input{
-			input_dataset, window_size, stride,
+			input_dataset, window_size, window_shift, window_stride,
 		},
 		Attrs: attrs,
 	}
@@ -4902,6 +4947,21 @@ func Add(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// Computes the derivative of a Gamma random sample w.r.t. `alpha`.
+func RandomGammaGrad(scope *Scope, alpha tf.Output, sample tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "RandomGammaGrad",
+		Input: []tf.Input{
+			alpha, sample,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes square of x element-wise.
 //
 // I.e., \\(y = x * x = x^2\\).
@@ -5650,7 +5710,7 @@ func LessEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 //
 // For each batch `i` and class `j` we have
 //
-//     softmax[i, j] = exp(logits[i, j]) / sum_j(exp(logits[i, j]))
+//     $$softmax[i, j] = exp(logits[i, j]) / sum_j(exp(logits[i, j]))$$
 //
 // Arguments:
 //	logits: 2-D with shape `[batch_size, num_classes]`.
@@ -6828,8 +6888,9 @@ type CropAndResizeAttr func(optionalAttr)
 
 // CropAndResizeMethod sets the optional method attribute to value.
 //
-// value: A string specifying the interpolation method. Only 'bilinear' is
-// supported for now.
+// value: A string specifying the sampling method for resizing. It can be either
+// `"bilinear"` or `"nearest"` and default to `"bilinear"`. Currently two sampling
+// methods are supported: Bilinear and Nearest Neighbor.
 // If not specified, defaults to "bilinear"
 func CropAndResizeMethod(value string) CropAndResizeAttr {
 	return func(m optionalAttr) {
@@ -6847,19 +6908,23 @@ func CropAndResizeExtrapolationValue(value float32) CropAndResizeAttr {
 	}
 }
 
-// Extracts crops from the input image tensor and bilinearly resizes them (possibly
+// Extracts crops from the input image tensor and resizes them.
 //
-// with aspect ratio change) to a common output size specified by `crop_size`. This
-// is more general than the `crop_to_bounding_box` op which extracts a fixed size
-// slice from the input image and does not allow resizing or aspect ratio change.
+// Extracts crops from the input image tensor and resizes them using bilinear
+// sampling or nearest neighbor sampling (possibly with aspect ratio change) to a
+// common output size specified by `crop_size`. This is more general than the
+// `crop_to_bounding_box` op which extracts a fixed size slice from the input image
+// and does not allow resizing or aspect ratio change.
 //
 // Returns a tensor with `crops` from the input `image` at positions defined at the
 // bounding box locations in `boxes`. The cropped boxes are all resized (with
-// bilinear interpolation) to a fixed `size = [crop_height, crop_width]`. The
-// result is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`. The
-// resizing is corner aligned. In particular, if `boxes = [[0, 0, 1, 1]]`, the
-// method will give identical results to using `tf.image.resize_bilinear()`
-// with `align_corners=True`.
+// bilinear or nearest neighbor interpolation) to a fixed
+// `size = [crop_height, crop_width]`. The result is a 4-D tensor
+// `[num_boxes, crop_height, crop_width, depth]`. The resizing is corner aligned.
+// In particular, if `boxes = [[0, 0, 1, 1]]`, the method will give identical
+// results to using `tf.image.resize_bilinear()` or
+// `tf.image.resize_nearest_neighbor()`(depends on the `method` argument) with
+// `align_corners=True`.
 //
 // Arguments:
 //	image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
@@ -7242,6 +7307,26 @@ func Min(scope *Scope, input tf.Output, axis tf.Output, optional ...MinAttr) (ou
 	return op.Output(0)
 }
 
+// Computes the Bessel i1e function of `x` element-wise.
+//
+// Exponentially scaled modified Bessel function of order 0 defined as
+// `bessel_i1e(x) = exp(-abs(x)) bessel_i1(x)`.
+//
+// This function is faster and numerically stabler than `bessel_i1(x)`.
+func BesselI1e(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BesselI1e",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Transforms a Tensor into a serialized TensorProto proto.
 //
 // Arguments:
@@ -8437,6 +8522,21 @@ func DataFormatVecPermute(scope *Scope, x tf.Output, optional ...DataFormatVecPe
 	return op.Output(0)
 }
 
+// Computes the gradient of `igamma(a, x)` wrt `a`.
+func IgammaGradA(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IgammaGradA",
+		Input: []tf.Input{
+			a, x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Converts each string in the input Tensor to its hash mod by a number of buckets.
 //
 // The hash function is deterministic on the content of the string within the
@@ -9101,6 +9201,85 @@ func ResourceScatterDiv(scope *Scope, resource tf.Output, indices tf.Output, upd
 	return scope.AddOperation(opspec)
 }
 
+// ResourceScatterNdAddAttr is an optional argument to ResourceScatterNdAdd.
+type ResourceScatterNdAddAttr func(optionalAttr)
+
+// ResourceScatterNdAddUseLocking sets the optional use_locking attribute to value.
+//
+// value: An optional bool. Defaults to True. If True, the assignment will
+// be protected by a lock; otherwise the behavior is undefined,
+// but may exhibit less contention.
+// If not specified, defaults to true
+func ResourceScatterNdAddUseLocking(value bool) ResourceScatterNdAddAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Adds sparse `updates` to individual values or slices within a given
+//
+// variable according to `indices`.
+//
+// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+//
+// `indices` must be integer tensor, containing indices into `ref`.
+// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
+//
+// The innermost dimension of `indices` (with length `K`) corresponds to
+// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
+// dimension of `ref`.
+//
+// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
+//
+// ```
+// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+// ```
+//
+// For example, say we want to update 4 scattered elements to a rank-1 tensor to
+// 8 elements. In Python, that update would look like this:
+//
+// ```python
+//     ref = tfe.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+//     indices = tf.constant([[4], [3], [1] ,[7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     update = tf.scatter_nd_add(ref, indices, updates)
+//     with tf.Session() as sess:
+//       print sess.run(update)
+// ```
+//
+// The resulting update to ref would look like this:
+//
+//     [1, 12, 3, 14, 14, 6, 7, 20]
+//
+// See @{tf.scatter_nd} for more details about how to make updates to
+// slices.
+//
+// Arguments:
+//	ref: A resource handle. Must be from a VarHandleOp.
+//	indices: A Tensor. Must be one of the following types: int32, int64.
+// A tensor of indices into ref.
+//	updates: A Tensor. Must have the same type as ref. A tensor of
+// values to add to ref.
+//
+// Returns the created operation.
+func ResourceScatterNdAdd(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdAddAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceScatterNdAdd",
+		Input: []tf.Input{
+			ref, indices, updates,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Mutually reduces multiple tensors of identical type and shape.
 func CollectiveReduce(scope *Scope, input tf.Output, group_size int64, group_key int64, instance_key int64, merge_op string, final_op string, subdiv_offsets []int64) (data tf.Output) {
 	if scope.Err() != nil {
@@ -9161,6 +9340,68 @@ func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, option
 	return op.Output(0)
 }
 
+// StringSplitV2Attr is an optional argument to StringSplitV2.
+type StringSplitV2Attr func(optionalAttr)
+
+// StringSplitV2Maxsplit sets the optional maxsplit attribute to value.
+//
+// value: An `int`. If `maxsplit > 0`, limit of the split of the result.
+// If not specified, defaults to -1
+func StringSplitV2Maxsplit(value int64) StringSplitV2Attr {
+	return func(m optionalAttr) {
+		m["maxsplit"] = value
+	}
+}
+
+// Split elements of `source` based on `sep` into a `SparseTensor`.
+//
+// Let N be the size of source (typically N will be the batch size). Split each
+// element of `source` based on `sep` and return a `SparseTensor`
+// containing the split tokens. Empty tokens are ignored.
+//
+// For example, N = 2, source[0] is 'hello world' and source[1] is 'a b c',
+// then the output will be
+// ```
+// st.indices = [0, 0;
+//               0, 1;
+//               1, 0;
+//               1, 1;
+//               1, 2]
+// st.shape = [2, 3]
+// st.values = ['hello', 'world', 'a', 'b', 'c']
+// ```
+//
+// If `sep` is given, consecutive delimiters are not grouped together and are
+// deemed to delimit empty strings. For example, source of `"1<>2<><>3"` and
+// sep of `"<>"` returns `["1", "2", "", "3"]`. If `sep` is None or an empty
+// string, consecutive whitespace are regarded as a single separator, and the
+// result will contain no empty strings at the startor end if the string has
+// leading or trailing whitespace.
+//
+// Note that the above mentioned behavior matches python's str.split.
+//
+// Arguments:
+//	input: `1-D` string `Tensor`, the strings to split.
+//	sep: `0-D` string `Tensor`, the delimiter character.
+func StringSplitV2(scope *Scope, input tf.Output, sep tf.Output, optional ...StringSplitV2Attr) (indices tf.Output, values tf.Output, shape tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StringSplitV2",
+		Input: []tf.Input{
+			input, sep,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
 // MaxPoolAttr is an optional argument to MaxPool.
 type MaxPoolAttr func(optionalAttr)
 
@@ -9245,9 +9486,11 @@ func SparseMatMulBIsSparse(value bool) SparseMatMulAttr {
 // Multiply matrix "a" by matrix "b".
 //
 // The inputs must be two-dimensional matrices and the inner dimension of "a" must
-// match the outer dimension of "b". This op is optimized for the case where at
-// least one of "a" or "b" is sparse. The breakeven for using this versus a dense
-// matrix multiply on one platform was 30% zero values in the sparse matrix.
+// match the outer dimension of "b". Both "a" and "b" must be `Tensor`s not
+// `SparseTensor`s.  This op is optimized for the case where at least one of "a" or
+// "b" is sparse, in the sense that they have a large proportion of zero values.
+// The breakeven for using this versus a dense matrix multiply on one platform was
+// 30% zero values in the sparse matrix.
 //
 // The gradient computation of this operation will only take advantage of sparsity
 // in the input gradient when that gradient comes from a Relu.
@@ -9878,6 +10121,51 @@ func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize
 	return op.Output(0)
 }
 
+// Greedily selects a subset of bounding boxes in descending order of score,
+//
+// pruning away boxes that have high overlaps
+// with previously selected boxes.  Bounding boxes with score less than
+// `score_threshold` are removed. N-by-n overlap values are supplied as square matrix,
+// which allows for defining a custom overlap criterium (eg. intersection over union,
+// intersection over area, etc.).
+//
+// The output of this operation is a set of integers indexing into the input
+// collection of bounding boxes representing the selected boxes.  The bounding
+// box coordinates corresponding to the selected indices can then be obtained
+// using the `tf.gather operation`.  For example:
+//
+//   selected_indices = tf.image.non_max_suppression_with_overlaps(
+//       overlaps, scores, max_output_size, overlap_threshold, score_threshold)
+//   selected_boxes = tf.gather(boxes, selected_indices)
+//
+// Arguments:
+//	overlaps: A 2-D float tensor of shape `[num_boxes, num_boxes]` representing
+// the n-by-n box overlap values.
+//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
+// score corresponding to each box (each row of boxes).
+//	max_output_size: A scalar integer tensor representing the maximum number of
+// boxes to be selected by non max suppression.
+//	overlap_threshold: A 0-D float tensor representing the threshold for deciding whether
+// boxes overlap too.
+//	score_threshold: A 0-D float tensor representing the threshold for deciding when to remove
+// boxes based on score.
+//
+// Returns A 1-D integer tensor of shape `[M]` representing the selected
+// indices from the boxes tensor, where `M <= max_output_size`.
+func NonMaxSuppressionWithOverlaps(scope *Scope, overlaps tf.Output, scores tf.Output, max_output_size tf.Output, overlap_threshold tf.Output, score_threshold tf.Output) (selected_indices tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "NonMaxSuppressionWithOverlaps",
+		Input: []tf.Input{
+			overlaps, scores, max_output_size, overlap_threshold, score_threshold,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // StageClearAttr is an optional argument to StageClear.
 type StageClearAttr func(optionalAttr)
 
@@ -10170,6 +10458,57 @@ func Atan(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// ResourceApplyAdaMaxAttr is an optional argument to ResourceApplyAdaMax.
+type ResourceApplyAdaMaxAttr func(optionalAttr)
+
+// ResourceApplyAdaMaxUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var, m, and v tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyAdaMaxUseLocking(value bool) ResourceApplyAdaMaxAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update '*var' according to the AdaMax algorithm.
+//
+// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
+// v_t <- max(beta2 * v_{t-1}, abs(g))
+// variable <- variable - learning_rate / (1 - beta1^t) * m_t / (v_t + epsilon)
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	m: Should be from a Variable().
+//	v: Should be from a Variable().
+//	beta1_power: Must be a scalar.
+//	lr: Scaling factor. Must be a scalar.
+//	beta1: Momentum factor. Must be a scalar.
+//	beta2: Momentum factor. Must be a scalar.
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyAdaMax(scope *Scope, var_ tf.Output, m tf.Output, v tf.Output, beta1_power tf.Output, lr tf.Output, beta1 tf.Output, beta2 tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdaMaxAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyAdaMax",
+		Input: []tf.Input{
+			var_, m, v, beta1_power, lr, beta1, beta2, epsilon, grad,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Encode audio data using the WAV file format.
 //
 // This operation will generate a string suitable to be saved out to create a .wav
@@ -10778,25 +11117,139 @@ func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Out
 	return scope.AddOperation(opspec)
 }
 
-// Locks a mutex resource.  The output is the lock.  So long as the lock tensor
-//
-// is alive, any other request to use `MutexLock` with this mutex will wait.
-//
-// This is particularly useful for creating a critical section when used in
-// conjunction with `MutexLockIdentity`:
-//
-// ```python
-//
-// mutex = mutex_v2(
-//   shared_name=handle_name, container=container, name=name)
-//
-// def execute_in_critical_section(fn, *args, **kwargs):
-//   lock = gen_resource_variable_ops.mutex_lock(mutex)
-//
-//   with ops.control_dependencies([lock]):
-//     r = fn(*args, **kwargs)
-//
-//   with ops.control_dependencies(nest.flatten(r)):
+// CudnnRNNBackpropV2Attr is an optional argument to CudnnRNNBackpropV2.
+type CudnnRNNBackpropV2Attr func(optionalAttr)
+
+// CudnnRNNBackpropV2RnnMode sets the optional rnn_mode attribute to value.
+// If not specified, defaults to "lstm"
+func CudnnRNNBackpropV2RnnMode(value string) CudnnRNNBackpropV2Attr {
+	return func(m optionalAttr) {
+		m["rnn_mode"] = value
+	}
+}
+
+// CudnnRNNBackpropV2InputMode sets the optional input_mode attribute to value.
+// If not specified, defaults to "linear_input"
+func CudnnRNNBackpropV2InputMode(value string) CudnnRNNBackpropV2Attr {
+	return func(m optionalAttr) {
+		m["input_mode"] = value
+	}
+}
+
+// CudnnRNNBackpropV2Direction sets the optional direction attribute to value.
+// If not specified, defaults to "unidirectional"
+func CudnnRNNBackpropV2Direction(value string) CudnnRNNBackpropV2Attr {
+	return func(m optionalAttr) {
+		m["direction"] = value
+	}
+}
+
+// CudnnRNNBackpropV2Dropout sets the optional dropout attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNBackpropV2Dropout(value float32) CudnnRNNBackpropV2Attr {
+	return func(m optionalAttr) {
+		m["dropout"] = value
+	}
+}
+
+// CudnnRNNBackpropV2Seed sets the optional seed attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNBackpropV2Seed(value int64) CudnnRNNBackpropV2Attr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// CudnnRNNBackpropV2Seed2 sets the optional seed2 attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNBackpropV2Seed2(value int64) CudnnRNNBackpropV2Attr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Backprop step of CudnnRNN.
+//
+// Compute the backprop of both data and weights in a RNN. Takes an extra
+//     "host_reserved" inupt than CudnnRNNBackprop, which is used to determine RNN
+//     cudnnRNNAlgo_t and cudnnMathType_t.
+//
+// rnn_mode: Indicates the type of the RNN model.
+// input_mode: Indicates whether there is a linear projection between the input and
+//     the actual computation before the first layer. 'skip_input' is only allowed
+//     when input_size == num_units; 'auto_select' implies 'skip_input' when
+//     input_size == num_units; otherwise, it implies 'linear_input'.
+// direction: Indicates whether a bidirectional model will be used. Should be
+//   "unidirectional" or "bidirectional".
+// dropout: Dropout probability. When set to 0., dropout is disabled.
+// seed: The 1st part of a seed to initialize dropout.
+// seed2: The 2nd part of a seed to initialize dropout.
+// input: A 3-D tensor with the shape of [seq_length, batch_size, input_size].
+// input_h: A 3-D tensor with the shape of [num_layer * dir, batch_size,
+//     num_units].
+// input_c: For LSTM, a 3-D tensor with the shape of
+//     [num_layer * dir, batch, num_units]. For other models, it is ignored.
+// params: A 1-D tensor that contains the weights and biases in an opaque layout.
+//     The size must be created through CudnnRNNParamsSize, and initialized
+//     separately. Note that they might not be compatible across different
+//     generations. So it is a good idea to save and restore
+// output: A 3-D tensor with the shape of [seq_length, batch_size,
+//     dir * num_units].
+// output_h: The same shape has input_h.
+// output_c: The same shape as input_c for LSTM. An empty tensor for other models.
+// output_backprop: A 3-D tensor with the same shape as output in the forward pass.
+// output_h_backprop: A 3-D tensor with the same shape as output_h in the forward
+//     pass.
+// output_c_backprop: A 3-D tensor with the same shape as output_c in the forward
+//     pass.
+// reserve_space: The same reserve_space produced in the forward operation.
+// host_reserved: The same host_reserved produced in the forward operation.
+// input_backprop: The backprop to input in the forward pass. Has the same shape
+//     as input.
+// input_h_backprop: The backprop to input_h in the forward pass. Has the same
+//     shape as input_h.
+// input_c_backprop: The backprop to input_c in the forward pass. Has the same
+//     shape as input_c.
+// params_backprop: The backprop to the params buffer in the forward pass. Has the
+//     same shape as params.
+func CudnnRNNBackpropV2(scope *Scope, input tf.Output, input_h tf.Output, input_c tf.Output, params tf.Output, output tf.Output, output_h tf.Output, output_c tf.Output, output_backprop tf.Output, output_h_backprop tf.Output, output_c_backprop tf.Output, reserve_space tf.Output, host_reserved tf.Output, optional ...CudnnRNNBackpropV2Attr) (input_backprop tf.Output, input_h_backprop tf.Output, input_c_backprop tf.Output, params_backprop tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "CudnnRNNBackpropV2",
+		Input: []tf.Input{
+			input, input_h, input_c, params, output, output_h, output_c, output_backprop, output_h_backprop, output_c_backprop, reserve_space, host_reserved,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
+}
+
+// Locks a mutex resource.  The output is the lock.  So long as the lock tensor
+//
+// is alive, any other request to use `MutexLock` with this mutex will wait.
+//
+// This is particularly useful for creating a critical section when used in
+// conjunction with `MutexLockIdentity`:
+//
+// ```python
+//
+// mutex = mutex_v2(
+//   shared_name=handle_name, container=container, name=name)
+//
+// def execute_in_critical_section(fn, *args, **kwargs):
+//   lock = gen_resource_variable_ops.mutex_lock(mutex)
+//
+//   with ops.control_dependencies([lock]):
+//     r = fn(*args, **kwargs)
+//
+//   with ops.control_dependencies(nest.flatten(r)):
 //     with ops.colocate_with(mutex):
 //       ensure_lock_exists = mutex_lock_identity(lock)
 //
@@ -10965,6 +11418,34 @@ func BatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, o
 	return op.Output(0)
 }
 
+// Check if the input matches the regex pattern.
+//
+// The input is a string tensor of any shape. The pattern is a scalar
+// string tensor which is applied to every element of the input tensor.
+// The boolean values (True or False) of the output tensor indicate
+// if the input matches the regex pattern provided.
+//
+// The pattern follows the re2 syntax (https://github.com/google/re2/wiki/Syntax)
+//
+// Arguments:
+//	input: A string tensor of the text to be processed.
+//	pattern: A 1-D string tensor of the regular expression to match the input.
+//
+// Returns A bool tensor with the same shape as `input`.
+func RegexFullMatch(scope *Scope, input tf.Output, pattern tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "RegexFullMatch",
+		Input: []tf.Input{
+			input, pattern,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Says whether the targets are in the top `K` predictions.
 //
 // This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
@@ -11457,7 +11938,7 @@ func SampleDistortedBoundingBoxAspectRatioRange(value []float32) SampleDistorted
 // SampleDistortedBoundingBoxAreaRange sets the optional area_range attribute to value.
 //
 // value: The cropped area of the image must contain a fraction of the
-// supplied image within in this range.
+// supplied image within this range.
 // If not specified, defaults to 
 func SampleDistortedBoundingBoxAreaRange(value []float32) SampleDistortedBoundingBoxAttr {
 	return func(m optionalAttr) {
@@ -12229,6 +12710,7 @@ func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Outp
 //                       [0, 0, 2, 2, 0, 0]
 //                       [0, 0, 0, 0, 0, 0]]
 // ```
+//
 func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -13547,9 +14029,11 @@ func ReduceJoinSeparator(value string) ReduceJoinAttr {
 // Joins a string Tensor across the given dimensions.
 //
 // Computes the string join across dimensions in the given string Tensor of shape
-// `[d_0, d_1, ..., d_n-1]`.  Returns a new Tensor created by joining the input
+// `[\\(d_0, d_1, ..., d_{n-1}\\)]`.  Returns a new Tensor created by joining the input
 // strings with the given separator (default: empty string).  Negative indices are
-// counted backwards from the end, with `-1` being equivalent to `n - 1`.
+// counted backwards from the end, with `-1` being equivalent to `n - 1`.  If
+// indices are not specified, joins across all dimensions beginning from `n - 1`
+// through `0`.
 //
 // For example:
 //
@@ -13562,9 +14046,10 @@ func ReduceJoinSeparator(value string) ReduceJoinAttr {
 // tf.reduce_join(a, 0, keep_dims=True) ==> [["ac", "bd"]]
 // tf.reduce_join(a, 1, keep_dims=True) ==> [["ab"], ["cd"]]
 // tf.reduce_join(a, 0, separator=".") ==> ["a.c", "b.d"]
-// tf.reduce_join(a, [0, 1]) ==> ["acbd"]
-// tf.reduce_join(a, [1, 0]) ==> ["abcd"]
-// tf.reduce_join(a, []) ==> ["abcd"]
+// tf.reduce_join(a, [0, 1]) ==> "acbd"
+// tf.reduce_join(a, [1, 0]) ==> "abcd"
+// tf.reduce_join(a, []) ==> [["a", "b"], ["c", "d"]]
+// tf.reduce_join(a) = tf.reduce_join(a, [1, 0]) ==> "abcd"
 // ```
 //
 // Arguments:
@@ -14654,27 +15139,27 @@ func CudnnRNNBackpropSeed2(value int64) CudnnRNNBackpropAttr {
 //
 // rnn_mode: Indicates the type of the RNN model.
 // input_mode: Indicate whether there is a linear projection between the input and
-//     The actual computation before the first layer. 'skip_input' is only allowed
+//     the actual computation before the first layer. 'skip_input' is only allowed
 //     when input_size == num_units; 'auto_select' implies 'skip_input' when
 //     input_size == num_units; otherwise, it implies 'linear_input'.
-// direction: Indicates whether a bidirectional model will be used.
-//     dir = (direction == bidirectional) ? 2 : 1
-// dropout: dropout probability. When set to 0., dropout is disabled.
-// seed: the 1st part of a seed to initialize dropout.
-// seed2: the 2nd part of a seed to initialize dropout.
-// input: a 3-D tensor with the shape of [seq_length, batch_size, input_size].
-// input_h: a 3-D tensor with the shape of [num_layer * dir, batch_size,
+// direction: Indicates whether a bidirectional model will be used. Should be
+//   "unidirectional" or "bidirectional".
+// dropout: Dropout probability. When set to 0., dropout is disabled.
+// seed: The 1st part of a seed to initialize dropout.
+// seed2: The 2nd part of a seed to initialize dropout.
+// input: A 3-D tensor with the shape of [seq_length, batch_size, input_size].
+// input_h: A 3-D tensor with the shape of [num_layer * dir, batch_size,
 //     num_units].
 // input_c: For LSTM, a 3-D tensor with the shape of
 //     [num_layer * dir, batch, num_units]. For other models, it is ignored.
-// params: a 1-D tensor that contains the weights and biases in an opaque layout.
+// params: A 1-D tensor that contains the weights and biases in an opaque layout.
 //     The size must be created through CudnnRNNParamsSize, and initialized
 //     separately. Note that they might not be compatible across different
 //     generations. So it is a good idea to save and restore
-// output: a 3-D tensor with the shape of [seq_length, batch_size,
+// output: A 3-D tensor with the shape of [seq_length, batch_size,
 //     dir * num_units].
-// output_h: the same shape has input_h.
-// output_c: the same shape as input_c for LSTM. An empty tensor for other models.
+// output_h: The same shape has input_h.
+// output_c: The same shape as input_c for LSTM. An empty tensor for other models.
 // output_backprop: A 3-D tensor with the same shape as output in the forward pass.
 // output_h_backprop: A 3-D tensor with the same shape as output_h in the forward
 //     pass.
@@ -15635,6 +16120,30 @@ func OrderedMapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataTyp
 	return key, values
 }
 
+// Calculates the prior from the training data (the bias) and fills in the first node with the logits' prior. Returns a boolean indicating whether to continue centering.
+//
+// Arguments:
+//	tree_ensemble_handle: Handle to the tree ensemble.
+//	mean_gradients: A tensor with shape=[logits_dimension] with mean of gradients for a first node.
+//	mean_hessians: A tensor with shape=[logits_dimension] mean of hessians for a first node.
+//	l1: l1 regularization factor on leaf weights, per instance based.
+//	l2: l2 regularization factor on leaf weights, per instance based.
+//
+// Returns Bool, whether to continue bias centering.
+func BoostedTreesCenterBias(scope *Scope, tree_ensemble_handle tf.Output, mean_gradients tf.Output, mean_hessians tf.Output, l1 tf.Output, l2 tf.Output) (continue_centering tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BoostedTreesCenterBias",
+		Input: []tf.Input{
+			tree_ensemble_handle, mean_gradients, mean_hessians, l1, l2,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SerializeManySparseAttr is an optional argument to SerializeManySparse.
 type SerializeManySparseAttr func(optionalAttr)
 
@@ -17203,6 +17712,7 @@ func QuantizeV2RoundMode(value string) QuantizeV2Attr {
 // out[i] = (in[i] - min_range) * range(T) / (max_range - min_range)
 // if T == qint8, out[i] -= (range(T) + 1) / 2.0
 // ```
+//
 // here `range(T) = numeric_limits::max() - numeric_limits::min()`
 //
 // *MIN_COMBINED Mode Example*
@@ -17246,6 +17756,7 @@ func QuantizeV2RoundMode(value string) QuantizeV2Attr {
 //
 // We first find the range of values in our tensor. The
 // range we use is always centered on 0, so we find m such that
+//
 // ```c++
 //   m = max(abs(input_min), abs(input_max))
 // ```
@@ -17254,6 +17765,7 @@ func QuantizeV2RoundMode(value string) QuantizeV2Attr {
 //
 // Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
 // If T is signed, this is
+//
 // ```
 //   num_bits = sizeof(T) * 8
 //   [min_fixed, max_fixed] =
@@ -17261,16 +17773,19 @@ func QuantizeV2RoundMode(value string) QuantizeV2Attr {
 // ```
 //
 // Otherwise, if T is unsigned, the fixed-point range is
+//
 // ```
 //   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
 // ```
 //
 // From this we compute our scaling factor, s:
+//
 // ```c++
 //   s = (max_fixed - min_fixed) / (2 * m)
 // ```
 //
 // Now we can quantize the elements of our tensor:
+//
 // ```c++
 // result = round(input * s)
 // ```
@@ -17367,6 +17882,31 @@ func QuantizedReluX(scope *Scope, features tf.Output, max_value tf.Output, min_f
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// Creates a dataset that batches `batch_size` elements from `input_dataset`.
+//
+// Arguments:
+//
+//	batch_size: A scalar representing the number of elements to accumulate in a batch.
+//	drop_remainder: A scalar representing whether the last batch should be dropped in case its size
+// is smaller than desired.
+//
+//
+func BatchDatasetV2(scope *Scope, input_dataset tf.Output, batch_size tf.Output, drop_remainder tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "BatchDatasetV2",
+		Input: []tf.Input{
+			input_dataset, batch_size, drop_remainder,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // QuantizedConv2DAttr is an optional argument to QuantizedConv2D.
 type QuantizedConv2DAttr func(optionalAttr)
 
@@ -18006,6 +18546,34 @@ func MutableHashTableOfTensorsV2(scope *Scope, key_dtype tf.DataType, value_dtyp
 	return op.Output(0)
 }
 
+// The gradient operator for the SparseSlice op.
+//
+// This op takes in the upstream gradient w.r.t. non-empty values of
+// the sliced `SparseTensor`, and outputs the gradients w.r.t.
+// the non-empty values of input `SparseTensor`.
+//
+// Arguments:
+//	backprop_val_grad: 1-D. The gradient with respect to
+// the non-empty values of the sliced `SparseTensor`.
+//	input_indices: 2-D.  The `indices` of the input `SparseTensor`.
+//	input_start: 1-D. tensor represents the start of the slice.
+//	output_indices: 2-D.  The `indices` of the sliced `SparseTensor`.
+//
+// Returns 1-D. The gradient with respect to the non-empty values of input `SparseTensor`.
+func SparseSliceGrad(scope *Scope, backprop_val_grad tf.Output, input_indices tf.Output, input_start tf.Output, output_indices tf.Output) (val_grad tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseSliceGrad",
+		Input: []tf.Input{
+			backprop_val_grad, input_indices, input_start, output_indices,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the gradient of the sigmoid of `x` wrt its input.
 //
 // Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and
@@ -18050,6 +18618,31 @@ func HSVToRGB(scope *Scope, images tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
+// Creates a dataset by applying optimizations to `input_dataset`.
+//
+// Creates a dataset by applying optimizations to `input_dataset`.
+//
+// Arguments:
+//	input_dataset: A variant tensor representing the input dataset.
+//	optimizations: A `tf.string` vector `tf.Tensor` identifying optimizations to use.
+//
+//
+func OptimizeDataset(scope *Scope, input_dataset tf.Output, optimizations tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "OptimizeDataset",
+		Input: []tf.Input{
+			input_dataset, optimizations,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Retrieves the tree ensemble resource stamp token, number of trees and growing statistics.
 //
 // Arguments:
@@ -18224,6 +18817,26 @@ func AssignVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.
 	return scope.AddOperation(opspec)
 }
 
+// Strip leading and trailing whitespaces from the Tensor.
+//
+// Arguments:
+//	input: A string `Tensor` of any shape.
+//
+// Returns A string `Tensor` of the same shape as the input.
+func StringStrip(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "StringStrip",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns a tensor of ones with the same shape and type as x.
 //
 // Arguments:
@@ -18278,6 +18891,10 @@ func SparseFillEmptyRowsGrad(scope *Scope, reverse_index_map tf.Output, grad_val
 //
 // if < 0, `scale * features` otherwise.
 //
+// To be used together with
+// `initializer = tf.variance_scaling_initializer(factor=1.0, mode='FAN_IN')`.
+// For correct dropout, use `tf.contrib.nn.alpha_dropout`.
+//
 // See [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515)
 func Selu(scope *Scope, features tf.Output) (activations tf.Output) {
 	if scope.Err() != nil {
@@ -18960,7 +19577,7 @@ func MatrixTriangularSolveLower(value bool) MatrixTriangularSolveAttr {
 //          adjoint.
 //
 // @compatibility(numpy)
-// Equivalent to np.linalg.triangular_solve
+// Equivalent to scipy.linalg.solve_triangular
 // @end_compatibility
 // If not specified, defaults to false
 func MatrixTriangularSolveAdjoint(value bool) MatrixTriangularSolveAttr {
@@ -19736,9 +20353,9 @@ func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyReso
 // ```
 //
 // Arguments:
-//	start: First entry in the range.
-//	stop: Last entry in the range.
-//	num: Number of values to generate.
+//	start: 0-D tensor. First entry in the range.
+//	stop: 0-D tensor. Last entry in the range.
+//	num: 0-D tensor. Number of values to generate.
 //
 // Returns 1-D. The generated values.
 func LinSpace(scope *Scope, start tf.Output, stop tf.Output, num tf.Output) (output tf.Output) {
@@ -20919,6 +21536,37 @@ func LookupTableInsertV2(scope *Scope, table_handle tf.Output, keys tf.Output, v
 	return scope.AddOperation(opspec)
 }
 
+// Creates a dataset that batches and pads `batch_size` elements from the input.
+//
+// Arguments:
+//
+//	batch_size: A scalar representing the number of elements to accumulate in a
+// batch.
+//	padded_shapes: A list of int64 tensors representing the desired padded shapes
+// of the corresponding output components. These shapes may be partially
+// specified, using `-1` to indicate that a particular dimension should be
+// padded to the maximum size of all batch elements.
+//	padding_values: A list of scalars containing the padding value to use for
+// each of the outputs.
+//	drop_remainder: A scalar representing whether the last batch should be dropped in case its size
+// is smaller than desired.
+//
+func PaddedBatchDatasetV2(scope *Scope, input_dataset tf.Output, batch_size tf.Output, padded_shapes []tf.Output, padding_values []tf.Output, drop_remainder tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "PaddedBatchDatasetV2",
+		Input: []tf.Input{
+			input_dataset, batch_size, tf.OutputList(padded_shapes), tf.OutputList(padding_values), drop_remainder,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns element-wise smallest integer in not less than x.
 func Ceil(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
@@ -21790,7 +22438,7 @@ func ImageSummaryBadColor(value tf.Tensor) ImageSummaryAttr {
 //    generated sequentially as '*tag*/image/0', '*tag*/image/1', etc.
 //
 // The `bad_color` argument is the color to use in the generated images for
-// non-finite input values.  It is a `unit8` 1-D tensor of length `channels`.
+// non-finite input values.  It is a `uint8` 1-D tensor of length `channels`.
 // Each element must be in the range `[0, 255]` (It represents the value of a
 // pixel in the output image).  Non-finite values in the input tensor are
 // replaced by this tensor in the output image.  The default value is the color
@@ -22248,7 +22896,7 @@ func TensorListSetItem(scope *Scope, input_handle tf.Output, index tf.Output, it
 
 // Computes the matrix exponential of one or more square matrices:
 //
-// exp(A) = \sum_{n=0}^\infty A^n/n!
+// \\(exp(A) = \sum_{n=0}^\infty A^n/n!\\)
 //
 // The exponential is computed using a combination of the scaling and squaring
 // method and the Pade approximation. Details can be founds in:
@@ -22628,6 +23276,28 @@ func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...Matr
 	return op.Output(0)
 }
 
+// Returns a serialized GraphDef representing `input_dataset`.
+//
+// Returns a graph representation for `input_dataset`.
+//
+// Arguments:
+//	input_dataset: A variant tensor representing the dataset to return the graph representation for.
+//
+// Returns The graph representation of the dataset (as serialized GraphDef).
+func DatasetToGraph(scope *Scope, input_dataset tf.Output) (graph tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "DatasetToGraph",
+		Input: []tf.Input{
+			input_dataset,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SvdAttr is an optional argument to Svd.
 type SvdAttr func(optionalAttr)
 
@@ -23651,10 +24321,10 @@ func ResourceApplyAdamUseNesterov(value bool) ResourceApplyAdamAttr {
 
 // Update '*var' according to the Adam algorithm.
 //
-// lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t)
-// m_t <- beta1 * m_{t-1} + (1 - beta1) * g_t
-// v_t <- beta2 * v_{t-1} + (1 - beta2) * g_t * g_t
-// variable <- variable - lr_t * m_t / (sqrt(v_t) + epsilon)
+// $$lr_t := \text{learning_rate} * \sqrt{(1 - beta_2^t) / (1 - beta_1^t)}$$
+// $$m_t := beta_1 * m_{t-1} + (1 - beta_1) * g$$
+// $$v_t := beta_2 * v_{t-1} + (1 - beta_2) * g * g$$
+// $$variable := variable - lr_t * m_t / (\sqrt{v_t} + \epsilon)$$
 //
 // Arguments:
 //	var_: Should be from a Variable().
@@ -24118,7 +24788,7 @@ func SampleDistortedBoundingBoxV2AspectRatioRange(value []float32) SampleDistort
 // SampleDistortedBoundingBoxV2AreaRange sets the optional area_range attribute to value.
 //
 // value: The cropped area of the image must contain a fraction of the
-// supplied image within in this range.
+// supplied image within this range.
 // If not specified, defaults to 
 func SampleDistortedBoundingBoxV2AreaRange(value []float32) SampleDistortedBoundingBoxV2Attr {
 	return func(m optionalAttr) {
@@ -24627,10 +25297,57 @@ func NonMaxSuppressionV2(scope *Scope, boxes tf.Output, scores tf.Output, max_ou
 	return op.Output(0)
 }
 
-// Computes the matrix logarithm of one or more square matrices:
-//
+// Greedily selects a subset of bounding boxes in descending order of score,
 //
-// log(exp(A)) = A
+// pruning away boxes that have high intersection-over-union (IOU) overlap
+// with previously selected boxes.  Bounding boxes with score less than
+// `score_threshold` are removed.  Bounding boxes are supplied as
+// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
+// diagonal pair of box corners and the coordinates can be provided as normalized
+// (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
+// is agnostic to where the origin is in the coordinate system and more
+// generally is invariant to orthogonal transformations and translations
+// of the coordinate system; thus translating or reflections of the coordinate
+// system result in the same boxes being selected by the algorithm.
+// The output of this operation is a set of integers indexing into the input
+// collection of bounding boxes representing the selected boxes.  The bounding
+// box coordinates corresponding to the selected indices can then be obtained
+// using the `tf.gather operation`.  For example:
+//   selected_indices = tf.image.non_max_suppression_v2(
+//       boxes, scores, max_output_size, iou_threshold, score_threshold)
+//   selected_boxes = tf.gather(boxes, selected_indices)
+//
+// Arguments:
+//	boxes: A 2-D float tensor of shape `[num_boxes, 4]`.
+//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
+// score corresponding to each box (each row of boxes).
+//	max_output_size: A scalar integer tensor representing the maximum number of
+// boxes to be selected by non max suppression.
+//	iou_threshold: A 0-D float tensor representing the threshold for deciding whether
+// boxes overlap too much with respect to IOU.
+//	score_threshold: A 0-D float tensor representing the threshold for deciding when to remove
+// boxes based on score.
+//
+// Returns A 1-D integer tensor of shape `[M]` representing the selected
+// indices from the boxes tensor, where `M <= max_output_size`.
+func NonMaxSuppressionV3(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, iou_threshold tf.Output, score_threshold tf.Output) (selected_indices tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "NonMaxSuppressionV3",
+		Input: []tf.Input{
+			boxes, scores, max_output_size, iou_threshold, score_threshold,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the matrix logarithm of one or more square matrices:
+//
+//
+// \\(log(exp(A)) = A\\)
 //
 // This op is only defined for complex matrices. If A is positive-definite and
 // real, then casting to a complex matrix, taking the logarithm and casting back
@@ -24667,6 +25384,31 @@ func MatrixLogarithm(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
+//   This op is used as a placeholder in If branch functions. It doesn't provide a
+//   valid output when run, so must either be removed (e.g. replaced with a
+//   function input) or guaranteed not to be used (e.g. if mirroring an
+//   intermediate output needed for the gradient computation of the other branch).
+//
+// Arguments:
+//	dtype: The type of the output.
+//	shape:     The purported shape of the output. This is only used for shape inference;
+//     the output will not necessarily have this shape. Can be a partial shape.
+//
+// Returns     \"Fake\" output value. This should not be consumed by another op.
+func FakeParam(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
+	opspec := tf.OpSpec{
+		Type: "FakeParam",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // EncodeProtoAttr is an optional argument to EncodeProto.
 type EncodeProtoAttr func(optionalAttr)
 
@@ -25008,6 +25750,23 @@ func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) {
 	return scope.AddOperation(opspec)
 }
 
+// A dataset that splits the elements of its input into multiple elements.
+func UnbatchDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "UnbatchDataset",
+		Input: []tf.Input{
+			input_dataset,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // RpcAttr is an optional argument to Rpc.
 type RpcAttr func(optionalAttr)
 
@@ -25260,6 +26019,36 @@ func ConcatenateDataset(scope *Scope, input_dataset tf.Output, another_dataset t
 	return op.Output(0)
 }
 
+// Debugging/model interpretability outputs for each example.
+//
+// It traverses all the trees and computes debug metrics for individual examples,
+// such as getting split feature ids and logits after each split along the decision
+// path used to compute directional feature contributions.
+//
+// Arguments:
+//
+//	bucketized_features: A list of rank 1 Tensors containing bucket id for each
+// feature.
+//	logits_dimension: scalar, dimension of the logits, to be used for constructing the protos in
+// examples_debug_outputs_serialized.
+//
+// Returns Output rank 1 Tensor containing a proto serialized as a string for each example.
+func BoostedTreesExampleDebugOutputs(scope *Scope, tree_ensemble_handle tf.Output, bucketized_features []tf.Output, logits_dimension int64) (examples_debug_outputs_serialized tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"logits_dimension": logits_dimension}
+	opspec := tf.OpSpec{
+		Type: "BoostedTreesExampleDebugOutputs",
+		Input: []tf.Input{
+			tree_ensemble_handle, tf.OutputList(bucketized_features),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Adds a value to the current value of a variable.
 //
 // Any ReadVariableOp with a control dependency on this op is guaranteed to
@@ -25959,6 +26748,26 @@ func TFRecordDataset(scope *Scope, filenames tf.Output, compression_type tf.Outp
 	return op.Output(0)
 }
 
+// A container for an iterator resource.
+//
+// Returns A handle to the iterator that can be passed to a "MakeIterator" or
+// "IteratorGetNext" op. In contrast to Iterator, AnonymousIterator prevents
+// resource sharing by name, and does not keep a reference to the resource
+// container.
+func AnonymousIterator(scope *Scope, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "AnonymousIterator",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // BatchToSpace for 4-D tensors of type T.
 //
 // This is a legacy version of the more general BatchToSpaceND.
@@ -26462,6 +27271,28 @@ func Cross(scope *Scope, a tf.Output, b tf.Output) (product tf.Output) {
 	return op.Output(0)
 }
 
+// Writes the given dataset to the given file using the TFRecord format.
+//
+// Arguments:
+//	input_dataset: A variant tensor representing the dataset to write.
+//	filename: A scalar string tensor representing the filename to use.
+//	compression_type: A scalar string tensor containing either (i) the empty string (no
+// compression), (ii) "ZLIB", or (iii) "GZIP".
+//
+// Returns the created operation.
+func DatasetToTFRecord(scope *Scope, input_dataset tf.Output, filename tf.Output, compression_type tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "DatasetToTFRecord",
+		Input: []tf.Input{
+			input_dataset, filename, compression_type,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // AvgPool3DAttr is an optional argument to AvgPool3D.
 type AvgPool3DAttr func(optionalAttr)
 
@@ -26509,6 +27340,26 @@ func AvgPool3D(scope *Scope, input tf.Output, ksize []int64, strides []int64, pa
 	return op.Output(0)
 }
 
+// A placeholder for input pipeline graph optimizations.
+//
+// A placeholder for input pipeline graph optimizations.
+//
+// Arguments:
+//	input_dataset: A variant tensor representing the input dataset.
+func SinkDataset(scope *Scope, input_dataset tf.Output) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SinkDataset",
+		Input: []tf.Input{
+			input_dataset,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Performs a padding as a preprocess during a convolution.
 //
 // Similar to FusedResizeAndPadConv2d, this op allows for an optimized
@@ -27064,6 +27915,26 @@ func QueueEnqueueV2(scope *Scope, handle tf.Output, components []tf.Output, opti
 	return scope.AddOperation(opspec)
 }
 
+// Computes the Bessel i0e function of `x` element-wise.
+//
+// Exponentially scaled modified Bessel function of order 0 defined as
+// `bessel_i0e(x) = exp(-abs(x)) bessel_i0(x)`.
+//
+// This function is faster and numerically stabler than `bessel_i0(x)`.
+func BesselI0e(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BesselI0e",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // QueueDequeueManyV2Attr is an optional argument to QueueDequeueManyV2.
 type QueueDequeueManyV2Attr func(optionalAttr)
 
@@ -27174,6 +28045,29 @@ func EncodeBase64(scope *Scope, input tf.Output, optional ...EncodeBase64Attr) (
 	return op.Output(0)
 }
 
+// A dataset that creates window datasets from the input dataset.
+//
+// Arguments:
+//
+//	window_size: A scalar representing the number of elements to accumulate in a window.
+//
+//
+func WindowDataset(scope *Scope, input_dataset tf.Output, window_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "WindowDataset",
+		Input: []tf.Input{
+			input_dataset, window_size,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Deprecated. Use TensorArrayCloseV3
 //
 // DEPRECATED at GraphDef version 26: Use TensorArrayCloseV3
@@ -27546,30 +28440,30 @@ func CudnnRNNIsTraining(value bool) CudnnRNNAttr {
 //
 // rnn_mode: Indicates the type of the RNN model.
 // input_mode: Indicate whether there is a linear projection between the input and
-//   The actual computation before the first layer. 'skip_input' is only allowed
+//   the actual computation before the first layer. 'skip_input' is only allowed
 //   when input_size == num_units; 'auto_select' implies 'skip_input' when
 //   input_size == num_units; otherwise, it implies 'linear_input'.
-// direction: Indicates whether a bidirectional model will be used.
-//   dir = (direction == bidirectional) ? 2 : 1
-// dropout: dropout probability. When set to 0., dropout is disabled.
-// seed: the 1st part of a seed to initialize dropout.
-// seed2: the 2nd part of a seed to initialize dropout.
-// input: a 3-D tensor with the shape of [seq_length, batch_size, input_size].
-// input_h: a 3-D tensor with the shape of [num_layer * dir, batch_size,
+// direction: Indicates whether a bidirectional model will be used. Should be
+//   "unidirectional" or "bidirectional".
+// dropout: Dropout probability. When set to 0., dropout is disabled.
+// seed: The 1st part of a seed to initialize dropout.
+// seed2: The 2nd part of a seed to initialize dropout.
+// input: A 3-D tensor with the shape of [seq_length, batch_size, input_size].
+// input_h: A 3-D tensor with the shape of [num_layer * dir, batch_size,
 //     num_units].
 // input_c: For LSTM, a 3-D tensor with the shape of
 //     [num_layer * dir, batch, num_units]. For other models, it is ignored.
-// params: a 1-D tensor that contains the weights and biases in an opaque layout.
+// params: A 1-D tensor that contains the weights and biases in an opaque layout.
 //     The size must be created through CudnnRNNParamsSize, and initialized
 //     separately. Note that they might not be compatible across different
 //     generations. So it is a good idea to save and restore
-// output: a 3-D tensor with the shape of [seq_length, batch_size,
+// output: A 3-D tensor with the shape of [seq_length, batch_size,
 //     dir * num_units].
-// output_h: the same shape has input_h.
-// output_c: the same shape as input_c for LSTM. An empty tensor for other models.
+// output_h: The same shape has input_h.
+// output_c: The same shape as input_c for LSTM. An empty tensor for other models.
 // is_training: Indicates whether this operation is used for inferenece or
 //   training.
-// reserve_space: an opaque tensor that can be used in backprop calculation. It
+// reserve_space: An opaque tensor that can be used in backprop calculation. It
 //   is only produced if is_training is false.
 func CudnnRNN(scope *Scope, input tf.Output, input_h tf.Output, input_c tf.Output, params tf.Output, optional ...CudnnRNNAttr) (output tf.Output, output_h tf.Output, output_c tf.Output, reserve_space tf.Output) {
 	if scope.Err() != nil {
@@ -27590,6 +28484,37 @@ func CudnnRNN(scope *Scope, input tf.Output, input_h tf.Output, input_c tf.Outpu
 	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
 }
 
+// Creates a TensorArray for storing multiple gradients of values in the given handle.
+//
+// Similar to TensorArrayGradV3. However it creates an accumulator with an
+// expanded shape compared to the input TensorArray whose gradient is being
+// computed. This enables multiple gradients for the same TensorArray to be
+// calculated using the same accumulator.
+//
+// Arguments:
+//	handle: The handle to the forward TensorArray.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	shape_to_prepend: An int32 vector representing a shape. Elements in the gradient accumulator will
+// have shape which is this shape_to_prepend value concatenated with shape of the
+// elements in the TensorArray corresponding to the input handle.
+//	source: The gradient source string, used to decide which gradient TensorArray
+// to return.
+func TensorArrayGradWithShape(scope *Scope, handle tf.Output, flow_in tf.Output, shape_to_prepend tf.Output, source string) (grad_handle tf.Output, flow_out tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"source": source}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayGradWithShape",
+		Input: []tf.Input{
+			handle, flow_in, shape_to_prepend,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // Compare values of `input` to `threshold` and pack resulting bits into a `uint8`.
 //
 // Each comparison returns a boolean `true` (if `input_value > threshold`)
@@ -27980,7 +28905,7 @@ func RandomShuffleQueueV2(scope *Scope, component_types []tf.DataType, optional
 //
 // For example, if an image is 100 x 200 pixels (height x width) and the bounding
 // box is `[0.1, 0.2, 0.5, 0.9]`, the upper-left and bottom-right coordinates of
-// the bounding box will be `(40, 10)` to `(100, 50)` (in (x,y) coordinates).
+// the bounding box will be `(40, 10)` to `(180, 50)` (in (x,y) coordinates).
 //
 // Parts of the bounding box may fall outside the image.
 //
@@ -28321,7 +29246,7 @@ func BoostedTreesCreateEnsemble(scope *Scope, tree_ensemble_handle tf.Output, st
 // `input` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
 //
 // `indices` must be integer tensor, containing indices into `input`.
-// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
+// It must be shape \\([d_0, ..., d_{Q-2}, K]\\) where `0 < K <= P`.
 //
 // The innermost dimension of `indices` (with length `K`) corresponds to
 // indices into elements (if `K = P`) or `(P-K)`-dimensional slices
@@ -28329,9 +29254,7 @@ func BoostedTreesCreateEnsemble(scope *Scope, tree_ensemble_handle tf.Output, st
 //
 // `updates` is `Tensor` of rank `Q-1+P-K` with shape:
 //
-// ```
-// [d_0, ..., d_{Q-2}, input.shape[K], ..., input.shape[P-1]].
-// ```
+// $$[d_0, ..., d_{Q-2}, input.shape[K], ..., input.shape[P-1]].$$
 //
 // For example, say we want to add 4 scattered elements to a rank-1 tensor to 8
 // elements. In Python, that addition would look like this:
@@ -29092,6 +30015,119 @@ func OrderedMapSize(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapSi
 	return op.Output(0)
 }
 
+// CudnnRNNV2Attr is an optional argument to CudnnRNNV2.
+type CudnnRNNV2Attr func(optionalAttr)
+
+// CudnnRNNV2RnnMode sets the optional rnn_mode attribute to value.
+// If not specified, defaults to "lstm"
+func CudnnRNNV2RnnMode(value string) CudnnRNNV2Attr {
+	return func(m optionalAttr) {
+		m["rnn_mode"] = value
+	}
+}
+
+// CudnnRNNV2InputMode sets the optional input_mode attribute to value.
+// If not specified, defaults to "linear_input"
+func CudnnRNNV2InputMode(value string) CudnnRNNV2Attr {
+	return func(m optionalAttr) {
+		m["input_mode"] = value
+	}
+}
+
+// CudnnRNNV2Direction sets the optional direction attribute to value.
+// If not specified, defaults to "unidirectional"
+func CudnnRNNV2Direction(value string) CudnnRNNV2Attr {
+	return func(m optionalAttr) {
+		m["direction"] = value
+	}
+}
+
+// CudnnRNNV2Dropout sets the optional dropout attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNV2Dropout(value float32) CudnnRNNV2Attr {
+	return func(m optionalAttr) {
+		m["dropout"] = value
+	}
+}
+
+// CudnnRNNV2Seed sets the optional seed attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNV2Seed(value int64) CudnnRNNV2Attr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// CudnnRNNV2Seed2 sets the optional seed2 attribute to value.
+// If not specified, defaults to 0
+func CudnnRNNV2Seed2(value int64) CudnnRNNV2Attr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// CudnnRNNV2IsTraining sets the optional is_training attribute to value.
+// If not specified, defaults to true
+func CudnnRNNV2IsTraining(value bool) CudnnRNNV2Attr {
+	return func(m optionalAttr) {
+		m["is_training"] = value
+	}
+}
+
+// A RNN backed by cuDNN.
+//
+// Computes the RNN from the input and initial states, with respect to the params
+// buffer. Produces one extra output "host_reserved" than CudnnRNN.
+//
+// rnn_mode: Indicates the type of the RNN model.
+// input_mode: Indicates whether there is a linear projection between the input and
+//   the actual computation before the first layer. 'skip_input' is only allowed
+//   when input_size == num_units; 'auto_select' implies 'skip_input' when
+//   input_size == num_units; otherwise, it implies 'linear_input'.
+// direction: Indicates whether a bidirectional model will be used. Should be
+//   "unidirectional" or "bidirectional".
+// dropout: Dropout probability. When set to 0., dropout is disabled.
+// seed: The 1st part of a seed to initialize dropout.
+// seed2: The 2nd part of a seed to initialize dropout.
+// input: A 3-D tensor with the shape of [seq_length, batch_size, input_size].
+// input_h: A 3-D tensor with the shape of [num_layer * dir, batch_size,
+//     num_units].
+// input_c: For LSTM, a 3-D tensor with the shape of
+//     [num_layer * dir, batch, num_units]. For other models, it is ignored.
+// params: A 1-D tensor that contains the weights and biases in an opaque layout.
+//     The size must be created through CudnnRNNParamsSize, and initialized
+//     separately. Note that they might not be compatible across different
+//     generations. So it is a good idea to save and restore
+// output: A 3-D tensor with the shape of [seq_length, batch_size,
+//     dir * num_units].
+// output_h: The same shape has input_h.
+// output_c: The same shape as input_c for LSTM. An empty tensor for other models.
+// is_training: Indicates whether this operation is used for inferenece or
+//   training.
+// reserve_space: An opaque tensor that can be used in backprop calculation. It
+//   is only produced if is_training is true.
+// host_reserved: An opaque tensor that can be used in backprop calculation. It is
+//   only produced if is_training is true. It is output on host memory rather than
+//   device memory.
+func CudnnRNNV2(scope *Scope, input tf.Output, input_h tf.Output, input_c tf.Output, params tf.Output, optional ...CudnnRNNV2Attr) (output tf.Output, output_h tf.Output, output_c tf.Output, reserve_space tf.Output, host_reserved tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "CudnnRNNV2",
+		Input: []tf.Input{
+			input, input_h, input_c, params,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+}
+
 // ShapeNAttr is an optional argument to ShapeN.
 type ShapeNAttr func(optionalAttr)
 
-- 
cgit v1.2.3


From 12f110c1a95d53c5e08180b6813ed65578c46822 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal 
Date: Tue, 17 Jul 2018 14:01:24 -0700
Subject: Removes comment for implemented feature.

PiperOrigin-RevId: 204970144
---
 tensorflow/core/kernels/data/stats_dataset_ops.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/data/stats_dataset_ops.cc b/tensorflow/core/kernels/data/stats_dataset_ops.cc
index a537e7e68f..754c32b6ca 100644
--- a/tensorflow/core/kernels/data/stats_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/stats_dataset_ops.cc
@@ -310,7 +310,7 @@ class FeatureStatsDatasetOp : public UnaryDatasetOpKernel {
           for (const Tensor& t : *out_tensors) {
             auto record_t = t.flat();
             Example example;
-            // TODO(shivaniagrawal): redundant parsing here, potential solutions
+            // TODO(b/111553342): redundant parsing here, potential solutions
             // to improve performance is to a) have a potential
             // ParseExampleDataset and collect stats from there and b) make
             // changes to parse_example() where it returns stats as well.
@@ -333,7 +333,6 @@ class FeatureStatsDatasetOp : public UnaryDatasetOpKernel {
         return s;
       }
 
-      // TODO(shivaniagrawal): Add features/feature-values to streamz metrics.
       int AddStatsFeatureValues(const Feature& feature) {
         int feature_values_list_size = 0;
         switch (feature.kind_case()) {
-- 
cgit v1.2.3


From 2f93ac4891f81137ce5fc40a8bbb2714b6cf2151 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar 
Date: Tue, 17 Jul 2018 14:15:39 -0700
Subject: Initialize profiler pointer to null.

PiperOrigin-RevId: 204972768
---
 tensorflow/contrib/lite/interpreter.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/interpreter.h b/tensorflow/contrib/lite/interpreter.h
index 1a1c3e272b..bc608e2fce 100644
--- a/tensorflow/contrib/lite/interpreter.h
+++ b/tensorflow/contrib/lite/interpreter.h
@@ -629,7 +629,7 @@ class Interpreter {
   bool tensor_resized_since_op_invoke_ = false;
 
   // Profiler for this interpreter instance.
-  profiling::Profiler* profiler_;
+  profiling::Profiler* profiler_ = nullptr;
 
   // List of active external contexts.
   TfLiteExternalContext* external_contexts_[kTfLiteMaxExternalContexts];
-- 
cgit v1.2.3


From 8c5d2127182e0fadc0dcd6e97cb4acfba3a4c343 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein 
Date: Tue, 17 Jul 2018 14:24:43 -0700
Subject: [XLA] Shape inference should verify the shapes of sort keys and sort
 values match.

PiperOrigin-RevId: 204974328
---
 tensorflow/compiler/xla/service/shape_inference.cc      |  8 ++++++++
 tensorflow/compiler/xla/service/shape_inference_test.cc | 12 ++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 214146cf68..35df792b07 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -970,6 +970,14 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(HloOpcode operation,
       if (operand_shapes.size() == 1) {
         return *operand_shapes[0];
       } else if (operand_shapes.size() == 2) {
+        if (!ShapeUtil::SameDimensions(*operand_shapes[0],
+                                       *operand_shapes[1])) {
+          return InvalidArgument(
+              "Sort keys and values dimensions must match. "
+              "Keys shape is: %s\n, Values shape is: %s",
+              ShapeUtil::HumanString(*operand_shapes[0]).c_str(),
+              ShapeUtil::HumanString(*operand_shapes[1]).c_str());
+        }
         return ShapeUtil::MakeTupleShape(
             {*operand_shapes[0], *operand_shapes[1]});
       }
diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc
index 9b1ce143c6..6046d50c6d 100644
--- a/tensorflow/compiler/xla/service/shape_inference_test.cc
+++ b/tensorflow/compiler/xla/service/shape_inference_test.cc
@@ -1524,6 +1524,18 @@ TEST_F(ShapeInferenceTest, BadSlice) {
       << statusor.status();
 }
 
+TEST_F(ShapeInferenceTest, BadSort) {
+  auto keys = ShapeUtil::MakeShape(F32, {4});
+  auto values = ShapeUtil::MakeShape(F32, {5});
+  StatusOr statusor =
+      ShapeInference::InferVariadicOpShape(HloOpcode::kSort, {&keys, &values});
+  ASSERT_FALSE(statusor.ok());
+
+  EXPECT_THAT(statusor.status().error_message(),
+              HasSubstr("dimensions must match"))
+      << statusor.status();
+}
+
 class GatherShapeInferenceTest : public ShapeInferenceTest {
  protected:
   const Shape s64_scalar_ = ShapeUtil::MakeShape(S64, {});
-- 
cgit v1.2.3


From c9fdfab3781bd55cf3d981b98982a1e8122de8d2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 14:32:36 -0700
Subject: Print kOutputYXInput.

PiperOrigin-RevId: 204975773
---
 tensorflow/stream_executor/dnn.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/stream_executor/dnn.cc b/tensorflow/stream_executor/dnn.cc
index 82aa8ceb32..2a30f922bc 100644
--- a/tensorflow/stream_executor/dnn.cc
+++ b/tensorflow/stream_executor/dnn.cc
@@ -117,6 +117,8 @@ string FilterLayoutString(FilterLayout layout) {
   switch (layout) {
     case FilterLayout::kOutputInputYX:
       return "OutputInputYX";
+    case FilterLayout::kOutputYXInput:
+      return "OutputYXInput";
     case FilterLayout::kOutputInputYX4:
       return "OutputInputYX4";
     case FilterLayout::kInputYXOutput:
-- 
cgit v1.2.3


From 67d7f85537bc7bb9638f3af0303f33f3f8990b6c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 14:45:12 -0700
Subject: Add 'no_oss' to all TF Lite tests that are not running in kokoro

PiperOrigin-RevId: 204978094
---
 tensorflow/contrib/lite/BUILD                      |   7 +
 tensorflow/contrib/lite/delegates/eager/BUILD      |   2 +
 tensorflow/contrib/lite/delegates/nnapi/BUILD      |   1 +
 tensorflow/contrib/lite/examples/label_image/BUILD |   1 +
 tensorflow/contrib/lite/java/BUILD                 |   5 +
 tensorflow/contrib/lite/java/ovic/BUILD            |   1 +
 tensorflow/contrib/lite/kernels/BUILD              | 181 +++++++++++++++++----
 tensorflow/contrib/lite/kernels/internal/BUILD     |  22 ++-
 tensorflow/contrib/lite/models/smartreply/BUILD    |   3 +
 tensorflow/contrib/lite/profiling/BUILD            |   3 +
 tensorflow/contrib/lite/python/BUILD               |  10 +-
 tensorflow/contrib/lite/schema/BUILD               |   6 +-
 .../contrib/lite/schema/builtin_ops_header/BUILD   |   2 +
 tensorflow/contrib/lite/testing/BUILD              |   8 +
 tensorflow/contrib/lite/toco/BUILD                 |   3 +
 .../lite/toco/graph_transformations/tests/BUILD    |   3 +
 tensorflow/contrib/lite/toco/python/BUILD          |   5 +-
 .../lite/toco/tensorflow_graph_matching/BUILD      |   1 +
 tensorflow/contrib/lite/toco/tflite/BUILD          |   4 +
 tensorflow/contrib/lite/tools/BUILD                |   2 +
 tensorflow/contrib/lite/tools/benchmark/BUILD      |   1 +
 21 files changed, 229 insertions(+), 42 deletions(-)

diff --git a/tensorflow/contrib/lite/BUILD b/tensorflow/contrib/lite/BUILD
index b95d4d0fce..7d7dd6b708 100644
--- a/tensorflow/contrib/lite/BUILD
+++ b/tensorflow/contrib/lite/BUILD
@@ -47,6 +47,10 @@ cc_test(
     name = "arena_planner_test",
     size = "small",
     srcs = ["arena_planner_test.cc"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable",
+    ],
     deps = [
         ":arena_planner",
         "//tensorflow/contrib/lite/testing:util",
@@ -200,6 +204,7 @@ cc_test(
     name = "graph_info_test",
     size = "small",
     srcs = ["graph_info_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":framework",
         ":string_util",
@@ -244,6 +249,7 @@ cc_test(
     name = "op_resolver_test",
     size = "small",
     srcs = ["op_resolver_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":framework",
         "//tensorflow/contrib/lite/testing:util",
@@ -276,6 +282,7 @@ cc_test(
     name = "util_test",
     size = "small",
     srcs = ["util_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":context",
         ":util",
diff --git a/tensorflow/contrib/lite/delegates/eager/BUILD b/tensorflow/contrib/lite/delegates/eager/BUILD
index 270d83d188..9d8c20e96f 100644
--- a/tensorflow/contrib/lite/delegates/eager/BUILD
+++ b/tensorflow/contrib/lite/delegates/eager/BUILD
@@ -26,6 +26,7 @@ cc_test(
     size = "small",
     srcs = ["buffer_map_test.cc"],
     tags = [
+        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
@@ -55,6 +56,7 @@ cc_test(
     size = "small",
     srcs = ["util_test.cc"],
     tags = [
+        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
diff --git a/tensorflow/contrib/lite/delegates/nnapi/BUILD b/tensorflow/contrib/lite/delegates/nnapi/BUILD
index 35a8f6ca41..091f8fbce7 100644
--- a/tensorflow/contrib/lite/delegates/nnapi/BUILD
+++ b/tensorflow/contrib/lite/delegates/nnapi/BUILD
@@ -22,6 +22,7 @@ tf_cc_test(
     name = "nnapi_delegate_test",
     size = "small",
     srcs = ["nnapi_delegate_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":nnapi_delegate",
         "//tensorflow/contrib/lite:framework",
diff --git a/tensorflow/contrib/lite/examples/label_image/BUILD b/tensorflow/contrib/lite/examples/label_image/BUILD
index c61445114e..fc55a78019 100644
--- a/tensorflow/contrib/lite/examples/label_image/BUILD
+++ b/tensorflow/contrib/lite/examples/label_image/BUILD
@@ -63,6 +63,7 @@ cc_test(
     data = [
         "testdata/grace_hopper.bmp",
     ],
+    tags = ["no_oss"],
     deps = [
         ":bitmap_helpers",
         "@com_google_googletest//:gtest",
diff --git a/tensorflow/contrib/lite/java/BUILD b/tensorflow/contrib/lite/java/BUILD
index 593af81a18..098ba7e773 100644
--- a/tensorflow/contrib/lite/java/BUILD
+++ b/tensorflow/contrib/lite/java/BUILD
@@ -69,6 +69,7 @@ java_test(
     size = "small",
     srcs = ["src/test/java/org/tensorflow/lite/TensorFlowLiteTest.java"],
     javacopts = JAVACOPTS,
+    tags = ["no_oss"],
     test_class = "org.tensorflow.lite.TensorFlowLiteTest",
     deps = [
         ":libtensorflowlite_jni.so",
@@ -83,6 +84,7 @@ java_test(
     size = "small",
     srcs = ["src/test/java/org/tensorflow/lite/DataTypeTest.java"],
     javacopts = JAVACOPTS,
+    tags = ["no_oss"],
     test_class = "org.tensorflow.lite.DataTypeTest",
     deps = [
         ":libtensorflowlite_jni.so",
@@ -105,6 +107,7 @@ java_test(
         "src/testdata/with_custom_op.lite",
     ],
     javacopts = JAVACOPTS,
+    tags = ["no_oss"],
     test_class = "org.tensorflow.lite.NativeInterpreterWrapperTest",
     deps = [
         ":libtensorflowlite_jni.so",
@@ -124,6 +127,7 @@ java_test(
         "src/testdata/mobilenet.tflite.bin",
     ],
     javacopts = JAVACOPTS,
+    tags = ["no_oss"],
     test_class = "org.tensorflow.lite.InterpreterTest",
     visibility = ["//visibility:private"],
     deps = [
@@ -142,6 +146,7 @@ java_test(
         "src/testdata/add.bin",
     ],
     javacopts = JAVACOPTS,
+    tags = ["no_oss"],
     test_class = "org.tensorflow.lite.TensorTest",
     deps = [
         ":tensorflowlitelib",
diff --git a/tensorflow/contrib/lite/java/ovic/BUILD b/tensorflow/contrib/lite/java/ovic/BUILD
index f232b00045..06f46fb923 100644
--- a/tensorflow/contrib/lite/java/ovic/BUILD
+++ b/tensorflow/contrib/lite/java/ovic/BUILD
@@ -18,6 +18,7 @@ java_test(
         "//tensorflow/contrib/lite/java/ovic/src/testdata:ovic_testdata",
     ],
     javacopts = JAVACOPTS,
+    tags = ["no_oss"],
     test_class = "org.tensorflow.ovic.OvicClassifierTest",
     visibility = ["//visibility:public"],
     deps = [
diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index 33594c138b..ad30624f40 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -12,7 +12,10 @@ tf_cc_test(
     name = "optional_tensor_test",
     size = "small",
     srcs = ["optional_tensor_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -112,7 +115,10 @@ tf_cc_test(
     name = "kernel_util_test",
     size = "small",
     srcs = ["kernel_util_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":kernel_util",
         "//tensorflow/contrib/lite/testing:util",
@@ -124,6 +130,7 @@ tf_cc_test(
     name = "test_util_test",
     size = "small",
     srcs = ["test_util_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":test_util",
         "//tensorflow/contrib/lite/testing:util",
@@ -232,7 +239,10 @@ tf_cc_test(
     name = "audio_spectrogram_test",
     size = "small",
     srcs = ["audio_spectrogram_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -246,7 +256,10 @@ tf_cc_test(
     name = "mfcc_test",
     size = "small",
     srcs = ["mfcc_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -260,7 +273,10 @@ tf_cc_test(
     name = "detection_postprocess_test",
     size = "small",
     srcs = ["detection_postprocess_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -301,6 +317,7 @@ tf_cc_test(
     size = "small",
     srcs = ["arg_min_max_test.cc"],
     tags = [
+        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -315,7 +332,10 @@ tf_cc_test(
     name = "div_test",
     size = "small",
     srcs = ["div_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -328,7 +348,10 @@ tf_cc_test(
     name = "sub_test",
     size = "small",
     srcs = ["sub_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -341,7 +364,10 @@ tf_cc_test(
     name = "transpose_test",
     size = "small",
     srcs = ["transpose_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -356,7 +382,10 @@ tf_cc_test(
     name = "space_to_batch_nd_test",
     size = "small",
     srcs = ["space_to_batch_nd_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -369,7 +398,10 @@ tf_cc_test(
     name = "batch_to_space_nd_test",
     size = "small",
     srcs = ["batch_to_space_nd_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -382,7 +414,10 @@ tf_cc_test(
     name = "cast_test",
     size = "small",
     srcs = ["cast_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -435,7 +470,10 @@ tf_cc_test(
     name = "dequantize_test",
     size = "small",
     srcs = ["dequantize_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -462,7 +500,10 @@ tf_cc_test(
     name = "bidirectional_sequence_lstm_test",
     size = "small",
     srcs = ["bidirectional_sequence_lstm_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -475,7 +516,10 @@ tf_cc_test(
     name = "floor_test",
     size = "small",
     srcs = ["floor_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -488,7 +532,10 @@ tf_cc_test(
     name = "elementwise_test",
     size = "small",
     srcs = ["elementwise_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -501,7 +548,10 @@ tf_cc_test(
     name = "unidirectional_sequence_lstm_test",
     size = "small",
     srcs = ["unidirectional_sequence_lstm_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -515,6 +565,7 @@ tf_cc_test(
     size = "small",
     srcs = ["bidirectional_sequence_rnn_test.cc"],
     tags = [
+        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
@@ -529,7 +580,10 @@ tf_cc_test(
     name = "unidirectional_sequence_rnn_test",
     size = "small",
     srcs = ["unidirectional_sequence_rnn_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -555,7 +609,10 @@ tf_cc_test(
     name = "exp_test",
     size = "small",
     srcs = ["exp_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -568,7 +625,10 @@ tf_cc_test(
     name = "fake_quant_test",
     size = "small",
     srcs = ["fake_quant_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -581,7 +641,10 @@ tf_cc_test(
     name = "maximum_minimum_test",
     size = "small",
     srcs = ["maximum_minimum_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -594,7 +657,10 @@ tf_cc_test(
     name = "reduce_test",
     size = "small",
     srcs = ["reduce_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -620,7 +686,10 @@ tf_cc_test(
     name = "pad_test",
     size = "small",
     srcs = ["pad_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -646,7 +715,10 @@ tf_cc_test(
     name = "gather_test",
     size = "small",
     srcs = ["gather_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:builtin_op_data",
@@ -660,7 +732,10 @@ tf_cc_test(
     name = "topk_v2_test",
     size = "small",
     srcs = ["topk_v2_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:builtin_op_data",
@@ -781,7 +856,10 @@ tf_cc_test(
     name = "log_softmax_test",
     size = "small",
     srcs = ["log_softmax_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -862,7 +940,10 @@ tf_cc_test(
     name = "split_test",
     size = "small",
     srcs = ["split_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -875,7 +956,10 @@ tf_cc_test(
     name = "squeeze_test",
     size = "small",
     srcs = ["squeeze_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -888,7 +972,10 @@ tf_cc_test(
     name = "strided_slice_test",
     size = "small",
     srcs = ["strided_slice_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -901,7 +988,10 @@ tf_cc_test(
     name = "tile_test",
     size = "small",
     srcs = ["tile_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:builtin_op_data",
@@ -918,6 +1008,7 @@ tf_cc_test(
         "comparisons_test.cc",
     ],
     tags = [
+        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -932,7 +1023,10 @@ tf_cc_test(
     name = "neg_test",
     size = "small",
     srcs = ["neg_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -948,6 +1042,7 @@ tf_cc_test(
         "select_test.cc",
     ],
     tags = [
+        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -965,6 +1060,7 @@ tf_cc_test(
         "slice_test.cc",
     ],
     tags = [
+        "no_oss",
         "tflite_not_portable_ios",
     ],
     deps = [
@@ -979,7 +1075,10 @@ tf_cc_test(
     name = "transpose_conv_test",
     size = "small",
     srcs = ["transpose_conv_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
@@ -992,7 +1091,10 @@ tf_cc_test(
     name = "expand_dims_test",
     size = "small",
     srcs = ["expand_dims_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:builtin_op_data",
@@ -1006,7 +1108,10 @@ tf_cc_test(
     name = "sparse_to_dense_test",
     size = "small",
     srcs = ["sparse_to_dense_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:builtin_op_data",
@@ -1020,7 +1125,10 @@ tf_cc_test(
     name = "shape_test",
     size = "small",
     srcs = ["shape_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:builtin_op_data",
@@ -1034,7 +1142,10 @@ tf_cc_test(
     name = "pow_test",
     size = "small",
     srcs = ["pow_test.cc"],
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":builtin_ops",
         "//tensorflow/contrib/lite:builtin_op_data",
diff --git a/tensorflow/contrib/lite/kernels/internal/BUILD b/tensorflow/contrib/lite/kernels/internal/BUILD
index 7962fcbc9d..3a855fe3dd 100644
--- a/tensorflow/contrib/lite/kernels/internal/BUILD
+++ b/tensorflow/contrib/lite/kernels/internal/BUILD
@@ -232,6 +232,7 @@ cc_library(
 cc_test(
     name = "tensor_test",
     srcs = ["tensor_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":reference",
         "@com_google_googletest//:gtest",
@@ -260,6 +261,7 @@ cc_library(
 cc_test(
     name = "quantization_util_test",
     srcs = ["quantization_util_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":quantization_util",
         "@com_google_googletest//:gtest",
@@ -505,7 +507,10 @@ cc_test(
         "//conditions:default": [],
     }),
     linkstatic = 1,
-    tags = ["tflite_not_portable_ios"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable_ios",
+    ],
     deps = [
         ":tensor_utils",
         "//tensorflow/contrib/lite:builtin_op_data",
@@ -517,6 +522,7 @@ cc_test(
 cc_test(
     name = "depthwiseconv_float_test",
     srcs = ["depthwiseconv_float_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -529,6 +535,7 @@ cc_test(
 cc_test(
     name = "depthwiseconv_quantized_test",
     srcs = ["depthwiseconv_quantized_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -541,7 +548,10 @@ cc_test(
 cc_test(
     name = "resize_bilinear_test",
     srcs = ["resize_bilinear_test.cc"],
-    tags = ["tflite_not_portable"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable",
+    ],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -557,6 +567,7 @@ cc_test(
     srcs = [
         "softmax_quantized_test.cc",
     ],
+    tags = ["no_oss"],
     deps = [
         ":optimized_base",
         ":quantization_util",
@@ -572,7 +583,10 @@ cc_test(
     srcs = [
         "logsoftmax_quantized_test.cc",
     ],
-    tags = ["tflite_not_portable"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable",
+    ],
     deps = [
         ":optimized_base",
         ":quantization_util",
@@ -585,6 +599,7 @@ cc_test(
 cc_test(
     name = "log_quantized_test",
     srcs = ["log_quantized_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":optimized_base",
         ":reference_base",
@@ -611,6 +626,7 @@ cc_library(
 cc_test(
     name = "batch_to_space_nd_test",
     srcs = ["batch_to_space_nd_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":optimized_base",
         "@com_google_googletest//:gtest_main",
diff --git a/tensorflow/contrib/lite/models/smartreply/BUILD b/tensorflow/contrib/lite/models/smartreply/BUILD
index 8b5fa240ac..9d88c396ba 100644
--- a/tensorflow/contrib/lite/models/smartreply/BUILD
+++ b/tensorflow/contrib/lite/models/smartreply/BUILD
@@ -47,6 +47,7 @@ cc_test(
     name = "extract_feature_op_test",
     size = "small",
     srcs = ["ops/extract_feature_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":custom_ops",
         "//tensorflow/contrib/lite:framework",
@@ -61,6 +62,7 @@ cc_test(
     name = "normalize_op_test",
     size = "small",
     srcs = ["ops/normalize_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":custom_ops",
         "//tensorflow/contrib/lite:framework",
@@ -75,6 +77,7 @@ cc_test(
     name = "predict_op_test",
     size = "small",
     srcs = ["ops/predict_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":custom_ops",
         "//tensorflow/contrib/lite:framework",
diff --git a/tensorflow/contrib/lite/profiling/BUILD b/tensorflow/contrib/lite/profiling/BUILD
index a162b87b8f..b29ca330dc 100644
--- a/tensorflow/contrib/lite/profiling/BUILD
+++ b/tensorflow/contrib/lite/profiling/BUILD
@@ -20,6 +20,7 @@ cc_test(
     srcs = ["profiler_test.cc"],
     copts = ["-DTFLITE_PROFILING_ENABLED"],
     defines = ["TFLITE_PROFILING_ENABLED"],
+    tags = ["no_oss"],
     deps = [
         ":profiler",
         "//tensorflow/contrib/lite/testing:util",
@@ -58,6 +59,7 @@ cc_test(
     name = "profile_summarizer_test",
     srcs = ["profile_summarizer_test.cc"],
     copts = common_copts,
+    tags = ["no_oss"],
     deps = [
         ":profile_summarizer",
         "//tensorflow/contrib/lite:framework",
@@ -75,6 +77,7 @@ cc_test(
     srcs = ["profile_buffer_test.cc"],
     copts = ["-DTFLITE_PROFILING_ENABLED"],
     defines = ["TFLITE_PROFILING_ENABLED"],
+    tags = ["no_oss"],
     deps = [
         ":profile_buffer",
         "//tensorflow/contrib/lite/testing:util",
diff --git a/tensorflow/contrib/lite/python/BUILD b/tensorflow/contrib/lite/python/BUILD
index 8c9608db04..727fbff38e 100644
--- a/tensorflow/contrib/lite/python/BUILD
+++ b/tensorflow/contrib/lite/python/BUILD
@@ -71,7 +71,10 @@ py_test(
     srcs = ["lite_test.py"],
     data = [":interpreter_test_data"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
+    tags = [
+        "no_oss",
+        "no_windows",
+    ],
     deps = [
         ":lite",
     ],
@@ -163,7 +166,10 @@ py_test(
     name = "convert_saved_model_test",
     srcs = ["convert_saved_model_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
+    tags = [
+        "no_oss",
+        "no_windows",
+    ],
     visibility = ["//visibility:public"],
     deps = [
         ":convert_saved_model",
diff --git a/tensorflow/contrib/lite/schema/BUILD b/tensorflow/contrib/lite/schema/BUILD
index f095151cae..b616e449e6 100644
--- a/tensorflow/contrib/lite/schema/BUILD
+++ b/tensorflow/contrib/lite/schema/BUILD
@@ -30,7 +30,10 @@ py_test(
     size = "small",
     srcs = ["upgrade_schema_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_pip"],
+    tags = [
+        "no_oss",
+        "no_pip",
+    ],
     deps = [
         ":upgrade_schema",
         "//tensorflow/python:client_testlib",
@@ -64,6 +67,7 @@ cc_test(
         "schema_v3.fbs",
     ],
     tags = [
+        "no_oss",
         "tflite_not_portable_android",
         "tflite_not_portable_ios",
     ],
diff --git a/tensorflow/contrib/lite/schema/builtin_ops_header/BUILD b/tensorflow/contrib/lite/schema/builtin_ops_header/BUILD
index 0148149a6a..4a627761da 100644
--- a/tensorflow/contrib/lite/schema/builtin_ops_header/BUILD
+++ b/tensorflow/contrib/lite/schema/builtin_ops_header/BUILD
@@ -24,6 +24,7 @@ cc_binary(
 cc_test(
     name = "generator_test",
     srcs = ["generator_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":generator",
         "@com_google_googletest//:gtest",
@@ -36,6 +37,7 @@ cc_test(
     data = [
         "//tensorflow/contrib/lite:builtin_ops.h",
     ],
+    tags = ["no_oss"],
     deps = [
         ":generator",
         "@com_google_googletest//:gtest",
diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index 789bc695f8..4c37bcb3c9 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -112,6 +112,7 @@ cc_library(
 cc_test(
     name = "message_test",
     srcs = ["message_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":message",
         "@com_google_googletest//:gtest_main",
@@ -131,6 +132,7 @@ cc_test(
     name = "split_test",
     size = "small",
     srcs = ["split_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":split",
         "@com_google_googletest//:gtest_main",
@@ -146,6 +148,7 @@ cc_test(
     name = "join_test",
     size = "small",
     srcs = ["join_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":join",
         "@com_google_googletest//:gtest_main",
@@ -171,6 +174,7 @@ cc_test(
     srcs = ["tflite_driver_test.cc"],
     data = ["//tensorflow/contrib/lite:testdata/multi_add.bin"],
     tags = [
+        "no_oss",
         "tflite_not_portable_android",
         "tflite_not_portable_ios",
     ],
@@ -192,6 +196,7 @@ cc_library(
 cc_test(
     name = "tokenize_test",
     srcs = ["tokenize_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":tokenize",
         "@com_google_googletest//:gtest_main",
@@ -214,6 +219,7 @@ cc_library(
 cc_test(
     name = "test_runner_test",
     srcs = ["test_runner_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":test_runner",
         "@com_google_googletest//:gtest_main",
@@ -252,6 +258,7 @@ cc_test(
     srcs = ["tf_driver_test.cc"],
     data = ["//tensorflow/contrib/lite:testdata/multi_add.pb"],
     tags = [
+        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
@@ -277,6 +284,7 @@ cc_test(
     size = "small",
     srcs = ["generate_testspec_test.cc"],
     tags = [
+        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index 2252fe175a..5e197e584c 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -337,6 +337,7 @@ cc_library(
 tf_cc_test(
     name = "import_tensorflow_test",
     srcs = ["import_tensorflow_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":toco_tooling",
         "//tensorflow/core:framework",
@@ -376,6 +377,7 @@ cc_library(
 tf_cc_test(
     name = "tooling_util_test",
     srcs = ["tooling_util_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":model",
         ":tooling_util",
@@ -410,6 +412,7 @@ tf_cc_test(
     data = [
         "toco_port_test.cc",
     ],
+    tags = ["no_oss"],
     deps = [
         ":toco_port",
         "@com_google_googletest//:gtest_main",
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD b/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
index 95e8433be2..e163fc9ae1 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
+++ b/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
@@ -10,6 +10,7 @@ load(
 tf_cc_test(
     name = "lstm_utils_test",
     srcs = ["lstm_utils_test.cc"],
+    tags = ["no_oss"],
     deps = [
         "//tensorflow/contrib/lite/toco:graph_transformations",
         "//tensorflow/contrib/lite/toco:model",
@@ -21,6 +22,7 @@ tf_cc_test(
 tf_cc_test(
     name = "quantize_weights_test",
     srcs = ["quantize_weights_test.cc"],
+    tags = ["no_oss"],
     deps = [
         "//tensorflow/contrib/lite/toco:graph_transformations",
         "//tensorflow/contrib/lite/toco:model",
@@ -33,6 +35,7 @@ tf_cc_test(
 tf_cc_test(
     name = "resolve_constant_concatenation_test",
     srcs = ["resolve_constant_concatenation_test.cc"],
+    tags = ["no_oss"],
     deps = [
         "//tensorflow/contrib/lite/toco:graph_transformations",
         "//tensorflow/contrib/lite/toco:model",
diff --git a/tensorflow/contrib/lite/toco/python/BUILD b/tensorflow/contrib/lite/toco/python/BUILD
index 93fe756a55..33c5b16462 100644
--- a/tensorflow/contrib/lite/toco/python/BUILD
+++ b/tensorflow/contrib/lite/toco/python/BUILD
@@ -53,5 +53,8 @@ tf_py_test(
     data = [
         ":toco_from_protos",
     ],
-    tags = ["no_pip"],
+    tags = [
+        "no_oss",
+        "no_pip",
+    ],
 )
diff --git a/tensorflow/contrib/lite/toco/tensorflow_graph_matching/BUILD b/tensorflow/contrib/lite/toco/tensorflow_graph_matching/BUILD
index 336e94de1e..ea1fc2827e 100644
--- a/tensorflow/contrib/lite/toco/tensorflow_graph_matching/BUILD
+++ b/tensorflow/contrib/lite/toco/tensorflow_graph_matching/BUILD
@@ -60,6 +60,7 @@ cc_library(
 tf_cc_test(
     name = "resolve_svdf_test",
     srcs = ["resolve_svdf_test.cc"],
+    tags = ["no_oss"],
     deps = [
         ":cluster",
         ":cluster_utils",
diff --git a/tensorflow/contrib/lite/toco/tflite/BUILD b/tensorflow/contrib/lite/toco/tflite/BUILD
index a02f90988b..83e977d7b3 100644
--- a/tensorflow/contrib/lite/toco/tflite/BUILD
+++ b/tensorflow/contrib/lite/toco/tflite/BUILD
@@ -37,6 +37,7 @@ tf_cc_test(
     srcs = [
         "operator_test.cc",
     ],
+    tags = ["no_oss"],
     deps = [
         ":operator",
         "//tensorflow/contrib/lite/toco:tooling_util",
@@ -66,6 +67,7 @@ tf_cc_test(
     srcs = [
         "types_test.cc",
     ],
+    tags = ["no_oss"],
     deps = [
         ":types",
         "@com_google_googletest//:gtest_main",
@@ -98,6 +100,7 @@ tf_cc_test(
     srcs = [
         "export_test.cc",
     ],
+    tags = ["no_oss"],
     deps = [
         ":export",
         "//tensorflow/contrib/lite/schema:schema_fbs",
@@ -131,6 +134,7 @@ tf_cc_test(
     srcs = [
         "import_test.cc",
     ],
+    tags = ["no_oss"],
     deps = [
         ":import",
         "//tensorflow/contrib/lite:schema_fbs_version",
diff --git a/tensorflow/contrib/lite/tools/BUILD b/tensorflow/contrib/lite/tools/BUILD
index d070018e83..0b26826403 100644
--- a/tensorflow/contrib/lite/tools/BUILD
+++ b/tensorflow/contrib/lite/tools/BUILD
@@ -53,6 +53,7 @@ cc_test(
         "//tensorflow/contrib/lite:testdata/test_model_broken.bin",
     ],
     tags = [
+        "no_oss",
         "tflite_not_portable_android",
         "tflite_not_portable_ios",
     ],
@@ -79,6 +80,7 @@ cc_test(
     size = "small",
     srcs = ["verifier_test.cc"],
     tags = [
+        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
diff --git a/tensorflow/contrib/lite/tools/benchmark/BUILD b/tensorflow/contrib/lite/tools/benchmark/BUILD
index 183a545295..810e25961f 100644
--- a/tensorflow/contrib/lite/tools/benchmark/BUILD
+++ b/tensorflow/contrib/lite/tools/benchmark/BUILD
@@ -40,6 +40,7 @@ cc_test(
     name = "command_line_flags_test",
     srcs = ["command_line_flags_test.cc"],
     copts = common_copts,
+    tags = ["no_oss"],
     visibility = ["//visibility:private"],
     deps = [
         ":command_line_flags",
-- 
cgit v1.2.3


From c8f0730484376c3020804fcf1639f55934b70cbe Mon Sep 17 00:00:00 2001
From: Jie 
Date: Tue, 17 Jul 2018 14:53:29 -0700
Subject: addressing review comments; changing set to unordered_set

---
 .../contrib/tensorrt/resources/trt_allocator.cc      | 20 +++++++++-----------
 .../contrib/tensorrt/resources/trt_allocator.h       |  2 +-
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
index e2bc5a61d0..d21f2783d4 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
@@ -41,17 +41,15 @@ void* TRTDeviceAllocator::allocate(uint64_t size, uint64_t alignment,
   alignment = 512;
   assert((alignment & (alignment - 1)) == 0);  // zero or a power of 2.
   void* mem = allocator_->AllocateRaw(alignment, size + alignment);
-
-  CHECK(mem_pool.count(mem) == 0);
-  mem_pool.insert(mem);
   CHECK(mem);
+
+  CHECK(mem_pool.insert(mem).second);
   void* alloc_mem = mem;
   uint64_t total_size = size + alignment;
   std::align(alignment, size, mem, total_size);
   CHECK(mem);
   if (mem != alloc_mem) {
-    CHECK(mem_map.count(mem) == 0);
-    mem_map[mem] = alloc_mem;
+    CHECK(mem_map.insert({mem, alloc_mem}).second);
   }
   VLOG(2) << "Allocated " << size << " bytes with alignment " << alignment
           << " @ " << mem;
@@ -66,13 +64,13 @@ TRTDeviceAllocator::TRTDeviceAllocator(tensorflow::Allocator* allocator)
 void TRTDeviceAllocator::free(void* memory) {
   VLOG(2) << "Deallocating @ " << memory;
   // allocated memory adjusted for alignment, restore the original pointer
-  if (mem_map.count(memory) != 0) {
-    auto alloc_mem = mem_map[memory];
-    mem_map.erase(memory);
-    memory = alloc_mem;
+  
+  auto alloc_mem = mem_map.find(memory);
+  if (alloc_mem != mem_map.end()) {
+    memory = alloc_mem->second;
+    mem_map.erase(alloc_mem->first);
   }
-  CHECK(mem_pool.count(memory) != 0);
-  mem_pool.erase(memory);
+  CHECK(mem_pool.erase(memory) != 0);
   allocator_->DeallocateRaw(memory);
 }
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.h b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
index 9ec0b3c4ff..e405d06cc4 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
@@ -69,7 +69,7 @@ class TRTDeviceAllocator : public TRTBaseAllocator {
 
   // supporting alignment from allocation request requires a map to free;
   std::unordered_map mem_map;
-  std::set mem_pool;
+  std::unordered_set mem_pool;
 };
 
 }  // namespace tensorrt
-- 
cgit v1.2.3


From dae05946b74222880fd4bf731ea5c8feab4e37c6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 15:00:03 -0700
Subject: [TF:XLA] Avoid special case copies for conditional HLOs.

PiperOrigin-RevId: 204980790
---
 tensorflow/compiler/xla/service/copy_insertion.cc | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tensorflow/compiler/xla/service/copy_insertion.cc b/tensorflow/compiler/xla/service/copy_insertion.cc
index ab3d846403..ca2a78da67 100644
--- a/tensorflow/compiler/xla/service/copy_insertion.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion.cc
@@ -76,15 +76,6 @@ SpecialCaseCopyPolicy GetSpecialCaseCopyPolicy(const CallGraphNode& node,
     policy.copy_parameters_and_constants = true;
     policy.copy_root_replicated_buffers = true;
   }
-  for (const CallSite& site : node.caller_callsites()) {
-    // The AddCopiesForConditional() already adds copies, but the copy remover
-    // removes them, so we re-add them by returning the policy here. But really
-    // the copy remover should not be removing them.
-    if (site.instruction()->opcode() == HloOpcode::kConditional) {
-      policy.copy_parameters_and_constants = true;
-      policy.copy_root_replicated_buffers = true;
-    }
-  }
   return policy;
 }
 
-- 
cgit v1.2.3


From 6d0c3acc5a728652b77914a38482f9592578e0df Mon Sep 17 00:00:00 2001
From: Jie 
Date: Tue, 17 Jul 2018 15:07:50 -0700
Subject: 1. rename class member variables with trailing underscore 2. change
 unique_ptr allocator type to base class.

---
 tensorflow/contrib/tensorrt/convert/convert_graph.cc   |  2 +-
 tensorflow/contrib/tensorrt/resources/trt_allocator.cc | 12 ++++++------
 tensorflow/contrib/tensorrt/resources/trt_allocator.h  |  4 ++--
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index bcc867efea..97a26f796c 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -829,7 +829,7 @@ tensorflow::Status ConvertAfterShapes(ConversionParams& params) {
     // The allocator is used to build the engine. The build and the built engine
     // will be destroyed after we get the serialized engine string, so it's fine
     // to use unique_ptr here.
-    std::unique_ptr alloc;
+    std::unique_ptr alloc;
     auto device_alloc = GetDeviceAndAllocator(params, engine);
     int cuda_device_id = 0;
     if (device_alloc.first >= 0) {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
index d21f2783d4..b55cba3bf3 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
@@ -43,13 +43,13 @@ void* TRTDeviceAllocator::allocate(uint64_t size, uint64_t alignment,
   void* mem = allocator_->AllocateRaw(alignment, size + alignment);
   CHECK(mem);
 
-  CHECK(mem_pool.insert(mem).second);
+  CHECK(mem_pool_.insert(mem).second);
   void* alloc_mem = mem;
   uint64_t total_size = size + alignment;
   std::align(alignment, size, mem, total_size);
   CHECK(mem);
   if (mem != alloc_mem) {
-    CHECK(mem_map.insert({mem, alloc_mem}).second);
+    CHECK(mem_map_.insert({mem, alloc_mem}).second);
   }
   VLOG(2) << "Allocated " << size << " bytes with alignment " << alignment
           << " @ " << mem;
@@ -65,12 +65,12 @@ void TRTDeviceAllocator::free(void* memory) {
   VLOG(2) << "Deallocating @ " << memory;
   // allocated memory adjusted for alignment, restore the original pointer
   
-  auto alloc_mem = mem_map.find(memory);
-  if (alloc_mem != mem_map.end()) {
+  auto alloc_mem = mem_map_.find(memory);
+  if (alloc_mem != mem_map_.end()) {
     memory = alloc_mem->second;
-    mem_map.erase(alloc_mem->first);
+    mem_map_.erase(alloc_mem->first);
   }
-  CHECK(mem_pool.erase(memory) != 0);
+  CHECK(mem_pool_.erase(memory) != 0);
   allocator_->DeallocateRaw(memory);
 }
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.h b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
index e405d06cc4..294b76a333 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
@@ -68,8 +68,8 @@ class TRTDeviceAllocator : public TRTBaseAllocator {
   tensorflow::Allocator* allocator_;
 
   // supporting alignment from allocation request requires a map to free;
-  std::unordered_map mem_map;
-  std::unordered_set mem_pool;
+  std::unordered_map mem_map_;
+  std::unordered_set mem_pool_;
 };
 
 }  // namespace tensorrt
-- 
cgit v1.2.3


From e56a5b8ec24bb3d3b9073bb6fcbaf57c70fcecb6 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne 
Date: Tue, 17 Jul 2018 15:03:38 -0700
Subject: Automated rollback of commit 2936833c7e22c102ff2b82e3f4e261b94602fbcc

PiperOrigin-RevId: 204981602
---
 tensorflow/core/common_runtime/direct_session.cc     | 14 ++++++--------
 tensorflow/core/common_runtime/session.cc            | 20 +++++++++++++-------
 tensorflow/core/common_runtime/session_factory.h     |  7 ++++++-
 tensorflow/core/common_runtime/session_test.cc       |  6 ++++--
 .../core/distributed_runtime/rpc/grpc_session.cc     | 15 ++++++---------
 tensorflow/core/public/session.h                     |  2 +-
 6 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 1732553abd..4c670820be 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -146,18 +146,15 @@ class DirectSessionFactory : public SessionFactory {
     return options.target.empty();
   }
 
-  Session* NewSession(const SessionOptions& options) override {
+  Status NewSession(const SessionOptions& options,
+                    Session** out_session) override {
     // Must do this before the CPU allocator is created.
     if (options.config.graph_options().build_cost_model() > 0) {
       EnableCPUAllocatorFullStats(true);
     }
     std::vector devices;
-    const Status s = DeviceFactory::AddDevices(
-        options, "/job:localhost/replica:0/task:0", &devices);
-    if (!s.ok()) {
-      LOG(ERROR) << s;
-      return nullptr;
-    }
+    TF_RETURN_IF_ERROR(DeviceFactory::AddDevices(
+        options, "/job:localhost/replica:0/task:0", &devices));
 
     DirectSession* session =
         new DirectSession(options, new DeviceMgr(devices), this);
@@ -165,7 +162,8 @@ class DirectSessionFactory : public SessionFactory {
       mutex_lock l(sessions_lock_);
       sessions_.push_back(session);
     }
-    return session;
+    *out_session = session;
+    return Status::OK();
   }
 
   Status Reset(const SessionOptions& options,
diff --git a/tensorflow/core/common_runtime/session.cc b/tensorflow/core/common_runtime/session.cc
index 4a9248171b..8c30beeec2 100644
--- a/tensorflow/core/common_runtime/session.cc
+++ b/tensorflow/core/common_runtime/session.cc
@@ -53,27 +53,33 @@ Status Session::PRun(const string& handle,
 
 Session* NewSession(const SessionOptions& options) {
   SessionFactory* factory;
-  const Status s = SessionFactory::GetFactory(options, &factory);
+  Status s = SessionFactory::GetFactory(options, &factory);
   if (!s.ok()) {
     LOG(ERROR) << s;
     return nullptr;
   }
-  return factory->NewSession(options);
+  Session* out_session;
+  s = NewSession(options, &out_session);
+  if (!s.ok()) {
+    LOG(ERROR) << "Failed to create session: " << s;
+    return nullptr;
+  }
+  return out_session;
 }
 
 Status NewSession(const SessionOptions& options, Session** out_session) {
   SessionFactory* factory;
-  const Status s = SessionFactory::GetFactory(options, &factory);
+  Status s = SessionFactory::GetFactory(options, &factory);
   if (!s.ok()) {
     *out_session = nullptr;
     LOG(ERROR) << s;
     return s;
   }
-  *out_session = factory->NewSession(options);
-  if (!*out_session) {
-    return errors::Internal("Failed to create session.");
+  s = factory->NewSession(options, out_session);
+  if (!s.ok()) {
+    *out_session = nullptr;
   }
-  return Status::OK();
+  return s;
 }
 
 Status Reset(const SessionOptions& options,
diff --git a/tensorflow/core/common_runtime/session_factory.h b/tensorflow/core/common_runtime/session_factory.h
index df3198a70d..81c172c6ae 100644
--- a/tensorflow/core/common_runtime/session_factory.h
+++ b/tensorflow/core/common_runtime/session_factory.h
@@ -30,7 +30,12 @@ struct SessionOptions;
 
 class SessionFactory {
  public:
-  virtual Session* NewSession(const SessionOptions& options) = 0;
+  // Creates a new session and stores it in *out_session, or fails with an error
+  // status if the Session could not be created. Caller takes ownership of
+  // *out_session if this returns Status::OK().
+  virtual Status NewSession(const SessionOptions& options,
+                            Session** out_session) = 0;
+
   virtual bool AcceptsOptions(const SessionOptions& options) = 0;
 
   // Abort and close all existing sessions, disconnecting their resources from
diff --git a/tensorflow/core/common_runtime/session_test.cc b/tensorflow/core/common_runtime/session_test.cc
index feaf29c7bb..1fa5aad60c 100644
--- a/tensorflow/core/common_runtime/session_test.cc
+++ b/tensorflow/core/common_runtime/session_test.cc
@@ -47,8 +47,10 @@ class FakeSessionFactory : public SessionFactory {
     return str_util::StartsWith(options.target, "fake");
   }
 
-  Session* NewSession(const SessionOptions& options) override {
-    return nullptr;
+  Status NewSession(const SessionOptions& options,
+                    Session** out_session) override {
+    *out_session = nullptr;
+    return Status::OK();
   }
 };
 class FakeSessionRegistrar {
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
index fd1c150fa7..fdce1b10e0 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
@@ -452,15 +452,12 @@ class GrpcSessionFactory : public SessionFactory {
     return str_util::StartsWith(options.target, kSchemePrefix);
   }
 
-  Session* NewSession(const SessionOptions& options) override {
-    std::unique_ptr ret;
-    Status s = GrpcSession::Create(options, &ret);
-    if (s.ok()) {
-      return ret.release();
-    } else {
-      LOG(ERROR) << "Error during session construction: " << s.ToString();
-      return nullptr;
-    }
+  Status NewSession(const SessionOptions& options,
+                    Session** out_session) override {
+    std::unique_ptr session;
+    TF_RETURN_IF_ERROR(GrpcSession::Create(options, &session));
+    *out_session = session.release();
+    return Status::OK();
   }
 
   // Invokes the session specific static method to reset containers.
diff --git a/tensorflow/core/public/session.h b/tensorflow/core/public/session.h
index d58c877cfd..cc8596ef3d 100644
--- a/tensorflow/core/public/session.h
+++ b/tensorflow/core/public/session.h
@@ -237,7 +237,7 @@ class Session {
 /// If session creation succeeds, the new `Session` will be stored in
 /// `*out_session`, the caller will take ownership of the returned
 /// `*out_session`, and this function will return `OK()`. Otherwise, this
-/// function will return an error status.
+/// function will return an error status and set *out_session to nullptr.
 Status NewSession(const SessionOptions& options, Session** out_session);
 
 /// \brief Resets resource containers associated with a target.
-- 
cgit v1.2.3


From 82768a48aab6839585bd97d68bdbfe42d84c02bc Mon Sep 17 00:00:00 2001
From: Zhenyu Tan 
Date: Tue, 17 Jul 2018 15:27:00 -0700
Subject: move session config creation logic to seperate function for reuse

PiperOrigin-RevId: 204985561
---
 tensorflow/python/estimator/estimator.py      |  8 +-------
 tensorflow/python/estimator/estimator_test.py |  5 +++++
 tensorflow/python/estimator/run_config.py     | 15 +++++++++++++--
 3 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 253716b43e..2fd6f6fab9 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -29,8 +29,6 @@ import six
 
 from google.protobuf import message
 from tensorflow.core.framework import summary_pb2
-from tensorflow.core.protobuf import config_pb2
-from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.client import session as tf_session
 from tensorflow.python.eager import context
 from tensorflow.python.estimator import model_fn as model_fn_lib
@@ -216,11 +214,7 @@ class Estimator(object):
     logging.info('Using config: %s', str(vars(self._config)))
 
     if self._config.session_config is None:
-      rewrite_opts = rewriter_config_pb2.RewriterConfig(
-          meta_optimizer_iterations=rewriter_config_pb2.RewriterConfig.ONE)
-      graph_opts = config_pb2.GraphOptions(rewrite_options=rewrite_opts)
-      self._session_config = config_pb2.ConfigProto(
-          allow_soft_placement=True, graph_options=graph_opts)
+      self._session_config = run_config.get_default_session_config()
     else:
       self._session_config = self._config.session_config
 
diff --git a/tensorflow/python/estimator/estimator_test.py b/tensorflow/python/estimator/estimator_test.py
index 495d019f26..8bc410ba0b 100644
--- a/tensorflow/python/estimator/estimator_test.py
+++ b/tensorflow/python/estimator/estimator_test.py
@@ -28,6 +28,7 @@ import six
 
 from google.protobuf import text_format
 
+from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.estimator import estimator
@@ -203,6 +204,10 @@ class EstimatorConstructorTest(test.TestCase):
 
     est = estimator.Estimator(model_fn=model_fn)
     self.assertTrue(isinstance(est.config, run_config.RunConfig))
+    self.assertTrue(est._session_config.allow_soft_placement)
+    rewrite_options = est._session_config.graph_options.rewrite_options
+    self.assertEqual(rewrite_options.meta_optimizer_iterations,
+                     rewriter_config_pb2.RewriterConfig.ONE)
 
   def test_default_model_dir(self):
 
diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py
index b495c4884d..6c1de166a4 100644
--- a/tensorflow/python/estimator/run_config.py
+++ b/tensorflow/python/estimator/run_config.py
@@ -294,6 +294,17 @@ def _validate_properties(run_config):
             message='protocol should be grpc or grpc+verbs')
 
 
+def get_default_session_config():
+  """Returns tf.ConfigProto instance."""
+
+  rewrite_opts = rewriter_config_pb2.RewriterConfig(
+      meta_optimizer_iterations=rewriter_config_pb2.RewriterConfig.ONE)
+  graph_opts = config_pb2.GraphOptions(rewrite_options=rewrite_opts)
+
+  return config_pb2.ConfigProto(allow_soft_placement=True,
+                                graph_options=graph_opts)
+
+
 class TaskType(object):
   MASTER = 'master'
   PS = 'ps'
@@ -508,9 +519,9 @@ class RunConfig(object):
       RunConfig._replace(
           self,
           allowed_properties_list=_DEFAULT_REPLACEABLE_LIST,
-          session_config=self._get_default_session_config())
+          session_config=self._get_default_session_config_distributed())
 
-  def _get_default_session_config(self):
+  def _get_default_session_config_distributed(self):
     """Returns None or tf.ConfigProto instance with default device_filters set.
 
     Device filters are set such that chief/master and worker communicates with
-- 
cgit v1.2.3


From cba4d6527c5b64eea86de15cf9a78f0ac0c39af9 Mon Sep 17 00:00:00 2001
From: Guangda Lai 
Date: Tue, 17 Jul 2018 15:27:07 -0700
Subject: Fix window bazel build failure by exclusing
 tensorflow/contrib/tensorrt:init_py from windows build.

PiperOrigin-RevId: 204985580
---
 tensorflow/contrib/BUILD          | 2 +-
 tensorflow/contrib/tensorrt/BUILD | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 1322056d80..a173c51879 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -102,7 +102,6 @@ py_library(
         "//tensorflow/contrib/summary:summary",
         "//tensorflow/contrib/tensor_forest:init_py",
         "//tensorflow/contrib/tensorboard",
-        "//tensorflow/contrib/tensorrt:init_py",
         "//tensorflow/contrib/testing:testing_py",
         "//tensorflow/contrib/text:text_py",
         "//tensorflow/contrib/tfprof",
@@ -132,6 +131,7 @@ py_library(
         "//tensorflow/contrib/cloud:cloud_py",  # doesn't compile on Windows
         "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py",
         "//tensorflow/contrib/lite/python:lite",  # unix dependency, need to fix code
+        "//tensorflow/contrib/tensorrt:init_py",  # doesn't compile on windows
     ]),
 )
 
diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index cb2daa7b12..70ce4a499c 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -311,6 +311,7 @@ tf_cuda_cc_test(
     name = "trt_plugin_factory_test",
     size = "small",
     srcs = ["plugin/trt_plugin_factory_test.cc"],
+    tags = ["no_windows"],
     deps = [
         ":trt_plugins",
         "//tensorflow/core:lib",
-- 
cgit v1.2.3


From 2196905591ada49e811a2fc30bcdadd02489dda1 Mon Sep 17 00:00:00 2001
From: Justin Lebar 
Date: Tue, 17 Jul 2018 15:52:00 -0700
Subject: [XLA:GPU] Warn if using driver jit < 396.20.0.

396.20 is the first driver version that contains ptxas 9.2.88, which is
the oldest known-working ptxas at the moment.

PiperOrigin-RevId: 204989646
---
 .../compiler/xla/service/gpu/nvptx_compiler.cc     | 23 ++++++++++------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index de5903f359..ad29862d83 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -362,8 +362,8 @@ void WarnIfBadPtxasVersion(const string& ptxas_path) {
   // b/70245379.
   //
   // ptxas 9.1.121 miscompiles some large multioutput fusions, again in a way
-  // that appears related to address calculations.  ptxas 9.2.88 appears to
-  // work, as far as we can tell.
+  // that appears related to address calculations, b/111107644.  ptxas 9.2.88
+  // appears to work, as far as we can tell.
   if (vmaj < 9) {
     LOG(ERROR)
         << "You are using ptxas 8.x, but XLA requires ptxas 9.x (and strongly "
@@ -406,20 +406,17 @@ void WarnIfBadDriverJITVersion() {
     //  - 387.x before 387.40
     //  - 390.x before 390.10.
     //
-    // TODO(jlebar): This list does not cover the address-calculation bug we've
-    // observed in ptxas 9.1.121.  Need to get a new safe range from nvidia
-    // corresponding to ptxas >= 9.2.88.
-    auto vmaj = std::get<0>(version);
-    auto vmin = std::get<1>(version);
-    if ((vmaj == 384 && vmin < 108) ||  //
-        (vmaj == 387 && vmin < 40) ||   //
-        (vmaj == 390 && vmin < 10)) {
+    // In addition, only >= 396.20 contains ptxas >= 9.2.88, which contains the
+    // fix for the "large multioutput fusions" miscompile, b/111107644.
+    if (version < std::make_tuple(396, 20, 0)) {
       LOG(WARNING)
           << "*** WARNING *** Invoking the PTX->SASS JIT from driver version "
           << se::cuda::DriverVersionToString(version)
-          << ", which is in range [384.0.0, 384.108.0) + [387.0.0, 387.40.0) + "
-             "[390.0.0, 390.10.0). These versions are known to miscompile XLA "
-             "code, leading to incorrect results or invalid-address errors.";
+          << ", which is older than 396.20.0. These versions are known to "
+             "miscompile XLA code, leading to incorrect results or "
+             "invalid-address errors.\nXLA only uses the driver JIT if it "
+             "cannot find ptxas; you don't need to update your driver if "
+             "you can point XLA to ptxas 9.2.88 or newer.";
     }
   });
 }
-- 
cgit v1.2.3


From efaadba01be04dc1f7f987d79519bfa1066c5945 Mon Sep 17 00:00:00 2001
From: Jie 
Date: Tue, 17 Jul 2018 16:23:55 -0700
Subject: check the status for std::align; removing memory allocate/free
 validation

---
 tensorflow/contrib/tensorrt/resources/trt_allocator.cc | 11 +++++------
 tensorflow/contrib/tensorrt/resources/trt_allocator.h  |  1 -
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
index b55cba3bf3..f733b3dd56 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
@@ -37,17 +37,17 @@ void TRTCudaAllocator::free(void* memory) { cudaFree(memory); }
 
 void* TRTDeviceAllocator::allocate(uint64_t size, uint64_t alignment,
                                    uint32_t flags) {
-  // WAR for allocator alignment requirement
+  // WAR for allocator alignment requirement. Certain cuda API calls require GPU
+  // memory with alignemtn to cudaDeviceProp::textureAlignment.
+  // See issue #20856
   alignment = 512;
   assert((alignment & (alignment - 1)) == 0);  // zero or a power of 2.
   void* mem = allocator_->AllocateRaw(alignment, size + alignment);
   CHECK(mem);
 
-  CHECK(mem_pool_.insert(mem).second);
   void* alloc_mem = mem;
   uint64_t total_size = size + alignment;
-  std::align(alignment, size, mem, total_size);
-  CHECK(mem);
+  CHECK(std::align(alignment, size, mem, total_size));
   if (mem != alloc_mem) {
     CHECK(mem_map_.insert({mem, alloc_mem}).second);
   }
@@ -64,13 +64,12 @@ TRTDeviceAllocator::TRTDeviceAllocator(tensorflow::Allocator* allocator)
 void TRTDeviceAllocator::free(void* memory) {
   VLOG(2) << "Deallocating @ " << memory;
   // allocated memory adjusted for alignment, restore the original pointer
-  
+
   auto alloc_mem = mem_map_.find(memory);
   if (alloc_mem != mem_map_.end()) {
     memory = alloc_mem->second;
     mem_map_.erase(alloc_mem->first);
   }
-  CHECK(mem_pool_.erase(memory) != 0);
   allocator_->DeallocateRaw(memory);
 }
 
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.h b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
index 294b76a333..cb00aee37d 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
@@ -69,7 +69,6 @@ class TRTDeviceAllocator : public TRTBaseAllocator {
 
   // supporting alignment from allocation request requires a map to free;
   std::unordered_map mem_map_;
-  std::unordered_set mem_pool_;
 };
 
 }  // namespace tensorrt
-- 
cgit v1.2.3


From 98772cfd22ca07fa693e8f4dc50337ce0edf7cf8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 16:29:52 -0700
Subject: Consolidate all Keras initializers into ops/init_ops

PiperOrigin-RevId: 204995701
---
 tensorflow/python/BUILD                            |  16 +-
 tensorflow/python/keras/initializers.py            | 100 ++---------
 tensorflow/python/keras/initializers_test.py       |  10 --
 tensorflow/python/ops/init_ops.py                  |  99 ++++++++++-
 tensorflow/python/ops/init_ops_test.py             | 196 +++++++++++++++++++++
 .../tools/api/golden/tensorflow.initializers.pbtxt |  24 +++
 6 files changed, 341 insertions(+), 104 deletions(-)
 create mode 100644 tensorflow/python/ops/init_ops_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 51e6d5aabf..a362dee97d 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2136,8 +2136,8 @@ py_library(
         ":linalg_ops_gen",
         ":linalg_ops_impl",
         ":math_ops",
-        ":nn_ops",
         ":random_ops",
+        ":util",
         "//third_party/py/numpy",
     ],
 )
@@ -3045,6 +3045,20 @@ cuda_py_test(
     shard_count = 5,
 )
 
+cuda_py_test(
+    name = "init_ops_test",
+    size = "small",
+    srcs = ["ops/init_ops_test.py"],
+    additional_deps = [
+        ":client_testlib",
+        ":init_ops",
+        ":framework_ops",
+        ":resource_variable_ops",
+        "//third_party/py/numpy",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
 cuda_py_test(
     name = "math_grad_test",
     size = "small",
diff --git a/tensorflow/python/keras/initializers.py b/tensorflow/python/keras/initializers.py
index ea104ea65d..b9d856efa8 100644
--- a/tensorflow/python/keras/initializers.py
+++ b/tensorflow/python/keras/initializers.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Keras initializer classes (soon to be replaced with core TF initializers).
+"""Keras initializer serialization / deserialization.
 """
 from __future__ import absolute_import
 from __future__ import division
@@ -22,107 +22,27 @@ import six
 
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
+
+# These imports are brought in so that keras.initializers.deserialize
+# has them available in module_objects.
 from tensorflow.python.ops.init_ops import Constant
 from tensorflow.python.ops.init_ops import glorot_normal_initializer
 from tensorflow.python.ops.init_ops import glorot_uniform_initializer
-
+from tensorflow.python.ops.init_ops import he_normal  # pylint: disable=unused-import
+from tensorflow.python.ops.init_ops import he_uniform  # pylint: disable=unused-import
 from tensorflow.python.ops.init_ops import Identity
 from tensorflow.python.ops.init_ops import Initializer  # pylint: disable=unused-import
+from tensorflow.python.ops.init_ops import lecun_normal  # pylint: disable=unused-import
+from tensorflow.python.ops.init_ops import lecun_uniform  # pylint: disable=unused-import
 from tensorflow.python.ops.init_ops import Ones
 from tensorflow.python.ops.init_ops import Orthogonal
 from tensorflow.python.ops.init_ops import RandomNormal
 from tensorflow.python.ops.init_ops import RandomUniform
 from tensorflow.python.ops.init_ops import TruncatedNormal
-from tensorflow.python.ops.init_ops import VarianceScaling
+from tensorflow.python.ops.init_ops import VarianceScaling  # pylint: disable=unused-import
 from tensorflow.python.ops.init_ops import Zeros
-from tensorflow.python.util.tf_export import tf_export
-
-
-@tf_export('keras.initializers.lecun_normal')
-def lecun_normal(seed=None):
-  """LeCun normal initializer.
-
-  It draws samples from a truncated normal distribution centered on 0
-  with `stddev = sqrt(1 / fan_in)`
-  where `fan_in` is the number of input units in the weight tensor.
-
-  Arguments:
-      seed: A Python integer. Used to seed the random generator.
-
-  Returns:
-      An initializer.
-
-  References:
-      - [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515)
-      - [Efficient
-      Backprop](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf)
-  """
-  return VarianceScaling(
-      scale=1., mode='fan_in', distribution='truncated_normal', seed=seed)
-
-
-@tf_export('keras.initializers.lecun_uniform')
-def lecun_uniform(seed=None):
-  """LeCun uniform initializer.
-
-  It draws samples from a uniform distribution within [-limit, limit]
-  where `limit` is `sqrt(3 / fan_in)`
-  where `fan_in` is the number of input units in the weight tensor.
-
-  Arguments:
-      seed: A Python integer. Used to seed the random generator.
-
-  Returns:
-      An initializer.
 
-  References:
-      LeCun 98, Efficient Backprop,
-      http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf
-  """
-  return VarianceScaling(
-      scale=1., mode='fan_in', distribution='uniform', seed=seed)
-
-
-@tf_export('keras.initializers.he_normal')
-def he_normal(seed=None):
-  """He normal initializer.
-
-  It draws samples from a truncated normal distribution centered on 0
-  with `stddev = sqrt(2 / fan_in)`
-  where `fan_in` is the number of input units in the weight tensor.
-
-  Arguments:
-      seed: A Python integer. Used to seed the random generator.
-
-  Returns:
-      An initializer.
-
-  References:
-      He et al., http://arxiv.org/abs/1502.01852
-  """
-  return VarianceScaling(
-      scale=2., mode='fan_in', distribution='truncated_normal', seed=seed)
-
-
-@tf_export('keras.initializers.he_uniform')
-def he_uniform(seed=None):
-  """He uniform variance scaling initializer.
-
-  It draws samples from a uniform distribution within [-limit, limit]
-  where `limit` is `sqrt(6 / fan_in)`
-  where `fan_in` is the number of input units in the weight tensor.
-
-  Arguments:
-      seed: A Python integer. Used to seed the random generator.
-
-  Returns:
-      An initializer.
-
-  References:
-      He et al., http://arxiv.org/abs/1502.01852
-  """
-  return VarianceScaling(
-      scale=2., mode='fan_in', distribution='uniform', seed=seed)
+from tensorflow.python.util.tf_export import tf_export
 
 
 # Compatibility aliases
diff --git a/tensorflow/python/keras/initializers_test.py b/tensorflow/python/keras/initializers_test.py
index c519e194bd..51725e03f2 100644
--- a/tensorflow/python/keras/initializers_test.py
+++ b/tensorflow/python/keras/initializers_test.py
@@ -31,16 +31,6 @@ class KerasInitializersTest(test.TestCase):
               target_max=None, target_min=None):
     variable = keras.backend.variable(init(shape))
     output = keras.backend.get_value(variable)
-    lim = 3e-2
-    if target_std is not None:
-      self.assertGreater(lim, abs(output.std() - target_std))
-    if target_mean is not None:
-      self.assertGreater(lim, abs(output.mean() - target_mean))
-    if target_max is not None:
-      self.assertGreater(lim, abs(output.max() - target_max))
-    if target_min is not None:
-      self.assertGreater(lim, abs(output.min() - target_min))
-
     # Test serialization (assumes deterministic behavior).
     config = init.get_config()
     reconstructed_init = init.__class__.from_config(config)
diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py
index 3132f7467f..c315722b6b 100644
--- a/tensorflow/python/ops/init_ops.py
+++ b/tensorflow/python/ops/init_ops.py
@@ -1136,7 +1136,8 @@ convolutional_orthogonal_3d = ConvolutionOrthogonal3D
 # pylint: enable=invalid-name
 
 
-@tf_export("glorot_uniform_initializer", "keras.initializers.glorot_uniform")
+@tf_export("glorot_uniform_initializer", "keras.initializers.glorot_uniform",
+           "initializers.glorot_uniform")
 def glorot_uniform_initializer(seed=None, dtype=dtypes.float32):
   """The Glorot uniform initializer, also called Xavier uniform initializer.
 
@@ -1160,7 +1161,8 @@ def glorot_uniform_initializer(seed=None, dtype=dtypes.float32):
       scale=1.0, mode="fan_avg", distribution="uniform", seed=seed, dtype=dtype)
 
 
-@tf_export("glorot_normal_initializer", "keras.initializers.glorot_normal")
+@tf_export("glorot_normal_initializer", "keras.initializers.glorot_normal",
+           "initializers.glorot_normal")
 def glorot_normal_initializer(seed=None, dtype=dtypes.float32):
   """The Glorot normal initializer, also called Xavier normal initializer.
 
@@ -1181,7 +1183,98 @@ def glorot_normal_initializer(seed=None, dtype=dtypes.float32):
     An initializer.
   """
   return variance_scaling_initializer(
-      scale=1.0, mode="fan_avg", distribution="normal", seed=seed, dtype=dtype)
+      scale=1.0,
+      mode="fan_avg",
+      distribution="truncated_normal",
+      seed=seed,
+      dtype=dtype)
+
+
+@tf_export("keras.initializers.lecun_normal", "initializers.lecun_normal")
+def lecun_normal(seed=None):
+  """LeCun normal initializer.
+
+  It draws samples from a truncated normal distribution centered on 0
+  with `stddev = sqrt(1 / fan_in)`
+  where `fan_in` is the number of input units in the weight tensor.
+
+  Arguments:
+      seed: A Python integer. Used to seed the random generator.
+
+  Returns:
+      An initializer.
+
+  References:
+      - [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515)
+      - [Efficient
+      Backprop](http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf)
+  """
+  return VarianceScaling(
+      scale=1., mode="fan_in", distribution="truncated_normal", seed=seed)
+
+
+@tf_export("keras.initializers.lecun_uniform", "initializers.lecun_uniform")
+def lecun_uniform(seed=None):
+  """LeCun uniform initializer.
+
+  It draws samples from a uniform distribution within [-limit, limit]
+  where `limit` is `sqrt(3 / fan_in)`
+  where `fan_in` is the number of input units in the weight tensor.
+
+  Arguments:
+      seed: A Python integer. Used to seed the random generator.
+
+  Returns:
+      An initializer.
+
+  References:
+      LeCun 98, Efficient Backprop,
+      http://yann.lecun.com/exdb/publis/pdf/lecun-98b.pdf
+  """
+  return VarianceScaling(
+      scale=1., mode="fan_in", distribution="uniform", seed=seed)
+
+
+@tf_export("keras.initializers.he_normal", "initializers.he_normal")
+def he_normal(seed=None):
+  """He normal initializer.
+
+  It draws samples from a truncated normal distribution centered on 0
+  with `stddev = sqrt(2 / fan_in)`
+  where `fan_in` is the number of input units in the weight tensor.
+
+  Arguments:
+      seed: A Python integer. Used to seed the random generator.
+
+  Returns:
+      An initializer.
+
+  References:
+      He et al., http://arxiv.org/abs/1502.01852
+  """
+  return VarianceScaling(
+      scale=2., mode="fan_in", distribution="truncated_normal", seed=seed)
+
+
+@tf_export("keras.initializers.he_uniform", "initializers.he_uniform")
+def he_uniform(seed=None):
+  """He uniform variance scaling initializer.
+
+  It draws samples from a uniform distribution within [-limit, limit]
+  where `limit` is `sqrt(6 / fan_in)`
+  where `fan_in` is the number of input units in the weight tensor.
+
+  Arguments:
+      seed: A Python integer. Used to seed the random generator.
+
+  Returns:
+      An initializer.
+
+  References:
+      He et al., http://arxiv.org/abs/1502.01852
+  """
+  return VarianceScaling(
+      scale=2., mode="fan_in", distribution="uniform", seed=seed)
 
 
 # Utility functions.
diff --git a/tensorflow/python/ops/init_ops_test.py b/tensorflow/python/ops/init_ops_test.py
new file mode 100644
index 0000000000..f6fffa9079
--- /dev/null
+++ b/tensorflow/python/ops/init_ops_test.py
@@ -0,0 +1,196 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for initializers in init_ops."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.platform import test
+
+
+class InitializersTest(test.TestCase):
+
+  def _runner(self,
+              init,
+              shape,
+              target_mean=None,
+              target_std=None,
+              target_max=None,
+              target_min=None):
+    variable = resource_variable_ops.ResourceVariable(init(shape))
+    if context.executing_eagerly():
+      output = variable.numpy()
+    else:
+      sess = ops.get_default_session()
+      sess.run(variable.initializer)
+      output = sess.run(variable)
+    lim = 3e-2
+    if target_std is not None:
+      self.assertGreater(lim, abs(output.std() - target_std))
+    if target_mean is not None:
+      self.assertGreater(lim, abs(output.mean() - target_mean))
+    if target_max is not None:
+      self.assertGreater(lim, abs(output.max() - target_max))
+    if target_min is not None:
+      self.assertGreater(lim, abs(output.min() - target_min))
+
+  def test_uniform(self):
+    tensor_shape = (9, 6, 7)
+    with self.test_session():
+      self._runner(
+          init_ops.RandomUniform(minval=-1, maxval=1, seed=124),
+          tensor_shape,
+          target_mean=0.,
+          target_max=1,
+          target_min=-1)
+
+  def test_normal(self):
+    tensor_shape = (8, 12, 99)
+    with self.test_session():
+      self._runner(
+          init_ops.RandomNormal(mean=0, stddev=1, seed=153),
+          tensor_shape,
+          target_mean=0.,
+          target_std=1)
+
+  def test_truncated_normal(self):
+    tensor_shape = (12, 99, 7)
+    with self.test_session():
+      self._runner(
+          init_ops.TruncatedNormal(mean=0, stddev=1, seed=126),
+          tensor_shape,
+          target_mean=0.,
+          target_max=2,
+          target_min=-2)
+
+  def test_constant(self):
+    tensor_shape = (5, 6, 4)
+    with self.test_session():
+      self._runner(
+          init_ops.Constant(2),
+          tensor_shape,
+          target_mean=2,
+          target_max=2,
+          target_min=2)
+
+  def test_lecun_uniform(self):
+    tensor_shape = (5, 6, 4, 2)
+    with self.test_session():
+      fan_in, _ = init_ops._compute_fans(tensor_shape)
+      std = np.sqrt(1. / fan_in)
+      self._runner(
+          init_ops.lecun_uniform(seed=123),
+          tensor_shape,
+          target_mean=0.,
+          target_std=std)
+
+  def test_glorot_uniform_initializer(self):
+    tensor_shape = (5, 6, 4, 2)
+    with self.test_session():
+      fan_in, fan_out = init_ops._compute_fans(tensor_shape)
+      std = np.sqrt(2. / (fan_in + fan_out))
+      self._runner(
+          init_ops.glorot_uniform_initializer(seed=123),
+          tensor_shape,
+          target_mean=0.,
+          target_std=std)
+
+  def test_he_uniform(self):
+    tensor_shape = (5, 6, 4, 2)
+    with self.test_session():
+      fan_in, _ = init_ops._compute_fans(tensor_shape)
+      std = np.sqrt(2. / fan_in)
+      self._runner(
+          init_ops.he_uniform(seed=123),
+          tensor_shape,
+          target_mean=0.,
+          target_std=std)
+
+  def test_lecun_normal(self):
+    tensor_shape = (5, 6, 4, 2)
+    with self.test_session():
+      fan_in, _ = init_ops._compute_fans(tensor_shape)
+      std = np.sqrt(1. / fan_in)
+      self._runner(
+          init_ops.lecun_normal(seed=123),
+          tensor_shape,
+          target_mean=0.,
+          target_std=std)
+
+  def test_glorot_normal_initializer(self):
+    tensor_shape = (5, 6, 4, 2)
+    with self.test_session():
+      fan_in, fan_out = init_ops._compute_fans(tensor_shape)
+      std = np.sqrt(2. / (fan_in + fan_out))
+      self._runner(
+          init_ops.glorot_normal_initializer(seed=123),
+          tensor_shape,
+          target_mean=0.,
+          target_std=std)
+
+  def test_he_normal(self):
+    tensor_shape = (5, 6, 4, 2)
+    with self.test_session():
+      fan_in, _ = init_ops._compute_fans(tensor_shape)
+      std = np.sqrt(2. / fan_in)
+      self._runner(
+          init_ops.he_normal(seed=123),
+          tensor_shape,
+          target_mean=0.,
+          target_std=std)
+
+  def test_Orthogonal(self):
+    tensor_shape = (20, 20)
+    with self.test_session():
+      self._runner(init_ops.Orthogonal(seed=123), tensor_shape, target_mean=0.)
+
+  def test_Identity(self):
+    with self.test_session():
+      tensor_shape = (3, 4, 5)
+      with self.assertRaises(ValueError):
+        self._runner(
+            init_ops.Identity(),
+            tensor_shape,
+            target_mean=1. / tensor_shape[0],
+            target_max=1.)
+
+      tensor_shape = (3, 3)
+      self._runner(
+          init_ops.Identity(),
+          tensor_shape,
+          target_mean=1. / tensor_shape[0],
+          target_max=1.)
+
+  def test_Zeros(self):
+    tensor_shape = (4, 5)
+    with self.test_session():
+      self._runner(
+          init_ops.Zeros(), tensor_shape, target_mean=0., target_max=0.)
+
+  def test_Ones(self):
+    tensor_shape = (4, 5)
+    with self.test_session():
+      self._runner(init_ops.Ones(), tensor_shape, target_mean=1., target_max=1.)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/tools/api/golden/tensorflow.initializers.pbtxt b/tensorflow/tools/api/golden/tensorflow.initializers.pbtxt
index eaf0036cac..bc0426f2f1 100644
--- a/tensorflow/tools/api/golden/tensorflow.initializers.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.initializers.pbtxt
@@ -44,6 +44,30 @@ tf_module {
     name: "global_variables"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "glorot_normal"
+    argspec: "args=[\'seed\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \"\"], "
+  }
+  member_method {
+    name: "glorot_uniform"
+    argspec: "args=[\'seed\', \'dtype\'], varargs=None, keywords=None, defaults=[\'None\', \"\"], "
+  }
+  member_method {
+    name: "he_normal"
+    argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "he_uniform"
+    argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "lecun_normal"
+    argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "lecun_uniform"
+    argspec: "args=[\'seed\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "local_variables"
     argspec: "args=[], varargs=None, keywords=None, defaults=None"
-- 
cgit v1.2.3


From 2f4044c800a9df60a77f86568feee987e670fec9 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy 
Date: Tue, 17 Jul 2018 16:35:24 -0700
Subject: Only run small and medium tests on windows.

PiperOrigin-RevId: 204996716
---
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 1 +
 tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index ed73401467..61dec249f3 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -111,6 +111,7 @@ bazel test --announce_rc --config=opt -k --test_output=errors \
   --define=no_tensorflow_py_deps=true --test_lang_filters=py \
   --test_tag_filters=-no_pip,-no_windows,-no_oss \
   --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \
+  --test_size_filters=small,medium \
   --jobs="${N_JOBS}" --test_timeout="300,450,1200,3600" \
   --flaky_test_attempts=3 \
   //${PY_TEST_DIR}/tensorflow/python/... \
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index 36b2142d95..e232306653 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -116,6 +116,7 @@ bazel test --announce_rc --config=opt -k --test_output=errors \
   --define=no_tensorflow_py_deps=true --test_lang_filters=py \
   --test_tag_filters=-no_pip,-no_windows,-no_windows_gpu,-no_gpu,-no_pip_gpu,-no_oss \
   --build_tag_filters=-no_pip,-no_windows,-no_windows_gpu,-no_gpu,-no_pip_gpu,-no_oss --build_tests_only \
+  --test_size_filters=small,medium \
   --local_test_jobs=$TF_GPU_COUNT --test_timeout="300,450,1200,3600" \
   --flaky_test_attempts=3 \
   //${PY_TEST_DIR}/tensorflow/python/... \
-- 
cgit v1.2.3


From 60da45fcabb8a1e096cc9b4b5c0d10c7f1ee2a13 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan 
Date: Tue, 17 Jul 2018 16:55:09 -0700
Subject: Support channels_first data format with crossentropy losses

PiperOrigin-RevId: 204999627
---
 tensorflow/python/keras/BUILD                      |  11 ++
 tensorflow/python/keras/backend.py                 |  31 +++--
 tensorflow/python/keras/engine/training.py         |   6 +-
 .../python/keras/engine/training_gpu_test.py       | 125 +++++++++++++++++++++
 tensorflow/python/keras/utils/np_utils.py          |   3 +-
 .../api/golden/tensorflow.keras.backend.pbtxt      |   4 +-
 6 files changed, 169 insertions(+), 11 deletions(-)
 create mode 100644 tensorflow/python/keras/engine/training_gpu_test.py

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 01f1184766..a495d48545 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -704,6 +704,17 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "training_gpu_test",
+    size = "small",
+    srcs = ["engine/training_gpu_test.py"],
+    additional_deps = [
+        ":keras",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+)
+
 py_test(
     name = "imagenet_utils_test",
     size = "small",
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index cb3423598b..333f927d2f 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3458,7 +3458,7 @@ def softsign(x):
 
 
 @tf_export('keras.backend.categorical_crossentropy')
-def categorical_crossentropy(target, output, from_logits=False):
+def categorical_crossentropy(target, output, from_logits=False, axis=-1):
   """Categorical crossentropy between an output tensor and a target tensor.
 
   Arguments:
@@ -3468,28 +3468,33 @@ def categorical_crossentropy(target, output, from_logits=False):
           case `output` is expected to be the logits).
       from_logits: Boolean, whether `output` is the
           result of a softmax, or is a tensor of logits.
+      axis: Int specifying the channels axis. `axis=-1` corresponds to data
+          format `channels_last', and `axis=1` corresponds to data format
+          `channels_first`.
 
   Returns:
       Output tensor.
+
+  Raises:
+      ValueError: if `axis` is neither -1 nor one of the axes of `output`.
   """
+  rank = len(output.get_shape())
+  axis = axis % rank
   # Note: nn.softmax_cross_entropy_with_logits_v2
   # expects logits, Keras expects probabilities.
   if not from_logits:
     # scale preds so that the class probas of each sample sum to 1
-    output = output / math_ops.reduce_sum(  # pylint: disable=g-no-augmented-assignment
-        output, len(output.get_shape()) - 1, True)
+    output = output / math_ops.reduce_sum(output, axis, True)
     # manual computation of crossentropy
     epsilon_ = _to_tensor(epsilon(), output.dtype.base_dtype)
     output = clip_ops.clip_by_value(output, epsilon_, 1. - epsilon_)
-    return -math_ops.reduce_sum(
-        target * math_ops.log(output),
-        axis=len(output.get_shape()) - 1)
+    return -math_ops.reduce_sum(target * math_ops.log(output), axis)
   else:
     return nn.softmax_cross_entropy_with_logits_v2(labels=target, logits=output)
 
 
 @tf_export('keras.backend.sparse_categorical_crossentropy')
-def sparse_categorical_crossentropy(target, output, from_logits=False):
+def sparse_categorical_crossentropy(target, output, from_logits=False, axis=-1):
   """Categorical crossentropy with integer targets.
 
   Arguments:
@@ -3499,10 +3504,22 @@ def sparse_categorical_crossentropy(target, output, from_logits=False):
           case `output` is expected to be the logits).
       from_logits: Boolean, whether `output` is the
           result of a softmax, or is a tensor of logits.
+      axis: Int specifying the channels axis. `axis=-1` corresponds to data
+          format `channels_last', and `axis=1` corresponds to data format
+          `channels_first`.
 
   Returns:
       Output tensor.
+
+  Raises:
+      ValueError: if `axis` is neither -1 nor one of the axes of `output`.
   """
+  rank = len(output.get_shape())
+  axis = axis % rank
+  if axis != rank - 1:
+    permutation = list(range(axis)) + list(range(axis + 1, rank)) + [axis]
+    output = array_ops.transpose(output, perm=permutation)
+
   # Note: nn.sparse_softmax_cross_entropy_with_logits
   # expects logits, Keras expects probabilities.
   if not from_logits:
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index bd03f4871f..573422e533 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -897,7 +897,11 @@ class Model(Network):
         for output_shape, loss_fn in zip(self._feed_output_shapes,
                                          self._feed_loss_fns):
           if loss_fn is losses.sparse_categorical_crossentropy:
-            feed_output_shapes.append(output_shape[:-1] + (1,))
+            if K.image_data_format() == 'channels_first':
+              feed_output_shapes.append(
+                  (output_shape[0], 1) + output_shape[2:])
+            else:
+              feed_output_shapes.append(output_shape[:-1] + (1,))
           elif (not hasattr(loss_fn, '__name__') or
                 getattr(losses, loss_fn.__name__, None) is None):
             # If `loss_fn` is not a function (e.g. callable class)
diff --git a/tensorflow/python/keras/engine/training_gpu_test.py b/tensorflow/python/keras/engine/training_gpu_test.py
new file mode 100644
index 0000000000..5825ce814f
--- /dev/null
+++ b/tensorflow/python/keras/engine/training_gpu_test.py
@@ -0,0 +1,125 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for training routines."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python import keras
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras import backend as K
+from tensorflow.python.keras.layers.convolutional import Conv2D
+from tensorflow.python.platform import test
+from tensorflow.python.training import rmsprop
+
+
+class TrainingGPUTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_model_with_crossentropy_losses_channels_first(self):
+    """Tests use of all crossentropy losses with `channels_first`.
+
+    Tests `sparse_categorical_crossentropy`, `categorical_crossentropy`,
+    and `binary_crossentropy`.
+    Verifies that evaluate gives the same result with either `channels_first`
+    or `channels_last` image_data_format.
+    """
+    def prepare_simple_model(input_tensor, loss_name, target):
+      axis = 1 if K.image_data_format() == 'channels_first' else -1
+      loss = None
+      num_channels = None
+      activation = None
+      if loss_name == 'sparse_categorical_crossentropy':
+        loss = lambda y_true, y_pred: K.sparse_categorical_crossentropy(  # pylint: disable=g-long-lambda
+            y_true, y_pred, axis=axis)
+        num_channels = np.amax(target) + 1
+        activation = 'softmax'
+      elif loss_name == 'categorical_crossentropy':
+        loss = lambda y_true, y_pred: K.categorical_crossentropy(  # pylint: disable=g-long-lambda
+            y_true, y_pred, axis=axis)
+        num_channels = target.shape[axis]
+        activation = 'softmax'
+      elif loss_name == 'binary_crossentropy':
+        loss = lambda y_true, y_pred: K.binary_crossentropy(y_true, y_pred)  # pylint: disable=unnecessary-lambda
+        num_channels = target.shape[axis]
+        activation = 'sigmoid'
+      predictions = Conv2D(num_channels,
+                           1,
+                           activation=activation,
+                           kernel_initializer='ones',
+                           bias_initializer='ones')(input_tensor)
+      simple_model = keras.models.Model(inputs=input_tensor,
+                                        outputs=predictions)
+      simple_model.compile(optimizer=rmsprop.RMSPropOptimizer(1e-3), loss=loss)
+      return simple_model
+
+    if test.is_gpu_available(cuda_only=True):
+      with self.test_session(use_gpu=True):
+        losses_to_test = ['sparse_categorical_crossentropy',
+                          'categorical_crossentropy', 'binary_crossentropy']
+
+        data_channels_first = np.array([[[[8., 7.1, 0.], [4.5, 2.6, 0.55],
+                                          [0.9, 4.2, 11.2]]]], dtype=np.float32)
+        # Labels for testing 4-class sparse_categorical_crossentropy, 4-class
+        # categorical_crossentropy, and 2-class binary_crossentropy:
+        labels_channels_first = [np.array([[[[0, 1, 3], [2, 1, 0], [2, 2, 1]]]], dtype=np.float32),  # pylint: disable=line-too-long
+                                 np.array([[[[0, 1, 0], [0, 1, 0], [0, 0, 0]],
+                                            [[1, 0, 0], [0, 0, 1], [0, 1, 0]],
+                                            [[0, 0, 0], [1, 0, 0], [0, 0, 1]],
+                                            [[0, 0, 1], [0, 0, 0], [1, 0, 0]]]], dtype=np.float32),  # pylint: disable=line-too-long
+                                 np.array([[[[0, 1, 0], [0, 1, 0], [0, 0, 1]],
+                                            [[1, 0, 1], [1, 0, 1], [1, 1, 0]]]], dtype=np.float32)]  # pylint: disable=line-too-long
+        # Compute one loss for each loss function in the list `losses_to_test`:
+        loss_channels_last = [0., 0., 0.]
+        loss_channels_first = [0., 0., 0.]
+
+        old_data_format = K.image_data_format()
+
+        # Evaluate a simple network with channels last, with all three loss
+        # functions:
+        K.set_image_data_format('channels_last')
+        data = np.moveaxis(data_channels_first, 1, -1)
+        for index, loss_function in enumerate(losses_to_test):
+          labels = np.moveaxis(labels_channels_first[index], 1, -1)
+          inputs = keras.Input(shape=(3, 3, 1))
+          model = prepare_simple_model(inputs, loss_function, labels)
+          loss_channels_last[index] = model.evaluate(x=data, y=labels,
+                                                     batch_size=1, verbose=0)
+
+        # Evaluate the same network with channels first, with all three loss
+        # functions:
+        K.set_image_data_format('channels_first')
+        data = data_channels_first
+        for index, loss_function in enumerate(losses_to_test):
+          labels = labels_channels_first[index]
+          inputs = keras.Input(shape=(1, 3, 3))
+          model = prepare_simple_model(inputs, loss_function, labels)
+          loss_channels_first[index] = model.evaluate(x=data, y=labels,
+                                                      batch_size=1, verbose=0)
+
+        K.set_image_data_format(old_data_format)
+
+        np.testing.assert_allclose(loss_channels_first,
+                                   loss_channels_last,
+                                   err_msg='{}{}'.format(
+                                       'Computed different losses for ',
+                                       'channels_first and channels_last'))
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/utils/np_utils.py b/tensorflow/python/keras/utils/np_utils.py
index 9d9c72b162..c24e87308b 100644
--- a/tensorflow/python/keras/utils/np_utils.py
+++ b/tensorflow/python/keras/utils/np_utils.py
@@ -33,7 +33,8 @@ def to_categorical(y, num_classes=None):
       num_classes: total number of classes.
 
   Returns:
-      A binary matrix representation of the input.
+      A binary matrix representation of the input. The classes axis is placed
+      last.
   """
   y = np.array(y, dtype='int')
   input_shape = y.shape
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt
index c6149e8aa7..fddac63b78 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt
@@ -70,7 +70,7 @@ tf_module {
   }
   member_method {
     name: "categorical_crossentropy"
-    argspec: "args=[\'target\', \'output\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'target\', \'output\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'False\', \'-1\'], "
   }
   member_method {
     name: "clear_session"
@@ -462,7 +462,7 @@ tf_module {
   }
   member_method {
     name: "sparse_categorical_crossentropy"
-    argspec: "args=[\'target\', \'output\', \'from_logits\'], varargs=None, keywords=None, defaults=[\'False\'], "
+    argspec: "args=[\'target\', \'output\', \'from_logits\', \'axis\'], varargs=None, keywords=None, defaults=[\'False\', \'-1\'], "
   }
   member_method {
     name: "spatial_2d_padding"
-- 
cgit v1.2.3


From 36a66347e8e344cddee4a8d9123ccbcae40011b1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 17:04:15 -0700
Subject: Error on some documented invalid Cudnn inputs. Cudnn should have
 returned errors, but crashes instead.

PiperOrigin-RevId: 205000883
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 9e24a4538c..08228034f7 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -2406,6 +2406,19 @@ port::Status CudnnSupport::DoConvolveImpl(
                           stream, cudnn, algorithm_config, input_nd, filter,
                           conv, output_nd, scratch_allocator, &scratch));
 
+  if (cudnn_type == CUDNN_DATA_HALF &&
+      filter_descriptor.layout() == dnn::FilterLayout::kOutputYXInput &&
+      (algo_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM ||
+       input_descriptor.layout() != dnn::DataLayout::kBatchYXDepth ||
+       output_descriptor.layout() != dnn::DataLayout::kBatchYXDepth)) {
+    // TODO(timshen): Attach a nvbugs number.
+    return port::Status(
+        port::error::INTERNAL,
+        "Cudnn doesn't return an error code on this documented unsupported "
+        "layout combination. Instead, it accesses out-of-bounds memory. "
+        "Being nice and returning an error instead.");
+  }
+
   std::unique_ptr timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -3074,9 +3087,21 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     }
   }
 
+  if (cudnn_type == CUDNN_DATA_HALF &&
+      filter_descriptor.layout() == dnn::FilterLayout::kOutputYXInput &&
+      ((algo_desc.algo_id() != CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0 &&
+        algo_desc.algo_id() != CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1) ||
+       input_descriptor.layout() != dnn::DataLayout::kBatchYXDepth ||
+       output_descriptor.layout() != dnn::DataLayout::kBatchYXDepth)) {
+    return port::Status(
+        port::error::INTERNAL,
+        "Cudnn doesn't return an error code on this documented unsupported "
+        "layout combination. Instead, it crashes. Being nice and returning an "
+        "error instead. See nvbugs/2260917");
+  }
+
   // Cudnn 7.1.4 has a bug if the workspace of the following convolution is not
-  // zero-initialized.
-  // TODO(timshen): Add an nvbugs/ link.
+  // zero-initialized. See nvbugs/2254619.
   if (CUDNN_VERSION >= 7000 &&
       algorithm_config.algorithm().algo_id() ==
           CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
-- 
cgit v1.2.3


From 0c656ad9d7de201a7280c9b4ed561c40cbd9c5f3 Mon Sep 17 00:00:00 2001
From: Rachel Lim 
Date: Tue, 17 Jul 2018 17:14:16 -0700
Subject: Automated rollback of commit 0ba51c741981c4f264dc06356a44b89ab9dbacd1

PiperOrigin-RevId: 205002413
---
 tensorflow/contrib/data/kernels/csv_dataset_op.cc  |  53 +-
 tensorflow/contrib/data/ops/dataset_ops.cc         |  12 +-
 .../python/kernel_tests/csv_dataset_op_test.py     | 143 ++--
 .../python/kernel_tests/reader_dataset_ops_test.py | 875 ++++++++++++---------
 tensorflow/contrib/data/python/ops/readers.py      |  17 +-
 .../contrib/makefile/proto_text_cc_files.txt       |   1 +
 tensorflow/core/lib/io/zlib_compression_options.cc |  32 +
 tensorflow/core/lib/io/zlib_compression_options.h  |  19 +-
 tensorflow/core/lib/io/zlib_inputstream.cc         | 109 ++-
 tensorflow/core/lib/io/zlib_inputstream.h          |  27 +-
 10 files changed, 770 insertions(+), 518 deletions(-)
 create mode 100644 tensorflow/core/lib/io/zlib_compression_options.cc

diff --git a/tensorflow/contrib/data/kernels/csv_dataset_op.cc b/tensorflow/contrib/data/kernels/csv_dataset_op.cc
index 4657807785..dadde705e1 100644
--- a/tensorflow/contrib/data/kernels/csv_dataset_op.cc
+++ b/tensorflow/contrib/data/kernels/csv_dataset_op.cc
@@ -18,7 +18,10 @@ limitations under the License.
 #include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/lib/io/inputstream_interface.h"
 #include "tensorflow/core/lib/io/random_inputstream.h"
+#include "tensorflow/core/lib/io/zlib_compression_options.h"
+#include "tensorflow/core/lib/io/zlib_inputstream.h"
 
 namespace tensorflow {
 namespace {
@@ -37,6 +40,10 @@ class CSVDatasetOp : public DatasetOpKernel {
         ctx, filenames_tensor->dims() <= 1,
         errors::InvalidArgument("`filenames` must be a scalar or a vector."));
 
+    string compression_type;
+    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "compression_type",
+                                                    &compression_type));
+
     OpInputList record_defaults_list;
     OP_REQUIRES_OK(ctx,
                    ctx->input_list("record_defaults", &record_defaults_list));
@@ -86,6 +93,19 @@ class CSVDatasetOp : public DatasetOpKernel {
       filenames.push_back(filenames_tensor->flat()(i));
     }
 
+    io::ZlibCompressionOptions zlib_compression_options =
+        io::ZlibCompressionOptions::DEFAULT();
+    if (compression_type == "ZLIB") {
+      zlib_compression_options = io::ZlibCompressionOptions::DEFAULT();
+    } else if (compression_type == "GZIP") {
+      zlib_compression_options = io::ZlibCompressionOptions::GZIP();
+    } else {
+      OP_REQUIRES(ctx, compression_type.empty(),
+                  errors::InvalidArgument(
+                      "Unsupported compression_type: ", compression_type, "."));
+    }
+    zlib_compression_options.input_buffer_size = buffer_size;
+
     std::vector select_cols;
     select_cols.reserve(select_cols_tensor->NumElements());
     for (int i = 0; i < select_cols_tensor->NumElements(); ++i) {
@@ -103,7 +123,8 @@ class CSVDatasetOp : public DatasetOpKernel {
         ctx, select_cols.empty() || select_cols.front() >= 0,
         errors::InvalidArgument("select_cols should be non-negative indices"));
 
-    *output = new Dataset(ctx, std::move(filenames), header, buffer_size,
+    *output = new Dataset(ctx, std::move(filenames), header,
+                          std::move(compression_type), zlib_compression_options,
                           output_types_, output_shapes_,
                           std::move(record_defaults), std::move(select_cols),
                           use_quote_delim, delim[0], std::move(na_value));
@@ -113,21 +134,23 @@ class CSVDatasetOp : public DatasetOpKernel {
   class Dataset : public GraphDatasetBase {
    public:
     Dataset(OpKernelContext* ctx, std::vector filenames, bool header,
-            int64 buffer_size, const DataTypeVector& output_types,
+            string compression_type, io::ZlibCompressionOptions options,
+            const DataTypeVector& output_types,
             const std::vector& output_shapes,
             std::vector record_defaults, std::vector select_cols,
             bool use_quote_delim, char delim, string na_value)
         : GraphDatasetBase(ctx),
           filenames_(std::move(filenames)),
           header_(header),
-          buffer_size_(buffer_size),
           out_type_(output_types),
           output_shapes_(output_shapes),
           record_defaults_(std::move(record_defaults)),
           select_cols_(std::move(select_cols)),
           use_quote_delim_(use_quote_delim),
           delim_(delim),
-          na_value_(std::move(na_value)) {}
+          na_value_(std::move(na_value)),
+          use_compression_(!compression_type.empty()),
+          options_(options) {}
 
     std::unique_ptr MakeIteratorInternal(
         const string& prefix) const override {
@@ -510,7 +533,8 @@ class CSVDatasetOp : public DatasetOpKernel {
 
       Status FillBuffer(string* result) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
         result->clear();
-        Status s = input_stream_->ReadNBytes(dataset()->buffer_size_, result);
+        Status s = input_stream_->ReadNBytes(
+            dataset()->options_.input_buffer_size, result);
 
         if (errors::IsOutOfRange(s) && !result->empty()) {
           // Ignore OutOfRange error when ReadNBytes read < N bytes.
@@ -675,8 +699,17 @@ class CSVDatasetOp : public DatasetOpKernel {
         // Actually move on to next file.
         TF_RETURN_IF_ERROR(env->NewRandomAccessFile(
             dataset()->filenames_[current_file_index_], &file_));
-        input_stream_.reset(
-            new io::RandomAccessInputStream(file_.get(), false));
+        random_access_input_stream_ =
+            std::make_shared(file_.get(), false);
+
+        if (dataset()->use_compression_) {
+          input_stream_ = std::make_shared(
+              random_access_input_stream_.get(),
+              dataset()->options_.input_buffer_size,
+              dataset()->options_.input_buffer_size, dataset()->options_);
+        } else {
+          input_stream_ = random_access_input_stream_;
+        }
         buffer_.clear();
         pos_ = 0;
         if (dataset()->header_) {
@@ -704,8 +737,9 @@ class CSVDatasetOp : public DatasetOpKernel {
       string buffer_ GUARDED_BY(mu_);  // Maintain our own buffer
       size_t pos_ GUARDED_BY(
           mu_);  // Index into the buffer must be maintained between iters
-      std::unique_ptr input_stream_
+      std::shared_ptr random_access_input_stream_
           GUARDED_BY(mu_);
+      std::shared_ptr input_stream_ GUARDED_BY(mu_);
       size_t current_file_index_ GUARDED_BY(mu_) = 0;
       std::unique_ptr file_
           GUARDED_BY(mu_);  // must outlive input_stream_
@@ -713,7 +747,6 @@ class CSVDatasetOp : public DatasetOpKernel {
 
     const std::vector filenames_;
     const bool header_;
-    const int64 buffer_size_;
     const DataTypeVector out_type_;
     const std::vector output_shapes_;
     const std::vector record_defaults_;
@@ -721,6 +754,8 @@ class CSVDatasetOp : public DatasetOpKernel {
     const bool use_quote_delim_;
     const char delim_;
     const string na_value_;
+    const bool use_compression_;
+    const io::ZlibCompressionOptions options_;
   };  // class Dataset
 
   DataTypeVector output_types_;
diff --git a/tensorflow/contrib/data/ops/dataset_ops.cc b/tensorflow/contrib/data/ops/dataset_ops.cc
index 8413fcaf87..a623c27ff8 100644
--- a/tensorflow/contrib/data/ops/dataset_ops.cc
+++ b/tensorflow/contrib/data/ops/dataset_ops.cc
@@ -36,6 +36,7 @@ data_input_datasets: `N` datasets with the same type that will be interleaved
 
 REGISTER_OP("CSVDataset")
     .Input("filenames: string")
+    .Input("compression_type: string")
     .Input("buffer_size: int64")
     .Input("header: bool")
     .Input("field_delim: string")
@@ -52,17 +53,18 @@ REGISTER_OP("CSVDataset")
       shape_inference::ShapeHandle unused;
       // `filenames` must be a scalar or a vector.
       TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(0), 1, &unused));
-      // `buffer_size`, `header`, `field_delim`, `use_quote_delim`,
-      // `na_value` must be scalars
+      // `compression_type`, `buffer_size`, `header`, `field_delim`,
+      // `use_quote_delim`, `na_value` must be scalars
       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(6), 0, &unused));
       // `select_cols` must be a vector
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(6), 1, &unused));
-      // `record_defaults` must be a list of scalars...?
-      for (size_t i = 7; i < c->num_inputs(); ++i) {
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(7), 1, &unused));
+      // `record_defaults` must be lists of scalars
+      for (size_t i = 8; i < c->num_inputs(); ++i) {
         TF_RETURN_IF_ERROR(c->WithRank(c->input(i), 1, &unused));
       }
       return shape_inference::ScalarShape(c);
diff --git a/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py
index df115175f5..2a0e64caeb 100644
--- a/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/csv_dataset_op_test.py
@@ -18,10 +18,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import gzip
 import os
 import string
 import tempfile
 import time
+import zlib
 
 import numpy as np
 
@@ -62,18 +64,29 @@ class CsvDatasetOpTest(test.TestCase):
         op2 = sess.run(next2)
         self.assertAllEqual(op1, op2)
 
-  def setup_files(self, inputs, linebreak='\n'):
+  def _setup_files(self, inputs, linebreak='\n', compression_type=None):
     filenames = []
     for i, ip in enumerate(inputs):
       fn = os.path.join(self.get_temp_dir(), 'temp_%d.csv' % i)
-      with open(fn, 'wb') as f:
-        f.write(linebreak.join(ip).encode('utf-8'))
+      contents = linebreak.join(ip).encode('utf-8')
+      if compression_type is None:
+        with open(fn, 'wb') as f:
+          f.write(contents)
+      elif compression_type == 'GZIP':
+        with gzip.GzipFile(fn, 'wb') as f:
+          f.write(contents)
+      elif compression_type == 'ZLIB':
+        contents = zlib.compress(contents)
+        with open(fn, 'wb') as f:
+          f.write(contents)
+      else:
+        raise ValueError('Unsupported compression_type', compression_type)
       filenames.append(fn)
     return filenames
 
   def _make_test_datasets(self, inputs, **kwargs):
     # Test by comparing its output to what we could get with map->decode_csv
-    filenames = self.setup_files(inputs)
+    filenames = self._setup_files(inputs)
     dataset_expected = core_readers.TextLineDataset(filenames)
     dataset_expected = dataset_expected.map(
         lambda l: parsing_ops.decode_csv(l, **kwargs))
@@ -112,15 +125,18 @@ class CsvDatasetOpTest(test.TestCase):
           except errors.OutOfRangeError:
             break
 
-  def _test_dataset(self,
-                    inputs,
-                    expected_output=None,
-                    expected_err_re=None,
-                    linebreak='\n',
-                    **kwargs):
+  def _test_dataset(
+      self,
+      inputs,
+      expected_output=None,
+      expected_err_re=None,
+      linebreak='\n',
+      compression_type=None,  # Used for both setup and parsing
+      **kwargs):
     """Checks that elements produced by CsvDataset match expected output."""
     # Convert str type because py3 tf strings are bytestrings
-    filenames = self.setup_files(inputs, linebreak)
+    filenames = self._setup_files(inputs, linebreak, compression_type)
+    kwargs['compression_type'] = compression_type
     with ops.Graph().as_default() as g:
       with self.test_session(graph=g) as sess:
         dataset = readers.CsvDataset(filenames, **kwargs)
@@ -174,7 +190,7 @@ class CsvDatasetOpTest(test.TestCase):
   def testCsvDataset_ignoreErrWithUnescapedQuotes(self):
     record_defaults = [['']] * 3
     inputs = [['1,"2"3",4', '1,"2"3",4",5,5', 'a,b,"c"d"', 'e,f,g']]
-    filenames = self.setup_files(inputs)
+    filenames = self._setup_files(inputs)
     with ops.Graph().as_default() as g:
       with self.test_session(graph=g) as sess:
         dataset = readers.CsvDataset(filenames, record_defaults=record_defaults)
@@ -184,7 +200,7 @@ class CsvDatasetOpTest(test.TestCase):
   def testCsvDataset_ignoreErrWithUnquotedQuotes(self):
     record_defaults = [['']] * 3
     inputs = [['1,2"3,4', 'a,b,c"d', '9,8"7,6,5', 'e,f,g']]
-    filenames = self.setup_files(inputs)
+    filenames = self._setup_files(inputs)
     with ops.Graph().as_default() as g:
       with self.test_session(graph=g) as sess:
         dataset = readers.CsvDataset(filenames, record_defaults=record_defaults)
@@ -355,7 +371,7 @@ class CsvDatasetOpTest(test.TestCase):
         '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19',
         '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19'
     ]]
-    file_path = self.setup_files(data)
+    file_path = self._setup_files(data)
 
     with ops.Graph().as_default() as g:
       ds = readers.make_csv_dataset(
@@ -432,14 +448,29 @@ class CsvDatasetOpTest(test.TestCase):
         record_defaults=record_defaults,
         buffer_size=0)
 
-  def testCsvDataset_withBufferSize(self):
+  def _test_dataset_on_buffer_sizes(self,
+                                    inputs,
+                                    expected,
+                                    linebreak,
+                                    record_defaults,
+                                    compression_type=None,
+                                    num_sizes_to_test=20):
+    # Testing reading with a range of buffer sizes that should all work.
+    for i in list(range(1, 1 + num_sizes_to_test)) + [None]:
+      self._test_dataset(
+          inputs,
+          expected,
+          linebreak=linebreak,
+          compression_type=compression_type,
+          record_defaults=record_defaults,
+          buffer_size=i)
+
+  def testCsvDataset_withLF(self):
     record_defaults = [['NA']] * 3
     inputs = [['abc,def,ghi', '0,1,2', ',,']]
     expected = [['abc', 'def', 'ghi'], ['0', '1', '2'], ['NA', 'NA', 'NA']]
-    for i in range(20):
-      # Test a range of buffer sizes that should all work
-      self._test_dataset(
-          inputs, expected, record_defaults=record_defaults, buffer_size=i + 1)
+    self._test_dataset_on_buffer_sizes(
+        inputs, expected, linebreak='\n', record_defaults=record_defaults)
 
   def testCsvDataset_withCR(self):
     # Test that when the line separator is '\r', parsing works with all buffer
@@ -447,14 +478,8 @@ class CsvDatasetOpTest(test.TestCase):
     record_defaults = [['NA']] * 3
     inputs = [['abc,def,ghi', '0,1,2', ',,']]
     expected = [['abc', 'def', 'ghi'], ['0', '1', '2'], ['NA', 'NA', 'NA']]
-    for i in range(20):
-      # Test a range of buffer sizes that should all work
-      self._test_dataset(
-          inputs,
-          expected,
-          linebreak='\r',
-          record_defaults=record_defaults,
-          buffer_size=i + 1)
+    self._test_dataset_on_buffer_sizes(
+        inputs, expected, linebreak='\r', record_defaults=record_defaults)
 
   def testCsvDataset_withCRLF(self):
     # Test that when the line separator is '\r\n', parsing works with all buffer
@@ -462,29 +487,15 @@ class CsvDatasetOpTest(test.TestCase):
     record_defaults = [['NA']] * 3
     inputs = [['abc,def,ghi', '0,1,2', ',,']]
     expected = [['abc', 'def', 'ghi'], ['0', '1', '2'], ['NA', 'NA', 'NA']]
-    for i in range(20):
-      # Test a range of buffer sizes that should all work
-      self._test_dataset(
-          inputs,
-          expected,
-          linebreak='\r\n',
-          record_defaults=record_defaults,
-          buffer_size=i + 1)
+    self._test_dataset_on_buffer_sizes(
+        inputs, expected, linebreak='\r\n', record_defaults=record_defaults)
 
   def testCsvDataset_withBufferSizeAndQuoted(self):
     record_defaults = [['NA']] * 3
     inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']]
     expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'],
                 ['NA', 'NA', 'NA']]
-    for i in range(20):
-      # Test a range of buffer sizes that should all work
-      self._test_dataset(
-          inputs,
-          expected,
-          linebreak='\n',
-          record_defaults=record_defaults,
-          buffer_size=i + 1)
-    self._test_dataset(
+    self._test_dataset_on_buffer_sizes(
         inputs, expected, linebreak='\n', record_defaults=record_defaults)
 
   def testCsvDataset_withCRAndQuoted(self):
@@ -494,15 +505,7 @@ class CsvDatasetOpTest(test.TestCase):
     inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']]
     expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'],
                 ['NA', 'NA', 'NA']]
-    for i in range(20):
-      # Test a range of buffer sizes that should all work
-      self._test_dataset(
-          inputs,
-          expected,
-          linebreak='\r',
-          record_defaults=record_defaults,
-          buffer_size=i + 1)
-    self._test_dataset(
+    self._test_dataset_on_buffer_sizes(
         inputs, expected, linebreak='\r', record_defaults=record_defaults)
 
   def testCsvDataset_withCRLFAndQuoted(self):
@@ -512,17 +515,33 @@ class CsvDatasetOpTest(test.TestCase):
     inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']]
     expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'],
                 ['NA', 'NA', 'NA']]
-    for i in range(20):
-      # Test a range of buffer sizes that should all work
-      self._test_dataset(
-          inputs,
-          expected,
-          linebreak='\r\n',
-          record_defaults=record_defaults,
-          buffer_size=i + 1)
-    self._test_dataset(
+    self._test_dataset_on_buffer_sizes(
         inputs, expected, linebreak='\r\n', record_defaults=record_defaults)
 
+  def testCsvDataset_withGzipCompressionType(self):
+    record_defaults = [['NA']] * 3
+    inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']]
+    expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'],
+                ['NA', 'NA', 'NA']]
+    self._test_dataset_on_buffer_sizes(
+        inputs,
+        expected,
+        linebreak='\r\n',
+        compression_type='GZIP',
+        record_defaults=record_defaults)
+
+  def testCsvDataset_withZlibCompressionType(self):
+    record_defaults = [['NA']] * 3
+    inputs = [['"\n\n\n","\r\r\r","abc"', '"0","1","2"', '"","",""']]
+    expected = [['\n\n\n', '\r\r\r', 'abc'], ['0', '1', '2'],
+                ['NA', 'NA', 'NA']]
+    self._test_dataset_on_buffer_sizes(
+        inputs,
+        expected,
+        linebreak='\r\n',
+        compression_type='ZLIB',
+        record_defaults=record_defaults)
+
 
 class CsvDatasetBenchmark(test.Benchmark):
   """Benchmarks for the various ways of creating a dataset from CSV files.
diff --git a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
index 9df403ef50..851a33dfc8 100644
--- a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
@@ -17,13 +17,16 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import gzip
 import os
+import zlib
 
 import numpy as np
 
 from tensorflow.contrib.data.python.kernel_tests import reader_dataset_ops_test_base
 from tensorflow.contrib.data.python.ops import readers
 from tensorflow.python.data.ops import readers as core_readers
+from tensorflow.python.data.util import nest
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -182,264 +185,363 @@ class ReadBatchFeaturesTest(
 
 class MakeCsvDatasetTest(test.TestCase):
 
-  COLUMN_TYPES = [
-      dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64, dtypes.string
-  ]
-  COLUMNS = ["col%d" % i for i in range(len(COLUMN_TYPES))]
-  DEFAULT_VALS = [[], [], [], [], ["NULL"]]
-  DEFAULTS = [
-      constant_op.constant([], dtype=dtypes.int32),
-      constant_op.constant([], dtype=dtypes.int64),
-      constant_op.constant([], dtype=dtypes.float32),
-      constant_op.constant([], dtype=dtypes.float64),
-      constant_op.constant(["NULL"], dtype=dtypes.string)
-  ]
-  LABEL = COLUMNS[0]
-
-  def setUp(self):
-    super(MakeCsvDatasetTest, self).setUp()
-    self._num_files = 2
-    self._num_records = 11
-    self._test_filenames = self._create_files()
-
-  def _csv_values(self, fileno, recordno):
-    return [
-        fileno,
-        recordno,
-        fileno * recordno * 0.5,
-        fileno * recordno + 0.5,
-        "record %d" % recordno if recordno % 2 == 1 else "",
-    ]
+  def _make_csv_dataset(self, filenames, batch_size, num_epochs=1, **kwargs):
+    return readers.make_csv_dataset(
+        filenames, batch_size=batch_size, num_epochs=num_epochs, **kwargs)
 
-  def _write_file(self, filename, rows):
-    for i in range(len(rows)):
-      if isinstance(rows[i], list):
-        rows[i] = ",".join(str(v) if v is not None else "" for v in rows[i])
-    fn = os.path.join(self.get_temp_dir(), filename)
-    f = open(fn, "w")
-    f.write("\n".join(rows))
-    f.close()
-    return fn
-
-  def _create_file(self, fileno, header=True):
-    rows = []
-    if header:
-      rows.append(self.COLUMNS)
-    for recno in range(self._num_records):
-      rows.append(self._csv_values(fileno, recno))
-    return self._write_file("csv_file%d.csv" % fileno, rows)
-
-  def _create_files(self):
+  def _setup_files(self, inputs, linebreak="\n", compression_type=None):
     filenames = []
-    for i in range(self._num_files):
-      filenames.append(self._create_file(i))
+    for i, ip in enumerate(inputs):
+      fn = os.path.join(self.get_temp_dir(), "temp_%d.csv" % i)
+      contents = linebreak.join(ip).encode("utf-8")
+      if compression_type is None:
+        with open(fn, "wb") as f:
+          f.write(contents)
+      elif compression_type == "GZIP":
+        with gzip.GzipFile(fn, "wb") as f:
+          f.write(contents)
+      elif compression_type == "ZLIB":
+        contents = zlib.compress(contents)
+        with open(fn, "wb") as f:
+          f.write(contents)
+      else:
+        raise ValueError("Unsupported compression_type", compression_type)
+      filenames.append(fn)
     return filenames
 
-  def _make_csv_dataset(
-      self,
-      filenames,
-      defaults,
-      column_names=COLUMNS,
-      label_name=LABEL,
-      select_cols=None,
-      batch_size=1,
-      num_epochs=1,
-      shuffle=False,
-      shuffle_seed=None,
-      header=True,
-      na_value="",
-  ):
-    return readers.make_csv_dataset(
-        filenames,
-        batch_size=batch_size,
-        column_names=column_names,
-        column_defaults=defaults,
-        label_name=label_name,
-        num_epochs=num_epochs,
-        shuffle=shuffle,
-        shuffle_seed=shuffle_seed,
-        header=header,
-        na_value=na_value,
-        select_columns=select_cols,
-    )
-
-  def _next_actual_batch(self, file_indices, batch_size, num_epochs, defaults):
-    features = {col: list() for col in self.COLUMNS}
+  def _next_expected_batch(self, expected_output, expected_keys, batch_size,
+                           num_epochs):
+    features = {k: [] for k in expected_keys}
     for _ in range(num_epochs):
-      for i in file_indices:
-        for j in range(self._num_records):
-          values = self._csv_values(i, j)
-          for n, v in enumerate(values):
-            if v == "":  # pylint: disable=g-explicit-bool-comparison
-              values[n] = defaults[n][0]
-          values[-1] = values[-1].encode("utf-8")
-
-          # Regroup lists by column instead of row
-          for n, col in enumerate(self.COLUMNS):
-            features[col].append(values[n])
-          if len(list(features.values())[0]) == batch_size:
-            yield features
-            features = {col: list() for col in self.COLUMNS}
-
-  def _run_actual_batch(self, outputs, sess):
-    features, labels = sess.run(outputs)
-    batch = [features[k] for k in self.COLUMNS if k != self.LABEL]
-    batch.append(labels)
-    return batch
-
-  def _verify_records(
+      for values in expected_output:
+        for n, key in enumerate(expected_keys):
+          features[key].append(values[n])
+        if len(features[expected_keys[0]]) == batch_size:
+          yield features
+          features = {k: [] for k in expected_keys}
+    if features[expected_keys[0]]:  # Leftover from the last batch
+      yield features
+
+  def _verify_output(
       self,
       sess,
       dataset,
-      file_indices,
-      defaults=tuple(DEFAULT_VALS),
-      label_name=LABEL,
-      batch_size=1,
-      num_epochs=1,
+      batch_size,
+      num_epochs,
+      label_name,
+      expected_output,
+      expected_keys,
   ):
-    iterator = dataset.make_one_shot_iterator()
-    get_next = iterator.get_next()
+    nxt = dataset.make_one_shot_iterator().get_next()
 
-    for expected_features in self._next_actual_batch(file_indices, batch_size,
-                                                     num_epochs, defaults):
-      actual_features = sess.run(get_next)
+    for expected_features in self._next_expected_batch(
+        expected_output,
+        expected_keys,
+        batch_size,
+        num_epochs,
+    ):
+      actual_features = sess.run(nxt)
 
       if label_name is not None:
         expected_labels = expected_features.pop(label_name)
-        # Compare labels
         self.assertAllEqual(expected_labels, actual_features[1])
-        actual_features = actual_features[0]  # Extract features dict from tuple
+        actual_features = actual_features[0]
 
       for k in expected_features.keys():
         # Compare features
         self.assertAllEqual(expected_features[k], actual_features[k])
 
     with self.assertRaises(errors.OutOfRangeError):
-      sess.run(get_next)
-
-  def testMakeCSVDataset(self):
-    defaults = self.DEFAULTS
-
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        # Basic test: read from file 0.
-        dataset = self._make_csv_dataset(self._test_filenames[0], defaults)
-        self._verify_records(sess, dataset, [0])
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        # Basic test: read from file 1.
-        dataset = self._make_csv_dataset(self._test_filenames[1], defaults)
-        self._verify_records(sess, dataset, [1])
+      sess.run(nxt)
+
+  def _test_dataset(self,
+                    inputs,
+                    expected_output,
+                    expected_keys,
+                    batch_size=1,
+                    num_epochs=1,
+                    label_name=None,
+                    **kwargs):
+    """Checks that elements produced by CsvDataset match expected output."""
+    # Convert str type because py3 tf strings are bytestrings
+    filenames = self._setup_files(
+        inputs, compression_type=kwargs.get("compression_type", None))
     with ops.Graph().as_default() as g:
       with self.test_session(graph=g) as sess:
-        # Read from both files.
-        dataset = self._make_csv_dataset(self._test_filenames, defaults)
-        self._verify_records(sess, dataset, range(self._num_files))
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        # Read from both files. Exercise the `batch` and `num_epochs` parameters
-        # of make_csv_dataset and make sure they work.
         dataset = self._make_csv_dataset(
-            self._test_filenames, defaults, batch_size=2, num_epochs=10)
-        self._verify_records(
-            sess, dataset, range(self._num_files), batch_size=2, num_epochs=10)
+            filenames,
+            batch_size=batch_size,
+            num_epochs=num_epochs,
+            label_name=label_name,
+            **kwargs)
+        self._verify_output(sess, dataset, batch_size, num_epochs, label_name,
+                            expected_output, expected_keys)
+
+  def testMakeCSVDataset(self):
+    """Tests making a CSV dataset with keys and defaults provided."""
+    record_defaults = [
+        constant_op.constant([], dtypes.int32),
+        constant_op.constant([], dtypes.int64),
+        constant_op.constant([], dtypes.float32),
+        constant_op.constant([], dtypes.float64),
+        constant_op.constant([], dtypes.string)
+    ]
+
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
+        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
+    ]]
+    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
+                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
+    label = "col0"
+
+    self._test_dataset(
+        inputs,
+        expected_output=expected_output,
+        expected_keys=column_names,
+        column_names=column_names,
+        label_name=label,
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=True,
+        column_defaults=record_defaults,
+    )
+
+  def testMakeCSVDataset_withBatchSizeAndEpochs(self):
+    """Tests making a CSV dataset with keys and defaults provided."""
+    record_defaults = [
+        constant_op.constant([], dtypes.int32),
+        constant_op.constant([], dtypes.int64),
+        constant_op.constant([], dtypes.float32),
+        constant_op.constant([], dtypes.float64),
+        constant_op.constant([], dtypes.string)
+    ]
+
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
+        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
+    ]]
+    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
+                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
+    label = "col0"
+
+    self._test_dataset(
+        inputs,
+        expected_output=expected_output,
+        expected_keys=column_names,
+        column_names=column_names,
+        label_name=label,
+        batch_size=3,
+        num_epochs=10,
+        shuffle=False,
+        header=True,
+        column_defaults=record_defaults,
+    )
 
-  def testMakeCSVDataset_withBadColumns(self):
+  def testMakeCSVDataset_withCompressionType(self):
+    """Tests `compression_type` argument."""
+    record_defaults = [
+        constant_op.constant([], dtypes.int32),
+        constant_op.constant([], dtypes.int64),
+        constant_op.constant([], dtypes.float32),
+        constant_op.constant([], dtypes.float64),
+        constant_op.constant([], dtypes.string)
+    ]
+
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
+        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
+    ]]
+    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
+                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
+    label = "col0"
+
+    for compression_type in ("GZIP", "ZLIB"):
+      self._test_dataset(
+          inputs,
+          expected_output=expected_output,
+          expected_keys=column_names,
+          column_names=column_names,
+          label_name=label,
+          batch_size=1,
+          num_epochs=1,
+          shuffle=False,
+          header=True,
+          column_defaults=record_defaults,
+          compression_type=compression_type,
+      )
+
+  def testMakeCSVDataset_withBadInputs(self):
     """Tests that exception is raised when input is malformed.
     """
-    dupe_columns = self.COLUMNS[:-1] + self.COLUMNS[:1]
-    defaults = self.DEFAULTS
+    record_defaults = [
+        constant_op.constant([], dtypes.int32),
+        constant_op.constant([], dtypes.int64),
+        constant_op.constant([], dtypes.float32),
+        constant_op.constant([], dtypes.float64),
+        constant_op.constant([], dtypes.string)
+    ]
+
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
+        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
+    ]]
+    filenames = self._setup_files(inputs)
 
     # Duplicate column names
     with self.assertRaises(ValueError):
       self._make_csv_dataset(
-          self._test_filenames, defaults, column_names=dupe_columns)
+          filenames,
+          batch_size=1,
+          column_defaults=record_defaults,
+          label_name="col0",
+          column_names=column_names * 2)
 
     # Label key not one of column names
     with self.assertRaises(ValueError):
       self._make_csv_dataset(
-          self._test_filenames, defaults, label_name="not_a_real_label")
+          filenames,
+          batch_size=1,
+          column_defaults=record_defaults,
+          label_name="not_a_real_label",
+          column_names=column_names)
 
   def testMakeCSVDataset_withNoLabel(self):
-    """Tests that CSV datasets can be created when no label is specified.
-    """
-    defaults = self.DEFAULTS
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        # Read from both files. Make sure this works with no label key supplied.
-        dataset = self._make_csv_dataset(
-            self._test_filenames,
-            defaults,
-            batch_size=2,
-            num_epochs=10,
-            label_name=None)
-        self._verify_records(
-            sess,
-            dataset,
-            range(self._num_files),
-            batch_size=2,
-            num_epochs=10,
-            label_name=None)
+    """Tests making a CSV dataset with no label provided."""
+    record_defaults = [
+        constant_op.constant([], dtypes.int32),
+        constant_op.constant([], dtypes.int64),
+        constant_op.constant([], dtypes.float32),
+        constant_op.constant([], dtypes.float64),
+        constant_op.constant([], dtypes.string)
+    ]
+
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
+        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
+    ]]
+    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
+                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
+
+    self._test_dataset(
+        inputs,
+        expected_output=expected_output,
+        expected_keys=column_names,
+        column_names=column_names,
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=True,
+        column_defaults=record_defaults,
+    )
 
   def testMakeCSVDataset_withNoHeader(self):
     """Tests that datasets can be created from CSV files with no header line.
     """
-    defaults = self.DEFAULTS
-    file_without_header = self._create_file(
-        len(self._test_filenames), header=False)
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        dataset = self._make_csv_dataset(
-            file_without_header,
-            defaults,
-            batch_size=2,
-            num_epochs=10,
-            header=False,
-        )
-        self._verify_records(
-            sess,
-            dataset,
-            [len(self._test_filenames)],
-            batch_size=2,
-            num_epochs=10,
-        )
+    record_defaults = [
+        constant_op.constant([], dtypes.int32),
+        constant_op.constant([], dtypes.int64),
+        constant_op.constant([], dtypes.float32),
+        constant_op.constant([], dtypes.float64),
+        constant_op.constant([], dtypes.string)
+    ]
+
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [["0,1,2,3,4", "5,6,7,8,9"], ["10,11,12,13,14", "15,16,17,18,19"]]
+    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
+                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
+    label = "col0"
+
+    self._test_dataset(
+        inputs,
+        expected_output=expected_output,
+        expected_keys=column_names,
+        column_names=column_names,
+        label_name=label,
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=False,
+        column_defaults=record_defaults,
+    )
 
   def testMakeCSVDataset_withTypes(self):
     """Tests that defaults can be a dtype instead of a Tensor for required vals.
     """
-    defaults = [d for d in self.COLUMN_TYPES[:-1]]
-    defaults.append(constant_op.constant(["NULL"], dtype=dtypes.string))
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        dataset = self._make_csv_dataset(self._test_filenames, defaults)
-        self._verify_records(sess, dataset, range(self._num_files))
+    record_defaults = [
+        dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64,
+        dtypes.string
+    ]
+
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [[",".join(x[0] for x in column_names), "0,1,2,3,4", "5,6,7,8,9"],
+              [
+                  ",".join(x[0] for x in column_names), "10,11,12,13,14",
+                  "15,16,17,18,19"
+              ]]
+    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
+                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
+    label = "col0"
+
+    self._test_dataset(
+        inputs,
+        expected_output=expected_output,
+        expected_keys=column_names,
+        column_names=column_names,
+        label_name=label,
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=True,
+        column_defaults=record_defaults,
+    )
 
   def testMakeCSVDataset_withNoColNames(self):
     """Tests that datasets can be created when column names are not specified.
 
     In that case, we should infer the column names from the header lines.
     """
-    defaults = self.DEFAULTS
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        # Read from both files. Exercise the `batch` and `num_epochs` parameters
-        # of make_csv_dataset and make sure they work.
-        dataset = self._make_csv_dataset(
-            self._test_filenames,
-            defaults,
-            column_names=None,
-            batch_size=2,
-            num_epochs=10)
-        self._verify_records(
-            sess, dataset, range(self._num_files), batch_size=2, num_epochs=10)
+    record_defaults = [
+        constant_op.constant([], dtypes.int32),
+        constant_op.constant([], dtypes.int64),
+        constant_op.constant([], dtypes.float32),
+        constant_op.constant([], dtypes.float64),
+        constant_op.constant([], dtypes.string)
+    ]
+
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
+        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
+    ]]
+    expected_output = [[0, 1, 2, 3, b"4"], [5, 6, 7, 8, b"9"],
+                       [10, 11, 12, 13, b"14"], [15, 16, 17, 18, b"19"]]
+    label = "col0"
+
+    self._test_dataset(
+        inputs,
+        expected_output=expected_output,
+        expected_keys=column_names,
+        label_name=label,
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=True,
+        column_defaults=record_defaults,
+    )
 
   def testMakeCSVDataset_withTypeInferenceMismatch(self):
     # Test that error is thrown when num fields doesn't match columns
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [[",".join(x for x in column_names), "0,1,2,3,4", "5,6,7,8,9"], [
+        ",".join(x for x in column_names), "10,11,12,13,14", "15,16,17,18,19"
+    ]]
+    filenames = self._setup_files(inputs)
     with self.assertRaises(ValueError):
       self._make_csv_dataset(
-          self._test_filenames,
-          column_names=self.COLUMNS + ["extra_name"],
-          defaults=None,
+          filenames,
+          column_names=column_names + ["extra_name"],
+          column_defaults=None,
           batch_size=2,
           num_epochs=10)
 
@@ -448,197 +550,215 @@ class MakeCsvDatasetTest(test.TestCase):
 
     In that case, we should infer the types from the first N records.
     """
-    # Test that it works with standard test files (with header, etc)
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        dataset = self._make_csv_dataset(
-            self._test_filenames, defaults=None, batch_size=2, num_epochs=10)
-        self._verify_records(
-            sess,
-            dataset,
-            range(self._num_files),
-            batch_size=2,
-            num_epochs=10,
-            defaults=[[], [], [], [], [""]])
-
-  def testMakeCSVDataset_withTypeInferenceTricky(self):
-    # Test on a deliberately tricky file (type changes as we read more rows, and
-    # there are null values)
-    fn = os.path.join(self.get_temp_dir(), "file.csv")
-    expected_dtypes = [
-        dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float32,
-        dtypes.string, dtypes.string
-    ]
-    col_names = ["col%d" % i for i in range(len(expected_dtypes))]
-    rows = [[None, None, None, "NAN", "",
-             "a"], [1, 2**31 + 1, 2**64, 123, "NAN", ""],
-            ['"123"', 2, 2**64, 123.4, "NAN", '"cd,efg"']]
-    expected = [[0, 0, 0, 0, "", "a"], [1, 2**31 + 1, 2**64, 123, "", ""],
-                [123, 2, 2**64, 123.4, "", "cd,efg"]]
-    for row in expected:
-      row[-1] = row[-1].encode("utf-8")  # py3 expects byte strings
-      row[-2] = row[-2].encode("utf-8")  # py3 expects byte strings
-    self._write_file("file.csv", [col_names] + rows)
+    column_names = ["col%d" % i for i in range(5)]
+    str_int32_max = str(2**33)
+    inputs = [[
+        ",".join(x for x in column_names),
+        "0,%s,2.0,3e50,rabbit" % str_int32_max
+    ]]
+    expected_output = [[0, 2**33, 2.0, 3e50, b"rabbit"]]
+    label = "col0"
 
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        dataset = self._make_csv_dataset(
-            fn,
-            defaults=None,
-            column_names=None,
-            label_name=None,
-            na_value="NAN",
-        )
-        features = dataset.make_one_shot_iterator().get_next()
-        # Check that types match
-        for i in range(len(expected_dtypes)):
-          print(features["col%d" % i].dtype, expected_dtypes[i])
-          assert features["col%d" % i].dtype == expected_dtypes[i]
-        for i in range(len(rows)):
-          assert sess.run(features) == dict(zip(col_names, expected[i]))
-
-  def testMakeCSVDataset_withTypeInferenceAllTypes(self):
-    # Test that we make the correct inference for all types with fallthrough
-    fn = os.path.join(self.get_temp_dir(), "file.csv")
-    expected_dtypes = [
-        dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64,
-        dtypes.string, dtypes.string
+    self._test_dataset(
+        inputs,
+        expected_output=expected_output,
+        expected_keys=column_names,
+        column_names=column_names,
+        label_name=label,
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=True,
+    )
+
+  def testMakeCSVDataset_withTypeInferenceFallthrough(self):
+    """Tests that datasets can be created when no defaults are specified.
+
+    Tests on a deliberately tricky file.
+    """
+    column_names = ["col%d" % i for i in range(5)]
+    str_int32_max = str(2**33)
+    inputs = [[
+        ",".join(x for x in column_names),
+        ",,,,",
+        "0,0,0.0,0.0,0.0",
+        "0,%s,2.0,3e50,rabbit" % str_int32_max,
+        ",,,,",
+    ]]
+    expected_output = [[0, 0, 0, 0, b""], [0, 0, 0, 0, b"0.0"],
+                       [0, 2**33, 2.0, 3e50, b"rabbit"], [0, 0, 0, 0, b""]]
+    label = "col0"
+
+    self._test_dataset(
+        inputs,
+        expected_output=expected_output,
+        expected_keys=column_names,
+        column_names=column_names,
+        label_name=label,
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=True,
+    )
+
+  def testMakeCSVDataset_withSelectCols(self):
+    record_defaults = [
+        constant_op.constant([], dtypes.int32),
+        constant_op.constant([], dtypes.int64),
+        constant_op.constant([], dtypes.float32),
+        constant_op.constant([], dtypes.float64),
+        constant_op.constant([], dtypes.string)
     ]
-    col_names = ["col%d" % i for i in range(len(expected_dtypes))]
-    rows = [[1, 2**31 + 1, 1.0, 4e40, "abc", ""]]
-    expected = [[
-        1, 2**31 + 1, 1.0, 4e40, "abc".encode("utf-8"), "".encode("utf-8")
+    column_names = ["col%d" % i for i in range(5)]
+    str_int32_max = str(2**33)
+    inputs = [[
+        ",".join(x for x in column_names),
+        "0,%s,2.0,3e50,rabbit" % str_int32_max
     ]]
-    self._write_file("file.csv", [col_names] + rows)
+    expected_output = [[0, 2**33, 2.0, 3e50, b"rabbit"]]
 
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        dataset = self._make_csv_dataset(
-            fn,
-            defaults=None,
-            column_names=None,
-            label_name=None,
-            na_value="NAN",
-        )
-        features = dataset.make_one_shot_iterator().get_next()
-        # Check that types match
-        for i in range(len(expected_dtypes)):
-          self.assertAllEqual(features["col%d" % i].dtype, expected_dtypes[i])
-        for i in range(len(rows)):
-          self.assertAllEqual(
-              sess.run(features), dict(zip(col_names, expected[i])))
+    select_cols = [1, 3, 4]
+    self._test_dataset(
+        inputs,
+        expected_output=[[x[i] for i in select_cols] for x in expected_output],
+        expected_keys=[column_names[i] for i in select_cols],
+        column_names=column_names,
+        column_defaults=[record_defaults[i] for i in select_cols],
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=True,
+        select_columns=select_cols,
+    )
+
+    # Can still do inference without provided defaults
+    self._test_dataset(
+        inputs,
+        expected_output=[[x[i] for i in select_cols] for x in expected_output],
+        expected_keys=[column_names[i] for i in select_cols],
+        column_names=column_names,
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=True,
+        select_columns=select_cols,
+    )
+
+    # Can still do column name inference
+    self._test_dataset(
+        inputs,
+        expected_output=[[x[i] for i in select_cols] for x in expected_output],
+        expected_keys=[column_names[i] for i in select_cols],
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=True,
+        select_columns=select_cols,
+    )
+
+    # Can specify column names instead of indices
+    self._test_dataset(
+        inputs,
+        expected_output=[[x[i] for i in select_cols] for x in expected_output],
+        expected_keys=[column_names[i] for i in select_cols],
+        column_names=column_names,
+        batch_size=1,
+        num_epochs=1,
+        shuffle=False,
+        header=True,
+        select_columns=[column_names[i] for i in select_cols],
+    )
 
   def testMakeCSVDataset_withSelectColsError(self):
-    data = [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
-    col_names = ["col%d" % i for i in range(5)]
-    fn = self._write_file("file.csv", [col_names] + data)
+    record_defaults = [
+        constant_op.constant([], dtypes.int32),
+        constant_op.constant([], dtypes.int64),
+        constant_op.constant([], dtypes.float32),
+        constant_op.constant([], dtypes.float64),
+        constant_op.constant([], dtypes.string)
+    ]
+    column_names = ["col%d" % i for i in range(5)]
+    str_int32_max = str(2**33)
+    inputs = [[
+        ",".join(x for x in column_names),
+        "0,%s,2.0,3e50,rabbit" % str_int32_max
+    ]]
+
+    select_cols = [1, 3, 4]
+    filenames = self._setup_files(inputs)
+
     with self.assertRaises(ValueError):
       # Mismatch in number of defaults and number of columns selected,
       # should raise an error
       self._make_csv_dataset(
-          fn,
-          defaults=[[0]] * 5,
-          column_names=col_names,
-          label_name=None,
-          select_cols=[1, 3])
+          filenames,
+          batch_size=1,
+          column_defaults=record_defaults,
+          column_names=column_names,
+          select_columns=select_cols)
+
     with self.assertRaises(ValueError):
       # Invalid column name should raise an error
       self._make_csv_dataset(
-          fn,
-          defaults=[[0]],
-          column_names=col_names,
+          filenames,
+          batch_size=1,
+          column_defaults=[[0]],
+          column_names=column_names,
           label_name=None,
-          select_cols=["invalid_col_name"])
-
-  def testMakeCSVDataset_withSelectCols(self):
-    data = [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
-    col_names = ["col%d" % i for i in range(5)]
-    fn = self._write_file("file.csv", [col_names] + data)
-    # If select_cols is specified, should only yield a subset of columns
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        dataset = self._make_csv_dataset(
-            fn,
-            defaults=[[0], [0]],
-            column_names=col_names,
-            label_name=None,
-            select_cols=[1, 3])
-        expected = [[1, 3], [6, 8]]
-        features = dataset.make_one_shot_iterator().get_next()
-        for i in range(len(data)):
-          self.assertAllEqual(
-              sess.run(features),
-              dict(zip([col_names[1], col_names[3]], expected[i])))
-    # Can still do default inference with select_cols
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        dataset = self._make_csv_dataset(
-            fn,
-            defaults=None,
-            column_names=col_names,
-            label_name=None,
-            select_cols=[1, 3])
-        expected = [[1, 3], [6, 8]]
-        features = dataset.make_one_shot_iterator().get_next()
-        for i in range(len(data)):
-          self.assertAllEqual(
-              sess.run(features),
-              dict(zip([col_names[1], col_names[3]], expected[i])))
-    # Can still do column name inference
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        dataset = self._make_csv_dataset(
-            fn,
-            defaults=None,
-            column_names=None,
-            label_name=None,
-            select_cols=[1, 3])
-        expected = [[1, 3], [6, 8]]
-        features = dataset.make_one_shot_iterator().get_next()
-        for i in range(len(data)):
-          self.assertAllEqual(
-              sess.run(features),
-              dict(zip([col_names[1], col_names[3]], expected[i])))
-    # Can specify column names instead of indices
-    with ops.Graph().as_default() as g:
-      with self.test_session(graph=g) as sess:
-        dataset = self._make_csv_dataset(
-            fn,
-            defaults=None,
-            column_names=None,
-            label_name=None,
-            select_cols=[col_names[1], col_names[3]])
-        expected = [[1, 3], [6, 8]]
-        features = dataset.make_one_shot_iterator().get_next()
-        for i in range(len(data)):
-          self.assertAllEqual(
-              sess.run(features),
-              dict(zip([col_names[1], col_names[3]], expected[i])))
+          select_columns=["invalid_col_name"])
 
   def testMakeCSVDataset_withShuffle(self):
-    total_records = self._num_files * self._num_records
-    defaults = self.DEFAULTS
+    record_defaults = [
+        constant_op.constant([], dtypes.int32),
+        constant_op.constant([], dtypes.int64),
+        constant_op.constant([], dtypes.float32),
+        constant_op.constant([], dtypes.float64),
+        constant_op.constant([], dtypes.string)
+    ]
+
+    def str_series(st):
+      return ",".join(str(i) for i in range(st, st + 5))
+
+    column_names = ["col%d" % i for i in range(5)]
+    inputs = [
+        [",".join(x for x in column_names)
+        ] + [str_series(5 * i) for i in range(15)],
+        [",".join(x for x in column_names)] +
+        [str_series(5 * i) for i in range(15, 20)],
+    ]
+
+    filenames = self._setup_files(inputs)
+
+    total_records = 20
     for batch_size in [1, 2]:
       with ops.Graph().as_default() as g:
         with self.test_session(graph=g) as sess:
           # Test that shuffling with the same seed produces the same result
           dataset1 = self._make_csv_dataset(
-              self._test_filenames,
-              defaults,
+              filenames,
+              column_defaults=record_defaults,
+              column_names=column_names,
               batch_size=batch_size,
+              header=True,
               shuffle=True,
-              shuffle_seed=5)
+              shuffle_seed=5,
+              num_epochs=2,
+          )
           dataset2 = self._make_csv_dataset(
-              self._test_filenames,
-              defaults,
+              filenames,
+              column_defaults=record_defaults,
+              column_names=column_names,
               batch_size=batch_size,
+              header=True,
               shuffle=True,
-              shuffle_seed=5)
+              shuffle_seed=5,
+              num_epochs=2,
+          )
           outputs1 = dataset1.make_one_shot_iterator().get_next()
           outputs2 = dataset2.make_one_shot_iterator().get_next()
           for _ in range(total_records // batch_size):
-            batch1 = self._run_actual_batch(outputs1, sess)
-            batch2 = self._run_actual_batch(outputs2, sess)
+            batch1 = nest.flatten(sess.run(outputs1))
+            batch2 = nest.flatten(sess.run(outputs2))
             for i in range(len(batch1)):
               self.assertAllEqual(batch1[i], batch2[i])
 
@@ -646,23 +766,31 @@ class MakeCsvDatasetTest(test.TestCase):
         with self.test_session(graph=g) as sess:
           # Test that shuffling with a different seed produces different results
           dataset1 = self._make_csv_dataset(
-              self._test_filenames,
-              defaults,
+              filenames,
+              column_defaults=record_defaults,
+              column_names=column_names,
               batch_size=batch_size,
+              header=True,
               shuffle=True,
-              shuffle_seed=5)
+              shuffle_seed=5,
+              num_epochs=2,
+          )
           dataset2 = self._make_csv_dataset(
-              self._test_filenames,
-              defaults,
+              filenames,
+              column_defaults=record_defaults,
+              column_names=column_names,
               batch_size=batch_size,
+              header=True,
               shuffle=True,
-              shuffle_seed=6)
+              shuffle_seed=6,
+              num_epochs=2,
+          )
           outputs1 = dataset1.make_one_shot_iterator().get_next()
           outputs2 = dataset2.make_one_shot_iterator().get_next()
           all_equal = False
           for _ in range(total_records // batch_size):
-            batch1 = self._run_actual_batch(outputs1, sess)
-            batch2 = self._run_actual_batch(outputs2, sess)
+            batch1 = nest.flatten(sess.run(outputs1))
+            batch2 = nest.flatten(sess.run(outputs2))
             for i in range(len(batch1)):
               all_equal = all_equal and np.array_equal(batch1[i], batch2[i])
           self.assertFalse(all_equal)
@@ -874,6 +1002,5 @@ class MakeTFRecordDatasetTest(
           self._shuffle_test(batch_size, num_epochs, num_parallel_reads,
                              seed=21345)
 
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py
index 9373e37f5f..f018dd02e6 100644
--- a/tensorflow/contrib/data/python/ops/readers.py
+++ b/tensorflow/contrib/data/python/ops/readers.py
@@ -326,6 +326,7 @@ def make_csv_dataset(
     num_parallel_parser_calls=2,
     sloppy=False,
     num_rows_for_inference=100,
+    compression_type=None,
 ):
   """Reads CSV files into a dataset.
 
@@ -399,6 +400,8 @@ def make_csv_dataset(
     num_rows_for_inference: Number of rows of a file to use for type inference
       if record_defaults is not provided. If None, reads all the rows of all
       the files. Defaults to 100.
+    compression_type: (Optional.) A `tf.string` scalar evaluating to one of
+      `""` (no compression), `"ZLIB"`, or `"GZIP"`. Defaults to no compression.
 
   Returns:
     A dataset, where each element is a (features, labels) tuple that corresponds
@@ -461,7 +464,9 @@ def make_csv_dataset(
         use_quote_delim=use_quote_delim,
         na_value=na_value,
         select_cols=select_columns,
-        header=header)
+        header=header,
+        compression_type=compression_type,
+    )
 
   def map_fn(*columns):
     """Organizes columns into a features dictionary.
@@ -505,6 +510,7 @@ class CsvDataset(dataset_ops.Dataset):
   def __init__(self,
                filenames,
                record_defaults,
+               compression_type=None,
                buffer_size=None,
                header=False,
                field_delim=",",
@@ -562,6 +568,9 @@ class CsvDataset(dataset_ops.Dataset):
         both this and `select_columns` are specified, these must have the same
         lengths, and `column_defaults` is assumed to be sorted in order of
         increasing column index.
+      compression_type: (Optional.) A `tf.string` scalar evaluating to one of
+        `""` (no compression), `"ZLIB"`, or `"GZIP"`. Defaults to no
+        compression.
       buffer_size: (Optional.) A `tf.int64` scalar denoting the number of bytes
         to buffer while reading files. Defaults to 4MB.
       header: (Optional.) A `tf.bool` scalar indicating whether the CSV file(s)
@@ -581,6 +590,11 @@ class CsvDataset(dataset_ops.Dataset):
     super(CsvDataset, self).__init__()
     self._filenames = ops.convert_to_tensor(
         filenames, dtype=dtypes.string, name="filenames")
+    self._compression_type = convert.optional_param_to_tensor(
+        "compression_type",
+        compression_type,
+        argument_default="",
+        argument_dtype=dtypes.string)
     record_defaults = [
         constant_op.constant([], dtype=x) if x in _ACCEPTABLE_CSV_TYPES else x
         for x in record_defaults
@@ -621,6 +635,7 @@ class CsvDataset(dataset_ops.Dataset):
         use_quote_delim=self._use_quote_delim,
         na_value=self._na_value,
         select_cols=self._select_cols,
+        compression_type=self._compression_type,
     )
 
   @property
diff --git a/tensorflow/contrib/makefile/proto_text_cc_files.txt b/tensorflow/contrib/makefile/proto_text_cc_files.txt
index 76428bc1d4..7d26429f9c 100644
--- a/tensorflow/contrib/makefile/proto_text_cc_files.txt
+++ b/tensorflow/contrib/makefile/proto_text_cc_files.txt
@@ -35,6 +35,7 @@ tensorflow/core/lib/random/random.cc
 tensorflow/core/lib/random/distribution_sampler.cc
 tensorflow/core/lib/io/zlib_outputbuffer.cc
 tensorflow/core/lib/io/zlib_inputstream.cc
+tensorflow/core/lib/io/zlib_compression_options.cc
 tensorflow/core/lib/io/two_level_iterator.cc
 tensorflow/core/lib/io/table_builder.cc
 tensorflow/core/lib/io/table.cc
diff --git a/tensorflow/core/lib/io/zlib_compression_options.cc b/tensorflow/core/lib/io/zlib_compression_options.cc
new file mode 100644
index 0000000000..fc54083be1
--- /dev/null
+++ b/tensorflow/core/lib/io/zlib_compression_options.cc
@@ -0,0 +1,32 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/lib/io/zlib_compression_options.h"
+
+#include 
+
+namespace tensorflow {
+namespace io {
+
+ZlibCompressionOptions::ZlibCompressionOptions() {
+  flush_mode = Z_NO_FLUSH;
+  window_bits = MAX_WBITS;
+  compression_level = Z_DEFAULT_COMPRESSION;
+  compression_method = Z_DEFLATED;
+  compression_strategy = Z_DEFAULT_STRATEGY;
+}
+
+}  // namespace io
+}  // namespace tensorflow
diff --git a/tensorflow/core/lib/io/zlib_compression_options.h b/tensorflow/core/lib/io/zlib_compression_options.h
index dc7218e866..238c1464fb 100644
--- a/tensorflow/core/lib/io/zlib_compression_options.h
+++ b/tensorflow/core/lib/io/zlib_compression_options.h
@@ -16,8 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_LIB_IO_ZLIB_COMPRESSION_OPTIONS_H_
 #define TENSORFLOW_LIB_IO_ZLIB_COMPRESSION_OPTIONS_H_
 
-#include 
-
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
@@ -25,11 +23,14 @@ namespace io {
 
 class ZlibCompressionOptions {
  public:
+  ZlibCompressionOptions();
+
   static ZlibCompressionOptions DEFAULT();
   static ZlibCompressionOptions RAW();
   static ZlibCompressionOptions GZIP();
 
-  int8 flush_mode = Z_NO_FLUSH;
+  // Defaults to Z_NO_FLUSH
+  int8 flush_mode;
 
   // Size of the buffer used for caching the data read from source file.
   int64 input_buffer_size = 256 << 10;
@@ -71,7 +72,9 @@ class ZlibCompressionOptions {
   // window_bits value provided used while compressing. If a compressed stream
   // with a larger window size is given as input, inflate() will return with the
   // error code Z_DATA_ERROR instead of trying to allocate a larger window.
-  int8 window_bits = MAX_WBITS;
+  //
+  // Defaults to MAX_WBITS
+  int8 window_bits;
 
   // From the zlib manual (http://www.zlib.net/manual.html):
   // The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9:
@@ -79,10 +82,10 @@ class ZlibCompressionOptions {
   // (the input data is simply copied a block at a time). Z_DEFAULT_COMPRESSION
   // requests a default compromise between speed and compression (currently
   // equivalent to level 6).
-  int8 compression_level = Z_DEFAULT_COMPRESSION;
+  int8 compression_level;
 
-  // The only one supported at this time.
-  int8 compression_method = Z_DEFLATED;
+  // Only Z_DEFLATED is supported at this time.
+  int8 compression_method;
 
   // From the zlib manual (http://www.zlib.net/manual.html):
   // The mem_level parameter specifies how much memory should be allocated for
@@ -106,7 +109,7 @@ class ZlibCompressionOptions {
   // but not the correctness of the compressed output even if it is not set
   // appropriately. Z_FIXED prevents the use of dynamic Huffman codes, allowing
   // for a simpler decoder for special applications.
-  int8 compression_strategy = Z_DEFAULT_STRATEGY;
+  int8 compression_strategy;
 };
 
 inline ZlibCompressionOptions ZlibCompressionOptions::DEFAULT() {
diff --git a/tensorflow/core/lib/io/zlib_inputstream.cc b/tensorflow/core/lib/io/zlib_inputstream.cc
index 47de36bf6c..d069db6d20 100644
--- a/tensorflow/core/lib/io/zlib_inputstream.cc
+++ b/tensorflow/core/lib/io/zlib_inputstream.cc
@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include 
+
 #include "tensorflow/core/lib/io/zlib_inputstream.h"
 
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -21,6 +23,35 @@ limitations under the License.
 namespace tensorflow {
 namespace io {
 
+struct ZStreamDef {
+  ZStreamDef(size_t input_buffer_capacity, size_t output_buffer_capacity)
+      : input(new Bytef[input_buffer_capacity]),
+        output(new Bytef[output_buffer_capacity]),
+        stream(new z_stream) {}
+
+  // Buffer for storing contents read from compressed stream.
+  // TODO(srbs): Consider using circular buffers. That would greatly simplify
+  // the implementation.
+  std::unique_ptr input;
+
+  // Buffer for storing inflated contents of `input_stream_`.
+  std::unique_ptr output;
+
+  // Configuration passed to `inflate`.
+  //
+  // z_stream_def_->stream->next_in:
+  //   Next byte to de-compress. Points to some byte in
+  //   z_stream_def_->streamdef_.input buffer.
+  // z_stream_def_->stream->avail_in:
+  //   Number of bytes available to be decompressed at this time.
+  // z_stream_def_->stream->next_out:
+  //   Next byte to write de-compressed data to. Points to some byte in
+  //   z_stream_def_->streamdef_.output buffer.
+  // z_stream_def_->stream->avail_out:
+  //   Number of free bytes available at write location.
+  std::unique_ptr stream;
+};
+
 ZlibInputStream::ZlibInputStream(
     InputStreamInterface* input_stream,
     size_t input_buffer_bytes,   // size of z_stream.next_in buffer
@@ -30,10 +61,9 @@ ZlibInputStream::ZlibInputStream(
       input_stream_(input_stream),
       input_buffer_capacity_(input_buffer_bytes),
       output_buffer_capacity_(output_buffer_bytes),
-      z_stream_input_(new Bytef[input_buffer_capacity_]),
-      z_stream_output_(new Bytef[output_buffer_capacity_]),
       zlib_options_(zlib_options),
-      z_stream_(new z_stream),
+      z_stream_def_(
+          new ZStreamDef(input_buffer_capacity_, output_buffer_capacity_)),
       bytes_read_(0) {
   InitZlibBuffer();
 }
@@ -46,8 +76,8 @@ ZlibInputStream::ZlibInputStream(InputStreamInterface* input_stream,
                       zlib_options, false) {}
 
 ZlibInputStream::~ZlibInputStream() {
-  if (z_stream_) {
-    inflateEnd(z_stream_.get());
+  if (z_stream_def_->stream) {
+    inflateEnd(z_stream_def_->stream.get());
   }
   if (owns_input_stream_) {
     delete input_stream_;
@@ -56,51 +86,54 @@ ZlibInputStream::~ZlibInputStream() {
 
 Status ZlibInputStream::Reset() {
   TF_RETURN_IF_ERROR(input_stream_->Reset());
-  inflateEnd(z_stream_.get());
+  inflateEnd(z_stream_def_->stream.get());
   InitZlibBuffer();
   bytes_read_ = 0;
   return Status::OK();
 }
 
 void ZlibInputStream::InitZlibBuffer() {
-  memset(z_stream_.get(), 0, sizeof(z_stream));
+  memset(z_stream_def_->stream.get(), 0, sizeof(z_stream));
 
-  z_stream_->zalloc = Z_NULL;
-  z_stream_->zfree = Z_NULL;
-  z_stream_->opaque = Z_NULL;
-  z_stream_->next_in = Z_NULL;
-  z_stream_->avail_in = 0;
+  z_stream_def_->stream->zalloc = Z_NULL;
+  z_stream_def_->stream->zfree = Z_NULL;
+  z_stream_def_->stream->opaque = Z_NULL;
+  z_stream_def_->stream->next_in = Z_NULL;
+  z_stream_def_->stream->avail_in = 0;
 
-  int status = inflateInit2(z_stream_.get(), zlib_options_.window_bits);
+  int status =
+      inflateInit2(z_stream_def_->stream.get(), zlib_options_.window_bits);
 
   CHECK_EQ(status, Z_OK) << "inflateInit failed with status " << status;
 
-  z_stream_->next_in = z_stream_input_.get();
-  z_stream_->next_out = z_stream_output_.get();
-  next_unread_byte_ = reinterpret_cast(z_stream_output_.get());
-  z_stream_->avail_in = 0;
-  z_stream_->avail_out = output_buffer_capacity_;
+  z_stream_def_->stream->next_in = z_stream_def_->input.get();
+  z_stream_def_->stream->next_out = z_stream_def_->output.get();
+  next_unread_byte_ = reinterpret_cast(z_stream_def_->output.get());
+  z_stream_def_->stream->avail_in = 0;
+  z_stream_def_->stream->avail_out = output_buffer_capacity_;
 }
 
 Status ZlibInputStream::ReadFromStream() {
   int bytes_to_read = input_buffer_capacity_;
-  char* read_location = reinterpret_cast(z_stream_input_.get());
+  char* read_location = reinterpret_cast(z_stream_def_->input.get());
 
   // If there are unread bytes in the input stream we move them to the head
   // of the stream to maximize the space available to read new data into.
-  if (z_stream_->avail_in > 0) {
-    uLong read_bytes = z_stream_->next_in - z_stream_input_.get();
+  if (z_stream_def_->stream->avail_in > 0) {
+    uLong read_bytes =
+        z_stream_def_->stream->next_in - z_stream_def_->input.get();
     // Remove `read_bytes` from the head of the input stream.
     // Move unread bytes to the head of the input stream.
     if (read_bytes > 0) {
-      memmove(z_stream_input_.get(), z_stream_->next_in, z_stream_->avail_in);
+      memmove(z_stream_def_->input.get(), z_stream_def_->stream->next_in,
+              z_stream_def_->stream->avail_in);
     }
 
-    bytes_to_read -= z_stream_->avail_in;
-    read_location += z_stream_->avail_in;
+    bytes_to_read -= z_stream_def_->stream->avail_in;
+    read_location += z_stream_def_->stream->avail_in;
   }
   string data;
-  // Try to read enough data to fill up z_stream_input_.
+  // Try to read enough data to fill up z_stream_def_->input.
   // TODO(rohanj): Add a char* version of ReadNBytes to InputStreamInterface
   // and use that instead to make this more efficient.
   Status s = input_stream_->ReadNBytes(bytes_to_read, &data);
@@ -108,10 +141,10 @@ Status ZlibInputStream::ReadFromStream() {
 
   // Since we moved unread data to the head of the input stream we can point
   // next_in to the head of the input stream.
-  z_stream_->next_in = z_stream_input_.get();
+  z_stream_def_->stream->next_in = z_stream_def_->input.get();
 
   // Note: data.size() could be different from bytes_to_read.
-  z_stream_->avail_in += data.size();
+  z_stream_def_->stream->avail_in += data.size();
 
   if (!s.ok() && !errors::IsOutOfRange(s)) {
     return s;
@@ -135,7 +168,8 @@ Status ZlibInputStream::ReadFromStream() {
 size_t ZlibInputStream::ReadBytesFromCache(size_t bytes_to_read,
                                            string* result) {
   size_t unread_bytes =
-      reinterpret_cast(z_stream_->next_out) - next_unread_byte_;
+      reinterpret_cast(z_stream_def_->stream->next_out) -
+      next_unread_byte_;
   size_t can_read_bytes = std::min(bytes_to_read, unread_bytes);
   if (can_read_bytes > 0) {
     result->append(next_unread_byte_, can_read_bytes);
@@ -147,8 +181,9 @@ size_t ZlibInputStream::ReadBytesFromCache(size_t bytes_to_read,
 
 size_t ZlibInputStream::NumUnreadBytes() const {
   size_t read_bytes =
-      next_unread_byte_ - reinterpret_cast(z_stream_output_.get());
-  return output_buffer_capacity_ - z_stream_->avail_out - read_bytes;
+      next_unread_byte_ - reinterpret_cast(z_stream_def_->output.get());
+  return output_buffer_capacity_ - z_stream_def_->stream->avail_out -
+         read_bytes;
 }
 
 Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) {
@@ -167,14 +202,14 @@ Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) {
     // completely consumed. This is an optimization and can be removed if
     // it causes problems. `ReadFromStream` is capable of handling partially
     // filled up buffers.
-    if (z_stream_->avail_in == 0) {
+    if (z_stream_def_->stream->avail_in == 0) {
       TF_RETURN_IF_ERROR(ReadFromStream());
     }
 
     // Step 2. Setup output stream.
-    z_stream_->next_out = z_stream_output_.get();
-    next_unread_byte_ = reinterpret_cast(z_stream_output_.get());
-    z_stream_->avail_out = output_buffer_capacity_;
+    z_stream_def_->stream->next_out = z_stream_def_->output.get();
+    next_unread_byte_ = reinterpret_cast(z_stream_def_->output.get());
+    z_stream_def_->stream->avail_out = output_buffer_capacity_;
 
     // Step 3. Inflate Inflate Inflate!
     TF_RETURN_IF_ERROR(Inflate());
@@ -188,12 +223,12 @@ Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) {
 int64 ZlibInputStream::Tell() const { return bytes_read_; }
 
 Status ZlibInputStream::Inflate() {
-  int error = inflate(z_stream_.get(), zlib_options_.flush_mode);
+  int error = inflate(z_stream_def_->stream.get(), zlib_options_.flush_mode);
   if (error != Z_OK && error != Z_STREAM_END) {
     string error_string =
         strings::StrCat("inflate() failed with error ", error);
-    if (z_stream_->msg != nullptr) {
-      strings::StrAppend(&error_string, ": ", z_stream_->msg);
+    if (z_stream_def_->stream->msg != nullptr) {
+      strings::StrAppend(&error_string, ": ", z_stream_def_->stream->msg);
     }
     return errors::DataLoss(error_string);
   }
diff --git a/tensorflow/core/lib/io/zlib_inputstream.h b/tensorflow/core/lib/io/zlib_inputstream.h
index 37339163ee..ac9e23ca97 100644
--- a/tensorflow/core/lib/io/zlib_inputstream.h
+++ b/tensorflow/core/lib/io/zlib_inputstream.h
@@ -16,8 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_LIB_IO_ZLIB_INPUTSTREAM_H_
 #define TENSORFLOW_LIB_IO_ZLIB_INPUTSTREAM_H_
 
-#include 
-
 #include 
 
 #include "tensorflow/core/lib/core/status.h"
@@ -30,6 +28,10 @@ limitations under the License.
 namespace tensorflow {
 namespace io {
 
+// Forward declare some members of zlib.h, which is only included in the
+// .cc file.
+struct ZStreamDef;
+
 // An ZlibInputStream provides support for reading from a stream compressed
 // using zlib (http://www.zlib.net/). Buffers the contents of the file.
 //
@@ -79,28 +81,9 @@ class ZlibInputStream : public InputStreamInterface {
   size_t output_buffer_capacity_;  // Size of z_stream_output_
   char* next_unread_byte_;         // Next unread byte in z_stream_output_
 
-  // Buffer for storing contents read from compressed stream.
-  // TODO(srbs): Consider using circular buffers. That would greatly simplify
-  // the implementation.
-  std::unique_ptr z_stream_input_;
-
-  // Buffer for storing inflated contents of `input_stream_`.
-  std::unique_ptr z_stream_output_;
-
   ZlibCompressionOptions const zlib_options_;
 
-  // Configuration passed to `inflate`.
-  //
-  // z_stream_->next_in:
-  //   Next byte to de-compress. Points to some byte in z_stream_input_ buffer.
-  // z_stream_->avail_in:
-  //   Number of bytes available to be decompressed at this time.
-  // z_stream_->next_out:
-  //   Next byte to write de-compressed data to. Points to some byte in
-  //   z_stream_output_ buffer.
-  // z_stream_->avail_out:
-  //   Number of free bytes available at write location.
-  std::unique_ptr z_stream_;
+  std::unique_ptr z_stream_def_;
 
   // Reads data from `input_stream_` and tries to fill up `z_stream_input_` if
   // enough unread data is left in `input_stream_`.
-- 
cgit v1.2.3


From 1662a105497e60d002e101161987cbbd48ba06c6 Mon Sep 17 00:00:00 2001
From: Peter Hawkins 
Date: Tue, 17 Jul 2018 17:19:48 -0700
Subject: [TF:XLA] Move implementations of primitive math functions out of
 TF/XLA and into xla/client/lib/math.{cc,h}.

PiperOrigin-RevId: 205003168
---
 tensorflow/compiler/tf2xla/kernels/unary_ops.cc |  44 ++------
 tensorflow/compiler/xla/client/lib/math.cc      | 130 +++++++++++++++---------
 tensorflow/compiler/xla/client/lib/math.h       |  31 ++++++
 3 files changed, 122 insertions(+), 83 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
index 4bb31f4117..e6ec794cfd 100644
--- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
@@ -51,43 +51,18 @@ XLAJIT_MAKE_UNARY(Conj, xla::Conj(x));
 
 // Return x if x>0, otherwise -x.
 XLAJIT_MAKE_UNARY(Abs, xla::Abs(x));
-
-// acos(x) = 2 * atan(sqrt(1 - x^2) / (1 + x))
-XLAJIT_MAKE_UNARY(Acos,
-                  xla::ScalarLike(x, 2.0) *
-                      xla::Atan2(xla::Sqrt(xla::ScalarLike(x, 1.0) - x * x),
-                                 xla::ScalarLike(x, 1.0) + x));
-
-// acosh(x) = log(x + sqrt(x^2 - 1))
-//          = log(x + sqrt((x+1)*(x-1)))
-XLAJIT_MAKE_UNARY(Acosh,
-                  xla::Log(x + xla::Sqrt((x + xla::ScalarLike(x, 1.0)) *
-                                         (x - xla::ScalarLike(x, 1.0)))));
-
-// asin(x) = 2 * atan(x / (1 + sqrt(1 - x^2)))
-XLAJIT_MAKE_UNARY(
-    Asin, xla::ScalarLike(x, 2.0) *
-              xla::Atan2(x, xla::ScalarLike(x, 1.0) +
-                                xla::Sqrt(xla::ScalarLike(x, 1.0) - x * x)));
-
-// asinh(x) = log(x + sqrt(x^2 + 1))
-XLAJIT_MAKE_UNARY(Asinh,
-                  xla::Log(x + xla::Sqrt(x * x + xla::ScalarLike(x, 1.0))));
-
-XLAJIT_MAKE_UNARY(Atan, xla::Atan2(x, xla::ScalarLike(x, 1.0)));
-
-// atanh(x) = 0.5 * log((1 + x) / (1 - x))
-XLAJIT_MAKE_UNARY(Atanh, xla::Log((xla::ScalarLike(x, 1.0) + x) /
-                                  (xla::ScalarLike(x, 1.0) - x)) *
-                             xla::ScalarLike(x, 0.5));
+XLAJIT_MAKE_UNARY(Acos, xla::Acos(x));
+XLAJIT_MAKE_UNARY(Acosh, xla::Acosh(x));
+XLAJIT_MAKE_UNARY(Asin, xla::Asin(x))
+XLAJIT_MAKE_UNARY(Asinh, xla::Asinh(x));
+XLAJIT_MAKE_UNARY(Atan, xla::Atan(x));
+XLAJIT_MAKE_UNARY(Atanh, xla::Atanh(x));
 XLAJIT_MAKE_UNARY(Ceil, xla::Ceil(x));
 XLAJIT_MAKE_UNARY(Cos, xla::Cos(x));
-XLAJIT_MAKE_UNARY(Cosh, (xla::Exp(x) + xla::Exp(-x)) * xla::ScalarLike(x, 0.5));
+XLAJIT_MAKE_UNARY(Cosh, xla::Cosh(x));
 XLAJIT_MAKE_UNARY(Sin, xla::Sin(x));
 XLAJIT_MAKE_UNARY(Exp, xla::Exp(x));
-
 XLAJIT_MAKE_UNARY(Expm1, xla::Expm1(x));
-
 XLAJIT_MAKE_UNARY(Floor, xla::Floor(x));
 XLAJIT_MAKE_UNARY(IsFinite, xla::IsFinite(x));
 XLAJIT_MAKE_UNARY(
@@ -99,7 +74,6 @@ XLAJIT_MAKE_UNARY(IsNan, xla::Ne(x, x));
 XLAJIT_MAKE_UNARY(Inv, xla::ScalarLike(x, 1.0) / x);
 XLAJIT_MAKE_UNARY(Reciprocal, xla::ScalarLike(x, 1.0) / x);
 XLAJIT_MAKE_UNARY(Log, xla::Log(x));
-
 XLAJIT_MAKE_UNARY(Log1p, xla::Log1p(x));
 
 XLAJIT_MAKE_UNARY(Invert, xla::Not(x));
@@ -136,7 +110,7 @@ XLAJIT_MAKE_UNARY(Sigmoid, Sigmoid(x));
 
 // Returns 0 if x is 0, -1 if x < 0 and 1 if x > 0.
 XLAJIT_MAKE_UNARY(Sign, xla::Sign(x));
-XLAJIT_MAKE_UNARY(Sinh, (xla::Exp(x) - xla::Exp(-x)) * xla::ScalarLike(x, 0.5));
+XLAJIT_MAKE_UNARY(Sinh, xla::Sinh(x));
 
 // softplus(x) = log(1 + exp(x))
 //
@@ -153,7 +127,7 @@ XLAJIT_MAKE_UNARY(Softplus, xla::Max(x, xla::ScalarLike(x, 0.0)) +
 XLAJIT_MAKE_UNARY(Softsign, x / (xla::Abs(x) + xla::ScalarLike(x, 1.0)));
 XLAJIT_MAKE_UNARY(Sqrt, xla::Sqrt(x));
 XLAJIT_MAKE_UNARY(Square, x* x);
-XLAJIT_MAKE_UNARY(Tan, xla::Sin(x) / xla::Cos(x));
+XLAJIT_MAKE_UNARY(Tan, xla::Tan(x));
 XLAJIT_MAKE_UNARY(Tanh, xla::Tanh(x));
 
 XLAJIT_MAKE_UNARY(Real, xla::Real(x));
diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index 2a7ac1d716..a6d606f944 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -172,32 +172,29 @@ static constexpr std::array kLanczosCoefficients = {
 // lgamma(z + 1) = (log(2) + log(pi)) / 2 + (z + 1/2) * log(t(z)) - t(z) + A(z)
 // t(z) = z + kLanczosGamma + 1/2
 // A(z) = kBaseLanczosCoeff + sigma(k = 1, n, kLanczosCoefficients[i] / (z + k))
-xla::XlaOp Lgamma(xla::XlaOp input) {
-  xla::XlaOp one_half = xla::ScalarLike(input, 0.5);
-  xla::XlaOp one = xla::ScalarLike(input, 1);
+XlaOp Lgamma(XlaOp input) {
+  XlaOp one_half = ScalarLike(input, 0.5);
+  XlaOp one = ScalarLike(input, 1);
 
-  xla::XlaOp pi = xla::ScalarLike(input, M_PI);
-  xla::XlaOp log_pi = xla::ScalarLike(input, std::log(M_PI));
-  xla::XlaOp log_sqrt_two_pi =
-      xla::ScalarLike(input, (std::log(2) + std::log(M_PI)) / 2);
+  XlaOp pi = ScalarLike(input, M_PI);
+  XlaOp log_pi = ScalarLike(input, std::log(M_PI));
+  XlaOp log_sqrt_two_pi = ScalarLike(input, (std::log(2) + std::log(M_PI)) / 2);
 
-  xla::XlaOp lanczos_gamma_plus_one_half =
-      xla::ScalarLike(input, kLanczosGamma + 0.5);
-  xla::XlaOp log_lanczos_gamma_plus_one_half =
-      xla::ScalarLike(input, std::log(kLanczosGamma + 0.5));
+  XlaOp lanczos_gamma_plus_one_half = ScalarLike(input, kLanczosGamma + 0.5);
+  XlaOp log_lanczos_gamma_plus_one_half =
+      ScalarLike(input, std::log(kLanczosGamma + 0.5));
 
-  xla::XlaOp base_lanczos_coeff = xla::ScalarLike(input, kBaseLanczosCoeff);
+  XlaOp base_lanczos_coeff = ScalarLike(input, kBaseLanczosCoeff);
 
   // If the input is less than 0.5 use Gauss's reflection formula:
   // gamma(x) = pi / sin(pi * x) * gamma(1 - x)
-  xla::XlaOp need_to_reflect = xla::Lt(xla::Real(input), one_half);
-  xla::XlaOp z = xla::Select(need_to_reflect, -input, input - one);
+  XlaOp need_to_reflect = Lt(Real(input), one_half);
+  XlaOp z = Select(need_to_reflect, -input, input - one);
 
-  xla::XlaOp x = base_lanczos_coeff;
+  XlaOp x = base_lanczos_coeff;
   for (int i = 0; i < kLanczosCoefficients.size(); ++i) {
-    xla::XlaOp lanczos_coefficient =
-        xla::ScalarLike(input, kLanczosCoefficients[i]);
-    xla::XlaOp index = xla::ScalarLike(input, i);
+    XlaOp lanczos_coefficient = ScalarLike(input, kLanczosCoefficients[i]);
+    XlaOp index = ScalarLike(input, i);
     x = x + lanczos_coefficient / (z + index + one);
   }
 
@@ -206,14 +203,14 @@ xla::XlaOp Lgamma(xla::XlaOp input) {
   // the device.
   // log(t) = log(kLanczosGamma + 0.5 + z)
   //        = log(kLanczosGamma + 0.5) + log1p(z / (kLanczosGamma + 0.5))
-  xla::XlaOp t = lanczos_gamma_plus_one_half + z;
-  xla::XlaOp log_t = log_lanczos_gamma_plus_one_half +
-                     xla::Log1p(z / lanczos_gamma_plus_one_half);
+  XlaOp t = lanczos_gamma_plus_one_half + z;
+  XlaOp log_t =
+      log_lanczos_gamma_plus_one_half + Log1p(z / lanczos_gamma_plus_one_half);
 
-  xla::XlaOp log_y = log_sqrt_two_pi + (z + one_half) * log_t - t + xla::Log(x);
+  XlaOp log_y = log_sqrt_two_pi + (z + one_half) * log_t - t + Log(x);
 
-  xla::XlaOp reflection = log_pi - xla::Log(xla::Sin(pi * input)) - log_y;
-  xla::XlaOp result = xla::Select(need_to_reflect, reflection, log_y);
+  XlaOp reflection = log_pi - Log(Sin(pi * input)) - log_y;
+  XlaOp result = Select(need_to_reflect, reflection, log_y);
   return result;
 }
 
@@ -224,32 +221,30 @@ xla::XlaOp Lgamma(xla::XlaOp input) {
 // t(z) = z + kLanczosGamma + 1/2
 // A(z) = kBaseLanczosCoeff + sigma(k = 1, n, kLanczosCoefficients[i] / (z + k))
 // A'(z) = sigma(k = 1, n, kLanczosCoefficients[i] / (z + k) / (z + k))
-xla::XlaOp Digamma(xla::XlaOp input) {
-  xla::XlaOp zero = xla::ScalarLike(input, 0);
-  xla::XlaOp one_half = xla::ScalarLike(input, 0.5);
-  xla::XlaOp one = xla::ScalarLike(input, 1);
+XlaOp Digamma(XlaOp input) {
+  XlaOp zero = ScalarLike(input, 0);
+  XlaOp one_half = ScalarLike(input, 0.5);
+  XlaOp one = ScalarLike(input, 1);
 
-  xla::XlaOp pi = xla::ScalarLike(input, M_PI);
+  XlaOp pi = ScalarLike(input, M_PI);
 
-  xla::XlaOp lanczos_gamma = xla::ScalarLike(input, kLanczosGamma);
-  xla::XlaOp lanczos_gamma_plus_one_half =
-      xla::ScalarLike(input, kLanczosGamma + 0.5);
-  xla::XlaOp log_lanczos_gamma_plus_one_half =
-      xla::ScalarLike(input, std::log(kLanczosGamma + 0.5));
+  XlaOp lanczos_gamma = ScalarLike(input, kLanczosGamma);
+  XlaOp lanczos_gamma_plus_one_half = ScalarLike(input, kLanczosGamma + 0.5);
+  XlaOp log_lanczos_gamma_plus_one_half =
+      ScalarLike(input, std::log(kLanczosGamma + 0.5));
 
-  xla::XlaOp base_lanczos_coeff = xla::ScalarLike(input, kBaseLanczosCoeff);
+  XlaOp base_lanczos_coeff = ScalarLike(input, kBaseLanczosCoeff);
 
   // If the input is less than 0.5 use Gauss's reflection formula:
   // digamma(x) = digamma(1 - x) - pi * cot(pi * x)
-  xla::XlaOp need_to_reflect = xla::Lt(xla::Real(input), one_half);
-  xla::XlaOp z = xla::Select(need_to_reflect, -input, input - one);
+  XlaOp need_to_reflect = Lt(Real(input), one_half);
+  XlaOp z = Select(need_to_reflect, -input, input - one);
 
-  xla::XlaOp num = zero;
-  xla::XlaOp denom = base_lanczos_coeff;
+  XlaOp num = zero;
+  XlaOp denom = base_lanczos_coeff;
   for (int i = 0; i < kLanczosCoefficients.size(); ++i) {
-    xla::XlaOp lanczos_coefficient =
-        xla::ScalarLike(input, kLanczosCoefficients[i]);
-    xla::XlaOp index = xla::ScalarLike(input, i);
+    XlaOp lanczos_coefficient = ScalarLike(input, kLanczosCoefficients[i]);
+    XlaOp index = ScalarLike(input, i);
     num = num - lanczos_coefficient / ((z + index + one) * (z + index + one));
     denom = denom + lanczos_coefficient / (z + index + one);
   }
@@ -259,14 +254,53 @@ xla::XlaOp Digamma(xla::XlaOp input) {
   // the device.
   // log(t) = log(kLanczosGamma + 0.5 + z)
   //        = log(kLanczosGamma + 0.5) + log1p(z / (kLanczosGamma + 0.5))
-  xla::XlaOp t = lanczos_gamma_plus_one_half + z;
-  xla::XlaOp log_t = log_lanczos_gamma_plus_one_half +
-                     xla::Log1p(z / lanczos_gamma_plus_one_half);
+  XlaOp t = lanczos_gamma_plus_one_half + z;
+  XlaOp log_t =
+      log_lanczos_gamma_plus_one_half + Log1p(z / lanczos_gamma_plus_one_half);
 
-  xla::XlaOp y = log_t + num / denom - lanczos_gamma / t;
-  xla::XlaOp reflection = y - pi * xla::Cos(pi * input) / xla::Sin(pi * input);
-  xla::XlaOp result = xla::Select(need_to_reflect, reflection, y);
+  XlaOp y = log_t + num / denom - lanczos_gamma / t;
+  XlaOp reflection = y - pi * Cos(pi * input) / Sin(pi * input);
+  XlaOp result = Select(need_to_reflect, reflection, y);
   return result;
 }
 
+// Trigonometric functions.
+
+// acos(x) = 2 * atan(sqrt(1 - x^2) / (1 + x))
+XlaOp Acos(XlaOp x) {
+  return ScalarLike(x, 2.0) *
+         Atan2(Sqrt(ScalarLike(x, 1.0) - x * x), ScalarLike(x, 1.0) + x);
+}
+
+// asin(x) = 2 * atan(x / (1 + sqrt(1 - x^2)))
+XlaOp Asin(XlaOp x) {
+  return ScalarLike(x, 2.0) *
+         Atan2(x, ScalarLike(x, 1.0) + Sqrt(ScalarLike(x, 1.0) - x * x));
+}
+
+XlaOp Atan(XlaOp x) { return Atan2(x, ScalarLike(x, 1.0)); }
+
+XlaOp Tan(XlaOp x) { return Sin(x) / Cos(x); }
+
+// Hyperbolic trigonometric functions.
+
+// acosh(x) = log(x + sqrt(x^2 - 1))
+//          = log(x + sqrt((x+1)*(x-1)))
+XlaOp Acosh(XlaOp x) {
+  return Log(x + Sqrt((x + ScalarLike(x, 1.0)) * (x - ScalarLike(x, 1.0))));
+}
+
+// asinh(x) = log(x + sqrt(x^2 + 1))
+XlaOp Asinh(XlaOp x) { return Log(x + Sqrt(x * x + ScalarLike(x, 1.0))); }
+
+// atanh(x) = 0.5 * log((1 + x) / (1 - x))
+XlaOp Atanh(XlaOp x) {
+  return Log((ScalarLike(x, 1.0) + x) / (ScalarLike(x, 1.0) - x)) *
+         ScalarLike(x, 0.5);
+}
+
+XlaOp Cosh(XlaOp x) { return (Exp(x) + Exp(-x)) * ScalarLike(x, 0.5); }
+
+XlaOp Sinh(XlaOp x) { return (Exp(x) - Exp(-x)) * ScalarLike(x, 0.5); }
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/math.h b/tensorflow/compiler/xla/client/lib/math.h
index e4c79b5f52..d003d529cc 100644
--- a/tensorflow/compiler/xla/client/lib/math.h
+++ b/tensorflow/compiler/xla/client/lib/math.h
@@ -52,6 +52,37 @@ XlaOp Lgamma(XlaOp input);
 // Computes an approximation of the digamma function.
 XlaOp Digamma(XlaOp input);
 
+// Trigonometric functions
+
+// Computes the arc cosine of 'x'.
+XlaOp Acos(XlaOp x);
+
+// Computes the arc sine of 'x'.
+XlaOp Asin(XlaOp x);
+
+// Computes the arc tangent of 'x'.
+XlaOp Atan(XlaOp x);
+
+// Computes the tangent of 'x'.
+XlaOp Tan(XlaOp x);
+
+// Hyperbolic trigonometric functions
+
+// Computes the inverse hyperbolic cosine of 'x'.
+XlaOp Acosh(XlaOp x);
+
+// Computes the inverse hyperbolic sine of 'x'.
+XlaOp Asinh(XlaOp x);
+
+// Computes the inverse hyperbolic tangent of 'x'.
+XlaOp Atanh(XlaOp x);
+
+// Computes the hyperbolic cosine of 'x'.
+XlaOp Cosh(XlaOp x);
+
+// Computes the hyperbolic sine of 'x'.
+XlaOp Sinh(XlaOp x);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_MATH_H_
-- 
cgit v1.2.3


From 07cc6474b219ee3ad9f55860e621f61b34bb6bd1 Mon Sep 17 00:00:00 2001
From: Mark Heffernan 
Date: Tue, 17 Jul 2018 18:01:52 -0700
Subject: Add single-sided host send and receive operations. Adds a bit on
 kSend/kReceive instructions and their Done variants indicated whether the
 operations communicates with the host or another device (the default). Host
 send/recv operations are single-sided without a complementary recv/send
 instruction in another module.

Host send/recv operations are exposed in the XLA builder API as SendToHost and RecvFromHost.

PiperOrigin-RevId: 205008138
---
 tensorflow/compiler/xla/client/client.cc           |  16 ++-
 tensorflow/compiler/xla/client/client.h            |  12 +-
 .../compiler/xla/client/xla_client/xla_builder.cc  | 132 +++++++++++++++-----
 .../compiler/xla/client/xla_client/xla_builder.h   |  45 +++++--
 tensorflow/compiler/xla/service/BUILD              |   2 +
 tensorflow/compiler/xla/service/channel_tracker.cc |  28 ++++-
 tensorflow/compiler/xla/service/channel_tracker.h  |   6 +-
 tensorflow/compiler/xla/service/hlo.proto          |   5 +
 tensorflow/compiler/xla/service/hlo_instruction.cc |  29 +++--
 tensorflow/compiler/xla/service/hlo_instruction.h  |  23 ++--
 .../compiler/xla/service/hlo_instructions.cc       |  44 ++++---
 tensorflow/compiler/xla/service/hlo_instructions.h |  18 ++-
 .../xla/service/hlo_module_group_metadata.cc       |  10 +-
 .../xla/service/hlo_module_group_metadata.h        |   2 +-
 tensorflow/compiler/xla/service/hlo_parser.cc      |  40 ++++--
 tensorflow/compiler/xla/service/hlo_parser_test.cc |  15 +++
 tensorflow/compiler/xla/service/hlo_scheduling.cc  |   1 +
 tensorflow/compiler/xla/service/hlo_verifier.cc    | 137 +++++++++++++++------
 tensorflow/compiler/xla/service/hlo_verifier.h     |   4 -
 .../compiler/xla/service/layout_assignment.cc      |  83 +++++++++++--
 .../compiler/xla/service/layout_assignment.h       |   7 ++
 tensorflow/compiler/xla/service/service.cc         |   3 +-
 tensorflow/compiler/xla/xla.proto                  |   1 +
 tensorflow/compiler/xla/xla_data.proto             |  16 +++
 24 files changed, 529 insertions(+), 150 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index 3a157c69cd..8e54311bad 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -409,8 +409,10 @@ StatusOr Client::ExecutionStatsAsString(
   return string("[Execution Statistics] not available.");
 }
 
-StatusOr Client::CreateChannelHandle() {
+StatusOr Client::CreateChannelHandleByType(
+    ChannelHandle::ChannelType type) {
   CreateChannelHandleRequest request;
+  request.set_channel_type(type);
   CreateChannelHandleResponse response;
 
   VLOG(1) << "making create channel handle request";
@@ -424,4 +426,16 @@ StatusOr Client::CreateChannelHandle() {
   return response.channel();
 }
 
+StatusOr Client::CreateChannelHandle() {
+  return CreateChannelHandleByType(ChannelHandle::DEVICE_TO_DEVICE);
+}
+
+StatusOr Client::CreateHostToDeviceChannelHandle() {
+  return CreateChannelHandleByType(ChannelHandle::HOST_TO_DEVICE);
+}
+
+StatusOr Client::CreateDeviceToHostChannelHandle() {
+  return CreateChannelHandleByType(ChannelHandle::DEVICE_TO_HOST);
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/client.h b/tensorflow/compiler/xla/client/client.h
index 69d4d300ca..d751e183dd 100644
--- a/tensorflow/compiler/xla/client/client.h
+++ b/tensorflow/compiler/xla/client/client.h
@@ -178,10 +178,15 @@ class Client {
   StatusOr> GetComputationShape(
       const XlaComputation& computation);
 
-  // Creates a channel handle that can be used to transfer data between
-  // two computations via a pair of Send and Recv instructions.
+  // Creates a channel handle that can be used to transfer data between two
+  // computations on different devices via a pair of Send and Recv instructions.
   StatusOr CreateChannelHandle();
 
+  // Create a channel for communicating with the host via a SendtoHost or
+  // RecvFromHost operation.
+  StatusOr CreateHostToDeviceChannelHandle();
+  StatusOr CreateDeviceToHostChannelHandle();
+
   StatusOr LoadSnapshot(const HloSnapshot& module);
 
   ServiceInterface* stub() { return stub_; }
@@ -192,6 +197,9 @@ class Client {
   StatusOr ExecutionStatsAsString(const XlaComputation& computation,
                                           const ExecutionProfile& profile);
 
+  StatusOr CreateChannelHandleByType(
+      ChannelHandle::ChannelType type);
+
   ServiceInterface* stub_;  // Stub that this client is connected on.
 
   TF_DISALLOW_COPY_AND_ASSIGN(Client);
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 3b4f9e1407..ced26fc2ed 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -1940,28 +1940,17 @@ void XlaBuilder::Send(const XlaOp& operand, const ChannelHandle& handle) {
     TF_ASSIGN_OR_RETURN(XlaOp token, AddInstruction(std::move(token_instr),
                                                     HloOpcode::kAfterAll, {}));
 
-    // Send instruction produces a tuple of {aliased operand, U32 context,
-    // token}.
-    HloInstructionProto send_instr;
-    TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(operand));
-    *send_instr.mutable_shape() = ShapeUtil::MakeTupleShape(
-        {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()});
-    send_instr.set_channel_id(handle.handle());
-    TF_ASSIGN_OR_RETURN(XlaOp send,
-                        AddInstruction(std::move(send_instr), HloOpcode::kSend,
-                                       {operand, token}));
-
-    HloInstructionProto send_done_instr;
-    *send_done_instr.mutable_shape() = ShapeUtil::MakeTokenShape();
-    send_done_instr.set_channel_id(handle.handle());
-    return AddInstruction(std::move(send_done_instr), HloOpcode::kSendDone,
-                          {send});
+    return SendWithToken(operand, token, handle);
   });
 }
 
 XlaOp XlaBuilder::SendWithToken(const XlaOp& operand, const XlaOp& token,
                                 const ChannelHandle& handle) {
   return ReportErrorOrReturn([&]() -> StatusOr {
+    if (handle.type() != ChannelHandle::DEVICE_TO_DEVICE) {
+      return InvalidArgument("Send must use a device-to-device channel");
+    }
+
     // Send instruction produces a tuple of {aliased operand, U32 context,
     // token}.
     HloInstructionProto send_instr;
@@ -1992,6 +1981,27 @@ XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) {
     TF_ASSIGN_OR_RETURN(XlaOp token, AddInstruction(std::move(token_instr),
                                                     HloOpcode::kAfterAll, {}));
 
+    XlaOp recv = RecvWithToken(token, shape, handle);
+
+    // The RecvDone instruction produces a tuple of the data and a token
+    // type. Return XLA op containing the data.
+    // TODO(b/80000000): Remove this when clients have been updated to handle
+    // tokens.
+    HloInstructionProto recv_data;
+    *recv_data.mutable_shape() = shape;
+    recv_data.set_tuple_index(0);
+    return AddInstruction(std::move(recv_data), HloOpcode::kGetTupleElement,
+                          {recv});
+  });
+}
+
+XlaOp XlaBuilder::RecvWithToken(const XlaOp& token, const Shape& shape,
+                                const ChannelHandle& handle) {
+  return ReportErrorOrReturn([&]() -> StatusOr {
+    if (handle.type() != ChannelHandle::DEVICE_TO_DEVICE) {
+      return InvalidArgument("Recv must use a device-to-device channel");
+    }
+
     // Recv instruction produces a tuple of {receive buffer, U32 context,
     // token}.
     HloInstructionProto recv_instr;
@@ -2005,31 +2015,81 @@ XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) {
     *recv_done_instr.mutable_shape() =
         ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()});
     recv_done_instr.set_channel_id(handle.handle());
-    TF_ASSIGN_OR_RETURN(XlaOp recv_done,
-                        AddInstruction(std::move(recv_done_instr),
-                                       HloOpcode::kRecvDone, {recv}));
+    return AddInstruction(std::move(recv_done_instr), HloOpcode::kRecvDone,
+                          {recv});
+  });
+}
 
-    // The RecvDone instruction produces a tuple of the data and a token
-    // type. Return XLA op containing the data.
-    // TODO(b/80000000): Remove this when clients have been updated to handle
-    // tokens.
-    HloInstructionProto recv_data;
-    *recv_data.mutable_shape() = shape;
-    recv_data.set_tuple_index(0);
-    return AddInstruction(std::move(recv_data), HloOpcode::kGetTupleElement,
-                          {recv_done});
+XlaOp XlaBuilder::SendToHost(const XlaOp& operand, const XlaOp& token,
+                             const Shape& shape_with_layout,
+                             const ChannelHandle& handle) {
+  return ReportErrorOrReturn([&]() -> StatusOr {
+    if (!LayoutUtil::HasLayout(shape_with_layout)) {
+      return InvalidArgument("Shape passed to SendToHost must have a layout");
+    }
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+    if (!ShapeUtil::Compatible(operand_shape, shape_with_layout)) {
+      return InvalidArgument(
+          "SendToHost shape %s must be compatible with operand shape %s",
+          ShapeUtil::HumanStringWithLayout(shape_with_layout).c_str(),
+          ShapeUtil::HumanStringWithLayout(operand_shape).c_str());
+    }
+    // TODO(b/111544877): Support tuple shapes.
+    if (!ShapeUtil::IsArray(operand_shape)) {
+      return InvalidArgument("SendToHost only supports array shapes, shape: %s",
+                             ShapeUtil::HumanString(operand_shape).c_str());
+    }
+
+    if (handle.type() != ChannelHandle::DEVICE_TO_HOST) {
+      return InvalidArgument("SendToHost must use a device-to-host channel");
+    }
+
+    // Send instruction produces a tuple of {aliased operand, U32 context,
+    // token}.
+    HloInstructionProto send_instr;
+    *send_instr.mutable_shape() = ShapeUtil::MakeTupleShape(
+        {shape_with_layout, ShapeUtil::MakeShape(U32, {}),
+         ShapeUtil::MakeTokenShape()});
+    send_instr.set_channel_id(handle.handle());
+    send_instr.set_is_host_transfer(true);
+    TF_ASSIGN_OR_RETURN(XlaOp send,
+                        AddInstruction(std::move(send_instr), HloOpcode::kSend,
+                                       {operand, token}));
+
+    HloInstructionProto send_done_instr;
+    *send_done_instr.mutable_shape() = ShapeUtil::MakeTokenShape();
+    send_done_instr.set_channel_id(handle.handle());
+    send_done_instr.set_is_host_transfer(true);
+    return AddInstruction(std::move(send_done_instr), HloOpcode::kSendDone,
+                          {send});
   });
 }
 
-XlaOp XlaBuilder::RecvWithToken(const XlaOp& token, const Shape& shape,
-                                const ChannelHandle& handle) {
+XlaOp XlaBuilder::RecvFromHost(const XlaOp& token, const Shape& shape,
+                               const ChannelHandle& handle) {
   return ReportErrorOrReturn([&]() -> StatusOr {
+    if (!LayoutUtil::HasLayout(shape)) {
+      return InvalidArgument("Shape passed to RecvFromHost must have a layout");
+    }
+
+    // TODO(b/111544877): Support tuple shapes.
+    if (!ShapeUtil::IsArray(shape)) {
+      return InvalidArgument(
+          "RecvFromHost only supports array shapes, shape: %s",
+          ShapeUtil::HumanString(shape).c_str());
+    }
+
+    if (handle.type() != ChannelHandle::HOST_TO_DEVICE) {
+      return InvalidArgument("RecvFromHost must use a host-to-device channel");
+    }
+
     // Recv instruction produces a tuple of {receive buffer, U32 context,
     // token}.
     HloInstructionProto recv_instr;
     *recv_instr.mutable_shape() = ShapeUtil::MakeTupleShape(
         {shape, ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()});
     recv_instr.set_channel_id(handle.handle());
+    recv_instr.set_is_host_transfer(true);
     TF_ASSIGN_OR_RETURN(XlaOp recv, AddInstruction(std::move(recv_instr),
                                                    HloOpcode::kRecv, {token}));
 
@@ -2037,6 +2097,7 @@ XlaOp XlaBuilder::RecvWithToken(const XlaOp& token, const Shape& shape,
     *recv_done_instr.mutable_shape() =
         ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeTokenShape()});
     recv_done_instr.set_channel_id(handle.handle());
+    recv_done_instr.set_is_host_transfer(true);
     return AddInstruction(std::move(recv_done_instr), HloOpcode::kRecvDone,
                           {recv});
   });
@@ -2760,6 +2821,17 @@ XlaOp RecvWithToken(const XlaOp& token, const Shape& shape,
   return token.builder()->RecvWithToken(token, shape, handle);
 }
 
+XlaOp SendToHost(const XlaOp& operand, const XlaOp& token,
+                 const Shape& shape_with_layout, const ChannelHandle& handle) {
+  return operand.builder()->SendToHost(operand, token, shape_with_layout,
+                                       handle);
+}
+
+XlaOp RecvFromHost(const XlaOp& token, const Shape& shape,
+                   const ChannelHandle& handle) {
+  return token.builder()->RecvFromHost(token, shape, handle);
+}
+
 XlaOp InfeedWithToken(const XlaOp& token, const Shape& shape,
                       const string& config) {
   return token.builder()->InfeedWithToken(token, shape, config);
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index 2be6f4a553..445c1e0d77 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -848,12 +848,21 @@ class XlaBuilder {
                const GatherDimensionNumbers& dimension_numbers,
                tensorflow::gtl::ArraySlice window_bounds);
 
-  // Enqueues a Send node onto the computation, to send the given operand to
-  // a Recv instruction that shares the same channel handle.
+  // Enqueues a Send node onto the computation for device-to-device
+  // communication, to send the given operand to a Recv instruction that shares
+  // the same channel handle.
   void Send(const XlaOp& operand, const ChannelHandle& handle);
   XlaOp SendWithToken(const XlaOp& operand, const XlaOp& token,
                       const ChannelHandle& handle);
 
+  // Enqueues a Send node which sends data to the host.
+  XlaOp SendToHost(const XlaOp& operand, const XlaOp& token,
+                   const Shape& shape_with_layout, const ChannelHandle& handle);
+
+  // Enqueues a Recv node which receives data from the host.
+  XlaOp RecvFromHost(const XlaOp& token, const Shape& shape,
+                     const ChannelHandle& handle);
+
   // Enqueues an AfterAll operation with no operands producing a token-shaped
   // value.
   XlaOp CreateToken();
@@ -1293,6 +1302,11 @@ class XlaBuilder {
                              const ChannelHandle& handle);
   friend XlaOp RecvWithToken(const XlaOp& token, const Shape& shape,
                              const ChannelHandle& handle);
+  friend XlaOp SendToHost(const XlaOp& operand, const XlaOp& token,
+                          const Shape& shape_with_layout,
+                          const ChannelHandle& handle);
+  friend XlaOp RecvFromHost(const XlaOp& token, const Shape& shape,
+                            const ChannelHandle& handle);
   friend XlaOp InfeedWithToken(const XlaOp& token, const Shape& shape,
                                const string& config);
   friend XlaOp OutfeedWithToken(const XlaOp& operand, const XlaOp& token,
@@ -1951,8 +1965,10 @@ XlaOp Gather(const XlaOp& input, const XlaOp& gather_indices,
              const GatherDimensionNumbers& dimension_numbers,
              tensorflow::gtl::ArraySlice window_bounds);
 
-// Enqueues a Send node onto the computation, to send the given operand to
-// a Recv instruction that shares the same channel handle.
+// Enqueues a Send node onto the computation for device-to-device
+// communication. This operation sends the given operand to
+// a Recv instruction in a different computation that shares the same channel
+// handle.
 void Send(const XlaOp& operand, const ChannelHandle& handle);
 
 // Variant of Send which takes a token-shaped operand and produces a
@@ -1961,9 +1977,10 @@ void Send(const XlaOp& operand, const ChannelHandle& handle);
 XlaOp SendWithToken(const XlaOp& operand, const XlaOp& token,
                     const ChannelHandle& handle);
 
-// Enqueues a Recv node onto the computation. The data comes from a Send
-// instruction that shares the same channel handle and its shape must
-// be the same as the given shape.
+// Enqueues a Recv node onto the computation for device-to-device
+// communication. The data comes from a Send instruction in a different
+// computation that shares the same channel handle and its shape must be the
+// same as the given shape.
 XlaOp Recv(XlaBuilder* builder, const Shape& shape,
            const ChannelHandle& handle);
 
@@ -1974,6 +1991,20 @@ XlaOp Recv(XlaBuilder* builder, const Shape& shape,
 XlaOp RecvWithToken(const XlaOp& token, const Shape& shape,
                     const ChannelHandle& handle);
 
+// Enqueues a Send node which transfers data from the device to the host. The
+// 'shape_with_layout' argument defines the layout of the data transferred; its
+// shape must be compatible with the shape of the operand. The operand must be
+// array-shaped.
+// TODO(b/111544877): Support tuple shapes.
+XlaOp SendToHost(const XlaOp& operand, const XlaOp& token,
+                 const Shape& shape_with_layout, const ChannelHandle& handle);
+
+// Enqueues a Recv node which transfers data from the host to the device. The
+// given shape must contain a layout and must be an array.
+// TODO(b/111544877): Support tuple shapes.
+XlaOp RecvFromHost(const XlaOp& token, const Shape& shape,
+                   const ChannelHandle& handle);
+
 // Enqueues an operation (AfterAll) with no operands that produces a
 // token-shaped value.  Tokens are used for ordering side-effecting operations.
 // This is a separate method from AfterAll to facility the removal of
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 989bb759e3..cba7883fde 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1057,6 +1057,7 @@ cc_library(
     hdrs = ["hlo_module_group_metadata.h"],
     deps = [
         ":hlo",
+        ":hlo_casting_utils",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status",
         "//tensorflow/compiler/xla:status_macros",
@@ -2010,6 +2011,7 @@ cc_library(
     deps = [
         ":computation_layout",
         ":hlo",
+        ":hlo_casting_utils",
         ":hlo_dce",
         ":hlo_graph_dumper",
         ":hlo_pass",
diff --git a/tensorflow/compiler/xla/service/channel_tracker.cc b/tensorflow/compiler/xla/service/channel_tracker.cc
index a5b392cbc3..13008efed1 100644
--- a/tensorflow/compiler/xla/service/channel_tracker.cc
+++ b/tensorflow/compiler/xla/service/channel_tracker.cc
@@ -31,16 +31,23 @@ namespace xla {
 
 ChannelTracker::ChannelTracker() : next_channel_(1) {}
 
-ChannelHandle ChannelTracker::NewChannel() {
+StatusOr ChannelTracker::NewChannel(
+    ChannelHandle::ChannelType type) {
+  if (type != ChannelHandle::DEVICE_TO_DEVICE &&
+      type != ChannelHandle::HOST_TO_DEVICE &&
+      type != ChannelHandle::DEVICE_TO_HOST) {
+    return InvalidArgument("Invalid channel type: %d", type);
+  }
   tensorflow::mutex_lock lock(channel_mutex_);
 
   // Create a new channel handle with a unique value.
-  const ChannelHandle new_handle = AllocateHandle();
+  ChannelHandle new_handle = AllocateHandle(type);
 
   // Register a channel object associated with the handle.
   Channel channel;
   channel.has_sender = false;
   channel.receiver_count = 0;
+  channel.type = type;
   opaque_to_channel_[new_handle.handle()] = channel;
 
   return new_handle;
@@ -56,10 +63,11 @@ Status ChannelTracker::RegisterRecv(const ChannelHandle& handle) {
   return RegisterRecvInternal(handle);
 }
 
-ChannelHandle ChannelTracker::AllocateHandle() {
+ChannelHandle ChannelTracker::AllocateHandle(ChannelHandle::ChannelType type) {
   int64 handle_value = next_channel_++;
   ChannelHandle result;
   result.set_handle(handle_value);
+  result.set_type(type);
   return result;
 }
 
@@ -68,6 +76,13 @@ Status ChannelTracker::RegisterSendInternal(const ChannelHandle& handle) {
     return NotFound("channel handle not found: %lld", handle.handle());
   }
   Channel& channel = opaque_to_channel_[handle.handle()];
+  if (channel.type == ChannelHandle::HOST_TO_DEVICE) {
+    return FailedPrecondition(
+        "host-to-device channels cannot be used with a Send operation; "
+        "channel handle: %lld",
+        handle.handle());
+  }
+
   if (channel.has_sender) {
     return FailedPrecondition(
         "when registering send, passed a channel handle that is already used "
@@ -83,6 +98,13 @@ Status ChannelTracker::RegisterRecvInternal(const ChannelHandle& handle) {
     return NotFound("channel handle not found: %lld", handle.handle());
   }
   Channel& channel = opaque_to_channel_[handle.handle()];
+  if (channel.type == ChannelHandle::DEVICE_TO_HOST) {
+    return FailedPrecondition(
+        "device-to-host channels cannot be used with a Recv operation; "
+        "channel handle: %lld",
+        handle.handle());
+  }
+
   // TODO(b/33942691): Allow more than 1 receivers for broadcast.
   if (channel.receiver_count >= 1) {
     return FailedPrecondition(
diff --git a/tensorflow/compiler/xla/service/channel_tracker.h b/tensorflow/compiler/xla/service/channel_tracker.h
index fac0afd672..d773558c28 100644
--- a/tensorflow/compiler/xla/service/channel_tracker.h
+++ b/tensorflow/compiler/xla/service/channel_tracker.h
@@ -48,11 +48,12 @@ class ChannelTracker {
   struct Channel {
     bool has_sender;
     int64 receiver_count;
+    ChannelHandle::ChannelType type;
   };
 
   // Creates a new Channel object and returns the corresponding
   // ChannelHandle for it.
-  ChannelHandle NewChannel();
+  StatusOr NewChannel(ChannelHandle::ChannelType type);
 
   // Informs that the given channel handle is used for a Send operation.
   // Returns an error status if the handle is already used by another Send.
@@ -65,7 +66,8 @@ class ChannelTracker {
  private:
   // Bumps the next_channel_ number and returns the allocated number
   // wrapped in a ChannelHandle.
-  ChannelHandle AllocateHandle() EXCLUSIVE_LOCKS_REQUIRED(channel_mutex_);
+  ChannelHandle AllocateHandle(ChannelHandle::ChannelType type)
+      EXCLUSIVE_LOCKS_REQUIRED(channel_mutex_);
 
   Status RegisterSendInternal(const ChannelHandle& handle)
       EXCLUSIVE_LOCKS_REQUIRED(channel_mutex_);
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index d241791060..87abc0e74f 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -155,6 +155,11 @@ message HloInstructionProto {
   repeated int64 replica_group_ids = 44;
   int64 all_reduce_id = 45;
   string cross_replica_sum_barrier = 46;
+
+  // Whether this Send/Recv instruction transfers data to/from the host. Only
+  // present for Send and Recv instructions and their SendDone and RecvDone
+  // partners.
+  bool is_host_transfer = 47;
 }
 
 // Serialization of HloComputation.
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 02139facdb..ae30d2ad8d 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -115,26 +115,27 @@ StatusOr> HloInstruction::CreateFromProto(
       TF_RET_CHECK(proto.operand_ids_size() == 2)
           << "Send instruction should have 2 operand but sees "
           << proto.operand_ids_size();
-      instruction = CreateSend(operands(0), operands(1), proto.channel_id());
+      instruction = CreateSend(operands(0), operands(1), proto.channel_id(),
+                               proto.is_host_transfer());
       break;
     case HloOpcode::kSendDone:
       TF_RET_CHECK(proto.operand_ids_size() == 1)
           << "SendDone instruction should have 1 operand but sees "
           << proto.operand_ids_size();
-      instruction = CreateSendDone(operands(0));
+      instruction = CreateSendDone(operands(0), proto.is_host_transfer());
       break;
     case HloOpcode::kRecv:
       TF_RET_CHECK(proto.operand_ids_size() == 1)
           << "Recv instruction should have 1 operand but sees "
           << proto.operand_ids_size();
       instruction = CreateRecv(proto.shape().tuple_shapes(0), operands(0),
-                               proto.channel_id());
+                               proto.channel_id(), proto.is_host_transfer());
       break;
     case HloOpcode::kRecvDone:
       TF_RET_CHECK(proto.operand_ids_size() == 1)
           << "RecvDone instruction should have 1 operand but sees "
           << proto.operand_ids_size();
-      instruction = CreateRecvDone(operands(0));
+      instruction = CreateRecvDone(operands(0), proto.is_host_transfer());
       break;
     case HloOpcode::kReverse:
       TF_RET_CHECK(proto.operand_ids_size() == 1)
@@ -675,29 +676,33 @@ HloInstruction::CreateCrossReplicaSum(
 }
 
 /* static */ std::unique_ptr HloInstruction::CreateSend(
-    HloInstruction* operand, HloInstruction* token, int64 channel_id) {
-  return MakeUnique(operand, token, channel_id);
+    HloInstruction* operand, HloInstruction* token, int64 channel_id,
+    bool is_host_transfer) {
+  return MakeUnique(operand, token, channel_id,
+                                        is_host_transfer);
 }
 
 /* static */ std::unique_ptr HloInstruction::CreateSendDone(
-    HloInstruction* operand) {
+    HloInstruction* operand, bool is_host_transfer) {
   auto send_operand = DynCast(operand);
   CHECK(send_operand != nullptr)
       << "SendDone must take the context operand from Send";
-  return MakeUnique(send_operand);
+  return MakeUnique(send_operand, is_host_transfer);
 }
 
 /* static */ std::unique_ptr HloInstruction::CreateRecv(
-    const Shape& shape, HloInstruction* token, int64 channel_id) {
-  return MakeUnique(shape, token, channel_id);
+    const Shape& shape, HloInstruction* token, int64 channel_id,
+    bool is_host_transfer) {
+  return MakeUnique(shape, token, channel_id,
+                                        is_host_transfer);
 }
 
 /* static */ std::unique_ptr HloInstruction::CreateRecvDone(
-    HloInstruction* operand) {
+    HloInstruction* operand, bool is_host_transfer) {
   auto recv_operand = DynCast(operand);
   CHECK(recv_operand != nullptr)
       << "RecvDone must take the context operand from Recv";
-  return MakeUnique(recv_operand);
+  return MakeUnique(recv_operand, is_host_transfer);
 }
 
 /* static */ std::unique_ptr HloInstruction::CreateReverse(
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 180b2fb359..c6faa69a78 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -485,27 +485,30 @@ class HloInstruction {
 
   // Creates an asynchronous send instruction with the given channel id, which
   // initiates sending the operand data to a unique receive instruction in
-  // another computation that has the same channel id.
-  static std::unique_ptr CreateSend(HloInstruction* operand,
-                                                    HloInstruction* token,
-                                                    int64 channel_id);
+  // another computation that has the same channel id. If is_host_transfer is
+  // true, then this Send operation transfers data to the host.
+  static std::unique_ptr CreateSend(
+      HloInstruction* operand, HloInstruction* token, int64 channel_id,
+      bool is_host_transfer = false);
 
   // Blocks until data transfer for the Send instruction (operand) is complete.
   // The operand must be kSend.
   static std::unique_ptr CreateSendDone(
-      HloInstruction* operand);
+      HloInstruction* operand, bool is_host_transfer = false);
 
   // Creates an asynchronous receive instruction with the given channel id,
   // which allocates resources to receive data of the given shape from a unique
-  // send instruction in another computation that has the same channel id.
-  static std::unique_ptr CreateRecv(const Shape& shape,
-                                                    HloInstruction* token,
-                                                    int64 channel_id);
+  // send instruction in another computation that has the same channel id.  If
+  // is_host_transfer is true, then this Send operation transfers data from the
+  // host.
+  static std::unique_ptr CreateRecv(
+      const Shape& shape, HloInstruction* token, int64 channel_id,
+      bool is_host_transfer = false);
 
   // Blocks until data transfer for the Recv instruction (operand) is complete
   // and returns the receive buffer. The operand must be kRecv.
   static std::unique_ptr CreateRecvDone(
-      HloInstruction* operand);
+      HloInstruction* operand, bool is_host_transfer = false);
 
   // Creates a slice instruction, where the operand is sliced by the given
   // start/limit indices.
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc
index 702f808449..df26a2c744 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.cc
+++ b/tensorflow/compiler/xla/service/hlo_instructions.cc
@@ -181,8 +181,11 @@ std::unique_ptr HloFftInstruction::CloneWithNewOperandsImpl(
 
 HloSendRecvInstruction::HloSendRecvInstruction(HloOpcode opcode,
                                                const Shape& shape,
-                                               int64 channel_id)
-    : HloInstruction(opcode, shape), channel_id_(channel_id) {}
+                                               int64 channel_id,
+                                               bool is_host_transfer)
+    : HloInstruction(opcode, shape),
+      channel_id_(channel_id),
+      is_host_transfer_(is_host_transfer) {}
 
 HloInstructionProto HloSendRecvInstruction::ToProto() const {
   HloInstructionProto proto = HloInstruction::ToProto();
@@ -192,7 +195,12 @@ HloInstructionProto HloSendRecvInstruction::ToProto() const {
 
 std::vector HloSendRecvInstruction::ExtraAttributesToStringImpl(
     const HloPrintOptions& options) const {
-  return {StrCat("channel_id=", channel_id_)};
+  std::vector attrs;
+  attrs.push_back(StrCat("channel_id=", channel_id_));
+  if (is_host_transfer()) {
+    attrs.push_back("is_host_transfer=true");
+  }
+  return attrs;
 }
 
 bool HloSendRecvInstruction::IdenticalSlowPath(
@@ -205,13 +213,14 @@ bool HloSendRecvInstruction::IdenticalSlowPath(
 
 // Send instruction produces a tuple of {aliased operand, U32 context}.
 HloSendInstruction::HloSendInstruction(HloInstruction* operand,
-                                       HloInstruction* token, int64 channel_id)
+                                       HloInstruction* token, int64 channel_id,
+                                       bool is_host_transfer)
     : HloSendRecvInstruction(
           HloOpcode::kSend,
           ShapeUtil::MakeTupleShape({CHECK_NOTNULL(operand)->shape(),
                                      ShapeUtil::MakeShape(U32, {}),
                                      ShapeUtil::MakeTokenShape()}),
-          channel_id) {
+          channel_id, is_host_transfer) {
   AppendOperand(operand);
   AppendOperand(token);
 }
@@ -222,12 +231,14 @@ std::unique_ptr HloSendInstruction::CloneWithNewOperandsImpl(
     HloCloneContext* context) const {
   CHECK_EQ(new_operands.size(), 2);
   return MakeUnique(new_operands[0], new_operands[1],
-                                        channel_id());
+                                        channel_id(), is_host_transfer());
 }
 
-HloSendDoneInstruction::HloSendDoneInstruction(HloSendInstruction* operand)
+HloSendDoneInstruction::HloSendDoneInstruction(HloSendInstruction* operand,
+                                               bool is_host_transfer)
     : HloSendRecvInstruction(HloOpcode::kSendDone, ShapeUtil::MakeTokenShape(),
-                             CHECK_NOTNULL(operand)->channel_id()) {
+                             CHECK_NOTNULL(operand)->channel_id(),
+                             is_host_transfer) {
   AppendOperand(operand);
 }
 
@@ -238,17 +249,18 @@ HloSendDoneInstruction::CloneWithNewOperandsImpl(
     HloCloneContext* context) const {
   CHECK_EQ(new_operands.size(), 1);
   return MakeUnique(
-      Cast(new_operands[0]));
+      Cast(new_operands[0]), is_host_transfer());
 }
 
 // Recv instruction produces a tuple of {receive buffer, U32 context}.
 HloRecvInstruction::HloRecvInstruction(const Shape& shape,
-                                       HloInstruction* token, int64 channel_id)
+                                       HloInstruction* token, int64 channel_id,
+                                       bool is_host_transfer)
     : HloSendRecvInstruction(
           HloOpcode::kRecv,
           ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeShape(U32, {}),
                                      ShapeUtil::MakeTokenShape()}),
-          channel_id) {
+          channel_id, is_host_transfer) {
   AppendOperand(token);
 }
 
@@ -258,16 +270,18 @@ std::unique_ptr HloRecvInstruction::CloneWithNewOperandsImpl(
     HloCloneContext* context) const {
   CHECK_EQ(new_operands.size(), 1);
   return MakeUnique(
-      ShapeUtil::GetTupleElementShape(shape, 0), new_operands[0], channel_id());
+      ShapeUtil::GetTupleElementShape(shape, 0), new_operands[0], channel_id(),
+      is_host_transfer());
 }
 
-HloRecvDoneInstruction::HloRecvDoneInstruction(HloRecvInstruction* operand)
+HloRecvDoneInstruction::HloRecvDoneInstruction(HloRecvInstruction* operand,
+                                               bool is_host_transfer)
     : HloSendRecvInstruction(
           HloOpcode::kRecvDone,
           ShapeUtil::MakeTupleShape(
               {ShapeUtil::GetTupleElementShape(operand->shape(), 0),
                ShapeUtil::MakeTokenShape()}),
-          CHECK_NOTNULL(operand)->channel_id()) {
+          CHECK_NOTNULL(operand)->channel_id(), is_host_transfer) {
   AppendOperand(operand);
 }
 
@@ -278,7 +292,7 @@ HloRecvDoneInstruction::CloneWithNewOperandsImpl(
     HloCloneContext* context) const {
   CHECK_EQ(new_operands.size(), 1);
   return MakeUnique(
-      Cast(new_operands[0]));
+      Cast(new_operands[0]), is_host_transfer());
 }
 
 HloAllReduceInstruction::HloAllReduceInstruction(
diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h
index 65a93cdcf1..e4031f04d5 100644
--- a/tensorflow/compiler/xla/service/hlo_instructions.h
+++ b/tensorflow/compiler/xla/service/hlo_instructions.h
@@ -141,12 +141,15 @@ class HloSendRecvInstruction : public HloInstruction {
   // channel.
   int64 channel_id() const { return channel_id_; }
 
+  // Returns whether this send/recv instruction sends data to/from the host.
+  bool is_host_transfer() const { return is_host_transfer_; }
+
   // Returns a serialized representation of this instruction.
   HloInstructionProto ToProto() const override;
 
  protected:
   explicit HloSendRecvInstruction(HloOpcode opcode, const Shape& shape,
-                                  int64 channel_id);
+                                  int64 channel_id, bool is_host_transfer);
 
  private:
   std::vector ExtraAttributesToStringImpl(
@@ -157,12 +160,15 @@ class HloSendRecvInstruction : public HloInstruction {
           eq_computations) const override;
   // Represents a unique identifier for each Send/Recv instruction pair.
   int64 channel_id_;
+
+  // Whether this send/recv instruction sends data to/from the host.
+  bool is_host_transfer_;
 };
 
 class HloSendInstruction : public HloSendRecvInstruction {
  public:
   explicit HloSendInstruction(HloInstruction* operand, HloInstruction* token,
-                              int64 channel_id);
+                              int64 channel_id, bool is_host_transfer);
 
  private:
   // Implementation for non-common logic of CloneWithNewOperands.
@@ -174,7 +180,8 @@ class HloSendInstruction : public HloSendRecvInstruction {
 
 class HloSendDoneInstruction : public HloSendRecvInstruction {
  public:
-  explicit HloSendDoneInstruction(HloSendInstruction* operand);
+  explicit HloSendDoneInstruction(HloSendInstruction* operand,
+                                  bool is_host_transfer);
 
  private:
   // Implementation for non-common logic of CloneWithNewOperands.
@@ -187,7 +194,7 @@ class HloSendDoneInstruction : public HloSendRecvInstruction {
 class HloRecvInstruction : public HloSendRecvInstruction {
  public:
   explicit HloRecvInstruction(const Shape& shape, HloInstruction* token,
-                              int64 channel_id);
+                              int64 channel_id, bool is_host_transfer);
 
  private:
   // Implementation for non-common logic of CloneWithNewOperands.
@@ -199,7 +206,8 @@ class HloRecvInstruction : public HloSendRecvInstruction {
 
 class HloRecvDoneInstruction : public HloSendRecvInstruction {
  public:
-  explicit HloRecvDoneInstruction(HloRecvInstruction* operand);
+  explicit HloRecvDoneInstruction(HloRecvInstruction* operand,
+                                  bool is_host_transfer);
 
  private:
   // Implementation for non-common logic of CloneWithNewOperands.
diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
index 3ffac2f413..10bf9ffd6c 100644
--- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
+++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
@@ -20,6 +20,8 @@ limitations under the License.
 #include 
 
 #include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
+#include "tensorflow/compiler/xla/service/hlo_instructions.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/util.h"
@@ -175,8 +177,12 @@ bool HloModuleGroupMetadata::IsChannelInstruction(
     case HloOpcode::kSend:
     case HloOpcode::kRecv:
     case HloOpcode::kSendDone:
-    case HloOpcode::kRecvDone:
-      return true;
+    case HloOpcode::kRecvDone: {
+      const HloSendRecvInstruction* send_recv_instr =
+          DynCast(instruction);
+      CHECK(send_recv_instr != nullptr);
+      return !send_recv_instr->is_host_transfer();
+    }
     default:
       return false;
   }
diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.h b/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
index 9eea5c6a3d..84f2d3f5fb 100644
--- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
+++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.h
@@ -92,7 +92,7 @@ class HloModuleGroupMetadata {
     ComputationKind kind_ = ComputationKind::kInvalid;
   };
 
-  // Represents a channel and the 4 instructions that form the channel.
+  // Represents a channel and the instructions that form the channel.
   struct Channel {
     int64 id = -1;
     HloInstruction* send = nullptr;
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index d387539350..496eca0739 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -119,6 +119,7 @@ class HloParser {
 
   // Types of attributes.
   enum class AttrTy {
+    kBool,
     kInt64,
     kInt32,
     kFloat,
@@ -681,18 +682,27 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder,
     }
     case HloOpcode::kRecv: {
       optional channel_id;
+      // If the is_host_transfer attribute is not present then default to false.
+      optional is_host_transfer = false;
       attrs["channel_id"] = {/*required=*/true, AttrTy::kInt64, &channel_id};
+      attrs["is_host_transfer"] = {/*required=*/false, AttrTy::kBool,
+                                   &is_host_transfer};
       if (!ParseOperands(&operands, /*expected_size=*/1) ||
           !ParseAttributes(attrs)) {
         return false;
       }
+      // If the is_host_transfer attribute is not present then default to false.
       instruction = builder->AddInstruction(HloInstruction::CreateRecv(
-          shape.tuple_shapes(0), operands[0], *channel_id));
+          shape.tuple_shapes(0), operands[0], *channel_id, *is_host_transfer));
       break;
     }
     case HloOpcode::kRecvDone: {
       optional channel_id;
+      // If the is_host_transfer attribute is not present then default to false.
+      optional is_host_transfer = false;
       attrs["channel_id"] = {/*required=*/true, AttrTy::kInt64, &channel_id};
+      attrs["is_host_transfer"] = {/*required=*/false, AttrTy::kBool,
+                                   &is_host_transfer};
       if (!ParseOperands(&operands, /*expected_size=*/1) ||
           !ParseAttributes(attrs)) {
         return false;
@@ -700,24 +710,32 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder,
       if (channel_id != operands[0]->channel_id()) {
         return false;
       }
-      instruction =
-          builder->AddInstruction(HloInstruction::CreateRecvDone(operands[0]));
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateRecvDone(operands[0], *is_host_transfer));
       break;
     }
     case HloOpcode::kSend: {
       optional channel_id;
+      // If the is_host_transfer attribute is not present then default to false.
+      optional is_host_transfer = false;
       attrs["channel_id"] = {/*required=*/true, AttrTy::kInt64, &channel_id};
+      attrs["is_host_transfer"] = {/*required=*/false, AttrTy::kBool,
+                                   &is_host_transfer};
       if (!ParseOperands(&operands, /*expected_size=*/2) ||
           !ParseAttributes(attrs)) {
         return false;
       }
-      instruction = builder->AddInstruction(
-          HloInstruction::CreateSend(operands[0], operands[1], *channel_id));
+      instruction = builder->AddInstruction(HloInstruction::CreateSend(
+          operands[0], operands[1], *channel_id, *is_host_transfer));
       break;
     }
     case HloOpcode::kSendDone: {
       optional channel_id;
+      // If the is_host_transfer attribute is not present then default to false.
+      optional is_host_transfer = false;
       attrs["channel_id"] = {/*required=*/true, AttrTy::kInt64, &channel_id};
+      attrs["is_host_transfer"] = {/*required=*/false, AttrTy::kBool,
+                                   &is_host_transfer};
       if (!ParseOperands(&operands, /*expected_size=*/1) ||
           !ParseAttributes(attrs)) {
         return false;
@@ -725,8 +743,8 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder,
       if (channel_id != operands[0]->channel_id()) {
         return false;
       }
-      instruction =
-          builder->AddInstruction(HloInstruction::CreateSendDone(operands[0]));
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateSendDone(operands[0], *is_host_transfer));
       break;
     }
     case HloOpcode::kGetTupleElement: {
@@ -2043,6 +2061,14 @@ bool HloParser::ParseAttributeHelper(
   bool success = [&] {
     LocTy attr_loc = lexer_.GetLoc();
     switch (attr_type) {
+      case AttrTy::kBool: {
+        bool result;
+        if (!ParseBool(&result)) {
+          return false;
+        }
+        static_cast*>(attr_out_ptr)->emplace(result);
+        return true;
+      }
       case AttrTy::kInt64: {
         tensorflow::int64 result;
         if (!ParseInt64(&result)) {
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index f06c705c42..6ba34cf22a 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -286,6 +286,21 @@ ENTRY %TwoSendRecvBothWayRecvFist.v3 () -> (f32[], token[]) {
   %send-done = token[] send-done((f32[], u32[], token[]) %send), channel_id=16, sharding={maximal device=0}
 }
 
+)"
+},
+{
+"SendRecvWithHostTransfer",
+R"(HloModule HostTransferSendRecv_module
+
+ENTRY %TwoSendRecvBothWayRecvFist.v3 () -> (f32[], token[]) {
+  %token = token[] after-all()
+  %recv = (f32[], u32[], token[]) recv(token[] %token), channel_id=15, is_host_transfer=true
+  ROOT %recv-done = (f32[], token[]) recv-done((f32[], u32[], token[]) %recv), channel_id=15, is_host_transfer=true
+  %constant = f32[] constant(2.1), sharding={maximal device=0}
+  %send = (f32[], u32[], token[]) send(f32[] %constant, token[] %token), channel_id=16, is_host_transfer=true
+  %send-done = token[] send-done((f32[], u32[], token[]) %send), channel_id=16, is_host_transfer=true
+}
+
 )"
 },
 // get-tuple-element
diff --git a/tensorflow/compiler/xla/service/hlo_scheduling.cc b/tensorflow/compiler/xla/service/hlo_scheduling.cc
index c6d3909af6..27cc5361cd 100644
--- a/tensorflow/compiler/xla/service/hlo_scheduling.cc
+++ b/tensorflow/compiler/xla/service/hlo_scheduling.cc
@@ -567,6 +567,7 @@ StatusOr ScheduleComputationsInModule(
       sequence[computation] = std::move(one_computation_sequence);
     }
   }
+  VLOG(1) << "Module schedule:\n" << sequence;
   return sequence;
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 48eeba6afd..6a32093b6e 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -382,11 +382,6 @@ Status ShapeVerifier::HandlePad(HloInstruction* pad) {
 }
 
 Status ShapeVerifier::HandleSend(HloInstruction* send) {
-  TF_RET_CHECK(send->users().size() == 1);
-  const HloInstruction* send_done = send->users().front();
-  TF_RET_CHECK(send_done->opcode() == HloOpcode::kSendDone);
-  TF_RETURN_IF_ERROR(CheckSameChannel(send, send_done));
-  TF_RETURN_IF_ERROR(CheckIsTokenOperand(send, 1));
   return CheckShape(send,
                     ShapeUtil::MakeTupleShape({send->operand(0)->shape(),
                                                ShapeUtil::MakeShape(U32, {}),
@@ -394,34 +389,22 @@ Status ShapeVerifier::HandleSend(HloInstruction* send) {
 }
 
 Status ShapeVerifier::HandleSendDone(HloInstruction* send_done) {
-  TF_RET_CHECK(send_done->operands().size() == 1);
-  const HloInstruction* send = send_done->operand(0);
-  TF_RET_CHECK(send->opcode() == HloOpcode::kSend);
-  TF_RETURN_IF_ERROR(CheckSameChannel(send, send_done));
-
   return CheckShape(send_done, ShapeUtil::MakeTokenShape());
 }
 
 Status ShapeVerifier::HandleRecv(HloInstruction* recv) {
-  TF_RET_CHECK(recv->users().size() == 1);
-  const HloInstruction* recv_done = recv->users().front();
-  TF_RET_CHECK(recv_done->opcode() == HloOpcode::kRecvDone);
-  TF_RETURN_IF_ERROR(CheckSameChannel(recv, recv_done));
-  TF_RETURN_IF_ERROR(CheckIsTokenOperand(recv, 0));
   return CheckShape(
       recv, ShapeUtil::MakeTupleShape(
-                {ShapeUtil::GetTupleElementShape(recv_done->shape(), 0),
+                {ShapeUtil::GetTupleElementShape(recv->shape(), 0),
                  ShapeUtil::MakeShape(U32, {}), ShapeUtil::MakeTokenShape()}));
 }
 
 Status ShapeVerifier::HandleRecvDone(HloInstruction* recv_done) {
-  TF_RET_CHECK(recv_done->operands().size() == 1);
-  const HloInstruction* recv = recv_done->operand(0);
-  TF_RET_CHECK(recv->opcode() == HloOpcode::kRecv);
-  TF_RETURN_IF_ERROR(CheckSameChannel(recv, recv_done));
-  return CheckShape(recv_done,
-                    ShapeUtil::MakeTupleShape({recv->shape().tuple_shapes(0),
-                                               ShapeUtil::MakeTokenShape()}));
+  return CheckShape(
+      recv_done,
+      ShapeUtil::MakeTupleShape(
+          {ShapeUtil::GetTupleElementShape(recv_done->operand(0)->shape(), 0),
+           ShapeUtil::MakeTokenShape()}));
 }
 
 Status ShapeVerifier::HandleBatchNormTraining(
@@ -627,19 +610,6 @@ Status ShapeVerifier::CheckVariadicShape(const HloInstruction* instruction) {
                         instruction->opcode(), instruction->operands()));
 }
 
-// Checks if the given two instructions shares the same channel id.
-Status ShapeVerifier::CheckSameChannel(const HloInstruction* instr1,
-                                       const HloInstruction* instr2) {
-  if (instr1->channel_id() != instr2->channel_id()) {
-    return InternalError(
-        "Expected to have the same channel id, actual channel ids are: %s "
-        "(%lld), %s (%lld)",
-        instr1->ToString().c_str(), instr1->channel_id(),
-        instr2->ToString().c_str(), instr2->channel_id());
-  }
-  return Status::OK();
-}
-
 string ComputationsToString(
     tensorflow::gtl::ArraySlice computations) {
   return tensorflow::str_util::Join(
@@ -908,10 +878,105 @@ Status VerifyEntryAndExitShapes(const HloModule& module) {
   return Status::OK();
 }
 
+// Checks if the given two instructions share the same channel id.
+Status CheckSameChannel(const HloInstruction* instr1,
+                        const HloInstruction* instr2) {
+  if (instr1->channel_id() != instr2->channel_id()) {
+    return InternalError(
+        "Expected to have the same channel id, actual channel ids are: %s "
+        "(%lld), %s (%lld)",
+        instr1->ToString().c_str(), instr1->channel_id(),
+        instr2->ToString().c_str(), instr2->channel_id());
+  }
+  return Status::OK();
+}
+
+// Checks if the given two instructions have the same is_host_transfer attribute
+// value. Intsructions must be send/recv instructions or their 'done' variant.
+Status CheckSameIsHostTransfer(const HloInstruction* instr1,
+                               const HloInstruction* instr2) {
+  const HloSendRecvInstruction* send_recv1 =
+      DynCast(instr1);
+  const HloSendRecvInstruction* send_recv2 =
+      DynCast(instr2);
+  TF_RET_CHECK(send_recv1 != nullptr);
+  TF_RET_CHECK(send_recv2 != nullptr);
+  if (send_recv1->is_host_transfer() != send_recv2->is_host_transfer()) {
+    return InternalError(
+        "Expected instructions to have the same is-host-transfer property: %s, "
+        "%s ",
+        instr1->ToString().c_str(), instr2->ToString().c_str());
+  }
+  return Status::OK();
+}
+
+// Checks various invariants of send and recv instructions.
+Status VerifySendsAndRecvs(const HloModule& module) {
+  tensorflow::gtl::FlatMap host_channels;
+  // Host send/recv instructions must have their own unique channel.
+  auto check_unique_host_channel = [&](const HloInstruction* instruction) {
+    const HloSendRecvInstruction* sendrecv =
+        DynCast(instruction);
+    if (sendrecv->is_host_transfer()) {
+      auto it_inserted =
+          host_channels.insert({sendrecv->channel_id(), sendrecv});
+      if (!it_inserted.second) {
+        return FailedPrecondition(
+            "Channel %lld is used for multiple host send/recv instructions: %s "
+            "and "
+            "%s",
+            sendrecv->channel_id(), sendrecv->ToString().c_str(),
+            it_inserted.first->second->ToString().c_str());
+      }
+    }
+
+    return Status::OK();
+  };
+
+  // Send/Recv instruction must have a single user: the corresponding
+  // SendDone/RecvDone. with matching channel.
+  for (const HloComputation* computation : module.computations()) {
+    for (const HloInstruction* instruction : computation->instructions()) {
+      switch (instruction->opcode()) {
+        case HloOpcode::kSend: {
+          TF_RETURN_IF_ERROR(check_unique_host_channel(instruction));
+          TF_RET_CHECK(instruction->users().size() == 1);
+          const HloInstruction* send_done = instruction->users().front();
+          TF_RET_CHECK(send_done->opcode() == HloOpcode::kSendDone);
+          TF_RETURN_IF_ERROR(CheckSameChannel(instruction, send_done));
+          TF_RETURN_IF_ERROR(CheckSameIsHostTransfer(instruction, send_done));
+          break;
+        }
+        case HloOpcode::kRecv: {
+          TF_RETURN_IF_ERROR(check_unique_host_channel(instruction));
+          TF_RET_CHECK(instruction->users().size() == 1);
+          const HloInstruction* recv_done = instruction->users().front();
+          TF_RET_CHECK(recv_done->opcode() == HloOpcode::kRecvDone);
+          TF_RETURN_IF_ERROR(CheckSameChannel(instruction, recv_done));
+          TF_RETURN_IF_ERROR(CheckSameIsHostTransfer(instruction, recv_done));
+          break;
+        }
+        case HloOpcode::kSendDone:
+          TF_RET_CHECK(instruction->operands().size() == 1);
+          TF_RET_CHECK(instruction->operand(0)->opcode() == HloOpcode::kSend);
+          break;
+        case HloOpcode::kRecvDone:
+          TF_RET_CHECK(instruction->operands().size() == 1);
+          TF_RET_CHECK(instruction->operand(0)->opcode() == HloOpcode::kRecv);
+          break;
+        default:
+          break;
+      }
+    }
+  }
+  return Status::OK();
+}
+
 }  // namespace
 
 StatusOr HloVerifier::Run(HloModule* module) {
   TF_RETURN_IF_ERROR(VerifyHloStructure(module));
+  TF_RETURN_IF_ERROR(VerifySendsAndRecvs(*module));
 
   tensorflow::gtl::FlatMap instructions;
 
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.h b/tensorflow/compiler/xla/service/hlo_verifier.h
index 9e62bdc8a9..810c66cf02 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.h
+++ b/tensorflow/compiler/xla/service/hlo_verifier.h
@@ -102,10 +102,6 @@ class ShapeVerifier : public DfsHloVisitor {
   Status CheckTernaryShape(const HloInstruction* instruction);
   Status CheckVariadicShape(const HloInstruction* instruction);
 
-  // Checks if the given two instructions share the same channel id.
-  Status CheckSameChannel(const HloInstruction* instr1,
-                          const HloInstruction* instr2);
-
  private:
   // Whether the inputs and output of an instruction can contain both F32s and
   // BF16s. Tuples that include both F32s and BF16s are allowed regardless of
diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 46a6d57353..9705687b00 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -30,10 +30,12 @@ limitations under the License.
 #include "tensorflow/compiler/xla/map_util.h"
 #include "tensorflow/compiler/xla/ptr_util.h"
 #include "tensorflow/compiler/xla/service/computation_layout.h"
+#include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_dce.h"
 #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_instructions.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/logical_buffer.h"
 #include "tensorflow/compiler/xla/service/tuple_simplifier.h"
@@ -395,6 +397,43 @@ string LayoutConstraints::ToString() const {
   return output;
 }
 
+namespace {
+
+bool IsHostSendRecv(const HloInstruction* instruction) {
+  const HloSendRecvInstruction* send_recv_instr =
+      DynCast(instruction);
+  return send_recv_instr != nullptr && send_recv_instr->is_host_transfer();
+}
+
+}  // namespace
+
+Status LayoutAssignment::BuildHostChannelConstraints(
+    HloComputation* computation) {
+  for (auto* instruction : computation->instructions()) {
+    const HloSendRecvInstruction* send_recv_instr =
+        DynCast(instruction);
+    if (send_recv_instr == nullptr || !send_recv_instr->is_host_transfer()) {
+      continue;
+    }
+
+    // For host transfers the Send and Recv instruction carry the layout.
+    if (instruction->opcode() == HloOpcode::kSend ||
+        instruction->opcode() == HloOpcode::kRecv) {
+      const Shape& data_shape =
+          ShapeUtil::GetTupleElementShape(send_recv_instr->shape(), 0);
+      TF_RET_CHECK(ShapeUtil::IsArray(data_shape));
+      TF_RET_CHECK(LayoutUtil::HasLayout(data_shape));
+      const Layout* prev_layout = host_channel_constraints_.ConstrainChannel(
+          send_recv_instr->channel_id(), data_shape.layout());
+      TF_RET_CHECK(prev_layout == nullptr)
+          << "Cannot constrain host transfer layout as it was set to "
+          << LayoutUtil::HumanString(*prev_layout) << ": "
+          << send_recv_instr->ToString();
+    }
+  }
+  return Status::OK();
+}
+
 Status LayoutAssignment::AddMandatoryConstraints(
     const ComputationLayout* computation_layout,
     ChannelLayoutConstraints* channel_constraints, HloComputation* computation,
@@ -402,6 +441,11 @@ Status LayoutAssignment::AddMandatoryConstraints(
   VLOG(3) << "Adding mandatory layout constraints to computation "
           << computation->name();
 
+  auto get_channel_constraints = [&](const HloInstruction* instruction) {
+    return IsHostSendRecv(instruction) ? &host_channel_constraints_
+                                       : channel_constraints;
+  };
+
   // Constrain layouts of instructions which define values with pre-existing
   // layouts.
   for (auto* instruction : computation->instructions()) {
@@ -438,18 +482,21 @@ Status LayoutAssignment::AddMandatoryConstraints(
 
     if (instruction->opcode() == HloOpcode::kSend ||
         instruction->opcode() == HloOpcode::kRecv) {
-      CHECK(channel_constraints)
+      CHECK(get_channel_constraints(instruction))
           << "Multi-module layout assignment requires ChannelLayoutConstraints";
       int64 channel_id = instruction->channel_id();
-      if (!channel_constraints->IsChannelConstrained(channel_id)) {
+      if (!get_channel_constraints(instruction)
+               ->IsChannelConstrained(channel_id)) {
         continue;
       }
       if (instruction->opcode() == HloOpcode::kSend) {
         // TODO(b/68493863): Change to use SetOperandLayout().
         const Shape send_buffer_shape = instruction->operand(0)->shape();
         TF_RET_CHECK(ShapeUtil::IsArray(send_buffer_shape));
-        Shape new_buffer_shape = channel_constraints->LayoutShapeForChannel(
-            send_buffer_shape, instruction->channel_id());
+        Shape new_buffer_shape =
+            get_channel_constraints(instruction)
+                ->LayoutShapeForChannel(send_buffer_shape,
+                                        instruction->channel_id());
         TF_RETURN_IF_ERROR(constraints->SetInstructionLayout(
             new_buffer_shape, instruction->operand(0)));
       } else {
@@ -460,8 +507,9 @@ Status LayoutAssignment::AddMandatoryConstraints(
             const LogicalBuffer* buffer,
             constraints->points_to_analysis().GetBufferDefinedAt(instruction,
                                                                  {0}));
-        Shape new_shape = channel_constraints->LayoutShapeForChannel(
-            recv_buffer_shape, instruction->channel_id());
+        Shape new_shape = get_channel_constraints(instruction)
+                              ->LayoutShapeForChannel(
+                                  recv_buffer_shape, instruction->channel_id());
         TF_RETURN_IF_ERROR(
             constraints->SetBufferLayout(new_shape.layout(), *buffer));
       }
@@ -1538,6 +1586,10 @@ Status LayoutAssignment::RunOnComputation(
     ChannelLayoutConstraints* channel_constraints) {
   VLOG(2) << "LayoutAssignment::RunOnComputation(" << computation->name()
           << ")";
+
+  // Must be run before clearing layouts.
+  TF_RETURN_IF_ERROR(BuildHostChannelConstraints(computation));
+
   TF_RETURN_IF_ERROR(ClearComputationLayouts(computation));
   if (computation_layout != nullptr) {
     auto it = computation_layouts_.find(computation);
@@ -1627,14 +1679,20 @@ Status LayoutAssignment::RunOnComputation(
 Status LayoutAssignment::ConstrainChannelLayouts(
     HloComputation* computation,
     ChannelLayoutConstraints* channel_constraints) {
+  auto get_channel_constraints = [&](const HloInstruction* instruction) {
+    return IsHostSendRecv(instruction) ? &host_channel_constraints_
+                                       : channel_constraints;
+  };
   // We go through the kRecvDone before. These must either impose their layout,
-  // of find a matching one already existing (ConstrainChannel() returns
+  // or find a matching one already existing (ConstrainChannel() returns
   // nullptr).
   for (HloInstruction* instruction : computation->instructions()) {
     if (instruction->opcode() == HloOpcode::kRecvDone) {
-      const Layout* layout = channel_constraints->ConstrainChannel(
-          instruction->channel_id(),
-          ShapeUtil::GetSubshape(instruction->shape(), {0}).layout());
+      const Layout* layout =
+          get_channel_constraints(instruction)
+              ->ConstrainChannel(
+                  instruction->channel_id(),
+                  ShapeUtil::GetSubshape(instruction->shape(), {0}).layout());
       TF_RET_CHECK(layout == nullptr)
           << instruction->ToString()
           << " cannot constrain layout as it was set to "
@@ -1647,8 +1705,9 @@ Status LayoutAssignment::ConstrainChannelLayouts(
   for (HloInstruction* instruction : computation->MakeInstructionPostOrder()) {
     if (instruction->opcode() == HloOpcode::kSend) {
       HloInstruction* operand = instruction->mutable_operand(0);
-      const Layout* layout = channel_constraints->ConstrainChannel(
-          instruction->channel_id(), operand->shape().layout());
+      const Layout* layout = get_channel_constraints(instruction)
+                                 ->ConstrainChannel(instruction->channel_id(),
+                                                    operand->shape().layout());
       if (layout != nullptr) {
         // We found an already constrained layout which does not match the one
         // the kSend wants to impose. Either add a new kCopy, or use the
diff --git a/tensorflow/compiler/xla/service/layout_assignment.h b/tensorflow/compiler/xla/service/layout_assignment.h
index b75ecb311a..f9e8dbea2f 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.h
+++ b/tensorflow/compiler/xla/service/layout_assignment.h
@@ -488,6 +488,9 @@ class LayoutAssignment : public HloPassInterface {
     }
   }
 
+  // Adds constraints related to host Send/Recv instructions.
+  Status BuildHostChannelConstraints(HloComputation* computation);
+
   // Map containing the layouts of all computations assigned so
   // far. Computations are handled in a topological sort where computations are
   // handled before their caller instructions so the layouts of caller
@@ -507,6 +510,10 @@ class LayoutAssignment : public HloPassInterface {
   // computations/instructions.
   ChannelLayoutConstraints channel_constraints_;
 
+  // Layout constraints for send/recv instructions which communicate with the
+  // host.
+  ChannelLayoutConstraints host_channel_constraints_;
+
   // The set of HLO instructions which lacked any layout constraint, thus
   // receiving propagated default layouts.
   tensorflow::gtl::FlatSet
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index da3b622bfa..636013cbb5 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -169,7 +169,8 @@ Service::Service(const ServiceOptions& options,
 
 Status Service::CreateChannelHandle(const CreateChannelHandleRequest* arg,
                                     CreateChannelHandleResponse* result) {
-  *result->mutable_channel() = channel_tracker_.NewChannel();
+  TF_ASSIGN_OR_RETURN(*result->mutable_channel(),
+                      channel_tracker_.NewChannel(arg->channel_type()));
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index 6f07e4606b..10c0adc670 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -293,6 +293,7 @@ message ComputationStatsResponse {
 }
 
 message CreateChannelHandleRequest {
+  ChannelHandle.ChannelType channel_type = 1;
 }
 
 message CreateChannelHandleResponse {
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index c7472173a7..0b300dc7b2 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -308,6 +308,22 @@ message DeviceHandle {
 // Send instructions will be blocked until the data is transferred.
 message ChannelHandle {
   int64 handle = 1;
+  enum ChannelType {
+    // Invalid primitive type to serve as default.
+    CHANNEL_TYPE_INVALID = 0;
+
+    // A channel for sending data between devices.
+    DEVICE_TO_DEVICE = 1;
+
+    // A channel for sending data from the device to the host. Can only be used
+    // with a Send operation.
+    DEVICE_TO_HOST = 2;
+
+    // A channel for sending data from the host to the device. Can only be used
+    // with a Recv operation.
+    HOST_TO_DEVICE = 3;
+  }
+  ChannelType type = 2;
 }
 
 // DeviceAssignmentProto is a serialized form of DeviceAssignment class, which
-- 
cgit v1.2.3


From 8238266c4fd433107f38eb126a5c5da05a4d338b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 18:08:14 -0700
Subject: Support identity activation function in Cudnn implementation of fused
 conv2d bias activation.

PiperOrigin-RevId: 205008958
---
 .../kernels/fused_conv2d_bias_activation_op.cc     | 30 ++++++++++++++++------
 .../ops/fused_conv2d_bias_activation_op.cc         |  4 +--
 .../python/ops/fused_conv2d_bias_activation_op.py  |  6 +++--
 .../ops/fused_conv2d_bias_activation_op_test.py    | 25 +++++++++++-------
 tensorflow/stream_executor/cuda/cuda_dnn.cc        | 10 ++++++--
 5 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
index 2458f7554a..4554a3d89a 100644
--- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
+++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
@@ -135,9 +135,12 @@ class FusedConv2DBiasActivationOp : public OpKernel {
                    context->GetAttr("activation_mode", &activation_mode_str));
     OP_REQUIRES_OK(context, GetActivationModeFromString(activation_mode_str,
                                                         &activation_mode_));
-    OP_REQUIRES(context, activation_mode_ == ActivationMode::RELU,
-                errors::InvalidArgument("Current implementation only supports "
-                                        "RELU as the activation function."));
+    OP_REQUIRES(context,
+                activation_mode_ == ActivationMode::RELU ||
+                    activation_mode_ == ActivationMode::NONE,
+                errors::InvalidArgument(
+                    "Current implementation only supports RELU or NONE "
+                    "as the activation function."));
     cudnn_use_autotune_ = CudnnUseAutotune();
   }
 
@@ -538,6 +541,18 @@ void LaunchFusedConv2DBiasActivationOp::
       activation_mode,
   };
 
+  dnn::ActivationMode dnn_activation_mode;
+  switch (activation_mode) {
+    case ActivationMode::NONE:
+      dnn_activation_mode = dnn::ActivationMode::kNone;
+      break;
+    case ActivationMode::RELU:
+      dnn_activation_mode = dnn::ActivationMode::kRelu;
+      break;
+    default:
+      LOG(FATAL) << "Activation mode " << activation_mode << " not supported";
+  }
+
   dnn::AlgorithmConfig algorithm_config;
   if (cudnn_use_autotune && !AutoTuneConvBiasActivation::GetInstance()->Find(
                                 fused_conv_parameters, &algorithm_config)) {
@@ -558,10 +573,9 @@ void LaunchFusedConv2DBiasActivationOp::
               ->ThenFusedConvolveWithAlgorithm(
                   conv_input_desc, conv_input_ptr, conv_input_scale,
                   filter_desc, filter_ptr, conv_desc, side_input_ptr,
-                  side_input_scale, bias_desc, bias_ptr,
-                  dnn::ActivationMode::kRelu, output_desc, &output_ptr,
-                  &scratch_allocator, dnn::AlgorithmConfig(profile_algorithm),
-                  &profile_result)
+                  side_input_scale, bias_desc, bias_ptr, dnn_activation_mode,
+                  output_desc, &output_ptr, &scratch_allocator,
+                  dnn::AlgorithmConfig(profile_algorithm), &profile_result)
               .ok();
       if (cudnn_launch_status) {
         if (profile_result.is_valid()) {
@@ -597,7 +611,7 @@ void LaunchFusedConv2DBiasActivationOp::
           ->ThenFusedConvolveWithAlgorithm(
               conv_input_desc, conv_input_ptr, conv_input_scale, filter_desc,
               filter_ptr, conv_desc, side_input_ptr, side_input_scale,
-              bias_desc, bias_ptr, dnn::ActivationMode::kRelu, output_desc,
+              bias_desc, bias_ptr, dnn_activation_mode, output_desc,
               &output_ptr, &scratch_allocator, algorithm_config,
               /*output_profile_result=*/nullptr)
           .ok();
diff --git a/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc
index bafd1d5941..410571f378 100644
--- a/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc
+++ b/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc
@@ -44,7 +44,7 @@ REGISTER_OP("FusedConv2DBiasActivation")
     .Attr(GetPaddingAttrString())
     .Attr("data_format: {'NHWC', 'NCHW', 'NCHW_VECT_C'} = 'NHWC'")
     .Attr("filter_format: {'HWIO', 'OIHW', 'OIHW_VECT_I'} = 'HWIO'")
-    .Attr("activation_mode: {'Relu'} = 'Relu'")
+    .Attr("activation_mode: {'Relu', 'None'} = 'Relu'")
     .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](shape_inference::InferenceContext* c) {
       using shape_inference::ShapeHandle;
@@ -144,7 +144,7 @@ REGISTER_OP("FusedConv2DBiasActivation")
             `qint8 [ output_channels, input_channels / 4,
                      kernel_height, kernel_width, input_channels % 4 ]`
     activation_mode: The activation applied to the output.
-        Currently must be "Relu".
+        Must be "Relu" or "None".
     dilations: 1-D tensor of length 4.  The dilation factor for each dimension
         of `input`. If set to k > 1, there will be k-1 skipped cells between
         each filter element on that dimension. The dimension order is determined
diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py
index 983b6dc8e5..cdc07b935d 100644
--- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py
+++ b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op.py
@@ -66,8 +66,10 @@ def fused_conv2d_bias_activation(conv_input,
         This is optional and defaults to 0.
     side_input: A `Tensor` of the format specified by `data_format`.
         This is useful for implementing ResNet blocks.
-    activation_mode: (optional) currently must be the default "Relu".
-        Note that in qint8 mode, it also clips to 127, so acts like ReluX.
+    activation_mode: (optional) currently supports the default "Relu", or
+        "None" activation function.
+        Note: in qint8 mode, "None" actually clips to the range [-128, 127],
+        while "Relu" clips to the range [0, 127].
     data_format: Specifies the data format.
         Possible values are:
         "NHWC" float [batch, height, width, channels]
diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py
index 4d62ac65ff..0185ef662c 100644
--- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py
+++ b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py
@@ -622,7 +622,7 @@ def HwioToOihw(in_tensor):
 
 def SimulateFusedConv2dBiasActivationInt8(conv_input_scale, conv_input, kernel,
                                           padding, strides, side_input_scale,
-                                          side_input, biases):
+                                          side_input, biases, apply_relu):
   """Simulates the int8 fused 2-D convolution op using separate float ops.
 
     The arguments and return values have the same format, meanings and
@@ -636,6 +636,9 @@ def SimulateFusedConv2dBiasActivationInt8(conv_input_scale, conv_input, kernel,
     side_input_scale: A scalar 'float'.
     side_input: A `Tensor` of type `qint8` in NCHW_VECT_C layout.
     biases: A `Tensor` of type `float32` in NCHW layout.
+    apply_relu: A boolean to specify whether to apply "Relu" activation function
+      that clips outputs to the range [0, 127], or "None" activation that clips
+      to the range [-128, 127].
   Returns:
     A `Tensor` of type `qint8` in NCHW_VECT_C layout.
   """
@@ -649,10 +652,12 @@ def SimulateFusedConv2dBiasActivationInt8(conv_input_scale, conv_input, kernel,
   conv_and_side_inputs = conv_result + side_input_scale * NchwVectCToNchw(
       gen_array_ops.dequantize(side_input, -128, 127))
 
-  logit = nn_ops.bias_add(conv_and_side_inputs, biases, data_format="NCHW")
+  output = nn_ops.bias_add(conv_and_side_inputs, biases, data_format="NCHW")
+  if apply_relu:
+    output = nn_ops.relu(output)
 
   result, _, _ = gen_array_ops.quantize_v2(
-      NchwToNchwVectC(nn_ops.relu(logit)), -128, 127, dtypes.qint8)
+      NchwToNchwVectC(output), -128, 127, dtypes.qint8)
   return result
 
 
@@ -795,7 +800,7 @@ class FusedConvInt8Tests(test.TestCase):
       },
   ]
 
-  def runTest(self, test_param):
+  def runTest(self, test_param, apply_relu):
     batch_size = test_param["batch_size"]
     input_channels = test_param["input_channels"]
     output_channels = test_param["output_channels"]
@@ -831,8 +836,8 @@ class FusedConvInt8Tests(test.TestCase):
                                                 vertical_stride, padding_type)
     output_width = CalculateConvolvedOutputDim(input_width, filter_width,
                                                horizontal_stride, padding_type)
-    tf_logging.info("output_height=", output_height, ", output_width=", 
-			                 output_width)
+    tf_logging.info("output_height=", output_height, ", output_width=",
+                    output_width)
 
     side_input, _, _ = gen_array_ops.quantize_v2(
         random_ops.random_uniform(
@@ -858,12 +863,13 @@ class FusedConvInt8Tests(test.TestCase):
         conv_input_scale=conv_input_scale,
         side_input_scale=side_input_scale,
         side_input=side_input,
+        activation_mode="Relu" if apply_relu else "None",
         data_format="NCHW_VECT_C",
         filter_format="OIHW_VECT_I")
 
     expected = SimulateFusedConv2dBiasActivationInt8(
         conv_input_scale, conv_input, kernel, padding_type, strides,
-        side_input_scale, side_input, biases)
+        side_input_scale, side_input, biases, apply_relu)
 
     with self.test_session(use_gpu=True) as sess:
       actual_y, expected_y = sess.run([actual, expected])
@@ -877,8 +883,9 @@ class FusedConvInt8Tests(test.TestCase):
       tf_logging.info("int8 test skipped because not run with --config=cuda or "
                       "no GPUs with compute capability >= 6.1 are available.")
       return
-    for test_param in self._test_params:
-      self.runTest(test_param)
+    for apply_relu in [True, False]:
+      for test_param in self._test_params:
+        self.runTest(test_param, apply_relu)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 08228034f7..e85b6db511 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -791,6 +791,11 @@ class CudnnActivationDescriptor {
     double relu_ceiling = 0.0;
     cudnnActivationMode_t mode;
     switch (activation_mode) {
+#if CUDNN_VERSION >= 7100
+      case dnn::ActivationMode::kNone:
+        mode = CUDNN_ACTIVATION_IDENTITY;
+        break;
+#endif
       case dnn::ActivationMode::kRelu6:
         relu_ceiling = 6.0;
         mode = CUDNN_ACTIVATION_CLIPPED_RELU;
@@ -2493,10 +2498,11 @@ port::Status CudnnSupport::DoFusedConvolveImpl(
     DeviceMemory* output_data, ScratchAllocator* scratch_allocator,
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
-  if (activation_mode != dnn::ActivationMode::kRelu) {
+  if (activation_mode != dnn::ActivationMode::kRelu &&
+      activation_mode != dnn::ActivationMode::kNone) {
     return port::Status(port::error::INVALID_ARGUMENT,
                         "cudnnConvolutionBiasActivationForward() only supports "
-                        "Relu activation.");
+                        "Relu or None activation.");
   }
 
   CudnnTensorDescriptor conv_input_nd(
-- 
cgit v1.2.3


From b5569c5be8bd311c4893f712f56a606f118406b4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 18:15:34 -0700
Subject: Extending batching capability to support rank 5 tensors.

PiperOrigin-RevId: 205009777
---
 tensorflow/core/util/batch_util.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/util/batch_util.cc b/tensorflow/core/util/batch_util.cc
index 7ea8851e65..45556d53a4 100644
--- a/tensorflow/core/util/batch_util.cc
+++ b/tensorflow/core/util/batch_util.cc
@@ -264,6 +264,7 @@ Status CopyElementToLargerSlice(const Tensor& element, Tensor* parent,
     HANDLE_DIMS(2);
     HANDLE_DIMS(3);
     HANDLE_DIMS(4);
+    HANDLE_DIMS(5);
 #undef HANDLE_DIMS
     default:
       return errors::Unimplemented("CopyElementToLargerSlice Unhandled rank: ",
-- 
cgit v1.2.3


From e420854397d9f43cf6ed9726c7e4d8cba6f380e6 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal 
Date: Tue, 17 Jul 2018 18:30:18 -0700
Subject: Removing redundant comment.

PiperOrigin-RevId: 205011204
---
 tensorflow/python/data/util/nest.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/python/data/util/nest.py b/tensorflow/python/data/util/nest.py
index 32e08021dc..1b596bdfc0 100644
--- a/tensorflow/python/data/util/nest.py
+++ b/tensorflow/python/data/util/nest.py
@@ -13,7 +13,6 @@
 # limitations under the License.
 # ==============================================================================
 
-# TODO(shivaniagrawal): Merge with core nest
 """## Functions for working with arbitrarily nested sequences of elements.
 
 NOTE(mrry): This fork of the `tensorflow.python.util.nest` module
-- 
cgit v1.2.3


From 8a984b115dba13c7a7dc2de4ab5890ec6bc7750e Mon Sep 17 00:00:00 2001
From: Michael Kuperstein 
Date: Tue, 17 Jul 2018 18:31:02 -0700
Subject: [XLA] Add strided R1 tests with large strides.

PiperOrigin-RevId: 205011270
---
 tensorflow/compiler/xla/tests/slice_test.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/tests/slice_test.cc b/tensorflow/compiler/xla/tests/slice_test.cc
index 48138e7b07..a593faca00 100644
--- a/tensorflow/compiler/xla/tests/slice_test.cc
+++ b/tensorflow/compiler/xla/tests/slice_test.cc
@@ -344,7 +344,11 @@ INSTANTIATE_TEST_CASE_P(
         R1Spec{1024 * 1024 + 71, 3, 1024 * 512 - 9, 2},
         R1Spec{1024 * 1024 + 71, 3, 1024 * 512 - 9, 8},
         R1Spec{1024 * 1024 + 71, 3, 1024 * 512 - 9, 7},
-        R1Spec{1024 * 1024 + 71, 3, 1024 * 512 - 9, 125}
+        R1Spec{1024 * 1024 + 71, 3, 1024 * 512 - 9, 125},
+        R1Spec{16 * 1024 * 1024, 0, 16 * 1024 * 1024, 4097},
+        R1Spec{16 * 1024 * 1024, 0, 16 * 1024 * 1024, 4093},
+        R1Spec{16 * 1024 * 1024, 12 * 1024 + 17, 16 * 1024 * 1024 - 231, 4097},
+        R1Spec{16 * 1024 * 1024, 12 * 1024 + 17, 16 * 1024 * 1024 - 231, 4093}
     ),
     SliceR1TestDataToString
 );
-- 
cgit v1.2.3


From ff4945f86e04d403cdf46c19392b2041bc75c2ad Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling 
Date: Tue, 17 Jul 2018 19:25:21 -0700
Subject: Align TFLite tensors to 16 bytes for EIGEN_DONT_ALIGN

PiperOrigin-RevId: 205015541
---
 tensorflow/contrib/lite/arena_planner.cc         | 25 +++++++++---------------
 tensorflow/contrib/lite/arena_planner.h          | 10 +++++++++-
 tensorflow/contrib/lite/arena_planner_test.cc    |  8 +++++---
 tensorflow/contrib/lite/kernels/BUILD            |  1 +
 tensorflow/contrib/lite/kernels/eigen_support.cc | 11 +++++++++++
 5 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/lite/arena_planner.cc b/tensorflow/contrib/lite/arena_planner.cc
index 16a0e71624..02442575b3 100644
--- a/tensorflow/contrib/lite/arena_planner.cc
+++ b/tensorflow/contrib/lite/arena_planner.cc
@@ -17,14 +17,6 @@ limitations under the License.
 
 namespace tflite {
 
-namespace {
-
-// Memory allocation tuning
-constexpr const int kDefaultArenaAlignment = 64;
-constexpr const int kDefaultTensorAlignment = 4;
-
-}  // namespace
-
 struct AllocationInfo {
   // The node index requesting this allocation.
   int node;
@@ -36,13 +28,16 @@ struct AllocationInfo {
 
 ArenaPlanner::ArenaPlanner(TfLiteContext* context,
                            std::unique_ptr graph_info,
-                           bool preserve_inputs, bool preserve_intermediates)
+                           bool preserve_inputs, bool preserve_intermediates,
+                           int tensor_alignment)
     : context_(context),
       graph_info_(std::move(graph_info)),
       arena_(kDefaultArenaAlignment),
       persistent_arena_(kDefaultArenaAlignment),
       preserve_inputs_(preserve_inputs),
-      preserve_intermediates_(preserve_intermediates) {}
+      preserve_intermediates_(preserve_intermediates),
+      tensor_alignment_(tensor_alignment) {}
+
 ArenaPlanner::~ArenaPlanner() {}
 
 int64_t ArenaPlanner::BasePointer(TfLiteAllocationType type) {
@@ -264,14 +259,12 @@ TfLiteStatus ArenaPlanner::ResolveTensorAllocation(int tensor_index) {
 TfLiteStatus ArenaPlanner::CalculateTensorAllocation(int tensor_index) {
   TfLiteTensor& tensor = *graph_info_->tensor(tensor_index);
   if (tensor.allocation_type == kTfLiteArenaRw) {
-    TF_LITE_ENSURE_STATUS(arena_.Allocate(context_, kDefaultTensorAlignment,
-                                          tensor.bytes,
-                                          &allocs_[tensor_index]));
+    TF_LITE_ENSURE_STATUS(arena_.Allocate(
+        context_, tensor_alignment_, tensor.bytes, &allocs_[tensor_index]));
   }
   if (tensor.allocation_type == kTfLiteArenaRwPersistent) {
-    TF_LITE_ENSURE_STATUS(
-        persistent_arena_.Allocate(context_, kDefaultTensorAlignment,
-                                   tensor.bytes, &allocs_[tensor_index]));
+    TF_LITE_ENSURE_STATUS(persistent_arena_.Allocate(
+        context_, tensor_alignment_, tensor.bytes, &allocs_[tensor_index]));
   }
   return kTfLiteOk;
 }
diff --git a/tensorflow/contrib/lite/arena_planner.h b/tensorflow/contrib/lite/arena_planner.h
index 82c866734f..65a9730b70 100644
--- a/tensorflow/contrib/lite/arena_planner.h
+++ b/tensorflow/contrib/lite/arena_planner.h
@@ -25,6 +25,10 @@ limitations under the License.
 
 namespace tflite {
 
+// Memory allocation tuning
+constexpr const int kDefaultArenaAlignment = 64;
+constexpr const int kDefaultTensorAlignment = 16;
+
 struct AllocationInfo;
 
 // A memory planner that makes all the allocations using arenas.
@@ -47,7 +51,8 @@ class ArenaPlanner : public MemoryPlanner {
   // graph will not share memory with any other tensor, effectively preserving
   // them until the end of inference.
   ArenaPlanner(TfLiteContext* context, std::unique_ptr graph_info,
-               bool preserve_inputs, bool preserve_intermediates);
+               bool preserve_inputs, bool preserve_intermediates,
+               int tensor_alignment = kDefaultTensorAlignment);
   ~ArenaPlanner() override;
   ArenaPlanner(const ArenaPlanner&) = delete;
   ArenaPlanner& operator=(const ArenaPlanner&) = delete;
@@ -112,6 +117,9 @@ class ArenaPlanner : public MemoryPlanner {
   // If true, then no overlapping of memory areas is done, meaning intermediates
   // results can be queried after running (modulo running delegates).
   bool preserve_intermediates_;
+
+  // Number of bytes that tensor buffers should be aligned to.
+  int tensor_alignment_;
 };
 
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/arena_planner_test.cc b/tensorflow/contrib/lite/arena_planner_test.cc
index 1adb426d58..7d7c41289c 100644
--- a/tensorflow/contrib/lite/arena_planner_test.cc
+++ b/tensorflow/contrib/lite/arena_planner_test.cc
@@ -24,6 +24,8 @@ limitations under the License.
 namespace tflite {
 namespace {
 
+constexpr const int kTensorAlignment = 4;
+
 // A simple op to be used in tests, as syntactic sugar.
 class TestOp {
  public:
@@ -156,7 +158,7 @@ class ArenaPlannerTest : public ::testing::Test {
     context_.ReportError = ReportError;
     planner_.reset(new ArenaPlanner(
         &context_, std::unique_ptr(new TestGraphInfo(graph)),
-        preserve_inputs, /*preserve intermediates*/ false));
+        preserve_inputs, /*preserve intermediates*/ false, kTensorAlignment));
     CHECK(planner_->ResetAllocations() == kTfLiteOk);
     CHECK(planner_->PlanAllocations() == kTfLiteOk);
   }
@@ -178,8 +180,8 @@ class ArenaPlannerTest : public ::testing::Test {
     const TfLiteTensor& tensor = (*graph_->tensors())[tensor_index];
     int64_t offset = GetOffset(tensor_index) + tensor.bytes;
     // We must make sure the offset is aligned to kDefaultArenaAlignment.
-    if (offset % 4 != 0) {
-      offset += 4 - offset % 4;
+    if (offset % kTensorAlignment != 0) {
+      offset += kTensorAlignment - offset % kTensorAlignment;
     }
     return offset;
   };
diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index ad30624f40..9549b4445d 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -58,6 +58,7 @@ cc_library(
     }),
     deps = [
         ":op_macros",
+        "//tensorflow/contrib/lite:arena_planner",
         "//tensorflow/contrib/lite:context",
         "//tensorflow/contrib/lite/kernels/internal:optimized",
     ],
diff --git a/tensorflow/contrib/lite/kernels/eigen_support.cc b/tensorflow/contrib/lite/kernels/eigen_support.cc
index 4f0d020793..e542ad0765 100644
--- a/tensorflow/contrib/lite/kernels/eigen_support.cc
+++ b/tensorflow/contrib/lite/kernels/eigen_support.cc
@@ -16,6 +16,7 @@ limitations under the License.
 
 #include 
 
+#include "tensorflow/contrib/lite/arena_planner.h"
 #include "tensorflow/contrib/lite/kernels/internal/optimized/eigen_spatial_convolutions.h"
 #include "tensorflow/contrib/lite/kernels/op_macros.h"
 
@@ -23,6 +24,16 @@ namespace tflite {
 namespace eigen_support {
 namespace {
 
+#ifndef EIGEN_DONT_ALIGN
+// Eigen may require buffers to be algiend to 16, 32 or 64 bytes depending on
+// hardware architecture and build configurations.
+// If the static assertion fails, try to increase `kDefaultTensorAlignment` to
+// in `arena_planner.h` to 32 or 64.
+static_assert(
+    kDefaultTensorAlignment % EIGEN_MAX_ALIGN_BYTES == 0,
+    "kDefaultArenaAlignment doesn't comply with Eigen alignment requirement.");
+#endif  // EIGEN_DONT_ALIGN
+
 // We have a single global threadpool for all convolution operations. This means
 // that inferences started from different threads may block each other, but
 // since the underlying resource of CPU cores should be consumed by the
-- 
cgit v1.2.3


From 9c74fa8bdac008beee196a6497a47e65a4ae7577 Mon Sep 17 00:00:00 2001
From: Derek Murray 
Date: Tue, 17 Jul 2018 19:43:16 -0700
Subject: [tf.data] Switch iterator ops to use a simple non-blocking background
 worker.

In rare cases (e.g. under heavy load) `tensorflow::thread::ThreadPool::Schedule()` can block, because its fixed-length queue is full. In the iterator ops, we would rather pay the cost of one additional dynamic allocation per op invocation and never block. This fixes a potential deadlock when a closure is executed synchronously, blocks a thread-pool thread, and depends on its own blocked thread (or another thread in the same state) to make progress.

PiperOrigin-RevId: 205016675
---
 tensorflow/core/kernels/data/iterator_ops.cc | 105 ++++++++++++++++++++++-----
 1 file changed, 85 insertions(+), 20 deletions(-)

diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc
index 2a94a54f3d..da489db7c8 100644
--- a/tensorflow/core/kernels/data/iterator_ops.cc
+++ b/tensorflow/core/kernels/data/iterator_ops.cc
@@ -662,21 +662,89 @@ class MakeIteratorOp : public OpKernel {
   }
 };
 
+// A simple background worker that executes closures asynchronously and without
+// blocking.
+//
+// A `BackgroundWorker` is used to offload blocking work from an `AsyncOpKernel`
+// to avoid blocking an executor thread that may be required by the blocking
+// work.
+//
+// NOTE(mrry): We do not use a regular `tensorflow::thread::ThreadPool` for this
+// purpose because its current implementation (in Eigen) uses a finite-length
+// queue and will block the caller when full. This can lead to deadlock under
+// heavy load. Since the number of concurrent work items in each user of a
+// `BackgroundWorker` is at most one per op invocation, the dynamic allocation
+// overhead is tolerable.
+class BackgroundWorker {
+ public:
+  BackgroundWorker(Env* env, const string& name) {
+    thread_.reset(env->StartThread({} /* thread_options */, name,
+                                   [this]() { WorkerLoop(); }));
+  }
+
+  ~BackgroundWorker() {
+    {
+      mutex_lock l(mu_);
+      cancelled_ = true;
+    }
+    cond_var_.notify_one();
+    // Block until the background thread has terminated.
+    //
+    // NOTE(mrry): We explicitly free and join the thread here because
+    // `WorkerLoop()` uses other members of this object, and so we must join
+    // the thread before destroying them.
+    thread_.reset();
+  }
+
+  void Schedule(std::function work_item) {
+    {
+      mutex_lock l(mu_);
+      work_queue_.push_back(std::move(work_item));
+    }
+    cond_var_.notify_one();
+  }
+
+ private:
+  void WorkerLoop() {
+    while (true) {
+      std::function work_item = nullptr;
+      {
+        mutex_lock l(mu_);
+        while (!cancelled_ && work_queue_.empty()) {
+          cond_var_.wait(l);
+        }
+        if (cancelled_) {
+          return;
+        }
+        DCHECK(!work_queue_.empty());
+        work_item = std::move(work_queue_.front());
+        work_queue_.pop_front();
+      }
+      DCHECK(work_item != nullptr);
+      work_item();
+    }
+  }
+
+  std::unique_ptr thread_;
+  mutex mu_;
+  condition_variable cond_var_;
+  bool cancelled_ GUARDED_BY(mu_) = false;
+  std::deque> work_queue_ GUARDED_BY(mu_);
+};
+
 class ToSingleElementOp : public AsyncOpKernel {
  public:
   explicit ToSingleElementOp(OpKernelConstruction* ctx)
       : AsyncOpKernel(ctx),
-        thread_pool_(new thread::ThreadPool(
-            ctx->env(), ThreadOptions(),
-            strings::StrCat("to_single_element_op_thread_",
-                            SanitizeThreadSuffix(name())),
-            1 /* num_threads */, false /* low_latency_hint */)) {}
+        background_worker_(ctx->env(),
+                           strings::StrCat("to_single_element_op_thread_",
+                                           SanitizeThreadSuffix(name()))) {}
 
   void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override {
     // The call to `iterator->GetNext()` may block and depend on an
     // inter-op thread pool thread, so we issue the call from the
     // owned thread pool.
-    thread_pool_->Schedule([ctx, done]() {
+    background_worker_.Schedule([ctx, done]() {
       DatasetBase* dataset;
       OP_REQUIRES_OK_ASYNC(
           ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset), done);
@@ -729,18 +797,17 @@ class ToSingleElementOp : public AsyncOpKernel {
   }
 
  private:
-  std::unique_ptr thread_pool_;
+  BackgroundWorker background_worker_;
 };
 
 class OneShotIteratorOp : public AsyncOpKernel {
  public:
   explicit OneShotIteratorOp(OpKernelConstruction* ctx)
       : AsyncOpKernel(ctx),
-        thread_pool_(new thread::ThreadPool(
-            ctx->env(), ThreadOptions(),
+        background_worker_(
+            ctx->env(),
             strings::StrCat("one_shot_iterator_initialization_thread_",
-                            SanitizeThreadSuffix(name())),
-            1 /* num_threads */, false /* low_latency_hint */)),
+                            SanitizeThreadSuffix(name()))),
         graph_def_version_(ctx->graph_def_version())
 
   {
@@ -782,7 +849,7 @@ class OneShotIteratorOp : public AsyncOpKernel {
         if (!initialization_started_) {
           // TODO(mrry): Convert the initialization code to use
           // callbacks instead of wasting a thread.
-          thread_pool_->Schedule([this, ctx, done]() { Init(ctx, done); });
+          background_worker_.Schedule([this, ctx, done]() { Init(ctx, done); });
           initialization_started_ = true;
         } else {
           done_callbacks_.emplace_back(ctx, std::move(done));
@@ -915,7 +982,7 @@ class OneShotIteratorOp : public AsyncOpKernel {
   DataTypeVector output_dtypes_;
   std::vector output_shapes_;
 
-  std::unique_ptr thread_pool_;
+  BackgroundWorker background_worker_;
 
   mutex mu_;
   ContainerInfo cinfo_ GUARDED_BY(mu_);
@@ -932,11 +999,9 @@ class IteratorGetNextOp : public AsyncOpKernel {
  public:
   explicit IteratorGetNextOp(OpKernelConstruction* ctx)
       : AsyncOpKernel(ctx),
-        thread_pool_(new thread::ThreadPool(
-            ctx->env(), ThreadOptions(),
-            strings::StrCat("iterator_get_next_thread_",
-                            SanitizeThreadSuffix(name())),
-            1 /* num_threads */, false /* low_latency_hint */)) {}
+        background_worker_(ctx->env(),
+                           strings::StrCat("iterator_get_next_thread_",
+                                           SanitizeThreadSuffix(name()))) {}
 
   void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override {
     IteratorResource* iterator;
@@ -945,7 +1010,7 @@ class IteratorGetNextOp : public AsyncOpKernel {
     // The call to `iterator->GetNext()` may block and depend on an
     // inter-op thread pool thread, so we issue the call from the
     // owned thread pool.
-    thread_pool_->Schedule(std::bind(
+    background_worker_.Schedule(std::bind(
         [ctx, iterator](DoneCallback done) {
           std::vector components;
           bool end_of_sequence = false;
@@ -982,7 +1047,7 @@ class IteratorGetNextOp : public AsyncOpKernel {
   }
 
  private:
-  std::unique_ptr thread_pool_;
+  BackgroundWorker background_worker_;
 };
 
 class IteratorGetNextSyncOp : public OpKernel {
-- 
cgit v1.2.3


From 527a5eb1759d682810f8155b9767bcc7984d3b3e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 20:00:11 -0700
Subject: Adding an OptimizationParameters proto to provide support for more
 optimizers. Please note that the API should not be considered stable and is
 subject to change.

PiperOrigin-RevId: 205017619
---
 tensorflow/contrib/tpu/proto/BUILD                 |  10 ++
 .../tpu/proto/optimization_parameters.proto        | 162 +++++++++++++++++++++
 .../contrib/tpu/proto/tpu_embedding_config.proto   |  16 +-
 3 files changed, 175 insertions(+), 13 deletions(-)
 create mode 100644 tensorflow/contrib/tpu/proto/optimization_parameters.proto

diff --git a/tensorflow/contrib/tpu/proto/BUILD b/tensorflow/contrib/tpu/proto/BUILD
index 26016f47df..598b73b438 100644
--- a/tensorflow/contrib/tpu/proto/BUILD
+++ b/tensorflow/contrib/tpu/proto/BUILD
@@ -15,6 +15,16 @@ tf_proto_library(
         "tpu_embedding_config.proto",
     ],
     cc_api_version = 2,
+    protodeps = [":optimization_parameters_proto"],
+    visibility = ["//visibility:public"],
+)
+
+tf_proto_library(
+    name = "optimization_parameters_proto",
+    srcs = [
+        "optimization_parameters.proto",
+    ],
+    cc_api_version = 2,
     visibility = ["//visibility:public"],
 )
 
diff --git a/tensorflow/contrib/tpu/proto/optimization_parameters.proto b/tensorflow/contrib/tpu/proto/optimization_parameters.proto
new file mode 100644
index 0000000000..9150606f5e
--- /dev/null
+++ b/tensorflow/contrib/tpu/proto/optimization_parameters.proto
@@ -0,0 +1,162 @@
+syntax = "proto2";
+
+package tensorflow.tpu;
+
+message ClippingLimits {
+  optional float lower = 1 [default = -inf];
+  optional float upper = 2 [default = inf];
+}
+
+// Get the learning rate from a  source that can change
+// dynamically.
+message DynamicLearningRate {
+}
+
+// Source of learning rate to use.
+message LearningRate {
+  oneof learning_rate {
+    float constant = 1;
+    DynamicLearningRate dynamic = 2;
+  }
+}
+
+message AdagradParameters {
+  optional float initial_accumulator = 1 [default = 0.];
+}
+
+message StochasticGradientDescentParameters {
+}
+
+message FtrlParameters {
+  optional float l1 = 1 [default = 0.];
+  optional float l2 = 2 [default = 0.];
+  optional float lr_power = 3 [default = 0.];
+  optional float initial_accum = 4 [default = 0.];
+  optional float initial_linear = 5 [default = 0.];
+}
+
+// The Adam optimizer does not implement hyper-parameter update; use the dynamic
+// learning rate feature instead, setting the learning rate to:
+// user learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t)
+// Here, t is the current timestep.
+// https://github.com/tensorflow/tensorflow/blob/ab51450c817674c8ff08a7ae4f8ac50cdc4bed8b/tensorflow/python/training/adam.py#L54
+message AdamParameters {
+  optional float beta1 = 3 [default = 0.];
+  optional float beta2 = 4 [default = 0.];
+  optional float epsilon = 5 [default = 0.];
+  optional float initial_m = 6 [default = 0.];
+  optional float initial_v = 7 [default = 0.];
+}
+
+message MomentumParameters {
+  optional float momentum = 1 [default = 0.];
+  optional bool use_nesterov = 2 [default = false];
+  optional float initial_accum = 3 [default = 0.];
+}
+
+message RmsPropParameters {
+  optional float rho = 1 [default = 0.];
+  optional float momentum = 2 [default = 0.];
+  optional float epsilon = 3 [default = 0.];
+  optional float initial_ms = 4 [default = 0.];
+  optional float initial_mom = 5 [default = 0.];
+}
+
+message CenteredRmsPropParameters {
+  optional float rho = 1 [default = 0.];
+  optional float momentum = 2 [default = 0.];
+  optional float epsilon = 3 [default = 0.];
+  optional float initial_ms = 4 [default = 0.];
+  optional float initial_mom = 5 [default = 0.];
+  optional float initial_mg = 6 [default = 0.];
+}
+
+message MdlAdagradLightParameters {
+  optional float l2 = 1;
+  optional float lr_power = 2;
+  optional float min_servable_mdl_benefit = 3;
+  optional float mdl_mix_in_margin = 4;
+  optional float mdl_benefit_rampup_coeff = 5;
+  optional float mdl_min_weight = 6;
+  optional float benefit_revisit_scale = 7;
+  optional float max_event_benefit = 8;
+  optional float max_total_benefit = 9;
+  optional float mdl_hard_limit = 10;
+  optional bool hard_limit_min_benefit = 11;
+  optional bool mdl_regularize = 12;
+  optional float initial_accumulator = 13;
+  optional float initial_weight = 14;
+  optional float initial_benefit = 15;
+}
+
+message AdadeltaParameters {
+  optional float rho = 1;
+  optional float epsilon = 2;
+  optional float initial_accumulator = 3 [default = 0.];
+  optional float initial_update = 4 [default = 0.];
+}
+
+message ProximalAdagradParameters {
+  optional float l1 = 1;
+  optional float l2 = 2;
+  optional float initial_accumulator = 3;
+}
+
+message OptimizationParameters {
+  // Learning rate used for updating the embedding layer parameters.
+  optional LearningRate learning_rate = 13;
+  reserved 1;  // Old learning rate tag.
+
+  // Limits to which to clip the weight values after the backward pass; not
+  // present means no limits are applied.
+  optional ClippingLimits clipping_limits = 2;
+
+  // Limits to which to clip the backward pass gradient before using it for
+  // updates; not present means no limits are applied.
+  optional ClippingLimits gradient_clipping_limits = 7;
+
+  // Whether to use gradient accumulation (do two passes over the input
+  // gradients: one to accumulate them into a temporary array and another to
+  // apply them using the actual optimization algorithm).
+  optional bool use_gradient_accumulation = 15 [default = false];
+
+  // Optimization algorithm parameters; which field is selected determines which
+  // algorithm to use.
+  oneof parameters {
+    AdagradParameters adagrad = 3;
+    StochasticGradientDescentParameters stochastic_gradient_descent = 4;
+    FtrlParameters ftrl = 5;
+    AdamParameters adam = 6;
+    MomentumParameters momentum = 8;
+    RmsPropParameters rms_prop = 9;
+    CenteredRmsPropParameters centered_rms_prop = 10;
+    MdlAdagradLightParameters mdl_adagrad_light = 11;
+    AdadeltaParameters adadelta = 12;
+    ProximalAdagradParameters proximal_adagrad = 14;
+  }
+}
+
+// Specification of an optimization algorithm's state variables (both the main
+// value vector and any extra accumulators, etc.).
+message StateVariableSpecification {
+  // Parameter name for the state variable.
+  optional string name = 1;
+
+  // A normal state variable that should be saved and restored in checkpoints
+  // and used as an input or output to non-debug TensorFlow ops.
+  message UserDefined {
+  }
+
+  // A state variable that should be filled with a constant and normally hidden
+  // from users (used for intermediate gradients being accumulated, for
+  // example).
+  message FillWithConstant {
+    optional double initial_value = 1;
+  }
+
+  // Usage type of this state variable.
+  oneof usage {
+    UserDefined user_defined = 2;
+    FillWithConstant fill_with_constant = 3;
+  }
+}
diff --git a/tensorflow/contrib/tpu/proto/tpu_embedding_config.proto b/tensorflow/contrib/tpu/proto/tpu_embedding_config.proto
index b0ec968d3a..3476cc8953 100644
--- a/tensorflow/contrib/tpu/proto/tpu_embedding_config.proto
+++ b/tensorflow/contrib/tpu/proto/tpu_embedding_config.proto
@@ -2,6 +2,8 @@ syntax = "proto3";
 
 package tensorflow.tpu;
 
+import "tensorflow/contrib/tpu/proto/optimization_parameters.proto";
+
 // The TPUEmbeddingConfiguration contains specification of TPU Embedding lookups
 // and gradient updates separate from the TF Graph.
 message TPUEmbeddingConfiguration {
@@ -30,15 +32,6 @@ message TPUEmbeddingConfiguration {
   // The number of training examples per TensorNode.
   int32 batch_size = 4;
 
-  message GradientDescentOptimizer {
-    float learning_rate = 1;
-  }
-
-  message AdagradOptimizer {
-    float learning_rate = 1;
-    float initial_accumulator = 2;
-  }
-
   // Each Embedding
   message TPUEmbeddingTable {
     // Name of the embedding table. This will be used to name Variables in the
@@ -66,10 +59,7 @@ message TPUEmbeddingConfiguration {
     // separately to the convolutional or recurrent network.
     int32 num_features = 5;
 
-    oneof optimizer {
-      GradientDescentOptimizer gradient_descent = 6;
-      AdagradOptimizer adagrad = 7;
-    }
+    OptimizationParameters optimization_parameters = 6;
   }
 
   repeated TPUEmbeddingTable table_config = 5;
-- 
cgit v1.2.3


From 37cf4e0783ad06e6cfc94c98a77a48734190ed48 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 21:02:37 -0700
Subject: Automated rollback of commit ff4945f86e04d403cdf46c19392b2041bc75c2ad

PiperOrigin-RevId: 205022167
---
 tensorflow/contrib/lite/arena_planner.cc         | 25 +++++++++++++++---------
 tensorflow/contrib/lite/arena_planner.h          | 10 +---------
 tensorflow/contrib/lite/arena_planner_test.cc    |  8 +++-----
 tensorflow/contrib/lite/kernels/BUILD            |  1 -
 tensorflow/contrib/lite/kernels/eigen_support.cc | 11 -----------
 5 files changed, 20 insertions(+), 35 deletions(-)

diff --git a/tensorflow/contrib/lite/arena_planner.cc b/tensorflow/contrib/lite/arena_planner.cc
index 02442575b3..16a0e71624 100644
--- a/tensorflow/contrib/lite/arena_planner.cc
+++ b/tensorflow/contrib/lite/arena_planner.cc
@@ -17,6 +17,14 @@ limitations under the License.
 
 namespace tflite {
 
+namespace {
+
+// Memory allocation tuning
+constexpr const int kDefaultArenaAlignment = 64;
+constexpr const int kDefaultTensorAlignment = 4;
+
+}  // namespace
+
 struct AllocationInfo {
   // The node index requesting this allocation.
   int node;
@@ -28,16 +36,13 @@ struct AllocationInfo {
 
 ArenaPlanner::ArenaPlanner(TfLiteContext* context,
                            std::unique_ptr graph_info,
-                           bool preserve_inputs, bool preserve_intermediates,
-                           int tensor_alignment)
+                           bool preserve_inputs, bool preserve_intermediates)
     : context_(context),
       graph_info_(std::move(graph_info)),
       arena_(kDefaultArenaAlignment),
       persistent_arena_(kDefaultArenaAlignment),
       preserve_inputs_(preserve_inputs),
-      preserve_intermediates_(preserve_intermediates),
-      tensor_alignment_(tensor_alignment) {}
-
+      preserve_intermediates_(preserve_intermediates) {}
 ArenaPlanner::~ArenaPlanner() {}
 
 int64_t ArenaPlanner::BasePointer(TfLiteAllocationType type) {
@@ -259,12 +264,14 @@ TfLiteStatus ArenaPlanner::ResolveTensorAllocation(int tensor_index) {
 TfLiteStatus ArenaPlanner::CalculateTensorAllocation(int tensor_index) {
   TfLiteTensor& tensor = *graph_info_->tensor(tensor_index);
   if (tensor.allocation_type == kTfLiteArenaRw) {
-    TF_LITE_ENSURE_STATUS(arena_.Allocate(
-        context_, tensor_alignment_, tensor.bytes, &allocs_[tensor_index]));
+    TF_LITE_ENSURE_STATUS(arena_.Allocate(context_, kDefaultTensorAlignment,
+                                          tensor.bytes,
+                                          &allocs_[tensor_index]));
   }
   if (tensor.allocation_type == kTfLiteArenaRwPersistent) {
-    TF_LITE_ENSURE_STATUS(persistent_arena_.Allocate(
-        context_, tensor_alignment_, tensor.bytes, &allocs_[tensor_index]));
+    TF_LITE_ENSURE_STATUS(
+        persistent_arena_.Allocate(context_, kDefaultTensorAlignment,
+                                   tensor.bytes, &allocs_[tensor_index]));
   }
   return kTfLiteOk;
 }
diff --git a/tensorflow/contrib/lite/arena_planner.h b/tensorflow/contrib/lite/arena_planner.h
index 65a9730b70..82c866734f 100644
--- a/tensorflow/contrib/lite/arena_planner.h
+++ b/tensorflow/contrib/lite/arena_planner.h
@@ -25,10 +25,6 @@ limitations under the License.
 
 namespace tflite {
 
-// Memory allocation tuning
-constexpr const int kDefaultArenaAlignment = 64;
-constexpr const int kDefaultTensorAlignment = 16;
-
 struct AllocationInfo;
 
 // A memory planner that makes all the allocations using arenas.
@@ -51,8 +47,7 @@ class ArenaPlanner : public MemoryPlanner {
   // graph will not share memory with any other tensor, effectively preserving
   // them until the end of inference.
   ArenaPlanner(TfLiteContext* context, std::unique_ptr graph_info,
-               bool preserve_inputs, bool preserve_intermediates,
-               int tensor_alignment = kDefaultTensorAlignment);
+               bool preserve_inputs, bool preserve_intermediates);
   ~ArenaPlanner() override;
   ArenaPlanner(const ArenaPlanner&) = delete;
   ArenaPlanner& operator=(const ArenaPlanner&) = delete;
@@ -117,9 +112,6 @@ class ArenaPlanner : public MemoryPlanner {
   // If true, then no overlapping of memory areas is done, meaning intermediates
   // results can be queried after running (modulo running delegates).
   bool preserve_intermediates_;
-
-  // Number of bytes that tensor buffers should be aligned to.
-  int tensor_alignment_;
 };
 
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/arena_planner_test.cc b/tensorflow/contrib/lite/arena_planner_test.cc
index 7d7c41289c..1adb426d58 100644
--- a/tensorflow/contrib/lite/arena_planner_test.cc
+++ b/tensorflow/contrib/lite/arena_planner_test.cc
@@ -24,8 +24,6 @@ limitations under the License.
 namespace tflite {
 namespace {
 
-constexpr const int kTensorAlignment = 4;
-
 // A simple op to be used in tests, as syntactic sugar.
 class TestOp {
  public:
@@ -158,7 +156,7 @@ class ArenaPlannerTest : public ::testing::Test {
     context_.ReportError = ReportError;
     planner_.reset(new ArenaPlanner(
         &context_, std::unique_ptr(new TestGraphInfo(graph)),
-        preserve_inputs, /*preserve intermediates*/ false, kTensorAlignment));
+        preserve_inputs, /*preserve intermediates*/ false));
     CHECK(planner_->ResetAllocations() == kTfLiteOk);
     CHECK(planner_->PlanAllocations() == kTfLiteOk);
   }
@@ -180,8 +178,8 @@ class ArenaPlannerTest : public ::testing::Test {
     const TfLiteTensor& tensor = (*graph_->tensors())[tensor_index];
     int64_t offset = GetOffset(tensor_index) + tensor.bytes;
     // We must make sure the offset is aligned to kDefaultArenaAlignment.
-    if (offset % kTensorAlignment != 0) {
-      offset += kTensorAlignment - offset % kTensorAlignment;
+    if (offset % 4 != 0) {
+      offset += 4 - offset % 4;
     }
     return offset;
   };
diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index 9549b4445d..ad30624f40 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -58,7 +58,6 @@ cc_library(
     }),
     deps = [
         ":op_macros",
-        "//tensorflow/contrib/lite:arena_planner",
         "//tensorflow/contrib/lite:context",
         "//tensorflow/contrib/lite/kernels/internal:optimized",
     ],
diff --git a/tensorflow/contrib/lite/kernels/eigen_support.cc b/tensorflow/contrib/lite/kernels/eigen_support.cc
index e542ad0765..4f0d020793 100644
--- a/tensorflow/contrib/lite/kernels/eigen_support.cc
+++ b/tensorflow/contrib/lite/kernels/eigen_support.cc
@@ -16,7 +16,6 @@ limitations under the License.
 
 #include 
 
-#include "tensorflow/contrib/lite/arena_planner.h"
 #include "tensorflow/contrib/lite/kernels/internal/optimized/eigen_spatial_convolutions.h"
 #include "tensorflow/contrib/lite/kernels/op_macros.h"
 
@@ -24,16 +23,6 @@ namespace tflite {
 namespace eigen_support {
 namespace {
 
-#ifndef EIGEN_DONT_ALIGN
-// Eigen may require buffers to be algiend to 16, 32 or 64 bytes depending on
-// hardware architecture and build configurations.
-// If the static assertion fails, try to increase `kDefaultTensorAlignment` to
-// in `arena_planner.h` to 32 or 64.
-static_assert(
-    kDefaultTensorAlignment % EIGEN_MAX_ALIGN_BYTES == 0,
-    "kDefaultArenaAlignment doesn't comply with Eigen alignment requirement.");
-#endif  // EIGEN_DONT_ALIGN
-
 // We have a single global threadpool for all convolution operations. This means
 // that inferences started from different threads may block each other, but
 // since the underlying resource of CPU cores should be consumed by the
-- 
cgit v1.2.3


From dee0908764c391b275c9eac737f6e480fc8e8310 Mon Sep 17 00:00:00 2001
From: Clayne Robison 
Date: Tue, 17 Jul 2018 21:36:45 -0700
Subject: Adding better support for avx- and avx2-class instruction sets.

-march=sandybridge enables MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, POPCNT, AVX, AES and PCLMUL instruction set support. See https://gcc.gnu.org/onlinedocs/gcc-5.4.0/gcc/x86-Options.html#x86-Options

-march=haswell enables MOVBE, MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, POPCNT, AVX, AVX2, AES, PCLMUL, FSGSBASE, RDRND, FMA, BMI, BMI2 and F16C instruction set support. See https://gcc.gnu.org/onlinedocs/gcc-5.4.0/gcc/x86-Options.html#x86-Options
---
 tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh b/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh
index 1d9c832d66..7de58ef625 100755
--- a/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh
+++ b/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh
@@ -35,7 +35,8 @@ echo "TF_DOCKER_BUILD_IMAGE_NAME=${TF_DOCKER_BUILD_IMAGE_NAME}"
 echo "TF_DOCKER_BUILD_VERSION=${TF_DOCKER_BUILD_VERSION}"
 
 # Build containers for AVX
-#"TF_BAZEL_BUILD_OPTIONS": "'{}'" (default build option= avx)
+# Include the instructions for sandybridge and later, but tune for ivybridge
+TF_BAZEL_BUILD_OPTIONS="--config=mkl --copt=-march=sandybridge --copt=-mtune=ivybridge --copt=-O3 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0"
 
 # build the python 2 container and whl
 TF_DOCKER_BUILD_TYPE="MKL" \
@@ -55,7 +56,8 @@ TF_DOCKER_BUILD_TYPE="MKL" \
   ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh
 
 # Build containers for AVX2
-TF_BAZEL_BUILD_OPTIONS="--config=mkl --copt=-mavx2 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0"
+# Include the instructions for haswell and later, but tune for broadwell
+TF_BAZEL_BUILD_OPTIONS="--config=mkl --copt=-march=haswell --copt=-mtune=broadwell --copt=-O3 --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0"
 
 # build the python 2 container and whl
 TF_DOCKER_BUILD_TYPE="MKL" \
-- 
cgit v1.2.3


From 81161f9d9987a8eb70793d95048c20be34292859 Mon Sep 17 00:00:00 2001
From: Yunxing Dai 
Date: Tue, 17 Jul 2018 21:36:16 -0700
Subject: - Use InlinedVector for ShapeIndex. - Use a separate IndexTable for
 lookups. This reduces the number of cachelines needed for storing
 ShapeTreeNodes. - Set up benchmark for flat tuples, useful as a benchmark for
 future optimizations.

name                          old time/op    new time/op    delta
BM_Construct/2/8              8.34?s ? 1%    7.57?s ? 2%      -9.26%  (p=0.008 n=5+5)
BM_Construct/1/1000            143?s ? 1%     132?s ? 1%      -7.29%  (p=0.008 n=5+5)
BM_ConstructUnowned/2/8       2.18?s ? 4%    1.31?s ? 1%     -39.99%  (p=0.008 n=5+5)
BM_ConstructUnowned/1/1000    23.0?s ? 7%    15.1?s ? 1%     -34.47%  (p=0.008 n=5+5)
BM_Copy/2/8                   1.52?s ? 5%    0.37?s ? 1%     -76.01%  (p=0.008 n=5+5)
BM_Copy/1/1000                18.7?s ? 3%     4.9?s ? 2%     -73.85%  (p=0.008 n=5+5)
BM_Move/2/8                    0.03ns ? 2%   13.42ns ? 1%  +40877.10%  (p=0.016 n=4+5)
BM_Move/1/1000                 0.03ns ? 0%   13.54ns ? 3%  +40930.30%  (p=0.016 n=4+5)
BM_ForEach/2/8                 26.4ns ? 1%    27.9ns ? 2%      +5.77%  (p=0.008 n=5+5)
BM_ForEach/1/1000               271ns ? 1%     273ns ? 0%      +0.81%  (p=0.016 n=5+4)
BM_Iterate/2/8                 25.5ns ? 3%    23.9ns ? 8%        ~     (p=0.151 n=5+5)
BM_Iterate/1/1000               272ns ? 2%     271ns ? 1%        ~     (p=0.984 n=5+5)

name                          old allocs/op  new allocs/op  delta
BM_Construct/2/8                  373 ? 0%       276 ? 0%     -26.01%  (p=0.008 n=5+5)
BM_Construct/1/1000             5.00k ? 0%     4.00k ? 0%     -20.00%  (p=0.008 n=5+5)
BM_ConstructUnowned/2/8          99.0 ? 0%       2.0 ? 0%     -97.98%  (p=0.008 n=5+5)
BM_ConstructUnowned/1/1000      1.00k ? 0%     0.00k ? 0%     -99.80%  (p=0.008 n=5+5)
BM_Copy/2/8                       105 ? 0%        19 ? 0%     -81.90%  (p=0.008 n=5+5)
BM_Copy/1/1000                  1.31k ? 0%     0.25k ? 0%     -80.84%  (p=0.008 n=5+5)
BM_Move/2/8                      23.0 ? 0%      17.0 ? 0%     -26.09%  (p=0.008 n=5+5)
BM_Move/1/1000                    313 ? 0%       250 ? 0%     -20.13%  (p=0.008 n=5+5)
BM_ForEach/2/8                   23.0 ? 0%      17.0 ? 0%     -26.09%  (p=0.008 n=5+5)
BM_ForEach/1/1000                 313 ? 0%       250 ? 0%     -20.13%  (p=0.008 n=5+5)
BM_Iterate/2/8                   23.0 ? 0%      17.0 ? 0%     -26.09%  (p=0.008 n=5+5)
BM_Iterate/1/1000                 313 ? 0%       250 ? 0%     -20.13%  (p=0.008 n=5+5)

PiperOrigin-RevId: 205024687
---
 tensorflow/compiler/xla/shape_tree.h       | 140 ++++++++++++++++++++++-------
 tensorflow/compiler/xla/shape_tree_test.cc |  21 +++--
 tensorflow/compiler/xla/shape_util.h       |  13 +--
 3 files changed, 131 insertions(+), 43 deletions(-)

diff --git a/tensorflow/compiler/xla/shape_tree.h b/tensorflow/compiler/xla/shape_tree.h
index 4aacc87b78..c74dd648ad 100644
--- a/tensorflow/compiler/xla/shape_tree.h
+++ b/tensorflow/compiler/xla/shape_tree.h
@@ -44,10 +44,6 @@ struct ShapeTreeNode {
   // Data corresponding to this node.
   std::pair data;
 
-  // Children of this node, as indices into the container's nodes_ array.
-  std::vector children;
-
-  // Tells whether this is a leaf node.
   bool is_leaf = true;
 
   explicit ShapeTreeNode(ShapeIndex index)
@@ -56,6 +52,20 @@ struct ShapeTreeNode {
       : data(std::move(index), std::move(data)) {}
 };
 
+// Internal representation of an index table entry.
+struct IndexTableEntry {
+  // Index of the node in the ShapeTreeNode vector.
+  uint32 index;
+  // Index of the first child in a IndexTableEntry vector. In the index
+  // table all children entries for a given node will be placed next to each
+  // other. This allows us to use a single field to index them.
+  uint32 children_start;
+#ifndef NDEBUG
+  // Number of children, used for bounds checking.
+  uint32 children_count;
+#endif
+};
+
 }  // namespace internal
 
 template 
@@ -84,6 +94,7 @@ template 
 class ShapeTree {
  public:
   using Node = internal::ShapeTreeNode;
+  using Index = internal::IndexTableEntry;
 
   // Default constructor creates a tree with a nil shape (i.e. an empty tuple).
   ShapeTree() : ShapeTree(ShapeUtil::MakeNil()) {}
@@ -267,11 +278,12 @@ class ShapeTree {
  private:
   // Initialize node->children based on 'shape'. All children are assigned the
   // the given 'init_value'.
-  void InitChildren(const Shape& shape, const T& init_value, Node* node);
+  void InitChildren(const Shape& shape, const T& init_value, Node* node,
+                    Index* index);
 
   // Initialize node->children based on 'shape'. All children have
   // default-constructed data values.
-  void InitChildren(const Shape& shape, Node* node);
+  void InitChildren(const Shape& shape, Node* node, Index* index);
 
   // Returns the number of subshapes, including interior nodes, in shape.
   int64 CountSubshapes(const Shape& shape);
@@ -291,6 +303,9 @@ class ShapeTree {
   // The nodes in this shape tree.
   std::vector nodes_;
 
+  // Index table for node lookups.
+  std::vector index_table_;
+
   // If we own our Shape, this field contains it, and shape_ is a pointer into
   // here.  Otherwise if we don't own our shape, this is nullptr.
   std::shared_ptr shape_storage_;
@@ -373,36 +388,74 @@ int64 ShapeTree::CountSubshapes(const Shape& shape) {
 
 template 
 void ShapeTree::InitChildren(const Shape& shape, const T& init_value,
-                                Node* node) {
+                                Node* node, Index* index) {
   if (ShapeUtil::IsTuple(shape)) {
     const int64 size = ShapeUtil::TupleElementCount(shape);
-    node->children.reserve(size);
+#ifndef NDEBUG
+    index->children_count = size;
+#endif
     node->is_leaf = false;
     ShapeIndex shape_index = node->data.first;
     shape_index.push_back(0);
+
+    // At the end of the index_table, reserve a continuous space to hold the
+    // children of current node. In order to enforce the invariant that all
+    // children of a given node are placed together, we need to do the
+    // reservation before we recurse into any of its children.
+    int64 children_start_position = index_table_.size();
+    index_table_.resize(index_table_.size() + size);
+
     for (int i = 0; i < size; ++i) {
       shape_index[shape_index.size() - 1] = i;
-      node->children.push_back(nodes_.size());
+      index_table_[children_start_position + i].index = nodes_.size();
+      // The first child of the node in the index table is placed at the end of
+      // the table.
+      index_table_[children_start_position + i].children_start =
+          index_table_.size();
       nodes_.emplace_back(shape_index, init_value);
-      InitChildren(shape.tuple_shapes(i), init_value, &nodes_.back());
+      InitChildren(shape.tuple_shapes(i), init_value, &nodes_.back(),
+                   &index_table_[children_start_position + i]);
     }
+  } else {
+#ifndef NDEBUG
+    index->children_count = 0;
+#endif
   }
 }
 
 template 
-void ShapeTree::InitChildren(const Shape& shape, Node* node) {
+void ShapeTree::InitChildren(const Shape& shape, Node* node, Index* index) {
   if (ShapeUtil::IsTuple(shape)) {
     const int64 size = ShapeUtil::TupleElementCount(shape);
-    node->children.reserve(size);
+#ifndef NDEBUG
+    index->children_count = size;
+#endif
     node->is_leaf = false;
     ShapeIndex shape_index = node->data.first;
     shape_index.push_back(0);
+
+    // At the end of the index_table, reserve a continuous space to hold the
+    // children of current node. In order to enforce the invariant that all
+    // children of a given node are placed together, we need to do the
+    // reservation before we recurse into any of its children.
+    int64 children_start_position = index_table_.size();
+    index_table_.resize(index_table_.size() + size);
+
     for (int i = 0; i < size; ++i) {
       shape_index[shape_index.size() - 1] = i;
-      node->children.push_back(nodes_.size());
+      index_table_[children_start_position + i].index = nodes_.size();
+      // The first child of the node in the index table is placed at the end of
+      // the table.
+      index_table_[children_start_position + i].children_start =
+          index_table_.size();
       nodes_.emplace_back(shape_index);
-      InitChildren(shape.tuple_shapes(i), &nodes_.back());
+      InitChildren(shape.tuple_shapes(i), &nodes_.back(),
+                   &index_table_[children_start_position + i]);
     }
+  } else {
+#ifndef NDEBUG
+    index->children_count = 0;
+#endif
   }
 }
 
@@ -413,24 +466,36 @@ ShapeTree::ShapeTree(Shape shape)
   // The shape_ field is just used to hold the structure of the shape.
   // It should not be relied upon to store layout information.
   LayoutUtil::ClearLayout(shape_storage_.get());
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{});
-  InitChildren(*shape_, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, &nodes_[0], &index_table_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const Shape* shape) : shape_(shape) {
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{});
-  InitChildren(*shape_, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, &nodes_[0], &index_table_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const std::shared_ptr& shape)
     : shape_storage_(shape), shape_(shape_storage_.get()) {
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{});
-  InitChildren(*shape_, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, &nodes_[0], &index_table_[0]);
 }
 
 template 
@@ -440,26 +505,38 @@ ShapeTree::ShapeTree(Shape shape, const T& init_value)
   // The shape_ field is just used to hold the structure of the shape.
   // It should not be relied upon to store layout information.
   LayoutUtil::ClearLayout(shape_storage_.get());
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{}, init_value);
-  InitChildren(*shape_, init_value, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, init_value, &nodes_[0], &index_table_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const Shape* shape, const T& init_value)
     : shape_(shape) {
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{}, init_value);
-  InitChildren(*shape_, init_value, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, init_value, &nodes_[0], &index_table_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const std::shared_ptr& shape,
                         const T& init_value)
     : shape_storage_(shape), shape_(shape_storage_.get()) {
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{}, init_value);
-  InitChildren(*shape_, init_value, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, init_value, &nodes_[0], &index_table_[0]);
 }
 
 template 
@@ -474,13 +551,16 @@ T* ShapeTree::mutable_element(ShapeIndexView index) {
 
 template 
 internal::ShapeTreeNode* ShapeTree::Lookup(ShapeIndexView index) {
-  Node* node = &nodes_[0];
+  Index* iter = &index_table_[0];
   for (const int64 i : index) {
     CHECK_GE(i, 0);
-    CHECK_LT(i, node->children.size());
-    node = &nodes_[node->children[i]];
+#ifndef NDEBUG
+    CHECK_LT(i, iter->children_count);
+#endif
+    iter = &index_table_[iter->children_start + i];
   }
-  return node;
+
+  return &nodes_[iter->index];
 }
 
 template 
diff --git a/tensorflow/compiler/xla/shape_tree_test.cc b/tensorflow/compiler/xla/shape_tree_test.cc
index 51de82e957..4391078b64 100644
--- a/tensorflow/compiler/xla/shape_tree_test.cc
+++ b/tensorflow/compiler/xla/shape_tree_test.cc
@@ -227,14 +227,16 @@ TEST_F(ShapeTreeTest, NestedTupleShape) {
 
 TEST_F(ShapeTreeTest, InvalidIndexingTuple) {
   ShapeTree shape_tree{tuple_shape_};
-
+#ifndef NDEBUG
   EXPECT_DEATH(shape_tree.element({4}), "");
+#endif
 }
 
 TEST_F(ShapeTreeTest, InvalidIndexingNestedTuple) {
   ShapeTree shape_tree{nested_tuple_shape_};
-
+#ifndef NDEBUG
   EXPECT_DEATH(shape_tree.element({0, 0}), "");
+#endif
 }
 
 TEST_F(ShapeTreeTest, ShapeTreeOfNonCopyableType) {
@@ -602,12 +604,15 @@ void BM_Iterate(int iters, int depth, int fan_out) {
   }
 }
 
-BENCHMARK(BM_Construct)->ArgPair(2, 8);
-BENCHMARK(BM_ConstructUnowned)->ArgPair(2, 8);
-BENCHMARK(BM_Copy)->ArgPair(2, 8);
-BENCHMARK(BM_Move)->ArgPair(2, 8);
-BENCHMARK(BM_ForEach)->ArgPair(2, 8);
-BENCHMARK(BM_Iterate)->ArgPair(2, 8);
+#define BENCHMARK_WITH_ARGS(name) \
+  BENCHMARK(name)->ArgPair(2, 8)->ArgPair(1, 1000)
+
+BENCHMARK_WITH_ARGS(BM_Construct);
+BENCHMARK_WITH_ARGS(BM_ConstructUnowned);
+BENCHMARK_WITH_ARGS(BM_Copy);
+BENCHMARK_WITH_ARGS(BM_Move);
+BENCHMARK_WITH_ARGS(BM_ForEach);
+BENCHMARK_WITH_ARGS(BM_Iterate);
 
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index 17c1d7b10a..83d15e8fe3 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/gtl/optional.h"
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/env.h"
@@ -73,10 +74,12 @@ class ShapeIndex {
   // push_front is O(n^2), but shapes don't usually have a ton of dimensions.
   void push_front(int64 value) { indices_.insert(indices_.begin(), value); }
 
-  std::vector::const_iterator begin() const { return indices_.begin(); }
-  std::vector::const_iterator end() const { return indices_.end(); }
-  std::vector::iterator begin() { return indices_.begin(); }
-  std::vector::iterator end() { return indices_.end(); }
+  using container_type = gtl::InlinedVector;
+
+  container_type::const_iterator begin() const { return indices_.begin(); }
+  container_type::const_iterator end() const { return indices_.end(); }
+  container_type::iterator begin() { return indices_.begin(); }
+  container_type::iterator end() { return indices_.end(); }
 
   const int64* data() const { return indices_.data(); }
 
@@ -97,7 +100,7 @@ class ShapeIndex {
   string ToString() const;
 
  private:
-  std::vector indices_;
+  container_type indices_;
 };
 
 // A view into a ShapeIndex as above, with the cheap/easy ability to consume the
-- 
cgit v1.2.3


From aa15692e54390cf3967d51bc60acf5f783df9c08 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 17 Jul 2018 22:29:31 -0700
Subject: Update documentation for using pruning and contrib/slim training
 utility

PiperOrigin-RevId: 205027982
---
 tensorflow/contrib/model_pruning/python/learning.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/model_pruning/python/learning.py b/tensorflow/contrib/model_pruning/python/learning.py
index 2b79c23cef..26695237c2 100644
--- a/tensorflow/contrib/model_pruning/python/learning.py
+++ b/tensorflow/contrib/model_pruning/python/learning.py
@@ -33,11 +33,14 @@ to support training of pruned models
   # Create the train_op
   train_op = slim.learning.create_train_op(total_loss, optimizer)
 
-  # Set up sparsity
-  sparsity = pruning.setup_gradual_sparsity(self.global_step)
+  # Parse pruning hyperparameters
+  pruning_hparams = pruning.get_pruning_hparams().parse(FLAGS.pruning_hparams)
 
-  # Create mask update op
-  mask_update_op = pruning.add_mask_update_ip(sparsity)
+  # Create a pruning object using the pruning_hparams
+  p = pruning.Pruning(pruning_hparams)
+
+  # Add mask update ops to the graph
+  mask_update_op = p.conditional_mask_update_op()
 
   # Run training.
   learning.train(train_op,
-- 
cgit v1.2.3


From f1de0ddd55dcae6237ea7d21ccddcc6467a6cf8b Mon Sep 17 00:00:00 2001
From: Priya Gupta 
Date: Tue, 17 Jul 2018 23:08:47 -0700
Subject: Add support for MirroredVariables in init_from_checkpoint and
 warm_start in estimator.

PiperOrigin-RevId: 205030626
---
 tensorflow/contrib/distribute/python/BUILD         | 37 +++++++++
 .../distribute/python/checkpoint_utils_test.py     | 72 ++++++++++++++++
 tensorflow/contrib/distribute/python/values.py     | 15 ++--
 .../distribute/python/warm_starting_util_test.py   | 97 ++++++++++++++++++++++
 tensorflow/python/training/checkpoint_utils.py     | 52 ++++++++++--
 tensorflow/python/training/warm_starting_util.py   | 18 ++--
 6 files changed, 266 insertions(+), 25 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/python/checkpoint_utils_test.py
 create mode 100644 tensorflow/contrib/distribute/python/warm_starting_util_test.py

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 40dbfa3dd2..f5d7e24ae2 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -610,3 +610,40 @@ cuda_py_test(
         "no_pip",
     ],
 )
+
+cuda_py_test(
+    name = "warm_starting_util_test",
+    size = "medium",
+    srcs = ["warm_starting_util_test.py"],
+    additional_deps = [
+        ":combinations",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+    ],
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
+)
+
+cuda_py_test(
+    name = "checkpoint_utils_test",
+    size = "medium",
+    srcs = ["checkpoint_utils_test.py"],
+    additional_deps = [
+        ":combinations",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:checkpoint_utils_test",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+    ],
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
+)
diff --git a/tensorflow/contrib/distribute/python/checkpoint_utils_test.py b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
new file mode 100644
index 0000000000..fe3df9cbb9
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/checkpoint_utils_test.py
@@ -0,0 +1,72 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for checkpoint_utils.init_from_checkpoint with Distribution Strategy.
+
+These tests are located here instead of as part of
+`python.training.CheckpointsTest` because they need access to distribution
+strategies which are only present in contrib right now.
+TODO(priyag): Move the tests to core `python.training.CheckpointsTest` when
+distribution strategy moves out of contrib.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+from tensorflow.python.training import checkpoint_utils
+from tensorflow.python.training import checkpoint_utils_test
+
+
+class CheckpointUtilsWithDistributionStrategyTest(
+    test.TestCase, parameterized.TestCase):
+
+  @combinations.generate(combinations.combine(
+      distribution=[combinations.default_strategy,
+                    combinations.one_device_strategy,
+                    combinations.mirrored_strategy_with_gpu_and_cpu,
+                    combinations.mirrored_strategy_with_two_gpus],
+      in_tower_mode=[True, False],
+      mode=["graph"]))
+  def testInitFromCheckpoint(self, distribution, in_tower_mode):
+    checkpoint_dir = self.get_temp_dir()
+    with self.test_session() as session:
+      v1_value, _, _, _ = checkpoint_utils_test._create_checkpoints(
+          session, checkpoint_dir)
+
+    def init_and_verify(g):
+      v1 = variable_scope.get_variable("new_var1", [1, 10])
+      checkpoint_utils.init_from_checkpoint(checkpoint_dir, {
+          "var1": "new_var1",
+      })
+      with self.test_session(graph=g) as session:
+        session.run(variables.global_variables_initializer())
+        self.assertAllEqual(v1_value, self.evaluate(v1))
+
+    with ops.Graph().as_default() as g, distribution.scope():
+      if in_tower_mode:
+        distribution.call_for_each_tower(init_and_verify, g)
+      else:
+        init_and_verify(g)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index 1b5e00bc79..1761a43251 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -33,7 +33,6 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope as vs
 from tensorflow.python.training import device_util
 from tensorflow.python.training import distribute as distribute_lib
@@ -336,23 +335,27 @@ class MirroredVariable(DistributedVariable, Mirrored,
         raise ValueError("You must specify an aggregation method to update a "
                          "MirroredVariable in Tower Context.")
 
-      def merge_fn(strategy, value):
+      def merge_fn(strategy, value, *other_args, **other_kwargs):
         return strategy.update(
             self, f,
             strategy.reduce(
-                aggregation=self._aggregation, value=value, destinations=self))
+                aggregation=self._aggregation, value=value, destinations=self),
+            *other_args, **other_kwargs)
 
       return distribute_lib.get_tower_context().merge_call(merge_fn, *args,
                                                            **kwargs)
 
   def assign_sub(self, *args, **kwargs):
-    return self._assign_func(f=state_ops.assign_sub, *args, **kwargs)
+    assign_sub_fn = lambda var, *a, **kw: var.assign_sub(*a, **kw)
+    return self._assign_func(f=assign_sub_fn, *args, **kwargs)
 
   def assign_add(self, *args, **kwargs):
-    return self._assign_func(f=state_ops.assign_add, *args, **kwargs)
+    assign_add_fn = lambda var, *a, **kw: var.assign_add(*a, **kw)
+    return self._assign_func(f=assign_add_fn, *args, **kwargs)
 
   def assign(self, *args, **kwargs):
-    return self._assign_func(f=state_ops.assign, *args, **kwargs)
+    assign_fn = lambda var, *a, **kw: var.assign(*a, **kw)
+    return self._assign_func(f=assign_fn, *args, **kwargs)
 
   def is_initialized(self, name=None):
     # We have to cast the self._index.values() to a `list` because when we
diff --git a/tensorflow/contrib/distribute/python/warm_starting_util_test.py b/tensorflow/contrib/distribute/python/warm_starting_util_test.py
new file mode 100644
index 0000000000..d8bacdb338
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/warm_starting_util_test.py
@@ -0,0 +1,97 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for warm_starting_util with Distribution Strategy.
+
+These tests are located here instead of as part of `WarmStartingUtilTest`
+because they need access to distribution strategies which are only present in
+contrib right now.
+TODO(priyag): Move the tests to core `WarmStartingUtilTest` when distribution
+strategy moves out of contrib.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+from absl.testing import parameterized
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+from tensorflow.python.training import saver as saver_lib
+from tensorflow.python.training import warm_starting_util as ws_util
+
+
+class WarmStartingUtilWithDistributionStrategyTest(
+    test.TestCase, parameterized.TestCase):
+
+  @combinations.generate(combinations.combine(
+      distribution=[combinations.default_strategy,
+                    combinations.one_device_strategy,
+                    combinations.mirrored_strategy_with_gpu_and_cpu,
+                    combinations.mirrored_strategy_with_two_gpus],
+      save_with_distribution=[True, False],
+      restore_with_distribution=[True, False],
+      mode=["graph"]))
+  def testWarmStart(self, distribution, save_with_distribution,
+                    restore_with_distribution):
+
+    var_name = "v"
+    original_value = [[1., 2.], [3., 4.]]
+
+    # Create variable and save checkpoint from which to warm-start.
+    def create_var(g):
+      with self.test_session(graph=g) as sess:
+        var = variable_scope.get_variable(var_name, initializer=original_value)
+        sess.run(variables.global_variables_initializer())
+        saver = saver_lib.Saver()
+        ckpt_prefix = os.path.join(self.get_temp_dir(), "model")
+        saver.save(sess, ckpt_prefix, global_step=0)
+        return var, sess.run(var)
+
+    if save_with_distribution:
+      with ops.Graph().as_default() as g, distribution.scope():
+        _, prev_init_val = create_var(g)
+    else:
+      with ops.Graph().as_default() as g:
+        _, prev_init_val = create_var(g)
+
+    # Verify we initialized the values correctly.
+    self.assertAllEqual(original_value, prev_init_val)
+
+    def warm_start(g):
+      with self.test_session(graph=g) as sess:
+        # Initialize with zeros.
+        var = variable_scope.get_variable(
+            var_name, initializer=[[0., 0.], [0., 0.]])
+        ws_util.warm_start(self.get_temp_dir())
+        sess.run(variables.global_variables_initializer())
+        # Verify weights were correctly warm-started to previous values.
+        self.assertAllEqual(original_value, self.evaluate(var))
+
+    # Warm start in a new graph.
+    if restore_with_distribution:
+      with ops.Graph().as_default() as g, distribution.scope():
+        warm_start(g)
+    else:
+      with ops.Graph().as_default() as g:
+        warm_start(g)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/training/checkpoint_utils.py b/tensorflow/python/training/checkpoint_utils.py
index 5b372e82b3..883f4fd910 100644
--- a/tensorflow/python/training/checkpoint_utils.py
+++ b/tensorflow/python/training/checkpoint_utils.py
@@ -29,6 +29,7 @@ from tensorflow.python.ops import variable_scope as vs
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import saver
 from tensorflow.python.util.tf_export import tf_export
 
@@ -179,6 +180,16 @@ def init_from_checkpoint(ckpt_dir_or_file, assignment_map):
     tf.errors.OpError: If missing checkpoints or tensors in checkpoints.
     ValueError: If missing variables in current graph.
   """
+  if distribute_lib.get_cross_tower_context():
+    _init_from_checkpoint(None, ckpt_dir_or_file, assignment_map)
+  else:
+    distribute_lib.get_tower_context().merge_call(
+        _init_from_checkpoint, ckpt_dir_or_file, assignment_map)
+
+
+def _init_from_checkpoint(_, ckpt_dir_or_file, assignment_map):
+  """See `init_from_checkpoint` for documentation."""
+
   ckpt_file = _get_checkpoint_filename(ckpt_dir_or_file)
   reader = load_checkpoint(ckpt_dir_or_file)
   variable_map = reader.get_variable_to_shape_map()
@@ -187,10 +198,9 @@ def init_from_checkpoint(ckpt_dir_or_file, assignment_map):
     var = None
     # Check if this is Variable object or list of Variable objects (in case of
     # partitioned variables).
-    is_var = lambda x: isinstance(x, variables.Variable)
-    if is_var(current_var_or_name) or (
+    if _is_variable(current_var_or_name) or (
         isinstance(current_var_or_name, list)
-        and all(is_var(v) for v in current_var_or_name)):
+        and all(_is_variable(v) for v in current_var_or_name)):
       var = current_var_or_name
     else:
       store_vars = vs._get_default_variable_store()._vars  # pylint:disable=protected-access
@@ -205,7 +215,7 @@ def init_from_checkpoint(ckpt_dir_or_file, assignment_map):
         raise ValueError("Tensor %s is not found in %s checkpoint %s" % (
             tensor_name_in_ckpt, ckpt_dir_or_file, variable_map
         ))
-      if is_var(var):
+      if _is_variable(var):
         # Additional at-call-time checks.
         if not var.get_shape().is_compatible_with(
             variable_map[tensor_name_in_ckpt]):
@@ -297,13 +307,34 @@ def _set_checkpoint_initializer(variable,
   with ops.device(variable.device), ops.device("/cpu:0"):
     restore_op = io_ops.restore_v2(
         ckpt_file, [tensor_name], [slice_spec], [base_type], name=name)[0]
-    if isinstance(variable, resource_variable_ops.ResourceVariable):
+
+    # TODO(priyag, allenl): Use `SaveableObject.restore` instead here.
+    if resource_variable_ops.is_resource_variable(variable):
       init_op = variable.assign(restore_op, read_value=False)
     else:
       init_op = state_ops.assign(variable, restore_op)
-    variable._initializer_op = init_op  # pylint:disable=protected-access
-    restore_op.set_shape(variable.shape)
-    variable._initial_value = restore_op  # pylint:disable=protected-access
+
+    # pylint:disable=protected-access
+    # We need special handling for `DistributedVariable`s as they contain
+    # mutliple actual variables. `assign` on a `DistributedVariable` returns a
+    # combined `init_op` which contains initializers for all the contained
+    # variables. We then set each underlying variable's `_initializer_op` using
+    # the corresponding `init_op`.
+    # TODO(priyag): Use `isinstance` checks when `DistributedVariable` class
+    # moves out of contrib.
+    if any(base.__name__ == "DistributedVariable"
+           for base in  variable.__class__.__bases__):
+      assert distribute_lib.get_cross_tower_context()
+      assert hasattr(variable, "_index")
+      for (d, v) in six.iteritems(variable._index):
+        v._initializer_op = init_op._index[d]
+        restore_op.set_shape(v.shape)
+        v._initial_value = restore_op
+    else:
+      variable._initializer_op = init_op
+      restore_op.set_shape(variable.shape)
+      variable._initial_value = restore_op
+    # pylint:enable=protected-access
 
 
 def _set_variable_or_list_initializer(variable_or_list, ckpt_file,
@@ -337,6 +368,11 @@ def _set_variable_or_list_initializer(variable_or_list, ckpt_file,
     _set_checkpoint_initializer(variable_or_list, ckpt_file, tensor_name, "")
 
 
+def _is_variable(x):
+  return (isinstance(x, variables.Variable) or
+          resource_variable_ops.is_resource_variable(x))
+
+
 def _collect_partitioned_variable(name, all_vars):
   """Returns list of `tf.Variable` that comprise the partitioned variable."""
   if name + "/part_0" in all_vars:
diff --git a/tensorflow/python/training/warm_starting_util.py b/tensorflow/python/training/warm_starting_util.py
index ec740abdd1..b1a7cfab83 100644
--- a/tensorflow/python/training/warm_starting_util.py
+++ b/tensorflow/python/training/warm_starting_util.py
@@ -22,7 +22,6 @@ import collections
 import six
 
 from tensorflow.python.framework import ops
-from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
@@ -83,11 +82,6 @@ class VocabInfo(
     )
 
 
-def _is_variable(x):
-  return (isinstance(x, variables_lib.Variable) or
-          isinstance(x, resource_variable_ops.ResourceVariable))
-
-
 def _infer_var_name(var):
   """Returns name of the `var`.
 
@@ -126,9 +120,10 @@ def _warm_start_var(var, prev_ckpt, prev_tensor_name=None):
     prev_tensor_name: Name of the tensor to lookup in provided `prev_ckpt`. If
       None, we lookup tensor with same name as given `var`.
   """
-  if _is_variable(var):
+  if checkpoint_utils._is_variable(var):  # pylint: disable=protected-access
     current_var_name = _infer_var_name([var])
-  elif isinstance(var, list) and all(_is_variable(v) for v in var):
+  elif (isinstance(var, list) and
+        all(checkpoint_utils._is_variable(v) for v in var)):  # pylint: disable=protected-access
     current_var_name = _infer_var_name(var)
   elif isinstance(var, variables_lib.PartitionedVariable):
     current_var_name = _infer_var_name([var])
@@ -193,9 +188,10 @@ def _warm_start_var_with_vocab(var,
           prev_vocab_path):
     raise ValueError("Invalid args: Must provide all of [current_vocab_path, "
                      "current_vocab_size, prev_ckpt, prev_vocab_path}.")
-  if _is_variable(var):
+  if checkpoint_utils._is_variable(var):
     var = [var]
-  elif isinstance(var, list) and all(_is_variable(v) for v in var):
+  elif (isinstance(var, list) and
+        all(checkpoint_utils._is_variable(v) for v in var)):
     var = var
   elif isinstance(var, variables_lib.PartitionedVariable):
     var = var._get_variable_list()
@@ -271,7 +267,7 @@ def _get_grouped_variables(vars_to_warm_start):
       for v in vars_to_warm_start:
         list_of_vars += ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES,
                                            scope=v)
-    elif all([_is_variable(v) for v in vars_to_warm_start]):
+    elif all([checkpoint_utils._is_variable(v) for v in vars_to_warm_start]):  # pylint: disable=protected-access
       list_of_vars = vars_to_warm_start
     else:
       raise ValueError("If `vars_to_warm_start` is a list, it must be all "
-- 
cgit v1.2.3


From d9d029f510dbbc92329bafcd6bf2fbd0d273a675 Mon Sep 17 00:00:00 2001
From: Thomas Joerg 
Date: Tue, 17 Jul 2018 23:50:05 -0700
Subject: [XLA:GPU] Generalize the column reduction algorithm to handle tile
 widths greater than 1.

Tiles of width 1 result in poor memory bandwidth for 16b inputs.

PiperOrigin-RevId: 205033124
---
 .../xla/service/gpu/ir_emitter_unnested.cc         | 218 ++++++++++++++-------
 1 file changed, 147 insertions(+), 71 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 75bbbbe8ef..f2597da4b9 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -984,8 +984,8 @@ Status IrEmitterUnnested::EmitColumnReduction(
     tensorflow::gtl::ArraySlice<
         std::pair>
         extra_output_gens) {
-  // Divide the input matrix into tiles of size Kx1. For example, when the
-  // input matrix is 4x4 and K=2, the tiled matrix looks like
+  // Divide the input matrix into tiles of size KxL. For example, when the
+  // input matrix is 4x4, K=2, and L=1 the tiled matrix looks like
   //
   //   0123
   //   0123
@@ -997,14 +997,20 @@ Status IrEmitterUnnested::EmitColumnReduction(
   //
   // We choose 128 as the tile size based on empirical evidence. It's big enough
   // to reduce the amount of atomic adds in the end, maximizing the memory
-  // bandwidth.
-  constexpr int64 kTileSize = 128;
+  // bandwidth. A tile width of 2 allows for high memory bandwidth utilization
+  // on 16b input data.
+  constexpr int64 kTileHeight = 128;
+  constexpr int64 kTileWidth = 2;
 
-  // If the height is not a multiple of the tile size, we pad the bottom of the
+  // If the height is not a multiple of kTileHeight, we pad the bottom of the
   // input matrix.
-  const int64 height_in_tiles = CeilOfRatio(height, kTileSize);
-  Shape tiled_input_shape = ShapeUtil::MakeShapeWithLayout(
-      reduce->shape().element_type(), {height_in_tiles, width}, {1, 0});
+  const int64 height_in_tiles = CeilOfRatio(height, kTileHeight);
+  // If width is not a multiple of kTileWidth the rightmost thread will process
+  // fewer input elements.
+  const int64 width_in_tiles = CeilOfRatio(width, kTileWidth);
+  Shape tiled_input_shape =
+      ShapeUtil::MakeShapeWithLayout(reduce->shape().element_type(),
+                                     {height_in_tiles, width_in_tiles}, {1, 0});
   LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
       tiled_input_shape, ir_emitter_context_->device_description());
 
@@ -1016,27 +1022,39 @@ Status IrEmitterUnnested::EmitColumnReduction(
   };
 
   // for (linear_index = threadIdx.x + blockIdx.x * blockDim.x;
-  //      linear_index < height_in_tiles * width;
+  //      linear_index < height_in_tiles * width_in_tiles;
   //      linear_index += blockDim.x * gridDim.x) {
-  //   y_in_tiles = linear_index / width;
-  //   x = linear_index % width;
+  //   y_in_tiles = linear_index / width_in_tiles;
+  //   x_in_tiles = linear_index % width_in_tiles;
   //
-  //   partial_result = init_value;
-  //   if (height % kTileSize == 0 ||
-  //       y_in_tiles * kTileSize + kTileSize <= height) {
-  //     for (element_id_in_tile : range(kTileSize)) {
-  //       y = y_in_tiles * kTileSize + element_id_in_tile;
-  //       partial_result = Reducer(partial_result, input[y][x]);
+  //   partial_results[kTileWidth] = init_values;
+  //   tile_in_y_bounds = height % kTileHeight == 0 ||
+  //       y_in_tiles * kTileHeight + kTileHeight <= height;
+  //   tile_in_x_bounds = width % kTileWidth == 0 ||
+  //       x_in_tiles * kTileWidth + kTileWidth <= width;
+  //   // The implementation handles y and x bound checks separately.
+  //   if (tile_in_y_bounds && tile_in_x_bounds) {
+  //     for (y_offset : range(kTileHeight)) {
+  //       y = y_in_tiles * kTileHeight + y_offset;
+  //       for (x_offset : range(kTileWidth)) {
+  //         x = x_in_tiles * kTileWidth + x_offset;
+  //         partial_result = Reducer(partial_result[x_offset], input[y][x]);
+  //       }
   //     }
   //   } else {
-  //     for (element_id_in_tile : range(kTileSize)) {
-  //       y = y_in_tiles * kTileSize + element_id_in_tile;
-  //       if (y < height) {
-  //         partial_result = Reducer(partial_result, input[y][x]);
+  //     for (y_offset : range(kTileHeight)) {
+  //       y = y_in_tiles * kTileHeight + y_offset;
+  //       for (y_offset : range(kTileHeight)) {
+  //         x = x_in_tiles * kTileWidth + x_offset;
+  //         if (y < height && x < width) {
+  //           partial_result = Reducer(partial_result, input[y][x]);
+  //         }
   //       }
   //     }
   //   }
-  //   AtomicReducer(&output[x], partial_result);
+  //   for (x_offset : range(kTileWidth)) {
+  //     AtomicReducer(&output[x + x_offset], partial_result[x_offset]);
+  //   }
   // }
   auto loop_body_emitter = [=](const IrArray::Index& tile_index) -> Status {
     const int num_reduces = reducers.size();
@@ -1045,41 +1063,48 @@ Status IrEmitterUnnested::EmitColumnReduction(
         llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_);
     std::vector partial_reduction_result_addresses;
     for (int i = 0; i != num_reduces; ++i) {
-      llvm::Value* partial_reduction_result_address = ir_builder_.CreateAlloca(
-          element_ir_type, /*ArraySize=*/nullptr,
-          "partial_reduction_result." + llvm::Twine(i));
-      TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
-                          init_value_gens[i](IrArray::Index(index_ty)));
-      ir_builder_.CreateStore(init_ir_value, partial_reduction_result_address);
-      partial_reduction_result_addresses.push_back(
-          partial_reduction_result_address);
+      for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
+        llvm::Value* partial_reduction_result_address =
+            ir_builder_.CreateAlloca(
+                element_ir_type, /*ArraySize=*/nullptr,
+                "partial_reduction_result." +
+                    llvm::Twine(i * kTileWidth + x_offset));
+        TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
+                            init_value_gens[i](IrArray::Index(index_ty)));
+        ir_builder_.CreateStore(init_ir_value,
+                                partial_reduction_result_address);
+        partial_reduction_result_addresses.push_back(
+            partial_reduction_result_address);
+      }
     }
 
     // Emit an inner for-loop that partially reduces the elements in the given
     // tile.
     llvm::Value* y_in_tiles = tile_index[0];
-    llvm::Value* x = tile_index[1];
+    llvm::Value* x_in_tiles = tile_index[1];
 
     y_in_tiles = ir_builder_.CreateZExtOrTrunc(y_in_tiles, index_ty);
-    x = ir_builder_.CreateZExtOrTrunc(x, index_ty);
+    x_in_tiles = ir_builder_.CreateZExtOrTrunc(x_in_tiles, index_ty);
 
-    auto emit_tile_element_loop = [=](bool tile_in_bounds) -> Status {
+    auto emit_tile_element_loop = [=](bool tile_in_y_bounds,
+                                      bool tile_in_x_bounds) -> Status {
       std::unique_ptr tile_element_loop =
           llvm_ir::ForLoop::EmitForLoop("element_id_in_tile",
                                         index_typed_constant(0),
-                                        index_typed_constant(kTileSize),
+                                        index_typed_constant(kTileHeight),
                                         index_typed_constant(1), &ir_builder_);
 
       // Emit the body of the partial reduction loop.
       llvm_ir::SetToFirstInsertPoint(tile_element_loop->GetBodyBasicBlock(),
                                      &ir_builder_);
       llvm::Value* y = ir_builder_.CreateNSWAdd(
-          ir_builder_.CreateNSWMul(y_in_tiles, index_typed_constant(kTileSize)),
+          ir_builder_.CreateNSWMul(y_in_tiles,
+                                   index_typed_constant(kTileHeight)),
           tile_element_loop->GetIndVarValue());
 
-      // Unless we know the tile is entirely in bounds, we have to emit a
-      // y-in-bounds check before reading from the input.
-      if (!tile_in_bounds) {
+      // Unless we know that y is in bounds, we have to emit a check before
+      // reading from the input.
+      if (!tile_in_y_bounds) {
         llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
             ir_builder_.CreateICmpULT(y, index_typed_constant(height)),
             "y_in_bounds", &ir_builder_);
@@ -1088,8 +1113,20 @@ Status IrEmitterUnnested::EmitColumnReduction(
         // the partial reduction result.
         llvm_ir::SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
       }
-      llvm::Value* input_address = ir_builder_.CreateAlloca(element_ir_type);
-      {
+      for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
+        llvm::Value* x = ir_builder_.CreateNSWAdd(
+            ir_builder_.CreateNSWMul(x_in_tiles,
+                                     index_typed_constant(kTileWidth)),
+            index_typed_constant(x_offset));
+        // Unless we know that x is in bounds, we have to emit a check before
+        // reading from the input.
+        if (!tile_in_x_bounds) {
+          llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
+              ir_builder_.CreateICmpULT(x, index_typed_constant(width)),
+              "x_in_bounds", &ir_builder_);
+          llvm_ir::SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
+        }
+        llvm::Value* input_address = ir_builder_.CreateAlloca(element_ir_type);
         // {y,x} is an index to input_matrix_shape [height,width]. We need to
         // convert that to an index to input_shape (the shape of the operand of
         // "reduce"). This conversion is composed of a transposition from
@@ -1120,51 +1157,90 @@ Status IrEmitterUnnested::EmitColumnReduction(
           ir_builder_.CreateStore(input_ir_value, input_address);
           TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
               *reducers[i],
-              {partial_reduction_result_addresses[i], input_address},
-              partial_reduction_result_addresses[i]));
+              {partial_reduction_result_addresses[i * kTileWidth + x_offset],
+               input_address},
+              partial_reduction_result_addresses[i * kTileWidth + x_offset]));
+          TF_RETURN_IF_ERROR(EmitExtraOutputsForReduce(reduce, input_index,
+                                                       extra_output_gens));
         }
-        return EmitExtraOutputsForReduce(reduce, input_index,
-                                         extra_output_gens);
       }
+      return Status::OK();
     };
 
-    // y_end = kTileSize + y_in_tiles * kTileSize, i.e., the y location that's
-    // immediately beyond the tile.
+    // y_end = kTileHeight + y_in_tiles * kTileHeight, i.e., the y location
+    // that's immediately beyond the tile.
     llvm::Value* y_end = ir_builder_.CreateNSWAdd(
-        index_typed_constant(kTileSize),
-        ir_builder_.CreateNSWMul(y_in_tiles, index_typed_constant(kTileSize)));
-    llvm::Value* tile_in_bounds = ir_builder_.CreateOr(
+        index_typed_constant(kTileHeight),
+        ir_builder_.CreateNSWMul(y_in_tiles,
+                                 index_typed_constant(kTileHeight)));
+    // x_end = kTileWidth + x_in_tiles * kTileWidth, i.e., the x location
+    // that's immediately beyond the tile.
+    llvm::Value* x_end = ir_builder_.CreateNSWAdd(
+        index_typed_constant(kTileWidth),
+        ir_builder_.CreateNSWMul(x_in_tiles, index_typed_constant(kTileWidth)));
+    llvm::Value* tile_in_y_bounds = ir_builder_.CreateOr(
         ir_builder_.CreateICmpULE(y_end, index_typed_constant(height)),
-        ir_builder_.getInt1(height % kTileSize == 0));
-    // The tile is entirely in bound if "height" is a multiple of kTileSize or
+        ir_builder_.getInt1(height % kTileHeight == 0));
+    llvm::Value* tile_in_x_bounds = ir_builder_.CreateOr(
+        ir_builder_.CreateICmpULE(x_end, index_typed_constant(width)),
+        ir_builder_.getInt1(width % kTileWidth == 0));
+    // The tile is in y bounds if "height" is a multiple of kTileHeight or
     // y_end <= height.
-    llvm_ir::LlvmIfData if_tile_in_bounds_data =
-        llvm_ir::EmitIfThenElse(tile_in_bounds, "tile_in_bounds", &ir_builder_);
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.true_block,
+    llvm_ir::LlvmIfData if_tile_in_y_bounds_data = llvm_ir::EmitIfThenElse(
+        tile_in_y_bounds, "tile_in_y_bounds", &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.true_block,
                                    &ir_builder_);
-    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_bounds=*/true));
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.false_block,
+    // The tile is in x bounds if "width" is a multiple of kTileWidth or
+    // x_end <= width.
+    llvm_ir::LlvmIfData if_tile_in_x_bounds_data = llvm_ir::EmitIfThenElse(
+        tile_in_x_bounds, "tile_in_x_bounds", &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block,
                                    &ir_builder_);
-    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_bounds=*/false));
+    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/true,
+                                              /*tile_in_x_bounds=*/true));
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block,
+                                   &ir_builder_);
+    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/true,
+                                              /*tile_in_x_bounds=*/false));
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.false_block,
+                                   &ir_builder_);
+    if_tile_in_x_bounds_data = llvm_ir::EmitIfThenElse(
+        tile_in_x_bounds, "tile_in_x_bounds", &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block,
+                                   &ir_builder_);
+    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/false,
+                                              /*tile_in_x_bounds=*/true));
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block,
+                                   &ir_builder_);
+    TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/false,
+                                              /*tile_in_x_bounds=*/false));
 
-    // After the if-then-else statement on tile_in_bounds, emit atomic
-    // operations to accumulate the partial reduction result to the output
-    // element.
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.after_block,
+    // After the nested if-then-else statement on tile_in_y_bounds and
+    // tile_in_x_bounds, emit atomic operations to accumulate the partial
+    // reduction result to the output element.
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.after_block,
                                    &ir_builder_);
     const HloInstruction* output =
         reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce;
     for (int i = 0; i != num_reduces; ++i) {
-      llvm::Value* output_address =
-          GetIrArray(*output, *output, reduce_output_shapes[i])
-              .EmitArrayElementAddress(
-                  IrArray::Index(x,
-                                 ShapeUtil::GetSubshape(
-                                     output->shape(), reduce_output_shapes[i]),
-                                 &ir_builder_),
-                  &ir_builder_, "output_element_address");
-      TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation(
-          *reducers[i], output_address, partial_reduction_result_addresses[i]));
+      for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
+        llvm::Value* x = ir_builder_.CreateNSWAdd(
+            ir_builder_.CreateNSWMul(x_in_tiles,
+                                     index_typed_constant(kTileWidth)),
+            index_typed_constant(x_offset));
+        llvm::Value* output_address =
+            GetIrArray(*output, *output, reduce_output_shapes[i])
+                .EmitArrayElementAddress(
+                    IrArray::Index(
+                        x,
+                        ShapeUtil::GetSubshape(output->shape(),
+                                               reduce_output_shapes[i]),
+                        &ir_builder_),
+                    &ir_builder_, "output_element_address");
+        TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation(
+            *reducers[i], output_address,
+            partial_reduction_result_addresses[i * kTileWidth + x_offset]));
+      }
     }
     return Status::OK();
   };
-- 
cgit v1.2.3


From 5d6aec5318fba138bdf47ff0fbee035dc4d6e04a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 00:18:21 -0700
Subject: Track colocation context manager locations (file:line) and add
 colocation information support to error interpolation.

This CL add a new private property on ops: Operation._colocation_dict.  This property will return a dictionary for which the keys are nodes with which this Operation is colocated, and for which the values are traceable_stack.TraceableObject instances.  The TraceableObject instances record the location of the relevant colocation context manager but have the "obj" field set to None to prevent leaking private data.

For example, suppose file_a contained these lines:

  file_a.py:
    14: node_a = tf.constant(3, name='NODE_A')
    15: with tf.colocate_with(node_a):
    16:   node_b = tf.constant(4, name='NODE_B')

Then a TraceableObject t_obj representing the colocation context manager would have these member values:

  t_obj.obj -> None
  t_obj.name = 'NODE_A'
  t_obj.filename = 'file_a.py'
  t_obj.lineno = 15

and node_b.op._colocation_dict would return the dictionary

  { 'NODE_A': t_obj }

PiperOrigin-RevId: 205035378
---
 tensorflow/python/BUILD                            |   1 +
 tensorflow/python/framework/error_interpolation.py | 118 +++++++++++++++++----
 .../python/framework/error_interpolation_test.py   |  94 ++++++++++++++++
 tensorflow/python/framework/ops.py                 |  57 +++++++++-
 tensorflow/python/framework/ops_test.py            |   8 ++
 tensorflow/python/framework/traceable_stack.py     |  11 +-
 6 files changed, 258 insertions(+), 31 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index a362dee97d..c33a579ad2 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1044,6 +1044,7 @@ py_test(
         ":client_testlib",
         ":constant_op",
         ":error_interpolation",
+        ":traceable_stack",
     ],
 )
 
diff --git a/tensorflow/python/framework/error_interpolation.py b/tensorflow/python/framework/error_interpolation.py
index 72d5dc99a8..a79073b748 100644
--- a/tensorflow/python/framework/error_interpolation.py
+++ b/tensorflow/python/framework/error_interpolation.py
@@ -60,6 +60,8 @@ def _parse_message(message):
   Supported tags after node:
     file: Replaced with the filename in which the node was defined.
     line: Replaced by the line number at which the node was defined.
+    colocations: Replaced by a multi-line message describing the file and
+        line numbers at which this node was colocated with other nodes.
 
   Args:
     message: String to parse
@@ -85,13 +87,53 @@ def _parse_message(message):
   return seps, tags
 
 
-def _get_field_dict_from_traceback(tf_traceback, frame_index):
-  """Convert traceback elements into interpolation dictionary and return."""
-  frame = tf_traceback[frame_index]
-  return {
-      "file": frame[tf_stack.TB_FILENAME],
-      "line": frame[tf_stack.TB_LINENO],
-  }
+def _compute_colocation_summary_from_dict(colocation_dict, prefix=""):
+  """Return a summary of an op's colocation stack.
+
+  Args:
+    colocation_dict: The op._colocation_dict.
+    prefix:  An optional string prefix used before each line of the multi-
+        line string returned by this function.
+
+  Returns:
+    A multi-line string similar to:
+        Node-device colocations active during op creation:
+          with tf.colocate_with(test_node_1): 
+          with tf.colocate_with(test_node_2): 
+    The first line will have no padding to its left by default.  Subsequent
+    lines will have two spaces of left-padding.  Use the prefix argument
+    to increase indentation.
+  """
+  if not colocation_dict:
+    message = "No node-device colocations were active during op creation."
+    return prefix + message
+
+  str_list = []
+  str_list.append("%sNode-device colocations active during op creation:"
+                  % prefix)
+
+  for name, location in colocation_dict.items():
+    location_summary = "<{file}:{line}>".format(file=location.filename,
+                                                line=location.lineno)
+    subs = {
+        "prefix": prefix,
+        "indent": "  ",
+        "name": name,
+        "loc": location_summary,
+    }
+    str_list.append(
+        "{prefix}{indent}with tf.colocate_with({name}): {loc}".format(**subs))
+
+  return "\n".join(str_list)
+
+
+def _compute_colocation_summary_from_op(op, prefix=""):
+  """Fetch colocation file, line, and nesting and return a summary string."""
+  if not op:
+    return ""
+  # pylint: disable=protected-access
+  return _compute_colocation_summary_from_dict(op._colocation_dict, prefix)
+  # pylint: enable=protected-access
 
 
 def _find_index_of_defining_frame_for_op(op):
@@ -125,6 +167,54 @@ def _find_index_of_defining_frame_for_op(op):
   return 0
 
 
+def _get_defining_frame_from_op(op):
+  """Find and return stack frame where op was defined."""
+  frame = None
+  if op:
+    # pylint: disable=protected-access
+    frame_index = _find_index_of_defining_frame_for_op(op)
+    frame = op._traceback[frame_index]
+    # pylint: enable=protected-access
+  return frame
+
+
+def _compute_field_dict(op):
+  """Return a dictionary mapping interpolation tokens to values.
+
+  Args:
+    op: op.Operation object having a _traceback member.
+
+  Returns:
+    A dictionary mapping string tokens to string values.  The keys are shown
+    below along with example values.
+    {
+      "file": "tool_utils.py",
+      "line": "124",
+      "colocations":
+          '''Node-device colocations active during op creation:
+               with tf.colocate_with(test_node_1): 
+               with tf.colocate_with(test_node_2): '''
+    }
+    If op is None or lacks a _traceback field, the returned values will be
+    "".
+  """
+  default_value = ""
+  field_dict = {
+      "file": default_value,
+      "line": default_value,
+      "colocations": default_value,
+  }
+  frame = _get_defining_frame_from_op(op)
+  if frame:
+    field_dict["file"] = frame[tf_stack.TB_FILENAME]
+    field_dict["line"] = frame[tf_stack.TB_LINENO]
+  colocation_summary = _compute_colocation_summary_from_op(op)
+  if colocation_summary:
+    field_dict["colocations"] = colocation_summary
+
+  return field_dict
+
+
 def interpolate(error_message, graph):
   """Interpolates an error message.
 
@@ -148,19 +238,7 @@ def interpolate(error_message, graph):
     except KeyError:
       op = None
 
-    if op:
-      frame_index = _find_index_of_defining_frame_for_op(op)
-      # pylint: disable=protected-access
-      field_dict = _get_field_dict_from_traceback(op._traceback, frame_index)
-      # pylint: enable=protected-access
-    else:
-      field_dict = {
-          "file": "",
-          "line": "",
-          "func": "",
-          "code": None,
-      }
-    node_name_to_substitution_dict[name] = field_dict
+    node_name_to_substitution_dict[name] = _compute_field_dict(op)
 
   subs = [
       string.Template(tag.format).safe_substitute(
diff --git a/tensorflow/python/framework/error_interpolation_test.py b/tensorflow/python/framework/error_interpolation_test.py
index b6615317d1..1e5cb73854 100644
--- a/tensorflow/python/framework/error_interpolation_test.py
+++ b/tensorflow/python/framework/error_interpolation_test.py
@@ -22,6 +22,8 @@ import os
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import error_interpolation
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import traceable_stack
 from tensorflow.python.platform import test
 from tensorflow.python.util import tf_stack
 
@@ -55,6 +57,47 @@ def _modify_op_stack_with_filenames(op, num_user_frames, user_filename,
   op._traceback = stack
 
 
+def assert_node_in_colocation_summary(test_obj, colocation_summary_string,
+                                      name, filename="", lineno=""):
+  lineno = str(lineno)
+  name_phrase = "colocate_with(%s)" % name
+  for term in [name_phrase, filename, lineno]:
+    test_obj.assertIn(term, colocation_summary_string)
+  test_obj.assertNotIn("loc:@", colocation_summary_string)
+
+
+class ComputeColocationSummaryFromOpTest(test.TestCase):
+
+  def testCorrectFormatWithActiveColocations(self):
+    t_obj_1 = traceable_stack.TraceableObject(None,
+                                              filename="test_1.py",
+                                              lineno=27)
+    t_obj_2 = traceable_stack.TraceableObject(None,
+                                              filename="test_2.py",
+                                              lineno=38)
+    colocation_dict = {
+        "test_node_1": t_obj_1,
+        "test_node_2": t_obj_2,
+    }
+    summary = error_interpolation._compute_colocation_summary_from_dict(
+        colocation_dict, prefix="  ")
+    assert_node_in_colocation_summary(self,
+                                      summary,
+                                      name="test_node_1",
+                                      filename="test_1.py",
+                                      lineno=27)
+    assert_node_in_colocation_summary(self, summary,
+                                      name="test_node_2",
+                                      filename="test_2.py",
+                                      lineno=38)
+
+  def testCorrectFormatWhenNoColocationsWereActive(self):
+    colocation_dict = {}
+    summary = error_interpolation._compute_colocation_summary_from_dict(
+        colocation_dict, prefix="  ")
+    self.assertIn("No node-device colocations", summary)
+
+
 class InterpolateTest(test.TestCase):
 
   def setUp(self):
@@ -134,5 +177,56 @@ class InterpolateTest(test.TestCase):
     self.assertRegexpMatches(interpolated_string, expected_regex)
 
 
+class InterpolateColocationSummaryTest(test.TestCase):
+
+  def setUp(self):
+    # Add nodes to the graph for retrieval by name later.
+    node_one = constant_op.constant(1, name="One")
+    node_two = constant_op.constant(2, name="Two")
+
+    # node_three has one colocation group, obviously.
+    with ops.colocate_with(node_one):
+      node_three = constant_op.constant(3, name="Three_with_one")
+
+    # node_four has one colocation group even though three is (transitively)
+    # colocated with one.
+    with ops.colocate_with(node_three):
+      constant_op.constant(4, name="Four_with_three")
+
+    # node_five has two colocation groups because one and two are not colocated.
+    with ops.colocate_with(node_two):
+      with ops.colocate_with(node_one):
+        constant_op.constant(5, name="Five_with_one_with_two")
+
+    self.graph = node_three.graph
+
+  def testNodeThreeHasColocationInterpolation(self):
+    message = "^^node:Three_with_one:${colocations}^^"
+    result = error_interpolation.interpolate(message, self.graph)
+    assert_node_in_colocation_summary(self, result, name="One")
+
+  def testNodeFourHasColocationInterpolationForNodeThreeOnly(self):
+    message = "^^node:Four_with_three:${colocations}^^"
+    result = error_interpolation.interpolate(message, self.graph)
+    assert_node_in_colocation_summary(self, result, name="Three_with_one")
+    self.assertNotIn(
+        "One", result,
+        "Node One should not appear in Four_with_three's summary:\n%s"
+        % result)
+
+  def testNodeFiveHasColocationInterpolationForNodeOneAndTwo(self):
+    message = "^^node:Five_with_one_with_two:${colocations}^^"
+    result = error_interpolation.interpolate(message, self.graph)
+    assert_node_in_colocation_summary(self, result, name="One")
+    assert_node_in_colocation_summary(self, result, name="Two")
+
+  def testColocationInterpolationForNodeLackingColocation(self):
+    message = "^^node:One:${colocations}^^"
+    result = error_interpolation.interpolate(message, self.graph)
+    self.assertIn("No node-device colocations", result)
+    self.assertNotIn("One", result)
+    self.assertNotIn("Two", result)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index ea7a9986fe..b813cd6c06 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -47,10 +47,10 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import op_def_registry
 from tensorflow.python.framework import registry
-from tensorflow.python.util import tf_stack
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import traceable_stack
 from tensorflow.python.framework import versions
+from tensorflow.python.util import tf_stack
 from tensorflow.python.ops import control_flow_util
 from tensorflow.python.platform import app
 from tensorflow.python.platform import tf_logging as logging
@@ -1712,10 +1712,14 @@ class Operation(object):
     # This will be set by self.inputs.
     self._inputs_val = None
 
-    self._id_value = self._graph._next_id()  # pylint: disable=protected-access
+    # pylint: disable=protected-access
+    self._id_value = self._graph._next_id()
     self._original_op = original_op
     self._traceback = tf_stack.extract_stack()
-    self._control_flow_context = self.graph._get_control_flow_context()  # pylint: disable=protected-access
+    # List of traceable_stack.TraceableObjects for colocation context managers.
+    self._colocation_code_locations = None
+    self._control_flow_context = self.graph._get_control_flow_context()
+    # pylint: enable=protected-access
 
     # Initialize self._c_op.
     if c_op:
@@ -1853,6 +1857,42 @@ class Operation(object):
     """
     return c_api.TF_OperationDevice(self._c_op)
 
+  @property
+  def _colocation_dict(self):
+    """Code locations for colocation context managers active at op creation.
+
+    This property will return a dictionary for which the keys are nodes with
+    which this Operation is colocated, and for which the values are
+    traceable_stack.TraceableObject instances.  The TraceableObject instances
+    record the location of the relevant colocation context manager but have the
+    "obj" field set to None to prevent leaking private data.
+
+    For example, suppose file_a contained these lines:
+
+      file_a.py:
+        14: node_a = tf.constant(3, name='NODE_A')
+        15: with tf.colocate_with(node_a):
+        16:   node_b = tf.constant(4, name='NODE_B')
+
+    Then a TraceableObject t_obj representing the colocation context manager
+    would have these member values:
+
+      t_obj.obj -> None
+      t_obj.name = 'NODE_A'
+      t_obj.filename = 'file_a.py'
+      t_obj.lineno = 15
+
+    and node_b.op._colocation_code_locations would return the dictionary
+
+      { 'NODE_A': t_obj }
+
+    Returns:
+      {str: traceable_stack.TraceableObject} as per this method's description,
+      above.
+    """
+    locations_dict = self._colocation_code_locations or {}
+    return locations_dict.copy()
+
   @property
   def _output_types(self):
     """List this operation's output types.
@@ -3249,6 +3289,7 @@ class Graph(object):
       # pylint: disable=protected-access
       op._set_attr("_class", attr_value_pb2.AttrValue(
           list=attr_value_pb2.AttrValue.ListValue(s=all_colocation_groups)))
+      op._colocation_code_locations = self._snapshot_colocation_stack_metadata()
       # pylint: enable=protected-access
 
     # Sets "container" attribute if
@@ -4010,7 +4051,10 @@ class Graph(object):
       self._colocation_stack = traceable_stack.TraceableStack()
 
     if op is not None:
-      self._colocation_stack.push_obj(op, name=op.name, offset=1)
+      # offset refers to the stack frame used for storing code location.
+      # We use 4, the sum of 1 to use our caller's stack frame and 3
+      # to jump over layers of context managers above us.
+      self._colocation_stack.push_obj(op, offset=4)
 
     try:
       yield
@@ -4658,6 +4702,11 @@ class Graph(object):
     else:
       return self._graph_colocation_stack
 
+  def _snapshot_colocation_stack_metadata(self):
+    """Return colocation stack metadata as a dictionary."""
+    traceable_objects = self._colocation_stack.peek_traceable_objs()
+    return {obj.obj.name: obj.copy_metadata() for obj in traceable_objects}
+
   @_colocation_stack.setter
   def _colocation_stack(self, colocation_stack):
     if self._stack_state_is_thread_local:
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 150100d771..f848b69782 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -2554,6 +2554,14 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
     with self.assertRaises(ValueError):
       c.op.get_attr("_class")
 
+    # Roughly test that stack information is being saved correctly for the op.
+    locations_dict = b.op._colocation_dict
+    self.assertIn("a", locations_dict)
+    metadata = locations_dict["a"]
+    self.assertIsNone(metadata.obj)
+    basename = metadata.filename.split("/")[-1]
+    self.assertEqual("ops_test.py", basename)
+
   def testColocationDeviceInteraction(self):
     with ops.device("/cpu:0"):
       with ops.device("/device:GPU:0"):
diff --git a/tensorflow/python/framework/traceable_stack.py b/tensorflow/python/framework/traceable_stack.py
index 1b7c6bd7c5..7f4d28237f 100644
--- a/tensorflow/python/framework/traceable_stack.py
+++ b/tensorflow/python/framework/traceable_stack.py
@@ -27,9 +27,8 @@ class TraceableObject(object):
   # Return codes for the set_filename_and_line_from_caller() method.
   SUCCESS, HEURISTIC_USED, FAILURE = (0, 1, 2)
 
-  def __init__(self, obj, name=None, filename=None, lineno=None):
+  def __init__(self, obj, filename=None, lineno=None):
     self.obj = obj
-    self.name = name
     self.filename = filename
     self.lineno = lineno
 
@@ -72,8 +71,7 @@ class TraceableObject(object):
 
   def copy_metadata(self):
     """Return a TraceableObject like this one, but without the object."""
-    return self.__class__(None, name=self.name, filename=self.filename,
-                          lineno=self.lineno)
+    return self.__class__(None, filename=self.filename, lineno=self.lineno)
 
 
 class TraceableStack(object):
@@ -88,12 +86,11 @@ class TraceableStack(object):
     """
     self._stack = existing_stack[:] if existing_stack else []
 
-  def push_obj(self, obj, name=None, offset=0):
+  def push_obj(self, obj, offset=0):
     """Add object to the stack and record its filename and line information.
 
     Args:
       obj: An object to store on the stack.
-      name: A name for the object, used for dict keys in get_item_metadata_dict.
       offset: Integer.  If 0, the caller's stack frame is used.  If 1,
           the caller's caller's stack frame is used.
 
@@ -102,7 +99,7 @@ class TraceableStack(object):
       TraceableObject.HEURISTIC_USED if the stack was smaller than expected,
       and TraceableObject.FAILURE if the stack was empty.
     """
-    traceable_obj = TraceableObject(obj, name=name)
+    traceable_obj = TraceableObject(obj)
     self._stack.append(traceable_obj)
     # Offset is defined in "Args" as relative to the caller.  We are 1 frame
     # beyond the caller and need to compensate.
-- 
cgit v1.2.3


From f843a6210073ddda131a9fbef520539e1329fcd7 Mon Sep 17 00:00:00 2001
From: Jongmin Park 
Date: Wed, 18 Jul 2018 16:44:20 +0900
Subject: Edit  a link to notebooks directory

Bottom of this document, a link to **notebooks directory** who has tutorial misslinked.
Need to change from `..` to `../notebooks`.
---
 tensorflow/tools/docker/notebooks/1_hello_tensorflow.ipynb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/docker/notebooks/1_hello_tensorflow.ipynb b/tensorflow/tools/docker/notebooks/1_hello_tensorflow.ipynb
index 0633b03259..8fa871ef77 100644
--- a/tensorflow/tools/docker/notebooks/1_hello_tensorflow.ipynb
+++ b/tensorflow/tools/docker/notebooks/1_hello_tensorflow.ipynb
@@ -665,7 +665,7 @@
       "source": [
         "## What's next?\n",
         "\n",
-        "This has been a gentle introduction to TensorFlow, focused on what TensorFlow is and the very basics of doing anything in TensorFlow. If you'd like more, the next tutorial in the series is Getting Started with TensorFlow, also available in the [notebooks directory](..)."
+        "This has been a gentle introduction to TensorFlow, focused on what TensorFlow is and the very basics of doing anything in TensorFlow. If you'd like more, the next tutorial in the series is Getting Started with TensorFlow, also available in the [notebooks directory](../notebooks)."
       ]
     }
   ],
-- 
cgit v1.2.3


From b74f7b71fad773dd90c8f48b66bc82fb07eb9bc0 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel 
Date: Wed, 18 Jul 2018 03:10:36 -0700
Subject: Implement BitonicSort for GPU.

This is a first version, several things are still missing:
- Support for key/value sorting.
- Support for other types than F32, S32 and U32.
- Parallelization of the inner loop.

PiperOrigin-RevId: 205052657
---
 tensorflow/compiler/xla/service/gpu/ir_emitter.cc  | 172 ++++++++++++++++++++-
 tensorflow/compiler/xla/service/gpu/ir_emitter.h   |   7 +
 .../xla/service/gpu/ir_emitter_unnested.cc         |  29 ++++
 .../compiler/xla/service/gpu/ir_emitter_unnested.h |   1 +
 4 files changed, 206 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
index fe83d017f4..a08b72e3af 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
@@ -44,6 +44,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/compiler/xla/window_util.h"
+#include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/core/errors.h"
 
 namespace xla {
@@ -123,9 +124,136 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) {
   return Status::OK();
 }
 
-Status IrEmitter::HandleSort(HloInstruction*) {
-  // TODO(b/26783907): Implement sort on GPU.
-  return Unimplemented("sort");
+Status IrEmitter::HandleSort(HloInstruction* sort) {
+  auto keys = sort->operand(0);
+  auto values = sort->operand_count() > 1 ? sort->operand(1) : nullptr;
+  if (values != nullptr) {
+    // TODO(b/26783907): Also sort the values by their corresponding key.
+    return Unimplemented("Key/Value Sort is not implemented on GPU");
+  }
+  int dimension_to_sort = sort->dimensions(0);
+  const llvm_ir::IrArray& keys_array = GetIrArray(*keys, *sort);
+  const llvm_ir::IrArray& target_array = GetIrArray(*sort, *sort);
+
+  const Shape& keys_shape = keys->shape();
+
+  // TODO(b/26783907): This case can probably be avoided with the Algebraic
+  // Simplifier.
+  if (ShapeUtil::IsScalar(keys_shape)) {
+    return Status::OK();
+  }
+
+  // Create loop nests which loop through the operand dimensions. The sort
+  // dimension is handled in three separate innermost loops which perform the
+  // sorting.
+  llvm_ir::ForLoopNest loop_nest(IrName(sort), &ir_builder_);
+  llvm_ir::IrArray::Index keys_index = EmitOperandArrayLoopNest(
+      keys_array, dimension_to_sort, "keys", &loop_nest);
+
+  // 'compare_keys_index' is the index of the element that 'keys_index' should
+  // be compared to.
+  llvm_ir::IrArray::Index compare_keys_index(keys_index.GetType());
+  for (size_t dimension = 0; dimension < keys_index.size(); ++dimension) {
+    if (dimension != dimension_to_sort) {
+      compare_keys_index.push_back(keys_index[dimension]);
+    } else {
+      compare_keys_index.push_back(nullptr);
+    }
+  }
+
+  // Create the sorting loops which do the sorting.
+  int64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort);
+  std::unique_ptr stages_loop = loop_nest.AddLoop(
+      /*start_index=*/0,
+      /*end_index=*/
+      tensorflow::Log2Ceiling64(dimension_to_sort_bound),
+      /*suffix=*/"sort_stages");
+  std::unique_ptr mask_loop = loop_nest.AddLoop(
+      /*suffix=*/"mask",
+      /*start_index=*/keys_index.GetConstantWithIndexType(0),
+      /*end_index=*/stages_loop->GetIndVarValue());
+  std::unique_ptr compare_loop = loop_nest.AddLoop(
+      /*start_index=*/0,
+      /*end_index=*/dimension_to_sort_bound,
+      /*suffix=*/"compare");
+
+  // Naive C++ code for the inner loops (without parallelization):
+  //
+  // for (int64 stage = 0; stage < Log2Ceiling(dimension_to_sort_bound);
+  //     ++stage) {
+  //   int64 first_xor_mask = (1LL << (stage + 1)) - 1;
+  //   for (int64 i = 0; i < dimension_to_sort_bound; ++i) {
+  //     int64 j = i ^ first_xor_mask;
+  //     if (i < j && j < dimension_to_sort_bound) {
+  //       int64 min_key = std::min(keys[i], keys[j]);
+  //       keys[j] = std::max(keys[i], keys[j]);
+  //       keys[i] = min_key;
+  //     }
+  //   }
+  //   for (int64 mask = 0; mask < stage; ++mask) {
+  //     int64 later_xor_mask = (1LL << (stage - (mask + 1));
+  //     for (int64 i = 0; i < dimension_to_sort_bound; ++i) {
+  //       int64 j = i ^ later_xor_mask;
+  //       if (i < j && j < dimension_to_sort_bound) {
+  //         int64 min_key = std::min(keys[i], keys[j]);
+  //         keys[j] = std::max(keys[i], keys[j]);
+  //         keys[i] = min_key;
+  //       }
+  //     }
+  //   }
+  // }
+  //
+  // This follows the algorithm described on Wikipedia:
+  // https://en.wikipedia.org/wiki/Bitonic_sorter
+
+  SetToFirstInsertPoint(stages_loop->GetBodyBasicBlock(), &ir_builder_);
+  // The first xor mask of a stage is 2^(stage + 1) - 1.
+  auto first_xor_mask = ir_builder_.CreateSub(
+      ir_builder_.CreateShl(
+          keys_index.GetConstantWithIndexType(1),
+          ir_builder_.CreateAdd(stages_loop->GetIndVarValue(),
+                                keys_index.GetConstantWithIndexType(1))),
+      keys_index.GetConstantWithIndexType(1));
+  std::unique_ptr first_compare_loop =
+      llvm_ir::ForLoop::EmitForLoop(
+          /*prefix=*/"first_compare",
+          /*start_index=*/keys_index.GetConstantWithIndexType(0),
+          /*end_index=*/
+          keys_index.GetConstantWithIndexType(
+              keys_shape.dimensions(dimension_to_sort)),
+          /*step=*/keys_index.GetConstantWithIndexType(1),
+          /*ir_builder=*/&ir_builder_);
+
+  SetToFirstInsertPoint(first_compare_loop->GetBodyBasicBlock(), &ir_builder_);
+  // 'first_compare_loop' iterates through the 'dimension_to_sort'.
+  keys_index[dimension_to_sort] = first_compare_loop->GetIndVarValue();
+  compare_keys_index[dimension_to_sort] = ir_builder_.CreateXor(
+      first_compare_loop->GetIndVarValue(), first_xor_mask);
+  EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index,
+                  target_array);
+
+  SetToFirstInsertPoint(compare_loop->GetPreheaderBasicBlock(), &ir_builder_);
+  // The later masks of a stage are 2^(stage - (mask_loop_ind_var + 1)).
+  auto later_xor_mask = ir_builder_.CreateShl(
+      keys_index.GetConstantWithIndexType(1),
+      ir_builder_.CreateSub(
+          stages_loop->GetIndVarValue(),
+          ir_builder_.CreateAdd(mask_loop->GetIndVarValue(),
+                                keys_index.GetConstantWithIndexType(1))));
+
+  SetToFirstInsertPoint(compare_loop->GetBodyBasicBlock(), &ir_builder_);
+  // 'compare_loop' iterates through the 'dimension_to_sort'.
+  keys_index[dimension_to_sort] = compare_loop->GetIndVarValue();
+  compare_keys_index[dimension_to_sort] =
+      ir_builder_.CreateXor(compare_loop->GetIndVarValue(), later_xor_mask);
+  EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index,
+                  target_array);
+
+  // Set the IR builder insert point to the exit basic block of the outer most
+  // loop. This ensures later instructions are inserted after this loop nest.
+  ir_builder_.SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
+
+  return Status::OK();
 }
 
 Status IrEmitter::HandleSend(HloInstruction*) {
@@ -399,6 +527,44 @@ Status IrEmitter::EmitAtomicOperationUsingCAS(const HloComputation& computation,
   return Status::OK();
 }
 
+void IrEmitter::EmitCompareLoop(
+    int64 dimension_to_sort, const llvm_ir::IrArray::Index& keys_index,
+    const llvm_ir::IrArray::Index& compare_keys_index,
+    const llvm_ir::IrArray& keys_array) {
+  // TODO(b/26783907): parallelize this loop.
+
+  // if (is_smaller_index &&
+  //     compare_keys[dimension_to_sort] < dimension_to_sort_bound)
+  llvm::Value* is_smaller_index = ir_builder_.CreateICmpSLT(
+      keys_index[dimension_to_sort], compare_keys_index[dimension_to_sort]);
+  int64 dimension_to_sort_bound =
+      keys_array.GetShape().dimensions(dimension_to_sort);
+  auto if_data = llvm_ir::EmitIfThenElse(
+      ir_builder_.CreateAnd(
+          is_smaller_index,
+          ir_builder_.CreateICmpSLT(
+              compare_keys_index[dimension_to_sort],
+              keys_index.GetConstantWithIndexType(dimension_to_sort_bound))),
+      "smaller_comparison_index", &ir_builder_, /*emit_else=*/false);
+  SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
+  auto key1 = keys_array.EmitReadArrayElement(keys_index, &ir_builder_);
+  auto key2 = keys_array.EmitReadArrayElement(compare_keys_index, &ir_builder_);
+  auto key_type = keys_array.GetShape().element_type();
+  auto comparison =
+      primitive_util::IsFloatingPointType(key_type)
+          // TODO(b/26783907): Figure out how to handle NaNs.
+          ? ir_builder_.CreateFCmp(llvm::FCmpInst::FCMP_ULT, key1, key2)
+          : ir_builder_.CreateICmp(
+                primitive_util::IsSignedIntegralType(key_type)
+                    ? llvm::ICmpInst::ICMP_SLT
+                    : llvm::ICmpInst::ICMP_ULT,
+                key1, key2);
+  auto min_key = ir_builder_.CreateSelect(comparison, key1, key2);
+  auto max_key = ir_builder_.CreateSelect(comparison, key2, key1);
+  keys_array.EmitWriteArrayElement(keys_index, min_key, &ir_builder_);
+  keys_array.EmitWriteArrayElement(compare_keys_index, max_key, &ir_builder_);
+}
+
 Status IrEmitter::EmitAtomicOperationForNestedComputation(
     const HloComputation& computation, llvm::Value* output_address,
     llvm::Value* source_address) {
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
index d2dd335f10..e9ad4a752b 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
@@ -198,6 +198,13 @@ class IrEmitter : public DfsHloVisitorWithDefault {
                                      llvm::Value* output_address,
                                      llvm::Value* source_address);
 
+  // A helper method for HandleSort(). It adds the inner comparison loop where
+  // we compare elements pointed to by 'keys_index' and 'compare_keys_index'.
+  void EmitCompareLoop(int64 dimension_to_sort,
+                       const llvm_ir::IrArray::Index& keys_index,
+                       const llvm_ir::IrArray::Index& compare_keys_index,
+                       const llvm_ir::IrArray& keys_array);
+
   StatusOr ComputeNestedElement(
       const HloComputation& computation,
       tensorflow::gtl::ArraySlice parameter_elements);
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index f2597da4b9..70a227ca4a 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -2046,6 +2046,35 @@ Status IrEmitterUnnested::HandleSelect(HloInstruction* select) {
   return IrEmitter::HandleSelect(select);
 }
 
+Status IrEmitterUnnested::HandleSort(HloInstruction* sort) {
+  std::vector> thunks;
+  auto values = sort->operand_count() > 1 ? sort->operand(1) : nullptr;
+  if (values != nullptr) {
+    // TODO(b/26783907): Also sort the values by their corresponding key.
+    return Unimplemented("Key/Value Sort is not implemented on GPU");
+  }
+
+  // First copy the operand to the output, so that we can sort in-place.
+  // TODO(b/26783907): Share buffer of output and operand when it is possible.
+  if (sort->operand(0)->IsConstant()) {
+    thunks.push_back(MakeUnique(
+        /*source_address=*/sort->operand(0)->literal().untyped_data(),
+        /*destination_buffer=*/GetAllocationSlice(*sort),
+        /*mem_size=*/ShapeUtil::ByteSizeOf(sort->shape()), sort));
+  } else {
+    thunks.push_back(MakeUnique(
+        /*source_address=*/GetAllocationSlice(*sort->operand(0)),
+        /*destination_buffer=*/GetAllocationSlice(*sort),
+        /*mem_size=*/ShapeUtil::ByteSizeOf(sort->shape()), sort));
+  }
+
+  thunks.push_back(
+      BuildKernelThunk(sort, /*implements_whole_instruction=*/false));
+  thunk_sequence_->emplace_back(
+      MakeUnique(std::move(thunks), sort));
+  return IrEmitter::HandleSort(sort);
+}
+
 Status IrEmitterUnnested::HandleTupleSelect(HloInstruction* tuple_select) {
   thunk_sequence_->push_back(
       BuildKernelThunk(tuple_select, /*implements_whole_instruction=*/true));
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index 59547c16d7..616d8a2206 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -77,6 +77,7 @@ class IrEmitterUnnested : public IrEmitter {
   Status HandleOutfeed(HloInstruction* outfeed) override;
   Status HandleRng(HloInstruction* random) override;
   Status HandleSelect(HloInstruction* select) override;
+  Status HandleSort(HloInstruction* sort) override;
   Status HandleTupleSelect(HloInstruction* tuple_select) override;
   Status HandleCrossReplicaSum(HloInstruction* crs) override;
   Status HandleAfterAll(HloInstruction* gen_token) override;
-- 
cgit v1.2.3


From ff791a7fde3605493bef70de8a9c9779541daf66 Mon Sep 17 00:00:00 2001
From: Thomas Joerg 
Date: Wed, 18 Jul 2018 03:42:57 -0700
Subject: [XLA:GPU] s/InstructionFusionTest/MultiOutputFusionTest in
 multi_output_fusion_test.cc.

PiperOrigin-RevId: 205055522
---
 .../xla/service/gpu/multi_output_fusion_test.cc    | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
index 979ea79243..a6dc635b52 100644
--- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
@@ -27,7 +27,7 @@ namespace op = xla::testing::opcode_matchers;
 namespace xla {
 namespace gpu {
 
-using InstructionFusionTest = HloTestBase;
+using MultiOutputFusionTest = HloTestBase;
 
 const char kModulePrefix[] = R"(
     HloModule test_module
@@ -43,7 +43,7 @@ const char kModulePrefix[] = R"(
       ROOT mul.1 = f32[] add(scalar_lhs.1, scalar_rhs.1)
     })";
 
-TEST_F(InstructionFusionTest, MultiOutputFusionSiblingReduceAndReduceFusion) {
+TEST_F(MultiOutputFusionTest, MultiOutputFusionSiblingReduceAndReduceFusion) {
   // Fusion with reduce instruction root and a sibling reduce instruction
   // sharing the same input param.
   auto module = ParseHloString(tensorflow::strings::StrCat(kModulePrefix, R"(
@@ -72,7 +72,7 @@ TEST_F(InstructionFusionTest, MultiOutputFusionSiblingReduceAndReduceFusion) {
               op::Tuple(op::Reduce(), op::Reduce()));
 }
 
-TEST_F(InstructionFusionTest, MultiOutputFusionDifferentReduceInputShapes) {
+TEST_F(MultiOutputFusionTest, MultiOutputFusionDifferentReduceInputShapes) {
   auto module = ParseHloString(tensorflow::strings::StrCat(kModulePrefix, R"(
     fused_computation_1 {
       p1.1 = f32[6400]{0} parameter(1)
@@ -99,7 +99,7 @@ TEST_F(InstructionFusionTest, MultiOutputFusionDifferentReduceInputShapes) {
   ASSERT_FALSE(GpuMultiOutputFusion().Run(module.get()).ValueOrDie());
 }
 
-TEST_F(InstructionFusionTest, MultiOutputFusionDifferentReduceOutputShapes) {
+TEST_F(MultiOutputFusionTest, MultiOutputFusionDifferentReduceOutputShapes) {
   auto module = ParseHloString(tensorflow::strings::StrCat(kModulePrefix, R"(
     fused_computation_1 {
       p1.1 = f32[10,10]{1,0} parameter(1)
@@ -126,7 +126,7 @@ TEST_F(InstructionFusionTest, MultiOutputFusionDifferentReduceOutputShapes) {
   ASSERT_FALSE(GpuMultiOutputFusion().Run(module.get()).ValueOrDie());
 }
 
-TEST_F(InstructionFusionTest, MultiOutputFusionSiblingReduceFusions) {
+TEST_F(MultiOutputFusionTest, MultiOutputFusionSiblingReduceFusions) {
   // Two sibling fusions with reduce instruction roots sharing the same input
   // param.
   auto module = ParseHloString(tensorflow::strings::StrCat(kModulePrefix, R"(
@@ -160,7 +160,7 @@ TEST_F(InstructionFusionTest, MultiOutputFusionSiblingReduceFusions) {
               op::Tuple(op::Reduce(), op::Reduce()));
 }
 
-TEST_F(InstructionFusionTest,
+TEST_F(MultiOutputFusionTest,
        MultiOutputFusionSiblingReduceAndReduceMultiOutputFusion) {
   // Multi-output fusion with two reduce instructions root and a sibling reduce
   // instruction sharing the same input param.
@@ -193,7 +193,7 @@ TEST_F(InstructionFusionTest,
               op::Tuple(op::Reduce(), op::Reduce(), op::Reduce()));
 }
 
-TEST_F(InstructionFusionTest,
+TEST_F(MultiOutputFusionTest,
        MultiOutputFusionSiblingFusionCheckAgainstReduceOperand) {
   // Verify that if we already have a multi-output fusion that we prefer to pick
   // a reduce op from its operands for checking shape compatibility.
@@ -226,7 +226,7 @@ TEST_F(InstructionFusionTest,
   ASSERT_FALSE(GpuMultiOutputFusion().Run(module.get()).ValueOrDie());
 }
 
-TEST_F(InstructionFusionTest, MultiOutputFusionTwoLoops) {
+TEST_F(MultiOutputFusionTest, MultiOutputFusionTwoLoops) {
   auto module = ParseHloString(tensorflow::strings::StrCat(kModulePrefix, R"(
     fused_computation_1 {
       p0.1 = f32[6400]{0} parameter(0)
@@ -255,7 +255,7 @@ TEST_F(InstructionFusionTest, MultiOutputFusionTwoLoops) {
               op::Tuple(op::Multiply(), op::Divide()));
 }
 
-TEST_F(InstructionFusionTest, ProducerConsumerFusionLoopFusionAndReduce) {
+TEST_F(MultiOutputFusionTest, ProducerConsumerFusionLoopFusionAndReduce) {
   auto module = ParseHloString(tensorflow::strings::StrCat(kModulePrefix, R"(
     fused_add {
       p0.1 = f32[2,2,2]{2,1,0} parameter(0)
@@ -282,7 +282,7 @@ TEST_F(InstructionFusionTest, ProducerConsumerFusionLoopFusionAndReduce) {
               op::Tuple(op::Reduce(), op::Add()));
 }
 
-TEST_F(InstructionFusionTest, ProducerConsumerFusionLoopFusionAndReduceFusion) {
+TEST_F(MultiOutputFusionTest, ProducerConsumerFusionLoopFusionAndReduceFusion) {
   auto module = ParseHloString(tensorflow::strings::StrCat(kModulePrefix, R"(
     fused_select {
       p1.1 = f32[2,2,2]{2,1,0} parameter(1)
@@ -323,7 +323,7 @@ TEST_F(InstructionFusionTest, ProducerConsumerFusionLoopFusionAndReduceFusion) {
               op::Tuple(op::Reduce(), op::Reduce(), op::Select()));
 }
 
-TEST_F(InstructionFusionTest, ProducerConsumerFusionDoNotFuseLoopReduceFusion) {
+TEST_F(MultiOutputFusionTest, ProducerConsumerFusionDoNotFuseLoopReduceFusion) {
   auto module = ParseHloString(tensorflow::strings::StrCat(kModulePrefix, R"(
     fused_element_wise {
       p0.1 = f32[2,2,2]{2,1,0} parameter(0)
-- 
cgit v1.2.3


From be8184f8e002576aa2ef3274436dea68e9173c5f Mon Sep 17 00:00:00 2001
From: Kenneth Blomqvist 
Date: Wed, 18 Jul 2018 15:44:31 +0300
Subject: Fix extract image patches float type issue

---
 tensorflow/python/ops/array_grad.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py
index fe459a96b9..a2b5f77f91 100644
--- a/tensorflow/python/ops/array_grad.py
+++ b/tensorflow/python/ops/array_grad.py
@@ -790,7 +790,7 @@ def _ExtractImagePatchesGrad(op, grad):
 
   sp_mat = sparse_tensor.SparseTensor(
       array_ops.constant(idx, dtype=ops.dtypes.int64),
-      array_ops.ones((len(idx),), dtype=ops.dtypes.float32), sp_shape)
+      array_ops.ones((len(idx),), dtype=grad.dtype), sp_shape)
 
   jac = sparse_ops.sparse_tensor_dense_matmul(sp_mat, grad_flat)
 
-- 
cgit v1.2.3


From a46c9ab4419402182c34404f0f57c1f7b6b51858 Mon Sep 17 00:00:00 2001
From: Adrian Kuegel 
Date: Wed, 18 Jul 2018 07:02:44 -0700
Subject: Support unsigned indices for in-place DynamicUpdateSlice.

For unsigned indices, we need to use unsigned comparisons when clamping the
start_indices.
Also rename the files from ops.* to dynamic_update_slice_util.*

PiperOrigin-RevId: 205072344
---
 tensorflow/compiler/xla/service/cpu/BUILD          |   2 +-
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc  |   2 +-
 tensorflow/compiler/xla/service/gpu/BUILD          |   2 +-
 .../xla/service/gpu/ir_emitter_unnested.cc         |   2 +-
 tensorflow/compiler/xla/service/llvm_ir/BUILD      |   6 +-
 .../service/llvm_ir/dynamic_update_slice_util.cc   | 205 +++++++++++++++++++++
 .../service/llvm_ir/dynamic_update_slice_util.h    |  93 ++++++++++
 tensorflow/compiler/xla/service/llvm_ir/ops.cc     | 200 --------------------
 tensorflow/compiler/xla/service/llvm_ir/ops.h      |  93 ----------
 9 files changed, 305 insertions(+), 300 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
 create mode 100644 tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h
 delete mode 100644 tensorflow/compiler/xla/service/llvm_ir/ops.cc
 delete mode 100644 tensorflow/compiler/xla/service/llvm_ir/ops.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index c45d914e93..ace9f96cfb 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -252,12 +252,12 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo_module_config",
         "//tensorflow/compiler/xla/service:name_uniquer",
         "//tensorflow/compiler/xla/service/llvm_ir:alias_analysis",
+        "//tensorflow/compiler/xla/service/llvm_ir:dynamic_update_slice_util",
         "//tensorflow/compiler/xla/service/llvm_ir:fused_ir_emitter",
         "//tensorflow/compiler/xla/service/llvm_ir:ir_array",
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_loop",
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
         "//tensorflow/compiler/xla/service/llvm_ir:loop_emitter",
-        "//tensorflow/compiler/xla/service/llvm_ir:ops",
         "//tensorflow/compiler/xla/service/llvm_ir:tuple_ops",
         "//tensorflow/core:lib",
         "@llvm//:code_gen",
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 2ad41374d3..05f431642c 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -51,10 +51,10 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_casting_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_instructions.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index ceb3b5b5df..a043795a21 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -162,6 +162,7 @@ cc_library(
         "//tensorflow/compiler/xla/service:elemental_ir_emitter",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:name_uniquer",
+        "//tensorflow/compiler/xla/service/llvm_ir:dynamic_update_slice_util",
         "//tensorflow/compiler/xla/service/llvm_ir:fused_ir_emitter",
         "//tensorflow/compiler/xla/service/llvm_ir:ir_array",
         "//tensorflow/compiler/xla/service/llvm_ir:kernel_support_library",
@@ -169,7 +170,6 @@ cc_library(
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_loop",
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
         "//tensorflow/compiler/xla/service/llvm_ir:loop_emitter",
-        "//tensorflow/compiler/xla/service/llvm_ir:ops",
         "//tensorflow/compiler/xla/service/llvm_ir:tuple_ops",
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_no_cuda",
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 70a227ca4a..1caf10a6c1 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -59,10 +59,10 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/service/name_uniquer.h"
 #include "tensorflow/compiler/xla/shape_util.h"
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index 6f1e04a1c6..c14a5bfb53 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -164,9 +164,9 @@ cc_library(
 )
 
 cc_library(
-    name = "ops",
-    srcs = ["ops.cc"],
-    hdrs = ["ops.h"],
+    name = "dynamic_update_slice_util",
+    srcs = ["dynamic_update_slice_util.cc"],
+    hdrs = ["dynamic_update_slice_util.h"],
     deps = [
         ":fused_ir_emitter",
         ":ir_array",
diff --git a/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
new file mode 100644
index 0000000000..7048fcfdc9
--- /dev/null
+++ b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
@@ -0,0 +1,205 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h"
+#include "tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h"
+#include "tensorflow/compiler/xla/service/gpu/partition_assignment.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h"
+
+namespace xla {
+namespace llvm_ir {
+
+bool CanUpdateDynamicSliceInPlace(HloInstruction* dynamic_update_slice,
+                                  const BufferAssignment& assignment) {
+  CHECK_EQ(HloOpcode::kDynamicUpdateSlice, dynamic_update_slice->opcode());
+  const HloInstruction* operand = dynamic_update_slice->operand(0);
+  return assignment.HasTopLevelAllocation(dynamic_update_slice) &&
+         assignment.HasTopLevelAllocation(operand) &&
+         assignment.SharesTopLevelSlice(dynamic_update_slice, operand);
+}
+
+// Shared implementation of EmitDynamicUpdateSliceInPlace and
+// EmitFusedDynamicUpdateSliceInPlace.
+//
+// Emits a sequential loop if launch_dimensions is null.
+static Status EmitDynamicUpdateSliceInPlaceImpl(
+    const Shape& update_shape, const ElementGenerator& start_indices_generator,
+    bool is_signed, ElementGenerator update_array_generator,
+    const IrArray& output_array, const gpu::LaunchDimensions* launch_dimensions,
+    tensorflow::StringPiece name, llvm::IRBuilder<>* ir_builder) {
+  const Shape& output_shape = output_array.GetShape();
+
+  // Read start indices from start_indices_generator.
+  const int64 rank = ShapeUtil::Rank(output_shape);
+  IrArray::Index start_index(ir_builder->getInt64Ty(), rank);
+  for (int64 i = 0; i < rank; ++i) {
+    IrArray::Index dim_index({ir_builder->getInt64(i)});
+    TF_ASSIGN_OR_RETURN(start_index[i], start_indices_generator(dim_index));
+    llvm::Value* output_dim_size = llvm::ConstantInt::get(
+        start_index[i]->getType(), output_shape.dimensions(i));
+    llvm::Value* update_dim_size = llvm::ConstantInt::get(
+        start_index[i]->getType(), update_shape.dimensions(i));
+
+    // Clamp the start index so that the update region fits in the operand.
+    // start_index = clamp(start_index, 0, output_dim_size - update_dim_size)
+
+    // TODO(b/74360564): This is implementation defined behavior, but is
+    // currently respected by all implementations. Change this if we ever decide
+    // to officially document different behavior.
+    llvm::Value* max_bound =
+        ir_builder->CreateSub(output_dim_size, update_dim_size);
+    llvm::Value* zero = llvm::ConstantInt::get(start_index[i]->getType(), 0);
+    start_index[i] = ir_builder->CreateSelect(
+        ir_builder->CreateICmp(
+            is_signed ? llvm::ICmpInst::ICMP_SGE : llvm::ICmpInst::ICMP_UGE,
+            zero, start_index[i]),
+        zero, start_index[i]);
+
+    start_index[i] = ir_builder->CreateSelect(
+        ir_builder->CreateICmp(
+            is_signed ? llvm::ICmpInst::ICMP_SLE : llvm::ICmpInst::ICMP_ULE,
+            max_bound, start_index[i]),
+        max_bound, start_index[i]);
+  }
+
+  auto loop_body_emitter = [&](const IrArray::Index& update_index) -> Status {
+    // Calculate output_index, where we'll write the value from update.  For
+    // each dimension,
+    //
+    //   output_index[dim] = start_index[dim] + update_index[dim]
+    //
+    IrArray::Index output_index(start_index.GetType(), rank);
+    for (int64 i = 0; i < rank; ++i) {
+      llvm::Value* start_index0 = ir_builder->CreateSExtOrBitCast(
+          start_index[i], update_index[i]->getType());
+      output_index[i] = ir_builder->CreateAdd(start_index0, update_index[i]);
+    }
+
+    // Do output[output_index] = update[update_index].
+    TF_ASSIGN_OR_RETURN(llvm::Value * update_data,
+                        update_array_generator(update_index));
+    output_array.EmitWriteArrayElement(output_index, update_data, ir_builder);
+    return Status::OK();
+  };
+
+  if (launch_dimensions != nullptr) {
+    return gpu::ParallelLoopEmitter(loop_body_emitter, update_shape,
+                                    *launch_dimensions, ir_builder)
+        .EmitLoop(name);
+  }
+  return LoopEmitter(loop_body_emitter, update_shape, ir_builder)
+      .EmitLoop(name);
+}
+
+Status EmitDynamicUpdateSliceInPlace(
+    tensorflow::gtl::ArraySlice operand_arrays,
+    const IrArray& output_array, tensorflow::StringPiece name,
+    llvm::IRBuilder<>* ir_builder) {
+  VLOG(2) << "EmitDynamicUpdateSliceInPlace for " << name;
+
+  // No need to use operand_arrays[0], the input array of the
+  // dynamic-update-slice, because we know it aliases the op's output.
+  IrArray update_array = operand_arrays[1];
+  IrArray start_indices_array = operand_arrays[2];
+  Shape output_shape = output_array.GetShape();
+  Shape update_shape = update_array.GetShape();
+
+  ElementGenerator start_indices_generator = [&](const IrArray::Index& index) {
+    return start_indices_array.EmitReadArrayElement(index, ir_builder);
+  };
+  ElementGenerator update_array_generator = [&](const IrArray::Index& index) {
+    return update_array.EmitReadArrayElement(index, ir_builder);
+  };
+
+  bool is_signed = ShapeUtil::ElementIsSigned(start_indices_array.GetShape());
+  return EmitDynamicUpdateSliceInPlaceImpl(
+      update_shape, start_indices_generator, is_signed, update_array_generator,
+      output_array, /*launch_dimensions=*/nullptr, name, ir_builder);
+}
+
+// Shared implementation for EmitFusedDynamicUpdateSliceInPlace and
+// EmitParallelFusedDynamicUpdateSliceInPlace.
+//
+// Emits a sequential loop if launch_dimensions is null.
+static Status EmitFusedDynamicUpdateSliceInPlaceImpl(
+    HloInstruction* fusion,
+    tensorflow::gtl::ArraySlice fusion_operand_arrays,
+    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
+    const gpu::LaunchDimensions* launch_dimensions,
+    llvm::IRBuilder<>* ir_builder) {
+  CHECK_EQ(fusion->opcode(), HloOpcode::kFusion);
+  VLOG(2) << "EmitFusedDynamicUpdateSliceInPlace for "
+          << fusion->ToShortString();
+
+  auto* dynamic_update_slice = fusion->fused_expression_root();
+
+  const auto* update = dynamic_update_slice->operand(1);
+  const auto* start_indices = dynamic_update_slice->operand(2);
+  Shape update_shape = update->shape();
+
+  // Our in-place dynamic-update-slice implementation emits a loop over
+  // update_shape.  To emit a cache-friendly loop, we need to know that shape's
+  // layout.
+  //
+  // update_shape is inside a fusion node -- it's never materialized in memory
+  // and thus doesn't have a layout.  In this case we use the layout of the
+  // fusion node for iteration, since that corresponds to the order in memory of
+  // the buffer we'll be writing to.
+  //
+  // (This isn't necessarily optimal; in some cases it might be faster to peek
+  // through the chain of ops that gives us the update operand and use the
+  // layout of its source buffer(s).  But this is no worse than we do with
+  // fusion elsewhere.)
+  TF_RETURN_IF_ERROR(
+      LayoutUtil::CopyLayoutBetweenShapes(fusion->shape(), &update_shape));
+
+  // Create element generators for update and start_indices.
+  FusedIrEmitter fused_emitter(fusion_operand_arrays, elemental_emitter);
+  TF_RETURN_IF_ERROR(dynamic_update_slice->Accept(&fused_emitter));
+  ElementGenerator update_array_generator = fused_emitter.GetGenerator(update);
+  ElementGenerator start_indices_generator =
+      fused_emitter.GetGenerator(start_indices);
+
+  bool is_signed = ShapeUtil::ElementIsSigned(start_indices->shape());
+  return EmitDynamicUpdateSliceInPlaceImpl(
+      update_shape, start_indices_generator, is_signed, update_array_generator,
+      fusion_output_array, launch_dimensions, IrName(fusion), ir_builder);
+}
+
+Status EmitFusedDynamicUpdateSliceInPlace(
+    HloInstruction* fusion,
+    tensorflow::gtl::ArraySlice fusion_operand_arrays,
+    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
+    llvm::IRBuilder<>* ir_builder) {
+  return EmitFusedDynamicUpdateSliceInPlaceImpl(
+      fusion, fusion_operand_arrays, fusion_output_array, elemental_emitter,
+      /*launch_dimensions=*/nullptr, ir_builder);
+}
+
+Status EmitParallelFusedDynamicUpdateSliceInPlace(
+    HloInstruction* fusion,
+    tensorflow::gtl::ArraySlice fusion_operand_arrays,
+    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
+    const gpu::LaunchDimensions& launch_dimensions,
+    llvm::IRBuilder<>* ir_builder) {
+  return EmitFusedDynamicUpdateSliceInPlaceImpl(
+      fusion, fusion_operand_arrays, fusion_output_array, elemental_emitter,
+      &launch_dimensions, ir_builder);
+}
+
+}  // namespace llvm_ir
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h
new file mode 100644
index 0000000000..7f73fb6b29
--- /dev/null
+++ b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h
@@ -0,0 +1,93 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_DYNAMIC_UPDATE_SLICE_UTIL_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_DYNAMIC_UPDATE_SLICE_UTIL_H_
+
+#include "tensorflow/compiler/xla/service/buffer_assignment.h"
+#include "tensorflow/compiler/xla/service/elemental_ir_emitter.h"
+#include "tensorflow/compiler/xla/service/gpu/partition_assignment.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/ir_array.h"
+
+// Utilities related to emitting LLVM IR for various HLO ops.
+
+namespace xla {
+namespace llvm_ir {
+
+// Checks if we can emit code for the given DynamicUpdateSlice node that updates
+// its input in place.  Returns true if the dynamic-update-slice's
+// array-to-be-updated and output share the same BufferAllocation::Slice.
+//
+// dynamic_update_slice must be a DynamicUpdateSlice op.
+bool CanUpdateDynamicSliceInPlace(HloInstruction* dynamic_update_slice,
+                                  const BufferAssignment& assignment);
+
+// Checks if the given fusion node is amenable to being implemented by
+// EmitFusedDynamicUpdateSliceInPlace.
+inline bool CanEmitFusedDynamicUpdateSliceInPlace(
+    HloInstruction* fusion, const BufferAssignment& assignment) {
+  CHECK_EQ(fusion->opcode(), HloOpcode::kFusion);
+  HloInstruction* fused_root = fusion->fused_expression_root();
+  if (fused_root->opcode() != HloOpcode::kDynamicUpdateSlice ||
+      fusion->fusion_kind() != HloInstruction::FusionKind::kLoop) {
+    return false;
+  }
+  // Walk DynamicUpdateSlice operand(0) to fused parameter and get its
+  // associated operand. See if it shares an allocation with this operand.
+  HloInstruction* fusion_operand;
+  ShapeIndex index;
+  std::tie(fusion_operand, index) =
+      fused_root->mutable_operand(0)->LatestNonGteAncestorAndIndex();
+  if (fusion_operand->opcode() != HloOpcode::kParameter) {
+    return false;
+  }
+  auto* operand = fusion->operand(fusion_operand->parameter_number());
+  return assignment.HasAllocationAt(operand, index) &&
+         assignment.HasAllocationAt(fusion, {}) &&
+         assignment.SharesSliceAtIndex(fusion, {}, operand, index);
+}
+
+// Emits IR for running the given dynamic-update-slice op in-place -- that is,
+// where the input and output buffers share the same slice, so we can simply
+// modify the input/output buffer without touching any of the other elements.
+Status EmitDynamicUpdateSliceInPlace(
+    tensorflow::gtl::ArraySlice operand_arrays,
+    const IrArray& output_array, tensorflow::StringPiece name,
+    llvm::IRBuilder<>* ir_builder);
+
+// Given a loop-fusion node whose root is a dynamic-update-slice op whose
+// array-to-be-updated and output share the same buffer slice, emits
+// (sequential) code for a fusion node that does the dynamic-update-slice in
+// place.
+Status EmitFusedDynamicUpdateSliceInPlace(
+    HloInstruction* fusion,
+    tensorflow::gtl::ArraySlice fusion_operand_arrays,
+    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
+    llvm::IRBuilder<>* ir_builder);
+
+// Same as EmitFusedDynamicUpdateSliceInPlace, except emits a parallel loop with
+// the given launch dimensions.
+Status EmitParallelFusedDynamicUpdateSliceInPlace(
+    HloInstruction* fusion,
+    tensorflow::gtl::ArraySlice fusion_operand_arrays,
+    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
+    const gpu::LaunchDimensions& launch_dimensions,
+    llvm::IRBuilder<>* ir_builder);
+
+}  // namespace llvm_ir
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_DYNAMIC_UPDATE_SLICE_UTIL_H_
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ops.cc b/tensorflow/compiler/xla/service/llvm_ir/ops.cc
deleted file mode 100644
index 3b298f4746..0000000000
--- a/tensorflow/compiler/xla/service/llvm_ir/ops.cc
+++ /dev/null
@@ -1,200 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
-#include "tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h"
-#include "tensorflow/compiler/xla/service/gpu/partition_assignment.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h"
-
-namespace xla {
-namespace llvm_ir {
-
-bool CanUpdateDynamicSliceInPlace(HloInstruction* dynamic_update_slice,
-                                  const BufferAssignment& assignment) {
-  CHECK_EQ(HloOpcode::kDynamicUpdateSlice, dynamic_update_slice->opcode());
-  const HloInstruction* operand = dynamic_update_slice->operand(0);
-  return assignment.HasTopLevelAllocation(dynamic_update_slice) &&
-         assignment.HasTopLevelAllocation(operand) &&
-         assignment.SharesTopLevelSlice(dynamic_update_slice, operand);
-}
-
-// Shared implementation of EmitDynamicUpdateSliceInPlace and
-// EmitFusedDynamicUpdateSliceInPlace.
-//
-// Emits a sequential loop if launch_dimensions is null.
-static Status EmitDynamicUpdateSliceInPlaceImpl(
-    const Shape& update_shape, const ElementGenerator& start_indices_generator,
-    ElementGenerator update_array_generator, const IrArray& output_array,
-    const gpu::LaunchDimensions* launch_dimensions,
-    tensorflow::StringPiece name, llvm::IRBuilder<>* ir_builder) {
-  const Shape& output_shape = output_array.GetShape();
-
-  // Read start indices from start_indices_generator.
-  const int64 rank = ShapeUtil::Rank(output_shape);
-  IrArray::Index start_index(ir_builder->getInt64Ty(), rank);
-  for (int64 i = 0; i < rank; ++i) {
-    IrArray::Index dim_index({ir_builder->getInt64(i)});
-    TF_ASSIGN_OR_RETURN(start_index[i], start_indices_generator(dim_index));
-    llvm::Value* output_dim_size = llvm::ConstantInt::get(
-        start_index[i]->getType(), output_shape.dimensions(i));
-    llvm::Value* update_dim_size = llvm::ConstantInt::get(
-        start_index[i]->getType(), update_shape.dimensions(i));
-
-    // Clamp the start index so that the update region fits in the operand.
-    // start_index = clamp(start_index, 0, output_dim_size - update_dim_size)
-
-    // TODO(b/74360564): This is implementation defined behavior, but is
-    // currently respected by all implementations. Change this if we ever decide
-    // to oficially document different behavior.
-    llvm::Value* max_bound =
-        ir_builder->CreateSub(output_dim_size, update_dim_size);
-    llvm::Value* zero = llvm::ConstantInt::get(start_index[i]->getType(), 0);
-    start_index[i] = ir_builder->CreateSelect(
-        ir_builder->CreateICmp(llvm::ICmpInst::ICMP_SGE, zero, start_index[i]),
-        zero, start_index[i]);
-
-    start_index[i] = ir_builder->CreateSelect(
-        ir_builder->CreateICmp(llvm::ICmpInst::ICMP_SLE, max_bound,
-                               start_index[i]),
-        max_bound, start_index[i]);
-  }
-
-  auto loop_body_emitter = [&](const IrArray::Index& update_index) -> Status {
-    // Calculate output_index, where we'll write the value from update.  For
-    // each dimension,
-    //
-    //   output_index[dim] = start_index[dim] + update_index[dim]
-    //
-    IrArray::Index output_index(start_index.GetType(), rank);
-    for (int64 i = 0; i < rank; ++i) {
-      llvm::Value* start_index0 = ir_builder->CreateSExtOrBitCast(
-          start_index[i], update_index[i]->getType());
-      output_index[i] = ir_builder->CreateAdd(start_index0, update_index[i]);
-    }
-
-    // Do output[output_index] = update[update_index].
-    TF_ASSIGN_OR_RETURN(llvm::Value * update_data,
-                        update_array_generator(update_index));
-    output_array.EmitWriteArrayElement(output_index, update_data, ir_builder);
-    return Status::OK();
-  };
-
-  if (launch_dimensions != nullptr) {
-    return gpu::ParallelLoopEmitter(loop_body_emitter, update_shape,
-                                    *launch_dimensions, ir_builder)
-        .EmitLoop(name);
-  }
-  return LoopEmitter(loop_body_emitter, update_shape, ir_builder)
-      .EmitLoop(name);
-}
-
-Status EmitDynamicUpdateSliceInPlace(
-    tensorflow::gtl::ArraySlice operand_arrays,
-    const IrArray& output_array, tensorflow::StringPiece name,
-    llvm::IRBuilder<>* ir_builder) {
-  VLOG(2) << "EmitDynamicUpdateSliceInPlace for " << name;
-
-  // No need to use operand_arrays[0], the input array of the
-  // dynamic-update-slice, because we know it aliases the op's output.
-  IrArray update_array = operand_arrays[1];
-  IrArray start_indices_array = operand_arrays[2];
-  Shape output_shape = output_array.GetShape();
-  Shape update_shape = update_array.GetShape();
-
-  ElementGenerator start_indices_generator = [&](const IrArray::Index& index) {
-    return start_indices_array.EmitReadArrayElement(index, ir_builder);
-  };
-  ElementGenerator update_array_generator = [&](const IrArray::Index& index) {
-    return update_array.EmitReadArrayElement(index, ir_builder);
-  };
-
-  return EmitDynamicUpdateSliceInPlaceImpl(
-      update_shape, start_indices_generator, update_array_generator,
-      output_array, /*launch_dimensions=*/nullptr, name, ir_builder);
-}
-
-// Shared implementation for EmitFusedDynamicUpdateSliceInPlace and
-// EmitParallelFusedDynamicUpdateSliceInPlace.
-//
-// Emits a sequential loop if launch_dimensions is null.
-static Status EmitFusedDynamicUpdateSliceInPlaceImpl(
-    HloInstruction* fusion,
-    tensorflow::gtl::ArraySlice fusion_operand_arrays,
-    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
-    const gpu::LaunchDimensions* launch_dimensions,
-    llvm::IRBuilder<>* ir_builder) {
-  CHECK_EQ(fusion->opcode(), HloOpcode::kFusion);
-  VLOG(2) << "EmitFusedDynamicUpdateSliceInPlace for "
-          << fusion->ToShortString();
-
-  auto* dynamic_update_slice = fusion->fused_expression_root();
-
-  const auto* update = dynamic_update_slice->operand(1);
-  const auto* start_indices = dynamic_update_slice->operand(2);
-  Shape update_shape = update->shape();
-
-  // Our in-place dynamic-update-slice implementation emits a loop over
-  // update_shape.  To emit a cache-friendly loop, we need to know that shape's
-  // layout.
-  //
-  // update_shape is inside a fusion node -- it's never materialized in memory
-  // and thus doesn't have a layout.  In this case we use the layout of the
-  // fusion node for iteration, since that corresponds to the order in memory of
-  // the buffer we'll be writing to.
-  //
-  // (This isn't necessarily optimal; in some cases it might be faster to peek
-  // through the chain of ops that gives us the update operand and use the
-  // layout of its source buffer(s).  But this is no worse than we do with
-  // fusion elsewhere.)
-  TF_RETURN_IF_ERROR(
-      LayoutUtil::CopyLayoutBetweenShapes(fusion->shape(), &update_shape));
-
-  // Create element generators for update and start_indices.
-  FusedIrEmitter fused_emitter(fusion_operand_arrays, elemental_emitter);
-  TF_RETURN_IF_ERROR(dynamic_update_slice->Accept(&fused_emitter));
-  ElementGenerator update_array_generator = fused_emitter.GetGenerator(update);
-  ElementGenerator start_indices_generator =
-      fused_emitter.GetGenerator(start_indices);
-
-  return EmitDynamicUpdateSliceInPlaceImpl(
-      update_shape, start_indices_generator, update_array_generator,
-      fusion_output_array, launch_dimensions, IrName(fusion), ir_builder);
-}
-
-Status EmitFusedDynamicUpdateSliceInPlace(
-    HloInstruction* fusion,
-    tensorflow::gtl::ArraySlice fusion_operand_arrays,
-    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
-    llvm::IRBuilder<>* ir_builder) {
-  return EmitFusedDynamicUpdateSliceInPlaceImpl(
-      fusion, fusion_operand_arrays, fusion_output_array, elemental_emitter,
-      /*launch_dimensions=*/nullptr, ir_builder);
-}
-
-Status EmitParallelFusedDynamicUpdateSliceInPlace(
-    HloInstruction* fusion,
-    tensorflow::gtl::ArraySlice fusion_operand_arrays,
-    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
-    const gpu::LaunchDimensions& launch_dimensions,
-    llvm::IRBuilder<>* ir_builder) {
-  return EmitFusedDynamicUpdateSliceInPlaceImpl(
-      fusion, fusion_operand_arrays, fusion_output_array, elemental_emitter,
-      &launch_dimensions, ir_builder);
-}
-
-}  // namespace llvm_ir
-}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ops.h b/tensorflow/compiler/xla/service/llvm_ir/ops.h
deleted file mode 100644
index 175b081e84..0000000000
--- a/tensorflow/compiler/xla/service/llvm_ir/ops.h
+++ /dev/null
@@ -1,93 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_OPS_H_
-#define TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_OPS_H_
-
-#include "tensorflow/compiler/xla/service/buffer_assignment.h"
-#include "tensorflow/compiler/xla/service/elemental_ir_emitter.h"
-#include "tensorflow/compiler/xla/service/gpu/partition_assignment.h"
-#include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/ir_array.h"
-
-// Utilities related to emitting LLVM IR for various HLO ops.
-
-namespace xla {
-namespace llvm_ir {
-
-// Checks if we can emit code for the given DynamicUpdateSlice node that updates
-// its input in place.  Returns true if the dynamic-update-slice's
-// array-to-be-updated and output share the same BufferAllocation::Slice.
-//
-// dynamic_update_slice must be a DynamicUpdateSlice op.
-bool CanUpdateDynamicSliceInPlace(HloInstruction* dynamic_update_slice,
-                                  const BufferAssignment& assignment);
-
-// Checks if the given fusion node is amenable to being implemented by
-// EmitFusedDynamicUpdateSliceInPlace.
-inline bool CanEmitFusedDynamicUpdateSliceInPlace(
-    HloInstruction* fusion, const BufferAssignment& assignment) {
-  CHECK_EQ(fusion->opcode(), HloOpcode::kFusion);
-  HloInstruction* fused_root = fusion->fused_expression_root();
-  if (fused_root->opcode() != HloOpcode::kDynamicUpdateSlice ||
-      fusion->fusion_kind() != HloInstruction::FusionKind::kLoop) {
-    return false;
-  }
-  // Walk DynamicUpdateSlice operand(0) to fused parameter and get its
-  // associated operand. See if it shares an allocation with this operand.
-  HloInstruction* fusion_operand;
-  ShapeIndex index;
-  std::tie(fusion_operand, index) =
-      fused_root->mutable_operand(0)->LatestNonGteAncestorAndIndex();
-  if (fusion_operand->opcode() != HloOpcode::kParameter) {
-    return false;
-  }
-  auto* operand = fusion->operand(fusion_operand->parameter_number());
-  return assignment.HasAllocationAt(operand, index) &&
-         assignment.HasAllocationAt(fusion, {}) &&
-         assignment.SharesSliceAtIndex(fusion, {}, operand, index);
-}
-
-// Emits IR for running the given dynamic-update-slice op in-place -- that is,
-// where the input and output buffers share the same slice, so we can simply
-// modify the input/output buffer without touching any of the other elements.
-Status EmitDynamicUpdateSliceInPlace(
-    tensorflow::gtl::ArraySlice operand_arrays,
-    const IrArray& output_array, tensorflow::StringPiece name,
-    llvm::IRBuilder<>* ir_builder);
-
-// Given a loop-fusion node whose root is a dynamic-update-slice op whose
-// array-to-be-updated and output share the same buffer slice, emits
-// (sequential) code for a fusion node that does the dynamic-update-slice in
-// place.
-Status EmitFusedDynamicUpdateSliceInPlace(
-    HloInstruction* fusion,
-    tensorflow::gtl::ArraySlice fusion_operand_arrays,
-    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
-    llvm::IRBuilder<>* ir_builder);
-
-// Same as EmitFusedDynamicUpdateSliceInPlace, except emits a parallel loop with
-// the given launch dimensions.
-Status EmitParallelFusedDynamicUpdateSliceInPlace(
-    HloInstruction* fusion,
-    tensorflow::gtl::ArraySlice fusion_operand_arrays,
-    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
-    const gpu::LaunchDimensions& launch_dimensions,
-    llvm::IRBuilder<>* ir_builder);
-
-}  // namespace llvm_ir
-}  // namespace xla
-
-#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_OPS_H_
-- 
cgit v1.2.3


From 9b3a29889f6533b2f3c8cba6eba9e2d653712750 Mon Sep 17 00:00:00 2001
From: Rachel Lim 
Date: Wed, 18 Jul 2018 07:55:00 -0700
Subject: [tf.data] Add checkpointing for CsvDataset

PiperOrigin-RevId: 205078174
---
 tensorflow/contrib/data/kernels/csv_dataset_op.cc  | 100 +++++++++++++++++++--
 .../data/python/kernel_tests/serialization/BUILD   |  14 +++
 .../csv_dataset_serialization_test.py              |  73 +++++++++++++++
 3 files changed, 179 insertions(+), 8 deletions(-)
 create mode 100644 tensorflow/contrib/data/python/kernel_tests/serialization/csv_dataset_serialization_test.py

diff --git a/tensorflow/contrib/data/kernels/csv_dataset_op.cc b/tensorflow/contrib/data/kernels/csv_dataset_op.cc
index dadde705e1..f7e3ed886c 100644
--- a/tensorflow/contrib/data/kernels/csv_dataset_op.cc
+++ b/tensorflow/contrib/data/kernels/csv_dataset_op.cc
@@ -150,6 +150,7 @@ class CSVDatasetOp : public DatasetOpKernel {
           delim_(delim),
           na_value_(std::move(na_value)),
           use_compression_(!compression_type.empty()),
+          compression_type_(std::move(compression_type)),
           options_(options) {}
 
     std::unique_ptr MakeIteratorInternal(
@@ -169,10 +170,45 @@ class CSVDatasetOp : public DatasetOpKernel {
    protected:
     Status AsGraphDefInternal(DatasetGraphDefBuilder* b,
                               Node** output) const override {
-      // TODO(rachelim): Implement this
-      std::vector input_tensors;
-      TF_RETURN_IF_ERROR(b->AddDataset(this, input_tensors, output));
-      return errors::Unimplemented("CSVDataset: AsGraphDefInternal");
+      Node* filenames = nullptr;
+      Node* compression_type = nullptr;
+      Node* buffer_size = nullptr;
+      Node* header = nullptr;
+      Node* delim = nullptr;
+      Node* use_quote_delim = nullptr;
+      Node* na_value = nullptr;
+      Node* select_cols = nullptr;
+
+      std::vector record_defaults;
+      record_defaults.reserve(record_defaults_.size());
+      for (const Tensor& t : record_defaults_) {
+        Node* node;
+        TF_RETURN_IF_ERROR(b->AddTensor(t, &node));
+        record_defaults.emplace_back(node);
+      }
+
+      TF_RETURN_IF_ERROR(b->AddVector(filenames_, &filenames));
+      TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type));
+      TF_RETURN_IF_ERROR(
+          b->AddScalar(options_.input_buffer_size, &buffer_size));
+      TF_RETURN_IF_ERROR(b->AddScalar(header_, &header));
+
+      string delim_string(1, delim_);
+      TF_RETURN_IF_ERROR(b->AddScalar(delim_string, &delim));
+      TF_RETURN_IF_ERROR(b->AddScalar(use_quote_delim_, &use_quote_delim));
+      TF_RETURN_IF_ERROR(b->AddScalar(na_value_, &na_value));
+      TF_RETURN_IF_ERROR(b->AddVector(select_cols_, &select_cols));
+
+      TF_RETURN_IF_ERROR(b->AddDataset(
+          this,
+          {std::make_pair(0, filenames), std::make_pair(1, compression_type),
+           std::make_pair(2, buffer_size), std::make_pair(3, header),
+           std::make_pair(4, delim), std::make_pair(5, use_quote_delim),
+           std::make_pair(6, na_value),
+           std::make_pair(7, select_cols)},      // Single tensor inputs
+          {std::make_pair(8, record_defaults)},  // Tensor list inputs
+          {}, output));
+      return Status::OK();
     }
 
    private:
@@ -224,14 +260,58 @@ class CSVDatasetOp : public DatasetOpKernel {
      protected:
       Status SaveInternal(IteratorStateWriter* writer) override {
         mutex_lock l(mu_);
-        // TODO(rachelim): Implement save
-        return errors::Unimplemented("CSVDataset: SaveInternal");
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("current_file_index"),
+                                               current_file_index_));
+        // `input_stream_` is empty if
+        // 1. GetNext has not been called even once.
+        // 2. All files have been read and the iterator has been exhausted.
+        if (input_stream_ && num_buffer_reads_ > 0) {
+          TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("pos"), pos_));
+          // If num_buffer_reads_ == 0, the buffer hasn't been filled even once.
+          TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("num_buffer_reads"),
+                                                 num_buffer_reads_));
+        }
+        return Status::OK();
       }
+
       Status RestoreInternal(IteratorContext* ctx,
                              IteratorStateReader* reader) override {
         mutex_lock l(mu_);
-        // TODO(rachelim): Implement restore
-        return errors::Unimplemented("CSVDataset: RestoreInternal");
+        ResetStreamsLocked();
+        int64 current_file_index;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("current_file_index"),
+                                              ¤t_file_index));
+        current_file_index_ = size_t(current_file_index);
+        // The keys "pos" and "num_buffer_reads" are written only if
+        // the iterator was saved with an open, partially read file.
+        if (reader->Contains(full_name("pos"))) {
+          int64 pos, num_buffer_reads;
+          TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("pos"), &pos));
+          TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("num_buffer_reads"),
+                                                &num_buffer_reads));
+
+          TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env()));
+
+          num_buffer_reads_ = size_t(num_buffer_reads - 1);
+
+          // Restores the most recently held buffer
+          Status s = input_stream_->SkipNBytes(
+              num_buffer_reads_ * dataset()->options_.input_buffer_size);
+          if (!s.ok() && !errors::IsOutOfRange(s)) {
+            // We might get out of range error here if the size of the file
+            // is not an exact multiple of the buffer size, and the last buffer
+            // read is < buffer_size. This is valid and we do not surface the
+            // error.
+            return s;
+          }
+
+          Status s2 = FillBuffer(&buffer_);
+          if (!s2.ok() && !errors::IsOutOfRange(s2)) {
+            return s2;
+          }
+          pos_ = size_t(pos);
+        }
+        return Status::OK();
       }
 
      private:
@@ -533,6 +613,7 @@ class CSVDatasetOp : public DatasetOpKernel {
 
       Status FillBuffer(string* result) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
         result->clear();
+        ++num_buffer_reads_;
         Status s = input_stream_->ReadNBytes(
             dataset()->options_.input_buffer_size, result);
 
@@ -712,6 +793,7 @@ class CSVDatasetOp : public DatasetOpKernel {
         }
         buffer_.clear();
         pos_ = 0;
+        num_buffer_reads_ = 0;
         if (dataset()->header_) {
           // Read one line, but don't include it. Pass nullptrs as dummy
           // pointers to objects that shouldn't be invoked anyway
@@ -737,6 +819,7 @@ class CSVDatasetOp : public DatasetOpKernel {
       string buffer_ GUARDED_BY(mu_);  // Maintain our own buffer
       size_t pos_ GUARDED_BY(
           mu_);  // Index into the buffer must be maintained between iters
+      size_t num_buffer_reads_ GUARDED_BY(mu_);
       std::shared_ptr random_access_input_stream_
           GUARDED_BY(mu_);
       std::shared_ptr input_stream_ GUARDED_BY(mu_);
@@ -755,6 +838,7 @@ class CSVDatasetOp : public DatasetOpKernel {
     const char delim_;
     const string na_value_;
     const bool use_compression_;
+    const string compression_type_;
     const io::ZlibCompressionOptions options_;
   };  // class Dataset
 
diff --git a/tensorflow/contrib/data/python/kernel_tests/serialization/BUILD b/tensorflow/contrib/data/python/kernel_tests/serialization/BUILD
index 686788522a..3c3f23f9a9 100644
--- a/tensorflow/contrib/data/python/kernel_tests/serialization/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/serialization/BUILD
@@ -72,6 +72,20 @@ py_test(
     ],
 )
 
+py_test(
+    name = "csv_dataset_serialization_test",
+    size = "small",
+    srcs = ["csv_dataset_serialization_test.py"],
+    srcs_version = "PY2AND3",
+    tags = ["no_pip"],
+    deps = [
+        ":dataset_serialization_test_base",
+        "//tensorflow/contrib/data/python/ops:readers",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+    ],
+)
+
 py_test(
     name = "dataset_constructor_serialization_test",
     size = "medium",
diff --git a/tensorflow/contrib/data/python/kernel_tests/serialization/csv_dataset_serialization_test.py b/tensorflow/contrib/data/python/kernel_tests/serialization/csv_dataset_serialization_test.py
new file mode 100644
index 0000000000..247f2046ea
--- /dev/null
+++ b/tensorflow/contrib/data/python/kernel_tests/serialization/csv_dataset_serialization_test.py
@@ -0,0 +1,73 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the CsvDataset serialization."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import gzip
+import os
+
+from tensorflow.contrib.data.python.kernel_tests.serialization import dataset_serialization_test_base
+from tensorflow.contrib.data.python.ops import readers
+from tensorflow.python.platform import test
+
+
+class CsvDatasetSerializationTest(
+    dataset_serialization_test_base.DatasetSerializationTestBase):
+
+  def setUp(self):
+    self._num_cols = 7
+    self._num_rows = 10
+    self._num_epochs = 14
+    self._num_outputs = self._num_rows * self._num_epochs
+
+    inputs = [
+        ",".join(str(self._num_cols * j + i)
+                 for i in range(self._num_cols))
+        for j in range(self._num_rows)
+    ]
+    contents = "\n".join(inputs).encode("utf-8")
+
+    self._filename = os.path.join(self.get_temp_dir(), "file.csv")
+    self._compressed = os.path.join(self.get_temp_dir(),
+                                    "comp.csv")  # GZip compressed
+
+    with open(self._filename, "wb") as f:
+      f.write(contents)
+    with gzip.GzipFile(self._compressed, "wb") as f:
+      f.write(contents)
+
+  def ds_func(self, **kwargs):
+    compression_type = kwargs.get("compression_type", None)
+    if compression_type == "GZIP":
+      filename = self._compressed
+    elif compression_type is None:
+      filename = self._filename
+    else:
+      raise ValueError("Invalid compression type:", compression_type)
+
+    return readers.CsvDataset(filename, **kwargs).repeat(self._num_epochs)
+
+  def testSerializationCore(self):
+    defs = [[0]] * self._num_cols
+    self.run_core_tests(
+        lambda: self.ds_func(record_defaults=defs, buffer_size=2),
+        lambda: self.ds_func(record_defaults=defs, buffer_size=12),
+        self._num_outputs)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
cgit v1.2.3


From aa7960d9187131039b7122f66e60dd89dd5a90bb Mon Sep 17 00:00:00 2001
From: wim glenn 
Date: Wed, 18 Jul 2018 10:12:16 -0500
Subject: Recommend the user site, no sudo

---
 tensorflow/docs_src/install/install_linux.md | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 7534d0fac1..0d9b6af093 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -65,7 +65,7 @@ We *recommend* using `pip` version 8.1 or higher. If using a release before
 version 8.1, upgrade `pip`:
 
 
    -  sudo pip install -U pip
+  pip install -U pip
 
    
 
 If not using Ubuntu and [setuptools](https://pypi.org/project/setuptools/) is
@@ -198,7 +198,7 @@ We *recommend* using `pip` version 8.1 or higher. If using a release before
 version 8.1, upgrade `pip`:
 
 
    -  sudo pip install -U pip
+  pip install -U pip
 
    
 
 If not using Ubuntu and [setuptools](https://pypi.org/project/setuptools/) is
@@ -220,8 +220,8 @@ Choose one of the available TensorFlow packages for installation:
 And use `pip` to install the package for Python 2 or 3:
 
 
    -  sudo pip install -U tensorflow   # Python 2.7
-  sudo pip3 install -U tensorflow  # Python 3.n
+  pip install -U --user tensorflow   # Python 2.7
+  pip3 install -U --user tensorflow  # Python 3.n
 
    
 
 Use `pip list` to show the packages installed on the system.
@@ -239,8 +239,8 @@ If the above steps failed, try installing the TensorFlow binary using the remote
 URL of the `pip` package:
 
 
    -  sudo pip install --upgrade remote-pkg-URL   # Python 2.7
-  sudo pip3 install --upgrade remote-pkg-URL  # Python 3.n
+  pip install --user --upgrade remote-pkg-URL   # Python 2.7
+  pip3 install --user --upgrade remote-pkg-URL  # Python 3.n
 
    
 
 The remote-pkg-URL depends on the operating system, Python version,
@@ -255,8 +255,8 @@ encounter problems.
 To uninstall TensorFlow on your system, use one of following commands:
 
 
    -  sudo pip uninstall tensorflow   # for Python 2.7
-  sudo pip3 uninstall tensorflow  # for Python 3.n
+  pip uninstall tensorflow   # for Python 2.7
+  pip3 uninstall tensorflow  # for Python 3.n
 
    
 
 
-- 
cgit v1.2.3


From cb297f637e461839ff85c4557116fa90003daec7 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Wed, 18 Jul 2018 08:42:13 -0700
Subject: Add support for multiple inputs.

---
 tensorflow/contrib/tensorrt/test/base_test.py      | 12 ++++---
 .../tensorrt/test/tf_trt_integration_test_base.py  | 40 ++++++++++++----------
 2 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/base_test.py b/tensorflow/contrib/tensorrt/test/base_test.py
index f057e377cb..5ec7c7094e 100644
--- a/tensorflow/contrib/tensorrt/test/base_test.py
+++ b/tensorflow/contrib/tensorrt/test/base_test.py
@@ -36,11 +36,12 @@ class SimpleSingleEngineGraphDefTest(trt_test.TfTrtIntegrationTestBase):
     """Create a graph containing single segment."""
     # TODO(aaroey): test graph with different dtypes.
     dtype = dtypes.float32
+    input_name = "input"
     input_dims = [100, 24, 24, 2]
     g = ops.Graph()
     with g.as_default():
       inp = array_ops.placeholder(
-          dtype=dtype, shape=[None] + input_dims[1:], name=self.input_name)
+          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
       with g.device("/GPU:0"):
         conv_filter = constant_op.constant(
             [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
@@ -62,7 +63,8 @@ class SimpleSingleEngineGraphDefTest(trt_test.TfTrtIntegrationTestBase):
       array_ops.squeeze(pool, name=self.output_name)
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_dims=input_dims,
+        input_names=[input_name],
+        input_dims=[input_dims],
         num_expected_engines=1,
         expected_output_dims=(100, 6, 6, 6),
         allclose_atol=1.e-03,
@@ -75,11 +77,12 @@ class SimpleMultiEngineGraphDefTest(trt_test.TfTrtIntegrationTestBase):
     """Create a graph containing multiple segment."""
     # TODO(aaroey): test graph with different dtypes.
     dtype = dtypes.float32
+    input_name = "input"
     input_dims = [100, 24, 24, 2]
     g = ops.Graph()
     with g.as_default():
       inp = array_ops.placeholder(
-          dtype=dtype, shape=[None] + input_dims[1:], name=self.input_name)
+          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
       with g.device("/GPU:0"):
         conv_filter = constant_op.constant(
             [[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
@@ -109,7 +112,8 @@ class SimpleMultiEngineGraphDefTest(trt_test.TfTrtIntegrationTestBase):
       array_ops.squeeze(s, name=self.output_name)
     return trt_test.TfTrtIntegrationTestParams(
         gdef=g.as_graph_def(),
-        input_dims=input_dims,
+        input_names=[input_name],
+        input_dims=[input_dims],
         num_expected_engines=2,
         expected_output_dims=(100, 12, 12, 6),
         allclose_atol=1.e-03,
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 0f3b1eb37d..d7a7c8d998 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -35,8 +35,8 @@ from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 
 TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [
-    "gdef", "input_dims", "num_expected_engines", "expected_output_dims",
-    "allclose_atol", "allclose_rtol"
+    "gdef", "input_names", "input_dims", "num_expected_engines",
+    "expected_output_dims", "allclose_atol", "allclose_rtol"
 ])
 
 PRECISION_MODES = ["FP32", "FP16", "INT8"]
@@ -49,10 +49,6 @@ def _IsQuantizationMode(mode):
 class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   """Class to test Tensorflow-TensorRT integration."""
 
-  @property
-  def input_name(self):
-    return "input"
-
   @property
   def output_name(self):
     return "output"
@@ -98,7 +94,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       custom_op = rewriter_cfg.custom_optimizers.add()
       custom_op.name = "TensorRTOptimizer"
       custom_op.parameter_map["minimum_segment_size"].i = 3
-      custom_op.parameter_map["max_batch_size"].i = params.input_dims[0]
+      custom_op.parameter_map["max_batch_size"].i = max(
+          [dims[0] for dims in params.input_dims])
       custom_op.parameter_map["is_dynamic_op"].b = is_dynamic_op
       custom_op.parameter_map["max_workspace_size_bytes"].i = 1 << 25
       custom_op.parameter_map["precision_mode"].s = self._ToBytes(
@@ -117,20 +114,23 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
 
   def _RunGraph(self, params, gdef, input_data, config, num_runs=2):
     """Run given graphdef multiple times."""
+    assert len(params.input_names) == len(input_data)
     g = ops.Graph()
     with g.as_default():
-      inp, out = importer.import_graph_def(
+      io_ops = importer.import_graph_def(
           graph_def=gdef,
-          return_elements=[self.input_name, self.output_name],
+          return_elements=params.input_names + [self.output_name],
           name="")
-      inp = inp.outputs[0]
-      out = out.outputs[0]
+      inp = [i.outputs[0] for i in io_ops[:-1]]
+      assert len(inp) == len(input_data)
+      out = io_ops[-1].outputs[0]
     with self.test_session(
         graph=g, config=config, use_gpu=True, force_gpu=True) as sess:
       val = None
       # Defaults to 2 runs to verify result across multiple runs is same.
       for _ in range(num_runs):
-        new_val = sess.run(out, {inp: input_data})
+        new_val = sess.run(out,
+                           {inp[i]: input_data[i] for i in range(len(inp))})
         self.assertEquals(params.expected_output_dims, new_val.shape)
         if val is not None:
           self.assertAllEqual(new_val, val)
@@ -148,7 +148,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     return trt.create_inference_graph(
         input_graph_def=gdef,
         outputs=[self.output_name],
-        max_batch_size=params.input_dims[0],
+        max_batch_size=max([dims[0] for dims in params.input_dims]),
         max_workspace_size_bytes=1 << 25,
         precision_mode=precision_mode,
         minimum_segment_size=2,
@@ -180,7 +180,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
   def _RunTest(self, params, use_optimizer, precision_mode,
                dynamic_infer_engine, dynamic_calib_engine):
     assert precision_mode in PRECISION_MODES
-    inp = np.random.random_sample(params.input_dims)
+    input_data = [np.random.random_sample(dims) for dims in params.input_dims]
     input_gdef = params.gdef
     self._VerifyGraphDef(params, input_gdef)
 
@@ -188,7 +188,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     config_no_trt = self._GetConfigProto(params, False)
     logging.info("Running original graph w/o trt, config:\n%s",
                  str(config_no_trt))
-    ref_result = self._RunGraph(params, input_gdef, inp, config_no_trt)
+    ref_result = self._RunGraph(params, input_gdef, input_data, config_no_trt)
 
     # Run calibration if necessary.
     if _IsQuantizationMode(precision_mode):
@@ -200,13 +200,15 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         self.assertTrue(False)
         # TODO(aaroey): uncomment this and get infer_gdef when this mode is
         # supported.
-        # result = self._RunCalibration(params, input_gdef, inp, calib_config)
+        # result = self._RunCalibration(params, input_gdef, input_data,
+        #                               calib_config)
       else:
         calib_gdef = self._GetTrtGraphDef(params, input_gdef, precision_mode,
                                           dynamic_calib_engine)
         self._VerifyGraphDef(params, calib_gdef, precision_mode, False,
                              dynamic_calib_engine)
-        result = self._RunCalibration(params, calib_gdef, inp, calib_config)
+        result = self._RunCalibration(params, calib_gdef, input_data,
+                                      calib_config)
         infer_gdef = trt.calib_graph_to_infer_graph(calib_gdef)
         self._VerifyGraphDef(params, infer_gdef, precision_mode, True,
                              dynamic_calib_engine)
@@ -225,13 +227,13 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
     logging.info("Running final inference graph, config:\n%s",
                  str(infer_config))
     if use_optimizer:
-      result = self._RunGraph(params, infer_gdef, inp, infer_config)
+      result = self._RunGraph(params, infer_gdef, input_data, infer_config)
     else:
       trt_infer_gdef = self._GetTrtGraphDef(params, infer_gdef, precision_mode,
                                             dynamic_infer_engine)
       self._VerifyGraphDef(params, trt_infer_gdef, precision_mode, True,
                            dynamic_infer_engine)
-      result = self._RunGraph(params, trt_infer_gdef, inp, infer_config)
+      result = self._RunGraph(params, trt_infer_gdef, input_data, infer_config)
 
     self.assertAllClose(
         ref_result,
-- 
cgit v1.2.3


From c0dbd7e456a18d8431930b18c399f78dc66dda0c Mon Sep 17 00:00:00 2001
From: Jacques Pienaar 
Date: Wed, 18 Jul 2018 08:47:16 -0700
Subject: Register If as alias for XlaIf in bridge.

Enables lowering the If op via XLA.

PiperOrigin-RevId: 205084401
---
 tensorflow/compiler/tf2xla/kernels/if_op.cc    | 1 +
 tensorflow/compiler/tf2xla/kernels/while_op.cc | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/compiler/tf2xla/kernels/if_op.cc b/tensorflow/compiler/tf2xla/kernels/if_op.cc
index f5fcf3cacd..e2160feba0 100644
--- a/tensorflow/compiler/tf2xla/kernels/if_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/if_op.cc
@@ -246,6 +246,7 @@ void XlaIfOp::Compile(XlaOpKernelContext* ctx) {
   VLOG(1) << "Done building If";
 }
 
+REGISTER_XLA_OP(Name("If").AllowResourceTypes(), XlaIfOp);
 REGISTER_XLA_OP(Name("XlaIf").AllowResourceTypes(), XlaIfOp);
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/kernels/while_op.cc b/tensorflow/compiler/tf2xla/kernels/while_op.cc
index 9413a30a6c..009fdd81b2 100644
--- a/tensorflow/compiler/tf2xla/kernels/while_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/while_op.cc
@@ -299,6 +299,7 @@ void XlaWhileOp::Compile(XlaOpKernelContext* ctx) {
   VLOG(1) << "Done building while loop";
 }
 
+REGISTER_XLA_OP(Name("While").AllowResourceTypes(), XlaWhileOp);
 REGISTER_XLA_OP(Name("XlaWhile").AllowResourceTypes(), XlaWhileOp);
 
 }  // namespace tensorflow
-- 
cgit v1.2.3


From 5362d9dbedea6f93b1065e31cd0ee16ab110c7e1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 09:16:00 -0700
Subject: Data holder for the eager delegate.

PiperOrigin-RevId: 205088322
---
 tensorflow/contrib/lite/delegates/eager/BUILD      | 28 +++++++++++++
 .../contrib/lite/delegates/eager/delegate_data.cc  | 44 +++++++++++++++++++++
 .../contrib/lite/delegates/eager/delegate_data.h   | 46 ++++++++++++++++++++++
 .../lite/delegates/eager/delegate_data_test.cc     | 42 ++++++++++++++++++++
 4 files changed, 160 insertions(+)
 create mode 100644 tensorflow/contrib/lite/delegates/eager/delegate_data.cc
 create mode 100644 tensorflow/contrib/lite/delegates/eager/delegate_data.h
 create mode 100644 tensorflow/contrib/lite/delegates/eager/delegate_data_test.cc

diff --git a/tensorflow/contrib/lite/delegates/eager/BUILD b/tensorflow/contrib/lite/delegates/eager/BUILD
index 9d8c20e96f..23d8f543e5 100644
--- a/tensorflow/contrib/lite/delegates/eager/BUILD
+++ b/tensorflow/contrib/lite/delegates/eager/BUILD
@@ -38,6 +38,34 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "delegate_data",
+    srcs = ["delegate_data.cc"],
+    hdrs = ["delegate_data.h"],
+    deps = [
+        ":buffer_map",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:lib",
+        "//tensorflow/core/common_runtime/eager:context",
+    ],
+)
+
+cc_test(
+    name = "delegate_data_test",
+    size = "small",
+    srcs = ["delegate_data_test.cc"],
+    tags = [
+        "tflite_not_portable",
+    ],
+    deps = [
+        ":delegate_data",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite:util",
+        "//tensorflow/contrib/lite/testing:util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 cc_library(
     name = "util",
     srcs = ["util.cc"],
diff --git a/tensorflow/contrib/lite/delegates/eager/delegate_data.cc b/tensorflow/contrib/lite/delegates/eager/delegate_data.cc
new file mode 100644
index 0000000000..b2516379e7
--- /dev/null
+++ b/tensorflow/contrib/lite/delegates/eager/delegate_data.cc
@@ -0,0 +1,44 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/contrib/lite/delegates/eager/delegate_data.h"
+
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tflite {
+tensorflow::Status DelegateData::Create(std::unique_ptr* data) {
+  std::vector devices;
+
+  TF_RETURN_IF_ERROR(tensorflow::DeviceFactory::AddDevices(
+      tensorflow::SessionOptions(), "/device:cpu:*", &devices));
+
+  std::unique_ptr device_mgr(
+      new tensorflow::DeviceMgr(devices));
+  // Note that Rendezvous is ref-counted so it will be automatically deleted.
+  tensorflow::Rendezvous* rendezvous =
+      new tensorflow::IntraProcessRendezvous(device_mgr.get());
+  data->reset(new DelegateData(new tensorflow::EagerContext(
+      tensorflow::SessionOptions(),
+      tensorflow::ContextDevicePlacementPolicy::DEVICE_PLACEMENT_SILENT,
+      /*async=*/false, std::move(device_mgr), rendezvous)));
+  return tensorflow::Status();
+}
+
+DelegateData::DelegateData(tensorflow::EagerContext* eager_context)
+    : eager_context_(eager_context) {}
+
+DelegateData::~DelegateData() {}
+
+}  // namespace tflite
diff --git a/tensorflow/contrib/lite/delegates/eager/delegate_data.h b/tensorflow/contrib/lite/delegates/eager/delegate_data.h
new file mode 100644
index 0000000000..053d174c08
--- /dev/null
+++ b/tensorflow/contrib/lite/delegates/eager/delegate_data.h
@@ -0,0 +1,46 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_DELEGATE_DATA_H_
+#define TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_DELEGATE_DATA_H_
+
+#include "tensorflow/contrib/lite/delegates/eager/buffer_map.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
+
+namespace tflite {
+
+// Data kept by the Eager delegate for the lifetime of an Interpreter.
+class DelegateData {
+ public:
+  // Create a new DelegateData, initialized with a newly-created EagerContext.
+  static tensorflow::Status Create(std::unique_ptr* data);
+
+  ~DelegateData();
+
+  // The EagerContext that is required for execution of Eager Ops.
+  tensorflow::EagerContext* GetEagerContext() { return eager_context_.get(); }
+
+  // Map from TF Lite tensor index to TensorFlow tensor.
+  BufferMap* GetBufferMap() { return &buffer_map_; }
+
+ private:
+  explicit DelegateData(tensorflow::EagerContext* eager_context);
+
+  std::unique_ptr eager_context_;
+  BufferMap buffer_map_;
+};
+
+}  // namespace tflite
+
+#endif  // TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_DELEGATE_DATA_H_
diff --git a/tensorflow/contrib/lite/delegates/eager/delegate_data_test.cc b/tensorflow/contrib/lite/delegates/eager/delegate_data_test.cc
new file mode 100644
index 0000000000..cf8bc27d04
--- /dev/null
+++ b/tensorflow/contrib/lite/delegates/eager/delegate_data_test.cc
@@ -0,0 +1,42 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/contrib/lite/delegates/eager/delegate_data.h"
+
+#include 
+#include 
+#include "tensorflow/contrib/lite/testing/util.h"
+
+namespace tflite {
+namespace {
+
+TEST(DelegateDataTest, Basic) {
+  std::unique_ptr data;
+  // We only check for success because it is hard to make initialization fail.
+  // It only happens if we manage to not link the CPU device factory into the
+  // binary.
+  EXPECT_TRUE(DelegateData::Create(&data).ok());
+
+  EXPECT_NE(data->GetEagerContext(), nullptr);
+  EXPECT_NE(data->GetBufferMap(), nullptr);
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
-- 
cgit v1.2.3


From 41d7dc4a02440cef93477445bb58d4ee932a34a4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 09:20:07 -0700
Subject: Change the return type of ProcessState::GetCPUAllocator() from
 Allocator* to VisitableAllocator*.  All implementations are already
 VisitableAllocators, this just makes that fact clear and useful.

PiperOrigin-RevId: 205088826
---
 tensorflow/core/common_runtime/gpu/gpu_device.cc | 1 +
 tensorflow/core/common_runtime/process_state.cc  | 2 +-
 tensorflow/core/common_runtime/process_state.h   | 4 ++--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 3cb51b0dbc..c87841e2dc 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -41,6 +41,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/gpu/gpu_util.h"
 #include "tensorflow/core/common_runtime/gpu_device_context.h"
 #include "tensorflow/core/common_runtime/local_device.h"
+#include "tensorflow/core/common_runtime/visitable_allocator.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/device_base.h"
 #include "tensorflow/core/framework/op_kernel.h"
diff --git a/tensorflow/core/common_runtime/process_state.cc b/tensorflow/core/common_runtime/process_state.cc
index 4d83b25ce6..447338e7bd 100644
--- a/tensorflow/core/common_runtime/process_state.cc
+++ b/tensorflow/core/common_runtime/process_state.cc
@@ -71,7 +71,7 @@ ProcessState::MemDesc ProcessState::PtrType(const void* ptr) {
   return MemDesc();
 }
 
-Allocator* ProcessState::GetCPUAllocator(int numa_node) {
+VisitableAllocator* ProcessState::GetCPUAllocator(int numa_node) {
   CHECK_GE(numa_node, 0);
   if (!numa_enabled_) numa_node = 0;
   mutex_lock lock(mu_);
diff --git a/tensorflow/core/common_runtime/process_state.h b/tensorflow/core/common_runtime/process_state.h
index 0f4ae230bb..2892677333 100644
--- a/tensorflow/core/common_runtime/process_state.h
+++ b/tensorflow/core/common_runtime/process_state.h
@@ -65,7 +65,7 @@ class ProcessState {
 
   // Returns the one CPUAllocator used for the given numa_node.
   // TEMPORARY: ignores numa_node.
-  Allocator* GetCPUAllocator(int numa_node);
+  VisitableAllocator* GetCPUAllocator(int numa_node);
 
   typedef std::unordered_map MDMap;
 
@@ -87,7 +87,7 @@ class ProcessState {
 
   mutex mu_;
 
-  std::vector cpu_allocators_ GUARDED_BY(mu_);
+  std::vector cpu_allocators_ GUARDED_BY(mu_);
 
   virtual ~ProcessState();
 
-- 
cgit v1.2.3


From 702253235917259e527df0838850f1e563a88ff8 Mon Sep 17 00:00:00 2001
From: Peter Hawkins 
Date: Wed, 18 Jul 2018 09:32:04 -0700
Subject: [TF:XLA] Fix bug where the wrong value could be returned if
 always_return_tuple=false and there is exactly one output value that was not
 the last thing built in the XlaBuilder.

Also register IdentityN with a resource type, which was found to be missing when debugging this problem (mostly unrelated otherwise).

PiperOrigin-RevId: 205090332
---
 tensorflow/compiler/jit/xla_device_ops.h        |  4 +-
 tensorflow/compiler/tf2xla/xla_compiler.cc      | 10 +++--
 tensorflow/compiler/tf2xla/xla_compiler_test.cc | 52 +++++++++++++++++++++++++
 3 files changed, 59 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_device_ops.h b/tensorflow/compiler/jit/xla_device_ops.h
index 134dcc1bb5..6adda327f1 100644
--- a/tensorflow/compiler/jit/xla_device_ops.h
+++ b/tensorflow/compiler/jit/xla_device_ops.h
@@ -77,9 +77,7 @@ class XlaAssignVariableOp : public AsyncOpKernel {
       ConstantOp);                                                             \
   REGISTER_KERNEL_BUILDER(                                                     \
       Name("Identity").Device(DEVICE).TypeConstraint("T", TYPES), IdentityOp); \
-  REGISTER_KERNEL_BUILDER(                                                     \
-      Name("IdentityN").Device(DEVICE).TypeConstraint("T", TYPES),             \
-      IdentityNOp);                                                            \
+  REGISTER_KERNEL_BUILDER(Name("IdentityN").Device(DEVICE), IdentityNOp);      \
   REGISTER_KERNEL_BUILDER(Name("Placeholder").Device(DEVICE), PlaceholderOp);  \
   REGISTER_KERNEL_BUILDER(Name("PlaceholderV2").Device(DEVICE),                \
                           PlaceholderOp);                                      \
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 319cbc74e9..cb47581e36 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -422,16 +422,18 @@ Status BuildComputation(
       // assignment will be placed on this value, which will cause the resource
       // update to be returned from the same device that provided the resource.
       handle = xla::GetTupleElement(xla::Tuple(builder, {handle}), 0);
-
       elems.push_back(handle);
     }
   }
 
   *num_computation_outputs = elems.size();
 
-  // Builds the XLA computation.
-  if (always_return_tuple || elems.size() != 1) {
-    xla::Tuple(builder, elems);
+  // Builds the XLA computation. We *always* form a tuple here to ensure that
+  // the output value is the last thing added into the XLA computation, even
+  // if there is only one output value.
+  auto tuple = xla::Tuple(builder, elems);
+  if (!always_return_tuple && elems.size() == 1) {
+    xla::GetTupleElement(tuple, 0);
   }
   builder->ClearOpMetadata();
 
diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
index 6f76816a86..2fb93be01d 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
@@ -228,6 +228,58 @@ TEST_F(XlaCompilerTest, Simple) {
   EXPECT_TRUE(xla::LiteralTestUtil::Equal(*expected_literal, *actual_literal));
 }
 
+// Tests compilation of a graph where the _Retval node is not necessarily last
+// amongst the graph nodes in construction order, and always_return_tuple is
+// false. Regression test for bug where the wrong value was returned.
+TEST_F(XlaCompilerTest, OutOfOrderGraph) {
+  Scope scope = Scope::NewRootScope().ExitOnError();
+  auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0);
+  auto b = ops::_Arg(scope.WithOpName("B"), DT_INT32, 1);
+  // The _Retval node is not last in construction order.
+  auto d = ops::_Retval(scope.WithOpName("D"), a, 0);
+  auto c = ops::Add(scope.WithOpName("C"), a, b);
+
+  std::unique_ptr graph(new Graph(OpRegistry::Global()));
+  TF_ASSERT_OK(scope.ToGraph(graph.get()));
+
+  // Builds a description of the arguments.
+  std::vector args(2);
+  args[0].kind = XlaCompiler::Argument::kParameter;
+  args[0].type = DT_INT32;
+  args[0].shape = TensorShape({2});
+  args[1].kind = XlaCompiler::Argument::kParameter;
+  args[1].type = DT_INT32;
+  args[1].shape = TensorShape({2});
+
+  // Compiles the graph.
+  XlaCompiler compiler(DefaultOptions());
+
+  XlaCompiler::CompileOptions compile_options;
+  compile_options.always_return_tuple = false;
+  XlaCompiler::CompilationResult result;
+  TF_ASSERT_OK(compiler.CompileGraph(compile_options, "add", std::move(graph),
+                                     args, &result));
+
+  // Tests that the generated computation works.
+  std::unique_ptr param0_literal =
+      xla::LiteralUtil::CreateR1({7, 42});
+  std::unique_ptr param1_literal =
+      xla::LiteralUtil::CreateR1({-3, 101});
+  std::unique_ptr param0_data =
+      client_->TransferToServer(*param0_literal).ConsumeValueOrDie();
+  std::unique_ptr param1_data =
+      client_->TransferToServer(*param1_literal).ConsumeValueOrDie();
+
+  std::unique_ptr actual =
+      client_
+          ->Execute(*result.computation, {param0_data.get(), param1_data.get()})
+          .ConsumeValueOrDie();
+  std::unique_ptr actual_literal =
+      client_->Transfer(*actual).ConsumeValueOrDie();
+
+  EXPECT_TRUE(xla::LiteralTestUtil::Equal(*param0_literal, *actual_literal));
+}
+
 TEST_F(XlaCompilerTest, HasSaneErrorOnNonCompileTimeConstantInputToReshape) {
   // Builds a graph that adds reshapes a tensor, but with the shape not
   // statically known.
-- 
cgit v1.2.3


From b78e0da14d29d7e9d3b51a7b633c960baacd710c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 09:35:33 -0700
Subject: [XLA] add bit shift operators to local Python client

PiperOrigin-RevId: 205090789
---
 .../compiler/xla/python/local_computation_builder.cc   |  3 +++
 .../compiler/xla/python/local_computation_builder.h    |  3 +++
 .../compiler/xla/python/local_computation_builder.i    |  3 +++
 tensorflow/compiler/xla/python/xla_client.py           |  3 +++
 tensorflow/compiler/xla/python/xla_client_test.py      | 18 ++++++++++++++++++
 5 files changed, 30 insertions(+)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index be55d50b23..66b1c08a39 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -614,6 +614,9 @@ _FORWARD_BINOP(Min)
 _FORWARD_BINOP(And)
 _FORWARD_BINOP(Or)
 _FORWARD_BINOP(Xor)
+_FORWARD_BINOP(ShiftLeft)
+_FORWARD_BINOP(ShiftRightArithmetic)
+_FORWARD_BINOP(ShiftRightLogical)
 _FORWARD_UNOP(Not)
 _FORWARD_UNOP(Abs)
 _FORWARD_UNOP(Exp)
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 690ff277e8..17ad044578 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -333,6 +333,9 @@ class LocalComputationBuilder {
   _FORWARD_BINOP(And)
   _FORWARD_BINOP(Or)
   _FORWARD_BINOP(Xor)
+  _FORWARD_BINOP(ShiftLeft)
+  _FORWARD_BINOP(ShiftRightArithmetic)
+  _FORWARD_BINOP(ShiftRightLogical)
   _FORWARD_UNOP(Not)
   _FORWARD_UNOP(Abs)
   _FORWARD_UNOP(Exp)
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index afdea88cb7..42bf76e5d8 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -989,6 +989,9 @@ tensorflow::ImportNumpy();
 %unignore xla::swig::LocalComputationBuilder::And;
 %unignore xla::swig::LocalComputationBuilder::Or;
 %unignore xla::swig::LocalComputationBuilder::Xor;
+%unignore xla::swig::LocalComputationBuilder::ShiftLeft;
+%unignore xla::swig::LocalComputationBuilder::ShiftRightArithmetic;
+%unignore xla::swig::LocalComputationBuilder::ShiftRightLogical;
 %unignore xla::swig::LocalComputationBuilder::Not;
 %unignore xla::swig::LocalComputationBuilder::Abs;
 %unignore xla::swig::LocalComputationBuilder::Exp;
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index e2b6eaa096..f93d7bda2d 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -125,6 +125,9 @@ _BINARY_OPS = [
     'Or',
     'Xor',
     'Pow',
+    'ShiftLeft',
+    'ShiftRightArithmetic',
+    'ShiftRightLogical',
 ]
 
 
diff --git a/tensorflow/compiler/xla/python/xla_client_test.py b/tensorflow/compiler/xla/python/xla_client_test.py
index 0564ddcb85..93177aa647 100644
--- a/tensorflow/compiler/xla/python/xla_client_test.py
+++ b/tensorflow/compiler/xla/python/xla_client_test.py
@@ -171,6 +171,24 @@ class ComputationsWithConstantsTest(LocalComputationTest):
         c.Constant(NumpyArrayF32([[1, -1, 1], [-1, 1, -1]])))
     self._ExecuteAndCompareClose(c, expected=[[2, 1, 4], [3, 6, 5]])
 
+  def testShiftLeft(self):
+    c = self._NewComputation()
+    c.ShiftLeft(c.Constant(NumpyArrayS32([3])),
+                c.Constant(NumpyArrayS32([2])))
+    self._ExecuteAndCompareClose(c, expected=[12])
+
+  def testShiftRightArithmetic(self):
+    c = self._NewComputation()
+    c.ShiftRightArithmetic(c.Constant(NumpyArrayS32([-2])),
+                           c.Constant(NumpyArrayS32([1])))
+    self._ExecuteAndCompareClose(c, expected=[-1])
+
+  def testShiftRightLogical(self):
+    c = self._NewComputation()
+    c.ShiftRightLogical(c.Constant(NumpyArrayS32([-1])),
+                        c.Constant(NumpyArrayS32([1])))
+    self._ExecuteAndCompareClose(c, expected=[2**31 - 1])
+
   def testGetProto(self):
     c = self._NewComputation()
     c.Add(
-- 
cgit v1.2.3


From 5a5eb900a4fca0a2f523691af3de6381b4cca23a Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Wed, 18 Jul 2018 09:40:31 -0700
Subject: Fix tf_trt_integration_test in py3

---
 .../tensorrt/test/tf_trt_integration_test.py       | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py
index 7c3ef498c9..035b112254 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test.py
@@ -186,8 +186,8 @@ class TfTrtIntegrationTest(test_util.TensorFlowTestCase):
       # Defaults to 2 runs to verify result across multiple runs is same.
       for _ in range(num_runs):
         new_val = sess.run(out, {inp: input_data})
-        self.assertEquals(TEST_GRAPHS[graph_key].expected_output_dims,
-                          new_val.shape)
+        self.assertEqual(TEST_GRAPHS[graph_key].expected_output_dims,
+                         new_val.shape)
         if val is not None:
           self.assertAllEqual(new_val, val)
         val = new_val
@@ -220,19 +220,19 @@ class TfTrtIntegrationTest(test_util.TensorFlowTestCase):
     for n in gdef.node:
       if n.op == "TRTEngineOp":
         num_engines += 1
-        self.assertNotEqual("", n.attr["serialized_segment"].s)
-        self.assertNotEqual("", n.attr["segment_funcdef_name"].s)
-        self.assertEquals(n.attr["precision_mode"].s, precision_mode)
-        self.assertEquals(n.attr["static_engine"].b, not dynamic_engine)
+        self.assertNotEqual(to_bytes(""), n.attr["serialized_segment"].s)
+        self.assertNotEqual(to_bytes(""), n.attr["segment_funcdef_name"].s)
+        self.assertEqual(n.attr["precision_mode"].s, to_bytes(precision_mode))
+        self.assertEqual(n.attr["static_engine"].b, not dynamic_engine)
         if precision_mode == MODE_INT8 and is_calibrated:
-          self.assertNotEqual("", n.attr["calibration_data"].s)
+          self.assertNotEqual(to_bytes(""), n.attr["calibration_data"].s)
         else:
-          self.assertEquals("", n.attr["calibration_data"].s)
+          self.assertEqual(to_bytes(""), n.attr["calibration_data"].s)
     if precision_mode is None:
-      self.assertEquals(num_engines, 0)
+      self.assertEqual(num_engines, 0)
     else:
-      self.assertEquals(num_engines,
-                        TEST_GRAPHS[graph_key].num_expected_engines)
+      self.assertEqual(num_engines,
+                       TEST_GRAPHS[graph_key].num_expected_engines)
 
   def _RunTest(self, graph_key, use_optimizer, precision_mode,
                dynamic_infer_engine, dynamic_calib_engine):
-- 
cgit v1.2.3


From 748ef1c2eb58b0e031d796ea0211a8c6d74531ff Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 09:55:53 -0700
Subject: Add back in stddev parameter inadvertently dropped from conv model.
 Use create_eval_graph() for graph to be frozen.

PiperOrigin-RevId: 205093459
---
 tensorflow/examples/speech_commands/freeze.py | 2 +-
 tensorflow/examples/speech_commands/models.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/examples/speech_commands/freeze.py b/tensorflow/examples/speech_commands/freeze.py
index 7657b23c60..89e790d4e4 100644
--- a/tensorflow/examples/speech_commands/freeze.py
+++ b/tensorflow/examples/speech_commands/freeze.py
@@ -130,7 +130,7 @@ def main(_):
       FLAGS.clip_stride_ms, FLAGS.window_size_ms, FLAGS.window_stride_ms,
       FLAGS.feature_bin_count, FLAGS.model_architecture, FLAGS.preprocess)
   if FLAGS.quantize:
-    tf.contrib.quantize.create_training_graph(quant_delay=0)
+    tf.contrib.quantize.create_eval_graph()
   models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint)
 
   # Turn all the variables into inline constants inside the graph and save it.
diff --git a/tensorflow/examples/speech_commands/models.py b/tensorflow/examples/speech_commands/models.py
index 65ae3b1511..4d1454be0d 100644
--- a/tensorflow/examples/speech_commands/models.py
+++ b/tensorflow/examples/speech_commands/models.py
@@ -302,7 +302,7 @@ def create_conv_model(fingerprint_input, model_settings, is_training):
   label_count = model_settings['label_count']
   final_fc_weights = tf.get_variable(
       name='final_fc_weights',
-      initializer=tf.truncated_normal_initializer,
+      initializer=tf.truncated_normal_initializer(stddev=0.01),
       shape=[second_conv_element_count, label_count])
   final_fc_bias = tf.get_variable(
       name='final_fc_bias',
-- 
cgit v1.2.3


From 5f332577163010ed0de62c514992aa14db6b607c Mon Sep 17 00:00:00 2001
From: Rachel Lim 
Date: Wed, 18 Jul 2018 10:01:10 -0700
Subject: [tf.data] remove unnecessary build dependencies in
 grappler/optimizers/data

PiperOrigin-RevId: 205094150
---
 tensorflow/core/grappler/optimizers/data/BUILD          | 3 ---
 tensorflow/core/grappler/optimizers/data/graph_utils.cc | 4 ----
 2 files changed, 7 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD
index 3cb9d4d61c..c8946c499c 100644
--- a/tensorflow/core/grappler/optimizers/data/BUILD
+++ b/tensorflow/core/grappler/optimizers/data/BUILD
@@ -48,10 +48,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core/grappler:graph_view",
         "//tensorflow/core/grappler:grappler_item",
-        "//tensorflow/core/grappler:grappler_item_builder",
         "//tensorflow/core/grappler:utils",
-        "//tensorflow/core/grappler/clusters:virtual_cluster",
-        "//tensorflow/core/grappler/optimizers:meta_optimizer",
     ] + tf_protos_all(),
 )
 
diff --git a/tensorflow/core/grappler/optimizers/data/graph_utils.cc b/tensorflow/core/grappler/optimizers/data/graph_utils.cc
index b5b46ccafe..ea5f450009 100644
--- a/tensorflow/core/grappler/optimizers/data/graph_utils.cc
+++ b/tensorflow/core/grappler/optimizers/data/graph_utils.cc
@@ -16,11 +16,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/data/graph_utils.h"
 
 #include "tensorflow/core/framework/device_base.h"
-#include "tensorflow/core/grappler/clusters/virtual_cluster.h"
 #include "tensorflow/core/grappler/graph_view.h"
-#include "tensorflow/core/grappler/grappler_item.h"
-#include "tensorflow/core/grappler/grappler_item_builder.h"
-#include "tensorflow/core/grappler/optimizers/meta_optimizer.h"
 #include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
-- 
cgit v1.2.3


From 491b2d61156333c44e6bf06e2ac0a7ac02c4d310 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 10:15:59 -0700
Subject: Remove duplicated Log converter

PiperOrigin-RevId: 205096505
---
 tensorflow/contrib/lite/toco/import_tensorflow.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc
index 576eb71534..0d7eff5db4 100644
--- a/tensorflow/contrib/lite/toco/import_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc
@@ -1862,7 +1862,6 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Less", ConvertSimpleOperator},
       {"LessEqual", ConvertSimpleOperator},
       {"Log", ConvertSimpleOperator},
-      {"Log", ConvertSimpleOperator},
       {"LogSoftmax", ConvertSimpleOperator},
       {"MatMul", ConvertMatMulOperator},
       {"Max", ConvertReduceOperator},
-- 
cgit v1.2.3


From 9cc29a75ce8131db67b48e92dac3c16a255b92ed Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Wed, 18 Jul 2018 10:16:16 -0700
Subject: Allows constructing resource variables from tf.Variable.

Also adds arguments to control distributed aggregation to the tf.Variable constructor.

Removes tfe.Variable from examples as it's now unnecessary.

PiperOrigin-RevId: 205096552
---
 tensorflow/contrib/checkpoint/python/containers.py |  6 +-
 .../contrib/eager/python/examples/l2hmc/l2hmc.py   |  2 +-
 .../eager/python/examples/l2hmc/neural_nets.py     |  5 +-
 .../examples/notebooks/custom_training.ipynb       |  7 +-
 .../eager/python/examples/revnet/revnet_test.py    |  2 +-
 .../eager/python/examples/rnn_ptb/rnn_ptb.py       |  2 +-
 .../contrib/eager/python/examples/sagan/sagan.py   |  2 +-
 .../eager/python/examples/workshop/2_models.ipynb  |  2 +-
 tensorflow/contrib/eager/python/tfe_test.py        |  7 --
 .../base_api/api_def_ResourceScatterNdUpdate.pbtxt |  2 +-
 tensorflow/docs_src/guide/eager.md                 | 22 +++----
 .../kernel_tests/resource_variable_ops_test.py     |  9 +++
 tensorflow/python/ops/resource_variable_ops.py     | 13 ++++
 tensorflow/python/ops/variable_scope.py            | 33 +---------
 tensorflow/python/ops/variables.py                 | 75 ++++++++++++++++------
 .../tools/api/golden/tensorflow.-variable.pbtxt    |  2 +-
 third_party/examples/eager/spinn/spinn.py          |  2 +-
 17 files changed, 106 insertions(+), 87 deletions(-)

diff --git a/tensorflow/contrib/checkpoint/python/containers.py b/tensorflow/contrib/checkpoint/python/containers.py
index 4d3d531299..242c1e8ba4 100644
--- a/tensorflow/contrib/checkpoint/python/containers.py
+++ b/tensorflow/contrib/checkpoint/python/containers.py
@@ -35,9 +35,9 @@ class UniqueNameTracker(data_structures.CheckpointableDataStructure):
       self.slotdeps = tf.contrib.checkpoint.UniqueNameTracker()
       slotdeps = self.slotdeps
       slots = []
-      slots.append(slotdeps.track(tfe.Variable(3.), "x"))  # Named "x"
-      slots.append(slotdeps.track(tfe.Variable(4.), "y"))
-      slots.append(slotdeps.track(tfe.Variable(5.), "x"))  # Named "x_1"
+      slots.append(slotdeps.track(tf.Variable(3.), "x"))  # Named "x"
+      slots.append(slotdeps.track(tf.Variable(4.), "y"))
+      slots.append(slotdeps.track(tf.Variable(5.), "x"))  # Named "x_1"
   ```
   """
 
diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py b/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py
index 729d8525fa..275aee5130 100644
--- a/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py
+++ b/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py
@@ -54,7 +54,7 @@ class Dynamics(tf.keras.Model):
     self.position_fn = neural_nets.GenericNet(x_dim, factor=2.)
     self.momentum_fn = neural_nets.GenericNet(x_dim, factor=1.)
 
-    self.eps = tfe.Variable(
+    self.eps = tf.Variable(
         initial_value=eps, name="eps", dtype=tf.float32, trainable=True)
 
   def apply_transition(self, position):
diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/neural_nets.py b/tensorflow/contrib/eager/python/examples/l2hmc/neural_nets.py
index e230ad5e25..68e0bc3123 100644
--- a/tensorflow/contrib/eager/python/examples/l2hmc/neural_nets.py
+++ b/tensorflow/contrib/eager/python/examples/l2hmc/neural_nets.py
@@ -25,7 +25,6 @@ from __future__ import division
 from __future__ import print_function
 
 import tensorflow as tf
-import tensorflow.contrib.eager as tfe
 
 
 class GenericNet(tf.keras.Model):
@@ -47,13 +46,13 @@ class GenericNet(tf.keras.Model):
 
     # Scale
     self.scale_layer = _custom_dense(x_dim, .001)
-    self.coeff_scale = tfe.Variable(
+    self.coeff_scale = tf.Variable(
         initial_value=tf.zeros([1, x_dim]), name='coeff_scale', trainable=True)
     # Translation
     self.translation_layer = _custom_dense(x_dim, factor=.001)
     # Transformation
     self.transformation_layer = _custom_dense(x_dim, .001)
-    self.coeff_transformation = tfe.Variable(
+    self.coeff_transformation = tf.Variable(
         initial_value=tf.zeros([1, x_dim]),
         name='coeff_transformation',
         trainable=True)
diff --git a/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb
index 591e2d0c85..5f1b48fa0d 100644
--- a/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb
+++ b/tensorflow/contrib/eager/python/examples/notebooks/custom_training.ipynb
@@ -118,7 +118,6 @@
       "cell_type": "code",
       "source": [
         "import tensorflow as tf\n",
-        "tfe = tf.contrib.eager # Shorthand for some symbols\n",
         "\n",
         "tf.enable_eager_execution()"
       ],
@@ -184,7 +183,7 @@
       },
       "cell_type": "code",
       "source": [
-        "v = tfe.Variable(1.0)\n",
+        "v = tf.Variable(1.0)\n",
         "assert v.numpy() == 1.0\n",
         "\n",
         "# Re-assign the value\n",
@@ -258,8 +257,8 @@
         "  def __init__(self):\n",
         "    # Initialize variable to (5.0, 0.0)\n",
         "    # In practice, these should be initialized to random values.\n",
-        "    self.W = tfe.Variable(5.0)\n",
-        "    self.b = tfe.Variable(0.0)\n",
+        "    self.W = tf.Variable(5.0)\n",
+        "    self.b = tf.Variable(0.0)\n",
         "    \n",
         "  def __call__(self, x):\n",
         "    return self.W * x + self.b\n",
diff --git a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
index b2ac4b67c9..b0d0a5486d 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
@@ -138,7 +138,7 @@ class RevNetTest(tf.test.TestCase):
           minval=0,
           maxval=self.config.n_classes,
           dtype=tf.int32)
-      global_step = tfe.Variable(0., trainable=False)
+      global_step = tf.Variable(0., trainable=False)
       model = revnet.RevNet(config=config)
       model(x)
       updates = model.get_updates_for(x)
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
index c2340a293a..d64bf5354e 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
@@ -310,7 +310,7 @@ def main(_):
   with tf.device("/device:GPU:0" if have_gpu else None):
     # Make learning_rate a Variable so it can be included in the checkpoint
     # and we can resume training with the last saved learning_rate.
-    learning_rate = tfe.Variable(20.0, name="learning_rate")
+    learning_rate = tf.Variable(20.0, name="learning_rate")
     model = PTBModel(corpus.vocab_size(), FLAGS.embedding_dim,
                      FLAGS.hidden_dim, FLAGS.num_layers, FLAGS.dropout,
                      use_cudnn_rnn)
diff --git a/tensorflow/contrib/eager/python/examples/sagan/sagan.py b/tensorflow/contrib/eager/python/examples/sagan/sagan.py
index 561be36c91..8130414985 100644
--- a/tensorflow/contrib/eager/python/examples/sagan/sagan.py
+++ b/tensorflow/contrib/eager/python/examples/sagan/sagan.py
@@ -62,7 +62,7 @@ class SelfAttentionModule(tf.keras.Model):
         kernel_size=1,
         strides=(1, 1),
         data_format=data_format)
-    self.scale = tfe.Variable(0., trainable=True)
+    self.scale = tf.Variable(0., trainable=True)
 
   def call(self, x):
     f = self.f(x)
diff --git a/tensorflow/contrib/eager/python/examples/workshop/2_models.ipynb b/tensorflow/contrib/eager/python/examples/workshop/2_models.ipynb
index 4f1410e00b..f3a65f5aab 100644
--- a/tensorflow/contrib/eager/python/examples/workshop/2_models.ipynb
+++ b/tensorflow/contrib/eager/python/examples/workshop/2_models.ipynb
@@ -69,7 +69,7 @@
       "cell_type": "code",
       "source": [
         "# Creating variables\n",
-        "v = tfe.Variable(1.0)\n",
+        "v = tf.Variable(1.0)\n",
         "v"
       ],
       "execution_count": 2,
diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py
index db50b33af2..4454abfb96 100644
--- a/tensorflow/contrib/eager/python/tfe_test.py
+++ b/tensorflow/contrib/eager/python/tfe_test.py
@@ -27,7 +27,6 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import numerics
-from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.summary import summary
 from tensorflow.python.summary.writer import writer
@@ -45,12 +44,6 @@ class TFETest(test_util.TensorFlowTestCase):
                                  r'indices = 7 is not in \[0, 3\)'):
       array_ops.gather([0, 1, 2], 7)
 
-  def testVariableError(self):
-    with self.assertRaisesRegexp(
-        RuntimeError,
-        r'Variable not supported when eager execution is enabled'):
-      variables.Variable(initial_value=1.0)
-
   def testGradients(self):
 
     def square(x):
diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdUpdate.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdUpdate.pbtxt
index b07ee9fda9..17b79ee30c 100644
--- a/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdUpdate.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdUpdate.pbtxt
@@ -51,7 +51,7 @@ For example, say we want to update 4 scattered elements to a rank-1 tensor to
 8 elements. In Python, that update would look like this:
 
 ```python
-    ref = tfe.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+    ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
     indices = tf.constant([[4], [3], [1] ,[7]])
     updates = tf.constant([9, 10, 11, 12])
     update = tf.scatter_nd_update(ref, indices, updates)
diff --git a/tensorflow/docs_src/guide/eager.md b/tensorflow/docs_src/guide/eager.md
index e98206eef9..42ad9652f8 100644
--- a/tensorflow/docs_src/guide/eager.md
+++ b/tensorflow/docs_src/guide/eager.md
@@ -225,7 +225,7 @@ the tape backwards and then discard. A particular `tf.GradientTape` can only
 compute one gradient; subsequent calls throw a runtime error.
 
 ```py
-w = tfe.Variable([[1.0]])
+w = tf.Variable([[1.0]])
 with tf.GradientTape() as tape:
   loss = w * w
 
@@ -260,8 +260,8 @@ def grad(weights, biases):
 train_steps = 200
 learning_rate = 0.01
 # Start with arbitrary values for W and B on the same batch of data
-W = tfe.Variable(5.)
-B = tfe.Variable(10.)
+W = tf.Variable(5.)
+B = tf.Variable(10.)
 
 print("Initial loss: {:.3f}".format(loss(W, B)))
 
@@ -407,11 +407,11 @@ with tf.device("/gpu:0"):
 
 ### Variables and optimizers
 
-`tfe.Variable` objects store mutable `tf.Tensor` values accessed during
+`tf.Variable` objects store mutable `tf.Tensor` values accessed during
 training to make automatic differentiation easier. The parameters of a model can
 be encapsulated in classes as variables.
 
-Better encapsulate model parameters by using `tfe.Variable` with
+Better encapsulate model parameters by using `tf.Variable` with
 `tf.GradientTape`. For example, the automatic differentiation example above
 can be rewritten:
 
@@ -419,8 +419,8 @@ can be rewritten:
 class Model(tf.keras.Model):
   def __init__(self):
     super(Model, self).__init__()
-    self.W = tfe.Variable(5., name='weight')
-    self.B = tfe.Variable(10., name='bias')
+    self.W = tf.Variable(5., name='weight')
+    self.B = tf.Variable(10., name='bias')
   def call(self, inputs):
     return inputs * self.W + self.B
 
@@ -498,17 +498,17 @@ is removed, and is then deleted.
 
 ```py
 with tf.device("gpu:0"):
-  v = tfe.Variable(tf.random_normal([1000, 1000]))
+  v = tf.Variable(tf.random_normal([1000, 1000]))
   v = None  # v no longer takes up GPU memory
 ```
 
 ### Object-based saving
 
-`tfe.Checkpoint` can save and restore `tfe.Variable`s to and from
+`tfe.Checkpoint` can save and restore `tf.Variable`s to and from
 checkpoints:
 
 ```py
-x = tfe.Variable(10.)
+x = tf.Variable(10.)
 
 checkpoint = tfe.Checkpoint(x=x)  # save as "x"
 
@@ -612,7 +612,7 @@ def line_search_step(fn, init_x, rate=1.0):
 `tf.GradientTape` is a powerful interface for computing gradients, but there
 is another [Autograd](https://github.com/HIPS/autograd)-style API available for
 automatic differentiation. These functions are useful if writing math code with
-only tensors and gradient functions, and without `tfe.Variables`:
+only tensors and gradient functions, and without `tf.Variables`:
 
 * `tfe.gradients_function` —Returns a function that computes the derivatives
   of its input function parameter with respect to its arguments. The input
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index e358293a90..c739cd2c0d 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -246,6 +246,15 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
     self.assertEqual(self.evaluate(read), [[2]])
 
+  def testUseResource(self):
+    v = variables.Variable(1.0, use_resource=True)
+    self.assertTrue(isinstance(v, resource_variable_ops.ResourceVariable))
+
+  def testEagerNoUseResource(self):
+    with context.eager_mode():
+      v = variables.Variable(1.0)
+      self.assertTrue(isinstance(v, resource_variable_ops.ResourceVariable))
+
   @test_util.run_in_graph_and_eager_modes
   def testScatterMin(self):
     with ops.device("cpu:0"):
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 1f56ad25bf..5979b76ff2 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -1294,3 +1294,16 @@ def is_resource_variable(var):
   """"Returns True if `var` is to be considered a ResourceVariable."""
   return isinstance(var, ResourceVariable) or hasattr(
       var, "_should_act_as_resource_variable")
+
+
+_DEFAULT_USE_RESOURCE = False
+
+
+def _default_variable_creator(_, *args, **kwds):
+  use_resource = kwds.pop("use_resource", _DEFAULT_USE_RESOURCE)
+  use_resource = use_resource or context.executing_eagerly()
+  if use_resource:
+    return ResourceVariable(*args, **kwds)
+  return variables.RefVariable(*args, **kwds)
+
+variables.default_variable_creator = _default_variable_creator
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 77f67c18ee..0f37dcc027 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -191,36 +191,9 @@ class _ReuseMode(enum.Enum):
   # REUSE_TRUE = 3
 
 
-@tf_export("VariableSynchronization")
-class VariableSynchronization(enum.Enum):
-  """Indicates when a distributed variable will be synced."""
-
-  # Indicates that the synchronization will be determined by the current
-  # `DistributionStrategy` (eg. With `MirroredStrategy` this would be
-  # `ON_WRITE`).
-  AUTO = 0
-
-  # Indicates that there will only be one copy of the variable, so there is no
-  # need to sync.
-  NONE = 1
-
-  # Indicates that the variable will be aggregated across devices
-  # every time it is updated.
-  ON_WRITE = 2
-
-  # Indicates that the variable will be aggregated across devices
-  # when it is read (eg. when checkpointing or when evaluating an op that uses
-  # the variable).
-  ON_READ = 3
-
-
-@tf_export("VariableAggregation")
-class VariableAggregation(enum.Enum):
-  """Indicates how a distributed variable will be aggregated."""
-  NONE = 0
-  SUM = 1
-  MEAN = 2
-
+# TODO(apassos) remove these forwarding symbols.
+VariableSynchronization = variables.VariableSynchronization  # pylint: disable=invalid-name
+VariableAggregation = variables.VariableAggregation  # pylint: disable=invalid-name
 
 AUTO_REUSE = _ReuseMode.AUTO_REUSE
 tf_export("AUTO_REUSE").export_constant(__name__, "AUTO_REUSE")
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 87e0de197c..6bb2d6f669 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -17,6 +17,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import enum  # pylint: disable=g-bad-import-order
+
 import six
 
 from tensorflow.core.framework import attr_value_pb2
@@ -38,8 +40,9 @@ from tensorflow.python.util.deprecation import deprecated
 from tensorflow.python.util.tf_export import tf_export
 
 
-def _default_variable_creator(_, *args, **kwds):
-  return RefVariable(*args, **kwds)
+def default_variable_creator(_, *args, **kwds):
+  del args, kwds
+  raise NotImplementedError("resource_variable_ops needs to be imported")
 
 
 def _make_getter(captured_getter, captured_previous):
@@ -49,12 +52,43 @@ def _make_getter(captured_getter, captured_previous):
   return getter
 
 
+@tf_export("VariableSynchronization")
+class VariableSynchronization(enum.Enum):
+  """Indicates when a distributed variable will be synced."""
+
+  # Indicates that the synchronization will be determined by the current
+  # `DistributionStrategy` (eg. With `MirroredStrategy` this would be
+  # `ON_WRITE`).
+  AUTO = 0
+
+  # Indicates that there will only be one copy of the variable, so there is no
+  # need to sync.
+  NONE = 1
+
+  # Indicates that the variable will be aggregated across devices
+  # every time it is updated.
+  ON_WRITE = 2
+
+  # Indicates that the variable will be aggregated across devices
+  # when it is read (eg. when checkpointing or when evaluating an op that uses
+  # the variable).
+  ON_READ = 3
+
+
+@tf_export("VariableAggregation")
+class VariableAggregation(enum.Enum):
+  """Indicates how a distributed variable will be aggregated."""
+  NONE = 0
+  SUM = 1
+  MEAN = 2
+
+
 class VariableMetaclass(type):
   """Metaclass to allow construction of tf.Variable to be overridden."""
 
   def __call__(cls, *args, **kwargs):
     if cls is Variable:
-      previous_getter = lambda *a, **k: _default_variable_creator(None, *a, **k)
+      previous_getter = lambda *a, **k: default_variable_creator(None, *a, **k)
       # TODO(apassos) use a stack of getters here
       return previous_getter(*args, **kwargs)
     else:
@@ -172,14 +206,6 @@ class Variable(six.with_metaclass(VariableMetaclass,
   * Replace `tf.Variable` with `tf.contrib.eager.Variable`;
   * Call `tf.get_variable_scope().set_use_resource(True)` inside a
     `tf.variable_scope` before the `tf.get_variable()` call.
-
-  @compatibility(eager)
-  `tf.Variable` is not compatible with eager execution.  Use
-  `tf.contrib.eager.Variable` instead which is compatible with both eager
-  execution and graph construction.  See [the TensorFlow Eager Execution
-  guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
-  for details on how variables work in eager execution.
-  @end_compatibility
   """
 
   def __init__(self,
@@ -193,7 +219,10 @@ class Variable(six.with_metaclass(VariableMetaclass,
                dtype=None,
                expected_shape=None,
                import_scope=None,
-               constraint=None):
+               constraint=None,
+               use_resource=None,
+               synchronization=VariableSynchronization.AUTO,
+               aggregation=VariableAggregation.NONE):
     """Creates a new variable with value `initial_value`.
 
     The new variable is added to the graph collections listed in `collections`,
@@ -245,20 +274,24 @@ class Variable(six.with_metaclass(VariableMetaclass,
         variable and return the Tensor for the projected value
         (which must have the same shape). Constraints are not safe to
         use when doing asynchronous distributed training.
+      use_resource: if True, a ResourceVariable is created; otherwise an
+       old-style ref-based variable is created. When eager execution is enabled
+       a resource variable is always created.
+      synchronization: Indicates when a distributed a variable will be
+        aggregated. Accepted values are constants defined in the class
+        @{tf.VariableSynchronization}. By default the synchronization is set to
+        `AUTO` and the current `DistributionStrategy` chooses
+        when to synchronize. If `synchronization` is set to `ON_READ`,
+        `trainable` must not be set to `True`.
+      aggregation: Indicates how a distributed variable will be aggregated.
+        Accepted values are constants defined in the class
+        @{tf.VariableAggregation}.
 
     Raises:
       ValueError: If both `variable_def` and initial_value are specified.
       ValueError: If the initial value is not specified, or does not have a
         shape and `validate_shape` is `True`.
       RuntimeError: If eager execution is enabled.
-
-    @compatibility(eager)
-    `tf.Variable` is not compatible with eager execution.  Use
-    `tfe.Variable` instead which is compatible with both eager execution
-    and graph construction.  See [the TensorFlow Eager Execution
-    guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
-    for details on how variables work in eager execution.
-    @end_compatibility
     """
     raise NotImplementedError
 
@@ -1714,7 +1747,7 @@ class PartitionedVariable(object):
   """A container for partitioned `Variable` objects.
 
   @compatibility(eager) `tf.PartitionedVariable` is not compatible with
-  eager execution.  Use `tfe.Variable` instead which is compatible
+  eager execution.  Use `tf.Variable` instead which is compatible
   with both eager execution and graph construction.  See [the
   TensorFlow Eager Execution
   guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
diff --git a/tensorflow/tools/api/golden/tensorflow.-variable.pbtxt b/tensorflow/tools/api/golden/tensorflow.-variable.pbtxt
index 23b552cc38..e841c4ad89 100644
--- a/tensorflow/tools/api/golden/tensorflow.-variable.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.-variable.pbtxt
@@ -49,7 +49,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'initial_value\', \'trainable\', \'collections\', \'validate_shape\', \'caching_device\', \'name\', \'variable_def\', \'dtype\', \'expected_shape\', \'import_scope\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'initial_value\', \'trainable\', \'collections\', \'validate_shape\', \'caching_device\', \'name\', \'variable_def\', \'dtype\', \'expected_shape\', \'import_scope\', \'constraint\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'None\', \'True\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], "
   }
   member_method {
     name: "assign"
diff --git a/third_party/examples/eager/spinn/spinn.py b/third_party/examples/eager/spinn/spinn.py
index 67456a5bdf..c242ef3fdd 100644
--- a/third_party/examples/eager/spinn/spinn.py
+++ b/third_party/examples/eager/spinn/spinn.py
@@ -419,7 +419,7 @@ class SNLIClassifierTrainer(tfe.Checkpointable):
     # Create a custom learning rate Variable for the RMSProp optimizer, because
     # the learning rate needs to be manually decayed later (see
     # decay_learning_rate()).
-    self._learning_rate = tfe.Variable(lr, name="learning_rate")
+    self._learning_rate = tf.Variable(lr, name="learning_rate")
     self._optimizer = tf.train.RMSPropOptimizer(self._learning_rate,
                                                 epsilon=1e-6)
 
-- 
cgit v1.2.3


From 0e6bb6e3358a741bd995cb9b0055091c6b42a632 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Wed, 18 Jul 2018 10:25:37 -0700
Subject: Set plugin_converter_ and enable tests for custom_plugin_example.

---
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc     | 2 ++
 tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD | 5 ++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 65fef27533..49e825151a 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -2588,6 +2588,8 @@ void Converter::register_op_converters() {
   op_registry_["BatchMatMul"] = ConvertBatchMatMul;
   op_registry_["TopKV2"] = ConvertTopK;
 #endif
+
+  plugin_converter_ = ConvertPlugin;
 }
 
 }  // namespace
diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
index a89cf3ab8b..1ef1c3de75 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
@@ -111,8 +111,7 @@ cuda_py_test(
         "//tensorflow/python:tf_optimizer",
     ],
     tags = [
-        "manual",
-        "noguitar",
-        "notap",
+        "no_windows",
+        "nomac",
     ],
 )
-- 
cgit v1.2.3


From ec85fc632651324cb674793ae9741fb9a9a9c4f6 Mon Sep 17 00:00:00 2001
From: Andrew Selle 
Date: Wed, 18 Jul 2018 10:25:51 -0700
Subject: Correct exception handling in TFLite Python interpreter.

We were incorrectly returning nullptr all the time. We need
to return None sometimes and check for it.

PiperOrigin-RevId: 205098110
---
 .../interpreter_wrapper/interpreter_wrapper.cc     | 23 +++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
index c38b692dcd..f97919363b 100644
--- a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
+++ b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
@@ -340,6 +340,8 @@ PyObject* InterpreterWrapper::SetTensor(int i, PyObject* value) {
 
 namespace {
 
+// Checks to see if a tensor access can succeed (returns nullptr on error).
+// Otherwise returns Py_None.
 PyObject* CheckGetTensorArgs(Interpreter* interpreter_, int tensor_index,
                              TfLiteTensor** tensor, int* type_num) {
   TFLITE_PY_ENSURE_VALID_INTERPRETER();
@@ -362,7 +364,7 @@ PyObject* CheckGetTensorArgs(Interpreter* interpreter_, int tensor_index,
     return nullptr;
   }
 
-  return nullptr;
+  Py_RETURN_NONE;
 }
 
 }  // namespace
@@ -371,10 +373,12 @@ PyObject* InterpreterWrapper::GetTensor(int i) const {
   // Sanity check accessor
   TfLiteTensor* tensor = nullptr;
   int type_num = 0;
-  if (PyObject* pynone_or_nullptr =
-          CheckGetTensorArgs(interpreter_.get(), i, &tensor, &type_num)) {
-    return pynone_or_nullptr;
-  }
+
+  PyObject* check_result =
+      CheckGetTensorArgs(interpreter_.get(), i, &tensor, &type_num);
+  if (check_result == nullptr) return check_result;
+  Py_XDECREF(check_result);
+
   std::vector dims(tensor->dims->data,
                              tensor->dims->data + tensor->dims->size);
   // Make a buffer copy but we must tell Numpy It owns that data or else
@@ -396,10 +400,11 @@ PyObject* InterpreterWrapper::tensor(PyObject* base_object, int i) {
   // Sanity check accessor
   TfLiteTensor* tensor = nullptr;
   int type_num = 0;
-  if (PyObject* pynone_or_nullptr =
-          CheckGetTensorArgs(interpreter_.get(), i, &tensor, &type_num)) {
-    return pynone_or_nullptr;
-  }
+
+  PyObject* check_result =
+      CheckGetTensorArgs(interpreter_.get(), i, &tensor, &type_num);
+  if (check_result == nullptr) return check_result;
+  Py_XDECREF(check_result);
 
   std::vector dims(tensor->dims->data,
                              tensor->dims->data + tensor->dims->size);
-- 
cgit v1.2.3


From 2a0958455799601068db054c130fa9573e7c1e22 Mon Sep 17 00:00:00 2001
From: Michael Case 
Date: Wed, 18 Jul 2018 10:37:01 -0700
Subject: Remove usage of remove_undocumented from core parallel_for.

remove_undocumented is causing issues with our pip tests.
remove_undocumented is not used anywhere else in core TF code
and we have a new mechanism for annotating the public TF API.
---
 tensorflow/python/ops/parallel_for/__init__.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/tensorflow/python/ops/parallel_for/__init__.py b/tensorflow/python/ops/parallel_for/__init__.py
index b49d865968..dd8bc6d487 100644
--- a/tensorflow/python/ops/parallel_for/__init__.py
+++ b/tensorflow/python/ops/parallel_for/__init__.py
@@ -23,13 +23,3 @@ from tensorflow.python.ops.parallel_for.control_flow_ops import for_loop
 from tensorflow.python.ops.parallel_for.control_flow_ops import pfor
 from tensorflow.python.ops.parallel_for.gradients import batch_jacobian
 from tensorflow.python.ops.parallel_for.gradients import jacobian
-from tensorflow.python.util.all_util import remove_undocumented
-
-_allowed_symbols = [
-    'pfor',
-    'for_loop',
-    'jacobian',
-    'batch_jacobian',
-]
-
-remove_undocumented(__name__, _allowed_symbols)
-- 
cgit v1.2.3


From 34b588e35dae4f36ce205b713bc0a47e98097585 Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Wed, 18 Jul 2018 17:37:45 +0000
Subject: Add additional shape validation for QuantizedAdd

This fix add additional shape validation for QuantizedAdd
with min_x, min_y, max_x, max_y.

Additional unit tests have been added in math_ops_test.cc.

Signed-off-by: Yong Tang 
---
 tensorflow/core/ops/math_ops.cc      |  7 +++++++
 tensorflow/core/ops/math_ops_test.cc | 15 +++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index 4b0591c6e8..386ae9635a 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -1504,6 +1504,13 @@ REGISTER_OP("QuantizedAdd")
     .SetIsCommutative()
     .SetShapeFn([](InferenceContext* c) {
       TF_RETURN_IF_ERROR(shape_inference::BroadcastBinaryOpShapeFn(c));
+      // min_x, max_x, min_y, max_y should be scalar.
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused));
+
       c->set_output(1, c->Scalar());
       c->set_output(2, c->Scalar());
       return Status::OK();
diff --git a/tensorflow/core/ops/math_ops_test.cc b/tensorflow/core/ops/math_ops_test.cc
index 25dc033065..23f1538912 100644
--- a/tensorflow/core/ops/math_ops_test.cc
+++ b/tensorflow/core/ops/math_ops_test.cc
@@ -543,4 +543,19 @@ TEST(MathOpsTest, HistogramFixedWidth_ShapeFn) {
   INFER_OK(op, "[?];[2];[]", "[?]");
   INFER_OK(op, "[?];[2];?", "[?]");
 }
+
+TEST(MathOpsTest, QuantizedAdd_ShapeFn) {
+  ShapeInferenceTestOp op("QuantizedAdd");
+
+  INFER_OK(op, "?;?;?;?;?;?", "?;[];[]");
+  INFER_OK(op, "?;?;[];[];[];[]", "?;[];[]");
+  INFER_OK(op, "[1,2];?;[];[];[];[]", "?;[];[]");
+  INFER_OK(op, "[];[2];[];[];[];[]", "[d1_0];[];[]");
+
+  // Rank checks on input scalars.
+  INFER_ERROR("must be rank 0", op, "?;?;[1];?;?;?");
+  INFER_ERROR("must be rank 0", op, "?;?;?;[2];?;?");
+  INFER_ERROR("must be rank 0", op, "?;?;?;?;[3];?");
+  INFER_ERROR("must be rank 0", op, "?;?;?;?;?;[4]");
+}
 }  // end namespace tensorflow
-- 
cgit v1.2.3


From bc8fd55cac9f10716b92883b83a36d54a1d7047b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 10:39:03 -0700
Subject: Add 'eager' namespace.

PiperOrigin-RevId: 205100305
---
 tensorflow/contrib/lite/delegates/eager/buffer_map.cc         | 2 ++
 tensorflow/contrib/lite/delegates/eager/buffer_map.h          | 2 ++
 tensorflow/contrib/lite/delegates/eager/buffer_map_test.cc    | 2 ++
 tensorflow/contrib/lite/delegates/eager/delegate_data.cc      | 2 ++
 tensorflow/contrib/lite/delegates/eager/delegate_data.h       | 2 ++
 tensorflow/contrib/lite/delegates/eager/delegate_data_test.cc | 2 ++
 tensorflow/contrib/lite/delegates/eager/util.cc               | 2 ++
 tensorflow/contrib/lite/delegates/eager/util.h                | 2 ++
 tensorflow/contrib/lite/delegates/eager/util_test.cc          | 2 ++
 9 files changed, 18 insertions(+)

diff --git a/tensorflow/contrib/lite/delegates/eager/buffer_map.cc b/tensorflow/contrib/lite/delegates/eager/buffer_map.cc
index e4a780b735..1d6453f498 100644
--- a/tensorflow/contrib/lite/delegates/eager/buffer_map.cc
+++ b/tensorflow/contrib/lite/delegates/eager/buffer_map.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/log_memory.h"
 
 namespace tflite {
+namespace eager {
 namespace {
 // A tensor buffer that is allocated, deallocated and populated by TF Lite.
 class TfLiteTensorBuffer : public tensorflow::TensorBuffer {
@@ -102,4 +103,5 @@ void BufferMap::SetFromTensorFlow(int tensor_index, tensorflow::Tensor tensor) {
   id_to_tensor_[tensor_index] = std::move(tensor);
 }
 
+}  // namespace eager
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/delegates/eager/buffer_map.h b/tensorflow/contrib/lite/delegates/eager/buffer_map.h
index 922f67f574..a28329ae7d 100644
--- a/tensorflow/contrib/lite/delegates/eager/buffer_map.h
+++ b/tensorflow/contrib/lite/delegates/eager/buffer_map.h
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 
 namespace tflite {
+namespace eager {
 
 // Maps a TF Lite tensor index into a TensorFlow tensor.
 //
@@ -54,6 +55,7 @@ class BufferMap {
   std::map id_to_tensor_;
 };
 
+}  // namespace eager
 }  // namespace tflite
 
 #endif  // TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_BUFFER_MAP_H_
diff --git a/tensorflow/contrib/lite/delegates/eager/buffer_map_test.cc b/tensorflow/contrib/lite/delegates/eager/buffer_map_test.cc
index c447eeaa05..dcb3f6c941 100644
--- a/tensorflow/contrib/lite/delegates/eager/buffer_map_test.cc
+++ b/tensorflow/contrib/lite/delegates/eager/buffer_map_test.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/contrib/lite/util.h"
 
 namespace tflite {
+namespace eager {
 namespace {
 
 using ::testing::ElementsAre;
@@ -163,6 +164,7 @@ TEST(BufferMapTest, TensorFlowOverwritesTfLite) {
 }
 
 }  // namespace
+}  // namespace eager
 }  // namespace tflite
 
 int main(int argc, char** argv) {
diff --git a/tensorflow/contrib/lite/delegates/eager/delegate_data.cc b/tensorflow/contrib/lite/delegates/eager/delegate_data.cc
index b2516379e7..29687694bd 100644
--- a/tensorflow/contrib/lite/delegates/eager/delegate_data.cc
+++ b/tensorflow/contrib/lite/delegates/eager/delegate_data.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tflite {
+namespace eager {
 tensorflow::Status DelegateData::Create(std::unique_ptr* data) {
   std::vector devices;
 
@@ -41,4 +42,5 @@ DelegateData::DelegateData(tensorflow::EagerContext* eager_context)
 
 DelegateData::~DelegateData() {}
 
+}  // namespace eager
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/delegates/eager/delegate_data.h b/tensorflow/contrib/lite/delegates/eager/delegate_data.h
index 053d174c08..8a0e8ba8bf 100644
--- a/tensorflow/contrib/lite/delegates/eager/delegate_data.h
+++ b/tensorflow/contrib/lite/delegates/eager/delegate_data.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/context.h"
 
 namespace tflite {
+namespace eager {
 
 // Data kept by the Eager delegate for the lifetime of an Interpreter.
 class DelegateData {
@@ -41,6 +42,7 @@ class DelegateData {
   BufferMap buffer_map_;
 };
 
+}  // namespace eager
 }  // namespace tflite
 
 #endif  // TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_DELEGATE_DATA_H_
diff --git a/tensorflow/contrib/lite/delegates/eager/delegate_data_test.cc b/tensorflow/contrib/lite/delegates/eager/delegate_data_test.cc
index cf8bc27d04..30251b8f82 100644
--- a/tensorflow/contrib/lite/delegates/eager/delegate_data_test.cc
+++ b/tensorflow/contrib/lite/delegates/eager/delegate_data_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/contrib/lite/testing/util.h"
 
 namespace tflite {
+namespace eager {
 namespace {
 
 TEST(DelegateDataTest, Basic) {
@@ -33,6 +34,7 @@ TEST(DelegateDataTest, Basic) {
 }
 
 }  // namespace
+}  // namespace eager
 }  // namespace tflite
 
 int main(int argc, char** argv) {
diff --git a/tensorflow/contrib/lite/delegates/eager/util.cc b/tensorflow/contrib/lite/delegates/eager/util.cc
index e1879bdaff..4426c653e6 100644
--- a/tensorflow/contrib/lite/delegates/eager/util.cc
+++ b/tensorflow/contrib/lite/delegates/eager/util.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include "tensorflow/contrib/lite/delegates/eager/util.h"
 
 namespace tflite {
+namespace eager {
 
 TfLiteStatus ConvertStatus(TfLiteContext* context,
                            const tensorflow::Status& status) {
@@ -67,4 +68,5 @@ TF_DataType GetTensorFlowDataType(TfLiteType type) {
   }
 }
 
+}  // namespace eager
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/delegates/eager/util.h b/tensorflow/contrib/lite/delegates/eager/util.h
index 12b33b9b49..a9407be071 100644
--- a/tensorflow/contrib/lite/delegates/eager/util.h
+++ b/tensorflow/contrib/lite/delegates/eager/util.h
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tflite {
+namespace eager {
 
 // Converts a tensorflow:Status into a TfLiteStatus. If the original status
 // represented an error, reports it using the given 'context'.
@@ -35,6 +36,7 @@ TfLiteStatus CopyShape(TfLiteContext* context, const tensorflow::Tensor& src,
 // Returns the TF C API Data type that corresponds to the given TfLiteType.
 TF_DataType GetTensorFlowDataType(TfLiteType type);
 
+}  // namespace eager
 }  // namespace tflite
 
 #endif  // TENSORFLOW_CONTRIB_LITE_DELEGATES_EAGER_UTIL_H_
diff --git a/tensorflow/contrib/lite/delegates/eager/util_test.cc b/tensorflow/contrib/lite/delegates/eager/util_test.cc
index 53ed4db972..c4fbf54127 100644
--- a/tensorflow/contrib/lite/delegates/eager/util_test.cc
+++ b/tensorflow/contrib/lite/delegates/eager/util_test.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/contrib/lite/testing/util.h"
 
 namespace tflite {
+namespace eager {
 namespace {
 
 using tensorflow::DT_FLOAT;
@@ -102,6 +103,7 @@ TEST(UtilTest, TypeConversions) {
 }
 
 }  // namespace
+}  // namespace eager
 }  // namespace tflite
 
 int main(int argc, char** argv) {
-- 
cgit v1.2.3


From fb70a5587395b1e68c08a4d396c63c5bd80fa1e1 Mon Sep 17 00:00:00 2001
From: Youlong Cheng 
Date: Wed, 18 Jul 2018 10:50:45 -0700
Subject: PUBLIC: Remove redundant set_configuration_from_sharded_input_tensors
 in input_pipeline and fix a bug for broadcast mode. RELNOTES: n/a

PiperOrigin-RevId: 205102082
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 718ea630a8..78b79b111e 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -701,8 +701,6 @@ def generate_per_core_enqueue_ops_fn_for_host(
     infeed_queue = tpu_feed.InfeedQueue(
         number_of_tuple_elements=len(per_host_sharded_inputs[0]))
     captured_infeed_queue.capture(infeed_queue)
-    infeed_queue.set_configuration_from_sharded_input_tensors(
-        per_host_sharded_inputs)
 
     per_host_enqueue_ops = infeed_queue.generate_enqueue_ops(
         per_host_sharded_inputs, tpu_ordinal_function=tpu_ordinal_function_impl)
@@ -837,8 +835,6 @@ def generate_per_host_v2_enqueue_ops_fn_for_host(
     infeed_queue = tpu_feed.InfeedQueue(
         number_of_tuple_elements=len(per_host_sharded_inputs[0]))
     captured_infeed_queue.capture(infeed_queue)
-    infeed_queue.set_configuration_from_sharded_input_tensors(
-        per_host_sharded_inputs)
 
     per_host_enqueue_ops = infeed_queue.generate_enqueue_ops(
         per_host_sharded_inputs, tpu_ordinal_function=tpu_ordinal_function_impl)
@@ -867,7 +863,7 @@ def generate_broadcast_enqueue_ops_fn(ctx, input_fn, inputs_structure_recorder,
 
   def tpu_ordinal_function_impl(replica_id):
     if ctx.device_assignment:
-      return ctx.device_assignment.tpu_ordinal(replica_id=replica_id)
+      return ctx.device_assignment.tpu_ordinal(replica=replica_id)
     else:
       return replica_id % num_replicas_per_host
 
-- 
cgit v1.2.3


From 80ab99ec746520faa763a7bb171ee8850a597ec1 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay 
Date: Wed, 18 Jul 2018 10:55:32 -0700
Subject: Create new metrics class and add mean metric.

PiperOrigin-RevId: 205102847
---
 tensorflow/python/keras/metrics.py      | 381 ++++++++++++++++++++++++++++++++
 tensorflow/python/keras/metrics_test.py | 196 +++++++++++++---
 2 files changed, 541 insertions(+), 36 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index e03d7dfe93..72e15763cb 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -19,9 +19,16 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from abc import ABCMeta
+from abc import abstractmethod
 import six
 
+from tensorflow.python.eager import context
+from tensorflow.python.eager import function
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.keras import backend as K
+from tensorflow.python.keras.engine.base_layer import Layer
 from tensorflow.python.keras.losses import binary_crossentropy
 from tensorflow.python.keras.losses import categorical_crossentropy
 from tensorflow.python.keras.losses import cosine_proximity
@@ -37,11 +44,385 @@ from tensorflow.python.keras.losses import sparse_categorical_crossentropy
 from tensorflow.python.keras.losses import squared_hinge
 from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras.utils.generic_utils import serialize_keras_object
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import confusion_matrix
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.ops import weights_broadcast_ops
+from tensorflow.python.training import distribute as distribute_lib
+from tensorflow.python.util import tf_decorator
 from tensorflow.python.util.tf_export import tf_export
 
 
+def update_state(update_state_fn):
+  """Decorator to wrap metric `update_state()` with `defun()`, `add_update()`.
+
+  Args:
+    update_state_fn: function that accumulates metric statistics.
+
+  Returns:
+    If eager execution is enabled, returns None.
+    If graph execution is enabled, returns an update op. This op should be
+      executed to update the metric state with the given inputs.
+  """
+
+  def decorated(*args, **kwargs):
+    """Decorated function with `defun()` and `add_update()`."""
+
+    # Converting update_state_fn() into a graph function, so that
+    # we can return a single op that performs all of the variable updates.
+    # Assigning to a different method name to avoid reference cycle.
+    defuned_update_state_fn = function.defun(update_state_fn)
+    update_op = defuned_update_state_fn(*args, **kwargs)
+    if update_op is not None:  # update_op will be None in eager execution.
+      metric_obj = args[0]
+      metric_obj.add_update(update_op, inputs=True)
+    return update_op
+
+  return tf_decorator.make_decorator(update_state_fn, decorated)
+
+
+def result(result_fn):
+  """Decorator to wrap metric `result()` function in `merge_call()`.
+
+  Result computation is an idempotent operation that simply calculates the
+  metric value using the state variables.
+
+  If metric state variables are distributed across towers/devices and
+  `result()` is requested from the context of one device - This function wraps
+  `result()` in a distribution strategy `merge_call()`. With this,
+  the metric state variables will be aggregated across devices.
+
+  Args:
+    result_fn: function that computes the metric result.
+
+  Returns:
+    The metric result tensor.
+  """
+
+  def decorated(*args):
+    """Decorated function with merge_call."""
+    tower_context = distribute_lib.get_tower_context()
+    if tower_context is None:  # if in cross tower context already
+      return result_fn()
+
+    # TODO(psv): Test distribution of metrics using different distribution
+    # strategies.
+
+    # Creating a wrapper for merge_fn. merge_call invokes the given merge_fn
+    # with distribution object as the first parameter. We create a wrapper here
+    # so that the result function need not have that parameter.
+    def merge_fn_wrapper(distribution, merge_fn, *args):
+      # We will get `PerDevice` merge function. Taking the first one as all are
+      # identical copies of the function that we had passed below.
+      return distribution.unwrap(merge_fn)[0](*args)
+
+    # Wrapping result in merge_call. merge_call is used when we want to leave
+    # tower mode and compute a value in cross tower mode.
+    return tower_context.merge_call(merge_fn_wrapper, result_fn, *args)
+
+  return tf_decorator.make_decorator(result_fn, decorated)
+
+
+def _safe_div(numerator, denominator):
+  """Divides two tensors element-wise, returning 0 if the denominator is <= 0.
+
+  Args:
+    numerator: A `Tensor`.
+    denominator: A `Tensor`, with dtype matching `numerator`.
+
+  Returns:
+    0 if `denominator` <= 0, else `numerator` / `denominator`
+  """
+  t = math_ops.truediv(numerator, denominator)
+  zero = array_ops.zeros_like(t, dtype=denominator.dtype)
+  condition = math_ops.greater(denominator, zero)
+  zero = math_ops.cast(zero, t.dtype)
+  return array_ops.where(condition, t, zero)
+
+
+def _squeeze_or_expand_dimensions(y_pred, y_true, sample_weight):
+  """Squeeze or expand last dimension if needed.
+
+  1. Squeezes last dim of `y_pred` or `y_true` if their rank differs by 1
+  (using `confusion_matrix.remove_squeezable_dimensions`).
+  2. Squeezes or expands last dim of `sample_weight` if its rank differs by 1
+  from the new rank of `y_pred`.
+  If `sample_weight` is scalar, it is kept scalar.
+
+  This will use static shape if available. Otherwise, it will add graph
+  operations, which could result in a performance hit.
+
+  Args:
+    y_pred: Predicted values, a `Tensor` of arbitrary dimensions.
+    y_true: Optional label `Tensor` whose dimensions match `y_pred`.
+    sample_weight: Optional weight scalar or `Tensor` whose dimensions match
+      `y_pred`.
+
+  Returns:
+    Tuple of `y_pred`, `y_true` and `sample_weight`. Each of them possibly has
+    the last dimension squeezed,
+    `sample_weight` could be extended by one dimension.
+  """
+  if y_true is not None:
+    # squeeze last dim of `y_pred` or `y_true` if their rank differs by 1
+    y_true, y_pred = confusion_matrix.remove_squeezable_dimensions(
+        y_true, y_pred)
+    y_pred.get_shape().assert_is_compatible_with(y_true.get_shape())
+
+  if sample_weight is None:
+    return y_pred, y_true, None
+
+  sample_weight = ops.convert_to_tensor(sample_weight)
+  weights_shape = sample_weight.get_shape()
+  weights_rank = weights_shape.ndims
+  if weights_rank == 0:  # If weights is scalar, do nothing.
+    return y_pred, y_true, sample_weight
+
+  y_pred_shape = y_pred.get_shape()
+  y_pred_rank = y_pred_shape.ndims
+  if (y_pred_rank is not None) and (weights_rank is not None):
+    # Use static rank.
+    if weights_rank - y_pred_rank == 1:
+      sample_weight = array_ops.squeeze(sample_weight, [-1])
+    elif y_pred_rank - weights_rank == 1:
+      sample_weight = array_ops.expand_dims(sample_weight, [-1])
+    return y_pred, y_true, sample_weight
+
+  # Use dynamic rank.
+  weights_rank_tensor = array_ops.rank(sample_weight)
+  rank_diff = weights_rank_tensor - array_ops.rank(y_pred)
+  maybe_squeeze_weights = lambda: array_ops.squeeze(sample_weight, [-1])
+
+  def _maybe_expand_weights():
+    return control_flow_ops.cond(
+        math_ops.equal(rank_diff,
+                       -1), lambda: array_ops.expand_dims(sample_weight, [-1]),
+        lambda: sample_weight)
+
+  def _maybe_adjust_weights():
+    return control_flow_ops.cond(
+        math_ops.equal(rank_diff, 1), maybe_squeeze_weights,
+        _maybe_expand_weights)
+
+  # squeeze or expand last dim of `sample_weight` if its rank differs by 1
+  # from the new rank of `y_pred`.
+  sample_weight = control_flow_ops.cond(
+      math_ops.equal(weights_rank_tensor, 0), lambda: sample_weight,
+      _maybe_adjust_weights)
+  return y_pred, y_true, sample_weight
+
+
+class Metric(Layer):
+  """Encapsulates metric logic and state.
+
+  Usage with eager execution:
+
+  ```python
+  m = SomeMetric(...)
+  for input in ...:
+    m.update_state(input)
+  print('Final result: ', m.result().numpy())
+  ```
+
+  Usage with graph execution:
+
+  ```python
+  m = SomeMetric(...)
+  init_op = tf.global_variables_initializer()  # Initialize variables
+  with tf.Session() as sess:
+    sess.run(init_op)
+    for input in ...:
+      update_op = m.update_state(input)
+      sess.run(update_op)
+    print('Final result: ', sess.run(m.result()))
+  ```
+
+  To be implemented by subclasses:
+  * `__init__()`: All state variables should be created in this method by
+    calling `self.add_weight()` like: `self.var = self.add_weight(...)`
+  * `update_state()`: Has all updates to the state variables like:
+    self.var.assign_add(...). Please decorate the function with:
+    @update_state: Converts `update_state()` into a graph function, so that
+    we can return a single op that performs all of the variable updates and
+    adds the update op to the metric layer.
+  * `result()`: Computes and returns a value for the metric
+    from the state variables. Please decorate the function with:
+    @result: Wraps `result()` in a distribution strategy merge_call().
+
+  Example subclass implementation:
+
+  ```
+  class BinaryTruePositives(Metric):
+    def __init__(self, name='binary-true-positives', dtype=dtypes.float64):
+      super(BinaryTruePositives, self).__init__(name=name, dtype=dtype)
+      self.true_positives = self.add_weight(
+          'true_positives', initializer=init_ops.zeros_initializer)
+
+    @update_state
+    def update_state(self, y_true, y_pred, sample_weight=None):
+      y_true = math_ops.cast(y_true, dtypes.bool)
+      y_pred = math_ops.cast(y_pred, dtypes.bool)
+      y_pred, y_true, sample_weight = _squeeze_or_expand_dimensions(
+          y_pred, y_true, sample_weight)
+
+      values = math_ops.logical_and(
+          math_ops.equal(y_true, True), math_ops.equal(y_pred, True))
+      values = math_ops.cast(values, self._dtype)
+      if sample_weight is not None:
+        sample_weight = math_ops.cast(sample_weight, self._dtype)
+        values = math_ops.multiply(values, sample_weight)
+      state_ops.assign_add(self.true_positives, math_ops.reduce_sum(values))
+
+    @result
+    def result(self):
+      return array_ops.identity(self.true_positives)
+  ```
+  """
+  __metaclass__ = ABCMeta
+
+  def __init__(self, name=None, dtype=dtypes.float64):
+    super(Metric, self).__init__(name=name, dtype=dtype)
+    self.stateful = True  # All metric layers are stateful.
+    self.built = True
+
+  def __call__(self, *args, **kwargs):
+    """Accumulates statistics and then computes metric result value.
+
+    Args:
+      *args:
+      **kwargs: A mini-batch of inputs to the Metric,
+        passed on to `update_state()`.
+
+    Returns:
+      The metric value tensor.
+    """
+    update_op = self.update_state(*args, **kwargs)
+    with ops.control_dependencies([update_op]):
+      return self.result()
+
+  def reset_states(self):
+    """Resets all of the metric state variables.
+
+    This function is called between epochs/steps,
+    when a metric is evaluated during training.
+    """
+    for v in self.variables:
+      K.set_value(v, 0)
+
+  @abstractmethod
+  def update_state(self, *args, **kwargs):
+    """Accumulates statistics for the metric.
+
+    Please decorate the function with:
+    @update_state: Converts `update_state()` into a graph function, so that
+    we can return a single op that performs all of the variable updates
+      This means:
+      a) Operations on the same resource are executed in textual order.
+         This should make it easier to do things like add the updated
+         value of a variable to another, for example.
+      b) You don't need to worry about collecting the update ops to execute.
+         All update ops added to the graph by this function will be executed.
+      As a result, code should generally work the same way with graph or
+      eager execution.
+    and adds the update op to the metric layer.
+
+    Args:
+      *args:
+      **kwargs: A mini-batch of inputs to the Metric.
+    """
+    NotImplementedError('Must be implemented in subclasses.')
+
+  @abstractmethod
+  def result(self):
+    """Computes and returns the metric value tensor.
+
+    Result computation is an idempotent operation that simply calculates the
+    metric value using the state variables.
+
+    Please decorate the function with:
+    @result: Wraps `result()` in a distribution strategy merge_call().
+    """
+    NotImplementedError('Must be implemented in subclasses.')
+
+  ### For use by subclasses ###
+  def add_weight(self,
+                 name,
+                 shape=(),
+                 aggregation=vs.VariableAggregation.SUM,
+                 synchronization=vs.VariableSynchronization.ON_READ,
+                 initializer=None):
+    """Adds state variable. Only for use by subclasses."""
+    return super(Metric, self).add_weight(
+        name=name,
+        shape=shape,
+        dtype=self._dtype,
+        trainable=False,
+        initializer=initializer,
+        synchronization=synchronization,
+        aggregation=aggregation)
+
+  ### End: For use by subclasses ###
+
+
+class Mean(Metric):
+  """Computes the (weighted) mean of the given values.
+
+  This metric creates two variables, `total` and `count` that are used to
+  compute the average of `values`. This average is ultimately returned as `mean`
+  which is an idempotent operation that simply divides `total` by `count`.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+  """
+
+  def __init__(self, name='mean', dtype=dtypes.float64):
+    super(Mean, self).__init__(name=name, dtype=dtype)
+    # Create new state variables
+    self.total = self.add_weight(
+        'total', initializer=init_ops.zeros_initializer)
+    self.count = self.add_weight(
+        'count', initializer=init_ops.zeros_initializer)
+
+  @update_state
+  def update_state(self, values, sample_weight=None):
+    """Accumulates statistics for computing the mean.
+
+    For example, if `values` is [1, 3, 5, 7] then the mean is 4. If
+    the `sample_weight` is specified as [1, 1, 0, 0] then the mean would be 2.
+
+    Args:
+      values: Per-example value.
+      sample_weight: Optional weighting of each example. Defaults to 1.
+    """
+    values = math_ops.cast(values, self._dtype)
+    if sample_weight is None:
+      num_values = math_ops.cast(array_ops.size(values), self._dtype)
+    else:
+      sample_weight = math_ops.cast(sample_weight, self._dtype)
+
+      # Update dimensions of weights to match with values.
+      values, _, sample_weight = _squeeze_or_expand_dimensions(
+          values, None, sample_weight)
+      sample_weight = weights_broadcast_ops.broadcast_weights(
+          sample_weight, values)
+      num_values = math_ops.reduce_sum(sample_weight)
+      values = math_ops.multiply(values, sample_weight)
+    values = math_ops.reduce_sum(values)
+
+    # Update state variables
+    state_ops.assign_add(self.total, values)
+    state_ops.assign_add(self.count, num_values)
+
+  @result
+  def result(self):
+    return _safe_div(self.total, self.count)
+
+
 @tf_export('keras.metrics.binary_accuracy')
 def binary_accuracy(y_true, y_pred):
   return K.mean(math_ops.equal(y_true, math_ops.round(y_pred)), axis=-1)
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 15e793f5fc..6d8269f34d 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -18,67 +18,72 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
 import numpy as np
 
-from tensorflow.python import keras
+from tensorflow.python.eager import context
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras import backend as K
+from tensorflow.python.keras import layers
+from tensorflow.python.keras import metrics
+from tensorflow.python.keras.engine.training import Model
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
+from tensorflow.python.training.checkpointable import util as checkpointable_utils
 
 
 class KerasMetricsTest(test.TestCase):
 
   def test_metrics(self):
     with self.test_session():
-      y_a = keras.backend.variable(np.random.random((6, 7)))
-      y_b = keras.backend.variable(np.random.random((6, 7)))
-      for metric in [keras.metrics.binary_accuracy,
-                     keras.metrics.categorical_accuracy]:
+      y_a = K.variable(np.random.random((6, 7)))
+      y_b = K.variable(np.random.random((6, 7)))
+      for metric in [metrics.binary_accuracy, metrics.categorical_accuracy]:
         output = metric(y_a, y_b)
-        self.assertEqual(keras.backend.eval(output).shape, (6,))
+        self.assertEqual(K.eval(output).shape, (6,))
 
   def test_sparse_categorical_accuracy(self):
     with self.test_session():
-      metric = keras.metrics.sparse_categorical_accuracy
-      y_a = keras.backend.variable(np.random.randint(0, 7, (6,)))
-      y_b = keras.backend.variable(np.random.random((6, 7)))
-      self.assertEqual(keras.backend.eval(metric(y_a, y_b)).shape, (6,))
+      metric = metrics.sparse_categorical_accuracy
+      y_a = K.variable(np.random.randint(0, 7, (6,)))
+      y_b = K.variable(np.random.random((6, 7)))
+      self.assertEqual(K.eval(metric(y_a, y_b)).shape, (6,))
 
   def test_sparse_top_k_categorical_accuracy(self):
     with self.test_session():
-      y_pred = keras.backend.variable(np.array([[0.3, 0.2, 0.1],
-                                                [0.1, 0.2, 0.7]]))
-      y_true = keras.backend.variable(np.array([[1], [0]]))
-      result = keras.backend.eval(
-          keras.metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=3))
+      y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]]))
+      y_true = K.variable(np.array([[1], [0]]))
+      result = K.eval(
+          metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=3))
       self.assertEqual(result, 1)
-      result = keras.backend.eval(
-          keras.metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=2))
+      result = K.eval(
+          metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=2))
       self.assertEqual(result, 0.5)
-      result = keras.backend.eval(
-          keras.metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=1))
+      result = K.eval(
+          metrics.sparse_top_k_categorical_accuracy(y_true, y_pred, k=1))
       self.assertEqual(result, 0.)
 
   def test_top_k_categorical_accuracy(self):
     with self.test_session():
-      y_pred = keras.backend.variable(np.array([[0.3, 0.2, 0.1],
-                                                [0.1, 0.2, 0.7]]))
-      y_true = keras.backend.variable(np.array([[0, 1, 0], [1, 0, 0]]))
-      result = keras.backend.eval(
-          keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=3))
+      y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]]))
+      y_true = K.variable(np.array([[0, 1, 0], [1, 0, 0]]))
+      result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=3))
       self.assertEqual(result, 1)
-      result = keras.backend.eval(
-          keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=2))
+      result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=2))
       self.assertEqual(result, 0.5)
-      result = keras.backend.eval(
-          keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=1))
+      result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=1))
       self.assertEqual(result, 0.)
 
   def test_stateful_metrics(self):
     with self.test_session():
       np.random.seed(1334)
 
-      class BinaryTruePositives(keras.layers.Layer):
+      class BinaryTruePositives(layers.Layer):
         """Stateful Metric to count the total true positives over all batches.
 
         Assumes predictions and targets of shape `(samples, 1)`.
@@ -91,11 +96,11 @@ class KerasMetricsTest(test.TestCase):
 
         def __init__(self, name='true_positives', **kwargs):
           super(BinaryTruePositives, self).__init__(name=name, **kwargs)
-          self.true_positives = keras.backend.variable(value=0, dtype='int32')
+          self.true_positives = K.variable(value=0, dtype='int32')
           self.stateful = True
 
         def reset_states(self):
-          keras.backend.set_value(self.true_positives, 0)
+          K.set_value(self.true_positives, 0)
 
         def __call__(self, y_true, y_pred):
           """Computes the number of true positives in a batch.
@@ -120,14 +125,14 @@ class KerasMetricsTest(test.TestCase):
           return current_true_pos + true_pos
 
       metric_fn = BinaryTruePositives()
-      config = keras.metrics.serialize(metric_fn)
-      metric_fn = keras.metrics.deserialize(
+      config = metrics.serialize(metric_fn)
+      metric_fn = metrics.deserialize(
           config, custom_objects={'BinaryTruePositives': BinaryTruePositives})
 
       # Test on simple model
-      inputs = keras.Input(shape=(2,))
-      outputs = keras.layers.Dense(1, activation='sigmoid')(inputs)
-      model = keras.Model(inputs, outputs)
+      inputs = layers.Input(shape=(2,))
+      outputs = layers.Dense(1, activation='sigmoid')(inputs)
+      model = Model(inputs, outputs)
       model.compile(optimizer='sgd',
                     loss='binary_crossentropy',
                     metrics=['acc', metric_fn])
@@ -184,6 +189,125 @@ class KerasMetricsTest(test.TestCase):
       self.assertAllClose(
           val_outs[2], history.history['val_true_positives'][-1], atol=1e-5)
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_mean(self):
+    m = metrics.Mean(name='my_mean')
+
+    # check config
+    self.assertEqual(m.name, 'my_mean')
+    self.assertTrue(m.stateful)
+    self.assertEqual(m.dtype, dtypes.float64)
+    self.assertEqual(len(m.variables), 2)
+    self.evaluate(variables.global_variables_initializer())
+
+    # check initial state
+    self.assertEqual(self.evaluate(m.total), 0)
+    self.assertEqual(self.evaluate(m.count), 0)
+
+    # check __call__()
+    self.assertEqual(self.evaluate(m(100)), 100)
+    self.assertEqual(self.evaluate(m.total), 100)
+    self.assertEqual(self.evaluate(m.count), 1)
+
+    # check update_state() and result() + state accumulation + tensor input
+    update_op = m.update_state(ops.convert_n_to_tensor([1, 5]))
+    self.evaluate(update_op)
+    self.assertEqual(self.evaluate(m.result()), 106 / 3)
+    self.assertEqual(self.evaluate(m.total), 106)  # 100 + 1 + 5
+    self.assertEqual(self.evaluate(m.count), 3)
+
+    # check reset_states()
+    m.reset_states()
+    self.assertEqual(self.evaluate(m.total), 0)
+    self.assertEqual(self.evaluate(m.count), 0)
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_mean_with_sample_weight(self):
+    m = metrics.Mean()
+    self.evaluate(variables.global_variables_initializer())
+
+    # check scalar weight
+    result_t = m(100, sample_weight=0.5)
+    self.assertEqual(self.evaluate(result_t), 50 / 0.5)
+    self.assertEqual(self.evaluate(m.total), 50)
+    self.assertEqual(self.evaluate(m.count), 0.5)
+
+    # check weights not scalar and weights rank matches values rank
+    result_t = m([1, 5], sample_weight=[1, 0.2])
+    result = self.evaluate(result_t)
+    self.assertAlmostEqual(result, 52 / 1.7, 2)
+    self.assertAlmostEqual(self.evaluate(m.total), 52, 2)  # 50 + 1 + 5 * 0.2
+    self.assertAlmostEqual(self.evaluate(m.count), 1.7, 2)  # 0.5 + 1.2
+
+    # check weights broadcast
+    result_t = m([1, 2], sample_weight=0.5)
+    self.assertAlmostEqual(self.evaluate(result_t), 53.5 / 2.7, 2)
+    self.assertAlmostEqual(self.evaluate(m.total), 53.5, 2)  # 52 + 0.5 + 1
+    self.assertAlmostEqual(self.evaluate(m.count), 2.7, 2)  # 1.7 + 0.5 + 0.5
+
+    # check weights squeeze
+    result_t = m([1, 5], sample_weight=[[1], [0.2]])
+    self.assertAlmostEqual(self.evaluate(result_t), 55.5 / 3.9, 2)
+    self.assertAlmostEqual(self.evaluate(m.total), 55.5, 2)  # 53.5 + 1 + 1
+    self.assertAlmostEqual(self.evaluate(m.count), 3.9, 2)  # 2.7 + 1.2
+
+    # check weights expand
+    result_t = m([[1], [5]], sample_weight=[1, 0.2])
+    self.assertAlmostEqual(self.evaluate(result_t), 57.5 / 5.1, 2)
+    self.assertAlmostEqual(self.evaluate(m.total), 57.5, 2)  # 55.5 + 1 + 1
+    self.assertAlmostEqual(self.evaluate(m.count), 5.1, 2)  # 3.9 + 1.2
+
+  def test_mean_graph_with_placeholder(self):
+    with context.graph_mode(), self.test_session() as sess:
+      m = metrics.Mean()
+      v = array_ops.placeholder(dtypes.float32)
+      w = array_ops.placeholder(dtypes.float32)
+      sess.run(variables.global_variables_initializer())
+
+      # check __call__()
+      result_t = m(v, sample_weight=w)
+      result = sess.run(result_t, feed_dict=({v: 100, w: 0.5}))
+      self.assertEqual(sess.run(m.total), 50)
+      self.assertEqual(sess.run(m.count), 0.5)
+      self.assertEqual(result, 50 / 0.5)
+
+      # check update_state() and result()
+      result = sess.run(result_t, feed_dict=({v: [1, 5], w: [1, 0.2]}))
+      self.assertAlmostEqual(sess.run(m.total), 52, 2)  # 50 + 1 + 5 * 0.2
+      self.assertAlmostEqual(sess.run(m.count), 1.7, 2)  # 0.5 + 1.2
+      self.assertAlmostEqual(result, 52 / 1.7, 2)
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_save_restore(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
+    m = metrics.Mean()
+    checkpoint = checkpointable_utils.Checkpoint(mean=m)
+    self.evaluate(variables.global_variables_initializer())
+
+    # update state
+    self.evaluate(m(100.))
+    self.evaluate(m(200.))
+
+    # save checkpoint and then add an update
+    save_path = checkpoint.save(checkpoint_prefix)
+    self.evaluate(m(1000.))
+
+    # restore to the same checkpoint mean object
+    checkpoint.restore(save_path).assert_consumed().run_restore_ops()
+    self.evaluate(m(300.))
+    self.assertEqual(200., self.evaluate(m.result()))
+
+    # restore to a different checkpoint mean object
+    restore_mean = metrics.Mean()
+    restore_checkpoint = checkpointable_utils.Checkpoint(mean=restore_mean)
+    status = restore_checkpoint.restore(save_path)
+    restore_update = restore_mean(300.)
+    status.assert_consumed().run_restore_ops()
+    self.evaluate(restore_update)
+    self.assertEqual(200., self.evaluate(restore_mean.result()))
+    self.assertEqual(3, self.evaluate(restore_mean.count))
+
 
 if __name__ == '__main__':
   test.main()
-- 
cgit v1.2.3


From df7cc25691312ec27d1b38c3cbf566c2e00aa92b Mon Sep 17 00:00:00 2001
From: Shashi Shekhar 
Date: Wed, 18 Jul 2018 10:56:11 -0700
Subject: Add mobile device field to the issue template.

PiperOrigin-RevId: 205102950
---
 ISSUE_TEMPLATE.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md
index 2f3df7cda9..52faed9297 100644
--- a/ISSUE_TEMPLATE.md
+++ b/ISSUE_TEMPLATE.md
@@ -15,9 +15,10 @@ If you open a GitHub issue, here is our policy:
 ### System information
 - **Have I written custom code (as opposed to using a stock example script provided in TensorFlow)**:
 - **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
+- **Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device**:
 - **TensorFlow installed from (source or binary)**:
 - **TensorFlow version (use command below)**:
-- **Python version**: 
+- **Python version**:
 - **Bazel version (if compiling from source)**:
 - **GCC/Compiler version (if compiling from source)**:
 - **CUDA/cuDNN version**:
-- 
cgit v1.2.3


From 6b40cba84e422bd9f2dca35e9c85af7851f7ccb7 Mon Sep 17 00:00:00 2001
From: Derek Murray 
Date: Wed, 18 Jul 2018 10:56:34 -0700
Subject: Remove unused lib_internal dependency in contrib/tensor_forest.

PiperOrigin-RevId: 205103025
---
 tensorflow/contrib/tensor_forest/BUILD | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD
index 136856c015..164f3e58e6 100644
--- a/tensorflow/contrib/tensor_forest/BUILD
+++ b/tensorflow/contrib/tensor_forest/BUILD
@@ -223,7 +223,6 @@ tf_kernel_library(
         ":model_ops_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
     ],
     alwayslink = 1,
 )
@@ -319,7 +318,6 @@ tf_kernel_library(
         ":stats_ops_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
     ],
     alwayslink = 1,
 )
-- 
cgit v1.2.3


From fbfc8db63ca6bbabcede9dcb8b2bd8989ebebcd9 Mon Sep 17 00:00:00 2001
From: Jie 
Date: Wed, 18 Jul 2018 11:16:33 -0700
Subject: [trt allocator]   return nullptr when allocation fails   abort free()
 when receiving nullptr as input

---
 .../contrib/tensorrt/resources/trt_allocator.cc     | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
index f733b3dd56..81d7330b49 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
@@ -42,11 +42,13 @@ void* TRTDeviceAllocator::allocate(uint64_t size, uint64_t alignment,
   // See issue #20856
   alignment = 512;
   assert((alignment & (alignment - 1)) == 0);  // zero or a power of 2.
-  void* mem = allocator_->AllocateRaw(alignment, size + alignment);
-  CHECK(mem);
+  size_t total_size = size + alignment;
+  void* mem = allocator_->AllocateRaw(alignment, total_size);
+  if (!mem) {
+    return nullptr;
+  }
 
   void* alloc_mem = mem;
-  uint64_t total_size = size + alignment;
   CHECK(std::align(alignment, size, mem, total_size));
   if (mem != alloc_mem) {
     CHECK(mem_map_.insert({mem, alloc_mem}).second);
@@ -64,13 +66,14 @@ TRTDeviceAllocator::TRTDeviceAllocator(tensorflow::Allocator* allocator)
 void TRTDeviceAllocator::free(void* memory) {
   VLOG(2) << "Deallocating @ " << memory;
   // allocated memory adjusted for alignment, restore the original pointer
-
-  auto alloc_mem = mem_map_.find(memory);
-  if (alloc_mem != mem_map_.end()) {
-    memory = alloc_mem->second;
-    mem_map_.erase(alloc_mem->first);
+  if (memory) {
+    auto alloc_mem = mem_map_.find(memory);
+    if (alloc_mem != mem_map_.end()) {
+      memory = alloc_mem->second;
+      mem_map_.erase(alloc_mem->first);
+    }
+    allocator_->DeallocateRaw(memory);
   }
-  allocator_->DeallocateRaw(memory);
 }
 
 }  // namespace tensorrt
-- 
cgit v1.2.3


From f5a830421f287208a51bd04a94842913eb1fc0d2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 11:21:19 -0700
Subject: BEGIN_PUBLIC Rollback of 81161f9d9987a8eb70793d95048c20be34292859 due
 internal breakage. END_PUBLIC

Automated rollback of commit 81161f9d9987a8eb70793d95048c20be34292859

PiperOrigin-RevId: 205107655
---
 tensorflow/compiler/xla/shape_tree.h       | 140 +++++++----------------------
 tensorflow/compiler/xla/shape_tree_test.cc |  21 ++---
 tensorflow/compiler/xla/shape_util.h       |  13 ++-
 3 files changed, 43 insertions(+), 131 deletions(-)

diff --git a/tensorflow/compiler/xla/shape_tree.h b/tensorflow/compiler/xla/shape_tree.h
index c74dd648ad..4aacc87b78 100644
--- a/tensorflow/compiler/xla/shape_tree.h
+++ b/tensorflow/compiler/xla/shape_tree.h
@@ -44,6 +44,10 @@ struct ShapeTreeNode {
   // Data corresponding to this node.
   std::pair data;
 
+  // Children of this node, as indices into the container's nodes_ array.
+  std::vector children;
+
+  // Tells whether this is a leaf node.
   bool is_leaf = true;
 
   explicit ShapeTreeNode(ShapeIndex index)
@@ -52,20 +56,6 @@ struct ShapeTreeNode {
       : data(std::move(index), std::move(data)) {}
 };
 
-// Internal representation of an index table entry.
-struct IndexTableEntry {
-  // Index of the node in the ShapeTreeNode vector.
-  uint32 index;
-  // Index of the first child in a IndexTableEntry vector. In the index
-  // table all children entries for a given node will be placed next to each
-  // other. This allows us to use a single field to index them.
-  uint32 children_start;
-#ifndef NDEBUG
-  // Number of children, used for bounds checking.
-  uint32 children_count;
-#endif
-};
-
 }  // namespace internal
 
 template 
@@ -94,7 +84,6 @@ template 
 class ShapeTree {
  public:
   using Node = internal::ShapeTreeNode;
-  using Index = internal::IndexTableEntry;
 
   // Default constructor creates a tree with a nil shape (i.e. an empty tuple).
   ShapeTree() : ShapeTree(ShapeUtil::MakeNil()) {}
@@ -278,12 +267,11 @@ class ShapeTree {
  private:
   // Initialize node->children based on 'shape'. All children are assigned the
   // the given 'init_value'.
-  void InitChildren(const Shape& shape, const T& init_value, Node* node,
-                    Index* index);
+  void InitChildren(const Shape& shape, const T& init_value, Node* node);
 
   // Initialize node->children based on 'shape'. All children have
   // default-constructed data values.
-  void InitChildren(const Shape& shape, Node* node, Index* index);
+  void InitChildren(const Shape& shape, Node* node);
 
   // Returns the number of subshapes, including interior nodes, in shape.
   int64 CountSubshapes(const Shape& shape);
@@ -303,9 +291,6 @@ class ShapeTree {
   // The nodes in this shape tree.
   std::vector nodes_;
 
-  // Index table for node lookups.
-  std::vector index_table_;
-
   // If we own our Shape, this field contains it, and shape_ is a pointer into
   // here.  Otherwise if we don't own our shape, this is nullptr.
   std::shared_ptr shape_storage_;
@@ -388,74 +373,36 @@ int64 ShapeTree::CountSubshapes(const Shape& shape) {
 
 template 
 void ShapeTree::InitChildren(const Shape& shape, const T& init_value,
-                                Node* node, Index* index) {
+                                Node* node) {
   if (ShapeUtil::IsTuple(shape)) {
     const int64 size = ShapeUtil::TupleElementCount(shape);
-#ifndef NDEBUG
-    index->children_count = size;
-#endif
+    node->children.reserve(size);
     node->is_leaf = false;
     ShapeIndex shape_index = node->data.first;
     shape_index.push_back(0);
-
-    // At the end of the index_table, reserve a continuous space to hold the
-    // children of current node. In order to enforce the invariant that all
-    // children of a given node are placed together, we need to do the
-    // reservation before we recurse into any of its children.
-    int64 children_start_position = index_table_.size();
-    index_table_.resize(index_table_.size() + size);
-
     for (int i = 0; i < size; ++i) {
       shape_index[shape_index.size() - 1] = i;
-      index_table_[children_start_position + i].index = nodes_.size();
-      // The first child of the node in the index table is placed at the end of
-      // the table.
-      index_table_[children_start_position + i].children_start =
-          index_table_.size();
+      node->children.push_back(nodes_.size());
       nodes_.emplace_back(shape_index, init_value);
-      InitChildren(shape.tuple_shapes(i), init_value, &nodes_.back(),
-                   &index_table_[children_start_position + i]);
+      InitChildren(shape.tuple_shapes(i), init_value, &nodes_.back());
     }
-  } else {
-#ifndef NDEBUG
-    index->children_count = 0;
-#endif
   }
 }
 
 template 
-void ShapeTree::InitChildren(const Shape& shape, Node* node, Index* index) {
+void ShapeTree::InitChildren(const Shape& shape, Node* node) {
   if (ShapeUtil::IsTuple(shape)) {
     const int64 size = ShapeUtil::TupleElementCount(shape);
-#ifndef NDEBUG
-    index->children_count = size;
-#endif
+    node->children.reserve(size);
     node->is_leaf = false;
     ShapeIndex shape_index = node->data.first;
     shape_index.push_back(0);
-
-    // At the end of the index_table, reserve a continuous space to hold the
-    // children of current node. In order to enforce the invariant that all
-    // children of a given node are placed together, we need to do the
-    // reservation before we recurse into any of its children.
-    int64 children_start_position = index_table_.size();
-    index_table_.resize(index_table_.size() + size);
-
     for (int i = 0; i < size; ++i) {
       shape_index[shape_index.size() - 1] = i;
-      index_table_[children_start_position + i].index = nodes_.size();
-      // The first child of the node in the index table is placed at the end of
-      // the table.
-      index_table_[children_start_position + i].children_start =
-          index_table_.size();
+      node->children.push_back(nodes_.size());
       nodes_.emplace_back(shape_index);
-      InitChildren(shape.tuple_shapes(i), &nodes_.back(),
-                   &index_table_[children_start_position + i]);
+      InitChildren(shape.tuple_shapes(i), &nodes_.back());
     }
-  } else {
-#ifndef NDEBUG
-    index->children_count = 0;
-#endif
   }
 }
 
@@ -466,36 +413,24 @@ ShapeTree::ShapeTree(Shape shape)
   // The shape_ field is just used to hold the structure of the shape.
   // It should not be relied upon to store layout information.
   LayoutUtil::ClearLayout(shape_storage_.get());
-  const int64 count = CountSubshapes(*shape_);
-  nodes_.reserve(count);
+  nodes_.reserve(CountSubshapes(*shape_));
   nodes_.emplace_back(ShapeIndex{});
-
-  index_table_.reserve(count);
-  index_table_.emplace_back(Index{0, 1});
-  InitChildren(*shape_, &nodes_[0], &index_table_[0]);
+  InitChildren(*shape_, &nodes_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const Shape* shape) : shape_(shape) {
-  const int64 count = CountSubshapes(*shape_);
-  nodes_.reserve(count);
+  nodes_.reserve(CountSubshapes(*shape_));
   nodes_.emplace_back(ShapeIndex{});
-
-  index_table_.reserve(count);
-  index_table_.emplace_back(Index{0, 1});
-  InitChildren(*shape_, &nodes_[0], &index_table_[0]);
+  InitChildren(*shape_, &nodes_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const std::shared_ptr& shape)
     : shape_storage_(shape), shape_(shape_storage_.get()) {
-  const int64 count = CountSubshapes(*shape_);
-  nodes_.reserve(count);
+  nodes_.reserve(CountSubshapes(*shape_));
   nodes_.emplace_back(ShapeIndex{});
-
-  index_table_.reserve(count);
-  index_table_.emplace_back(Index{0, 1});
-  InitChildren(*shape_, &nodes_[0], &index_table_[0]);
+  InitChildren(*shape_, &nodes_[0]);
 }
 
 template 
@@ -505,38 +440,26 @@ ShapeTree::ShapeTree(Shape shape, const T& init_value)
   // The shape_ field is just used to hold the structure of the shape.
   // It should not be relied upon to store layout information.
   LayoutUtil::ClearLayout(shape_storage_.get());
-  const int64 count = CountSubshapes(*shape_);
-  nodes_.reserve(count);
+  nodes_.reserve(CountSubshapes(*shape_));
   nodes_.emplace_back(ShapeIndex{}, init_value);
-
-  index_table_.reserve(count);
-  index_table_.emplace_back(Index{0, 1});
-  InitChildren(*shape_, init_value, &nodes_[0], &index_table_[0]);
+  InitChildren(*shape_, init_value, &nodes_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const Shape* shape, const T& init_value)
     : shape_(shape) {
-  const int64 count = CountSubshapes(*shape_);
-  nodes_.reserve(count);
+  nodes_.reserve(CountSubshapes(*shape_));
   nodes_.emplace_back(ShapeIndex{}, init_value);
-
-  index_table_.reserve(count);
-  index_table_.emplace_back(Index{0, 1});
-  InitChildren(*shape_, init_value, &nodes_[0], &index_table_[0]);
+  InitChildren(*shape_, init_value, &nodes_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const std::shared_ptr& shape,
                         const T& init_value)
     : shape_storage_(shape), shape_(shape_storage_.get()) {
-  const int64 count = CountSubshapes(*shape_);
-  nodes_.reserve(count);
+  nodes_.reserve(CountSubshapes(*shape_));
   nodes_.emplace_back(ShapeIndex{}, init_value);
-
-  index_table_.reserve(count);
-  index_table_.emplace_back(Index{0, 1});
-  InitChildren(*shape_, init_value, &nodes_[0], &index_table_[0]);
+  InitChildren(*shape_, init_value, &nodes_[0]);
 }
 
 template 
@@ -551,16 +474,13 @@ T* ShapeTree::mutable_element(ShapeIndexView index) {
 
 template 
 internal::ShapeTreeNode* ShapeTree::Lookup(ShapeIndexView index) {
-  Index* iter = &index_table_[0];
+  Node* node = &nodes_[0];
   for (const int64 i : index) {
     CHECK_GE(i, 0);
-#ifndef NDEBUG
-    CHECK_LT(i, iter->children_count);
-#endif
-    iter = &index_table_[iter->children_start + i];
+    CHECK_LT(i, node->children.size());
+    node = &nodes_[node->children[i]];
   }
-
-  return &nodes_[iter->index];
+  return node;
 }
 
 template 
diff --git a/tensorflow/compiler/xla/shape_tree_test.cc b/tensorflow/compiler/xla/shape_tree_test.cc
index 4391078b64..51de82e957 100644
--- a/tensorflow/compiler/xla/shape_tree_test.cc
+++ b/tensorflow/compiler/xla/shape_tree_test.cc
@@ -227,16 +227,14 @@ TEST_F(ShapeTreeTest, NestedTupleShape) {
 
 TEST_F(ShapeTreeTest, InvalidIndexingTuple) {
   ShapeTree shape_tree{tuple_shape_};
-#ifndef NDEBUG
+
   EXPECT_DEATH(shape_tree.element({4}), "");
-#endif
 }
 
 TEST_F(ShapeTreeTest, InvalidIndexingNestedTuple) {
   ShapeTree shape_tree{nested_tuple_shape_};
-#ifndef NDEBUG
+
   EXPECT_DEATH(shape_tree.element({0, 0}), "");
-#endif
 }
 
 TEST_F(ShapeTreeTest, ShapeTreeOfNonCopyableType) {
@@ -604,15 +602,12 @@ void BM_Iterate(int iters, int depth, int fan_out) {
   }
 }
 
-#define BENCHMARK_WITH_ARGS(name) \
-  BENCHMARK(name)->ArgPair(2, 8)->ArgPair(1, 1000)
-
-BENCHMARK_WITH_ARGS(BM_Construct);
-BENCHMARK_WITH_ARGS(BM_ConstructUnowned);
-BENCHMARK_WITH_ARGS(BM_Copy);
-BENCHMARK_WITH_ARGS(BM_Move);
-BENCHMARK_WITH_ARGS(BM_ForEach);
-BENCHMARK_WITH_ARGS(BM_Iterate);
+BENCHMARK(BM_Construct)->ArgPair(2, 8);
+BENCHMARK(BM_ConstructUnowned)->ArgPair(2, 8);
+BENCHMARK(BM_Copy)->ArgPair(2, 8);
+BENCHMARK(BM_Move)->ArgPair(2, 8);
+BENCHMARK(BM_ForEach)->ArgPair(2, 8);
+BENCHMARK(BM_Iterate)->ArgPair(2, 8);
 
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index 83d15e8fe3..17c1d7b10a 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -31,7 +31,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
-#include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/gtl/optional.h"
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/env.h"
@@ -74,12 +73,10 @@ class ShapeIndex {
   // push_front is O(n^2), but shapes don't usually have a ton of dimensions.
   void push_front(int64 value) { indices_.insert(indices_.begin(), value); }
 
-  using container_type = gtl::InlinedVector;
-
-  container_type::const_iterator begin() const { return indices_.begin(); }
-  container_type::const_iterator end() const { return indices_.end(); }
-  container_type::iterator begin() { return indices_.begin(); }
-  container_type::iterator end() { return indices_.end(); }
+  std::vector::const_iterator begin() const { return indices_.begin(); }
+  std::vector::const_iterator end() const { return indices_.end(); }
+  std::vector::iterator begin() { return indices_.begin(); }
+  std::vector::iterator end() { return indices_.end(); }
 
   const int64* data() const { return indices_.data(); }
 
@@ -100,7 +97,7 @@ class ShapeIndex {
   string ToString() const;
 
  private:
-  container_type indices_;
+  std::vector indices_;
 };
 
 // A view into a ShapeIndex as above, with the cheap/easy ability to consume the
-- 
cgit v1.2.3


From b062b8e92376012383fe55ceb660d83af08a8686 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 11:42:38 -0700
Subject: Fixing bugs in graphdef export of bool types and a few broken ops.

PiperOrigin-RevId: 205111208
---
 tensorflow/contrib/lite/toco/export_tensorflow.cc  | 86 +++++++++++++++-------
 .../convert_expanddims_to_reshape.cc               |  5 --
 .../graph_transformations/propagate_fixed_sizes.cc |  7 ++
 .../unfuse_activation_functions.cc                 |  9 ++-
 tensorflow/contrib/lite/toco/tooling_util.cc       |  8 +-
 5 files changed, 81 insertions(+), 34 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/export_tensorflow.cc b/tensorflow/contrib/lite/toco/export_tensorflow.cc
index 4508aa6632..5c112ffc38 100644
--- a/tensorflow/contrib/lite/toco/export_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/export_tensorflow.cc
@@ -215,6 +215,30 @@ void ConvertFloatTensorConst(const Model& model, const string& name,
                    LegacyScalarPolicy::kAvoidLegacyScalars);
 }
 
+void ConvertBoolTensorConst(const Model& model, const string& name,
+                            GraphDef* tensorflow_graph) {
+  if (HasAlreadyExportedConst(name, *tensorflow_graph)) {
+    return;
+  }
+  CHECK(model.HasArray(name));
+  const auto& array = model.GetArray(name);
+  tensorflow::NodeDef* const_op = tensorflow_graph->add_node();
+  const_op->set_op("Const");
+  const_op->set_name(name);
+  (*const_op->mutable_attr())["dtype"].set_type(DT_BOOL);
+  auto* tensor = (*const_op->mutable_attr())["value"].mutable_tensor();
+  tensor->set_dtype(DT_BOOL);
+  const auto& data = array.GetBuffer().data;
+  for (auto index : data) {
+    tensor->add_bool_val(index);
+  }
+  const auto& array_shape = array.shape();
+  auto* shape = tensor->mutable_tensor_shape();
+  for (int i = 0; i < array_shape.dimensions_count(); i++) {
+    shape->add_dim()->set_size(array_shape.dims(i));
+  }
+}
+
 void ConvertIntTensorConst(const Model& model, const string& name,
                            GraphDef* tensorflow_graph) {
   if (HasAlreadyExportedConst(name, *tensorflow_graph)) {
@@ -621,7 +645,8 @@ void ConvertAddOperator(const Model& model, const AddOperator& src_op,
   CHECK_EQ(src_op.inputs.size(), 2);
   *add_op->add_input() = src_op.inputs[0];
   *add_op->add_input() = src_op.inputs[1];
-  (*add_op->mutable_attr())["T"].set_type(DT_FLOAT);
+  (*add_op->mutable_attr())["T"].set_type(
+      GetTensorFlowDataType(model, src_op.outputs[0]));
 }
 
 void ConvertAddNOperator(const Model& model, const AddNOperator& src_op,
@@ -633,7 +658,8 @@ void ConvertAddNOperator(const Model& model, const AddNOperator& src_op,
     *add_op->add_input() = input;
   }
   (*add_op->mutable_attr())["N"].set_i(src_op.inputs.size());
-  (*add_op->mutable_attr())["T"].set_type(DT_FLOAT);
+  (*add_op->mutable_attr())["T"].set_type(
+      GetTensorFlowDataType(model, src_op.outputs[0]));
 }
 
 void ConvertMulOperator(const Model& model, const MulOperator& src_op,
@@ -644,16 +670,18 @@ void ConvertMulOperator(const Model& model, const MulOperator& src_op,
   CHECK_EQ(src_op.inputs.size(), 2);
   *add_op->add_input() = src_op.inputs[0];
   *add_op->add_input() = src_op.inputs[1];
-  (*add_op->mutable_attr())["T"].set_type(DT_FLOAT);
+  (*add_op->mutable_attr())["T"].set_type(
+      GetTensorFlowDataType(model, src_op.outputs[0]));
 }
 
-void ConvertReluOperator(const ReluOperator& src_op,
+void ConvertReluOperator(const Model& model, const ReluOperator& src_op,
                          GraphDef* tensorflow_graph) {
   tensorflow::NodeDef* relu_op = tensorflow_graph->add_node();
   relu_op->set_op("Relu");
   relu_op->set_name(src_op.outputs[0]);
   *relu_op->add_input() = src_op.inputs[0];
-  (*relu_op->mutable_attr())["T"].set_type(DT_FLOAT);
+  (*relu_op->mutable_attr())["T"].set_type(
+      GetTensorFlowDataType(model, src_op.outputs[0]));
 }
 
 void ConvertRelu1Operator(const Relu1Operator& src_op,
@@ -1694,43 +1722,43 @@ void ConvertSubOperator(const Model& model, const SubOperator& src_op,
 void ConvertTensorFlowMinimumOperator(const Model& model,
                                       const TensorFlowMinimumOperator& src_op,
                                       GraphDef* tensorflow_graph) {
-  tensorflow::NodeDef* sub_op = tensorflow_graph->add_node();
-  sub_op->set_op("Minimum");
-  sub_op->set_name(src_op.outputs[0]);
+  tensorflow::NodeDef* min_op = tensorflow_graph->add_node();
+  min_op->set_op("Minimum");
+  min_op->set_name(src_op.outputs[0]);
   CHECK_EQ(src_op.inputs.size(), 2);
-  *sub_op->add_input() = src_op.inputs[0];
-  *sub_op->add_input() = src_op.inputs[1];
+  *min_op->add_input() = src_op.inputs[0];
+  *min_op->add_input() = src_op.inputs[1];
   const tensorflow::DataType data_type =
       GetTensorFlowDataType(model, src_op.inputs[0]);
-  (*sub_op->mutable_attr())["T"].set_type(data_type);
+  (*min_op->mutable_attr())["T"].set_type(data_type);
 }
 
 void ConvertTensorFlowMaximumOperator(const Model& model,
                                       const TensorFlowMaximumOperator& src_op,
                                       GraphDef* tensorflow_graph) {
-  tensorflow::NodeDef* sub_op = tensorflow_graph->add_node();
-  sub_op->set_op("Maximum");
-  sub_op->set_name(src_op.outputs[0]);
+  tensorflow::NodeDef* max_op = tensorflow_graph->add_node();
+  max_op->set_op("Maximum");
+  max_op->set_name(src_op.outputs[0]);
   CHECK_EQ(src_op.inputs.size(), 2);
-  *sub_op->add_input() = src_op.inputs[0];
-  *sub_op->add_input() = src_op.inputs[1];
+  *max_op->add_input() = src_op.inputs[0];
+  *max_op->add_input() = src_op.inputs[1];
   const tensorflow::DataType data_type =
       GetTensorFlowDataType(model, src_op.inputs[0]);
-  (*sub_op->mutable_attr())["T"].set_type(data_type);
+  (*max_op->mutable_attr())["T"].set_type(data_type);
 }
 
 void ConvertSelectOperator(const Model& model, const SelectOperator& src_op,
                            GraphDef* tensorflow_graph) {
-  tensorflow::NodeDef* sub_op = tensorflow_graph->add_node();
-  sub_op->set_op("Select");
-  sub_op->set_name(src_op.outputs[0]);
+  tensorflow::NodeDef* select_op = tensorflow_graph->add_node();
+  select_op->set_op("Select");
+  select_op->set_name(src_op.outputs[0]);
   CHECK_EQ(src_op.inputs.size(), 3);
-  *sub_op->add_input() = src_op.inputs[0];
-  *sub_op->add_input() = src_op.inputs[1];
-  *sub_op->add_input() = src_op.inputs[2];
+  *select_op->add_input() = src_op.inputs[0];
+  *select_op->add_input() = src_op.inputs[1];
+  *select_op->add_input() = src_op.inputs[2];
   const tensorflow::DataType data_type =
       GetTensorFlowDataType(model, src_op.inputs[1]);
-  (*sub_op->mutable_attr())["T"].set_type(data_type);
+  (*select_op->mutable_attr())["T"].set_type(data_type);
 }
 
 void ConvertTileOperator(const Model& model,
@@ -1753,11 +1781,14 @@ void ConvertTileOperator(const Model& model,
 void ConvertTopKV2Operator(const Model& model, const TopKV2Operator& src_op,
                            GraphDef* tensorflow_graph) {
   tensorflow::NodeDef* topk_op = tensorflow_graph->add_node();
-  topk_op->set_op("TOPKV2");
+  topk_op->set_op("TopKV2");
   topk_op->set_name(src_op.outputs[0]);
   CHECK_EQ(src_op.inputs.size(), 2);
   *topk_op->add_input() = src_op.inputs[0];
   *topk_op->add_input() = src_op.inputs[1];
+  const tensorflow::DataType data_type =
+      GetTensorFlowDataType(model, src_op.inputs[0]);
+  (*topk_op->mutable_attr())["T"].set_type(data_type);
   (*topk_op->mutable_attr())["sorted"].set_b(true);
 }
 
@@ -1864,7 +1895,7 @@ void ConvertOperator(const Model& model, const Operator& src_op,
     ConvertMulOperator(model, static_cast(src_op),
                        tensorflow_graph);
   } else if (src_op.type == OperatorType::kRelu) {
-    ConvertReluOperator(static_cast(src_op),
+    ConvertReluOperator(model, static_cast(src_op),
                         tensorflow_graph);
   } else if (src_op.type == OperatorType::kRelu1) {
     ConvertRelu1Operator(static_cast(src_op),
@@ -2138,6 +2169,9 @@ void ExportTensorFlowGraphDefImplementation(const Model& model,
     const auto& array = *array_pair.second;
     if (array.buffer) {
       switch (array.data_type) {
+        case ArrayDataType::kBool:
+          ConvertBoolTensorConst(model, array_name, tensorflow_graph);
+          break;
         case ArrayDataType::kFloat:
           ConvertFloatTensorConst(model, array_name, tensorflow_graph);
           break;
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/convert_expanddims_to_reshape.cc b/tensorflow/contrib/lite/toco/graph_transformations/convert_expanddims_to_reshape.cc
index 56f48d47de..310a88484c 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/convert_expanddims_to_reshape.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/convert_expanddims_to_reshape.cc
@@ -40,11 +40,6 @@ bool ConvertExpandDimsToReshape::Run(Model* model, std::size_t op_index) {
     // Yield until input dims have been resolved.
     return false;
   }
-  if (input_array.shape().dimensions_count() == 0) {
-    // Input array cannot be 0-D.
-    // (Unsure if this is TF behavior, but was required to get a test to pass.)
-    return false;
-  }
 
   const auto& axis_array = model->GetArray(expand_op->inputs[1]);
   if (!axis_array.has_shape()) {
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 5e2ba0eca7..a250db9975 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -437,6 +437,7 @@ void ProcessTensorFlowReshapeOperator(Model* model,
       product_non_wildcard_dims *= shape_data[i];
     }
   }
+
   const int input_flat_size = RequiredBufferSizeForShape(input_shape);
   if (has_wildcard) {
     CHECK_GE(input_flat_size, product_non_wildcard_dims)
@@ -445,6 +446,12 @@ void ProcessTensorFlowReshapeOperator(Model* model,
         << op->outputs[0] << "\". Are your input shapes correct?";
     shape_data[wildcard_index] = input_flat_size / product_non_wildcard_dims;
   }
+
+  if (shape_data.size() == 1 && shape_data[0] == 0) {
+    // We have reshaped a scalar, so preserve as a scalar.
+    shape_data.clear();
+  }
+
   auto& output_shape = *output_array.mutable_shape();
   *output_shape.mutable_dims() = shape_data;
   CHECK_EQ(input_flat_size, RequiredBufferSizeForShape(output_shape))
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/unfuse_activation_functions.cc b/tensorflow/contrib/lite/toco/graph_transformations/unfuse_activation_functions.cc
index 2c7046c8c7..69bad2fa89 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/unfuse_activation_functions.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/unfuse_activation_functions.cc
@@ -64,7 +64,14 @@ bool UnfuseActivationFunctions::Run(Model* model, std::size_t op_index) {
   const string& tmp_array_name =
       AvailableArrayName(*model, op->outputs[0] + "_unfused");
   CHECK(!model->HasArray(tmp_array_name));
-  model->GetOrCreateArray(tmp_array_name);
+
+  const auto& output_array = model->GetArray(op->outputs[0]);
+  auto& tmp_array = model->GetOrCreateArray(tmp_array_name);
+  if (output_array.quantization_params) {
+    tmp_array.GetOrCreateQuantizationParams() =
+        output_array.GetQuantizationParams();
+  }
+
   ac_op->inputs = {tmp_array_name};
   op->outputs = {tmp_array_name};
   return true;
diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index 4305727c8c..edcdd8f8cc 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -940,8 +940,12 @@ void CheckEachArray(const Model& model) {
       // shape.
       CHECK(array->has_shape());
       // Constant buffer should has a valid shape.
-      for (int d : array->shape().dims()) {
-        CHECK_GE(d, 1);
+      bool is_scalar =
+          array->shape().dimensions_count() == 1 && array->shape().dims(0) == 0;
+      if (!is_scalar) {
+        for (int d : array->shape().dims()) {
+          CHECK_GE(d, 1);
+        }
       }
       // The shape flat-size should agree with the buffer length.
       CHECK_EQ(array->buffer->Length(),
-- 
cgit v1.2.3


From 128a67df506370312b60c6dfa2e48881b71b627c Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Wed, 18 Jul 2018 11:57:56 -0700
Subject: [TF:XLA] Bump open source llvm revision to r337361

PiperOrigin-RevId: 205113604
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 378de4261c..4b4f31813c 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -487,11 +487,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "llvm",
       urls = [
-          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/bd8c8d759852871609ba2e4e79868420f751949d.tar.gz",
-	  "https://github.com/llvm-mirror/llvm/archive/bd8c8d759852871609ba2e4e79868420f751949d.tar.gz",
+          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/10c3b3d15ed6a788ac12221b784caf81fb8248b5.tar.gz",
+	  "https://github.com/llvm-mirror/llvm/archive/10c3b3d15ed6a788ac12221b784caf81fb8248b5.tar.gz",
       ],
-      sha256 = "0c63e8583b213543309e8577ffe87a0cf34cc22269630d2c5c2f0a2345fda4a8",
-      strip_prefix = "llvm-bd8c8d759852871609ba2e4e79868420f751949d",
+      sha256 = "a9feb6b47267c30fd7c19ebfdf4dbde6757054f716fa77c09bcb1106799c3253",
+      strip_prefix = "llvm-10c3b3d15ed6a788ac12221b784caf81fb8248b5",
       build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
   )
 
-- 
cgit v1.2.3


From a10592ef7741d858466a980239fc95e65d7c66b6 Mon Sep 17 00:00:00 2001
From: Geoffrey Irving 
Date: Wed, 11 Jul 2018 13:05:14 -0700
Subject: Improve error messages for gather_nd and scatter_nd

Use SliceDebugString to produce nice error messages using
multidimensional indexes.
---
 tensorflow/core/kernels/gather_nd_op.cc               |  7 ++++---
 tensorflow/core/kernels/scatter_nd_op.cc              |  6 ++++--
 tensorflow/core/kernels/scatter_nd_op_test.cc         |  2 +-
 tensorflow/python/kernel_tests/gather_nd_op_test.py   | 12 ++++--------
 tensorflow/python/kernel_tests/scatter_nd_ops_test.py |  4 ++--
 5 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/tensorflow/core/kernels/gather_nd_op.cc b/tensorflow/core/kernels/gather_nd_op.cc
index 4e53291b7f..e50b7fe3bf 100644
--- a/tensorflow/core/kernels/gather_nd_op.cc
+++ b/tensorflow/core/kernels/gather_nd_op.cc
@@ -188,12 +188,13 @@ Status DoGatherNd(OpKernelContext* c, const Tensor& params,
 
     // bad_i will only return >= 0 on CPUs right now.
     if (bad_i >= 0) {
+      auto shape = indices.shape();
+      shape.RemoveLastDims(1);
       return errors::InvalidArgument(
-          "flat indices[", bad_i, ", :] = [",
+          "indices", SliceDebugString(shape, bad_i), " = [",
           str_util::Join(
               gtl::ArraySlice(&indices_mat(bad_i, 0), indices_nd), ", "),
-          "] does not index into param (shape: ", params.shape().DebugString(),
-          ").");
+          "] does not index into param shape ", params.shape().DebugString());
     }
   }
   return Status::OK();
diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc
index e1fc2ea128..5f300fb64d 100644
--- a/tensorflow/core/kernels/scatter_nd_op.cc
+++ b/tensorflow/core/kernels/scatter_nd_op.cc
@@ -537,11 +537,13 @@ Status DoScatterNd(OpKernelContext* c, const Tensor& indices,
     }
   }
   if (bad_i >= 0) {
+    auto slice_shape = indices.shape();
+    slice_shape.RemoveLastDims(1);
     return errors::InvalidArgument(
-        "Invalid indices: ", SliceDebugString(indices.shape(), bad_i), " = [",
+        "indices", SliceDebugString(slice_shape, bad_i), " = [",
         str_util::Join(
             gtl::ArraySlice(&indices_flat(bad_i, 0), slice_dim), ", "),
-        "] does not index into ", shape.DebugString());
+        "] does not index into shape ", shape.DebugString());
   }
   return Status::OK();
 }
diff --git a/tensorflow/core/kernels/scatter_nd_op_test.cc b/tensorflow/core/kernels/scatter_nd_op_test.cc
index c134a8dd5b..95ecc69c95 100644
--- a/tensorflow/core/kernels/scatter_nd_op_test.cc
+++ b/tensorflow/core/kernels/scatter_nd_op_test.cc
@@ -185,7 +185,7 @@ TEST_F(ScatterNdUpdateOpTest, Error_IndexOutOfRange) {
                            {100, 101, 102, 777, 778, 779, 10000, 10001, 10002});
   Status s = RunOpKernel();
   EXPECT_TRUE(str_util::StrContains(
-      s.ToString(), "Invalid indices: [2,0] = [99] does not index into [5,3]"))
+      s.ToString(), "indices[2] = [99] does not index into shape [5,3]"))
       << s;
 }
 
diff --git a/tensorflow/python/kernel_tests/gather_nd_op_test.py b/tensorflow/python/kernel_tests/gather_nd_op_test.py
index 58e2a8ac2a..c0b419e1d1 100644
--- a/tensorflow/python/kernel_tests/gather_nd_op_test.py
+++ b/tensorflow/python/kernel_tests/gather_nd_op_test.py
@@ -203,8 +203,7 @@ class GatherNdTest(test.TestCase):
       indices = [[[0], [7]]]  # Make this one higher rank
       gather_nd = array_ops.gather_nd(params, indices)
       with self.assertRaisesOpError(
-          r"flat indices\[1, :\] = \[7\] does not index into param "
-          r"\(shape: \[3\]\)"):
+          r"indices\[0,1\] = \[7\] does not index into param shape \[3\]"):
         gather_nd.eval()
 
   def _disabledTestBadIndicesGPU(self):
@@ -217,8 +216,7 @@ class GatherNdTest(test.TestCase):
       indices = [[[0], [7]]]  # Make this one higher rank
       gather_nd = array_ops.gather_nd(params, indices)
       with self.assertRaisesOpError(
-          r"flat indices\[1, :\] = \[7\] does not index into param "
-          r"\(shape: \[3\]\)"):
+          r"indices\[0,1\] = \[7\] does not index into param shape \[3\]"):
         gather_nd.eval()
 
   def testBadIndicesWithSlicesCPU(self):
@@ -227,8 +225,7 @@ class GatherNdTest(test.TestCase):
       indices = [[[0], [0], [1]]]  # Make this one higher rank
       gather_nd = array_ops.gather_nd(params, indices)
       with self.assertRaisesOpError(
-          r"flat indices\[2, :\] = \[1\] does not index into param "
-          r"\(shape: \[1,3\]\)"):
+          r"indices\[0,2\] = \[1\] does not index into param shape \[1,3\]"):
         gather_nd.eval()
 
   def _disabledTestBadIndicesWithSlicesGPU(self):
@@ -241,8 +238,7 @@ class GatherNdTest(test.TestCase):
       indices = [[[0], [0], [1]]]  # Make this one higher rank
       gather_nd = array_ops.gather_nd(params, indices)
       with self.assertRaisesOpError(
-          r"flat indices\[2, :\] = \[1\] does not index into param "
-          r"\(shape: \[1,3\]\)"):
+          r"indices\[0,2\] = \[1\] does not index into param shape \[1,3\]"):
         gather_nd.eval()
 
   def testGradientsRank2Elements(self):
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index f9b9c77bbf..c31499e52d 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -268,12 +268,12 @@ class StatefulScatterNdTest(test.TestCase):
         # Test some out of range errors.
         indices = np.array([[-1], [0], [5]])
         with self.assertRaisesOpError(
-            r"Invalid indices: \[0,0\] = \[-1\] does not index into \[6\]"):
+            r"indices\[0\] = \[-1\] does not index into shape \[6\]"):
           op(ref, indices, updates).eval()
 
         indices = np.array([[2], [0], [6]])
         with self.assertRaisesOpError(
-            r"Invalid indices: \[2,0\] = \[6\] does not index into \[6\]"):
+            r"indices\[2\] = \[6\] does not index into shape \[6\]"):
           op(ref, indices, updates).eval()
 
   def testRank3ValidShape(self):
-- 
cgit v1.2.3


From a4a2d0a9654a5c2c75faf6dd91ee82dcd37cc004 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 12:23:16 -0700
Subject: Avoid ambiguous 'detail' namespace.

PiperOrigin-RevId: 205117524
---
 .../core/kernels/depthwise_conv_op_gpu.cu.cc       | 28 +++++++++++-----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc b/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
index 5472a192d9..2a25459194 100644
--- a/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/depthwise_conv_op_gpu.cu.cc
@@ -641,20 +641,6 @@ Status LaunchDepthwiseConv2dGPUSmall(OpKernelContext* ctx,
   return Status::OK();
 }
 
-namespace {
-// Returns whether the context's GPU supports efficient fp16 math.
-bool HasFastHalfMath(OpKernelContext* ctx) {
-  int major, minor;
-  ctx->op_device_context()
-      ->stream()
-      ->parent()
-      ->GetDeviceDescription()
-      .cuda_compute_capability(&major, &minor);
-  auto cuda_arch = major * 100 + minor * 10;
-  // GPUs before sm_53 don't support fp16 math, and sm_61's fp16 math is slow.
-  return cuda_arch >= 530 && cuda_arch != 610;
-}
-
 namespace detail {
 template 
 struct PseudoHalfType {
@@ -666,9 +652,23 @@ struct PseudoHalfType {
 };
 }  // namespace detail
 
+namespace {
 // Maps to float if T is __half, and to T otherwise.
 template 
 using PseudoHalfType = typename detail::PseudoHalfType::Type;
+
+// Returns whether the context's GPU supports efficient fp16 math.
+bool HasFastHalfMath(OpKernelContext* ctx) {
+  int major, minor;
+  ctx->op_device_context()
+      ->stream()
+      ->parent()
+      ->GetDeviceDescription()
+      .cuda_compute_capability(&major, &minor);
+  auto cuda_arch = major * 100 + minor * 10;
+  // GPUs before sm_53 don't support fp16 math, and sm_61's fp16 math is slow.
+  return cuda_arch >= 530 && cuda_arch != 610;
+}
 }  // namespace
 
 template 
Date: Wed, 18 Jul 2018 12:25:53 -0700
Subject: Remove mentions of developer preview in TFLite docs.

PiperOrigin-RevId: 205117878
---
 tensorflow/docs_src/mobile/index.md        |  3 ---
 tensorflow/docs_src/mobile/tflite/index.md | 16 ++++------------
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/tensorflow/docs_src/mobile/index.md b/tensorflow/docs_src/mobile/index.md
index 419ae7094a..6032fcad02 100644
--- a/tensorflow/docs_src/mobile/index.md
+++ b/tensorflow/docs_src/mobile/index.md
@@ -13,9 +13,6 @@ Here are a few of the differences between the two:
   developed with TensorFlow Lite will have a smaller binary size, fewer
   dependencies, and better performance.
 
-- TensorFlow Lite is in developer preview, so not all use cases are covered yet.
-  We expect you to use TensorFlow Mobile to cover production cases.
-
 - TensorFlow Lite supports only a limited set of operators, so not all models
   will work on it by default. TensorFlow for Mobile has a fuller set of
   supported functionality.
diff --git a/tensorflow/docs_src/mobile/tflite/index.md b/tensorflow/docs_src/mobile/tflite/index.md
index 3d1733024e..cc4af2a875 100644
--- a/tensorflow/docs_src/mobile/tflite/index.md
+++ b/tensorflow/docs_src/mobile/tflite/index.md
@@ -70,10 +70,9 @@ There are several factors which are fueling interest in this domain:
 We believe the next wave of machine learning applications will have significant
 processing on mobile and embedded devices.
 
-## TensorFlow Lite developer preview highlights
+## TensorFlow Lite highlights
 
-TensorFlow Lite is available as a developer preview and includes the
-following:
+TensorFlow Lite provides:
 
 - A set of core operators, both quantized and float, many of which have been
   tuned for mobile platforms.  These can be used to create and run custom
@@ -129,9 +128,6 @@ following:
 
 - Java and C++ API support
 
-Note: This is a developer release, and it’s likely that there will be changes in
-the API in upcoming versions. We do not guarantee backward or forward
-compatibility with this release.
 
 ## Getting Started
 
@@ -201,9 +197,5 @@ possible performance for a particular model on a particular device.
 
 ## Next Steps
 
-For the developer preview, most of our documentation is on GitHub. Please take a
-look at the [TensorFlow Lite
-repository](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite)
-on GitHub for more information and for code samples, demo applications, and
-more.
-
+The TensorFlow Lite [GitHub repository](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite).
+contains additional docs, code samples, and demo applications.
-- 
cgit v1.2.3


From f1a1496148ea8a828e37201b8d0ab5d7e4979a1a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 12:55:42 -0700
Subject: Fixing gather to support round-tripping non-zero axis.

PiperOrigin-RevId: 205122011
---
 tensorflow/contrib/lite/toco/BUILD                 |  2 +
 tensorflow/contrib/lite/toco/export_tensorflow.cc  | 18 +++++++-
 .../graph_transformations/graph_transformations.h  |  1 +
 .../graph_transformations/propagate_fixed_sizes.cc | 19 ++++++--
 .../resolve_constant_gather.cc                     | 12 +++--
 .../resolve_gather_attributes.cc                   | 53 ++++++++++++++++++++++
 .../unpartition_embedding_lookup.cc                |  2 +
 tensorflow/contrib/lite/toco/import_tensorflow.cc  | 13 +++++-
 tensorflow/contrib/lite/toco/model.h               |  7 ++-
 tensorflow/contrib/lite/toco/tflite/operator.cc    |  5 +-
 tensorflow/contrib/lite/toco/toco_tooling.cc       |  1 +
 11 files changed, 118 insertions(+), 15 deletions(-)
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/resolve_gather_attributes.cc

diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index 5e197e584c..c88079717d 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -93,6 +93,7 @@ cc_library(
         ":runtime",
         ":toco_port",
         "//tensorflow/core:lib",
+        "@com_google_absl//absl/types:optional",
     ],
 )
 
@@ -246,6 +247,7 @@ cc_library(
         "graph_transformations/resolve_constant_transpose.cc",
         "graph_transformations/resolve_constant_unary.cc",
         "graph_transformations/resolve_fake_quant_args_from_vars.cc",
+        "graph_transformations/resolve_gather_attributes.cc",
         "graph_transformations/resolve_multiply_by_zero.cc",
         "graph_transformations/resolve_pad_attributes.cc",
         "graph_transformations/resolve_padv2_attributes.cc",
diff --git a/tensorflow/contrib/lite/toco/export_tensorflow.cc b/tensorflow/contrib/lite/toco/export_tensorflow.cc
index 5c112ffc38..91bfb401e8 100644
--- a/tensorflow/contrib/lite/toco/export_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/export_tensorflow.cc
@@ -1138,13 +1138,27 @@ void ConvertFloorOperator(const Model& model, const FloorOperator& src_op,
 void ConvertGatherOperator(const Model& model, const GatherOperator& src_op,
                            GraphDef* tensorflow_graph) {
   tensorflow::NodeDef* gather_op = tensorflow_graph->add_node();
-  gather_op->set_op("Gather");
+  gather_op->set_op("GatherV2");
   gather_op->set_name(src_op.outputs[0]);
-  CHECK_EQ(src_op.inputs.size(), 2);
   *gather_op->add_input() = src_op.inputs[0];
   *gather_op->add_input() = src_op.inputs[1];
 
+  if (!src_op.axis) {
+    // Dynamic axis.
+    CHECK_EQ(src_op.inputs.size(), 3);
+    *gather_op->add_input() = src_op.inputs[2];
+  } else {
+    // Constant axis.
+    CHECK_EQ(src_op.inputs.size(), 2);
+    const string gather_axis =
+        AvailableArrayName(model, gather_op->name() + "/axis");
+    CreateIntTensorConst(gather_axis, {src_op.axis.value()}, {},
+                         tensorflow_graph);
+    *gather_op->add_input() = gather_axis;
+  }
+
   (*gather_op->mutable_attr())["Tindices"].set_type(DT_INT32);
+  (*gather_op->mutable_attr())["Taxis"].set_type(DT_INT32);
   const tensorflow::DataType params_type =
       GetTensorFlowDataType(model, src_op.inputs[0]);
   (*gather_op->mutable_attr())["Tparams"].set_type(params_type);
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
index 5cee08fd4c..b7634e28c6 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
@@ -195,6 +195,7 @@ DECLARE_GRAPH_TRANSFORMATION(Dequantize)
 DECLARE_GRAPH_TRANSFORMATION(UnpartitionEmbeddingLookup)
 DECLARE_GRAPH_TRANSFORMATION(ShuffleFCWeights)
 DECLARE_GRAPH_TRANSFORMATION(ResolveFakeQuantArgsFromVars)
+DECLARE_GRAPH_TRANSFORMATION(ResolveGatherAttributes)
 
 class PropagateDefaultMinMax : public GraphTransformation {
  public:
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index a250db9975..4275ee9a03 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1043,17 +1043,28 @@ void ProcessGatherOperator(Model* model, GatherOperator* op) {
     return;
   }
 
+  // Yield until the axis has been resolved.
+  if (!op->axis) {
+    return;
+  }
+  int axis = op->axis.value();
+
   const auto& input_shape = input_array.shape();
   const auto& indices_shape = indices_array.shape();
   QCHECK_GE(input_shape.dimensions_count(), 1);
   op->input_rank = input_shape.dimensions_count();
+  QCHECK_LT(axis, op->input_rank);
 
-  // Copy the input dimensions to the output except for dimension 0,
+  // Copy the input dimensions to the output except for the axis dimensions
   // where the dimension of indices_shape is used.
-  // TODO(mgubin): if axis != 0 this is not true, change when it's supported.
   auto output_dims = output_array.mutable_shape()->mutable_dims();
-  output_dims->push_back(indices_shape.dims(0));
-  for (int dim = 1; dim < input_shape.dimensions_count(); dim++) {
+  for (int dim = 0; dim < axis; ++dim) {
+    output_dims->push_back(input_shape.dims(dim));
+  }
+  for (int dim = 0; dim < indices_shape.dimensions_count(); ++dim) {
+    output_dims->push_back(indices_shape.dims(dim));
+  }
+  for (int dim = axis + 1; dim < input_shape.dimensions_count(); ++dim) {
     output_dims->push_back(input_shape.dims(dim));
   }
 }
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_gather.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_gather.cc
index debe298a5a..36d7dad0ce 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_gather.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_gather.cc
@@ -69,7 +69,7 @@ bool ResolveConstantGather::Run(Model* model, std::size_t op_index) {
   }
   const auto* op = static_cast(base_op);
 
-  CHECK_EQ(op->inputs.size(), 2);
+  CHECK_GE(op->inputs.size(), 2);
   CHECK_EQ(op->outputs.size(), 1);
   auto& output_array = model->GetArray(op->outputs[0]);
   if (output_array.data_type == ArrayDataType::kNone) {
@@ -81,10 +81,14 @@ bool ResolveConstantGather::Run(Model* model, std::size_t op_index) {
     return false;
   }
 
-  // Only handling axis=0 for now.
-  if (op->axis != 0) {
+  if (!op->axis) {
+    // Yield until axis has been set by ResolveGatherAttributes.
+    return false;
+  }
+  if (op->axis.value() != 0) {
+    // Only handling axis=0 for now.
     AddMessageF("%s has axis %d; only axis=0 is supported", LogName(*op),
-                op->axis);
+                op->axis.value());
     return false;
   }
 
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_gather_attributes.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_gather_attributes.cc
new file mode 100644
index 0000000000..ce825c91af
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_gather_attributes.cc
@@ -0,0 +1,53 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include 
+#include 
+#include 
+#include 
+
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace toco {
+
+bool ResolveGatherAttributes::Run(Model* model, std::size_t op_index) {
+  auto* gather_op = model->operators[op_index].get();
+  if (gather_op->type != OperatorType::kGather) return false;
+  auto* op = static_cast(gather_op);
+
+  if (op->axis) {
+    // Attributes already resolved
+    return false;
+  }
+  if (op->inputs.size() != 3) return false;
+  if (!IsConstantParameterArray(*model, op->inputs[2])) return false;
+
+  const auto& indices_array = model->GetArray(op->inputs[2]);
+  if (!indices_array.has_shape()) return false;
+  const auto& axis_data = indices_array.GetBuffer().data;
+  CHECK_EQ(axis_data.size(), 1)
+      << "Multidimensional gather not supported on " << LogName(*op);
+  op->axis = {axis_data[0]};
+
+  // Drop the axis array as we no longer need it.
+  DeleteArrayIfUsedOnce(op->inputs[2], model);
+  op->inputs.resize(2);
+
+  return true;
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc b/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc
index cbea39bcc0..dd9e26e68b 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc
@@ -187,6 +187,7 @@ bool UnpartitionEmbeddingLookup::Run(Model* model, std::size_t op_index) {
       AvailableArrayName(*model, gather_ops[0]->inputs[0] + "_permuted/perm"));
   gather_params_permute_op->outputs.push_back(
       AvailableArrayName(*model, gather_ops[0]->inputs[0] + "_permuted"));
+  gather_params_permute_op->axis = {0};
   op_it = model->operators.emplace(op_it, gather_params_permute_op) + 1;
   model->GetOrCreateArray(gather_params_permute_op->outputs[0]);
   const auto& partition_array = model->GetArray(gather_ops[0]->inputs[0]);
@@ -212,6 +213,7 @@ bool UnpartitionEmbeddingLookup::Run(Model* model, std::size_t op_index) {
                               mod_op->inputs[0]};
   merged_gather_op->outputs = {stitch_op->outputs[0]};
   merged_gather_op->input_rank = partition_array.shape().dimensions_count();
+  merged_gather_op->axis = {0};
   model->operators.emplace(op_it, merged_gather_op);
 
   AddMessageF(
diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc
index 0d7eff5db4..9dde7a8bd6 100644
--- a/tensorflow/contrib/lite/toco/import_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc
@@ -1197,8 +1197,17 @@ tensorflow::Status ConvertGatherOperator(
   auto* op = new GatherOperator;
   op->inputs.push_back(node.input(0));
   op->inputs.push_back(node.input(1));
-  // TODO(ahentz): we currently ignore the third tensor in GatherV2 but we
-  // should read it an pass it on to the TF Lite Interpreter.
+  if (node.input_size() >= 3) {
+    // GatherV2 form where we are provided an axis. It may be either a constant
+    // or runtime defined value, so we just wire up the array and let
+    // ResolveGatherAttributes take care of it later on.
+    const auto axis_data_type = GetDataTypeAttr(node, "Taxis");
+    CHECK(axis_data_type == DT_INT32 || axis_data_type == DT_INT64);
+    op->inputs.push_back(node.input(2));
+  } else {
+    // Gather form that assumes axis=0.
+    op->axis = {0};
+  }
   op->outputs.push_back(node.name());
   model->operators.emplace_back(op);
   return tensorflow::Status::OK();
diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h
index 8fff68cf47..15f5d9c354 100644
--- a/tensorflow/contrib/lite/toco/model.h
+++ b/tensorflow/contrib/lite/toco/model.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include 
 #include 
 
+#include "absl/types/optional.h"
 #include "tensorflow/contrib/lite/toco/model_flags.pb.h"
 #include "tensorflow/contrib/lite/toco/runtime/types.h"
 #include "tensorflow/contrib/lite/toco/toco_port.h"
@@ -1525,11 +1526,15 @@ struct FloorOperator : Operator {
 // Inputs:
 //   inputs[0]: required: the params array
 //   inputs[1]: required: the indices to gather
+//   inputs[2]: optional: axis
 //
 // TensorFlow equivalent: Gather
 struct GatherOperator : Operator {
   GatherOperator() : Operator(OperatorType::kGather) {}
-  int axis = 0;
+  // Axis is populated explicitly or implicitly from the axis input by
+  // ResolveGatherAttributes. An empty axis indicates that the axis has not yet
+  // be resolved.
+  absl::optional axis;
   int input_rank = 0;
 };
 
diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc
index 68d13586f1..1a1c4b8944 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator.cc
@@ -370,12 +370,13 @@ class Gather : public BuiltinOperator WriteOptions(
       const TocoOperator& op,
       flatbuffers::FlatBufferBuilder* builder) const override {
-    return ::tflite::CreateGatherOptions(*builder, op.axis);
+    int axis = op.axis ? op.axis.value() : 0;
+    return ::tflite::CreateGatherOptions(*builder, axis);
   }
 
   void ReadOptions(const TfLiteOptions& options,
                    TocoOperator* op) const override {
-    op->axis = options.axis();
+    op->axis = {options.axis()};
   }
 
   int GetVersion(const Operator& op) const override { return 1; }
diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc
index d8964ebc13..aa7f6996eb 100644
--- a/tensorflow/contrib/lite/toco/toco_tooling.cc
+++ b/tensorflow/contrib/lite/toco/toco_tooling.cc
@@ -117,6 +117,7 @@ void MakeGeneralGraphTransformationsSet(
   transformations->Add(new ResolveConstantShapeOrRank);
   transformations->Add(new MakeInitialDequantizeOperator);
   transformations->Add(new UnpartitionEmbeddingLookup);
+  transformations->Add(new ResolveGatherAttributes);
 }
 
 bool SupportsQuantization(FileFormat format) {
-- 
cgit v1.2.3


From f836031f97d4b2ef2160f71332ee9c96a20ce84a Mon Sep 17 00:00:00 2001
From: Pavithra Vijay 
Date: Wed, 18 Jul 2018 13:19:15 -0700
Subject: Fix warning message when output not in loss dictionary. Output may be
 used just in summary so this message can be misleading.

PiperOrigin-RevId: 205125644
---
 tensorflow/python/keras/engine/training.py      |  7 +++----
 tensorflow/python/keras/engine/training_test.py | 23 +++++++++++++++++++++++
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 573422e533..fbc2a11eda 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -217,10 +217,9 @@ class Model(Network):
       for name in self.output_names:
         if name not in loss:
           logging.warning(
-              'Output "' + name + '" missing from loss dictionary. '
-              'We assume this was done on purpose, '
-              'and we will not be expecting '
-              'any data to be passed to "' + name + '" during training.')
+              'Output "' + name + '" missing from loss dictionary. We assume '
+              'this was done on purpose. The fit and evaluate APIs will not be '
+              'expecting any data to be passed to "' + name + '".')
         loss_functions.append(losses.get(loss.get(name)))
     elif isinstance(loss, list):
       if len(loss) != len(self.outputs):
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index d9e548f01f..c621a88fb3 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import logging
 import os
 import unittest
 
@@ -415,6 +416,28 @@ class TrainingTest(test.TestCase):
       x2 = model.predict(val_a)
       self.assertAllClose(x1, x2, atol=1e-7)
 
+  def test_compile_warning_for_loss_missing_output(self):
+    with self.test_session():
+      inp = keras.layers.Input(shape=(16,), name='input_a')
+      out_1 = keras.layers.Dense(8, name='dense_1')(inp)
+      out_2 = keras.layers.Dense(3, activation='softmax', name='dense_2')(out_1)
+      model = keras.models.Model(inputs=[inp], outputs=[out_1, out_2])
+
+      with test.mock.patch.object(logging, 'warning') as mock_log:
+        model.compile(
+            loss={
+                'dense_2': 'categorical_crossentropy',
+            },
+            optimizer='rmsprop',
+            metrics={
+                'dense_2': 'categorical_accuracy',
+                'dense_1': 'categorical_accuracy',
+            })
+        msg = ('Output "dense_1" missing from loss dictionary. We assume this '
+               'was done on purpose. The fit and evaluate APIs will not be '
+               'expecting any data to be passed to "dense_1".')
+        self.assertRegexpMatches(str(mock_log.call_args), msg)
+
 
 class LossWeightingTest(test.TestCase):
 
-- 
cgit v1.2.3


From d5b6a46986cfeaa852a0ff512bd1e6464886e58e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 13:44:34 -0700
Subject: Fix bug when converting branches that have no direct effects (only
 side-effects).

PiperOrigin-RevId: 205129625
---
 tensorflow/contrib/autograph/converters/control_flow.py      | 4 ++--
 tensorflow/contrib/autograph/converters/control_flow_test.py | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/autograph/converters/control_flow.py b/tensorflow/contrib/autograph/converters/control_flow.py
index a25232f713..5a5a6ad63a 100644
--- a/tensorflow/contrib/autograph/converters/control_flow.py
+++ b/tensorflow/contrib/autograph/converters/control_flow.py
@@ -171,8 +171,8 @@ class ControlFlowTransformer(converter.Base):
       # actually has some return value as well.
       cond_results = None
       # TODO(mdan): This doesn't belong here; it's specific to the operator.
-      returned_from_body = templates.replace_as_expression('1')
-      returned_from_orelse = templates.replace_as_expression('1')
+      returned_from_body = templates.replace_as_expression('tf.constant(1)')
+      returned_from_orelse = templates.replace_as_expression('tf.constant(1)')
 
     body_name = self.ctx.namer.new_symbol('if_true', body_scope.referenced)
     orelse_name = self.ctx.namer.new_symbol('if_false', orelse_scope.referenced)
diff --git a/tensorflow/contrib/autograph/converters/control_flow_test.py b/tensorflow/contrib/autograph/converters/control_flow_test.py
index 6670b8a66f..ade3501426 100644
--- a/tensorflow/contrib/autograph/converters/control_flow_test.py
+++ b/tensorflow/contrib/autograph/converters/control_flow_test.py
@@ -31,7 +31,8 @@ class ControlFlowTest(converter_testing.TestCase):
   def assertTransformedResult(self, test_fn, inputs, expected):
     if not isinstance(inputs, tuple):
       inputs = (inputs,)
-    with self.converted(test_fn, control_flow, {}) as result:
+    with self.converted(test_fn, control_flow, {},
+                        constant_op.constant) as result:
       with self.test_session() as sess:
         self.assertEqual(sess.run(result.test_fn(*inputs)), expected)
 
-- 
cgit v1.2.3


From e7c38706e68fad461a2e894dbf7b64dcf607f488 Mon Sep 17 00:00:00 2001
From: Jiri Simsa 
Date: Wed, 18 Jul 2018 13:45:51 -0700
Subject: Adding CPU and GPU kernels for casting to and from uint32 and uint64.

PiperOrigin-RevId: 205129878
---
 tensorflow/contrib/makefile/tf_op_files.txt    |  2 ++
 tensorflow/core/kernels/BUILD                  |  2 ++
 tensorflow/core/kernels/cast_op.cc             | 24 ++++++++++----
 tensorflow/core/kernels/cast_op_gpu.cu.cc      |  8 +++--
 tensorflow/core/kernels/cast_op_impl.h         | 30 ++++++++++++++---
 tensorflow/core/kernels/cast_op_impl_uint32.cc | 46 ++++++++++++++++++++++++++
 tensorflow/core/kernels/cast_op_impl_uint64.cc | 46 ++++++++++++++++++++++++++
 tensorflow/core/kernels/cast_op_test.cc        |  4 +++
 8 files changed, 149 insertions(+), 13 deletions(-)
 create mode 100644 tensorflow/core/kernels/cast_op_impl_uint32.cc
 create mode 100644 tensorflow/core/kernels/cast_op_impl_uint64.cc

diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index 6e7423f85e..ecf2e120df 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -229,6 +229,8 @@ tensorflow/core/kernels/cast_op_impl_int32.cc
 tensorflow/core/kernels/cast_op_impl_int64.cc
 tensorflow/core/kernels/cast_op_impl_int8.cc
 tensorflow/core/kernels/cast_op_impl_uint16.cc
+tensorflow/core/kernels/cast_op_impl_uint32.cc
+tensorflow/core/kernels/cast_op_impl_uint64.cc
 tensorflow/core/kernels/cast_op_impl_uint8.cc
 tensorflow/core/kernels/boosted_trees/prediction_ops.cc
 tensorflow/core/kernels/boosted_trees/resource_ops.cc
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 1c842150fd..99e5e3cfca 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4855,6 +4855,8 @@ filegroup(
         "cast_op_impl_int64.cc",
         "cast_op_impl_int8.cc",
         "cast_op_impl_uint16.cc",
+        "cast_op_impl_uint32.cc",
+        "cast_op_impl_uint64.cc",
         "cast_op_impl_uint8.cc",
         "concat_lib.h",
         "concat_lib_cpu.cc",
diff --git a/tensorflow/core/kernels/cast_op.cc b/tensorflow/core/kernels/cast_op.cc
index 626db9131a..e6e388b3d1 100644
--- a/tensorflow/core/kernels/cast_op.cc
+++ b/tensorflow/core/kernels/cast_op.cc
@@ -41,8 +41,10 @@ typedef Eigen::SyclDevice SYCLDevice;
 #define CURRY_TYPES2(FN, arg0)   \
   FN(arg0, bool);                \
   FN(arg0, uint8);               \
-  FN(arg0, int8);                \
   FN(arg0, uint16);              \
+  FN(arg0, uint32);              \
+  FN(arg0, uint64);              \
+  FN(arg0, int8);                \
   FN(arg0, int16);               \
   FN(arg0, int32);               \
   FN(arg0, int64);               \
@@ -86,10 +88,14 @@ Status CpuCastOp::Prepare() {
     work_ = GetCpuCastFromBool(dst_dtype_);
   } else if (src_dtype_ == DT_UINT8) {
     work_ = GetCpuCastFromUint8(dst_dtype_);
-  } else if (src_dtype_ == DT_INT8) {
-    work_ = GetCpuCastFromInt8(dst_dtype_);
   } else if (src_dtype_ == DT_UINT16) {
     work_ = GetCpuCastFromUint16(dst_dtype_);
+  } else if (src_dtype_ == DT_UINT32) {
+    work_ = GetCpuCastFromUint32(dst_dtype_);
+  } else if (src_dtype_ == DT_UINT64) {
+    work_ = GetCpuCastFromUint64(dst_dtype_);
+  } else if (src_dtype_ == DT_INT8) {
+    work_ = GetCpuCastFromInt8(dst_dtype_);
   } else if (src_dtype_ == DT_INT16) {
     work_ = GetCpuCastFromInt16(dst_dtype_);
   } else if (src_dtype_ == DT_INT32) {
@@ -135,10 +141,14 @@ class GpuCastOp : public CastOpBase {
       work_ = GetGpuCastFromBool(dst_dtype_);
     } else if (src_dtype_ == DT_UINT8) {
       work_ = GetGpuCastFromUint8(dst_dtype_);
-    } else if (src_dtype_ == DT_INT8) {
-      work_ = GetGpuCastFromInt8(dst_dtype_);
     } else if (src_dtype_ == DT_UINT16) {
       work_ = GetGpuCastFromUint16(dst_dtype_);
+    } else if (src_dtype_ == DT_UINT32) {
+      work_ = GetGpuCastFromUint32(dst_dtype_);
+    } else if (src_dtype_ == DT_UINT64) {
+      work_ = GetGpuCastFromUint64(dst_dtype_);
+    } else if (src_dtype_ == DT_INT8) {
+      work_ = GetGpuCastFromInt8(dst_dtype_);
     } else if (src_dtype_ == DT_INT16) {
       work_ = GetGpuCastFromInt16(dst_dtype_);
     } else if (src_dtype_ == DT_INT32) {
@@ -178,8 +188,10 @@ REGISTER_KERNEL_BUILDER(Name("Cast").Device(DEVICE_CPU), CpuCastOp);
 
 CURRY_TYPES2(REGISTER_CAST_GPU, bool);
 CURRY_TYPES2(REGISTER_CAST_GPU, uint8);
-CURRY_TYPES2(REGISTER_CAST_GPU, int8);
 CURRY_TYPES2(REGISTER_CAST_GPU, uint16);
+CURRY_TYPES2(REGISTER_CAST_GPU, uint32);
+CURRY_TYPES2(REGISTER_CAST_GPU, uint64);
+CURRY_TYPES2(REGISTER_CAST_GPU, int8);
 CURRY_TYPES2(REGISTER_CAST_GPU, int16);
 CURRY_TYPES2(REGISTER_CAST_GPU, int32);
 CURRY_TYPES2(REGISTER_CAST_GPU, int64);
diff --git a/tensorflow/core/kernels/cast_op_gpu.cu.cc b/tensorflow/core/kernels/cast_op_gpu.cu.cc
index 9c9e9e7658..607e7f5efd 100644
--- a/tensorflow/core/kernels/cast_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/cast_op_gpu.cu.cc
@@ -37,8 +37,10 @@ struct CastFunctor {
 #define DEFINE_ALL_FROM(in_type)        \
   DEFINE(in_type, bool);                \
   DEFINE(in_type, uint8);               \
-  DEFINE(in_type, int8);                \
   DEFINE(in_type, uint16);              \
+  DEFINE(in_type, uint32);              \
+  DEFINE(in_type, uint64);              \
+  DEFINE(in_type, int8);                \
   DEFINE(in_type, int16);               \
   DEFINE(in_type, int32);               \
   DEFINE(in_type, int64);               \
@@ -50,8 +52,10 @@ struct CastFunctor {
 
 DEFINE_ALL_FROM(bool);
 DEFINE_ALL_FROM(uint8);
-DEFINE_ALL_FROM(int8);
 DEFINE_ALL_FROM(uint16);
+DEFINE_ALL_FROM(uint32);
+DEFINE_ALL_FROM(uint64);
+DEFINE_ALL_FROM(int8);
 DEFINE_ALL_FROM(int16);
 DEFINE_ALL_FROM(int32);
 DEFINE_ALL_FROM(int64);
diff --git a/tensorflow/core/kernels/cast_op_impl.h b/tensorflow/core/kernels/cast_op_impl.h
index 382e5440e1..fe821b25df 100644
--- a/tensorflow/core/kernels/cast_op_impl.h
+++ b/tensorflow/core/kernels/cast_op_impl.h
@@ -48,8 +48,10 @@ struct CastFunctor {
 #define CURRY_TYPES3_NO_HALF(FN, arg0, arg1) \
   FN(arg0, arg1, bool);                      \
   FN(arg0, arg1, uint8);                     \
-  FN(arg0, arg1, int8);                      \
   FN(arg0, arg1, uint16);                    \
+  FN(arg0, arg1, uint32);                    \
+  FN(arg0, arg1, uint64);                    \
+  FN(arg0, arg1, int8);                      \
   FN(arg0, arg1, int16);                     \
   FN(arg0, arg1, int32);                     \
   FN(arg0, arg1, int64);                     \
@@ -82,10 +84,16 @@ std::function
 GetCpuCastFromUint8(DataType dst_dtype);
 
 std::function
-GetCpuCastFromInt8(DataType dst_dtype);
+GetCpuCastFromUint16(DataType dst_dtype);
 
 std::function
-GetCpuCastFromUint16(DataType dst_dtype);
+GetCpuCastFromUint32(DataType dst_dtype);
+
+std::function
+GetCpuCastFromUint64(DataType dst_dtype);
+
+std::function
+GetCpuCastFromInt8(DataType dst_dtype);
 
 std::function
 GetCpuCastFromInt16(DataType dst_dtype);
@@ -123,10 +131,16 @@ std::function
 GetGpuCastFromUint8(DataType dst_dtype);
 
 std::function
-GetGpuCastFromInt8(DataType dst_dtype);
+GetGpuCastFromUint16(DataType dst_dtype);
 
 std::function
-GetGpuCastFromUint16(DataType dst_dtype);
+GetGpuCastFromUint32(DataType dst_dtype);
+
+std::function
+GetGpuCastFromUint64(DataType dst_dtype);
+
+std::function
+GetGpuCastFromInt8(DataType dst_dtype);
 
 std::function
 GetGpuCastFromInt16(DataType dst_dtype);
@@ -167,6 +181,12 @@ GetSyclCastFromUint8(DataType dst_dtype);
 std::function
 GetSyclCastFromUint16(DataType dst_dtype);
 
+std::function
+GetSyclCastFromUint32(DataType dst_dtype);
+
+std::function
+GetSyclCastFromUint64(DataType dst_dtype);
+
 std::function
 GetSyclCastFromInt16(DataType dst_dtype);
 
diff --git a/tensorflow/core/kernels/cast_op_impl_uint32.cc b/tensorflow/core/kernels/cast_op_impl_uint32.cc
new file mode 100644
index 0000000000..d1a854d98b
--- /dev/null
+++ b/tensorflow/core/kernels/cast_op_impl_uint32.cc
@@ -0,0 +1,46 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/cast_op_impl.h"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
+
+std::function
+GetCpuCastFromUint32(DataType dst_dtype) {
+  CURRY_TYPES3(CAST_CASE, CPUDevice, uint32);
+  return nullptr;
+}
+
+#if GOOGLE_CUDA
+std::function
+GetGpuCastFromUint32(DataType dst_dtype) {
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, uint32);
+  return nullptr;
+}
+#endif  // GOOGLE_CUDA
+
+#ifdef TENSORFLOW_USE_SYCL
+typedef Eigen::SyclDevice SYCLDevice;
+std::function
+GetSyclCastFromUint32(DataType dst_dtype) {
+  CURRY_TYPES3_NO_HALF(CAST_CASE, SYCLDevice, uint32);
+  return nullptr;
+}
+#endif  // TENSORFLOW_USE_SYCL
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/cast_op_impl_uint64.cc b/tensorflow/core/kernels/cast_op_impl_uint64.cc
new file mode 100644
index 0000000000..604e0424fc
--- /dev/null
+++ b/tensorflow/core/kernels/cast_op_impl_uint64.cc
@@ -0,0 +1,46 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/cast_op_impl.h"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
+
+std::function
+GetCpuCastFromUint64(DataType dst_dtype) {
+  CURRY_TYPES3(CAST_CASE, CPUDevice, uint64);
+  return nullptr;
+}
+
+#if GOOGLE_CUDA
+std::function
+GetGpuCastFromUint64(DataType dst_dtype) {
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, uint64);
+  return nullptr;
+}
+#endif  // GOOGLE_CUDA
+
+#ifdef TENSORFLOW_USE_SYCL
+typedef Eigen::SyclDevice SYCLDevice;
+std::function
+GetSyclCastFromUint64(DataType dst_dtype) {
+  CURRY_TYPES3_NO_HALF(CAST_CASE, SYCLDevice, uint64);
+  return nullptr;
+}
+#endif  // TENSORFLOW_USE_SYCL
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/cast_op_test.cc b/tensorflow/core/kernels/cast_op_test.cc
index 7da9d28a3d..75e21802c0 100644
--- a/tensorflow/core/kernels/cast_op_test.cc
+++ b/tensorflow/core/kernels/cast_op_test.cc
@@ -70,6 +70,8 @@ class CastOpTest : public OpsTestBase {
 #define TEST_ALL_CASTS_FROM(in) \
   TEST_CAST(in, uint8);         \
   TEST_CAST(in, uint16);        \
+  TEST_CAST(in, uint32);        \
+  TEST_CAST(in, uint64);        \
   TEST_CAST(in, int16);         \
   TEST_CAST(in, int32);         \
   TEST_CAST(in, int64);         \
@@ -80,6 +82,8 @@ class CastOpTest : public OpsTestBase {
 
 TEST_ALL_CASTS_FROM(uint8)
 TEST_ALL_CASTS_FROM(uint16)
+TEST_ALL_CASTS_FROM(uint32)
+TEST_ALL_CASTS_FROM(uint64)
 TEST_ALL_CASTS_FROM(int16)
 TEST_ALL_CASTS_FROM(int32)
 TEST_ALL_CASTS_FROM(int64)
-- 
cgit v1.2.3


From c9d4dddab10f92f98c1038f0fac136ab4f3724b6 Mon Sep 17 00:00:00 2001
From: Tong Shen 
Date: Wed, 18 Jul 2018 13:46:54 -0700
Subject: Add some debug information.

PiperOrigin-RevId: 205130083
---
 .../compiler/jit/encapsulate_subgraphs_pass.cc     | 40 +++++++++++++++++++---
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
index 9c424b201e..fdd71c6a58 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
@@ -138,7 +138,7 @@ class Encapsulator {
 
   // Find subgraphs marked with 'group_attribute', and build a new
   // subgraph, one for each value of 'group_attribute'.
-  Status SplitIntoSubgraphs();
+  Status SplitIntoSubgraphs(FunctionLibraryDefinition* library);
 
   // Build a FunctionDef for each subgraph, and add it 'library'. The values of
   // the 'group_attribute' annotations become the function names.
@@ -1478,7 +1478,7 @@ Status Encapsulator::CopySubgraphEdges(
   return Status::OK();
 }
 
-Status Encapsulator::SplitIntoSubgraphs() {
+Status Encapsulator::SplitIntoSubgraphs(FunctionLibraryDefinition* library) {
   Status s;
 
   // Map from input graph nodes to subgraph nodes.
@@ -1513,6 +1513,15 @@ Status Encapsulator::SplitIntoSubgraphs() {
     TF_RETURN_IF_ERROR(BuildControlFlowInfo(subgraph.GetGraph(), &dummy));
   }
 
+  if (VLOG_IS_ON(1)) {
+    // Dump subgraphs.
+    for (auto& entry : subgraphs_) {
+      dump_graph::DumpGraphToFile(
+          strings::StrCat("encapsulate_subgraphs_subgraph_", entry.first),
+          *entry.second.GetGraph(), library);
+    }
+  }
+
   return s;
 }
 
@@ -1936,6 +1945,8 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend(
             // continue.
             TensorShapeProto proto;
             context->ShapeHandleToProto(shape, &proto);
+            VLOG(2) << "Node " << src_node->name()
+                    << " has known shape: " << proto.DebugString();
             if (dummy_node_images.find(src_node) == dummy_node_images.end()) {
               dummy_node_images[src_node] =
                   AddDummyShapedNode(src_node, src_port, control_flow_info,
@@ -1953,6 +1964,8 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend(
               if (VLOG_IS_ON(2)) {
                 TensorShapeProto proto;
                 context->ShapeHandleToProto(shape, &proto);
+                VLOG(2) << "Node " << src_node->name()
+                        << " has unknown shape: " << proto.DebugString();
               }
               stack.push_back({src_node, false});
             }
@@ -2195,6 +2208,23 @@ Status Encapsulator::FindClusterDependencies() {
       }
     }
   }
+  if (VLOG_IS_ON(2)) {
+    // Print debug information.
+    VLOG(2) << "node_ancestors_map:";
+    for (const auto& node_iter : node_ancestors_map) {
+      VLOG(2) << "\t" << node_iter.first->name() << ": subgraph = '"
+              << node_iter.second.subgraph
+              << "', outside_compilation_cluster = '"
+              << node_iter.second.outside_compilation_cluster
+              << "', ancestor_clusters: "
+              << (node_iter.second.ancestor_clusters.empty() ? "(empty)" : "");
+      for (const auto& cluster_iter : node_iter.second.ancestor_clusters) {
+        VLOG(2) << "\t\tsubgraph = '" << cluster_iter.subgraph
+                << "', outside_compilation_cluster = '"
+                << cluster_iter.outside_compilation_cluster << "'";
+      }
+    }
+  }
   return Status::OK();
 }
 
@@ -2402,7 +2432,7 @@ Status EncapsulateSubgraphsInFunctions(
                             std::move(outside_compilation_attribute),
                             &graph_in);
   TF_RETURN_IF_ERROR(encapsulator.FindClusterDependencies());
-  TF_RETURN_IF_ERROR(encapsulator.SplitIntoSubgraphs());
+  TF_RETURN_IF_ERROR(encapsulator.SplitIntoSubgraphs(library));
 
   TF_RETURN_IF_ERROR(encapsulator.BuildFunctionDefs(
       rewrite_subgraph_fn, reuse_existing_functions, library));
@@ -2451,7 +2481,7 @@ Status EncapsulateSubgraphsPass::Run(
     const GraphOptimizationPassOptions& options) {
   VLOG(1) << "EncapsulateSubgraphsPass::Run";
   if (VLOG_IS_ON(1)) {
-    dump_graph::DumpGraphToFile("before_encapsulate_subgraphs", **options.graph,
+    dump_graph::DumpGraphToFile("encapsulate_subgraphs_before", **options.graph,
                                 options.flib_def);
   }
 
@@ -2534,7 +2564,7 @@ Status EncapsulateSubgraphsPass::Run(
       "EncapsulateSubgraphsPass failed");
 
   if (VLOG_IS_ON(1)) {
-    dump_graph::DumpGraphToFile("after_encapsulate_subgraphs", *graph_out,
+    dump_graph::DumpGraphToFile("encapsulate_subgraphs_after", *graph_out,
                                 options.flib_def);
   }
 
-- 
cgit v1.2.3


From 44af531d952a35c887770ecc4cfddfb0431c2478 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 13:54:27 -0700
Subject: Prevent removal of reshapes that need to remain for proper shape
 information.

PiperOrigin-RevId: 205131437
---
 .../lite/toco/graph_transformations/remove_trivial_reshape.cc    | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_reshape.cc b/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_reshape.cc
index 404f27e067..5295eeccec 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_reshape.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_reshape.cc
@@ -59,6 +59,15 @@ bool IsReshapeTrivial(const Model& model, const Operator& op,
   if (CountOpsWithInput(model, op.outputs[0]) == 1) {
     const auto* next_op = GetOpWithInput(model, op.outputs[0]);
     if (next_op->type == OperatorType::kReshape) {
+      if (!IsDiscardableArray(model, next_op->outputs[0])) {
+        // If the |next_op| output is used as a model output we need to preserve
+        // its shape.
+        transformation->AddMessageF(
+            "%s cannot be merged into following reshape %s as it is "
+            "non-discardable and must keep the specified shape",
+            LogName(op), LogName(*next_op));
+        return false;
+      }
       transformation->AddMessageF(
           "%s is trivial because its output is only consumed by another "
           "Reshape op %s",
-- 
cgit v1.2.3


From 4ca04537c0d1d75ea37944aa3bb2dc749428031a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 14:12:13 -0700
Subject: Import/export support for Any, LogicalAnd, and LogicalNot ops.

PiperOrigin-RevId: 205134621
---
 tensorflow/contrib/lite/toco/export_tensorflow.cc  | 55 +++++++++++++++++
 .../propagate_array_data_types.cc                  |  3 +
 .../graph_transformations/propagate_fixed_sizes.cc | 68 +++++++++++++++++++++-
 .../resolve_constant_unary.cc                      |  4 +-
 .../resolve_reduce_attributes.cc                   |  2 +
 tensorflow/contrib/lite/toco/import_tensorflow.cc  | 39 +++++++------
 tensorflow/contrib/lite/toco/model.h               | 47 +++++++++++++--
 tensorflow/contrib/lite/toco/tooling_util.cc       |  9 ++-
 8 files changed, 197 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/export_tensorflow.cc b/tensorflow/contrib/lite/toco/export_tensorflow.cc
index 91bfb401e8..f9a6d31d60 100644
--- a/tensorflow/contrib/lite/toco/export_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/export_tensorflow.cc
@@ -1680,6 +1680,9 @@ void ConvertReduceOperator(const Model& model, const T& src_op,
   const tensorflow::DataType params_type =
       GetTensorFlowDataType(model, src_op.inputs[0]);
   (*new_op->mutable_attr())["T"].set_type(params_type);
+  const tensorflow::DataType indices_type =
+      GetTensorFlowDataType(model, src_op.inputs[1]);
+  (*new_op->mutable_attr())["Tidx"].set_type(indices_type);
 
   if (src_op.keep_dims) {
     (*new_op->mutable_attr())["keep_dims"].set_b(true);
@@ -1873,6 +1876,43 @@ void ConvertPowOperator(const Model& model, const PowOperator& src_op,
   (*pow_op->mutable_attr())["T"].set_type(data_type);
 }
 
+void ConvertAnyOperator(const Model& model, const AnyOperator& src_op,
+                        GraphDef* tensorflow_graph) {
+  tensorflow::NodeDef* any_op = tensorflow_graph->add_node();
+  any_op->set_op("Any");
+  any_op->set_name(src_op.outputs[0]);
+  CHECK_EQ(src_op.inputs.size(), 2);
+  for (int i = 0; i < 2; ++i) {
+    *any_op->add_input() = src_op.inputs[i];
+  }
+  const tensorflow::DataType data_type =
+      GetTensorFlowDataType(model, src_op.inputs[1]);
+  (*any_op->mutable_attr())["Tidx"].set_type(data_type);
+  (*any_op->mutable_attr())["keep_dims"].set_b(src_op.keep_dims);
+}
+
+void ConvertLogicalAndOperator(const Model& model,
+                               const LogicalAndOperator& src_op,
+                               GraphDef* tensorflow_graph) {
+  tensorflow::NodeDef* logical_op = tensorflow_graph->add_node();
+  logical_op->set_op("LogicalAnd");
+  logical_op->set_name(src_op.outputs[0]);
+  CHECK_EQ(src_op.inputs.size(), 2);
+  for (int i = 0; i < 2; ++i) {
+    *logical_op->add_input() = src_op.inputs[i];
+  }
+}
+
+void ConvertLogicalNotOperator(const Model& model,
+                               const LogicalNotOperator& src_op,
+                               GraphDef* tensorflow_graph) {
+  tensorflow::NodeDef* logical_op = tensorflow_graph->add_node();
+  logical_op->set_op("LogicalNot");
+  logical_op->set_name(src_op.outputs[0]);
+  CHECK_EQ(src_op.inputs.size(), 1);
+  *logical_op->add_input() = src_op.inputs[0];
+}
+
 void ConvertOperator(const Model& model, const Operator& src_op,
                      GraphDef* tensorflow_graph) {
   if (src_op.fused_activation_function != FusedActivationFunctionType::kNone) {
@@ -2019,6 +2059,10 @@ void ConvertOperator(const Model& model, const Operator& src_op,
     ConvertReduceOperator(model,
                           static_cast(src_op),
                           tensorflow_graph, "Prod");
+  } else if (src_op.type == OperatorType::kReduceMin) {
+    ConvertReduceOperator(model,
+                          static_cast(src_op),
+                          tensorflow_graph, "Min");
   } else if (src_op.type == OperatorType::kReduceMax) {
     ConvertReduceOperator(model,
                           static_cast(src_op),
@@ -2105,6 +2149,17 @@ void ConvertOperator(const Model& model, const Operator& src_op,
   } else if (src_op.type == OperatorType::kPow) {
     ConvertPowOperator(model, static_cast(src_op), "Pow",
                        tensorflow_graph);
+  } else if (src_op.type == OperatorType::kAny) {
+    ConvertAnyOperator(model, static_cast(src_op),
+                       tensorflow_graph);
+  } else if (src_op.type == OperatorType::kLogicalAnd) {
+    ConvertLogicalAndOperator(model,
+                              static_cast(src_op),
+                              tensorflow_graph);
+  } else if (src_op.type == OperatorType::kLogicalNot) {
+    ConvertLogicalNotOperator(model,
+                              static_cast(src_op),
+                              tensorflow_graph);
   } else {
     LOG(FATAL) << "Unhandled operator type " << OperatorTypeName(src_op.type);
   }
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc
index 670bcf64e7..3dda536ef7 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -62,6 +62,9 @@ bool PropagateArrayDataTypes::Run(Model* model, std::size_t op_index) {
     case OperatorType::kGreaterEqual:
     case OperatorType::kEqual:
     case OperatorType::kNotEqual:
+    case OperatorType::kAny:
+    case OperatorType::kLogicalAnd:
+    case OperatorType::kLogicalNot:
       // These operators unconditionally produce bool outputs
       SetDataTypeForAllOutputs(model, op, ArrayDataType::kBool);
       break;
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 4275ee9a03..62ed5c46e9 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -529,7 +529,7 @@ void ProcessAddNOperator(Model* model, Operator* op) {
 
 bool KeepDims(const Operator& op) {
   switch (op.type) {
-    case OperatorType::kMin:  //  Reduction Min
+    case OperatorType::kReduceMin:  //  Reduction Min
       return static_cast(op).keep_dims;
     case OperatorType::kReduceMax:  //  Reduction Max
       return static_cast(op).keep_dims;
@@ -1519,6 +1519,65 @@ void ProcessTileOperator(Model* model, TensorFlowTileOperator* op) {
   }
 }
 
+void ProcessAnyOperator(Model* model, AnyOperator* op) {
+  CHECK_EQ(op->inputs.size(), 2);
+  CHECK_EQ(op->outputs.size(), 1);
+
+  auto& output_array = model->GetArray(op->outputs[0]);
+  if (output_array.has_shape()) {
+    // We have already run.
+    return;
+  }
+
+  const auto& input_array = model->GetArray(op->inputs[0]);
+  if (!input_array.has_shape()) {
+    // Yield until input dims have been resolved.
+    return;
+  }
+  const auto& input_shape = input_array.shape();
+
+  auto& reduction_indices_array = model->GetArray(op->inputs[1]);
+  if (!reduction_indices_array.has_shape()) {
+    // Yield until reduction indices shape been resolved.
+    return;
+  }
+  if (!reduction_indices_array.buffer) {
+    // Yield until the reduction indices are constant.
+    return;
+  }
+  CHECK(reduction_indices_array.data_type == ArrayDataType::kInt32)
+      << "Any reduction input must be int32";
+
+  int input_rank = input_shape.dimensions_count();
+  std::set true_indices;
+  const auto& reduction_indices =
+      reduction_indices_array.GetBuffer().data;
+  for (int i = 0; i < reduction_indices.size(); ++i) {
+    const int32 reduction_index = reduction_indices[i];
+    if (reduction_index < -input_rank || reduction_index >= input_rank) {
+      CHECK(false) << "Invalid reduction dimension " << reduction_index
+                   << " for input with " << input_rank << " dimensions";
+    }
+    int32 wrapped_index = reduction_index;
+    if (wrapped_index < 0) {
+      wrapped_index += input_rank;
+    }
+    true_indices.insert(wrapped_index);
+  }
+
+  auto* mutable_dims = output_array.mutable_shape()->mutable_dims();
+  mutable_dims->clear();
+  for (int i = 0; i < input_rank; ++i) {
+    if (true_indices.count(i) > 0) {
+      if (op->keep_dims) {
+        mutable_dims->emplace_back(1);
+      }
+    } else {
+      mutable_dims->emplace_back(input_shape.dims(i));
+    }
+  }
+}
+
 }  // namespace
 
 bool PropagateFixedSizes::Run(Model* model, std::size_t op_index) {
@@ -1557,6 +1616,8 @@ bool PropagateFixedSizes::Run(Model* model, std::size_t op_index) {
     case OperatorType::kFloor:
     case OperatorType::kExp:
     case OperatorType::kSin:
+    case OperatorType::kLogicalAnd:
+    case OperatorType::kLogicalNot:
       ProcessSimpleOperator(model, op, 0);
       break;
     case OperatorType::kGather:
@@ -1625,7 +1686,7 @@ bool PropagateFixedSizes::Run(Model* model, std::size_t op_index) {
     case OperatorType::kL2Pool:
       ProcessL2PoolOperator(model, static_cast(op));
       break;
-    case OperatorType::kMin:  //  Reduction Min
+    case OperatorType::kReduceMin:  //  Reduction Min
     case OperatorType::kReduceMax:  //  Reduction Max
     case OperatorType::kSum:
     case OperatorType::kReduceProd:
@@ -1750,6 +1811,9 @@ bool PropagateFixedSizes::Run(Model* model, std::size_t op_index) {
     case OperatorType::kTile:
       ProcessTileOperator(model, static_cast(op));
       break;
+    case OperatorType::kAny:
+      ProcessAnyOperator(model, static_cast(op));
+      break;
     default:
       // Unimplemented, another graph transformation should drop it.
       LOG(FATAL) << "Unhandled operator type " << OperatorTypeName(op->type);
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc
index 51099cf74a..fe3882c28d 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc
@@ -57,7 +57,7 @@ bool ResolveConstantUnaryOperator::Run(Model* model, std::size_t op_index) {
     case OperatorType::kSqrt:
     case OperatorType::kSquare:
     case OperatorType::kSum:
-    case OperatorType::kMin:  //  Reduction Min
+    case OperatorType::kReduceMin:  //  Reduction Min
     case OperatorType::kReduceMax:  //  Reduction Max
     case OperatorType::kReshape:
     case OperatorType::kRelu6:
@@ -196,7 +196,7 @@ bool ResolveConstantUnaryOperator::Run(Model* model, std::size_t op_index) {
       }
       output_float_data[i] = sum;
     }
-  } else if (unary_op->type == OperatorType::kMin) {
+  } else if (unary_op->type == OperatorType::kReduceMin) {
     // At the moment only full reduction across all dimensions is supported.
     // TODO(starka): Output should not be padded.
     for (int i = 0; i < output_dims_count; i++) {
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_reduce_attributes.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_reduce_attributes.cc
index 5f8a06ba92..7d456af2fb 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_reduce_attributes.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_reduce_attributes.cc
@@ -48,6 +48,8 @@ bool ResolveReduceAttributes::Run(Model* model, std::size_t op_index) {
       return ResolveAttributes(model, static_cast(op));
     case OperatorType::kReduceProd:
       return ResolveAttributes(model, static_cast(op));
+    case OperatorType::kReduceMin:
+      return ResolveAttributes(model, static_cast(op));
     case OperatorType::kReduceMax:
       return ResolveAttributes(model, static_cast(op));
     default:
diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc
index 9dde7a8bd6..8bb797fe0f 100644
--- a/tensorflow/contrib/lite/toco/import_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc
@@ -1042,22 +1042,6 @@ tensorflow::Status ConvertSimpleOperator(
   return ConvertSimpleOperator(node, tf_import_flags, model);
 }
 
-tensorflow::Status ConvertMinOperator(
-    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
-    Model* model) {
-  CHECK_EQ(node.op(), "Min");
-  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 2));
-  auto* op = new TensorFlowMinOperator;
-  op->inputs.push_back(node.input(0));
-  op->inputs.push_back(node.input(1));
-  op->outputs.push_back(node.name());
-  model->operators.emplace_back(op);
-  if (HasAttr(node, "keep_dims")) {
-    op->keep_dims = GetBoolAttr(node, "keep_dims");
-  }
-  return tensorflow::Status::OK();
-}
-
 tensorflow::Status ConvertUnsupportedOperator(
     const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
     Model* model) {
@@ -1594,6 +1578,24 @@ tensorflow::Status ConvertShapeOperator(
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertAnyOperator(
+    const NodeDef& node, const TensorFlowImportFlags& tf_import_flags,
+    Model* model) {
+  CHECK_EQ(node.op(), "Any");
+  TF_QCHECK_OK(CheckInputsCount(node, tf_import_flags, 2));
+  const auto idx_type =
+      HasAttr(node, "Tidx") ? GetDataTypeAttr(node, "Tidx") : DT_INT32;
+  CHECK(idx_type == DT_INT32);
+  auto op = absl::make_unique();
+  op->inputs.push_back(node.input(0));
+  op->inputs.push_back(node.input(1));
+  op->outputs.push_back(node.name());
+  op->keep_dims =
+      HasAttr(node, "keep_dims") ? GetBoolAttr(node, "keep_dims") : false;
+  model->operators.push_back(std::move(op));
+  return tensorflow::Status::OK();
+}
+
 void StripCaretFromArrayNames(Model* model) {
   for (auto& op : model->operators) {
     for (auto& input : op->inputs) {
@@ -1829,6 +1831,7 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"Add", ConvertSimpleOperator},
       {"AddN", ConvertSimpleOperator},
       {"All", ConvertSimpleOperator},
+      {"Any", ConvertAnyOperator},
       {"ArgMax", ConvertArgMinMaxOperator},
       {"ArgMin", ConvertArgMinMaxOperator},
       {"Assert", ConvertSimpleOperator},
@@ -1872,13 +1875,15 @@ ConverterMapType GetTensorFlowNodeConverterMap() {
       {"LessEqual", ConvertSimpleOperator},
       {"Log", ConvertSimpleOperator},
       {"LogSoftmax", ConvertSimpleOperator},
+      {"LogicalAnd", ConvertSimpleOperator},
+      {"LogicalNot", ConvertSimpleOperator},
       {"MatMul", ConvertMatMulOperator},
       {"Max", ConvertReduceOperator},
       {"MaxPool", ConvertMaxPoolOperator},
       {"Maximum", ConvertSimpleOperator},
       {"Mean", ConvertReduceOperator},
       {"Merge", ConvertSimpleOperator},
-      {"Min", ConvertMinOperator},
+      {"Min", ConvertReduceOperator},
       {"Minimum", ConvertSimpleOperator},
       {"Mul", ConvertSimpleOperator},
       {"Neg", ConvertSimpleOperator},
diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h
index 15f5d9c354..6fe194516d 100644
--- a/tensorflow/contrib/lite/toco/model.h
+++ b/tensorflow/contrib/lite/toco/model.h
@@ -110,7 +110,7 @@ enum class OperatorType : uint8 {
   kLessEqual,
   kReduceMax,  //  Reduction Max
   kMaximum,    //  Element-wise Maximum
-  kMin,        //  Reduction Min
+  kReduceMin,  //  Reduction Min
   kMinimum,    //  Element-wise Minimum
   kMatMul,
   kMerge,
@@ -143,6 +143,9 @@ enum class OperatorType : uint8 {
   kNotEqual,
   kPow,
   kArgMin,
+  kAny,
+  kLogicalAnd,
+  kLogicalNot,
 };
 
 // Helper to deal with TensorFlow arrays using a different ordering of
@@ -1416,16 +1419,15 @@ struct TensorFlowMaxOperator : Operator {
   bool keep_dims = false;
 };
 
-// Global min reduction: computes the min of all of entries in the input array.
-// Thus the output is "0-dimensional": it consists of a single scalar value.
+// Min reduction: computes the min of all of entries across the axes.
 //
 // Inputs:
 //   inputs[0]: required: the input array
 //
-// TensorFlow equivalent: Min --- except that we only support the special case
-// of global reduction across all dimensions.
+// TensorFlow equivalent: Min
 struct TensorFlowMinOperator : Operator {
-  TensorFlowMinOperator() : Operator(OperatorType::kMin) {}
+  TensorFlowMinOperator() : Operator(OperatorType::kReduceMin) {}
+  std::vector axis;
   bool keep_dims = false;
 };
 
@@ -1690,6 +1692,39 @@ struct PowOperator : Operator {
   PowOperator() : Operator(OperatorType::kPow) {}
 };
 
+// Any operator:
+//
+// Inputs:
+// Inputs[0]: required: A boolean input tensor.
+// Inputs[1]: required: reduction_indices.
+//
+// TensorFlow equivalent: tf.reduce_any.
+struct AnyOperator : Operator {
+  AnyOperator() : Operator(OperatorType::kAny) {}
+  bool keep_dims = false;
+};
+
+// LogicalAnd operator:
+//
+// Inputs:
+// Inputs[0]: required: A boolean tensor.
+// Inputs[1]: required: A boolean tensor.
+//
+// TensorFlow equivalent: tf.logical_and.
+struct LogicalAndOperator : Operator {
+  LogicalAndOperator() : Operator(OperatorType::kLogicalAnd) {}
+};
+
+// LogicalNot operator:
+//
+// Inputs:
+// Inputs[0]: required: A boolean tensor.
+//
+// TensorFlow equivalent: tf.logical_not.
+struct LogicalNotOperator : Operator {
+  LogicalNotOperator() : Operator(OperatorType::kLogicalNot) {}
+};
+
 // Alloc's are used for transient arrays only. An Alloc specifies which interval
 // of the "transient_data" workspace buffer passed to inference functions, is to
 // be used for the transient array at hand. The 'start' and 'end' values are
diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index edcdd8f8cc..52f8df45a2 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -351,10 +351,10 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(LessEqual)
     HANDLE_OPERATORTYPENAME_CASE(MatMul)
     HANDLE_OPERATORTYPENAME_CASE(ReduceMax)  //  Reduction Max
-    HANDLE_OPERATORTYPENAME_CASE(Maximum)  //  Element-wise Maximum
+    HANDLE_OPERATORTYPENAME_CASE(Maximum)    //  Element-wise Maximum
     HANDLE_OPERATORTYPENAME_CASE(Merge)
-    HANDLE_OPERATORTYPENAME_CASE(Min)      //  Reduction Min
-    HANDLE_OPERATORTYPENAME_CASE(Minimum)  //  Element-wise Minimum
+    HANDLE_OPERATORTYPENAME_CASE(ReduceMin)  //  Reduction Min
+    HANDLE_OPERATORTYPENAME_CASE(Minimum)    //  Element-wise Minimum
     HANDLE_OPERATORTYPENAME_CASE(Neg)
     HANDLE_OPERATORTYPENAME_CASE(Pack)
     HANDLE_OPERATORTYPENAME_CASE(Pad)
@@ -399,6 +399,9 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(Equal)
     HANDLE_OPERATORTYPENAME_CASE(NotEqual)
     HANDLE_OPERATORTYPENAME_CASE(Pow)
+    HANDLE_OPERATORTYPENAME_CASE(Any)
+    HANDLE_OPERATORTYPENAME_CASE(LogicalAnd)
+    HANDLE_OPERATORTYPENAME_CASE(LogicalNot)
     default:
       LOG(FATAL) << "Unhandled op type";
 #undef HANDLE_OPERATORTYPENAME_CASE
-- 
cgit v1.2.3


From 1ce63a90ed6a346d62db653ba0ec43accead797d Mon Sep 17 00:00:00 2001
From: Clayne Robison 
Date: Wed, 18 Jul 2018 14:43:38 -0700
Subject: Passing full sandybridge build params to the container build script

---
 tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh b/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh
index 7de58ef625..a1d91a6123 100755
--- a/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh
+++ b/tensorflow/tools/ci_build/linux/mkl/build-dev-container.sh
@@ -44,6 +44,7 @@ TF_DOCKER_BUILD_TYPE="MKL" \
   TF_DOCKER_BUILD_DEVEL_BRANCH="${TF_DOCKER_BUILD_DEVEL_BRANCH}" \
   TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
   TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \
+  TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \
   ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh 
 
 # build the python 3 container and whl
@@ -53,6 +54,7 @@ TF_DOCKER_BUILD_TYPE="MKL" \
   TF_DOCKER_BUILD_IMAGE_NAME="${TF_DOCKER_BUILD_IMAGE_NAME}" \
   TF_DOCKER_BUILD_VERSION="${TF_DOCKER_BUILD_VERSION}" \
   TF_DOCKER_BUILD_PYTHON_VERSION="PYTHON3" \
+  TF_BAZEL_BUILD_OPTIONS="${TF_BAZEL_BUILD_OPTIONS}" \
   ${WORKSPACE}/tensorflow/tools/docker/parameterized_docker_build.sh
 
 # Build containers for AVX2
-- 
cgit v1.2.3


From 0bac40657d2f7d5f879cbaa7f5d4f14c41a3d585 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 14:47:33 -0700
Subject: Interal changel

PiperOrigin-RevId: 205140721
---
 tensorflow/contrib/lite/delegates/eager/BUILD | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/contrib/lite/delegates/eager/BUILD b/tensorflow/contrib/lite/delegates/eager/BUILD
index 23d8f543e5..9f31ffdf67 100644
--- a/tensorflow/contrib/lite/delegates/eager/BUILD
+++ b/tensorflow/contrib/lite/delegates/eager/BUILD
@@ -42,6 +42,10 @@ cc_library(
     name = "delegate_data",
     srcs = ["delegate_data.cc"],
     hdrs = ["delegate_data.h"],
+    tags = [
+        "no_oss",
+        "tflite_not_portable",
+    ],
     deps = [
         ":buffer_map",
         "//tensorflow/core:core_cpu",
-- 
cgit v1.2.3


From a186bcdcb0d3b85909560d85167cda55ccbc973b Mon Sep 17 00:00:00 2001
From: Jiri Simsa 
Date: Wed, 18 Jul 2018 15:02:45 -0700
Subject: [tf.data] Transformation for asserting which transformations happen
 next in a (possibly optimized) pipeline.

PiperOrigin-RevId: 205143256
---
 tensorflow/contrib/cmake/tf_core_kernels.cmake     |   1 +
 tensorflow/contrib/data/kernels/BUILD              |  11 ++
 .../contrib/data/kernels/assert_next_dataset_op.cc | 152 +++++++++++++++++++++
 tensorflow/contrib/data/ops/dataset_ops.cc         |  13 ++
 tensorflow/contrib/data/python/kernel_tests/BUILD  |   1 -
 .../kernel_tests/optimize_dataset_op_test.py       |  60 ++++++--
 tensorflow/contrib/data/python/ops/optimization.py |  53 +++++++
 7 files changed, 277 insertions(+), 14 deletions(-)
 create mode 100644 tensorflow/contrib/data/kernels/assert_next_dataset_op.cc

diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake
index 844f62649d..7b892ba248 100644
--- a/tensorflow/contrib/cmake/tf_core_kernels.cmake
+++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake
@@ -68,6 +68,7 @@ if(tensorflow_BUILD_CONTRIB_KERNELS)
       "${tensorflow_source_dir}/tensorflow/contrib/coder/kernels/range_coder_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/coder/kernels/range_coder_ops_util.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/coder/ops/coder_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/data/kernels/assert_next_dataset_op.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/data/kernels/csv_dataset_op.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/data/kernels/directed_interleave_dataset_op.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/data/kernels/ignore_errors_dataset_op.cc"
diff --git a/tensorflow/contrib/data/kernels/BUILD b/tensorflow/contrib/data/kernels/BUILD
index 7b69e10441..566cbb246a 100644
--- a/tensorflow/contrib/data/kernels/BUILD
+++ b/tensorflow/contrib/data/kernels/BUILD
@@ -70,9 +70,20 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "assert_next_dataset_op",
+    srcs = ["assert_next_dataset_op.cc"],
+    deps = [
+        "//tensorflow/core:framework_headers_lib",
+        "//third_party/eigen3",
+        "@protobuf_archive//:protobuf_headers",
+    ],
+)
+
 cc_library(
     name = "dataset_kernels",
     deps = [
+        ":assert_next_dataset_op",
         ":csv_dataset_op",
         ":directed_interleave_dataset_op",
         ":ignore_errors_dataset_op",
diff --git a/tensorflow/contrib/data/kernels/assert_next_dataset_op.cc b/tensorflow/contrib/data/kernels/assert_next_dataset_op.cc
new file mode 100644
index 0000000000..95b8e1f7fd
--- /dev/null
+++ b/tensorflow/contrib/data/kernels/assert_next_dataset_op.cc
@@ -0,0 +1,152 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include 
+
+#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/tensor.h"
+
+namespace tensorflow {
+namespace {
+
+// See documentation in ../ops/dataset_ops.cc for a high-level
+// description of the following op.
+class AssertNextDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit AssertNextDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+  }
+
+ protected:
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    std::vector transformations;
+    OP_REQUIRES_OK(ctx, ParseVectorArgument(ctx, "transformations",
+                                                    &transformations));
+    *output =
+        new Dataset(ctx, input, transformations, output_types_, output_shapes_);
+  }
+
+ private:
+  class Dataset : public GraphDatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, const DatasetBase* input,
+            const std::vector& transformations,
+            const DataTypeVector& output_types,
+            const std::vector& output_shapes)
+        : GraphDatasetBase(ctx),
+          input_(input),
+          transformations_(transformations),
+          output_types_(output_types),
+          output_shapes_(output_shapes) {
+      input_->Ref();
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr MakeIteratorInternal(
+        const string& prefix) const override {
+      return std::unique_ptr(
+          new Iterator({this, strings::StrCat(prefix, "::Assert")}));
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return output_types_;
+    }
+    const std::vector& output_shapes() const override {
+      return output_shapes_;
+    }
+
+    string DebugString() const override {
+      return "AssertNextDatasetOp::Dataset";
+    }
+
+   protected:
+    Status AsGraphDefInternal(OpKernelContext* ctx, DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      Node* input_graph_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddParentDataset(ctx, input_, &input_graph_node));
+      Node* transformations_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddVector(transformations_, &transformations_node));
+      TF_RETURN_IF_ERROR(b->AddDataset(
+          this, {input_graph_node, transformations_node}, output));
+      return Status::OK();
+    }
+
+   private:
+    class Iterator : public DatasetIterator {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator(params) {}
+
+      Status Initialize(IteratorContext* ctx) override {
+        std::vector tokens =
+            str_util::Split(prefix(), ':', str_util::SkipEmpty());
+        if (dataset()->transformations_.size() > tokens.size() - 2) {
+          return errors::InvalidArgument(
+              "Asserted next ", dataset()->transformations_.size(),
+              " transformations but encountered only ", tokens.size() - 2, ".");
+        }
+        int n = tokens.size();
+        for (size_t i = 0; i < dataset()->transformations_.size(); ++i) {
+          if (dataset()->transformations_[i] != tokens[n - 2 - i]) {
+            return errors::InvalidArgument(
+                "Asserted ", dataset()->transformations_[i],
+                " transformation at offset ", i, " but encountered ",
+                tokens[n - 2 - i], " transformation instead.");
+          }
+        }
+        return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_);
+      }
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector* out_tensors,
+                             bool* end_of_sequence) override {
+        return input_impl_->GetNext(ctx, out_tensors, end_of_sequence);
+      }
+
+     protected:
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_));
+        return Status::OK();
+      }
+
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        TF_RETURN_IF_ERROR(RestoreParent(ctx, reader, input_impl_));
+        return Status::OK();
+      }
+
+     private:
+      std::unique_ptr input_impl_;
+    };
+
+    const DatasetBase* input_;
+    const std::vector transformations_;
+    const DataTypeVector output_types_;
+    const std::vector output_shapes_;
+  };
+
+  DataTypeVector output_types_;
+  std::vector output_shapes_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("AssertNextDataset").Device(DEVICE_CPU),
+                        AssertNextDatasetOp);
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/data/ops/dataset_ops.cc b/tensorflow/contrib/data/ops/dataset_ops.cc
index a623c27ff8..b5c6f2e241 100644
--- a/tensorflow/contrib/data/ops/dataset_ops.cc
+++ b/tensorflow/contrib/data/ops/dataset_ops.cc
@@ -177,4 +177,17 @@ display_name: A human-readable name for the threads that may be visible in
   some visualizations.
 )doc");
 
+REGISTER_OP("AssertNextDataset")
+    .Input("input_dataset: variant")
+    .Input("transformations: string")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused;
+      // transformations should be a vector.
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      return shape_inference::ScalarShape(c);
+    });
+
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 18457320b9..d372bed479 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -208,7 +208,6 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/contrib/data/python/ops:optimization",
-        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:errors",
         "//tensorflow/python/data/ops:dataset_ops",
diff --git a/tensorflow/contrib/data/python/kernel_tests/optimize_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/optimize_dataset_op_test.py
index 21eebccd11..cfef40e192 100644
--- a/tensorflow/contrib/data/python/kernel_tests/optimize_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/optimize_dataset_op_test.py
@@ -18,7 +18,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.data.python.ops import optimization
-from tensorflow.core.framework import graph_pb2
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import errors
 from tensorflow.python.platform import test
@@ -26,41 +25,76 @@ from tensorflow.python.platform import test
 
 class OptimizeDatasetTest(test.TestCase):
 
+  def testAssertSuffix(self):
+    dataset = dataset_ops.Dataset.from_tensors(0).apply(
+        optimization.assert_next(["Map"])).map(lambda x: x)
+    iterator = dataset.make_one_shot_iterator()
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      self.assertEqual(0, sess.run(get_next))
+
+  def testAssertSuffixInvalid(self):
+    dataset = dataset_ops.Dataset.from_tensors(0).apply(
+        optimization.assert_next(["Whoops"])).map(lambda x: x)
+    iterator = dataset.make_one_shot_iterator()
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          "Asserted Whoops transformation at offset 0 but encountered "
+          "Map transformation instead."
+      ):
+        sess.run(get_next)
+
+  def testAssertSuffixShort(self):
+    dataset = dataset_ops.Dataset.from_tensors(0).apply(
+        optimization.assert_next(["Map", "Whoops"])).map(lambda x: x)
+    iterator = dataset.make_one_shot_iterator()
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          "Asserted next 2 transformations but encountered only 1."):
+        sess.run(get_next)
+
   def testDefaultOptimizations(self):
-    dataset = dataset_ops.Dataset.range(10).map(lambda x: x * x).batch(
-        10).apply(optimization.optimize())
+    dataset = dataset_ops.Dataset.range(10).apply(
+        optimization.assert_next(
+            ["Map", "Batch"])).map(lambda x: x * x).batch(10).apply(
+                optimization.optimize())
     iterator = dataset.make_one_shot_iterator()
     get_next = iterator.get_next()
 
     with self.test_session() as sess:
-      graph = graph_pb2.GraphDef().FromString(
-          sess.run(dataset._as_serialized_graph()))
       self.assertAllEqual([x * x for x in range(10)], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
   def testEmptyOptimizations(self):
-    dataset = dataset_ops.Dataset.range(10).map(lambda x: x * x).batch(
-        10).apply(optimization.optimize([]))
+    dataset = dataset_ops.Dataset.range(10).apply(
+        optimization.assert_next(
+            ["Map", "Batch"])).map(lambda x: x * x).batch(10).apply(
+                optimization.optimize([]))
     iterator = dataset.make_one_shot_iterator()
     get_next = iterator.get_next()
 
     with self.test_session() as sess:
-      graph = graph_pb2.GraphDef().FromString(
-          sess.run(dataset._as_serialized_graph()))
       self.assertAllEqual([x * x for x in range(10)], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
   def testOptimization(self):
-    dataset = dataset_ops.Dataset.range(10).map(lambda x: x * x).batch(
-        10).apply(optimization.optimize(["map_and_batch_fusion"]))
+    dataset = dataset_ops.Dataset.range(10).apply(
+        optimization.assert_next(
+            ["MapAndBatch"])).map(lambda x: x * x).batch(10).apply(
+                optimization.optimize(["map_and_batch_fusion"]))
     iterator = dataset.make_one_shot_iterator()
     get_next = iterator.get_next()
 
     with self.test_session() as sess:
-      graph = graph_pb2.GraphDef().FromString(
-          sess.run(dataset._as_serialized_graph()))
       self.assertAllEqual([x * x for x in range(10)], sess.run(get_next))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
diff --git a/tensorflow/contrib/data/python/ops/optimization.py b/tensorflow/contrib/data/python/ops/optimization.py
index cf89657226..018c5115e1 100644
--- a/tensorflow/contrib/data/python/ops/optimization.py
+++ b/tensorflow/contrib/data/python/ops/optimization.py
@@ -18,12 +18,34 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.data.python.ops import contrib_op_loader  # pylint: disable=unused-import
+from tensorflow.contrib.data.python.ops import gen_dataset_ops as contrib_gen_dataset_ops
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_dataset_ops
 
 
+# TODO(jsimsa): Support RE matching for both individual transformation (e.g. to
+# account for indexing) and transformation sequence.
+def assert_next(transformations):
+  """A transformation that asserts which transformations happen next.
+
+  Args:
+    transformations: A `tf.string` vector `tf.Tensor` identifying the
+      transformations that are expected to happen next.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    @{tf.data.Dataset.apply}.
+  """
+
+  def _apply_fn(dataset):
+    """Function from `Dataset` to `Dataset` that applies the transformation."""
+    return _AssertNextDataset(dataset, transformations)
+
+  return _apply_fn
+
+
 def optimize(optimizations=None):
   """A transformation that applies optimizations.
 
@@ -44,6 +66,37 @@ def optimize(optimizations=None):
   return _apply_fn
 
 
+class _AssertNextDataset(dataset_ops.Dataset):
+  """A `Dataset` that asserts which transformations happen next."""
+
+  def __init__(self, input_dataset, transformations):
+    """See `assert_next()` for details."""
+    super(_AssertNextDataset, self).__init__()
+    self._input_dataset = input_dataset
+    if transformations is None:
+      raise ValueError("At least one transformation should be specified")
+    self._transformations = ops.convert_to_tensor(
+        transformations, dtype=dtypes.string, name="transformations")
+
+  def _as_variant_tensor(self):
+    return contrib_gen_dataset_ops.assert_next_dataset(
+        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
+        self._transformations,
+        **dataset_ops.flat_structure(self))
+
+  @property
+  def output_classes(self):
+    return self._input_dataset.output_classes
+
+  @property
+  def output_shapes(self):
+    return self._input_dataset.output_shapes
+
+  @property
+  def output_types(self):
+    return self._input_dataset.output_types
+
+
 class _OptimizeDataset(dataset_ops.Dataset):
   """A `Dataset` that acts as an identity, and applies optimizations."""
 
-- 
cgit v1.2.3


From 6619dd5fdcad02f087f5758083e2585bdfef9e78 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Wed, 18 Jul 2018 15:03:02 -0700
Subject: Don't cluster nodes that have inputs with mismatching deadness

TensorFlow allows nodes to have some live inputs and some dead inputs.  The
executor does not execute these nodes but instead propagates a dead signal to
all their outputs (i.e. these nodes are treated as fully dead).

This is a problem for auto-clustering because it means auto-clustering can kill
nodes that used to be alive.  For instance say before clustering we have a graph
like

digraph {
  Alive0 -> P
  Alive1 -> Q
  Dead -> R
  P -> X
  Q -> X
  Q -> Y
  R -> Y
}

and we cluster P, Q, R, X and Y into a single XLA cluster.

Then after clustering both X and Y are dead because the cluster is a single node
as far as the executor is concerned and said node won't get scheduled if any of
its inputs are dead.

This CL introduces a static analysis pass that our auto-clustering code can use
to ensure nodes that have inputs with mismatching deadness (like "Y" in the
example graph) are not included in XLA clusters.

PiperOrigin-RevId: 205143316
---
 tensorflow/compiler/jit/BUILD                      |   6 +
 tensorflow/compiler/jit/deadness_analysis.cc       | 546 +++++++++++++++++++++
 tensorflow/compiler/jit/deadness_analysis.h        |  68 +++
 tensorflow/compiler/jit/deadness_analysis_test.cc  | 443 +++++++++++++++++
 .../compiler/jit/mark_for_compilation_pass.cc      |  26 +-
 tensorflow/compiler/jit/xla_fusion_optimizer.cc    |  12 +
 tensorflow/core/graph/algorithm.cc                 |  37 +-
 tensorflow/core/graph/algorithm.h                  |  16 +-
 tensorflow/core/graph/algorithm_test.cc            |  52 ++
 9 files changed, 1184 insertions(+), 22 deletions(-)
 create mode 100644 tensorflow/compiler/jit/deadness_analysis.cc
 create mode 100644 tensorflow/compiler/jit/deadness_analysis.h
 create mode 100644 tensorflow/compiler/jit/deadness_analysis_test.cc

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index c2245b8eae..9174a67cc6 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -304,11 +304,13 @@ cc_library(
     name = "compilation_passes",
     srcs = [
         "build_xla_launch_ops_pass.cc",
+        "deadness_analysis.cc",
         "encapsulate_subgraphs_pass.cc",
         "mark_for_compilation_pass.cc",
     ],
     hdrs = [
         "build_xla_launch_ops_pass.h",
+        "deadness_analysis.h",
         "encapsulate_subgraphs_pass.h",
         "mark_for_compilation_pass.h",
     ],
@@ -325,6 +327,7 @@ cc_library(
         "//tensorflow/compiler/tf2xla:dump_graph",
         "//tensorflow/compiler/tf2xla:xla_compiler",
         "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:util",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
@@ -377,6 +380,7 @@ tf_cc_test(
     name = "compilation_passes_test",
     size = "small",
     srcs = [
+        "deadness_analysis_test.cc",
         "encapsulate_subgraphs_pass_test.cc",
         "mark_for_compilation_pass_test.cc",
     ],
@@ -387,6 +391,7 @@ tf_cc_test(
         "//tensorflow/cc:cc_ops_internal",
         "//tensorflow/cc:function_ops",
         "//tensorflow/cc:ops",
+        "//tensorflow/cc:sendrecv_ops",
         "//tensorflow/compiler/jit/kernels:xla_launch_op",
         "//tensorflow/compiler/tf2xla:xla_compiler",
         "//tensorflow/compiler/tf2xla/kernels:xla_ops",
@@ -458,6 +463,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":common",
+        ":compilation_passes",
         ":union_find",
         ":xla_cluster_util",
         "//tensorflow/compiler/jit/graphcycles",
diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc
new file mode 100644
index 0000000000..b2d119029a
--- /dev/null
+++ b/tensorflow/compiler/jit/deadness_analysis.cc
@@ -0,0 +1,546 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/jit/deadness_analysis.h"
+#include "tensorflow/core/graph/algorithm.h"
+#include "tensorflow/core/graph/tensor_id.h"
+#include "tensorflow/core/lib/gtl/flatset.h"
+#include "tensorflow/core/lib/hash/hash.h"
+
+// ALGORITHM OVERVIEW
+//
+// We map every output produced by each node in the TensorFlow graph (including
+// control dependence) into an instance of the Predicate class.  Instances of
+// Predicate denote logical formulas and mapping a node `n` to a predicate
+// `pred` implies that `n` is executed whenver `pred` is true.  Then we can
+// deduce mismatching liveness in the inputs to node by comparing the predicate
+// those inputs are mapped to.
+//
+// Loops are handled pessimistically -- we map Merge nodes with backedges to
+// uninterpreted symbols (the same kind we use to represent Switch and _Recv).
+// Predicate equality has to hold over all possible assignments to these
+// uninterpreted symbols.
+
+namespace tensorflow {
+
+namespace {
+
+// Represents a logical predicate, used as described in the algorithm overview
+// above.
+class Predicate {
+ public:
+  enum class Kind { kAnd, kOr, kNot, kSymbol };
+
+  virtual string ToString() const = 0;
+  virtual bool operator==(const Predicate& other) const = 0;
+  virtual bool operator!=(const Predicate& other) const {
+    return !(*this == other);
+  }
+  int64 hash() const { return hash_; }
+
+  virtual Kind kind() const = 0;
+  virtual ~Predicate() {}
+
+ protected:
+  explicit Predicate(int64 hash) : hash_(hash) {}
+
+ private:
+  const int64 hash_;
+};
+
+int64 HashPredicateSequence(Predicate::Kind kind,
+                            gtl::ArraySlice preds) {
+  int64 hash = ::tensorflow::hash()(kind);
+  for (Predicate* pred : preds) {
+    hash = Hash64Combine(hash, pred->hash());
+  }
+  return hash;
+}
+
+bool PredicateSequenceEqual(gtl::ArraySlice lhs,
+                            gtl::ArraySlice rhs) {
+  if (lhs.size() != rhs.size()) {
+    return false;
+  }
+  for (int64 i = 0; i < lhs.size(); i++) {
+    if (*lhs[i] != *rhs[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+// Represents a logical conjunction of a set of predicates.
+class AndPredicate : public Predicate {
+ public:
+  explicit AndPredicate(std::vector operands)
+      : Predicate(HashPredicateSequence(Kind::kAnd, operands)),
+        operands_(std::move(operands)) {}
+
+  string ToString() const override {
+    if (operands().empty()) {
+      return "#true";
+    }
+
+    std::vector operands_str;
+    std::transform(operands().begin(), operands().end(),
+                   std::back_inserter(operands_str),
+                   [](Predicate* pred) { return pred->ToString(); });
+
+    return strings::StrCat("(", str_util::Join(operands_str, " & "), ")");
+  }
+
+  bool operator==(const Predicate& other) const override {
+    return other.kind() == Kind::kAnd &&
+           PredicateSequenceEqual(
+               dynamic_cast(other).operands(), operands());
+  }
+
+  Kind kind() const override { return Kind::kAnd; }
+
+  const tensorflow::gtl::ArraySlice operands() const {
+    return operands_;
+  }
+
+ private:
+  std::vector operands_;
+};
+
+// Represents a logical disjunction of a set of predicates.
+class OrPredicate : public Predicate {
+ public:
+  explicit OrPredicate(std::vector operands)
+      : Predicate(HashPredicateSequence(Kind::kOr, operands)),
+        operands_(std::move(operands)) {}
+
+  string ToString() const override {
+    if (operands().empty()) {
+      return "#false";
+    }
+
+    std::vector operands_str;
+    std::transform(operands().begin(), operands().end(),
+                   std::back_inserter(operands_str),
+                   [](Predicate* pred) { return pred->ToString(); });
+
+    return strings::StrCat("(", str_util::Join(operands_str, " | "), ")");
+  }
+
+  bool operator==(const Predicate& other) const override {
+    return other.kind() == Kind::kOr &&
+           PredicateSequenceEqual(
+               dynamic_cast(other).operands(), operands());
+  }
+
+  Kind kind() const override { return Kind::kOr; }
+  const tensorflow::gtl::ArraySlice operands() const {
+    return operands_;
+  }
+
+ private:
+  std::vector operands_;
+};
+
+// Represents a logical negation of a set of predicates.
+class NotPredicate : public Predicate {
+ public:
+  explicit NotPredicate(Predicate* operand)
+      : Predicate(HashPredicateSequence(Kind::kNot, {operand})),
+        operand_(operand) {}
+
+  string ToString() const override {
+    return strings::StrCat("~", operand()->ToString());
+  }
+
+  bool operator==(const Predicate& other) const override {
+    return other.kind() == Kind::kNot &&
+           *dynamic_cast(other).operand() == *operand();
+  }
+
+  Kind kind() const override { return Kind::kNot; }
+  Predicate* operand() const { return operand_; }
+
+ private:
+  Predicate* operand_;
+};
+
+// Represents an uninterpreted symbol in a logical predicate.
+//
+// Two predicates are equivalent iff they are equivalent for all assignments to
+// the symbols contained in them.
+class SymbolPredicate : public Predicate {
+ public:
+  explicit SymbolPredicate(TensorId tensor_id, bool must_be_true)
+      : Predicate(Hash(tensor_id, must_be_true)),
+        tensor_id_(std::move(tensor_id)),
+        must_be_true_(must_be_true) {}
+
+  string ToString() const override { return tensor_id_.ToString(); }
+  bool operator==(const Predicate& other) const override {
+    return other.kind() == Kind::kSymbol &&
+           must_be_true() ==
+               dynamic_cast(other).must_be_true() &&
+           dynamic_cast(other).tensor_id() ==
+               tensor_id();
+  }
+
+  Kind kind() const override { return Kind::kSymbol; }
+
+  // If `must_be_true()` is true this SymbolPredicate represents the proposition
+  // "tensor_id() is live and evaluates to true".
+  //
+  // If `must_be_true()` is false then this SymbolPredicate represents the
+  // proposition "tensor_id() is live (and may evalutate to any value)"
+  TensorId tensor_id() const { return tensor_id_; }
+  bool must_be_true() const { return must_be_true_; }
+
+ private:
+  TensorId tensor_id_;
+  bool must_be_true_;
+
+  static int64 Hash(const TensorId tensor_id, bool must_be_true) {
+    return Hash64Combine(
+        ::tensorflow::hash()(must_be_true),
+        Hash64Combine(::tensorflow::hash()(Kind::kSymbol),
+                      TensorId::Hasher{}(tensor_id)));
+  }
+};
+
+// Creates and owns Predicate instances.  Simplifies predicates as it creates
+// them.
+class PredicateFactory {
+ public:
+  Predicate* MakeAndPredicate(gtl::ArraySlice operands) {
+    return MakeAndOrImpl(operands, /*is_and=*/true);
+  }
+  Predicate* MakeOrPredicate(gtl::ArraySlice operands) {
+    return MakeAndOrImpl(operands, /*is_and=*/false);
+  }
+
+  Predicate* MakeNotPredicate(Predicate* pred) {
+    return Make(pred);
+  }
+
+  Predicate* MakeSymbolPredicate(TensorId tensor_id, bool must_be_true) {
+    return Make(tensor_id, must_be_true);
+  }
+
+  Predicate* MakeTrue() { return MakeAndPredicate({}); }
+  Predicate* MakeFalse() { return MakeOrPredicate({}); }
+
+ private:
+  template 
+  Predicate* Make(Args... args) {
+    std::unique_ptr pred(
+        new PredicateT(std::forward(args)...));
+    predicate_storage_.emplace_back(std::move(pred));
+    return predicate_storage_.back().get();
+  }
+
+  Predicate* MakeAndOrImpl(gtl::ArraySlice operands, bool is_and);
+
+  struct PredicatePtrHash {
+    size_t operator()(const Predicate* pred) const { return pred->hash(); }
+  };
+
+  struct PredicatePtrEq {
+    size_t operator()(const Predicate* a, const Predicate* b) const {
+      return *a == *b;
+    }
+  };
+
+  using PredicateSet =
+      gtl::FlatSet;
+
+  std::vector> predicate_storage_;
+};
+
+// Common code to create AndPredicate or OrPredicate instances.
+Predicate* PredicateFactory::MakeAndOrImpl(gtl::ArraySlice operands,
+                                           bool is_and) {
+  Predicate::Kind pred_kind =
+      is_and ? Predicate::Kind::kAnd : Predicate::Kind::kOr;
+  PredicateSet simplified_ops_set;
+  std::vector simplified_ops;
+  for (Predicate* op : operands) {
+    // Simplify A&A => A and  A|A => A.
+    if (!simplified_ops_set.insert(op).second) {
+      continue;
+    }
+
+    if (op->kind() == pred_kind) {
+      // "Inline" the operands of an inner And/Or into the parent And/Or.
+      gtl::ArraySlice operands =
+          is_and ? dynamic_cast(op)->operands()
+                 : dynamic_cast(op)->operands();
+      for (Predicate* subop : operands) {
+        if (simplified_ops_set.insert(subop).second) {
+          simplified_ops.push_back(subop);
+        }
+      }
+    } else {
+      simplified_ops.push_back(op);
+    }
+  }
+
+  if (simplified_ops.size() == 1) {
+    return simplified_ops[0];
+  }
+
+  // Simplify "A&~A=>False" and "A|~A=>True".
+  PredicateSet negated_ops;
+  for (Predicate* op : simplified_ops) {
+    if (op->kind() == Predicate::Kind::kNot) {
+      negated_ops.insert(dynamic_cast(*op).operand());
+    }
+  }
+
+  for (Predicate* op : simplified_ops) {
+    if (negated_ops.count(op)) {
+      return is_and ? MakeFalse() : MakeTrue();
+    }
+  }
+
+  std::stable_sort(
+      simplified_ops.begin(), simplified_ops.end(),
+      [](Predicate* a, Predicate* b) { return a->hash() < b->hash(); });
+
+  return is_and ? Make(std::move(simplified_ops))
+                : Make(std::move(simplified_ops));
+}
+
+class DeadnessAnalysisImpl : public DeadnessAnalysis {
+ public:
+  explicit DeadnessAnalysisImpl(const Graph* graph)
+      : graph_(*graph), vlog_(VLOG_IS_ON(2)) {}
+
+  Status Populate();
+  bool HasInputsWithMismatchingDeadness(const Node& node) override;
+  void Print() const override;
+
+ private:
+  enum class EdgeKind { kDataAndControl, kDataOnly, kControlOnly };
+
+  std::vector GetIncomingPreds(Node* n, EdgeKind edge_kind);
+  void SetPred(Node* n, int output_idx, Predicate* pred) {
+    CHECK(
+        predicate_map_.insert({TensorId(n->name(), output_idx), pred}).second);
+  }
+  void SetPred(Node* n, gtl::ArraySlice output_idxs, Predicate* pred) {
+    for (int output_idx : output_idxs) {
+      SetPred(n, output_idx, pred);
+    }
+  }
+
+  Status HandleSwitch(Node* n);
+  Status HandleMerge(Node* n);
+  Status HandleRecv(Node* n);
+  Status HandleGeneric(Node* n);
+
+  const Graph& graph_;
+  gtl::FlatMap predicate_map_;
+  PredicateFactory predicate_factory_;
+  bool vlog_;
+};
+
+TensorId InputEdgeToTensorId(const Edge* e) {
+  return TensorId(e->src()->name(), e->src_output());
+}
+
+std::vector DeadnessAnalysisImpl::GetIncomingPreds(
+    Node* n, DeadnessAnalysisImpl::EdgeKind edge_kind) {
+  std::vector incoming_preds;
+  for (const Edge* in_edge : n->in_edges()) {
+    bool should_process =
+        edge_kind == EdgeKind::kDataAndControl ||
+        (in_edge->IsControlEdge() && edge_kind == EdgeKind::kControlOnly) ||
+        (!in_edge->IsControlEdge() && edge_kind == EdgeKind::kDataOnly);
+
+    if (should_process) {
+      auto it = predicate_map_.find(InputEdgeToTensorId(in_edge));
+      CHECK(it != predicate_map_.end());
+      incoming_preds.push_back(it->second);
+    }
+  }
+  return incoming_preds;
+}
+
+Status DeadnessAnalysisImpl::HandleSwitch(Node* n) {
+  std::vector input_preds =
+      GetIncomingPreds(n, EdgeKind::kDataAndControl);
+  const Edge* pred_edge;
+  TF_RETURN_IF_ERROR(n->input_edge(1, &pred_edge));
+  Predicate* true_switch = predicate_factory_.MakeSymbolPredicate(
+      TensorId(pred_edge->src()->name(), pred_edge->src_output()),
+      /*must_be_true=*/true);
+  Predicate* false_switch = predicate_factory_.MakeNotPredicate(true_switch);
+
+  // Output 0 is alive iff all inputs are alive and the condition is false.
+  input_preds.push_back(false_switch);
+  SetPred(n, 0, predicate_factory_.MakeAndPredicate(input_preds));
+  input_preds.pop_back();
+
+  // Output 1 is alive iff all inputs are alive and the condition is true.
+  input_preds.push_back(true_switch);
+  SetPred(n, 1, predicate_factory_.MakeAndPredicate(input_preds));
+  input_preds.pop_back();
+
+  // Control is alive iff any inputs are alive.
+  SetPred(n, Graph::kControlSlot,
+          predicate_factory_.MakeAndPredicate(input_preds));
+
+  return Status::OK();
+}
+
+Status DeadnessAnalysisImpl::HandleMerge(Node* n) {
+  // Merge ignores deadness of its control inputs.  A merge that isn't the
+  // target of a backedge has is alive iff any of its data inputs are.  We treat
+  // the liveness of a merge that is the target of a backedge symbolically.
+
+  bool has_backedge = std::any_of(
+      n->in_edges().begin(), n->in_edges().end(), [](const Edge* e) {
+        return !e->IsControlEdge() && e->src()->IsNextIteration();
+      });
+
+  Predicate* input_data_pred =
+      has_backedge ? predicate_factory_.MakeSymbolPredicate(
+                         TensorId(n->name(), 0), /*must_be_true=*/false)
+                   : predicate_factory_.MakeOrPredicate(
+                         GetIncomingPreds(n, EdgeKind::kDataOnly));
+
+  SetPred(n, {0, 1, Graph::kControlSlot}, input_data_pred);
+  return Status::OK();
+}
+
+Status DeadnessAnalysisImpl::HandleRecv(Node* n) {
+  // In addition to being alive or dead based on the inputs, a _Recv can also
+  // acquire a dead signal from a _Send.
+  std::vector input_preds =
+      GetIncomingPreds(n, EdgeKind::kDataAndControl);
+  input_preds.push_back(predicate_factory_.MakeSymbolPredicate(
+      TensorId(n->name(), 0), /*must_be_true=*/false));
+  SetPred(n, {0, Graph::kControlSlot},
+          predicate_factory_.MakeAndPredicate(input_preds));
+  return Status::OK();
+}
+
+Status DeadnessAnalysisImpl::HandleGeneric(Node* n) {
+  // Generally nodes are alive iff all their inputs are alive.
+  Predicate* pred = predicate_factory_.MakeAndPredicate(
+      GetIncomingPreds(n, EdgeKind::kDataAndControl));
+  for (int output_idx = 0; output_idx < n->num_outputs(); output_idx++) {
+    SetPred(n, output_idx, pred);
+  }
+  SetPred(n, Graph::kControlSlot, pred);
+  return Status::OK();
+}
+
+Status DeadnessAnalysisImpl::Populate() {
+  std::vector rpo;
+  GetReversePostOrder(graph_, &rpo, /*stable_comparator=*/{},
+                      /*edge_filter=*/[](const Edge& edge) {
+                        return !edge.src()->IsNextIteration();
+                      });
+
+  // This an abstract interpretation over the deadness propagation semantics of
+  // the graph executor.
+  for (Node* n : rpo) {
+    if (n->IsSwitch()) {
+      TF_RETURN_IF_ERROR(HandleSwitch(n));
+    } else if (n->IsMerge()) {
+      TF_RETURN_IF_ERROR(HandleMerge(n));
+    } else if (n->IsControlTrigger()) {
+      SetPred(n, Graph::kControlSlot, predicate_factory_.MakeTrue());
+    } else if (n->IsRecv() || n->IsHostRecv()) {
+      TF_RETURN_IF_ERROR(HandleRecv(n));
+    } else {
+      TF_RETURN_IF_ERROR(HandleGeneric(n));
+    }
+  }
+
+  return Status::OK();
+}
+
+bool DeadnessAnalysisImpl::HasInputsWithMismatchingDeadness(const Node& node) {
+  CHECK(!node.IsMerge());
+
+  if (vlog_) {
+    VLOG(2) << "HasInputsWithMismatchingDeadness(" << node.name() << ")";
+  }
+
+  Predicate* pred = nullptr;
+  for (const Edge* edge : node.in_edges()) {
+    auto it = predicate_map_.find(InputEdgeToTensorId(edge));
+    CHECK(it != predicate_map_.end());
+    if (vlog_) {
+      VLOG(2) << "  " << InputEdgeToTensorId(edge).ToString() << ": "
+              << it->second->ToString();
+    }
+
+    // Today we just compare the predicates for equality (with some
+    // canonicalization/simplification happening before) but we could be more
+    // sophisticated here if need be.
+    if (pred != nullptr && *pred != *it->second) {
+      if (vlog_) {
+        VLOG(2) << "HasInputsWithMismatchingDeadness(" << node.name()
+                << ") -> true";
+      }
+      return true;
+    }
+    pred = it->second;
+  }
+
+  if (vlog_) {
+    VLOG(2) << "HasInputsWithMismatchingDeadness(" << node.name()
+            << ") -> false";
+  }
+
+  return false;
+}
+
+void DeadnessAnalysisImpl::Print() const {
+  std::vector tensor_ids;
+  for (const auto& kv_pair : predicate_map_) {
+    tensor_ids.push_back(kv_pair.first);
+  }
+
+  std::sort(tensor_ids.begin(), tensor_ids.end());
+
+  for (TensorId tensor_id : tensor_ids) {
+    auto it = predicate_map_.find(tensor_id);
+    CHECK(it != predicate_map_.end()) << tensor_id.ToString();
+    VLOG(2) << tensor_id.ToString() << " -> " << it->second->ToString();
+  }
+}
+
+}  // namespace
+
+DeadnessAnalysis::~DeadnessAnalysis() {}
+
+/*static*/ Status DeadnessAnalysis::Run(
+    const Graph& graph, std::unique_ptr* result) {
+  std::unique_ptr analysis(
+      new DeadnessAnalysisImpl(&graph));
+  TF_RETURN_IF_ERROR(analysis->Populate());
+
+  if (VLOG_IS_ON(2)) {
+    analysis->Print();
+  }
+
+  *result = std::move(analysis);
+  return Status::OK();
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/deadness_analysis.h b/tensorflow/compiler/jit/deadness_analysis.h
new file mode 100644
index 0000000000..6e7ab41161
--- /dev/null
+++ b/tensorflow/compiler/jit/deadness_analysis.h
@@ -0,0 +1,68 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_JIT_DEADNESS_ANALYSIS_H_
+#define TENSORFLOW_COMPILER_JIT_DEADNESS_ANALYSIS_H_
+
+#include "tensorflow/core/graph/graph.h"
+
+namespace tensorflow {
+
+// This analyzes a TensorFlow graph to identify nodes which may have partially
+// dead inputs (i.e. these nodes may have some dead inputs and some alive
+// inputs).
+//
+// For example, the ADD node in the following graph
+//
+//      V0  PRED0    V1  PRED1
+//       |    |       |    |
+//       v    v       v    v
+//       SWITCH       SWITCH
+//          |            |
+//          +---+   + ---+
+//              |   |
+//              v   v
+//               ADD
+//
+// can have its inputs independently dead or alive based on the runtime values
+// of PRED0 and PRED1.
+//
+// It is tempting to call this a liveness analysis but I avoided that because
+// "liveness" already has other connotations.
+class DeadnessAnalysis {
+ public:
+  // Returns true if `node` may have some live inputs and some dead inputs.
+  //
+  // This is a conservatively correct routine -- if it returns false then `node`
+  // is guaranteed to not have inputs with mismatching liveness, but not the
+  // converse.
+  //
+  // REQUIRES: node is not a Merge operation.
+  virtual bool HasInputsWithMismatchingDeadness(const Node& node) = 0;
+
+  // Prints out the internal state of this instance.  For debugging purposes
+  // only.
+  virtual void Print() const = 0;
+  virtual ~DeadnessAnalysis();
+
+  // Run the deadness analysis over `graph` and returns an error or a populated
+  // instance of DeadnessAnalysis in `result`.
+  static Status Run(const Graph& graph,
+                    std::unique_ptr* result);
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_COMPILER_JIT_DEADNESS_ANALYSIS_H_
diff --git a/tensorflow/compiler/jit/deadness_analysis_test.cc b/tensorflow/compiler/jit/deadness_analysis_test.cc
new file mode 100644
index 0000000000..584385cab7
--- /dev/null
+++ b/tensorflow/compiler/jit/deadness_analysis_test.cc
@@ -0,0 +1,443 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/jit/deadness_analysis.h"
+
+#include "tensorflow/cc/framework/ops.h"
+#include "tensorflow/cc/ops/array_ops.h"
+#include "tensorflow/cc/ops/control_flow_ops_internal.h"
+#include "tensorflow/cc/ops/function_ops.h"
+#include "tensorflow/cc/ops/sendrecv_ops.h"
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/compiler/jit/defs.h"
+#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
+#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/core/framework/node_def_util.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/graph/algorithm.h"
+#include "tensorflow/core/graph/graph_constructor.h"
+#include "tensorflow/core/graph/graph_def_builder.h"
+#include "tensorflow/core/graph/graph_def_builder_util.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace {
+
+Status AnalyzeDeadness(Graph* graph,
+                       std::unique_ptr* result) {
+  FixupSourceAndSinkEdges(graph);
+  return DeadnessAnalysis::Run(*graph, result);
+}
+
+ops::Switch CreateSwitch(const Scope& root, const string& prefix) {
+  Output value = ops::Placeholder(root.WithOpName(prefix + "/value"), DT_FLOAT);
+  Output predicate =
+      ops::Placeholder(root.WithOpName(prefix + "/pred"), DT_BOOL);
+  return ops::Switch(root.WithOpName(prefix + "/switch"), value, predicate);
+}
+
+Output CreateInductionVariable(const Scope& root, const string& prefix,
+                               const string& frame_name, int32 init) {
+  Output initial_value = ops::Const(root.WithOpName(prefix + "/init"), init);
+  Output enter_initial_value = ops::internal::Enter(
+      root.WithOpName(prefix + "/enter"), initial_value, frame_name);
+
+  ops::Merge iv(root.WithOpName(prefix + "/iv"), {enter_initial_value});
+  Output increment_by = ops::Const(root.WithOpName(prefix + "/incr"), 1);
+  Output final_value = ops::Const(root.WithOpName(prefix + "/final"), 10);
+  Output loop_cond_expr =
+      ops::Less(root.WithOpName(prefix + "/less"), iv.output, final_value);
+  Output loop_cond =
+      ops::LoopCond(root.WithOpName(prefix + "/cond"), loop_cond_expr);
+  ops::Switch latch(root.WithOpName(prefix + "/latch"), iv.output, loop_cond);
+  ops::internal::Exit exit(root.WithOpName(prefix + "/exit"), iv.output);
+  Output iv_next =
+      ops::Add(root.WithOpName(prefix + "/ivnext"), iv.output, increment_by);
+  Output next_iteration =
+      ops::NextIteration(root.WithOpName(prefix + "next_iteration"), iv_next);
+
+  root.graph()->AddEdge(next_iteration.node(), 0, iv.output.node(), 1);
+  root.graph()->AddControlEdge(iv.output.node(), increment_by.node());
+  root.graph()->AddControlEdge(iv.output.node(), final_value.node());
+
+  return iv.output;
+}
+
+TEST(DeadnessAnalysisTest, BasicPositive) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  ops::Switch sw = CreateSwitch(root, "0");
+  Output add =
+      ops::Add(root.WithOpName("add"), sw.output_true, sw.output_false);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*add.node()));
+}
+
+TEST(DeadnessAnalysisTest, BasicNegative) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  Output a = ops::Placeholder(root.WithOpName("a"), DT_FLOAT);
+  Output b = ops::Placeholder(root.WithOpName("b"), DT_FLOAT);
+  Output add = ops::Add(root.WithOpName("add"), a, b);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*add.node()));
+}
+
+TEST(DeadnessAnalysisTest, AndIsCommutative) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  ops::Switch sw_0 = CreateSwitch(root, "0");
+  ops::Switch sw_1 = CreateSwitch(root, "1");
+
+  Output a0 =
+      ops::Add(root.WithOpName("a0"), sw_0.output_false, sw_1.output_false);
+  Output a1 =
+      ops::Add(root.WithOpName("a1"), sw_1.output_false, sw_0.output_false);
+
+  Output b0 =
+      ops::Add(root.WithOpName("b0"), sw_0.output_false, sw_1.output_true);
+  Output b1 =
+      ops::Add(root.WithOpName("b1"), sw_1.output_true, sw_0.output_false);
+
+  Output live0 = ops::Add(root.WithOpName("live0"), a0, a1);
+  Output live1 = ops::Add(root.WithOpName("live1"), b0, b1);
+
+  Output halfdead0 = ops::Add(root.WithOpName("halfdead0"), a0, b0);
+  Output halfdead1 = ops::Add(root.WithOpName("halfdead1"), a1, b1);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*live0.node()));
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*live1.node()));
+
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*halfdead0.node()));
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*halfdead1.node()));
+}
+
+TEST(DeadnessAnalysisTest, AndIsAssociative) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  ops::Switch sw_0 = CreateSwitch(root, "0");
+  ops::Switch sw_1 = CreateSwitch(root, "1");
+  ops::Switch sw_2 = CreateSwitch(root, "2");
+
+  Output a0 =
+      ops::Add(root.WithOpName("a0"), sw_0.output_false, sw_1.output_false);
+  Output a1 = ops::Add(root.WithOpName("a1"), a0, sw_2.output_false);
+
+  Output b0 =
+      ops::Add(root.WithOpName("b0"), sw_1.output_false, sw_2.output_false);
+  Output b1 = ops::Add(root.WithOpName("b1"), sw_0.output_false, b0);
+
+  Output add = ops::Add(root.WithOpName("add"), a1, b1);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*add.node()));
+}
+
+TEST(DeadnessAnalysisTest, OrIsCommutative) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  ops::Switch sw_0 = CreateSwitch(root, "0");
+  ops::Switch sw_1 = CreateSwitch(root, "1");
+
+  ops::Merge m0(root.WithOpName("m0"), {sw_0.output_false, sw_1.output_false});
+  ops::Merge m1(root.WithOpName("m1"), {sw_1.output_false, sw_0.output_false});
+  ops::Merge m2(root.WithOpName("m2"), {sw_0.output_false, sw_1.output_true});
+  ops::Merge m3(root.WithOpName("m3"), {sw_1.output_true, sw_0.output_false});
+
+  Output live0 = ops::Add(root.WithOpName("live0"), m0.output, m1.output);
+  Output live1 = ops::Add(root.WithOpName("live1"), m2.output, m3.output);
+
+  Output halfdead0 =
+      ops::Add(root.WithOpName("halfdead0"), m0.output, m2.output);
+  Output halfdead1 =
+      ops::Add(root.WithOpName("halfdead1"), m1.output, m3.output);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*live0.node()));
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*live1.node()));
+
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*halfdead0.node()));
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*halfdead1.node()));
+}
+
+TEST(DeadnessAnalysisTest, OrIsAssociative) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  ops::Switch sw_0 = CreateSwitch(root, "0");
+  ops::Switch sw_1 = CreateSwitch(root, "1");
+  ops::Switch sw_2 = CreateSwitch(root, "2");
+
+  ops::Merge m0(root.WithOpName("m0"), {sw_0.output_false, sw_1.output_false});
+  ops::Merge m1(root.WithOpName("m1"), {m0.output, sw_2.output_false});
+  ops::Merge m2(root.WithOpName("m2"), {sw_1.output_false, sw_2.output_false});
+  ops::Merge m3(root.WithOpName("m3"), {sw_0.output_false, m2.output});
+
+  Output add = ops::Add(root.WithOpName("add"), m1.output, m3.output);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*add.node()));
+}
+
+TEST(DeadnessAnalysisTest, AndOfOr) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  ops::Switch sw_0 = CreateSwitch(root, "0");
+  ops::Switch sw_1 = CreateSwitch(root, "1");
+  ops::Switch sw_2 = CreateSwitch(root, "2");
+  ops::Switch sw_3 = CreateSwitch(root, "3");
+
+  ops::Merge m0(root.WithOpName("m0"), {sw_0.output_false, sw_1.output_false});
+  ops::Merge m1(root.WithOpName("m1"), {sw_2.output_false, sw_3.output_false});
+
+  Output add0 = ops::Add(root.WithOpName("add0"), m0.output, m1.output);
+  Output add1 = ops::Add(root.WithOpName("add1"), m0.output, m1.output);
+
+  Output add2 = ops::Add(root.WithOpName("add2"), add0, add1);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*add2.node()));
+}
+
+TEST(DeadnessAnalysisTest, OrOfAnd) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  ops::Switch sw_0 = CreateSwitch(root, "0");
+  ops::Switch sw_1 = CreateSwitch(root, "1");
+  ops::Switch sw_2 = CreateSwitch(root, "2");
+  ops::Switch sw_3 = CreateSwitch(root, "3");
+
+  Output add0 =
+      ops::Add(root.WithOpName("add0"), sw_0.output_false, sw_1.output_false);
+  Output add1 =
+      ops::Add(root.WithOpName("add1"), sw_2.output_false, sw_3.output_false);
+
+  ops::Merge m0(root.WithOpName("m0"), {add0, add1});
+  ops::Merge m1(root.WithOpName("m1"), {add0, add1});
+
+  Output add2 = ops::Add(root.WithOpName("add2"), m0.output, m1.output);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*add2.node()));
+}
+
+TEST(DeadnessAnalysisTest, NEGATIVE_AndOrDistributive) {
+  // This demonstrates one of the weaknesses in the current approach -- since we
+  // only do some basic simplifications we can't see that "(A|B)&C" ==
+  // "(A&C)|(B&C)".
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  ops::Switch sw_0 = CreateSwitch(root, "0");
+  ops::Switch sw_1 = CreateSwitch(root, "1");
+  ops::Switch sw_2 = CreateSwitch(root, "2");
+
+  ops::Merge m0(root.WithOpName("m0"), {sw_0.output_false, sw_1.output_false});
+  Output add0 = ops::Add(root.WithOpName("add0"), m0.output, sw_2.output_false);
+
+  Output add1 =
+      ops::Add(root.WithOpName("add1"), sw_0.output_false, sw_2.output_false);
+  Output add2 =
+      ops::Add(root.WithOpName("add2"), sw_1.output_false, sw_2.output_false);
+  ops::Merge m1(root.WithOpName("m1"), {add1, add2});
+
+  Output add3 = ops::Add(root.WithOpName("add3"), add0, m1.output);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*add2.node()));
+}
+
+TEST(DeadnessAnalysisTest, Ternary) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  Output predicate = ops::Placeholder(root.WithOpName("predicate"), DT_BOOL);
+  Output true_value = ops::Placeholder(root.WithOpName("true_value"), DT_FLOAT);
+  Output false_value =
+      ops::Placeholder(root.WithOpName("false_value"), DT_FLOAT);
+
+  ops::Switch predicated_true(root.WithOpName("predicated_true"), true_value,
+                              predicate);
+
+  ops::Switch predicated_false(root.WithOpName("predicated_false"), true_value,
+                               predicate);
+  ops::Merge merge(root.WithOpName("ternary"), {predicated_true.output_true,
+                                                predicated_false.output_false});
+  Output addend = ops::Placeholder(root.WithOpName("addend"), DT_FLOAT);
+  Output add = ops::Add(root.WithOpName("add"), merge.output, addend);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*add.node()));
+}
+
+TEST(DeadnessAnalysisTest, Recv) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  Output recv_a = ops::_Recv(root.WithOpName("recv_a"), DT_FLOAT, "tensor_a",
+                             "sender", 0, "receiver");
+  Output recv_b = ops::_Recv(root.WithOpName("recv_b"), DT_FLOAT, "tensor_b",
+                             "sender", 0, "receiver");
+  Output add = ops::Add(root.WithOpName("add"), recv_a, recv_b);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*add.node()));
+}
+
+TEST(DeadnessAnalysisTest, HostRecv) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  Output recv_a = ops::_HostRecv(root.WithOpName("recv_a"), DT_FLOAT,
+                                 "tensor_a", "sender", 0, "receiver");
+  Output recv_b = ops::_HostRecv(root.WithOpName("recv_b"), DT_FLOAT,
+                                 "tensor_b", "sender", 0, "receiver");
+  Output add = ops::Add(root.WithOpName("add"), recv_a, recv_b);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*add.node()));
+}
+
+TEST(DeadnessAnalysisTest, Loop) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+  Output iv0 = CreateInductionVariable(root, "iv0", "fr0", 0);
+  Output iv1 = CreateInductionVariable(root, "iv1", "fr0", 0);
+  Output iv2 = CreateInductionVariable(root, "iv2", "fr0", 1);
+  Output add0 = ops::Add(root.WithOpName("add0"), iv0, iv1);
+  Output add1 = ops::Add(root.WithOpName("add1"), iv1, iv2);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  // NB!  iv0 and iv1 are equivalent and a smarter deadness analysis would have
+  // noticed that.  Today we are pessimistic here because we assign an
+  // uninterpreted symbol to merges with backedges.
+
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*add0.node()));
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*add1.node()));
+}
+
+TEST(DeadnessAnalysisTest, ControlInputs) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+  ops::Switch sw = CreateSwitch(root, "0");
+
+  Output id0 = ops::Identity(root.WithOpName("id0"), sw.output_false);
+  Output id1 = ops::Identity(root.WithOpName("id1"), sw.output_true);
+
+  Output const0 = ops::Const(root.WithOpName("const0"), 1);
+  Output const1 = ops::Const(root.WithOpName("const1"), 2);
+
+  Output add = ops::Add(root.WithOpName("add"), const0, const1);
+
+  root.graph()->AddControlEdge(id0.node(), const0.node());
+  root.graph()->AddControlEdge(id1.node(), const1.node());
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*add.node()));
+}
+
+TEST(DeadnessAnalysisTest, ControlTrigger) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+  ops::Switch sw = CreateSwitch(root, "0");
+
+  Output id0 = ops::Identity(root.WithOpName("id0"), sw.output_false);
+  Output id1 = ops::Identity(root.WithOpName("id1"), sw.output_true);
+
+  ops::ControlTrigger ctrl_trigger0(root.WithOpName("ctrl_trigger0"));
+  ops::ControlTrigger ctrl_trigger1(root.WithOpName("ctrl_trigger1"));
+
+  Output const0 = ops::Const(root.WithOpName("const0"), 1);
+  Output const1 = ops::Const(root.WithOpName("const1"), 2);
+
+  Output add = ops::Add(root.WithOpName("add"), const0, const1);
+
+  root.graph()->AddControlEdge(id0.node(), ctrl_trigger0.operation.node());
+  root.graph()->AddControlEdge(ctrl_trigger0.operation.node(), const0.node());
+
+  root.graph()->AddControlEdge(id1.node(), ctrl_trigger1.operation.node());
+  root.graph()->AddControlEdge(ctrl_trigger1.operation.node(), const1.node());
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*add.node()));
+}
+
+TEST(DeadnessAnalysisTest, ControlInputsToMerge) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+  ops::Switch sw = CreateSwitch(root, "0");
+
+  Output id0 = ops::Identity(root.WithOpName("id0"), sw.output_false);
+  Output id1 = ops::Identity(root.WithOpName("id1"), sw.output_true);
+
+  Output constant = ops::Const(root.WithOpName("constant"), 5);
+  ops::Merge m0(root.WithOpName("m0"), {constant});
+  ops::Merge m1(root.WithOpName("m0"), {constant});
+  Output add = ops::Add(root.WithOpName("add"), m0.output, m1.output);
+
+  root.graph()->AddControlEdge(id0.node(), m0.output.node());
+  root.graph()->AddControlEdge(id1.node(), m1.output.node());
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_FALSE(result->HasInputsWithMismatchingDeadness(*add.node()));
+}
+
+TEST(DeadnessAnalysisTest, RecvVsSwitch) {
+  // Demonstrates why we need the must_be_true bit on SymbolP.
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  Output recv = ops::_Recv(root.WithOpName("recv"), DT_BOOL, "tensor", "sender",
+                           0, "receiver");
+  Output value = ops::Placeholder(root.WithOpName("value"), DT_BOOL);
+  ops::Switch sw(root.WithOpName("switch"), value, recv);
+  Output logical_and =
+      ops::LogicalAnd(root.WithOpName("and"), recv, sw.output_true);
+
+  std::unique_ptr result;
+  TF_ASSERT_OK(AnalyzeDeadness(root.graph(), &result));
+
+  EXPECT_TRUE(result->HasInputsWithMismatchingDeadness(*logical_and.node()));
+}
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 8c3882116d..6558f14dd6 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include 
 #include 
 
+#include "tensorflow/compiler/jit/deadness_analysis.h"
 #include "tensorflow/compiler/jit/defs.h"
 #include "tensorflow/compiler/jit/graphcycles/graphcycles.h"
 #include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h"
@@ -28,6 +29,7 @@ limitations under the License.
 #include "tensorflow/compiler/jit/xla_cluster_util.h"
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/graph_def_util.h"
 #include "tensorflow/core/framework/memory_types.h"
@@ -462,17 +464,27 @@ Status MarkForCompilationPass::Run(
   VLOG(1) << "flags->tf_xla_fusion_only = " << flags->tf_xla_fusion_only;
   const FunctionLibraryDefinition* fld = options.flib_def;
 
-  auto is_compilable = [global_jit_level, cpu_global_jit, fusion_only, fld](
-                           const Node* node, const DeviceType& device_type) {
+  std::unique_ptr deadness;
+  {
+    XLA_SCOPED_LOGGING_TIMER_LEVEL("DeadnessAnalysis", 0);
+    TF_RETURN_IF_ERROR(DeadnessAnalysis::Run(**options.graph, &deadness));
+  }
+
+  auto is_compilable = [&](const Node* node, const DeviceType& device_type) {
     const XlaOpRegistry::DeviceRegistration* registration;
     if (!XlaOpRegistry::GetCompilationDevice(device_type.type(),
                                              ®istration)) {
       return false;
     }
 
+    // TODO(b/111570009): This bailout for ControlTrigger is probably not
+    // needed.
+    //
     // Don't compile control trigger nodes. We won't preserve their deadness
     // semantics correctly, so it's safest not to compile them.
-    if (node->IsControlTrigger()) return false;
+    if (node->IsControlTrigger()) {
+      return false;
+    }
 
     // If this device requires a JIT, we must say yes.
     if (registration->requires_compilation) return true;
@@ -485,6 +497,14 @@ Status MarkForCompilationPass::Run(
     status = fld->GetAttr(*node, kXlaCompileAttr, &compile);
     if (status.ok()) return compile;
 
+    // If inputs to `node` can have conflicting deadness (i.e. some are alive
+    // and some are dead) then don't compile it.  XLA cannot represent the
+    // deadness semantics of these nodes correctly and auto-clustering these
+    // nodes can cause deadness propagate to nodes that should be live.
+    if (node->IsMerge() || deadness->HasInputsWithMismatchingDeadness(*node)) {
+      return false;
+    }
+
     // Check for fusable ops only if requested.
     if (global_jit_level > 0 && fusion_only && !IsXlaFusable(node->def())) {
       return false;
diff --git a/tensorflow/compiler/jit/xla_fusion_optimizer.cc b/tensorflow/compiler/jit/xla_fusion_optimizer.cc
index 74257b09a8..b70e1cf52b 100644
--- a/tensorflow/compiler/jit/xla_fusion_optimizer.cc
+++ b/tensorflow/compiler/jit/xla_fusion_optimizer.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include 
 #include 
 
+#include "tensorflow/compiler/jit/deadness_analysis.h"
 #include "tensorflow/compiler/jit/defs.h"
 #include "tensorflow/compiler/jit/graphcycles/graphcycles.h"
 #include "tensorflow/compiler/jit/union_find.h"
@@ -146,6 +147,9 @@ Status XlaFusionOptimizer::Optimize(grappler::Cluster* cluster,
   TF_RETURN_IF_ERROR(
       ImportGraphDef(options, item.graph, &graph, &shape_refiner));
 
+  std::unique_ptr deadness;
+  TF_RETURN_IF_ERROR(DeadnessAnalysis::Run(graph, &deadness));
+
   // Collect nodes that can be fused via XLA, while ignoring those that
   // explicitly ask for XLA: (*) nodes that are marked to be compiled
   // explicitly. (*) nodes assigned to XLA device.
@@ -185,6 +189,14 @@ Status XlaFusionOptimizer::Optimize(grappler::Cluster* cluster,
       continue;
     }
 
+    // If inputs to `node` can have conflicting deadness (i.e. some are alive
+    // and some are dead) then don't compile it.  XLA cannot represent the
+    // deadness semantics of these nodes correctly and auto-clustering these
+    // nodes can cause deadness propagate to nodes that should be live.
+    if (node->IsMerge() || deadness->HasInputsWithMismatchingDeadness(*node)) {
+      continue;
+    }
+
     compilation_candidates.insert(node);
   }
 
diff --git a/tensorflow/core/graph/algorithm.cc b/tensorflow/core/graph/algorithm.cc
index 4652fbe406..9b4200e0b4 100644
--- a/tensorflow/core/graph/algorithm.cc
+++ b/tensorflow/core/graph/algorithm.cc
@@ -25,7 +25,8 @@ namespace tensorflow {
 
 void DFS(const Graph& g, const std::function& enter,
          const std::function& leave,
-         const NodeComparator& stable_comparator) {
+         const NodeComparator& stable_comparator,
+         const EdgeFilter& edge_filter) {
   // Stack of work to do.
   struct Work {
     Node* node;
@@ -52,7 +53,6 @@ void DFS(const Graph& g, const std::function& enter,
     // Arrange to call leave(n) when all done with descendants.
     if (leave) stack.push_back(Work{n, true});
 
-    gtl::iterator_range nodes = n->out_nodes();
     auto add_work = [&visited, &stack](Node* out) {
       if (!visited[out->id()]) {
         // Note; we must not mark as visited until we actually process it.
@@ -62,16 +62,20 @@ void DFS(const Graph& g, const std::function& enter,
 
     if (stable_comparator) {
       std::vector nodes_sorted;
-      for (Node* out : nodes) {
-        nodes_sorted.emplace_back(out);
+      for (const Edge* out_edge : n->out_edges()) {
+        if (!edge_filter || edge_filter(*out_edge)) {
+          nodes_sorted.emplace_back(out_edge->dst());
+        }
       }
       std::sort(nodes_sorted.begin(), nodes_sorted.end(), stable_comparator);
       for (Node* out : nodes_sorted) {
         add_work(out);
       }
     } else {
-      for (Node* out : nodes) {
-        add_work(out);
+      for (const Edge* out_edge : n->out_edges()) {
+        if (!edge_filter || edge_filter(*out_edge)) {
+          add_work(out_edge->dst());
+        }
       }
     }
   }
@@ -118,8 +122,6 @@ void ReverseDFSFromHelper(const Graph& g, gtl::ArraySlice start,
     // Arrange to call leave(n) when all done with descendants.
     if (leave) stack.push_back(Work{n, true});
 
-    gtl::iterator_range nodes = n->in_nodes();
-
     auto add_work = [&visited, &stack](T out) {
       if (!visited[out->id()]) {
         // Note; we must not mark as visited until we actually process it.
@@ -129,16 +131,16 @@ void ReverseDFSFromHelper(const Graph& g, gtl::ArraySlice start,
 
     if (stable_comparator) {
       std::vector nodes_sorted;
-      for (T in : nodes) {
-        nodes_sorted.emplace_back(in);
+      for (const Edge* in_edge : n->in_edges()) {
+        nodes_sorted.emplace_back(in_edge->src());
       }
       std::sort(nodes_sorted.begin(), nodes_sorted.end(), stable_comparator);
       for (T in : nodes_sorted) {
         add_work(in);
       }
     } else {
-      for (T in : nodes) {
-        add_work(in);
+      for (const Edge* in_edge : n->in_edges()) {
+        add_work(in_edge->src());
       }
     }
   }
@@ -161,14 +163,17 @@ void ReverseDFSFrom(const Graph& g, gtl::ArraySlice start,
 }
 
 void GetPostOrder(const Graph& g, std::vector* order,
-                  const NodeComparator& stable_comparator) {
+                  const NodeComparator& stable_comparator,
+                  const EdgeFilter& edge_filter) {
   order->clear();
-  DFS(g, nullptr, [order](Node* n) { order->push_back(n); }, stable_comparator);
+  DFS(g, nullptr, [order](Node* n) { order->push_back(n); }, stable_comparator,
+      edge_filter);
 }
 
 void GetReversePostOrder(const Graph& g, std::vector* order,
-                         const NodeComparator& stable_comparator) {
-  GetPostOrder(g, order, stable_comparator);
+                         const NodeComparator& stable_comparator,
+                         const EdgeFilter& edge_filter) {
+  GetPostOrder(g, order, stable_comparator, edge_filter);
   std::reverse(order->begin(), order->end());
 }
 
diff --git a/tensorflow/core/graph/algorithm.h b/tensorflow/core/graph/algorithm.h
index ac4a099013..5bbbc6f6dc 100644
--- a/tensorflow/core/graph/algorithm.h
+++ b/tensorflow/core/graph/algorithm.h
@@ -28,6 +28,8 @@ namespace tensorflow {
 // Comparator for two nodes. This is used in order to get a stable ording.
 using NodeComparator = std::function;
 
+using EdgeFilter = std::function;
+
 // Compares two node based on their ids.
 struct NodeComparatorID {
   bool operator()(const Node* n1, const Node* n2) const {
@@ -47,9 +49,11 @@ struct NodeComparatorName {
 // If leave is not empty, calls leave(n) after visiting all children of n.
 // If stable_comparator is set, a stable ordering of visit is achieved by
 // sorting a node's neighbors first before visiting them.
+// If edge_filter is set then ignores edges for which edge_filter returns false.
 extern void DFS(const Graph& g, const std::function& enter,
                 const std::function& leave,
-                const NodeComparator& stable_comparator = {});
+                const NodeComparator& stable_comparator = {},
+                const EdgeFilter& edge_filter = {});
 
 // Perform a reverse depth-first-search on g starting at the sink node.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
@@ -83,15 +87,21 @@ extern void ReverseDFSFrom(const Graph& g, gtl::ArraySlice start,
 // If stable_comparator is set, a stable ordering of visit is achieved by
 // sorting a node's neighbors first before visiting them.
 //
+// If edge_filter is set then ignores edges for which edge_filter returns false.
+//
 // REQUIRES: order is not NULL.
 void GetPostOrder(const Graph& g, std::vector* order,
-                  const NodeComparator& stable_comparator = {});
+                  const NodeComparator& stable_comparator = {},
+                  const EdgeFilter& edge_filter = {});
 
 // Stores in *order the reverse post-order numbering of all nodes
 // If stable_comparator is set, a stable ordering of visit is achieved by
 // sorting a node's neighbors first before visiting them.
+//
+// If edge_filter is set then ignores edges for which edge_filter returns false.
 void GetReversePostOrder(const Graph& g, std::vector* order,
-                         const NodeComparator& stable_comparator = {});
+                         const NodeComparator& stable_comparator = {},
+                         const EdgeFilter& edge_filter = {});
 
 // Prune nodes in "g" that are not in some path from the source node
 // to any node in 'nodes'. Returns true if changes were made to the graph.
diff --git a/tensorflow/core/graph/algorithm_test.cc b/tensorflow/core/graph/algorithm_test.cc
index f67d5a2fd2..60a3e66aa1 100644
--- a/tensorflow/core/graph/algorithm_test.cc
+++ b/tensorflow/core/graph/algorithm_test.cc
@@ -36,6 +36,11 @@ namespace {
 REGISTER_OP("TestParams").Output("o: float");
 REGISTER_OP("TestInput").Output("a: float").Output("b: float");
 REGISTER_OP("TestMul").Input("a: float").Input("b: float").Output("o: float");
+REGISTER_OP("TestUnary").Input("a: float").Output("o: float");
+REGISTER_OP("TestBinary")
+    .Input("a: float")
+    .Input("b: float")
+    .Output("o: float");
 
 // Compares that the order of nodes in 'inputs' respects the
 // pair orders described in 'ordered_pairs'.
@@ -148,5 +153,52 @@ TEST(AlgorithmTest, ReversePostOrderStable) {
     EXPECT_TRUE(ExpectBefore({{"t2", "t3"}}, order, &error));
   }
 }
+
+TEST(AlgorithmTest, PostOrderWithEdgeFilter) {
+  GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+  string error;
+  Node* n0 = ops::SourceOp("TestParams", b.opts().WithName("n0"));
+  Node* n1 = ops::UnaryOp("TestUnary", n0, b.opts().WithName("n1"));
+  Node* n2 = ops::UnaryOp("TestUnary", n1, b.opts().WithName("n2"));
+  Node* n3 = ops::BinaryOp("TestBinary", n2, n0, b.opts().WithName("n3"));
+
+  Graph g(OpRegistry::Global());
+  TF_ASSERT_OK(GraphDefBuilderToGraph(b, &g));
+
+  g.AddEdge(g.FindNodeId(n3->id()), 0, g.FindNodeId(n1->id()), 1);
+
+  std::vector post_order;
+  auto edge_filter = [&](const Edge& e) {
+    return !(e.src()->id() == n3->id() && e.dst()->id() == n1->id());
+  };
+
+  std::vector expected_post_order = {
+      g.sink_node(),          g.FindNodeId(n3->id()), g.FindNodeId(n2->id()),
+      g.FindNodeId(n1->id()), g.FindNodeId(n0->id()), g.source_node()};
+
+  std::vector expected_reverse_post_order = expected_post_order;
+  std::reverse(expected_reverse_post_order.begin(),
+               expected_reverse_post_order.end());
+
+  GetPostOrder(g, &post_order, /*stable_comparator=*/{},
+               /*edge_filter=*/edge_filter);
+
+  ASSERT_EQ(expected_post_order.size(), post_order.size());
+  for (int i = 0; i < post_order.size(); i++) {
+    CHECK_EQ(post_order[i], expected_post_order[i])
+        << post_order[i]->name() << " vs. " << expected_post_order[i]->name();
+  }
+
+  std::vector reverse_post_order;
+  GetReversePostOrder(g, &reverse_post_order, /*stable_comparator=*/{},
+                      /*edge_filter=*/edge_filter);
+
+  ASSERT_EQ(expected_reverse_post_order.size(), reverse_post_order.size());
+  for (int i = 0; i < reverse_post_order.size(); i++) {
+    CHECK_EQ(reverse_post_order[i], expected_reverse_post_order[i])
+        << reverse_post_order[i]->name() << " vs. "
+        << expected_reverse_post_order[i]->name();
+  }
+}
 }  // namespace
 }  // namespace tensorflow
-- 
cgit v1.2.3


From 7d79c72ba025026a908e5ebd15c4bb6817a90e6e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 15:21:35 -0700
Subject: Add function to set inter-op thread pool size in cluster after
 initialization of session-specific threadpools

PiperOrigin-RevId: 205146586
---
 tensorflow/core/grappler/clusters/cluster.cc | 8 ++++++++
 tensorflow/core/grappler/clusters/cluster.h  | 3 +++
 2 files changed, 11 insertions(+)

diff --git a/tensorflow/core/grappler/clusters/cluster.cc b/tensorflow/core/grappler/clusters/cluster.cc
index 8d8c6084ec..6d84283e68 100644
--- a/tensorflow/core/grappler/clusters/cluster.cc
+++ b/tensorflow/core/grappler/clusters/cluster.cc
@@ -29,6 +29,14 @@ void Cluster::AllowSoftPlacement(bool soft_placement_state) {
   options_.config.set_allow_soft_placement(soft_placement_state);
 }
 
+void Cluster::SetNumInterOpThreads(int num_threads) {
+  for (int i = 0; i < options_.config.session_inter_op_thread_pool_size();
+       ++i) {
+    options_.config.mutable_session_inter_op_thread_pool(i)->set_num_threads(
+        num_threads);
+  }
+}
+
 void Cluster::SetNumWarmupSteps(int num_steps) {
   options_.config.mutable_graph_options()->set_build_cost_model_after(
       num_steps);
diff --git a/tensorflow/core/grappler/clusters/cluster.h b/tensorflow/core/grappler/clusters/cluster.h
index 06db36b3aa..e94fb900c0 100644
--- a/tensorflow/core/grappler/clusters/cluster.h
+++ b/tensorflow/core/grappler/clusters/cluster.h
@@ -65,6 +65,9 @@ class Cluster {
   // with reftype input(s) which are from CPU.
   void AllowSoftPlacement(bool soft_placement_state);
 
+  // Update the number of inter-op threads for each per-session threadpool
+  void SetNumInterOpThreads(int num_threads);
+
   // Set the number of steps required to warmup TensorFlow. Must be called
   // before Provision().
   void SetNumWarmupSteps(int num_steps);
-- 
cgit v1.2.3


From e3006b1d706fb171525cdd5cfe3a2305d6a5d879 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 15:21:45 -0700
Subject: Add support for rank >= 1 tensors for XLA top_k_v2.

PiperOrigin-RevId: 205146612
---
 tensorflow/compiler/tests/sort_ops_test.py    | 32 +++++++++++++++++++++
 tensorflow/compiler/tf2xla/kernels/topk_op.cc | 40 ++++++++++++++-------------
 2 files changed, 53 insertions(+), 19 deletions(-)

diff --git a/tensorflow/compiler/tests/sort_ops_test.py b/tensorflow/compiler/tests/sort_ops_test.py
index 9e2ef964a1..7ff01be3cb 100644
--- a/tensorflow/compiler/tests/sort_ops_test.py
+++ b/tensorflow/compiler/tests/sort_ops_test.py
@@ -88,6 +88,38 @@ class XlaSortOpTest(xla_test.XLATestCase):
               topk, [x.astype(dtype)],
               expected=[x[indices].astype(dtype), indices])
 
+  def testTopK2D(self):
+    # TODO(b/26783907): The Sort HLO is not implemented on CPU or GPU.
+    if self.device in ["XLA_CPU", "XLA_GPU"]:
+      return
+
+    supported_types = set(
+        [dtypes.bfloat16.as_numpy_dtype, np.float32, np.int32, np.uint32])
+    for dtype in supported_types.intersection(self.numeric_types):
+      # Use small input size for bfloat16. Otherwise, we'll get duplicate values
+      # after conversion to bfloat16, so the possible resulting index array is
+      # no longer unique.
+      if dtype == dtypes.bfloat16.as_numpy_dtype:
+        array_size = 10
+        k_options = [0, 1, 2, 10]
+      else:
+        array_size = 200 * 1000
+        k_options = [0, 1, 2, 10, 20, 100, 1000, 200 * 1000]
+      batch = 16
+      for x in [np.arange(batch * array_size)]:
+        np.random.shuffle(x)
+        x = np.reshape(x, [batch, array_size])
+        for k in k_options:
+          indices = x.argsort(axis=1)[::, -1:-k - 1:-1]
+          expected = np.sort(x, axis=1)[::, -1:-k - 1:-1]
+
+          def topk(v, k=k):
+            return nn_ops.top_k(v, k=k, sorted=True)
+
+          self._assertOpOutputMatchesExpected(
+              topk, [x.astype(dtype)],
+              expected=[expected.astype(dtype), indices])
+
   def testTopKZeros(self):
     """Tests that positive and negative zeros sort correctly."""
     # TODO(b/26783907): The Sort HLO is not implemented on CPU or GPU.
diff --git a/tensorflow/compiler/tf2xla/kernels/topk_op.cc b/tensorflow/compiler/tf2xla/kernels/topk_op.cc
index 1ddcb08c8e..82d4a69777 100644
--- a/tensorflow/compiler/tf2xla/kernels/topk_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/topk_op.cc
@@ -41,33 +41,35 @@ class TopKOp : public XlaOpKernel {
     OP_REQUIRES(context, input_shape.dims() >= 1,
                 errors::InvalidArgument("input must be >= 1-D, got shape ",
                                         input_shape.DebugString()));
+    int last_dim = input_shape.dims() - 1;
+    int last_dim_size = input_shape.dim_size(last_dim);
     OP_REQUIRES(
-        context, input_shape.dim_size(input_shape.dims() - 1) >= k,
+        context, last_dim_size >= k,
         errors::InvalidArgument("input must have at least k columns. Had ",
-                                input_shape.dim_size(input_shape.dims() - 1),
-                                ", needed ", k));
-
-    OP_REQUIRES(
-        context, input_shape.dims() == 1,
-        errors::Unimplemented("TopK is implemented for 1-D inputs, got shape ",
-                              input_shape.DebugString()));
+                                last_dim_size, ", needed ", k));
 
     xla::XlaBuilder* const b = context->builder();
-    if (input_shape.dim_size(0) < k) {
-      k = input_shape.dim_size(0);
+    if (last_dim_size < k) {
+      k = last_dim_size;
     }
     const xla::XlaOp input = context->Input(0);
-    xla::XlaOp iota_s32 = xla::Iota(b, xla::S32, input_shape.dim_size(0));
-    xla::XlaOp sort_result = xla::Sort(xla::Neg(input), iota_s32);
+
+    xla::XlaOp iota_s32 = xla::Iota(b, xla::S32, last_dim_size);
+    auto input_dims = input_shape.dim_sizes();
+    std::vector broadcast_dims(input_dims.begin(), input_dims.end() - 1);
+    xla::XlaOp broadcast_s32 = xla::Broadcast(iota_s32, broadcast_dims);
+    xla::XlaOp sort_result = xla::Sort(xla::Neg(input), broadcast_s32);
+
+    std::vector start_indices(input_shape.dims(), 0);
+    std::vector limit_indices(input_dims.begin(), input_dims.end());
+    limit_indices[last_dim] = k;
+    std::vector strides(input_shape.dims(), 1);
+
     xla::XlaOp values =
-        xla::Neg(xla::Slice(xla::GetTupleElement(sort_result, 0),
-                            /*start_indices=*/{0},
-                            /*limit_indices=*/{k},
-                            /*strides=*/{1}));
+        xla::Neg(xla::Slice(xla::GetTupleElement(sort_result, 0), start_indices,
+                            limit_indices, strides));
     xla::XlaOp indices = xla::Slice(xla::GetTupleElement(sort_result, 1),
-                                    /*start_indices=*/{0},
-                                    /*limit_indices=*/{k},
-                                    /*strides=*/{1});
+                                    start_indices, limit_indices, strides);
     context->SetOutput(0, values);
     context->SetOutput(1, indices);
   }
-- 
cgit v1.2.3


From c80f1493222720c2601c7d68f0ec0dd236ceb621 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Wed, 18 Jul 2018 15:23:40 -0700
Subject: Add some LOGging when we detect nodes in a cycle

PiperOrigin-RevId: 205146924
---
 tensorflow/core/graph/graph_constructor.cc           |  8 ++++++++
 tensorflow/tools/graph_transforms/transform_utils.cc | 13 ++++++++++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc
index add26f3b71..8c73f8f712 100644
--- a/tensorflow/core/graph/graph_constructor.cc
+++ b/tensorflow/core/graph/graph_constructor.cc
@@ -1042,6 +1042,14 @@ Status GraphConstructor::Convert() {
   }
 
   if (processed < node_defs_.size()) {
+    LOG(WARNING) << "IN " << __func__ << (node_defs_.size() - processed)
+                 << " NODES IN A CYCLE";
+    for (int64 i = 0; i < node_defs_.size(); i++) {
+      if (pending_count_[i] != 0) {
+        LOG(WARNING) << "PENDING: " << SummarizeNodeDef(*node_defs_[i])
+                     << "WITH PENDING COUNT = " << pending_count_[i];
+      }
+    }
     return errors::InvalidArgument(node_defs_.size() - processed,
                                    " nodes in a cycle");
   }
diff --git a/tensorflow/tools/graph_transforms/transform_utils.cc b/tensorflow/tools/graph_transforms/transform_utils.cc
index af17fd75bc..cb084e49b7 100644
--- a/tensorflow/tools/graph_transforms/transform_utils.cc
+++ b/tensorflow/tools/graph_transforms/transform_utils.cc
@@ -247,9 +247,16 @@ Status SortByExecutionOrder(const GraphDef& input_graph_def,
     }
   }
 
-  if (processed < input_graph_def.node_size()) {
-    return errors::InvalidArgument(input_graph_def.node_size() - processed,
-                                   " nodes in a cycle");
+  if (processed < num_nodes) {
+    LOG(WARNING) << "IN " << __func__ << (num_nodes - processed)
+                 << " NODES IN A CYCLE";
+    for (int64 i = 0; i < num_nodes; i++) {
+      if (pending_count[i] != 0) {
+        LOG(WARNING) << "PENDING: " << SummarizeNodeDef(input_graph_def.node(i))
+                     << "WITH PENDING COUNT = " << pending_count[i];
+      }
+    }
+    return errors::InvalidArgument(num_nodes - processed, " nodes in a cycle");
   }
   return Status::OK();
 }
-- 
cgit v1.2.3


From 5e7178ddb7bc8b863469f7240d0cf5a74c77b543 Mon Sep 17 00:00:00 2001
From: Yifei Feng 
Date: Wed, 18 Jul 2018 15:27:50 -0700
Subject: Upgrade bazel to 0.15.0.

PiperOrigin-RevId: 205147588
---
 WORKSPACE                                                      | 2 +-
 configure.py                                                   | 2 +-
 tensorflow/tools/ci_build/ci_sanity.sh                         | 2 +-
 tensorflow/tools/ci_build/install/install_bazel.sh             | 2 +-
 tensorflow/tools/ci_build/install/install_bazel_from_source.sh | 2 +-
 tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh      | 8 ++++----
 tensorflow/tools/docker/Dockerfile.devel                       | 2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu                   | 2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7      | 2 +-
 9 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index fd7570a80a..17961829a6 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -18,7 +18,7 @@ closure_repositories()
 # files, in case the parsing of those build files depends on the bazel
 # version we require here.
 load("//tensorflow:version_check.bzl", "check_bazel_version_at_least")
-check_bazel_version_at_least("0.10.0")
+check_bazel_version_at_least("0.15.0")
 
 load("//tensorflow:workspace.bzl", "tf_workspace")
 
diff --git a/configure.py b/configure.py
index c482628ec8..25729adf36 100644
--- a/configure.py
+++ b/configure.py
@@ -1429,7 +1429,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.10.0')
+  check_bazel_version('0.15.0')
 
   reset_tf_configure_bazelrc(args.workspace)
   cleanup_makefile()
diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index db37edf809..866fe95d2b 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -354,7 +354,7 @@ do_external_licenses_check(){
 
   # Whitelist
   echo ${EXTRA_LICENSE_FILE}
-  grep -e "@bazel_tools//src" -e "@bazel_tools//tools/" -e "@com_google_absl//" -e "//external" -e "@local" -e "@com_github_googlecloudplatform_google_cloud_cpp//" -v ${EXTRA_LICENSES_FILE} > temp.txt
+  grep -e "@bazel_tools//src" -e "@bazel_tools//tools/" -e "@com_google_absl//" -e "//external" -e "@local" -e "@com_github_googlecloudplatform_google_cloud_cpp//" -e "@embedded_jdk//" -v ${EXTRA_LICENSES_FILE} > temp.txt
   mv temp.txt ${EXTRA_LICENSES_FILE}
 
 
diff --git a/tensorflow/tools/ci_build/install/install_bazel.sh b/tensorflow/tools/ci_build/install/install_bazel.sh
index adbff8f6ef..e284401b8a 100755
--- a/tensorflow/tools/ci_build/install/install_bazel.sh
+++ b/tensorflow/tools/ci_build/install/install_bazel.sh
@@ -15,7 +15,7 @@
 # ==============================================================================
 
 # Select bazel version.
-BAZEL_VERSION="0.14.1"
+BAZEL_VERSION="0.15.0"
 
 set +e
 local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}')
diff --git a/tensorflow/tools/ci_build/install/install_bazel_from_source.sh b/tensorflow/tools/ci_build/install/install_bazel_from_source.sh
index 9d24b3e421..87be81577d 100755
--- a/tensorflow/tools/ci_build/install/install_bazel_from_source.sh
+++ b/tensorflow/tools/ci_build/install/install_bazel_from_source.sh
@@ -18,7 +18,7 @@
 # It will compile bazel from source and install it in /usr/local/bin
 
 # Select bazel version.
-BAZEL_VERSION="0.14.1"
+BAZEL_VERSION="0.15.0"
 
 set +e
 local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}')
diff --git a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
index c03cbd9c66..0482cf619a 100644
--- a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
+++ b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
@@ -33,10 +33,10 @@ function set_remote_cache_options {
   echo "build --tls_enabled=true" >> "${TMP_BAZELRC}"
   echo "build --remote_timeout=3600" >> "${TMP_BAZELRC}"
   echo "build --auth_enabled=true" >> "${TMP_BAZELRC}"
-  echo "build --spawn_strategy=remote" >> "${TMP_BAZELRC}"
-  echo "build --strategy=Javac=remote" >> "${TMP_BAZELRC}"
-  echo "build --strategy=Closure=remote" >> "${TMP_BAZELRC}"
-  echo "build --genrule_strategy=remote" >> "${TMP_BAZELRC}"
+  echo "build --spawn_strategy=standalone" >> "${TMP_BAZELRC}"
+  echo "build --strategy=Javac=standalone" >> "${TMP_BAZELRC}"
+  echo "build --strategy=Closure=standalone" >> "${TMP_BAZELRC}"
+  echo "build --genrule_strategy=standalone" >> "${TMP_BAZELRC}"
   echo "build --google_credentials=$GOOGLE_CLOUD_CREDENTIAL" >> "${TMP_BAZELRC}"
 }
 
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index fd94d64268..f7fe4119da 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -63,7 +63,7 @@ RUN echo "startup --batch" >>/etc/bazel.bazelrc
 RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
     >>/etc/bazel.bazelrc
 # Install the most recent bazel release.
-ENV BAZEL_VERSION 0.14.1
+ENV BAZEL_VERSION 0.15.0
 WORKDIR /
 RUN mkdir /bazel && \
     cd /bazel && \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 44120bf274..957a7ed799 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -83,7 +83,7 @@ RUN echo "startup --batch" >>/etc/bazel.bazelrc
 RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
     >>/etc/bazel.bazelrc
 # Install the most recent bazel release.
-ENV BAZEL_VERSION 0.14.1
+ENV BAZEL_VERSION 0.15.0
 WORKDIR /
 RUN mkdir /bazel && \
     cd /bazel && \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
index 3bedc8cf34..30bc2d2806 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
@@ -4,7 +4,7 @@ LABEL maintainer="Gunhan Gulsoy "
 
 # It is possible to override these for releases.
 ARG TF_BRANCH=master
-ARG BAZEL_VERSION=0.5.4
+ARG BAZEL_VERSION=0.15.0
 ARG TF_AVAILABLE_CPUS=32
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-- 
cgit v1.2.3


From 7648454c49b397fba7fc2c73c5d7d2149af3481a Mon Sep 17 00:00:00 2001
From: Sunitha Kambhampati 
Date: Wed, 18 Jul 2018 15:37:21 -0700
Subject: Add SQLITE_OPEN_URI flag to sqlite open to support db_uri and add a
 unit test

---
 tensorflow/contrib/summary/summary_ops_test.py | 19 +++++++++++++++++++
 tensorflow/core/lib/db/sqlite.cc               |  1 +
 2 files changed, 20 insertions(+)

diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py
index 3e41e3d0b4..77b1c93ff2 100644
--- a/tensorflow/contrib/summary/summary_ops_test.py
+++ b/tensorflow/contrib/summary/summary_ops_test.py
@@ -17,9 +17,12 @@ from __future__ import division
 from __future__ import print_function
 
 import os
+import pathlib
 import tempfile
 import time
 
+import sqlite3
+
 import numpy as np
 import six
 
@@ -275,6 +278,22 @@ class EagerFileTest(test_util.TensorFlowTestCase):
 
 class EagerDbTest(summary_test_util.SummaryDbTest):
 
+  def testDbURIOpen(self):
+    tmpdb_path = os.path.join(self.get_temp_dir(), 'tmpDbURITest.sqlite')
+    tmpdb_uri = pathlib.Path(tmpdb_path).as_uri()
+    tmpdb_writer = summary_ops.create_db_writer(
+        tmpdb_uri,
+        "experimentA",
+        "run1",
+        "user1")
+    with summary_ops.always_record_summaries():
+      with tmpdb_writer.as_default():
+        summary_ops.scalar('t1', 2.0)
+    tmpdb = sqlite3.connect(tmpdb_path)
+    num = get_one(tmpdb, 'SELECT count(*) FROM Tags WHERE tag_name = "t1"')
+    self.assertEqual(num, 1)
+    tmpdb.close()
+
   def testIntegerSummaries(self):
     step = training_util.create_global_step()
     writer = self.create_db_writer()
diff --git a/tensorflow/core/lib/db/sqlite.cc b/tensorflow/core/lib/db/sqlite.cc
index cb6943379d..cf11f3a331 100644
--- a/tensorflow/core/lib/db/sqlite.cc
+++ b/tensorflow/core/lib/db/sqlite.cc
@@ -112,6 +112,7 @@ Status EnvPragma(Sqlite* db, const char* pragma, const char* var) {
 /* static */
 Status Sqlite::Open(const string& path, int flags, Sqlite** db) {
   flags |= SQLITE_OPEN_PRIVATECACHE;
+  flags |= SQLITE_OPEN_URI;
   sqlite3* sqlite = nullptr;
   int rc = sqlite3_open_v2(path.c_str(), &sqlite, flags, nullptr);
   if (rc != SQLITE_OK) {
-- 
cgit v1.2.3


From 06a9805c336242ceded4907da9159bf518e6623a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 15:35:52 -0700
Subject: DCGAN using tf.keras and eager execution.

PiperOrigin-RevId: 205149061
---
 .../examples/generative_examples/dcgan.ipynb       | 711 +++++++++++++++++++++
 1 file changed, 711 insertions(+)
 create mode 100644 tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb

diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
new file mode 100644
index 0000000000..43c8c355dc
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
@@ -0,0 +1,711 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "0TD5ZrvEMbhZ"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\").\n",
+        "\n",
+        "# DCGAN: An example with tf.keras and eager\n",
+        "\n",
+        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\u003ctd\u003e\n",
+        "\u003ca target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb\"\u003e\n",
+        "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e  \n",
+        "\u003c/td\u003e\u003ctd\u003e\n",
+        "\u003ca target=\"_blank\"  href=\"https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb\"\u003e\u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\u003c/td\u003e\u003c/table\u003e"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "ITZuApL56Mny"
+      },
+      "source": [
+        "This notebook demonstrates how to generate images of handwritten digits using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). To do this, we use Deep Convolutional Generative Adverserial Networks ([DCGAN](https://arxiv.org/pdf/1511.06434.pdf)).\n",
+        "\n",
+        "On a colab GPU(Tesla K80), the model takes around 40 seconds per epoch to train.\n",
+        "\n",
+        "Below is the output generated after training the generator and discriminator models for 100 epochs.\n",
+        "\n",
+        "![sample output](https://tensorflow.org/images/gan/dcgan.gif)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "u_2z-B3piVsw"
+      },
+      "outputs": [],
+      "source": [
+        "# to generate gifs\n",
+        "!pip install imageio"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "e1_Y75QXJS6h"
+      },
+      "source": [
+        "## Import TensorFlow and enable eager execution"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "YfIk2es3hJEd"
+      },
+      "outputs": [],
+      "source": [
+        "# Import TensorFlow \u003e= 1.9 and enable eager execution\n",
+        "import tensorflow as tf\n",
+        "tf.enable_eager_execution()\n",
+        "\n",
+        "import os\n",
+        "import time\n",
+        "import numpy as np\n",
+        "import glob\n",
+        "import matplotlib.pyplot as plt\n",
+        "import PIL\n",
+        "import imageio\n",
+        "from IPython import display"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "iYn4MdZnKCey"
+      },
+      "source": [
+        "## Load the dataset\n",
+        "\n",
+        "We are going to use the MNIST dataset to train the generator and the discriminator. The generator will then generate handwritten digits."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "a4fYMGxGhrna"
+      },
+      "outputs": [],
+      "source": [
+        "(train_images, train_labels), (_, _) = tf.keras.datasets.mnist.load_data()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "NFC2ghIdiZYE"
+      },
+      "outputs": [],
+      "source": [
+        "train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')\n",
+        "# We are normalizing the images to the range of [-1, 1]\n",
+        "train_images = (train_images - 127.5) / 127.5"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "S4PIDhoDLbsZ"
+      },
+      "outputs": [],
+      "source": [
+        "BUFFER_SIZE = 60000\n",
+        "BATCH_SIZE = 256"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "PIGN6ouoQxt3"
+      },
+      "source": [
+        "## Use tf.data to create batches and shuffle the dataset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "-yKCCQOoJ7cn"
+      },
+      "outputs": [],
+      "source": [
+        "train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "THY-sZMiQ4UV"
+      },
+      "source": [
+        "## Write the generator and discriminator models\n",
+        "\n",
+        "* **Generator** \n",
+        "  * It is responsible for **creating the convincing images good enough to fool the discriminator**.\n",
+        "  * It consists of Conv2DTranspose(Upsampling) layers. We start with a fully connected layer and upsample the image 2 times so as to reach the desired image size(mnist image size) which is (28, 28, 1). \n",
+        "  * We use **leaky relu** activation except for the **last layer** which uses **tanh** activation.\n",
+        "  \n",
+        "* **Discriminator**\n",
+        "  * **The discriminator is responsible for classifying the fake images from the real images.**\n",
+        "  * In other words, the discriminator is given generated images(from the generator) and the real MNIST images. The job of the discriminator is to classify these images into fake(generated) and real(MNIST images).\n",
+        "  * **Basically the generator should be good enough to fool the discriminator that the generated images are real**."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "VGLbvBEmjK0a"
+      },
+      "outputs": [],
+      "source": [
+        "class Generator(tf.keras.Model):\n",
+        "  def __init__(self):\n",
+        "    super(Generator, self).__init__()\n",
+        "    self.fc1 = tf.keras.layers.Dense(7*7*64, use_bias=False)\n",
+        "    self.batchnorm1 = tf.keras.layers.BatchNormalization()\n",
+        "    \n",
+        "    self.conv1 = tf.keras.layers.Conv2DTranspose(64, (5, 5), strides=(1, 1), padding='same', use_bias=False)\n",
+        "    self.batchnorm2 = tf.keras.layers.BatchNormalization()\n",
+        "    \n",
+        "    self.conv2 = tf.keras.layers.Conv2DTranspose(32, (5, 5), strides=(2, 2), padding='same', use_bias=False)\n",
+        "    self.batchnorm3 = tf.keras.layers.BatchNormalization()\n",
+        "    \n",
+        "    self.conv3 = tf.keras.layers.Conv2DTranspose(1, (5, 5), strides=(2, 2), padding='same', use_bias=False)\n",
+        "\n",
+        "  def call(self, x, training=True):\n",
+        "    x = self.fc1(x)\n",
+        "    x = self.batchnorm1(x, training=training)\n",
+        "    x = tf.nn.relu(x)\n",
+        "\n",
+        "    x = tf.reshape(x, shape=(-1, 7, 7, 64))\n",
+        "\n",
+        "    x = self.conv1(x)\n",
+        "    x = self.batchnorm2(x, training=training)\n",
+        "    x = tf.nn.relu(x)\n",
+        "\n",
+        "    x = self.conv2(x)\n",
+        "    x = self.batchnorm3(x, training=training)\n",
+        "    x = tf.nn.relu(x)\n",
+        "\n",
+        "    x = tf.nn.tanh(self.conv3(x))  \n",
+        "    return x"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "bkOfJxk5j5Hi"
+      },
+      "outputs": [],
+      "source": [
+        "class Discriminator(tf.keras.Model):\n",
+        "  def __init__(self):\n",
+        "    super(Discriminator, self).__init__()\n",
+        "    self.conv1 = tf.keras.layers.Conv2D(64, (5, 5), strides=(2, 2), padding='same')\n",
+        "    self.conv2 = tf.keras.layers.Conv2D(128, (5, 5), strides=(2, 2), padding='same')\n",
+        "    self.dropout = tf.keras.layers.Dropout(0.3)\n",
+        "    self.flatten = tf.keras.layers.Flatten()\n",
+        "    self.fc1 = tf.keras.layers.Dense(1)\n",
+        "\n",
+        "  def call(self, x, training=True):\n",
+        "    x = tf.nn.leaky_relu(self.conv1(x))\n",
+        "    x = self.dropout(x, training=training)\n",
+        "    x = tf.nn.leaky_relu(self.conv2(x))\n",
+        "    x = self.dropout(x, training=training)\n",
+        "    x = self.flatten(x)\n",
+        "    x = self.fc1(x)\n",
+        "    return x"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "gDkA05NE6QMs"
+      },
+      "outputs": [],
+      "source": [
+        "generator = Generator()\n",
+        "discriminator = Discriminator()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "0FMYgY_mPfTi"
+      },
+      "source": [
+        "## Define the loss functions and the optimizer\n",
+        "\n",
+        "* **Discriminator loss**\n",
+        "  * The discriminator loss function takes 2 inputs; **real images, generated images**\n",
+        "  * real_loss is a sigmoid cross entropy loss of the **real images** and an **array of ones(since these are the real images)**\n",
+        "  * generated_loss is a sigmoid cross entropy loss of the **generated images** and an **array of zeros(since these are the fake images)**\n",
+        "  * Then the total_loss is the sum of real_loss and the generated_loss\n",
+        "  \n",
+        "* **Generator loss**\n",
+        "  * It is a sigmoid cross entropy loss of the generated images and an **array of ones**\n",
+        "  \n",
+        "\n",
+        "* The discriminator and the generator optimizers are different since we will train them separately."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "wkMNfBWlT-PV"
+      },
+      "outputs": [],
+      "source": [
+        "def discriminator_loss(real_output, generated_output):\n",
+        "    # [1,1,...,1] with real output since it is true and we want\n",
+        "    # our generated examples to look like it\n",
+        "    real_loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=tf.ones_like(real_output), logits=real_output)\n",
+        "\n",
+        "    # [0,0,...,0] with generated images since they are fake\n",
+        "    generated_loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=tf.zeros_like(generated_output), logits=generated_output)\n",
+        "\n",
+        "    total_loss = real_loss + generated_loss\n",
+        "\n",
+        "    return total_loss"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "90BIcCKcDMxz"
+      },
+      "outputs": [],
+      "source": [
+        "def generator_loss(generated_output):\n",
+        "    return tf.losses.sigmoid_cross_entropy(tf.ones_like(generated_output), generated_output)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "iWCn_PVdEJZ7"
+      },
+      "outputs": [],
+      "source": [
+        "discriminator_optimizer = tf.train.AdamOptimizer(1e-4)\n",
+        "generator_optimizer = tf.train.AdamOptimizer(1e-4)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "Rw1fkAczTQYh"
+      },
+      "source": [
+        "## Training\n",
+        "\n",
+        "* We start by iterating over the dataset\n",
+        "* The generator is given **noise as an input** which when passed through the generator model will output a image looking like a handwritten digit\n",
+        "* The discriminator is given the **real MNIST images as well as the generated images(from the generator)**.\n",
+        "* Next, we calculate the generator and the discriminator loss.\n",
+        "* Then, we calculate the gradients of loss with respect to both the generator and the discriminator variables(inputs) and apply those to the optimizer.\n",
+        "\n",
+        "## Generate Images\n",
+        "\n",
+        "* After training, its time to generate some images!\n",
+        "* We start by creating noise array as an input to the generator\n",
+        "* The generator will then convert the noise into handwritten images.\n",
+        "* Last step is to plot the predictions and **voila!**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "NS2GWywBbAWo"
+      },
+      "outputs": [],
+      "source": [
+        "EPOCHS = 150\n",
+        "noise_dim = 100\n",
+        "num_examples_to_generate = 100\n",
+        "\n",
+        "# keeping the random vector constant for generation(prediction) so\n",
+        "# it will be easier to see the improvement of the gan.\n",
+        "random_vector_for_generation = tf.random_normal([num_examples_to_generate,\n",
+        "                                                 noise_dim])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "RmdVsmvhPxyy"
+      },
+      "outputs": [],
+      "source": [
+        "def generate_and_save_images(model, epoch, test_input):\n",
+        "  # make sure the training parameter is set to False because we\n",
+        "  # don't want to train the batchnorm layer when doing inference.\n",
+        "  predictions = model(test_input, training=False)\n",
+        "\n",
+        "  fig = plt.figure(figsize=(10,10))\n",
+        "  \n",
+        "  for i in range(predictions.shape[0]):\n",
+        "      plt.subplot(10, 10, i+1)\n",
+        "      plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5, cmap='gray')\n",
+        "      plt.axis('off')\n",
+        "        \n",
+        "  # tight_layout minimizes the overlap between 2 sub-plots\n",
+        "  plt.tight_layout()\n",
+        "  plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))\n",
+        "  plt.show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "2M7LmLtGEMQJ"
+      },
+      "outputs": [],
+      "source": [
+        "def train(dataset, epochs, noise_dim):  \n",
+        "  for epoch in range(epochs):\n",
+        "    start = time.time()\n",
+        "    \n",
+        "    for images in dataset:\n",
+        "      # generating noise from a uniform distribution\n",
+        "      noise = tf.random_normal([BATCH_SIZE, noise_dim])\n",
+        "      \n",
+        "      with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n",
+        "        generated_images = generator(noise, training=True)\n",
+        "      \n",
+        "        real_output = discriminator(images, training=True)\n",
+        "        generated_output = discriminator(generated_images, training=True)\n",
+        "        \n",
+        "        gen_loss = generator_loss(generated_output)\n",
+        "        disc_loss = discriminator_loss(real_output, generated_output)\n",
+        "        \n",
+        "      gradients_of_generator = gen_tape.gradient(gen_loss, generator.variables)\n",
+        "      gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.variables)\n",
+        "      \n",
+        "      generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.variables))\n",
+        "      discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.variables))\n",
+        "\n",
+        "      \n",
+        "    if epoch % 10 == 0:\n",
+        "      display.clear_output(wait=True)\n",
+        "      generate_and_save_images(generator,\n",
+        "                               epoch + 1,\n",
+        "                               random_vector_for_generation)\n",
+        "\n",
+        "    print ('Time taken for epoch {} is {} sec'.format(epoch + 1,\n",
+        "                                                      time.time()-start))\n",
+        "  # generating after the final epoch\n",
+        "  generate_and_save_images(generator,\n",
+        "                           epochs,\n",
+        "                           random_vector_for_generation)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "Ly3UN0SLLY2l"
+      },
+      "outputs": [],
+      "source": [
+        "train(train_dataset, EPOCHS, noise_dim)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "P4M_vIbUi7c0"
+      },
+      "source": [
+        "# Display an image using the epoch number"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "WfO5wCdclHGL"
+      },
+      "outputs": [],
+      "source": [
+        "def display_image(epoch_no):\n",
+        "  plt.figure(figsize=(15,15))\n",
+        "  plt.imshow(np.array(PIL.Image.open('image_at_epoch_{:04d}.png'.format(epoch_no))))\n",
+        "  plt.axis('off')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "5x3q9_Oe5q0A"
+      },
+      "outputs": [],
+      "source": [
+        "display_image(EPOCHS)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "NywiH3nL8guF"
+      },
+      "source": [
+        "## Generate a GIF of all the saved images."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "xmO0Dmu2WICn"
+      },
+      "source": [
+        "\u003c!-- TODO(markdaoust): Remove the hack when Ipython version is updated --\u003e\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "IGKQgENQ8lEI"
+      },
+      "outputs": [],
+      "source": [
+        "with imageio.get_writer('dcgan.gif', mode='I') as writer:\n",
+        "  filenames = glob.glob('image*.png')\n",
+        "  filenames = sorted(filenames)\n",
+        "  for filename in filenames:\n",
+        "    image = imageio.imread(filename)\n",
+        "    writer.append_data(image)\n",
+        "  # this is a hack to display the gif inside the notebook\n",
+        "  os.system('mv dcgan.gif dcgan.gif.png')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "uV0yiKpzNP1b"
+      },
+      "outputs": [],
+      "source": [
+        "display.Image(filename=\"dcgan.gif.png\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "4UJjSnIMOzOJ"
+      },
+      "outputs": [],
+      "source": [
+        ""
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "default_view": {},
+      "name": "dcgan.ipynb",
+      "private_outputs": true,
+      "provenance": [
+        {
+          "file_id": "1eb0NOTQapkYs3X0v-zL1x5_LFKgDISnp",
+          "timestamp": 1527173385672
+        }
+      ],
+      "toc_visible": true,
+      "version": "0.3.2",
+      "views": {}
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
-- 
cgit v1.2.3


From 36c4bf1143220819a18ebdcd4f8aaca864a1644c Mon Sep 17 00:00:00 2001
From: Allen Lavoie 
Date: Wed, 18 Jul 2018 16:10:06 -0700
Subject: Tweak tf.keras.Model layer collection to auto-flatten non-Layer
 container types.

This makes model.layers consistent between "model.l = Dense(2)" and "model.l = [Dense(2)]", and makes introspection slightly more pleasant (non-empty but useless automatically generated container types will be omitted).

PiperOrigin-RevId: 205154485
---
 .../checkpointable/data_structures_test.py         | 25 +++++++++++++---------
 .../python/training/checkpointable/layer_utils.py  | 13 +++++++----
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py
index ec8c9da809..7bee00a927 100644
--- a/tensorflow/python/training/checkpointable/data_structures_test.py
+++ b/tensorflow/python/training/checkpointable/data_structures_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 import os
 
 import numpy
+import six
 
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
@@ -72,11 +73,14 @@ class ListTests(test.TestCase):
     model = HasList()
     output = model(array_ops.ones([32, 2]))
     self.assertAllEqual([32, 12], output.shape)
-    self.assertEqual(2, len(model.layers))
-    self.assertIs(model.layer_list, model.layers[0])
-    self.assertEqual(10, len(model.layers[0].layers))
+    self.assertEqual(11, len(model.layers))
+    self.assertEqual(10, len(model.layer_list.layers))
+    six.assertCountEqual(
+        self,
+        model.layers,
+        model.layer_list.layers + model.layers_with_updates)
     for index in range(10):
-      self.assertEqual(3 + index, model.layers[0].layers[index].units)
+      self.assertEqual(3 + index, model.layer_list.layers[index].units)
     self.assertEqual(2, len(model._checkpoint_dependencies))
     self.assertIs(model.layer_list, model._checkpoint_dependencies[0].ref)
     self.assertIs(model.layers_with_updates,
@@ -123,9 +127,11 @@ class ListTests(test.TestCase):
         self.l2 = []
 
     model = HasEqualContainers()
-    model.l1.append(HasEqualContainers())
-    model.l2.append(HasEqualContainers())
-    self.assertEqual([model.l1, model.l2], model.layers)
+    first_layer = HasEqualContainers()
+    model.l1.append(first_layer)
+    second_layer = HasEqualContainers()
+    model.l2.append(second_layer)
+    self.assertEqual([first_layer, second_layer], model.layers)
 
   def testNotCheckpointable(self):
     class NotCheckpointable(object):
@@ -260,9 +266,8 @@ class MappingTests(test.TestCase):
     model = HasMapping()
     output = model(array_ops.ones([32, 2]))
     self.assertAllEqual([32, 7], output.shape)
-    self.assertEqual(1, len(model.layers))
-    self.assertIs(model.layer_dict, model.layers[0])
-    self.assertEqual(3, len(model.layers[0].layers))
+    self.assertEqual(5, len(model.layers))
+    six.assertCountEqual(self, model.layers, model.layer_dict.layers)
     self.assertEqual(1, len(model._checkpoint_dependencies))
     self.assertIs(model.layer_dict, model._checkpoint_dependencies[0].ref)
     self.evaluate([v.initializer for v in model.variables])
diff --git a/tensorflow/python/training/checkpointable/layer_utils.py b/tensorflow/python/training/checkpointable/layer_utils.py
index 978fcb2252..d65b631fe9 100644
--- a/tensorflow/python/training/checkpointable/layer_utils.py
+++ b/tensorflow/python/training/checkpointable/layer_utils.py
@@ -32,10 +32,15 @@ def is_layer(obj):
 
 def filter_empty_layer_containers(layer_list):
   """Filter out empty Layer-like containers."""
-  return [layer for layer in layer_list
-          # Filter out only empty Checkpointable data structures. Empty Networks
-          # will still show up in Model.layers.
-          if is_layer(layer) or getattr(layer, "layers", True)]
+  filtered = []
+  for obj in layer_list:
+    if is_layer(obj):
+      filtered.append(obj)
+    else:
+      # Checkpointable data structures will not show up in ".layers" lists, but
+      # the layers they contain will.
+      filtered.extend(obj.layers)
+  return filtered
 
 
 def gather_trainable_weights(trainable, sub_layers, extra_variables):
-- 
cgit v1.2.3


From 02725138f41befb78573e6abc177baed06cc1b3f Mon Sep 17 00:00:00 2001
From: Yunxing Dai 
Date: Wed, 18 Jul 2018 16:11:57 -0700
Subject: - Use InlinedVector for ShapeIndex. - Use a separate IndexTable for
 lookups. This reduces the number of cachelines needed for storing
 ShapeTreeNodes. - Set up benchmark for flat tuples, useful as a benchmark for
 future optimizations.

name                          old time/op    new time/op    delta
BM_Construct/2/8              8.34?s ? 1%    7.57?s ? 2%      -9.26%  (p=0.008 n=5+5)
BM_Construct/1/1000            143?s ? 1%     132?s ? 1%      -7.29%  (p=0.008 n=5+5)
BM_ConstructUnowned/2/8       2.18?s ? 4%    1.31?s ? 1%     -39.99%  (p=0.008 n=5+5)
BM_ConstructUnowned/1/1000    23.0?s ? 7%    15.1?s ? 1%     -34.47%  (p=0.008 n=5+5)
BM_Copy/2/8                   1.52?s ? 5%    0.37?s ? 1%     -76.01%  (p=0.008 n=5+5)
BM_Copy/1/1000                18.7?s ? 3%     4.9?s ? 2%     -73.85%  (p=0.008 n=5+5)
BM_Move/2/8                    0.03ns ? 2%   13.42ns ? 1%  +40877.10%  (p=0.016 n=4+5)
BM_Move/1/1000                 0.03ns ? 0%   13.54ns ? 3%  +40930.30%  (p=0.016 n=4+5)
BM_ForEach/2/8                 26.4ns ? 1%    27.9ns ? 2%      +5.77%  (p=0.008 n=5+5)
BM_ForEach/1/1000               271ns ? 1%     273ns ? 0%      +0.81%  (p=0.016 n=5+4)
BM_Iterate/2/8                 25.5ns ? 3%    23.9ns ? 8%        ~     (p=0.151 n=5+5)
BM_Iterate/1/1000               272ns ? 2%     271ns ? 1%        ~     (p=0.984 n=5+5)

name                          old allocs/op  new allocs/op  delta
BM_Construct/2/8                  373 ? 0%       276 ? 0%     -26.01%  (p=0.008 n=5+5)
BM_Construct/1/1000             5.00k ? 0%     4.00k ? 0%     -20.00%  (p=0.008 n=5+5)
BM_ConstructUnowned/2/8          99.0 ? 0%       2.0 ? 0%     -97.98%  (p=0.008 n=5+5)
BM_ConstructUnowned/1/1000      1.00k ? 0%     0.00k ? 0%     -99.80%  (p=0.008 n=5+5)
BM_Copy/2/8                       105 ? 0%        19 ? 0%     -81.90%  (p=0.008 n=5+5)
BM_Copy/1/1000                  1.31k ? 0%     0.25k ? 0%     -80.84%  (p=0.008 n=5+5)
BM_Move/2/8                      23.0 ? 0%      17.0 ? 0%     -26.09%  (p=0.008 n=5+5)
BM_Move/1/1000                    313 ? 0%       250 ? 0%     -20.13%  (p=0.008 n=5+5)
BM_ForEach/2/8                   23.0 ? 0%      17.0 ? 0%     -26.09%  (p=0.008 n=5+5)
BM_ForEach/1/1000                 313 ? 0%       250 ? 0%     -20.13%  (p=0.008 n=5+5)
BM_Iterate/2/8                   23.0 ? 0%      17.0 ? 0%     -26.09%  (p=0.008 n=5+5)
BM_Iterate/1/1000                 313 ? 0%       250 ? 0%     -20.13%  (p=0.008 n=5+5)

PiperOrigin-RevId: 205154861
---
 tensorflow/compiler/xla/shape_tree.h       | 140 ++++++++++++++++++++++-------
 tensorflow/compiler/xla/shape_tree_test.cc |  21 +++--
 tensorflow/compiler/xla/shape_util.h       |  13 +--
 3 files changed, 131 insertions(+), 43 deletions(-)

diff --git a/tensorflow/compiler/xla/shape_tree.h b/tensorflow/compiler/xla/shape_tree.h
index 4aacc87b78..c74dd648ad 100644
--- a/tensorflow/compiler/xla/shape_tree.h
+++ b/tensorflow/compiler/xla/shape_tree.h
@@ -44,10 +44,6 @@ struct ShapeTreeNode {
   // Data corresponding to this node.
   std::pair data;
 
-  // Children of this node, as indices into the container's nodes_ array.
-  std::vector children;
-
-  // Tells whether this is a leaf node.
   bool is_leaf = true;
 
   explicit ShapeTreeNode(ShapeIndex index)
@@ -56,6 +52,20 @@ struct ShapeTreeNode {
       : data(std::move(index), std::move(data)) {}
 };
 
+// Internal representation of an index table entry.
+struct IndexTableEntry {
+  // Index of the node in the ShapeTreeNode vector.
+  uint32 index;
+  // Index of the first child in a IndexTableEntry vector. In the index
+  // table all children entries for a given node will be placed next to each
+  // other. This allows us to use a single field to index them.
+  uint32 children_start;
+#ifndef NDEBUG
+  // Number of children, used for bounds checking.
+  uint32 children_count;
+#endif
+};
+
 }  // namespace internal
 
 template 
@@ -84,6 +94,7 @@ template 
 class ShapeTree {
  public:
   using Node = internal::ShapeTreeNode;
+  using Index = internal::IndexTableEntry;
 
   // Default constructor creates a tree with a nil shape (i.e. an empty tuple).
   ShapeTree() : ShapeTree(ShapeUtil::MakeNil()) {}
@@ -267,11 +278,12 @@ class ShapeTree {
  private:
   // Initialize node->children based on 'shape'. All children are assigned the
   // the given 'init_value'.
-  void InitChildren(const Shape& shape, const T& init_value, Node* node);
+  void InitChildren(const Shape& shape, const T& init_value, Node* node,
+                    Index* index);
 
   // Initialize node->children based on 'shape'. All children have
   // default-constructed data values.
-  void InitChildren(const Shape& shape, Node* node);
+  void InitChildren(const Shape& shape, Node* node, Index* index);
 
   // Returns the number of subshapes, including interior nodes, in shape.
   int64 CountSubshapes(const Shape& shape);
@@ -291,6 +303,9 @@ class ShapeTree {
   // The nodes in this shape tree.
   std::vector nodes_;
 
+  // Index table for node lookups.
+  std::vector index_table_;
+
   // If we own our Shape, this field contains it, and shape_ is a pointer into
   // here.  Otherwise if we don't own our shape, this is nullptr.
   std::shared_ptr shape_storage_;
@@ -373,36 +388,74 @@ int64 ShapeTree::CountSubshapes(const Shape& shape) {
 
 template 
 void ShapeTree::InitChildren(const Shape& shape, const T& init_value,
-                                Node* node) {
+                                Node* node, Index* index) {
   if (ShapeUtil::IsTuple(shape)) {
     const int64 size = ShapeUtil::TupleElementCount(shape);
-    node->children.reserve(size);
+#ifndef NDEBUG
+    index->children_count = size;
+#endif
     node->is_leaf = false;
     ShapeIndex shape_index = node->data.first;
     shape_index.push_back(0);
+
+    // At the end of the index_table, reserve a continuous space to hold the
+    // children of current node. In order to enforce the invariant that all
+    // children of a given node are placed together, we need to do the
+    // reservation before we recurse into any of its children.
+    int64 children_start_position = index_table_.size();
+    index_table_.resize(index_table_.size() + size);
+
     for (int i = 0; i < size; ++i) {
       shape_index[shape_index.size() - 1] = i;
-      node->children.push_back(nodes_.size());
+      index_table_[children_start_position + i].index = nodes_.size();
+      // The first child of the node in the index table is placed at the end of
+      // the table.
+      index_table_[children_start_position + i].children_start =
+          index_table_.size();
       nodes_.emplace_back(shape_index, init_value);
-      InitChildren(shape.tuple_shapes(i), init_value, &nodes_.back());
+      InitChildren(shape.tuple_shapes(i), init_value, &nodes_.back(),
+                   &index_table_[children_start_position + i]);
     }
+  } else {
+#ifndef NDEBUG
+    index->children_count = 0;
+#endif
   }
 }
 
 template 
-void ShapeTree::InitChildren(const Shape& shape, Node* node) {
+void ShapeTree::InitChildren(const Shape& shape, Node* node, Index* index) {
   if (ShapeUtil::IsTuple(shape)) {
     const int64 size = ShapeUtil::TupleElementCount(shape);
-    node->children.reserve(size);
+#ifndef NDEBUG
+    index->children_count = size;
+#endif
     node->is_leaf = false;
     ShapeIndex shape_index = node->data.first;
     shape_index.push_back(0);
+
+    // At the end of the index_table, reserve a continuous space to hold the
+    // children of current node. In order to enforce the invariant that all
+    // children of a given node are placed together, we need to do the
+    // reservation before we recurse into any of its children.
+    int64 children_start_position = index_table_.size();
+    index_table_.resize(index_table_.size() + size);
+
     for (int i = 0; i < size; ++i) {
       shape_index[shape_index.size() - 1] = i;
-      node->children.push_back(nodes_.size());
+      index_table_[children_start_position + i].index = nodes_.size();
+      // The first child of the node in the index table is placed at the end of
+      // the table.
+      index_table_[children_start_position + i].children_start =
+          index_table_.size();
       nodes_.emplace_back(shape_index);
-      InitChildren(shape.tuple_shapes(i), &nodes_.back());
+      InitChildren(shape.tuple_shapes(i), &nodes_.back(),
+                   &index_table_[children_start_position + i]);
     }
+  } else {
+#ifndef NDEBUG
+    index->children_count = 0;
+#endif
   }
 }
 
@@ -413,24 +466,36 @@ ShapeTree::ShapeTree(Shape shape)
   // The shape_ field is just used to hold the structure of the shape.
   // It should not be relied upon to store layout information.
   LayoutUtil::ClearLayout(shape_storage_.get());
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{});
-  InitChildren(*shape_, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, &nodes_[0], &index_table_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const Shape* shape) : shape_(shape) {
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{});
-  InitChildren(*shape_, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, &nodes_[0], &index_table_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const std::shared_ptr& shape)
     : shape_storage_(shape), shape_(shape_storage_.get()) {
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{});
-  InitChildren(*shape_, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, &nodes_[0], &index_table_[0]);
 }
 
 template 
@@ -440,26 +505,38 @@ ShapeTree::ShapeTree(Shape shape, const T& init_value)
   // The shape_ field is just used to hold the structure of the shape.
   // It should not be relied upon to store layout information.
   LayoutUtil::ClearLayout(shape_storage_.get());
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{}, init_value);
-  InitChildren(*shape_, init_value, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, init_value, &nodes_[0], &index_table_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const Shape* shape, const T& init_value)
     : shape_(shape) {
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{}, init_value);
-  InitChildren(*shape_, init_value, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, init_value, &nodes_[0], &index_table_[0]);
 }
 
 template 
 ShapeTree::ShapeTree(const std::shared_ptr& shape,
                         const T& init_value)
     : shape_storage_(shape), shape_(shape_storage_.get()) {
-  nodes_.reserve(CountSubshapes(*shape_));
+  const int64 count = CountSubshapes(*shape_);
+  nodes_.reserve(count);
   nodes_.emplace_back(ShapeIndex{}, init_value);
-  InitChildren(*shape_, init_value, &nodes_[0]);
+
+  index_table_.reserve(count);
+  index_table_.emplace_back(Index{0, 1});
+  InitChildren(*shape_, init_value, &nodes_[0], &index_table_[0]);
 }
 
 template 
@@ -474,13 +551,16 @@ T* ShapeTree::mutable_element(ShapeIndexView index) {
 
 template 
 internal::ShapeTreeNode* ShapeTree::Lookup(ShapeIndexView index) {
-  Node* node = &nodes_[0];
+  Index* iter = &index_table_[0];
   for (const int64 i : index) {
     CHECK_GE(i, 0);
-    CHECK_LT(i, node->children.size());
-    node = &nodes_[node->children[i]];
+#ifndef NDEBUG
+    CHECK_LT(i, iter->children_count);
+#endif
+    iter = &index_table_[iter->children_start + i];
   }
-  return node;
+
+  return &nodes_[iter->index];
 }
 
 template 
diff --git a/tensorflow/compiler/xla/shape_tree_test.cc b/tensorflow/compiler/xla/shape_tree_test.cc
index 51de82e957..4391078b64 100644
--- a/tensorflow/compiler/xla/shape_tree_test.cc
+++ b/tensorflow/compiler/xla/shape_tree_test.cc
@@ -227,14 +227,16 @@ TEST_F(ShapeTreeTest, NestedTupleShape) {
 
 TEST_F(ShapeTreeTest, InvalidIndexingTuple) {
   ShapeTree shape_tree{tuple_shape_};
-
+#ifndef NDEBUG
   EXPECT_DEATH(shape_tree.element({4}), "");
+#endif
 }
 
 TEST_F(ShapeTreeTest, InvalidIndexingNestedTuple) {
   ShapeTree shape_tree{nested_tuple_shape_};
-
+#ifndef NDEBUG
   EXPECT_DEATH(shape_tree.element({0, 0}), "");
+#endif
 }
 
 TEST_F(ShapeTreeTest, ShapeTreeOfNonCopyableType) {
@@ -602,12 +604,15 @@ void BM_Iterate(int iters, int depth, int fan_out) {
   }
 }
 
-BENCHMARK(BM_Construct)->ArgPair(2, 8);
-BENCHMARK(BM_ConstructUnowned)->ArgPair(2, 8);
-BENCHMARK(BM_Copy)->ArgPair(2, 8);
-BENCHMARK(BM_Move)->ArgPair(2, 8);
-BENCHMARK(BM_ForEach)->ArgPair(2, 8);
-BENCHMARK(BM_Iterate)->ArgPair(2, 8);
+#define BENCHMARK_WITH_ARGS(name) \
+  BENCHMARK(name)->ArgPair(2, 8)->ArgPair(1, 1000)
+
+BENCHMARK_WITH_ARGS(BM_Construct);
+BENCHMARK_WITH_ARGS(BM_ConstructUnowned);
+BENCHMARK_WITH_ARGS(BM_Copy);
+BENCHMARK_WITH_ARGS(BM_Move);
+BENCHMARK_WITH_ARGS(BM_ForEach);
+BENCHMARK_WITH_ARGS(BM_Iterate);
 
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index 17c1d7b10a..d6f17fc965 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/gtl/optional.h"
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/env.h"
@@ -73,10 +74,12 @@ class ShapeIndex {
   // push_front is O(n^2), but shapes don't usually have a ton of dimensions.
   void push_front(int64 value) { indices_.insert(indices_.begin(), value); }
 
-  std::vector::const_iterator begin() const { return indices_.begin(); }
-  std::vector::const_iterator end() const { return indices_.end(); }
-  std::vector::iterator begin() { return indices_.begin(); }
-  std::vector::iterator end() { return indices_.end(); }
+  using container_type = tensorflow::gtl::InlinedVector;
+
+  container_type::const_iterator begin() const { return indices_.begin(); }
+  container_type::const_iterator end() const { return indices_.end(); }
+  container_type::iterator begin() { return indices_.begin(); }
+  container_type::iterator end() { return indices_.end(); }
 
   const int64* data() const { return indices_.data(); }
 
@@ -97,7 +100,7 @@ class ShapeIndex {
   string ToString() const;
 
  private:
-  std::vector indices_;
+  container_type indices_;
 };
 
 // A view into a ShapeIndex as above, with the cheap/easy ability to consume the
-- 
cgit v1.2.3


From 4cca48f61d0f81c4966e0b83e0f8269b5f9b84e8 Mon Sep 17 00:00:00 2001
From: Yanan Cao 
Date: Wed, 18 Jul 2018 16:18:15 -0700
Subject: Allow eval_metrics to be None in TPUEstimatorSpec

PiperOrigin-RevId: 205155840
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 78b79b111e..c7cd7896e0 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -1336,7 +1336,8 @@ class _ModelFnWrapper(object):
       loss = tpu_estimator_spec.loss
       captured_scaffold_fn.capture(tpu_estimator_spec.scaffold_fn)
       to_record = {}
-      to_record['eval_metrics'] = tpu_estimator_spec.eval_metrics
+      if tpu_estimator_spec.eval_metrics:
+        to_record['eval_metrics'] = tpu_estimator_spec.eval_metrics
       if tpu_estimator_spec.host_call is not None:
         # We assume that evaluate won't update global step, so we don't wrap
         # this host_call.
@@ -1639,7 +1640,7 @@ class _OutfeedHostCall(object):
       RuntimeError: If outfeed tensor is scalar.
     """
     if not self._names:
-      return []
+      return {}
 
     ret = {}
     # For each i, dequeue_ops[i] is a list containing the tensors from all
@@ -2514,7 +2515,8 @@ class TPUEstimator(estimator_lib.Estimator):
           host_call_ret = host_calls.create_tpu_hostcall()
           eval_metric_ops = {}
           eval_update_ops = []
-          for k, v in host_call_ret['eval_metrics'].items():
+
+          for k, v in host_call_ret.get('eval_metrics', {}).items():
             eval_metric_ops[k] = (v[0], dummy_update_op)
             eval_update_ops.append(v[1])
 
-- 
cgit v1.2.3


From bb52a6663a0141de53ddaf844f6c7087c0ddf7f7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 16:29:35 -0700
Subject: ConfigProto.experimental.num_dev_to_dev_copy_streams defaults to 0 by
 proto3 semantics.  Silently correct that value to 1, without logging an
 error.

PiperOrigin-RevId: 205157429
---
 tensorflow/core/common_runtime/gpu/gpu_device.cc | 1 +
 tensorflow/core/protobuf/config.proto            | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 7110ffd40c..3292ef2f62 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -225,6 +225,7 @@ class BaseGPUDevice::StreamGroupFactory {
 
       int num_d2d_streams =
           options.experimental().num_dev_to_dev_copy_streams();
+      if (num_d2d_streams == 0) num_d2d_streams = 1;
       if (num_d2d_streams < 1 || num_d2d_streams > 4) {
         LOG(ERROR)
             << "Illegal GPUOptions.experimental.num_dev_to_dev_copy_streams="
diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto
index 77639461d9..22a2691dcc 100644
--- a/tensorflow/core/protobuf/config.proto
+++ b/tensorflow/core/protobuf/config.proto
@@ -145,7 +145,8 @@ message GPUOptions {
     bool use_unified_memory = 2;
 
     // If > 1, the number of device-to-device copy streams to create
-    // for each GPUDevice.
+    // for each GPUDevice.  Default value is 0, which is automatically
+    // converted to 1.
     int32 num_dev_to_dev_copy_streams = 3;
   }
 
-- 
cgit v1.2.3


From 474b40bc7cb33d25f9bdc187d021e94a807bf1bd Mon Sep 17 00:00:00 2001
From: Shashi Shekhar 
Date: Wed, 18 Jul 2018 16:53:53 -0700
Subject: Fix iOS build. - Fix flatbuffers to use 1.8 version. - Fix cc flags
 to not include --std=c++11. - Drop i386 build in fat binary.

PiperOrigin-RevId: 205160898
---
 tensorflow/contrib/lite/Makefile                   | 3 ++-
 tensorflow/contrib/lite/build_ios_universal_lib.sh | 3 +--
 tensorflow/contrib/lite/download_dependencies.sh   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/lite/Makefile b/tensorflow/contrib/lite/Makefile
index a616138d33..df5954744a 100644
--- a/tensorflow/contrib/lite/Makefile
+++ b/tensorflow/contrib/lite/Makefile
@@ -82,8 +82,9 @@ endif
 
 # Settings for the host compiler.
 CXX := $(CC_PREFIX) ${TARGET_TOOLCHAIN_PREFIX}g++
-CXXFLAGS += --std=c++11 -O3 -DNDEBUG
+CXXFLAGS += -O3 -DNDEBUG
 CCFLAGS := ${CXXFLAGS}
+CXXFLAGS += --std=c++11
 CC := $(CC_PREFIX) ${TARGET_TOOLCHAIN_PREFIX}gcc
 AR := $(CC_PREFIX) ${TARGET_TOOLCHAIN_PREFIX}ar
 CFLAGS :=
diff --git a/tensorflow/contrib/lite/build_ios_universal_lib.sh b/tensorflow/contrib/lite/build_ios_universal_lib.sh
index e9531aef19..31df43a175 100755
--- a/tensorflow/contrib/lite/build_ios_universal_lib.sh
+++ b/tensorflow/contrib/lite/build_ios_universal_lib.sh
@@ -21,7 +21,7 @@ cd "$SCRIPT_DIR/../../.."
 
 # Build library for supported architectures and packs them in a fat binary.
 make_library() {
-    for arch in x86_64 i386 armv7 armv7s arm64
+    for arch in x86_64 armv7 armv7s arm64
     do
         make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=${arch} \
         -j 8 \
@@ -29,7 +29,6 @@ make_library() {
     done
     lipo \
     tensorflow/contrib/lite/gen/lib/ios_x86_64/${1} \
-    tensorflow/contrib/lite/gen/lib/ios_i386/${1} \
     tensorflow/contrib/lite/gen/lib/ios_armv7/${1} \
     tensorflow/contrib/lite/gen/lib/ios_armv7s/${1} \
     tensorflow/contrib/lite/gen/lib/ios_arm64/${1} \
diff --git a/tensorflow/contrib/lite/download_dependencies.sh b/tensorflow/contrib/lite/download_dependencies.sh
index 840015a7fa..8c7df474d5 100755
--- a/tensorflow/contrib/lite/download_dependencies.sh
+++ b/tensorflow/contrib/lite/download_dependencies.sh
@@ -35,7 +35,7 @@ GOOGLETEST_URL="https://github.com/google/googletest/archive/release-1.8.0.tar.g
 ABSL_URL="$(grep -o 'https://github.com/abseil/abseil-cpp/.*tar.gz' "${BZL_FILE_PATH}" | head -n1)"
 NEON_2_SSE_URL="https://github.com/intel/ARM_NEON_2_x86_SSE/archive/master.zip"
 FARMHASH_URL="https://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz"
-FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/master.zip"
+FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/v1.8.0.zip"
 FFT2D_URL="https://mirror.bazel.build/www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz"
 
 # TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64,
-- 
cgit v1.2.3


From ae62a692ddb53253462c1f79702fbc45baeb4ae3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 16:58:22 -0700
Subject: Windows: Update in common_env.sh

Respect TMPDIR if it's already set.

PiperOrigin-RevId: 205161568
---
 tensorflow/tools/ci_build/windows/bazel/common_env.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/windows/bazel/common_env.sh b/tensorflow/tools/ci_build/windows/bazel/common_env.sh
index 3af132217e..333a89d3f5 100644
--- a/tensorflow/tools/ci_build/windows/bazel/common_env.sh
+++ b/tensorflow/tools/ci_build/windows/bazel/common_env.sh
@@ -26,7 +26,8 @@
 # * Bazel windows executable copied as "bazel.exe" and included in PATH.
 
 # Use a temporary directory with a short name.
-export TMPDIR="C:/tmp"
+export TMPDIR=${TMPDIR:-"C:/tmp"}
+export TMPDIR=$(cygpath -m "$TMPDIR")
 mkdir -p "$TMPDIR"
 
 # Set bash path
-- 
cgit v1.2.3


From f55028af4861bc78516164975a43f259507adf60 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 17:15:56 -0700
Subject: Automated rollback of commit 36a66347e8e344cddee4a8d9123ccbcae40011b1

PiperOrigin-RevId: 205164273
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 29 ++---------------------------
 1 file changed, 2 insertions(+), 27 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index e85b6db511..766a0dafb5 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -2411,19 +2411,6 @@ port::Status CudnnSupport::DoConvolveImpl(
                           stream, cudnn, algorithm_config, input_nd, filter,
                           conv, output_nd, scratch_allocator, &scratch));
 
-  if (cudnn_type == CUDNN_DATA_HALF &&
-      filter_descriptor.layout() == dnn::FilterLayout::kOutputYXInput &&
-      (algo_desc.algo_id() != CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM ||
-       input_descriptor.layout() != dnn::DataLayout::kBatchYXDepth ||
-       output_descriptor.layout() != dnn::DataLayout::kBatchYXDepth)) {
-    // TODO(timshen): Attach a nvbugs number.
-    return port::Status(
-        port::error::INTERNAL,
-        "Cudnn doesn't return an error code on this documented unsupported "
-        "layout combination. Instead, it accesses out-of-bounds memory. "
-        "Being nice and returning an error instead.");
-  }
-
   std::unique_ptr timer;
   if (is_profiling) {
     timer.reset(new CUDATimer(parent_));  // NOLINT
@@ -3093,21 +3080,9 @@ port::Status CudnnSupport::DoConvolveBackwardDataImpl(
     }
   }
 
-  if (cudnn_type == CUDNN_DATA_HALF &&
-      filter_descriptor.layout() == dnn::FilterLayout::kOutputYXInput &&
-      ((algo_desc.algo_id() != CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0 &&
-        algo_desc.algo_id() != CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1) ||
-       input_descriptor.layout() != dnn::DataLayout::kBatchYXDepth ||
-       output_descriptor.layout() != dnn::DataLayout::kBatchYXDepth)) {
-    return port::Status(
-        port::error::INTERNAL,
-        "Cudnn doesn't return an error code on this documented unsupported "
-        "layout combination. Instead, it crashes. Being nice and returning an "
-        "error instead. See nvbugs/2260917");
-  }
-
   // Cudnn 7.1.4 has a bug if the workspace of the following convolution is not
-  // zero-initialized. See nvbugs/2254619.
+  // zero-initialized.
+  // TODO(timshen): Add an nvbugs/ link.
   if (CUDNN_VERSION >= 7000 &&
       algorithm_config.algorithm().algo_id() ==
           CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 &&
-- 
cgit v1.2.3


From 6bd8f7d241a1bb7492c8bb552ac7df5c22a3edc1 Mon Sep 17 00:00:00 2001
From: Rachel Lim 
Date: Wed, 18 Jul 2018 17:16:03 -0700
Subject: [tf.data] Update size of csv_dataset_op_test to medium

PiperOrigin-RevId: 205164288
---
 tensorflow/contrib/data/python/kernel_tests/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index d372bed479..f805027727 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -60,7 +60,7 @@ py_test(
 
 py_test(
     name = "csv_dataset_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["csv_dataset_op_test.py"],
     srcs_version = "PY2AND3",
     tags = ["no_pip"],
-- 
cgit v1.2.3


From 4a547aa15545aa71bbbeca2ea444f41996730246 Mon Sep 17 00:00:00 2001
From: Jiri Simsa 
Date: Wed, 18 Jul 2018 17:22:58 -0700
Subject: Adding rationale for not including uint32 and uint64 as part of
 TF_CALL_INTEGRAL_TYPES.

PiperOrigin-RevId: 205165086
---
 tensorflow/core/framework/register_types.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/core/framework/register_types.h b/tensorflow/core/framework/register_types.h
index e90596980f..f1cd37ecda 100644
--- a/tensorflow/core/framework/register_types.h
+++ b/tensorflow/core/framework/register_types.h
@@ -151,6 +151,12 @@ limitations under the License.
 
 // Defines for sets of types.
 
+// TODO(b/111604096): Add uint32 and uint64 to TF_CALL_INTEGRAL_TYPES.
+//
+// The uint32 and uint64 types were introduced in 10/2017 to be used via XLA and
+// thus were not included in TF_CALL_INTEGRAL_TYPES. Including them in
+// TF_CALL_INTEGRAL_TYPES should only happen after evaluating the effect on the
+// TF binary size and performance.
 #define TF_CALL_INTEGRAL_TYPES(m)                                      \
   TF_CALL_int64(m) TF_CALL_int32(m) TF_CALL_uint16(m) TF_CALL_int16(m) \
       TF_CALL_uint8(m) TF_CALL_int8(m)
-- 
cgit v1.2.3


From 11020427c924ee21453feccdb7d4d8384006de10 Mon Sep 17 00:00:00 2001
From: Yifei Feng 
Date: Wed, 18 Jul 2018 17:23:28 -0700
Subject: Automated rollback of commit 5e7178ddb7bc8b863469f7240d0cf5a74c77b543

PiperOrigin-RevId: 205165137
---
 WORKSPACE                                                      | 2 +-
 configure.py                                                   | 2 +-
 tensorflow/tools/ci_build/ci_sanity.sh                         | 2 +-
 tensorflow/tools/ci_build/install/install_bazel.sh             | 2 +-
 tensorflow/tools/ci_build/install/install_bazel_from_source.sh | 2 +-
 tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh      | 8 ++++----
 tensorflow/tools/docker/Dockerfile.devel                       | 2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu                   | 2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7      | 2 +-
 9 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index 17961829a6..fd7570a80a 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -18,7 +18,7 @@ closure_repositories()
 # files, in case the parsing of those build files depends on the bazel
 # version we require here.
 load("//tensorflow:version_check.bzl", "check_bazel_version_at_least")
-check_bazel_version_at_least("0.15.0")
+check_bazel_version_at_least("0.10.0")
 
 load("//tensorflow:workspace.bzl", "tf_workspace")
 
diff --git a/configure.py b/configure.py
index 25729adf36..c482628ec8 100644
--- a/configure.py
+++ b/configure.py
@@ -1429,7 +1429,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.15.0')
+  check_bazel_version('0.10.0')
 
   reset_tf_configure_bazelrc(args.workspace)
   cleanup_makefile()
diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 866fe95d2b..db37edf809 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -354,7 +354,7 @@ do_external_licenses_check(){
 
   # Whitelist
   echo ${EXTRA_LICENSE_FILE}
-  grep -e "@bazel_tools//src" -e "@bazel_tools//tools/" -e "@com_google_absl//" -e "//external" -e "@local" -e "@com_github_googlecloudplatform_google_cloud_cpp//" -e "@embedded_jdk//" -v ${EXTRA_LICENSES_FILE} > temp.txt
+  grep -e "@bazel_tools//src" -e "@bazel_tools//tools/" -e "@com_google_absl//" -e "//external" -e "@local" -e "@com_github_googlecloudplatform_google_cloud_cpp//" -v ${EXTRA_LICENSES_FILE} > temp.txt
   mv temp.txt ${EXTRA_LICENSES_FILE}
 
 
diff --git a/tensorflow/tools/ci_build/install/install_bazel.sh b/tensorflow/tools/ci_build/install/install_bazel.sh
index e284401b8a..adbff8f6ef 100755
--- a/tensorflow/tools/ci_build/install/install_bazel.sh
+++ b/tensorflow/tools/ci_build/install/install_bazel.sh
@@ -15,7 +15,7 @@
 # ==============================================================================
 
 # Select bazel version.
-BAZEL_VERSION="0.15.0"
+BAZEL_VERSION="0.14.1"
 
 set +e
 local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}')
diff --git a/tensorflow/tools/ci_build/install/install_bazel_from_source.sh b/tensorflow/tools/ci_build/install/install_bazel_from_source.sh
index 87be81577d..9d24b3e421 100755
--- a/tensorflow/tools/ci_build/install/install_bazel_from_source.sh
+++ b/tensorflow/tools/ci_build/install/install_bazel_from_source.sh
@@ -18,7 +18,7 @@
 # It will compile bazel from source and install it in /usr/local/bin
 
 # Select bazel version.
-BAZEL_VERSION="0.15.0"
+BAZEL_VERSION="0.14.1"
 
 set +e
 local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}')
diff --git a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
index 0482cf619a..c03cbd9c66 100644
--- a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
+++ b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
@@ -33,10 +33,10 @@ function set_remote_cache_options {
   echo "build --tls_enabled=true" >> "${TMP_BAZELRC}"
   echo "build --remote_timeout=3600" >> "${TMP_BAZELRC}"
   echo "build --auth_enabled=true" >> "${TMP_BAZELRC}"
-  echo "build --spawn_strategy=standalone" >> "${TMP_BAZELRC}"
-  echo "build --strategy=Javac=standalone" >> "${TMP_BAZELRC}"
-  echo "build --strategy=Closure=standalone" >> "${TMP_BAZELRC}"
-  echo "build --genrule_strategy=standalone" >> "${TMP_BAZELRC}"
+  echo "build --spawn_strategy=remote" >> "${TMP_BAZELRC}"
+  echo "build --strategy=Javac=remote" >> "${TMP_BAZELRC}"
+  echo "build --strategy=Closure=remote" >> "${TMP_BAZELRC}"
+  echo "build --genrule_strategy=remote" >> "${TMP_BAZELRC}"
   echo "build --google_credentials=$GOOGLE_CLOUD_CREDENTIAL" >> "${TMP_BAZELRC}"
 }
 
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index f7fe4119da..fd94d64268 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -63,7 +63,7 @@ RUN echo "startup --batch" >>/etc/bazel.bazelrc
 RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
     >>/etc/bazel.bazelrc
 # Install the most recent bazel release.
-ENV BAZEL_VERSION 0.15.0
+ENV BAZEL_VERSION 0.14.1
 WORKDIR /
 RUN mkdir /bazel && \
     cd /bazel && \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 957a7ed799..44120bf274 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -83,7 +83,7 @@ RUN echo "startup --batch" >>/etc/bazel.bazelrc
 RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
     >>/etc/bazel.bazelrc
 # Install the most recent bazel release.
-ENV BAZEL_VERSION 0.15.0
+ENV BAZEL_VERSION 0.14.1
 WORKDIR /
 RUN mkdir /bazel && \
     cd /bazel && \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
index 30bc2d2806..3bedc8cf34 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
@@ -4,7 +4,7 @@ LABEL maintainer="Gunhan Gulsoy "
 
 # It is possible to override these for releases.
 ARG TF_BRANCH=master
-ARG BAZEL_VERSION=0.15.0
+ARG BAZEL_VERSION=0.5.4
 ARG TF_AVAILABLE_CPUS=32
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-- 
cgit v1.2.3


From 3dbbf2943fbbbac5f1b0b97f0e03aa51a703a611 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar 
Date: Wed, 18 Jul 2018 17:26:01 -0700
Subject: Register Identity kernel of resource type for GPU.

PiperOrigin-RevId: 205165413
---
 tensorflow/core/kernels/identity_op.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/kernels/identity_op.cc b/tensorflow/core/kernels/identity_op.cc
index dffb4d7171..6f79729883 100644
--- a/tensorflow/core/kernels/identity_op.cc
+++ b/tensorflow/core/kernels/identity_op.cc
@@ -145,6 +145,7 @@ REGISTER_GPU_KERNEL(Variant);
 REGISTER_GPU_HOST_KERNEL(int32);
 REGISTER_GPU_HOST_KERNEL(bool);
 REGISTER_GPU_HOST_KERNEL(string);
+REGISTER_GPU_HOST_KERNEL(ResourceHandle);
 
 #undef REGISTER_GPU_HOST_KERNEL
 
-- 
cgit v1.2.3


From ff222c19fd642a6001ef8ba0ba7a823f37886a5d Mon Sep 17 00:00:00 2001
From: Michael Case 
Date: Wed, 18 Jul 2018 17:43:34 -0700
Subject: Internal Change.

PiperOrigin-RevId: 205167512
---
 tensorflow/python/debug/examples/examples_test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/debug/examples/examples_test.sh b/tensorflow/python/debug/examples/examples_test.sh
index 2d35b2d8bb..f7d597c8c0 100755
--- a/tensorflow/python/debug/examples/examples_test.sh
+++ b/tensorflow/python/debug/examples/examples_test.sh
@@ -99,7 +99,7 @@ if [[ -d "${CUSTOM_DUMP_ROOT}" ]]; then
 fi
 
 # Test debugging of tf.keras.
-cat << EOF | "${DEBUG_KERAS_BIN}" --debug --ui_type=readline
+cat << EOF | ${DEBUG_KERAS_BIN} --debug --ui_type=readline
 run -f has_inf_or_nan
 EOF
 
-- 
cgit v1.2.3


From e7278e47c1c683bcbc710bc27fe19b995cc0ca51 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Wed, 18 Jul 2018 18:01:21 -0700
Subject: Disable the deadness analysis

This is a semantic rollback which should make CL's fixing the deadness analysis
pass easier to make / review.

PiperOrigin-RevId: 205169229
---
 tensorflow/compiler/jit/mark_for_compilation_pass.cc | 15 ---------------
 tensorflow/compiler/jit/xla_fusion_optimizer.cc      | 12 ------------
 2 files changed, 27 deletions(-)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 6558f14dd6..73db0d5952 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -21,7 +21,6 @@ limitations under the License.
 #include 
 #include 
 
-#include "tensorflow/compiler/jit/deadness_analysis.h"
 #include "tensorflow/compiler/jit/defs.h"
 #include "tensorflow/compiler/jit/graphcycles/graphcycles.h"
 #include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h"
@@ -464,12 +463,6 @@ Status MarkForCompilationPass::Run(
   VLOG(1) << "flags->tf_xla_fusion_only = " << flags->tf_xla_fusion_only;
   const FunctionLibraryDefinition* fld = options.flib_def;
 
-  std::unique_ptr deadness;
-  {
-    XLA_SCOPED_LOGGING_TIMER_LEVEL("DeadnessAnalysis", 0);
-    TF_RETURN_IF_ERROR(DeadnessAnalysis::Run(**options.graph, &deadness));
-  }
-
   auto is_compilable = [&](const Node* node, const DeviceType& device_type) {
     const XlaOpRegistry::DeviceRegistration* registration;
     if (!XlaOpRegistry::GetCompilationDevice(device_type.type(),
@@ -497,14 +490,6 @@ Status MarkForCompilationPass::Run(
     status = fld->GetAttr(*node, kXlaCompileAttr, &compile);
     if (status.ok()) return compile;
 
-    // If inputs to `node` can have conflicting deadness (i.e. some are alive
-    // and some are dead) then don't compile it.  XLA cannot represent the
-    // deadness semantics of these nodes correctly and auto-clustering these
-    // nodes can cause deadness propagate to nodes that should be live.
-    if (node->IsMerge() || deadness->HasInputsWithMismatchingDeadness(*node)) {
-      return false;
-    }
-
     // Check for fusable ops only if requested.
     if (global_jit_level > 0 && fusion_only && !IsXlaFusable(node->def())) {
       return false;
diff --git a/tensorflow/compiler/jit/xla_fusion_optimizer.cc b/tensorflow/compiler/jit/xla_fusion_optimizer.cc
index b70e1cf52b..74257b09a8 100644
--- a/tensorflow/compiler/jit/xla_fusion_optimizer.cc
+++ b/tensorflow/compiler/jit/xla_fusion_optimizer.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include 
 #include 
 
-#include "tensorflow/compiler/jit/deadness_analysis.h"
 #include "tensorflow/compiler/jit/defs.h"
 #include "tensorflow/compiler/jit/graphcycles/graphcycles.h"
 #include "tensorflow/compiler/jit/union_find.h"
@@ -147,9 +146,6 @@ Status XlaFusionOptimizer::Optimize(grappler::Cluster* cluster,
   TF_RETURN_IF_ERROR(
       ImportGraphDef(options, item.graph, &graph, &shape_refiner));
 
-  std::unique_ptr deadness;
-  TF_RETURN_IF_ERROR(DeadnessAnalysis::Run(graph, &deadness));
-
   // Collect nodes that can be fused via XLA, while ignoring those that
   // explicitly ask for XLA: (*) nodes that are marked to be compiled
   // explicitly. (*) nodes assigned to XLA device.
@@ -189,14 +185,6 @@ Status XlaFusionOptimizer::Optimize(grappler::Cluster* cluster,
       continue;
     }
 
-    // If inputs to `node` can have conflicting deadness (i.e. some are alive
-    // and some are dead) then don't compile it.  XLA cannot represent the
-    // deadness semantics of these nodes correctly and auto-clustering these
-    // nodes can cause deadness propagate to nodes that should be live.
-    if (node->IsMerge() || deadness->HasInputsWithMismatchingDeadness(*node)) {
-      continue;
-    }
-
     compilation_candidates.insert(node);
   }
 
-- 
cgit v1.2.3


From e6fb3baeed99543b3ffd22522c30b08c94835e15 Mon Sep 17 00:00:00 2001
From: RJ Ryan 
Date: Wed, 18 Jul 2018 18:40:27 -0700
Subject: Update tf.strided_slice documentation to reflect that end and strides
 are ignored when shrink_axis_mask is set.

PiperOrigin-RevId: 205173137
---
 tensorflow/python/ops/array_ops.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 361667ec49..ec6488ea63 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -636,10 +636,10 @@ def strided_slice(input_,
   `foo[:4, tf.newaxis, :2]` would produce a shape `(4, 1, 2)` tensor.
 
   If the ith bit of `shrink_axis_mask` is set, it implies that the ith
-  specification shrinks the dimensionality by 1. `begin[i]`, `end[i]` and
-  `strides[i]` must imply a slice of size 1 in the dimension. For example in
-  Python one might do `foo[:, 3, :]` which would result in
-  `shrink_axis_mask` equal to 2.
+  specification shrinks the dimensionality by 1, taking on the value at index
+  `begin[i]`. `end[i]` and `strides[i]` are ignored in this case. For example in
+  Python one might do `foo[:, 3, :]` which would result in `shrink_axis_mask`
+  equal to 2.
 
 
   NOTE: `begin` and `end` are zero-indexed.
-- 
cgit v1.2.3


From 6fbfa767b659468b408cacee4aeb2934aa50949e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 18:51:11 -0700
Subject: add int32 support for div

PiperOrigin-RevId: 205173981
---
 tensorflow/contrib/lite/kernels/div.cc             | 62 ++++++++++++++--------
 tensorflow/contrib/lite/kernels/div_test.cc        | 61 ++++++++++++++++++++-
 .../kernels/internal/reference/reference_ops.h     |  9 ++--
 .../lite/testing/generated_examples_zip_test.cc    |  2 -
 4 files changed, 103 insertions(+), 31 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/div.cc b/tensorflow/contrib/lite/kernels/div.cc
index bc5c3783fd..d7420ddd8e 100644
--- a/tensorflow/contrib/lite/kernels/div.cc
+++ b/tensorflow/contrib/lite/kernels/div.cc
@@ -78,29 +78,44 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 }
 
 template 
-void EvalFloat(TfLiteContext* context, TfLiteNode* node,
-               TfLiteDivParams* params, const OpData* data,
-               const TfLiteTensor* input1, const TfLiteTensor* input2,
-               TfLiteTensor* output) {
-  float output_activation_min, output_activation_max;
-  CalculateActivationRange(params->activation, &output_activation_min,
-                           &output_activation_max);
-#define TF_LITE_DIV(type, opname)                                   \
-  type::opname(GetTensorData(input1), GetTensorDims(input1), \
-               GetTensorData(input2), GetTensorDims(input2), \
-               output_activation_min, output_activation_max,        \
-               GetTensorData(output), GetTensorDims(output))
-  if (kernel_type == kReference) {
-    if (data->requires_broadcast) {
-      TF_LITE_DIV(reference_ops, BroadcastDiv);
+void EvalDiv(TfLiteContext* context, TfLiteNode* node, TfLiteDivParams* params,
+             const OpData* data, const TfLiteTensor* input1,
+             const TfLiteTensor* input2, TfLiteTensor* output) {
+#define TF_LITE_DIV(type, opname, data_type)                            \
+  data_type output_activation_min, output_activation_max;               \
+  CalculateActivationRange(params->activation, &output_activation_min,  \
+                           &output_activation_max);                     \
+  type::opname(GetTensorData(input1), GetTensorDims(input1), \
+               GetTensorData(input2), GetTensorDims(input2), \
+               output_activation_min, output_activation_max,            \
+               GetTensorData(output), GetTensorDims(output))
+  if (output->type == kTfLiteInt32) {
+    if (kernel_type == kReference) {
+      if (data->requires_broadcast) {
+        TF_LITE_DIV(reference_ops, BroadcastDiv, int32_t);
+      } else {
+        TF_LITE_DIV(reference_ops, Div, int32_t);
+      }
     } else {
-      TF_LITE_DIV(reference_ops, Div);
+      if (data->requires_broadcast) {
+        TF_LITE_DIV(optimized_ops, BroadcastDiv, int32_t);
+      } else {
+        TF_LITE_DIV(optimized_ops, Div, int32_t);
+      }
     }
-  } else {
-    if (data->requires_broadcast) {
-      TF_LITE_DIV(optimized_ops, BroadcastDiv);
+  } else if (output->type == kTfLiteFloat32) {
+    if (kernel_type == kReference) {
+      if (data->requires_broadcast) {
+        TF_LITE_DIV(reference_ops, BroadcastDiv, float);
+      } else {
+        TF_LITE_DIV(reference_ops, Div, float);
+      }
     } else {
-      TF_LITE_DIV(optimized_ops, Div);
+      if (data->requires_broadcast) {
+        TF_LITE_DIV(optimized_ops, BroadcastDiv, float);
+      } else {
+        TF_LITE_DIV(optimized_ops, Div, float);
+      }
     }
   }
 #undef TF_LITE_DIV
@@ -115,11 +130,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* input2 = GetInput(context, node, kInputTensor2);
   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
 
-  if (output->type == kTfLiteFloat32) {
-    EvalFloat(context, node, params, data, input1, input2, output);
+  if (output->type == kTfLiteFloat32 || output->type == kTfLiteInt32) {
+    EvalDiv(context, node, params, data, input1, input2, output);
   } else {
     context->ReportError(
-        context, "Div only supports FLOAT32 and quantized UINT8 now, got %d.",
+        context,
+        "Div only supports FLOAT32, INT32 and quantized UINT8 now, got %d.",
         output->type);
     return kTfLiteError;
   }
diff --git a/tensorflow/contrib/lite/kernels/div_test.cc b/tensorflow/contrib/lite/kernels/div_test.cc
index 276b8289fb..97aa2fe04e 100644
--- a/tensorflow/contrib/lite/kernels/div_test.cc
+++ b/tensorflow/contrib/lite/kernels/div_test.cc
@@ -52,6 +52,13 @@ class FloatDivOpModel : public BaseDivOpModel {
   std::vector GetOutput() { return ExtractVector(output_); }
 };
 
+class IntegerDivOpModel : public BaseDivOpModel {
+ public:
+  using BaseDivOpModel::BaseDivOpModel;
+
+  std::vector GetOutput() { return ExtractVector(output_); }
+};
+
 TEST(FloatDivOpTest, NoActivation) {
   FloatDivOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}},
                     {TensorType_FLOAT32, {1, 2, 2, 1}},
@@ -75,7 +82,7 @@ TEST(FloatDivOpTest, ActivationRELU_N1_TO_1) {
 }
 
 TEST(FloatDivOpTest, VariousInputShapes) {
-  std::vector> test_shapes = {
+  std::vector> test_shapes = {
       {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
   for (int i = 0; i < test_shapes.size(); ++i) {
     FloatDivOpModel m({TensorType_FLOAT32, test_shapes[i]},
@@ -92,7 +99,7 @@ TEST(FloatDivOpTest, VariousInputShapes) {
 }
 
 TEST(FloatDivOpTest, WithBroadcast) {
-  std::vector> test_shapes = {
+  std::vector> test_shapes = {
       {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
   for (int i = 0; i < test_shapes.size(); ++i) {
     FloatDivOpModel m({TensorType_FLOAT32, test_shapes[i]},
@@ -108,6 +115,56 @@ TEST(FloatDivOpTest, WithBroadcast) {
   }
 }
 
+TEST(IntegerDivOpTest, NoActivation) {
+  IntegerDivOpModel m({TensorType_INT32, {1, 2, 2, 1}},
+                      {TensorType_INT32, {1, 2, 2, 1}}, {TensorType_INT32, {}},
+                      ActivationFunctionType_NONE);
+  m.PopulateTensor(m.input1(), {-2, 2, -15, 8});
+  m.PopulateTensor(m.input2(), {5, -2, -3, 5});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({0, -1, 5, 1}));
+}
+
+TEST(IntegerDivOpTest, ActivationRELU_N1_TO_1) {
+  IntegerDivOpModel m({TensorType_INT32, {1, 2, 2, 1}},
+                      {TensorType_INT32, {1, 2, 2, 1}}, {TensorType_INT32, {}},
+                      ActivationFunctionType_RELU_N1_TO_1);
+  m.PopulateTensor(m.input1(), {-2, 2, -12, 8});
+  m.PopulateTensor(m.input2(), {1, 2, -15, 5});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({-1, 1, 0, 1}));
+}
+
+TEST(IntegerDivOpTest, VariousInputShapes) {
+  std::vector> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    IntegerDivOpModel m({TensorType_INT32, test_shapes[i]},
+                        {TensorType_INT32, test_shapes[i]},
+                        {TensorType_INT32, {}}, ActivationFunctionType_NONE);
+    m.PopulateTensor(m.input1(), {-20, 2, 3, 8, 11, -20});
+    m.PopulateTensor(m.input2(), {1, 2, 6, 5, -11, -1});
+    m.Invoke();
+    EXPECT_THAT(m.GetOutput(), ElementsAreArray({-20, 1, 0, 1, -1, 20}))
+        << "With shape number " << i;
+  }
+}
+
+TEST(IntegerDivOpTest, WithBroadcast) {
+  std::vector> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    IntegerDivOpModel m({TensorType_INT32, test_shapes[i]},
+                        {TensorType_INT32, {}},  // always a scalar
+                        {TensorType_INT32, {}}, ActivationFunctionType_NONE);
+    m.PopulateTensor(m.input1(), {-20, 21, 7, 8, 11, -123});
+    m.PopulateTensor(m.input2(), {3});
+    m.Invoke();
+    EXPECT_THAT(m.GetOutput(), ElementsAreArray({-6, 7, 2, 2, 3, -41}))
+        << "With shape number " << i;
+  }
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index 6fabb9c268..04f61c7434 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -1654,10 +1654,11 @@ void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims,
   }
 }
 
-inline void Div(const float* input1_data, const Dims<4>& input1_dims,
-                const float* input2_data, const Dims<4>& input2_dims,
-                float output_activation_min, float output_activation_max,
-                float* output_data, const Dims<4>& output_dims) {
+template 
+inline void Div(const T* input1_data, const Dims<4>& input1_dims,
+                const T* input2_data, const Dims<4>& input2_dims,
+                T output_activation_min, T output_activation_max,
+                T* output_data, const Dims<4>& output_dims) {
   const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims);
   for (int i = 0; i < flat_size; ++i) {
     output_data[i] = ActivationFunctionWithMinMax(
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index ba36017baf..770092e12c 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -53,8 +53,6 @@ tensorflow::Env* env = tensorflow::Env::Default();
 // Key is a substring of the test name and value is a bug number.
 // TODO(ahentz): make sure we clean this list up frequently.
 std::map kBrokenTests = {
-    {R"(^\/div.*int32)", "68808744"},
-
     // Pad and PadV2 only supports 4D tensors.
     {R"(^\/pad.*,input_shape=\[.,.\],paddings=\[\[.,.\],\[.,.\]\])",
      "70527055"},
-- 
cgit v1.2.3


From a84859ad52d2cff7cb5b7d6f98086f28c85a14c9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 19:12:55 -0700
Subject: Adding pack op to schema.

PiperOrigin-RevId: 205175758
---
 tensorflow/contrib/lite/builtin_ops.h             |   1 +
 tensorflow/contrib/lite/model.cc                  |   1 +
 tensorflow/contrib/lite/nnapi_delegate.cc         |   1 +
 tensorflow/contrib/lite/schema/schema.fbs         |   7 +
 tensorflow/contrib/lite/schema/schema_generated.h | 156 +++++++++++++++++++++-
 5 files changed, 160 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h
index 4c7b27c4e0..558e547121 100644
--- a/tensorflow/contrib/lite/builtin_ops.h
+++ b/tensorflow/contrib/lite/builtin_ops.h
@@ -108,6 +108,7 @@ typedef enum {
   kTfLiteBuiltinFakeQuant = 80,
   kTfLiteBuiltinReduceProd = 81,
   kTfLiteBuiltinReduceMax = 82,
+  kTfLiteBuiltinPack = 83,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index 6c1ba3694a..5e6106a87e 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -764,6 +764,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_TOPK_V2:
     case BuiltinOperator_TRANSPOSE:
     case BuiltinOperator_POW:
+    case BuiltinOperator_PACK:
       break;
   }
   return kTfLiteOk;
diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc
index 5950840e8a..710ce1632e 100644
--- a/tensorflow/contrib/lite/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/nnapi_delegate.cc
@@ -614,6 +614,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_SHAPE:
       case tflite::BuiltinOperator_POW:
       case tflite::BuiltinOperator_FAKE_QUANT:
+      case tflite::BuiltinOperator_PACK:
         logError("Op code %d is currently not delegated to NNAPI", builtin);
         return kTfLiteError;
         break;
diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs
index 6c3189a884..0434199a08 100644
--- a/tensorflow/contrib/lite/schema/schema.fbs
+++ b/tensorflow/contrib/lite/schema/schema.fbs
@@ -164,6 +164,7 @@ enum BuiltinOperator : byte {
   FAKE_QUANT = 80,
   REDUCE_PROD = 81,
   REDUCE_MAX = 82,
+  PACK = 83,
 }
 
 // Options for the builtin operators.
@@ -226,6 +227,7 @@ union BuiltinOptions {
   PowOptions,
   ArgMinOptions,
   FakeQuantOptions,
+  PackOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -537,6 +539,11 @@ table FakeQuantOptions {
   narrow_range:bool;
 }
 
+table PackOptions {
+  values_count:int;
+  axis:int;
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h
index 8052404319..9b84030938 100755
--- a/tensorflow/contrib/lite/schema/schema_generated.h
+++ b/tensorflow/contrib/lite/schema/schema_generated.h
@@ -205,6 +205,9 @@ struct PowOptionsT;
 struct FakeQuantOptions;
 struct FakeQuantOptionsT;
 
+struct PackOptions;
+struct PackOptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -353,11 +356,12 @@ enum BuiltinOperator {
   BuiltinOperator_FAKE_QUANT = 80,
   BuiltinOperator_REDUCE_PROD = 81,
   BuiltinOperator_REDUCE_MAX = 82,
+  BuiltinOperator_PACK = 83,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_REDUCE_MAX
+  BuiltinOperator_MAX = BuiltinOperator_PACK
 };
 
-inline BuiltinOperator (&EnumValuesBuiltinOperator())[82] {
+inline BuiltinOperator (&EnumValuesBuiltinOperator())[83] {
   static BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -440,7 +444,8 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[82] {
     BuiltinOperator_ARG_MIN,
     BuiltinOperator_FAKE_QUANT,
     BuiltinOperator_REDUCE_PROD,
-    BuiltinOperator_REDUCE_MAX
+    BuiltinOperator_REDUCE_MAX,
+    BuiltinOperator_PACK
   };
   return values;
 }
@@ -530,6 +535,7 @@ inline const char **EnumNamesBuiltinOperator() {
     "FAKE_QUANT",
     "REDUCE_PROD",
     "REDUCE_MAX",
+    "PACK",
     nullptr
   };
   return names;
@@ -600,11 +606,12 @@ enum BuiltinOptions {
   BuiltinOptions_PowOptions = 56,
   BuiltinOptions_ArgMinOptions = 57,
   BuiltinOptions_FakeQuantOptions = 58,
+  BuiltinOptions_PackOptions = 59,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_FakeQuantOptions
+  BuiltinOptions_MAX = BuiltinOptions_PackOptions
 };
 
-inline BuiltinOptions (&EnumValuesBuiltinOptions())[59] {
+inline BuiltinOptions (&EnumValuesBuiltinOptions())[60] {
   static BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -664,7 +671,8 @@ inline BuiltinOptions (&EnumValuesBuiltinOptions())[59] {
     BuiltinOptions_ShapeOptions,
     BuiltinOptions_PowOptions,
     BuiltinOptions_ArgMinOptions,
-    BuiltinOptions_FakeQuantOptions
+    BuiltinOptions_FakeQuantOptions,
+    BuiltinOptions_PackOptions
   };
   return values;
 }
@@ -730,6 +738,7 @@ inline const char **EnumNamesBuiltinOptions() {
     "PowOptions",
     "ArgMinOptions",
     "FakeQuantOptions",
+    "PackOptions",
     nullptr
   };
   return names;
@@ -976,6 +985,10 @@ template<> struct BuiltinOptionsTraits {
   static const BuiltinOptions enum_value = BuiltinOptions_FakeQuantOptions;
 };
 
+template<> struct BuiltinOptionsTraits {
+  static const BuiltinOptions enum_value = BuiltinOptions_PackOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -1471,6 +1484,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_FakeQuantOptions ?
       reinterpret_cast(value) : nullptr;
   }
+  PackOptionsT *AsPackOptions() {
+    return type == BuiltinOptions_PackOptions ?
+      reinterpret_cast(value) : nullptr;
+  }
+  const PackOptionsT *AsPackOptions() const {
+    return type == BuiltinOptions_PackOptions ?
+      reinterpret_cast(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -5304,6 +5325,72 @@ inline flatbuffers::Offset CreateFakeQuantOptions(
 
 flatbuffers::Offset CreateFakeQuantOptions(flatbuffers::FlatBufferBuilder &_fbb, const FakeQuantOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct PackOptionsT : public flatbuffers::NativeTable {
+  typedef PackOptions TableType;
+  int32_t values_count;
+  int32_t axis;
+  PackOptionsT()
+      : values_count(0),
+        axis(0) {
+  }
+};
+
+struct PackOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef PackOptionsT NativeTableType;
+  enum {
+    VT_VALUES_COUNT = 4,
+    VT_AXIS = 6
+  };
+  int32_t values_count() const {
+    return GetField(VT_VALUES_COUNT, 0);
+  }
+  int32_t axis() const {
+    return GetField(VT_AXIS, 0);
+  }
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           VerifyField(verifier, VT_VALUES_COUNT) &&
+           VerifyField(verifier, VT_AXIS) &&
+           verifier.EndTable();
+  }
+  PackOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(PackOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const PackOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct PackOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  void add_values_count(int32_t values_count) {
+    fbb_.AddElement(PackOptions::VT_VALUES_COUNT, values_count, 0);
+  }
+  void add_axis(int32_t axis) {
+    fbb_.AddElement(PackOptions::VT_AXIS, axis, 0);
+  }
+  explicit PackOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  PackOptionsBuilder &operator=(const PackOptionsBuilder &);
+  flatbuffers::Offset Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset CreatePackOptions(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    int32_t values_count = 0,
+    int32_t axis = 0) {
+  PackOptionsBuilder builder_(_fbb);
+  builder_.add_axis(axis);
+  builder_.add_values_count(values_count);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset CreatePackOptions(flatbuffers::FlatBufferBuilder &_fbb, const PackOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -5611,6 +5698,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const FakeQuantOptions *builtin_options_as_FakeQuantOptions() const {
     return builtin_options_type() == BuiltinOptions_FakeQuantOptions ? static_cast(builtin_options()) : nullptr;
   }
+  const PackOptions *builtin_options_as_PackOptions() const {
+    return builtin_options_type() == BuiltinOptions_PackOptions ? static_cast(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector *custom_options() const {
     return GetPointer *>(VT_CUSTOM_OPTIONS);
   }
@@ -5874,6 +5964,10 @@ template<> inline const FakeQuantOptions *Operator::builtin_options_as inline const PackOptions *Operator::builtin_options_as() const {
+  return builtin_options_as_PackOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -7937,6 +8031,35 @@ inline flatbuffers::Offset CreateFakeQuantOptions(flatbuffers:
       _narrow_range);
 }
 
+inline PackOptionsT *PackOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new PackOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void PackOptions::UnPackTo(PackOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+  { auto _e = values_count(); _o->values_count = _e; };
+  { auto _e = axis(); _o->axis = _e; };
+}
+
+inline flatbuffers::Offset PackOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const PackOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreatePackOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset CreatePackOptions(flatbuffers::FlatBufferBuilder &_fbb, const PackOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const PackOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  auto _values_count = _o->values_count;
+  auto _axis = _o->axis;
+  return tflite::CreatePackOptions(
+      _fbb,
+      _values_count,
+      _axis);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -8358,6 +8481,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_PackOptions: {
+      auto ptr = reinterpret_cast(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -8608,6 +8735,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_PackOptions: {
+      auto ptr = reinterpret_cast(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -8846,6 +8977,10 @@ inline flatbuffers::Offset BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast(value);
       return CreateFakeQuantOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_PackOptions: {
+      auto ptr = reinterpret_cast(value);
+      return CreatePackOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -9084,6 +9219,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new FakeQuantOptionsT(*reinterpret_cast(u.value));
       break;
     }
+    case BuiltinOptions_PackOptions: {
+      value = new PackOptionsT(*reinterpret_cast(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -9381,6 +9520,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_PackOptions: {
+      auto ptr = reinterpret_cast(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
-- 
cgit v1.2.3


From 996b078c45879ad0ea6ee56f58fcbeb6c03c3246 Mon Sep 17 00:00:00 2001
From: Jiri Simsa 
Date: Wed, 18 Jul 2018 20:27:42 -0700
Subject: Add support for bitcasting to/from uint32 and uint64, and update
 out-of-date tf.cast documentation.

PiperOrigin-RevId: 205180661
---
 tensorflow/core/ops/array_ops.cc                  | 12 +++++++-----
 tensorflow/python/kernel_tests/bitcast_op_test.py |  8 +++++++-
 tensorflow/python/ops/math_ops.py                 | 15 ++++++++-------
 3 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index fce0b93cd7..02989f8d3d 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -2549,14 +2549,16 @@ REGISTER_OP("ExtractImagePatches")
 REGISTER_OP("Bitcast")
     .Input("input: T")
     .Output("output: type")
-    // All supported dtypes are listed here to include qint16 and quint16.
+    // All supported dtypes are listed here to include qint16, quint16, uint32,
+    // and uint64.
     .Attr(
-        "T: {bfloat16, half, float, double, int64, int32, uint8, uint16, int8, "
-        "int16, complex64, complex128, qint8, quint8, qint16, quint16, qint32}")
+        "T: {bfloat16, half, float, double, int64, int32, uint8, uint16, "
+        "uint32, uint64, int8, int16, complex64, complex128, qint8, quint8, "
+        "qint16, quint16, qint32}")
     .Attr(
         "type: {bfloat16, half, float, double, int64, int32, uint8, uint16, "
-        "int8, int16, complex64, complex128, qint8, quint8, qint16, quint16, "
-        "qint32}")
+        "uint32, uint64, int8, int16, complex64, complex128, qint8, quint8, "
+        "qint16, quint16, qint32}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input = c->input(0);
       if (!c->RankKnown(input)) {
diff --git a/tensorflow/python/kernel_tests/bitcast_op_test.py b/tensorflow/python/kernel_tests/bitcast_op_test.py
index a535468b05..a2c6b54273 100644
--- a/tensorflow/python/kernel_tests/bitcast_op_test.py
+++ b/tensorflow/python/kernel_tests/bitcast_op_test.py
@@ -76,12 +76,18 @@ class BitcastTest(test.TestCase):
     datatype = dtypes.int8
     array_ops.bitcast(x, datatype, None)
 
-  def testQuantizeType(self):
+  def testQuantizedType(self):
     shape = [3, 4]
     x = np.zeros(shape, np.uint16)
     datatype = dtypes.quint16
     self._testBitcast(x, datatype, shape)
 
+  def testUnsignedType(self):
+    shape = [3, 4]
+    x = np.zeros(shape, np.int64)
+    datatype = dtypes.uint64
+    self._testBitcast(x, datatype, shape)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index c28dca5137..fbe6b62302 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -628,16 +628,17 @@ def cast(x, dtype, name=None):
   ```
 
   The operation supports data types (for `x` and `dtype`) of
-  `uint8`, `int8`, `uint16`, `int16`, `int32`, `int64`, `float16`, `float32`,
-  `float64`, `complex64`, `complex128`, `bfloat16`. In case of casting from
-  complex types (`complex64`, `complex128`) to real types, only the real part
-  of `x` is returned. In case of casting from real types to complex types
-  (`complex64`, `complex128`), the imaginary part of the returned value is set
-  to `0`. The handling of complex types here matches the behavior of numpy.
+  `uint8`, `uint16`, `uint32`, `uint64`, `int8`, `int16`, `int32`, `int64`,
+  `float16`, `float32`, `float64`, `complex64`, `complex128`, `bfloat16`.
+  In case of casting from complex types (`complex64`, `complex128`) to real
+  types, only the real part of `x` is returned. In case of casting from real
+  types to complex types (`complex64`, `complex128`), the imaginary part of the
+  returned value is set to `0`. The handling of complex types here matches the
+  behavior of numpy.
 
   Args:
     x: A `Tensor` or `SparseTensor` of numeric type. It could be
-      `uint8`, `int8`, `uint16`, `int16`, `int32`, `int64`,
+      `uint8`, `uint16`, `uint32`, `uint64`, `int8`, `int16`, `int32`, `int64`,
       `float16`, `float32`, `float64`, `complex64`, `complex128`, `bfloat16`.
     dtype: The destination type. The list of supported dtypes is the same
       as `x`.
-- 
cgit v1.2.3


From 2e9d8fd8a27cf716ce91d1022fc3154cecad9e1d Mon Sep 17 00:00:00 2001
From: Jie 
Date: Wed, 18 Jul 2018 20:47:56 -0700
Subject: [tftrt unit tests]   1. unit tests refactored to
 tf_trt_integration_test style   2. disabled failed test (tests passed in
 static conversion but failed in      dynamic conversion)   3. comment on
 _VerifyGraphDef:      we should cover cases for failed conversion in dynamic
 conversion,      where func def (TF fallback) is used.

---
 tensorflow/contrib/tensorrt/BUILD                  |  44 +++------
 .../contrib/tensorrt/test/batch_matmul_test.py     | 107 +++++++++------------
 .../contrib/tensorrt/test/biasadd_matmul_test.py   |  80 ++++++++-------
 .../test/binary_tensor_weight_broadcast_test.py    |  95 +++++++-----------
 .../contrib/tensorrt/test/concatenation_test.py    |  68 ++++++-------
 .../contrib/tensorrt/test/const_broadcast_test.py  |  48 +++++----
 .../test/multi_connection_neighbor_engine_test.py  |  54 +++++------
 .../tensorrt/test/neighboring_engine_test.py       |  49 +++++-----
 .../tensorrt/test/tf_trt_integration_test_base.py  |   3 +
 tensorflow/contrib/tensorrt/test/unary_test.py     |  66 +++++--------
 .../contrib/tensorrt/test/vgg_block_nchw_test.py   |  44 ++++-----
 tensorflow/contrib/tensorrt/test/vgg_block_test.py |  44 ++++-----
 12 files changed, 303 insertions(+), 399 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index dd2554c81e..fa47f51b66 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -335,48 +335,28 @@ py_library(
 
 cuda_py_tests(
     name = "tf_trt_integration_test",
-    srcs = ["test/base_test.py"],
-    additional_deps = [
-        ":tf_trt_integration_test_base",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-    ],
-    prefix = "integration_test",
-    tags = [
-        "no_windows",
-        "nomac",
-    ],
-)
-
-py_test(
-    name = "converter_unit_tests",
     srcs = [
-        "test/base_unit_test.py",
-        "test/batch_matmul_test.py",
-        "test/biasadd_matmul_test.py",
+        "test/base_test.py",
+       #"test/batch_matmul_test.py",
+       #"test/biasadd_matmul_test.py",
         "test/binary_tensor_weight_broadcast_test.py",
         "test/concatenation_test.py",
         "test/const_broadcast_test.py",
         "test/multi_connection_neighbor_engine_test.py",
         "test/neighboring_engine_test.py",
-        "test/run_test.py",
         "test/unary_test.py",
-        "test/unit_tests.py",
-        "test/utilities.py",
-        "test/vgg_block_nchw_test.py",
-        "test/vgg_block_test.py",
+       #"test/vgg_block_nchw_test.py",
+       #"test/vgg_block_test.py",
     ],
-    main = "test/unit_tests.py",
-    srcs_version = "PY2AND3",
-    tags = [
-        "notap",
-    ],
-    deps = [
-        ":init_py",
+    additional_deps = [
+        ":tf_trt_integration_test_base",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:layers",
-        "//tensorflow/python:training",
+    ],
+    prefix = "integration_test",
+    tags = [
+        "no_windows",
+        "nomac",
     ],
 )
 
diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
index 3c83a3a562..163af54184 100644
--- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
@@ -20,78 +20,59 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops import variables
-from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.training import training
-from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest
-from tensorflow.contrib.tensorrt.test.utilities import get_all_variables
+from tensorflow.python.platform import test
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 
 
-class BatchMatMulTest(BaseUnitTest):
-  """Testing BatchMatMul in TF-TRT conversion"""
+class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase):
 
-  def __init__(self, log_file='log.txt'):
-    super(BatchMatMulTest, self).__init__()
-    self.static_mode_list = {"FP32", "FP16"}
-    self.debug = True
-    self.dynamic_mode_list = {}
-    self.inp_dims = (12, 5, 8, 12)
-    self.dummy_input = np.random.random_sample(self.inp_dims)
-    self.get_network = self.matmul_test
-    self.expect_nb_nodes = 16
-    self.log_file = log_file
-    self.test_name = self.__class__.__name__
-    self.ckpt = "./tmp.ckpt"
-    sess = session.Session()
-
-  def matmul_test(self):
+  def GetParams(self):
+    """Testing conversion of BatchMatMul in TF-TRT conversion"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [12, 5, 8, 12]
+    w1_name = "matmul_w1"
+    w1_dims = [12, 5, 12, 7]
+    w2_name = "matmul_w2"
+    w2_dims = [12, 12, 7]
     g = ops.Graph()
-    gpu_options = config_pb2.GPUOptions()
-    sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options)
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtypes.float32, shape=self.inp_dims, name="input")
-
-      b = constant_op.constant(
-          np.random.randn(12, 5, 12, 7), dtype=dtypes.float32)
-      x1 = math_ops.matmul(x, b)
-      b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32)
-      x1 = x1 + b
-
-      var = variable_scope.get_variable(
-          "test", [12, 5, 12, 7],
-          dtype=dtypes.float32,
-          initializer=init_ops.truncated_normal_initializer)
-      x2 = math_ops.matmul(x, var)
-      b = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtypes.float32)
-      x2 = x2 * b
-
-      var = variable_scope.get_variable(
-          "test2", [12, 84],
-          dtype=dtypes.float32,
-          initializer=init_ops.truncated_normal_initializer)
-      c = gen_array_ops.reshape(x, [12, 40, 12])
-      b = gen_array_ops.reshape(var, [12, 12, 7])
-      x3 = math_ops.matmul(c, b)
-      b = constant_op.constant(np.random.randn(40, 1), dtype=dtypes.float32)
-      x3 = x3 + b
-      x3 = gen_array_ops.reshape(x3, [12, 5, 8, 7])
-
-      out = x3 + x1
-      array_ops.squeeze(out, name="output")
+      inp = array_ops.placeholder(
+          dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
+      w1 = array_ops.placeholder(
+         dtype=dtype, shape=w1_dims, name=w1_name)
+      w2 = array_ops.placeholder(
+         dtype=dtype, shape=w2_dims, name=w2_name)
+      with g.device("/GPU:0"):
+        b = constant_op.constant(
+            np.random.randn(12, 5, 12, 7), dtype=dtype)
+        c = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtype)
+        d = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtype)
+        x1 = math_ops.matmul(inp, b)
+        x1 = x1 + c
+        x2 = math_ops.matmul(inp, w1)
+        x2 = x2 * d
+        e = gen_array_ops.reshape(inp, [12, 40, 12])
+        x3 = math_ops.matmul(e, w2)
+        f = constant_op.constant(np.random.randn(40, 1), dtype=dtype)
+        x3 = x3 + f
+        x3 = gen_array_ops.reshape(x3, [12, 5, 8, 7])
+        out = x1 + x2 + x3
+      array_ops.squeeze(out, name=self.output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name, w1_name, w2_name],
+        input_dims=[input_dims, w1_dims, w2_dims],
+        num_expected_engines=1,
+        expected_output_dims=(12, 5, 8, 7),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
 
-      with session.Session(config=sessconfig, graph=g) as sess:
-        names_var_list = get_all_variables(sess)
-        saver = training.Saver(names_var_list)
-        sess.run(variables.global_variables_initializer())
-        saver.save(sess, self.ckpt)
-    return g.as_graph_def()
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
index 1ac6f5cb6a..9b153ada05 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -28,89 +27,86 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.layers import core
-from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest
-
-
-class BiasaddMatMulTest(BaseUnitTest):
-  """Testing BiasAdd MatMul in TF-TRT conversion"""
-
-  def __init__(self, log_file='log.txt'):
-    super(BiasaddMatMulTest, self).__init__()
-    self.static_mode_list = {"FP32", "FP16"}
-    self.debug = True
-    self.dynamic_mode_list = {}
-    self.inp_dims = (48, 12)
-    self.dummy_input = np.random.random_sample(self.inp_dims)
-    self.get_network = self.matmul_test
-    self.expect_nb_nodes = 53
-    self.log_file = log_file
-    self.test_name = self.__class__.__name__
-
-  def matmul_test(self):
+from tensorflow.python.platform import test
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
+
+
+class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
+
+  def GetParams(self):
+    """Testing conversion of BiasAdd MatMul in TF-TRT conversion"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [48, 12]
     g = ops.Graph()
-    gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-    sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options)
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.float32, shape=self.inp_dims, name="input")
+          dtype=dtype, shape=input_dims, name=input_name)
 
-      b = constant_op.constant(np.random.randn(12, 4), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(12, 4), dtype=dtype)
       x1 = math_ops.matmul(x, b)
-      b = constant_op.constant(np.random.randn(1, 4), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(1, 4), dtype=dtype)
       x1 = x1 + b
 
-      b = constant_op.constant(np.random.randn(48, 4), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(48, 4), dtype=dtype)
       x2 = math_ops.matmul(x, b, transpose_a=True)
       x2 = gen_array_ops.reshape(x2, [48, 1])
 
-      b = constant_op.constant(np.random.randn(4, 12), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(4, 12), dtype=dtype)
       x3 = math_ops.matmul(x, b, transpose_b=True)
 
-      b = constant_op.constant(np.random.randn(16, 48), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(16, 48), dtype=dtype)
       x4 = math_ops.matmul(x, b, transpose_b=True, transpose_a=True)
       x4 = gen_array_ops.reshape(x4, [48, 4])
 
-      x5 = gen_array_ops.reshape(x, [4, 12, 12])
-      x5 = core.flatten(x5)
-      b = constant_op.constant(np.random.randn(144, 48), dtype=dtypes.float32)
+      x5 = gen_array_ops.reshape(x, [4, 144])
+      b = constant_op.constant(np.random.randn(144, 48), dtype=dtype)
       x5 = math_ops.matmul(x5, b)
-      b = constant_op.constant(np.random.randn(48), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(48), dtype=dtype)
       x5 = nn.bias_add(x5, b)
       x5 = gen_array_ops.reshape(x5, [48, 4])
 
       x6 = gen_array_ops.reshape(x, [4, 12, 12])
-      b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(12), dtype=dtype)
       x6 = nn.bias_add(x6, b, data_format="NHWC")
       x6 = gen_array_ops.reshape(x6, [48, -1])
 
       x7 = gen_array_ops.reshape(x, [4, 12, 3, 4])
-      b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(4), dtype=dtype)
       x7 = nn.bias_add(x7, b, data_format="NHWC")
       x7 = gen_array_ops.reshape(x7, [48, -1])
 
       x8 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2])
-      b = constant_op.constant(np.random.randn(2), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(2), dtype=dtype)
       x8 = nn.bias_add(x8, b, data_format="NHWC")
       x8 = gen_array_ops.reshape(x8, [48, -1])
 
       x9 = gen_array_ops.reshape(x, [4, 12, 3, 2, 2])
-      b = constant_op.constant(np.random.randn(3), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(3), dtype=dtype)
       x9 = nn.bias_add(x9, b, data_format="NCHW")
       x9 = gen_array_ops.reshape(x9, [48, -1])
 
       x10 = gen_array_ops.reshape(x, [4, 12, 3, 4])
-      b = constant_op.constant(np.random.randn(12), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(12), dtype=dtype)
       x10 = nn.bias_add(x10, b, data_format="NCHW")
       x10 = gen_array_ops.reshape(x10, [48, -1])
 
       x11 = gen_array_ops.reshape(x, [4, 12, 12])
-      b = constant_op.constant(np.random.randn(4), dtype=dtypes.float32)
+      b = constant_op.constant(np.random.randn(4), dtype=dtype)
       x11 = nn.bias_add(x11, b, data_format="NCHW")
       x11 = gen_array_ops.reshape(x11, [48, -1])
 
       out = array_ops.concat(
           [x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11], axis=-1)
-      out = array_ops.squeeze(out, name="output")
-
-    return g.as_graph_def()
+      out = array_ops.squeeze(out, name=self.output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        num_expected_engines=7,
+        expected_output_dims=(48, 89),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
index 5233a493d0..e80712731d 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -20,129 +20,108 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest
+from tensorflow.python.platform import test
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 
 
-class BinaryTensorWeightBroadcastTest(BaseUnitTest):
-  """unit tests for scale & elementwise layers in TF-TRT"""
+class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
 
-  def __init__(self, log_file='log.txt'):
-    super(BinaryTensorWeightBroadcastTest, self).__init__()
-    self.static_mode_list = {"FP32", "FP16"}
-    self.debug = True
-    self.dynamic_mode_list = {}
-    self.inp_dims = (10, 24, 24, 20)
-    self.dummy_input = np.random.random_sample(self.inp_dims)
-    self.get_network = self.get_simple_graph_def
-    self.expect_nb_nodes = 35
-    self.log_file = log_file
-    self.test_name = self.__class__.__name__
-    self.allclose_rtol = 0.1
-    self.allclose_atol = 0.05
-
-  def get_simple_graph_def(self):
+  def GetParams(self):
+    """unit tests for scale & elementwise layers in TF-TRT"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [10, 24, 24, 20]
     g = ops.Graph()
-    gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-    sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options)
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.float32, shape=self.inp_dims, name="input")
-
+          dtype=dtype, shape=input_dims, name=input_name)
       # scale
-      a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(1), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
-
       # scale
-      a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(1), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
-
       # scale
-      a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
-
       # scale
-      a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(24, 1, 1), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
-
       # scale
       a = constant_op.constant(
-          np.random.randn(24, 24, 20), dtype=dtypes.float32)
+          np.random.randn(24, 24, 20), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
-
       # scale
       a = constant_op.constant(
-          np.random.randn(24, 24, 20), dtype=dtypes.float32)
+          np.random.randn(24, 24, 20), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
-
       # elementwise
-      a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(20), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
-
       # elementwise
-      a = constant_op.constant(np.random.randn(20), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(20), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
-
       # elementwise
       a = constant_op.constant(
-          np.random.randn(1, 24, 1, 1), dtype=dtypes.float32)
+          np.random.randn(1, 24, 1, 1), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
-
       # elementwise
       a = constant_op.constant(
-          np.random.randn(1, 24, 1, 1), dtype=dtypes.float32)
+          np.random.randn(1, 24, 1, 1), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
-
       # elementwise
       a = constant_op.constant(
-          np.random.randn(1, 24, 24, 1), dtype=dtypes.float32)
+          np.random.randn(1, 24, 24, 1), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
-
       # elementwise
       a = constant_op.constant(
-          np.random.randn(1, 24, 24, 1), dtype=dtypes.float32)
+          np.random.randn(1, 24, 24, 1), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
-
       # elementwise
       a = constant_op.constant(
-          np.random.randn(1, 24, 24, 20), dtype=dtypes.float32)
+          np.random.randn(1, 24, 24, 20), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
-
       # elementwise
       a = constant_op.constant(
-          np.random.randn(1, 24, 24, 20), dtype=dtypes.float32)
+          np.random.randn(1, 24, 24, 20), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
-
       # elementwise
-      a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(24, 20), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
-
       # elementwise
-      a = constant_op.constant(np.random.randn(24, 20), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(24, 20), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
-
-      gen_array_ops.reshape(x, [5, -1], name="output")
-
-    return g.as_graph_def()
+      gen_array_ops.reshape(x, [5, -1], name=self.output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        num_expected_engines=16,
+        expected_output_dims=(5, 23040),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py
index de0817d2e8..cf0bfeeb00 100644
--- a/tensorflow/contrib/tensorrt/test/concatenation_test.py
+++ b/tensorflow/contrib/tensorrt/test/concatenation_test.py
@@ -20,68 +20,64 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_math_ops
-from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest
+from tensorflow.python.platform import test
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 
 
-class ConcatenationTest(BaseUnitTest):
-  """Testing Concatenation in TF-TRT conversion"""
+class ConcatenationTest(trt_test.TfTrtIntegrationTestBase):
 
-  def __init__(self, log_file='log.txt'):
-    super(ConcatenationTest, self).__init__()
-    self.static_mode_list = {"FP32", "FP16"}
-    self.debug = True
-    self.dynamic_mode_list = {}
-    self.inp_dims = (2, 3, 3, 1)
-    self.dummy_input = np.random.random_sample(self.inp_dims)
-    self.get_network = self.get_simple_graph_def
-    self.expect_nb_nodes = 4
-    self.log_file = log_file
-    self.test_name = self.__class__.__name__
-
-  def get_simple_graph_def(self):
+  def GetParams(self):
+    """Testing Concatenation in TF-TRT conversion"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [2, 3, 3, 1]
     g = ops.Graph()
-    gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-    sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options)
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.float32, shape=self.inp_dims, name="input")
-
+          dtype=dtype, shape=input_dims, name=input_name)
       # scale
-      a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
       r1 = x / a
-      a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
       r2 = a / x
-      a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtype)
       r3 = a + x
-      a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtype)
       r4 = x * a
-      a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
       r5 = x - a
-      a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
       r6 = a - x
-      a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(3, 1), dtype=dtype)
       r7 = x - a
-      a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(3, 1), dtype=dtype)
       r8 = a - x
-      a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
       r9 = gen_math_ops.maximum(x, a)
-      a = constant_op.constant(np.random.randn(3, 1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(3, 1), dtype=dtype)
       r10 = gen_math_ops.minimum(a, x)
-      a = constant_op.constant(np.random.randn(3), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(3), dtype=dtype)
       r11 = x * a
-      a = constant_op.constant(np.random.randn(1), dtype=dtypes.float32)
+      a = constant_op.constant(np.random.randn(1), dtype=dtype)
       r12 = a * x
       concat1 = array_ops.concat([r1, r2, r3, r4, r5, r6], axis=-1)
       concat2 = array_ops.concat([r7, r8, r9, r10, r11, r12], axis=3)
       x = array_ops.concat([concat1, concat2], axis=-1)
+      gen_array_ops.reshape(x, [2, -1], name=self.output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        num_expected_engines=1,
+        expected_output_dims=(2, 126),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
 
-      gen_array_ops.reshape(x, [2, -1], name="output")
-
-    return g.as_graph_def()
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
index 74d39d9015..97f5580ac0 100644
--- a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
@@ -20,56 +20,52 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
-from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest
+from tensorflow.python.platform import test
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 
 
-class ConstBroadcastTest(BaseUnitTest):
-  """Testing Constant broadcasting in TF-TRT"""
+class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase):
 
-  def __init__(self, log_file='log.txt'):
-    super(ConstBroadcastTest, self).__init__()
-    self.static_mode_list = {"FP32", "FP16"}
-    self.debug = True
-    self.dynamic_mode_list = {}
-    self.inp_dims = (5, 12, 12, 2)
-    self.dummy_input = np.random.random_sample(self.inp_dims)
-    self.get_network = self.conv_broadcast
-    self.expect_nb_nodes = 7
-    self.log_file = log_file
-    self.test_name = self.__class__.__name__
-    self.allclose_rtol = 0.05
-    self.allclose_atol = 0.05
-
-  def conv_broadcast(self):
+  def GetParams(self):
+    """unit test for Constant broadcasting in TF-TRT"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [5, 12, 12, 2]
     g = ops.Graph()
-    gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-    sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options)
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.float32, shape=self.inp_dims, name="input")
+          dtype=dtype, shape=input_dims, name=input_name)
       filt1 = constant_op.constant(
-          1, shape=(3, 3, 2, 1), dtype=dtypes.float32, name='filt1')
+          0.3, shape=(3, 3, 2, 1), dtype=dtype, name='filt1')
       y1 = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='SAME', name='y1')
       z1 = nn.relu(y1, name='z1')
       filt2 = constant_op.constant(
           np.random.randn(9),
           shape=(3, 3, 1, 1),
-          dtype=dtypes.float32,
+          dtype=dtype,
           name='filt2')
       y2 = nn.conv2d(z1, filt2, strides=[1, 1, 1, 1], padding='SAME', name='y2')
       z2 = nn.relu(y2, name='z')
       filt3 = constant_op.constant(
           np.random.randn(3, 3, 1, 1),
           shape=(3, 3, 1, 1),
-          dtype=dtypes.float32,
+          dtype=dtype,
           name='filt3')
       y3 = nn.conv2d(z2, filt3, strides=[1, 1, 1, 1], padding='SAME', name='y3')
       z = nn.relu(y3, name='output')
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        num_expected_engines=1,
+        expected_output_dims=(5, 12, 12, 1),
+        allclose_atol=1.e-02,
+        allclose_rtol=1.e-02)
 
-    return g.as_graph_def()
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
index 291b4d16c1..e62f9e479e 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -28,37 +27,25 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest
+from tensorflow.python.platform import test
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 
 
-class MultiConnectionNeighborEngineTest(BaseUnitTest):
-  """Multi connection neighboring nodes wiring tests in TF-TRT"""
+class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
 
-  def __init__(self, log_file='log.txt'):
-    super(MultiConnectionNeighborEngineTest, self).__init__()
-    self.static_mode_list = {"FP32", "FP16"}
-    self.debug = True
-    self.dynamic_mode_list = {}
-    self.inp_dims = (2, 3, 7, 5)
-    self.dummy_input = np.random.normal(1.0, 0.5, self.inp_dims)
-    self.get_network = self.neighboring_tensor_test
-    self.expect_nb_nodes = 7
-    self.log_file = log_file
-    self.test_name = self.__class__.__name__
-    self.allclose_rtol = 0.05
-    self.allclose_atol = 0.05
-
-  def neighboring_tensor_test(self):
+  def GetParams(self):
+    """unit test for multi connection neighboring nodes wiring tests in TF-TRT"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [2, 3, 7, 5]
     g = ops.Graph()
-    gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-    sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options)
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.float32, shape=self.inp_dims, name="input")
+          dtype=dtype, shape=input_dims, name=input_name)
       e = constant_op.constant(
           np.random.normal(.05, .005, [3, 2, 3, 4]),
           name="weights",
-          dtype=dtypes.float32)
+          dtype=dtype)
       conv = nn.conv2d(
           input=x,
           filter=e,
@@ -69,33 +56,42 @@ class MultiConnectionNeighborEngineTest(BaseUnitTest):
       b = constant_op.constant(
           np.random.normal(2.0, 1.0, [1, 4, 1, 1]),
           name="bias",
-          dtype=dtypes.float32)
+          dtype=dtype)
       t = conv + b
 
       b = constant_op.constant(
           np.random.normal(5.0, 1.0, [1, 4, 1, 1]),
           name="bias",
-          dtype=dtypes.float32)
+          dtype=dtype)
       q = conv - b
       edge = math_ops.sigmoid(q)
 
       b = constant_op.constant(
           np.random.normal(5.0, 1.0, [1, 4, 1, 1]),
           name="bias",
-          dtype=dtypes.float32)
+          dtype=dtype)
       d = b + conv
       edge3 = math_ops.sigmoid(d)
 
       c = constant_op.constant(
           np.random.normal(1.0, 1.0, [1, 4, 1, 1]),
           name="bias",
-          dtype=dtypes.float32)
+          dtype=dtype)
       edge1 = gen_math_ops.tan(conv)
       t = t - edge1
       q = q + edge
       t = t + q
       t = t + d
       t = t - edge3
-      array_ops.squeeze(t, name="output")
+      array_ops.squeeze(t, name=self.output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        num_expected_engines=2,
+        expected_output_dims=(2, 4, 5, 4),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
 
-    return g.as_graph_def()
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
index f916db3504..bbe8823552 100644
--- a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
@@ -20,44 +20,31 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import gen_math_ops
-from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest
+from tensorflow.python.platform import test
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 
 
-class NeighboringEngineTest(BaseUnitTest):
-  """Neighboring node wiring tests in TF-TRT conversion"""
+class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase):
 
-  def __init__(self, log_file='log.txt'):
-    super(NeighboringEngineTest, self).__init__()
-    self.static_mode_list = {"FP32", "FP16"}
-    self.debug = True
-    self.dynamic_mode_list = {}
-    self.inp_dims = (2, 3, 7, 5)
-    self.dummy_input = np.random.random_sample(self.inp_dims)
-    self.get_network = self.neighboring_tensor_test
-    self.expect_nb_nodes = 5
-    self.log_file = log_file
-    self.test_name = self.__class__.__name__
-    self.allclose_rtol = 0.05
-    self.allclose_atol = 0.05
-
-  def neighboring_tensor_test(self):
+  def GetParams(self):
+    """Neighboring node wiring tests in TF-TRT conversion"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [2, 3, 7, 5]
     g = ops.Graph()
-    gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-    sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options)
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.float32, shape=self.inp_dims, name="input")
+          dtype=dtype, shape=input_dims, name=input_name)
       e = constant_op.constant(
           np.random.normal(.3, 0.05, [3, 2, 3, 4]),
           name="weights",
-          dtype=dtypes.float32)
+          dtype=dtype)
       conv = nn.conv2d(
           input=x,
           filter=e,
@@ -68,11 +55,19 @@ class NeighboringEngineTest(BaseUnitTest):
       b = constant_op.constant(
           np.random.normal(1.0, 1.0, [1, 4, 1, 1]),
           name="bias",
-          dtype=dtypes.float32)
+          dtype=dtype)
       t = conv * b
-
       e = gen_math_ops.tan(conv)
       t = t - e
-      array_ops.squeeze(t, name="output")
+      array_ops.squeeze(t, name=self.output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        num_expected_engines=2,
+        expected_output_dims=(2, 4, 5, 4),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
 
-    return g.as_graph_def()
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 02a9280542..48890ad413 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -162,6 +162,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
                       dynamic_engine=None):
     num_engines = 0
     for n in gdef.node:
+      # TODO(jie): we should have coverage for failed conversion (TF fallback).
+      # where the conversion will fail and we shouldn't count this engine as the
+      # converted engines.
       if n.op == "TRTEngineOp":
         num_engines += 1
         self.assertNotEqual(self._ToBytes(""), n.attr["serialized_segment"].s)
diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py
index a054939ce2..4c10c50e85 100644
--- a/tensorflow/contrib/tensorrt/test/unary_test.py
+++ b/tensorflow/contrib/tensorrt/test/unary_test.py
@@ -20,46 +20,31 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops import variables
-from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.training import training
-from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest
-from tensorflow.contrib.tensorrt.test.utilities import get_all_variables
+from tensorflow.python.platform import test
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 
 
-class UnaryTest(BaseUnitTest):
-  """Unit tests for unary operations in TF-TRT"""
+class UnaryTest(trt_test.TfTrtIntegrationTestBase):
+    
 
-  def __init__(self, log_file='log.txt'):
-    super(UnaryTest, self).__init__()
-    self.static_mode_list = {"FP32", "FP16"}
-    self.debug = True
-    self.dynamic_mode_list = {}
-    self.inp_dims = (12, 5, 8, 1, 1, 12)
-    self.dummy_input = np.random.random_sample(self.inp_dims)
-    self.get_network = self.unary_test
-    self.expect_nb_nodes = 17
-    self.log_file = log_file
-    self.test_name = self.__class__.__name__
-    self.ckpt = "./tmp.ckpt"
-
-  def unary_test(self):
+  def GetParams(self):
+    """unit test for unary operations in TF-TRT"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [12, 5, 8, 1, 1, 12]
+    input2_name = "input_2"
+    input2_dims = [12, 5, 8, 1, 12, 1, 1]
     g = ops.Graph()
-    gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-    sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options)
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.float32, shape=self.inp_dims, name="input")
+          dtype=dtype, shape=input_dims, name=input_name)
       q = math_ops.abs(x)
       q = q + 1.0
       q = gen_math_ops.exp(q)
@@ -75,7 +60,7 @@ class UnaryTest(BaseUnitTest):
       q = q + 3.0
       a = gen_math_ops.reciprocal(q)
 
-      x = constant_op.constant(np.random.randn(5, 8, 12), dtype=dtypes.float32)
+      x = constant_op.constant(np.random.randn(5, 8, 12), dtype=dtype)
       q = math_ops.abs(x)
       q = q + 2.0
       q = gen_math_ops.exp(q)
@@ -90,11 +75,8 @@ class UnaryTest(BaseUnitTest):
       b = gen_math_ops.reciprocal(q)
 
       # TODO(jie): this one will break, broadcasting on batch.
-      x = variable_scope.get_variable(
-          "test", [12, 40, 12],
-          dtype=dtypes.float32,
-          initializer=init_ops.truncated_normal_initializer)
-      x = gen_array_ops.reshape(x, [12, 5, 8, 1, 12, 1, 1])
+      x = array_ops.placeholder(
+          dtype=dtype, shape=input2_dims, name=input2_name)
       q = math_ops.abs(x)
       q = q + 5.0
       q = gen_math_ops.exp(q)
@@ -115,11 +97,15 @@ class UnaryTest(BaseUnitTest):
 
       q = a * b
       q = q / c
-      array_ops.squeeze(q, name="output")
+      array_ops.squeeze(q, name=self.output_name)
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name, input2_name],
+        input_dims=[input_dims, input2_dims],
+        num_expected_engines=5,
+        expected_output_dims=(12, 5, 8, 12),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
 
-      with session.Session(config=sessconfig, graph=g) as sess:
-        names_var_list = get_all_variables(sess)
-        saver = training.Saver(names_var_list)
-        sess.run(variables.global_variables_initializer())
-        saver.save(sess, self.ckpt)
-    return g.as_graph_def()
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
index 9a759eb994..3621c13bc9 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -28,31 +27,21 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import nn_impl
-from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest
+from tensorflow.python.platform import test
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 
 
-class VGGBlockNCHWTest(BaseUnitTest):
-  """single vgg layer in NCHW unit tests in TF-TRT"""
+class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase):
 
-  def __init__(self, log_file='log.txt'):
-    super(VGGBlockNCHWTest, self).__init__()
-    self.static_mode_list = {"FP32", "FP16"}
-    self.debug = True
-    self.dynamic_mode_list = {}
-    self.inp_dims = (5, 2, 8, 8)
-    self.dummy_input = np.random.random_sample(self.inp_dims)
-    self.get_network = self.get_simple_graph_def
-    self.expect_nb_nodes = 3
-    self.log_file = log_file
-    self.test_name = self.__class__.__name__
-
-  def get_simple_graph_def(self):
+  def GetParams(self):
+    """single vgg layer in NCHW unit tests in TF-TRT"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [5, 2, 8, 8]
     g = ops.Graph()
-    gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-    sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options)
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.float32, shape=self.inp_dims, name="input")
+          dtype=dtype, shape=input_dims, name=input_name)
       x, mean_x, var_x = nn_impl.fused_batch_norm(
           x,
           np.random.randn(2).astype(np.float32),
@@ -62,7 +51,7 @@ class VGGBlockNCHWTest(BaseUnitTest):
           data_format="NCHW",
           is_training=False)
       e = constant_op.constant(
-          np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32)
+          np.random.randn(1, 1, 2, 6), name="weights", dtype=dtype)
       conv = nn.conv2d(
           input=x,
           filter=e,
@@ -71,7 +60,7 @@ class VGGBlockNCHWTest(BaseUnitTest):
           padding="SAME",
           name="conv")
       b = constant_op.constant(
-          np.random.randn(6), name="bias", dtype=dtypes.float32)
+          np.random.randn(6), name="bias", dtype=dtype)
       t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd")
       relu = nn.relu(t, "relu")
       idty = array_ops.identity(relu, "ID")
@@ -81,5 +70,14 @@ class VGGBlockNCHWTest(BaseUnitTest):
           data_format="NCHW",
           name="max_pool")
       array_ops.squeeze(v, name="output")
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        num_expected_engines=1,
+        expected_output_dims=(5, 6, 2, 2),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
 
-    return g.as_graph_def()
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
index 04176d58ca..1ef32fe52f 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -28,31 +27,21 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import nn_impl
-from tensorflow.contrib.tensorrt.test.base_unit_test import BaseUnitTest
+from tensorflow.python.platform import test
+from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt_test
 
 
-class VGGBlockTest(BaseUnitTest):
-  """single vgg layer test in TF-TRT conversion"""
+class VGGBlockTest(trt_test.TfTrtIntegrationTestBase):
 
-  def __init__(self, log_file='log.txt'):
-    super(VGGBlockTest, self).__init__()
-    self.static_mode_list = {"FP32", "FP16"}
-    self.debug = True
-    self.dynamic_mode_list = {}
-    self.inp_dims = (5, 8, 8, 2)
-    self.dummy_input = np.random.random_sample(self.inp_dims)
-    self.get_network = self.get_simple_graph_def
-    self.expect_nb_nodes = 7
-    self.log_file = log_file
-    self.test_name = self.__class__.__name__
-
-  def get_simple_graph_def(self):
+  def GetParams(self):
+    """single vgg layer test in TF-TRT conversion"""
+    dtype = dtypes.float32
+    input_name = "input"
+    input_dims = [5, 8, 8, 2]
     g = ops.Graph()
-    gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-    sessconfig = config_pb2.ConfigProto(gpu_options=gpu_options)
     with g.as_default():
       x = array_ops.placeholder(
-          dtype=dtypes.float32, shape=self.inp_dims, name="input")
+          dtype=dtype, shape=input_dims, name=input_name)
       x, mean_x, var_x = nn_impl.fused_batch_norm(
           x,
           np.random.randn(2).astype(np.float32),
@@ -61,16 +50,25 @@ class VGGBlockTest(BaseUnitTest):
           variance=np.random.randn(2).astype(np.float32),
           is_training=False)
       e = constant_op.constant(
-          np.random.randn(1, 1, 2, 6), name="weights", dtype=dtypes.float32)
+          np.random.randn(1, 1, 2, 6), name="weights", dtype=dtype)
       conv = nn.conv2d(
           input=x, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv")
       b = constant_op.constant(
-          np.random.randn(6), name="bias", dtype=dtypes.float32)
+          np.random.randn(6), name="bias", dtype=dtype)
       t = nn.bias_add(conv, b, name="biasAdd")
       relu = nn.relu(t, "relu")
       idty = array_ops.identity(relu, "ID")
       v = nn_ops.max_pool(
           idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool")
       array_ops.squeeze(v, name="output")
+    return trt_test.TfTrtIntegrationTestParams(
+        gdef=g.as_graph_def(),
+        input_names=[input_name],
+        input_dims=[input_dims],
+        num_expected_engines=1,
+        expected_output_dims=(5, 2, 2, 6),
+        allclose_atol=1.e-03,
+        allclose_rtol=1.e-03)
 
-    return g.as_graph_def()
+if __name__ == "__main__":
+  test.main()
-- 
cgit v1.2.3


From 2422a250654757480e1c3e301a2a4d3564e9ff25 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Wed, 18 Jul 2018 21:18:10 -0700
Subject: Update ops-related pbtxt files.

PiperOrigin-RevId: 205184208
---
 tensorflow/core/ops/compat/ops_history.v1.pbtxt | 65 +++++++++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                   |  4 ++
 2 files changed, 69 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index d94fa2cad7..e91089e627 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -11045,6 +11045,71 @@ op {
     }
   }
 }
+op {
+  name: "Bitcast"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "type"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_BFLOAT16
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "type"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_BFLOAT16
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+}
 op {
   name: "BitwiseAnd"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 4f24ab480f..6f07dd612e 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -4132,6 +4132,8 @@ op {
         type: DT_INT32
         type: DT_UINT8
         type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
         type: DT_INT8
         type: DT_INT16
         type: DT_COMPLEX64
@@ -4157,6 +4159,8 @@ op {
         type: DT_INT32
         type: DT_UINT8
         type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
         type: DT_INT8
         type: DT_INT16
         type: DT_COMPLEX64
-- 
cgit v1.2.3


From 874de86fe803823589d6b1c1e2dbe4adc5d3408c Mon Sep 17 00:00:00 2001
From: Jiri Simsa 
Date: Wed, 18 Jul 2018 21:33:42 -0700
Subject: Changing the mapping between proto and TF types.

PiperOrigin-RevId: 205185039
---
 .../kernel_tests/decode_proto_op_test_base.py      |  31 +-
 .../python/kernel_tests/proto_op_test_base.py      |  72 ++--
 tensorflow/core/kernels/BUILD                      |   2 +
 tensorflow/core/kernels/decode_proto_op.cc         | 367 +++++++++------------
 tensorflow/core/kernels/encode_proto_op.cc         | 284 +++++++++-------
 tensorflow/core/util/proto/BUILD                   |  10 +
 tensorflow/core/util/proto/decode.h                | 298 +++++++++++------
 tensorflow/core/util/proto/proto_utils.cc          |  70 ++++
 tensorflow/core/util/proto/proto_utils.h           |  33 ++
 9 files changed, 682 insertions(+), 485 deletions(-)
 create mode 100644 tensorflow/core/util/proto/proto_utils.cc
 create mode 100644 tensorflow/core/util/proto/proto_utils.h

diff --git a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py
index 5f7f510352..e3570e38a3 100644
--- a/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py
+++ b/tensorflow/contrib/proto/python/kernel_tests/decode_proto_op_test_base.py
@@ -106,34 +106,27 @@ class DecodeProtoOpTestBase(test_base.ProtoOpTestBase, parameterized.TestCase):
           self.assertEqual(v, ev)
         continue
 
-      # This can be a little confusing. For testing we are using TestValue in
-      # two ways: it's the proto that we decode for testing, and it's used in
-      # the expected value as a union type.
-      #
-      # The two cases are slightly different: this is the second case. We may be
-      # fetching the uint64_value from the test proto, but in the expected proto
-      # we store it in the int64_value field because TensorFlow doesn't support
-      # unsigned int64.
       tf_type_to_primitive_value_field = {
+          dtypes.bool:
+              'bool_value',
           dtypes.float32:
               'float_value',
           dtypes.float64:
               'double_value',
-          dtypes.int32:
-              'int32_value',
-          dtypes.uint8:
-              'uint8_value',
           dtypes.int8:
               'int8_value',
-          dtypes.string:
-              'string_value',
+          dtypes.int32:
+              'int32_value',
           dtypes.int64:
               'int64_value',
-          dtypes.bool:
-              'bool_value',
-          # Unhandled TensorFlow types:
-          # DT_INT16 DT_COMPLEX64 DT_QINT8 DT_QUINT8 DT_QINT32
-          # DT_BFLOAT16 DT_QINT16 DT_QUINT16 DT_UINT16
+          dtypes.string:
+              'string_value',
+          dtypes.uint8:
+              'uint8_value',
+          dtypes.uint32:
+              'uint32_value',
+          dtypes.uint64:
+              'uint64_value',
       }
       tf_field_name = tf_type_to_primitive_value_field.get(field.dtype)
       if tf_field_name is None:
diff --git a/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py b/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py
index cbc7b3d3f8..2950c7dfdc 100644
--- a/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py
+++ b/tensorflow/contrib/proto/python/kernel_tests/proto_op_test_base.py
@@ -44,7 +44,7 @@ class ProtoOpTestBase(test.TestCase):
         ("minmax", ProtoOpTestBase.minmax_test_case()),
         ("nested", ProtoOpTestBase.nested_test_case()),
         ("optional", ProtoOpTestBase.optional_test_case()),
-        ("promote_unsigned", ProtoOpTestBase.promote_unsigned_test_case()),
+        ("promote", ProtoOpTestBase.promote_test_case()),
         ("ragged", ProtoOpTestBase.ragged_test_case()),
         ("shaped_batch", ProtoOpTestBase.shaped_batch_test_case()),
         ("simple", ProtoOpTestBase.simple_test_case()),
@@ -83,13 +83,13 @@ class ProtoOpTestBase(test.TestCase):
     test_case.sizes.append(0)
     field = test_case.fields.add()
     field.name = "uint64_value_with_default"
-    field.dtype = types_pb2.DT_INT64
-    field.value.int64_value.append(4)
+    field.dtype = types_pb2.DT_UINT64
+    field.value.uint64_value.append(4)
     test_case.sizes.append(0)
     field = test_case.fields.add()
     field.name = "fixed64_value_with_default"
-    field.dtype = types_pb2.DT_INT64
-    field.value.int64_value.append(6)
+    field.dtype = types_pb2.DT_UINT64
+    field.value.uint64_value.append(6)
     test_case.sizes.append(0)
     field = test_case.fields.add()
     field.name = "int32_value_with_default"
@@ -108,13 +108,13 @@ class ProtoOpTestBase(test.TestCase):
     test_case.sizes.append(0)
     field = test_case.fields.add()
     field.name = "uint32_value_with_default"
-    field.dtype = types_pb2.DT_INT32
-    field.value.int32_value.append(9)
+    field.dtype = types_pb2.DT_UINT32
+    field.value.uint32_value.append(9)
     test_case.sizes.append(0)
     field = test_case.fields.add()
     field.name = "fixed32_value_with_default"
-    field.dtype = types_pb2.DT_INT32
-    field.value.int32_value.append(7)
+    field.dtype = types_pb2.DT_UINT32
+    field.value.uint32_value.append(7)
     test_case.sizes.append(0)
     field = test_case.fields.add()
     field.name = "bool_value_with_default"
@@ -202,15 +202,15 @@ class ProtoOpTestBase(test.TestCase):
     test_case.sizes.append(2)
     field = test_case.fields.add()
     field.name = "uint64_value"
-    field.dtype = types_pb2.DT_INT64
-    field.value.int64_value.append(0)
-    field.value.int64_value.append(-1)
+    field.dtype = types_pb2.DT_UINT64
+    field.value.uint64_value.append(0)
+    field.value.uint64_value.append(18446744073709551615)
     test_case.sizes.append(2)
     field = test_case.fields.add()
     field.name = "fixed64_value"
-    field.dtype = types_pb2.DT_INT64
-    field.value.int64_value.append(0)
-    field.value.int64_value.append(-1)
+    field.dtype = types_pb2.DT_UINT64
+    field.value.uint64_value.append(0)
+    field.value.uint64_value.append(18446744073709551615)
     test_case.sizes.append(2)
     field = test_case.fields.add()
     field.name = "int32_value"
@@ -232,15 +232,15 @@ class ProtoOpTestBase(test.TestCase):
     test_case.sizes.append(2)
     field = test_case.fields.add()
     field.name = "uint32_value"
-    field.dtype = types_pb2.DT_INT32
-    field.value.int32_value.append(0)
-    field.value.int32_value.append(-1)
+    field.dtype = types_pb2.DT_UINT32
+    field.value.uint32_value.append(0)
+    field.value.uint32_value.append(4294967295)
     test_case.sizes.append(2)
     field = test_case.fields.add()
     field.name = "fixed32_value"
-    field.dtype = types_pb2.DT_INT32
-    field.value.int32_value.append(0)
-    field.value.int32_value.append(-1)
+    field.dtype = types_pb2.DT_UINT32
+    field.value.uint32_value.append(0)
+    field.value.uint32_value.append(4294967295)
     test_case.sizes.append(2)
     field = test_case.fields.add()
     field.name = "bool_value"
@@ -289,28 +289,40 @@ class ProtoOpTestBase(test.TestCase):
     return test_case
 
   @staticmethod
-  def promote_unsigned_test_case():
+  def promote_test_case():
     test_case = test_example_pb2.TestCase()
     value = test_case.values.add()
+    value.sint32_value.append(2147483647)
+    value.sfixed32_value.append(2147483647)
+    value.int32_value.append(2147483647)
     value.fixed32_value.append(4294967295)
     value.uint32_value.append(4294967295)
     test_case.shapes.append(1)
     test_case.sizes.append(1)
     field = test_case.fields.add()
-    field.name = "fixed32_value"
+    field.name = "sint32_value"
     field.dtype = types_pb2.DT_INT64
-    field.value.int64_value.append(4294967295)
+    field.value.int64_value.append(2147483647)
     test_case.sizes.append(1)
     field = test_case.fields.add()
-    field.name = "uint32_value"
+    field.name = "sfixed32_value"
     field.dtype = types_pb2.DT_INT64
-    field.value.int64_value.append(4294967295)
-    # Comes from an explicitly-specified default
-    test_case.sizes.append(0)
+    field.value.int64_value.append(2147483647)
+    test_case.sizes.append(1)
     field = test_case.fields.add()
-    field.name = "uint32_value_with_default"
+    field.name = "int32_value"
     field.dtype = types_pb2.DT_INT64
-    field.value.int64_value.append(9)
+    field.value.int64_value.append(2147483647)
+    test_case.sizes.append(1)
+    field = test_case.fields.add()
+    field.name = "fixed32_value"
+    field.dtype = types_pb2.DT_UINT64
+    field.value.uint64_value.append(4294967295)
+    test_case.sizes.append(1)
+    field = test_case.fields.add()
+    field.name = "uint32_value"
+    field.dtype = types_pb2.DT_UINT64
+    field.value.uint64_value.append(4294967295)
     return test_case
 
   @staticmethod
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 99e5e3cfca..10cbcdecc8 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6320,6 +6320,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core/util/proto:decode",
         "//tensorflow/core/util/proto:descriptors",
+        "//tensorflow/core/util/proto:proto_utils",
         "//third_party/eigen3",
     ],
 )
@@ -6332,6 +6333,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core/util/proto:descriptors",
+        "//tensorflow/core/util/proto:proto_utils",
         "//third_party/eigen3",
     ],
 )
diff --git a/tensorflow/core/kernels/decode_proto_op.cc b/tensorflow/core/kernels/decode_proto_op.cc
index 6d3dcc1c59..b54e1ea8ac 100644
--- a/tensorflow/core/kernels/decode_proto_op.cc
+++ b/tensorflow/core/kernels/decode_proto_op.cc
@@ -13,21 +13,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// DecodeProto is a TensorFlow Op which extracts arbitrary fields
-// from protos serialized as strings.
+// DecodeProto is a TensorFlow op which extracts arbitrary fields from protos
+// serialized as strings.
 //
 // See docs in ../ops/decode_proto_op.cc.
 //
-// This implementation reads the serialized format using a handful of
-// calls from the WireFormatLite API used by generated proto code.
-// WireFormatLite is marked as an "internal" proto API but is widely
-// used in practice and highly unlikely to change.
-// This will be much faster than the previous implementation based on
-// constructing a temporary dynamic message in memory and using the
-// proto reflection api to read it.
-// It can be used with any proto whose descriptors are available at
-// runtime but should be competitive in speed with approaches that
-// compile in the proto definitions.
+// This implementation reads the serialized format using a handful of calls from
+// the WireFormatLite API used by generated proto code. WireFormatLite is marked
+// as an "internal" proto API but is widely used in practice and highly unlikely
+// to change. This will be much faster than the previous implementation based on
+// constructing a temporary dynamic message in memory and using the proto
+// reflection api to read it. It can be used with any proto whose descriptors
+// are available at runtime but should be competitive in speed with approaches
+// that compile in the proto definitions.
 
 #include 
 #include 
@@ -36,11 +34,13 @@ limitations under the License.
 #include "third_party/eigen3/Eigen/Core"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/util/proto/decode.h"
 #include "tensorflow/core/util/proto/descriptors.h"
+#include "tensorflow/core/util/proto/proto_utils.h"
 #include "tensorflow/core/util/ptr_util.h"
 
 namespace tensorflow {
@@ -58,53 +58,6 @@ using ::tensorflow::protobuf::io::CodedInputStream;
 
 const bool kFailOnDecodeError = true;
 
-// Returns true if the proto field type can be converted to the
-// tensorflow::DataType.
-bool CheckOutputType(FieldDescriptor::Type field_type, DataType output_type) {
-  switch (field_type) {
-    case WireFormatLite::TYPE_DOUBLE:
-      return output_type == tensorflow::DT_DOUBLE;
-    case WireFormatLite::TYPE_FLOAT:
-      return output_type == tensorflow::DT_FLOAT ||
-             output_type == tensorflow::DT_DOUBLE;
-    case WireFormatLite::TYPE_INT64:
-      return output_type == tensorflow::DT_INT64;
-    case WireFormatLite::TYPE_UINT64:
-      return output_type == tensorflow::DT_INT64;
-    case WireFormatLite::TYPE_INT32:
-      return output_type == tensorflow::DT_INT32;
-    case WireFormatLite::TYPE_FIXED64:
-      return output_type == tensorflow::DT_INT64;
-    case WireFormatLite::TYPE_FIXED32:
-      return output_type == tensorflow::DT_INT32 ||
-             output_type == tensorflow::DT_INT64;
-    case WireFormatLite::TYPE_BOOL:
-      return output_type == tensorflow::DT_BOOL;
-    case WireFormatLite::TYPE_STRING:
-      return output_type == tensorflow::DT_STRING;
-    case WireFormatLite::TYPE_GROUP:
-      return output_type == tensorflow::DT_STRING;
-    case WireFormatLite::TYPE_MESSAGE:
-      return output_type == tensorflow::DT_STRING;
-    case WireFormatLite::TYPE_BYTES:
-      return output_type == tensorflow::DT_STRING;
-    case WireFormatLite::TYPE_UINT32:
-      return output_type == tensorflow::DT_INT32 ||
-             output_type == tensorflow::DT_INT64;
-    case WireFormatLite::TYPE_ENUM:
-      return output_type == tensorflow::DT_INT32;
-    case WireFormatLite::TYPE_SFIXED32:
-      return output_type == tensorflow::DT_INT32;
-    case WireFormatLite::TYPE_SFIXED64:
-      return output_type == tensorflow::DT_INT64;
-    case WireFormatLite::TYPE_SINT32:
-      return output_type == tensorflow::DT_INT32;
-    case WireFormatLite::TYPE_SINT64:
-      return output_type == tensorflow::DT_INT64;
-      // default: intentionally omitted in order to enable static checking.
-  }
-}
-
 // Used to store the default value of a protocol message field, casted to the
 // type of the output tensor.
 //
@@ -113,13 +66,15 @@ struct DefaultValue {
   DataType dtype = DataType::DT_INVALID;
   union Value {
     bool v_bool;           // DT_BOOL
-    uint8 v_uint8;         // DT_UINT8
+    double v_double;       // DT_DOUBLE
+    float v_float;         // DT_FLOAT
     int8 v_int8;           // DT_INT8
     int32 v_int32;         // DT_INT32
     int64 v_int64;         // DT_INT64
-    float v_float;         // DT_FLOAT
-    double v_double;       // DT_DOUBLE
     const char* v_string;  // DT_STRING
+    uint8 v_uint8;         // DT_UINT8
+    uint8 v_uint32;        // DT_UINT32
+    uint8 v_uint64;        // DT_UINT64
   };
   Value value;
 };
@@ -138,23 +93,29 @@ Status InitDefaultValue(DataType dtype, const T value, DefaultValue* result) {
     case DT_BOOL:
       result->value.v_bool = static_cast(value);
       break;
-    case DT_INT32:
-      result->value.v_int32 = static_cast(value);
+    case DT_DOUBLE:
+      result->value.v_double = static_cast(value);
+      break;
+    case DT_FLOAT:
+      result->value.v_float = static_cast(value);
       break;
     case DT_INT8:
       result->value.v_int8 = static_cast(value);
       break;
-    case DT_UINT8:
-      result->value.v_uint8 = static_cast(value);
+    case DT_INT32:
+      result->value.v_int32 = static_cast(value);
       break;
     case DT_INT64:
       result->value.v_int64 = static_cast(value);
       break;
-    case DT_FLOAT:
-      result->value.v_float = static_cast(value);
+    case DT_UINT8:
+      result->value.v_uint8 = static_cast(value);
       break;
-    case DT_DOUBLE:
-      result->value.v_double = static_cast(value);
+    case DT_UINT32:
+      result->value.v_uint32 = static_cast(value);
+      break;
+    case DT_UINT64:
+      result->value.v_uint64 = static_cast(value);
       break;
     default:
       // We should never get here, given the type checking that occurs earlier.
@@ -241,13 +202,11 @@ struct FieldInfo {
     number = field_desc->number();
 
     // The wire format library defines the same constants used in
-    // descriptor.proto. This static_cast is safe because they
-    // are guaranteed to stay in sync.
-    // We need the field type from the FieldDescriptor here
-    // because the wire format doesn't tell us anything about
-    // what happens inside a packed repeated field: there is
-    // enough information in the wire format to skip the
-    // whole field but not enough to know how to parse what's
+    // descriptor.proto. This static_cast is safe because they are guaranteed to
+    // stay in sync. We need the field type from the FieldDescriptor here
+    // because the wire format doesn't tell us anything about what happens
+    // inside a packed repeated field: there is enough information in the wire
+    // format to skip the whole field but not enough to know how to parse what's
     // inside. For that we go to the schema.
     type = static_cast(field_desc->type());
     is_repeated = field_desc->is_repeated();
@@ -257,16 +216,15 @@ struct FieldInfo {
   FieldInfo(const FieldInfo&) = delete;
   FieldInfo& operator=(const FieldInfo&) = delete;
 
-  // Internally we sort field descriptors by wire number for
-  // fast lookup. In general this is different from the order
-  // given by the user. Output_index gives the index into
-  // the field_names and output_types attributes and into
+  // Internally we sort field descriptors by wire number for fast lookup. In
+  // general this is different from the order given by the user. Output_index
+  // gives the index into the field_names and output_types attributes and into
   // the output tensor list.
   int output_index = -1;
 
-  // This is a cache of the relevant fields from `FieldDescriptorProto`.
-  // This was added after noticing that FieldDescriptor->type() was
-  // using 6% of the cpu profile.
+  // This is a cache of the relevant fields from `FieldDescriptorProto`. This
+  // was added after noticing that FieldDescriptor->type() was using 6% of the
+  // cpu profile.
   WireFormatLite::FieldType type;
   int number;
   bool is_repeated;
@@ -275,16 +233,16 @@ struct FieldInfo {
 
 // A CountCollector counts sizes of repeated and optional fields in a proto.
 //
-// Each field is tracked by a single CountCollector instance. The
-// instance manages a single count, which is stored as a pointer (it
-// is intended to be a reference to the `sizes` output which is being
-// filled in). The pointer is passed in at initialization.
+// Each field is tracked by a single CountCollector instance. The instance
+// manages a single count, which is stored as a pointer (it is intended to be a
+// reference to the `sizes` output which is being filled in). The pointer is
+// passed in at initialization.
 //
-// Counting is done as a separate pass in order to allocate output tensors
-// all at once. This allows the TensorFlow runtime to optimize allocation
-// for the consumer, while removing the need for copying inside this op.
-// After this pass, the DenseCollector class (below) gathers the data:
-// It is more complex and provides better motivation for the API here.
+// Counting is done as a separate pass in order to allocate output tensors all
+// at once. This allows the TensorFlow runtime to optimize allocation for the
+// consumer, while removing the need for copying inside this op. After this
+// pass, the DenseCollector class (below) gathers the data: it is more complex
+// and provides better motivation for the API here.
 class CountCollector {
  public:
   CountCollector() = delete;
@@ -298,8 +256,8 @@ class CountCollector {
     if (*count_ptr_ == 0 || field.is_repeated) {
       (*count_ptr_)++;
     }
-    // We expect a wire type based on the schema field_type, to allow
-    // a little more checking.
+    // We expect a wire type based on the schema field_type, to allow a little
+    // more checking.
     if (!SkipValue(input, field)) {
       return errors::DataLoss("ReadValue: Failed skipping field when counting");
     }
@@ -329,8 +287,8 @@ class CountCollector {
       return errors::DataLoss("ReadPackedValues: Skipping packed field failed");
     }
 
-    // Dispatch to the appropriately typed field reader based on the
-    // schema type.
+    // Dispatch to the appropriately typed field reader based on the schema
+    // type.
     Status st;
     switch (field.type) {
       case WireFormatLite::TYPE_DOUBLE:
@@ -409,18 +367,17 @@ class CountCollector {
     return input->Skip(length);
   }
 
-  // Counts the number of packed varints in an array.
-  // The end of a varint is signaled by a value < 0x80,
-  // so counting them requires parsing the bytestream.
-  // It is the caller's responsibility to ensure that len > 0.
+  // Counts the number of packed varints in an array. The end of a varint is
+  // signaled by a value < 0x80, so counting them requires parsing the
+  // bytestream. It is the caller's responsibility to ensure that len > 0.
   Status CountPackedVarint(const uint8* buf, size_t len) {
     const uint8* bound = buf + len;
     int count;
 
-    // The last byte in a valid encoded varint is guaranteed to have
-    // the high bit unset. We rely on this property to prevent
-    // ReadVarint64FromArray from going out of bounds, so validate
-    // the end of the buf before scanning anything.
+    // The last byte in a valid encoded varint is guaranteed to have the high
+    // bit unset. We rely on this property to prevent ReadVarint64FromArray from
+    // going out of bounds, so validate the end of the buf before scanning
+    // anything.
     if (bound[-1] & 0x80) {
       return errors::DataLoss("Corrupt packed varint");
     }
@@ -439,8 +396,8 @@ class CountCollector {
     return Status::OK();
   }
 
-  // Counts the number of fixed-size values in a packed field.
-  // This can be done without actually parsing anything.
+  // Counts the number of fixed-size values in a packed field. This can be done
+  // without actually parsing anything.
   template 
   Status CountPackedFixed(const uint8* unused_buf, size_t len) {
     int count = len / sizeof(T);
@@ -452,10 +409,9 @@ class CountCollector {
     return Status::OK();
   }
 
-  // Skips a single value in the input stream.
-  // Dispatches to the appropriately typed field skipper based on the
-  // schema type tag.
-  // This is not as permissive as just handling the wire type.
+  // Skips a single value in the input stream. Dispatches to the appropriately
+  // typed field skipper based on the schema type tag. This is not as permissive
+  // as just handling the wire type.
   static bool SkipValue(CodedInputStream* input, const FieldInfo& field) {
     uint32 tmp32;
     protobuf_uint64 tmp64;
@@ -507,13 +463,13 @@ class CountCollector {
 
 // A DenseCollector accumulates values from a proto into a tensor.
 //
-// There is an instance of DenseCollector for each field of each
-// proto. The DenseCollector deserializes the value from the wire
-// directly into the preallocated output Tensor.
+// There is an instance of DenseCollector for each field of each proto. The
+// DenseCollector deserializes the value from the wire directly into the
+// preallocated output Tensor.
 //
-// This class is named DenseCollector because in the future there should
-// be a SparseCollector that accumulates field data into sparse tensors if
-// the user requests it.
+// This class is named DenseCollector because in the future there should be a
+// SparseCollector that accumulates field data into sparse tensors if the user
+// requests it.
 class DenseCollector {
  public:
   DenseCollector() = delete;
@@ -578,40 +534,43 @@ class DenseCollector {
     }
   }
 
-  // Fills in any missing values in the output array with defaults.
-  // Dispatches to the appropriately typed field default based on the
-  // runtime type tag.
+  // Fills in any missing values in the output array with defaults. Dispatches
+  // to the appropriately typed field default based on the runtime type tag.
   Status FillWithDefaults() {
     switch (default_value_.dtype) {
+      case DataType::DT_BOOL:
+        return FillDefault(default_value_.value.v_bool);
       case DataType::DT_FLOAT:
         return FillDefault(default_value_.value.v_float);
       case DataType::DT_DOUBLE:
         return FillDefault(default_value_.value.v_double);
-      case DataType::DT_INT32:
-        return FillDefault(default_value_.value.v_int32);
-      case DataType::DT_UINT8:
-        return FillDefault(default_value_.value.v_uint8);
       case DataType::DT_INT8:
         return FillDefault(default_value_.value.v_int8);
-      case DataType::DT_STRING:
-        return FillDefault(default_value_.value.v_string);
+      case DataType::DT_INT32:
+        return FillDefault(default_value_.value.v_int32);
       case DataType::DT_INT64:
         return FillDefault(default_value_.value.v_int64);
-      case DataType::DT_BOOL:
-        return FillDefault(default_value_.value.v_bool);
+      case DataType::DT_STRING:
+        return FillDefault(default_value_.value.v_string);
+      case DataType::DT_UINT8:
+        return FillDefault(default_value_.value.v_uint8);
+      case DataType::DT_UINT32:
+        return FillDefault(default_value_.value.v_uint32);
+      case DataType::DT_UINT64:
+        return FillDefault(default_value_.value.v_uint64);
       default:
         // There are many tensorflow dtypes not handled here, but they
         // should not come up unless type casting is added to the Op.
         // Chaining with tf.cast() should do the right thing until then.
-        return errors::DataLoss(
-            "Failed filling defaults in unknown tf::DataType");
+        return errors::DataLoss("Failed filling defaults for ",
+                                DataTypeString(default_value_.dtype));
     }
   }
 
  private:
-  // Fills empty values in the dense representation with a
-  // default value. This uses next_repeat_index_ which counts the number
-  // of parsed values for the field.
+  // Fills empty values in the dense representation with a default value. This
+  // uses next_repeat_index_ which counts the number of parsed values for the
+  // field.
   template 
   Status FillDefault(const T& default_value) {
     for (int i = next_repeat_index_; i < max_repeat_count_; i++) {
@@ -622,11 +581,10 @@ class DenseCollector {
 
   int32 next_repeat_index_ = 0;
 
-  // This is a pointer to data_[message_index_].
-  // There is no bounds checking at this level: we computed the max
-  // repeat size for each field in CountCollector and use the same
-  // code to traverse it here, so we are guaranteed not to be called
-  // for more items than we have allocated space.
+  // This is a pointer to data_[message_index_]. There is no bounds checking at
+  // this level: we computed the max repeat size for each field in
+  // CountCollector and use the same code to traverse it here, so we are
+  // guaranteed not to be called for more items than we have allocated space.
   void* const datap_ = nullptr;
 
   const DefaultValue default_value_;
@@ -665,7 +623,6 @@ class DecodeProtoOp : public OpKernel {
                                 "have the same length"));
 
     // Gather the field descriptors and check that requested output types match.
-
     int field_index = 0;
     std::vector field_descs;
     for (const string& name : field_names) {
@@ -673,18 +630,16 @@ class DecodeProtoOp : public OpKernel {
       OP_REQUIRES(context, fd != nullptr,
                   errors::InvalidArgument("Unknown field: ", name,
                                           " in message type ", message_type));
-      OP_REQUIRES(context,
-                  CheckOutputType(fd->type(), output_types[field_index]),
-                  // Many TensorFlow types don't have corresponding proto types
-                  // and the user will get an error if they are requested. It
-                  // would be nice to allow conversions here, but tf.cast
-                  // already exists so we don't duplicate the functionality.
-                  // Known unhandled types:
-                  //   DT_INT16 DT_COMPLEX64 DT_QINT8 DT_QUINT8 DT_QINT32
-                  //   DT_BFLOAT16 DT_QINT16 DT_QUINT16 DT_UINT16
-                  errors::InvalidArgument("Unexpected output type for ",
-                                          fd->full_name(), ": ", fd->cpp_type(),
-                                          " to ", output_types[field_index]));
+      OP_REQUIRES(
+          context,
+          proto_utils::IsCompatibleType(fd->type(), output_types[field_index]),
+          // Many TensorFlow types don't have corresponding proto types and the
+          // user will get an error if they are requested. It would be nice to
+          // allow conversions here, but tf.cast already exists so we don't
+          // duplicate the functionality.
+          errors::InvalidArgument("Unexpected output type for ",
+                                  fd->full_name(), ": ", fd->cpp_type(), " to ",
+                                  output_types[field_index]));
 
       field_index++;
       field_descs.push_back(fd);
@@ -726,10 +681,9 @@ class DecodeProtoOp : public OpKernel {
         errors::InvalidArgument("format must be one of binary or text"));
     is_binary_ = format == "binary";
 
-    // Enable the initial protobuf sanitizer, which is much
-    // more expensive than the decoder.
-    // TODO(nix): Remove this once the fast decoder
-    // has passed security review.
+    // Enable the initial protobuf sanitizer, which is much more expensive than
+    // the decoder.
+    // TODO(nix): Remove this once the fast decoder has passed security review.
     OP_REQUIRES_OK(context, context->GetAttr("sanitize", &sanitize_));
   }
 
@@ -742,9 +696,9 @@ class DecodeProtoOp : public OpKernel {
 
     int field_count = fields_.size();
 
-    // Save the argument shape for later, then flatten the input
-    // Tensor since we are working componentwise. We will restore
-    // the same shape in the returned Tensor.
+    // Save the argument shape for later, then flatten the input Tensor since we
+    // are working componentwise. We will restore the same shape in the returned
+    // Tensor.
     const TensorShape& shape_prefix = buf_tensor.shape();
 
     TensorShape sizes_shape = shape_prefix;
@@ -752,8 +706,8 @@ class DecodeProtoOp : public OpKernel {
     Tensor* sizes_tensor = nullptr;
     OP_REQUIRES_OK(ctx, ctx->allocate_output(0, sizes_shape, &sizes_tensor));
 
-    // This is used to allocate binary bufs if used. It serves only
-    // to define memory ownership.
+    // This is used to allocate binary bufs if used. It serves only to define
+    // memory ownership.
     std::vector tmp_binary_bufs(message_count);
 
     // These are the actual buffers to use, which may be in tmp_binary_bufs
@@ -768,8 +722,8 @@ class DecodeProtoOp : public OpKernel {
         bufs.push_back(buf);
       }
     } else {
-      // We will have to allocate a copy, either to convert from text to
-      // binary or to sanitize a binary proto.
+      // We will have to allocate a copy, either to convert from text to binary
+      // or to sanitize a binary proto.
       for (int mi = 0; mi < message_count; ++mi) {
         ReserializeMessage(ctx, buf_tensor.flat()(mi),
                            &tmp_binary_bufs[mi]);
@@ -780,16 +734,14 @@ class DecodeProtoOp : public OpKernel {
       }
     }
 
-    // Walk through all the strings in the input tensor, counting
-    // the number of fields in each.
-    // We can't allocate our actual output Tensor until we know the
-    // maximum repeat count, so we do a first pass through the serialized
-    // proto just counting fields.
-    // We always allocate at least one value so that optional fields
-    // are populated with default values - this avoids a TF
-    // conditional when handling the output data.
-    // The caller can distinguish between real data and defaults
-    // using the repeat count matrix that is returned by decode_proto.
+    // Walk through all the strings in the input tensor, counting the number of
+    // fields in each. We can't allocate our actual output Tensor until we know
+    // the maximum repeat count, so we do a first pass through the serialized
+    // proto just counting fields. We always allocate at least one value so that
+    // optional fields are populated with default values - this avoids a TF
+    // conditional when handling the output data. The caller can distinguish
+    // between real data and defaults using the repeat count matrix that is
+    // returned by decode_proto.
     std::vector max_sizes(field_count, 1);
     for (int mi = 0; mi < message_count; ++mi) {
       CountFields(ctx, mi, *bufs[mi], sizes_tensor, &max_sizes);
@@ -814,14 +766,12 @@ class DecodeProtoOp : public OpKernel {
       //  REGISTER_OP(...)
       //    .Attr("output_types: list(type) >= 0")
       //    .Output("values: output_types")
-      OP_REQUIRES_OK(ctx,
-                     // ctx->allocate_output(output_indices_[fi] + 1,
-                     ctx->allocate_output(fields_[fi]->output_index + 1,
-                                          out_shape, &outputs[fi]));
+      OP_REQUIRES_OK(ctx, ctx->allocate_output(fields_[fi]->output_index + 1,
+                                               out_shape, &outputs[fi]));
     }
 
-    // Make the second pass through the serialized proto, decoding
-    // into preallocated tensors.
+    // Make the second pass through the serialized proto, decoding into
+    // preallocated tensors.
     AccumulateFields(ctx, bufs, outputs);
   }
 
@@ -976,6 +926,7 @@ class DecodeProtoOp : public OpKernel {
   // Look up the FieldDescriptor for a particular field number.
   bool LookupField(int field_number, int* field_index) {
     // Look up the FieldDescriptor using linear search.
+    //
     // TODO(nix): this could be sped up with binary search, but we are
     // already way off the fastpath at this point. If you see a hotspot
     // here, somebody is sending you very inefficient protos.
@@ -1010,6 +961,7 @@ class DecodeProtoOp : public OpKernel {
       // This takes advantage of the sorted field numbers in most serialized
       // protos: it tries the next expected field first rather than doing
       // a lookup by field number.
+      //
       // TODO(nix): haberman@ suggests a hybrid approach with a lookup table
       // for small field numbers and a hash table for larger ones. This would
       // be a simpler approach that should offer comparable speed in most
@@ -1029,9 +981,9 @@ class DecodeProtoOp : public OpKernel {
             last_good_field_index = field_index;
           }
         } else {
-          // If we see a field that is past the next field we want,
-          // it was empty. Look for the one after that.
-          // Repeat until we run out of fields that we care about.
+          // If we see a field that is past the next field we want, it was
+          // empty. Look for the one after that. Repeat until we run out of
+          // fields that we care about.
           while (field_number >= next_good_field_number) {
             if (field_number == next_good_field_number) {
               last_good_field_number = field_number;
@@ -1044,10 +996,9 @@ class DecodeProtoOp : public OpKernel {
               next_good_field_number =
                   fields_[last_good_field_index + 1]->number;
             } else {
-              // Saw something past the last field we care about.
-              // Continue parsing the message just in case there
-              // are disordered fields later, but any remaining
-              // ordered fields will have no effect.
+              // Saw something past the last field we care about. Continue
+              // parsing the message just in case there are disordered fields
+              // later, but any remaining ordered fields will have no effect.
               next_good_field_number = INT_MAX;
             }
           }
@@ -1077,20 +1028,20 @@ class DecodeProtoOp : public OpKernel {
                       WireFormatLite::WireType wire_type,
                       CodedInputStream* input, CollectorClass* collector) {
     // The wire format library defines the same constants used in
-    // descriptor.proto. This static_cast is safe because they
-    // are guaranteed to stay in sync.
-    // We need the field type from the FieldDescriptor here
-    // because the wire format doesn't tell us anything about
-    // what happens inside a packed repeated field: there is
-    // enough information in the wire format to skip the
-    // whole field but not enough to know how to parse what's
-    // inside. For that we go to the schema.
+    // descriptor.proto. This static_cast is safe because they are guaranteed to
+    // stay in sync.
+    //
+    // We need the field type from the FieldDescriptor here because the wire
+    // format doesn't tell us anything about what happens inside a packed
+    // repeated field: there is enough information in the wire format to skip
+    // the whole field but not enough to know how to parse what's inside. For
+    // that we go to the schema.
     WireFormatLite::WireType schema_wire_type =
         WireFormatLite::WireTypeForFieldType(field.type);
 
-    // Handle packed repeated fields. SkipField would skip the
-    // whole length-delimited blob without letting us count the
-    // values, so we have to scan them ourselves.
+    // Handle packed repeated fields. SkipField would skip the whole
+    // length-delimited blob without letting us count the values, so we have to
+    // scan them ourselves.
     if (wire_type == WireFormatLite::WIRETYPE_LENGTH_DELIMITED &&
         schema_wire_type != WireFormatLite::WIRETYPE_LENGTH_DELIMITED) {
       // Handle packed repeated primitives.
@@ -1098,11 +1049,7 @@ class DecodeProtoOp : public OpKernel {
       if (!input->ReadVarintSizeAsInt(&length)) {
         return errors::DataLoss("CollectField: Failed reading packed size");
       }
-      Status st = collector->ReadPackedValues(input, field, length);
-      if (!st.ok()) {
-        return st;
-      }
-      return Status::OK();
+      return collector->ReadPackedValues(input, field, length);
     }
 
     // Read ordinary values, including strings, bytes, and messages.
@@ -1118,9 +1065,9 @@ class DecodeProtoOp : public OpKernel {
   }
 
   string message_type_;
-  // Note that fields are sorted by increasing field number,
-  // which is not in general the order given by the user-specified
-  // field_names and output_types Op attributes.
+  // Note that fields are sorted by increasing field number, which is not in
+  // general the order given by the user-specified field_names and output_types
+  // Op attributes.
   std::vector> fields_;
 
   // Owned_desc_pool_ is null when using descriptor_source=local.
@@ -1131,12 +1078,12 @@ class DecodeProtoOp : public OpKernel {
   // True if decoding binary format, false if decoding text format.
   bool is_binary_;
 
-  // True if the protos should be sanitized before parsing.
-  // Enables the initial protobuf sanitizer, which is much
-  // more expensive than the decoder. The flag defaults to true
-  // but can be set to false for trusted sources.
-  // TODO(nix): flip the default to false when the fast decoder
-  // has passed security review.
+  // True if the protos should be sanitized before parsing. Enables the initial
+  // protobuf sanitizer, which is much more expensive than the decoder. The flag
+  // defaults to true but can be set to false for trusted sources.
+  //
+  // TODO(nix): Flip the default to false when the fast decoder has passed
+  // security review.
   bool sanitize_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(DecodeProtoOp);
diff --git a/tensorflow/core/kernels/encode_proto_op.cc b/tensorflow/core/kernels/encode_proto_op.cc
index 3b02ae52a2..4a0c1943e5 100644
--- a/tensorflow/core/kernels/encode_proto_op.cc
+++ b/tensorflow/core/kernels/encode_proto_op.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/util/proto/descriptors.h"
+#include "tensorflow/core/util/proto/proto_utils.h"
 
 namespace tensorflow {
 namespace {
@@ -42,9 +43,9 @@ using ::tensorflow::protobuf::internal::WireFormatLite;
 using ::tensorflow::protobuf::io::CodedOutputStream;
 using ::tensorflow::protobuf::io::StringOutputStream;
 
-// Computes the total serialized size for a packed repeated field.
-// For fixed-size types this can just multiply, but for variable-sized
-// types it has to iterate through the values in the tensor.
+// Computes the total serialized size for a packed repeated field. For
+// fixed-size types this can just multiply, but for variable-sized types it has
+// to iterate through the values in the tensor.
 template 
 size_t TotalPackedSize(const Tensor& input, int message_index, int size);
 
@@ -83,11 +84,11 @@ size_t TotalPackedSize(const Tensor& input,
 }
 
 template <>
-size_t TotalPackedSize(const Tensor& input,
-                                                           int message_index,
-                                                           int size) {
+size_t TotalPackedSize(const Tensor& input,
+                                                            int message_index,
+                                                            int size) {
   size_t data_size = 0;
-  auto input_t = input.flat_inner_dims();
+  auto input_t = input.flat_inner_dims();
   for (int64 i = 0; i < size; i++) {
     data_size += WireFormatLite::UInt64Size(
         input_t(static_cast(message_index), i));
@@ -95,6 +96,19 @@ size_t TotalPackedSize(const Tensor& input,
   return data_size;
 }
 
+template <>
+size_t TotalPackedSize(const Tensor& input,
+                                                          int message_index,
+                                                          int size) {
+  size_t data_size = 0;
+  auto input_t = input.flat_inner_dims();
+  for (int64 i = 0; i < size; i++) {
+    data_size += WireFormatLite::Int32Size(
+        input_t(static_cast(message_index), i));
+  }
+  return data_size;
+}
+
 template <>
 size_t TotalPackedSize(const Tensor& input,
                                                           int message_index,
@@ -109,23 +123,20 @@ size_t TotalPackedSize(const Tensor& input,
 }
 
 template <>
-size_t TotalPackedSize(const Tensor& input,
-                                                            int message_index,
-                                                            int size) {
+size_t TotalPackedSize(
+    const Tensor& input, int message_index, int size) {
   return size * WireFormatLite::kFixed64Size;
 }
 
 template <>
-size_t TotalPackedSize(const Tensor& input,
-                                                            int message_index,
-                                                            int size) {
+size_t TotalPackedSize(
+    const Tensor& input, int message_index, int size) {
   return size * WireFormatLite::kFixed32Size;
 }
 
 template <>
-size_t TotalPackedSize(const Tensor& input,
-                                                            int message_index,
-                                                            int size) {
+size_t TotalPackedSize(
+    const Tensor& input, int message_index, int size) {
   return size * WireFormatLite::kFixed32Size;
 }
 
@@ -137,11 +148,11 @@ size_t TotalPackedSize(const Tensor& input,
 }
 
 template <>
-size_t TotalPackedSize(const Tensor& input,
-                                                           int message_index,
-                                                           int size) {
+size_t TotalPackedSize(const Tensor& input,
+                                                            int message_index,
+                                                            int size) {
   size_t data_size = 0;
-  auto input_t = input.flat_inner_dims();
+  auto input_t = input.flat_inner_dims();
   for (int64 i = 0; i < size; i++) {
     data_size += WireFormatLite::UInt32Size(
         input_t(static_cast(message_index), i));
@@ -150,11 +161,11 @@ size_t TotalPackedSize(const Tensor& input,
 }
 
 template <>
-size_t TotalPackedSize(const Tensor& input,
-                                                           int message_index,
-                                                           int size) {
+size_t TotalPackedSize(const Tensor& input,
+                                                            int message_index,
+                                                            int size) {
   size_t data_size = 0;
-  auto input_t = input.flat_inner_dims();
+  auto input_t = input.flat_inner_dims();
   for (int64 i = 0; i < size; i++) {
     data_size += WireFormatLite::UInt32Size(
         input_t(static_cast(message_index), i));
@@ -181,6 +192,12 @@ size_t TotalPackedSize(
   return size * WireFormatLite::kSFixed32Size;
 }
 
+template <>
+size_t TotalPackedSize(
+    const Tensor& input, int message_index, int size) {
+  return size * WireFormatLite::kSFixed32Size;
+}
+
 template <>
 size_t TotalPackedSize(
     const Tensor& input, int message_index, int size) {
@@ -200,6 +217,19 @@ size_t TotalPackedSize(const Tensor& input,
   return data_size;
 }
 
+template <>
+size_t TotalPackedSize(const Tensor& input,
+                                                           int message_index,
+                                                           int size) {
+  size_t data_size = 0;
+  auto input_t = input.flat_inner_dims();
+  for (int64 i = 0; i < size; i++) {
+    data_size += WireFormatLite::SInt32Size(
+        input_t(static_cast(message_index), i));
+  }
+  return data_size;
+}
+
 template <>
 size_t TotalPackedSize(const Tensor& input,
                                                            int message_index,
@@ -213,14 +243,13 @@ size_t TotalPackedSize(const Tensor& input,
   return data_size;
 }
 
-// Writes a possibly repeated primitive field.
-// TensorFlow does not have unsigned types, so we decode them to signed and
-// encode them back to unsigned.
+// Writes a possibly repeated primitive field. TensorFlow does not have unsigned
+// types, so we decode them to signed and encode them back to unsigned.
 template 
-void WriteField(const FieldDescriptor& field_desc, const Tensor& input,
-                int message_index, int size, CodedOutputStream* output) {
+Status WriteField(const FieldDescriptor& field_desc, const Tensor& input,
+                  int message_index, int size, CodedOutputStream* output) {
   auto wire_type = WireFormatLite::WireTypeForFieldType(
       WireFormatLite::FieldType(field_desc.type()));
 
@@ -250,12 +279,14 @@ void WriteField(const FieldDescriptor& field_desc, const Tensor& input,
       Writer(value, output);
     }
   }
+  return Status::OK();
 }
 
 // Writes a possibly repeated string, bytes, or message field.
 template 
-void WriteVarLenField(const FieldDescriptor& field_desc, const Tensor& input,
-                      int message_index, int size, CodedOutputStream* output) {
+Status WriteVarLenField(const FieldDescriptor& field_desc, const Tensor& input,
+                        int message_index, int size,
+                        CodedOutputStream* output) {
   auto input_t = input.flat_inner_dims();
   for (int64 i = 0; i < size; i++) {
     const T& value = input_t(static_cast(message_index), i);
@@ -264,14 +295,14 @@ void WriteVarLenField(const FieldDescriptor& field_desc, const Tensor& input,
     // small speedup.
     Writer(field_desc.number(), value, output);
   }
+  return Status::OK();
 }
 
-// Writes a group field.
-// Groups are treated like submessages, but tag-delimited
-// instead of length-delimited. WireFormatLite handles this
-// differently so we code it ourselves.
-void WriteGroup(const FieldDescriptor& field_desc, const Tensor& input,
-                int message_index, int size, CodedOutputStream* output) {
+// Writes a group field. Groups are treated like submessages, but tag-delimited
+// instead of length-delimited. WireFormatLite handles this differently so we
+// code it ourselves.
+Status WriteGroup(const FieldDescriptor& field_desc, const Tensor& input,
+                  int message_index, int size, CodedOutputStream* output) {
   auto input_t = input.flat_inner_dims();
   for (int64 i = 0; i < size; i++) {
     const string& value = input_t(static_cast(message_index), i);
@@ -282,16 +313,16 @@ void WriteGroup(const FieldDescriptor& field_desc, const Tensor& input,
     WireFormatLite::WriteTag(field_desc.number(),
                              WireFormatLite::WIRETYPE_END_GROUP, output);
   }
+  return Status::OK();
 }
 
-// Writes a (possibly repeated) field into an output stream.
-// It is the caller's responsibility to ensure that the type of
-// the input tensor is compatible with the type of the proto
-// field descriptor, and that (message_index, size-1) is within
-// bounds.
-void WriteField(const FieldDescriptor& field_desc, const Tensor& input,
-                int message_index, int size, CodedOutputStream* output) {
-  DataType tf_type = input.dtype();
+// Writes a (possibly repeated) field into an output stream. It is the caller's
+// responsibility to ensure that the type of the input tensor is compatible with
+// the type of the proto field descriptor, and that (message_index, size-1) is
+// within bounds.
+Status WriteField(const FieldDescriptor& field_desc, const Tensor& input,
+                  int message_index, int size, CodedOutputStream* output) {
+  DataType dtype = input.dtype();
 
   switch (field_desc.type()) {
     case WireFormatLite::TYPE_DOUBLE:
@@ -299,7 +330,7 @@ void WriteField(const FieldDescriptor& field_desc, const Tensor& input,
                         WireFormatLite::WriteDoubleNoTag>(
           field_desc, input, message_index, size, output);
     case WireFormatLite::TYPE_FLOAT:
-      switch (tf_type) {
+      switch (dtype) {
         case DataType::DT_FLOAT:
           return WriteField(
@@ -309,36 +340,48 @@ void WriteField(const FieldDescriptor& field_desc, const Tensor& input,
                             WireFormatLite::WriteFloatNoTag>(
               field_desc, input, message_index, size, output);
         default:
-          return;
+          return errors::DataLoss("Failed writing TYPE_FLOAT for ",
+                                  DataTypeString(dtype));
       }
     case WireFormatLite::TYPE_INT64:
       return WriteField(
           field_desc, input, message_index, size, output);
     case WireFormatLite::TYPE_UINT64:
-      return WriteField(
           field_desc, input, message_index, size, output);
     case WireFormatLite::TYPE_INT32:
-      return WriteField(
-          field_desc, input, message_index, size, output);
+      switch (dtype) {
+        case DataType::DT_INT64:
+          return WriteField(
+              field_desc, input, message_index, size, output);
+        case DataType::DT_INT32:
+          return WriteField(
+              field_desc, input, message_index, size, output);
+        default:
+          return errors::DataLoss("Failed writing TYPE_INT32 for ",
+                                  DataTypeString(dtype));
+      }
     case WireFormatLite::TYPE_FIXED64:
-      return WriteField(
           field_desc, input, message_index, size, output);
     case WireFormatLite::TYPE_FIXED32:
-      switch (tf_type) {
-        case DataType::DT_INT64:
-          return WriteField(
               field_desc, input, message_index, size, output);
-        case DataType::DT_INT32:
-          return WriteField(
               field_desc, input, message_index, size, output);
         default:
-          return;
+          return errors::DataLoss("Failed writing TYPE_FIXED32 for ",
+                                  DataTypeString(dtype));
       }
     case WireFormatLite::TYPE_BOOL:
       return WriteField(
           field_desc, input, message_index, size, output);
     case WireFormatLite::TYPE_UINT32:
-      switch (tf_type) {
-        case DataType::DT_INT64:
-          return WriteField(
               field_desc, input, message_index, size, output);
-        case DataType::DT_INT32:
-          return WriteField(
               field_desc, input, message_index, size, output);
         default:
-          return;
+          return errors::DataLoss("Failed writing TYPE_UINT32 for ",
+                                  DataTypeString(dtype));
       }
     case WireFormatLite::TYPE_ENUM:
       return WriteField(
           field_desc, input, message_index, size, output);
     case WireFormatLite::TYPE_SFIXED32:
-      return WriteField(
-          field_desc, input, message_index, size, output);
+      switch (dtype) {
+        case DataType::DT_INT64:
+          return WriteField(
+              field_desc, input, message_index, size, output);
+        case DataType::DT_INT32:
+          return WriteField(
+              field_desc, input, message_index, size, output);
+        default:
+          return errors::DataLoss("Failed writing TYPE_SFIXED32 for ",
+                                  DataTypeString(dtype));
+      }
     case WireFormatLite::TYPE_SFIXED64:
       return WriteField(
           field_desc, input, message_index, size, output);
     case WireFormatLite::TYPE_SINT32:
-      return WriteField(
-          field_desc, input, message_index, size, output);
+      switch (dtype) {
+        case DataType::DT_INT64:
+          return WriteField(
+              field_desc, input, message_index, size, output);
+        case DataType::DT_INT32:
+          return WriteField(
+              field_desc, input, message_index, size, output);
+        default:
+          return errors::DataLoss("Failed writing TYPE_SINT32 for ",
+                                  DataTypeString(dtype));
+      }
     case WireFormatLite::TYPE_SINT64:
       return WriteField(
@@ -392,42 +456,6 @@ void WriteField(const FieldDescriptor& field_desc, const Tensor& input,
   }
 }
 
-// Checks that a Protobuf field is compatible with a TensorFlow datatype.
-// This is separated from WriteField to lift it out of the inner loop.
-bool IsCompatibleType(const FieldDescriptor& field_desc, DataType tf_type) {
-  switch (field_desc.type()) {
-    case WireFormatLite::TYPE_DOUBLE:
-      return tf_type == DataType::DT_DOUBLE;
-    case WireFormatLite::TYPE_FLOAT:
-      return tf_type == DataType::DT_FLOAT || tf_type == DataType::DT_DOUBLE;
-    case WireFormatLite::TYPE_INT64:
-    case WireFormatLite::TYPE_SFIXED64:
-    case WireFormatLite::TYPE_SINT64:
-      return tf_type == DataType::DT_INT64;
-    case WireFormatLite::TYPE_UINT64:
-      return tf_type == DataType::DT_INT64;
-    case WireFormatLite::TYPE_INT32:
-    case WireFormatLite::TYPE_ENUM:
-    case WireFormatLite::TYPE_SFIXED32:
-    case WireFormatLite::TYPE_SINT32:
-      return tf_type == DataType::DT_INT32;
-    case WireFormatLite::TYPE_FIXED64:
-      return tf_type == DataType::DT_INT64;
-    case WireFormatLite::TYPE_FIXED32:
-    case WireFormatLite::TYPE_UINT32:
-      return tf_type == DataType::DT_INT64 || tf_type == DataType::DT_INT32;
-    case WireFormatLite::TYPE_BOOL:
-      return tf_type == DataType::DT_BOOL;
-    case WireFormatLite::TYPE_STRING:
-    case WireFormatLite::TYPE_GROUP:
-    case WireFormatLite::TYPE_MESSAGE:
-    case WireFormatLite::TYPE_BYTES:
-      return tf_type == DataType::DT_STRING;
-      // default: intentionally omitted in order to enable static checking.
-  }
-  return false;
-}
-
 class EncodeProtoOp : public OpKernel {
  public:
   explicit EncodeProtoOp(OpKernelConstruction* context) : OpKernel(context) {
@@ -475,14 +503,14 @@ class EncodeProtoOp : public OpKernel {
               });
   }
 
-  void Compute(OpKernelContext* cx) override {
+  void Compute(OpKernelContext* ctx) override {
     const Tensor* sizes_tensor;
-    OP_REQUIRES_OK(cx, cx->input("sizes", &sizes_tensor));
+    OP_REQUIRES_OK(ctx, ctx->input("sizes", &sizes_tensor));
 
     OpInputList values;
-    OP_REQUIRES_OK(cx, cx->input_list("values", &values));
+    OP_REQUIRES_OK(ctx, ctx->input_list("values", &values));
 
-    OP_REQUIRES(cx, field_descs_.size() == values.size(),
+    OP_REQUIRES(ctx, field_descs_.size() == values.size(),
                 errors::InvalidArgument(
                     "Length of inputs list must match field_names"));
 
@@ -493,12 +521,14 @@ class EncodeProtoOp : public OpKernel {
       const Tensor& v = values[i];
 
       // The type of each value tensor must match the corresponding field.
-      OP_REQUIRES(cx, IsCompatibleType(*field_descs_[i], v.dtype()),
-                  errors::InvalidArgument(
-                      "Incompatible type for field " + field_names_[i] +
-                          ".  Saw dtype: ",
-                      DataTypeString(v.dtype()),
-                      " but field type is: ", field_descs_[i]->type_name()));
+      OP_REQUIRES(
+          ctx,
+          proto_utils::IsCompatibleType(field_descs_[i]->type(), v.dtype()),
+          errors::InvalidArgument(
+              "Incompatible type for field " + field_names_[i] +
+                  ".  Saw dtype: ",
+              DataTypeString(v.dtype()),
+              " but field type is: ", field_descs_[i]->type_name()));
 
       // All value tensors must have the same shape prefix (i.e. batch size).
       TensorShape shape_prefix = v.shape();
@@ -507,14 +537,14 @@ class EncodeProtoOp : public OpKernel {
       // Do some initialization on the first input value. The rest will
       // have to match this one.
       if (i == 0) {
-        OP_REQUIRES(cx, v.dims() >= 1,
+        OP_REQUIRES(ctx, v.dims() >= 1,
                     errors::InvalidArgument(
                         "Expected value to be at least a vector, saw shape: ",
                         v.shape().DebugString()));
         common_prefix = shape_prefix;
         message_count = common_prefix.num_elements();
       } else {
-        OP_REQUIRES(cx, shape_prefix == common_prefix,
+        OP_REQUIRES(ctx, shape_prefix == common_prefix,
                     errors::InvalidArgument(
                         "Values must match up to the last dimension"));
       }
@@ -523,7 +553,7 @@ class EncodeProtoOp : public OpKernel {
     TensorShape expected_sizes_shape = common_prefix;
     expected_sizes_shape.AddDim(field_descs_.size());
 
-    OP_REQUIRES(cx, sizes_tensor->shape() == expected_sizes_shape,
+    OP_REQUIRES(ctx, sizes_tensor->shape() == expected_sizes_shape,
                 errors::InvalidArgument(
                     "sizes should be batch_size + [len(field_names)].  Saw: ",
                     sizes_tensor->shape().DebugString(),
@@ -536,12 +566,11 @@ class EncodeProtoOp : public OpKernel {
       int max_size = v.dim_size(v.dims() - 1);
 
       // The last dimension of a value tensor must be greater than the
-      // corresponding
-      // size in the sizes tensor.
+      // corresponding size in the sizes tensor.
       for (int message_index = 0; message_index < message_count;
            message_index++) {
         OP_REQUIRES(
-            cx, sizes(message_index, i) <= max_size,
+            ctx, sizes(message_index, i) <= max_size,
             errors::InvalidArgument(
                 "Size to write must not be larger than value tensor; but saw: ",
                 sizes(message_index, i), " > ", max_size, " at message ",
@@ -551,13 +580,13 @@ class EncodeProtoOp : public OpKernel {
 
     // This pointer is owned by the context.
     Tensor* output_tensor;
-    OP_REQUIRES_OK(cx, cx->allocate_output(0, common_prefix, &output_tensor));
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, common_prefix, &output_tensor));
 
     auto bufs = output_tensor->flat();
     for (int message_index = 0; message_index < message_count;
          message_index++) {
       // TODO(nix): possibly optimize allocation here by calling
-      //   bufs(message_index).reserve(DEFAULT_BUF_SIZE);
+      // `bufs(message_index).reserve(DEFAULT_BUF_SIZE)`.
       StringOutputStream output_string(&bufs(message_index));
       CodedOutputStream out(&output_string);
       // Write fields in ascending field_number order.
@@ -566,7 +595,8 @@ class EncodeProtoOp : public OpKernel {
         const Tensor& v = values[i];
         int size = sizes(message_index, i);
         if (!size) continue;
-        WriteField(field_desc, v, message_index, size, &out);
+        OP_REQUIRES_OK(ctx,
+                       WriteField(field_desc, v, message_index, size, &out));
       }
     }
   }
@@ -578,8 +608,8 @@ class EncodeProtoOp : public OpKernel {
   // Owned_desc_pool_ is null when using descriptor_source=local.
   std::unique_ptr owned_desc_pool_;
 
-  // Contains indices into field_names_, sorted by field number since
-  // that's the order of writing.
+  // Contains indices into field_names_, sorted by field number since that's the
+  // order of writing.
   std::vector sorted_field_index_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(EncodeProtoOp);
diff --git a/tensorflow/core/util/proto/BUILD b/tensorflow/core/util/proto/BUILD
index ade14ed162..7e549c7764 100644
--- a/tensorflow/core/util/proto/BUILD
+++ b/tensorflow/core/util/proto/BUILD
@@ -60,3 +60,13 @@ cc_library(
     ],
     alwayslink = 1,
 )
+
+cc_library(
+    name = "proto_utils",
+    srcs = ["proto_utils.cc"],
+    hdrs = ["proto_utils.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+    ],
+)
diff --git a/tensorflow/core/util/proto/decode.h b/tensorflow/core/util/proto/decode.h
index 74634a356a..cbcb203ee7 100644
--- a/tensorflow/core/util/proto/decode.h
+++ b/tensorflow/core/util/proto/decode.h
@@ -27,6 +27,7 @@ limitations under the License.
 #define TENSORFLOW_CORE_UTIL_PROTO_DECODE_H_
 
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -102,6 +103,16 @@ inline const uint8* ReadVarint32FromArray(const uint8* buffer, bool* ok,
 template 
 const uint8* ReadFromArray(const uint8* buf, TensorType* value);
 
+template <>
+inline const uint8* ReadFromArray(
+    const uint8* buf, int64* value) {
+  uint32 temp;
+  bool unused_ok;  // The Counting pass would have failed if this were corrupt.
+  buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
+  *value = static_cast(temp);
+  return buf;
+}
+
 template <>
 inline const uint8* ReadFromArray(
     const uint8* buf, int32* value) {
@@ -123,8 +134,8 @@ inline const uint8* ReadFromArray(
 }
 
 template <>
-inline const uint8* ReadFromArray(
-    const uint8* buf, int64* value) {
+inline const uint8* ReadFromArray(
+    const uint8* buf, uint64* value) {
   uint32 temp;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
@@ -133,22 +144,26 @@ inline const uint8* ReadFromArray(
 }
 
 template <>
-inline const uint8* ReadFromArray(
-    const uint8* buf, int32* value) {
-  uint32 temp;
+inline const uint8* ReadFromArray(
+    const uint8* buf, uint32* value) {
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
-  buf = ReadVarint32FromArray(buf, &unused_ok, &temp);
-  *value = WrapUnsignedAsSigned32(temp);
-  return buf;
+  return ReadVarint32FromArray(buf, &unused_ok, value);
+}
+
+template <>
+inline const uint8* ReadFromArray(
+    const uint8* buf, uint64* value) {
+  bool unused_ok;  // The Counting pass would have failed if this were corrupt.
+  return ReadVarint64FromArray(buf, &unused_ok, value);
 }
 
 template <>
-inline const uint8* ReadFromArray(
+inline const uint8* ReadFromArray(
     const uint8* buf, int64* value) {
   uint64 temp;
   bool unused_ok;  // The Counting pass would have failed if this were corrupt.
   buf = ReadVarint64FromArray(buf, &unused_ok, &temp);
-  *value = static_cast(temp);
+  *value = WireFormatLite::ZigZagDecode32(temp);
   return buf;
 }
 
@@ -173,8 +188,8 @@ inline const uint8* ReadFromArray(
 }
 
 template <>
-inline const uint8* ReadFromArray(
-    const uint8* buf, int64* value) {
+inline const uint8* ReadFromArray(
+    const uint8* buf, uint64* value) {
   uint32 temp;
   buf = WireFormatLite::ReadPrimitiveFromArray(
@@ -184,8 +199,8 @@ inline const uint8* ReadFromArray(
 }
 
 template <>
-inline const uint8* ReadFromArray(
-    const uint8* buf, int32* value) {
+inline const uint8* ReadFromArray(
+    const uint8* buf, uint32* value) {
   uint32 temp;
   buf = WireFormatLite::ReadPrimitiveFromArray(
@@ -195,8 +210,8 @@ inline const uint8* ReadFromArray(
 }
 
 template <>
-inline const uint8* ReadFromArray(
-    const uint8* buf, int64* value) {
+inline const uint8* ReadFromArray(
+    const uint8* buf, uint64* value) {
   protobuf_uint64 temp;
   buf = WireFormatLite::ReadPrimitiveFromArray(
@@ -205,6 +220,17 @@ inline const uint8* ReadFromArray(
   return buf;
 }
 
+template <>
+inline const uint8* ReadFromArray(
+    const uint8* buf, int64* value) {
+  int32 temp;
+  buf = WireFormatLite::ReadPrimitiveFromArray(
+      buf, &temp);
+  *value = temp;
+  return buf;
+}
+
 template <>
 inline const uint8* ReadFromArray(
     const uint8* buf, int32* value) {
@@ -232,6 +258,17 @@ inline const uint8* ReadFromArray(
       buf, value);
 }
 
+template <>
+inline const uint8* ReadFromArray(
+    const uint8* buf, double* value) {
+  float temp;
+  buf =
+      WireFormatLite::ReadPrimitiveFromArray(
+          buf, &temp);
+  *value = temp;
+  return buf;
+}
+
 template <>
 inline const uint8* ReadFromArray(
     const uint8* buf, double* value) {
@@ -334,48 +371,56 @@ inline Status ReadGroupBytes(CodedInputStream* input, int field_number,
 inline Status ReadValue(CodedInputStream* input,
                         WireFormatLite::FieldType field_type, int field_number,
                         DataType dtype, int index, void* datap) {
-  // Dispatch to the appropriately typed field reader based on the
-  // schema type.
+  // Dispatch to the appropriately typed field reader based on the schema type.
   switch (field_type) {
     case WireFormatLite::TYPE_DOUBLE:
       return ReadPrimitive(
           input, index, datap);
     case WireFormatLite::TYPE_FLOAT:
-      if (dtype == DataType::DT_FLOAT) {
-        return ReadPrimitive(
-            input, index, datap);
-      }
-      if (dtype == DataType::DT_DOUBLE) {
-        return ReadPrimitive(
-            input, index, datap);
+      switch (dtype) {
+        case DataType::DT_DOUBLE:
+          return ReadPrimitive(
+              input, index, datap);
+        case DataType::DT_FLOAT:
+          return ReadPrimitive(
+              input, index, datap);
+        default:
+          return errors::DataLoss("Failed reading TYPE_FLOAT for ",
+                                  DataTypeString(dtype));
       }
-      // Any case that reaches this point should have triggered an error
-      // already.
-      return errors::DataLoss("Failed reading TYPE_FLOAT");
     case WireFormatLite::TYPE_INT64:
       return ReadPrimitive(
           input, index, datap);
     case WireFormatLite::TYPE_UINT64:
-      return ReadPrimitive(
-          input, index, datap);
+      return ReadPrimitive(input, index, datap);
     case WireFormatLite::TYPE_INT32:
-      return ReadPrimitive(
-          input, index, datap);
+      switch (dtype) {
+        case DataType::DT_INT64:
+          return ReadPrimitive(
+              input, index, datap);
+        case DataType::DT_INT32:
+          return ReadPrimitive(
+              input, index, datap);
+        default:
+          return errors::DataLoss("Failed reading TYPE_INT32 for ",
+                                  DataTypeString(dtype));
+      }
     case WireFormatLite::TYPE_FIXED64:
-      return ReadPrimitive(input, index, datap);
     case WireFormatLite::TYPE_FIXED32:
-      if (dtype == DataType::DT_INT64) {
-        return ReadPrimitive(
-            input, index, datap);
-      }
-      if (dtype == DataType::DT_INT32) {
-        return ReadPrimitive(
-            input, index, datap);
+      switch (dtype) {
+        case DataType::DT_UINT64:
+          return ReadPrimitive(
+              input, index, datap);
+        case DataType::DT_UINT32:
+          return ReadPrimitive(
+              input, index, datap);
+        default:
+          return errors::DataLoss("Failed reading TYPE_FIXED32 for ",
+                                  DataTypeString(dtype));
       }
-      // Any case that reaches this point should have triggered an error
-      // already.
-      return errors::DataLoss("Failed reading TYPE_FIXED32");
     case WireFormatLite::TYPE_BOOL:
       return ReadPrimitive(input, index,
                                                                   datap);
@@ -388,29 +433,47 @@ inline Status ReadValue(CodedInputStream* input,
     case WireFormatLite::TYPE_BYTES:
       return ReadBytes(input, index, datap);
     case WireFormatLite::TYPE_UINT32:
-      if (dtype == DataType::DT_INT64) {
-        return ReadPrimitive(
-            input, index, datap);
+      switch (dtype) {
+        case DataType::DT_UINT64:
+          return ReadPrimitive(
+              input, index, datap);
+        case DataType::DT_UINT32:
+          return ReadPrimitive(
+              input, index, datap);
+        default:
+          return errors::DataLoss("Failed reading TYPE_UINT32 for ",
+                                  DataTypeString(dtype));
       }
-      if (dtype == DataType::DT_INT32) {
-        return ReadPrimitive(
-            input, index, datap);
-      }
-      // Any case that reaches this point should have triggered an error
-      // already.
-      return errors::DataLoss("Failed reading TYPE_UINT32");
     case WireFormatLite::TYPE_ENUM:
       return ReadPrimitive(
           input, index, datap);
     case WireFormatLite::TYPE_SFIXED32:
-      return ReadPrimitive(
-          input, index, datap);
+      switch (dtype) {
+        case DataType::DT_INT64:
+          return ReadPrimitive(
+              input, index, datap);
+        case DataType::DT_INT32:
+          return ReadPrimitive(
+              input, index, datap);
+        default:
+          return errors::DataLoss("Failed reading TYPE_SFIXED32 for ",
+                                  DataTypeString(dtype));
+      }
     case WireFormatLite::TYPE_SFIXED64:
       return ReadPrimitive(input, index, datap);
     case WireFormatLite::TYPE_SINT32:
-      return ReadPrimitive(
-          input, index, datap);
+      switch (dtype) {
+        case DataType::DT_INT64:
+          return ReadPrimitive(
+              input, index, datap);
+        case DataType::DT_INT32:
+          return ReadPrimitive(
+              input, index, datap);
+        default:
+          return errors::DataLoss("Failed reading TYPE_SINT32 for ",
+                                  DataTypeString(dtype));
+      }
     case WireFormatLite::TYPE_SINT64:
       return ReadPrimitive(
           input, index, datap);
@@ -425,47 +488,66 @@ inline Status ReadPackedFromArray(const void* buf, size_t buf_size,
                                   const WireFormatLite::FieldType field_type,
                                   const int field_number, const DataType dtype,
                                   const int stride, int* index, void* data) {
-  // Dispatch to the appropriately typed field reader based on the
-  // schema type.
+  // Dispatch to the appropriately typed field reader based on the schema type.
   switch (field_type) {
     case WireFormatLite::TYPE_DOUBLE:
       *index += ReadPackedPrimitives(
           buf, buf_size, *index, stride, data);
       return Status::OK();
     case WireFormatLite::TYPE_FLOAT:
-      *index += ReadPackedPrimitives(
-          buf, buf_size, *index, stride, data);
-      return Status::OK();
+      switch (dtype) {
+        case DataType::DT_DOUBLE:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        case DataType::DT_FLOAT:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        default:
+          return errors::DataLoss("Failed reading TYPE_FLOAT for ",
+                                  DataTypeString(dtype));
+      }
     case WireFormatLite::TYPE_INT64:
       *index += ReadPackedPrimitives(
           buf, buf_size, *index, stride, data);
       return Status::OK();
     case WireFormatLite::TYPE_UINT64:
-      *index += ReadPackedPrimitives(
+      *index += ReadPackedPrimitives(
           buf, buf_size, *index, stride, data);
       return Status::OK();
     case WireFormatLite::TYPE_INT32:
-      *index += ReadPackedPrimitives(
-          buf, buf_size, *index, stride, data);
-      return Status::OK();
+      switch (dtype) {
+        case DataType::DT_INT64:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        case DataType::DT_INT32:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        default:
+          return errors::DataLoss("Failed reading TYPE_INT32 for ",
+                                  DataTypeString(dtype));
+      }
     case WireFormatLite::TYPE_FIXED64:
-      *index += ReadPackedPrimitives(
+      *index += ReadPackedPrimitives(
           buf, buf_size, *index, stride, data);
       return Status::OK();
     case WireFormatLite::TYPE_FIXED32:
-      if (dtype == DataType::DT_INT64) {
-        *index += ReadPackedPrimitives(
-            buf, buf_size, *index, stride, data);
-        return Status::OK();
-      }
-      if (dtype == DataType::DT_INT32) {
-        *index += ReadPackedPrimitives(
-            buf, buf_size, *index, stride, data);
-        return Status::OK();
+      switch (dtype) {
+        case DataType::DT_UINT64:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        case DataType::DT_UINT32:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        default:
+          return errors::DataLoss("Failed reading TYPE_FIXED32 for ",
+                                  DataTypeString(dtype));
       }
-      // Any case that reaches this point should have triggered an error
-      // already.
-      return errors::DataLoss("Failed reading TYPE_FIXED32");
     case WireFormatLite::TYPE_BOOL:
       *index += ReadPackedPrimitives(
           buf, buf_size, *index, stride, data);
@@ -476,38 +558,56 @@ inline Status ReadPackedFromArray(const void* buf, size_t buf_size,
     case WireFormatLite::TYPE_BYTES:
       return errors::DataLoss("Non-primitive type encountered as packed");
     case WireFormatLite::TYPE_UINT32:
-      if (dtype == DataType::DT_INT64) {
-        *index += ReadPackedPrimitives(
-            buf, buf_size, *index, stride, data);
-        return Status::OK();
+      switch (dtype) {
+        case DataType::DT_UINT64:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        case DataType::DT_UINT32:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        default:
+          return errors::DataLoss("Failed reading TYPE_UINT32 for ",
+                                  DataTypeString(dtype));
       }
-      if (dtype == DataType::DT_INT32) {
-        *index += ReadPackedPrimitives(
-            buf, buf_size, *index, stride, data);
-        return Status::OK();
-      }
-      // Any case that reaches this point should have triggered an error
-      // already.
-      return errors::DataLoss("Failed reading TYPE_UINT32");
     case WireFormatLite::TYPE_ENUM:
       *index += ReadPackedPrimitives(
           buf, buf_size, *index, stride, data);
       return Status::OK();
     case WireFormatLite::TYPE_SFIXED32:
-      *index += ReadPackedPrimitives(
-          buf, buf_size, *index, stride, data);
-      return Status::OK();
-
+      switch (dtype) {
+        case DataType::DT_INT64:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        case DataType::DT_INT32:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        default:
+          return errors::DataLoss("Failed reading TYPE_INT32 for ",
+                                  DataTypeString(dtype));
+      }
     case WireFormatLite::TYPE_SFIXED64:
       *index += ReadPackedPrimitives(
           buf, buf_size, *index, stride, data);
       return Status::OK();
 
     case WireFormatLite::TYPE_SINT32:
-      *index += ReadPackedPrimitives(
-          buf, buf_size, *index, stride, data);
-      return Status::OK();
-
+      switch (dtype) {
+        case DataType::DT_INT64:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        case DataType::DT_INT32:
+          *index += ReadPackedPrimitives(
+              buf, buf_size, *index, stride, data);
+          return Status::OK();
+        default:
+          return errors::DataLoss("Failed reading TYPE_SINT32 for ",
+                                  DataTypeString(dtype));
+      }
     case WireFormatLite::TYPE_SINT64:
       *index += ReadPackedPrimitives(
           buf, buf_size, *index, stride, data);
diff --git a/tensorflow/core/util/proto/proto_utils.cc b/tensorflow/core/util/proto/proto_utils.cc
new file mode 100644
index 0000000000..201f05a129
--- /dev/null
+++ b/tensorflow/core/util/proto/proto_utils.cc
@@ -0,0 +1,70 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/platform/protobuf.h"
+
+#include "tensorflow/core/util/proto/proto_utils.h"
+
+namespace tensorflow {
+namespace proto_utils {
+
+using tensorflow::protobuf::FieldDescriptor;
+using tensorflow::protobuf::internal::WireFormatLite;
+
+bool IsCompatibleType(FieldDescriptor::Type field_type, DataType dtype) {
+  switch (field_type) {
+    case WireFormatLite::TYPE_DOUBLE:
+      return dtype == tensorflow::DT_DOUBLE;
+    case WireFormatLite::TYPE_FLOAT:
+      return dtype == tensorflow::DT_FLOAT || dtype == tensorflow::DT_DOUBLE;
+    case WireFormatLite::TYPE_INT64:
+      return dtype == tensorflow::DT_INT64;
+    case WireFormatLite::TYPE_UINT64:
+      return dtype == tensorflow::DT_UINT64;
+    case WireFormatLite::TYPE_INT32:
+      return dtype == tensorflow::DT_INT32 || dtype == tensorflow::DT_INT64;
+    case WireFormatLite::TYPE_FIXED64:
+      return dtype == tensorflow::DT_UINT64;
+    case WireFormatLite::TYPE_FIXED32:
+      return dtype == tensorflow::DT_UINT32 || dtype == tensorflow::DT_UINT64;
+    case WireFormatLite::TYPE_BOOL:
+      return dtype == tensorflow::DT_BOOL;
+    case WireFormatLite::TYPE_STRING:
+      return dtype == tensorflow::DT_STRING;
+    case WireFormatLite::TYPE_GROUP:
+      return dtype == tensorflow::DT_STRING;
+    case WireFormatLite::TYPE_MESSAGE:
+      return dtype == tensorflow::DT_STRING;
+    case WireFormatLite::TYPE_BYTES:
+      return dtype == tensorflow::DT_STRING;
+    case WireFormatLite::TYPE_UINT32:
+      return dtype == tensorflow::DT_UINT32 || dtype == tensorflow::DT_UINT64;
+    case WireFormatLite::TYPE_ENUM:
+      return dtype == tensorflow::DT_INT32;
+    case WireFormatLite::TYPE_SFIXED32:
+      return dtype == tensorflow::DT_INT32 || dtype == tensorflow::DT_INT64;
+    case WireFormatLite::TYPE_SFIXED64:
+      return dtype == tensorflow::DT_INT64;
+    case WireFormatLite::TYPE_SINT32:
+      return dtype == tensorflow::DT_INT32 || dtype == tensorflow::DT_INT64;
+    case WireFormatLite::TYPE_SINT64:
+      return dtype == tensorflow::DT_INT64;
+      // default: intentionally omitted in order to enable static checking.
+  }
+}
+
+}  // namespace proto_utils
+}  // namespace tensorflow
diff --git a/tensorflow/core/util/proto/proto_utils.h b/tensorflow/core/util/proto/proto_utils.h
new file mode 100644
index 0000000000..d5e0b9006c
--- /dev/null
+++ b/tensorflow/core/util/proto/proto_utils.h
@@ -0,0 +1,33 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_UTIL_PROTO_PROTO_UTILS_H_
+#define TENSORFLOW_CORE_UTIL_PROTO_PROTO_UTILS_H_
+
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/platform/protobuf.h"
+
+namespace tensorflow {
+namespace proto_utils {
+
+using tensorflow::protobuf::FieldDescriptor;
+
+// Returns true if the proto field type can be converted to the tensor dtype.
+bool IsCompatibleType(FieldDescriptor::Type field_type, DataType dtype);
+
+}  // namespace proto_utils
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_UTIL_PROTO_PROTO_UTILS_H_
-- 
cgit v1.2.3


From 3e5dbd6a34e3a069f27e33de341e5f8d4cfdd7b4 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Wed, 18 Jul 2018 22:24:34 -0700
Subject: Fix a bug where plugin factory didn't reset the unique_ptr but
 release it (thus cause mem leak).

---
 tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc b/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc
index 2bc591484d..cccc912262 100644
--- a/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc
+++ b/tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.cc
@@ -65,9 +65,6 @@ bool PluginFactoryTensorRT::RegisterPlugin(
 
 void PluginFactoryTensorRT::DestroyPlugins() {
   tensorflow::mutex_lock lock(instance_m_);
-  for (auto& owned_plugin_ptr : owned_plugins_) {
-    owned_plugin_ptr.release();
-  }
   owned_plugins_.clear();
 }
 
-- 
cgit v1.2.3


From 9a87590da3876b38af946ab363c9d94b8d46e0f9 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Wed, 18 Jul 2018 22:43:41 -0700
Subject: Temporarily fix the undefined symbols problem

---
 tensorflow/core/graph/algorithm.cc | 16 ++++++++++++++++
 tensorflow/core/graph/algorithm.h  | 22 ++++++++++++++++------
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/graph/algorithm.cc b/tensorflow/core/graph/algorithm.cc
index 9b4200e0b4..548096078f 100644
--- a/tensorflow/core/graph/algorithm.cc
+++ b/tensorflow/core/graph/algorithm.cc
@@ -23,6 +23,12 @@ limitations under the License.
 
 namespace tensorflow {
 
+void DFS(const Graph& g, const std::function& enter,
+         const std::function& leave,
+         const NodeComparator& stable_comparator) {
+  DFS(g, enter, leave, stable_comparator, {});
+}
+
 void DFS(const Graph& g, const std::function& enter,
          const std::function& leave,
          const NodeComparator& stable_comparator,
@@ -162,6 +168,11 @@ void ReverseDFSFrom(const Graph& g, gtl::ArraySlice start,
   ReverseDFSFromHelper(g, start, enter, leave, stable_comparator);
 }
 
+void GetPostOrder(const Graph& g, std::vector* order,
+                  const NodeComparator& stable_comparator) {
+  GetPostOrder(g, order, stable_comparator, {});
+}
+
 void GetPostOrder(const Graph& g, std::vector* order,
                   const NodeComparator& stable_comparator,
                   const EdgeFilter& edge_filter) {
@@ -170,6 +181,11 @@ void GetPostOrder(const Graph& g, std::vector* order,
       edge_filter);
 }
 
+void GetReversePostOrder(const Graph& g, std::vector* order,
+                         const NodeComparator& stable_comparator) {
+  GetReversePostOrder(g, order, stable_comparator, {});
+}
+
 void GetReversePostOrder(const Graph& g, std::vector* order,
                          const NodeComparator& stable_comparator,
                          const EdgeFilter& edge_filter) {
diff --git a/tensorflow/core/graph/algorithm.h b/tensorflow/core/graph/algorithm.h
index 5bbbc6f6dc..7d8a3456e4 100644
--- a/tensorflow/core/graph/algorithm.h
+++ b/tensorflow/core/graph/algorithm.h
@@ -52,8 +52,12 @@ struct NodeComparatorName {
 // If edge_filter is set then ignores edges for which edge_filter returns false.
 extern void DFS(const Graph& g, const std::function& enter,
                 const std::function& leave,
-                const NodeComparator& stable_comparator = {},
-                const EdgeFilter& edge_filter = {});
+                const NodeComparator& stable_comparator = {});
+
+extern void DFS(const Graph& g, const std::function& enter,
+                const std::function& leave,
+                const NodeComparator& stable_comparator,
+                const EdgeFilter& edge_filter);
 
 // Perform a reverse depth-first-search on g starting at the sink node.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
@@ -91,8 +95,11 @@ extern void ReverseDFSFrom(const Graph& g, gtl::ArraySlice start,
 //
 // REQUIRES: order is not NULL.
 void GetPostOrder(const Graph& g, std::vector* order,
-                  const NodeComparator& stable_comparator = {},
-                  const EdgeFilter& edge_filter = {});
+                  const NodeComparator& stable_comparator = {});
+
+void GetPostOrder(const Graph& g, std::vector* order,
+                  const NodeComparator& stable_comparator,
+                  const EdgeFilter& edge_filter);
 
 // Stores in *order the reverse post-order numbering of all nodes
 // If stable_comparator is set, a stable ordering of visit is achieved by
@@ -100,8 +107,11 @@ void GetPostOrder(const Graph& g, std::vector* order,
 //
 // If edge_filter is set then ignores edges for which edge_filter returns false.
 void GetReversePostOrder(const Graph& g, std::vector* order,
-                         const NodeComparator& stable_comparator = {},
-                         const EdgeFilter& edge_filter = {});
+                         const NodeComparator& stable_comparator);
+
+void GetReversePostOrder(const Graph& g, std::vector* order,
+                         const NodeComparator& stable_comparator,
+                         const EdgeFilter& edge_filter);
 
 // Prune nodes in "g" that are not in some path from the source node
 // to any node in 'nodes'. Returns true if changes were made to the graph.
-- 
cgit v1.2.3


From 694dd61fcdfb2019af3e28b4151d93bdd690c94a Mon Sep 17 00:00:00 2001
From: cheerss 
Date: Thu, 19 Jul 2018 14:57:43 +0800
Subject: Update keras.md

tf.keras.models should load model configuration with `model_from_json` or `model_from_yaml`, not `from_json` or `from_yaml`
---
 tensorflow/docs_src/guide/keras.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/docs_src/guide/keras.md b/tensorflow/docs_src/guide/keras.md
index 1d846df104..2330fa03c7 100644
--- a/tensorflow/docs_src/guide/keras.md
+++ b/tensorflow/docs_src/guide/keras.md
@@ -467,13 +467,13 @@ JSON and YAML serialization formats:
 json_string = model.to_json()
 
 # Recreate the model (freshly initialized)
-fresh_model = keras.models.from_json(json_string)
+fresh_model = keras.models.model_from_json(json_string)
 
 # Serializes a model to YAML format
 yaml_string = model.to_yaml()
 
 # Recreate the model
-fresh_model = keras.models.from_yaml(yaml_string)
+fresh_model = keras.models.model_from_yaml(yaml_string)
 ```
 
 Caution: Subclassed models are not serializable because their architecture is
-- 
cgit v1.2.3


From a847fc55483a6300079e14f3a8c05963fc2c3337 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Wed, 18 Jul 2018 23:58:55 -0700
Subject: Fix deserializeCudaEngine to take the real plugin factory pointer,
 introduced by #19871

---
 tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 54009179a8..646d62483f 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -15,9 +15,11 @@ limitations under the License.
 #include "tensorflow/contrib/tensorrt/kernels/trt_engine_op.h"
 
 #include 
+
 #include "tensorflow/contrib/tensorrt/convert/convert_nodes.h"
 #include "tensorflow/contrib/tensorrt/convert/utils.h"
 #include "tensorflow/contrib/tensorrt/log/trt_logger.h"
+#include "tensorflow/contrib/tensorrt/plugin/trt_plugin_factory.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
 #include "tensorflow/core/framework/graph_to_functiondef.h"
@@ -457,7 +459,8 @@ TRTEngineOp::EngineCtxPair& TRTEngineOp::GetEngine(int batch_size,
 #endif
     TrtUniquePtrType static_engine(
         infer->deserializeCudaEngine(serialized_segment_.c_str(),
-                                     serialized_segment_.size(), nullptr));
+                                     serialized_segment_.size(),
+                                     PluginFactoryTensorRT::GetInstance()));
     auto raw_static_engine = static_engine.get();
     const auto max_batch_size = raw_static_engine->getMaxBatchSize();
     engine_map_[max_batch_size] = {
-- 
cgit v1.2.3


From 6e94fd7b5eebc506202e1ae3b6c0de73e3b727bf Mon Sep 17 00:00:00 2001
From: Adrian Kuegel 
Date: Thu, 19 Jul 2018 00:58:11 -0700
Subject: Move Bitonic sort emitter logic into a helper file.

This will allow to reuse it for the CPU backend.
Also move the method EmitOperandArrayLoopNest to the LoopNest class.
This makes more sense than having it as part of IrEmitter.

PiperOrigin-RevId: 205200030
---
 tensorflow/compiler/xla/service/gpu/BUILD          |   1 +
 tensorflow/compiler/xla/service/gpu/ir_emitter.cc  | 203 +--------------------
 tensorflow/compiler/xla/service/gpu/ir_emitter.h   |  11 --
 tensorflow/compiler/xla/service/llvm_ir/BUILD      |  14 ++
 .../compiler/xla/service/llvm_ir/llvm_loop.cc      |  30 +++
 .../compiler/xla/service/llvm_ir/llvm_loop.h       |  11 ++
 .../compiler/xla/service/llvm_ir/sort_util.cc      | 201 ++++++++++++++++++++
 .../compiler/xla/service/llvm_ir/sort_util.h       |  34 ++++
 8 files changed, 298 insertions(+), 207 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
 create mode 100644 tensorflow/compiler/xla/service/llvm_ir/sort_util.h

diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index a043795a21..ca39797e81 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -170,6 +170,7 @@ cc_library(
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_loop",
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
         "//tensorflow/compiler/xla/service/llvm_ir:loop_emitter",
+        "//tensorflow/compiler/xla/service/llvm_ir:sort_util",
         "//tensorflow/compiler/xla/service/llvm_ir:tuple_ops",
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_no_cuda",
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
index a08b72e3af..449a18e710 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/sort_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/service/name_uniquer.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -44,7 +45,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/compiler/xla/window_util.h"
-#include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/core/errors.h"
 
 namespace xla {
@@ -125,135 +125,14 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) {
 }
 
 Status IrEmitter::HandleSort(HloInstruction* sort) {
-  auto keys = sort->operand(0);
   auto values = sort->operand_count() > 1 ? sort->operand(1) : nullptr;
   if (values != nullptr) {
     // TODO(b/26783907): Also sort the values by their corresponding key.
     return Unimplemented("Key/Value Sort is not implemented on GPU");
   }
   int dimension_to_sort = sort->dimensions(0);
-  const llvm_ir::IrArray& keys_array = GetIrArray(*keys, *sort);
-  const llvm_ir::IrArray& target_array = GetIrArray(*sort, *sort);
-
-  const Shape& keys_shape = keys->shape();
-
-  // TODO(b/26783907): This case can probably be avoided with the Algebraic
-  // Simplifier.
-  if (ShapeUtil::IsScalar(keys_shape)) {
-    return Status::OK();
-  }
-
-  // Create loop nests which loop through the operand dimensions. The sort
-  // dimension is handled in three separate innermost loops which perform the
-  // sorting.
-  llvm_ir::ForLoopNest loop_nest(IrName(sort), &ir_builder_);
-  llvm_ir::IrArray::Index keys_index = EmitOperandArrayLoopNest(
-      keys_array, dimension_to_sort, "keys", &loop_nest);
-
-  // 'compare_keys_index' is the index of the element that 'keys_index' should
-  // be compared to.
-  llvm_ir::IrArray::Index compare_keys_index(keys_index.GetType());
-  for (size_t dimension = 0; dimension < keys_index.size(); ++dimension) {
-    if (dimension != dimension_to_sort) {
-      compare_keys_index.push_back(keys_index[dimension]);
-    } else {
-      compare_keys_index.push_back(nullptr);
-    }
-  }
-
-  // Create the sorting loops which do the sorting.
-  int64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort);
-  std::unique_ptr stages_loop = loop_nest.AddLoop(
-      /*start_index=*/0,
-      /*end_index=*/
-      tensorflow::Log2Ceiling64(dimension_to_sort_bound),
-      /*suffix=*/"sort_stages");
-  std::unique_ptr mask_loop = loop_nest.AddLoop(
-      /*suffix=*/"mask",
-      /*start_index=*/keys_index.GetConstantWithIndexType(0),
-      /*end_index=*/stages_loop->GetIndVarValue());
-  std::unique_ptr compare_loop = loop_nest.AddLoop(
-      /*start_index=*/0,
-      /*end_index=*/dimension_to_sort_bound,
-      /*suffix=*/"compare");
-
-  // Naive C++ code for the inner loops (without parallelization):
-  //
-  // for (int64 stage = 0; stage < Log2Ceiling(dimension_to_sort_bound);
-  //     ++stage) {
-  //   int64 first_xor_mask = (1LL << (stage + 1)) - 1;
-  //   for (int64 i = 0; i < dimension_to_sort_bound; ++i) {
-  //     int64 j = i ^ first_xor_mask;
-  //     if (i < j && j < dimension_to_sort_bound) {
-  //       int64 min_key = std::min(keys[i], keys[j]);
-  //       keys[j] = std::max(keys[i], keys[j]);
-  //       keys[i] = min_key;
-  //     }
-  //   }
-  //   for (int64 mask = 0; mask < stage; ++mask) {
-  //     int64 later_xor_mask = (1LL << (stage - (mask + 1));
-  //     for (int64 i = 0; i < dimension_to_sort_bound; ++i) {
-  //       int64 j = i ^ later_xor_mask;
-  //       if (i < j && j < dimension_to_sort_bound) {
-  //         int64 min_key = std::min(keys[i], keys[j]);
-  //         keys[j] = std::max(keys[i], keys[j]);
-  //         keys[i] = min_key;
-  //       }
-  //     }
-  //   }
-  // }
-  //
-  // This follows the algorithm described on Wikipedia:
-  // https://en.wikipedia.org/wiki/Bitonic_sorter
-
-  SetToFirstInsertPoint(stages_loop->GetBodyBasicBlock(), &ir_builder_);
-  // The first xor mask of a stage is 2^(stage + 1) - 1.
-  auto first_xor_mask = ir_builder_.CreateSub(
-      ir_builder_.CreateShl(
-          keys_index.GetConstantWithIndexType(1),
-          ir_builder_.CreateAdd(stages_loop->GetIndVarValue(),
-                                keys_index.GetConstantWithIndexType(1))),
-      keys_index.GetConstantWithIndexType(1));
-  std::unique_ptr first_compare_loop =
-      llvm_ir::ForLoop::EmitForLoop(
-          /*prefix=*/"first_compare",
-          /*start_index=*/keys_index.GetConstantWithIndexType(0),
-          /*end_index=*/
-          keys_index.GetConstantWithIndexType(
-              keys_shape.dimensions(dimension_to_sort)),
-          /*step=*/keys_index.GetConstantWithIndexType(1),
-          /*ir_builder=*/&ir_builder_);
-
-  SetToFirstInsertPoint(first_compare_loop->GetBodyBasicBlock(), &ir_builder_);
-  // 'first_compare_loop' iterates through the 'dimension_to_sort'.
-  keys_index[dimension_to_sort] = first_compare_loop->GetIndVarValue();
-  compare_keys_index[dimension_to_sort] = ir_builder_.CreateXor(
-      first_compare_loop->GetIndVarValue(), first_xor_mask);
-  EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index,
-                  target_array);
-
-  SetToFirstInsertPoint(compare_loop->GetPreheaderBasicBlock(), &ir_builder_);
-  // The later masks of a stage are 2^(stage - (mask_loop_ind_var + 1)).
-  auto later_xor_mask = ir_builder_.CreateShl(
-      keys_index.GetConstantWithIndexType(1),
-      ir_builder_.CreateSub(
-          stages_loop->GetIndVarValue(),
-          ir_builder_.CreateAdd(mask_loop->GetIndVarValue(),
-                                keys_index.GetConstantWithIndexType(1))));
-
-  SetToFirstInsertPoint(compare_loop->GetBodyBasicBlock(), &ir_builder_);
-  // 'compare_loop' iterates through the 'dimension_to_sort'.
-  keys_index[dimension_to_sort] = compare_loop->GetIndVarValue();
-  compare_keys_index[dimension_to_sort] =
-      ir_builder_.CreateXor(compare_loop->GetIndVarValue(), later_xor_mask);
-  EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index,
-                  target_array);
-
-  // Set the IR builder insert point to the exit basic block of the outer most
-  // loop. This ensures later instructions are inserted after this loop nest.
-  ir_builder_.SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
-
-  return Status::OK();
+  return llvm_ir::EmitSortInPlace(dimension_to_sort, GetIrArray(*sort, *sort),
+                                  IrName(sort), &ir_builder_);
 }
 
 Status IrEmitter::HandleSend(HloInstruction*) {
@@ -527,44 +406,6 @@ Status IrEmitter::EmitAtomicOperationUsingCAS(const HloComputation& computation,
   return Status::OK();
 }
 
-void IrEmitter::EmitCompareLoop(
-    int64 dimension_to_sort, const llvm_ir::IrArray::Index& keys_index,
-    const llvm_ir::IrArray::Index& compare_keys_index,
-    const llvm_ir::IrArray& keys_array) {
-  // TODO(b/26783907): parallelize this loop.
-
-  // if (is_smaller_index &&
-  //     compare_keys[dimension_to_sort] < dimension_to_sort_bound)
-  llvm::Value* is_smaller_index = ir_builder_.CreateICmpSLT(
-      keys_index[dimension_to_sort], compare_keys_index[dimension_to_sort]);
-  int64 dimension_to_sort_bound =
-      keys_array.GetShape().dimensions(dimension_to_sort);
-  auto if_data = llvm_ir::EmitIfThenElse(
-      ir_builder_.CreateAnd(
-          is_smaller_index,
-          ir_builder_.CreateICmpSLT(
-              compare_keys_index[dimension_to_sort],
-              keys_index.GetConstantWithIndexType(dimension_to_sort_bound))),
-      "smaller_comparison_index", &ir_builder_, /*emit_else=*/false);
-  SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
-  auto key1 = keys_array.EmitReadArrayElement(keys_index, &ir_builder_);
-  auto key2 = keys_array.EmitReadArrayElement(compare_keys_index, &ir_builder_);
-  auto key_type = keys_array.GetShape().element_type();
-  auto comparison =
-      primitive_util::IsFloatingPointType(key_type)
-          // TODO(b/26783907): Figure out how to handle NaNs.
-          ? ir_builder_.CreateFCmp(llvm::FCmpInst::FCMP_ULT, key1, key2)
-          : ir_builder_.CreateICmp(
-                primitive_util::IsSignedIntegralType(key_type)
-                    ? llvm::ICmpInst::ICMP_SLT
-                    : llvm::ICmpInst::ICMP_ULT,
-                key1, key2);
-  auto min_key = ir_builder_.CreateSelect(comparison, key1, key2);
-  auto max_key = ir_builder_.CreateSelect(comparison, key2, key1);
-  keys_array.EmitWriteArrayElement(keys_index, min_key, &ir_builder_);
-  keys_array.EmitWriteArrayElement(compare_keys_index, max_key, &ir_builder_);
-}
-
 Status IrEmitter::EmitAtomicOperationForNestedComputation(
     const HloComputation& computation, llvm::Value* output_address,
     llvm::Value* source_address) {
@@ -691,10 +532,10 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
   // operand dimensions. The reduction dimension of the LHS and RHS are handled
   // in a separate innermost loop which performs the sum of products.
   llvm_ir::ForLoopNest loop_nest(IrName(dot), &ir_builder_);
-  llvm_ir::IrArray::Index lhs_index = EmitOperandArrayLoopNest(
-      lhs_array, lhs_reduction_dimension, "lhs", &loop_nest);
-  llvm_ir::IrArray::Index rhs_index = EmitOperandArrayLoopNest(
-      rhs_array, rhs_reduction_dimension, "rhs", &loop_nest);
+  llvm_ir::IrArray::Index lhs_index = loop_nest.EmitOperandArrayLoopNest(
+      lhs_array, /*dimension_to_skip=*/lhs_reduction_dimension, "lhs");
+  llvm_ir::IrArray::Index rhs_index = loop_nest.EmitOperandArrayLoopNest(
+      rhs_array, /*dimension_to_skip=*/rhs_reduction_dimension, "rhs");
 
   // Create the reduction loop which does the sum of products reduction.
   std::unique_ptr reduction_loop = loop_nest.AddLoop(
@@ -943,36 +784,6 @@ Status IrEmitter::HandleBatchNormGrad(HloInstruction*) {
       "to a cudnn CustomCall using CudnnBatchNormRewriter.");
 }
 
-llvm_ir::IrArray::Index IrEmitter::EmitOperandArrayLoopNest(
-    const llvm_ir::IrArray& operand_array, int64 reduction_dimension,
-    tensorflow::StringPiece name_suffix, llvm_ir::ForLoopNest* loop_nest) {
-  // Prepares the dimension list we will use to emit the loop nest. Outermost
-  // loops are added first. Add loops in major-to-minor order, and skip the
-  // reduction dimension.
-  std::vector dimensions;
-  const Shape& shape = operand_array.GetShape();
-  for (int i = 0; i < LayoutUtil::MinorToMajor(shape).size(); ++i) {
-    int64 dimension = LayoutUtil::Major(shape.layout(), i);
-    if (dimension != reduction_dimension) {
-      dimensions.push_back(dimension);
-    }
-  }
-
-  // Create loop nest with one for-loop for each dimension of the
-  // output.
-  llvm_ir::IrArray::Index index =
-      loop_nest->AddLoopsForShapeOnDimensions(shape, dimensions, name_suffix);
-  // Verify every dimension except the reduction dimension was set in the index.
-  for (size_t dimension = 0; dimension < index.size(); ++dimension) {
-    if (dimension == reduction_dimension) {
-      DCHECK_EQ(nullptr, index[dimension]);
-    } else {
-      DCHECK_NE(nullptr, index[dimension]);
-    }
-  }
-  return index;
-}
-
 StatusOr IrEmitter::ComputeNestedElement(
     const HloComputation& computation,
     tensorflow::gtl::ArraySlice parameter_elements) {
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
index e9ad4a752b..77e48d729c 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
@@ -171,17 +171,6 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   const HloModuleConfig& hlo_module_config_;
 
  private:
-  // Emits a series of nested loops for iterating over an operand array in the
-  // dot operation. Loops are constructed in major to minor dimension layout
-  // order. No loop is emitted for the given reduction_dimension. The function
-  // returns an IrArray index for the given operand_array containing the indvars
-  // of the loops. All dimensions of the index are filled except for the
-  // reduction dimension. name_suffix is the string to append to the names of
-  // LLVM constructs (eg, basic blocks) constructed by this method.
-  llvm_ir::IrArray::Index EmitOperandArrayLoopNest(
-      const llvm_ir::IrArray& operand_array, int64 reduction_dimension,
-      tensorflow::StringPiece name_suffix, llvm_ir::ForLoopNest* loop_nest);
-
   // A helper method for EmitAtomicOperationForNestedComputation. Certain
   // computations, such as floating-point addition and integer maximization, can
   // be simply implemented using an LLVM atomic instruction. If "computation" is
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index c14a5bfb53..462be543bc 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -180,6 +180,20 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "sort_util",
+    srcs = ["sort_util.cc"],
+    hdrs = ["sort_util.h"],
+    deps = [
+        ":ir_array",
+        ":llvm_loop",
+        ":llvm_util",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/core:lib",
+        "@llvm//:core",
+    ],
+)
+
 cc_library(
     name = "tuple_ops",
     srcs = ["tuple_ops.cc"],
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc
index c9ae7d3afd..1227534779 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc
@@ -262,5 +262,35 @@ IrArray::Index ForLoopNest::AddLoopsForShapeOnDimensions(
   return index;
 }
 
+IrArray::Index ForLoopNest::EmitOperandArrayLoopNest(
+    const llvm_ir::IrArray& operand_array, int64 dimension_to_skip,
+    tensorflow::StringPiece name_suffix) {
+  // Prepares the dimension list we will use to emit the loop nest. Outermost
+  // loops are added first. Add loops in major-to-minor order, and skip the
+  // 'dimension_to_skip' dimension.
+  std::vector dimensions;
+  const Shape& shape = operand_array.GetShape();
+  for (int64 dimension : LayoutUtil::MinorToMajor(shape)) {
+    if (dimension != dimension_to_skip) {
+      dimensions.push_back(dimension);
+    }
+  }
+
+  // Create loop nest with one for-loop for each dimension of the
+  // output.
+  llvm_ir::IrArray::Index index =
+      AddLoopsForShapeOnDimensions(shape, dimensions, name_suffix);
+  // Verify every dimension except the 'dimension_to_skip' dimension was set in
+  // the index.
+  for (size_t dimension = 0; dimension < index.size(); ++dimension) {
+    if (dimension == dimension_to_skip) {
+      DCHECK_EQ(nullptr, index[dimension]);
+    } else {
+      DCHECK_NE(nullptr, index[dimension]);
+    }
+  }
+  return index;
+}
+
 }  // namespace llvm_ir
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h
index 0dd5b9d3b2..b3266022db 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h
@@ -248,6 +248,17 @@ class ForLoopNest {
       const Shape& shape, tensorflow::gtl::ArraySlice dimensions,
       tensorflow::StringPiece suffix);
 
+  // Emits a series of nested loops for iterating over an operand array. Loops
+  // are constructed in major to minor dimension layout order. No loop is
+  // emitted for the given 'dimension_to_skip'. The function returns an IrArray
+  // index for the given operand_array containing the indvars of the loops. All
+  // dimensions of the index are filled except for 'dimension_to_skip'.
+  // name_suffix is the string to append to the names of LLVM constructs (eg,
+  // basic blocks) constructed by this method.
+  IrArray::Index EmitOperandArrayLoopNest(const llvm_ir::IrArray& operand_array,
+                                          int64 dimension_to_skip,
+                                          tensorflow::StringPiece name_suffix);
+
   // Convenience methods which return particular basic blocks of the outermost
   // or innermost loops. These methods return nullptr if no loops have been
   // added yet.
diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
new file mode 100644
index 0000000000..16a9a5aaeb
--- /dev/null
+++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
@@ -0,0 +1,201 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/llvm_ir/sort_util.h"
+
+// IWYU pragma: no_include "llvm/IR/Intrinsics.gen.inc"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Instructions.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/ir_array.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/core/lib/core/bits.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+namespace llvm_ir {
+
+namespace {
+// Adds the inner comparison loop where we compare elements pointed to by
+// 'keys_index' and 'compare_keys_index'.
+void EmitCompareLoop(int64 dimension_to_sort,
+                     const llvm_ir::IrArray::Index& keys_index,
+                     const llvm_ir::IrArray::Index& compare_keys_index,
+                     const llvm_ir::IrArray& keys_array,
+                     llvm::IRBuilder<>* ir_builder) {
+  // TODO(b/26783907): parallelize this loop.
+
+  // if (is_smaller_index &&
+  //     compare_keys[dimension_to_sort] < dimension_to_sort_bound)
+  llvm::Value* is_smaller_index = ir_builder->CreateICmpSLT(
+      keys_index[dimension_to_sort], compare_keys_index[dimension_to_sort]);
+  int64 dimension_to_sort_bound =
+      keys_array.GetShape().dimensions(dimension_to_sort);
+  auto if_data = llvm_ir::EmitIfThenElse(
+      ir_builder->CreateAnd(
+          is_smaller_index,
+          ir_builder->CreateICmpSLT(
+              compare_keys_index[dimension_to_sort],
+              keys_index.GetConstantWithIndexType(dimension_to_sort_bound))),
+      "smaller_comparison_index", ir_builder, /*emit_else=*/false);
+  SetToFirstInsertPoint(if_data.true_block, ir_builder);
+  auto key1 = keys_array.EmitReadArrayElement(keys_index, ir_builder);
+  auto key2 = keys_array.EmitReadArrayElement(compare_keys_index, ir_builder);
+  auto key_type = keys_array.GetShape().element_type();
+  auto comparison =
+      primitive_util::IsFloatingPointType(key_type)
+          // TODO(b/26783907): Figure out how to handle NaNs.
+          ? ir_builder->CreateFCmp(llvm::FCmpInst::FCMP_ULT, key1, key2)
+          : ir_builder->CreateICmp(
+                primitive_util::IsSignedIntegralType(key_type)
+                    ? llvm::ICmpInst::ICMP_SLT
+                    : llvm::ICmpInst::ICMP_ULT,
+                key1, key2);
+  auto min_key = ir_builder->CreateSelect(comparison, key1, key2);
+  auto max_key = ir_builder->CreateSelect(comparison, key2, key1);
+  keys_array.EmitWriteArrayElement(keys_index, min_key, ir_builder);
+  keys_array.EmitWriteArrayElement(compare_keys_index, max_key, ir_builder);
+}
+}  // namespace
+
+Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
+                       tensorflow::StringPiece name,
+                       llvm::IRBuilder<>* ir_builder) {
+  const Shape& keys_shape = keys_array.GetShape();
+
+  // TODO(b/26783907): This case can probably be avoided with the Algebraic
+  // Simplifier.
+  if (ShapeUtil::IsScalar(keys_shape)) {
+    return Status::OK();
+  }
+
+  // Create loop nests which loop through the operand dimensions. The sort
+  // dimension is handled in three separate innermost loops which perform the
+  // sorting.
+  ForLoopNest loop_nest(name, ir_builder);
+  IrArray::Index keys_index =
+      loop_nest.EmitOperandArrayLoopNest(keys_array, dimension_to_sort, "keys");
+
+  // 'compare_keys_index' is the index of the element that 'keys_index' should
+  // be compared to.
+  IrArray::Index compare_keys_index(keys_index.GetType());
+  for (size_t dimension = 0; dimension < keys_index.size(); ++dimension) {
+    if (dimension != dimension_to_sort) {
+      compare_keys_index.push_back(keys_index[dimension]);
+    } else {
+      compare_keys_index.push_back(nullptr);
+    }
+  }
+
+  // Create the sorting loops which do the sorting.
+  int64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort);
+  std::unique_ptr stages_loop = loop_nest.AddLoop(
+      /*start_index=*/0,
+      /*end_index=*/
+      tensorflow::Log2Ceiling64(dimension_to_sort_bound),
+      /*suffix=*/"sort_stages");
+  std::unique_ptr mask_loop = loop_nest.AddLoop(
+      /*suffix=*/"mask",
+      /*start_index=*/keys_index.GetConstantWithIndexType(0),
+      /*end_index=*/stages_loop->GetIndVarValue());
+  std::unique_ptr compare_loop = loop_nest.AddLoop(
+      /*start_index=*/0,
+      /*end_index=*/dimension_to_sort_bound,
+      /*suffix=*/"compare");
+
+  // Naive C++ code for the inner loops (without parallelization):
+  //
+  // for (int64 stage = 0; stage < Log2Ceiling(dimension_to_sort_bound);
+  //     ++stage) {
+  //   int64 first_xor_mask = (1LL << (stage + 1)) - 1;
+  //   for (int64 i = 0; i < dimension_to_sort_bound; ++i) {
+  //     int64 j = i ^ first_xor_mask;
+  //     if (i < j && j < dimension_to_sort_bound) {
+  //       int64 min_key = std::min(keys[i], keys[j]);
+  //       keys[j] = std::max(keys[i], keys[j]);
+  //       keys[i] = min_key;
+  //     }
+  //   }
+  //   for (int64 mask = 0; mask < stage; ++mask) {
+  //     int64 later_xor_mask = (1LL << (stage - (mask + 1));
+  //     for (int64 i = 0; i < dimension_to_sort_bound; ++i) {
+  //       int64 j = i ^ later_xor_mask;
+  //       if (i < j && j < dimension_to_sort_bound) {
+  //         int64 min_key = std::min(keys[i], keys[j]);
+  //         keys[j] = std::max(keys[i], keys[j]);
+  //         keys[i] = min_key;
+  //       }
+  //     }
+  //   }
+  // }
+  //
+  // This follows the algorithm described on Wikipedia:
+  // https://en.wikipedia.org/wiki/Bitonic_sorter
+
+  SetToFirstInsertPoint(stages_loop->GetBodyBasicBlock(), ir_builder);
+  // The first xor mask of a stage is 2^(stage + 1) - 1.
+  auto first_xor_mask = ir_builder->CreateSub(
+      ir_builder->CreateShl(
+          keys_index.GetConstantWithIndexType(1),
+          ir_builder->CreateAdd(stages_loop->GetIndVarValue(),
+                                keys_index.GetConstantWithIndexType(1))),
+      keys_index.GetConstantWithIndexType(1));
+  std::unique_ptr first_compare_loop = ForLoop::EmitForLoop(
+      /*prefix=*/"first_compare",
+      /*start_index=*/keys_index.GetConstantWithIndexType(0),
+      /*end_index=*/
+      keys_index.GetConstantWithIndexType(dimension_to_sort_bound),
+      /*step=*/keys_index.GetConstantWithIndexType(1),
+      /*ir_builder=*/ir_builder);
+
+  SetToFirstInsertPoint(first_compare_loop->GetBodyBasicBlock(), ir_builder);
+  // 'first_compare_loop' iterates through the 'dimension_to_sort'.
+  keys_index[dimension_to_sort] = first_compare_loop->GetIndVarValue();
+  compare_keys_index[dimension_to_sort] = ir_builder->CreateXor(
+      first_compare_loop->GetIndVarValue(), first_xor_mask);
+  EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index, keys_array,
+                  ir_builder);
+
+  SetToFirstInsertPoint(compare_loop->GetPreheaderBasicBlock(), ir_builder);
+  // The later masks of a stage are 2^(stage - (mask_loop_ind_var + 1)).
+  auto later_xor_mask = ir_builder->CreateShl(
+      keys_index.GetConstantWithIndexType(1),
+      ir_builder->CreateSub(
+          stages_loop->GetIndVarValue(),
+          ir_builder->CreateAdd(mask_loop->GetIndVarValue(),
+                                keys_index.GetConstantWithIndexType(1))));
+
+  SetToFirstInsertPoint(compare_loop->GetBodyBasicBlock(), ir_builder);
+  // 'compare_loop' iterates through the 'dimension_to_sort'.
+  keys_index[dimension_to_sort] = compare_loop->GetIndVarValue();
+  compare_keys_index[dimension_to_sort] =
+      ir_builder->CreateXor(compare_loop->GetIndVarValue(), later_xor_mask);
+  EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index, keys_array,
+                  ir_builder);
+
+  // Set the IR builder insert point to the exit basic block of the outer most
+  // loop. This ensures later instructions are inserted after this loop nest.
+  ir_builder->SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
+
+  return Status::OK();
+}
+
+}  // namespace llvm_ir
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
new file mode 100644
index 0000000000..fc45bfab12
--- /dev/null
+++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
@@ -0,0 +1,34 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_SORT_UTIL_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_SORT_UTIL_H_
+
+#include "tensorflow/compiler/xla/service/llvm_ir/ir_array.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+namespace llvm_ir {
+// Emits llvm IR to sort the 'dimension_to_sort' dimension of 'keys_array' into
+// ascending order.
+Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
+                       tensorflow::StringPiece name,
+                       llvm::IRBuilder<>* ir_builder);
+}  // namespace llvm_ir
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_SORT_UTIL_H_
-- 
cgit v1.2.3


From c818bf016d4b48838d943338fd0c8581ab95ada1 Mon Sep 17 00:00:00 2001
From: Thomas Joerg 
Date: Thu, 19 Jul 2018 01:23:33 -0700
Subject: [XLA:GPU] Make the scalar_mul_computation in the test fixture
 actually use 'multiply'.

PiperOrigin-RevId: 205202802
---
 tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
index a6dc635b52..49b075be5e 100644
--- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
@@ -40,7 +40,7 @@ const char kModulePrefix[] = R"(
     scalar_mul_computation {
       scalar_lhs.1 = f32[] parameter(0)
       scalar_rhs.1 = f32[] parameter(1)
-      ROOT mul.1 = f32[] add(scalar_lhs.1, scalar_rhs.1)
+      ROOT mul.1 = f32[] multiply(scalar_lhs.1, scalar_rhs.1)
     })";
 
 TEST_F(MultiOutputFusionTest, MultiOutputFusionSiblingReduceAndReduceFusion) {
-- 
cgit v1.2.3


From dbf10397ed25001f03ed0eac879e328f625fe0d1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 03:53:59 -0700
Subject: A deleter class that calls RefCounted::Unref, and a unique_ptr alias
 RefCountPtr that uses this deleter.

This class can be used to automate the management of ref-owned objects.

PiperOrigin-RevId: 205217510
---
 tensorflow/core/lib/core/refcount.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tensorflow/core/lib/core/refcount.h b/tensorflow/core/lib/core/refcount.h
index eb41f9ff36..87bcfec411 100644
--- a/tensorflow/core/lib/core/refcount.h
+++ b/tensorflow/core/lib/core/refcount.h
@@ -17,6 +17,8 @@ limitations under the License.
 #define TENSORFLOW_LIB_CORE_REFCOUNT_H_
 
 #include 
+#include 
+
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
@@ -58,6 +60,15 @@ class RefCounted {
   void operator=(const RefCounted&) = delete;
 };
 
+// A deleter class to form a std::unique_ptr that unrefs objects.
+struct RefCountDeleter {
+  void operator()(tensorflow::core::RefCounted* o) const { o->Unref(); }
+};
+
+// A unique_ptr that unrefs the owned object on destruction.
+template 
+using RefCountPtr = std::unique_ptr;
+
 // Helper class to unref an object when out-of-scope.
 class ScopedUnref {
  public:
-- 
cgit v1.2.3


From bdd4871cd0d3159f709de1588532096d32db1390 Mon Sep 17 00:00:00 2001
From: vilmar-hillow 
Date: Thu, 19 Jul 2018 14:52:13 +0300
Subject: Typo in tf.Session fixed

---
 tensorflow/docs_src/performance/performance_guide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/performance/performance_guide.md b/tensorflow/docs_src/performance/performance_guide.md
index cb0f5ca924..dafacbe379 100644
--- a/tensorflow/docs_src/performance/performance_guide.md
+++ b/tensorflow/docs_src/performance/performance_guide.md
@@ -464,7 +464,7 @@ equal to the number of physical cores rather than logical cores.
   config = tf.ConfigProto()
   config.intra_op_parallelism_threads = 44
   config.inter_op_parallelism_threads = 44
-  tf.session(config=config)
+  tf.Session(config=config)
 
 ```
 
-- 
cgit v1.2.3


From 68353db31b981057d612a46571ee244a2aca6840 Mon Sep 17 00:00:00 2001
From: Thomas Joerg 
Date: Thu, 19 Jul 2018 05:23:36 -0700
Subject: [XLA:GPU] Ignore fp precision for multi-output fusion.

This allows fusing producers with fp16 outputs into reduce fusions which are always fp32.

PiperOrigin-RevId: 205224989
---
 .../xla/service/gpu/multi_output_fusion.cc         | 40 ++++++++++++-
 .../xla/service/gpu/multi_output_fusion_test.cc    | 70 ++++++++++++++++++++++
 2 files changed, 107 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
index ea661b3c2c..f95fbb01f9 100644
--- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
+++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include 
 #include 
 
+#include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -71,7 +72,6 @@ bool GpuMultiOutputFusion::ShapesCompatibleForFusion(HloInstruction* instr1,
   // In that case, the operand of the reduce needs to have the same shape
   // as the other tuple operands, but also we need to compare the output
   // shapes of the reduces.
-  // TODO(tjoerg): Allow differences in fp precision.
   auto* element_instr_1 = get_element_instr(instr1);
   auto* element_instr_2 = get_element_instr(instr2);
   if (element_instr_1->opcode() == HloOpcode::kReduce &&
@@ -80,8 +80,8 @@ bool GpuMultiOutputFusion::ShapesCompatibleForFusion(HloInstruction* instr1,
     return false;
   }
   // The elementwise output shapes must be the same (including layout).
-  return ShapeUtil::Equal(get_element_shape(element_instr_1),
-                          get_element_shape(element_instr_2));
+  return ShapeUtil::EqualIgnoringFpPrecision(
+      get_element_shape(element_instr_1), get_element_shape(element_instr_2));
 }
 
 namespace {
@@ -107,6 +107,27 @@ bool IsInputFusibleReduction(HloInstruction* instr) {
     return IsReductionToVector(*instr);
   }
 }
+
+// The code emitted for reduction suffers from poor data locality if the layouts
+// of input parameters differ. In such situtations it is beneficial not to fuse.
+// We consider input params with maximum rank only. Params with smaller ranks
+// will be broadcasted and have not been observed to cause data locality issues.
+// TODO(b/110927656): Improve reduce emitters to remove this limitation.
+bool ReduceFriendlyInputLayouts(HloInstruction* instr) {
+  int64 max_rank = 0;
+  const Layout* max_rank_layout;
+  for (HloInstruction* param : instr->fused_parameters()) {
+    if (ShapeUtil::Rank(param->shape()) > max_rank) {
+      max_rank = ShapeUtil::Rank(param->shape());
+      max_rank_layout = ¶m->shape().layout();
+    }
+  }
+  return c_all_of(instr->fused_parameters(), [&](HloInstruction* param) {
+    return (ShapeUtil::Rank(param->shape()) < max_rank) ||
+           (LayoutUtil::Equal(param->shape().layout(), *max_rank_layout));
+  });
+}
+
 }  // namespace
 
 bool GpuMultiOutputFusion::IsFusible(HloInstruction* instr) {
@@ -173,29 +194,41 @@ bool GpuMultiOutputFusion::DoProducerConsumerMultiOutputFusion() {
   // fusions operands.
   for (HloInstruction* consumer : computation()->MakeInstructionPostOrder()) {
     if (consumer->user_count() == 0) {
+      VLOG(3) << consumer->name() << " has no users.";
       continue;
     }
     if (!IsInputFusibleReduction(consumer)) {
+      VLOG(3) << consumer->name() << " is not an input-fusable reduction.";
       continue;
     }
+    VLOG(3) << consumer->name()
+            << " is a fusion candidate. Looking for fuseable operands.";
 
     auto consumer_operands = consumer->operands();
     for (size_t i = 0; i < consumer_operands.size(); ++i) {
       HloInstruction* producer = consumer_operands[i];
       if (!producer->IsFusable()) {
+        VLOG(3) << producer->name() << " is not fusable.";
         continue;
       }
       const bool is_loop_fusion =
           producer->opcode() == HloOpcode::kFusion &&
           producer->fusion_kind() == HloInstruction::FusionKind::kLoop;
       if (!is_loop_fusion) {
+        VLOG(3) << producer->name() << " is not a loop fusion.";
         continue;
       }
       if (!ShapesCompatibleForFusion(producer, consumer)) {
+        VLOG(3) << producer->name() << " has an incompatible shape.";
+        continue;
+      }
+      if (!ReduceFriendlyInputLayouts(producer)) {
+        VLOG(3) << producer->name() << " has inputs with mixed layouts.";
         continue;
       }
       // If we have already decided to fuse this producer, skip it.
       if (ContainsKey(to_fuse, producer)) {
+        VLOG(3) << producer->name() << " will be fused with another consumer.";
         continue;
       }
       // Do not fuse a producer if the other operands of the fusion are
@@ -204,6 +237,7 @@ bool GpuMultiOutputFusion::DoProducerConsumerMultiOutputFusion() {
             return producer != operand &&
                    reachability()->IsReachable(producer, operand);
           })) {
+        VLOG(3) << producer->name() << " would introduce a cycle when fused.";
         break;
       }
       to_fuse.insert(producer);
diff --git a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
index 49b075be5e..451e49f23a 100644
--- a/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/multi_output_fusion_test.cc
@@ -349,5 +349,75 @@ TEST_F(MultiOutputFusionTest, ProducerConsumerFusionDoNotFuseLoopReduceFusion) {
   ASSERT_FALSE(GpuMultiOutputFusion().Run(module.get()).ValueOrDie());
 }
 
+TEST_F(MultiOutputFusionTest,
+       ProducerConsumerFusionFp16LoopFusionAndReduceFusion) {
+  auto module = ParseHloString(tensorflow::strings::StrCat(kModulePrefix, R"(
+    fused_select {
+      p1.1 = f16[2,2,2]{2,1,0} parameter(1)
+      c0 = f16[] constant(0)
+      broadcast = f16[2,2,2]{2,1,0} broadcast(f16[] c0), dimensions={}
+      greater-than = pred[2,2,2]{2,1,0} greater-than(f32[2,2,2]{2,1,0} p1.1, f32[2,2,2]{2,1,0} broadcast)
+      p0.1 = f16[2,2,2]{2,1,0} parameter(0)
+      ROOT select = f16[2,2,2]{2,1,0} select(pred[2,2,2]{2,1,0} greater-than, f16[2,2,2]{2,1,0} p0.1, f16[2,2,2]{2,1,0} broadcast)
+    }
+    fused_reduce {
+      p0.2 = f16[2,2,2]{2,1,0} parameter(0)
+      convert = f32[2,2,2]{2,1,0} convert(p0.2)
+      c1 = f32[] constant(0)
+      r1 = f32[2,2]{1,0} reduce(convert, c1), dimensions={2}, to_apply=scalar_add_computation
+      mul = f32[2,2,2]{2,1,0} multiply(convert, convert)
+      r2 = f32[2,2]{1,0} reduce(mul, c1), dimensions={2}, to_apply=scalar_add_computation
+      ROOT tuple = (f32[2,2]{1,0}, f32[2,2]{1,0}) tuple(r1, r2)
+    }
+    ENTRY reduce {
+      p0 = f16[2,2,2]{2,1,0} parameter(0)
+      p1 = f16[2,2,2]{2,1,0} parameter(1)
+      select = f16[2,2,2]{2,1,0} fusion(p0, p1), kind=kLoop, calls=fused_select
+      fusion = (f32[2,2]{1,0}, f32[2,2]{1,0}) fusion(select), kind=kInput, calls=fused_reduce
+      gte0 = f32[2,2]{1,0} get-tuple-element(fusion), index=0
+      gte1 = f32[2,2]{1,0} get-tuple-element(fusion), index=1
+      ROOT root = (f32[2,2]{1,0}, f32[2,2]{1,0}, f16[2,2,2]{2,1,0}) tuple(gte1, gte1, select)
+    })"))
+                    .ValueOrDie();
+  ASSERT_TRUE(GpuMultiOutputFusion().Run(module.get()).ValueOrDie());
+  SCOPED_TRACE(module->ToString());
+  const HloInstruction* root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, op::Tuple(op::GetTupleElement(), op::GetTupleElement(),
+                              op::GetTupleElement()));
+  const HloInstruction* fusion = root->operand(0)->operand(0);
+  ASSERT_TRUE(fusion->IsMultiOutputFusion());
+  EXPECT_THAT(fusion->fused_expression_root(),
+              op::Tuple(op::Reduce(), op::Reduce(), op::Select()));
+}
+
+TEST_F(MultiOutputFusionTest,
+       ProducerConsumerFusionReduceUnfriendlyLoopFusion) {
+  auto module = ParseHloString(tensorflow::strings::StrCat(kModulePrefix, R"(
+    mixed_input_layouts_computation {
+      p0.1 = f16[128,1024,32,32]{1,3,2,0} parameter(0)
+      p1.1 = f16[128,1024,32,32]{3,2,1,0} parameter(1)
+      copy = f16[128,1024,32,32]{1,3,2,0} copy(p1.1)
+      c0 = f16[] constant(0)
+      broadcast = f16[128,1024,32,32]{1,3,2,0} broadcast(c0), dimensions={}
+      greater-than = pred[128,1024,32,32]{1,3,2,0} greater-than(copy, broadcast)
+      ROOT root = f16[128,1024,32,32]{1,3,2,0} select(greater-than, p0.1, broadcast)
+    }
+    fused_reduce {
+      p0.2 = f16[128,1024,32,32]{1,3,2,0} parameter(0)
+      convert = f32[128,1024,32,32]{1,3,2,0} convert(p0.2)
+      c0.2 = f32[] constant(0)
+      ROOT reduce = f32[1024]{0} reduce(convert, c0.2), dimensions={0,2,3}, to_apply=scalar_add_computation
+    }
+    ENTRY reduce {
+      p0 = f16[128,1024,32,32]{3,2,1,0} parameter(0)
+      p1 = f16[128,1024,32,32]{1,3,2,0} parameter(1)
+      loop_fusion = f16[128,1024,32,32]{1,3,2,0} fusion(p0, p1), kind=kLoop, calls=mixed_input_layouts_computation
+      reduce_fusion = f32[1024]{0} fusion(loop_fusion), kind=kInput, calls=fused_reduce
+      ROOT root = (f32[1024]{0}, f16[128,1024,32,32]{1,3,2,0}) tuple(reduce_fusion, loop_fusion)
+    })"))
+                    .ValueOrDie();
+  ASSERT_FALSE(GpuMultiOutputFusion().Run(module.get()).ValueOrDie());
+}
+
 }  // namespace gpu
 }  // namespace xla
-- 
cgit v1.2.3


From 4e3b7baca38aa93657272b2e80128d0552247f87 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 06:07:49 -0700
Subject: Docstrings in compiler.py

PiperOrigin-RevId: 205228977
---
 tensorflow/contrib/autograph/pyct/compiler.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/autograph/pyct/compiler.py b/tensorflow/contrib/autograph/pyct/compiler.py
index c172ab21f6..c90a5e89c2 100644
--- a/tensorflow/contrib/autograph/pyct/compiler.py
+++ b/tensorflow/contrib/autograph/pyct/compiler.py
@@ -71,7 +71,16 @@ def _build_source_map(node, code):
 
 
 def ast_to_source(node, indentation='  '):
-  """Return the source code of given AST."""
+  """Return the source code of given AST.
+
+  Args:
+    node: The code to compile, as an AST object.
+    indentation: The string to use for indentation.
+
+  Returns:
+    code: The source code generated from the AST object
+    source_mapping: A mapping between the user and AutoGraph generated code.
+  """
   original_node = node
   if isinstance(node, gast.AST):
     node = gast.gast_to_ast(node)
@@ -105,7 +114,8 @@ def ast_to_object(node,
       exit.
 
   Returns:
-    A module object containing the compiled source code.
+    compiled_node: A module object containing the compiled source code.
+    source: The source code of the compiled object
   Raises:
     ValueError: If ag_source_map__ is already in the namespace of the compiled
     node.
-- 
cgit v1.2.3


From 616a42e95e179c8300b8983e1c534bc03097a869 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Thu, 19 Jul 2018 06:58:35 -0700
Subject: Revert "Temporarily fix the undefined symbols problem"

This reverts commit 9a87590da3876b38af946ab363c9d94b8d46e0f9.
---
 tensorflow/core/graph/algorithm.cc | 16 ----------------
 tensorflow/core/graph/algorithm.h  | 22 ++++++----------------
 2 files changed, 6 insertions(+), 32 deletions(-)

diff --git a/tensorflow/core/graph/algorithm.cc b/tensorflow/core/graph/algorithm.cc
index 548096078f..9b4200e0b4 100644
--- a/tensorflow/core/graph/algorithm.cc
+++ b/tensorflow/core/graph/algorithm.cc
@@ -23,12 +23,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-void DFS(const Graph& g, const std::function& enter,
-         const std::function& leave,
-         const NodeComparator& stable_comparator) {
-  DFS(g, enter, leave, stable_comparator, {});
-}
-
 void DFS(const Graph& g, const std::function& enter,
          const std::function& leave,
          const NodeComparator& stable_comparator,
@@ -168,11 +162,6 @@ void ReverseDFSFrom(const Graph& g, gtl::ArraySlice start,
   ReverseDFSFromHelper(g, start, enter, leave, stable_comparator);
 }
 
-void GetPostOrder(const Graph& g, std::vector* order,
-                  const NodeComparator& stable_comparator) {
-  GetPostOrder(g, order, stable_comparator, {});
-}
-
 void GetPostOrder(const Graph& g, std::vector* order,
                   const NodeComparator& stable_comparator,
                   const EdgeFilter& edge_filter) {
@@ -181,11 +170,6 @@ void GetPostOrder(const Graph& g, std::vector* order,
       edge_filter);
 }
 
-void GetReversePostOrder(const Graph& g, std::vector* order,
-                         const NodeComparator& stable_comparator) {
-  GetReversePostOrder(g, order, stable_comparator, {});
-}
-
 void GetReversePostOrder(const Graph& g, std::vector* order,
                          const NodeComparator& stable_comparator,
                          const EdgeFilter& edge_filter) {
diff --git a/tensorflow/core/graph/algorithm.h b/tensorflow/core/graph/algorithm.h
index 7d8a3456e4..5bbbc6f6dc 100644
--- a/tensorflow/core/graph/algorithm.h
+++ b/tensorflow/core/graph/algorithm.h
@@ -52,12 +52,8 @@ struct NodeComparatorName {
 // If edge_filter is set then ignores edges for which edge_filter returns false.
 extern void DFS(const Graph& g, const std::function& enter,
                 const std::function& leave,
-                const NodeComparator& stable_comparator = {});
-
-extern void DFS(const Graph& g, const std::function& enter,
-                const std::function& leave,
-                const NodeComparator& stable_comparator,
-                const EdgeFilter& edge_filter);
+                const NodeComparator& stable_comparator = {},
+                const EdgeFilter& edge_filter = {});
 
 // Perform a reverse depth-first-search on g starting at the sink node.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
@@ -95,11 +91,8 @@ extern void ReverseDFSFrom(const Graph& g, gtl::ArraySlice start,
 //
 // REQUIRES: order is not NULL.
 void GetPostOrder(const Graph& g, std::vector* order,
-                  const NodeComparator& stable_comparator = {});
-
-void GetPostOrder(const Graph& g, std::vector* order,
-                  const NodeComparator& stable_comparator,
-                  const EdgeFilter& edge_filter);
+                  const NodeComparator& stable_comparator = {},
+                  const EdgeFilter& edge_filter = {});
 
 // Stores in *order the reverse post-order numbering of all nodes
 // If stable_comparator is set, a stable ordering of visit is achieved by
@@ -107,11 +100,8 @@ void GetPostOrder(const Graph& g, std::vector* order,
 //
 // If edge_filter is set then ignores edges for which edge_filter returns false.
 void GetReversePostOrder(const Graph& g, std::vector* order,
-                         const NodeComparator& stable_comparator);
-
-void GetReversePostOrder(const Graph& g, std::vector* order,
-                         const NodeComparator& stable_comparator,
-                         const EdgeFilter& edge_filter);
+                         const NodeComparator& stable_comparator = {},
+                         const EdgeFilter& edge_filter = {});
 
 // Prune nodes in "g" that are not in some path from the source node
 // to any node in 'nodes'. Returns true if changes were made to the graph.
-- 
cgit v1.2.3


From 1e1f3b0c69aa716834be55e311e512363107f1df Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 15 Jul 2018 15:41:06 +0000
Subject: Update or-tools to v6.7.2

This fix updates or-tools from 253f795 (dated 03/21/2017)
to the latest versioned release version of v6.7.2

Signed-off-by: Yong Tang 
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 4b4f31813c..0b2bf83259 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -144,13 +144,13 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "ortools_archive",
       urls = [
-          "https://mirror.bazel.build/github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz",
+          "https://mirror.bazel.build/github.com/google/or-tools/archive/v6.7.2.tar.gz",
           # Please uncomment me, when the next upgrade happens. Then
           # remove the whitelist entry in third_party/repo.bzl.
-          # "https://github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz",
+          # "https://github.com/google/or-tools/archive/v6.7.2.tar.gz",
       ],
-      sha256 = "932075525642b04ac6f1b50589f1df5cd72ec2f448b721fd32234cf183f0e755",
-      strip_prefix = "or-tools-253f7955c6a1fd805408fba2e42ac6d45b312d15/src",
+      sha256 = "d025a95f78b5fc5eaa4da5f395f23d11c23cf7dbd5069f1f627f002de87b86b9",
+      strip_prefix = "or-tools-6.7.2/src",
       build_file = clean_dep("//third_party:ortools.BUILD"),
   )
 
-- 
cgit v1.2.3


From 4ef7d900fb11d3e361b62a4a6c8a645fbdb8efb7 Mon Sep 17 00:00:00 2001
From: Yong Tang 
Date: Sun, 15 Jul 2018 15:42:15 +0000
Subject: Remove unneeded comments

Signed-off-by: Yong Tang 
---
 tensorflow/workspace.bzl | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 0b2bf83259..06916e924c 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -145,9 +145,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "ortools_archive",
       urls = [
           "https://mirror.bazel.build/github.com/google/or-tools/archive/v6.7.2.tar.gz",
-          # Please uncomment me, when the next upgrade happens. Then
-          # remove the whitelist entry in third_party/repo.bzl.
-          # "https://github.com/google/or-tools/archive/v6.7.2.tar.gz",
+          "https://github.com/google/or-tools/archive/v6.7.2.tar.gz",
       ],
       sha256 = "d025a95f78b5fc5eaa4da5f395f23d11c23cf7dbd5069f1f627f002de87b86b9",
       strip_prefix = "or-tools-6.7.2/src",
-- 
cgit v1.2.3


From cb299834dbe8469f8b54c129e6831e42eed399a2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 07:10:21 -0700
Subject: Adds class name to the multi_label per class metrics when
 label_vocabulary is provided.

PiperOrigin-RevId: 205235131
---
 tensorflow/contrib/estimator/python/estimator/head.py    | 16 +++++++++++++---
 .../contrib/estimator/python/estimator/head_test.py      | 14 ++++++++------
 tensorflow/python/estimator/canned/metric_keys.py        |  5 +++++
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py
index c9d86ef4ab..34f765d565 100644
--- a/tensorflow/contrib/estimator/python/estimator/head.py
+++ b/tensorflow/contrib/estimator/python/estimator/head.py
@@ -943,20 +943,30 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
         class_probabilities = array_ops.slice(
             probabilities, begin=begin, size=size)
         class_labels = array_ops.slice(labels, begin=begin, size=size)
-        prob_key = keys.PROBABILITY_MEAN_AT_CLASS % class_id
+        if self._label_vocabulary is None:
+          prob_key = keys.PROBABILITY_MEAN_AT_CLASS % class_id
+        else:
+          prob_key = (
+              keys.PROBABILITY_MEAN_AT_NAME % self._label_vocabulary[class_id])
         metric_ops[head_lib._summary_key(self._name, prob_key)] = (  # pylint:disable=protected-access
             head_lib._predictions_mean(  # pylint:disable=protected-access
                 predictions=class_probabilities,
                 weights=weights,
                 name=prob_key))
-        auc_key = keys.AUC_AT_CLASS % class_id
+        if self._label_vocabulary is None:
+          auc_key = keys.AUC_AT_CLASS % class_id
+        else:
+          auc_key = keys.AUC_AT_NAME % self._label_vocabulary[class_id]
         metric_ops[head_lib._summary_key(self._name, auc_key)] = (  # pylint:disable=protected-access
             head_lib._auc(  # pylint:disable=protected-access
                 labels=class_labels,
                 predictions=class_probabilities,
                 weights=weights,
                 name=auc_key))
-        auc_pr_key = keys.AUC_PR_AT_CLASS % class_id
+        if self._label_vocabulary is None:
+          auc_pr_key = keys.AUC_PR_AT_CLASS % class_id
+        else:
+          auc_pr_key = keys.AUC_PR_AT_NAME % self._label_vocabulary[class_id]
         metric_ops[head_lib._summary_key(self._name, auc_pr_key)] = (  # pylint:disable=protected-access
             head_lib._auc(  # pylint:disable=protected-access
                 labels=class_labels,
diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py
index 7b884402d4..2d367adb47 100644
--- a/tensorflow/contrib/estimator/python/estimator/head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/head_test.py
@@ -694,12 +694,14 @@ class MultiLabelHead(test.TestCase):
         # this assert tests that the algorithm remains consistent.
         keys.AUC: 0.3333,
         keys.AUC_PR: 0.7639,
-        keys.PROBABILITY_MEAN_AT_CLASS % 0: np.sum(_sigmoid(logits[:, 0])) / 2.,
-        keys.AUC_AT_CLASS % 0: 0.,
-        keys.AUC_PR_AT_CLASS % 0: 1.,
-        keys.PROBABILITY_MEAN_AT_CLASS % 1: np.sum(_sigmoid(logits[:, 1])) / 2.,
-        keys.AUC_AT_CLASS % 1: 1.,
-        keys.AUC_PR_AT_CLASS % 1: 1.,
+        keys.PROBABILITY_MEAN_AT_NAME % 'a':
+            np.sum(_sigmoid(logits[:, 0])) / 2.,
+        keys.AUC_AT_NAME % 'a': 0.,
+        keys.AUC_PR_AT_NAME % 'a': 1.,
+        keys.PROBABILITY_MEAN_AT_NAME % 'b':
+            np.sum(_sigmoid(logits[:, 1])) / 2.,
+        keys.AUC_AT_NAME % 'b': 1.,
+        keys.AUC_PR_AT_NAME % 'b': 1.,
     }
 
     self._test_eval(
diff --git a/tensorflow/python/estimator/canned/metric_keys.py b/tensorflow/python/estimator/canned/metric_keys.py
index 4f7c849ba4..9d49240fea 100644
--- a/tensorflow/python/estimator/canned/metric_keys.py
+++ b/tensorflow/python/estimator/canned/metric_keys.py
@@ -47,3 +47,8 @@ class MetricKeys(object):
   PROBABILITY_MEAN_AT_CLASS = 'probability_mean/class%d'
   AUC_AT_CLASS = 'auc/class%d'
   AUC_PR_AT_CLASS = 'auc_precision_recall/class%d'
+
+  # The following require a class name applied.
+  PROBABILITY_MEAN_AT_NAME = 'probability_mean/%s'
+  AUC_AT_NAME = 'auc/%s'
+  AUC_PR_AT_NAME = 'auc_precision_recall/%s'
-- 
cgit v1.2.3


From 5bf5f09450f153b1f35030e91b18bf56499a85d7 Mon Sep 17 00:00:00 2001
From: Mark Daoust 
Date: Thu, 19 Jul 2018 07:26:50 -0700
Subject: Consistently use `--upgrade` instead of `-U`

---
 tensorflow/docs_src/install/install_linux.md | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 0d9b6af093..84f7cc1c31 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -65,7 +65,7 @@ We *recommend* using `pip` version 8.1 or higher. If using a release before
 version 8.1, upgrade `pip`:
 
 
    -  pip install -U pip
+  pip install --upgrade pip
 
    
 
 If not using Ubuntu and [setuptools](https://pypi.org/project/setuptools/) is
@@ -102,7 +102,7 @@ When the Virtualenv is activated, the shell prompt displays as `(venv) $`.
 Within the active virtual environment, upgrade `pip`:
 
 
    -(venv)$ pip install -U pip
+(venv)$ pip install --upgrade pip
 
    
 
 You can install other Python packages within the virtual environment without
@@ -120,7 +120,7 @@ Choose one of the available TensorFlow packages for installation:
 Within an active Virtualenv environment, use `pip` to install the package:
 
 
    -  pip install -U tensorflow
+  pip install --upgrade tensorflow
 
    
 
 Use `pip list` to show the packages installed in the virtual environment.
@@ -198,7 +198,7 @@ We *recommend* using `pip` version 8.1 or higher. If using a release before
 version 8.1, upgrade `pip`:
 
 
    -  pip install -U pip
+  pip install --upgrade pip
 
    
 
 If not using Ubuntu and [setuptools](https://pypi.org/project/setuptools/) is
@@ -220,8 +220,8 @@ Choose one of the available TensorFlow packages for installation:
 And use `pip` to install the package for Python 2 or 3:
 
 
    -  pip install -U --user tensorflow   # Python 2.7
-  pip3 install -U --user tensorflow  # Python 3.n
+  pip install --upgrade --user tensorflow   # Python 2.7
+  pip3 install -upgrade --user tensorflow  # Python 3.n
 
    
 
 Use `pip list` to show the packages installed on the system.
-- 
cgit v1.2.3


From 7b0c47e59e2af9d28b453609497548cb4fbdc6df Mon Sep 17 00:00:00 2001
From: Mark Daoust 
Date: Thu, 19 Jul 2018 07:36:49 -0700
Subject: fix typo I added.

---
 tensorflow/docs_src/install/install_linux.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 84f7cc1c31..b0106ad481 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -221,7 +221,7 @@ And use `pip` to install the package for Python 2 or 3:
 
 
       pip install --upgrade --user tensorflow   # Python 2.7
-  pip3 install -upgrade --user tensorflow  # Python 3.n
+  pip3 install --upgrade --user tensorflow  # Python 3.n
 
    
 
 Use `pip list` to show the packages installed on the system.
-- 
cgit v1.2.3


From 7c2e16f92a13762a50d37049bd8c80fc439b03ab Mon Sep 17 00:00:00 2001
From: James Keeling 
Date: Thu, 19 Jul 2018 07:47:51 -0700
Subject: Fix argument comment in c_api_function_test.cc

PiperOrigin-RevId: 205239285
---
 tensorflow/c/c_api_function_test.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/c/c_api_function_test.cc b/tensorflow/c/c_api_function_test.cc
index 610274696f..f7ca219c89 100644
--- a/tensorflow/c/c_api_function_test.cc
+++ b/tensorflow/c/c_api_function_test.cc
@@ -1516,7 +1516,8 @@ void DefineStatefulFunction(const char* name, TF_Function** func) {
 
   TF_Output inputs[] = {};
   TF_Output outputs[] = {{random, 0}};
-  *func = TF_GraphToFunction(func_graph.get(), name, /*append_hash=*/false, -1,
+  *func = TF_GraphToFunction(func_graph.get(), name,
+                             /*append_hash_to_fn_name=*/false, -1,
                              /*opers=*/nullptr, 0, inputs, 1, outputs,
                              /*output_names=*/nullptr,
                              /*opts=*/nullptr, "", s.get());
-- 
cgit v1.2.3


From da12c366bd55cd797e433f78034d3848736ebe6f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 08:09:45 -0700
Subject: Imported from GitHub PR #20884 Add int32/string k/v support for
 tf.contrib.lookup.HashTable This fix tries to address the issue raised in
 #20869 where there were no int32/string k/v support for
 tf.contrib.lookup.HashTable. This fix adds the int32/string for the kernel.

This fix fixes #20869.

Signed-off-by: Yong Tang 

Copybara import of the project:

  - 2cdbb1e05e349b985ace42abd9a6a5140fa38b9f Add int32/string k/v support for tf.contrib.lookup.HashTa... by Yong Tang 
  - 54653810d18ea28c4d9fb82736b3529e84d1128c Add test case for int32/string(k/v) of tf.contrib.lookup.... by Yong Tang 
  - 27599c2915cde87a3c550c8876519e95b3f828db Fix string vs byte mismatch in python 3 by Yong Tang 
  - b58849aac511dcc0fc95218e011e945d3dac86b5 Merge 27599c2915cde87a3c550c8876519e95b3f828db into e7278... by Yong Tang 

COPYBARA_INTEGRATE_REVIEW=https://github.com/tensorflow/tensorflow/pull/20884 from yongtang:20869-tf.contrib.lookup.HashTable 27599c2915cde87a3c550c8876519e95b3f828db
PiperOrigin-RevId: 205242516
---
 tensorflow/contrib/lookup/lookup_ops_test.py | 15 +++++++++++++++
 tensorflow/core/kernels/lookup_table_op.cc   |  1 +
 2 files changed, 16 insertions(+)

diff --git a/tensorflow/contrib/lookup/lookup_ops_test.py b/tensorflow/contrib/lookup/lookup_ops_test.py
index 889accdd5a..8d510ede58 100644
--- a/tensorflow/contrib/lookup/lookup_ops_test.py
+++ b/tensorflow/contrib/lookup/lookup_ops_test.py
@@ -280,6 +280,21 @@ class HashTableOpTest(test.TestCase):
       table.init.run()
       self.assertAllEqual(3, table.size().eval())
 
+  def testHashTableInt32String(self):
+    with self.test_session():
+      default_val = "n/a"
+      keys = constant_op.constant([0, 1, 2], dtypes.int32)
+      values = constant_op.constant(["brain", "salad", "surgery"])
+      table = lookup.HashTable(
+          lookup.KeyValueTensorInitializer(keys, values), default_val)
+      table.init.run()
+
+      input_tensor = constant_op.constant([0, 1, -1])
+      output = table.lookup(input_tensor)
+
+      result = output.eval()
+      self.assertAllEqual([b"brain", b"salad", b"n/a"], result)
+
 
 class MutableHashTableOpTest(test.TestCase):
 
diff --git a/tensorflow/core/kernels/lookup_table_op.cc b/tensorflow/core/kernels/lookup_table_op.cc
index 57b7798ba0..07e754a6ef 100644
--- a/tensorflow/core/kernels/lookup_table_op.cc
+++ b/tensorflow/core/kernels/lookup_table_op.cc
@@ -822,6 +822,7 @@ REGISTER_KERNEL(int64, float);
 REGISTER_KERNEL(string, string);
 REGISTER_KERNEL(string, bool);
 REGISTER_KERNEL(int32, int32);
+REGISTER_KERNEL(int32, string);
 
 #undef REGISTER_KERNEL
 
-- 
cgit v1.2.3


From 6f0e971c30654b02e3ed2f1bc4d3f09b584668e7 Mon Sep 17 00:00:00 2001
From: Peter Hawkins 
Date: Thu, 19 Jul 2018 08:21:34 -0700
Subject: [TF:XLA] Rename xla::Diagonal to xla::GetMatrixDiagonal. Fix its
 handling of rectangular matrices. Switch the TF DiagPart and MatrixDiagPart
 operators to use GetMatrixDiagonal.

Extend CreateScalar{And,Or}Computation to support non-PRED types.

PiperOrigin-RevId: 205244201
---
 tensorflow/compiler/tf2xla/kernels/diag_op.cc      | 105 ++-------------------
 tensorflow/compiler/tf2xla/lib/scatter.cc          |   2 +-
 tensorflow/compiler/tf2xla/lib/triangular_solve.cc |   4 +-
 tensorflow/compiler/xla/client/lib/arithmetic.cc   |  12 ++-
 tensorflow/compiler/xla/client/lib/arithmetic.h    |   6 +-
 tensorflow/compiler/xla/client/lib/numeric.cc      |  21 +++--
 tensorflow/compiler/xla/client/lib/numeric.h       |   6 +-
 tensorflow/compiler/xla/client/lib/numeric_test.cc |  26 ++++-
 tensorflow/compiler/xla/tests/reduce_test.cc       |  43 +++++++--
 9 files changed, 100 insertions(+), 125 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/diag_op.cc b/tensorflow/compiler/tf2xla/kernels/diag_op.cc
index 6dec414c53..22cda27567 100644
--- a/tensorflow/compiler/tf2xla/kernels/diag_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/diag_op.cc
@@ -123,8 +123,6 @@ class DiagPartOp : public XlaOpKernel {
   explicit DiagPartOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
 
   void Compile(XlaOpKernelContext* ctx) override {
-    xla::XlaBuilder* builder = ctx->builder();
-
     const TensorShape input_shape = ctx->InputShape(0);
     auto dims = input_shape.dim_sizes();
 
@@ -150,37 +148,13 @@ class DiagPartOp : public XlaOpKernel {
       new_dims.push_back(dims[i]);
     }
 
-    xla::XlaOp diag = ctx->Input(0);
-
-    // TODO(b/30878775): use Slice with strides when supported, in place of
-    // the Pad -> Reshape -> Slice.
-
-    // Picture:
-    // [[1, 0, 0, 0]  pad and reshape to [[1, 0, 0, 0, 0],
-    //  [0, 2, 0, 0]  =================>  [2, 0, 0, 0, 0],
-    //  [0, 0, 3, 0]                      [3, 0, 0, 0, 0],
-    //  [0, 0, 0, 4]]                     [4, 0, 0, 0, 0]]
-    // and then slice out the first column.
-
-    // Flattens the input to 1D.
-    int64 size = input_shape.num_elements();
-    diag = xla::Reshape(diag, {size});
-
-    // Adds padding after the last element of 'new_size'.
-    xla::PaddingConfig config;
-    auto* dim = config.add_dimensions();
-    dim->set_edge_padding_high(new_size);
-    auto zero = XlaHelpers::Zero(builder, input_type(0));
-    diag = xla::Pad(diag, zero, config);
-
-    // Reshapes so the diagonal is now in the first column.
-    diag = xla::Reshape(diag, {new_size, new_size + 1});
+    xla::XlaOp input = ctx->Input(0);
 
-    // Slices out the first column and reshapes to the final shape.
-    diag = xla::Slice(diag, {0, 0}, {new_size, 1}, {1, 1});
-    diag = xla::Reshape(diag, new_dims);
+    xla::XlaOp output = xla::Reshape(
+        xla::GetMatrixDiagonal(xla::Reshape(input, {new_size, new_size})),
+        new_dims);
 
-    ctx->SetOutput(0, diag);
+    ctx->SetOutput(0, output);
   }
 };
 
@@ -220,8 +194,6 @@ class MatrixDiagPartOp : public XlaOpKernel {
   explicit MatrixDiagPartOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
 
   void Compile(XlaOpKernelContext* ctx) override {
-    xla::XlaBuilder* builder = ctx->builder();
-
     const TensorShape input_shape = ctx->InputShape(0);
     auto dims = input_shape.dim_sizes();
 
@@ -229,71 +201,8 @@ class MatrixDiagPartOp : public XlaOpKernel {
                 errors::InvalidArgument("Expected 2 <= dims, got shape ",
                                         input_shape.DebugString()));
 
-    xla::XlaOp diag = ctx->Input(0);
-
-    int last_dim = dims.size() - 1;
-    int64 last_dim_size = dims[last_dim];
-
-    // The smaller of the last two dimension sizes.
-    int64 smaller_dim_size = std::min(dims[last_dim - 1], dims[last_dim]);
-
-    // TODO(b/30878775): use Slice with strides when supported, in place of
-    // the Pad -> Reshape -> Slice.
-
-    // Picture: for each 2D matrix in the tensor's last two dimensions:
-    // [[1, 0, 0, 0]  pad and reshape to [[1, 0, 0, 0, 0],
-    //  [0, 2, 0, 0]  =================>  [2, 0, 0, 0, 0],
-    //  [0, 0, 3, 0]]                     [3, 0, 0, 0, 0],
-    // and then slice out the first column.
-    //
-    // Another example, with tall and narrow input.
-    // [[1, 0]  pad and reshape to [[1, 0, 0],
-    //  [0, 2]  =================>  [2, 0, 0]]
-    //  [0, 0]
-    //  [0, 0]]
-
-    // Collapses the last two dimensions.
-    std::vector flattened_dims(dims.begin(), dims.end() - 1);
-    flattened_dims.back() *= dims.back();
-    diag = xla::Reshape(diag, flattened_dims);
-
-    // Slices or pads the last dimension to 'target_size'.
-    int64 actual_size = flattened_dims.back();
-    int64 target_size = smaller_dim_size * (last_dim_size + 1);
-    if (actual_size < target_size) {
-      xla::PaddingConfig config =
-          xla::MakeNoPaddingConfig(flattened_dims.size());
-      auto* dim = config.mutable_dimensions(flattened_dims.size() - 1);
-      dim->set_edge_padding_high(target_size - actual_size);
-      auto zero = XlaHelpers::Zero(builder, input_type(0));
-      diag = xla::Pad(diag, zero, config);
-    } else if (actual_size > target_size) {
-      std::vector start(flattened_dims.size(), 0);
-      std::vector limits(flattened_dims.begin(), flattened_dims.end());
-      std::vector strides(flattened_dims.size(), 1);
-      limits[flattened_dims.size() - 1] = target_size;
-      diag = xla::Slice(diag, start, limits, strides);
-    }
-
-    // Reshape so the target values are in the first position of the last
-    // dimension.
-    std::vector unflattened_dims(dims.begin(), dims.end());
-    dims[last_dim - 1] = smaller_dim_size;
-    dims[last_dim] = last_dim_size + 1;
-    diag = xla::Reshape(diag, dims);
-
-    // Slices out the first column and reshapes to the final shape.
-    std::vector start(dims.size(), 0);
-    std::vector limits(dims.begin(), dims.end());
-    std::vector strides(dims.size(), 1);
-    limits[last_dim] = 1;
-    diag = xla::Slice(diag, start, limits, strides);
-
-    // Collapses away the last dimension.
-    dims.pop_back();
-    diag = xla::Reshape(diag, dims);
-
-    ctx->SetOutput(0, diag);
+    xla::XlaOp input = ctx->Input(0);
+    ctx->SetOutput(0, xla::GetMatrixDiagonal(input));
   }
 };
 
diff --git a/tensorflow/compiler/tf2xla/lib/scatter.cc b/tensorflow/compiler/tf2xla/lib/scatter.cc
index 6a5be1c2be..739032fef7 100644
--- a/tensorflow/compiler/tf2xla/lib/scatter.cc
+++ b/tensorflow/compiler/tf2xla/lib/scatter.cc
@@ -132,7 +132,7 @@ xla::StatusOr XlaScatter(
     // Discard updates with negative indices, since some users expect this.
     auto index_in_range = xla::ReduceAll(
         xla::Le(zero_index, index), xla::ConstantR0(body_builder, true),
-        xla::CreateScalarAndComputation(body_builder));
+        xla::CreateScalarAndComputation(xla::PRED, body_builder));
 
     // Make the index in bounds to prevent implementation defined behavior.
     index = xla::Max(index, zero_index);
diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
index e405f8dfaa..a2dd5a0d57 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
+++ b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
@@ -325,7 +325,7 @@ xla::XlaOp TriangularSolveLeftLooking(xla::XlaOp a, xla::XlaOp b,
     }
 
     // Rescale the input to be unit triangular
-    auto diag = Diagonal(a);
+    auto diag = xla::GetMatrixDiagonal(a);
     xla::XlaOp scaled_a;
     std::vector broadcast_dimensions(ndims - 1);
     std::iota(broadcast_dimensions.begin(), broadcast_dimensions.end(), 0);
@@ -490,7 +490,7 @@ xla::XlaOp TriangularSolveRightLooking(xla::XlaOp a, xla::XlaOp b,
     }
 
     // Rescale the input to be unit triangular
-    auto diag = Diagonal(a);
+    auto diag = xla::GetMatrixDiagonal(a);
     xla::XlaOp scaled_a;
     std::vector broadcast_dimensions(ndims - 1);
     std::iota(broadcast_dimensions.begin(), broadcast_dimensions.end(), 0);
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.cc b/tensorflow/compiler/xla/client/lib/arithmetic.cc
index 978fc40f34..de1d785e19 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.cc
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.cc
@@ -94,16 +94,18 @@ XlaComputation CreateScalarMinComputation(PrimitiveType type,
       });
 }
 
-XlaComputation CreateScalarAndComputation(XlaBuilder* builder) {
+XlaComputation CreateScalarAndComputation(PrimitiveType type,
+                                          XlaBuilder* builder) {
   return CreateScalarComputation(
-      "and", PRED, builder,
+      "and", type, builder,
       [](XlaBuilder* b, const XlaOp& lhs, const XlaOp& rhs) {
         return And(lhs, rhs);
       });
 }
 
-XlaComputation CreateScalarOrComputation(XlaBuilder* builder) {
-  return CreateScalarComputation("or", PRED, builder,
+XlaComputation CreateScalarOrComputation(PrimitiveType type,
+                                         XlaBuilder* builder) {
+  return CreateScalarComputation("or", type, builder,
                                  [](XlaBuilder* b, const XlaOp& lhs,
                                     const XlaOp& rhs) { return Or(lhs, rhs); });
 }
@@ -112,7 +114,7 @@ XlaOp Any(XlaOp predicates) {
   XlaBuilder* builder = predicates.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr {
     auto f = ConstantR0(builder, false);
-    XlaComputation logical_or = CreateScalarOrComputation(builder);
+    XlaComputation logical_or = CreateScalarOrComputation(PRED, builder);
     TF_ASSIGN_OR_RETURN(const Shape& predicates_shape,
                         builder->GetShape(predicates));
     std::vector all_dimensions(ShapeUtil::Rank(predicates_shape));
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.h b/tensorflow/compiler/xla/client/lib/arithmetic.h
index d0b916e8c8..8367e09450 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.h
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.h
@@ -45,10 +45,12 @@ XlaComputation CreateScalarMinComputation(PrimitiveType type,
                                           XlaBuilder* builder);
 
 // Creates a scalar logical AND computation and returns it.
-XlaComputation CreateScalarAndComputation(XlaBuilder* builder);
+XlaComputation CreateScalarAndComputation(PrimitiveType type,
+                                          XlaBuilder* builder);
 
 // Creates a scalar logical OR computation and returns it.
-XlaComputation CreateScalarOrComputation(XlaBuilder* builder);
+XlaComputation CreateScalarOrComputation(PrimitiveType type,
+                                         XlaBuilder* builder);
 
 // Returns whether any predicate in "predicates" is set.
 //
diff --git a/tensorflow/compiler/xla/client/lib/numeric.cc b/tensorflow/compiler/xla/client/lib/numeric.cc
index cdbeb189f4..a6e460aa75 100644
--- a/tensorflow/compiler/xla/client/lib/numeric.cc
+++ b/tensorflow/compiler/xla/client/lib/numeric.cc
@@ -79,25 +79,30 @@ XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m,
   return ConvertElementType(indicator, type);
 }
 
-XlaOp Diagonal(XlaOp x) {
+XlaOp GetMatrixDiagonal(XlaOp x) {
   XlaBuilder* builder = x.builder();
   return builder->ReportErrorOrReturn([&]() -> StatusOr {
     TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
     const int64 n_dims = ShapeUtil::Rank(shape);
     TF_RET_CHECK(n_dims >= 2);
-    const int64 n = shape.dimensions(n_dims - 1);
     const int64 m = shape.dimensions(n_dims - 2);
+    const int64 n = shape.dimensions(n_dims - 1);
     tensorflow::gtl::ArraySlice major_dims(
         AsInt64Slice(shape.dimensions()), /*pos=*/0, /*len=*/n_dims - 2);
     auto a = Iota(builder, U32, n);
     auto b = Iota(builder, U32, m);
-    auto indicator = Eq(a, Broadcast(b, {n}), /*broadcast_dimensions=*/{0});
+    auto indicator = Eq(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0});
     auto mask = Broadcast(indicator, major_dims);
-    XlaComputation add =
-        CreateScalarAddComputation(shape.element_type(), builder);
-    auto diag = Reduce(Select(mask, x, Zeros(builder, shape)), ScalarLike(x, 0),
-                       add, {n_dims - 1});
-    return diag;
+
+    // TPUs don't support S64 add reduction at the moment. But fortunately
+    // OR-reductions work just as well for integers.
+    XlaComputation reducer =
+        primitive_util::IsIntegralType(shape.element_type())
+            ? CreateScalarOrComputation(shape.element_type(), builder)
+            : CreateScalarAddComputation(shape.element_type(), builder);
+
+    return Reduce(Select(mask, x, Zeros(builder, shape)), ScalarLike(x, 0),
+                  reducer, {m >= n ? n_dims - 2 : n_dims - 1});
   });
 }
 
diff --git a/tensorflow/compiler/xla/client/lib/numeric.h b/tensorflow/compiler/xla/client/lib/numeric.h
index 3ec084636b..e9037b722c 100644
--- a/tensorflow/compiler/xla/client/lib/numeric.h
+++ b/tensorflow/compiler/xla/client/lib/numeric.h
@@ -29,8 +29,10 @@ XlaOp Iota(XlaBuilder* builder, PrimitiveType type, int64 size);
 // else.
 XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m, int64 n);
 
-// Get the diagonals of the last two dimensions.
-XlaOp Diagonal(XlaOp x);
+// Get the diagonals of the last two dimensions. If 'x' has shape
+// [..., M, N], then the output has shape [..., min(M, N)], containing the
+// diagonal elements (i.e., with indices [..., i, i]).
+XlaOp GetMatrixDiagonal(XlaOp x);
 
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/client/lib/numeric_test.cc b/tensorflow/compiler/xla/client/lib/numeric_test.cc
index bc8a73e9d7..bfea3f539d 100644
--- a/tensorflow/compiler/xla/client/lib/numeric_test.cc
+++ b/tensorflow/compiler/xla/client/lib/numeric_test.cc
@@ -24,7 +24,11 @@ limitations under the License.
 namespace xla {
 namespace {
 
-using NumericTest = ClientLibraryTestBase;
+class NumericTest : public ClientLibraryTestBase {
+ protected:
+  template 
+  void TestMatrixDiagonal();
+};
 
 XLA_TEST_F(NumericTest, Iota) {
   XlaBuilder builder(TestName());
@@ -33,5 +37,25 @@ XLA_TEST_F(NumericTest, Iota) {
   ComputeAndCompareR1(&builder, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, {});
 }
 
+template 
+void NumericTest::TestMatrixDiagonal() {
+  XlaBuilder builder("GetMatrixDiagonal");
+  Array3D input(2, 3, 4);
+  input.FillIota(0);
+
+  XlaOp a;
+  auto a_data = CreateR3Parameter(input, 0, "a", &builder, &a);
+  GetMatrixDiagonal(a);
+  Array2D expected({{0, 5, 10}, {12, 17, 22}});
+
+  ComputeAndCompareR2(&builder, expected, {a_data.get()});
+}
+
+XLA_TEST_F(NumericTest, GetMatrixDiagonal_S32) { TestMatrixDiagonal(); }
+
+XLA_TEST_F(NumericTest, GetMatrixDiagonal_S64) { TestMatrixDiagonal(); }
+
+XLA_TEST_F(NumericTest, GetMatrixDiagonal_F32) { TestMatrixDiagonal(); }
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/reduce_test.cc b/tensorflow/compiler/xla/tests/reduce_test.cc
index 1407fca72f..e4a8ddf86a 100644
--- a/tensorflow/compiler/xla/tests/reduce_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_test.cc
@@ -125,10 +125,10 @@ class ReduceTest : public ClientLibraryTestBase {
     XlaComputation reduce;
     if (and_reduce) {
       init_value = ConstantR0(&builder, true);
-      reduce = CreateScalarAndComputation(&builder);
+      reduce = CreateScalarAndComputation(PRED, &builder);
     } else {
       init_value = ConstantR0(&builder, false);
-      reduce = CreateScalarOrComputation(&builder);
+      reduce = CreateScalarOrComputation(PRED, &builder);
     }
     Reduce(pred_values, init_value, reduce,
            /*dimensions_to_reduce=*/{0});
@@ -163,10 +163,10 @@ class ReduceTest : public ClientLibraryTestBase {
     XlaComputation reduce_op;
     if (and_reduce) {
       init_value = ConstantR0(&builder, true);
-      reduce_op = CreateScalarAndComputation(&builder);
+      reduce_op = CreateScalarAndComputation(PRED, &builder);
     } else {
       init_value = ConstantR0(&builder, false);
-      reduce_op = CreateScalarOrComputation(&builder);
+      reduce_op = CreateScalarOrComputation(PRED, &builder);
     }
 
     Reduce(input_pred, init_value, reduce_op,
@@ -798,13 +798,17 @@ XLA_TEST_F(ReduceTest, VectorizedReduce_Min) {
 
 XLA_TEST_F(ReduceTest, VectorizedReduce_BooleanAnd) {
   RunVectorizedReduceTestForType(
-      static_cast(CreateScalarAndComputation),
+      static_cast([](XlaBuilder* builder) {
+        return CreateScalarAndComputation(PRED, builder);
+      }),
       [](bool a, bool b) { return a && b; }, true);
 }
 
 XLA_TEST_F(ReduceTest, VectorizedReduce_BooleanOr) {
   RunVectorizedReduceTestForType(
-      static_cast(CreateScalarOrComputation),
+      static_cast([](XlaBuilder* builder) {
+        return CreateScalarOrComputation(PRED, builder);
+      }),
       [](bool a, bool b) { return a || b; }, false);
 }
 
@@ -963,5 +967,32 @@ XLA_TEST_F(ReduceTest, ReduceIdentity) {
       ErrorSpec(0.0001));
 }
 
+XLA_TEST_F(ReduceTest, AndReduceU64) {
+  XlaBuilder builder(TestName());
+  Array2D initializer = {{0x123456789ABCDEF0LL, 0x3BCDEF12A4567890LL},
+                                 {0XFFFFFFFFFFFFFFD6LL, 101},
+                                 {1, 0XFFFFFFFFFFFFFFFFLL}};
+  auto reducer = CreateScalarAndComputation(U64, &builder);
+  auto m = ConstantR2FromArray2D(&builder, initializer);
+  Reduce(m, ConstantR0(&builder, 0xFFFFFFFFFFFFFFFFLL), reducer, {1});
+
+  std::vector expected = {0x1204461080145890LL, 68, 1};
+  ComputeAndCompareR1(&builder, expected, {});
+}
+
+XLA_TEST_F(ReduceTest, OrReduceU64) {
+  XlaBuilder builder(TestName());
+  Array2D initializer = {{0x123456789ABCDEF0LL, 0x3BCDEF12A4567890LL},
+                                 {0xFFFFFFFFFFFFFFD6LL, 101},
+                                 {1, 0xCAFEBEEFABABABABLL}};
+  auto reducer = CreateScalarOrComputation(U64, &builder);
+  auto m = ConstantR2FromArray2D(&builder, initializer);
+  Reduce(m, ConstantR0(&builder, 0), reducer, {1});
+
+  std::vector expected = {0X3BFDFF7ABEFEFEF0LL, 0XFFFFFFFFFFFFFFF7LL,
+                                  0xCAFEBEEFABABABABLL};
+  ComputeAndCompareR1(&builder, expected, {});
+}
+
 }  // namespace
 }  // namespace xla
-- 
cgit v1.2.3


From daafc6571a4817b1313b7c243fbd35e3a9f12dab Mon Sep 17 00:00:00 2001
From: David Majnemer 
Date: Thu, 19 Jul 2018 08:51:32 -0700
Subject: [XLA] Don't use Pow for simple expressions

Using Pow to handle squaring or taking the reciprocal is overkill, Pow is not going to be as accurate as the straightforward formulation without relying on optimization in the compiler or the Pow implementation to kick in.

PiperOrigin-RevId: 205247912
---
 tensorflow/compiler/xla/client/lib/math.cc | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/math.cc b/tensorflow/compiler/xla/client/lib/math.cc
index a6d606f944..0221de7672 100644
--- a/tensorflow/compiler/xla/client/lib/math.cc
+++ b/tensorflow/compiler/xla/client/lib/math.cc
@@ -25,11 +25,9 @@ XlaOp Sqrt(XlaOp operand) { return Pow(operand, ScalarLike(operand, 0.5)); }
 
 XlaOp Rsqrt(XlaOp operand) { return Pow(operand, ScalarLike(operand, -0.5)); }
 
-XlaOp Square(XlaOp operand) { return Pow(operand, ScalarLike(operand, 2.0)); }
+XlaOp Square(XlaOp operand) { return operand * operand; }
 
-XlaOp Reciprocal(XlaOp operand) {
-  return Pow(operand, ScalarLike(operand, -1.0));
-}
+XlaOp Reciprocal(XlaOp operand) { return ScalarLike(operand, 1.0) / operand; }
 
 namespace {
 
-- 
cgit v1.2.3


From e9e48b963b1ad1274ad8a0ad7d07d7fa990fe6b9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 08:52:49 -0700
Subject: Update `reader` dependencies such that the SavedModel loader still
 works on mobile.

PiperOrigin-RevId: 205248073
---
 tensorflow/cc/saved_model/BUILD | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
index 730b1b669b..3d3895c8fa 100644
--- a/tensorflow/cc/saved_model/BUILD
+++ b/tensorflow/cc/saved_model/BUILD
@@ -39,9 +39,20 @@ cc_library(
     hdrs = ["reader.h"],
     deps = [
         ":constants",
+    ] + if_not_mobile([
+        # TODO(b/111634734): :lib and :protos_all contain dependencies that
+        # cannot be built on mobile platforms. Instead, include the appropriate
+        # tf_lib depending on the build platform.
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
-    ],
+    ]) + if_mobile([
+        # Mobile-friendly SavedModel proto. See go/portable-proto for more info.
+        "//tensorflow/core:saved_model_portable_proto",
+    ]) + if_android([
+        "//tensorflow/core:android_tensorflow_lib",
+    ]) + if_ios([
+        "//tensorflow/core:ios_tensorflow_lib",
+    ]),
 )
 
 tf_cc_test(
-- 
cgit v1.2.3


From 2509b3a2152c8dda9fff8ed58f414c1316fa5379 Mon Sep 17 00:00:00 2001
From: Asim Shankar 
Date: Thu, 19 Jul 2018 08:56:07 -0700
Subject: eager guide: s/tfe.Checkpoint/tf.train.Checkpoint/

PiperOrigin-RevId: 205248470
---
 tensorflow/contrib/eager/python/examples/gan/mnist.py    |  5 ++---
 .../contrib/eager/python/examples/rnn_ptb/rnn_ptb.py     |  2 +-
 tensorflow/docs_src/guide/eager.md                       | 16 ++++++++--------
 third_party/examples/eager/spinn/spinn.py                |  2 +-
 4 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/gan/mnist.py b/tensorflow/contrib/eager/python/examples/gan/mnist.py
index b33243021b..9a42179299 100644
--- a/tensorflow/contrib/eager/python/examples/gan/mnist.py
+++ b/tensorflow/contrib/eager/python/examples/gan/mnist.py
@@ -29,7 +29,6 @@ import time
 
 import tensorflow as tf
 
-import tensorflow.contrib.eager as tfe
 from tensorflow.examples.tutorials.mnist import input_data
 
 layers = tf.keras.layers
@@ -265,7 +264,7 @@ def train_one_epoch(generator, discriminator, generator_optimizer,
 
 def main(_):
   (device, data_format) = ('/gpu:0', 'channels_first')
-  if FLAGS.no_gpu or tfe.num_gpus() <= 0:
+  if FLAGS.no_gpu or tf.contrib.eager.num_gpus() <= 0:
     (device, data_format) = ('/cpu:0', 'channels_last')
   print('Using device %s, and data format %s.' % (device, data_format))
 
@@ -291,7 +290,7 @@ def main(_):
   latest_cpkt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
   if latest_cpkt:
     print('Using latest checkpoint at ' + latest_cpkt)
-  checkpoint = tfe.Checkpoint(**model_objects)
+  checkpoint = tf.train.Checkpoint(**model_objects)
   # Restore variables on creation if a checkpoint exists.
   checkpoint.restore(latest_cpkt)
 
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
index d64bf5354e..15776c694e 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
@@ -315,7 +315,7 @@ def main(_):
                      FLAGS.hidden_dim, FLAGS.num_layers, FLAGS.dropout,
                      use_cudnn_rnn)
     optimizer = tf.train.GradientDescentOptimizer(learning_rate)
-    checkpoint = tfe.Checkpoint(
+    checkpoint = tf.train.Checkpoint(
         learning_rate=learning_rate, model=model,
         # GradientDescentOptimizer has no state to checkpoint, but noting it
         # here lets us swap in an optimizer that does.
diff --git a/tensorflow/docs_src/guide/eager.md b/tensorflow/docs_src/guide/eager.md
index 42ad9652f8..3b54d6d2bb 100644
--- a/tensorflow/docs_src/guide/eager.md
+++ b/tensorflow/docs_src/guide/eager.md
@@ -504,13 +504,13 @@ with tf.device("gpu:0"):
 
 ### Object-based saving
 
-`tfe.Checkpoint` can save and restore `tf.Variable`s to and from
+`tf.train.Checkpoint` can save and restore `tf.Variable`s to and from
 checkpoints:
 
 ```py
 x = tf.Variable(10.)
 
-checkpoint = tfe.Checkpoint(x=x)  # save as "x"
+checkpoint = tf.train.Checkpoint(x=x)  # save as "x"
 
 x.assign(2.)   # Assign a new value to the variables and save.
 save_path = checkpoint.save('./ckpt/')
@@ -523,18 +523,18 @@ checkpoint.restore(save_path)
 print(x)  # => 2.0
 ```
 
-To save and load models, `tfe.Checkpoint` stores the internal state of objects,
+To save and load models, `tf.train.Checkpoint` stores the internal state of objects,
 without requiring hidden variables. To record the state of a `model`,
-an `optimizer`, and a global step, pass them to a `tfe.Checkpoint`:
+an `optimizer`, and a global step, pass them to a `tf.train.Checkpoint`:
 
 ```py
 model = MyModel()
 optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
 checkpoint_dir = ‘/path/to/model_dir’
 checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
-root = tfe.Checkpoint(optimizer=optimizer,
-                      model=model,
-                      optimizer_step=tf.train.get_or_create_global_step())
+root = tf.train.Checkpoint(optimizer=optimizer,
+                           model=model,
+                           optimizer_step=tf.train.get_or_create_global_step())
 
 root.save(file_prefix=checkpoint_prefix)
 # or
@@ -824,7 +824,7 @@ gives you eager's interactive experimentation and debuggability with the
 distributed performance benefits of graph execution.
 
 Write, debug, and iterate in eager execution, then import the model graph for
-production deployment. Use `tfe.Checkpoint` to save and restore model
+production deployment. Use `tf.train.Checkpoint` to save and restore model
 variables, this allows movement between eager and graph execution environments.
 See the examples in:
 [tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).
diff --git a/third_party/examples/eager/spinn/spinn.py b/third_party/examples/eager/spinn/spinn.py
index c242ef3fdd..de63ebe9e6 100644
--- a/third_party/examples/eager/spinn/spinn.py
+++ b/third_party/examples/eager/spinn/spinn.py
@@ -626,7 +626,7 @@ def train_or_infer_spinn(embed,
     model = SNLIClassifier(config, embed)
     global_step = tf.train.get_or_create_global_step()
     trainer = SNLIClassifierTrainer(model, config.lr)
-    checkpoint = tfe.Checkpoint(trainer=trainer, global_step=global_step)
+    checkpoint = tf.train.Checkpoint(trainer=trainer, global_step=global_step)
     checkpoint.restore(tf.train.latest_checkpoint(config.logdir))
 
     if inference_sentence_pair:
-- 
cgit v1.2.3


From 15f3a087693a75243962b346d31a013d15990921 Mon Sep 17 00:00:00 2001
From: Dan Moldovan 
Date: Thu, 19 Jul 2018 08:57:50 -0700
Subject: Add a few more links to the notebook.

PiperOrigin-RevId: 205248656
---
 tensorflow/contrib/autograph/README.md | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/autograph/README.md b/tensorflow/contrib/autograph/README.md
index 679ab48e5c..cc54da4daa 100644
--- a/tensorflow/contrib/autograph/README.md
+++ b/tensorflow/contrib/autograph/README.md
@@ -1,6 +1,6 @@
 # AutoGraph
 
-IMPORTANT: AutoGraph is alpha software, and under active development. Expect rough edges and bugs, but if you try it, we appreciate early feedback! We'd also love contributions ([please see our contributing guidelines](CONTRIBUTING.md) and our [style guide](STYLE_GUIDE.md)).
+IMPORTANT: AutoGraph is beta software, and under active development. Expect rough edges and bugs, but if you try it, we appreciate early feedback! We'd also love contributions ([please see our contributing guidelines](CONTRIBUTING.md) and our [style guide](STYLE_GUIDE.md)).
 
 AutoGraph is a Python to TensorFlow compiler.
 
@@ -68,12 +68,21 @@ Then import the `autograph` module from `tf.contrib`:
 from tensorflow.contrib import autograph as ag
 ```
 
-### Interactive demo notebooks
+### Related links
 
-For more extensive examples, check out these interactive notebooks:
+Articles:
 
- * [RNN trained using Keras and Estimators](https://colab.sandbox.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb)
+ * [TensorFlow blog post](https://medium.com/tensorflow/autograph-converts-python-into-tensorflow-graphs-b2a871f87ec7)
+
+Interactive notebooks:
+
+ * [Quick guide](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/guide/autograph.ipynb)
+ * [RNN trained using Keras and Estimators](https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/autograph/examples/notebooks/rnn_keras_estimator.ipynb)
  * [Demo from the TF Dev Summit 2018](https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb)
+ * [Basic control flow speed test](https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_collatz_speed_test.ipynb)
+ * [MNIST training speed test](https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/autograph/examples/notebooks/ag_vs_eager_mnist_speed_test.ipynb)
+ * [Basic algorithm samples](https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/autograph/examples/notebooks/algorithms.ipynb)
+ * [Introductory workshop support notebook](https://colab.research.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/contrib/autograph/examples/notebooks/workshop.ipynb)
 
 ## Using with annotations
 
-- 
cgit v1.2.3


From e240aa301afc57a63366638af4cf92c823e8084a Mon Sep 17 00:00:00 2001
From: Eugene Brevdo 
Date: Thu, 19 Jul 2018 09:09:45 -0700
Subject: Add tf.bool support back to tf.scatter_nd.

PiperOrigin-RevId: 205250376
---
 tensorflow/core/kernels/scatter_nd_op.cc           |  4 ++++
 tensorflow/core/kernels/scatter_nd_op_cpu_impl.h   |  7 ++++---
 tensorflow/core/ops/array_ops.cc                   |  2 +-
 .../python/kernel_tests/scatter_nd_ops_test.py     | 23 ++++++++++++++++++++++
 4 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc
index e1fc2ea128..c44753e25e 100644
--- a/tensorflow/core/kernels/scatter_nd_op.cc
+++ b/tensorflow/core/kernels/scatter_nd_op.cc
@@ -277,6 +277,9 @@ TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_ADD_SUB_CPU);
 TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_UPDATE_CPU);
 TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_CPU);
 TF_CALL_string(REGISTER_SCATTER_ND_CPU);
+TF_CALL_bool(REGISTER_SCATTER_ND_ADD_SUB_CPU);
+TF_CALL_bool(REGISTER_SCATTER_ND_UPDATE_CPU);
+TF_CALL_bool(REGISTER_SCATTER_ND_CPU);
 
 // Registers GPU kernels.
 #if GOOGLE_CUDA
@@ -309,6 +312,7 @@ TF_CALL_complex128(REGISTER_SCATTER_ND_ALL_GPU);
 
 TF_CALL_int32(REGISTER_SCATTER_ND_ADD_SUB_SYCL);
 TF_CALL_int32(REGISTER_SCATTER_ND_UPDATE_SYCL);
+TF_CALL_bool(REGISTER_SCATTER_ND_UPDATE_SYCL);
 TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ND_ADD_SUB_SYCL);
 TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ND_UPDATE_SYCL);
 #undef REGISTER_SCATTER_ND_ADD_SUB_SYCL
diff --git a/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h b/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h
index 7cfffa20c5..472f5a3547 100644
--- a/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h
+++ b/tensorflow/core/kernels/scatter_nd_op_cpu_impl.h
@@ -161,15 +161,16 @@ struct ScatterNdFunctor {
 
 TF_CALL_ALL_TYPES(REGISTER_SCATTER_ND_UPDATE);
 REGISTER_SCATTER_ND_INDEX(string, scatter_nd_op::UpdateOp::ADD);
-TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_MATH)
-
+TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ND_MATH);
+TF_CALL_bool(REGISTER_SCATTER_ND_MATH);
 #undef REGISTER_SCATTER_ND_MATH
 #undef REGISTER_SCATTER_ND_UPDATE
 #undef REGISTER_SCATTER_ND_INDEX
 #undef REGISTER_SCATTER_ND_FULL
 
-#ifdef TENSORFLOW_USE_SYCL
 // Implementation of update functor for SYCL.
+#ifdef TENSORFLOW_USE_SYCL
+
 template 
 struct ScatterNdFunctor {
   Index operator()(
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 02989f8d3d..d6ae75473f 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -2881,7 +2881,7 @@ REGISTER_OP("ScatterNdNonAliasingAdd")
     .Input("indices: Tindices")
     .Input("updates: T")
     .Output("output: T")
-    .Attr("T: numbertype")
+    .Attr("T: {numbertype, bool}")
     .Attr("Tindices: {int32, int64}")
     .SetShapeFn(shape_inference::ScatterNdUpdateShape);
 
diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
index f9b9c77bbf..080319f6e8 100644
--- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py
@@ -369,6 +369,29 @@ class ScatterNdTest(test.TestCase):
     del input_  # input_ is not used in scatter_nd
     return array_ops.scatter_nd(indices, updates, shape)
 
+  @test_util.run_in_graph_and_eager_modes
+  def testBool(self):
+    indices = constant_op.constant(
+        [[4], [3], [1], [7]], dtype=dtypes.int32)
+    updates = constant_op.constant(
+        [False, True, False, True], dtype=dtypes.bool)
+    expected = np.array(
+        [False, False, False, True, False, False, False, True])
+    scatter = self.scatter_nd(indices, updates, shape=(8,))
+    result = self.evaluate(scatter)
+    self.assertAllEqual(expected, result)
+
+    # Same indice is updated twice by same value.
+    indices = constant_op.constant(
+        [[4], [3], [3], [7]], dtype=dtypes.int32)
+    updates = constant_op.constant(
+        [False, True, True, True], dtype=dtypes.bool)
+    expected = np.array([
+        False, False, False, True, False, False, False, True])
+    scatter = self.scatter_nd(indices, updates, shape=(8,))
+    result = self.evaluate(scatter)
+    self.assertAllEqual(expected, result)
+
   @test_util.run_in_graph_and_eager_modes
   def testInvalidShape(self):
     # TODO(apassos) figure out how to unify these errors
-- 
cgit v1.2.3


From b81144bbfb9a7887b0cefb2802ef3cc6df4860ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 09:18:03 -0700
Subject: Update ops-related pbtxt files.

PiperOrigin-RevId: 205251410
---
 tensorflow/core/ops/compat/ops_history.v1.pbtxt | 55 +++++++++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                   |  1 +
 2 files changed, 56 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index e91089e627..69351cd392 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -55242,6 +55242,61 @@ op {
     }
   }
 }
+op {
+  name: "ScatterNdNonAliasingAdd"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+        type: DT_BOOL
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "ScatterNdSub"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 6f07dd612e..978bb0bbf4 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -26182,6 +26182,7 @@ op {
         type: DT_HALF
         type: DT_UINT32
         type: DT_UINT64
+        type: DT_BOOL
       }
     }
   }
-- 
cgit v1.2.3


From 7ce817b2b386787b504c7652cc8877135409a8f4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 10:12:15 -0700
Subject: Rework Add / Sub kernels signatures.

PiperOrigin-RevId: 205259359
---
 tensorflow/contrib/lite/kernels/add.cc             |  85 +-
 tensorflow/contrib/lite/kernels/internal/common.h  | 133 +++
 .../internal/optimized/legacy_optimized_ops.h      | 239 ++++++
 .../kernels/internal/optimized/optimized_ops.h     | 802 +++++-------------
 .../internal/reference/legacy_reference_ops.h      | 234 ++++++
 .../kernels/internal/reference/reference_ops.h     | 907 +++++++++++----------
 tensorflow/contrib/lite/kernels/internal/types.h   | 112 ++-
 tensorflow/contrib/lite/kernels/sub.cc             |  68 +-
 8 files changed, 1452 insertions(+), 1128 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/add.cc b/tensorflow/contrib/lite/kernels/add.cc
index f44d531cbf..af9b5c7013 100644
--- a/tensorflow/contrib/lite/kernels/add.cc
+++ b/tensorflow/contrib/lite/kernels/add.cc
@@ -110,15 +110,12 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 
     QuantizeMultiplierSmallerThanOneExp(
         real_input1_multiplier, &data->input1_multiplier, &data->input1_shift);
-    data->input1_shift *= -1;
 
     QuantizeMultiplierSmallerThanOneExp(
         real_input2_multiplier, &data->input2_multiplier, &data->input2_shift);
-    data->input2_shift *= -1;
 
     QuantizeMultiplierSmallerThanOneExp(
         real_output_multiplier, &data->output_multiplier, &data->output_shift);
-    data->output_shift *= -1;
 
     CalculateActivationRangeUint8(params->activation, output,
                                   &data->output_activation_min,
@@ -152,14 +149,14 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
         CheckedLog2(output->params.scale, &output_scale_log2_rounded);
     TF_LITE_ENSURE(context, output_scale_is_pot);
 
-    data->input1_shift = output_scale_log2_rounded - input1_scale_log2_rounded;
-    data->input2_shift = output_scale_log2_rounded - input2_scale_log2_rounded;
+    data->input1_shift = input1_scale_log2_rounded - output_scale_log2_rounded;
+    data->input2_shift = input2_scale_log2_rounded - output_scale_log2_rounded;
 
     // Shifting of one input is supported. The graph quantization should ensure
     // that the other input matches the output.
     TF_LITE_ENSURE(context, data->input1_shift == 0 || data->input2_shift == 0);
-    TF_LITE_ENSURE(context, data->input1_shift >= 0);
-    TF_LITE_ENSURE(context, data->input2_shift >= 0);
+    TF_LITE_ENSURE(context, data->input1_shift <= 0);
+    TF_LITE_ENSURE(context, data->input2_shift <= 0);
 
     CalculateActivationRangeQuantized(context, params->activation, output,
                                       &data->output_activation_min,
@@ -173,24 +170,27 @@ template 
 void EvalAdd(TfLiteContext* context, TfLiteNode* node, TfLiteAddParams* params,
              const OpData* data, const TfLiteTensor* input1,
              const TfLiteTensor* input2, TfLiteTensor* output) {
-#define TF_LITE_ADD(type, opname, data_type)                            \
-  data_type output_activation_min, output_activation_max;               \
-  CalculateActivationRange(params->activation, &output_activation_min,  \
-                           &output_activation_max);                     \
-  type::opname(GetTensorData(input1), GetTensorDims(input1), \
-               GetTensorData(input2), GetTensorDims(input2), \
-               output_activation_min, output_activation_max,            \
-               GetTensorData(output), GetTensorDims(output))
+#define TF_LITE_ADD(type, opname, data_type)                             \
+  data_type output_activation_min, output_activation_max;                \
+  CalculateActivationRange(params->activation, &output_activation_min,   \
+                           &output_activation_max);                      \
+  tflite::ArithmeticParams op_params;                                    \
+  SetActivationParams(output_activation_min, output_activation_max,      \
+                      &op_params);                                       \
+  type::opname(op_params, GetTensorShape(input1),                        \
+               GetTensorData(input1), GetTensorShape(input2), \
+               GetTensorData(input2), GetTensorShape(output), \
+               GetTensorData(output))
   if (output->type == kTfLiteInt32) {
     if (kernel_type == kReference) {
       if (data->requires_broadcast) {
-        TF_LITE_ADD(reference_ops, BroadcastAdd, int32_t);
+        TF_LITE_ADD(reference_ops, BroadcastAdd4DSlow, int32_t);
       } else {
         TF_LITE_ADD(reference_ops, Add, int32_t);
       }
     } else {
       if (data->requires_broadcast) {
-        TF_LITE_ADD(optimized_ops, BroadcastAdd, int32_t);
+        TF_LITE_ADD(optimized_ops, BroadcastAdd4DSlow, int32_t);
       } else {
         TF_LITE_ADD(optimized_ops, Add, int32_t);
       }
@@ -198,13 +198,13 @@ void EvalAdd(TfLiteContext* context, TfLiteNode* node, TfLiteAddParams* params,
   } else if (output->type == kTfLiteFloat32) {
     if (kernel_type == kReference) {
       if (data->requires_broadcast) {
-        TF_LITE_ADD(reference_ops, BroadcastAdd, float);
+        TF_LITE_ADD(reference_ops, BroadcastAdd4DSlow, float);
       } else {
         TF_LITE_ADD(reference_ops, Add, float);
       }
     } else {
       if (data->requires_broadcast) {
-        TF_LITE_ADD(optimized_ops, BroadcastAdd, float);
+        TF_LITE_ADD(optimized_ops, BroadcastAdd4DSlow, float);
       } else {
         TF_LITE_ADD(optimized_ops, Add, float);
       }
@@ -220,30 +220,43 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node,
                               const TfLiteTensor* input2,
                               TfLiteTensor* output) {
   if (output->type == kTfLiteUInt8) {
-#define TF_LITE_ADD(type, opname)                                              \
-  type::opname(                                                                \
-      data->left_shift, GetTensorData(input1), GetTensorDims(input1), \
-      data->input1_offset, data->input1_multiplier, data->input1_shift,        \
-      GetTensorData(input2), GetTensorDims(input2),                   \
-      data->input2_offset, data->input2_multiplier, data->input2_shift,        \
-      data->output_offset, data->output_multiplier, data->output_shift,        \
-      data->output_activation_min, data->output_activation_max,                \
-      GetTensorData(output), GetTensorDims(output));
+#define TF_LITE_ADD(type, opname)                                      \
+  tflite::ArithmeticParams op_params;                                  \
+  op_params.left_shift = data->left_shift;                             \
+  op_params.input1_offset = data->input1_offset;                       \
+  op_params.input1_multiplier = data->input1_multiplier;               \
+  op_params.input1_shift = data->input1_shift;                         \
+  op_params.input2_offset = data->input2_offset;                       \
+  op_params.input2_multiplier = data->input2_multiplier;               \
+  op_params.input2_shift = data->input2_shift;                         \
+  op_params.output_offset = data->output_offset;                       \
+  op_params.output_multiplier = data->output_multiplier;               \
+  op_params.output_shift = data->output_shift;                         \
+  SetActivationParams(data->output_activation_min,                     \
+                      data->output_activation_max, &op_params);        \
+  type::opname(op_params, GetTensorShape(input1),                      \
+               GetTensorData(input1), GetTensorShape(input2), \
+               GetTensorData(input2), GetTensorShape(output), \
+               GetTensorData(output))
     // The quantized version of Add doesn't support activations, so we
     // always use BroadcastAdd.
     if (kernel_type == kReference) {
-      TF_LITE_ADD(reference_ops, BroadcastAdd);
+      TF_LITE_ADD(reference_ops, BroadcastAdd4DSlow);
     } else {
-      TF_LITE_ADD(optimized_ops, BroadcastAdd);
+      TF_LITE_ADD(optimized_ops, BroadcastAdd4DSlow);
     }
 #undef TF_LITE_ADD
   } else if (output->type == kTfLiteInt16) {
-#define TF_LITE_ADD(type, opname)                                        \
-  type::opname(GetTensorData(input1), GetTensorDims(input1),    \
-               data->input1_shift, GetTensorData(input2),       \
-               GetTensorDims(input2), data->input2_shift,                \
-               data->output_activation_min, data->output_activation_max, \
-               GetTensorData(output), GetTensorDims(output));
+#define TF_LITE_ADD(type, opname)                                      \
+  tflite::ArithmeticParams op_params;                                  \
+  op_params.input1_shift = data->input1_shift;                         \
+  op_params.input2_shift = data->input2_shift;                         \
+  SetActivationParams(data->output_activation_min,                     \
+                      data->output_activation_max, &op_params);        \
+  type::opname(op_params, GetTensorShape(input1),                      \
+               GetTensorData(input1), GetTensorShape(input2), \
+               GetTensorData(input2), GetTensorShape(output), \
+               GetTensorData(output))
     // The quantized version of Add doesn't support activations, so we
     // always use BroadcastAdd.
     if (kernel_type == kReference) {
diff --git a/tensorflow/contrib/lite/kernels/internal/common.h b/tensorflow/contrib/lite/kernels/internal/common.h
index b86ca49c11..310a8980e6 100644
--- a/tensorflow/contrib/lite/kernels/internal/common.h
+++ b/tensorflow/contrib/lite/kernels/internal/common.h
@@ -127,6 +127,139 @@ int CountLeadingZeros(T integer_input) {
   return leading_zeros;
 }
 
+// DO NOT USE THIS STRUCT FOR NEW FUNCTIONALITY BEYOND IMPLEMENTING
+// BROADCASTING.
+//
+// NdArrayDesc describes the shape and memory layout of an N-dimensional
+// rectangular array of numbers.
+//
+// NdArrayDesc is basically identical to Dims defined in types.h.
+// However, as Dims is to be deprecated, this class exists as an adaptor
+// to enable simple unoptimized implementations of element-wise broadcasting
+// operations.
+template 
+struct NdArrayDesc {
+  // The "extent" of each dimension. Indices along dimension d must be in the
+  // half-open interval [0, extents[d]).
+  int extents[N];
+
+  // The number of *elements* (not bytes) between consecutive indices of each
+  // dimension.
+  int strides[N];
+};
+
+// DO NOT USE THIS FUNCTION FOR NEW FUNCTIONALITY BEYOND IMPLEMENTING
+// BROADCASTING.
+//
+// Same as Offset(), except takes as NdArrayDesc instead of Dims.
+inline int SubscriptToIndex(const NdArrayDesc<4>& desc, int i0, int i1, int i2,
+                            int i3) {
+  TFLITE_DCHECK(i0 >= 0 && i0 < desc.extents[0]);
+  TFLITE_DCHECK(i1 >= 0 && i1 < desc.extents[1]);
+  TFLITE_DCHECK(i2 >= 0 && i2 < desc.extents[2]);
+  TFLITE_DCHECK(i3 >= 0 && i3 < desc.extents[3]);
+  return i0 * desc.strides[0] + i1 * desc.strides[1] + i2 * desc.strides[2] +
+         i3 * desc.strides[3];
+}
+
+// Given the dimensions of the operands for an element-wise binary broadcast,
+// adjusts them so that they can be directly iterated over with simple loops.
+// Returns the adjusted dims as instances of NdArrayDesc in 'desc0_out' and
+// 'desc1_out'. 'desc0_out' and 'desc1_out' cannot be nullptr.
+//
+// This function assumes that the two input shapes are compatible up to
+// broadcasting and the shorter one has already been prepended with 1s to be the
+// same length. E.g., if shape0 is (1, 16, 16, 64) and shape1 is (1, 64),
+// shape1 must already have been prepended to be (1, 1, 1, 64). Recall that
+// Dims refer to shapes in reverse order. In this case, input0_dims will be
+// (64, 16, 16, 1) and input1_dims will be (64, 1, 1, 1).
+//
+// When two shapes are compatible up to broadcasting, for each dimension d,
+// the input extents are either equal, or one of them is 1.
+//
+// This function performs the following for each dimension d:
+// - If the extents are equal, then do nothing since the loop that walks over
+//   both of the input arrays is correct.
+// - Otherwise, one (and only one) of the extents must be 1. Say extent0 is 1
+//   and extent1 is e1. Then set extent0 to e1 and stride0 *to 0*. This allows
+//   array0 to be referenced *at any index* in dimension d and still access the
+//   same slice.
+template 
+inline void NdArrayDescsForElementwiseBroadcast(const Dims& input0_dims,
+                                                const Dims& input1_dims,
+                                                NdArrayDesc* desc0_out,
+                                                NdArrayDesc* desc1_out) {
+  TFLITE_DCHECK(desc0_out != nullptr);
+  TFLITE_DCHECK(desc1_out != nullptr);
+
+  // Copy dims to desc.
+  for (int i = 0; i < N; ++i) {
+    desc0_out->extents[i] = input0_dims.sizes[i];
+    desc0_out->strides[i] = input0_dims.strides[i];
+    desc1_out->extents[i] = input1_dims.sizes[i];
+    desc1_out->strides[i] = input1_dims.strides[i];
+  }
+
+  // Walk over each dimension. If the extents are equal do nothing.
+  // Otherwise, set the desc with extent 1 to have extent equal to the other and
+  // stride 0.
+  for (int i = 0; i < N; ++i) {
+    const int extent0 = ArraySize(input0_dims, i);
+    const int extent1 = ArraySize(input1_dims, i);
+    if (extent0 != extent1) {
+      if (extent0 == 1) {
+        desc0_out->strides[i] = 0;
+        desc0_out->extents[i] = extent1;
+      } else {
+        TFLITE_DCHECK_EQ(extent1, 1);
+        desc1_out->strides[i] = 0;
+        desc1_out->extents[i] = extent0;
+      }
+    }
+  }
+}
+
+template 
+inline void NdArrayDescsForElementwiseBroadcast(
+    const RuntimeShape& input0_shape, const RuntimeShape& input1_shape,
+    NdArrayDesc* desc0_out, NdArrayDesc* desc1_out) {
+  TFLITE_DCHECK(desc0_out != nullptr);
+  TFLITE_DCHECK(desc1_out != nullptr);
+
+  auto extended_input0_shape = RuntimeShape::ExtendedShape(N, input0_shape);
+  auto extended_input1_shape = RuntimeShape::ExtendedShape(N, input1_shape);
+
+  // Copy dims to desc, calculating strides.
+  int desc0_stride = 1;
+  int desc1_stride = 1;
+  for (int i = N - 1; i >= 0; --i) {
+    desc0_out->extents[i] = extended_input0_shape.Dims(i);
+    desc0_out->strides[i] = desc0_stride;
+    desc0_stride *= extended_input0_shape.Dims(i);
+    desc1_out->extents[i] = extended_input1_shape.Dims(i);
+    desc1_out->strides[i] = desc1_stride;
+    desc1_stride *= extended_input1_shape.Dims(i);
+  }
+
+  // Walk over each dimension. If the extents are equal do nothing.
+  // Otherwise, set the desc with extent 1 to have extent equal to the other and
+  // stride 0.
+  for (int i = 0; i < N; ++i) {
+    const int extent0 = extended_input0_shape.Dims(i);
+    const int extent1 = extended_input1_shape.Dims(i);
+    if (extent0 != extent1) {
+      if (extent0 == 1) {
+        desc0_out->strides[i] = 0;
+        desc0_out->extents[i] = extent1;
+      } else {
+        TFLITE_DCHECK_EQ(extent1, 1);
+        desc1_out->strides[i] = 0;
+        desc1_out->extents[i] = extent0;
+      }
+    }
+  }
+}
+
 }  // namespace tflite
 
 #endif  // TENSORFLOW_CONTRIB_LITE_KERNELS_INTERNAL_COMMON_H_
diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/legacy_optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/legacy_optimized_ops.h
index 6db41d7961..d5503073a7 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/legacy_optimized_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/legacy_optimized_ops.h
@@ -55,6 +55,245 @@ inline void Relu(const float* input_data, const Dims<4>& input_dims,
        DimsToShape(output_dims));
 }
 
+// legacy, for compatibility with old checked-in code
+template 
+void Add(const float* input1_data, const Dims<4>& input1_dims,
+         const float* input2_data, const Dims<4>& input2_dims,
+         float* output_data, const Dims<4>& output_dims) {
+  float output_activation_min, output_activation_max;
+  GetActivationMinMax(Ac, &output_activation_min, &output_activation_max);
+
+  tflite::ArithmeticParams op_params;
+  op_params.float_activation_min = output_activation_min;
+  op_params.float_activation_max = output_activation_max;
+  Add(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
+template 
+inline void Add(int left_shift, const uint8* input1_data,
+                const Dims<4>& input1_dims, int32 input1_offset,
+                int32 input1_multiplier, int input1_shift,
+                const uint8* input2_data, const Dims<4>& input2_dims,
+                int32 input2_offset, int32 input2_multiplier, int input2_shift,
+                int32 output_offset, int32 output_multiplier, int output_shift,
+                int32 output_activation_min, int32 output_activation_max,
+                uint8* output_data, const Dims<4>& output_dims) {
+  constexpr int kReverseShift = -1;
+  static_assert(Ac == FusedActivationFunctionType::kNone ||
+                    Ac == FusedActivationFunctionType::kRelu ||
+                    Ac == FusedActivationFunctionType::kRelu6 ||
+                    Ac == FusedActivationFunctionType::kRelu1,
+                "");
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  if (Ac == FusedActivationFunctionType::kNone) {
+    TFLITE_DCHECK_EQ(output_activation_min, 0);
+    TFLITE_DCHECK_EQ(output_activation_max, 255);
+  }
+
+  tflite::ArithmeticParams op_params;
+  op_params.left_shift = left_shift;
+  op_params.input1_offset = input1_offset;
+  op_params.input1_multiplier = input1_multiplier;
+  op_params.input1_shift = kReverseShift * input1_shift;
+  op_params.input2_offset = input2_offset;
+  op_params.input2_multiplier = input2_multiplier;
+  op_params.input2_shift = kReverseShift * input2_shift;
+  op_params.output_offset = output_offset;
+  op_params.output_multiplier = output_multiplier;
+  op_params.output_shift = kReverseShift * output_shift;
+  op_params.quantized_activation_min = output_activation_min;
+  op_params.quantized_activation_max = output_activation_max;
+  Add(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
+template 
+void Add(const int32* input1_data, const Dims<4>& input1_dims,
+         const int32* input2_data, const Dims<4>& input2_dims,
+         int32* output_data, const Dims<4>& output_dims) {
+  gemmlowp::ScopedProfilingLabel label("Add/int32");
+  TFLITE_DCHECK(Ac == FusedActivationFunctionType::kNone);
+
+  tflite::ArithmeticParams op_params;
+  op_params.quantized_activation_min = std::numeric_limits::min();
+  op_params.quantized_activation_max = std::numeric_limits::max();
+  Add(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
+template 
+void BroadcastAdd(const T* input1_data, const Dims<4>& input1_dims,
+                  const T* input2_data, const Dims<4>& input2_dims,
+                  T output_activation_min, T output_activation_max,
+                  T* output_data, const Dims<4>& output_dims) {
+  tflite::ArithmeticParams op_params;
+  op_params.float_activation_min = output_activation_min;
+  op_params.float_activation_max = output_activation_max;
+  BroadcastAdd4DSlow(op_params, DimsToShape(input1_dims), input1_data,
+                     DimsToShape(input2_dims), input2_data,
+                     DimsToShape(output_dims), output_data);
+}
+
+template 
+inline void BroadcastAdd(int left_shift, const uint8* input1_data,
+                         const Dims<4>& input1_dims, int32 input1_offset,
+                         int32 input1_multiplier, int input1_shift,
+                         const uint8* input2_data, const Dims<4>& input2_dims,
+                         int32 input2_offset, int32 input2_multiplier,
+                         int input2_shift, int32 output_offset,
+                         int32 output_multiplier, int output_shift,
+                         int32 output_activation_min,
+                         int32 output_activation_max, uint8* output_data,
+                         const Dims<4>& output_dims) {
+  constexpr int kReverseShift = -1;
+  static_assert(Ac == FusedActivationFunctionType::kNone ||
+                    Ac == FusedActivationFunctionType::kRelu ||
+                    Ac == FusedActivationFunctionType::kRelu6 ||
+                    Ac == FusedActivationFunctionType::kRelu1,
+                "");
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  if (Ac == FusedActivationFunctionType::kNone) {
+    TFLITE_DCHECK_EQ(output_activation_min, 0);
+    TFLITE_DCHECK_EQ(output_activation_max, 255);
+  }
+
+  tflite::ArithmeticParams op_params;
+  op_params.left_shift = left_shift;
+  op_params.input1_offset = input1_offset;
+  op_params.input1_multiplier = input1_multiplier;
+  op_params.input1_shift = kReverseShift * input1_shift;
+  op_params.input2_offset = input2_offset;
+  op_params.input2_multiplier = input2_multiplier;
+  op_params.input2_shift = kReverseShift * input2_shift;
+  op_params.output_offset = output_offset;
+  op_params.output_multiplier = output_multiplier;
+  op_params.output_shift = kReverseShift * output_shift;
+  op_params.quantized_activation_min = output_activation_min;
+  op_params.quantized_activation_max = output_activation_max;
+  BroadcastAdd4DSlow(op_params, DimsToShape(input1_dims), input1_data,
+                     DimsToShape(input2_dims), input2_data,
+                     DimsToShape(output_dims), output_data);
+}
+
+template 
+inline void BroadcastAddFivefold(
+    int y0, int y1, int y2, int y3, int y4, int left_shift,
+    const uint8* input1_data, const Dims<4>& input1_dims, int32 input1_offset,
+    int32 input1_multiplier, int input1_shift, const uint8* input2_data,
+    const Dims<4>& input2_dims, int32 input2_offset, int32 input2_multiplier,
+    int input2_shift, int32 output_offset, int32 output_multiplier,
+    int output_shift, int32 output_activation_min, int32 output_activation_max,
+    uint8* output_data, const Dims<4>& output_dims) {
+  constexpr int kReverseShift = -1;
+  static_assert(Ac == FusedActivationFunctionType::kNone ||
+                    Ac == FusedActivationFunctionType::kRelu ||
+                    Ac == FusedActivationFunctionType::kRelu6 ||
+                    Ac == FusedActivationFunctionType::kRelu1,
+                "");
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  if (Ac == FusedActivationFunctionType::kNone) {
+    TFLITE_DCHECK_EQ(output_activation_min, 0);
+    TFLITE_DCHECK_EQ(output_activation_max, 255);
+  }
+  tflite::ArithmeticParams op_params;
+  op_params.broadcast_category =
+      tflite::BroadcastableOpCategory::kFirstInputBroadcastsFast;
+  op_params.left_shift = left_shift;
+  op_params.input1_offset = input1_offset;
+  op_params.input1_multiplier = input1_multiplier;
+  op_params.input1_shift = kReverseShift * input1_shift;
+  op_params.input2_offset = input2_offset;
+  op_params.input2_multiplier = input2_multiplier;
+  op_params.input2_shift = kReverseShift * input2_shift;
+  op_params.output_offset = output_offset;
+  op_params.output_multiplier = output_multiplier;
+  op_params.output_shift = kReverseShift * output_shift;
+  op_params.quantized_activation_min = output_activation_min;
+  op_params.quantized_activation_max = output_activation_max;
+  op_params.broadcast_shape[4] = y0;
+  op_params.broadcast_shape[3] = y1;
+  op_params.broadcast_shape[2] = y2;
+  op_params.broadcast_shape[1] = y3;
+  op_params.broadcast_shape[0] = y4;
+  BroadcastAddFivefold(op_params, DimsToShape(input1_dims), input1_data,
+                       DimsToShape(input2_dims), input2_data,
+                       DimsToShape(output_dims), output_data);
+}
+
+// legacy, for compatibility with old checked-in code
+template 
+void BroadcastAdd(const T* input1_data, const Dims<4>& input1_dims,
+                  const T* input2_data, const Dims<4>& input2_dims,
+                  T* output_data, const Dims<4>& output_dims) {
+  T output_activation_min, output_activation_max;
+  GetActivationMinMax(Ac, &output_activation_min, &output_activation_max);
+
+  BroadcastAdd(input1_data, input1_dims, input2_data, input2_dims,
+               output_activation_min, output_activation_max, output_data,
+               output_dims);
+}
+
+template 
+inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
+                int input1_shift, const int16* input2_data,
+                const Dims<4>& input2_dims, int input2_shift,
+                int16 output_activation_min, int16 output_activation_max,
+                int16* output_data, const Dims<4>& output_dims) {
+  constexpr int kReverseShift = -1;
+  static_assert(Ac == FusedActivationFunctionType::kNone ||
+                    Ac == FusedActivationFunctionType::kRelu ||
+                    Ac == FusedActivationFunctionType::kRelu6 ||
+                    Ac == FusedActivationFunctionType::kRelu1,
+                "");
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  if (Ac == FusedActivationFunctionType::kNone) {
+    TFLITE_DCHECK_EQ(output_activation_min, -32768);
+    TFLITE_DCHECK_EQ(output_activation_max, 32767);
+  }
+
+  tflite::ArithmeticParams op_params;
+  op_params.input1_shift = kReverseShift * input1_shift;
+  op_params.input2_shift = kReverseShift * input2_shift;
+  op_params.quantized_activation_min = output_activation_min;
+  op_params.quantized_activation_max = output_activation_max;
+  Add(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
+inline void Sub(const float* input1_data, const Dims<4>& input1_dims,
+                const float* input2_data, const Dims<4>& input2_dims,
+                float* output_data, const Dims<4>& output_dims) {
+  float output_activation_min, output_activation_max;
+  GetActivationMinMax(FusedActivationFunctionType::kNone,
+                      &output_activation_min, &output_activation_max);
+  tflite::ArithmeticParams op_params;
+  op_params.float_activation_min = output_activation_min;
+  op_params.float_activation_max = output_activation_max;
+  Sub(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
+template 
+void Sub(const T* input1_data, const Dims<4>& input1_dims, const T* input2_data,
+         const Dims<4>& input2_dims, T* output_data,
+         const Dims<4>& output_dims) {
+  T output_activation_min, output_activation_max;
+  GetActivationMinMax(FusedActivationFunctionType::kNone,
+                      &output_activation_min, &output_activation_max);
+  tflite::ArithmeticParams op_params;
+  op_params.quantized_activation_min = output_activation_min;
+  op_params.quantized_activation_max = output_activation_max;
+  Sub(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
 inline void AveragePool(const float* input_data, const Dims<4>& input_dims,
                         int stride_width, int stride_height, int pad_width,
                         int pad_height, int kwidth, int kheight,
diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
index 2f73036e03..78567d52ea 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
@@ -42,10 +42,12 @@ namespace optimized_ops {
 // Unoptimized reference ops:
 using reference_ops::ArgMax;
 using reference_ops::ArgMinMax;
+using reference_ops::BroadcastAdd4DSlow;
 using reference_ops::BroadcastGreater;
 using reference_ops::BroadcastGreaterEqual;
 using reference_ops::BroadcastLess;
 using reference_ops::BroadcastLessEqual;
+using reference_ops::BroadcastSub4DSlow;
 using reference_ops::Concatenation;
 using reference_ops::DepthConcatenation;
 using reference_ops::Dequantize;
@@ -217,98 +219,6 @@ SaturatingRoundingMultiplyByPOTParam(
       SaturatingRoundingMultiplyByPOTParam(a.raw(), exponent));
 }
 
-// DO NOT USE THIS STRUCT FOR NEW FUNCTIONALITY BEYOND IMPLEMENTING ELEMENT-WISE
-// BROADCASTING.
-//
-// NdArrayDesc describes the shape and memory layout of an N-dimensional
-// rectangular array of numbers.
-//
-// NdArrayDesc is basically identical to Dims defined in types.h.
-// However, as Dims is to be deprecated, this class exists as an adaptor
-// to enable simple unoptimized implementations of element-wise broadcasting
-// operations.
-template 
-struct NdArrayDesc {
-  // The "extent" of each dimension. Indices along dimension d must be in the
-  // half-open interval [0, extents[d]).
-  int extents[N];
-
-  // The number of *elements* (not bytes) between consecutive indices of each
-  // dimension.
-  int strides[N];
-};
-
-// DO NOT USE THIS FUNCTION FOR NEW FUNCTIONALITY BEYOND IMPLEMENTING
-// ELEMENT-WISE BROADCASTING.
-//
-// Same as Offset(), except takes as NdArrayDesc instead of Dims.
-inline int SubscriptToIndex(const NdArrayDesc<4>& desc, int i0, int i1, int i2,
-                            int i3) {
-  TFLITE_DCHECK(i0 >= 0 && i0 < desc.extents[0]);
-  TFLITE_DCHECK(i1 >= 0 && i1 < desc.extents[1]);
-  TFLITE_DCHECK(i2 >= 0 && i2 < desc.extents[2]);
-  TFLITE_DCHECK(i3 >= 0 && i3 < desc.extents[3]);
-  return i0 * desc.strides[0] + i1 * desc.strides[1] + i2 * desc.strides[2] +
-         i3 * desc.strides[3];
-}
-
-// Given the dimensions of the operands for an element-wise binary broadcast,
-// adjusts them so that they can be directly iterated over with simple loops.
-// Returns the adjusted dims as instances of NdArrayDesc in 'desc0_out' and
-// 'desc1_out'. 'desc0_out' and 'desc1_out' cannot be nullptr.
-//
-// This function assumes that the two input shapes are compatible up to
-// broadcasting and the shorter one has already been prepended with 1s to be the
-// same length. E.g., if shape0 is (1, 16, 16, 64) and shape1 is (1, 64),
-// shape1 must already have been prepended to be (1, 1, 1, 64). Recall that
-// Dims refer to shapes in reverse order. In this case, input0_dims will be
-// (64, 16, 16, 1) and input1_dims will be (64, 1, 1, 1).
-//
-// When two shapes are compatible up to broadcasting, for each dimension d,
-// the input extents are either equal, or one of them is 1.
-//
-// This function performs the following for each dimension d:
-// - If the extents are equal, then do nothing since the loop that walks over
-//   both of the input arrays is correct.
-// - Otherwise, one (and only one) of the extents must be 1. Say extent0 is 1
-//   and extent1 is e1. Then set extent0 to e1 and stride0 *to 0*. This allows
-//   array0 to be referenced *at any index* in dimension d and still access the
-//   same slice.
-template 
-inline void NdArrayDescsForElementwiseBroadcast(const Dims& input0_dims,
-                                                const Dims& input1_dims,
-                                                NdArrayDesc* desc0_out,
-                                                NdArrayDesc* desc1_out) {
-  TFLITE_DCHECK(desc0_out != nullptr);
-  TFLITE_DCHECK(desc1_out != nullptr);
-
-  // Copy dims to desc.
-  for (int i = 0; i < N; ++i) {
-    desc0_out->extents[i] = input0_dims.sizes[i];
-    desc0_out->strides[i] = input0_dims.strides[i];
-    desc1_out->extents[i] = input1_dims.sizes[i];
-    desc1_out->strides[i] = input1_dims.strides[i];
-  }
-
-  // Walk over each dimension. If the extents are equal do nothing.
-  // Otherwise, set the desc with extent 1 to have extent equal to the other and
-  // stride 0.
-  for (int i = 0; i < N; ++i) {
-    const int extent0 = ArraySize(input0_dims, i);
-    const int extent1 = ArraySize(input1_dims, i);
-    if (extent0 != extent1) {
-      if (extent0 == 1) {
-        desc0_out->strides[i] = 0;
-        desc0_out->extents[i] = extent1;
-      } else {
-        TFLITE_DCHECK_EQ(extent1, 1);
-        desc1_out->strides[i] = 0;
-        desc1_out->extents[i] = extent0;
-      }
-    }
-  }
-}
-
 inline bool AreSameDims(const Dims<4>& dims1, const Dims<4>& dims2) {
   for (int i = 0; i < 4; i++) {
     if (dims1.sizes[i] != dims2.sizes[i]) {
@@ -2478,20 +2388,17 @@ inline void L2Normalization(const uint8* input_data,
   }
 }
 
-inline void Add(const float* input1_data, const Dims<4>& input1_dims,
-                const float* input2_data, const Dims<4>& input2_dims,
-                float output_activation_min, float output_activation_max,
-                float* output_data, const Dims<4>& output_dims) {
+inline void Add(const ArithmeticParams& params,
+                const RuntimeShape& input1_shape, const float* input1_data,
+                const RuntimeShape& input2_shape, const float* input2_data,
+                const RuntimeShape& output_shape, float* output_data) {
   gemmlowp::ScopedProfilingLabel label("Add");
-  TFLITE_DCHECK(IsPackedWithoutStrides(input1_dims));
-  TFLITE_DCHECK(IsPackedWithoutStrides(input2_dims));
-  TFLITE_DCHECK(IsPackedWithoutStrides(output_dims));
 
   int i = 0;
-  const int size = MatchingFlatSize(input1_dims, input2_dims, output_dims);
+  const int size = MatchingFlatSize(input1_shape, input2_shape, output_shape);
 #ifdef USE_NEON
-  const auto activation_min = vdupq_n_f32(output_activation_min);
-  const auto activation_max = vdupq_n_f32(output_activation_max);
+  const auto activation_min = vdupq_n_f32(params.float_activation_min);
+  const auto activation_max = vdupq_n_f32(params.float_activation_max);
   for (; i <= size - 16; i += 16) {
     auto a10 = vld1q_f32(input1_data + i);
     auto a11 = vld1q_f32(input1_data + i + 4);
@@ -2530,29 +2437,26 @@ inline void Add(const float* input1_data, const Dims<4>& input1_dims,
 
   for (; i < size; i++) {
     auto x = input1_data[i] + input2_data[i];
-    output_data[i] = ActivationFunctionWithMinMax(x, output_activation_min,
-                                                  output_activation_max);
+    output_data[i] = ActivationFunctionWithMinMax(
+        x, params.float_activation_min, params.float_activation_max);
   }
 }
 
 // Element-wise add that can often be used for inner loop of broadcast add as
 // well as the non-broadcast add.
-inline void AddElementwise(int size, int left_shift, const uint8* input1_data,
-                           int32 input1_offset, int32 input1_multiplier,
-                           int input1_shift, const uint8* input2_data,
-                           int32 input2_offset, int32 input2_multiplier,
-                           int input2_shift, int32 output_offset,
-                           int32 output_multiplier, int output_shift,
-                           int32 output_activation_min,
-                           int32 output_activation_max, uint8* output_data) {
+inline void AddElementwise(int size, const ArithmeticParams& params,
+                           const uint8* input1_data, const uint8* input2_data,
+                           uint8* output_data) {
   int i = 0;
-  TFLITE_DCHECK_GT(input1_offset, -256);
-  TFLITE_DCHECK_GT(input2_offset, -256);
-  TFLITE_DCHECK_LT(input1_offset, 256);
-  TFLITE_DCHECK_LT(input2_offset, 256);
+  TFLITE_DCHECK_GT(params.input1_offset, -256);
+  TFLITE_DCHECK_GT(params.input2_offset, -256);
+  TFLITE_DCHECK_LT(params.input1_offset, 256);
+  TFLITE_DCHECK_LT(params.input2_offset, 256);
 #ifdef USE_NEON
-  const auto output_activation_min_vector = vdup_n_u8(output_activation_min);
-  const auto output_activation_max_vector = vdup_n_u8(output_activation_max);
+  const auto output_activation_min_vector =
+      vdup_n_u8(params.quantized_activation_min);
+  const auto output_activation_max_vector =
+      vdup_n_u8(params.quantized_activation_max);
   for (; i <= size - 8; i += 8) {
     const auto input1_val_original = vld1_u8(input1_data + i);
     const auto input2_val_original = vld1_u8(input2_data + i);
@@ -2561,9 +2465,9 @@ inline void AddElementwise(int size, int left_shift, const uint8* input1_data,
     const auto input2_val_s16 =
         vreinterpretq_s16_u16(vmovl_u8(input2_val_original));
     const auto input1_val =
-        vaddq_s16(input1_val_s16, vdupq_n_s16(input1_offset));
+        vaddq_s16(input1_val_s16, vdupq_n_s16(params.input1_offset));
     const auto input2_val =
-        vaddq_s16(input2_val_s16, vdupq_n_s16(input2_offset));
+        vaddq_s16(input2_val_s16, vdupq_n_s16(params.input2_offset));
     const auto input1_val_high = vget_high_s16(input1_val);
     const auto input1_val_low = vget_low_s16(input1_val);
     const auto input2_val_high = vget_high_s16(input2_val);
@@ -2572,32 +2476,32 @@ inline void AddElementwise(int size, int left_shift, const uint8* input1_data,
     auto x12 = vmovl_s16(input1_val_high);
     auto x21 = vmovl_s16(input2_val_low);
     auto x22 = vmovl_s16(input2_val_high);
-    const auto left_shift_dup = vdupq_n_s32(left_shift);
+    const auto left_shift_dup = vdupq_n_s32(params.left_shift);
     x11 = vshlq_s32(x11, left_shift_dup);
     x12 = vshlq_s32(x12, left_shift_dup);
     x21 = vshlq_s32(x21, left_shift_dup);
     x22 = vshlq_s32(x22, left_shift_dup);
-    x11 = vqrdmulhq_n_s32(x11, input1_multiplier);
-    x12 = vqrdmulhq_n_s32(x12, input1_multiplier);
-    x21 = vqrdmulhq_n_s32(x21, input2_multiplier);
-    x22 = vqrdmulhq_n_s32(x22, input2_multiplier);
-    const auto input1_shift_dup = vdupq_n_s32(-input1_shift);
-    const auto input2_shift_dup = vdupq_n_s32(-input2_shift);
+    x11 = vqrdmulhq_n_s32(x11, params.input1_multiplier);
+    x12 = vqrdmulhq_n_s32(x12, params.input1_multiplier);
+    x21 = vqrdmulhq_n_s32(x21, params.input2_multiplier);
+    x22 = vqrdmulhq_n_s32(x22, params.input2_multiplier);
+    const auto input1_shift_dup = vdupq_n_s32(params.input1_shift);
+    const auto input2_shift_dup = vdupq_n_s32(params.input2_shift);
     x11 = vshlq_s32(x11, input1_shift_dup);
     x12 = vshlq_s32(x12, input1_shift_dup);
     x21 = vshlq_s32(x21, input2_shift_dup);
     x22 = vshlq_s32(x22, input2_shift_dup);
     auto s1 = vaddq_s32(x11, x21);
     auto s2 = vaddq_s32(x12, x22);
-    s1 = vqrdmulhq_n_s32(s1, output_multiplier);
-    s2 = vqrdmulhq_n_s32(s2, output_multiplier);
+    s1 = vqrdmulhq_n_s32(s1, params.output_multiplier);
+    s2 = vqrdmulhq_n_s32(s2, params.output_multiplier);
     using gemmlowp::RoundingDivideByPOT;
-    s1 = RoundingDivideByPOT(s1, output_shift);
-    s2 = RoundingDivideByPOT(s2, output_shift);
+    s1 = RoundingDivideByPOT(s1, -params.output_shift);
+    s2 = RoundingDivideByPOT(s2, -params.output_shift);
     const auto s1_narrowed = vmovn_s32(s1);
     const auto s2_narrowed = vmovn_s32(s2);
     const auto s = vaddq_s16(vcombine_s16(s1_narrowed, s2_narrowed),
-                             vdupq_n_s16(output_offset));
+                             vdupq_n_s16(params.output_offset));
     const auto clamped =
         vmax_u8(output_activation_min_vector,
                 vmin_u8(output_activation_max_vector, vqmovun_s16(s)));
@@ -2606,101 +2510,74 @@ inline void AddElementwise(int size, int left_shift, const uint8* input1_data,
 #endif  // NEON
 
   for (; i < size; ++i) {
-    const int32 input1_val = input1_offset + input1_data[i];
-    const int32 input2_val = input2_offset + input2_data[i];
-    const int32 shifted_input1_val = input1_val * (1 << left_shift);
-    const int32 shifted_input2_val = input2_val * (1 << left_shift);
+    const int32 input1_val = params.input1_offset + input1_data[i];
+    const int32 input2_val = params.input2_offset + input2_data[i];
+    const int32 shifted_input1_val = input1_val * (1 << params.left_shift);
+    const int32 shifted_input2_val = input2_val * (1 << params.left_shift);
     const int32 scaled_input1_val =
         MultiplyByQuantizedMultiplierSmallerThanOneExp(
-            shifted_input1_val, input1_multiplier,
-            kReverseShift * input1_shift);
+            shifted_input1_val, params.input1_multiplier, params.input1_shift);
     const int32 scaled_input2_val =
         MultiplyByQuantizedMultiplierSmallerThanOneExp(
-            shifted_input2_val, input2_multiplier,
-            kReverseShift * input2_shift);
+            shifted_input2_val, params.input2_multiplier, params.input2_shift);
     const int32 raw_sum = scaled_input1_val + scaled_input2_val;
     const int32 raw_output =
         MultiplyByQuantizedMultiplierSmallerThanOneExp(
-            raw_sum, output_multiplier, kReverseShift * output_shift) +
-        output_offset;
-    const int32 clamped_output = std::min(
-        output_activation_max, std::max(output_activation_min, raw_output));
+            raw_sum, params.output_multiplier, params.output_shift) +
+        params.output_offset;
+    const int32 clamped_output =
+        std::min(params.quantized_activation_max,
+                 std::max(params.quantized_activation_min, raw_output));
     output_data[i] = static_cast(clamped_output);
   }
 }
 
-// legacy, for compatibility with old checked-in code
-template 
-void Add(const float* input1_data, const Dims<4>& input1_dims,
-         const float* input2_data, const Dims<4>& input2_dims,
-         float* output_data, const Dims<4>& output_dims) {
-  float output_activation_min, output_activation_max;
-  GetActivationMinMax(Ac, &output_activation_min, &output_activation_max);
-
-  Add(input1_data, input1_dims, input2_data, input2_dims, output_activation_min,
-      output_activation_max, output_data, output_dims);
-}
-
-template 
-inline void Add(int left_shift, const uint8* input1_data,
-                const Dims<4>& input1_dims, int32 input1_offset,
-                int32 input1_multiplier, int input1_shift,
-                const uint8* input2_data, const Dims<4>& input2_dims,
-                int32 input2_offset, int32 input2_multiplier, int input2_shift,
-                int32 output_offset, int32 output_multiplier, int output_shift,
-                int32 output_activation_min, int32 output_activation_max,
-                uint8* output_data, const Dims<4>& output_dims) {
-  static_assert(Ac == FusedActivationFunctionType::kNone ||
-                    Ac == FusedActivationFunctionType::kRelu ||
-                    Ac == FusedActivationFunctionType::kRelu6 ||
-                    Ac == FusedActivationFunctionType::kRelu1,
-                "");
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
-  if (Ac == FusedActivationFunctionType::kNone) {
-    TFLITE_DCHECK_EQ(output_activation_min, 0);
-    TFLITE_DCHECK_EQ(output_activation_max, 255);
-  }
+inline void Add(const ArithmeticParams& params,
+                const RuntimeShape& input1_shape, const uint8* input1_data,
+                const RuntimeShape& input2_shape, const uint8* input2_data,
+                const RuntimeShape& output_shape, uint8* output_data) {
+  TFLITE_DCHECK_LE(params.quantized_activation_min,
+                   params.quantized_activation_max);
   gemmlowp::ScopedProfilingLabel label("Add/8bit");
-  const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims);
-  TFLITE_DCHECK(IsPackedWithoutStrides(input1_dims));
-  TFLITE_DCHECK(IsPackedWithoutStrides(input2_dims));
-  TFLITE_DCHECK(IsPackedWithoutStrides(output_dims));
-
-  TFLITE_DCHECK_GT(input1_offset, -256);
-  TFLITE_DCHECK_GT(input2_offset, -256);
-  TFLITE_DCHECK_LT(input1_offset, 256);
-  TFLITE_DCHECK_LT(input2_offset, 256);
-  AddElementwise(flat_size, left_shift, input1_data, input1_offset,
-                 input1_multiplier, input1_shift, input2_data, input2_offset,
-                 input2_multiplier, input2_shift, output_offset,
-                 output_multiplier, output_shift, output_activation_min,
-                 output_activation_max, output_data);
+  const int flat_size =
+      MatchingFlatSize(input1_shape, input2_shape, output_shape);
+
+  TFLITE_DCHECK_GT(params.input1_offset, -256);
+  TFLITE_DCHECK_GT(params.input2_offset, -256);
+  TFLITE_DCHECK_LT(params.input1_offset, 256);
+  TFLITE_DCHECK_LT(params.input2_offset, 256);
+  AddElementwise(flat_size, params, input1_data, input2_data, output_data);
 }
 
-inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
-                int input1_shift, const int16* input2_data,
-                const Dims<4>& input2_dims, int input2_shift,
-                int16 output_activation_min, int16 output_activation_max,
-                int16* output_data, const Dims<4>& output_dims) {
+inline void Add(const ArithmeticParams& params,
+                const RuntimeShape& input1_shape, const int16* input1_data,
+                const RuntimeShape& input2_shape, const int16* input2_data,
+                const RuntimeShape& output_shape, int16* output_data) {
   gemmlowp::ScopedProfilingLabel label("Add/Int16");
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  TFLITE_DCHECK_LE(params.quantized_activation_min,
+                   params.quantized_activation_max);
 
-  const int flat_size = MatchingFlatSize(output_dims, input1_dims, input2_dims);
+  const int input1_shift = params.input1_shift;
+  const int flat_size =
+      MatchingFlatSize(output_shape, input1_shape, input2_shape);
+  const int16 output_activation_min = params.quantized_activation_min;
+  const int16 output_activation_max = params.quantized_activation_max;
 
-  TFLITE_DCHECK(input1_shift == 0 || input2_shift == 0);
-  TFLITE_DCHECK_GE(input1_shift, 0);
-  TFLITE_DCHECK_GE(input2_shift, 0);
+  TFLITE_DCHECK(input1_shift == 0 || params.input2_shift == 0);
+  TFLITE_DCHECK_LE(input1_shift, 0);
+  TFLITE_DCHECK_LE(params.input2_shift, 0);
   const int16* not_shift_input = input1_shift == 0 ? input1_data : input2_data;
   const int16* shift_input = input1_shift == 0 ? input2_data : input1_data;
-  const int input_shift = input1_shift == 0 ? input2_shift : input1_shift;
+  const int input_right_shift =
+      input1_shift == 0 ? -params.input2_shift : -input1_shift;
 
   for (int i = 0; i < flat_size; i++) {
     // F0 uses 0 integer bits, range [-1, 1].
     using F0 = gemmlowp::FixedPoint;
 
     F0 input_ready_scaled = F0::FromRaw(not_shift_input[i]);
-    F0 scaled_input =
-        F0::FromRaw(gemmlowp::RoundingDivideByPOT(shift_input[i], input_shift));
+    F0 scaled_input = F0::FromRaw(
+        gemmlowp::RoundingDivideByPOT(shift_input[i], input_right_shift));
     F0 result = gemmlowp::SaturatingAdd(scaled_input, input_ready_scaled);
     const int16 raw_output = result.raw();
     const int16 clamped_output = std::min(
@@ -2709,195 +2586,59 @@ inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
   }
 }
 
-inline void Add(const int32* input1_data, const Dims<4>& input1_dims,
-                const int32* input2_data, const Dims<4>& input2_dims,
-                int32 output_activation_min, int32 output_activation_max,
-                int32* output_data, const Dims<4>& output_dims) {
+inline void Add(const ArithmeticParams& params,
+                const RuntimeShape& input1_shape, const int32* input1_data,
+                const RuntimeShape& input2_shape, const int32* input2_data,
+                const RuntimeShape& output_shape, int32* output_data) {
   gemmlowp::ScopedProfilingLabel label("Add/int32");
 
-  const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims);
-  for (int i = 0; i < flat_size; ++i) {
-    output_data[i] = ActivationFunctionWithMinMax(
-        input1_data[i] + input2_data[i], output_activation_min,
-        output_activation_max);
-  }
-}
-
-template 
-inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
-                int input1_shift, const int16* input2_data,
-                const Dims<4>& input2_dims, int input2_shift,
-                int16 output_activation_min, int16 output_activation_max,
-                int16* output_data, const Dims<4>& output_dims) {
-  static_assert(Ac == FusedActivationFunctionType::kNone ||
-                    Ac == FusedActivationFunctionType::kRelu ||
-                    Ac == FusedActivationFunctionType::kRelu6 ||
-                    Ac == FusedActivationFunctionType::kRelu1,
-                "");
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
-  if (Ac == FusedActivationFunctionType::kNone) {
-    TFLITE_DCHECK_EQ(output_activation_min, -32768);
-    TFLITE_DCHECK_EQ(output_activation_max, 32767);
-  }
-
-  Add(input1_data, input1_dims, input1_shift, input2_data, input2_dims,
-      input2_shift, output_activation_min, output_activation_max, output_data,
-      output_dims);
-}
-
-template 
-void Add(const int32* input1_data, const Dims<4>& input1_dims,
-         const int32* input2_data, const Dims<4>& input2_dims,
-         int32* output_data, const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("Add/int32");
-  TFLITE_DCHECK(Ac == FusedActivationFunctionType::kNone);
-
-  auto input1_map = MapAsVector(input1_data, input1_dims);
-  auto input2_map = MapAsVector(input2_data, input2_dims);
-  auto output_map = MapAsVector(output_data, output_dims);
-  if (AreSameDims(input1_dims, input2_dims)) {
+  auto input1_map = MapAsVector(input1_data, input1_shape);
+  auto input2_map = MapAsVector(input2_data, input2_shape);
+  auto output_map = MapAsVector(output_data, output_shape);
+  if (input1_shape == input2_shape) {
     output_map.array() = input1_map.array() + input2_map.array();
-  } else if (FlatSize(input2_dims) == 1) {
+  } else if (input2_shape.FlatSize() == 1) {
     auto scalar = input2_data[0];
     output_map.array() = input1_map.array() + scalar;
-  } else if (FlatSize(input1_dims) == 1) {
+  } else if (input1_shape.FlatSize() == 1) {
     auto scalar = input1_data[0];
     output_map.array() = scalar + input2_map.array();
   } else {
     // Should not come here.
     TFLITE_DCHECK(false);
   }
+  output_map = output_map.cwiseMax(params.quantized_activation_min);
+  output_map = output_map.cwiseMin(params.quantized_activation_max);
 }
 
-// TODO(jiawen): We can implement BroadcastAdd on buffers of arbitrary
-// dimensionality if the runtime code does a single loop over one dimension
-// that handles broadcasting as the base case. The code generator would then
-// generate max(D1, D2) nested for loops.
-// TODO(benoitjacob): BroadcastAdd is intentionally duplicated from
-// reference_ops.h. Once an optimized version is implemented and NdArrayDesc
-// is no longer referenced in this file, move NdArrayDesc from types.h to
-// reference_ops.h.
-template 
-void BroadcastAdd(const T* input1_data, const Dims<4>& input1_dims,
-                  const T* input2_data, const Dims<4>& input2_dims,
-                  T output_activation_min, T output_activation_max,
-                  T* output_data, const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("BroadcastAdd");
-
-  NdArrayDesc<4> desc1;
-  NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
-
-  // In Tensorflow, the dimensions are canonically named (batch_number, row,
-  // col, channel), with extents (batches, height, width, depth), with the
-  // trailing dimension changing most rapidly (channels has the smallest stride,
-  // typically 1 element).
-  //
-  // In generated C code, we store arrays with the dimensions reversed. The
-  // first dimension has smallest stride.
-  //
-  // We name our variables by their Tensorflow convention, but generate C code
-  // nesting loops such that the innermost loop has the smallest stride for the
-  // best cache behavior.
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
-          output_data[Offset(output_dims, c, x, y, b)] =
-              ActivationFunctionWithMinMax(
-                  input1_data[SubscriptToIndex(desc1, c, x, y, b)] +
-                      input2_data[SubscriptToIndex(desc2, c, x, y, b)],
-                  output_activation_min, output_activation_max);
-        }
-      }
-    }
-  }
-}
-
-// legacy, for compatibility with old checked-in code
-template 
-void BroadcastAdd(const T* input1_data, const Dims<4>& input1_dims,
-                  const T* input2_data, const Dims<4>& input2_dims,
-                  T* output_data, const Dims<4>& output_dims) {
-  T output_activation_min, output_activation_max;
-  GetActivationMinMax(Ac, &output_activation_min, &output_activation_max);
-
-  BroadcastAdd(input1_data, input1_dims, input2_data, input2_dims,
-               output_activation_min, output_activation_max, output_data,
-               output_dims);
-}
-
-inline void BroadcastAdd(int left_shift, const uint8* input1_data,
-                         const Dims<4>& input1_dims, int32 input1_offset,
-                         int32 input1_multiplier, int input1_shift,
-                         const uint8* input2_data, const Dims<4>& input2_dims,
-                         int32 input2_offset, int32 input2_multiplier,
-                         int input2_shift, int32 output_offset,
-                         int32 output_multiplier, int output_shift,
-                         int32 output_activation_min,
-                         int32 output_activation_max, uint8* output_data,
-                         const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("BroadcastAddGeneric/8bit");
-
-  NdArrayDesc<4> desc1;
-  NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
-
-  // In Tensorflow, the dimensions are canonically named (batch_number, row,
-  // col, channel), with extents (batches, height, width, depth), with the
-  // trailing dimension changing most rapidly (channels has the smallest stride,
-  // typically 1 element).
-  //
-  // In generated C code, we store arrays with the dimensions reversed. The
-  // first dimension has smallest stride.
-  //
-  // We name our variables by their Tensorflow convention, but generate C code
-  // nesting loops such that the innermost loop has the smallest stride for the
-  // best cache behavior.
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
-          const int32 input1_val =
-              input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)];
-          const int32 input2_val =
-              input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)];
-          const int32 shifted_input1_val = input1_val * (1 << left_shift);
-          const int32 shifted_input2_val = input2_val * (1 << left_shift);
-          const int32 scaled_input1_val =
-              MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  shifted_input1_val, input1_multiplier,
-                  kReverseShift * input1_shift);
-          const int32 scaled_input2_val =
-              MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  shifted_input2_val, input2_multiplier,
-                  kReverseShift * input2_shift);
-          const int32 raw_sum = scaled_input1_val + scaled_input2_val;
-          const int32 raw_output =
-              MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  raw_sum, output_multiplier, kReverseShift * output_shift) +
-              output_offset;
-          const int32 clamped_output =
-              std::min(output_activation_max,
-                       std::max(output_activation_min, raw_output));
-          output_data[Offset(output_dims, c, x, y, b)] =
-              static_cast(clamped_output);
-        }
-      }
-    }
-  }
-}
-
-inline void BroadcastAddFivefold(
-    int y0, int y1, int y2, int y3, int y4, int left_shift,
-    const uint8* input1_data, const Dims<4>& input1_dims, int32 input1_offset,
-    int32 input1_multiplier, int input1_shift, const uint8* input2_data,
-    const Dims<4>& input2_dims, int32 input2_offset, int32 input2_multiplier,
-    int input2_shift, int32 output_offset, int32 output_multiplier,
-    int output_shift, int32 output_activation_min, int32 output_activation_max,
-    uint8* output_data, const Dims<4>& output_dims) {
+inline void BroadcastAddFivefold(const ArithmeticParams& unswitched_params,
+                                 const RuntimeShape& unswitched_input1_shape,
+                                 const uint8* unswitched_input1_data,
+                                 const RuntimeShape& unswitched_input2_shape,
+                                 const uint8* unswitched_input2_data,
+                                 const RuntimeShape& output_shape,
+                                 uint8* output_data) {
   gemmlowp::ScopedProfilingLabel label("BroadcastAddFivefold/8bit");
 
+  ArithmeticParams switched_params = unswitched_params;
+  switched_params.input1_offset = unswitched_params.input2_offset;
+  switched_params.input1_multiplier = unswitched_params.input2_multiplier;
+  switched_params.input1_shift = unswitched_params.input2_shift;
+  switched_params.input2_offset = unswitched_params.input1_offset;
+  switched_params.input2_multiplier = unswitched_params.input1_multiplier;
+  switched_params.input2_shift = unswitched_params.input1_shift;
+
+  const bool use_unswitched =
+      unswitched_params.broadcast_category ==
+      tflite::BroadcastableOpCategory::kFirstInputBroadcastsFast;
+
+  const ArithmeticParams& params =
+      use_unswitched ? unswitched_params : switched_params;
+  const uint8* input1_data =
+      use_unswitched ? unswitched_input1_data : unswitched_input2_data;
+  const uint8* input2_data =
+      use_unswitched ? unswitched_input2_data : unswitched_input1_data;
+
   // Fivefold nested loops. The second input resets its position for each
   // iteration of the second loop. The first input resets its position at the
   // beginning of the fourth loop. The innermost loop is an elementwise add of
@@ -2905,82 +2646,29 @@ inline void BroadcastAddFivefold(
   uint8* output_data_ptr = output_data;
   const uint8* input1_data_ptr = input1_data;
   const uint8* input2_data_reset = input2_data;
-  for (int i4 = 0; i4 < y4; ++i4) {
+  int y0 = params.broadcast_shape[0];
+  int y1 = params.broadcast_shape[1];
+  int y2 = params.broadcast_shape[2];
+  int y3 = params.broadcast_shape[3];
+  int y4 = params.broadcast_shape[4];
+  for (int i0 = 0; i0 < y0; ++i0) {
     const uint8* input2_data_ptr;
-    for (int i3 = 0; i3 < y3; ++i3) {
+    for (int i1 = 0; i1 < y1; ++i1) {
       input2_data_ptr = input2_data_reset;
       for (int i2 = 0; i2 < y2; ++i2) {
-        for (int i1 = 0; i1 < y1; ++i1) {
-          AddElementwise(
-              y0, left_shift, input1_data_ptr, input1_offset, input1_multiplier,
-              input1_shift, input2_data_ptr, input2_offset, input2_multiplier,
-              input2_shift, output_offset, output_multiplier, output_shift,
-              output_activation_min, output_activation_max, output_data_ptr);
-          input2_data_ptr += y0;
-          output_data_ptr += y0;
+        for (int i3 = 0; i3 < y3; ++i3) {
+          AddElementwise(y4, params, input1_data_ptr, input2_data_ptr,
+                         output_data_ptr);
+          input2_data_ptr += y4;
+          output_data_ptr += y4;
         }
-        input1_data_ptr += y0;
+        input1_data_ptr += y4;
       }
     }
     input2_data_reset = input2_data_ptr;
   }
 }
 
-template 
-inline void BroadcastAdd(int left_shift, const uint8* input1_data,
-                         const Dims<4>& input1_dims, int32 input1_offset,
-                         int32 input1_multiplier, int input1_shift,
-                         const uint8* input2_data, const Dims<4>& input2_dims,
-                         int32 input2_offset, int32 input2_multiplier,
-                         int input2_shift, int32 output_offset,
-                         int32 output_multiplier, int output_shift,
-                         int32 output_activation_min,
-                         int32 output_activation_max, uint8* output_data,
-                         const Dims<4>& output_dims) {
-  static_assert(Ac == FusedActivationFunctionType::kNone ||
-                    Ac == FusedActivationFunctionType::kRelu ||
-                    Ac == FusedActivationFunctionType::kRelu6 ||
-                    Ac == FusedActivationFunctionType::kRelu1,
-                "");
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
-  if (Ac == FusedActivationFunctionType::kNone) {
-    TFLITE_DCHECK_EQ(output_activation_min, 0);
-    TFLITE_DCHECK_EQ(output_activation_max, 255);
-  }
-  BroadcastAdd(left_shift, input1_data, input1_dims, input1_offset,
-               input1_multiplier, input1_shift, input2_data, input2_dims,
-               input2_offset, input2_multiplier, input2_shift, output_offset,
-               output_multiplier, output_shift, output_activation_min,
-               output_activation_max, output_data, output_dims);
-}
-
-template 
-inline void BroadcastAddFivefold(
-    int y0, int y1, int y2, int y3, int y4, int left_shift,
-    const uint8* input1_data, const Dims<4>& input1_dims, int32 input1_offset,
-    int32 input1_multiplier, int input1_shift, const uint8* input2_data,
-    const Dims<4>& input2_dims, int32 input2_offset, int32 input2_multiplier,
-    int input2_shift, int32 output_offset, int32 output_multiplier,
-    int output_shift, int32 output_activation_min, int32 output_activation_max,
-    uint8* output_data, const Dims<4>& output_dims) {
-  static_assert(Ac == FusedActivationFunctionType::kNone ||
-                    Ac == FusedActivationFunctionType::kRelu ||
-                    Ac == FusedActivationFunctionType::kRelu6 ||
-                    Ac == FusedActivationFunctionType::kRelu1,
-                "");
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
-  if (Ac == FusedActivationFunctionType::kNone) {
-    TFLITE_DCHECK_EQ(output_activation_min, 0);
-    TFLITE_DCHECK_EQ(output_activation_max, 255);
-  }
-  BroadcastAddFivefold(y0, y1, y2, y3, y4, left_shift, input1_data, input1_dims,
-                       input1_offset, input1_multiplier, input1_shift,
-                       input2_data, input2_dims, input2_offset,
-                       input2_multiplier, input2_shift, output_offset,
-                       output_multiplier, output_shift, output_activation_min,
-                       output_activation_max, output_data, output_dims);
-}
-
 inline void Mul(const float* input1_data, const Dims<4>& input1_dims,
                 const float* input2_data, const Dims<4>& input2_dims,
                 float output_activation_min, float output_activation_max,
@@ -3305,135 +2993,78 @@ void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims,
 }
 
 // TODO(aselle): This is not actually optimized yet.
-inline void Sub(const float* input1_data, const Dims<4>& input1_dims,
-                const float* input2_data, const Dims<4>& input2_dims,
-                float output_activation_min, float output_activation_max,
-                float* output_data, const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("Sub");
-  const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims);
+inline void SubNonBroadcast(const ArithmeticParams& params,
+                            const RuntimeShape& input1_shape,
+                            const float* input1_data,
+                            const RuntimeShape& input2_shape,
+                            const float* input2_data,
+                            const RuntimeShape& output_shape,
+                            float* output_data) {
+  gemmlowp::ScopedProfilingLabel label("SubNonBroadcast");
+  const int flat_size =
+      MatchingFlatSize(input1_shape, input2_shape, output_shape);
   for (int i = 0; i < flat_size; ++i) {
     output_data[i] = ActivationFunctionWithMinMax(
-        input1_data[i] - input2_data[i], output_activation_min,
-        output_activation_max);
+        input1_data[i] - input2_data[i], params.float_activation_min,
+        params.float_activation_max);
   }
 }
 
-inline void Sub(const int32* input1_data, const Dims<4>& input1_dims,
-                const int32* input2_data, const Dims<4>& input2_dims,
-                int32 output_activation_min, int32 output_activation_max,
-                int32* output_data, const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("Sub/int32");
-  const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims);
+inline void SubWithActivation(const ArithmeticParams& params,
+                              const RuntimeShape& input1_shape,
+                              const int32* input1_data,
+                              const RuntimeShape& input2_shape,
+                              const int32* input2_data,
+                              const RuntimeShape& output_shape,
+                              int32* output_data) {
+  gemmlowp::ScopedProfilingLabel label("SubWithActivation/int32");
+  const int flat_size =
+      MatchingFlatSize(input1_shape, input2_shape, input2_shape);
   for (int i = 0; i < flat_size; ++i) {
     output_data[i] = ActivationFunctionWithMinMax(
-        input1_data[i] - input2_data[i], output_activation_min,
-        output_activation_max);
+        input1_data[i] - input2_data[i], params.quantized_activation_min,
+        params.quantized_activation_max);
   }
 }
 
-// TODO(jiawen): We can implement BroadcastSub on buffers of arbitrary
-// dimensionality if the runtime code does a single loop over one dimension
-// that handles broadcasting as the base case. The code generator would then
-// generate max(D1, D2) nested for loops.
-// TODO(benoitjacob): BroadcastSub is intentionally duplicated from
-// reference_ops.h. Once an optimized version is implemented and NdArrayDesc
-// is no longer referenced in this file, move NdArrayDesc from types.h to
-// reference_ops.h.
-template 
-void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims,
-                  const T* input2_data, const Dims<4>& input2_dims,
-                  T output_activation_min, T output_activation_max,
-                  T* output_data, const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("BroadcastSub");
-
-  NdArrayDesc<4> desc1;
-  NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
-
-  // In Tensorflow, the dimensions are canonically named (batch_number, row,
-  // col, channel), with extents (batches, height, width, depth), with the
-  // trailing dimension changing most rapidly (channels has the smallest stride,
-  // typically 1 element).
-  //
-  // In generated C code, we store arrays with the dimensions reversed. The
-  // first dimension has smallest stride.
-  //
-  // We name our variables by their Tensorflow convention, but generate C code
-  // nesting loops such that the innermost loop has the smallest stride for the
-  // best cache behavior.
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
-          output_data[Offset(output_dims, c, x, y, b)] =
-              ActivationFunctionWithMinMax(
-                  input1_data[SubscriptToIndex(desc1, c, x, y, b)] -
-                      input2_data[SubscriptToIndex(desc2, c, x, y, b)],
-                  output_activation_min, output_activation_max);
-        }
-      }
-    }
+inline void SubWithActivation(const ArithmeticParams& params,
+                              const RuntimeShape& input1_shape,
+                              const float* input1_data,
+                              const RuntimeShape& input2_shape,
+                              const float* input2_data,
+                              const RuntimeShape& output_shape,
+                              float* output_data) {
+  gemmlowp::ScopedProfilingLabel label("SubWithActivation/float");
+  const int flat_size =
+      MatchingFlatSize(input1_shape, input2_shape, input2_shape);
+  for (int i = 0; i < flat_size; ++i) {
+    output_data[i] = ActivationFunctionWithMinMax(
+        input1_data[i] - input2_data[i], params.float_activation_min,
+        params.float_activation_max);
   }
 }
 
-inline void BroadcastSub(int left_shift, const uint8* input1_data,
-                         const Dims<4>& input1_dims, int32 input1_offset,
-                         int32 input1_multiplier, int input1_shift,
-                         const uint8* input2_data, const Dims<4>& input2_dims,
-                         int32 input2_offset, int32 input2_multiplier,
-                         int input2_shift, int32 output_offset,
-                         int32 output_multiplier, int output_shift,
-                         int32 output_activation_min,
-                         int32 output_activation_max, uint8* output_data,
-                         const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("BroadcastSub/8bit");
-
-  NdArrayDesc<4> desc1;
-  NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+template 
+void Sub(const ArithmeticParams& params, const RuntimeShape& input1_shape,
+         const T* input1_data, const RuntimeShape& input2_shape,
+         const T* input2_data, const RuntimeShape& output_shape,
+         T* output_data) {
+  gemmlowp::ScopedProfilingLabel label("Sub");
 
-  // In Tensorflow, the dimensions are canonically named (batch_number, row,
-  // col, channel), with extents (batches, height, width, depth), with the
-  // trailing dimension changing most rapidly (channels has the smallest stride,
-  // typically 1 element).
-  //
-  // In generated C code, we store arrays with the dimensions reversed. The
-  // first dimension has smallest stride.
-  //
-  // We name our variables by their Tensorflow convention, but generate C code
-  // nesting loops such that the innermost loop has the smallest stride for the
-  // best cache behavior.
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
-          const int32 input1_val =
-              input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)];
-          const int32 input2_val =
-              input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)];
-          const int32 shifted_input1_val = input1_val * (1 << left_shift);
-          const int32 shifted_input2_val = input2_val * (1 << left_shift);
-          const int32 scaled_input1_val =
-              MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  shifted_input1_val, input1_multiplier,
-                  kReverseShift * input1_shift);
-          const int32 scaled_input2_val =
-              MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  shifted_input2_val, input2_multiplier,
-                  kReverseShift * input2_shift);
-          const int32 raw_sub = scaled_input1_val - scaled_input2_val;
-          const int32 raw_output =
-              MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  raw_sub, output_multiplier, kReverseShift * output_shift) +
-              output_offset;
-          const int32 clamped_output =
-              std::min(output_activation_max,
-                       std::max(output_activation_min, raw_output));
-          output_data[Offset(output_dims, c, x, y, b)] =
-              static_cast(clamped_output);
-        }
-      }
-    }
+  auto input1_map = MapAsVector(input1_data, input1_shape);
+  auto input2_map = MapAsVector(input2_data, input2_shape);
+  auto output_map = MapAsVector(output_data, output_shape);
+  if (input1_shape == input2_shape) {
+    output_map.array() = input1_map.array() - input2_map.array();
+  } else if (input1_shape.FlatSize() == 1) {
+    auto scalar = input1_data[0];
+    output_map.array() = scalar - input2_map.array();
+  } else if (input2_shape.FlatSize() == 1) {
+    auto scalar = input2_data[0];
+    output_map.array() = input1_map.array() - scalar;
+  } else {
+    BroadcastSub4DSlow(params, input1_shape, input1_data, input2_shape,
+                       input2_data, output_shape, output_data);
   }
 }
 
@@ -5875,63 +5506,6 @@ inline void Slice(const T* input_data, const Dims<4>& input_dims,
   }
 }
 
-template 
-void GenericBroadcastSub(const T* input1_data, const Dims<4>& input1_dims,
-                         const T* input2_data, const Dims<4>& input2_dims,
-                         T* output_data, const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("GenericBroadcastSub");
-
-  NdArrayDesc<4> desc1;
-  NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
-
-  // In Tensorflow, the dimensions are canonically named (batch_number, row,
-  // col, channel), with extents (batches, height, width, depth), with the
-  // trailing dimension changing most rapidly (channels has the smallest stride,
-  // typically 1 element).
-  //
-  // In generated C code, we store arrays with the dimensions reversed. The
-  // first dimension has smallest stride.
-  //
-  // We name our variables by their Tensorflow convention, but generate C code
-  // nesting loops such that the innermost loop has the smallest stride for the
-  // best cache behavior.
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
-          output_data[Offset(output_dims, c, x, y, b)] =
-              input1_data[SubscriptToIndex(desc1, c, x, y, b)] -
-              input2_data[SubscriptToIndex(desc2, c, x, y, b)];
-        }
-      }
-    }
-  }
-}
-
-template 
-void Sub(const T* input1_data, const Dims<4>& input1_dims, const T* input2_data,
-         const Dims<4>& input2_dims, T* output_data,
-         const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("Sub");
-
-  auto input1_map = MapAsVector(input1_data, input1_dims);
-  auto input2_map = MapAsVector(input2_data, input2_dims);
-  auto output_map = MapAsVector(output_data, output_dims);
-  if (AreSameDims(input1_dims, input2_dims)) {
-    output_map.array() = input1_map.array() - input2_map.array();
-  } else if (FlatSize(input1_dims) == 1) {
-    auto scalar = input1_data[0];
-    output_map.array() = scalar - input2_map.array();
-  } else if (FlatSize(input2_dims) == 1) {
-    auto scalar = input2_data[0];
-    output_map.array() = input1_map.array() - scalar;
-  } else {
-    GenericBroadcastSub(input1_data, input1_dims, input2_data, input2_dims,
-                        output_data, output_dims);
-  }
-}
-
 template 
 void TensorFlowMinimum(const T* input1_data, const Dims<4>& input1_dims,
                        const T* input2_data, T* output_data,
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/legacy_reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/legacy_reference_ops.h
index f715d34bc1..bcf5e4e4f6 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/legacy_reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/legacy_reference_ops.h
@@ -63,6 +63,240 @@ inline void Relu6(const float* input_data, const Dims<4>& input_dims,
         DimsToShape(output_dims));
 }
 
+template 
+inline void Add(int left_shift, const uint8* input1_data,
+                const Dims<4>& input1_dims, int32 input1_offset,
+                int32 input1_multiplier, int input1_shift,
+                const uint8* input2_data, const Dims<4>& input2_dims,
+                int32 input2_offset, int32 input2_multiplier, int input2_shift,
+                int32 output_offset, int32 output_multiplier, int output_shift,
+                int32 output_activation_min, int32 output_activation_max,
+                uint8* output_data, const Dims<4>& output_dims) {
+  constexpr int kReverseShift = -1;
+  static_assert(Ac == FusedActivationFunctionType::kNone ||
+                    Ac == FusedActivationFunctionType::kRelu ||
+                    Ac == FusedActivationFunctionType::kRelu6 ||
+                    Ac == FusedActivationFunctionType::kRelu1,
+                "");
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  if (Ac == FusedActivationFunctionType::kNone) {
+    TFLITE_DCHECK_EQ(output_activation_min, 0);
+    TFLITE_DCHECK_EQ(output_activation_max, 255);
+  }
+
+  tflite::ArithmeticParams op_params;
+  op_params.left_shift = left_shift;
+  op_params.input1_offset = input1_offset;
+  op_params.input1_multiplier = input1_multiplier;
+  op_params.input1_shift = kReverseShift * input1_shift;
+  op_params.input2_offset = input2_offset;
+  op_params.input2_multiplier = input2_multiplier;
+  op_params.input2_shift = kReverseShift * input2_shift;
+  op_params.output_offset = output_offset;
+  op_params.output_multiplier = output_multiplier;
+  op_params.output_shift = kReverseShift * output_shift;
+  op_params.quantized_activation_min = output_activation_min;
+  op_params.quantized_activation_max = output_activation_max;
+  Add(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
+template 
+void Add(const int32* input1_data, const Dims<4>& input1_dims,
+         const int32* input2_data, const Dims<4>& input2_dims,
+         int32* output_data, const Dims<4>& output_dims) {
+  gemmlowp::ScopedProfilingLabel label("Add/int32");
+  TFLITE_DCHECK(Ac == FusedActivationFunctionType::kNone);
+
+  tflite::ArithmeticParams op_params;
+  op_params.quantized_activation_min = std::numeric_limits::min();
+  op_params.quantized_activation_max = std::numeric_limits::max();
+  Add(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
+template 
+inline void BroadcastAdd(int left_shift, const uint8* input1_data,
+                         const Dims<4>& input1_dims, int32 input1_offset,
+                         int32 input1_multiplier, int input1_shift,
+                         const uint8* input2_data, const Dims<4>& input2_dims,
+                         int32 input2_offset, int32 input2_multiplier,
+                         int input2_shift, int32 output_offset,
+                         int32 output_multiplier, int output_shift,
+                         int32 output_activation_min,
+                         int32 output_activation_max, uint8* output_data,
+                         const Dims<4>& output_dims) {
+  constexpr int kReverseShift = -1;
+  static_assert(Ac == FusedActivationFunctionType::kNone ||
+                    Ac == FusedActivationFunctionType::kRelu ||
+                    Ac == FusedActivationFunctionType::kRelu6 ||
+                    Ac == FusedActivationFunctionType::kRelu1,
+                "");
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  if (Ac == FusedActivationFunctionType::kNone) {
+    TFLITE_DCHECK_EQ(output_activation_min, 0);
+    TFLITE_DCHECK_EQ(output_activation_max, 255);
+  }
+
+  tflite::ArithmeticParams op_params;
+  op_params.left_shift = left_shift;
+  op_params.input1_offset = input1_offset;
+  op_params.input1_multiplier = input1_multiplier;
+  op_params.input1_shift = kReverseShift * input1_shift;
+  op_params.input2_offset = input2_offset;
+  op_params.input2_multiplier = input2_multiplier;
+  op_params.input2_shift = kReverseShift * input2_shift;
+  op_params.output_offset = output_offset;
+  op_params.output_multiplier = output_multiplier;
+  op_params.output_shift = kReverseShift * output_shift;
+  op_params.quantized_activation_min = output_activation_min;
+  op_params.quantized_activation_max = output_activation_max;
+  BroadcastAdd4DSlow(op_params, DimsToShape(input1_dims), input1_data,
+                     DimsToShape(input2_dims), input2_data,
+                     DimsToShape(output_dims), output_data);
+}
+
+template 
+void Add(const float* input1_data, const Dims<4>& input1_dims,
+         const float* input2_data, const Dims<4>& input2_dims,
+         float* output_data, const Dims<4>& output_dims) {
+  float output_activation_min, output_activation_max;
+  GetActivationMinMax(Ac, &output_activation_min, &output_activation_max);
+
+  tflite::ArithmeticParams op_params;
+  op_params.float_activation_min = output_activation_min;
+  op_params.float_activation_max = output_activation_max;
+  Add(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
+template 
+void BroadcastAdd(const T* input1_data, const Dims<4>& input1_dims,
+                  const T* input2_data, const Dims<4>& input2_dims,
+                  T output_activation_min, T output_activation_max,
+                  T* output_data, const Dims<4>& output_dims) {
+  tflite::ArithmeticParams op_params;
+  op_params.float_activation_min = output_activation_min;
+  op_params.float_activation_max = output_activation_max;
+  BroadcastAdd4DSlow(op_params, DimsToShape(input1_dims), input1_data,
+                     DimsToShape(input2_dims), input2_data,
+                     DimsToShape(output_dims), output_data);
+}
+
+template 
+inline void BroadcastAddFivefold(
+    int y0, int y1, int y2, int y3, int y4, int left_shift,
+    const uint8* input1_data, const Dims<4>& input1_dims, int32 input1_offset,
+    int32 input1_multiplier, int input1_shift, const uint8* input2_data,
+    const Dims<4>& input2_dims, int32 input2_offset, int32 input2_multiplier,
+    int input2_shift, int32 output_offset, int32 output_multiplier,
+    int output_shift, int32 output_activation_min, int32 output_activation_max,
+    uint8* output_data, const Dims<4>& output_dims) {
+  constexpr int kReverseShift = -1;
+  static_assert(Ac == FusedActivationFunctionType::kNone ||
+                    Ac == FusedActivationFunctionType::kRelu ||
+                    Ac == FusedActivationFunctionType::kRelu6 ||
+                    Ac == FusedActivationFunctionType::kRelu1,
+                "");
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  if (Ac == FusedActivationFunctionType::kNone) {
+    TFLITE_DCHECK_EQ(output_activation_min, 0);
+    TFLITE_DCHECK_EQ(output_activation_max, 255);
+  }
+  tflite::ArithmeticParams op_params;
+  op_params.broadcast_category =
+      tflite::BroadcastableOpCategory::kFirstInputBroadcastsFast;
+  op_params.left_shift = left_shift;
+  op_params.input1_offset = input1_offset;
+  op_params.input1_multiplier = input1_multiplier;
+  op_params.input1_shift = kReverseShift * input1_shift;
+  op_params.input2_offset = input2_offset;
+  op_params.input2_multiplier = input2_multiplier;
+  op_params.input2_shift = kReverseShift * input2_shift;
+  op_params.output_offset = output_offset;
+  op_params.output_multiplier = output_multiplier;
+  op_params.output_shift = kReverseShift * output_shift;
+  op_params.quantized_activation_min = output_activation_min;
+  op_params.quantized_activation_max = output_activation_max;
+  op_params.broadcast_shape[4] = y0;
+  op_params.broadcast_shape[3] = y1;
+  op_params.broadcast_shape[2] = y2;
+  op_params.broadcast_shape[1] = y3;
+  op_params.broadcast_shape[0] = y4;
+  BroadcastAddFivefold(op_params, DimsToShape(input1_dims), input1_data,
+                       DimsToShape(input2_dims), input2_data,
+                       DimsToShape(output_dims), output_data);
+}
+
+// legacy, for compatibility with old checked-in code
+template 
+void BroadcastAdd(const T* input1_data, const Dims<4>& input1_dims,
+                  const T* input2_data, const Dims<4>& input2_dims,
+                  T* output_data, const Dims<4>& output_dims) {
+  T output_activation_min, output_activation_max;
+  GetActivationMinMax(Ac, &output_activation_min, &output_activation_max);
+
+  BroadcastAdd(input1_data, input1_dims, input2_data, input2_dims,
+               output_activation_min, output_activation_max, output_data,
+               output_dims);
+}
+
+template 
+inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
+                int input1_shift, const int16* input2_data,
+                const Dims<4>& input2_dims, int input2_shift,
+                int16 output_activation_min, int16 output_activation_max,
+                int16* output_data, const Dims<4>& output_dims) {
+  static_assert(Ac == FusedActivationFunctionType::kNone ||
+                    Ac == FusedActivationFunctionType::kRelu ||
+                    Ac == FusedActivationFunctionType::kRelu6 ||
+                    Ac == FusedActivationFunctionType::kRelu1,
+                "");
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  if (Ac == FusedActivationFunctionType::kNone) {
+    TFLITE_DCHECK_EQ(output_activation_min, -32768);
+    TFLITE_DCHECK_EQ(output_activation_max, 32767);
+  }
+
+  tflite::ArithmeticParams op_params;
+  op_params.input1_shift = kReverseShift * input1_shift;
+  op_params.input2_shift = kReverseShift * input2_shift;
+  op_params.quantized_activation_min = output_activation_min;
+  op_params.quantized_activation_max = output_activation_max;
+  Add(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
+inline void Sub(const float* input1_data, const Dims<4>& input1_dims,
+                const float* input2_data, const Dims<4>& input2_dims,
+                float* output_data, const Dims<4>& output_dims) {
+  float output_activation_min, output_activation_max;
+  GetActivationMinMax(FusedActivationFunctionType::kNone,
+                      &output_activation_min, &output_activation_max);
+  tflite::ArithmeticParams op_params;
+  op_params.float_activation_min = output_activation_min;
+  op_params.float_activation_max = output_activation_max;
+  Sub(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
+template 
+void Sub(const T* input1_data, const Dims<4>& input1_dims, const T* input2_data,
+         const Dims<4>& input2_dims, T* output_data,
+         const Dims<4>& output_dims) {
+  tflite::ArithmeticParams op_params;
+  op_params.quantized_activation_min = std::numeric_limits::min();
+  op_params.quantized_activation_max = std::numeric_limits::max();
+  Sub(op_params, DimsToShape(input1_dims), input1_data,
+      DimsToShape(input2_dims), input2_data, DimsToShape(output_dims),
+      output_data);
+}
+
 inline void AveragePool(const float* input_data, const Dims<4>& input_dims,
                         int stride_width, int stride_height, int pad_width,
                         int pad_height, int kwidth, int kheight,
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index 04f61c7434..10e23f0b41 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -158,98 +158,6 @@ SaturatingRoundingMultiplyByPOTParam(
       SaturatingRoundingMultiplyByPOTParam(a.raw(), exponent));
 }
 
-// DO NOT USE THIS STRUCT FOR NEW FUNCTIONALITY BEYOND IMPLEMENTING ELEMENT-WISE
-// BROADCASTING.
-//
-// NdArrayDesc describes the shape and memory layout of an N-dimensional
-// rectangular array of numbers.
-//
-// NdArrayDesc is basically identical to Dims defined in types.h.
-// However, as Dims is to be deprecated, this class exists as an adaptor
-// to enable simple unoptimized implementations of element-wise broadcasting
-// operations.
-template 
-struct NdArrayDesc {
-  // The "extent" of each dimension. Indices along dimension d must be in the
-  // half-open interval [0, extents[d]).
-  int extents[N];
-
-  // The number of *elements* (not bytes) between consecutive indices of each
-  // dimension.
-  int strides[N];
-};
-
-// DO NOT USE THIS FUNCTION FOR NEW FUNCTIONALITY BEYOND IMPLEMENTING
-// ELEMENT-WISE BROADCASTING.
-//
-// Same as Offset(), except takes as NdArrayDesc instead of Dims.
-inline int SubscriptToIndex(const NdArrayDesc<4>& desc, int i0, int i1, int i2,
-                            int i3) {
-  TFLITE_DCHECK(i0 >= 0 && i0 < desc.extents[0]);
-  TFLITE_DCHECK(i1 >= 0 && i1 < desc.extents[1]);
-  TFLITE_DCHECK(i2 >= 0 && i2 < desc.extents[2]);
-  TFLITE_DCHECK(i3 >= 0 && i3 < desc.extents[3]);
-  return i0 * desc.strides[0] + i1 * desc.strides[1] + i2 * desc.strides[2] +
-         i3 * desc.strides[3];
-}
-
-// Given the dimensions of the operands for an element-wise binary broadcast,
-// adjusts them so that they can be directly iterated over with simple loops.
-// Returns the adjusted dims as instances of NdArrayDesc in 'desc0_out' and
-// 'desc1_out'. 'desc0_out' and 'desc1_out' cannot be nullptr.
-//
-// This function assumes that the two input shapes are compatible up to
-// broadcasting and the shorter one has already been prepended with 1s to be the
-// same length. E.g., if shape0 is (1, 16, 16, 64) and shape1 is (1, 64),
-// shape1 must already have been prepended to be (1, 1, 1, 64). Recall that
-// Dims refer to shapes in reverse order. In this case, input0_dims will be
-// (64, 16, 16, 1) and input1_dims will be (64, 1, 1, 1).
-//
-// When two shapes are compatible up to broadcasting, for each dimension d,
-// the input extents are either equal, or one of them is 1.
-//
-// This function performs the following for each dimension d:
-// - If the extents are equal, then do nothing since the loop that walks over
-//   both of the input arrays is correct.
-// - Otherwise, one (and only one) of the extents must be 1. Say extent0 is 1
-//   and extent1 is e1. Then set extent0 to e1 and stride0 *to 0*. This allows
-//   array0 to be referenced *at any index* in dimension d and still access the
-//   same slice.
-template 
-inline void NdArrayDescsForElementwiseBroadcast(const Dims& input0_dims,
-                                                const Dims& input1_dims,
-                                                NdArrayDesc* desc0_out,
-                                                NdArrayDesc* desc1_out) {
-  TFLITE_DCHECK(desc0_out != nullptr);
-  TFLITE_DCHECK(desc1_out != nullptr);
-
-  // Copy dims to desc.
-  for (int i = 0; i < N; ++i) {
-    desc0_out->extents[i] = input0_dims.sizes[i];
-    desc0_out->strides[i] = input0_dims.strides[i];
-    desc1_out->extents[i] = input1_dims.sizes[i];
-    desc1_out->strides[i] = input1_dims.strides[i];
-  }
-
-  // Walk over each dimension. If the extents are equal do nothing.
-  // Otherwise, set the desc with extent 1 to have extent equal to the other and
-  // stride 0.
-  for (int i = 0; i < N; ++i) {
-    const int extent0 = ArraySize(input0_dims, i);
-    const int extent1 = ArraySize(input1_dims, i);
-    if (extent0 != extent1) {
-      if (extent0 == 1) {
-        desc0_out->strides[i] = 0;
-        desc0_out->extents[i] = extent1;
-      } else {
-        TFLITE_DCHECK_EQ(extent1, 1);
-        desc1_out->strides[i] = 0;
-        desc1_out->extents[i] = extent0;
-      }
-    }
-  }
-}
-
 inline void Conv(const float* input_data, const Dims<4>& input_dims,
                  const float* filter_data, const Dims<4>& filter_dims,
                  const float* bias_data, const Dims<4>& bias_dims,
@@ -1065,114 +973,108 @@ inline void L2Normalization(const uint8* input_data,
 }
 
 template 
-inline void Add(const T* input1_data, const Dims<4>& input1_dims,
-                const T* input2_data, const Dims<4>& input2_dims,
-                T output_activation_min, T output_activation_max,
-                T* output_data, const Dims<4>& output_dims) {
-  const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims);
+inline void Add(const ArithmeticParams& params,
+                const RuntimeShape& input1_shape, const T* input1_data,
+                const RuntimeShape& input2_shape, const T* input2_data,
+                const RuntimeShape& output_shape, T* output_data) {
+  const int flat_size =
+      MatchingFlatSize(input1_shape, input2_shape, output_shape);
   for (int i = 0; i < flat_size; ++i) {
     output_data[i] = ActivationFunctionWithMinMax(
-        input1_data[i] + input2_data[i], output_activation_min,
-        output_activation_max);
+        input1_data[i] + input2_data[i], params.quantized_activation_min,
+        params.quantized_activation_max);
   }
 }
 
-// legacy, for compatibility with old checked-in code
-template 
-void Add(const float* input1_data, const Dims<4>& input1_dims,
-         const float* input2_data, const Dims<4>& input2_dims,
-         float* output_data, const Dims<4>& output_dims) {
-  float output_activation_min, output_activation_max;
-  GetActivationMinMax(Ac, &output_activation_min, &output_activation_max);
-
-  Add(input1_data, input1_dims, input2_data, input2_dims, output_activation_min,
-      output_activation_max, output_data, output_dims);
+inline void Add(const ArithmeticParams& params,
+                const RuntimeShape& input1_shape, const float* input1_data,
+                const RuntimeShape& input2_shape, const float* input2_data,
+                const RuntimeShape& output_shape, float* output_data) {
+  const int size = MatchingFlatSize(input1_shape, input2_shape, output_shape);
+  for (int i = 0; i < size; i++) {
+    auto x = input1_data[i] + input2_data[i];
+    output_data[i] = ActivationFunctionWithMinMax(
+        x, params.float_activation_min, params.float_activation_max);
+  }
 }
 
-template 
-inline void Add(int left_shift, const uint8* input1_data,
-                const Dims<4>& input1_dims, int32 input1_offset,
-                int32 input1_multiplier, int input1_shift,
-                const uint8* input2_data, const Dims<4>& input2_dims,
-                int32 input2_offset, int32 input2_multiplier, int input2_shift,
-                int32 output_offset, int32 output_multiplier, int output_shift,
-                int32 output_activation_min, int32 output_activation_max,
-                uint8* output_data, const Dims<4>& output_dims) {
-  static_assert(Ac == FusedActivationFunctionType::kNone ||
-                    Ac == FusedActivationFunctionType::kRelu ||
-                    Ac == FusedActivationFunctionType::kRelu6 ||
-                    Ac == FusedActivationFunctionType::kRelu1,
-                "");
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
-  if (Ac == FusedActivationFunctionType::kNone) {
-    TFLITE_DCHECK_EQ(output_activation_min, 0);
-    TFLITE_DCHECK_EQ(output_activation_max, 255);
-  }
-  const int batches =
-      MatchingArraySize(input1_dims, 3, input2_dims, 3, output_dims, 3);
-  const int height =
-      MatchingArraySize(input1_dims, 2, input2_dims, 2, output_dims, 2);
-  const int width =
-      MatchingArraySize(input1_dims, 1, input2_dims, 1, output_dims, 1);
-  const int depth =
-      MatchingArraySize(input1_dims, 0, input2_dims, 0, output_dims, 0);
-  for (int b = 0; b < batches; ++b) {
-    for (int y = 0; y < height; ++y) {
-      for (int x = 0; x < width; ++x) {
-        for (int c = 0; c < depth; ++c) {
-          const int32 input1_val =
-              input1_offset + input1_data[Offset(input1_dims, c, x, y, b)];
-          const int32 input2_val =
-              input2_offset + input2_data[Offset(input2_dims, c, x, y, b)];
-          const int32 shifted_input1_val = input1_val * (1 << left_shift);
-          const int32 shifted_input2_val = input2_val * (1 << left_shift);
-          const int32 scaled_input1_val =
-              MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  shifted_input1_val, input1_multiplier,
-                  kReverseShift * input1_shift);
-          const int32 scaled_input2_val =
-              MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  shifted_input2_val, input2_multiplier,
-                  kReverseShift * input2_shift);
-          const int32 raw_sum = scaled_input1_val + scaled_input2_val;
-          const int32 raw_output =
-              MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  raw_sum, output_multiplier, kReverseShift * output_shift) +
-              output_offset;
-          const int32 clamped_output =
-              std::min(output_activation_max,
-                       std::max(output_activation_min, raw_output));
-          output_data[Offset(output_dims, c, x, y, b)] =
-              static_cast(clamped_output);
-        }
-      }
-    }
+// Element-wise add that can often be used for inner loop of broadcast add as
+// well as the non-broadcast add.
+inline void AddElementwise(int size, const ArithmeticParams& params,
+                           const uint8* input1_data, const uint8* input2_data,
+                           uint8* output_data) {
+  TFLITE_DCHECK_GT(params.input1_offset, -256);
+  TFLITE_DCHECK_GT(params.input2_offset, -256);
+  TFLITE_DCHECK_LT(params.input1_offset, 256);
+  TFLITE_DCHECK_LT(params.input2_offset, 256);
+
+  for (int i = 0; i < size; ++i) {
+    const int32 input1_val = params.input1_offset + input1_data[i];
+    const int32 input2_val = params.input2_offset + input2_data[i];
+    const int32 shifted_input1_val = input1_val * (1 << params.left_shift);
+    const int32 shifted_input2_val = input2_val * (1 << params.left_shift);
+    const int32 scaled_input1_val =
+        MultiplyByQuantizedMultiplierSmallerThanOneExp(
+            shifted_input1_val, params.input1_multiplier, params.input1_shift);
+    const int32 scaled_input2_val =
+        MultiplyByQuantizedMultiplierSmallerThanOneExp(
+            shifted_input2_val, params.input2_multiplier, params.input2_shift);
+    const int32 raw_sum = scaled_input1_val + scaled_input2_val;
+    const int32 raw_output =
+        MultiplyByQuantizedMultiplierSmallerThanOneExp(
+            raw_sum, params.output_multiplier, params.output_shift) +
+        params.output_offset;
+    const int32 clamped_output =
+        std::min(params.quantized_activation_max,
+                 std::max(params.quantized_activation_min, raw_output));
+    output_data[i] = static_cast(clamped_output);
   }
 }
 
-inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
-                int input1_shift, const int16* input2_data,
-                const Dims<4>& input2_dims, int input2_shift,
-                int16 output_activation_min, int16 output_activation_max,
-                int16* output_data, const Dims<4>& output_dims) {
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+inline void Add(const ArithmeticParams& params,
+                const RuntimeShape& input1_shape, const uint8* input1_data,
+                const RuntimeShape& input2_shape, const uint8* input2_data,
+                const RuntimeShape& output_shape, uint8* output_data) {
+  TFLITE_DCHECK_LE(params.quantized_activation_min,
+                   params.quantized_activation_max);
+  const int flat_size =
+      MatchingFlatSize(input1_shape, input2_shape, output_shape);
 
-  const int flat_size = MatchingFlatSize(output_dims, input1_dims, input2_dims);
+  TFLITE_DCHECK_GT(params.input1_offset, -256);
+  TFLITE_DCHECK_GT(params.input2_offset, -256);
+  TFLITE_DCHECK_LT(params.input1_offset, 256);
+  TFLITE_DCHECK_LT(params.input2_offset, 256);
+  AddElementwise(flat_size, params, input1_data, input2_data, output_data);
+}
+
+inline void Add(const ArithmeticParams& params,
+                const RuntimeShape& input1_shape, const int16* input1_data,
+                const RuntimeShape& input2_shape, const int16* input2_data,
+                const RuntimeShape& output_shape, int16* output_data) {
+  TFLITE_DCHECK_LE(params.quantized_activation_min,
+                   params.quantized_activation_max);
+
+  const int input1_shift = params.input1_shift;
+  const int flat_size =
+      MatchingFlatSize(output_shape, input1_shape, input2_shape);
+  const int16 output_activation_min = params.quantized_activation_min;
+  const int16 output_activation_max = params.quantized_activation_max;
 
-  TFLITE_DCHECK(input1_shift == 0 || input2_shift == 0);
-  TFLITE_DCHECK_GE(input1_shift, 0);
-  TFLITE_DCHECK_GE(input2_shift, 0);
+  TFLITE_DCHECK(input1_shift == 0 || params.input2_shift == 0);
+  TFLITE_DCHECK_LE(input1_shift, 0);
+  TFLITE_DCHECK_LE(params.input2_shift, 0);
   const int16* not_shift_input = input1_shift == 0 ? input1_data : input2_data;
   const int16* shift_input = input1_shift == 0 ? input2_data : input1_data;
-  const int input_shift = input1_shift == 0 ? input2_shift : input1_shift;
+  const int input_right_shift =
+      input1_shift == 0 ? -params.input2_shift : -input1_shift;
 
   for (int i = 0; i < flat_size; i++) {
     // F0 uses 0 integer bits, range [-1, 1].
     using F0 = gemmlowp::FixedPoint;
 
     F0 input_ready_scaled = F0::FromRaw(not_shift_input[i]);
-    F0 scaled_input =
-        F0::FromRaw(gemmlowp::RoundingDivideByPOT(shift_input[i], input_shift));
+    F0 scaled_input = F0::FromRaw(
+        gemmlowp::RoundingDivideByPOT(shift_input[i], input_right_shift));
     F0 result = gemmlowp::SaturatingAdd(scaled_input, input_ready_scaled);
     const int16 raw_output = result.raw();
     const int16 clamped_output = std::min(
@@ -1181,42 +1083,28 @@ inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
   }
 }
 
-template 
-inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
-                int input1_shift, const int16* input2_data,
-                const Dims<4>& input2_dims, int input2_shift,
-                int16 output_activation_min, int16 output_activation_max,
-                int16* output_data, const Dims<4>& output_dims) {
-  static_assert(Ac == FusedActivationFunctionType::kNone ||
-                    Ac == FusedActivationFunctionType::kRelu ||
-                    Ac == FusedActivationFunctionType::kRelu6 ||
-                    Ac == FusedActivationFunctionType::kRelu1,
-                "");
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
-  if (Ac == FusedActivationFunctionType::kNone) {
-    TFLITE_DCHECK_EQ(output_activation_min, -32768);
-    TFLITE_DCHECK_EQ(output_activation_max, 32767);
-  }
-
-  Add(input1_data, input1_dims, input1_shift, input2_data, input2_dims,
-      input2_shift, output_activation_min, output_activation_max, output_data,
-      output_dims);
-}
-
 // TODO(jiawen): We can implement BroadcastAdd on buffers of arbitrary
 // dimensionality if the runtime code does a single loop over one dimension
 // that handles broadcasting as the base case. The code generator would then
 // generate max(D1, D2) nested for loops.
-template 
-void BroadcastAdd(const T* input1_data, const Dims<4>& input1_dims,
-                  const T* input2_data, const Dims<4>& input2_dims,
-                  T output_activation_min, T output_activation_max,
-                  T* output_data, const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("BroadcastAdd");
-
+// TODO(benoitjacob): BroadcastAdd is intentionally duplicated from
+// reference_ops.h. Once an optimized version is implemented and NdArrayDesc
+// is no longer referenced in this file, move NdArrayDesc from types.h to
+// reference_ops.h.
+inline void BroadcastAdd4DSlow(const ArithmeticParams& params,
+                               const RuntimeShape& input1_shape,
+                               const float* input1_data,
+                               const RuntimeShape& input2_shape,
+                               const float* input2_data,
+                               const RuntimeShape& output_shape,
+                               float* output_data) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastAdd4DSlow/float");
   NdArrayDesc<4> desc1;
   NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+  NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1,
+                                      &desc2);
+  RuntimeShape extended_output_shape =
+      RuntimeShape::ExtendedShape(4, output_shape);
 
   // In Tensorflow, the dimensions are canonically named (batch_number, row,
   // col, channel), with extents (batches, height, width, depth), with the
@@ -1229,49 +1117,77 @@ void BroadcastAdd(const T* input1_data, const Dims<4>& input1_dims,
   // We name our variables by their Tensorflow convention, but generate C code
   // nesting loops such that the innermost loop has the smallest stride for the
   // best cache behavior.
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
-          output_data[Offset(output_dims, c, x, y, b)] =
+  for (int b = 0; b < extended_output_shape.Dims(0); ++b) {
+    for (int y = 0; y < extended_output_shape.Dims(1); ++y) {
+      for (int x = 0; x < extended_output_shape.Dims(2); ++x) {
+        for (int c = 0; c < extended_output_shape.Dims(3); ++c) {
+          output_data[Offset(extended_output_shape, b, y, x, c)] =
               ActivationFunctionWithMinMax(
-                  input1_data[SubscriptToIndex(desc1, c, x, y, b)] +
-                      input2_data[SubscriptToIndex(desc2, c, x, y, b)],
-                  output_activation_min, output_activation_max);
+                  input1_data[SubscriptToIndex(desc1, b, y, x, c)] +
+                      input2_data[SubscriptToIndex(desc2, b, y, x, c)],
+                  params.float_activation_min, params.float_activation_max);
         }
       }
     }
   }
 }
 
-// legacy, for compatibility with old checked-in code
-template 
-void BroadcastAdd(const T* input1_data, const Dims<4>& input1_dims,
-                  const T* input2_data, const Dims<4>& input2_dims,
-                  T* output_data, const Dims<4>& output_dims) {
-  T output_activation_min, output_activation_max;
-  GetActivationMinMax(Ac, &output_activation_min, &output_activation_max);
+inline void BroadcastAdd4DSlow(const ArithmeticParams& params,
+                               const RuntimeShape& input1_shape,
+                               const int32* input1_data,
+                               const RuntimeShape& input2_shape,
+                               const int32* input2_data,
+                               const RuntimeShape& output_shape,
+                               int32* output_data) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastAdd4DSlow/int32");
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1,
+                                      &desc2);
+  RuntimeShape extended_output_shape =
+      RuntimeShape::ExtendedShape(4, output_shape);
 
-  BroadcastAdd(input1_data, input1_dims, input2_data, input2_dims,
-               output_activation_min, output_activation_max, output_data,
-               output_dims);
+  // In Tensorflow, the dimensions are canonically named (batch_number, row,
+  // col, channel), with extents (batches, height, width, depth), with the
+  // trailing dimension changing most rapidly (channels has the smallest stride,
+  // typically 1 element).
+  //
+  // In generated C code, we store arrays with the dimensions reversed. The
+  // first dimension has smallest stride.
+  //
+  // We name our variables by their Tensorflow convention, but generate C code
+  // nesting loops such that the innermost loop has the smallest stride for the
+  // best cache behavior.
+  for (int b = 0; b < extended_output_shape.Dims(0); ++b) {
+    for (int y = 0; y < extended_output_shape.Dims(1); ++y) {
+      for (int x = 0; x < extended_output_shape.Dims(2); ++x) {
+        for (int c = 0; c < extended_output_shape.Dims(3); ++c) {
+          output_data[Offset(extended_output_shape, b, y, x, c)] =
+              ActivationFunctionWithMinMax(
+                  input1_data[SubscriptToIndex(desc1, b, y, x, c)] +
+                      input2_data[SubscriptToIndex(desc2, b, y, x, c)],
+                  params.quantized_activation_min,
+                  params.quantized_activation_max);
+        }
+      }
+    }
+  }
 }
 
-inline void BroadcastAdd(int left_shift, const uint8* input1_data,
-                         const Dims<4>& input1_dims, int32 input1_offset,
-                         int32 input1_multiplier, int input1_shift,
-                         const uint8* input2_data, const Dims<4>& input2_dims,
-                         int32 input2_offset, int32 input2_multiplier,
-                         int input2_shift, int32 output_offset,
-                         int32 output_multiplier, int output_shift,
-                         int32 output_activation_min,
-                         int32 output_activation_max, uint8* output_data,
-                         const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("BroadcastAdd/8bit");
-
+inline void BroadcastAdd4DSlow(const ArithmeticParams& params,
+                               const RuntimeShape& input1_shape,
+                               const uint8* input1_data,
+                               const RuntimeShape& input2_shape,
+                               const uint8* input2_data,
+                               const RuntimeShape& output_shape,
+                               uint8* output_data) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastAdd4DSlow/uint8");
   NdArrayDesc<4> desc1;
   NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+  NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1,
+                                      &desc2);
+  RuntimeShape extended_output_shape =
+      RuntimeShape::ExtendedShape(4, output_shape);
 
   // In Tensorflow, the dimensions are canonically named (batch_number, row,
   // col, channel), with extents (batches, height, width, depth), with the
@@ -1284,33 +1200,37 @@ inline void BroadcastAdd(int left_shift, const uint8* input1_data,
   // We name our variables by their Tensorflow convention, but generate C code
   // nesting loops such that the innermost loop has the smallest stride for the
   // best cache behavior.
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+  for (int b = 0; b < extended_output_shape.Dims(0); ++b) {
+    for (int y = 0; y < extended_output_shape.Dims(1); ++y) {
+      for (int x = 0; x < extended_output_shape.Dims(2); ++x) {
+        for (int c = 0; c < extended_output_shape.Dims(3); ++c) {
           const int32 input1_val =
-              input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)];
+              params.input1_offset +
+              input1_data[SubscriptToIndex(desc1, b, y, x, c)];
           const int32 input2_val =
-              input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)];
-          const int32 shifted_input1_val = input1_val * (1 << left_shift);
-          const int32 shifted_input2_val = input2_val * (1 << left_shift);
+              params.input2_offset +
+              input2_data[SubscriptToIndex(desc2, b, y, x, c)];
+          const int32 shifted_input1_val =
+              input1_val * (1 << params.left_shift);
+          const int32 shifted_input2_val =
+              input2_val * (1 << params.left_shift);
           const int32 scaled_input1_val =
               MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  shifted_input1_val, input1_multiplier,
-                  kReverseShift * input1_shift);
+                  shifted_input1_val, params.input1_multiplier,
+                  params.input1_shift);
           const int32 scaled_input2_val =
               MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  shifted_input2_val, input2_multiplier,
-                  kReverseShift * input2_shift);
+                  shifted_input2_val, params.input2_multiplier,
+                  params.input2_shift);
           const int32 raw_sum = scaled_input1_val + scaled_input2_val;
           const int32 raw_output =
               MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  raw_sum, output_multiplier, kReverseShift * output_shift) +
-              output_offset;
+                  raw_sum, params.output_multiplier, params.output_shift) +
+              params.output_offset;
           const int32 clamped_output =
-              std::min(output_activation_max,
-                       std::max(output_activation_min, raw_output));
-          output_data[Offset(output_dims, c, x, y, b)] =
+              std::min(params.quantized_activation_max,
+                       std::max(params.quantized_activation_min, raw_output));
+          output_data[Offset(extended_output_shape, b, y, x, c)] =
               static_cast(clamped_output);
         }
       }
@@ -1318,117 +1238,62 @@ inline void BroadcastAdd(int left_shift, const uint8* input1_data,
   }
 }
 
-inline void BroadcastAddFivefold(
-    int y0, int y1, int y2, int y3, int y4, int left_shift,
-    const uint8* input1_data, const Dims<4>& input1_dims, int32 input1_offset,
-    int32 input1_multiplier, int input1_shift, const uint8* input2_data,
-    const Dims<4>& input2_dims, int32 input2_offset, int32 input2_multiplier,
-    int input2_shift, int32 output_offset, int32 output_multiplier,
-    int output_shift, int32 output_activation_min, int32 output_activation_max,
-    uint8* output_data, const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("BroadcastAddFivefold/8bit");
-
-  int sb1 = y0;
-  int sa2 = y0;
-  int sb2 = y0 * y1;
-  int sa3 = y0 * y2;
-  int sa4 = y0 * y2 * y3;
-  int sb4 = y0 * y1 * y2;
-
+inline void BroadcastAddFivefold(const ArithmeticParams& unswitched_params,
+                                 const RuntimeShape& unswitched_input1_shape,
+                                 const uint8* unswitched_input1_data,
+                                 const RuntimeShape& unswitched_input2_shape,
+                                 const uint8* unswitched_input2_data,
+                                 const RuntimeShape& output_shape,
+                                 uint8* output_data) {
+  ArithmeticParams switched_params = unswitched_params;
+  switched_params.input1_offset = unswitched_params.input2_offset;
+  switched_params.input1_multiplier = unswitched_params.input2_multiplier;
+  switched_params.input1_shift = unswitched_params.input2_shift;
+  switched_params.input2_offset = unswitched_params.input1_offset;
+  switched_params.input2_multiplier = unswitched_params.input1_multiplier;
+  switched_params.input2_shift = unswitched_params.input1_shift;
+
+  const bool use_unswitched =
+      unswitched_params.broadcast_category ==
+      tflite::BroadcastableOpCategory::kFirstInputBroadcastsFast;
+
+  const ArithmeticParams& params =
+      use_unswitched ? unswitched_params : switched_params;
+  const uint8* input1_data =
+      use_unswitched ? unswitched_input1_data : unswitched_input2_data;
+  const uint8* input2_data =
+      use_unswitched ? unswitched_input2_data : unswitched_input1_data;
+
+  // Fivefold nested loops. The second input resets its position for each
+  // iteration of the second loop. The first input resets its position at the
+  // beginning of the fourth loop. The innermost loop is an elementwise add of
+  // sections of the arrays.
   uint8* output_data_ptr = output_data;
-  for (int i4 = 0; i4 < y4; ++i4) {
-    for (int i3 = 0; i3 < y3; ++i3) {
+  const uint8* input1_data_ptr = input1_data;
+  const uint8* input2_data_reset = input2_data;
+  int y0 = params.broadcast_shape[0];
+  int y1 = params.broadcast_shape[1];
+  int y2 = params.broadcast_shape[2];
+  int y3 = params.broadcast_shape[3];
+  int y4 = params.broadcast_shape[4];
+  for (int i0 = 0; i0 < y0; ++i0) {
+    const uint8* input2_data_ptr;
+    for (int i1 = 0; i1 < y1; ++i1) {
+      input2_data_ptr = input2_data_reset;
       for (int i2 = 0; i2 < y2; ++i2) {
-        for (int i1 = 0; i1 < y1; ++i1) {
-          for (int i0 = 0; i0 < y0; ++i0) {
-            const int32 input1_val =
-                input1_offset +
-                input1_data[i4 * sa4 + i3 * sa3 + i2 * sa2 + i0];
-            const int32 input2_val =
-                input2_offset +
-                input2_data[i4 * sb4 + i2 * sb2 + i1 * sb1 + i0];
-            const int32 shifted_input1_val = input1_val * (1 << left_shift);
-            const int32 shifted_input2_val = input2_val * (1 << left_shift);
-            const int32 scaled_input1_val =
-                MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                    shifted_input1_val, input1_multiplier,
-                    kReverseShift * input1_shift);
-            const int32 scaled_input2_val =
-                MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                    shifted_input2_val, input2_multiplier,
-                    kReverseShift * input2_shift);
-            const int32 raw_sum = scaled_input1_val + scaled_input2_val;
-            const int32 raw_output =
-                MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                    raw_sum, output_multiplier, kReverseShift * output_shift) +
-                output_offset;
-            const int32 clamped_output =
-                std::min(output_activation_max,
-                         std::max(output_activation_min, raw_output));
-            *output_data_ptr = static_cast(clamped_output);
-            ++output_data_ptr;
-          }
+        for (int i3 = 0; i3 < y3; ++i3) {
+          AddElementwise(y4, params, input1_data_ptr, input2_data_ptr,
+                         output_data_ptr);
+          input2_data_ptr += y4;
+          output_data_ptr += y4;
         }
+        input1_data_ptr += y4;
       }
     }
+    input2_data_reset = input2_data_ptr;
   }
 }
 
-template 
-inline void BroadcastAdd(int left_shift, const uint8* input1_data,
-                         const Dims<4>& input1_dims, int32 input1_offset,
-                         int32 input1_multiplier, int input1_shift,
-                         const uint8* input2_data, const Dims<4>& input2_dims,
-                         int32 input2_offset, int32 input2_multiplier,
-                         int input2_shift, int32 output_offset,
-                         int32 output_multiplier, int output_shift,
-                         int32 output_activation_min,
-                         int32 output_activation_max, uint8* output_data,
-                         const Dims<4>& output_dims) {
-  static_assert(Ac == FusedActivationFunctionType::kNone ||
-                    Ac == FusedActivationFunctionType::kRelu ||
-                    Ac == FusedActivationFunctionType::kRelu6 ||
-                    Ac == FusedActivationFunctionType::kRelu1,
-                "");
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
-  if (Ac == FusedActivationFunctionType::kNone) {
-    TFLITE_DCHECK_EQ(output_activation_min, 0);
-    TFLITE_DCHECK_EQ(output_activation_max, 255);
-  }
-  BroadcastAdd(left_shift, input1_data, input1_dims, input1_offset,
-               input1_multiplier, input1_shift, input2_data, input2_dims,
-               input2_offset, input2_multiplier, input2_shift, output_offset,
-               output_multiplier, output_shift, output_activation_min,
-               output_activation_max, output_data, output_dims);
-}
-
-template 
-inline void BroadcastAddFivefold(
-    int y0, int y1, int y2, int y3, int y4, int left_shift,
-    const uint8* input1_data, const Dims<4>& input1_dims, int32 input1_offset,
-    int32 input1_multiplier, int input1_shift, const uint8* input2_data,
-    const Dims<4>& input2_dims, int32 input2_offset, int32 input2_multiplier,
-    int input2_shift, int32 output_offset, int32 output_multiplier,
-    int output_shift, int32 output_activation_min, int32 output_activation_max,
-    uint8* output_data, const Dims<4>& output_dims) {
-  static_assert(Ac == FusedActivationFunctionType::kNone ||
-                    Ac == FusedActivationFunctionType::kRelu ||
-                    Ac == FusedActivationFunctionType::kRelu6 ||
-                    Ac == FusedActivationFunctionType::kRelu1,
-                "");
-  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
-  if (Ac == FusedActivationFunctionType::kNone) {
-    TFLITE_DCHECK_EQ(output_activation_min, 0);
-    TFLITE_DCHECK_EQ(output_activation_max, 255);
-  }
-  BroadcastAddFivefold(y0, y1, y2, y3, y4, left_shift, input1_data, input1_dims,
-                       input1_offset, input1_multiplier, input1_shift,
-                       input2_data, input2_dims, input2_offset,
-                       input2_multiplier, input2_shift, output_offset,
-                       output_multiplier, output_shift, output_activation_min,
-                       output_activation_max, output_data, output_dims);
-}
-
 template 
 inline void Mul(const T* input1_data, const Dims<4>& input1_dims,
                 const T* input2_data, const Dims<4>& input2_dims,
@@ -1667,16 +1532,35 @@ inline void Div(const T* input1_data, const Dims<4>& input1_dims,
   }
 }
 
-template 
-inline void Sub(const T* input1_data, const Dims<4>& input1_dims,
-                const T* input2_data, const Dims<4>& input2_dims,
-                T output_activation_min, T output_activation_max,
-                T* output_data, const Dims<4>& output_dims) {
-  const int flat_size = MatchingFlatSize(input1_dims, input2_dims, output_dims);
+inline void SubNonBroadcast(const ArithmeticParams& params,
+                            const RuntimeShape& input1_shape,
+                            const float* input1_data,
+                            const RuntimeShape& input2_shape,
+                            const float* input2_data,
+                            const RuntimeShape& output_shape,
+                            float* output_data) {
+  const int flat_size =
+      MatchingFlatSize(input1_shape, input2_shape, output_shape);
   for (int i = 0; i < flat_size; ++i) {
     output_data[i] = ActivationFunctionWithMinMax(
-        input1_data[i] - input2_data[i], output_activation_min,
-        output_activation_max);
+        input1_data[i] - input2_data[i], params.float_activation_min,
+        params.float_activation_max);
+  }
+}
+
+inline void SubNonBroadcast(const ArithmeticParams& params,
+                            const RuntimeShape& input1_shape,
+                            const int32* input1_data,
+                            const RuntimeShape& input2_shape,
+                            const int32* input2_data,
+                            const RuntimeShape& output_shape,
+                            int32* output_data) {
+  const int flat_size =
+      MatchingFlatSize(input1_shape, input2_shape, output_shape);
+  for (int i = 0; i < flat_size; ++i) {
+    output_data[i] = ActivationFunctionWithMinMax(
+        input1_data[i] - input2_data[i], params.quantized_activation_min,
+        params.quantized_activation_max);
   }
 }
 
@@ -1684,16 +1568,24 @@ inline void Sub(const T* input1_data, const Dims<4>& input1_dims,
 // dimensionality if the runtime code does a single loop over one dimension
 // that handles broadcasting as the base case. The code generator would then
 // generate max(D1, D2) nested for loops.
-template 
-void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims,
-                  const T* input2_data, const Dims<4>& input2_dims,
-                  T output_activation_min, T output_activation_max,
-                  T* output_data, const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("BroadcastSub");
-
+// TODO(benoitjacob): BroadcastSub is intentionally duplicated from
+// reference_ops.h. Once an optimized version is implemented and NdArrayDesc
+// is no longer referenced in this file, move NdArrayDesc from types.h to
+// reference_ops.h.
+inline void BroadcastSub4DSlow(const ArithmeticParams& params,
+                               const RuntimeShape& input1_shape,
+                               const float* input1_data,
+                               const RuntimeShape& input2_shape,
+                               const float* input2_data,
+                               const RuntimeShape& output_shape,
+                               float* output_data) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastAdd4DSlow/float");
   NdArrayDesc<4> desc1;
   NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+  NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1,
+                                      &desc2);
+  RuntimeShape extended_output_shape =
+      RuntimeShape::ExtendedShape(4, output_shape);
 
   // In Tensorflow, the dimensions are canonically named (batch_number, row,
   // col, channel), with extents (batches, height, width, depth), with the
@@ -1706,36 +1598,35 @@ void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims,
   // We name our variables by their Tensorflow convention, but generate C code
   // nesting loops such that the innermost loop has the smallest stride for the
   // best cache behavior.
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
-          output_data[Offset(output_dims, c, x, y, b)] =
+  for (int b = 0; b < extended_output_shape.Dims(0); ++b) {
+    for (int y = 0; y < extended_output_shape.Dims(1); ++y) {
+      for (int x = 0; x < extended_output_shape.Dims(2); ++x) {
+        for (int c = 0; c < extended_output_shape.Dims(3); ++c) {
+          output_data[Offset(extended_output_shape, b, y, x, c)] =
               ActivationFunctionWithMinMax(
-                  input1_data[SubscriptToIndex(desc1, c, x, y, b)] -
-                      input2_data[SubscriptToIndex(desc2, c, x, y, b)],
-                  output_activation_min, output_activation_max);
+                  input1_data[SubscriptToIndex(desc1, b, y, x, c)] -
+                      input2_data[SubscriptToIndex(desc2, b, y, x, c)],
+                  params.float_activation_min, params.float_activation_max);
         }
       }
     }
   }
 }
 
-inline void BroadcastSub(int left_shift, const uint8* input1_data,
-                         const Dims<4>& input1_dims, int32 input1_offset,
-                         int32 input1_multiplier, int input1_shift,
-                         const uint8* input2_data, const Dims<4>& input2_dims,
-                         int32 input2_offset, int32 input2_multiplier,
-                         int input2_shift, int32 output_offset,
-                         int32 output_multiplier, int output_shift,
-                         int32 output_activation_min,
-                         int32 output_activation_max, uint8* output_data,
-                         const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("BroadcastSub/8bit");
-
+inline void BroadcastSub4DSlow(const ArithmeticParams& params,
+                               const RuntimeShape& input1_shape,
+                               const uint8* input1_data,
+                               const RuntimeShape& input2_shape,
+                               const uint8* input2_data,
+                               const RuntimeShape& output_shape,
+                               uint8* output_data) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastAdd4DSlow/uint8");
   NdArrayDesc<4> desc1;
   NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+  NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1,
+                                      &desc2);
+  RuntimeShape extended_output_shape =
+      RuntimeShape::ExtendedShape(4, output_shape);
 
   // In Tensorflow, the dimensions are canonically named (batch_number, row,
   // col, channel), with extents (batches, height, width, depth), with the
@@ -1748,33 +1639,37 @@ inline void BroadcastSub(int left_shift, const uint8* input1_data,
   // We name our variables by their Tensorflow convention, but generate C code
   // nesting loops such that the innermost loop has the smallest stride for the
   // best cache behavior.
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+  for (int b = 0; b < extended_output_shape.Dims(0); ++b) {
+    for (int y = 0; y < extended_output_shape.Dims(1); ++y) {
+      for (int x = 0; x < extended_output_shape.Dims(2); ++x) {
+        for (int c = 0; c < extended_output_shape.Dims(3); ++c) {
           const int32 input1_val =
-              input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)];
+              params.input1_offset +
+              input1_data[SubscriptToIndex(desc1, b, y, x, c)];
           const int32 input2_val =
-              input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)];
-          const int32 shifted_input1_val = input1_val * (1 << left_shift);
-          const int32 shifted_input2_val = input2_val * (1 << left_shift);
+              params.input2_offset +
+              input2_data[SubscriptToIndex(desc2, b, y, x, c)];
+          const int32 shifted_input1_val =
+              input1_val * (1 << params.left_shift);
+          const int32 shifted_input2_val =
+              input2_val * (1 << params.left_shift);
           const int32 scaled_input1_val =
               MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  shifted_input1_val, input1_multiplier,
-                  kReverseShift * input1_shift);
+                  shifted_input1_val, params.input1_multiplier,
+                  params.input1_shift);
           const int32 scaled_input2_val =
               MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  shifted_input2_val, input2_multiplier,
-                  kReverseShift * input2_shift);
+                  shifted_input2_val, params.input2_multiplier,
+                  params.input2_shift);
           const int32 raw_sub = scaled_input1_val - scaled_input2_val;
           const int32 raw_output =
               MultiplyByQuantizedMultiplierSmallerThanOneExp(
-                  raw_sub, output_multiplier, kReverseShift * output_shift) +
-              output_offset;
+                  raw_sub, params.output_multiplier, params.output_shift) +
+              params.output_offset;
           const int32 clamped_output =
-              std::min(output_activation_max,
-                       std::max(output_activation_min, raw_output));
-          output_data[Offset(output_dims, c, x, y, b)] =
+              std::min(params.quantized_activation_max,
+                       std::max(params.quantized_activation_min, raw_output));
+          output_data[Offset(extended_output_shape, b, y, x, c)] =
               static_cast(clamped_output);
         }
       }
@@ -1782,6 +1677,156 @@ inline void BroadcastSub(int left_shift, const uint8* input1_data,
   }
 }
 
+inline void BroadcastSub4DSlow(const ArithmeticParams& params,
+                               const RuntimeShape& input1_shape,
+                               const int32* input1_data,
+                               const RuntimeShape& input2_shape,
+                               const int32* input2_data,
+                               const RuntimeShape& output_shape,
+                               int32* output_data) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastAdd4DSlow/int32");
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1,
+                                      &desc2);
+  RuntimeShape extended_output_shape =
+      RuntimeShape::ExtendedShape(4, output_shape);
+
+  // In Tensorflow, the dimensions are canonically named (batch_number, row,
+  // col, channel), with extents (batches, height, width, depth), with the
+  // trailing dimension changing most rapidly (channels has the smallest stride,
+  // typically 1 element).
+  //
+  // In generated C code, we store arrays with the dimensions reversed. The
+  // first dimension has smallest stride.
+  //
+  // We name our variables by their Tensorflow convention, but generate C code
+  // nesting loops such that the innermost loop has the smallest stride for the
+  // best cache behavior.
+  for (int b = 0; b < extended_output_shape.Dims(0); ++b) {
+    for (int y = 0; y < extended_output_shape.Dims(1); ++y) {
+      for (int x = 0; x < extended_output_shape.Dims(2); ++x) {
+        for (int c = 0; c < extended_output_shape.Dims(3); ++c) {
+          output_data[Offset(extended_output_shape, b, y, x, c)] =
+              ActivationFunctionWithMinMax(
+                  input1_data[SubscriptToIndex(desc1, b, y, x, c)] -
+                      input2_data[SubscriptToIndex(desc2, b, y, x, c)],
+                  params.quantized_activation_min,
+                  params.quantized_activation_max);
+        }
+      }
+    }
+  }
+}
+
+template 
+void BroadcastSub4DSlow(const ArithmeticParams& params,
+                        const RuntimeShape& input1_shape, const T* input1_data,
+                        const RuntimeShape& input2_shape, const T* input2_data,
+                        const RuntimeShape& output_shape, T* output_data) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastAdd4DSlow/templated");
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1,
+                                      &desc2);
+  RuntimeShape extended_output_shape =
+      RuntimeShape::ExtendedShape(4, output_shape);
+
+  // In Tensorflow, the dimensions are canonically named (batch_number, row,
+  // col, channel), with extents (batches, height, width, depth), with the
+  // trailing dimension changing most rapidly (channels has the smallest stride,
+  // typically 1 element).
+  //
+  // In generated C code, we store arrays with the dimensions reversed. The
+  // first dimension has smallest stride.
+  //
+  // We name our variables by their Tensorflow convention, but generate C code
+  // nesting loops such that the innermost loop has the smallest stride for the
+  // best cache behavior.
+  for (int b = 0; b < extended_output_shape.Dims(0); ++b) {
+    for (int y = 0; y < extended_output_shape.Dims(1); ++y) {
+      for (int x = 0; x < extended_output_shape.Dims(2); ++x) {
+        for (int c = 0; c < extended_output_shape.Dims(3); ++c) {
+          output_data[Offset(extended_output_shape, b, y, x, c)] =
+              ActivationFunctionWithMinMax(
+                  input1_data[SubscriptToIndex(desc1, b, y, x, c)] -
+                      input2_data[SubscriptToIndex(desc2, b, y, x, c)],
+                  params.quantized_activation_min,
+                  params.quantized_activation_max);
+        }
+      }
+    }
+  }
+}
+
+template 
+void Sub(const ArithmeticParams& params, const RuntimeShape& input1_shape,
+         const T* input1_data, const RuntimeShape& input2_shape,
+         const T* input2_data, const RuntimeShape& output_shape,
+         T* output_data) {
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1,
+                                      &desc2);
+  RuntimeShape extended_output_shape =
+      RuntimeShape::ExtendedShape(4, output_shape);
+
+  // In Tensorflow, the dimensions are canonically named (batch_number, row,
+  // col, channel), with extents (batches, height, width, depth), with the
+  // trailing dimension changing most rapidly (channels has the smallest stride,
+  // typically 1 element).
+  //
+  // In generated C code, we store arrays with the dimensions reversed. The
+  // first dimension has smallest stride.
+  //
+  // We name our variables by their Tensorflow convention, but generate C code
+  // nesting loops such that the innermost loop has the smallest stride for the
+  // best cache behavior.
+  for (int b = 0; b < extended_output_shape.Dims(0); ++b) {
+    for (int y = 0; y < extended_output_shape.Dims(1); ++y) {
+      for (int x = 0; x < extended_output_shape.Dims(2); ++x) {
+        for (int c = 0; c < extended_output_shape.Dims(3); ++c) {
+          output_data[Offset(extended_output_shape, b, y, x, c)] =
+              input1_data[SubscriptToIndex(desc1, b, y, x, c)] -
+              input2_data[SubscriptToIndex(desc2, b, y, x, c)];
+        }
+      }
+    }
+  }
+}
+
+inline void SubWithActivation(const ArithmeticParams& params,
+                              const RuntimeShape& input1_shape,
+                              const int32* input1_data,
+                              const RuntimeShape& input2_shape,
+                              const int32* input2_data,
+                              const RuntimeShape& output_shape,
+                              int32* output_data) {
+  const int flat_size =
+      MatchingFlatSize(input1_shape, input2_shape, input2_shape);
+  for (int i = 0; i < flat_size; ++i) {
+    output_data[i] = ActivationFunctionWithMinMax(
+        input1_data[i] - input2_data[i], params.quantized_activation_min,
+        params.quantized_activation_max);
+  }
+}
+
+inline void SubWithActivation(const ArithmeticParams& params,
+                              const RuntimeShape& input1_shape,
+                              const float* input1_data,
+                              const RuntimeShape& input2_shape,
+                              const float* input2_data,
+                              const RuntimeShape& output_shape,
+                              float* output_data) {
+  const int flat_size =
+      MatchingFlatSize(input1_shape, input2_shape, input2_shape);
+  for (int i = 0; i < flat_size; ++i) {
+    output_data[i] = ActivationFunctionWithMinMax(
+        input1_data[i] - input2_data[i], params.float_activation_min,
+        params.float_activation_max);
+  }
+}
+
 template 
 void Concatenation(int concat_dim, const Scalar* const* input_data,
                    const Dims<4>* const* input_dims, int inputs_count,
@@ -3717,38 +3762,6 @@ inline void Mean(const T* input_data, const Dims<4>& input_dims,
   }
 }
 
-template 
-void Sub(const T* input1_data, const Dims<4>& input1_dims, const T* input2_data,
-         const Dims<4>& input2_dims, T* output_data,
-         const Dims<4>& output_dims) {
-  NdArrayDesc<4> desc1;
-  NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
-
-  // In Tensorflow, the dimensions are canonically named (batch_number, row,
-  // col, channel), with extents (batches, height, width, depth), with the
-  // trailing dimension changing most rapidly (channels has the smallest stride,
-  // typically 1 element).
-  //
-  // In generated C code, we store arrays with the dimensions reversed. The
-  // first dimension has smallest stride.
-  //
-  // We name our variables by their Tensorflow convention, but generate C code
-  // nesting loops such that the innermost loop has the smallest stride for the
-  // best cache behavior.
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
-          output_data[Offset(output_dims, c, x, y, b)] =
-              input1_data[SubscriptToIndex(desc1, c, x, y, b)] -
-              input2_data[SubscriptToIndex(desc2, c, x, y, b)];
-        }
-      }
-    }
-  }
-}
-
 template 
 void TensorFlowMinimum(const T* input1_data, const Dims<4>& input1_dims,
                        const T* input2_data, T* output_data,
diff --git a/tensorflow/contrib/lite/kernels/internal/types.h b/tensorflow/contrib/lite/kernels/internal/types.h
index 737cfb69c9..fe113dfdd3 100644
--- a/tensorflow/contrib/lite/kernels/internal/types.h
+++ b/tensorflow/contrib/lite/kernels/internal/types.h
@@ -119,6 +119,8 @@ class RuntimeShape {
   // larger shapes are separately allocated.
   static constexpr int kMaxSmallSize = 4;
 
+  RuntimeShape& operator=(RuntimeShape const&) = delete;
+
   RuntimeShape() : size_(0) {}
 
   explicit RuntimeShape(int dimensions_count) : size_(dimensions_count) {
@@ -135,6 +137,20 @@ class RuntimeShape {
     BuildFrom(init_list);
   }
 
+  // Avoid using this constructor.  We should be able to delete it when C++17
+  // rolls out.
+  RuntimeShape(RuntimeShape const& other) : size_(other.DimensionsCount()) {
+    if (size_ > kMaxSmallSize) {
+      dims_pointer_ = new int32[size_];
+    }
+    std::memcpy(DimsData(), other.DimsData(), sizeof(int32) * size_);
+  }
+
+  bool operator==(const RuntimeShape& comp) const {
+    return this->size_ == comp.size_ &&
+           std::memcmp(DimsData(), comp.DimsData(), size_ * sizeof(int32)) == 0;
+  }
+
   ~RuntimeShape() {
     if (size_ > kMaxSmallSize) {
       delete[] dims_pointer_;
@@ -191,6 +207,16 @@ class RuntimeShape {
     }
   }
 
+  // This will probably be factored out. Old code made substantial use of 4-D
+  // shapes, and so this function is used to extend smaller shapes. Note that
+  // (a) as Dims<4>-dependent code is eliminated, the reliance on this should be
+  // reduced, and (b) some kernels are stricly 4-D, but then the shapes of their
+  // inputs should already be 4-D, so this function should not be needed.
+  inline static RuntimeShape ExtendedShape(int new_shape_size,
+                                           const RuntimeShape& shape) {
+    return RuntimeShape(new_shape_size, shape, 1);
+  }
+
   inline void BuildFrom(const std::initializer_list init_list) {
     BuildFrom>(init_list);
   }
@@ -208,7 +234,25 @@ class RuntimeShape {
     return buffer_size;
   }
 
+  bool operator!=(const RuntimeShape& comp) const { return !((*this) == comp); }
+
  private:
+  // For use only by ExtendFrom(), written to guarantee (return-value) copy
+  // elision in C++17.
+  // This creates a shape padded to the desired size with the specified value.
+  RuntimeShape(int new_shape_size, const RuntimeShape& shape, int pad_value)
+      : size_(0) {
+    TFLITE_CHECK_GE(new_shape_size, shape.DimensionsCount());
+    TFLITE_CHECK_LE(new_shape_size, kMaxSmallSize);
+    Resize(new_shape_size);
+    const int size_increase = new_shape_size - shape.DimensionsCount();
+    for (int i = 0; i < size_increase; ++i) {
+      SetDim(i, pad_value);
+    }
+    std::memcpy(DimsData() + size_increase, shape.DimsData(),
+                sizeof(int32) * shape.DimensionsCount());
+  }
+
   int32 size_;
   union {
     int32 dims_[kMaxSmallSize];
@@ -364,6 +408,7 @@ inline int RequiredBufferSizeForDims(const Dims<4>& dims) {
 // arrays.
 inline int MatchingFlatSize(const RuntimeShape& shape,
                             const RuntimeShape& check_shape_0) {
+  TFLITE_DCHECK_EQ(shape.DimensionsCount(), check_shape_0.DimensionsCount());
   const int dims_count = shape.DimensionsCount();
   for (int i = 0; i < dims_count; ++i) {
     TFLITE_DCHECK_EQ(shape.Dims(i), check_shape_0.Dims(i));
@@ -374,6 +419,7 @@ inline int MatchingFlatSize(const RuntimeShape& shape,
 inline int MatchingFlatSize(const RuntimeShape& shape,
                             const RuntimeShape& check_shape_0,
                             const RuntimeShape& check_shape_1) {
+  TFLITE_DCHECK_EQ(shape.DimensionsCount(), check_shape_0.DimensionsCount());
   const int dims_count = shape.DimensionsCount();
   for (int i = 0; i < dims_count; ++i) {
     TFLITE_DCHECK_EQ(shape.Dims(i), check_shape_0.Dims(i));
@@ -385,6 +431,7 @@ inline int MatchingFlatSize(const RuntimeShape& shape,
                             const RuntimeShape& check_shape_0,
                             const RuntimeShape& check_shape_1,
                             const RuntimeShape& check_shape_2) {
+  TFLITE_DCHECK_EQ(shape.DimensionsCount(), check_shape_0.DimensionsCount());
   const int dims_count = shape.DimensionsCount();
   for (int i = 0; i < dims_count; ++i) {
     TFLITE_DCHECK_EQ(shape.Dims(i), check_shape_0.Dims(i));
@@ -397,6 +444,7 @@ inline int MatchingFlatSize(const RuntimeShape& shape,
                             const RuntimeShape& check_shape_1,
                             const RuntimeShape& check_shape_2,
                             const RuntimeShape& check_shape_3) {
+  TFLITE_DCHECK_EQ(shape.DimensionsCount(), check_shape_0.DimensionsCount());
   const int dims_count = shape.DimensionsCount();
   for (int i = 0; i < dims_count; ++i) {
     TFLITE_DCHECK_EQ(shape.Dims(i), check_shape_0.Dims(i));
@@ -601,14 +649,74 @@ struct PoolParams {
   int stride_width;
   int filter_height;
   int filter_width;
-  // uint8, etc, inference params.
+  // uint8, etc, activation params.
   int32 quantized_activation_min;
   int32 quantized_activation_max;
-  // float inference params.
+  // float activation params.
   float float_activation_min;
   float float_activation_max;
 };
 
+enum class BroadcastableOpCategory : uint8 {
+  kNone,
+  kNonBroadcast,               // Matching input shapes.
+  kFirstInputBroadcastsFast,   // Fivefold nested loops.
+  kSecondInputBroadcastsFast,  // Fivefold nested loops.
+  kGenericBroadcast,           // Fall-back.
+};
+
+// For Add, Sub, Mul ops.
+struct ArithmeticParams {
+  // Shape dependent / common to data / op types.
+  BroadcastableOpCategory broadcast_category;
+  // uint8 inference params.
+  int32 input1_offset;
+  int32 input2_offset;
+  int32 output_offset;
+  int32 output_multiplier;
+  int output_shift;
+  // Add / Sub, not Mul, uint8 inference params.
+  int left_shift;
+  int32 input1_multiplier;
+  int input1_shift;
+  int32 input2_multiplier;
+  int input2_shift;
+  // uint8, etc, activation params.
+  int32 quantized_activation_min;
+  int32 quantized_activation_max;
+  // float activation params.
+  float float_activation_min;
+  float float_activation_max;
+
+  // Processed output dimensions.
+  // Let input "a" be the one that broadcasts in the faster-changing dimension.
+  // Then, after coalescing, for shapes {a0, a1, a2, a3, a4} and
+  // {b0, b1, b2, b3, b4},
+  // broadcast_shape[4] = b0 = a0.
+  // broadcast_shape[3] = b1; a1 = 1.
+  // broadcast_shape[2] = b2 = a2.
+  // broadcast_shape[1] = a3; b3 = 1.
+  // broadcast_shape[0] = b4 = a4.
+  int broadcast_shape[5];
+};
+
+template 
+inline void SetActivationParams(T min, T max, ArithmeticParams* params);
+
+template <>
+inline void SetActivationParams(float min, float max,
+                                ArithmeticParams* params) {
+  params->float_activation_min = min;
+  params->float_activation_max = max;
+}
+
+template <>
+inline void SetActivationParams(int32 min, int32 max,
+                                ArithmeticParams* params) {
+  params->quantized_activation_min = min;
+  params->quantized_activation_max = max;
+}
+
 }  // namespace tflite
 
 #endif  // TENSORFLOW_CONTRIB_LITE_KERNELS_INTERNAL_TYPES_H_
diff --git a/tensorflow/contrib/lite/kernels/sub.cc b/tensorflow/contrib/lite/kernels/sub.cc
index 541c85f756..77a1f59689 100644
--- a/tensorflow/contrib/lite/kernels/sub.cc
+++ b/tensorflow/contrib/lite/kernels/sub.cc
@@ -81,40 +81,43 @@ template 
 void EvalSub(TfLiteContext* context, TfLiteNode* node, TfLiteSubParams* params,
              const OpData* data, const TfLiteTensor* input1,
              const TfLiteTensor* input2, TfLiteTensor* output) {
-#define TF_LITE_SUB(type, opname, data_type)                            \
-  data_type output_activation_min, output_activation_max;               \
-  CalculateActivationRange(params->activation, &output_activation_min,  \
-                           &output_activation_max);                     \
-  type::opname(GetTensorData(input1), GetTensorDims(input1), \
-               GetTensorData(input2), GetTensorDims(input2), \
-               output_activation_min, output_activation_max,            \
-               GetTensorData(output), GetTensorDims(output))
+#define TF_LITE_SUB(type, opname, data_type)                             \
+  data_type output_activation_min, output_activation_max;                \
+  CalculateActivationRange(params->activation, &output_activation_min,   \
+                           &output_activation_max);                      \
+  tflite::ArithmeticParams op_params;                                    \
+  SetActivationParams(output_activation_min, output_activation_max,      \
+                      &op_params);                                       \
+  type::opname(op_params, GetTensorShape(input1),                        \
+               GetTensorData(input1), GetTensorShape(input2), \
+               GetTensorData(input2), GetTensorShape(output), \
+               GetTensorData(output))
   if (output->type == kTfLiteInt32) {
     if (kernel_type == kReference) {
       if (data->requires_broadcast) {
-        TF_LITE_SUB(reference_ops, BroadcastSub, int32_t);
+        TF_LITE_SUB(reference_ops, BroadcastSub4DSlow, int32_t);
       } else {
-        TF_LITE_SUB(reference_ops, Sub, int32_t);
+        TF_LITE_SUB(reference_ops, SubWithActivation, int32_t);
       }
     } else {
       if (data->requires_broadcast) {
-        TF_LITE_SUB(optimized_ops, BroadcastSub, int32_t);
+        TF_LITE_SUB(optimized_ops, BroadcastSub4DSlow, int32_t);
       } else {
-        TF_LITE_SUB(optimized_ops, Sub, int32_t);
+        TF_LITE_SUB(optimized_ops, SubWithActivation, int32_t);
       }
     }
   } else if (output->type == kTfLiteFloat32) {
     if (kernel_type == kReference) {
       if (data->requires_broadcast) {
-        TF_LITE_SUB(reference_ops, BroadcastSub, float);
+        TF_LITE_SUB(reference_ops, BroadcastSub4DSlow, float);
       } else {
-        TF_LITE_SUB(reference_ops, Sub, float);
+        TF_LITE_SUB(reference_ops, SubWithActivation, float);
       }
     } else {
       if (data->requires_broadcast) {
-        TF_LITE_SUB(optimized_ops, BroadcastSub, float);
+        TF_LITE_SUB(optimized_ops, BroadcastSub4DSlow, float);
       } else {
-        TF_LITE_SUB(optimized_ops, Sub, float);
+        TF_LITE_SUB(optimized_ops, SubWithActivation, float);
       }
     }
   }
@@ -143,36 +146,43 @@ void EvalQuantized(TfLiteContext* context, TfLiteNode* node,
   int input1_shift;
   QuantizeMultiplierSmallerThanOneExp(real_input1_multiplier,
                                       &input1_multiplier, &input1_shift);
-  input1_shift *= -1;
   int32 input2_multiplier;
   int input2_shift;
   QuantizeMultiplierSmallerThanOneExp(real_input2_multiplier,
                                       &input2_multiplier, &input2_shift);
-  input2_shift *= -1;
   int32 output_multiplier;
   int output_shift;
   QuantizeMultiplierSmallerThanOneExp(real_output_multiplier,
                                       &output_multiplier, &output_shift);
-  output_shift *= -1;
 
   int32 output_activation_min, output_activation_max;
   CalculateActivationRangeUint8(params->activation, output,
                                 &output_activation_min, &output_activation_max);
 
-#define TF_LITE_SUB(type, opname)                                            \
-  type::opname(left_shift, GetTensorData(input1),                   \
-               GetTensorDims(input1), input1_offset, input1_multiplier,      \
-               input1_shift, GetTensorData(input2),                 \
-               GetTensorDims(input2), input2_offset, input2_multiplier,      \
-               input2_shift, output_offset, output_multiplier, output_shift, \
-               output_activation_min, output_activation_max,                 \
-               GetTensorData(output), GetTensorDims(output));
+#define TF_LITE_SUB(type, opname)                                      \
+  tflite::ArithmeticParams op_params;                                  \
+  op_params.left_shift = left_shift;                                   \
+  op_params.input1_offset = input1_offset;                             \
+  op_params.input1_multiplier = input1_multiplier;                     \
+  op_params.input1_shift = input1_shift;                               \
+  op_params.input2_offset = input2_offset;                             \
+  op_params.input2_multiplier = input2_multiplier;                     \
+  op_params.input2_shift = input2_shift;                               \
+  op_params.output_offset = output_offset;                             \
+  op_params.output_multiplier = output_multiplier;                     \
+  op_params.output_shift = output_shift;                               \
+  SetActivationParams(output_activation_min, output_activation_max,    \
+                      &op_params);                                     \
+  type::opname(op_params, GetTensorShape(input1),                      \
+               GetTensorData(input1), GetTensorShape(input2), \
+               GetTensorData(input2), GetTensorShape(output), \
+               GetTensorData(output))
   // The quantized version of Sub doesn't support activations, so we
   // always use BroadcastSub.
   if (kernel_type == kReference) {
-    TF_LITE_SUB(reference_ops, BroadcastSub);
+    TF_LITE_SUB(reference_ops, BroadcastSub4DSlow);
   } else {
-    TF_LITE_SUB(optimized_ops, BroadcastSub);
+    TF_LITE_SUB(optimized_ops, BroadcastSub4DSlow);
   }
 #undef TF_LITE_SUB
 }
-- 
cgit v1.2.3


From 8eb773d6c23de29dccfc3cf3da441a8552ed13ed Mon Sep 17 00:00:00 2001
From: Michael Kuperstein 
Date: Thu, 19 Jul 2018 10:30:58 -0700
Subject: [XLA] Better shape size validation for sparse arrays.

PiperOrigin-RevId: 205262376
---
 tensorflow/compiler/xla/shape_util.cc | 51 +++++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 20 deletions(-)

diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index f4668c0f55..6480148336 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -883,40 +883,51 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) {
   }
 
   int64 shape_size = [&shape]() {
-    int64 shape_size;
     if (LayoutUtil::IsSparseArray(shape)) {
-      shape_size = LayoutUtil::MaxSparseElements(shape.layout());
-      if (shape_size < 0) {
-        return shape_size;
+      int64 max_sparse_elements = LayoutUtil::MaxSparseElements(shape.layout());
+      if (max_sparse_elements < 0) {
+        return max_sparse_elements;
       }
-      shape_size = MultiplyWithoutOverflow(shape_size, ShapeUtil::Rank(shape));
-      if (shape_size < 0) {
-        return shape_size;
+      int64 sparse_elements_size = MultiplyWithoutOverflow(
+          max_sparse_elements, ByteSizeOfPrimitiveType(shape.element_type()));
+      if (sparse_elements_size < 0) {
+        return sparse_elements_size;
       }
-      shape_size = MultiplyWithoutOverflow(shape_size, sizeof(int64));
-      if (shape_size < 0) {
-        return shape_size;
+      int64 sparse_indices_size =
+          MultiplyWithoutOverflow(max_sparse_elements, ShapeUtil::Rank(shape));
+      if (sparse_indices_size < 0) {
+        return sparse_indices_size;
+      }
+      sparse_indices_size =
+          MultiplyWithoutOverflow(sparse_indices_size, sizeof(int64));
+      if (sparse_indices_size < 0) {
+        return sparse_indices_size;
+      }
+      // At this point, both sparse_indices_size and sparse_elements_size are
+      // non-negative, so we can easily check if adding them wraps.
+      if (static_cast(sparse_elements_size) +
+              static_cast(sparse_indices_size) >
+          INT64_MAX) {
+        return static_cast(-1);
       }
     }
 
-    shape_size = 1;
-
     // This is intentionally unconditional: even if the shape is sparse, we want
     // to verify the densified version has a reasonable size.
+    int64 dense_shape_size = 1;
     if (shape.dimensions().empty()) {
-      return shape_size;
+      return dense_shape_size;
     }
 
     for (int64 dim : shape.dimensions()) {
-      shape_size = MultiplyWithoutOverflow(shape_size, dim);
-      if (shape_size < 0) {
-        return shape_size;
+      dense_shape_size = MultiplyWithoutOverflow(dense_shape_size, dim);
+      if (dense_shape_size < 0) {
+        return dense_shape_size;
       }
     }
-    shape_size = MultiplyWithoutOverflow(
-        shape_size, ByteSizeOfPrimitiveType(shape.element_type()));
-
-    return shape_size;
+    dense_shape_size = MultiplyWithoutOverflow(
+        dense_shape_size, ByteSizeOfPrimitiveType(shape.element_type()));
+    return dense_shape_size;
   }();
 
   if (shape_size < 0) {
-- 
cgit v1.2.3


From 9dfa333cc8e78b9a533562448d67d48ec568622d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 10:41:29 -0700
Subject: Small text change

PiperOrigin-RevId: 205264291
---
 .../contrib/eager/python/examples/generative_examples/dcgan.ipynb       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
index 43c8c355dc..232f9a8ef0 100644
--- a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
@@ -31,7 +31,7 @@
         "\n",
         "On a colab GPU(Tesla K80), the model takes around 40 seconds per epoch to train.\n",
         "\n",
-        "Below is the output generated after training the generator and discriminator models for 100 epochs.\n",
+        "Below is the output generated after training the generator and discriminator models for 150 epochs.\n",
         "\n",
         "![sample output](https://tensorflow.org/images/gan/dcgan.gif)"
       ]
-- 
cgit v1.2.3


From 1044888430b34353f54266bf0674144dfe675687 Mon Sep 17 00:00:00 2001
From: Rohan Jain 
Date: Thu, 19 Jul 2018 10:44:24 -0700
Subject: Making the Eager iterator use the new copy_to_device.

This CL gets rid of the forced placement of all eager datasets / iterators on the CPU since now we can have some datasets on the GPU.

PiperOrigin-RevId: 205264791
---
 .../contrib/data/python/ops/prefetching_ops.py     |  5 ++
 tensorflow/contrib/eager/python/datasets.py        | 64 +++-------------------
 tensorflow/python/data/ops/iterator_ops.py         | 26 ++++-----
 3 files changed, 26 insertions(+), 69 deletions(-)

diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py
index 50212d3b52..45abd6376c 100644
--- a/tensorflow/contrib/data/python/ops/prefetching_ops.py
+++ b/tensorflow/contrib/data/python/ops/prefetching_ops.py
@@ -480,6 +480,11 @@ class _CopyToDeviceDataset(dataset_ops.Dataset):
 
     self._finalize_func = _remote_finalize_func
     self._finalize_captured_args = _remote_finalize_func.captured_inputs
+
+    g = ops.get_default_graph()
+    _remote_init_func.add_to_graph(g)
+    _remote_next_func.add_to_graph(g)
+    _remote_finalize_func.add_to_graph(g)
     # pylint: enable=protected-scope
 
   # The one_shot_iterator implementation needs a 0 arg _make_dataset function
diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py
index 58c548d798..e31dbbe80f 100644
--- a/tensorflow/contrib/eager/python/datasets.py
+++ b/tensorflow/contrib/eager/python/datasets.py
@@ -18,33 +18,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import threading
-
 from tensorflow.contrib.data.python.ops import prefetching_ops
 from tensorflow.python.data.ops import iterator_ops
-from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import sparse
 from tensorflow.python.eager import context
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_dataset_ops
-from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.training.checkpointable import base as checkpointable
 from tensorflow.python.training.saver import BaseSaverBuilder
 
-_uid_counter = 0
-_uid_lock = threading.Lock()
-
-
-def _generate_shared_name(prefix):
-  with _uid_lock:
-    global _uid_counter
-    uid = _uid_counter
-    _uid_counter += 1
-  return "{}{}".format(prefix, uid)
-
 
 class Iterator(iterator_ops.EagerIterator, checkpointable.CheckpointableBase):
   """An iterator producing tf.Tensor objects from a tf.data.Dataset.
@@ -80,38 +61,18 @@ class Iterator(iterator_ops.EagerIterator, checkpointable.CheckpointableBase):
           "`tf.contrib.eager.Iterator`. Use `for ... in dataset:` to iterate "
           "over the dataset instead.")
 
-    super(Iterator, self).__init__(dataset)
     if not context.context().device_spec.device_type:
       is_remote_device = False
     else:
       is_remote_device = context.context().device_spec.device_type != "CPU"
-    self._buffer_resource_handle = None
     if is_remote_device:
-      with ops.device("/device:CPU:0"):
-        iter_string_handle = gen_dataset_ops.iterator_to_string_handle(
-            self._resource)
-
-        @function.Defun(dtypes.string)
-        def remote_fn(h):
-          remote_iterator = iterator_ops.Iterator.from_string_handle(
-              h, self.output_types, self.output_shapes, self.output_classes)
-          return remote_iterator.get_next()
-
-        remote_fn.add_to_graph(None)
-        target = constant_op.constant("/device:CPU:0")
-      with ops.device(self._device):
-        self._buffer_resource_handle = prefetching_ops.function_buffering_resource(  # pylint: disable=line-too-long
-            string_arg=iter_string_handle,
-            output_types=self._flat_output_types,
-            f=remote_fn,
-            target_device=target,
-            buffer_size=10,
-            container="",
-            shared_name=_generate_shared_name(
-                "contrib_eager_iterator_function_buffer_resource"))
-        self._buffer_resource_deleter = resource_variable_ops.EagerResourceDeleter(  # pylint: disable=line-too-long
-            handle=self._buffer_resource_handle,
-            handle_device=self._device)
+      with ops.device(None):
+        # Let the placer figure out where to place the various functions etc.
+        # created by the CopyToDeviceDataset.
+        dataset = dataset.apply(prefetching_ops.copy_to_device(
+            context.context().device_name))
+        dataset = dataset.prefetch(1)
+    super(Iterator, self).__init__(dataset)
 
   def _next_internal(self):
     """Returns a nested structure of `tf.Tensor`s containing the next element.
@@ -120,16 +81,7 @@ class Iterator(iterator_ops.EagerIterator, checkpointable.CheckpointableBase):
     # that there is no more data to iterate over.
     # TODO(b/77291417): Fix
     with context.execution_mode(context.SYNC):
-      if self._buffer_resource_handle is not None:
-        with ops.device(self._device):
-          ret = prefetching_ops.function_buffering_resource_get_next(
-              function_buffer_resource=self._buffer_resource_handle,
-              output_types=self._flat_output_types)
-        return sparse.deserialize_sparse_tensors(
-            nest.pack_sequence_as(self._output_types, ret), self._output_types,
-            self._output_shapes, self._output_classes)
-      else:
-        return super(Iterator, self)._next_internal()
+      return super(Iterator, self)._next_internal()
 
   # TODO(shivaniagrawal): Expose checkpointable stateful objects from dataset
   # attributes(potential).
diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py
index 35de2f2841..f0784ed3d0 100644
--- a/tensorflow/python/data/ops/iterator_ops.py
+++ b/tensorflow/python/data/ops/iterator_ops.py
@@ -499,23 +499,23 @@ class EagerIterator(object):
           "tf.data.Dataset.make_initializable_iterator or "
           "tf.data.Dataset.make_one_shot_iterator for graph construction".
           format(type(self)))
-    with ops.device("/device:CPU:0"):
-      ds_variant = dataset._as_variant_tensor()  # pylint: disable=protected-access
-      self._output_classes = dataset.output_classes
-      self._output_types = dataset.output_types
-      self._output_shapes = dataset.output_shapes
-      self._flat_output_types = nest.flatten(
-          sparse.as_dense_types(self._output_types, self._output_classes))
-      self._flat_output_shapes = nest.flatten(
-          sparse.as_dense_shapes(self._output_shapes, self._output_classes))
+    self._device = context.context().device_name
+    ds_variant = dataset._as_variant_tensor()  # pylint: disable=protected-access
+    self._output_classes = dataset.output_classes
+    self._output_types = dataset.output_types
+    self._output_shapes = dataset.output_shapes
+    self._flat_output_types = nest.flatten(
+        sparse.as_dense_types(self._output_types, self._output_classes))
+    self._flat_output_shapes = nest.flatten(
+        sparse.as_dense_shapes(self._output_shapes, self._output_classes))
+    with ops.colocate_with(ds_variant):
       self._resource = gen_dataset_ops.anonymous_iterator(
           output_types=self._flat_output_types,
           output_shapes=self._flat_output_shapes)
       gen_dataset_ops.make_iterator(ds_variant, self._resource)
-      # Delete the resource when this object is deleted
-      self._resource_deleter = resource_variable_ops.EagerResourceDeleter(
-          handle=self._resource, handle_device="/device:CPU:0")
-    self._device = context.context().device_name
+    # Delete the resource when this object is deleted
+    self._resource_deleter = resource_variable_ops.EagerResourceDeleter(
+        handle=self._resource, handle_device=self._device)
 
   def __iter__(self):
     return self
-- 
cgit v1.2.3


From 88c520b3f12a1ee5e63d9f05094ca9f84700ea6e Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Thu, 19 Jul 2018 11:02:05 -0700
Subject: Merges variable_scope.variable and tf.Variable

PiperOrigin-RevId: 205267974
---
 tensorflow/python/ops/resource_variable_ops.py    | 13 ------
 tensorflow/python/ops/variable_scope.py           | 54 ++++++---------------
 tensorflow/python/ops/variables.py                | 57 +++++++++++++++++++----
 tensorflow/python/training/checkpointable/util.py |  2 +-
 4 files changed, 63 insertions(+), 63 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 5979b76ff2..1f56ad25bf 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -1294,16 +1294,3 @@ def is_resource_variable(var):
   """"Returns True if `var` is to be considered a ResourceVariable."""
   return isinstance(var, ResourceVariable) or hasattr(
       var, "_should_act_as_resource_variable")
-
-
-_DEFAULT_USE_RESOURCE = False
-
-
-def _default_variable_creator(_, *args, **kwds):
-  use_resource = kwds.pop("use_resource", _DEFAULT_USE_RESOURCE)
-  use_resource = use_resource or context.executing_eagerly()
-  if use_resource:
-    return ResourceVariable(*args, **kwds)
-  return variables.RefVariable(*args, **kwds)
-
-variables.default_variable_creator = _default_variable_creator
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 0f37dcc027..aca44bcd44 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -2349,7 +2349,10 @@ def default_variable_creator(next_creator=None, **kwargs):
   validate_shape = kwargs.get("validate_shape", True)
   caching_device = kwargs.get("caching_device", None)
   name = kwargs.get("name", None)
+  variable_def = kwargs.get("variable_def", None)
   dtype = kwargs.get("dtype", None)
+  expected_shape = kwargs.get("expected_shape", None)
+  import_scope = kwargs.get("import_scope", None)
   constraint = kwargs.get("constraint", None)
   use_resource = kwargs.get("use_resource", None)
 
@@ -2360,23 +2363,24 @@ def default_variable_creator(next_creator=None, **kwargs):
 
   if use_resource is None:
     use_resource = get_variable_scope().use_resource
-  if use_resource or (use_resource is None and context.executing_eagerly()):
+  use_resource = use_resource or context.executing_eagerly()
+  if use_resource:
     return resource_variable_ops.ResourceVariable(
         initial_value=initial_value, trainable=trainable,
         collections=collections, validate_shape=validate_shape,
         caching_device=caching_device, name=name, dtype=dtype,
-        constraint=constraint)
-  elif not use_resource and context.executing_eagerly():
-    raise RuntimeError(
-        "VariableScope should use resource variable when eager execution is"
-        " enabled, but use_resource is False."
-    )
+        constraint=constraint, variable_def=variable_def,
+        import_scope=import_scope)
   else:
-    return variables.Variable(
+    return variables.RefVariable(
         initial_value=initial_value, trainable=trainable,
         collections=collections, validate_shape=validate_shape,
         caching_device=caching_device, name=name, dtype=dtype,
-        constraint=constraint)
+        constraint=constraint, variable_def=variable_def,
+        expected_shape=expected_shape, import_scope=import_scope)
+
+
+variables.default_variable_creator = default_variable_creator
 
 
 def _make_getter(captured_getter, captured_previous):
@@ -2384,36 +2388,8 @@ def _make_getter(captured_getter, captured_previous):
   return lambda **kwargs: captured_getter(captured_previous, **kwargs)
 
 
-def variable(initial_value=None,
-             trainable=None,
-             collections=None,
-             validate_shape=True,
-             caching_device=None,
-             name=None,
-             dtype=None,
-             constraint=None,
-             use_resource=None,
-             synchronization=VariableSynchronization.AUTO,
-             aggregation=VariableAggregation.NONE):
-  previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
-  for getter in ops.get_default_graph()._variable_creator_stack:  # pylint: disable=protected-access
-    previous_getter = _make_getter(getter, previous_getter)
-
-  # Reset `aggregation` that is explicitly set as `None` to the enum None value.
-  if aggregation is None:
-    aggregation = VariableAggregation.NONE
-  return previous_getter(
-      initial_value=initial_value,
-      trainable=trainable,
-      collections=collections,
-      validate_shape=validate_shape,
-      caching_device=caching_device,
-      name=name,
-      dtype=dtype,
-      constraint=constraint,
-      use_resource=use_resource,
-      synchronization=synchronization,
-      aggregation=aggregation)
+# TODO(apassos) remove forwarding symbol
+variable = variables.Variable
 
 
 @tf_contextlib.contextmanager
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 6bb2d6f669..d03d93beeb 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -40,15 +40,15 @@ from tensorflow.python.util.deprecation import deprecated
 from tensorflow.python.util.tf_export import tf_export
 
 
-def default_variable_creator(_, *args, **kwds):
-  del args, kwds
-  raise NotImplementedError("resource_variable_ops needs to be imported")
+def default_variable_creator(_, **kwds):
+  del kwds
+  raise NotImplementedError("variable_scope needs to be imported")
 
 
 def _make_getter(captured_getter, captured_previous):
   """To avoid capturing loop variables."""
-  def getter(*args, **kwargs):
-    return captured_getter(captured_previous, *args, **kwargs)
+  def getter(**kwargs):
+    return captured_getter(captured_previous, **kwargs)
   return getter
 
 
@@ -86,11 +86,48 @@ class VariableAggregation(enum.Enum):
 class VariableMetaclass(type):
   """Metaclass to allow construction of tf.Variable to be overridden."""
 
+  def _variable_call(cls,
+                     initial_value=None,
+                     trainable=None,
+                     collections=None,
+                     validate_shape=True,
+                     caching_device=None,
+                     name=None,
+                     variable_def=None,
+                     dtype=None,
+                     expected_shape=None,
+                     import_scope=None,
+                     constraint=None,
+                     use_resource=None,
+                     synchronization=VariableSynchronization.AUTO,
+                     aggregation=VariableAggregation.NONE):
+    """Call on Variable class. Useful to force the signature."""
+    previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
+    for getter in ops.get_default_graph()._variable_creator_stack:  # pylint: disable=protected-access
+      previous_getter = _make_getter(getter, previous_getter)
+
+    # Reset `aggregation` that is explicitly set as `None` to the enum NONE.
+    if aggregation is None:
+      aggregation = VariableAggregation.NONE
+    return previous_getter(
+        initial_value=initial_value,
+        trainable=trainable,
+        collections=collections,
+        validate_shape=validate_shape,
+        caching_device=caching_device,
+        name=name,
+        variable_def=variable_def,
+        dtype=dtype,
+        expected_shape=expected_shape,
+        import_scope=import_scope,
+        constraint=constraint,
+        use_resource=use_resource,
+        synchronization=synchronization,
+        aggregation=aggregation)
+
   def __call__(cls, *args, **kwargs):
     if cls is Variable:
-      previous_getter = lambda *a, **k: default_variable_creator(None, *a, **k)
-      # TODO(apassos) use a stack of getters here
-      return previous_getter(*args, **kwargs)
+      return cls._variable_call(*args, **kwargs)
     else:
       return super(VariableMetaclass, cls).__call__(*args, **kwargs)
 
@@ -650,8 +687,8 @@ class Variable(six.with_metaclass(VariableMetaclass,
   @staticmethod
   def from_proto(variable_def, import_scope=None):
     """Returns a `Variable` object created from `variable_def`."""
-    return Variable(variable_def=variable_def,
-                    import_scope=import_scope)
+    return RefVariable(variable_def=variable_def,
+                       import_scope=import_scope)
 
   class SaveSliceInfo(object):
     """Information on how to save this Variable as a slice.
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index 6ae5765b13..686232fe27 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -747,7 +747,7 @@ def capture_dependencies(template):
           initial_value=initializer,
           name=name,
           **inner_kwargs)
-    if name.startswith(name_prefix):
+    if name is not None and name.startswith(name_prefix):
       scope_stripped_name = name[len(name_prefix) + 1:]
       if not checkpointable_parent:
         return template._add_variable_with_custom_getter(  # pylint: disable=protected-access
-- 
cgit v1.2.3


From 109ae67a7e99e3dcb4d93cc22df5b3912f4558c9 Mon Sep 17 00:00:00 2001
From: Derek Murray 
Date: Thu, 19 Jul 2018 11:27:39 -0700
Subject: Expose each device's incarnation via `Session.list_devices()`.

PiperOrigin-RevId: 205273020
---
 tensorflow/c/c_api.cc                               |  1 +
 tensorflow/c/c_api.h                                |  7 +++++++
 tensorflow/python/client/session.py                 | 14 +++++++++++---
 .../python/client/session_list_devices_test.py      |  8 +++++++-
 tensorflow/python/client/session_test.py            | 21 +++++++++++++--------
 tensorflow/python/client/tf_session.i               |  5 +++++
 6 files changed, 44 insertions(+), 12 deletions(-)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 5c218d3f25..a3003953a3 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -963,6 +963,7 @@ TF_DEVICELIST_METHOD(const char*, TF_DeviceListName, name().c_str(), nullptr);
 TF_DEVICELIST_METHOD(const char*, TF_DeviceListType, device_type().c_str(),
                      nullptr);
 TF_DEVICELIST_METHOD(int64_t, TF_DeviceListMemoryBytes, memory_limit(), -1);
+TF_DEVICELIST_METHOD(uint64_t, TF_DeviceListIncarnation, incarnation(), 0);
 
 #undef TF_DEVICELIST_METHOD
 
diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index 1eb75ef11f..fddc09d45e 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -1521,6 +1521,13 @@ TF_CAPI_EXPORT extern const char* TF_DeviceListType(const TF_DeviceList* list,
 TF_CAPI_EXPORT extern int64_t TF_DeviceListMemoryBytes(
     const TF_DeviceList* list, int index, TF_Status* status);
 
+// Retrieve the incarnation number of a given device.
+//
+// If index is out of bounds, an error code will be set in the status object,
+// and 0 will be returned.
+TF_CAPI_EXPORT extern uint64_t TF_DeviceListIncarnation(
+    const TF_DeviceList* list, int index, TF_Status* status);
+
 // --------------------------------------------------------------------------
 // Load plugins containing custom ops and kernels
 
diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index e037925961..8ede6ab54c 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -540,10 +540,11 @@ class _DeviceAttributes(object):
         (in bytes).
   """
 
-  def __init__(self, name, device_type, memory_limit_bytes):
+  def __init__(self, name, device_type, memory_limit_bytes, incarnation):
     self._name = device.canonical_name(name)
     self._device_type = device_type
     self._memory_limit_bytes = memory_limit_bytes
+    self._incarnation = incarnation
 
   @property
   def name(self):
@@ -557,11 +558,16 @@ class _DeviceAttributes(object):
   def memory_limit_bytes(self):
     return self._memory_limit_bytes
 
+  @property
+  def incarnation(self):
+    return self._incarnation
+
   def __repr__(self):
-    return '_DeviceAttributes(%s, %s, %d)' % (
+    return '_DeviceAttributes(%s, %s, %d, %d)' % (
         self.name,
         self.device_type,
         self.memory_limit_bytes,
+        self.incarnation,
     )
 
 
@@ -658,7 +664,9 @@ class BaseSession(SessionInterface):
       name = tf_session.TF_DeviceListName(raw_device_list, i)
       device_type = tf_session.TF_DeviceListType(raw_device_list, i)
       memory = tf_session.TF_DeviceListMemoryBytes(raw_device_list, i)
-      device_list.append(_DeviceAttributes(name, device_type, memory))
+      incarnation = tf_session.TF_DeviceListIncarnation(raw_device_list, i)
+      device_list.append(
+          _DeviceAttributes(name, device_type, memory, incarnation))
     tf_session.TF_DeleteDeviceList(raw_device_list)
     return device_list
 
diff --git a/tensorflow/python/client/session_list_devices_test.py b/tensorflow/python/client/session_list_devices_test.py
index c5d82c213a..dd381c689f 100644
--- a/tensorflow/python/client/session_list_devices_test.py
+++ b/tensorflow/python/client/session_list_devices_test.py
@@ -37,6 +37,8 @@ class SessionListDevicesTest(test_util.TensorFlowTestCase):
       devices = sess.list_devices()
       self.assertTrue('/job:localhost/replica:0/task:0/device:CPU:0' in set(
           [d.name for d in devices]), devices)
+      # All valid device incarnations must be non-zero.
+      self.assertTrue(all(d.incarnation != 0 for d in devices))
 
   def testInvalidDeviceNumber(self):
     opts = tf_session.TF_NewSessionOptions()
@@ -54,6 +56,8 @@ class SessionListDevicesTest(test_util.TensorFlowTestCase):
       devices = sess.list_devices()
       self.assertTrue('/job:local/replica:0/task:0/device:CPU:0' in set(
           [d.name for d in devices]), devices)
+      # All valid device incarnations must be non-zero.
+      self.assertTrue(all(d.incarnation != 0 for d in devices))
 
   def testListDevicesClusterSpecPropagation(self):
     server1 = server_lib.Server.create_local_server()
@@ -67,11 +71,13 @@ class SessionListDevicesTest(test_util.TensorFlowTestCase):
     config = config_pb2.ConfigProto(cluster_def=cluster_def)
     with session.Session(server1.target, config=config) as sess:
       devices = sess.list_devices()
-      device_names = set([d.name for d in devices])
+      device_names = set(d.name for d in devices)
       self.assertTrue(
           '/job:worker/replica:0/task:0/device:CPU:0' in device_names)
       self.assertTrue(
           '/job:worker/replica:0/task:1/device:CPU:0' in device_names)
+      # All valid device incarnations must be non-zero.
+      self.assertTrue(all(d.incarnation != 0 for d in devices))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index b72e029d1c..052be68385 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -35,6 +35,7 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.framework import common_shapes
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import device as framework_device_lib
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import function
@@ -104,18 +105,20 @@ class SessionTest(test_util.TensorFlowTestCase):
           copy_val)
 
   def testManyCPUs(self):
-    # TODO(keveman): Implement ListDevices and test for the number of
-    # devices returned by ListDevices.
     with session.Session(
         config=config_pb2.ConfigProto(device_count={
-            'CPU': 2
-        })):
+            'CPU': 2, 'GPU': 0
+        })) as sess:
       inp = constant_op.constant(10.0, name='W1')
       self.assertAllEqual(inp.eval(), 10.0)
 
+      devices = sess.list_devices()
+      self.assertEqual(2, len(devices))
+      for device in devices:
+        self.assertEqual('CPU', framework_device_lib.DeviceSpec.from_string(
+            device.name).device_type)
+
   def testPerSessionThreads(self):
-    # TODO(keveman): Implement ListDevices and test for the number of
-    # devices returned by ListDevices.
     with session.Session(
         config=config_pb2.ConfigProto(use_per_session_threads=True)):
       inp = constant_op.constant(10.0, name='W1')
@@ -1868,19 +1871,21 @@ class SessionTest(test_util.TensorFlowTestCase):
 
   def testDeviceAttributes(self):
     attrs = session._DeviceAttributes(
-        '/job:worker/replica:0/task:3/device:CPU:2', 'TYPE', 1337)
+        '/job:worker/replica:0/task:3/device:CPU:2', 'TYPE', 1337, 1000000)
     self.assertEqual(1337, attrs.memory_limit_bytes)
     self.assertEqual('/job:worker/replica:0/task:3/device:CPU:2', attrs.name)
     self.assertEqual('TYPE', attrs.device_type)
+    self.assertEqual(1000000, attrs.incarnation)
     str_repr = '%s' % attrs
     self.assertTrue(str_repr.startswith('_DeviceAttributes'), str_repr)
 
   def testDeviceAttributesCanonicalization(self):
     attrs = session._DeviceAttributes('/job:worker/replica:0/task:3/cpu:1',
-                                      'TYPE', 1337)
+                                      'TYPE', 1337, 1000000)
     self.assertEqual(1337, attrs.memory_limit_bytes)
     self.assertEqual('/job:worker/replica:0/task:3/device:CPU:1', attrs.name)
     self.assertEqual('TYPE', attrs.device_type)
+    self.assertEqual(1000000, attrs.incarnation)
     str_repr = '%s' % attrs
     self.assertTrue(str_repr.startswith('_DeviceAttributes'), str_repr)
 
diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index 985cb90436..1cdd8e0b6a 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -138,6 +138,11 @@ tensorflow::ImportNumpy();
   $result = PyLong_FromLongLong($1);
 }
 
+// Convert TF_DeviceListIncarnation uint64_t output to Python integer
+%typemap(out) uint64_t {
+  $result = PyLong_FromUnsignedLongLong($1);
+}
+
 // We use TF_OperationGetControlInputs_wrapper instead of
 // TF_OperationGetControlInputs
 %ignore TF_OperationGetControlInputs;
-- 
cgit v1.2.3


From 8571b4f3d2d06caefd8f714d7ea98a3701ea3a96 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 11:37:18 -0700
Subject: Internal Change

PiperOrigin-RevId: 205274579
---
 tensorflow/contrib/lite/delegates/eager/BUILD | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tensorflow/contrib/lite/delegates/eager/BUILD b/tensorflow/contrib/lite/delegates/eager/BUILD
index 9f31ffdf67..03a4b7bf1d 100644
--- a/tensorflow/contrib/lite/delegates/eager/BUILD
+++ b/tensorflow/contrib/lite/delegates/eager/BUILD
@@ -42,10 +42,6 @@ cc_library(
     name = "delegate_data",
     srcs = ["delegate_data.cc"],
     hdrs = ["delegate_data.h"],
-    tags = [
-        "no_oss",
-        "tflite_not_portable",
-    ],
     deps = [
         ":buffer_map",
         "//tensorflow/core:core_cpu",
@@ -59,6 +55,7 @@ cc_test(
     size = "small",
     srcs = ["delegate_data_test.cc"],
     tags = [
+        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
-- 
cgit v1.2.3


From 3d6c8b8aae8433b16af61097641b9958e679fb3d Mon Sep 17 00:00:00 2001
From: Smit Hinsu 
Date: Thu, 19 Jul 2018 11:49:44 -0700
Subject: Assert closeness of output values instead of equality in layout
 optimizer tests using convolutions

Convolution output may differ across convolution algorithms and is not guaranteed to match exactly.

PiperOrigin-RevId: 205276671
---
 tensorflow/python/grappler/layout_optimizer_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/grappler/layout_optimizer_test.py b/tensorflow/python/grappler/layout_optimizer_test.py
index 7d07c77c79..8cc971c61d 100644
--- a/tensorflow/python/grappler/layout_optimizer_test.py
+++ b/tensorflow/python/grappler/layout_optimizer_test.py
@@ -1340,7 +1340,7 @@ class LayoutOptimizerTest(test.TestCase):
       expected_num_transposes = 2
       self.assertEqual(expected_num_transposes, num_transposes)
       self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
-      self.assertAllEqual(output_val_ref, output_val)
+      self.assertAllClose(output_val_ref, output_val, atol=1e-3)
 
   def testLoop(self):
     if test.is_gpu_available(cuda_only=True):
-- 
cgit v1.2.3


From cb11b60da0e2d8e2730e9bb096f40aa2ed1f2b56 Mon Sep 17 00:00:00 2001
From: Mark Daoust 
Date: Thu, 19 Jul 2018 11:50:09 -0700
Subject: Add redirects to point api duplicates to the canonical doc location.

PiperOrigin-RevId: 205276722
---
 tensorflow/tools/docs/generate.py          |  5 +++++
 tensorflow/tools/docs/generate_lib.py      | 30 ++++++++++++++++++++++++++++--
 tensorflow/tools/docs/generate_lib_test.py | 13 ++++++++++++-
 3 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/docs/generate.py b/tensorflow/tools/docs/generate.py
index fc93085e3e..f96887e4c7 100644
--- a/tensorflow/tools/docs/generate.py
+++ b/tensorflow/tools/docs/generate.py
@@ -31,6 +31,11 @@ if __name__ == '__main__':
   doc_generator = generate_lib.DocGenerator()
   doc_generator.add_output_dir_argument()
   doc_generator.add_src_dir_argument()
+  doc_generator.argument_parser.add_argument(
+      '--site_api_path',
+      type=str, default='api_docs/python',
+      help='The path from the site-root to api_docs'
+           'directory for this project')
 
   # This doc generator works on the TensorFlow codebase. Since this script lives
   # at tensorflow/tools/docs, and all code is defined somewhere inside
diff --git a/tensorflow/tools/docs/generate_lib.py b/tensorflow/tools/docs/generate_lib.py
index e7634cd5dc..4f70a69364 100644
--- a/tensorflow/tools/docs/generate_lib.py
+++ b/tensorflow/tools/docs/generate_lib.py
@@ -55,7 +55,8 @@ def write_docs(output_dir,
                parser_config,
                yaml_toc,
                root_title='TensorFlow',
-               search_hints=True):
+               search_hints=True,
+               site_api_path=None):
   """Write previously extracted docs to disk.
 
   Write a docs page for each symbol included in the indices of parser_config to
@@ -73,6 +74,8 @@ def write_docs(output_dir,
     root_title: The title name for the root level index.md.
     search_hints: (bool) include meta-data search hints at the top of each
       output file.
+    site_api_path: Used to write the api-duplicates _redirects.yaml file. if
+      None (the default) the file is not generated.
 
   Raises:
     ValueError: if `output_dir` is not an absolute path
@@ -92,6 +95,9 @@ def write_docs(output_dir,
   #  - symbol name(string):pathname (string)
   symbol_to_file = {}
 
+  # Collect redirects for an api _redirects.yaml file.
+  redirects = ['redirects:\n']
+
   # Parse and write Markdown pages, resolving cross-links (@{symbol}).
   for full_name, py_object in six.iteritems(parser_config.index):
     parser_config.reference_resolver.current_doc_full_name = full_name
@@ -150,6 +156,25 @@ def write_docs(output_dir,
       raise OSError(
           'Cannot write documentation for %s to %s' % (full_name, directory))
 
+    if site_api_path:
+      duplicates = parser_config.duplicates.get(full_name, [])
+      if not duplicates:
+        continue
+
+      duplicates = [item for item in duplicates if item != full_name]
+      template = ('- from: /{}\n'
+                  '  to: /{}\n')
+      for dup in duplicates:
+        from_path = os.path.join(site_api_path, dup.replace('.', '/'))
+        to_path = os.path.join(site_api_path, full_name.replace('.', '/'))
+        redirects.append(
+            template.format(from_path, to_path))
+
+  if site_api_path:
+    api_redirects_path = os.path.join(output_dir, '_redirects.yaml')
+    with open(api_redirects_path, 'w') as redirect_file:
+      redirect_file.write(''.join(redirects))
+
   if yaml_toc:
     # Generate table of contents
 
@@ -608,7 +633,8 @@ class DocGenerator(object):
         parser_config,
         yaml_toc=self.yaml_toc,
         root_title=root_title,
-        search_hints=getattr(flags, 'search_hints', True))
+        search_hints=getattr(flags, 'search_hints', True),
+        site_api_path=getattr(flags, 'site_api_path', None))
 
     # Replace all the @{} references in files under `FLAGS.src_dir`
     replace_refs(flags.src_dir, flags.output_dir, reference_resolver, '*.md')
diff --git a/tensorflow/tools/docs/generate_lib_test.py b/tensorflow/tools/docs/generate_lib_test.py
index 7a6f9fd9f7..de18b13254 100644
--- a/tensorflow/tools/docs/generate_lib_test.py
+++ b/tensorflow/tools/docs/generate_lib_test.py
@@ -107,7 +107,18 @@ class GenerateTest(googletest.TestCase):
 
     output_dir = googletest.GetTempDir()
 
-    generate_lib.write_docs(output_dir, parser_config, yaml_toc=True)
+    generate_lib.write_docs(output_dir, parser_config, yaml_toc=True,
+                            site_api_path='api_docs/python')
+
+    # Check redirects
+    redirects_file = os.path.join(output_dir, '_redirects.yaml')
+    self.assertTrue(os.path.exists(redirects_file))
+    with open(redirects_file) as f:
+      redirects = f.read()
+    self.assertEqual(redirects.split(), [
+        'redirects:', '-', 'from:', '/api_docs/python/tf/test_function', 'to:',
+        '/api_docs/python/tf/TestModule/test_function'
+    ])
 
     # Make sure that the right files are written to disk.
     self.assertTrue(os.path.exists(os.path.join(output_dir, 'index.md')))
-- 
cgit v1.2.3


From cb5f7b3f72bd113ffa8a5179ce289e3ec90fb908 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Thu, 19 Jul 2018 12:06:19 -0700
Subject: Fix python formatting issues.

---
 tensorflow/contrib/tensorrt/BUILD                  |  8 +++----
 .../contrib/tensorrt/test/batch_matmul_test.py     | 10 ++++----
 .../contrib/tensorrt/test/biasadd_matmul_test.py   |  4 ++--
 .../test/binary_tensor_weight_broadcast_test.py    | 28 ++++++++--------------
 .../contrib/tensorrt/test/concatenation_test.py    |  4 ++--
 .../contrib/tensorrt/test/const_broadcast_test.py  | 13 ++++------
 .../test/multi_connection_neighbor_engine_test.py  | 20 +++++-----------
 .../tensorrt/test/neighboring_engine_test.py       | 12 ++++------
 .../tensorrt/test/tf_trt_integration_test_base.py  |  3 ++-
 tensorflow/contrib/tensorrt/test/unary_test.py     |  5 ++--
 .../contrib/tensorrt/test/vgg_block_nchw_test.py   |  7 +++---
 tensorflow/contrib/tensorrt/test/vgg_block_test.py |  7 +++---
 12 files changed, 47 insertions(+), 74 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index fa47f51b66..dea9c0a4ae 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -337,16 +337,16 @@ cuda_py_tests(
     name = "tf_trt_integration_test",
     srcs = [
         "test/base_test.py",
-       #"test/batch_matmul_test.py",
-       #"test/biasadd_matmul_test.py",
+        # "test/batch_matmul_test.py",
+        # "test/biasadd_matmul_test.py",
         "test/binary_tensor_weight_broadcast_test.py",
         "test/concatenation_test.py",
         "test/const_broadcast_test.py",
         "test/multi_connection_neighbor_engine_test.py",
         "test/neighboring_engine_test.py",
         "test/unary_test.py",
-       #"test/vgg_block_nchw_test.py",
-       #"test/vgg_block_test.py",
+        # "test/vgg_block_nchw_test.py",
+        # "test/vgg_block_test.py",
     ],
     additional_deps = [
         ":tf_trt_integration_test_base",
diff --git a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
index 163af54184..e47daae3ee 100644
--- a/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/batch_matmul_test.py
@@ -45,13 +45,10 @@ class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase):
     with g.as_default():
       inp = array_ops.placeholder(
           dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
-      w1 = array_ops.placeholder(
-         dtype=dtype, shape=w1_dims, name=w1_name)
-      w2 = array_ops.placeholder(
-         dtype=dtype, shape=w2_dims, name=w2_name)
+      w1 = array_ops.placeholder(dtype=dtype, shape=w1_dims, name=w1_name)
+      w2 = array_ops.placeholder(dtype=dtype, shape=w2_dims, name=w2_name)
       with g.device("/GPU:0"):
-        b = constant_op.constant(
-            np.random.randn(12, 5, 12, 7), dtype=dtype)
+        b = constant_op.constant(np.random.randn(12, 5, 12, 7), dtype=dtype)
         c = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtype)
         d = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtype)
         x1 = math_ops.matmul(inp, b)
@@ -74,5 +71,6 @@ class BatchMatMulTest(trt_test.TfTrtIntegrationTestBase):
         allclose_atol=1.e-03,
         allclose_rtol=1.e-03)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
index 9b153ada05..2de99c5d5c 100644
--- a/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
+++ b/tensorflow/contrib/tensorrt/test/biasadd_matmul_test.py
@@ -40,8 +40,7 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
     input_dims = [48, 12]
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
 
       b = constant_op.constant(np.random.randn(12, 4), dtype=dtype)
       x1 = math_ops.matmul(x, b)
@@ -108,5 +107,6 @@ class BiasaddMatMulTest(trt_test.TfTrtIntegrationTestBase):
         allclose_atol=1.e-03,
         allclose_rtol=1.e-03)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
index e80712731d..0a3e00afb0 100644
--- a/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/binary_tensor_weight_broadcast_test.py
@@ -39,8 +39,7 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
     input_dims = [10, 24, 24, 20]
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       # scale
       a = constant_op.constant(np.random.randn(1), dtype=dtype)
       f = x + a
@@ -58,13 +57,11 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
       f = a + x
       x = math_ops.sigmoid(f)
       # scale
-      a = constant_op.constant(
-          np.random.randn(24, 24, 20), dtype=dtype)
+      a = constant_op.constant(np.random.randn(24, 24, 20), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
       # scale
-      a = constant_op.constant(
-          np.random.randn(24, 24, 20), dtype=dtype)
+      a = constant_op.constant(np.random.randn(24, 24, 20), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
       # elementwise
@@ -76,33 +73,27 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
       f = a + x
       x = math_ops.sigmoid(f)
       # elementwise
-      a = constant_op.constant(
-          np.random.randn(1, 24, 1, 1), dtype=dtype)
+      a = constant_op.constant(np.random.randn(1, 24, 1, 1), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
       # elementwise
-      a = constant_op.constant(
-          np.random.randn(1, 24, 1, 1), dtype=dtype)
+      a = constant_op.constant(np.random.randn(1, 24, 1, 1), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
       # elementwise
-      a = constant_op.constant(
-          np.random.randn(1, 24, 24, 1), dtype=dtype)
+      a = constant_op.constant(np.random.randn(1, 24, 24, 1), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
       # elementwise
-      a = constant_op.constant(
-          np.random.randn(1, 24, 24, 1), dtype=dtype)
+      a = constant_op.constant(np.random.randn(1, 24, 24, 1), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
       # elementwise
-      a = constant_op.constant(
-          np.random.randn(1, 24, 24, 20), dtype=dtype)
+      a = constant_op.constant(np.random.randn(1, 24, 24, 20), dtype=dtype)
       f = a + x
       x = math_ops.sigmoid(f)
       # elementwise
-      a = constant_op.constant(
-          np.random.randn(1, 24, 24, 20), dtype=dtype)
+      a = constant_op.constant(np.random.randn(1, 24, 24, 20), dtype=dtype)
       f = x + a
       x = math_ops.sigmoid(f)
       # elementwise
@@ -123,5 +114,6 @@ class BinaryTensorWeightBroadcastTest(trt_test.TfTrtIntegrationTestBase):
         allclose_atol=1.e-03,
         allclose_rtol=1.e-03)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/concatenation_test.py b/tensorflow/contrib/tensorrt/test/concatenation_test.py
index cf0bfeeb00..222167f8d5 100644
--- a/tensorflow/contrib/tensorrt/test/concatenation_test.py
+++ b/tensorflow/contrib/tensorrt/test/concatenation_test.py
@@ -39,8 +39,7 @@ class ConcatenationTest(trt_test.TfTrtIntegrationTestBase):
     input_dims = [2, 3, 3, 1]
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       # scale
       a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
       r1 = x / a
@@ -79,5 +78,6 @@ class ConcatenationTest(trt_test.TfTrtIntegrationTestBase):
         allclose_atol=1.e-03,
         allclose_rtol=1.e-03)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
index 97f5580ac0..6e5f546fc7 100644
--- a/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
+++ b/tensorflow/contrib/tensorrt/test/const_broadcast_test.py
@@ -34,21 +34,17 @@ class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase):
   def GetParams(self):
     """unit test for Constant broadcasting in TF-TRT"""
     dtype = dtypes.float32
-    input_name = "input"
+    input_name = 'input'
     input_dims = [5, 12, 12, 2]
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       filt1 = constant_op.constant(
           0.3, shape=(3, 3, 2, 1), dtype=dtype, name='filt1')
       y1 = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='SAME', name='y1')
       z1 = nn.relu(y1, name='z1')
       filt2 = constant_op.constant(
-          np.random.randn(9),
-          shape=(3, 3, 1, 1),
-          dtype=dtype,
-          name='filt2')
+          np.random.randn(9), shape=(3, 3, 1, 1), dtype=dtype, name='filt2')
       y2 = nn.conv2d(z1, filt2, strides=[1, 1, 1, 1], padding='SAME', name='y2')
       z2 = nn.relu(y2, name='z')
       filt3 = constant_op.constant(
@@ -67,5 +63,6 @@ class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase):
         allclose_atol=1.e-02,
         allclose_rtol=1.e-02)
 
-if __name__ == "__main__":
+
+if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
index e62f9e479e..7fa798fb45 100644
--- a/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/multi_connection_neighbor_engine_test.py
@@ -40,8 +40,7 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
     input_dims = [2, 3, 7, 5]
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       e = constant_op.constant(
           np.random.normal(.05, .005, [3, 2, 3, 4]),
           name="weights",
@@ -54,29 +53,21 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
           padding="VALID",
           name="conv")
       b = constant_op.constant(
-          np.random.normal(2.0, 1.0, [1, 4, 1, 1]),
-          name="bias",
-          dtype=dtype)
+          np.random.normal(2.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
       t = conv + b
 
       b = constant_op.constant(
-          np.random.normal(5.0, 1.0, [1, 4, 1, 1]),
-          name="bias",
-          dtype=dtype)
+          np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
       q = conv - b
       edge = math_ops.sigmoid(q)
 
       b = constant_op.constant(
-          np.random.normal(5.0, 1.0, [1, 4, 1, 1]),
-          name="bias",
-          dtype=dtype)
+          np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
       d = b + conv
       edge3 = math_ops.sigmoid(d)
 
       c = constant_op.constant(
-          np.random.normal(1.0, 1.0, [1, 4, 1, 1]),
-          name="bias",
-          dtype=dtype)
+          np.random.normal(1.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
       edge1 = gen_math_ops.tan(conv)
       t = t - edge1
       q = q + edge
@@ -93,5 +84,6 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase):
         allclose_atol=1.e-03,
         allclose_rtol=1.e-03)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
index bbe8823552..439af81664 100644
--- a/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
+++ b/tensorflow/contrib/tensorrt/test/neighboring_engine_test.py
@@ -39,12 +39,9 @@ class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase):
     input_dims = [2, 3, 7, 5]
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       e = constant_op.constant(
-          np.random.normal(.3, 0.05, [3, 2, 3, 4]),
-          name="weights",
-          dtype=dtype)
+          np.random.normal(.3, 0.05, [3, 2, 3, 4]), name="weights", dtype=dtype)
       conv = nn.conv2d(
           input=x,
           filter=e,
@@ -53,9 +50,7 @@ class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase):
           padding="VALID",
           name="conv")
       b = constant_op.constant(
-          np.random.normal(1.0, 1.0, [1, 4, 1, 1]),
-          name="bias",
-          dtype=dtype)
+          np.random.normal(1.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
       t = conv * b
       e = gen_math_ops.tan(conv)
       t = t - e
@@ -69,5 +64,6 @@ class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase):
         allclose_atol=1.e-03,
         allclose_rtol=1.e-03)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 48890ad413..6e12e7e026 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -169,7 +169,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
         num_engines += 1
         self.assertNotEqual(self._ToBytes(""), n.attr["serialized_segment"].s)
         self.assertNotEqual(self._ToBytes(""), n.attr["segment_funcdef_name"].s)
-        self.assertEqual(self._ToBytes(precision_mode), n.attr["precision_mode"].s)
+        self.assertEqual(
+            self._ToBytes(precision_mode), n.attr["precision_mode"].s)
         self.assertEqual(not dynamic_engine, n.attr["static_engine"].b)
         if _IsQuantizationMode(precision_mode) and is_calibrated:
           self.assertNotEqual(self._ToBytes(""), n.attr["calibration_data"].s)
diff --git a/tensorflow/contrib/tensorrt/test/unary_test.py b/tensorflow/contrib/tensorrt/test/unary_test.py
index 4c10c50e85..7395c4a311 100644
--- a/tensorflow/contrib/tensorrt/test/unary_test.py
+++ b/tensorflow/contrib/tensorrt/test/unary_test.py
@@ -32,7 +32,6 @@ from tensorflow.contrib.tensorrt.test import tf_trt_integration_test_base as trt
 
 
 class UnaryTest(trt_test.TfTrtIntegrationTestBase):
-    
 
   def GetParams(self):
     """unit test for unary operations in TF-TRT"""
@@ -43,8 +42,7 @@ class UnaryTest(trt_test.TfTrtIntegrationTestBase):
     input2_dims = [12, 5, 8, 1, 12, 1, 1]
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       q = math_ops.abs(x)
       q = q + 1.0
       q = gen_math_ops.exp(q)
@@ -107,5 +105,6 @@ class UnaryTest(trt_test.TfTrtIntegrationTestBase):
         allclose_atol=1.e-03,
         allclose_rtol=1.e-03)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
index 3621c13bc9..0e28afeaf4 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_nchw_test.py
@@ -40,8 +40,7 @@ class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase):
     input_dims = [5, 2, 8, 8]
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       x, mean_x, var_x = nn_impl.fused_batch_norm(
           x,
           np.random.randn(2).astype(np.float32),
@@ -59,8 +58,7 @@ class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase):
           strides=[1, 1, 2, 2],
           padding="SAME",
           name="conv")
-      b = constant_op.constant(
-          np.random.randn(6), name="bias", dtype=dtype)
+      b = constant_op.constant(np.random.randn(6), name="bias", dtype=dtype)
       t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd")
       relu = nn.relu(t, "relu")
       idty = array_ops.identity(relu, "ID")
@@ -79,5 +77,6 @@ class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase):
         allclose_atol=1.e-03,
         allclose_rtol=1.e-03)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/tensorrt/test/vgg_block_test.py b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
index 1ef32fe52f..f2aacddd87 100644
--- a/tensorflow/contrib/tensorrt/test/vgg_block_test.py
+++ b/tensorflow/contrib/tensorrt/test/vgg_block_test.py
@@ -40,8 +40,7 @@ class VGGBlockTest(trt_test.TfTrtIntegrationTestBase):
     input_dims = [5, 8, 8, 2]
     g = ops.Graph()
     with g.as_default():
-      x = array_ops.placeholder(
-          dtype=dtype, shape=input_dims, name=input_name)
+      x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
       x, mean_x, var_x = nn_impl.fused_batch_norm(
           x,
           np.random.randn(2).astype(np.float32),
@@ -53,8 +52,7 @@ class VGGBlockTest(trt_test.TfTrtIntegrationTestBase):
           np.random.randn(1, 1, 2, 6), name="weights", dtype=dtype)
       conv = nn.conv2d(
           input=x, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv")
-      b = constant_op.constant(
-          np.random.randn(6), name="bias", dtype=dtype)
+      b = constant_op.constant(np.random.randn(6), name="bias", dtype=dtype)
       t = nn.bias_add(conv, b, name="biasAdd")
       relu = nn.relu(t, "relu")
       idty = array_ops.identity(relu, "ID")
@@ -70,5 +68,6 @@ class VGGBlockTest(trt_test.TfTrtIntegrationTestBase):
         allclose_atol=1.e-03,
         allclose_rtol=1.e-03)
 
+
 if __name__ == "__main__":
   test.main()
-- 
cgit v1.2.3


From f330a1c8925a4a33bd0ea451656cfd80772979c3 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Thu, 19 Jul 2018 12:11:20 -0700
Subject: Intern predicate pointers

This is a performance optimization.

PiperOrigin-RevId: 205280010
---
 tensorflow/compiler/jit/deadness_analysis.cc | 158 +++++++++++++++------------
 1 file changed, 89 insertions(+), 69 deletions(-)

diff --git a/tensorflow/compiler/jit/deadness_analysis.cc b/tensorflow/compiler/jit/deadness_analysis.cc
index b2d119029a..d81e5fe900 100644
--- a/tensorflow/compiler/jit/deadness_analysis.cc
+++ b/tensorflow/compiler/jit/deadness_analysis.cc
@@ -44,10 +44,6 @@ class Predicate {
   enum class Kind { kAnd, kOr, kNot, kSymbol };
 
   virtual string ToString() const = 0;
-  virtual bool operator==(const Predicate& other) const = 0;
-  virtual bool operator!=(const Predicate& other) const {
-    return !(*this == other);
-  }
   int64 hash() const { return hash_; }
 
   virtual Kind kind() const = 0;
@@ -58,6 +54,8 @@ class Predicate {
 
  private:
   const int64 hash_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(Predicate);
 };
 
 int64 HashPredicateSequence(Predicate::Kind kind,
@@ -69,19 +67,6 @@ int64 HashPredicateSequence(Predicate::Kind kind,
   return hash;
 }
 
-bool PredicateSequenceEqual(gtl::ArraySlice lhs,
-                            gtl::ArraySlice rhs) {
-  if (lhs.size() != rhs.size()) {
-    return false;
-  }
-  for (int64 i = 0; i < lhs.size(); i++) {
-    if (*lhs[i] != *rhs[i]) {
-      return false;
-    }
-  }
-  return true;
-}
-
 // Represents a logical conjunction of a set of predicates.
 class AndPredicate : public Predicate {
  public:
@@ -102,17 +87,9 @@ class AndPredicate : public Predicate {
     return strings::StrCat("(", str_util::Join(operands_str, " & "), ")");
   }
 
-  bool operator==(const Predicate& other) const override {
-    return other.kind() == Kind::kAnd &&
-           PredicateSequenceEqual(
-               dynamic_cast(other).operands(), operands());
-  }
-
   Kind kind() const override { return Kind::kAnd; }
 
-  const tensorflow::gtl::ArraySlice operands() const {
-    return operands_;
-  }
+  const gtl::ArraySlice operands() const { return operands_; }
 
  private:
   std::vector operands_;
@@ -138,16 +115,8 @@ class OrPredicate : public Predicate {
     return strings::StrCat("(", str_util::Join(operands_str, " | "), ")");
   }
 
-  bool operator==(const Predicate& other) const override {
-    return other.kind() == Kind::kOr &&
-           PredicateSequenceEqual(
-               dynamic_cast(other).operands(), operands());
-  }
-
   Kind kind() const override { return Kind::kOr; }
-  const tensorflow::gtl::ArraySlice operands() const {
-    return operands_;
-  }
+  const gtl::ArraySlice operands() const { return operands_; }
 
  private:
   std::vector operands_;
@@ -164,11 +133,6 @@ class NotPredicate : public Predicate {
     return strings::StrCat("~", operand()->ToString());
   }
 
-  bool operator==(const Predicate& other) const override {
-    return other.kind() == Kind::kNot &&
-           *dynamic_cast(other).operand() == *operand();
-  }
-
   Kind kind() const override { return Kind::kNot; }
   Predicate* operand() const { return operand_; }
 
@@ -188,14 +152,6 @@ class SymbolPredicate : public Predicate {
         must_be_true_(must_be_true) {}
 
   string ToString() const override { return tensor_id_.ToString(); }
-  bool operator==(const Predicate& other) const override {
-    return other.kind() == Kind::kSymbol &&
-           must_be_true() ==
-               dynamic_cast(other).must_be_true() &&
-           dynamic_cast(other).tensor_id() ==
-               tensor_id();
-  }
-
   Kind kind() const override { return Kind::kSymbol; }
 
   // If `must_be_true()` is true this SymbolPredicate represents the proposition
@@ -225,16 +181,37 @@ class PredicateFactory {
   Predicate* MakeAndPredicate(gtl::ArraySlice operands) {
     return MakeAndOrImpl(operands, /*is_and=*/true);
   }
+
   Predicate* MakeOrPredicate(gtl::ArraySlice operands) {
     return MakeAndOrImpl(operands, /*is_and=*/false);
   }
 
   Predicate* MakeNotPredicate(Predicate* pred) {
-    return Make(pred);
+    SignatureForNot signature = pred;
+    auto it = interned_not_instances_.find(signature);
+    if (it == interned_not_instances_.end()) {
+      std::unique_ptr new_pred = Make(pred);
+      Predicate* new_pred_ptr = new_pred.get();
+      interned_not_instances_.emplace(signature, std::move(new_pred));
+      return new_pred_ptr;
+    } else {
+      return it->second.get();
+    }
   }
 
   Predicate* MakeSymbolPredicate(TensorId tensor_id, bool must_be_true) {
-    return Make(tensor_id, must_be_true);
+    SignatureForSymbol signature = {tensor_id, must_be_true};
+    auto it = interned_symbol_instances_.find(signature);
+    if (it == interned_symbol_instances_.end()) {
+      std::unique_ptr new_pred =
+          Make(tensor_id, must_be_true);
+      Predicate* new_pred_ptr = new_pred.get();
+      interned_symbol_instances_.emplace(std::move(signature),
+                                         std::move(new_pred));
+      return new_pred_ptr;
+    } else {
+      return it->second.get();
+    }
   }
 
   Predicate* MakeTrue() { return MakeAndPredicate({}); }
@@ -242,29 +219,53 @@ class PredicateFactory {
 
  private:
   template 
-  Predicate* Make(Args... args) {
-    std::unique_ptr pred(
+  std::unique_ptr Make(Args&&... args) {
+    return std::unique_ptr(
         new PredicateT(std::forward(args)...));
-    predicate_storage_.emplace_back(std::move(pred));
-    return predicate_storage_.back().get();
   }
 
   Predicate* MakeAndOrImpl(gtl::ArraySlice operands, bool is_and);
 
-  struct PredicatePtrHash {
-    size_t operator()(const Predicate* pred) const { return pred->hash(); }
+  // Predicate instances are interned, meaning that there is only a single
+  // instance of a Predicate object with a given content.  This makes checking
+  // for structural equality super-cheap -- we can just compare pointers.
+  //
+  // We intern predicates by maintaining a map from the content of a Predicate
+  // to the only instance of said predicate we allow to exist in the
+  // interned_and_or_instances_, interned_not_instances_ and
+  // interned_symbol_instances_ fields.  These maps also double up as storage
+  // for the owning pointers to predicate instances.
+
+  using SignatureForAndOr =
+      std::pair>;
+  using SignatureForNot = Predicate*;
+  using SignatureForSymbol = std::pair;
+
+  struct HashSignatureForAndOr {
+    size_t operator()(const SignatureForAndOr& signature) const {
+      size_t hash = ::tensorflow::hash()(signature.first);
+      for (Predicate* p : signature.second) {
+        hash = Hash64Combine(hash, ::tensorflow::hash()(p));
+      }
+      return hash;
+    }
   };
 
-  struct PredicatePtrEq {
-    size_t operator()(const Predicate* a, const Predicate* b) const {
-      return *a == *b;
+  struct HashSignatureForSymbol {
+    size_t operator()(const SignatureForSymbol& signature) const {
+      return Hash64Combine(SafeTensorId::Hasher()(signature.first),
+                           ::tensorflow::hash()(signature.second));
     }
   };
 
-  using PredicateSet =
-      gtl::FlatSet;
-
-  std::vector> predicate_storage_;
+  gtl::FlatMap,
+               HashSignatureForAndOr>
+      interned_and_or_instances_;
+  gtl::FlatMap>
+      interned_not_instances_;
+  gtl::FlatMap,
+               HashSignatureForSymbol>
+      interned_symbol_instances_;
 };
 
 // Common code to create AndPredicate or OrPredicate instances.
@@ -272,7 +273,7 @@ Predicate* PredicateFactory::MakeAndOrImpl(gtl::ArraySlice operands,
                                            bool is_and) {
   Predicate::Kind pred_kind =
       is_and ? Predicate::Kind::kAnd : Predicate::Kind::kOr;
-  PredicateSet simplified_ops_set;
+  gtl::FlatSet simplified_ops_set;
   std::vector simplified_ops;
   for (Predicate* op : operands) {
     // Simplify A&A => A and  A|A => A.
@@ -300,7 +301,7 @@ Predicate* PredicateFactory::MakeAndOrImpl(gtl::ArraySlice operands,
   }
 
   // Simplify "A&~A=>False" and "A|~A=>True".
-  PredicateSet negated_ops;
+  gtl::FlatSet negated_ops;
   for (Predicate* op : simplified_ops) {
     if (op->kind() == Predicate::Kind::kNot) {
       negated_ops.insert(dynamic_cast(*op).operand());
@@ -317,8 +318,26 @@ Predicate* PredicateFactory::MakeAndOrImpl(gtl::ArraySlice operands,
       simplified_ops.begin(), simplified_ops.end(),
       [](Predicate* a, Predicate* b) { return a->hash() < b->hash(); });
 
-  return is_and ? Make(std::move(simplified_ops))
-                : Make(std::move(simplified_ops));
+  auto it = interned_and_or_instances_.find({pred_kind, simplified_ops});
+  if (it == interned_and_or_instances_.end()) {
+    simplified_ops.shrink_to_fit();
+    // NB!  Because we'll use a non-owning reference to simplified_ops in the
+    // key for interned_and_or_instances_ we need to be careful to std::move()
+    // it all the way through.
+    gtl::ArraySlice operands_slice = simplified_ops;
+    std::unique_ptr new_pred =
+        is_and ? Make(std::move(simplified_ops))
+               : Make(std::move(simplified_ops));
+
+    Predicate* new_pred_ptr = new_pred.get();
+    CHECK(interned_and_or_instances_
+              .emplace(SignatureForAndOr(pred_kind, operands_slice),
+                       std::move(new_pred))
+              .second);
+    return new_pred_ptr;
+  } else {
+    return it->second.get();
+  }
 }
 
 class DeadnessAnalysisImpl : public DeadnessAnalysis {
@@ -491,8 +510,9 @@ bool DeadnessAnalysisImpl::HasInputsWithMismatchingDeadness(const Node& node) {
 
     // Today we just compare the predicates for equality (with some
     // canonicalization/simplification happening before) but we could be more
-    // sophisticated here if need be.
-    if (pred != nullptr && *pred != *it->second) {
+    // sophisticated here if need be.  Comparing pointers is sufficient because
+    // we intern Predicate instances by their content.
+    if (pred != nullptr && pred != it->second) {
       if (vlog_) {
         VLOG(2) << "HasInputsWithMismatchingDeadness(" << node.name()
                 << ") -> true";
-- 
cgit v1.2.3


From bbe9364b22ce78f0ab4ec03895bd7178802ef893 Mon Sep 17 00:00:00 2001
From: Michael Case 
Date: Thu, 19 Jul 2018 12:16:26 -0700
Subject: Internal Change.

PiperOrigin-RevId: 205280699
---
 tensorflow/contrib/estimator/BUILD | 204 +++++--------------------------------
 tensorflow/python/estimator/BUILD  |   8 +-
 2 files changed, 29 insertions(+), 183 deletions(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 11d40f5982..1aa3df8d8d 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -28,7 +28,7 @@ py_library(
         ":multi_head",
         ":replicate_model_fn",
         ":rnn",
-        "//tensorflow/python:util",
+        "//tensorflow:tensorflow_py_no_contrib",
     ],
 )
 
@@ -54,22 +54,10 @@ py_test(
     deps = [
         ":baseline",
         ":head",
-        "//tensorflow/python:check_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:session",
-        "//tensorflow/python:summary",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variables",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:export_export",
         "//tensorflow/python/estimator:metric_keys",
         "//tensorflow/python/estimator:numpy_io",
-        "//tensorflow/python/feature_column",
-        "//tensorflow/python/ops/losses",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -96,11 +84,8 @@ py_test(
     ],
     deps = [
         ":boosted_trees",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:training",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:numpy_io",
-        "//tensorflow/python/feature_column",
         "//third_party/py/numpy",
     ],
 )
@@ -110,7 +95,7 @@ py_library(
     srcs = ["python/estimator/dnn.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:nn",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator",
         "//tensorflow/python/estimator:dnn",
     ],
@@ -129,16 +114,11 @@ py_test(
     deps = [
         ":dnn",
         ":head",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:summary",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:dnn_testing_utils",
         "//tensorflow/python/estimator:export_export",
         "//tensorflow/python/estimator:numpy_io",
         "//tensorflow/python/estimator:prediction_keys",
-        "//tensorflow/python/feature_column",
-        "//tensorflow/python/ops/losses",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -149,7 +129,7 @@ py_library(
     srcs = ["python/estimator/dnn_linear_combined.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:nn",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator",
         "//tensorflow/python/estimator:dnn_linear_combined",
     ],
@@ -168,18 +148,12 @@ py_test(
     deps = [
         ":dnn_linear_combined",
         ":head",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:nn",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:summary",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:dnn_testing_utils",
         "//tensorflow/python/estimator:export_export",
         "//tensorflow/python/estimator:linear_testing_utils",
         "//tensorflow/python/estimator:numpy_io",
         "//tensorflow/python/estimator:prediction_keys",
-        "//tensorflow/python/feature_column",
-        "//tensorflow/python/ops/losses",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -192,10 +166,7 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:clip_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:sparse_tensor",
-        "//tensorflow/python:training",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator",
         "//tensorflow/python/estimator:model_fn",
         "//tensorflow/python/estimator:util",
@@ -211,18 +182,11 @@ py_test(
     tags = ["notsan"],  # b/62863147
     deps = [
         ":extenders",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/predictor",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:metrics",
-        "//tensorflow/python:sparse_tensor",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variables",
         "//tensorflow/python/estimator:estimator_py",
         "//tensorflow/python/estimator:linear",
-        "//tensorflow/python/feature_column",
         "//third_party/py/numpy",
     ],
 )
@@ -246,21 +210,11 @@ py_test(
     tags = ["notsan"],  # b/62863147
     deps = [
         ":export",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:metrics",
-        "//tensorflow/python:parsing_ops",
-        "//tensorflow/python:session",
-        "//tensorflow/python:state_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python:util",
-        "//tensorflow/python:variables",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator",
         "//tensorflow/python/estimator:export_export",
         "//tensorflow/python/estimator:export_output",
         "//tensorflow/python/estimator:model_fn",
-        "//tensorflow/python/saved_model:loader",
-        "//tensorflow/python/saved_model:tag_constants",
     ],
 )
 
@@ -271,25 +225,12 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:check_ops",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:lookup_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:metrics",
-        "//tensorflow/python:nn",
-        "//tensorflow/python:sparse_ops",
-        "//tensorflow/python:sparse_tensor",
-        "//tensorflow/python:summary",
-        "//tensorflow/python:training",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:export_output",
         "//tensorflow/python/estimator:head",
         "//tensorflow/python/estimator:metric_keys",
         "//tensorflow/python/estimator:model_fn",
         "//tensorflow/python/estimator:prediction_keys",
-        "//tensorflow/python/ops/losses",
-        "//tensorflow/python/saved_model:signature_constants",
     ],
 )
 
@@ -300,25 +241,10 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":head",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:check_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:sparse_tensor",
-        "//tensorflow/python:string_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variables",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:metric_keys",
         "//tensorflow/python/estimator:model_fn",
         "//tensorflow/python/estimator:prediction_keys",
-        "//tensorflow/python/ops/losses",
-        "//tensorflow/python/saved_model:signature_constants",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -331,8 +257,7 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:training",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:estimator_py",
     ],
 )
@@ -345,10 +270,7 @@ py_test(
     tags = ["notsan"],
     deps = [
         ":hooks",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:estimator_py",
         "//third_party/py/numpy",
         "@six_archive//:six",
@@ -377,16 +299,11 @@ py_test(
     deps = [
         ":head",
         ":linear",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:summary",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:export_export",
         "//tensorflow/python/estimator:linear_testing_utils",
         "//tensorflow/python/estimator:numpy_io",
         "//tensorflow/python/estimator:prediction_keys",
-        "//tensorflow/python/feature_column",
-        "//tensorflow/python/ops/losses",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -399,8 +316,7 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:util",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:dnn",
         "//tensorflow/python/estimator:linear",
     ],
@@ -413,9 +329,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":logit_fns",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:session",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:model_fn",
     ],
 )
@@ -427,18 +341,11 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:metrics",
-        "//tensorflow/python:summary",
-        "//tensorflow/python:training",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:export_output",
         "//tensorflow/python/estimator:head",
         "//tensorflow/python/estimator:metric_keys",
         "//tensorflow/python/estimator:model_fn",
-        "//tensorflow/python/saved_model:signature_constants",
         "@six_archive//:six",
     ],
 )
@@ -451,15 +358,10 @@ py_test(
     deps = [
         ":head",
         ":multi_head",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:string_ops",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:metric_keys",
         "//tensorflow/python/estimator:model_fn",
         "//tensorflow/python/estimator:prediction_keys",
-        "//tensorflow/python/saved_model:signature_constants",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -472,24 +374,10 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:device",
-        "//tensorflow/python:device_lib",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:sparse_ops",
-        "//tensorflow/python:sparse_tensor",
-        "//tensorflow/python:state_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python:util",
-        "//tensorflow/python:variable_scope",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator:export_output",
         "//tensorflow/python/estimator:model_fn",
         "//tensorflow/python/estimator:util",
-        "//tensorflow/python/ops/losses",
         "@six_archive//:six",
     ],
 )
@@ -500,6 +388,7 @@ cuda_py_test(
     srcs = ["python/estimator/replicate_model_fn_test.py"],
     additional_deps = [
         "@absl_py//absl/testing:parameterized",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator",
         "//tensorflow/python/estimator:dnn",
         "//tensorflow/python/estimator:export_export",
@@ -508,21 +397,6 @@ cuda_py_test(
         "//tensorflow/python/estimator:numpy_io",
         "//tensorflow/python/estimator:optimizers",
         "//tensorflow/python/estimator:prediction_keys",
-        "//tensorflow/python/feature_column",
-        "//tensorflow/python/ops/losses",
-        "//tensorflow/python/saved_model:signature_constants",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:metrics",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:summary",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variable_scope",
-        "//tensorflow/python:variables",
         ":replicate_model_fn",
     ],
     tags = [
@@ -538,22 +412,11 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":extenders",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/contrib/feature_column:feature_column_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:check_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:init_ops",
-        "//tensorflow/python:layers",
-        "//tensorflow/python:partitioned_variables",
-        "//tensorflow/python:rnn",
-        "//tensorflow/python:rnn_cell",
-        "//tensorflow/python:summary",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variable_scope",
         "//tensorflow/python/estimator",
         "//tensorflow/python/estimator:head",
         "//tensorflow/python/estimator:optimizers",
-        "//tensorflow/python/feature_column",
         "@six_archive//:six",
     ],
 )
@@ -572,21 +435,10 @@ py_test(
     deps = [
         ":head",
         ":rnn",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/contrib/data",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:check_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:state_ops",
-        "//tensorflow/python:summary",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variables",
         "//tensorflow/python/estimator:numpy_io",
         "//tensorflow/python/estimator:parsing_utils",
-        "//tensorflow/python/feature_column",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -597,13 +449,7 @@ py_library(
     srcs = ["python/estimator/early_stopping.py"],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:init_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:state_ops",
-        "//tensorflow/python:summary",
-        "//tensorflow/python:training",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator",
     ],
 )
@@ -614,7 +460,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":early_stopping",
-        "//tensorflow/python:client_testlib",
+        "//tensorflow:tensorflow_py_no_contrib",
         "//tensorflow/python/estimator",
         "@absl_py//absl/testing:parameterized",
     ],
diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index 6c415b1bf2..fd46163050 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -40,9 +40,9 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":gc",
+        ":metric_keys",
+        ":util",
         "//tensorflow:tensorflow_py_no_contrib",
-        "//tensorflow/python/estimator:metric_keys",
-        "//tensorflow/python/estimator:util",
     ],
 )
 
@@ -683,9 +683,9 @@ py_test(
     ],
     deps = [
         ":keras",
+        ":numpy_io",
+        ":run_config",
         "//tensorflow:tensorflow_py_no_contrib",
-        "//tensorflow/python/estimator:numpy_io",
-        "//tensorflow/python/estimator:run_config",
         "//third_party/py/numpy",
     ],
 )
-- 
cgit v1.2.3


From e9869ece182be721dc07fe8ecb7c7288f2fce90f Mon Sep 17 00:00:00 2001
From: Jiri Simsa 
Date: Thu, 19 Jul 2018 12:19:37 -0700
Subject: [tf.data] Adding `tf.contrib.data.reduce_dataset` which can be used
 to reduce a dataset to a single element.

PiperOrigin-RevId: 205281140
---
 tensorflow/contrib/data/__init__.py                |  2 +
 tensorflow/contrib/data/python/kernel_tests/BUILD  |  2 +
 .../python/kernel_tests/get_single_element_test.py | 82 +++++++++++++++-------
 tensorflow/contrib/data/python/ops/BUILD           |  3 +
 .../contrib/data/python/ops/get_single_element.py  | 30 ++++++++
 5 files changed, 94 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index 675330716b..7878e46e88 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -52,6 +52,7 @@ See @{$guide/datasets$Importing Data} for an overview.
 @@prefetch_to_device
 @@read_batch_features
 @@rejection_resample
+@@reduce_dataset
 @@sample_from_datasets
 @@scan
 @@shuffle_and_repeat
@@ -77,6 +78,7 @@ from tensorflow.contrib.data.python.ops.counter import Counter
 from tensorflow.contrib.data.python.ops.enumerate_ops import enumerate_dataset
 from tensorflow.contrib.data.python.ops.error_ops import ignore_errors
 from tensorflow.contrib.data.python.ops.get_single_element import get_single_element
+from tensorflow.contrib.data.python.ops.get_single_element import reduce_dataset
 from tensorflow.contrib.data.python.ops.grouping import bucket_by_sequence_length
 from tensorflow.contrib.data.python.ops.grouping import group_by_reducer
 from tensorflow.contrib.data.python.ops.grouping import group_by_window
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index f805027727..036dc795bb 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -121,6 +121,7 @@ py_test(
     srcs = ["get_single_element_test.py"],
     deps = [
         "//tensorflow/contrib/data/python/ops:get_single_element",
+        "//tensorflow/contrib/data/python/ops:grouping",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
@@ -128,6 +129,7 @@ py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python/data/ops:dataset_ops",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/contrib/data/python/kernel_tests/get_single_element_test.py b/tensorflow/contrib/data/python/kernel_tests/get_single_element_test.py
index 87b7c6ddb7..e6883d53e0 100644
--- a/tensorflow/contrib/data/python/kernel_tests/get_single_element_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/get_single_element_test.py
@@ -17,9 +17,12 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
+import numpy as np
+
 from tensorflow.contrib.data.python.ops import get_single_element
+from tensorflow.contrib.data.python.ops import grouping
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
@@ -27,40 +30,69 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
-class GetSingleElementTest(test.TestCase):
+class GetSingleElementTest(test.TestCase, parameterized.TestCase):
 
-  def testGetSingleElement(self):
-    skip_value = array_ops.placeholder(dtypes.int64, shape=[])
-    take_value = array_ops.placeholder_with_default(
-        constant_op.constant(1, dtype=dtypes.int64), shape=[])
+  @parameterized.named_parameters(
+      ("Zero", 0, 1),
+      ("Five", 5, 1),
+      ("Ten", 10, 1),
+      ("Empty", 100, 1, errors.InvalidArgumentError, "Dataset was empty."),
+      ("MoreThanOne", 0, 2, errors.InvalidArgumentError,
+       "Dataset had more than one element."),
+  )
+  def testGetSingleElement(self, skip, take, error=None, error_msg=None):
+    skip_t = array_ops.placeholder(dtypes.int64, shape=[])
+    take_t = array_ops.placeholder(dtypes.int64, shape=[])
 
     def make_sparse(x):
       x_1d = array_ops.reshape(x, [1])
       x_2d = array_ops.reshape(x, [1, 1])
       return sparse_tensor.SparseTensor(x_2d, x_1d, x_1d)
 
-    dataset = (dataset_ops.Dataset.range(100)
-               .skip(skip_value)
-               .map(lambda x: (x * x, make_sparse(x)))
-               .take(take_value))
-
+    dataset = dataset_ops.Dataset.range(100).skip(skip_t).map(
+        lambda x: (x * x, make_sparse(x))).take(take_t)
     element = get_single_element.get_single_element(dataset)
 
     with self.test_session() as sess:
-      for x in [0, 5, 10]:
-        dense_val, sparse_val = sess.run(element, feed_dict={skip_value: x})
-        self.assertEqual(x * x, dense_val)
-        self.assertAllEqual([[x]], sparse_val.indices)
-        self.assertAllEqual([x], sparse_val.values)
-        self.assertAllEqual([x], sparse_val.dense_shape)
-
-      with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                   "Dataset was empty."):
-        sess.run(element, feed_dict={skip_value: 100})
-
-      with self.assertRaisesRegexp(errors.InvalidArgumentError,
-                                   "Dataset had more than one element."):
-        sess.run(element, feed_dict={skip_value: 0, take_value: 2})
+      if error is None:
+        dense_val, sparse_val = sess.run(
+            element, feed_dict={
+                skip_t: skip,
+                take_t: take
+            })
+        self.assertEqual(skip * skip, dense_val)
+        self.assertAllEqual([[skip]], sparse_val.indices)
+        self.assertAllEqual([skip], sparse_val.values)
+        self.assertAllEqual([skip], sparse_val.dense_shape)
+      else:
+        with self.assertRaisesRegexp(error, error_msg):
+          sess.run(element, feed_dict={skip_t: skip, take_t: take})
+
+  @parameterized.named_parameters(
+      ("SumZero", 0),
+      ("SumOne", 1),
+      ("SumFive", 5),
+      ("SumTen", 10),
+  )
+  def testReduceDataset(self, stop):
+    def init_fn(_):
+      return np.int64(0)
+
+    def reduce_fn(state, value):
+      return state + value
+
+    def finalize_fn(state):
+      return state
+
+    sum_reducer = grouping.Reducer(init_fn, reduce_fn, finalize_fn)
+
+    stop_t = array_ops.placeholder(dtypes.int64, shape=[])
+    dataset = dataset_ops.Dataset.range(stop_t)
+    element = get_single_element.reduce_dataset(dataset, sum_reducer)
+
+    with self.test_session() as sess:
+      value = sess.run(element, feed_dict={stop_t: stop})
+      self.assertEqual(stop * (stop - 1) / 2, value)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index 160d7fe22a..1ad021ea03 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -28,10 +28,12 @@ py_library(
     srcs = ["get_single_element.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":grouping",
         "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/util:nest",
         "//tensorflow/python/data/util:sparse",
+        "//third_party/py/numpy",
     ],
 )
 
@@ -129,6 +131,7 @@ py_library(
         "//tensorflow/python/data/util:convert",
         "//tensorflow/python/data/util:nest",
         "//tensorflow/python/data/util:sparse",
+        "//third_party/py/numpy",
     ],
 )
 
diff --git a/tensorflow/contrib/data/python/ops/get_single_element.py b/tensorflow/contrib/data/python/ops/get_single_element.py
index 0f4cd8e20c..ef9284456e 100644
--- a/tensorflow/contrib/data/python/ops/get_single_element.py
+++ b/tensorflow/contrib/data/python/ops/get_single_element.py
@@ -17,6 +17,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
+from tensorflow.contrib.data.python.ops import grouping
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import sparse
@@ -68,3 +71,30 @@ def get_single_element(dataset):
   return sparse.deserialize_sparse_tensors(
       nested_ret, dataset.output_types, dataset.output_shapes,
       dataset.output_classes)
+
+
+def reduce_dataset(dataset, reducer):
+  """Returns the result of reducing the `dataset` using `reducer`.
+
+  Args:
+    dataset: A @{tf.data.Dataset} object.
+    reducer: A @{tf.contrib.data.Reducer} object representing the reduce logic.
+
+  Returns:
+    A nested structure of @{tf.Tensor} objects, corresponding to the result
+    of reducing `dataset` using `reducer`.
+
+  Raises:
+    TypeError: if `dataset` is not a `tf.data.Dataset` object.
+  """
+  if not isinstance(dataset, dataset_ops.Dataset):
+    raise TypeError("`dataset` must be a `tf.data.Dataset` object.")
+
+  # The sentinel dataset is used in case the reduced dataset is empty.
+  sentinel_dataset = dataset_ops.Dataset.from_tensors(
+      reducer.finalize_func(reducer.init_func(np.int64(0))))
+  reduced_dataset = dataset.apply(
+      grouping.group_by_reducer(lambda x: np.int64(0), reducer))
+
+  return get_single_element(
+      reduced_dataset.concatenate(sentinel_dataset).take(1))
-- 
cgit v1.2.3


From 8ec87f55008982eb939d963c1d4a4ff7ef9ab3d3 Mon Sep 17 00:00:00 2001
From: Smit Hinsu 
Date: Thu, 19 Jul 2018 13:06:41 -0700
Subject: Mark tensorflow_lingvo directory as internal to tensorflow

PiperOrigin-RevId: 205288716
---
 tensorflow/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 518c2b0489..0b08f2093d 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -415,6 +415,7 @@ package_group(
         "//learning/meta_rank/...",
         "//tensorflow/...",
         "//tensorflow_fold/llgtm/...",
+        "//tensorflow_lingvo/...",
         "//third_party/py/tensor2tensor/...",
     ],
 )
-- 
cgit v1.2.3


From afae286739dbfd6339cde505ae573f2776b80afc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 13:21:37 -0700
Subject: [XLA] add XLA math library primitives to local Python client

PiperOrigin-RevId: 205291033
---
 .../xla/python/local_computation_builder.cc        | 24 ++++++++++++++++++----
 .../xla/python/local_computation_builder.h         | 24 ++++++++++++++++++----
 .../xla/python/local_computation_builder.i         | 24 ++++++++++++++++++----
 tensorflow/compiler/xla/python/xla_client.py       | 18 +++++++++++++++-
 4 files changed, 77 insertions(+), 13 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index 66b1c08a39..f25348e735 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -617,6 +617,8 @@ _FORWARD_BINOP(Xor)
 _FORWARD_BINOP(ShiftLeft)
 _FORWARD_BINOP(ShiftRightArithmetic)
 _FORWARD_BINOP(ShiftRightLogical)
+_FORWARD_BINOP(Atan2)
+_FORWARD_BINOP(Pow)
 _FORWARD_UNOP(Not)
 _FORWARD_UNOP(Abs)
 _FORWARD_UNOP(Exp)
@@ -630,13 +632,27 @@ _FORWARD_UNOP(Sign)
 _FORWARD_UNOP(Cos)
 _FORWARD_UNOP(Sin)
 _FORWARD_UNOP(Tanh)
-_FORWARD_UNOP(Sqrt)
-_FORWARD_UNOP(Square)
-_FORWARD_BINOP(Pow)
 _FORWARD_UNOP(IsFinite)
-_FORWARD_UNOP(Reciprocal)
 _FORWARD_UNOP(Neg)
 _FORWARD_UNOP(Sort)
+_FORWARD_UNOP(Sqrt)
+_FORWARD_UNOP(Rsqrt)
+_FORWARD_UNOP(Square)
+_FORWARD_UNOP(Reciprocal)
+_FORWARD_UNOP(Erfc)
+_FORWARD_UNOP(Erf)
+_FORWARD_UNOP(ErfInv)
+_FORWARD_UNOP(Lgamma)
+_FORWARD_UNOP(Digamma)
+_FORWARD_UNOP(Acos)
+_FORWARD_UNOP(Asin)
+_FORWARD_UNOP(Atan)
+_FORWARD_UNOP(Tan)
+_FORWARD_UNOP(Acosh)
+_FORWARD_UNOP(Asinh)
+_FORWARD_UNOP(Atanh)
+_FORWARD_UNOP(Cosh)
+_FORWARD_UNOP(Sinh)
 
 #undef _FORWARD
 #undef _FORWARD_UNOP
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 17ad044578..0e0d8ac29a 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -336,6 +336,8 @@ class LocalComputationBuilder {
   _FORWARD_BINOP(ShiftLeft)
   _FORWARD_BINOP(ShiftRightArithmetic)
   _FORWARD_BINOP(ShiftRightLogical)
+  _FORWARD_BINOP(Atan2)
+  _FORWARD_BINOP(Pow)
   _FORWARD_UNOP(Not)
   _FORWARD_UNOP(Abs)
   _FORWARD_UNOP(Exp)
@@ -349,13 +351,27 @@ class LocalComputationBuilder {
   _FORWARD_UNOP(Cos)
   _FORWARD_UNOP(Sin)
   _FORWARD_UNOP(Tanh)
-  _FORWARD_UNOP(Sqrt)
-  _FORWARD_UNOP(Square)
-  _FORWARD_BINOP(Pow)
   _FORWARD_UNOP(IsFinite)
-  _FORWARD_UNOP(Reciprocal)
   _FORWARD_UNOP(Neg)
   _FORWARD_UNOP(Sort)
+  _FORWARD_UNOP(Sqrt)
+  _FORWARD_UNOP(Rsqrt)
+  _FORWARD_UNOP(Square)
+  _FORWARD_UNOP(Reciprocal)
+  _FORWARD_UNOP(Erfc)
+  _FORWARD_UNOP(Erf)
+  _FORWARD_UNOP(ErfInv)
+  _FORWARD_UNOP(Lgamma)
+  _FORWARD_UNOP(Digamma)
+  _FORWARD_UNOP(Acos)
+  _FORWARD_UNOP(Asin)
+  _FORWARD_UNOP(Atan)
+  _FORWARD_UNOP(Tan)
+  _FORWARD_UNOP(Acosh)
+  _FORWARD_UNOP(Asinh)
+  _FORWARD_UNOP(Atanh)
+  _FORWARD_UNOP(Cosh)
+  _FORWARD_UNOP(Sinh)
 
 #undef _FORWARD
 #undef _FORWARD_UNOP
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 42bf76e5d8..eeccbd7cfa 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -1005,13 +1005,29 @@ tensorflow::ImportNumpy();
 %unignore xla::swig::LocalComputationBuilder::Cos;
 %unignore xla::swig::LocalComputationBuilder::Sin;
 %unignore xla::swig::LocalComputationBuilder::Tanh;
-%unignore xla::swig::LocalComputationBuilder::Sqrt;
-%unignore xla::swig::LocalComputationBuilder::Square;
-%unignore xla::swig::LocalComputationBuilder::Pow;
+%unignore xla::swig::LocalComputationBuilder::Atan2;
 %unignore xla::swig::LocalComputationBuilder::IsFinite;
-%unignore xla::swig::LocalComputationBuilder::Reciprocal;
+%unignore xla::swig::LocalComputationBuilder::Pow;
 %unignore xla::swig::LocalComputationBuilder::Neg;
 %unignore xla::swig::LocalComputationBuilder::Sort;
+%unignore xla::swig::LocalComputationBuilder::Sqrt;
+%unignore xla::swig::LocalComputationBuilder::Rsqrt;
+%unignore xla::swig::LocalComputationBuilder::Square;
+%unignore xla::swig::LocalComputationBuilder::Reciprocal;
+%unignore xla::swig::LocalComputationBuilder::Erfc;
+%unignore xla::swig::LocalComputationBuilder::Erf;
+%unignore xla::swig::LocalComputationBuilder::ErfInv;
+%unignore xla::swig::LocalComputationBuilder::Lgamma;
+%unignore xla::swig::LocalComputationBuilder::Digamma;
+%unignore xla::swig::LocalComputationBuilder::Acos;
+%unignore xla::swig::LocalComputationBuilder::Asin;
+%unignore xla::swig::LocalComputationBuilder::Atan;
+%unignore xla::swig::LocalComputationBuilder::Tan;
+%unignore xla::swig::LocalComputationBuilder::Acosh;
+%unignore xla::swig::LocalComputationBuilder::Asinh;
+%unignore xla::swig::LocalComputationBuilder::Atanh;
+%unignore xla::swig::LocalComputationBuilder::Cosh;
+%unignore xla::swig::LocalComputationBuilder::Sinh;
 %unignore xla::swig::DestructureLocalShapedBufferTuple;
 %unignore xla::swig::DeleteLocalShapedBuffer;
 %unignore xla::swig::DeleteLocalComputation;
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index f93d7bda2d..ef043e4ca0 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -99,12 +99,27 @@ _UNARY_OPS = [
     'Cos',
     'Sin',
     'Tanh',
+    'IsFinite',
     'Sqrt',
+    'Rsqrt',
     'Square',
-    'IsFinite',
     'Reciprocal',
     'Neg',
     'Sort',
+    'Erf',
+    'Erfc',
+    'ErfInv',
+    'Lgamma',
+    'Digamma',
+    'Acos',
+    'Asin',
+    'Atan',
+    'Tan',
+    'Acosh',
+    'Asinh',
+    'Atanh',
+    'Cosh',
+    'Sinh',
 ]
 
 _BINARY_OPS = [
@@ -128,6 +143,7 @@ _BINARY_OPS = [
     'ShiftLeft',
     'ShiftRightArithmetic',
     'ShiftRightLogical',
+    'Atan2',
 ]
 
 
-- 
cgit v1.2.3


From 6de6c2c8c00d947f08c40f37f563b35292dddf48 Mon Sep 17 00:00:00 2001
From: Justin Lebar 
Date: Thu, 19 Jul 2018 13:25:25 -0700
Subject: Use std::unique_ptr in BFCAllocator::AllocationRegion.

PiperOrigin-RevId: 205291721
---
 tensorflow/core/common_runtime/bfc_allocator.h | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/common_runtime/bfc_allocator.h b/tensorflow/core/common_runtime/bfc_allocator.h
index 52aedb1e9c..cd8ff6e5c0 100644
--- a/tensorflow/core/common_runtime/bfc_allocator.h
+++ b/tensorflow/core/common_runtime/bfc_allocator.h
@@ -191,18 +191,14 @@ class BFCAllocator : public VisitableAllocator {
       DCHECK_EQ(0, memory_size % kMinAllocationSize);
       const size_t n_handles =
           (memory_size + kMinAllocationSize - 1) / kMinAllocationSize;
-      handles_ = new ChunkHandle[n_handles];
+      handles_.reset(new ChunkHandle[n_handles]);
       for (size_t i = 0; i < n_handles; i++) {
         handles_[i] = kInvalidChunkHandle;
       }
     }
 
-    AllocationRegion() {}
-
-    ~AllocationRegion() { delete[] handles_; }
-
+    AllocationRegion() = default;
     AllocationRegion(AllocationRegion&& other) { Swap(other); }
-
     AllocationRegion& operator=(AllocationRegion&& other) {
       Swap(other);
       return *this;
@@ -241,7 +237,7 @@ class BFCAllocator : public VisitableAllocator {
     // Array of size "memory_size / kMinAllocationSize".  It is
     // indexed by (p-base) / kMinAllocationSize, contains ChunkHandle
     // for the memory allocation represented by "p"
-    ChunkHandle* handles_ = nullptr;
+    std::unique_ptr handles_;
 
     TF_DISALLOW_COPY_AND_ASSIGN(AllocationRegion);
   };
-- 
cgit v1.2.3


From c7aabf29c098176fdd4cbb4d32327e989505b054 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 13:33:09 -0700
Subject: Stop itemizing TF Lite tests to run in continuous builds.

PiperOrigin-RevId: 205292867
---
 .../tools/ci_build/ci_parameterized_build.sh       | 31 +-------------------
 .../tools/ci_build/linux/cpu/run_py3_contrib.sh    | 33 +---------------------
 2 files changed, 2 insertions(+), 62 deletions(-)

diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh
index 08e2c3edd2..5115be8c6d 100755
--- a/tensorflow/tools/ci_build/ci_parameterized_build.sh
+++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh
@@ -150,36 +150,7 @@ BAZEL_TARGET="//tensorflow/... -//tensorflow/compiler/..."
 if [[ -n "$TF_SKIP_CONTRIB_TESTS" ]]; then
   BAZEL_TARGET="$BAZEL_TARGET -//tensorflow/contrib/..."
 else
-  BAZEL_TARGET="${BAZEL_TARGET} -//tensorflow/contrib/lite/..."
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:context_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:framework"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:interpreter_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:model_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/toco:toco"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:simple_memory_arena_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite:string_util_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:activations_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:add_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:basic_rnn_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:concatenation_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:conv_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:depthwise_conv_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:embedding_lookup_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:embedding_lookup_sparse_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:fully_connected_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:hashtable_lookup_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:local_response_norm_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:lsh_projection_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:lstm_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:l2norm_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:mul_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:pooling_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:reshape_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:resize_bilinear_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:skip_gram_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:softmax_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:space_to_depth_test"
-  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/kernels:svdf_test"
+  BAZEL_TARGET="${BAZEL_TARGET} //tensorflow/contrib/lite/..."
 fi
 
 TUT_TEST_DATA_DIR="/tmp/tf_tutorial_test_data"
diff --git a/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh b/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh
index 2b68de3c5b..f6fa9251d4 100755
--- a/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh
+++ b/tensorflow/tools/ci_build/linux/cpu/run_py3_contrib.sh
@@ -34,35 +34,4 @@ yes "" | $PYTHON_BIN_PATH configure.py
 bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test -k \
     --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --config=opt \
     --test_size_filters=small,medium --test_output=errors -- \
-    //tensorflow/contrib/... \
-    -//tensorflow/contrib/lite/... \
-    //tensorflow/contrib/lite:context_test \
-    //tensorflow/contrib/lite:framework \
-    //tensorflow/contrib/lite:interpreter_test \
-    //tensorflow/contrib/lite:model_test \
-    //tensorflow/contrib/lite/toco:toco \
-    //tensorflow/contrib/lite:simple_memory_arena_test \
-    //tensorflow/contrib/lite:string_util_test \
-    //tensorflow/contrib/lite/kernels:activations_test \
-    //tensorflow/contrib/lite/kernels:add_test \
-    //tensorflow/contrib/lite/kernels:basic_rnn_test \
-    //tensorflow/contrib/lite/kernels:concatenation_test \
-    //tensorflow/contrib/lite/kernels:conv_test \
-    //tensorflow/contrib/lite/kernels:depthwise_conv_test \
-    //tensorflow/contrib/lite/kernels:embedding_lookup_test \
-    //tensorflow/contrib/lite/kernels:embedding_lookup_sparse_test \
-    //tensorflow/contrib/lite/kernels:fully_connected_test \
-    //tensorflow/contrib/lite/testing:generated_zip_tests \
-    //tensorflow/contrib/lite/kernels:hashtable_lookup_test \
-    //tensorflow/contrib/lite/kernels:local_response_norm_test \
-    //tensorflow/contrib/lite/kernels:lsh_projection_test \
-    //tensorflow/contrib/lite/kernels:lstm_test \
-    //tensorflow/contrib/lite/kernels:l2norm_test \
-    //tensorflow/contrib/lite/kernels:mul_test \
-    //tensorflow/contrib/lite/kernels:pooling_test \
-    //tensorflow/contrib/lite/kernels:reshape_test \
-    //tensorflow/contrib/lite/kernels:resize_bilinear_test \
-    //tensorflow/contrib/lite/kernels:skip_gram_test \
-    //tensorflow/contrib/lite/kernels:softmax_test \
-    //tensorflow/contrib/lite/kernels:space_to_depth_test \
-    //tensorflow/contrib/lite/kernels:svdf_test
+    //tensorflow/contrib/...
-- 
cgit v1.2.3


From 661ad6be85fa611fa297bc8b8bacef752bef7ffc Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Thu, 19 Jul 2018 13:34:14 -0700
Subject: [TF:XLA] Bump open source llvm revision to r337441

PiperOrigin-RevId: 205293013
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 4b4f31813c..2c8658fc59 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -487,11 +487,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "llvm",
       urls = [
-          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/10c3b3d15ed6a788ac12221b784caf81fb8248b5.tar.gz",
-	  "https://github.com/llvm-mirror/llvm/archive/10c3b3d15ed6a788ac12221b784caf81fb8248b5.tar.gz",
+          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/62b518b75a780a3bc75982cbe54b0e7bc262aa6e.tar.gz",
+	  "https://github.com/llvm-mirror/llvm/archive/62b518b75a780a3bc75982cbe54b0e7bc262aa6e.tar.gz",
       ],
-      sha256 = "a9feb6b47267c30fd7c19ebfdf4dbde6757054f716fa77c09bcb1106799c3253",
-      strip_prefix = "llvm-10c3b3d15ed6a788ac12221b784caf81fb8248b5",
+      sha256 = "51ab0edcf7dde0207f5cf141aec16b14fcac5290112cdf1ea671a2757f719f8b",
+      strip_prefix = "llvm-62b518b75a780a3bc75982cbe54b0e7bc262aa6e",
       build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
   )
 
-- 
cgit v1.2.3


From 58fdd0dfce6d4c71fa7d381190987ccad33da0b6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 13:35:40 -0700
Subject: Test io::RecordWriter.flush()

Requires changing flush_mode from default Z_NO_FLUSH
See tensorflow/core/lib/io/zlib_compression_options.h

PiperOrigin-RevId: 205293231
---
 tensorflow/core/BUILD                              |  1 +
 .../core/lib/io/record_reader_writer_test.cc       | 84 ++++++++++++++++++++++
 2 files changed, 85 insertions(+)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 514713bb96..fc12027291 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3226,6 +3226,7 @@ tf_cc_tests(
         ":test",
         ":test_main",
         "//third_party/eigen3",
+        "@zlib_archive//:zlib",
     ],
 )
 
diff --git a/tensorflow/core/lib/io/record_reader_writer_test.cc b/tensorflow/core/lib/io/record_reader_writer_test.cc
index 95ac040602..c36c909399 100644
--- a/tensorflow/core/lib/io/record_reader_writer_test.cc
+++ b/tensorflow/core/lib/io/record_reader_writer_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/lib/io/record_reader.h"
 #include "tensorflow/core/lib/io/record_writer.h"
 
+#include 
 #include 
 #include "tensorflow/core/platform/env.h"
 
@@ -33,6 +34,89 @@ static std::vector BufferSizes() {
           12, 13, 14, 15, 16, 17, 18, 19, 20, 65536};
 }
 
+namespace {
+
+io::RecordReaderOptions GetMatchingReaderOptions(
+    const io::RecordWriterOptions& options) {
+  if (options.compression_type == io::RecordWriterOptions::ZLIB_COMPRESSION) {
+    return io::RecordReaderOptions::CreateRecordReaderOptions("ZLIB");
+  }
+  return io::RecordReaderOptions::CreateRecordReaderOptions("");
+}
+
+uint64 GetFileSize(const string& fname) {
+  Env* env = Env::Default();
+  uint64 fsize;
+  TF_CHECK_OK(env->GetFileSize(fname, &fsize));
+  return fsize;
+}
+
+void VerifyFlush(const io::RecordWriterOptions& options) {
+  std::vector records = {
+      "abcdefghijklmnopqrstuvwxyz",
+      "ZYXWVUTSRQPONMLKJIHGFEDCBA0123456789!@#$%^&*()",
+      "G5SyohOL9UmXofSOOwWDrv9hoLLMYPJbG9r38t3uBRcHxHj2PdKcPDuZmKW62RIY",
+      "aaaaaaaaaaaaaaaaaaaaaaaaaa",
+  };
+
+  Env* env = Env::Default();
+  string fname = testing::TmpDir() + "/record_reader_writer_flush_test";
+
+  std::unique_ptr file;
+  TF_CHECK_OK(env->NewWritableFile(fname, &file));
+  io::RecordWriter writer(file.get(), options);
+
+  std::unique_ptr read_file;
+  TF_CHECK_OK(env->NewRandomAccessFile(fname, &read_file));
+  io::RecordReaderOptions read_options = GetMatchingReaderOptions(options);
+  io::RecordReader reader(read_file.get(), read_options);
+
+  EXPECT_EQ(GetFileSize(fname), 0);
+  for (size_t i = 0; i < records.size(); i++) {
+    uint64 start_size = GetFileSize(fname);
+
+    // Write a new record.
+    TF_EXPECT_OK(writer.WriteRecord(records[i]));
+    TF_CHECK_OK(writer.Flush());
+    TF_CHECK_OK(file->Flush());
+
+    // Verify that file size has changed after file flush.
+    uint64 new_size = GetFileSize(fname);
+    EXPECT_GT(new_size, start_size);
+
+    // Verify that file has all records written so far and no more.
+    uint64 offset = 0;
+    string record;
+    for (size_t j = 0; j <= i; j++) {
+      // Check that j'th record is written correctly.
+      TF_CHECK_OK(reader.ReadRecord(&offset, &record));
+      EXPECT_EQ(record, records[j]);
+    }
+
+    // Verify that file has no more records.
+    CHECK_EQ(reader.ReadRecord(&offset, &record).code(), error::OUT_OF_RANGE);
+  }
+}
+
+}  // namespace
+
+TEST(RecordReaderWriterTest, TestFlush) {
+  io::RecordWriterOptions options;
+  VerifyFlush(options);
+}
+
+TEST(RecordReaderWriterTest, TestZlibSyncFlush) {
+  io::RecordWriterOptions options;
+  options.compression_type = io::RecordWriterOptions::ZLIB_COMPRESSION;
+  // The default flush_mode is Z_NO_FLUSH and only writes to the file when the
+  // buffer is full or the file is closed, which makes testing harder.
+  // By using Z_SYNC_FLUSH the test can verify Flush does write out records of
+  // approximately the right size at the right times.
+  options.zlib_options.flush_mode = Z_SYNC_FLUSH;
+
+  VerifyFlush(options);
+}
+
 TEST(RecordReaderWriterTest, TestBasics) {
   Env* env = Env::Default();
   string fname = testing::TmpDir() + "/record_reader_writer_test";
-- 
cgit v1.2.3


From 1b21235444eb12429ee41d185b6f594778f7c30a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 13:48:50 -0700
Subject: Improve Windows build process

After this change, the windows build steps should be like:
https://docs.google.com/document/d/1oVYzPJVv8r5N9PecqwG74rY_QbqPH70IxBouBHdq5EI/edit?usp=sharing

PiperOrigin-RevId: 205295588
---
 configure.py                                       | 59 ++++++++++++++++------
 .../ci_build/windows/cpu/pip/build_tf_windows.sh   | 10 ++--
 .../ci_build/windows/gpu/pip/build_tf_windows.sh   | 10 ++--
 tensorflow/tools/pip_package/build_pip_package.sh  |  6 ++-
 4 files changed, 54 insertions(+), 31 deletions(-)

diff --git a/configure.py b/configure.py
index c482628ec8..60fe54b2f6 100644
--- a/configure.py
+++ b/configure.py
@@ -882,7 +882,7 @@ def set_tf_cudnn_version(environ_cp):
     default_cudnn_path = environ_cp.get('CUDA_TOOLKIT_PATH')
     ask_cudnn_path = (r'Please specify the location where cuDNN %s library is '
                       'installed. Refer to README.md for more details. [Default'
-                      ' is %s]:') % (tf_cudnn_version, default_cudnn_path)
+                      ' is %s]: ') % (tf_cudnn_version, default_cudnn_path)
     cudnn_install_path = get_from_env_or_user_or_default(
         environ_cp, 'CUDNN_INSTALL_PATH', ask_cudnn_path, default_cudnn_path)
 
@@ -1201,7 +1201,7 @@ def set_tf_cuda_compute_capabilities(environ_cp):
         'https://developer.nvidia.com/cuda-gpus.\nPlease'
         ' note that each additional compute '
         'capability significantly increases your '
-        'build time and binary size. [Default is: %s]' %
+        'build time and binary size. [Default is: %s]: ' %
         default_cuda_compute_capabilities)
     tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
         environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
@@ -1402,14 +1402,36 @@ def set_build_strip_flag():
   write_to_bazelrc('build --strip=always')
 
 
-def set_windows_build_flags():
-  if is_windows():
-    # The non-monolithic build is not supported yet
-    write_to_bazelrc('build --config monolithic')
-    # Suppress warning messages
-    write_to_bazelrc('build --copt=-w --host_copt=-w')
-    # Output more verbose information when something goes wrong
-    write_to_bazelrc('build --verbose_failures')
+def set_windows_build_flags(environ_cp):
+  """Set Windows specific build options."""
+  # The non-monolithic build is not supported yet
+  write_to_bazelrc('build --config monolithic')
+  # Suppress warning messages
+  write_to_bazelrc('build --copt=-w --host_copt=-w')
+  # Output more verbose information when something goes wrong
+  write_to_bazelrc('build --verbose_failures')
+  # The host and target platforms are the same in Windows build. So we don't
+  # have to distinct them. This avoids building the same targets twice.
+  write_to_bazelrc('build --distinct_host_configuration=false')
+  # Enable short object file path to avoid long path issue on Windows.
+  # TODO(pcloudy): Remove this flag when upgrading Bazel to 0.16.0
+  # Short object file path will be enabled by default.
+  write_to_bazelrc('build --experimental_shortened_obj_file_path=true')
+
+  if get_var(
+      environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline',
+      True,
+      ('Would you like to override eigen strong inline for some C++ '
+       'compilation to reduce the compiling time?'),
+      'Eigen strong inline overridden.',
+      'Not overriding eigen strong inline, '
+      'some compilations could take more than 20 mins.'):
+    # Due to a known MSVC compiler issue
+    # https://github.com/tensorflow/tensorflow/issues/10521
+    # Overriding eigen strong inline speeds up the compiling of
+    # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes,
+    # but this also hurts the performance. Let users decide what they want.
+    write_to_bazelrc('build --define=override_eigen_strong_inline=true')
 
 
 def config_info_line(name, help_text):
@@ -1537,7 +1559,8 @@ def main():
   set_grpc_build_flags()
   set_cc_opt_flags(environ_cp)
   set_build_strip_flag()
-  set_windows_build_flags()
+  if is_windows():
+    set_windows_build_flags(environ_cp)
 
   if get_var(
       environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace',
@@ -1549,11 +1572,15 @@ def main():
     create_android_ndk_rule(environ_cp)
     create_android_sdk_rule(environ_cp)
 
-  print('Preconfigured Bazel build configs. You can use any of the below by '
-        'adding "--config=<>" to your build command. See tools/bazel.rc for '
-        'more details.')
-  config_info_line('mkl', 'Build with MKL support.')
-  config_info_line('monolithic', 'Config for mostly static monolithic build.')
+  # On Windows, we don't have MKL support and the build is always monolithic.
+  # So no need to print the following message.
+  # TODO(pcloudy): remove the following if check when they make sense on Windows
+  if not is_windows():
+    print('Preconfigured Bazel build configs. You can use any of the below by '
+          'adding "--config=<>" to your build command. See tools/bazel.rc for '
+          'more details.')
+    config_info_line('mkl', 'Build with MKL support.')
+    config_info_line('monolithic', 'Config for mostly static monolithic build.')
 
 if __name__ == '__main__':
   main()
diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 61dec249f3..dc7ea1dc57 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -67,16 +67,12 @@ for ARG in "$@"; do
 done
 
 if [[ "$release_build" != 1 ]]; then
-  # --define=override_eigen_strong_inline=true speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc
+  # Overriding eigen strong inline speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc
   # by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521
-  # Because this hurts the performance of TF, we don't enable it in release build.
-  echo "build --define=override_eigen_strong_inline=true" >> "${TMP_BAZELRC}"
+  # Because this hurts the performance of TF, we don't override it in release build.
+  export TF_OVERRIDE_EIGEN_STRONG_INLINE=0
 fi
 
-# The host and target platforms are the same in Windows build. So we don't have
-# to distinct them. This helps avoid building the same targets twice.
-echo "build --distinct_host_configuration=false" >> "${TMP_BAZELRC}"
-
 # Enable short object file path to avoid long path issue on Windows.
 echo "startup --output_user_root=${TMPDIR}" >> "${TMP_BAZELRC}"
 
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index e232306653..a4175a0e81 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -67,16 +67,12 @@ for ARG in "$@"; do
 done
 
 if [[ "$release_build" != 1 ]]; then
-  # --define=override_eigen_strong_inline=true speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc
+  # Overriding eigen strong inline speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc
   # by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521
-  # Because this hurts the performance of TF, we don't enable it in release build.
-  echo "build --define=override_eigen_strong_inline=true" >> "${TMP_BAZELRC}"
+  # Because this hurts the performance of TF, we don't override it in release build.
+  export TF_OVERRIDE_EIGEN_STRONG_INLINE=0
 fi
 
-# The host and target platforms are the same in Windows build. So we don't have
-# to distinct them. This helps avoid building the same targets twice.
-echo "build --distinct_host_configuration=false" >> "${TMP_BAZELRC}"
-
 # Enable short object file path to avoid long path issue on Windows.
 echo "startup --output_user_root=${TMPDIR}" >> "${TMP_BAZELRC}"
 
diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh
index 4101b34a11..ca40f2eaa8 100755
--- a/tensorflow/tools/pip_package/build_pip_package.sh
+++ b/tensorflow/tools/pip_package/build_pip_package.sh
@@ -17,8 +17,12 @@
 
 set -e
 
+function is_absolute {
+  [[ "$1" = /* ]] || [[ "$1" =~ ^[a-zA-Z]:[/\\].* ]]
+}
+
 function real_path() {
-  [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"
+  is_absolute "$1" && echo "$1" || echo "$PWD/${1#./}"
 }
 
 function cp_external() {
-- 
cgit v1.2.3


From 498fed9be6cc556b08c1d3ffd31565497daaa8c1 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy 
Date: Thu, 19 Jul 2018 13:59:20 -0700
Subject: Automated rollback of commit 8ec87f55008982eb939d963c1d4a4ff7ef9ab3d3

PiperOrigin-RevId: 205297172
---
 tensorflow/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 0b08f2093d..518c2b0489 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -415,7 +415,6 @@ package_group(
         "//learning/meta_rank/...",
         "//tensorflow/...",
         "//tensorflow_fold/llgtm/...",
-        "//tensorflow_lingvo/...",
         "//third_party/py/tensor2tensor/...",
     ],
 )
-- 
cgit v1.2.3


From 6e02d79ba0179a23679e65b31405c591726bc552 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 14:34:20 -0700
Subject: Make count metric consistent with other metrics by converting
 variable to tensor (_aggregate_variable() returns Tensor).

PiperOrigin-RevId: 205303531
---
 tensorflow/contrib/metrics/python/ops/metric_ops.py    | 18 ++++++++++++------
 .../contrib/metrics/python/ops/metric_ops_test.py      |  5 +++++
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index b14202ff9e..a328670526 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -3715,6 +3715,7 @@ def count(values,
           name=None):
   """Computes the number of examples, or sum of `weights`.
 
+  This metric keeps track of the denominator in `tf.metrics.mean`.
   When evaluating some metric (e.g. mean) on one or more subsets of the data,
   this auxiliary metric is useful for keeping track of how many examples there
   are in each subset.
@@ -3741,15 +3742,21 @@ def count(values,
     ValueError: If `weights` is not `None` and its shape doesn't match `values`,
       or if either `metrics_collections` or `updates_collections` are not a list
       or tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.executing_eagerly():
+    raise RuntimeError('tf.contrib.metrics.count is not supported when eager '
+                       'execution is enabled.')
 
   with variable_scope.variable_scope(name, 'count', (values, weights)):
+
     count_ = metrics_impl.metric_variable([], dtypes.float32, name='count')
 
     if weights is None:
       num_values = math_ops.to_float(array_ops.size(values))
     else:
-      _, _, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
+      values = math_ops.to_float(values)
+      values, _, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
           predictions=values,
           labels=None,
           weights=weights)
@@ -3758,15 +3765,14 @@ def count(values,
       num_values = math_ops.reduce_sum(weights)
 
     with ops.control_dependencies([values]):
-      update_op = state_ops.assign_add(count_, num_values)
+      update_count_op = state_ops.assign_add(count_, num_values)
 
-    if metrics_collections:
-      ops.add_to_collections(metrics_collections, count_)
+    count_ = metrics_impl._aggregate_variable(count_, metrics_collections)  # pylint: disable=protected-access
 
     if updates_collections:
-      ops.add_to_collections(updates_collections, update_op)
+      ops.add_to_collections(updates_collections, update_count_op)
 
-    return count_, update_op
+    return count_, update_count_op
 
 
 def cohen_kappa(labels,
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index a09fc4abd4..401fedcbed 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -6854,6 +6854,11 @@ class CountTest(test.TestCase):
         array_ops.ones([4, 3]), updates_collections=[my_collection_name])
     self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
 
+  def testReturnType(self):
+    c, op = metrics.count(array_ops.ones([4, 3]))
+    self.assertTrue(isinstance(c, ops.Tensor))
+    self.assertTrue(isinstance(op, ops.Operation) or isinstance(op, ops.Tensor))
+
   def testBasic(self):
     with self.test_session() as sess:
       values_queue = data_flow_ops.FIFOQueue(
-- 
cgit v1.2.3


From 056c971faf024d38160f0e593f37b16be66da666 Mon Sep 17 00:00:00 2001
From: Youlong Cheng 
Date: Thu, 19 Jul 2018 14:55:15 -0700
Subject: PUBLIC: Relax batch_size checking for broadcast mode and fix a bug in
 broadcast mode. RELNOTES: n/a

PiperOrigin-RevId: 205307114
---
 tensorflow/contrib/tpu/python/tpu/tpu_context.py   | 9 ++++++---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 3 ++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
index 211c59cb90..750e677263 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
@@ -595,7 +595,8 @@ class _InternalTPUContext(object):
         raise ValueError(message)
 
     if mode == model_fn_lib.ModeKeys.TRAIN:
-      if self._train_batch_size % num_replicas != 0:
+      if (self._train_batch_size % num_replicas != 0 and
+          not self.is_input_broadcast_with_iterators()):
         raise ValueError(
             'train batch size {} must be divisible by number of replicas {}'
             .format(self._train_batch_size, num_replicas))
@@ -605,7 +606,8 @@ class _InternalTPUContext(object):
         raise ValueError(
             'eval_batch_size in TPUEstimator constructor cannot be `None`'
             'if .evaluate is running on TPU.')
-      if self._eval_batch_size % num_replicas != 0:
+      if (self._eval_batch_size % num_replicas != 0 and
+          not self.is_input_broadcast_with_iterators()):
         raise ValueError(
             'eval batch size {} must be divisible by number of replicas {}'
             .format(self._eval_batch_size, num_replicas))
@@ -619,7 +621,8 @@ class _InternalTPUContext(object):
         raise ValueError(
             'predict_batch_size in TPUEstimator constructor should not be '
             '`None` if .predict is running on TPU.')
-      if self._predict_batch_size % num_replicas != 0:
+      if (self._predict_batch_size % num_replicas != 0 and
+          not self.is_input_broadcast_with_iterators()):
         raise ValueError(
             'predict batch size {} must be divisible by number of replicas {}'
             .format(self._predict_batch_size, num_replicas))
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index c7cd7896e0..73dfefd19c 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -858,7 +858,8 @@ def generate_broadcast_enqueue_ops_fn(ctx, input_fn, inputs_structure_recorder,
     if ctx.mode == model_fn_lib.ModeKeys.PREDICT:
       raise TypeError('Mode PREDICT not yet supported in BROADCAST mode.')
 
-    hooks.append(inputs.dataset_initializer_hook())
+    if is_dataset:
+      hooks.append(inputs.dataset_initializer_hook())
     num_replicas_per_host = ctx.num_of_replicas_per_host
 
   def tpu_ordinal_function_impl(replica_id):
-- 
cgit v1.2.3


From 5f34e4ded7ac519c4102029eb2f22fb9b4b27aba Mon Sep 17 00:00:00 2001
From: Justin Lebar 
Date: Thu, 19 Jul 2018 15:04:57 -0700
Subject: Cleanups to BFCAllocator::FreeAndMaybeCoalesce.

No functional change.

PiperOrigin-RevId: 205308949
---
 tensorflow/core/common_runtime/bfc_allocator.cc | 48 +++++++++----------------
 1 file changed, 16 insertions(+), 32 deletions(-)

diff --git a/tensorflow/core/common_runtime/bfc_allocator.cc b/tensorflow/core/common_runtime/bfc_allocator.cc
index 9cda17867b..f8ca039d15 100644
--- a/tensorflow/core/common_runtime/bfc_allocator.cc
+++ b/tensorflow/core/common_runtime/bfc_allocator.cc
@@ -465,49 +465,33 @@ void BFCAllocator::FreeAndMaybeCoalesce(BFCAllocator::ChunkHandle h) {
   Chunk* c = ChunkFromHandle(h);
   CHECK(c->in_use() && (c->bin_num == kInvalidBinNum));
 
-  // Mark the chunk as no longer in use
+  // Mark the chunk as no longer in use.
   c->allocation_id = -1;
 
   // Updates the stats.
   stats_.bytes_in_use -= c->size;
 
-  // This chunk is no longer in-use, consider coalescing the chunk
-  // with adjacent chunks.
-  ChunkHandle chunk_to_reassign = h;
+  ChunkHandle coalesced_chunk = h;
 
-  // If the next chunk is free, coalesce the two
-  if (c->next != kInvalidChunkHandle) {
-    Chunk* cnext = ChunkFromHandle(c->next);
-    if (!cnext->in_use()) {
-      //      VLOG(8) << "Chunk at " << cnext->ptr << " merging with c " <<
-      //      c->ptr;
-
-      chunk_to_reassign = h;
-
-      // Deletes c->next
-      RemoveFreeChunkFromBin(c->next);
-      Merge(h, ChunkFromHandle(h)->next);
-    }
+  // If the next chunk is free, merge it into c and delete it.
+  if (c->next != kInvalidChunkHandle && !ChunkFromHandle(c->next)->in_use()) {
+    // VLOG(8) << "Merging c->next " << ChunkFromHandle(c->next)->ptr
+    //         << " with c " << c->ptr;
+    RemoveFreeChunkFromBin(c->next);
+    Merge(h, c->next);
   }
 
-  // If the previous chunk is free, coalesce the two
-  c = ChunkFromHandle(h);
-  if (c->prev != kInvalidChunkHandle) {
-    Chunk* cprev = ChunkFromHandle(c->prev);
-    if (!cprev->in_use()) {
-      //      VLOG(8) << "Chunk at " << c->ptr << " merging into c->prev "
-      //       << cprev->ptr;
+  // If the previous chunk is free, merge c into it and delete c.
+  if (c->prev != kInvalidChunkHandle && !ChunkFromHandle(c->prev)->in_use()) {
+    // VLOG(8) << "Merging c " << c->ptr << " into c->prev "
+    //         << ChunkFromHandle(c->prev)->ptr;
 
-      chunk_to_reassign = c->prev;
-
-      // Deletes c
-      RemoveFreeChunkFromBin(c->prev);
-      Merge(ChunkFromHandle(h)->prev, h);
-      c = ChunkFromHandle(h);
-    }
+    coalesced_chunk = c->prev;
+    RemoveFreeChunkFromBin(c->prev);
+    Merge(c->prev, h);
   }
 
-  InsertFreeChunkIntoBin(chunk_to_reassign);
+  InsertFreeChunkIntoBin(coalesced_chunk);
 }
 
 void BFCAllocator::AddAllocVisitor(Visitor visitor) {
-- 
cgit v1.2.3


From 6860bf17aa7fc4a4ff323d8913cee6dccd54a52b Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Thu, 19 Jul 2018 15:22:22 -0700
Subject: Fix dependency issues.

---
 tensorflow/contrib/tensorrt/BUILD                            |  4 +++-
 .../contrib/tensorrt/test/tf_trt_integration_test_base.py    | 12 ++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index dea9c0a4ae..317041b87d 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -153,6 +153,7 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
+        ":tf_trt_integration_test_base",
         ":trt_convert_py",
         ":trt_ops_py",
         "//tensorflow/python:errors",
@@ -327,7 +328,8 @@ py_library(
     name = "tf_trt_integration_test_base",
     srcs = ["test/tf_trt_integration_test_base.py"],
     deps = [
-        ":init_py",
+        ":trt_convert_py",
+        ":trt_ops_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_test_lib",
     ],
diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 6e12e7e026..560dc256fa 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -24,14 +24,14 @@ import warnings
 import numpy as np
 import six
 
-from tensorflow.contrib import tensorrt as trt
+from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
-from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 
 TfTrtIntegrationTestParams = namedtuple("TfTrtIntegrationTestParams", [
@@ -105,7 +105,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
       graph_options = config_pb2.GraphOptions()
 
     gpu_options = config_pb2.GPUOptions()
-    if trt.trt_convert.get_linked_tensorrt_version()[0] == 3:
+    if trt_convert.get_linked_tensorrt_version()[0] == 3:
       gpu_options.per_process_gpu_memory_fraction = 0.50
 
     config = config_pb2.ConfigProto(
@@ -145,7 +145,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
 
   def _GetTrtGraphDef(self, params, gdef, precision_mode, is_dynamic_op):
     """Return trt converted graphdef."""
-    return trt.create_inference_graph(
+    return trt_convert.create_inference_graph(
         input_graph_def=gdef,
         outputs=[self.output_name],
         max_batch_size=max([dims[0] for dims in params.input_dims]),
@@ -213,7 +213,7 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase):
                              dynamic_calib_engine)
         result = self._RunCalibration(params, calib_gdef, input_data,
                                       calib_config)
-        infer_gdef = trt.calib_graph_to_infer_graph(calib_gdef)
+        infer_gdef = trt_convert.calib_graph_to_infer_graph(calib_gdef)
         self._VerifyGraphDef(params, infer_gdef, precision_mode, True,
                              dynamic_calib_engine)
 
@@ -320,5 +320,5 @@ def _AddTests(test_class):
                  dynamic_calib_engine))
 
 
-if trt.is_tensorrt_enabled():
+if trt_convert.is_tensorrt_enabled():
   _AddTests(TfTrtIntegrationTestBase)
-- 
cgit v1.2.3


From 97f89dcd6ec02d40f6aae1d5ce5ffa377a40c110 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer 
Date: Thu, 19 Jul 2018 15:26:32 -0700
Subject: [XLA:CPU] Don't create dead constants in WhileLoopSinking

We run this in a fixpoint pipeline together with DCE, dead constants mean that
the fixpoint will never be reached.

PiperOrigin-RevId: 205312251
---
 .../xla/service/while_loop_constant_sinking.cc     |  6 +++
 .../service/while_loop_constant_sinking_test.cc    | 45 ++++++++++++++++++++++
 2 files changed, 51 insertions(+)

diff --git a/tensorflow/compiler/xla/service/while_loop_constant_sinking.cc b/tensorflow/compiler/xla/service/while_loop_constant_sinking.cc
index 10fc4958fa..62af45128a 100644
--- a/tensorflow/compiler/xla/service/while_loop_constant_sinking.cc
+++ b/tensorflow/compiler/xla/service/while_loop_constant_sinking.cc
@@ -61,6 +61,12 @@ StatusOr WhileLoopConstantSinking::TrySinkingConstantsIntoWhileBody(
        WhileUtil::GetInvariantGTEsForWhileBody(*while_body)) {
     int64 index = invariant_gte->tuple_index();
     const HloInstruction& invariant_value = *init_value.operand(index);
+
+    // Should have at least one user that's not while_body_root.
+    if (invariant_gte->user_count() <= 1) {
+      continue;
+    }
+
     if (invariant_value.opcode() == HloOpcode::kConstant) {
       auto* constant_instr =
           while_body->AddInstruction(invariant_value.Clone(/*suffix=*/".sunk"));
diff --git a/tensorflow/compiler/xla/service/while_loop_constant_sinking_test.cc b/tensorflow/compiler/xla/service/while_loop_constant_sinking_test.cc
index 393e758038..266039d2ff 100644
--- a/tensorflow/compiler/xla/service/while_loop_constant_sinking_test.cc
+++ b/tensorflow/compiler/xla/service/while_loop_constant_sinking_test.cc
@@ -196,5 +196,50 @@ ENTRY entry {
                         op::GetTupleElement(op::Parameter(0)),
                         op::GetTupleElement(op::Parameter(0))));
 }
+
+TEST_F(WhileLoopConstantSinkingTest, DontCreateDeadConstant) {
+  const char* const hlo_string = R"(
+HloModule ModuleWithWhile
+
+body {
+  p_body = (f32[2],f32[2]) parameter(0)
+  p_body.0 = f32[2] get-tuple-element((f32[2],f32[2]) p_body), index=0
+  p_body.1 = f32[2] get-tuple-element((f32[2],f32[2]) p_body), index=1
+
+  outfeed = token[] outfeed(p_body.0)
+  ROOT root = (f32[2],f32[2],f32[2]) tuple(p_body.0, p_body.1, p_body.1)
+}
+
+condition {
+  p_cond = (f32[2],f32[2]) parameter(0)
+  ROOT result = pred[] constant(true)
+}
+
+ENTRY entry {
+  const_0 = f32[2] constant({1, 2})
+  const_1 = f32[2] constant({2, 1})
+  while_init = (f32[2],f32[2]) tuple(const_0, const_1)
+  ROOT while = (f32[2],f32[2],f32[2]) while(while_init), condition=condition,
+                                      body=body
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module,
+                          ParseHloString(hlo_string));
+
+  TF_ASSERT_OK_AND_ASSIGN(bool changed,
+                          WhileLoopConstantSinking{}.Run(module.get()));
+  ASSERT_TRUE(changed);
+
+  auto* while_body = module->GetComputationWithName("body");
+  EXPECT_THAT(while_body->root_instruction(),
+              op::Tuple(op::GetTupleElement(), op::GetTupleElement(),
+                        op::GetTupleElement()));
+  for (const HloInstruction* inst : while_body->instructions()) {
+    if (inst->opcode() == HloOpcode::kConstant) {
+      EXPECT_GT(inst->user_count(), 0);
+    }
+  }
+}
 }  // namespace
 }  // namespace xla
-- 
cgit v1.2.3


From c0d998e3a772b0021b6283145e2aa24b701a0e7f Mon Sep 17 00:00:00 2001
From: Toby Boyd 
Date: Tue, 17 Jul 2018 14:26:44 -0700
Subject: Link lib and header where ./configure expects

---
 tensorflow/tools/ci_build/Dockerfile.gpu | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu
index 7591ecc04e..46538d9ec7 100644
--- a/tensorflow/tools/ci_build/Dockerfile.gpu
+++ b/tensorflow/tools/ci_build/Dockerfile.gpu
@@ -22,6 +22,11 @@ RUN /install/install_golang.sh
 COPY install/.bazelrc /etc/bazel.bazelrc
 ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
 
+# Link NCCL libray and header where the build script expects them.
+RUN mkdir /usr/local/cuda-9.0/lib &&  \
+    ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \
+    ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h
+
 # Configure the build for our CUDA configuration.
 ENV TF_NEED_CUDA 1
 ENV TF_CUDA_COMPUTE_CAPABILITIES 3.0
-- 
cgit v1.2.3


From c519794c7cca51d2c75aa53b56a1448804f68647 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Thu, 19 Jul 2018 15:32:00 -0700
Subject: Disable the plugin test by default.

---
 tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
index 1ef1c3de75..69058c5826 100644
--- a/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
+++ b/tensorflow/contrib/tensorrt/custom_plugin_examples/BUILD
@@ -111,7 +111,10 @@ cuda_py_test(
         "//tensorflow/python:tf_optimizer",
     ],
     tags = [
+        "manual",
         "no_windows",
+        "noguitar",
         "nomac",
+        "notap",
     ],
 )
-- 
cgit v1.2.3


From a103552156432bcda7e29e5588e83c62d5154b88 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Thu, 19 Jul 2018 15:35:28 -0700
Subject: Disable unused import errors.

---
 tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
index 560dc256fa..301f7b44c4 100644
--- a/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
+++ b/tensorflow/contrib/tensorrt/test/tf_trt_integration_test_base.py
@@ -24,7 +24,9 @@ import warnings
 import numpy as np
 import six
 
+# pylint: disable=unused-import
 from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+# pylint: enable=unused-import
 from tensorflow.contrib.tensorrt.python import trt_convert
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
-- 
cgit v1.2.3


From b7b86af2425f166c586fef80e6ae46991cdedde9 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Thu, 19 Jul 2018 15:39:25 -0700
Subject: Teach XlaTensorBuffer to not deallocate null pointers

Zero sized XlaTensorBuffer can have null as the backing storage.  De-allocating
a null pointer produces an annoying warning from the BFCAllocator.

PiperOrigin-RevId: 205314271
---
 tensorflow/compiler/jit/xla_launch_util.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/jit/xla_launch_util.h b/tensorflow/compiler/jit/xla_launch_util.h
index 90531174ff..1ea3fa4cf2 100644
--- a/tensorflow/compiler/jit/xla_launch_util.h
+++ b/tensorflow/compiler/jit/xla_launch_util.h
@@ -122,7 +122,11 @@ class XlaTensorBuffer : public TensorBuffer {
     data_ = const_cast(ptr);
   }
 
-  ~XlaTensorBuffer() override { allocator_->DeallocateRaw(data_); }
+  ~XlaTensorBuffer() override {
+    if (data_) {
+      allocator_->DeallocateRaw(data_);
+    }
+  }
 
   void* data() const override { return data_; }
   size_t size() const override { return expected_size_; }
-- 
cgit v1.2.3


From 716d15118f62c17c29bbec4d006fd3055bb56812 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne 
Date: Thu, 19 Jul 2018 15:52:35 -0700
Subject: Update minimum bazel version to 0.13.0.

I was using bazel 0.12.0 and was getting the error:

file '@bazel_tools//tools/cpp:windows_cc_configure.bzl' does not contain symbol 'setup_vc_env_vars'

PiperOrigin-RevId: 205316270
---
 WORKSPACE    | 2 +-
 configure.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index fd7570a80a..e7cf23a159 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -18,7 +18,7 @@ closure_repositories()
 # files, in case the parsing of those build files depends on the bazel
 # version we require here.
 load("//tensorflow:version_check.bzl", "check_bazel_version_at_least")
-check_bazel_version_at_least("0.10.0")
+check_bazel_version_at_least("0.13.0")
 
 load("//tensorflow:workspace.bzl", "tf_workspace")
 
diff --git a/configure.py b/configure.py
index 60fe54b2f6..251bebc2e1 100644
--- a/configure.py
+++ b/configure.py
@@ -1451,7 +1451,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.10.0')
+  check_bazel_version('0.13.0')
 
   reset_tf_configure_bazelrc(args.workspace)
   cleanup_makefile()
-- 
cgit v1.2.3


From 3647625e531e713ad9a7fb0f3c5b68863ae4e7b8 Mon Sep 17 00:00:00 2001
From: Guangda Lai 
Date: Thu, 19 Jul 2018 15:53:13 -0700
Subject: Implement something similar to std::align since it's not supported.

PiperOrigin-RevId: 205316355
---
 tensorflow/contrib/tensorrt/BUILD                  | 30 +++++++-
 .../contrib/tensorrt/convert/convert_nodes.h       |  1 +
 .../contrib/tensorrt/resources/trt_allocator.cc    | 49 +++++++++++---
 .../contrib/tensorrt/resources/trt_allocator.h     | 14 +++-
 .../tensorrt/resources/trt_allocator_test.cc       | 79 ++++++++++++++++++++++
 5 files changed, 160 insertions(+), 13 deletions(-)
 create mode 100644 tensorflow/contrib/tensorrt/resources/trt_allocator_test.cc

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 70ce4a499c..a9378e9ad6 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -80,6 +80,7 @@ cc_library(
     copts = tf_copts(),
     visibility = ["//visibility:public"],
     deps = [
+        ":trt_allocator",
         ":trt_logging",
         ":trt_plugins",
         ":trt_resources",
@@ -195,17 +196,16 @@ tf_py_wrap_cc(
 tf_cuda_library(
     name = "trt_resources",
     srcs = [
-        "resources/trt_allocator.cc",
         "resources/trt_int8_calibrator.cc",
         "resources/trt_resource_manager.cc",
     ],
     hdrs = [
-        "resources/trt_allocator.h",
         "resources/trt_int8_calibrator.h",
         "resources/trt_resource_manager.h",
         "resources/trt_resources.h",
     ],
     deps = [
+        ":trt_allocator",
         ":trt_logging",
         ":utils",
         "//tensorflow/core:framework_headers_lib",
@@ -216,6 +216,31 @@ tf_cuda_library(
     ]),
 )
 
+tf_cuda_library(
+    name = "trt_allocator",
+    srcs = ["resources/trt_allocator.cc"],
+    hdrs = ["resources/trt_allocator.h"],
+    deps = [
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib_proto_parsing",
+    ] + if_tensorrt([
+        "@local_config_tensorrt//:nv_infer",
+    ]),
+)
+
+tf_cc_test(
+    name = "trt_allocator_test",
+    size = "small",
+    srcs = ["resources/trt_allocator_test.cc"],
+    tags = ["no_windows"],
+    deps = [
+        ":trt_allocator",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 # Library for the node-level conversion portion of TensorRT operation creation
 tf_cuda_library(
     name = "trt_conversion",
@@ -231,6 +256,7 @@ tf_cuda_library(
     ],
     deps = [
         ":segment",
+        ":trt_allocator",
         ":trt_plugins",
         ":trt_logging",
         ":trt_resources",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 1a4c0e755d..81baf8e7c1 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include 
 
 #include "tensorflow/contrib/tensorrt/convert/utils.h"
+#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
 #include "tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h"
 #include "tensorflow/core/framework/graph.pb.h"
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
index 81d7330b49..d8f97bfbbc 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.cc
@@ -19,12 +19,42 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
+#include "cuda/include/cuda_runtime_api.h"
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
+
+namespace tensorflow {
+namespace tensorrt {
+
+// std::align is not supported, so this method mimic its behavior.
+void* Align(size_t alignment, size_t size, void*& ptr, size_t& space) {
+  QCHECK_GT(alignment, 0) << "alignment must be greater than 0.";
+  QCHECK_EQ(0, alignment & (alignment - 1)) << "Alignment must be power of 2.";
+  QCHECK_GT(size, 0) << "size must be greater than 0.";
+  QCHECK(ptr) << "ptr must not be nullptr.";
+  QCHECK_GT(space, 0) << "space must be greater than 0.";
+  const uintptr_t ptr_val = reinterpret_cast(ptr);
+  QCHECK_GE(ptr_val + space, ptr_val) << "Provided space overflows.";
 
+  if (size > space) return nullptr;
+  const uintptr_t aligned_ptr_val = ((ptr_val + alignment - 1) & -alignment);
+  if (aligned_ptr_val > ptr_val + space - size) return nullptr;
+  ptr = reinterpret_cast(aligned_ptr_val);
+  const uintptr_t diff = aligned_ptr_val - ptr_val;
+  space -= diff;
+  return ptr;
+}
+
+}  // namespace tensorrt
+}  // namespace tensorflow
+
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
 #if NV_TENSORRT_MAJOR > 2
-#include "cuda/include/cuda_runtime_api.h"
 
 namespace tensorflow {
 namespace tensorrt {
+
 void* TRTCudaAllocator::allocate(uint64_t size, uint64_t alignment,
                                  uint32_t flags) {
   assert((alignment & (alignment - 1)) == 0);  // zero or a power of 2.
@@ -44,17 +74,16 @@ void* TRTDeviceAllocator::allocate(uint64_t size, uint64_t alignment,
   assert((alignment & (alignment - 1)) == 0);  // zero or a power of 2.
   size_t total_size = size + alignment;
   void* mem = allocator_->AllocateRaw(alignment, total_size);
-  if (!mem) {
-    return nullptr;
-  }
+  if (!mem) return nullptr;
 
   void* alloc_mem = mem;
-  CHECK(std::align(alignment, size, mem, total_size));
+  QCHECK(Align(alignment, size, mem, total_size));
   if (mem != alloc_mem) {
-    CHECK(mem_map_.insert({mem, alloc_mem}).second);
+    QCHECK(mem_map_.insert({mem, alloc_mem}).second);
   }
-  VLOG(2) << "Allocated " << size << " bytes with alignment " << alignment
-          << " @ " << mem;
+  VLOG(2) << "Allocated " << total_size << " bytes memory @" << alloc_mem
+          << "; aligned to " << size << " bytes @" << mem << " with alignment "
+          << alignment;
   return mem;
 }
 
@@ -80,5 +109,5 @@ void TRTDeviceAllocator::free(void* memory) {
 }  // namespace tensorflow
 
 #endif
-#endif
-#endif
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator.h b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
index b8825b108d..6f94492083 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_allocator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator.h
@@ -16,13 +16,25 @@ limitations under the License.
 #ifndef TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_ALLOCATOR_H_
 #define TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_ALLOCATOR_H_
 
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
+#include 
+
 #include "tensorflow/core/framework/allocator.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
 #include "tensorrt/include/NvInfer.h"
+#endif  // GOOGLE_TENSORRT
+#endif  // GOOGLE_CUDA
+
+namespace tensorflow {
+namespace tensorrt {
+// std::align is not supported, so this function mimic its behavior.
+void* Align(size_t alignment, size_t size, void*& ptr, size_t& space);
+}  // namespace tensorrt
+}  // namespace tensorflow
 
+#if GOOGLE_CUDA
+#if GOOGLE_TENSORRT
 #if NV_TENSORRT_MAJOR == 3
 // Define interface here temporarily until TRT 4.0 is released
 namespace nvinfer1 {
diff --git a/tensorflow/contrib/tensorrt/resources/trt_allocator_test.cc b/tensorflow/contrib/tensorrt/resources/trt_allocator_test.cc
new file mode 100644
index 0000000000..f515ed03f2
--- /dev/null
+++ b/tensorflow/contrib/tensorrt/resources/trt_allocator_test.cc
@@ -0,0 +1,79 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/tensorrt/resources/trt_allocator.h"
+
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace tensorrt {
+
+bool RunTest(const size_t alignment, const size_t size,
+             const intptr_t orig_ptr_val, const size_t orig_space) {
+  void* const orig_ptr = reinterpret_cast(orig_ptr_val);
+  void* ptr = orig_ptr;
+  size_t space = orig_space;
+  void* result = Align(alignment, size, ptr, space);
+  if (result == nullptr) {
+    EXPECT_EQ(orig_ptr, ptr);
+    EXPECT_EQ(orig_space, space);
+    return false;
+  } else {
+    EXPECT_EQ(result, ptr);
+    const intptr_t ptr_val = reinterpret_cast(ptr);
+    EXPECT_EQ(0, ptr_val % alignment);
+    EXPECT_GE(ptr_val, orig_ptr_val);
+    EXPECT_GE(space, size);
+    EXPECT_LE(space, orig_space);
+    EXPECT_EQ(ptr_val + space, orig_ptr_val + orig_space);
+    return true;
+  }
+}
+
+TEST(TRTAllocatorTest, Align) {
+  for (const size_t space :
+       {1, 2, 3, 4, 7, 8, 9, 10, 16, 32, 511, 512, 513, 700, 12345}) {
+    for (size_t alignment = 1; alignment <= space * 4; alignment *= 2) {
+      for (const intptr_t ptr_val :
+           {1ul, alignment == 1 ? 1ul : alignment - 1, alignment, alignment + 1,
+            alignment + (alignment / 2)}) {
+        if (ptr_val % alignment == 0) {
+          for (const size_t size :
+               {1ul, space == 1 ? 1ul : space - 1, space, space + 1}) {
+            EXPECT_EQ(space >= size, RunTest(alignment, size, ptr_val, space));
+          }
+        } else {
+          EXPECT_FALSE(RunTest(alignment, space, ptr_val, space));
+          const size_t diff = alignment - ptr_val % alignment;
+          if (space > diff) {
+            EXPECT_TRUE(
+                RunTest(alignment, space - diff, ptr_val + diff, space - diff));
+            for (const size_t size :
+                 {1ul, space - diff > 1 ? space - diff - 1 : 1ul, space - diff,
+                  space - diff + 1, space - 1}) {
+              EXPECT_EQ(space - diff >= size,
+                        RunTest(alignment, size, ptr_val, space));
+            }
+          } else {
+            EXPECT_FALSE(RunTest(alignment, 1, ptr_val, space));
+          }
+        }
+      }
+    }
+  }
+}
+
+}  // namespace tensorrt
+}  // namespace tensorflow
-- 
cgit v1.2.3


From 8f130ff5b021efb94946ed9deb1341890763fd3f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 15:58:10 -0700
Subject: Fix ResourceVariable placement during checkpointing to correctly
 colocate the copy of the variable on the same machine. Addresses Issue
 #20914.

PiperOrigin-RevId: 205317119
---
 tensorflow/python/training/saver.py      |  7 ++++++-
 tensorflow/python/training/saver_test.py | 18 ++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 1ee975fbe4..11510d9928 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -126,7 +126,12 @@ class BaseSaverBuilder(object):
           def f():
             with ops.device(v.device):
               x = v.read_value()
-            with ops.device("/device:CPU:0"):
+            # To allow variables placed on non-CPU devices to be checkpointed,
+            # we copy them to CPU on the same machine first.
+            device_spec = pydev.DeviceSpec().parse_from_string(v.device)
+            device_spec.merge_from(
+                pydev.DeviceSpec().parse_from_string("/device:CPU:0"))
+            with ops.device(device_spec.to_string()):
               return array_ops.identity(x)
           return f
 
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index ae9c244aaf..ecce8ae6bd 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -174,6 +174,24 @@ class SaverTest(test.TestCase):
   def testResourceBasic(self):
     self.basicSaveRestore(resource_variable_ops.ResourceVariable)
 
+  def testResourceColocation(self):
+    partitioner = partitioned_variables.fixed_size_partitioner(num_shards=2)
+    with ops_lib.device("/job:ps/device:GPU:0"):
+      v = variable_scope.get_variable("v0",
+                                      shape=[10, 2],
+                                      partitioner=partitioner,
+                                      use_resource=True)
+    saver_module.Saver({"v0": v}).build()
+    save_op = None
+    for op in ops_lib.get_default_graph().get_operations():
+      if op.type == "SaveV2":
+        save_op = op
+        break
+    assert save_op is not None
+    for save_inp in save_op.inputs[3:]:
+      # Input to SaveV2 op is placed on CPU of the same device as the Variable.
+      self.assertEqual("/job:ps/device:CPU:0", save_inp.device)
+
   def testResourceVariableReadOpsAddedDeterministically(self):
     graph_defs = []
     num_graphs = 10
-- 
cgit v1.2.3


From 00affecfa57c7ad20e3438aadb3de5686bfce9d0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 16:48:04 -0700
Subject: Remove elem_type from tensorflow Pack export, tensorflow doesn't like
 graphs that use this attribute in Pack.

PiperOrigin-RevId: 205324618
---
 tensorflow/contrib/lite/toco/export_tensorflow.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/export_tensorflow.cc b/tensorflow/contrib/lite/toco/export_tensorflow.cc
index f9a6d31d60..b79bb300f0 100644
--- a/tensorflow/contrib/lite/toco/export_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/export_tensorflow.cc
@@ -1257,8 +1257,6 @@ void ConvertPackOperator(const Model& model, const PackOperator& src_op,
   for (const auto& input : src_op.inputs) {
     *pack_op->add_input() = input;
   }
-  (*pack_op->mutable_attr())["elem_type"].set_type(
-      GetTensorFlowDataType(model, src_op.outputs[0]));
   (*pack_op->mutable_attr())["axis"].set_i(src_op.axis);
   (*pack_op->mutable_attr())["N"].set_i(src_op.inputs.size());
   (*pack_op->mutable_attr())["T"].set_type(GetTensorFlowDataType(src_op.dtype));
-- 
cgit v1.2.3


From ea06626358edc5fd3cbdae4839b3d51cd7b0dfa4 Mon Sep 17 00:00:00 2001
From: Justin Lebar 
Date: Thu, 19 Jul 2018 17:10:08 -0700
Subject: Update comments in BFCAllocator.

PiperOrigin-RevId: 205327986
---
 tensorflow/core/common_runtime/bfc_allocator.cc |  4 ----
 tensorflow/core/common_runtime/bfc_allocator.h  | 27 ++++++++++++++++++-------
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/common_runtime/bfc_allocator.cc b/tensorflow/core/common_runtime/bfc_allocator.cc
index f8ca039d15..3bf0532491 100644
--- a/tensorflow/core/common_runtime/bfc_allocator.cc
+++ b/tensorflow/core/common_runtime/bfc_allocator.cc
@@ -155,10 +155,6 @@ bool BFCAllocator::Extend(size_t alignment, size_t rounded_bytes) {
 
   region_manager_.set_handle(c->ptr, h);
 
-  // TODO(vrv): Try to merge this new region with an existing region,
-  // if the address space is contiguous, to avoid fragmentation
-  // across regions.
-
   // Insert the chunk into the right bin.
   InsertFreeChunkIntoBin(h);
 
diff --git a/tensorflow/core/common_runtime/bfc_allocator.h b/tensorflow/core/common_runtime/bfc_allocator.h
index cd8ff6e5c0..580e61e2ea 100644
--- a/tensorflow/core/common_runtime/bfc_allocator.h
+++ b/tensorflow/core/common_runtime/bfc_allocator.h
@@ -88,11 +88,20 @@ class BFCAllocator : public VisitableAllocator {
   static const int kInvalidBinNum = -1;
   static const int kNumBins = 21;
 
-  // Chunks point to memory.  Their prev/next pointers form a
-  // doubly-linked list of addresses sorted by base address that
-  // must be contiguous.  Chunks contain information about whether
-  // they are in use or whether they are free, and contain a pointer
-  // to the bin they are in.
+  // A Chunk points to a piece of memory that's either entirely free or entirely
+  // in use by one user memory allocation.
+  //
+  // An AllocationRegion's memory is split up into one or more disjoint Chunks,
+  // which together cover the whole region without gaps.  Chunks participate in
+  // a doubly-linked list, and the prev/next pointers point to the physically
+  // adjacent chunks.
+  //
+  // Since a chunk cannot be partially in use, we may need to split a free chunk
+  // in order to service a user allocation.  We always merge adjacent free
+  // chunks.
+  //
+  // Chunks contain information about whether they are in use or whether they
+  // are free, and contain a pointer to the bin they are in.
   struct Chunk {
     size_t size = 0;  // Full size of buffer.
 
@@ -177,8 +186,12 @@ class BFCAllocator : public VisitableAllocator {
   static const size_t kMinAllocationBits = 8;
   static const size_t kMinAllocationSize = 1 << kMinAllocationBits;
 
-  // AllocationRegion maps pointers to ChunkHandles for a single
-  // contiguous memory region.
+  // BFCAllocator allocates memory into a collection of disjoint
+  // AllocationRegions.  Each AllocationRegion corresponds to one call to
+  // SubAllocator::Alloc().
+  //
+  // An AllocationRegion contains one or more Chunks, covering all of its
+  // memory.  Its primary job is to map a pointers to ChunkHandles.
   //
   // This class is thread-compatible.
   class AllocationRegion {
-- 
cgit v1.2.3


From 78121905a545ca8e91ab1dbc899b5509933331d7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 17:22:25 -0700
Subject: Set correct TF_OVERRIDE_EIGEN_STRONG_INLINE value in
 build_tf_windows.sh

PiperOrigin-RevId: 205329361
---
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 4 +++-
 tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index dc7ea1dc57..42f58deb42 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -66,11 +66,13 @@ for ARG in "$@"; do
   fi
 done
 
-if [[ "$release_build" != 1 ]]; then
+if [[ "$release_build" == 1 ]]; then
   # Overriding eigen strong inline speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc
   # by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521
   # Because this hurts the performance of TF, we don't override it in release build.
   export TF_OVERRIDE_EIGEN_STRONG_INLINE=0
+else
+  export TF_OVERRIDE_EIGEN_STRONG_INLINE=1
 fi
 
 # Enable short object file path to avoid long path issue on Windows.
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index a4175a0e81..2a8c2d9167 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -66,11 +66,13 @@ for ARG in "$@"; do
   fi
 done
 
-if [[ "$release_build" != 1 ]]; then
+if [[ "$release_build" == 1 ]]; then
   # Overriding eigen strong inline speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc
   # by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521
   # Because this hurts the performance of TF, we don't override it in release build.
   export TF_OVERRIDE_EIGEN_STRONG_INLINE=0
+else
+  export TF_OVERRIDE_EIGEN_STRONG_INLINE=1
 fi
 
 # Enable short object file path to avoid long path issue on Windows.
-- 
cgit v1.2.3


From 95ec73a1c8b5174ec2221c2b5ecaf179c9deef48 Mon Sep 17 00:00:00 2001
From: Amit Patankar 
Date: Thu, 19 Jul 2018 17:28:52 -0700
Subject: Fix clang builds for NCCL error.

PiperOrigin-RevId: 205330050
---
 tensorflow/tools/ci_build/Dockerfile.gpu | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu
index 7591ecc04e..6ab703d93d 100644
--- a/tensorflow/tools/ci_build/Dockerfile.gpu
+++ b/tensorflow/tools/ci_build/Dockerfile.gpu
@@ -14,6 +14,11 @@ RUN /install/install_bootstrap_deb_packages.sh
 RUN add-apt-repository -y ppa:openjdk-r/ppa && \
     add-apt-repository -y ppa:george-edison55/cmake-3.x
 RUN /install/install_deb_packages.sh
+
+# Install NCCL
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libnccl2=2.2.13-1+cuda9.0
+
 RUN /install/install_pip_packages.sh
 RUN /install/install_bazel.sh
 RUN /install/install_golang.sh
-- 
cgit v1.2.3


From 37bbf89920f013ef1d59f0eaef65431d4f4a4a28 Mon Sep 17 00:00:00 2001
From: Rohan Jain 
Date: Thu, 19 Jul 2018 17:37:22 -0700
Subject: Fixing bug where in Eager mode datasets not intended to be on the GPU
 end up being there.

PiperOrigin-RevId: 205331171
---
 tensorflow/contrib/eager/python/datasets_test.py | 14 ++++++++++
 tensorflow/python/data/ops/iterator_ops.py       | 33 ++++++++++++------------
 2 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/eager/python/datasets_test.py b/tensorflow/contrib/eager/python/datasets_test.py
index 68bec9aee8..acc605247f 100644
--- a/tensorflow/contrib/eager/python/datasets_test.py
+++ b/tensorflow/contrib/eager/python/datasets_test.py
@@ -193,6 +193,20 @@ class IteratorTest(test.TestCase):
       x = math_ops.add(x, x)
     self.assertAllEqual([0., 2.], x.numpy())
 
+  def testGpuTensor(self):
+    ds = Dataset.from_tensors([0., 1.])
+    with ops.device(test.gpu_device_name()):
+      for x in ds:
+        y = math_ops.add(x, x)
+    self.assertAllEqual([0., 2.], y.numpy())
+
+  def testGpuDefinedDataset(self):
+    with ops.device(test.gpu_device_name()):
+      ds = Dataset.from_tensors([0., 1.])
+      for x in ds:
+        y = math_ops.add(x, x)
+    self.assertAllEqual([0., 2.], y.numpy())
+
   def testTensorsExplicitPrefetchToDevice(self):
     ds = Dataset.from_tensor_slices([0., 1.])
     ds = ds.apply(prefetching_ops.prefetch_to_device(test.gpu_device_name()))
diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py
index f0784ed3d0..3ef22cf981 100644
--- a/tensorflow/python/data/ops/iterator_ops.py
+++ b/tensorflow/python/data/ops/iterator_ops.py
@@ -500,22 +500,23 @@ class EagerIterator(object):
           "tf.data.Dataset.make_one_shot_iterator for graph construction".
           format(type(self)))
     self._device = context.context().device_name
-    ds_variant = dataset._as_variant_tensor()  # pylint: disable=protected-access
-    self._output_classes = dataset.output_classes
-    self._output_types = dataset.output_types
-    self._output_shapes = dataset.output_shapes
-    self._flat_output_types = nest.flatten(
-        sparse.as_dense_types(self._output_types, self._output_classes))
-    self._flat_output_shapes = nest.flatten(
-        sparse.as_dense_shapes(self._output_shapes, self._output_classes))
-    with ops.colocate_with(ds_variant):
-      self._resource = gen_dataset_ops.anonymous_iterator(
-          output_types=self._flat_output_types,
-          output_shapes=self._flat_output_shapes)
-      gen_dataset_ops.make_iterator(ds_variant, self._resource)
-    # Delete the resource when this object is deleted
-    self._resource_deleter = resource_variable_ops.EagerResourceDeleter(
-        handle=self._resource, handle_device=self._device)
+    with ops.device("/cpu:0"):
+      ds_variant = dataset._as_variant_tensor()  # pylint: disable=protected-access
+      self._output_classes = dataset.output_classes
+      self._output_types = dataset.output_types
+      self._output_shapes = dataset.output_shapes
+      self._flat_output_types = nest.flatten(
+          sparse.as_dense_types(self._output_types, self._output_classes))
+      self._flat_output_shapes = nest.flatten(
+          sparse.as_dense_shapes(self._output_shapes, self._output_classes))
+      with ops.colocate_with(ds_variant):
+        self._resource = gen_dataset_ops.anonymous_iterator(
+            output_types=self._flat_output_types,
+            output_shapes=self._flat_output_shapes)
+        gen_dataset_ops.make_iterator(ds_variant, self._resource)
+        # Delete the resource when this object is deleted
+        self._resource_deleter = resource_variable_ops.EagerResourceDeleter(
+            handle=self._resource, handle_device=self._device)
 
   def __iter__(self):
     return self
-- 
cgit v1.2.3


From 77c7b1112210beb3f0752f206bfa519f22aaf5c6 Mon Sep 17 00:00:00 2001
From: Allen Lavoie 
Date: Thu, 19 Jul 2018 18:12:34 -0700
Subject: Fix AdamOptimizer usage in a defun

Adds init_scope around Adam's non-slot variable lookup so it's always accessed in a consistent context.

PiperOrigin-RevId: 205335191
---
 tensorflow/python/eager/function_test.py | 18 ++++++++++++++++++
 tensorflow/python/training/adam.py       | 13 +++++++------
 tensorflow/python/training/adam_test.py  |  6 ++++++
 3 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index cdd9fe1760..e6592b2e37 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -45,6 +45,7 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
+from tensorflow.python.training import adam
 from tensorflow.python.training import momentum
 from tensorflow.python.training import training_ops
 from tensorflow.python.util import compat
@@ -1166,6 +1167,23 @@ class AutomaticControlDependenciesTest(test.TestCase):
     value = train()
     self.assertEqual(value.numpy(), -1.0)
 
+  # TODO(b/111663004): This should work when the outer context is graph
+  # building.
+  def testOptimizerNonSlotVarsInDefunNoError(self):
+    def loss(v):
+      return v**2
+
+    optimizer = adam.AdamOptimizer(learning_rate=1.0)
+
+    @function.defun
+    def train():
+      v = resource_variable_ops.ResourceVariable(1.0)
+      grad = backprop.implicit_grad(loss)(v)
+      optimizer.apply_gradients(grad)
+      return v.read_value()
+
+    train()
+
   def testOptimizerInDefunWithCapturedVariable(self):
     v = resource_variable_ops.ResourceVariable(1.0)
     def loss():
diff --git a/tensorflow/python/training/adam.py b/tensorflow/python/training/adam.py
index b65c88e972..bcbe5907d6 100644
--- a/tensorflow/python/training/adam.py
+++ b/tensorflow/python/training/adam.py
@@ -109,12 +109,13 @@ class AdamOptimizer(optimizer.Optimizer):
     self._updated_lr = None
 
   def _get_beta_accumulators(self):
-    if context.executing_eagerly():
-      graph = None
-    else:
-      graph = ops.get_default_graph()
-    return (self._get_non_slot_variable("beta1_power", graph=graph),
-            self._get_non_slot_variable("beta2_power", graph=graph))
+    with ops.init_scope():
+      if context.executing_eagerly():
+        graph = None
+      else:
+        graph = ops.get_default_graph()
+      return (self._get_non_slot_variable("beta1_power", graph=graph),
+              self._get_non_slot_variable("beta2_power", graph=graph))
 
   def _create_slots(self, var_list):
     # Create the beta1 and beta2 accumulators on the same device as the first
diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py
index ccdc7e384d..8f84427654 100644
--- a/tensorflow/python/training/adam_test.py
+++ b/tensorflow/python/training/adam_test.py
@@ -315,6 +315,12 @@ class AdamOptimizerTest(test.TestCase):
 
   def testTwoSessions(self):
     optimizer = adam.AdamOptimizer()
+
+    with context.eager_mode():
+      var0 = variables.Variable(np.array([1.0, 2.0]), name="v0")
+      grads0 = constant_op.constant(np.array([0.1, 0.1]))
+      optimizer.apply_gradients([(grads0, var0)])
+
     g = ops.Graph()
     with g.as_default():
       with session.Session():
-- 
cgit v1.2.3


From 0c11bcb5f3443ce870f31f5ba013ae8bc375ad2d Mon Sep 17 00:00:00 2001
From: Jacker 
Date: Fri, 20 Jul 2018 10:09:16 +0800
Subject: Update saver.py

Fix device placement of save_op for ResourceVariable.
---
 tensorflow/python/training/saver.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 11510d9928..60885e9292 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -126,13 +126,10 @@ class BaseSaverBuilder(object):
           def f():
             with ops.device(v.device):
               x = v.read_value()
-            # To allow variables placed on non-CPU devices to be checkpointed,
-            # we copy them to CPU on the same machine first.
-            device_spec = pydev.DeviceSpec().parse_from_string(v.device)
-            device_spec.merge_from(
-                pydev.DeviceSpec().parse_from_string("/device:CPU:0"))
-            with ops.device(device_spec.to_string()):
-              return array_ops.identity(x)
+              # To allow variables placed on non-CPU devices to be checkpointed,
+              # we copy them to CPU on the same machine first.
+              with ops.device("/device:CPU:0"):
+                return array_ops.identity(x)
           return f
 
         self.handle_op = var.handle
-- 
cgit v1.2.3


From db308efbf4e95a7362fde90d35447091349b548e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Thu, 19 Jul 2018 19:45:33 -0700
Subject: Allow input_shape to be specified in TOCO python converter

PiperOrigin-RevId: 205342464
---
 tensorflow/contrib/lite/python/convert.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/python/convert.py b/tensorflow/contrib/lite/python/convert.py
index 0ea2630f71..ec49738fb5 100644
--- a/tensorflow/contrib/lite/python/convert.py
+++ b/tensorflow/contrib/lite/python/convert.py
@@ -115,6 +115,7 @@ def build_toco_convert_protos(input_tensors,
                               inference_type=lite_constants.FLOAT,
                               inference_input_type=None,
                               input_format=lite_constants.TENSORFLOW_GRAPHDEF,
+                              input_shapes=None,
                               output_format=lite_constants.TFLITE,
                               quantized_input_stats=None,
                               default_ranges_stats=None,
@@ -141,6 +142,8 @@ def build_toco_convert_protos(input_tensors,
       Must be `{FLOAT, QUANTIZED_UINT8}`. (default `inference_type`)
     input_format: Type of data to read Currently must be
       `{TENSORFLOW_GRAPHDEF}`. (default TENSORFLOW_GRAPHDEF)
+    input_shapes: Input array shape. It needs to be a list of the same length
+      as `input_tensors`, or None. (default None)
     output_format: Output file format. Currently must be `{TFLITE,
       GRAPHVIZ_DOT}`. (default TFLITE)
     quantized_input_stats: List of tuples of integers representing the mean and
@@ -209,7 +212,11 @@ def build_toco_convert_protos(input_tensors,
     if inference_type == lite_constants.QUANTIZED_UINT8:
       input_array.mean_value, input_array.std_value = quantized_input_stats[idx]
     input_array.name = tensor_name(input_tensor)
-    input_array.shape.dims.extend(map(int, input_tensor.get_shape()))
+    if input_shapes is None:
+      shape = input_tensor.get_shape()
+    else:
+      shape = input_shapes[idx]
+    input_array.shape.dims.extend(map(int, shape))
 
   for output_tensor in output_tensors:
     model.output_arrays.append(tensor_name(output_tensor))
-- 
cgit v1.2.3


From ee5271968e17b1bc0b852b0285dad4a38e1a97a0 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena 
Date: Thu, 19 Jul 2018 23:16:08 -0700
Subject: Support Defuns and nested Defuns inside cond_v2 branches. Support
 nested cond_v2s.

PiperOrigin-RevId: 205356562
---
 tensorflow/core/kernels/functional_ops.cc          |   1 +
 tensorflow/python/BUILD                            |   4 +-
 .../python/framework/function_def_to_graph.py      |  32 +-
 .../python/framework/function_def_to_graph_test.py |  34 +-
 tensorflow/python/kernel_tests/BUILD               |   3 +-
 tensorflow/python/kernel_tests/cond_v2_test.py     | 372 ++++++++++++++++++++-
 tensorflow/python/ops/cond_v2_impl.py              | 139 +++++---
 7 files changed, 515 insertions(+), 70 deletions(-)

diff --git a/tensorflow/core/kernels/functional_ops.cc b/tensorflow/core/kernels/functional_ops.cc
index 519c475332..cb285bf732 100644
--- a/tensorflow/core/kernels/functional_ops.cc
+++ b/tensorflow/core/kernels/functional_ops.cc
@@ -536,6 +536,7 @@ class FakeParamOp : public OpKernel {
 };
 
 REGISTER_KERNEL_BUILDER(Name("FakeParam").Device(DEVICE_CPU), FakeParamOp);
+REGISTER_KERNEL_BUILDER(Name("FakeParam").Device(DEVICE_GPU), FakeParamOp);
 
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index c33a579ad2..9c7f3b7b25 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -745,8 +745,8 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":framework",
+        ":framework_ops",
         ":function",
-        ":op_def_registry",
         ":tensor_shape",
         ":versions",
         "//tensorflow/core:protos_all_py",
@@ -762,8 +762,10 @@ py_test(
     deps = [
         ":array_ops",
         ":client_testlib",
+        ":constant_op",
         ":dtypes",
         ":framework_ops",
+        ":function",
         ":function_def_to_graph",
         ":graph_to_function_def",
         ":math_ops",
diff --git a/tensorflow/python/framework/function_def_to_graph.py b/tensorflow/python/framework/function_def_to_graph.py
index 46c9c4c14a..1b09506662 100644
--- a/tensorflow/python/framework/function_def_to_graph.py
+++ b/tensorflow/python/framework/function_def_to_graph.py
@@ -25,7 +25,7 @@ from tensorflow.core.framework import types_pb2
 from tensorflow.core.framework import versions_pb2
 from tensorflow.python.framework import function
 from tensorflow.python.framework import importer
-from tensorflow.python.framework import op_def_registry
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import versions
 from tensorflow.python.ops import cond_v2_impl
 
@@ -114,6 +114,10 @@ def function_def_to_graph_def(fdef, input_shapes=None):
           producer=versions.GRAPH_DEF_VERSION,
           min_consumer=versions.GRAPH_DEF_VERSION_MIN_CONSUMER))
 
+  # Copy *all* functions from outer graph to `graph_def` so that both direct
+  # and indirect references are safely handled.
+  ops.get_default_graph()._copy_functions_to_graph_def(graph_def, 0)  # pylint: disable=protected-access
+
   if input_shapes and len(input_shapes) != len(fdef.signature.input_arg):
     raise ValueError("Length of input_shapes must match the number of " +
                      "input_args. len(input_shapes): {} len(input_arg): {}".
@@ -142,24 +146,18 @@ def function_def_to_graph_def(fdef, input_shapes=None):
     nested_to_flat_tensor_name[arg_def.name] = "{}:0".format(arg_def.name)
 
   for node_def in fdef.node_def:
-    op_def = op_def_registry.get_registered_ops().get(node_def.op)
-    if not op_def:
-      # TODO(b/80470245): Support functions which refer other functions.
-      raise NotImplementedError(
-          "No op registered for {},".format(node_def.op) +
-          " it may be a function. function_def_to_graph_def " +
-          "currently does not support converting functions with " +
-          "references to other graph functions.")
+    op_def = ops.get_default_graph()._get_op_def(node_def.op)  # pylint: disable=protected-access
 
     for attr in op_def.attr:
-      if attr.type in ("func", "list(func)"):
-        # TODO(b/80470245): Support functions which refer other functions.
-        raise NotImplementedError("Unsupported attr {} ".format(attr.name) +
-                                  " with type {}".format(attr.type) +
-                                  " in op {}. ".format(op_def.name) +
-                                  "function_def_to_graph_def currently does " +
-                                  "not support converting functions with " +
-                                  "references to other graph functions.")
+      if attr.type == "func":
+        fname = node_def.attr[attr.name].func.name
+        if not ops.get_default_graph()._is_function(fname):  # pylint: disable=protected-access
+          raise ValueError("%s function not found." % fname)
+      elif attr.type == "list(func)":
+        for fn in node_def.attr[attr.name].list.func:
+          fname = fn.name
+          if not ops.get_default_graph()._is_function(fname):  # pylint: disable=protected-access
+            raise ValueError("%s function not found." % fname)
 
     # Iterate over output_args in op_def to build the map.
     # Index of the output tensor in the flattened list of *all* output
diff --git a/tensorflow/python/framework/function_def_to_graph_test.py b/tensorflow/python/framework/function_def_to_graph_test.py
index 0f4e6ef54f..cd2a16ed5a 100644
--- a/tensorflow/python/framework/function_def_to_graph_test.py
+++ b/tensorflow/python/framework/function_def_to_graph_test.py
@@ -18,7 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import function
 from tensorflow.python.framework import function_def_to_graph
 from tensorflow.python.framework import graph_to_function_def
 from tensorflow.python.framework import ops
@@ -79,7 +81,6 @@ class FunctionDefToGraphTest(test.TestCase):
 
     g = function_def_to_graph.function_def_to_graph(
         fdef, input_shapes=[None, tensor_shape.matrix(5, 7)])
-    print(g.as_graph_def())
     self.assertIsNone(g.inputs[0].shape.dims)
     self.assertSequenceEqual(g.inputs[1].shape.dims, [5, 7])
     self.assertSequenceEqual(g.outputs[0].shape.dims, [5, 7])
@@ -179,6 +180,37 @@ class FunctionDefToGraphDefTest(test.TestCase):
     self.assertEqual(g.node[0].attr["shape"].shape.unknown_rank, False)
     self.assertFalse("shape" in g.node[2].attr)
 
+  def testFunctionCallsFromFunction(self):
+    x = constant_op.constant(5.0)
+    y = constant_op.constant(10.0)
+
+    @function.Defun()
+    def fn():
+
+      @function.Defun()
+      def inner_fn():
+        return x + y
+
+      return inner_fn()
+
+    # Instantiate the function in this graph so that
+    # `function_def_to_graph` can find it.
+    fn()
+
+    def fn2():
+      return 2 * fn()
+
+    fdef = function._DefinedFunction(fn2, [], []).definition
+    func_graph = function_def_to_graph.function_def_to_graph(fdef)
+    with func_graph.as_default():
+      x_ph, y_ph = func_graph.inputs
+      with self.test_session(graph=func_graph) as sess:
+        self.assertEqual(
+            sess.run(func_graph.outputs[0], feed_dict={
+                x_ph: 5.0,
+                y_ph: 10.0
+            }), 30.0)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 838cf836f1..db2e7e2c2a 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -3095,7 +3095,7 @@ tf_py_test(
 
 tf_py_test(
     name = "cond_v2_test",
-    size = "small",
+    size = "medium",
     srcs = ["cond_v2_test.py"],
     additional_deps = [
         "//tensorflow/python:array_ops",
@@ -3110,4 +3110,5 @@ tf_py_test(
         "//tensorflow/python:training",
     ],
     grpc_enabled = True,
+    tags = ["no_gpu"],  # TODO(b/111656070)
 )
diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py
index 759db5d5f4..97ce245fc8 100644
--- a/tensorflow/python/kernel_tests/cond_v2_test.py
+++ b/tensorflow/python/kernel_tests/cond_v2_test.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import cond_v2
@@ -35,10 +36,12 @@ from tensorflow.python.training import saver
 from tensorflow.python.util import compat
 
 
-class NewCondTest(test.TestCase):
+class CondV2Test(test.TestCase):
 
-  def _testCond(self, true_fn, false_fn, train_vals):
-    with self.test_session() as sess:
+  def _testCond(self, true_fn, false_fn, train_vals, feed_dict=None):
+    if not feed_dict:
+      feed_dict = {}
+    with self.test_session(graph=ops.get_default_graph()) as sess:
       pred = array_ops.placeholder(dtypes.bool, name="pred")
 
       expected = control_flow_ops.cond(pred, true_fn, false_fn, name="expected")
@@ -47,13 +50,17 @@ class NewCondTest(test.TestCase):
       expected_grad = gradients_impl.gradients(expected, train_vals)
       actual_grad = gradients_impl.gradients(actual, train_vals)
 
+      sess_run_args = {pred: True}
+      sess_run_args.update(feed_dict)
       expected_val, actual_val, expected_grad_val, actual_grad_val = sess.run(
-          (expected, actual, expected_grad, actual_grad), {pred: True})
+          (expected, actual, expected_grad, actual_grad), sess_run_args)
       self.assertEqual(expected_val, actual_val)
       self.assertEqual(expected_grad_val, actual_grad_val)
 
+      sess_run_args = {pred: False}
+      sess_run_args.update(feed_dict)
       expected_val, actual_val, expected_grad_val, actual_grad_val = sess.run(
-          (expected, actual, expected_grad, actual_grad), {pred: False})
+          (expected, actual, expected_grad, actual_grad), sess_run_args)
       self.assertEqual(expected_val, actual_val)
       self.assertEqual(expected_grad_val, actual_grad_val)
 
@@ -131,6 +138,349 @@ class NewCondTest(test.TestCase):
         self.assertIn("foo_cond_1_true", ops.get_default_graph()._functions)
         self.assertIn("foo_cond_1_false", ops.get_default_graph()._functions)
 
+  def testDefunInCond(self):
+    x = constant_op.constant(1.0, name="x")
+    y = constant_op.constant(2.0, name="y")
+
+    def true_fn():
+
+      @function.Defun()
+      def fn():
+        return x * y * 2.0
+
+      return fn()
+
+    def false_fn():
+      return 2.0
+
+    self._testCond(true_fn, false_fn, [x])
+    self._testCond(true_fn, false_fn, [x, y])
+    self._testCond(true_fn, false_fn, [y])
+
+  def testNestedDefunInCond(self):
+    x = constant_op.constant(1.0, name="x")
+    y = constant_op.constant(2.0, name="y")
+
+    def true_fn():
+      return 2.0
+
+    def false_fn():
+
+      @function.Defun()
+      def fn():
+
+        @function.Defun()
+        def nested_fn():
+          return x * y * 2.0
+
+        return nested_fn()
+
+      return fn()
+
+    self._testCond(true_fn, false_fn, [x])
+    self._testCond(true_fn, false_fn, [x, y])
+    self._testCond(true_fn, false_fn, [y])
+
+  def testDoubleNestedDefunInCond(self):
+    x = constant_op.constant(1.0, name="x")
+    y = constant_op.constant(2.0, name="y")
+
+    def true_fn():
+
+      @function.Defun()
+      def fn():
+
+        @function.Defun()
+        def nested_fn():
+
+          @function.Defun()
+          def nested_nested_fn():
+            return x * y * 2.0
+
+          return nested_nested_fn()
+
+        return nested_fn()
+
+      return fn()
+
+    def false_fn():
+      return 2.0
+
+    self._testCond(true_fn, false_fn, [x])
+    self._testCond(true_fn, false_fn, [x, y])
+    self._testCond(true_fn, false_fn, [y])
+
+  def testNestedCond(self):
+
+    def run_test(pred_value):
+
+      def build_graph():
+        pred = array_ops.placeholder(dtypes.bool, name="pred")
+        x = constant_op.constant(1.0, name="x")
+        y = constant_op.constant(2.0, name="y")
+
+        def true_fn():
+          return 2.0
+
+        def false_fn():
+
+          def false_true_fn():
+            return x * y * 2.0
+
+          def false_false_fn():
+            return x * 5.0
+
+          return _cond(pred, false_true_fn, false_false_fn, "inside_false_fn")
+
+        return x, y, pred, true_fn, false_fn
+
+      with ops.Graph().as_default():
+        x, y, pred, true_fn, false_fn = build_graph()
+        self._testCond(true_fn, false_fn, [x, y], {pred: pred_value})
+        self._testCond(true_fn, false_fn, [x], {pred: pred_value})
+        self._testCond(true_fn, false_fn, [y], {pred: pred_value})
+
+    run_test(True)
+    run_test(False)
+
+  def testDoubleNestedCond(self):
+
+    def run_test(pred1_value, pred2_value):
+
+      def build_graph():
+        pred1 = array_ops.placeholder(dtypes.bool, name="pred1")
+        pred2 = array_ops.placeholder(dtypes.bool, name="pred2")
+        x = constant_op.constant(1.0, name="x")
+        y = constant_op.constant(2.0, name="y")
+
+        def true_fn():
+          return 2.0
+
+        def false_fn():
+
+          def false_true_fn():
+
+            def false_true_true_fn():
+              return x * y * 2.0
+
+            def false_true_false_fn():
+              return x * 10.0
+
+            return _cond(
+                pred1,
+                false_true_true_fn,
+                false_true_false_fn,
+                name="inside_false_true_fn")
+
+          def false_false_fn():
+            return x * 5.0
+
+          return _cond(
+              pred2, false_true_fn, false_false_fn, name="inside_false_fn")
+
+        return x, y, pred1, pred2, true_fn, false_fn
+
+      with ops.Graph().as_default():
+        x, y, pred1, pred2, true_fn, false_fn = build_graph()
+        self._testCond(true_fn, false_fn, [x, y], {
+            pred1: pred1_value,
+            pred2: pred2_value
+        })
+        x, y, pred1, pred2, true_fn, false_fn = build_graph()
+        self._testCond(true_fn, false_fn, [x], {
+            pred1: pred1_value,
+            pred2: pred2_value
+        })
+        x, y, pred1, pred2, true_fn, false_fn = build_graph()
+        self._testCond(true_fn, false_fn, [y], {
+            pred1: pred1_value,
+            pred2: pred2_value
+        })
+
+    run_test(True, True)
+    run_test(True, False)
+    run_test(False, False)
+    run_test(False, True)
+
+  def testGradientFromInsideDefun(self):
+
+    def build_graph():
+      pred_outer = array_ops.placeholder(dtypes.bool, name="pred_outer")
+      pred_inner = array_ops.placeholder(dtypes.bool, name="pred_inner")
+      x = constant_op.constant(1.0, name="x")
+      y = constant_op.constant(2.0, name="y")
+
+      def true_fn():
+        return 2.0
+
+      def false_fn():
+
+        def inner_true_fn():
+          return x * y * 2.0
+
+        def inner_false_fn():
+          return x * 5.0
+
+        return cond_v2.cond_v2(
+            pred_inner, inner_true_fn, inner_false_fn, name="inner_cond")
+
+      cond_outer = cond_v2.cond_v2(
+          pred_outer, true_fn, false_fn, name="outer_cond")
+
+      # Compute grads inside a Defun.
+      @function.Defun()
+      def nesting_fn():
+        return gradients_impl.gradients(cond_outer, [x, y])
+
+      grads = nesting_fn()
+
+      return grads, pred_outer, pred_inner
+
+    with ops.Graph().as_default():
+      grads, pred_outer, pred_inner = build_graph()
+      with self.test_session(graph=ops.get_default_graph()) as sess:
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: True,
+                pred_inner: True
+            }), [0., 0.])
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: True,
+                pred_inner: False
+            }), [0., 0.])
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: False,
+                pred_inner: True
+            }), [4., 2.])
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: False,
+                pred_inner: False
+            }), [5., 0.])
+
+  def testGradientFromInsideNestedDefun(self):
+
+    def build_graph():
+      pred_outer = array_ops.placeholder(dtypes.bool, name="pred_outer")
+      pred_inner = array_ops.placeholder(dtypes.bool, name="pred_inner")
+      x = constant_op.constant(1.0, name="x")
+      y = constant_op.constant(2.0, name="y")
+
+      def true_fn():
+        return 2.0
+
+      def false_fn():
+
+        def inner_true_fn():
+          return x * y * 2.0
+
+        def inner_false_fn():
+          return x * 5.0
+
+        return cond_v2.cond_v2(
+            pred_inner, inner_true_fn, inner_false_fn, name="inner_cond")
+
+      cond_outer = cond_v2.cond_v2(
+          pred_outer, true_fn, false_fn, name="outer_cond")
+
+      # Compute grads inside a Defun.
+      @function.Defun()
+      def nesting_fn():
+
+        @function.Defun()
+        def inner_nesting_fn():
+          return gradients_impl.gradients(cond_outer, [x, y])
+
+        return inner_nesting_fn()
+
+      grads = nesting_fn()
+
+      return grads, pred_outer, pred_inner
+
+    with ops.Graph().as_default():
+      grads, pred_outer, pred_inner = build_graph()
+      with self.test_session(graph=ops.get_default_graph()) as sess:
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: True,
+                pred_inner: True
+            }), [0., 0.])
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: True,
+                pred_inner: False
+            }), [0., 0.])
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: False,
+                pred_inner: True
+            }), [4., 2.])
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: False,
+                pred_inner: False
+            }), [5., 0.])
+
+  def testBuildCondAndGradientInsideDefun(self):
+
+    def build_graph():
+      pred_outer = array_ops.placeholder(dtypes.bool, name="pred_outer")
+      pred_inner = array_ops.placeholder(dtypes.bool, name="pred_inner")
+      x = constant_op.constant(1.0, name="x")
+      y = constant_op.constant(2.0, name="y")
+
+      # Build cond and its gradient inside a Defun.
+      @function.Defun()
+      def fn():
+
+        def true_fn():
+          return 2.0
+
+        def false_fn():
+
+          def inner_true_fn():
+            return x * y * 2.0
+
+          def inner_false_fn():
+            return x * 5.0
+
+          return cond_v2.cond_v2(
+              pred_inner, inner_true_fn, inner_false_fn, name="inner_cond")
+
+        cond_outer = cond_v2.cond_v2(
+            pred_outer, true_fn, false_fn, name="outer_cond")
+        return gradients_impl.gradients(cond_outer, [x, y])
+
+      grads = fn()
+
+      return grads, pred_outer, pred_inner
+
+    with ops.Graph().as_default():
+      grads, pred_outer, pred_inner = build_graph()
+      with self.test_session(graph=ops.get_default_graph()) as sess:
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: True,
+                pred_inner: True
+            }), [0., 0.])
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: True,
+                pred_inner: False
+            }), [0., 0.])
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: False,
+                pred_inner: True
+            }), [4., 2.])
+        self.assertSequenceEqual(
+            sess.run(grads, {
+                pred_outer: False,
+                pred_inner: False
+            }), [5., 0.])
+
   def testSecondDerivative(self):
     with self.test_session() as sess:
       pred = array_ops.placeholder(dtypes.bool, name="pred")
@@ -532,5 +882,17 @@ class CondV2ColocationGroupAndDeviceTest(test.TestCase):
         self.assertTrue(len(run_metadata.partition_graphs) >= 2)
 
 
+def _cond(pred, true_fn, false_fn, name):
+  if _is_old_cond():
+    return control_flow_ops.cond(pred, true_fn, false_fn, name=name)
+  else:
+    return cond_v2.cond_v2(pred, true_fn, false_fn, name=name)
+
+
+def _is_old_cond():
+  return isinstance(ops.get_default_graph()._get_control_flow_context(),
+                    control_flow_ops.CondContext)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/cond_v2_impl.py b/tensorflow/python/ops/cond_v2_impl.py
index d310f83dca..5cd0cb34de 100644
--- a/tensorflow/python/ops/cond_v2_impl.py
+++ b/tensorflow/python/ops/cond_v2_impl.py
@@ -135,6 +135,10 @@ def cond_v2(pred, true_fn, false_fn, name="cond"):
 def _IfGrad(op, *grads):  # pylint: disable=invalid-name
   """The gradient of an If op produced by cond_v2."""
   true_graph, false_graph = _get_func_graphs(op)
+  # Note: op.graph != ops.get_default_graph() when we are computing the gradient
+  # of a nested cond.
+  assert true_graph._outer_graph == op.graph
+  assert false_graph._outer_graph == op.graph
 
   # Create grad functions that compute the gradient of the true/false forward
   # graphs. These functions will capture tensors from the forward pass
@@ -147,15 +151,16 @@ def _IfGrad(op, *grads):  # pylint: disable=invalid-name
   assert ([t.dtype for t in true_grad_graph.outputs] ==
           [t.dtype for t in false_grad_graph.outputs])
 
-  # Match up the captured grad function inputs with outputs of 'op' and other
-  # external tensors.
-  true_grad_inputs = _get_grad_inputs(op, true_graph, true_grad_graph)
-  false_grad_inputs = _get_grad_inputs(op, false_graph, false_grad_graph)
+  # Resolve references to forward graph tensors in grad graphs and ensure
+  # they are in-scope, i.e., belong to one of outer graphs of the grad graph.
+  true_grad_extra_inputs = _resolve_grad_inputs(true_graph, true_grad_graph)
+  false_grad_extra_inputs = _resolve_grad_inputs(false_graph, false_grad_graph)
 
   # Make the inputs to true_grad_graph and false_grad_graph match. Note that
   # this modifies true_grad_graph and false_grad_graph.
   grad_inputs = _make_inputs_match(true_grad_graph, false_grad_graph,
-                                   true_grad_inputs, false_grad_inputs)
+                                   true_grad_extra_inputs,
+                                   false_grad_extra_inputs)
 
   # Add all intermediate tensors as function outputs so they're available for
   # higher-order gradient computations.
@@ -199,11 +204,20 @@ def _get_func_graphs(if_op):
     input_shapes = [t.shape for t in extra_inputs]
     func_name = if_op.get_attr(branch_name).name
     fdef = if_op.graph._get_function(func_name).definition
-    func_graph = _function_def_to_graph.function_def_to_graph(
-        fdef, input_shapes)
+    # `if_op.graph` may not be the same as `ops.get_default_graph()` e.g.
+    # in the case of nested if ops or when the gradient is being computed
+    # from inside a Defun. We build the `func_graph` with `if_op.graph` as its
+    # `outer_graph`. This resembles how the `_FuncGraph` was built in the
+    # forward pass. We need this so that we can resolve references to tensors
+    # in `func_graph` from its gradient graph in `_resolve_grad_inputs`.
+    with if_op.graph.as_default():
+      func_graph = _function_def_to_graph.function_def_to_graph(
+          fdef, input_shapes)
     func_graph.extra_inputs = extra_inputs
     func_graph.extra_args = func_graph.inputs
     func_graph._captured = dict(zip(extra_inputs, func_graph.inputs))
+    # Set the if op so that the gradient code can use it.
+    func_graph._if = if_op
     return func_graph
 
   return (_get_func_graph_for_branch("then_branch"),
@@ -240,7 +254,7 @@ def _grad_fn(func_graph, grads):
   # Build the gradient graph. Note that this builds the gradient computation of
   # func_graph in the current graph, which requires capturing tensors from
   # func_graph. The captured func_graph tensors are resolved to external tensors
-  # in _get_grad_inputs.
+  # in _resolve_grad_inputs.
   result = _gradients_impl._GradientsHelper(
       ys, func_graph.inputs, grad_ys=grad_ys,
       src_graph=func_graph)
@@ -261,43 +275,49 @@ def _create_grad_func(func_graph, grads, name):
                                            [], [], name)
 
 
-def _get_grad_inputs(if_op, cond_graph, grad_graph):
-  """Returns the tensors we should pass to grad_graph.
+def _resolve_grad_inputs(cond_graph, grad_graph):
+  """Returns the tensors to pass as `extra_inputs` to `grad_graph`.
 
-  This method handles tensors captured from cond_graph in grad_graph. It
-  converts these to suitable input tensors from the outer graph.
+  The `grad_graph` may have external references to
+  1. Its outer graph containing the input gradients. These references are kept
+     as is.
+  2. Tensors in the forward pass graph. These tensors may not be "live"
+     when the gradient is being computed. We replace such references by their
+     corresponding tensor in the least common ancestor graph of `grad_graph` and
+     `cond_graph`. Since we export intermediate tensors for all branch
+     functions, this is always possible.
 
   Args:
-    if_op: Operation. The forward-pass If op that uses cond_graph.
     cond_graph: function._FuncGraph. The forward-pass function.
     grad_graph: function._FuncGraph. The gradients function.
 
   Returns:
     A list of inputs tensors to be passed to grad_graph.
   """
-  inputs = []
-
-  # Maps placeholders in cond_graph -> input tensor in outer graph.
-  forward_input_map = {v: k for k, v in cond_graph._captured.items()}
+  new_extra_inputs = []
 
   for t in grad_graph.extra_inputs:
-    if t.graph == ops.get_default_graph():
-      # t is in the outer graph (e.g. one of the input gradients).
-      inputs.append(t)
-    elif t in forward_input_map:
-      # t is an input placeholder in cond_graph. Get the corresponding input
-      # tensor in the outer graph.
-      assert t.graph == cond_graph
-      assert forward_input_map[t].graph == ops.get_default_graph()
-      inputs.append(forward_input_map[t])
-    else:
-      # t is an intermediate value in cond_graph. Get the corresponding output
-      # of 'if_op' (note that all intermediate values are outputs).
-      assert t.graph == cond_graph
-      output_idx = cond_graph.outputs.index(t)
-      inputs.append(if_op.outputs[output_idx])
-
-  return inputs
+    if t.graph != grad_graph._outer_graph:
+      # `t` is a tensor in `cond_graph` or one of its ancestors. We bubble this
+      # tensor to the least common ancestor of the `cond_graph` and
+      # `grad_graph` so that it is "in-scope" for `grad_graph`.
+      # TODO(srbs): `_is_ancestor` calls may be expensive. Compute the least
+      # common ancestor once and re-use.
+      assert _is_ancestor(cond_graph, t.graph)
+      while not _is_ancestor(grad_graph, t.graph):
+        assert isinstance(t.graph, _function._FuncGraph)
+        if t in t.graph.extra_args:
+          # TODO(srbs): Consider building a map of extra_args -> extra_inputs.
+          # instead of searching for `t` twice.
+          t = t.graph.extra_inputs[t.graph.extra_args.index(t)]
+        else:
+          # Note: All intermediate tensors are output by the If op.
+          # TODO(srbs): .index() calls may be expensive. Optimize.
+          t = t.graph._if.outputs[t.graph.outputs.index(t)]
+      assert _is_ancestor(grad_graph, t.graph)
+    new_extra_inputs.append(t)
+
+  return new_extra_inputs
 
 
 def _create_new_tf_function(func_graph):
@@ -326,7 +346,8 @@ def _create_new_tf_function(func_graph):
   # a new TF_Function that we add to the graph.
   fdef = _function.function_def_from_tf_function(c_func)
   defined_func = _function._from_definition(fdef)
-  defined_func.add_to_graph(ops.get_default_graph())
+  defined_func._sub_functions = func_graph._functions
+  defined_func.add_to_graph(func_graph._outer_graph)
 
   return func_graph.name
 
@@ -389,7 +410,8 @@ def _pad_params(true_graph, false_graph, true_params, false_params):
   return new_true_params, new_false_inputs
 
 
-def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs):
+def _make_inputs_match(true_graph, false_graph, true_extra_inputs,
+                       false_extra_inputs):
   """Modifies true_graph and false_graph so they have the same input signature.
 
   This method reorders and/or adds parameters to true_graph and false_graph so
@@ -400,9 +422,9 @@ def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs):
   Args:
     true_graph: function._FuncGraph
     false_graph: function._FuncGraph
-    true_inputs: a list of Tensors in the outer graph. The inputs for
+    true_extra_inputs: a list of Tensors in the outer graph. The inputs for
       true_graph.
-    false_inputs: a list of Tensors in the outer graph. The inputs for
+    false_extra_inputs: a list of Tensors in the outer graph. The inputs for
       false_graph.
 
   Returns:
@@ -411,12 +433,12 @@ def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs):
     false_inputs.
   """
   shared_inputs, true_only_inputs, false_only_inputs = _separate_unique_inputs(
-      true_inputs, false_inputs)
+      true_extra_inputs, false_extra_inputs)
 
   new_inputs = shared_inputs + true_only_inputs + false_only_inputs
 
-  true_input_to_param = dict(zip(true_inputs, true_graph.inputs))
-  false_input_to_param = dict(zip(false_inputs, false_graph.inputs))
+  true_input_to_param = dict(zip(true_extra_inputs, true_graph.inputs))
+  false_input_to_param = dict(zip(false_extra_inputs, false_graph.inputs))
 
   true_graph.inputs = (
       [true_input_to_param[t] for t in shared_inputs] +
@@ -432,6 +454,9 @@ def _make_inputs_match(true_graph, false_graph, true_inputs, false_inputs):
   true_graph.extra_inputs = new_inputs
   false_graph.extra_inputs = new_inputs
 
+  true_graph.extra_args = true_graph.inputs
+  false_graph.extra_args = false_graph.inputs
+
   true_graph._captured = dict(zip(new_inputs, true_graph.inputs))
   false_graph._captured = dict(zip(new_inputs, false_graph.inputs))
 
@@ -454,14 +479,30 @@ def _create_dummy_params(func_graph, template_tensors):
 
 
 def _get_grad_fn_name(func_graph):
-  """Returns a unique name to use for the grad function of `func_graph`."""
+  """Returns a unique name to use for the grad function of `func_graph`.
+
+  Ensures this name is unique in the entire hierarchy.
+
+  Args:
+    func_graph: The _FuncGraph.
+
+  Returns:
+    A string, the name to use for the gradient function.
+  """
   name = "%s_grad" % func_graph.name
 
   base_name = name
   counter = 1
-  if ops.get_default_graph()._is_function(name):
-    name = "%s_%s" % (base_name, counter)
-    counter += 1
+  has_conflict = True
+  while has_conflict:
+    curr_graph = func_graph._outer_graph
+    has_conflict = curr_graph._is_function(name)
+    while not has_conflict and isinstance(curr_graph, _function._FuncGraph):
+      curr_graph = curr_graph._outer_graph
+      has_conflict = curr_graph._is_function(name)
+    if has_conflict:
+      name = "%s_%s" % (base_name, counter)
+      counter += 1
 
   return name
 
@@ -477,3 +518,11 @@ def _check_same_outputs(true_graph, false_graph):
         "arguments, got:\n"
         "  true_fn: %s\n"
         "  false_fn: %s" % (true_output_types, false_output_types))
+
+
+def _is_ancestor(graph, maybe_ancestor):
+  if maybe_ancestor == graph:
+    return True
+  if isinstance(graph, _function._FuncGraph):
+    return _is_ancestor(graph._outer_graph, maybe_ancestor)
+  return False
-- 
cgit v1.2.3


From 6533c5a8ade658568a82c3c7bb9d1368a641c0a3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 01:44:48 -0700
Subject: Describe what happens on case of duplicate indices on scatter_nd.

PiperOrigin-RevId: 205367181
---
 tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt
index 58753a651a..ad1c527b01 100644
--- a/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ScatterNd.pbtxt
@@ -32,8 +32,12 @@ slices within a tensor (initially zero for numeric, empty for string) of
 the given `shape` according to indices.  This operator is the inverse of the
 @{tf.gather_nd} operator which extracts values or slices from a given tensor.
 
+If `indices` contains duplicates, then their updates are accumulated (summed).
+
 **WARNING**: The order in which updates are applied is nondeterministic, so the
-output will be nondeterministic if `indices` contains duplicates.
+output will be nondeterministic if `indices` contains duplicates -- because
+of some numerical approximation issues, numbers summed in different order
+may yield different results.
 
 `indices` is an integer tensor containing indices into a new tensor of shape
 `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
-- 
cgit v1.2.3


From a641057c719aa95175a9ad3f9e26044f0c31416e Mon Sep 17 00:00:00 2001
From: Peter Hawkins 
Date: Fri, 20 Jul 2018 05:13:01 -0700
Subject: [TPU] Delete attr_scope that does nothing, now that the forward
 compatibility window has expired.

PiperOrigin-RevId: 205383196
---
 tensorflow/contrib/tpu/python/tpu/tpu.py | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index 7216626a58..06885bbc25 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -633,23 +633,14 @@ def split_compile_and_replicate(computation,
     with tpu_function.tpu_shard_context(
         num_replicas), ops.control_dependencies([metadata]):
 
-      # For backward compatibility reasons, we tag replicated inputs with the
-      # _tpu_replicated_input attribute. This does nothing and exists only for
-      # backward compatibility.
-      # TODO(phawkins): delete the attr_scope after 6/28/2018.
-      # pylint: disable=protected-access
-      with graph._attr_scope({
-          "_tpu_replicated_input": attr_value_pb2.AttrValue(b=True)
-      }):
-        # Add identity ops so even unused inputs are "consumed" by the
-        # computation. This is to avoid orphaned TPUReplicatedInput nodes.
-        # TODO(phawkins): consider instead pruning unused TPUReplicatedInput
-        # and eliding trivial TPUReplicatedInput/TPUReplicatedOutput pairs.
-        computation_inputs = [
-            array_ops.identity(x, name="replicated_input_{}".format(i))
-            for i, x in enumerate(computation_inputs)
-        ]
-      # pylint: enable=protected-access
+      # Add identity ops so even unused inputs are "consumed" by the
+      # computation. This is to avoid orphaned TPUReplicatedInput nodes.
+      # TODO(phawkins): consider instead pruning unused TPUReplicatedInput
+      # and eliding trivial TPUReplicatedInput/TPUReplicatedOutput pairs.
+      computation_inputs = [
+          array_ops.identity(x, name="replicated_input_{}".format(i))
+          for i, x in enumerate(computation_inputs)
+      ]
 
       # If there is an infeed queue, adds the dequeued values to the
       # computation's inputs.
-- 
cgit v1.2.3


From aede46743b0256ef382f23b0db0370fe0777116d Mon Sep 17 00:00:00 2001
From: Joe Yearsley 
Date: Fri, 20 Jul 2018 15:00:29 +0100
Subject: Update fold_old_batch_norms.cc

Fixed my previous fix.
---
 tensorflow/tools/graph_transforms/fold_old_batch_norms.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
index f1d361e07d..156636ab82 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
@@ -159,7 +159,7 @@ Status FuseScaleOffsetToConvWeights(const std::vector& scale_values,
   NodeDef bias_add_node;
   bias_add_node.set_op("BiasAdd");
   bias_add_node.set_name(conv_output_name);
-  if (!conv_node.attr().count("data_format")) {
+  if (conv_node.attr().count("data_format") > 0) {
     CopyNodeAttr(conv_node, "data_format", "data_format", &bias_add_node);
   }
   CopyNodeAttr(conv_node, "T", "T", &bias_add_node);
-- 
cgit v1.2.3


From 8d14663dbe9446ba50a36f64aaecfb5c06ea26d3 Mon Sep 17 00:00:00 2001
From: Jianwei Xie 
Date: Fri, 20 Jul 2018 08:52:47 -0700
Subject: Fixed  keras_support related dependency in mnist example.

PiperOrigin-RevId: 205403669
---
 tensorflow/contrib/tpu/BUILD | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index ef6c752851..14e4e9cc2b 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -165,6 +165,17 @@ py_library(
         "python/tpu/keras_support.py",
     ],
     srcs_version = "PY2AND3",
+    visibility = [
+        "//cloud/vmm/testing/tests/tpu:__subpackages__",
+        "//learning/brain:__subpackages__",
+        # TODO(b/111651964): Clean special visibility for keras_support.
+        #
+        # Note: If you are an end user, please do not add your project to this
+        # visibility. This feature is experimental, and will be made public
+        # when ready.
+        "//third_party/cloud_tpu/models/keras:__subpackages__",
+        "//tensorflow:__subpackages__",
+    ],
     deps = [
         ":tpu_lib",
         ":tpu_py",
-- 
cgit v1.2.3


From 2ff8c85dfca8afb2a4129e8fa86802bd5f25a1c6 Mon Sep 17 00:00:00 2001
From: Asim Shankar 
Date: Fri, 20 Jul 2018 09:28:37 -0700
Subject: [eager]: Correctly handle operation arguments of mixed types in the
 slow path.

Consider the following:

import tensorflow as tf
tf.enable_eager_execution()
x = tf.Variable(1.0)
tf.Print(x, ["foo", x])

Prior to this commit, this snippet would fail with an error:

ValueError: exceptions.TypeError: object of type 'ResourceVariable' has no len()

raised from the call to ops.EagerTensor in convert_to_mixed_eager_tensors.

With this commit, the tf.Print call works correctly.
Note that convert_to_mixed_eager_tensors is only called in the slow path of operation execution (i.e., when TFE_Py_FastPathExecute fails). Which happens rarely (e.g., when mixing primitive string and EagerTensor/ResourceVariable arguments).

PiperOrigin-RevId: 205408407
---
 tensorflow/python/eager/core_test.py | 8 ++++++++
 tensorflow/python/eager/execute.py   | 6 +-----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py
index 3fabe7060e..cc765725a4 100644
--- a/tensorflow/python/eager/core_test.py
+++ b/tensorflow/python/eager/core_test.py
@@ -610,6 +610,14 @@ class TFETest(test_util.TensorFlowTestCase):
       self.assertEquals(typ, dtypes.float32)
       self.assertIsInstance(t, ops.EagerTensor)
 
+  def testConvertMixedEagerTensorsWithVariables(self):
+    var = resource_variable_ops.ResourceVariable(1.0)
+    types, tensors = execute_lib.convert_to_mixed_eager_tensors(
+        ['foo', var], context.context())
+    self.assertAllEqual([dtypes.string, dtypes.float32], types)
+    for t in tensors:
+      self.assertIsInstance(t, ops.EagerTensor)
+
 
 class SendRecvTest(test_util.TensorFlowTestCase):
 
diff --git a/tensorflow/python/eager/execute.py b/tensorflow/python/eager/execute.py
index 2ff5b8d8f4..f9b8d2cb5d 100644
--- a/tensorflow/python/eager/execute.py
+++ b/tensorflow/python/eager/execute.py
@@ -198,11 +198,7 @@ def args_to_matching_eager(l, ctx, default_dtype=None):
 
 
 def convert_to_mixed_eager_tensors(values, ctx):
-  v = [
-      t if isinstance(t, ops.EagerTensor) else ops.EagerTensor(
-          t, context=ctx._handle, device=ctx.device_name)  # pylint: disable=protected-access
-      for t in values
-  ]
+  v = [ops.internal_convert_to_tensor(t, ctx=ctx) for t in values]
   types = [t._datatype_enum() for t in v]  # pylint: disable=protected-access
   return types, v
 
-- 
cgit v1.2.3


From 75ca1d8df21cd4c7904ceecb12c0cfc268da361f Mon Sep 17 00:00:00 2001
From: Dan Moldovan 
Date: Fri, 20 Jul 2018 09:33:03 -0700
Subject: A notebook containing simple algorithms.

PiperOrigin-RevId: 205408982
---
 .../autograph/examples/notebooks/algorithms.ipynb  | 1512 ++++++++++++++++++++
 1 file changed, 1512 insertions(+)
 create mode 100644 tensorflow/contrib/autograph/examples/notebooks/algorithms.ipynb

diff --git a/tensorflow/contrib/autograph/examples/notebooks/algorithms.ipynb b/tensorflow/contrib/autograph/examples/notebooks/algorithms.ipynb
new file mode 100644
index 0000000000..bf824e2760
--- /dev/null
+++ b/tensorflow/contrib/autograph/examples/notebooks/algorithms.ipynb
@@ -0,0 +1,1512 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "b9R-4ezU3NH0"
+      },
+      "source": [
+        "## AutoGraph: examples of simple algorithms\n",
+        "\n",
+        "This notebook shows how you can use AutoGraph to compile simple algorithms and run them in TensorFlow.\n",
+        "\n",
+        "It requires the nightly build of TensorFlow, which is installed below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "TuWj26KWz1fZ"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -U -q tf-nightly"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "3kudk1elq0Gh"
+      },
+      "source": [
+        "### Fibonacci numbers\n",
+        "\n",
+        "https://en.wikipedia.org/wiki/Fibonacci_number"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 197
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 7512,
+          "status": "ok",
+          "timestamp": 1532101577266,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "H7olFlMXqrHe",
+        "outputId": "472dbfe0-9449-4f93-e908-1a0785188a92"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "0 :  1\n",
+            "1 :  2\n",
+            "2 :  3\n",
+            "3 :  5\n",
+            "4 :  8\n",
+            "5 :  13\n",
+            "6 :  21\n",
+            "7 :  34\n",
+            "8 :  55\n",
+            "9 :  89\n"
+          ]
+        }
+      ],
+      "source": [
+        "import tensorflow as tf\n",
+        "from tensorflow.contrib import autograph as ag\n",
+        "\n",
+        "\n",
+        "def fib(n):\n",
+        "  f1 = 0\n",
+        "  f2 = 1\n",
+        "  for i in range(n):\n",
+        "    tmp = f2\n",
+        "    f2 = f2 + f1\n",
+        "    f1 = tmp\n",
+        "    print(i, ': ', f2)\n",
+        "  return f2\n",
+        "\n",
+        "\n",
+        "with tf.Graph().as_default():\n",
+        "  final_fib = ag.to_graph(fib)(tf.constant(10))\n",
+        "  with tf.Session() as sess:\n",
+        "    sess.run(final_fib)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "p8zZyj-tq4K3"
+      },
+      "source": [
+        "#### Generated code"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 541
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 103,
+          "status": "ok",
+          "timestamp": 1532101577412,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "UeWjK8rHq6Cj",
+        "outputId": "73ece895-12fb-489a-e52c-032945d7ed7a"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "from __future__ import print_function\n",
+            "import tensorflow as tf\n",
+            "\n",
+            "def tf__fib(n):\n",
+            "  try:\n",
+            "    with tf.name_scope('fib'):\n",
+            "      f1 = 0\n",
+            "      f2 = 1\n",
+            "\n",
+            "      def extra_test(f1_1, f2_1):\n",
+            "        with tf.name_scope('extra_test'):\n",
+            "          return True\n",
+            "\n",
+            "      def loop_body(i, f1_1, f2_1):\n",
+            "        with tf.name_scope('loop_body'):\n",
+            "          tmp = f2_1\n",
+            "          f2_1 = f2_1 + f1_1\n",
+            "          f1_1 = tmp\n",
+            "          with ag__.utils.control_dependency_on_returns(ag__.utils.\n",
+            "              dynamic_print(i, ': ', f2_1)):\n",
+            "            f2, i_1 = ag__.utils.alias_tensors(f2_1, i)\n",
+            "            return f1_1, f2\n",
+            "      f1, f2 = ag__.for_stmt(ag__.utils.dynamic_builtin(range, n),\n",
+            "          extra_test, loop_body, (f1, f2))\n",
+            "      return f2\n",
+            "  except:\n",
+            "    ag__.rewrite_graph_construction_error(ag_source_map__)\n",
+            "\n"
+          ]
+        }
+      ],
+      "source": [
+        "print(ag.to_code(fib))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "eIfVy6ZTrFEH"
+      },
+      "source": [
+        "### Fizz Buzz\n",
+        "\n",
+        "https://en.wikipedia.org/wiki/Fizz_buzz"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 125
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 233,
+          "status": "ok",
+          "timestamp": 1532101577681,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "33CAheYsrEQ7",
+        "outputId": "82a493ee-15b5-419d-8c9c-5f4159090a05"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Buzz\n",
+            "11\n",
+            "Fizz\n",
+            "13\n",
+            "14\n",
+            "FizzBuzz\n"
+          ]
+        }
+      ],
+      "source": [
+        "import tensorflow as tf\n",
+        "from tensorflow.contrib import autograph as ag\n",
+        "\n",
+        "def fizzbuzz(i, n):\n",
+        "  while i \u003c n:\n",
+        "    msg = ''\n",
+        "    if i % 3 == 0:\n",
+        "      msg += 'Fizz'\n",
+        "    if i % 5 == 0:\n",
+        "      msg += 'Buzz'\n",
+        "    if msg == '':\n",
+        "      msg = tf.as_string(i)\n",
+        "    print(msg)\n",
+        "    i += 1\n",
+        "  return i\n",
+        "\n",
+        "with tf.Graph().as_default():\n",
+        "  final_i = ag.to_graph(fizzbuzz)(tf.constant(10), tf.constant(16))\n",
+        "  with tf.Session() as sess:\n",
+        "    sess.run(final_i)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "Lkq3DBGOv3fA"
+      },
+      "source": [
+        "#### Generated code"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 1081
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 289,
+          "status": "ok",
+          "timestamp": 1532101578003,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "bBhFIIaZrxvx",
+        "outputId": "d076a7ea-e643-4689-f90a-57f5d086dedc"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "from __future__ import print_function\n",
+            "import tensorflow as tf\n",
+            "\n",
+            "def tf__fizzbuzz(i, n):\n",
+            "  try:\n",
+            "    with tf.name_scope('fizzbuzz'):\n",
+            "\n",
+            "      def loop_test(i_1):\n",
+            "        with tf.name_scope('loop_test'):\n",
+            "          return tf.less(i_1, n)\n",
+            "\n",
+            "      def loop_body(i_1):\n",
+            "        with tf.name_scope('loop_body'):\n",
+            "          msg = ''\n",
+            "\n",
+            "          def if_true():\n",
+            "            with tf.name_scope('if_true'):\n",
+            "              msg_1, = msg,\n",
+            "              msg_1 += 'Fizz'\n",
+            "              return msg_1,\n",
+            "\n",
+            "          def if_false():\n",
+            "            with tf.name_scope('if_false'):\n",
+            "              return msg,\n",
+            "          msg = ag__.utils.run_cond(tf.equal(i_1 % 3, 0), if_true, if_false)\n",
+            "\n",
+            "          def if_true_1():\n",
+            "            with tf.name_scope('if_true_1'):\n",
+            "              msg_2, = msg,\n",
+            "              msg_2 += 'Buzz'\n",
+            "              return msg_2,\n",
+            "\n",
+            "          def if_false_1():\n",
+            "            with tf.name_scope('if_false_1'):\n",
+            "              return msg,\n",
+            "          msg = ag__.utils.run_cond(tf.equal(i_1 % 5, 0), if_true_1, if_false_1\n",
+            "              )\n",
+            "\n",
+            "          def if_true_2():\n",
+            "            with tf.name_scope('if_true_2'):\n",
+            "              msg_3, = msg,\n",
+            "              msg_3 = tf.as_string(i_1)\n",
+            "              return msg_3,\n",
+            "\n",
+            "          def if_false_2():\n",
+            "            with tf.name_scope('if_false_2'):\n",
+            "              return msg,\n",
+            "          msg = ag__.utils.run_cond(tf.equal(msg, ''), if_true_2, if_false_2)\n",
+            "          with ag__.utils.control_dependency_on_returns(ag__.utils.\n",
+            "              dynamic_print(msg)):\n",
+            "            msg_4 = ag__.utils.alias_tensors(msg)\n",
+            "            i_1 += 1\n",
+            "            return i_1,\n",
+            "      i = ag__.while_stmt(loop_test, loop_body, (i,), (tf, n, ag__, i))\n",
+            "      return i\n",
+            "  except:\n",
+            "    ag__.rewrite_graph_construction_error(ag_source_map__)\n",
+            "\n"
+          ]
+        }
+      ],
+      "source": [
+        "print(ag.to_code(fizzbuzz))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "BNRtprSvwJgk"
+      },
+      "source": [
+        "### Conway's Game of Life\n",
+        "\n",
+        "https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "r8_0ioEuAI-a"
+      },
+      "source": [
+        "#### Testing boilerplate"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "7moIlf8VABkl"
+      },
+      "outputs": [],
+      "source": [
+        "NUM_STEPS = 1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "QlEvfIQPAYF5"
+      },
+      "source": [
+        "#### Game of Life for AutoGraph"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "5pCK2qQSAAK4"
+      },
+      "outputs": [],
+      "source": [
+        "#@test {\"skip\": true} \n",
+        "NUM_STEPS = 100"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 8,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 308
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 14892,
+          "status": "ok",
+          "timestamp": 1532101593030,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "hC3qMqryPDHS",
+        "outputId": "8405c0e9-e518-41d6-f5bc-e78df6474169"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/html": [
+              "\u003cvideo width=\"432.0\" height=\"288.0\" controls autoplay loop\u003e\n",
+              "  \u003csource type=\"video/mp4\" src=\"data:video/mp4;base64,AAAAHGZ0eXBNNFYgAAACAGlzb21pc28yYXZjMQAAAAhmcmVlAACZUm1kYXQAAAKuBgX//6rcRem9\n",
+              "5tlIt5Ys2CDZI+7veDI2NCAtIGNvcmUgMTQ4IHIyNzk1IGFhYTlhYTggLSBILjI2NC9NUEVHLTQg\n",
+              "QVZDIGNvZGVjIC0gQ29weWxlZnQgMjAwMy0yMDE3IC0gaHR0cDovL3d3dy52aWRlb2xhbi5vcmcv\n",
+              "eDI2NC5odG1sIC0gb3B0aW9uczogY2FiYWM9MSByZWY9MyBkZWJsb2NrPTE6MDowIGFuYWx5c2U9\n",
+              "MHgzOjB4MTEzIG1lPWhleCBzdWJtZT03IHBzeT0xIHBzeV9yZD0xLjAwOjAuMDAgbWl4ZWRfcmVm\n",
+              "PTEgbWVfcmFuZ2U9MTYgY2hyb21hX21lPTEgdHJlbGxpcz0xIDh4OGRjdD0xIGNxbT0wIGRlYWR6\n",
+              "b25lPTIxLDExIGZhc3RfcHNraXA9MSBjaHJvbWFfcXBfb2Zmc2V0PS0yIHRocmVhZHM9OSBsb29r\n",
+              "YWhlYWRfdGhyZWFkcz0xIHNsaWNlZF90aHJlYWRzPTAgbnI9MCBkZWNpbWF0ZT0xIGludGVybGFj\n",
+              "ZWQ9MCBibHVyYXlfY29tcGF0PTAgY29uc3RyYWluZWRfaW50cmE9MCBiZnJhbWVzPTMgYl9weXJh\n",
+              "bWlkPTIgYl9hZGFwdD0xIGJfYmlhcz0wIGRpcmVjdD0xIHdlaWdodGI9MSBvcGVuX2dvcD0wIHdl\n",
+              "aWdodHA9MiBrZXlpbnQ9MjUwIGtleWludF9taW49MTAgc2NlbmVjdXQ9NDAgaW50cmFfcmVmcmVz\n",
+              "aD0wIHJjX2xvb2thaGVhZD00MCByYz1jcmYgbWJ0cmVlPTEgY3JmPTIzLjAgcWNvbXA9MC42MCBx\n",
+              "cG1pbj0wIHFwbWF4PTY5IHFwc3RlcD00IGlwX3JhdGlvPTEuNDAgYXE9MToxLjAwAIAAAAPQZYiE\n",
+              "ABH//veIHzLLafk613IR560urR9Q7kZxXqS9/iAAAAMAFpyZZ6/h5MpYA5/oqv4s2qPbYpW3jfK6\n",
+              "zQ6q7WMrNj7Hy8jZzmBpfHCwAAO1W4riBNsrapcCk+5V1W0XkkFULR4Qe+H3uGA2HgNW0zFAAUgt\n",
+              "W4tdpXv2OEg0Vuy5W5l/xGRmEGKDyeXyrM0S6q/1EKbad0x2mcHseUqNmeOGLy1N3b376XZKZcPY\n",
+              "IXC5F2332tNMj8CwOQiXM9PiCLyCVfZ3rQSkKBTZErkpS5kXUyoJG3FdIqLjRFKEapbUjcW64HIo\n",
+              "BeIbtRyWV9FyZfcTakx2KW3eB4ZI//MDykSe8CRgN76uBEqZFXwO63wmUREhHOb5AdaLV3xyGl/I\n",
+              "RV70rU/3t9t1aq5mFD3hy1aLTAV2U7nG072dyX87F7NgCxZHT2kFxu44fxf6gqVzE3PEbGr5fx9x\n",
+              "7TKXtmY53VP8UaeCd2HJiZ/sd165SutTnfiWvaLuCnmmXGF0AGqbj9S19kgOhTubZIJBydTTqQOV\n",
+              "YRlxbgKn2nzvunv9+NDG0/2ikyyp73W15QClmjyt8dUeynoN8CwtEQ59DdrAPZe4ARZTwWAfsRXw\n",
+              "1vcZ6Gr1nCNWllQw5IyZyxQtXrfc5p4wjPvGaltciG7d3FG1SGk6HDsZy5i/PsnkjRXLUvGbzYp2\n",
+              "2gs7ZSGfSJbEifctcMGeSqhOOYORKy6f/9omoieCVEEkniBXwWZ/eImb3nxF7SFIaBjgG2j9w5ut\n",
+              "BY6zSuQ5zRCdajzJ1loNO0havI8mp5yViAeAlLKYCxeK0Lha1FskL67W1YsARZVZ5EkhqAYEeTNI\n",
+              "M38Og48OXmj6QBN7c1b9uDUTacYEXO88ZQ1gCIREIMnm2Fgkir8pN4gtSeQ12sfOVz5x5KX7sa95\n",
+              "L4LyFQPDrFZcDBr4PWLeEEv8yzk0cYHE97GmAlA6WQ0HlWsS42cnXefvTPXnx4vcq8pbEo/slAuH\n",
+              "IBsrJEN1+aMCc9FNxwUPVbZVaWVjwLY0qh+mNWEaiNGRmacDXrYWw0NjqMPiLiFHacY5oGELRgym\n",
+              "S2mSo6zhsD1wKQ3EUQtwrjKPiDYc/HCqhkVwoWKUdI8xTS60kn4f5UqB0L77Yevh/wt7AnvQKQAq\n",
+              "QAEEevggRl1uigbOBTtscnYRnAj0edW4QExAzdo+RwLWXTzW/l3cBWTrh3ORzZQlxJ8jQTvPLB+f\n",
+              "bLazJZWFQQDcWhuhQ3gYcP1ruNwIroINRIr8px0UOgAhnk6CllxMN6gA5S0YPhFVFKd3n0AAAC9f\n",
+              "vYgISQAAAltBmiRsQR/+tSqC8p1IAOZemTPutEfx0mzK8zG8tdIxonBsDpoLZ+NnIOp4qK6idP1s\n",
+              "vbGvZz/zHM86Bg3q0yx2atmtgoo/Trt3YRy3se4HTjou+tCi7oJt2d7A8vEhVDu33JNJx+WCOgP0\n",
+              "03nVdg9lBs15v/0w7qMc3zqqJXCOy/Whl9aRhcaeOEWcD7uK6mCV8a6MpDJ959xBRfv2i/qFOFbL\n",
+              "Grs58WiGJcq4MQJI+rVWuFN50oiqBgiunfUrRmdviPYpNN11V9pwcOJwssWfIE3agnor/RC7vfLY\n",
+              "YoXzaJjtWLEL92OOaHLZT0j555xfb4FZcoJee+RXovB9IaoDdYRusngtBXPMUvnO+g2Z5Qdo9P8q\n",
+              "Zb8ItBAeHT8IBZAD/Z2nEA6qbxqOBSBtQNW6ZFYLtCTIoP/bLjCDHgtZk3cf+N1CpXs15pUIYWDW\n",
+              "elZtlTkM4w4EJlLdjLZyQPAeaBx/qoLmKyTKAEhm0hU8EcTq00f6fwkWgz2J6GTGtL/vJXgC8u4o\n",
+              "nTnf+Ou7sVJGVaouXxrzx+yGVHEcp/eV4gaFA95rInngQAOZWbA3558nK61JBPZl3NjEv5B9r9pg\n",
+              "2+SYY3wBAUeu2fgAB2+yYGw82pkoJJKpzYWORs6i1vn3GEgUTcwlYsdJcraYC5SnGvqSZhX7KM72\n",
+              "uE1e9bkpvpVyG/mkACn5R4jwX3xc2utCjjZgM101rirIF/7VfDtmJsSTDes+UVhbSr3SeMSI9ixJ\n",
+              "+fVuFZ5bnQPoRIfPc+Erw+K99JiGN+HE98/eq4pPlMY9oCfVPSdNyOAAAAFfQZ5CeId/AUuqOi5D\n",
+              "jlKfxuJGZZ1+rVyomjOIykvxtsjsuCiGtElbraCSFWcn3aIYWLrF3fPovVLcOnroBkiRMsdf5yJA\n",
+              "F87MQuoKeTaGOrxojCCCS64RiHrqNsE+7mfRRUDuB4sAEHFQHxBorgTukPSvrdFr5QDq+BhZj/6H\n",
+              "KN+IutwFWKX3ZX9pO3sI8My78TgRY5AA6FEcT91WcvnMypB/OWXzK6M8fYuhVVWipAZigjVOYhcF\n",
+              "9i6GweQFX9AV9EUQOp2qFbkrT5jceBRFLX6j4JUQ781/UGTekv1fcpCmzlpNpp8GdSeWxRL4gasp\n",
+              "F5uO5KW63rlhYccBo1cFwIN8txHNnwyQNiP00XC0PWDRZfaWSxsACRWrISow71IyUfcL7JNhjTII\n",
+              "rwDYATS0xZ9ep8siFC3JTxg1eNaroYfeI4tbkRHok47Vk+CUOQPuagVBtFMOOcy2OUbw8AWlAAAA\n",
+              "ugGeYXRDfwHM79ghzBo9nMnzfQPPIuvorxBb6AC8F4fYGD/t93kNSKNSEuhUXq9FKGtxnCkxN880\n",
+              "BPb/uTbjLTQVyPNuYlGl/gTlyLcVA/cDoLrl5TvaR/AcSLFE7C/t3kLx0STNibmdAf4TsHWKSblH\n",
+              "VWB4X7oQHrrDdhwIivRgUZf7f63j2XaGB+cbp5aHCCwJoovY51YTqsZZTz70FlSnypPHQBNzif7h\n",
+              "uvZkXhtEzpu9rYMo3YECkgAAAXIBnmNqQ38BDchAitLfY16mYQAQlVmv7062W8KLpIS1/zhS50Ib\n",
+              "b3ERigmkZKZMPaCsAi+zsLcku/gHGHnVZpuCZMFs72gmyuL4JFo6VjWcr5FtBvzIgD26rBNvP73P\n",
+              "nJjl3JImmFHiKjNez/gG3zTuYyCACuJCEYXyuEmzCM13hdCPHKg5GZtso0Z1qk6T1k2oiqF/3RIn\n",
+              "kyjRWuxBlHHmJ46TXULiUY14G+RAGoXI+u/G6muNclld2bq+6Zztuy+5ynaDWNNjuN1Ag9KUIx2F\n",
+              "XwNdepmp52/rOvISNPbMJ0U26OvqplXi+qHTbg8MLpUSIGCY8w9FZ5woLAENgvgu9M79yGlL20e7\n",
+              "ypJ4RMBqHYDpEz6Z+SSjXD8LsJ7VKlwo22A5Yukp1vTp6HHA35nV+PXK09DuRWKKdQUzmXVihF51\n",
+              "/+bB0PEFdoNxGdbbM7WveaCJN8XI7JgQWvw2nPlHX8M5QyPGSJ2HEexumoFrABvRAAAB70GaaEmo\n",
+              "QWiZTAgj//61KoCPNGHq/MxnjqmxxQAEHvTwibmyMZGX3ES9Abh1tMR+/DjR+6dnqRr/VxCl6gEP\n",
+              "wJ/5EYCYfGaGmQYsLOeM3v2SZjdvqQBwrwKk5A/63kFm8fc3QCLe93Mldv3KWXHdFT7/mudSntDc\n",
+              "vJwStG4jgi5LKlWdSrVaAxOmElsF+zWNzaCIQ1dOiZqi3JKj64hOeq1XIWyGvRvh6OLKBpB4rL6W\n",
+              "ugf7H/IPbSQuF5jWV7zL5LhxWiTiI+kAZTUMfO2YOLzmhCUSN9GAmNzgY4D2awYB4V4QTDjI7kdQ\n",
+              "tL+3Pmfl1HVilu7nC9CzQSvWIosiwv4btyHTL7IPT2gusybyNfW8QO133L6KbDhhXSDWUtcIFCgn\n",
+              "QUm36C9hvgGjorpKYr5VnErpJX6fRJm76fFYs8/nt763alyqdcSrqaTOLaf/72Wkkmlwbq3nLOIw\n",
+              "ADFDkkAPwzaM811K11iK/3HaYRT3nEhjJQFk5v4WBXwIVLAZeKdtC8YoGN9K6isN142fOG3s6fm4\n",
+              "J1nMtOEZHIwep8In4slLmHh39qBzhGZO3igiVpgz7u+JMBeFkVHe72vduBjIy+1dqvxL/TPics3s\n",
+              "+alwfTMNQKave1qW+5Uj8jZQTjcLAtKvzoako9VMIOfQUQAAAQpBnoZFESw7/wC9ZU4P+UeGsidW\n",
+              "4n5tFkXmtxppYvKQ+WGj/x3AAdl6+9c9x7N2b/yJykTvVggfpMnFUWtxla4sr1ouwANom+Uf4IBJ\n",
+              "/zXPovndpGdy98nJbZxFU4rrWpr8aI4YmRX65+IGTn756CZWwXKY5DyMgKnDcCtk0HEuoHgdGhh7\n",
+              "1PG8+nue+pE9pBHqiBNWAjPd90qfMtABmMShLoXtUObqYbqXhJvVjjFhKdPS03IF24fu9Z0ax15V\n",
+              "DnkiLmgyOCvJmcdIX70L2ZEECd/hxrSq9JUVjC41OX0F/ayI6GtkPMUuZ2xWkMFo5rqOAo7v0Zlk\n",
+              "ke/79TjeY13FNiowqcbhMwfDuwAAATIBnqV0Q38BDXNpg2t4nJdhAA5ru/5Co2KbB/AnQt7fa959\n",
+              "0crOQgtTxL36jtVyKPmfuQMYuWbJ/7bYTEV8sEjceHvN6B0CSEZzVCjaPLzOQJZMQpQ4K4WKPlGc\n",
+              "lnEwYAC9Dsejj7Fbk2RyCFiJinyU2HOscjUR6fW2jRsAFpVq/PtZDVPvesPG3AqooVaKHp9Ex+Da\n",
+              "AH0OvccSugyDKsRBAEiYR8645aXxbFSzraQsELDsIIr6HRN8F3lUNVBvzNO3mxBhq4th/kgZSjjJ\n",
+              "JZrYmg3UfIUO/jn4xs2XQ9Pa7Uy5K3JhuIQwAOUKDmAMC0p6fgz2on4ceyEcfiCGDPZpPyL3391F\n",
+              "dXID0ctPQ1a+Hk7UcAc9gSDL8CZKz59YyO0ACPjfAKV3Y2dbTAKdWBsUU0EAAAFEAZ6nakN/AItk\n",
+              "aaqbMCcBE0iEIDnEBfRZN0neHQxaz5DPSzK0ZSL640q0AA5jkP0YAYAumNCN0MxJYpWFoQ9r43H0\n",
+              "i9SZLdv1UbgpG3aX6KESZW7AgdlevaBngH/w8xYsqWx5t90zzi7x9VyRYpIAD+XTrxvgBoFILNCs\n",
+              "gd+zDA9uvbAPlLMwG/qFltlwvLokMt344erv3a/C/ySOwZHFzpakInpJ7MQHkmKi1KHZB5KrfqwF\n",
+              "FnglZJwWbe7LtVojTdwQnAksziDNlEWCkMQQJwziY1KYtlXMNX8mZ3MtYR1KNf/CNin7/ys9ZQyx\n",
+              "4Zlk//H5KDc/8O2+JaxH20CAaAABxgSxo+yJal1LnRHYfOQ1TygNueW/rPAA37g/6fLS7mbYKz7k\n",
+              "dsiSiy1mAV7n/qq81UHJPShQSXK+E4Y5XKuXEWG4AAAB8UGarEmoQWyZTAgj//61KoAW7kO9JCjl\n",
+              "XSE6nAngAJVxWWFl/YDS0gZ32xjwUFed4hmI6rj18z16nS3Mz1iMmFblrtaE4zGXS046COODiIwH\n",
+              "QG5lRmcBExMKlnynQruQtA8n/NitzdP/ysLrucGyp5nKV+XyJURULfxk4kwNp0a5TFlJ1fusOOJm\n",
+              "y0hvsvEg+d4Jz3anvWT6M9n5A84CGucNifV+WlN9gI9gs3qSoCZdU/gglcFYM5u8YchzhQFyMKxn\n",
+              "kpfWK2LU7aaZHt6xLbqjuv74523K9/dtrrsFq/LySiv1P9Wk6/6d5RC72z4cyaUq6hMMn4IWWRo0\n",
+              "zJIM1/lSYsWxt5/M1Mkv00Rt8OZvmLxuFfd1BIVlANlpgZ39RYhqqzU6v1HwaW0EudelFBGhr5mf\n",
+              "GaDE05Z8ywp5rN4Qq4D4GNAGD/qgEjtaDDf4ZBAD/TAHBwxfNjm2nPAdbbbIuWSkkv8NK6EMlKqH\n",
+              "mOktd+CB3P6Szd1+HPnUsyQ3659r3XLnoi0cvM4usfW+BgxqT0mgHSgn/F6ajdTNM+a8xJQnT036\n",
+              "7195r0uF5vwi7PIviCQ2E4Vs4Wx80/8tBDEJS4qOY1YJ5aNV1OV82fB3HOimLHd2vU/d4Cv7OBh8\n",
+              "k3gNFcjeBGh+3lQcDCLZrG1mAAAA3kGeykUVLDv/AGVBMHxAlJYGEpFnv2bb0ADrwvVKxe7+SIJI\n",
+              "g0dPJdL0s9Hd2mGX7rpdIiUH9ZgtnBO+m3uPNae/YtN3u2p0kkCez2KiPNqgSoEcHM+ePgq7afkq\n",
+              "0HHTSZl/+QbjsyfbI/0lv1mLAJUd3u7VZPPHSdXK3vwLfAwOe3Nid72slU892DijWVvanzM1IzDQ\n",
+              "XfN6x6GH2qfaLrHePrJTJxXC/RSxcAol7x2JJ5OA8VjN8jXu0yKirBiYqgcdFf9odG8j4bRmE2wD\n",
+              "MG0SKuGrJfd91b6B7hbRUwAAAPYBnul0Q38Ahz7YAbwPIqnkAA5sEIcKo2/sVUP0LEeFOLjKjaet\n",
+              "5YFAjDbL5BIdGqWouG/H8ozoec2ZpUbIZu0ELtG5yXc/5opSZlnqbOpqdTQkLs6gr9dv5GbFvVjS\n",
+              "Os1j9FIMQsdc8pttosNtygWB8gLxr65El6umAZE5CVU9Mc8Xxg/tenmTduGK9Cd7qRDiu1sLYR2f\n",
+              "or3KBMo8ebz5q5EmWucvREbYSziQIIycIwJg9OG+aH+ZUEQbjbfHfaiX7yoxGJGP78aNOHP7GvC+\n",
+              "JwM6DxnSyowUBAqkW8ckgrhet8gYYrt8MIe1MPJQB6sv8hHuAXkAAAFWAZ7rakN/AI9XvmYGr0rf\n",
+              "QEvrPPTQWEAA5ru3wBCXPJiC8OaE25OBvVl2wRXqp61wQU4HxGJCAxkSOz+G3Yzvg36uCK8bPZTq\n",
+              "avaOG/H9WxjsuwAl/bIYJdnyD151CiUZ34aErVIixKJ53oKrLeHr3xLgxuH+y3w5uH5lQRsL0Pmp\n",
+              "0jQItTBkKwlPywxFk55pROuYZWi/h/N19QaFlF7WPobUElLlr+nCH+pVt1nW9/YwVGz/cO8zwmWe\n",
+              "Fb0OnFji7CYSsi9ScC3a50GjUP7IpaY5NAHv33V57bkO/BD6dnreymTbSmQdcj7PAJkvz610fMqn\n",
+              "mDGTMB31oxAIE5eWeH7mBZouSgmtxEamul7sYaTPe7mP6FqNCz0h6wLot/zAFwx9/D2+XB0x8mmS\n",
+              "b086o+gqkoYoHQeQm2Sb3MU1Bz0KHDGo9jCmsBmecxs3oNHV4KaIoLKAAAABrEGa8EmoQWyZTAgj\n",
+              "//61KoAcdmk2P6doyaR4wEHxsIcmssCD5f+3/v8PGtlbWZ+A0oGGFPTAdgmU2TFbrRxlmwUCouNe\n",
+              "8freV7blHDodFImzwP3saA3AZT6NUl7vDGH/tw5n9y8rP4XGnhEXBHK+6jIhoAYc6G1CDX0mqczJ\n",
+              "7tbei5I0YSkDjza4rJSbAF6cRoJQH3s2Q+ggBQR0BfH6N3QlPVwd9YFvP6++J+XrbNU56Pxu6Wey\n",
+              "51asar4AaARXHregTXL4xn/VNt8Ppk2xD3/1jXAVXdqMlS0tYGM/TtrcuTC63Lx21RQtklG6k0xA\n",
+              "eWm6W0oL0KTvxuyegpC2ySp5v6zpSEYvzWR4IYirfT0RYU+jLtX0t4M/L/0k8xOLTHbouoUPD6DN\n",
+              "dYYLYlVX5noJzjCAVCiS21OCcIKqWD/YiU/+dTZpdFFNdHEa/MPvUEq7cJD7ANJ0YUweepq2Eqdh\n",
+              "57SC4Tpg6jyEnFgMaHQLSz1nJNh4lxM1TPouGZ9bmQdDr9WY+nwzRBa+ZLnaqBSYKWSKEs/TNtNZ\n",
+              "ev7d+EnJUf9G9CAmmiSDlRAvAAAAz0GfDkUVLDv/AGU2nAwHHyQlvUxuENDSO8vXFIAPilnMlQWb\n",
+              "nTHwb8wkIo6JKOaIP9blrrNXcWeeQDVprB1Bn//+nbSDHls1apJcUyMHUmojA58P91gutTiF40zp\n",
+              "fDaF096G01gcvpH5Za4+DfUvxQpt/wH5PntJzggww1tLhP1NyH5U2TTgrnA/BevK2aCa9xCuCVgA\n",
+              "JJZF4uqHE//COeWbJ6LIFJPoadxAxbrAcxPQQHMzEG5G5S3Yfd+YJBLrdO35JvVrsUTYO4AfvJeC\n",
+              "zwAAAe8Bny10Q38Aj03WPPyvISnWAC7KM5WfLH925SBeAKcvJaYOa5WZCzX9H5nU/7qAFTCgAnl3\n",
+              "rAoSnKk1337XDAnLfPYAAOSIcqQwF++e4HouwNVAWCEsVyl7Y6DnBaBT2mD1H8560KoMvm3kKNNC\n",
+              "oxFCc4BdAIXk45JUbGFNGYAjCbBbJInMjwa41HA404yKnJG7rNXdBctnsSL/36UoXvVx3J2tGX84\n",
+              "+FHk7e72CsAyB49ajd62idmFQji9Jj1GaiqtCIjWs5o6Mz8s5QfrvipNYYD0YZ7gBBGm4AEz17d8\n",
+              "isscgsp4QI2odbuEJDq1nfJbW6+1HGcN1XfDC1Xfa5IptM5UYHm5zIT4rSPBIDE6l8/NhVxlFP21\n",
+              "JPQ0DZxnZFvxIBznQbqkhaGZjMafgFoRzC9Nl17x+K6e75RlplRZtXaUIbjAUFBJIQPkoIrT6/O9\n",
+              "NtkAmnl8qqUC1RktW/RjiJqOyRTTITHqNKvKy/0gb88xEvvGPgzcSs2KpkbHJWmCGIlSWEkuqcCE\n",
+              "jBn3Y8XOQxMUxEYeLPJ/9s/F2fT5NAnko+RFlv75fWLekZZP2s17yJ5ccFGhZyrkGX6u7xXK7N8G\n",
+              "Qlz8qfOHvgMQrlB8p4j7qtnPgBPf8mcsM295CuAZxkK+sut074W+0hM24VMAAADaAZ8vakN/AI9G\n",
+              "UrhSy/Rrhc/LGXguupji5cAHC2DVoxU1gWUkKeMT366GcmuxH5O8lBZJeHl8r2KNT0EaVARyW7pN\n",
+              "L4uNsKKl/WAzLJ1OZWTQf4NaAfodQGO9KzZS0j6oGvr/urKiQwbP44Tv//glYQyyCFeq+8nnrHBj\n",
+              "aACu2w1otySh0DYMX412uY6EYcx3GtQaRpNPiKQniWdVV2KH48fVxDy0uLS0SmCZEAWLVNvtWqO+\n",
+              "q2OwCBr1m50s0i8eRTlSP9xoKtxWC4ZqL77eAW3kYEBJOAywYUAAAAH6QZs0SahBbJlMCCP//rUq\n",
+              "gBY3NzYDjVIwwAKbp/vtZn3NtK6t0V/4sA0MV4ijJVoTZ+e36T0E9eQ0LOyzsqR0ULZJUDRy41oM\n",
+              "RdsBwM4wyEJC67daWmuDEXKhZo862uqAH8A0QJ5u5RKBPFpngChYYJdWzP3onEWImG8Yryy/SXt0\n",
+              "jQ5te76AagLius72bzwZ4AZfLm/04ID6oXhPwqkf1cNsu4/kIt7oCOETiL+lzwHLEnEsdPSz3DxD\n",
+              "uLGkH8o6jHofDxEXcB6cOS43aUxGKPYPtHCj2gw6RzcRoX5lD5mwqtoCTxk6N8TxyipSUyNnbA2b\n",
+              "G5NuBUVLHTce3QKY3SdkbyH/wzdOpT3YHUE+FYQwMKCF6SMyMBxp2gI9k4yUZYljUiekF2XIFkfv\n",
+              "TFy1RUmikOycLKkTYTreTarsMD5JfjZ2FJWrroj/YX+uNeGtKNZl9Zyt+k8u4Htq1bPYEjCrLHds\n",
+              "qeIuFWmvxTYEQblStjDXmWfITtxy8KvOgn9iV+KlidrnVhlE7Dz30fuHXxxFZvIzhgU9uv6sSC7T\n",
+              "vZuGMsKGBGTYmSe0P9hLI2VyM/8GUWwG/AITiU4a7OVDjUNRPaiIEt8jt2oImPIY8qcrJ82CVd+P\n",
+              "mSjoppoeHUTHmeo+koGqjhwT7ueVHNT5VZ4yuGKEDdFfEIkAAAEMQZ9SRRUsO/8AYrbCELHs5dcg\n",
+              "AyOPuRHZUWtdXLx9XaNQixO/8Cc4Q2MgEa/wKETsHiR8C1XOv7rI3JB0rg46JfjEArbHaTHmANKo\n",
+              "+czcI/sIduYNFOE3TvObMh/KtGpZSdF+qnDDtY8zD+7RQUdzmkG5zeDj3u4Vq+f3qnKCwgbU+U0R\n",
+              "dQR9Q60wXqL03p/iYVxkI8jJqvkECuxT7efJI+5rmzyP1yn+WKY2EsjjB7bwwVfe6RxBmzR9Ed/9\n",
+              "CA95ILUJxNg4HsmCO2Ko+MqZAH3wMlG18kUm2ogL3cKIkVXogjofyKhbsSpKLpFFk71DzB6NrY/3\n",
+              "HfknWM2yn9yeQB/joufGEf/bvMAS8QAAAN4Bn3F0Q38Ado97WJWiqN4XS53kTA5YWsnJBdebpf+9\n",
+              "lcN5zPySAC6fH/XzBsBKbxdm4pTiPFVrmGXyhaRiB6dxtlwj8MyI40Do8AXHq41BAunk4K4PTgzR\n",
+              "rFycWqaL549wB2C5jNCLXlq6Tuytik3ijlMSkx9noeIG2Lc83eWkRkQieksQSO4xI1tzzkdqaNhG\n",
+              "ExZARu3MauZwrBopslb/ZLdR5ZS0G6p8o9DD5cphJjxJoSV/70/0Gr+woS8Zj0JpVvvpygE5bXQp\n",
+              "/YBCqjmq4uOCyt9SvCzPelUEwXEAAAGyAZ9zakN/AHZ6+HiwE6fxvgA5rqP9zmI+FShvhJS43N4N\n",
+              "sc5a7qq0DK7DHadXkQxf+APmeqLrIGM9X5aCQgeyxdoAlcQoyNsm6ol85w5z6JV8A3YntmCae+s8\n",
+              "+8/Yheg1ctJWrSharoeypUyemQeq9Rm5cIkSOS9Ej0hbIHyFhPQW6K3SawgMNVKQ0s1BpJvXDQSY\n",
+              "x3jIEdIgEtwe7zce/DjcO3RNN3g+SlPoM7cl0qJbM44NIDG9JGXcwVrY/YKNrpChX0yegP2ZHDI1\n",
+              "MzOs5eWP/2l5loJrLid2mK4Qhw6EGFrIadsV8rSjzgHRNuzJ4U3JdubidEobU0ehkU0P6MYRK/XM\n",
+              "58mVywGbsw6LPu56h1S4w3zHGYMd1zPKOsnCUhaRfrSZTxvjerNQ22prVPqBstk4JgHdnSScrwGw\n",
+              "eQcqvIw7gKhonPDKM4fJtO4n2EsI5Cd0iGMjmgPw/PU3FL8ZP3QbYLMwZ81Wd7BLLBDf+ngKiFIe\n",
+              "it4neyhhaE/a71b8TxeM/ZrgH9+D76dlgPI1ZJW6CCVyIs6Y5gK2plkcgRYa0MwWF+1A6zPtBEgA\n",
+              "LOAAAAIIQZt4SahBbJlMCCP//rUqgBY9we30eRuAA2kMf/9/gX2SHKs8Uq31+W7Vx4LugxILnhMT\n",
+              "6icG5WQzdpL8yjIXjBq99nVaYweUdJE3LrdOpsVxNJ3kODVBkposYOoRuOMi/SNhcjrJwShp6ljG\n",
+              "Qs7tSeRJSYDkvm+SI2ckjbManbEesw6wo2ZffuryaLuWkU9SNALC+2QbPJD4bFy7sTmB9+6VOdMm\n",
+              "rnLvYN4ZyAJz7OhQG85P+JnxdgXgvSv66sWBs05p3vOE+53H+HQCMTLVgvoYmHNTIYtZ5CIln4hA\n",
+              "GrjLg53unVVQTiYlSzZrRE2vmtsqac+v6CrcbtgC4HktflvPTsvgqWNHri9NWa+EuXgx/AgGkZVJ\n",
+              "r1n6gAd3jtjLtv6YvbPiBBo2AhBUxCbYyroAjcvjwUBtRjXTdDEvdYfItmTKA7W3+KvVi/PCtod6\n",
+              "/3gOoaA7zRdO+8+MHlGl/c2xzQhj2O1n8eJkOu+NcsBkpmxyosDi11EOEaiQ6vfnOvH9MSM+7D/v\n",
+              "k91SLlwv/nF+5eDPHSLZQIoFUjHjwVoSGCdOLqmIe6tsfTERCeAhC+1bhRhe0612KIL6izjolsR2\n",
+              "nUgrl1o39HqnKAVqQ/HguEezLTgmGW27Df2kp4E1wRl/EQgEcsMfBPga1ndY4uHPYq84ArNCWk+c\n",
+              "YwxlHAPVC3PK3Zp2kQAAAWFBn5ZFFSw7/wBXFVHDEfqz5TAg6AmqzzGCl9B1ICKhB+tKz4Y9Km1L\n",
+              "/vZyZ1OR5rO815FlrTgGoncUDKVNjpKrVerCm+HleHb1b4FhYQG8B61zGq10uLuoQHIyL4Cv2/mm\n",
+              "s5Mi7ZftErBt64oWYphUyh0Hmn9dYYheGFzLdE9gvqcAEGJDyLZq+nfiK0Px8pHIgaIfsEdSUYcC\n",
+              "8Otyxta0EKY+Dm2m8AtQ8jjuDmkSHm/uLhgf1uCnztOKFhkR+ydRCeR9tnIlTfiv3gJbsPT8swjP\n",
+              "0OUm6yT8LhwwCJU0AGI9hN0/kTkz+NeSHjSPaBx26MAfS2Y5NEtva844h4B/RttjqxMsNDiDrfB4\n",
+              "5xn/Cl/3XrcF40eivyUSC+FHzx3M4BoLQLOKf7iz8hKiUrqRGVkGToUMxkr5192x9xCjbuvLRMd8\n",
+              "9Pel4WIOhSi52xuSf1eEhC5VVAp4lHpZmHCbgAAAAaABn7V0Q38AdnTaV3jxqK844c19uepGJJSA\n",
+              "C7DQuTz6pWfCzxcMbX5JwHItpyM9y3YT46z61a7h5Lyukp+nSKoO0zQhT0EB/u6ILUCNvVbb/89X\n",
+              "7TVI5UN6EFwYYfi4uoFmqb+5Cd0J/+d2405yTsK/f6WH/T+vNB1DYWrW67ctgHOgMHAWDLG9mitl\n",
+              "16bXmPVSi2sWzpWYg3147nlnaD00aZHqQlrMPzYTLLFwWHOLNqCoWpNLMMEevc8AnQWeykk9VNTU\n",
+              "NXzAXhrKDXl1tLQTxZG7GX3K9cQyeUnjfH3rMBGDD2zCLGXrMfPVl9EJ/F5M49Rjn38sXUf2JvF8\n",
+              "D9r9tV1APCHN27+egfFIMDg9OhrQMtjAe3WEfpYS7pl5yHh7ZZ2CedEo/Wf/ygYTAQFI72AaUTrV\n",
+              "n47d9OSqAdYs7lkgV0864auRyPQeTKK1Sp3ADeIFS134VGBNG1VnrfyZuznYkI2r0FVkGFrAXpUu\n",
+              "ZJmyKqqILhJ1OTBM8C0VBV2QXBYa2aSn2jj9t40/wJJWc9IGAVR0vj/u+wFocjwf4QAAAZYBn7dq\n",
+              "Q38AeUc/pR5QUuADgu7/kKjYlIf8yn+MfKKvFMJ4eRJz/DRqteBIBJsZW3T3phi3NzuSw0zOvEhr\n",
+              "CHz7xEUteyaR+fa6YCBeiCtangbUerW/UGoCobzV/74XB/lXH53NcEw+6x9o3/ZgwG/7l4psK3P0\n",
+              "EqSwtCrcKAAv8Wi0Z88mFp3Sp19shMF41mqYa8pNsyefrruQONS60LHg/1GySbrTeTWW74lCDwnt\n",
+              "BGXpwghp/QF087PP7hxkE8lvu8APh5F1FTiOCBSvJFm6yFC/tz24gmveLoV4Rq/qtYWRE09VDCDH\n",
+              "yjftToPMsyi4DoCtXsPRk5Jxr9Mn6xDxGjfz8uMmOKJ15ejPi/Sx9cR1QrBsU9dhcYifdB+c0AMF\n",
+              "PolB3N4pBZAASP6m7EzaTer6yZ2sIKcQdlGt9xsZ0SHtS2313gpdJkLEVrHpO5/BTcfUTTcK1+bC\n",
+              "PwRYX+iIyInP1m6htprdy84ySZ5IaGCpRKFxMCf5w22wXyyon+dlMPKACguyEPTCCZQ2MqEuC+sa\n",
+              "uB/hAAABxUGbvEmoQWyZTAgj//61KoAXgR9s4tVmwJ9HTza3s57iAAoQf/wjqzjlXnP+29f12EfR\n",
+              "S7B+4I2epG2qM/uoQ7VlrfXFlhjyX/aTq0n55QXAKa2xUKolKsuMfmZFFc6+GP96b13JiSidvPgt\n",
+              "2SSGnq9Yw4MfceFmgOaZRcwoMnpdb0UpI73YdP+DfypKyrkDqKWcBc/BGhrH8+XdnpCNDXfg5rMl\n",
+              "b0uFlQ11yUxnDYOfRwLbdjJA6FYddawSEVorFtY7jkSQx+OUBUgWkKC9rhKB+uV/yqQsvbuFiyYV\n",
+              "MviBpsZgSSN0TOC5JedQ5H38ENVBLjXnWZD9PQyueLoT4qwtI+7lodFSnBG3zboWdj6P7XDbgKT/\n",
+              "zKkFObUjwhstiQtohzxd5AXhBH3DQqNv6mRzuMxFDcTEo5ut/0/1HrPGOF4R3sJ/eQT+YnYseqvc\n",
+              "0m5njpgI3qkLmn8efBB4q3zWGpHCxBwC84HKjuugMICuXfcJHKn0aWkn65aEjT8AdxDWE09InGyo\n",
+              "EM1wsU0JgJ/qq/6MdHWfQW6+bt5xWlpYJ4axi9wZc3Aoz+Rixn8UVM2e/bd31+W37ucz9udquxnL\n",
+              "2JdNUAAAARlBn9pFFSw7/wBZVXkLa/7xg9HEtDOpc+GkSv0gCD3x6eQNkROUaCyL6QH8m/0USPLW\n",
+              "nllgC+uXg2X8kUpaUiErsLvwKd9y+trtKwV7xlvkAn0JqEnToCvptE1Sb8eF86DTi2ywy7WE/imn\n",
+              "jNBYQny1cV38ScnZp/V3phWQAYBG3kUdNNuj/FyVB7DgbQbTLK48AO5nLYv8B3LvBNBfBJ+ym1yg\n",
+              "YJXKwjm8kt8xUjO2UGKeggZOs7YHWr5Fj8OX4jV/B3/cMzP+f6YyrayA/80F6f9vgrbTlhWdlFQ8\n",
+              "QtrHKjmrl874OSSPJYH5wfQfF/1NrQd6soxjmSWYI9/FqOPoy6ujUPxQvg1fUda+wK31Cv8gD96H\n",
+              "LPqpgQAAAXkBn/l0Q38AeBaU9hYCjxV6lA176iBcJKIHTfhwkqkAB+a0LmdvcgdK3vyEsSkCI+8U\n",
+              "up3OQ4OQId/B45+Mf5P4Fc2VsfnQAACxyzNkvgEEYwZk+TyOR6/VZmeFNYMrBdqc2NNBlh56ISK/\n",
+              "h5V9lagvsX7yv0p9Hk6RXo3uoMgKhKOv/QgBAqhUvAKDw4DS7G31tehd/myRMmCPxIJ79bZsQe2/\n",
+              "iq7Nquzc/VDpPXFZHPvOmiyfyrt6Fxc2jLHZJGpvacPTIeLJiSaBxgRTEKBr/xXaKQjc5nLhlwgc\n",
+              "HSz1WRlyOsXOkob3rY8KoGVETaaIvHEl7sVHsV3QN7iR2rIGzf6YHv+c3l8OW1b7tAMShtcCLifl\n",
+              "8k1OtS8Z5o7MNTObuLXIONSPGo1fC97qRzqHFEfMZntEMqsFjjWPM6JduvRiAv8p/h0kRdcTeRox\n",
+              "t4PEdFJikYgCJgtFa00LDpNvd6Vv6MImiivCAgL9L7zEaNCr8p/p5ZiDugAAAO8Bn/tqQ38AfAnX\n",
+              "r+Rl0wYAC9kEZglKr0YEZPxbFiynbDVLyUoB5/4mwbggJCKqWcWLXkOc702XkfuMANGy7OD7QUCV\n",
+              "nopFHkp77AuzGvvM2JQndhYVkdbX30/kmHQDID1DcpthKQBbzUjm7wgAOqbulxKDc1OUw1plN1OA\n",
+              "iXs8Ju+zQDtZelKPfekDEF5iPA8IQMn3LLocZ168PVHW73hdmgfMFTsqduJxZ1oiezDuUBPUKdNQ\n",
+              "1lGg5KUsS5A9iNuo+n1shJKCmk20FfXGeNEywAjYeaq4bao/dd8nZn//htlIayY083IymAgdHbKW\n",
+              "UQAAAW1Bm/5JqEFsmUwUTBH//rUqgBbB5O6qXkABRezeefAxp9PjwxeDBuTTFSUNk2voPSz0T3Lj\n",
+              "1K/LmQtEI6YkskJKgxvIXHGf8LHTV/h2Mg/qV3IQ4zvBygOQs98iZyR5jgV+hQ58R6xIcus/6y5a\n",
+              "HrkViRrv8Sk7So3LYWmfkLzyR6vcCKhF/sCJsY8RS8BK5OOGU2Ll4Qs1n4jPQwTLDELf8SF2+07z\n",
+              "zB5hexERnOHmWZ9THKXS8j6NXPrj2p32k0gvmlI4b/Of9evEX9mDBp5GtQHOvTswQ/VYUajAUXz4\n",
+              "5w6EHuB/k+FBz9pe+B69syJ2X5MYn7Qi9rKpCl2kZv4uAWXuNo7oIaU7hr6elcFz53tdL9AEjCAb\n",
+              "BlT3p448134hjvo9lj95CHF5teK1w+R310Gc3NQ0eeJcsiYD2EoVrHHjVDF/m8I8JtTUFdJ3xm+G\n",
+              "muADOcIpcqYbeqyKWwHmgvRze+DMQbkLo4AlgQAAAR4Bnh1qQ38AfBSmnoPKZzTuFWeZOcrkeWeU\n",
+              "yVIALsozlefbqRZf6f7w7fkPoFSkdlxkJJsnO6qzfbc/Kotbm2yeFrIQw5yspszQL8gAAvMHKSnw\n",
+              "f4CTQ2vfLY55MADj1baDD7LZtn0UK1Eh1HnwXobc+mdHd/JEl/a2Tszf/EZ9+J7oMl+BYsjWKwNY\n",
+              "vOv5flnnPLcex/hWFIF4n+hpBybvasl5hI9mV0CeAAyAclftj8N9n7hadcpM/TOVmHbSkJ3cr/k+\n",
+              "StSwI8gY9k3tmbMSZc42caMpFr6YdNCCIj52zmNBccPNFxW+UT/4qCqtX1gc2j7obKDaWzC1yj1A\n",
+              "td8/VAjqVn+FzuuEokhhvubRT3RCdxeWnBTCG0CxwC7gAAACMkGaAknhClJlMCCP//61KoAXgkIw\n",
+              "VJpvAgAqN7f+5rJJcY8tkjj7p4LozjswOy2dTydK33mOBGS+NojRzBOlwt3ro+/vdQIUTIVrXKwh\n",
+              "2SrHPCPJXQoCjJUPkRODCmqbZeBHsv1r7iIOZPpX66HYYhWgPLvPzAb/Nqu9nQqKoyphhNy32+S5\n",
+              "qAFvjRKLSjPAx7GoKGUNMbYduhsBsrvVTwhrV8uWAls2mxYggJzVuRUZSL9cSt+tjl44BXjlbo1a\n",
+              "I7ybNHG97GCzcbSNcg0RA+iqwDsdnrZCO0zsNdWK1qVmER0PsSf0dicSrZwIcxZWy6JbkwQn5TnO\n",
+              "kAah3wAs6pJvW+a5ZiJHl6sVlU3yCOlrECAESqWu0YR75WfiMXgesBOuXGGNsC3icmPYNzM93us1\n",
+              "7GQTI6RmmFHGo+B2yAB2YJiK1YN/T0ltUuXfFAvL4UdHgEXOVIqVj+S+YpITMKy740IvYQ5zuZPD\n",
+              "ahdXF7HIU7xE0W12w+6qkuyZwxUMXLXdgx6svudMor1GNfDCdymcKIidhuuXh7vdQrgbivH7usVC\n",
+              "zjMqgjGahkW1YlmytCooEIoULx5ux9DK360iAi4u/nAomESdiosanRfQ9jQdJSpo4rurLfeCLF1Z\n",
+              "XsQAQRTcezHlxp1tz3A3WsYMA9urPBB8pUlDdB63MfZDCBphVx/Ddv1AMvPXFEPu18oREsV3BdKx\n",
+              "e3lxLWWpytzF3zXttYGgBb90j9DgRGE1uaAWyEAAAAEiQZ4gRTRMO/8AWVV6uU/hFqUNYqrP23yu\n",
+              "FpB+ECoAQNVnJ92i7ZF1i7u1D6K4L4gxm2RaiGsRDmf2iYWEjO8yGHAqwpcDep1/+H221WMh98AE\n",
+              "VV9Ferf+hy0D7Zu5rX4Hp3s1TpcNcEBIKPHVSHIzaZKKfPXkqE/ga/eepp8Bzdc39OW6g91hVVvf\n",
+              "WJxrnf77rapWbmivuJFfeO9u+RRykk/agdEi5E/5a475KGQprA2yl390PNrCvoamPyXbETwtbYAQ\n",
+              "pF9uDZkHdN/NQ1P4rz+zQLJx21eQsP9WBLswpDFYg9BjPw+3VrVEzeid2j5wJBlq+56Hw+Ex6fI6\n",
+              "1O0GbWSAC5/5Zg+kGX0Yx7/We9PseMWGwXWIVwqI7oHPEnK6wUkAAADgAZ5fdEN/AHk02mburIzA\n",
+              "1V5U+8CauxZABexQ9zxvy3GIkNn2+19EyZqnRm0DMMsXP4ZwiY8vW/qdBTlATfbmIFDxCTzt76+L\n",
+              "X3WaNfG+rqTfzj6gLFFHl5IJDtQmIC9KAmTgQM0Lp8TEDdYJnPYGFybq0Xdyl74+130DteV0SYTD\n",
+              "hgB6230zJvCx8ZW04pZHmYvtJ1LZAxF3BAWKPXcstkh7/Er8zYdPblR7K6t0r3b/sIHpME53VRBk\n",
+              "ggj1uN/p+iN4KwToxjP8kZ1opB7xpkyOQpicygiGnwjU7EpZpywAAAF2AZ5BakN/AIdka2Wer/IA\n",
+              "EJVZr+9KNmiS7zXHA/5uJU6D0CbJOrsLPWcfwAUCZZjhlCsnAlgzrrGOONmuxU3En1TfTKb/7Pu5\n",
+              "1R8PfIYkV/dZFitvMyRPMvzwXX1OcxtjbhM+M0LCh6zNEWJFi2Pi95t8cspIknD4iXNUblA3oEFp\n",
+              "VGuXt+8S3Upf64YqAxWADhb5zxXL+O/gnWiyawM9fyRrYcExecMkEiv5MHRsJs8Euzdps1vwxzNA\n",
+              "Zu4bu6ic2K2ueNja78qXGaHz7xLoPIVJv/T4KAuseyOhznfFtKf0Ey0eSBVK9qutGGF83lfe5Wtv\n",
+              "xb73lHTKLAyiyJassoDHBSQLAcUPb4nB6xWNr9G9gWtqEIp4Or9tKJzZIZ1tnIKZFZGb0ELAlV2+\n",
+              "pKKDz5nW+syHi871Soc3HtgomT3Y1cp83yQG1GdKkcJPkU1uJVzsVPzbXbSU7/z2Q7cikc4seN2D\n",
+              "ryQ1l58HjUs0ikCXV/V/CDkAAAH6QZpGSahBaJlMCCP//rUqgBbmS0XBN5gNQAaCJTjyhVwVkMwl\n",
+              "GF6KXnd0XUyzqjFCJEv0D2xQiJu8if6sKo6qHl+BP/MZw8ss5OKq407INzCjWOsjf2HTKyC5fNLK\n",
+              "wiJv+PzieOozn64ZK7RRud2QUaDe0kuhk4uCClSYQBImrxmWeEf/X9zH3+ilYhfoZigVm0IoMiuu\n",
+              "YX1ERVdg0Ld9E6wxbYMiQAGJU1qeeTwc8vb3w3kiJheTA2PNXtrJ98RwtpnhN6QxMe1dw+aQWI7S\n",
+              "j0oQ9iNx73N93RuNVRxXj/57S9VltjA0RTZBjLvYS81QDA3fBgaNHNzOBZ7dztz/rTxxOpumjTTw\n",
+              "x9FgnvlMsjx7FYPKUcXD5quVKd8lwTlOiGVI7X1HEv3Hh4EvpYVt6azhUBI1qGunVb3X1lyMhWJ9\n",
+              "p3muqcicwInEt+BuHY92HoNXaaJJbbQmNX5s3QJbI28Pg4gc2gaUF4SQRcBgM8uwcYUzxEkBS06L\n",
+              "0moZm8bwMsLYCLj3fgXOyFudpfg6jkYPDeVK811WbzEz8Hcd42XVL0EwE3bwDc+i2I4+NERo6J6l\n",
+              "d4d7nOIvqUuorZnDPtlYcfSWgBqdP0tQHvFb4Sv9QUCBvXlH2IEiNzo/daaHVtbFRNZ3cag2HOiP\n",
+              "lMxyt8xYJMnG7di2JiwAAAD7QZ5kRREsO/8AVwwP3fRRACC0tQoY45xe6yfL8KMHlR1wbd4HcPUC\n",
+              "+4PcnqOzdoNv80ufRyOopFYryJahX+qWFUVKK+nDtdvegTv/PqvENcT8ykEwwQ7z2oNUdaMITYi5\n",
+              "4tC5YA9FaLSBorMGx3aocAbiF8065MBqyaTkiW7FtGRHVSPubGixAl7hiQRoBoEipfCxkE/EBoII\n",
+              "omSCNrFRyjd8oY66cDfZt+iBI44uLDeP6eHMEpBALsV0FY7iWjBLaYO1t2PsklOb93SAExoyIX1I\n",
+              "TiPXiUgrCYe7dgepAF31BCnOuxiIAPWKLDHZLhGOJBLqdemk1EZoKCEAAAE5AZ6DdEN/AIteG4cJ\n",
+              "hGXgWAAHNd3/IaNiUh/zKhTXYgf+UKkbUvWJoLo7whMXByWkvy3MotNcPaSHeaKS5vKy/hBJIgk5\n",
+              "CWcdsbd5QzFHyjOIZiaEAA1AziqRPTDRRVYKhcrm181rAlAdaYmvKZAOu92pmI39/PSQjhiMouSe\n",
+              "XVT3pg0s+/zN7WMQCHqTmey2TTctwD0YnAH9CK4EMAw1jPCCTXgop9epuL/iXjup2S+LS3pGE3iO\n",
+              "oIHon+1ERGRC2Vp3b2QAstSXzK/2zI+bVnxf0PhgKqa/NeuEaF2SBGZ/TyqGPDnQfJRorCp1s+mw\n",
+              "tm/3aVbjKRTXeSwl+OCfF6rMqjf/Zw8/4yrjLNmiyOgD8OWqATkM50NFqOShrrTCaHdcxgVW70ss\n",
+              "cCXKxvzAUCe+4nK4C3zP8QAAAWMBnoVqQ38Ai2Rc7ISR6q0L0pberS7nbElvP1eAuajd6ehFPCEk\n",
+              "va4007gA4DkP0YAYAumNCN0kma3A2DvFPa+NTDmrilkXNhiNVTFRLzynsy8rdgQPBH6k5DFr/4eZ\n",
+              "jmJjfYPWB5+2eEYYc9uJ5Ni70hsVFfV+T8zp+ZkLZnd2wv7AZ7A8baF9R5O9oQlCkoVPxkDHTrmt\n",
+              "rElQhX8Fi0yj2+BVP5O9UNPGQU0+M3KYUTg9yTBG2cCw6Drt49/5M/86NN03F5R9JS9KGOfJjIlA\n",
+              "koCavGpTFqq7OYU0RM3ilfXBmxvL5QoIK28Uvs71J3h/IvKmg4v/14n3/eoSpqNUCC77ty2SgAAi\n",
+              "rxQNIHz2GF/lpTynlwsORrYNT1lJMVud8AAQb+/SaHWQXmhJ+8cZTt8XuMgG/t/hdF6GqyG0A/Pn\n",
+              "hWRq+asN+zBaeyQUWZrjl8ry0h3WPkAZksFb/gV7ABWxAAAB/0GaikmoQWyZTAgj//61KoAWw9mB\n",
+              "34Nmlq4DQoTYIkneVdOFHxDDrFwsv7yxZXXwNkGuLMduj7QGT/7lr2bNfzApMJfo9/ffM5g789Cz\n",
+              "1Mn0zxePHMHBL6IHHRVXWyqDMhVLYnQ9xFtc1jml18If/8STBCOf+AZjMnARcFmX1IwLt/ziVSoN\n",
+              "e4GPKKZqfZWytoW7461OuaeZ9dvtxrCL+W45zobgR5vOrVM+Opl+w/eFlupHlgpQBWgJcPy8sZC4\n",
+              "/O9laiYA63xx6M701UUvGFsRI+RM6anXyjKc7TVrmZ/YQKRjqB6Mejs2G1mTDkBn7T2ZURI2vZ3u\n",
+              "VXRNsQnGYDxRUokS3YRHs9LEF/gxKSdLEEiHDqcoIHyS2FPM+cIJRSvB7sxIA3hgfN/O4qDK6VO+\n",
+              "t71oi1H0Bkz1ugONnVTpQr+WeMS5AtXXNBMXU+ycO0+R9eRe9BwSk0V6tHm/HJ45oIYvyWTj3yZa\n",
+              "JQ6q+o4isbf26PsTbuSAcvQoMnzEXJkqElGJ8Z3rZtdkIzQW0DDnXeNRbj2wQmuUNBknMsWOw2/t\n",
+              "fD8BErzYLXI65PwTY+6R5c6RWYzF9HNMLBaO1c6cI4yEu1DMKtZW5FrmVuc6hg7VnWxgAgOdFKFA\n",
+              "QvmmcrbHsqCH4rkez1y5GoMlxeOuW5WKa/JdcefAflYgakEAAAEQQZ6oRRUsO/8AZUEtmg0dqwLy\n",
+              "ubLYtABfXw0ri+bvSnwBqWW9hB3/jYP94x5LyZNY560IvuBe5T4EX3/71Gbqj7BS5SJLQ7X1JK0z\n",
+              "I9iR6McwRU2BDEhu+2JQm1RA2fBVxnzCyNr1JVnfyyuumlkNzE8n1UgnkIbS/FMxc8DghB7zqZzK\n",
+              "rkagW0hHwSjNf+LJf3DnbXyvnzmB1lcv8Z9QlsnPKDef2giSgbZeTNWRMfeu91kckRy0SSKkaYVK\n",
+              "KUUpf450Vl2TzPLRaNhk7Du1IJzIJRf9supxssXD9v31LAVibgyznyLU/cS57Vr8KEXG+WpKysV+\n",
+              "6iQmQ/hCoRg82drzuniAPltxm8MMUZwVMGAAAAEzAZ7HdEN/AHUKF3WsfCAA7NAZyuGlRySXJzA8\n",
+              "WtPYIqCp+udF6BaVoG3w794kSqeP3syNbVlr+uFhruNMOOzTsNGrbATFZMl9DU6mhIXZ1HEAskmI\n",
+              "VVSgXlz4sVX35JqYrDPP8r9Bsg/O9tAp7LnTMjWlqOdgOPhHpyqf/hmokPsCwqtKfsDhxP/tmX60\n",
+              "fhM4KsfvpygzK8jmUmY/GDBCISRQeW6U8uaq8guf+cvy+sP09JLJ4HsULhIsm6kyYO04HBdOFUDr\n",
+              "/8IzlOKX3w/FCxhimlJIduY8iySAFQmALOuag1Ry1Z3p7NpGIGhZp/q5hzsMAsH2jpHXQPdtFNFH\n",
+              "4VkqDlRDeGqieCr6gwu3hPQQfF9yauq4qf5R+bfPha9tZ3XjpRO4eqNaj2xEQrcb5cIJOAAAAUsB\n",
+              "nslqQ38Aj1e+ZhXsJE07lvgA5ryx/X3Tt1hQ2T/wP93u+Km2fQtCsS47kHT/v+BMMbdxEWzwYvcd\n",
+              "d3NYalS7o/aUthPBRfYGmx2hUIQijLOXN4leC3SONeoCputIRor3Lgsy985K8UL4nvf1+pFmRQg0\n",
+              "eJgJ9ubt7jVqU4S6enDDZ82+hYwxDWOROomkxsOv8nlizRgAHHE1n42Dq5sLIu8oVYp/4M1h4rCy\n",
+              "m7AmDrR9dbHlpV6pqPLshIJSKr7R6XCF5H/mgt+78ttEoS2XxbrmVQj6DQtTzcYF1gqzE9DaiXTc\n",
+              "rKcf1aBAFclenBiNHhbAMEE20Br4FIkr51a0ynzJocMgaUhstOH+7gKJGCsTPkykOiVzQeIGOfi6\n",
+              "AmLkbzIds0NOnV21ExFbxIFAMu1BymG8Kjwvo1cLb7372R2f+Qt5Z8LjmGrBAAABxUGazkmoQWyZ\n",
+              "TAgj//61KoAWP/AeMmkxh4qDG8hcZFMZjYIY//v8PGtlbWZ+A0oGGFPTAdgmU2TFbrR0QmwUCouN\n",
+              "e8fq+V7LhZ4IhSGjAEZXRALCc6lvXQaVk4Hy29vGup69bTfpCSIWWGXFW7WfQjL50GRbZZRZHQ2m\n",
+              "pjAJ2N9/bloCCNQEfrVxCeDkKfJqKlRpIdnOUaiQpsnEysqkLqMfxaCLAtiv1vFXcLPLizzlMPs7\n",
+              "NIiiAuhD4+CMokPsODEut5yq6fM1zRym2P9iids6rfyvN0EtWlvUXkAIdmS8HfE5DlX5rtipWZ2i\n",
+              "d9rb+tQcwCfWN6erokI6tARQJu2c+ZSF/sI7qofDkfNVCHii2Msza0cnJEbLkEfdF+gBET2KrdRv\n",
+              "E5mgO+6ICEAI6O/h7r7DxvTQ9Wxzo3mHNo6898yojVZYUAEyiEUBn5+alz6XfA0d5GcOXFRjv906\n",
+              "SVSt5h/ZyjXd+HmcrubYPlDuxhjCrkqyrKcbhfJHp/Mq+DI065H9OXdNO/+uDSHvPcKkibqiAVhI\n",
+              "DqTA+NZM5+PbtXMsqU6iKpSzqr3AN5mBITP84n9JoTkmCR2U/+5h8eajZc3UcAAAAOdBnuxFFSw7\n",
+              "/wBlSP3uCsGGoV8bqfG+TF6JTvUuRSAD4pZzJUFnxrFOJYnshFJtjPOw7rAcguf7FPJIlPqbN5qs\n",
+              "fqCPl7TU74m2w4/OJHMnDpS1+crxo620hZORUqqaN/UeMSuSm/KKx2/MSsIgkvOy0fYS1MAD67Fk\n",
+              "Z5FUhBYQOPZatG+Xc3Icj+kvLjp5v9fX+nJsaNN4CCl0quEK1R//8eZO87p6DKKxlnRfV62uCNE9\n",
+              "o2MWYwf9qwHYbtyqG6I4xWPTngQnrsOmiw1Sy0bIvHiKKw6nsCsKdLVPqCFU/q5rppy8Ah4AAAIT\n",
+              "AZ8LdEN/AI9CIO0JMMhrV/0AB0HLuqwUdobO4BdVbPV1Ioua5WZC0IWTaPE/7qAFTCgAnl3rAoSn\n",
+              "Kk1336t4zGyyPYAAOSIcqQwF8zee7dn7XFk1tvgy6W/qOMTmkEiEdwceoRsnhNmrNp/TK9OoMIUg\n",
+              "ShyIuwXG8nP6tDCpAEYSuvpzo5kchXf9jICMUEGqQZjLulIdzbNUEecLTDRk1r3gpdToPPcXdXTM\n",
+              "AElxf3acmkXSo1kx4tBmKJrXm4kNQ2oDIaqLOc1dGZ+ccoProxsI+jQiCldj17rGF1/E4alcIa3L\n",
+              "dIofRLGOPkev2msNj9eN+tELiQktxoUq9fKnDsRx9Nbc5IkysRYA/KsIu02gpfPyisLPQwjLSjpr\n",
+              "jTxnZViCfPC6UCMSLVKUvso8AB0eV8Q+lldoHmqd+EeBeeJOkPU3vuU/GQacMWsLnKmVt/65Nw0r\n",
+              "y1AnL9+YKkDmvNgpqgQANfZvj5NhddHche/p4la1cXWhY3W/jmtWxMTkOC4tX16bao5sNwcVWRvt\n",
+              "UHjkDIOIXB+3akBV5Lzaef6YjjT1MeUeFh/FB0tOMV3Bhvdw35krP/ItZ1RF5hRCk1oYqz0ykGZW\n",
+              "YkciBlvCsweWM2wXwX55h7SZHtxiKM3rO4Aff+TOWGbe8hXaapPE+4wKof+j5KoQ530gP62KsQIG\n",
+              "BV49pf0LYkAEd7yVzO9dhYYFAAAA+QGfDWpDfwCPWoxxjdaiaFtca/OwfG9dSAC6jYuqYuZmzKSC\n",
+              "kzbTtnf9idy9v7frgKuFjQymibohZCHRXBQdujo9Laqcw233I4Za+//Mdf06kxHe/IBTsCsxcSfV\n",
+              "ksVUEdqCe9dEwWwg//4Ee8Le2gLXqz21e4jiFyBOjP5GsM1hpupcfwZtr5Mo/ou28BY4QZExXJ0H\n",
+              "FzCqK0jKq6c//ut1tsd+kiOyZUVGRAFVkS8bi0vvjrj3zga9Zaa6Mt7yQii43DdcrobbVIWdc0QI\n",
+              "3+rsc8fgmOnJ+GJGdWYzpFLd5zMjS5ofw5IMBt0GmHVcG82Z6YQkqKJHzQAAAe9BmxJJqEFsmUwI\n",
+              "I//+tSqAFjc3NgONUfiwAKbp/vtZn3NtK6t0V/4sA0MV4unWIJlE1N72EjQeUPmvxOpceaVXIrAK\n",
+              "21oMRdsBwM4wyEJDPiji6fXmMlmmsCvOtr78Aj8gA+xKnVDFjoVlH7PPNvnMo0iZJruZeFy1B4T9\n",
+              "/2iVnlLy1r3LZhoykeyNXqaKEANWeqYl2HjpH92g+fHSONko5D2m4SRKJwFWFllUBg2RTQ3etVYS\n",
+              "PdQGNCLeaZwhH8zjnIe5Vuu46VBC79Le/PF0x5A18FileZQS8Adcvcamp8leUQ9dML537b7ARaSt\n",
+              "9Lyu3Sdke9BouNe3+hTyxzxAi1Setn//aNMjVtdKZIT0wLvPIMCsfe3gvhpNMtez9cWJYRUO4qU0\n",
+              "Dlg6h/pUIog+BzidDDvn6SZ9WUgEXhGZOFeOBYowQfwTGI3ac1V8O93aTpJwa/om7scQbOrwAjjK\n",
+              "gaYt9yqViBt3FWYRIoJJGYqmGJkf0tLvcymA+Hyayho8kg3J33tLzi7Gkd8xVzsn0AbjvoJ9u5le\n",
+              "OKsB4L1kcStddnytXouu9GStBCQSRLPeb+iGeZTwQ5uYY8D5fTAcb3C6Ob+B7IWRbbytzq93Kz0y\n",
+              "yYvbeUq1qJCNW3/zJeXeH+8yV69x5FRyM+55j6UAAAEdQZ8wRRUsO/8AYsUcQvOGOSSADI46r94B\n",
+              "/W+PEO3biH5wUahFid/4E5wZcJb1S+5KPsyD0qQEL2HibG5BPsDLysut2eDJfU6ijjP6zrYmNEWR\n",
+              "huQfgh9NsMVuoggiphkYt9ccXxVhYHn++9K8YAnkm28Kzp0jUWHgD2VeIoDjCfJPNnBqH+CERm3s\n",
+              "nubUQ9LmttVf/+MNJAJgtOFW5A6IBAcBpJtd5kPS+zJ8VxzguhOiD6Pf/zfgjMDUsehmT57QUanw\n",
+              "gbdNgBf1mSXZw3Czfs4swXmaj+42V39PQblTRJ5hVxxBfyBMHdtD+eP+pUlQP8pBAAnf3v75+Q0T\n",
+              "L19oeS5dx79IIwiodA3vtFf2KOiU2gODZqY3kJGizWNAAAAA3AGfT3RDfwB2j3tYlaKo3hdLneRM\n",
+              "Dlhayh8NourV4B4kYRi+kgAOdUf8hAGAI5XCPTeroAwXn8G2yGEphnv3FPeZqmLNmvgLgUkPciaQ\n",
+              "A3x0WVLvMk+lZn6cJdklOXHEnjNKsClw6wU0RbMDBk1zQUzYb/75rZ2h0N0KqL096XGATDutyhUZ\n",
+              "RVkyTgfbEgHdPAmzdroStgpcOUEN4xVVZX2E+XrryGs2/tIi+iUaglsBszkGSHUeEuoEpHc8PRHH\n",
+              "tDc+6s5rO2oABm+Gux/PUd+4yoXEBbF4DtdMIooAAAHGAZ9RakN/AHaNgkMVTymoPnXABzXUf7nM\n",
+              "R8KlDfCSlxubwbY5y13VVoGV2GO0t+vExf+APmeqLrIGM9X5aCQgGSaQJX4OQoECqyNRzFZQDLhW\n",
+              "KA4dfYJp7oYRPF8AMOzGYqm7AO7w7FtM2J0yD1XqM3LrKYS1dGZTAzMM0YXyhFuS7+8HWwRTCnl1\n",
+              "B1MtLMYaA8qvJY/AATH13D2takXBcx78I1sCsI+P57X6Q2Nh62/bggQuV3uhAAN0tyrIgbNQYVBH\n",
+              "gFwoUmXrxaEApAv0P2E40tM9SJDDcZe8DyE7ljCyxGjQA+gKJHzTkZCCQsmlxDg5It6wsdQ6cusN\n",
+              "DyWnlyoq3MMo7ugMYcm1YMEY73l36Y/R5wo4wUzuNvV2tJ3rSYBCfXsVjc5o1oA8OllKUpgpBG5u\n",
+              "9AavXOqCqjA07sUF9WlQ9JPrhiXa9bThYRp0lNBazKKlKwsBPK9zJ1/OayuptCCUOtFLyDYWpp2k\n",
+              "qNXWH8r0IpnJjxnQFcNmI3LKk+rH0vqX+48vd2BUqTcJ4rwX4e+V6oU1+lJyU8fmS4Kj/iQFUx5A\n",
+              "ntiGKLVWwqfkoYN2YexrEPVBTpKi81wf61aU8NAxYQAAAjdBm1ZJqEFsmUwII//+tSqAFj3B7fR5\n",
+              "G4ADaQx//3+BfZIcqzxSrotcVc8CLm7cBBc8JifUTg3KyGbsl0UtvUGR3t77PRffuzjjVfcKeiAp\n",
+              "EmDpLoqmMXTQU5wmHksjapt36fasfEiGyN1dOKyOI9nT0TFFL0pzQSss7Ux5GajOaQUF29zSIoeo\n",
+              "7hOusjWiFyZylISVuEBU8nCgDYn9P601XpFko2u3FAuYp/svCLJOzc9W7b14FY05eVZdhfmiv0Wm\n",
+              "d+i5ZPIv9mhB+8Cb50V0LQeFfsyfPeAABtfp/HIPaN+amWONE9vQ2YbC1JsqKljPbi6Vrd258gHB\n",
+              "PNyXvESqATfkK1Gnk0AWxo7XFr5y0Ce95pJr1n6gAd91M5RV5lL/XAgE7sYG4524aA+cXAa2XPdd\n",
+              "1BugfbN6YGWbktwAoVIXoUq7TnrmhBrw2FHa1aE9uMJerl9x/Rs847iKP+iuBUD2VIUOVa/G9Po0\n",
+              "ksPo1bHVIsITIKnrhXV1NabDgHAc5kIv+PJk6IroGA19oMw2I1d4rGiaYQZE9dmK1VRARJ9VXDBJ\n",
+              "Vlz3aoQhCyQZvwzvxWhVA1iU1RO1TWnJsppajNeO4Vg4/b+BSviIvrSwwqmjaRr8iuCpVTgz+ZJ6\n",
+              "95zLiSdnoIFqQJA1Hz4YR/KIOmAfhTTnHcdDelso1m8Bx2oHlzAOiYwR4NhSSRD6EhhCU2kXf5vn\n",
+              "vYdShk1Y3/pp+Wd9yZwIwTneJB0AoI0bbmfrtbbWj1oAAAFQQZ90RRUsO/8AVxVRwqizyog1fzvw\n",
+              "w3oFk0s5kH60rPhj0qbUv+9nJnU5H1hbksC+yivmpdt3FAylOp/Re8NoooEKQr4q7MX/kjNCB5zj\n",
+              "aCmG5E3TxVGWGCYMCsdEF1I+HuXX2a3wLCwf1iqCfznNMRG46GE6nIgxc91oY/zfMduLLCzyb8AQ\n",
+              "b20W2eRODsXd4+7XC1RndLreJ7Km543AdL1iUo99hYdoASXjyWRNv6wvJrmyFngIDlQOrLluZf/9\n",
+              "T8Y21pcggXpfTtvdj+B+3lZv29AFHkL2xGPZvyL4UyVUgb3U1DWd/iySeGzlK1IbRNu7obP1czi4\n",
+              "Rchm1nI/pS+cSuamJbhlQHIreF0u2/zcrSGkuOpbObSfAY//5j6RVfcQovw5wL1RQN0tcA1GtFxu\n",
+              "ZpovaLthGUkeOPh8iV5bEpupJR1R79Ew1sEkTDugAAABwQGfk3RDfwB2dNpntdq7wHtHkfExb8Mi\n",
+              "4AOIW+6weDVD4WeLhja/JOA5FtORnuW7CfHWfWrXcPJWyNJJfpx2maEKeggtR3RVEAdA1a1truYO\n",
+              "N3PBvt2C5hri51AyWveiUQtRNh8OhcT8b+NVPo5dLHlfN2wr8ZipKDuUP3k1md+EiPqVCrK5TuMQ\n",
+              "knvfHHEV8fXqrrFiHhWYrAGbSJdOrXgrQTN4JDv0LMwXs1Nl1nmEdfSgT5BF3DohYi4r2xGfiJcJ\n",
+              "KMZ1oPHaRBjgxhu40ZP5HqUG5rQWHD92UCH/Terh0cf4e0554mxHgDF9CBXD2Ey6LaV8LB9Jb9nA\n",
+              "f7tFFMQRIVaLiP+uig+B5OoeaCY5+GdEeHuY+ZE9jNToZ4yOUwNfysZaXJBrtfqEkQosI3EYRZQA\n",
+              "COu9BHjZjXsKjEmWe9Jj9yWusbXq4WMANyEJEPNSeDcqy2nLsc2OqSE4CgyCqy8blbRZqycUiZt/\n",
+              "3NpFflI5dk/7eeQ8Uo727U5FhceNm/3Tv/0N3CZNlPGV4f+3/HHJknpIjibzMw4AkTq3Lkxy1XZ+\n",
+              "FA9yAR3cZ0/eN1EscyudULe5dTvs1EvlYMWBAAABtgGflWpDfwB5Rz+lHWcxYALocP/IVGxKQ/5l\n",
+              "P8Y+UVeKYTw8iTn+GjVV8vbhgCZ5cI/70wvHdrfJYaZZyRIawh8+61+/vwo8HAkEyAQL0QVrU8Db\n",
+              "Z7+ORIRATWUQyS/LIyP8q4/O5rf7OuybqgrrJ5JQm3dvb5EYgnYLHCULt4xtpfvTsT5gEynxu9HL\n",
+              "Km20sO4q1oqcF4MPx2dj7xETa3veUfVJqfvwop/9NWsmPrdhY/wz7rinYt2HcWm7+ulSBZtWIRv3\n",
+              "yMRoNM+lyCvZDr0PaN2HfwYWOYr/NgyLM3qvI6TujkJkGWBIPuiFK/SHsSPx7iAMcrZ3CQvQC1rq\n",
+              "psLEx1Lx0vtWsdQAcjEYe6l7VHqUFbgcjcHAYPQIIgi8NauIxLhxUOQnkJo1mXO/e5w2N9AAHA22\n",
+              "RlXXsFU92TGe3GmYdLlI4OC3IklyabPhxs95veQzY6n0a2BnyANXxWrQG1vVVVAYgtb88NEdo6By\n",
+              "gCh1aEE1VpUTP0of4shaZpNk/2gd6T34r4uIClLqdADAAdaA4/epPc357p2Ro8OkrT9okATGaQDM\n",
+              "AYBiPC2kAQBkyn5ImAAAAdBBm5pJqEFsmUwII//+tSqAF4In0o7iUdIU6DQAMu59v/f4eNbK2my3\n",
+              "LFfU4bVvmOXvurgANJp+yhdNshfKZWyf1yiq02eNo25TtXkBg+c9UZquU5KtxkSr2wTyRJb5fWbg\n",
+              "+NL8Fosje7XYkSxYEiB3sVwPhHSvNWh2d4v6fN1lP9qvuUnfb1Bn+TdruqmJdM2vx9efbO5Th2CP\n",
+              "KiH3jeuRzoCzSIUG7cY38FVzT4nUIJdz+2KjjjJ0E7ZNKQ6lROaPqjFN4utrXaZfqGFX2nWmlL+h\n",
+              "PxS7plcEcSC1oWpbRWphWgodqD5c2VmFV0yO9NkxWYeDoEeaPVORAB/gqWAbIHdoZVHMBBV6fLyv\n",
+              "D3u5FppjGB4tzB+WC5jnXJKg0Sk3SkInESay6cwWUVJt/G4Tfg6wbMdEkCvCKlRosg/RTpp5P6wR\n",
+              "Z2iZfctuN2EQi36vtriULh4PVI/bw9ZXWlyhMpAYPlW3C1NvZrlJMNaSqGSSnh5cJMfrxHquXcAN\n",
+              "CTgojRhZ3tMe14Ny/HV3UfnpEJgrqxN8KZxlRpYS28Q96uqEu6NBBsBIIz0ei/Mg1x57c0aguL4j\n",
+              "dVBDXATm12Zi0uXfiRBRiIror0O2CDrlUQAAAPNBn7hFFSw7/wBgSQL3wIE2Tv5B6OJXPcoXMcSb\n",
+              "cE8qv/1v/uy5HaAJNUQCTSWlcVovOwe/GLZOdN2BNEgb1OlzNEinzyASzg3GuZ9zFeyJHe/zvxXW\n",
+              "qHgQlhmuH8QdE1M1s5tXy5mwAyoAiCrzupaN60ez6jWL/yRvGdGiPt3qJJLeMG60zAMKa7QhUJFJ\n",
+              "FMWUFrcLW6iQXx7VTZR7Qo0gz/aCe+BxT2h34J4bdpQTH59SHjOd2X4DMr2kpW5buE3EQBEKSUD8\n",
+              "yEiNy7MVRtsZHXt1V4Pb6TljTGXtC9pzGwEXtgadiRP8dhtDjxgpVN3IyoEAAAFOAZ/XdEN/AHkx\n",
+              "u7J3fsEfo6cXtbkNOd4swcOB3voAJyKHu0c0/MGiiYXv+2wca3XUwSOEG+s8df2rHPxj/J/Armyt\n",
+              "j86AAAWOWZsl8AgjGF9fWv1mQf9jrWNuA4APvfeLBFbZJZm7otp6Fc0DFqB0XCbEvLTkRU5ySc7e\n",
+              "Y4CD3ziWyxgWkLgxNxAV0V3rzOqUGhFxcTbBCJI75knYyulzgB9+SazwgLVSR2N8nND844Y7GLCN\n",
+              "0aeRWZgNIAWJkPPhP1VnSRo1jOpV+axgAXL8ExpNwIvLk+O8lekZ0/1o7sI+uJ46XyI2SuA6uJHd\n",
+              "bwUKNMI2qDKAM6f4kKlJLSQWqzXAi8hAQzI017i25Vpi5npQJ4TsJeyOHRvmO1wY5ZnIEZHyhgB4\n",
+              "IoLWrdA5opbAou9XxH6m1F6osqepeJLd97Dr7+5BqWzoHoOLhOxNwAAAAQ4Bn9lqQ38Ah1fDGltb\n",
+              "SoFNBABy4LNe514R+dnaDTYn5E46OmsRrJgYyAm1lSXdflAXI1+CFQXE0A4eKb0poyZSLaaXfRBJ\n",
+              "r/tA3jW8xYt/UxFDszVrqnPHP/Ny6pw3mJ+pwWr+YYAHxNaLyZj85nxRNPFMUkOr96iCB+MslYrg\n",
+              "cr/vUoZCrrFka9nw08yFJlyN4Ky9KHUYJOXDrBIiz8KQQaHFalCe3rENKk9raHLB9E2PdI37xydW\n",
+              "9R3Ktqa3KW5rMJCOoArO2/3trkkCh+/FDlbsei4VdbDQ32DjCaAkDFjCyuqOJNsi8nSI2KDSRFCB\n",
+              "83l81kCObhPemVMTlMBQzSDvOtDFUtuVwHtirD8AAAFqQZvcSahBbJlMFEwR//61KoAWweTusUEY\n",
+              "AFR7WLigAceU/KgvW9LBBRTRioW652v1Xpv5tYMFhkRmmlUca4/8lM9NJwOZFgbdLq3dhRjr1SQ+\n",
+              "iitgTnIKVe77qt/yWy3INzcVxffYfGucVy2ypyvLSUZVvVzu37Ufe4d1uKQAC1EE3Wwzkx7sEK4N\n",
+              "QwJyCdTZZnLiyrlEXcLAMbB36CvMtmCiaP8XPpa1U2RaJxnBB9qYeP0+JCORflaC8m/hyWfMppd0\n",
+              "XeCFuAYTEakC9vO4HVF02QH4GZZigg7j7bXnvstEtP5QgYZViZcOoAaQGKtWm3PCHoS8mKWfCUk8\n",
+              "ZLC6z2a10V0U2DavVH2m02W1Lc4/2WzrwUTHr66DOaP+urnPdabeHdXruv1HJ087InGSipJtxGko\n",
+              "4rppNbdlP4z6g2o/ksCKcSZ76uS1diKM/39wzVYDu1tkCD1lomve9NoQwUToKqCn30PDqMAAAAEr\n",
+              "AZ/7akN/AIdka2XuDkeawxOj/BZhZtP+kNbRABb4RmWT8vSOMSH2HVKuz5/n3pn38gQM6YQqY5bV\n",
+              "v8KsLMWKt//3BpX7BUiSjA/GsXEpiGachc2o+KqjjRfujy3SLc+TvzNfgePwT9w0Jj9Y8j6ORxA7\n",
+              "13x9/iM5Lx1s2OQQyRluiOYKxXDE9QjNulPCcMLJFKpvAfnZmzl0pzzHw/ANcBEDhABHQ9ftCkUs\n",
+              "Q4pQOQF20mJ1++bXoRcUz/lR79ACwohpzpGuaQCknCVhUL3lnnyQzloB0PAIRq1VnOd+y8D18t8/\n",
+              "IEva3L9FTrRi90eT/2pNxjMaqrOmFzrhjd2kmSd3YBlll+A3KrjDn/HtXx8SDjztM7Km7BEd2LVO\n",
+              "U1pVGn0+C8gCov9gxoEAAAIMQZvgSeEKUmUwII///rUqgBet471BV4xl2QAFRvb+6Uilj9hVaCt9\n",
+              "oXOXB19FM5G4bNDJAOl9w7HrxMOF2dPOUf977Rp9NoBObCR9cN42Ht77Y+l36qfp5SrWPFz3DG9k\n",
+              "Uks1s5yfRvMME5RxPYk9+qohbe5TR7z2WNWBJjaTvhnu4485WU3BaTyIbA4BRRdj0/JwsbCXRVZy\n",
+              "OMmFdXnFdxhNGZ5JMCQy+ip435WTv8KevLzG3OUTxX5d8x0gaiQZdaPwNC9GVrgmtqTc0z7He5Hx\n",
+              "p/UnXiE+WgHU095CwXga4AbeOtQbj0tjxKUoS9sAoJ5fyTlHv9FnU0ujgUuoA3Kj0ma5qF69zgnv\n",
+              "MTXEIqf8zuYuInk435YB6s5Aa1W77q49/ZLR70JdKU9F42nWnuaGIFvaX8JNp0NTGvA0s1VSOWIl\n",
+              "YVdpY6hSPbDqLYXO/LE7X1D3sWpexh+/kcA2B6pYDzx14bD7OD1f9pMDWxIrW6BpNH75M54gOMY1\n",
+              "SxoTsfh6KVoyFK4Yqd6lPKCLY4O17tm0vzqLEva8zNeuM7b2yHKwMHpqK8FV5yaEer9Zd+uSgIqd\n",
+              "eftECExc0GDPrda1mDLPyRR8iDjZRvRS/EElnceTaWiUEonB934ThxItQqnJINdKSyNdNwx44Jgq\n",
+              "H9/Zh55FLA3sdVDr+1aesKMfNmYnbwaje7GN0y0AAAENQZ4eRTRMO/8AYEUc98FD5/CYkGD6VZTK\n",
+              "7qaMD8JeD5Yvz1s+LaCSFWcn3aLtkXWLu76WBTjEp2boTz2lISGgYIiIhTqGBdSAvn4GaApcqQ2+\n",
+              "sy0LjwIg9aZXDdjP9AWFTV1H8wY3dWCf+Rn8X8p7dsAFRxXZ4015PG0t6STtIq5DOqARSPJ32oCq\n",
+              "OenP2L2rQhT0bU7kBXZqDOvuedMFko4K8dbR3EOKtstAjt1gHGNubjQIVeNhJsdrdMtXEY7juX3P\n",
+              "NuPteAILXrR8S3R5mIOtuZ+vWEUdS+Inr7FnZsbQiIv9i7KDzU2m3LJLNdjmArFBBLgFXYHDvQmL\n",
+              "9VT51Mb8gx1TyNar/CPWDggAAADyAZ49dEN/AInJdfYNr4ilmYSAMFB4GADpypoeWWXE3q20mGL8\n",
+              "wfGmH6ZgcbtTXJWZn5/uB2IPeQFG/rqNYZ/bmIUcKhccFRuPa9wOgu4Qnm9oi81y+ChWQK1KoKDK\n",
+              "TWWDeg/SDhV8w/q9dFY0rcekgnjPKbKFgzK+IO7hoMF7vhpMoVCqvwMtBaesBfF4bzxIufyftMba\n",
+              "VRaJWuZpM22/FtH8FxujQ6EjGNr9PHZg3rsxXbkYHRqZvH6RGypNdfKRL4serPMKtCeuCWEKaj1Z\n",
+              "h+pr+ULdNvwpLLHfA3OCu3Ql8v/sLDD/O1LVB9ug+l/wHpAAAAGVAZ4/akN/AInJdjcgUcZACEqh\n",
+              "GvWiTtr19IbQdv8WE1dBOa+lNipi00vM+C9W8F7IDH0aaS+KKFaekfOwUNG520lVemVKNYbjnPl7\n",
+              "LimE+s4N2NJ5SYT5+XRMb+vTvKCkG/By5wQO/WbZo9HorEm10+Tu4CVIj+2Ky5hDZl+kA6mkBK7E\n",
+              "3LwAW+4rGYiO9JH1BLFQj0ZOJq0ybrdVynOYOw8TudsCI+I3fiT5nmYCkIO1N7h++s67fASBLfgP\n",
+              "CYo7yLNwfifRM3ay+JhoRmwX5tGJ8l9w676Zo1wDaqZ0Q5guAYSxSJk2jHShR6LxlZmIVJnq7S00\n",
+              "iBOM0mxomzMhjpxeX6zqy/aA2SEREi4ulxZsEvlIWhLQ5YFv6LMkVEh9RITRQOsKGEls7Y4eSRWc\n",
+              "f23FGWOVxL2MZUmPGVh++Xygx19XCiXwoatt/s2T7zGfLkQ2IBiMKXoeDb7yiR4q+0v6UjACWT2H\n",
+              "kOIRMpG/B4KQPsfMRT0Rk3cAwV9dNnKm4XTlo9P9TmyT71B/Greq+KvhEBDxAAACJkGaJEmoQWiZ\n",
+              "TAgj//61KoAW5ktFwTkgtAAhBassVgP2a7WSOTniW7GlpUC5YARIimzpboyDKn/53KIxVBS+A0NS\n",
+              "3NuuWMzq53zfHvhoSdYO4dYooBUDN2VkLpVK3v3kQo1FoE02X3cyV2j6ziOTJORgWGzqU5k0XKJO\n",
+              "1VCPDS1gJclQYem5NlGAENmSiR9I8XvNQLGvpLGF/2+aU31xCZzIPp4tUxyLu/gVqq+6L5DezfDz\n",
+              "gPP3+vv4JFttE5Nyc7LysmCaQfUhi6zPymHmdLjs3bZdma4hV61UMMsGBNZfYf2GUkV1dVZ9kkfz\n",
+              "RyUYJPFdwjA5S++T8sc03o81MYXnXYkO9hGiG6RRLRRV2fPSgGhghnaqxRhYVQiuVS0ENIpjxqqc\n",
+              "KBEaAMs1VoaLKEOrNhZ8yB1VLLV9KSiM7/prkkNKRuNLp0WeTv2eHtXhIdAfhKb+ic7Pb48CqpOl\n",
+              "FnnbgphlxDaS1dplrA4VxMNzEL/27xNMQzhuRvnSDNb60j/kSJHw5x2JG6G/VwCoVAfFrZll45AB\n",
+              "Puajv4y9+7flMd/pR8Rg9UAn+cey+vNCcCbbn7FNSWq2hl9cymk4fwW6iqBgiFEQ7YZtyDoNCyYz\n",
+              "KAnW0gvHCg+5n6+qxC+xDS291Y4JfSW927ZZudU0tXxvupwcKf6fDXxz/bqsOMvxj6Y81+e6Dezh\n",
+              "B2/8nCpk1Qc7N5s0JoStEQ8+K2ir0vIXayhFQIgAAAEeQZ5CRREsO/8AZTZTJbuKD3PiQhYpzA/Q\n",
+              "3Iqsld8XUz3sHppFsAHZevvXPBLN2cIUd+YCbEEH6MplVFEcbuDDV0dnlBcrCNrbp3+CAOdBsr6h\n",
+              "0YfLGDPxHlFlUCi4qTS1o0TT2Jzkq8/O+TU7SSImG1EjEmOGpKvxjn7KxERq2Pbd/0y1sNHk5hiQ\n",
+              "eJwHwc7Z19aIrWes4h3UYQqHeU6kfCpUHVgnGubU2A0Xjg0UrouNSumFogz0StLk4fuhL5slF3Bb\n",
+              "3NpP7YhgiVLV0FNM21/pfbXvRQFzmliOaZuScgePqa02nvOdEHEpGVRPLCGL/tvzSkZqhXResmQg\n",
+              "1qZ/TxlvqjWYqPRThBIk2nP66jbd6NLagdWz1BtbrwB3TQAAAVkBnmF0Q38Ajz7dDL7wKLyRAA5r\n",
+              "u/5Co2KbB/AnQg3XvWeaImUuto8KuobiZ5Rpi0jf/+r5lFprj/mYxpQ5OwqjQqFG0eXwqi1D6M23\n",
+              "HLH/3LvgYXkbAAGr9uWkQaEU+TeJ38WNXodDC29t8Y0uYEpwNzyC6FqtgkCyDYDpd/nESpdVRRJh\n",
+              "15SV0TP88AKwZsT7yWH2r5gpJv8AhXnnWmKJ/WMwiS/2+Kf3ikj614P+BDohXhMYGO4GSZ19EkRI\n",
+              "RjwO1zoy3Umd4iOMuBBPzevAs74sU7IUdkUF24rNAstoyqnAUgY510L3SgPXbZmJYMv+tRpT7ZuM\n",
+              "oLxE5ACIQ+eHStmGZgh2P1nvrIaZRiBxoWZ1B+DDOtu5OZpc7LbajGP/oy8HbEFyJIcGXHGB5VXY\n",
+              "HnskMmabuu5xyFIJcVaqbGg3TlqrbBE29OX6xO7K38oavU/okVlIM+AAAAGEAZ5jakN/AIdXv9ZL\n",
+              "/wCpeCQF0zyG8897iu+TVNq8xXl3pE8eXm424VBKoADmOQ/RgBgC6Y0IzpqUKPVKwCZafdEIuhUv\n",
+              "zhgtxewRpr3F4VdMy9NUqqvPfGroLPxDW64Af18RtCEv8t7amX9ezvEWK8AgZjHjHXeVi2k8dp4r\n",
+              "TuMjdngEOGe6y0V0qXE0vJudyGSblaiStnW6rV0e34JxbdN3Qbajy6ozlLfOkq7Wqx1iLXxa4foY\n",
+              "IPBIjzxdye8gOjZW7bP0axd+wppVHkXrrvuxUf9dp18AanJIIFv6MCm6ujRO2wyu4ZfSbZp/KVFm\n",
+              "xvxpBAJyjKSdCoPxWylEDyms9NAmwAADmUiy6WUOIsiAC130X9MRKfeLHi3miJh/YDGeINuX+P+e\n",
+              "NWBXxp3RqAzo1eISPcPztmgXUHCSN2VRpnCOFQoF4yyryK4v7s2U4a7V5e2sVJBhb7kguiVFACK3\n",
+              "rbLSCnWI4OCs6u017nghnGW3Juq0rF80iqmo5QCt19S62wAAAkZBmmhJqEFsmUwII//+tSqAFu/w\n",
+              "HjJpMYeKfGxaFh4NwH9VzFzipiNnWLhZf3lim8qQP0NcWviT9hCfSjxxrnYEE59yPQn7u6+tCr/u\n",
+              "vn8/iyWB73TxWIDTyqwOWzo0R8Wj7McP4QWP8yE0svd//Wkug5+3cHmcpP/ONbeBn+TAQ0VzErlc\n",
+              "2hXFLnmGW7EB004qvGi/S7JfG21T+V5Sx9Nre0PuomioWltV0uJSYiMg18UwZktQhoyeO+qpPgky\n",
+              "U9/xX6NUrUyAfCz03v4wSV58lpzV7BxftApX8ZGWBx2zWQV/YeOCEWbmbHqvN18Jd5FxK1iHRqe+\n",
+              "nBGg6SyBQEQQfCMxCo37AXM212ulRN9X2fE3P9HkhvkaOxQZ5AElyFJ4BlaM9J8bcUgOX6NS6Cqb\n",
+              "n7IHMcCIPjAIJ36atWVr0EheDYyrwatT/sRxqfSoF0RgoVqtGqstMXZF7XACu2N9LDV5Ss0B+mSl\n",
+              "kJJqGxc50wazbtpofP341QOLrRCoQigLO2IFkJyqTpln4FgoWIMbx8x6cKkFmIESXv7mZEx6LOrL\n",
+              "ggZa/EdzllkBPCO/+zBjmey1Y55MrbMpoidNDpdQ6yZ4UDU0ai3HtghNjtrUaVDC+dCrSCASLB02\n",
+              "bO819PX27qwUTWW1MCrVhUzQkUkht4Xa4bdnUW7zTudPa++EPxUMVY36vPDJoCGilCgIXzTOV6S9\n",
+              "OVTh4+OA6S/XkcoA6ZjbQLERX5kZSQMoFJs4bPot93titzpDSKAhc1QMx6eKK6Ol2IEAAAEkQZ6G\n",
+              "RRUsO/8AZUEFdKFRxHYcrgnLV1IJewAc5dAL6/Pr5YWcZb4ejev9b/lpY1ea5Xk1AlTe44c3rPkF\n",
+              "DXI6yAdEC7kxPh5StAse03AARSF2nro+Dr5bfPJyYF/ERJ9NScPmUIVihvTCsyh5qmuoAH9P7eCu\n",
+              "Y8rdH1hF/pTSa+Z1tzZc8gwGtgV/YsMtlWLs3VbLWxt2KTDW5Y2b0HA6zgNn25rXu72r6iiN5aw7\n",
+              "sjFipq/8rjgHE9K0EK2Opn+0SPK2Rbo28aoNdC9V8VxW1CpMNxKjFOs8YmQmJE6Qtkw+Uo5mh3ic\n",
+              "7Ng6Xje5wAF7a8Iyr8DMIwvMZnnVp6ilQ1B/LSGEPncviRIHH8w83Grtt0CsL1L2isuyMboY11N9\n",
+              "lxQPpwAAAUABnqV0Q38Aiz6zZgMl5b2XXQAXQ9yHCqNv7FVD9CxHdTnw5pqRTLAoFiba5ss3lqXG\n",
+              "QCf4/o32jzmzNKjZDN2ghdo3OS7n/NFKTMs4yX0NTqaEhdnVRvrbcGvcKo0NYMgzE8UNwneueU22\n",
+              "1vpuKbOkae4P82iS9XSi8TlOPcF8mmD+n9qfVTXzL4r0M/s5xxZempvnxqhz38EgmSM/Zw7kEyiv\n",
+              "giyuP/YjNhFl3FVcOSLiQTCj+F0nLUE7lia+UkuO/YNBXwUKZKD8Add8BG6ZTC4bD/RSktc7uv8w\n",
+              "NB82AXgnpuELTB2xZFOLAYJncjo03/3uAK678Cl8cw8fzlbnSpp5eUkHacCUtAY9LPrz/OMf2bA9\n",
+              "vBE2eUwrxz/W0Sg0tjzkUrpnJSF+xYsA2fgRolT6A0NA++mVN8PJVhaGzQAAAX4BnqdqQ38Aj1eg\n",
+              "HO2BrhbSJp3bjAA7Lyx/X3Tt1hQ2T/wP93u+Km2fQtCsS47kHT/v6cxSu0EEWzwOVr17m7uMIt8s\n",
+              "rOS2NL0s+wNbNsQiUhFGWcubxLdtukca9QFTdaQjRXuW15l7gz2QnuVPe/r9SLMinrQ8TAT7c4JB\n",
+              "GrUpwbYY2wvPKUw4NOIKdjGz2TGxM02Yhqm+YQD7nu+MPeXg/5dBf+XeKfPK+RchTbfnRfx28pUm\n",
+              "+MUq+ynmpWVmmfO3TbD8gZCbZRUeK4LOH5lP3nvVvkbZlQVhN5vPlxxNouZsDfsmprxmWrHzH3vb\n",
+              "E+c7VsDA88L9wCH+ZmQGzxFjyOQ8cz4P9rsZSuU8vQS1h6fmk4XXUosrmweEGKJT/Sv5qb0OG8e9\n",
+              "voRxFaPrroiqkALWSnA5n4zcQMwfY/xXX1aR5rslt9ItB406qJIsbsrkl8pXUe2CwOVm9B72bhd1\n",
+              "lqsCRNktqyPMF/Ek4JsxscPvDjbSqbQZL+uT8zjgAAAB5EGarEmoQWyZTAgj//61KoAZQB+OVG5p\n",
+              "SZHABUb2//v8PGtlbWZ+A0oGGFPTAdgmU2TFbsuJ6mwUCouNe8f1I2ythN04JSJ5lx+ik6KpnC91\n",
+              "1FD3eD5Jit+kJIg5holbnldcijL50GRMV+Tt0L65TPBxqSAUdrQu+eLUTHPpJCL4CV5RJau8pEIv\n",
+              "uK3a7QA/UMQ/nrDjeZ6jqf1BF3JjbyaeIc5drvnYbR6lQ0gBIzp/QRU9xrHm8FESnIe42aooWDJ9\n",
+              "bVMccs59QBQd45WisW0MXV7NFtyepgfK7biPJN57MDsWL2A4LYHAXH6f6In3GVsSrYQ2HUKGlxpv\n",
+              "Yf/Xvk0pBnHsuIEsslXTjxwTTzuRb2YT7QCJp6yHiUVL67n8RfvHMNoHfUzP4rVgPSXcPL8FOP2d\n",
+              "F8GxovHNOmsOSUyc+t9OZXQFF+4FJNSN23FsgARohBEJ3c1u0ax3ACLYlwfCd3/U1mT29ftZkWMR\n",
+              "uj01t9v2AGHvgKM29X2Vs/ALzLNDd2OM9z+AC4TlcpgcRujIhnjHf17Je/8RMBqJCZtdfrFmz6AW\n",
+              "Z/aNIv/p/WX6adpvStFWxoDAnf+Tai9COS20TO4GHDviQkpMo6tbNTk4tiYWsmvBNq5u/aO08r2y\n",
+              "Bs1eH2kAAAD6QZ7KRRUsO/8AZUj9pUTz7rNMoHjJ4gSsLw2wABNFEVCVBZ8at73oa3C8UmeDMVba\n",
+              "M3uHP8p2EFDXTkl9EiChbxZZgpuvefKfc50lYhoTJ/7H62X0Z9NX2I7S32WT1XJeJtD32zfVBu3K\n",
+              "VmE+30x6+W2pKnyMM0ZejDKLq8WyIyi+9rC0QVVyU0N739nDCyt6aqRfMfSdljqTnwOmgDB5pHyK\n",
+              "U8Nf/BZxnIET5uBVX/VcS4bjmT9sCYYwmAz5vBy8cv5J53FYPh0/wF7kP2myhm8SfTnmNtpTej0y\n",
+              "JjLbrdGSBUAu+lwbCsr/YdOCYrxvvrklZP4j4s5VlQAAAgYBnul0Q38Aiz6zZf6skuDOogA4jl3V\n",
+              "YKO0NncAuqtob34dJ/eVmQtCFk2jxP+6gBUwoAJ5d6wKEpypNd+AlIf83kNIAAC8trXyGAv3zzzV\n",
+              "tAa7kzCHOXS39Rxic+qZEHcHH0Hx0iIZnH1UNeoS6dQYQqolDkQpOXG8nP6tDCpAEYSQsJzo5kch\n",
+              "Xf9jICMUCBjMQXeVS1i3FdA07mrKCBowVzEdee9WvqvXV7KuMTufiL0hA8BHvtD6VFvEZ6eiqgvN\n",
+              "8RNM5cYXQ2i+4Lx4R2QlAIN1NNxqM8GvSjSh/rgipqY8DwHJh8p9Jbu0Zs+w86pgxJN8m/cvWxRZ\n",
+              "yFAtI7sBhDbJnNXx83ll0o93YVJhxi0TxWXPf6PlHZeEyvr6QOF2VVafQjsZUg34P/p6tj3lkAer\n",
+              "aZouLIrbfbTrpoGdtXuXR2qC418s780GZsUBVTlvppC7dgGYqQzB5daoV61BoiIg6tQyG20Yk/Ib\n",
+              "TtwSJmeU5Eiu/zRo0bpbU2jgV79WVCB/SVzxsmoD1jJEhzN1FHxsbajOijl9Vp76GofsezNr+37n\n",
+              "UWWhPPzCk1rCLQgaI34ekcMUWq/vBK2WDe7wKACe/5M5UglN5Ct9Orsd3SfYPc0336usW56marFA\n",
+              "xW2XgVLc1GludnoFyQrT+oASHSl68jJc1j3I4WTIeU/p+eW8RtUF4AAAAR4BnutqQ38Ai1egJmdK\n",
+              "YqnGBlYUAF9obzNVJ+s4Wyt0Rq0YuZmzKSClvCu/741bUzMW9+2RqBxHf8xROd9WCD2DFO6m3iiG\n",
+              "ZOgLMC6WQsGlrWDKBATBQkW8M70y/ztO1ZzNQj1ow5FREW75+T8qWeYnaEkP0sDPfhS/8A++EHpT\n",
+              "ONUZpoNHugOpCj8EFvE/MnQhkWbqDB+V4zYJeD+V1h9PGTTPeM5Ykyq4ZMi+8E5Gka9dd2CFXMaQ\n",
+              "M99mRo+FOH0+y87A4U4JusoMgrnGwBHn7tNdR1Jgk+wKYqmIwBj2jGPnQFJXhHhE3ZkpIjaeakM2\n",
+              "8MH5c8xC359KRjK1nfiZHGSkxS98YPps7lGGiAJ2WdM/l0XaVpItX1VPHy/wAAACGUGa8EmoQWyZ\n",
+              "TAgj//61KoAWNzc2A41R+LAApun++OIZUz7EikV/szjfxvYPLx+f9K2/F/he8DHawkBMdV2wRLxA\n",
+              "t50GIuRUSWE/39Xo4nAQqkjDTJdufKMgNIx0erMAcY2QA5ejjVo1tlzncJOxCqGpuGwA+5/4IKyu\n",
+              "bmTzdPecTw0ZdpVPq5j/sb/uUTmyS5oriK2QJUn4uMhurpWU0pM90BFHxmx/55iJQnC/E4AiRjGv\n",
+              "TSfvy9eol7L6q3/AmWDGKQmta5h6TQecJSS7keMMTmFMkcgh+dQEUTFbphGIZpTz6vxfkWPPyqpQ\n",
+              "VmS0gectGBeLssajkGiu1ivhXeMUvGnpqjpc6XSD8FJ8sVdfwdsse9JozsVq/t5YFq5+AnEYcopl\n",
+              "mlIiLVwif6/glDa/FvPVZyUrYuYY9L3TA7eEHe1IcHWSOPxpnafEFBrVGoeZPrbfymiVcHOQ/3CX\n",
+              "aGrpVwdWrmOHr8jLuajUxWOW37ajHobcyT1hYWMxRTx80fZmsfvsrNw/Nztdx7LidHGE8jPZ4gQZ\n",
+              "DABlByR/bof6mTmjqkfbsR1PCXy4RDNnn9nCnaSnb8pCApsF6YsDTv0+UmVzx2ZPSdm2LhZIqOim\n",
+              "mhiXHWt+ZE1dnYkLwTdsgNYEeAUTjY5XG25CAykSMfKGwGWeeOwqKmLAqTmb7mCXXxxpy4+bbELo\n",
+              "RAxOLFOR7z+Rlt4VIVMH4QAAASRBnw5FFSw7/wBiyP2mEJvZyVx6ACpM7CM8ZBKHKR5j7ndOem+L\n",
+              "X5lQTliSlHrc19blDxI+BarmPxVVRFr/CorqLGvI+vHNUfF9L5rOth1seL+LchCRD6bYXJMlctoQ\n",
+              "KBnrSfN8OsFA3rCX0rxhgXIKgdEDuCNRYd4XCiw0AyO8VPwgQ3UKQOwN4T9AdwOVZht3xWSjlGSY\n",
+              "LTfR+DOcni9vpFUI/V99yTFNeriW/Ezi0Mmb4Xp+UrrTAn+/oqePQryHATZ97i1I4TzdZJ6ol421\n",
+              "ZZiGDIa6I2z+mz36WJISXYfn5PcaqZon5evy7wkHdXdLSXQuyy6RoW3UMK1kv4eYGMx6MEUBV881\n",
+              "1DxJ4Az2tfQhJ60iq3lK6xGARpoGTWiGA3pBAAABAwGfLXRDfwCHPtdry+v+2nyY2Sk+gF5YW5HN\n",
+              "XoAL6QRR4alJgXnPRJGLu1H/XzBsCOVwj2OHZ7/Befz18ioG7PdTUWTo/DFmzXwFwKSHq5MESJ/K\n",
+              "+czoaBaMU0SilMUvvgF9NaNkzEcYOJjCpUUkl+lvc9iWY7aNcNT0YkO2YuPLl1ZJa6XpXyzgvJfC\n",
+              "YABMMMlHP4hWdgac8C4JyYJle4OEiXwhanMhhDIkpZpmZqqPP6iXGzuSTb+0ZDMJHqoDGqJmkb8S\n",
+              "IJuvyZGNE4panvJTPVd9f7g4/aXxMPm3Cn3wfT3mTthI056NzanOEWKjM1qGy4olpTOi0cV3zUKu\n",
+              "VGl1k7sAAAHXAZ8vakN/AInJcXImIY9AsY+/nZAB2XUf7nMR8KlDfCSlxubwbY5yyAvaK6FdhjtI\n",
+              "iTEMX/gD5nqi6yBjPV+WgerMVdQiwmsTWCh4ZDRMTEvRNiTK06p6H4BM93iWfwAaKh8Gz9Gaukwy\n",
+              "InHLEZ0yD1XqM2twrrM9K/zMIWUOeN0Z6Qpdges4mCaPjYBUMA0KTxEuHmES85gUYlt0s0Ks9Nu+\n",
+              "2hfyb2t0rmyvRs70WgBBgYrdeTZMCwmoCbRHPK4oxsSlCang/p1gu/DmbjnwYRln/v7ufz7R3gdP\n",
+              "Fr7XrHKEZc+f98DBxQMF82PBbmDGtLAQXHwptz6g5mqHfaJhvvgj78jkqTGrQ4WXMBaKzHGNvGYe\n",
+              "XIR0bHtcMMQd0uz0UHs+NS8bhlZ93PGBn0DI4S7X4qFOiND2PCIg5ogjbfFqU4Kuh5oLH4L3vi2E\n",
+              "bzWP7DaofhwjMqjCqAvZAgznNJDsvnJzQxJ6Pqjj2ny04t1drdQRUisSLN+PcLenLQZbe401Xg2H\n",
+              "yhW845ouHrITGSqb9EOEeoN97gj42PjsdYRMVLRDVvCV2BOAqdLbEmICPHZnyy75qPsejK7duPuc\n",
+              "fJ9rEnjynB/HxYz7zf/RM6xyYbzIoc3AAAACEkGbNEmoQWyZTAgj//61KoAbj1lLPyvb6PAZgAh9\n",
+              "7f/9/gX2SHKs8Uq31kdycpXc3bf6XPCYn1E4Nyshm7SbxYTXwR3t77AgzFtBuE6fBgZeY48yXmAW\n",
+              "rqOr3iMlgArjVOjemrjz47grY/T9rKmhvhaqPi8pvZTzkzZCl+tV6nzXVbBFw15yZW9xk2z611V7\n",
+              "GITjv5GH4Oi/06B5IbjEMVKEcRpvt893HwIyUBXniM9I90uh0TBxOedvsxxE2iLZsr/m/GNXryb+\n",
+              "9as6btju6GU5FfXHAHKy97PxI2Rac5Rx/FoPiuKEecRx7EQrDfRmlggPPP63oMY4jkBeTzC7Drwp\n",
+              "8ik2Z4rhoAMWlcRPfXCI56oe4Jt09oRInuaD3ww9/jGDjhHIXGbNYM/s5UG1XuYLCqaLxESIyPG/\n",
+              "eNnETthXX/QZDvDCFX3YINANkqDvHlUQ+vcUvksaWF/g1aVcMu45c8BoP1coWBAVWVE6iyDMwfYl\n",
+              "RYTcnNfp26mpOfqiSJnYH+AFj0qGJttgeZBuJCzdV4F5EDreo0WWAiq/0jdXljJ+ZxDij/UazQOM\n",
+              "0ct15Q7rTOqLKy+lpOVa/koSWj06e8eyy0wY1FBSVaROGYbDgXze1QzYiVyP6+WTk1fjz+Do+J+/\n",
+              "TxVlHJsfUOz0tbPJ3R4cSjRVigTxPg9VAYynpzzMlIr0/pCOGd4XYyl3SGTwAAABOUGfUkUVLDv/\n",
+              "AGU2ltMhgssRVFnYDYHdfwUIOpARUIP1pWfDHpU2pf97OTOpyP7SrW+j72yMHgCy10/KQJvVenOE\n",
+              "eMrSHUfyq6lVIsdEDgl0M+/NXx5VMpg+IZB+I7xozsY2f0ARjiAjA8ZSqG32YEqaGwpGp+vfKL3P\n",
+              "hav1CfnyaUmopPCa0Y5ww/PZN4YINPOwE+Gg36kaKP/ME/B0d8v00CzvLXmI8pIa3TqrGIa7PF4X\n",
+              "8miGO6oXkRH45ag0gFdgkGj+BD1PvtIptIkuqTa5jzG/NewDN9cCfws/hjc474K6NoCTyr++7Tth\n",
+              "LSIM60DcVje0csuhEMwOmCNob99l/AJp/9hMVsVsEaxUNsWBZFMKnZoLJU/ljkNlTtF1zcUwJoZD\n",
+              "oLTT6FmWVzlFnyfjiJdVIqMAAYsAAAIPAZ9xdEN/AI8+s1VkrBucudR5tN1L4cUDsugAOgW+6weD\n",
+              "VD4WeLhja/JOA5FtORnuW7CfHWfWrXcPJlwit0rQdaNL8wYmpMOBxVMKErdopYTnWfb0EZST9ZFP\n",
+              "kGeAI5wBNyE7pmk7U/hz6/Uncd5yONsvInzdtLdlFGIUuwPsZsiC4nxcPKJ4ER73zqMcPC62dMwB\n",
+              "YeP2JTSzcWxmsY8AuUeSUMff3wugzCWo2dZWIqj8MEevc9dnI6e4RX4rfqOmeKfJ7QFxuPllAOzz\n",
+              "FkyERujhdmr2mdRExctZgI01tg+iF/NwBCqP+hQ0BZaq12BgDPwBcWyuj8PXGo/75aroqbic3atK\n",
+              "78lcQoP6TccBH3q4TpJbdFKZCXZFrS7Hh71ZQxzuADlZ8DDRzGHyvFJs8+7LX0Z3SVEeli/7hzNR\n",
+              "3en2BovQV52x/rwTox00ojUHS89/I6QK5rr9xZ5z1Evdog7ewBETCofR8FQPxE+2X576ofb9SYpa\n",
+              "RU+FFWJ4WPQBj/u1ljXdmoINHOgs90YcpGG37DHSgRaxKh3h9samVWdsr/7ZPH7Krx9nfE8zJoXc\n",
+              "5Frf0sUOO22BhUTf6MatKarbA54SuNAmIi3ejRZKQJ4XCjhpsLBrmw33yy9Nk6OT0LCi0ELysL29\n",
+              "OvbOK/J+/iRz4bP6v+/3ppYXG9MzSEeggmS96wm6yOsevJy9wrAAAAHWAZ9zakN/AIdXwVSZADwX\n",
+              "ZeAC6HD/yFRsSkP+ZT/GPlFXimE8PIk5/ho1VfL2NNL2pqViOd6YYnwc7ksNMs5IkNYQ+fdC2XMm\n",
+              "GpZcBQdS+anJcAkZpOHFxqdIo1pLhI3h3bcsWXXBd+BTXZhbA2JSmhm8EWBGqSBNaO0U3Qcdcea5\n",
+              "428f3xthr08dSK0oFN+HNErgBuKfL3JZNShDHaW66u0MaG1B/cF2Go8z1F6LGKUAmsy0D/C2CM25\n",
+              "q38c827dgYTnZjZnTFxlPuxm+JuWvYpOeWyy3J/wjV/USVL+4BKz61/Ccy+EH/JkQUqRmUOtvYei\n",
+              "XxTdexyug9nI6kyTGc2H3hy0C3uFxKKFKo9PfiwDCQWhQ1+vZIsII4FYexn+pQbkz5kmdlWKB5Lx\n",
+              "ONpNVggWvIuTYEFI34NTLTOf285YYkebB68ywIJ5f1uX/OXMZ5RxH3gjNZ8mKLNX9suvs06qOt/Q\n",
+              "e2ZfZ7Orgt/l3O7GLxwWvzugIsO88I1KhpZhgYDdYZ//1lVBcwG/tKVYjF1obqjtyFctY9LPGIag\n",
+              "318ehZmIvkhW9djj90e+pnWknudbQDv3Os17s3l7qFADdqSGqYyGaSU47a6O12HCRSwmepV1bewA\n",
+              "AAIrQZt4SahBbJlMCCH//qpVAC8LE+AX+ndLRI9AAL65x3/f4eNbK2tvWi3seP5qm31GHdf4edmk\n",
+              "0/ZKv9BuxjUGH/qoYxXDUlaWZFHb65x0lomfbckqRBtklU+1LGTmYtvnPAbKnUSAh/jTBATZpFND\n",
+              "l6V6ofQ5PTBcFjOWwgI6YqalXUkmqnN6g77O4xvodhM7XQWhsA44ADmvatn61wvReF9d9MqoCN9N\n",
+              "Twpkx2kbbrSoHJrSyqidCsv+e2gnLoWDEdLGn/42++dseweQBj40iKRQ7paDrpDRwTZVjGQJ+52c\n",
+              "gaUSUp5A/cAn4FgESmp/sZ0NpfD9/7ZAmCbSUfPUar6ndxZ3XG2DXWcNFu473rzFQZNpJnXg/Pfh\n",
+              "QCQDuu/iX2Vi2NjGs1QVI3BReUxvD8Z/YeLy6w0jDh9dcJGJdKoNjb9Epdy5r0lFeFb9L8AWhdEd\n",
+              "sGreMPdTiMRlq+JOqjdogseyQTcuDo5iesxIsb0dhY+P9VqSJtTxyPO42dn6TXPZDgt1vROlp+Ic\n",
+              "VTutbib7FY5U+jSckVQsLzLRwDuIoa+HpEcHjzuwHMaHrKVljgiPeRI3Afdpqx3nHgy0MFCOhGEr\n",
+              "Jkw+Dadh5qrWjCGOX2K5HPLV0E5qw7krTDhpWX8sTsYsIqvxr/V2EjIFiKwnheBvunmhlbHNUKTl\n",
+              "ykWRC9Afa8QE+vO8sLJHYNqVh5kOrsn0+NP1Mm4JPbYiahSDJa4o8TJzkXFBAAABAkGflkUVLDv/\n",
+              "AGBJAvfAgTZO/kHo4lc9yaSVZkgaxkXEQAgySaAqoJy8U1XmJXFaLzsHv4KqZnckX0gP1AYFUr5X\n",
+              "3Zof5zltHp7OQG87KhkyMuJLOz4diYjf3ctsH2KA3/S29L1hP4qjZ9kfgNEsjrH/nSlX3ikiiFcQ\n",
+              "/2mu5vwlzQMTIUj5/0pAslvbULpI2rwxcgfjtpeW3qe/Q0sCZXyJ3L7VhEaeyKZo/ALUAi114xdn\n",
+              "Gao6fyKpZhWohGCsI53i8XO3Y7Dq+aD4ONx4A265BL770fTZiNNw+oM7dwTK1vcPMdOTVjz4fi6j\n",
+              "bCMBPzMCGM7CsAz7OQTIKiUTlOi8YAAAAakBn7V0Q38AeTG7snd+wR+ioRwfka+slSBm7w4HiigA\n",
+              "mYoe7RzT8waKJhe/5/xyHdk2lI4Qb6yur2vWdYx/k/gVzZWx+dAAALHLM2W5kE06MD+/WY8W9vMg\n",
+              "jgsWx+NCob+sUo3r0m3kC7Z6vE5pa/kp8NVK1XizBU/gSaY6/S/NP+nzZeAUHhvnb6LPnQnTmhI7\n",
+              "+CLAa1UiK6P+lwPbKP0S0Q5RWiopmhls/AKTmwxXB+WRWyrrFglLMCCi/H7yBlZCPn3f1nUi1WXW\n",
+              "txmtCNftDVTPLfu3fbw+YSszpG0LQoe/d+Hn14JtNEXcVveVKgdRtrJ2SZSzkDZoD5uTokEopKbG\n",
+              "geSmsxJSe6mDenK/tstnSjFiozTKWgyJb1mTK9iBWStV+uPeceDypkgatRgkwgz17Zgn457UL8xo\n",
+              "RIb3Rzvhn1PaM6KKHv4wQMqvpqRXKRm+SScKgBhgUzc706tHx+sk3QXrFbfmTj3VwEqpASdMV8SQ\n",
+              "Rc7Pl7VdiwexHM38nPcgZguGyvH4NF1CZay1mT9d+wee9MfU3VHZJgMp057sUGFJIJZNmQAAASYB\n",
+              "n7dqQ38Ah1fDGltbSoFNBABy4LNfpqaOuQiA03rsvInHR01iNZMDGQE2sq9jRvjWYcCsjv8TgHDx\n",
+              "TelM9UgK8aIkbW5xZBO7YH31DMzHB/HcoCKmBUni45/7i/CIo8gF1pGPr0DAA7wV6D09MIgWLTIz\n",
+              "u2RlgzWHXLOhQSqpesq6gEgghz4eO+szzJWiaji2cgnbFYV7gS1iXMpBIisJc8i3U9gywhFgtGxt\n",
+              "IPW/7TiYEwGOLwxyjZX1HkROuSI8lAAdZBpungwbYVpPKSngzu3PnOIcBqes7c29MHD8jRPn7Zrt\n",
+              "720E/jZ4jB2yT62h5AEs+TCYeJmiY6lwGwXm58hIVqeMFafCwAYhd3vDCtfE6mymrvYwtLYQ0YeE\n",
+              "Ebj2MbA5+zEAAAFwQZu6SahBbJlMFEwR//61KoAWx89GABUe1i4OfaowcQHQyqHCv9PnwkHOB5jh\n",
+              "ZaY1nqaJvfgMHLxnx0HRU319XsFiIgZ3fycxZ7MoTbod+V6rFy2y2Qtld8RvCt0Ug4PVQuLFLU9x\n",
+              "N6gbeWntqj92UVkXYHO8rtnoyHbc5vkyDRwK85+1rEknOmV2fCPAJQWJQHZKzqn/akJ6R91HlWya\n",
+              "u/8GgP8q7KTtX0XyZMALsB3jT/UhmW5AlGIwNHeW1rtDiMG/Xy+69i+m2kTOjww4y5o0/8WfwLLR\n",
+              "RKlhEE1LYjJQjoy3+hNy7YguxzdtR0GOg0UsPQLFZIBnnCwGmFharg9MSkzKoZck80tBnNzVcu5F\n",
+              "Ot8W+bdDLv2E/9UTXci1RXlM26z5jearPa/9d/CciU6kElsImbzJ5J2YpzVs+pvW89XbvAJMExZq\n",
+              "wXD26iUkefzti1p2cc2CbM5qN5CGCTCmR13du1Y9J/JQwXkxhEAAAAFiAZ/ZakN/AHwUpp6Dymc0\n",
+              "2L536BR5shJlFypABdlGcrzfdaw/6f5GB/atQKmEnLjISTsAvG6zfbdBMs7bm2yeFrIQxXuK81kC\n",
+              "9pAAAXcBlvswH72knWeKBsU0Ht1g5h3YcKtQv4e82ah693wXobc+mdHgPA3TBKIFWUv/iM+/E90G\n",
+              "S/NmTeZC+lgt/zT/+HMt/QSFK9C1+AMdH9l6Wmy5eJzA8pumBNuqAArwclv8LW1AC9Ryj7J7dIqZ\n",
+              "2nhKIYQ08cavMFAGExrDHt7RiTs4Auer+jpijDT1MWhCFcQjNZn9nbOp1MdYUZ3batlHR94YKH39\n",
+              "SB9iaEe1H+vDrSDRsP3b0PfVLevCUtQQ7tTMju5YxLigI0SkXHby6oMGwH35DOmYdZ/QEHihEbbH\n",
+              "ljlaWypqm6TR7b/zNBCPoaZiHS0IlbTr/gzMbXxGasP7GssB89XtUV2jZihKJYcij8456L2VAAAC\n",
+              "WkGb3knhClJlMCCH//6qVQAvW48vGhnpxPcAFRvWsRQfCH0ZQNKlkI/Fmy/VFBZqjdqwlFWyRDRU\n",
+              "ATa/x8nSCThm/LYIboN0iejGj3Uchm8nyLv3P3+HOOnCw7+XGsyycSpaT/SKI8hu4RwjrdDxqaYn\n",
+              "k6pZ6qjZtX+IZ04XS8X44piBkZKHHklQnddyez3eJG0JjT0fN5b/c72jAD+sOeXlR6iPKkSUzu0o\n",
+              "3ha2oHN6UEDmISbP1cbB3piI/SHrisHlFNjIuHiEdkqSzG95tlcEE5RmJMFHyIZtmV+VUnHUg//H\n",
+              "WOVjyT0+oFlaS4c8th8dtoQJgchjo9u+OPpSDxEJgWI6zeeh28ogNTGzlwRqjfRSsrTItvjA1MD/\n",
+              "oBFhKLk5Gm5LLSkMpDHu9T5I2IaoH3PKDFRJp5FswrHAqK+C6EMiKJRw3UfQ++e71IzTL0xpDNJL\n",
+              "z6AeitOHT7WHH1q0lcaxtRKIXyzlri2FOeAU+zEh7DbcM3wvbzCPYrbD4ePmP1flYALif0DM+F20\n",
+              "woqO1ciEp6KvfcdLwkVhOi6HukmunTXGsruYaqjkaLT2QlUIMJVPTAaXGvEAsJSG/0vfsDXKkk6Z\n",
+              "sB3ElNrSO3yHej1aIEgW5xnCNisEQsWn6TKnOYGilPN4ZN8EB64V0F8PWNB9Aq0baX+T8kKesmFw\n",
+              "2y/668NRP8ypn4s+0TEew3V5nLH+An+XxWolypflMoVnWhEhG2W+IIgxfWfPuSgDmqBKtSemnfnO\n",
+              "mj2z1HJ4yEmqNoBjJwYnWfK8e0PHHb381Mk1zGGJOgWAAAABUEGf/EU0TDv/AFlVerlP4Rak+BQA\n",
+              "rfH1MAekqKZtO9rI3YpPu0XbIusXd4D2mikBBjNWCs5ZCx1/nIkAW78LpHSyCScRX686DgqeELvg\n",
+              "+6gjEvz9oPv/Q5SyPMBeMNrb/QJ3ato+Qw19nLJWjl0bduh+HilMsrklIYKHCWBaC/dNC4s7Xl/r\n",
+              "RCzM7ZJuRKmUY/D5sEAdr/H6TIVmiD0u2jiehC8y8Gw6flB5fdlWyz5ArpMes88RS9cHH1n4Dp5A\n",
+              "9YiKoxa6XsjMVtwy/Q1CE1CcjEE8nX1x2wi3FF+AiuFwqQsSRlHtfUsVksDBdXLvE8zjbyOIuIMV\n",
+              "pnJU22cEHHqRAVAAAQz/a8I3JUwtCYefKDlHQuITIdlhxtkj1S9/MOKY0At1R1tnioLMWN7HUVCo\n",
+              "b6XS9uoGwS6oOJgKcTFbR1vNa4wchWq0XCPds0DBwQAAAPYBnht0Q38AeTSjvudgsbkOLNHOwJSE\n",
+              "7MIAOT4Tae/DlzyAOhFcKHSt+XmND2K3krM1WAe1ksxoXOx8R5ib25iI4yoXHAvjcPvcDoLvQIYy\n",
+              "rfzkEj8FCsgVqTty2M7mcrrsvBMmGI/tSEAq1Wpq/wSUg2I4oZj0GjiChzewD+uw3YnWAi/Ntf5Y\n",
+              "Cv2dU9qEo9e3jPCavhxnj6HVQyqcvxekJ6cEcAGQvRh8PwiQyys4LYMz+Th6jmnZO6zDQlY1h459\n",
+              "aXiX/1NPDVjhvbOibPxdXy1nW8ZFN/ZpmMtUtTAz4mvuGfLCJYTZv8r0n1cztBPRieehovEAAAGy\n",
+              "AZ4dakN/AHwTrqiSAEDVZr7cfUIfCi6SEtf6z4BBmn/qEvCbGFYoG0hJzipIIEfgPxGLOPb5hgYo\n",
+              "3EqlxYfhyi3ADlPB0rSvUe/2K1c1bOHHkBdbN7v2fRCe6cTgBUViIyBzKbW8+YVzs1NjLsftvDLF\n",
+              "Jws+AVbFUOsz2XZO6+tJqS4okplORVfI8Zh8pjE7ly6+HI7Omo301kEp6VZks8VHiVKJOuTRsuFe\n",
+              "1lak9cDIgZS7IV3MkEjdmu8V6wPVTOui5KhgRegdKpe7dvKwiZROacSHUyEpgoiQ49NAkgd9ICSC\n",
+              "nOG96XtcVUK5qLGXI1ECEXtJcuaFVMtCmmOBBiFL8jC1MpHbxQ+4k2qRSUjP3JvFi0NfrsxeXbrH\n",
+              "Ebg5vBmNpJE6T+wdC73c70xC+Mtp+wYFzu5kfTKcL8d+Nzu4GlIr338e6SWwNSpXRGjfdLp9o3Ic\n",
+              "2PzMtQmrlpbEeUDp1vnkaZoqSF5M9xanIk/zohgoPX5++NN/ebYvr56WROjUeIUdsOf6nrJlmboT\n",
+              "DZEat6r4aY15lVCgiz4Mpb/mqSazxzrszmdRYRxGsW8DnzAAAAHfQZoCSahBaJlMCHf//qmWALFy\n",
+              "5oM61QiAB+cxK4+jNCOHXw6RALujtnWF0llKsvjvaSIz+44BdTBn8Dqmduydu0Ab2yYLL8rBa9BR\n",
+              "bM/WBrO6FCt4pfpaT57HiAbORTevnWHgnUCdwsiqbddvhjkiuJYbgCMD0kEP1SURu/b2Z5hWsq5s\n",
+              "eIdJwlVUmffx/GFsHH2OVg2kldaudIzyWEsMXsnZccvZ4+1TTMECSDKdUtlhUW9AAgPUraaePKP1\n",
+              "hatMAsKbsEP5g1nzjTlmyHjs7FjRbwjKng4/qsqVQ+s9Z8Le9mq44VPerxrlkKxdRgf8PQXTEpxP\n",
+              "gMR8UP9I/vRSJBbzTafYsMhPytfC8ESUe9ySga0pNZKSvC+bN1h7zO9OEjqF3rsnXJU2SZN7NAbS\n",
+              "01WCPkWQIdWN39TZ8BwhuM2E1/XfXA9OxCI/7PAG40Z8M1rKVJPTY+iwZnIQA6cEF3rnJVasn/JZ\n",
+              "rircnzzi1JQr5NiwthCEkD02k7GAoyHtF8lIKArvw+GqH7Ox1Tpd6DhPPJm2hmyijeFH6E+9UCJk\n",
+              "Iiolc9K3UW1rmUlHlF/p9jHAvsiiJUpuG/KCfna2LEYj9yn6P2oNlWfqq5P2HNtctaJeVRZv9Qb/\n",
+              "mNVjyjAAAAErQZ4gRREsO/8AZUEtk8LzOoS4AAhIFC88oI10PfUAs3UxxCOOtSzHREgn4/jgVfHt\n",
+              "0r483Tf2Y8D+zGlycQw2lUV6Nidlo0k0sASUCm4dEwF8Hb0+IzseFE0dYexJdLqvhcI7IIUIH6RG\n",
+              "uv8cjTXFD8CTksvYGpGc+uBYXhlwc3/jHhNGtm8G24uHniey+Zy/NtEpSl5dub3bE324kx+/N1gF\n",
+              "sU/CxkQF6UQWvd6Br4nL+i2L6udCLqM/JAVJhScc01UR/bE+NX2i3upx0qofgxfWL8unNZ/BP9Vc\n",
+              "CvVXAtxPw+0JopAnWMlwtBFG9wd+oP4zOIJ88u/VEvyZQd0JJP1Y3qhYk13Deyiv0C1r6ci1z7CQ\n",
+              "UwYqgUT64pT/hlIvHeCzEZxqH+WbUbEAAAGYAZ5fdEN/AIteE+hbrZmAAHNd3/IVGxTYP4E6C+Wr\n",
+              "63le3xAHjzqOqEil1tIAAUY3LvF62/277H30QskV8sEjceHvPe7bE0mfZ44avBY2gS0AAAMByRDk\n",
+              "EKOyh31Y2H0mdsy+zcGsPrGm3pHtO2riBcgILxHO0F5398HG90hK8UgtDUfp9CQyPOvDSyEU4WTb\n",
+              "6/WT9Z3aca6tb4C53W6p8Geyjq/mwbvNpnCVbbqIcx1ZT2+dencovmeYmPlI7jrhk6KwLYEd+5gO\n",
+              "J2YeKk4iWai6BsaO9+Tb5P52jBVHcSZ+Vws5QhTxkBSpdHlWJRcbh50V4ViVltwUN//XNx+jx2bk\n",
+              "KsfglI41FGmS2xAJtr8ZhKDk1VRRL2tGsNB5nztuRXCFd8q4MIuVVWGjim0ntcxZ/R18mzJZN+sI\n",
+              "qKUvfsxoaeZp+oIaU1hLeXzgcHEe+3/6emdZeJWoDNhUqhkfWzWzVZbEzUKpDBS9AbVIA5KR27LD\n",
+              "3HEfRMw9yt8eYILg7m/Rm2ubtU8u6V2QuxVXq1OHry5oY2TAAAABvQGeQWpDfwCPV5unds/RGF4o\n",
+              "aWlq+XwTSVpG+igacFOApaqyNJIXSXT4q7gA4DkP0YAYAumNCN0MwD7HSEeIsv3Q3L9kZ2RagxvU\n",
+              "jle4yQq6Zl5W7AgdlZnaBngH/w8xYsqWx5t90zzi7s9VyRY9jaNshfxuJAZcRgFILNTmQNCPoCtl\n",
+              "wyo5Ht91VCy2qSby6JDLeTD096PzM4KOK7/I+amuefuT0S/QnDNs952oi11JV2mbadqtKDqJE9x4\n",
+              "nX/OjU9PBP1uhsFLNkjsz6ZHlTOcsZvWUxabbw0HBNFuLXWIYqtAYdWN7c/QUoqY2IlVBR//v+NN\n",
+              "Bxf/rxPv+9QlTTeUOAVhzyU/kQACorW+VEL2KFNUPF85LUxlbSGEYQv/98/fAQAu6hKRw3yoJoPy\n",
+              "tyr7S7Za9gGurMYseuvuasNoB+fPCmp37VWgm4yNZQ0LM+8CPtaQgShVMs2/RIG2cXksHuYVqEB7\n",
+              "PJtzP2tl8EYDen8RohIb2UO5d/Xdc8aoi/Nu4IzGq8ApuZIxjC5J9bUYtMDEDA6eChGKPjb20vqg\n",
+              "2PRBI2fSXJrcSROGTC4m+VsF+VagO1LnjrakndEAAAHtQZpDSahBbJlMCG///qeEAVH55ayIAL6z\n",
+              "9D9Go2JR/VsPgULYIy+HM1JNQWUio64eqKV59gHDbxQ77xKGvVi/RlMeepNHF+Cplpp4rKqgivaK\n",
+              "14o0jVVjKwdzXmYfm8QJck76NrSj9rXzMi3Th9DbQ5HQHvlFr1+Ft6fGVXaubVoF+Bx3J4nvsWO+\n",
+              "FhXDphKaWh9geM/3PqX1TK4zqhRL2wKgDCWdLvIi2s2e48RSWR1zksj0SjkMINJfgjA7wVj0dW8Z\n",
+              "NZGlcRPjgkoSgpomI+x9/l7dJ5fHEj4WOkMQMTJnj+KOqaXfgtXbhBachZ0Av1Z6rh+qw/iObJOy\n",
+              "7q2gUdlftEWI7In7KZjqqg18Bg+z35wI2FmknOyXdEiDAPaFiRrhqkKOLfgLssw1BdohiuTGWlKn\n",
+              "NvPL4EzIbAUeS+0qv5cFdXvRjnn1zOMYTMpyN1CZYg4pqjj8mGtGdm1F7w0Xo4Mnm3hRmvZyyOaW\n",
+              "yf38s1SCwyOkhQcwJhrAAebvkxMWrAUWrTq9K9PdCUqFbMVB9+93aovoux8zBfM/WLangtLLXd/D\n",
+              "T9TcgY0eosWGZeAhQk2sxNC3bgvMT328AT2T2XCg2nG4jsOakPWfscwbc0zKfItj/1eXvyR2tk+K\n",
+              "fpgdg9dJ/OdcXINTUAAAB95tb292AAAAbG12aGQAAAAAAAAAAAAAAAAAAAPoAAAnEAABAAABAAAA\n",
+              "AAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAA\n",
+              "AAAAAAAAAAAAAAAAAAACAAAHCHRyYWsAAABcdGtoZAAAAAMAAAAAAAAAAAAAAAEAAAAAAAAnEAAA\n",
+              "AAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAABsAAAASAA\n",
+              "AAAAACRlZHRzAAAAHGVsc3QAAAAAAAAAAQAAJxAAAAgAAAEAAAAABoBtZGlhAAAAIG1kaGQAAAAA\n",
+              "AAAAAAAAAAAAACgAAAGQAFXEAAAAAAAtaGRscgAAAAAAAAAAdmlkZQAAAAAAAAAAAAAAAFZpZGVv\n",
+              "SGFuZGxlcgAAAAYrbWluZgAAABR2bWhkAAAAAQAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAA\n",
+              "AAABAAAADHVybCAAAAABAAAF63N0YmwAAACzc3RzZAAAAAAAAAABAAAAo2F2YzEAAAAAAAAAAQAA\n",
+              "AAAAAAAAAAAAAAAAAAABsAEgAEgAAABIAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n",
+              "AAAAAAAAAAAY//8AAAAxYXZjQwFkABX/4QAYZ2QAFazZQbCWhAAAAwAEAAADAFA8WLZYAQAGaOvj\n",
+              "yyLAAAAAHHV1aWRraEDyXyRPxbo5pRvPAyPzAAAAAAAAABhzdHRzAAAAAAAAAAEAAABkAAAEAAAA\n",
+              "ABRzdHNzAAAAAAAAAAEAAAABAAADMGN0dHMAAAAAAAAAZAAAAAEAAAgAAAAAAQAAFAAAAAABAAAI\n",
+              "AAAAAAEAAAAAAAAAAQAABAAAAAABAAAUAAAAAAEAAAgAAAAAAQAAAAAAAAABAAAEAAAAAAEAABQA\n",
+              "AAAAAQAACAAAAAABAAAAAAAAAAEAAAQAAAAAAQAAFAAAAAABAAAIAAAAAAEAAAAAAAAAAQAABAAA\n",
+              "AAABAAAUAAAAAAEAAAgAAAAAAQAAAAAAAAABAAAEAAAAAAEAABQAAAAAAQAACAAAAAABAAAAAAAA\n",
+              "AAEAAAQAAAAAAQAAFAAAAAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAABAAAMAAAAAAEAAAQAAAAA\n",
+              "AQAAFAAAAAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAABAAAUAAAAAAEAAAgAAAAAAQAAAAAAAAAB\n",
+              "AAAEAAAAAAEAABQAAAAAAQAACAAAAAABAAAAAAAAAAEAAAQAAAAAAQAAFAAAAAABAAAIAAAAAAEA\n",
+              "AAAAAAAAAQAABAAAAAABAAAUAAAAAAEAAAgAAAAAAQAAAAAAAAABAAAEAAAAAAEAABQAAAAAAQAA\n",
+              "CAAAAAABAAAAAAAAAAEAAAQAAAAAAQAAFAAAAAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAABAAAM\n",
+              "AAAAAAEAAAQAAAAAAQAAFAAAAAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAABAAAUAAAAAAEAAAgA\n",
+              "AAAAAQAAAAAAAAABAAAEAAAAAAEAABQAAAAAAQAACAAAAAABAAAAAAAAAAEAAAQAAAAAAQAAFAAA\n",
+              "AAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAABAAAUAAAAAAEAAAgAAAAAAQAAAAAAAAABAAAEAAAA\n",
+              "AAEAABQAAAAAAQAACAAAAAABAAAAAAAAAAEAAAQAAAAAAQAAFAAAAAABAAAIAAAAAAEAAAAAAAAA\n",
+              "AQAABAAAAAABAAAMAAAAAAEAAAQAAAAAAQAAFAAAAAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAAB\n",
+              "AAAUAAAAAAEAAAgAAAAAAQAAAAAAAAABAAAEAAAAAAEAAAgAAAAAHHN0c2MAAAAAAAAAAQAAAAEA\n",
+              "AABkAAAAAQAAAaRzdHN6AAAAAAAAAAAAAABkAAAGhgAAAl8AAAFjAAAAvgAAAXYAAAHzAAABDgAA\n",
+              "ATYAAAFIAAAB9QAAAOIAAAD6AAABWgAAAbAAAADTAAAB8wAAAN4AAAH+AAABEAAAAOIAAAG2AAAC\n",
+              "DAAAAWUAAAGkAAABmgAAAckAAAEdAAABfQAAAPMAAAFxAAABIgAAAjYAAAEmAAAA5AAAAXoAAAH+\n",
+              "AAAA/wAAAT0AAAFnAAACAwAAARQAAAE3AAABTwAAAckAAADrAAACFwAAAP0AAAHzAAABIQAAAOAA\n",
+              "AAHKAAACOwAAAVQAAAHFAAABugAAAdQAAAD3AAABUgAAARIAAAFuAAABLwAAAhAAAAERAAAA9gAA\n",
+              "AZkAAAIqAAABIgAAAV0AAAGIAAACSgAAASgAAAFEAAABggAAAegAAAD+AAACCgAAASIAAAIdAAAB\n",
+              "KAAAAQcAAAHbAAACFgAAAT0AAAITAAAB2gAAAi8AAAEGAAABrQAAASoAAAF0AAABZgAAAl4AAAFU\n",
+              "AAAA+gAAAbYAAAHjAAABLwAAAZwAAAHBAAAB8QAAABRzdGNvAAAAAAAAAAEAAAAsAAAAYnVkdGEA\n",
+              "AABabWV0YQAAAAAAAAAhaGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAtaWxzdAAAACWp\n",
+              "dG9vAAAAHWRhdGEAAAABAAAAAExhdmY1Ny44My4xMDA=\n",
+              "\"\u003e\n",
+              "  Your browser does not support the video tag.\n",
+              "\u003c/video\u003e"
+            ],
+            "text/plain": [
+              "\u003cIPython.core.display.HTML at 0x7f84b2253b50\u003e"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "import time\n",
+        "import traceback\n",
+        "\n",
+        "from matplotlib import pyplot as plt\n",
+        "from matplotlib import animation as anim\n",
+        "import tensorflow as tf\n",
+        "from tensorflow.contrib import autograph as ag\n",
+        "from IPython import display\n",
+        "\n",
+        "\n",
+        "@ag.do_not_convert(ag.RunMode.PY_FUNC)\n",
+        "def render(boards):\n",
+        "  fig = plt.figure()\n",
+        "\n",
+        "  ims = []\n",
+        "  for b in boards:\n",
+        "    im = plt.imshow(b, interpolation='none')\n",
+        "    im.axes.get_xaxis().set_visible(False)\n",
+        "    im.axes.get_yaxis().set_visible(False)\n",
+        "    ims.append([im])\n",
+        "\n",
+        "  try:\n",
+        "    ani = anim.ArtistAnimation(\n",
+        "        fig, ims, interval=100, blit=True, repeat_delay=5000)\n",
+        "    plt.close()\n",
+        "\n",
+        "    display.display(display.HTML(ani.to_html5_video()))\n",
+        "  except RuntimeError:\n",
+        "    print('Coult not render animation:')\n",
+        "    traceback.print_exc()\n",
+        "\n",
+        "\n",
+        "def gol_episode(board):\n",
+        "  directions = tf.constant(\n",
+        "      ((-1, -1), (-1, 0), (-1, 1), (0, -1), (0, 1), (1, -1), (1, 0), (1, 1)))\n",
+        "\n",
+        "  new_board = []\n",
+        "  ag.set_element_type(new_board, tf.int32)\n",
+        "\n",
+        "  for i in range(len(board)):\n",
+        "    for j in range(len(board[i])):\n",
+        "      num_neighbors = 0\n",
+        "      for d in directions:\n",
+        "        ni = i + d[0]\n",
+        "        nj = j + d[1]\n",
+        "        if ni \u003e= 0 and nj \u003e= 0 and ni \u003c len(board) and nj \u003c len(board[i]):\n",
+        "          num_neighbors += board[ni][nj]\n",
+        "      \n",
+        "      new_cell = 0\n",
+        "      if num_neighbors == 2:\n",
+        "        new_cell = board[i][j]\n",
+        "      elif num_neighbors == 3:\n",
+        "        new_cell = 1\n",
+        "      \n",
+        "      new_board.append(new_cell)\n",
+        "  final_board = ag.stack(new_board)\n",
+        "  final_board = tf.reshape(final_board, board.shape)\n",
+        "  return final_board\n",
+        "  \n",
+        "\n",
+        "def gol(initial_board):\n",
+        "  board = initial_board\n",
+        "  boards = []\n",
+        "  ag.set_element_type(boards, tf.int32)\n",
+        "  # We are being explicit about tensor constants to ensure the loop\n",
+        "  # is not unrolled in the graph. This may change in the future.\n",
+        "  for i in range(tf.constant(NUM_STEPS)):\n",
+        "    board = gol_episode(board)\n",
+        "    boards.append(board)\n",
+        "  boards = ag.stack(boards)\n",
+        "  render(boards)\n",
+        "  return tf.no_op()\n",
+        " \n",
+        "\n",
+        "with tf.Graph().as_default():\n",
+        "  # Gosper glider gun\n",
+        "  # Adapted from http://www.cplusplus.com/forum/lounge/75168/\n",
+        "  _ = 0\n",
+        "  initial_board = tf.constant((\n",
+        "      ( _,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n",
+        "      ( _,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,1,_,_,_,_,_,_,_,_,_,_,_,_ ),\n",
+        "      ( _,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,1,_,1,_,_,_,_,_,_,_,_,_,_,_,_ ),\n",
+        "      ( _,_,_,_,_,_,_,_,_,_,_,_,_,1,1,_,_,_,_,_,_,1,1,_,_,_,_,_,_,_,_,_,_,_,_,1,1,_ ),\n",
+        "      ( _,_,_,_,_,_,_,_,_,_,_,_,1,_,_,_,1,_,_,_,_,1,1,_,_,_,_,_,_,_,_,_,_,_,_,1,1,_ ),\n",
+        "      ( _,1,1,_,_,_,_,_,_,_,_,1,_,_,_,_,_,1,_,_,_,1,1,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n",
+        "      ( _,1,1,_,_,_,_,_,_,_,_,1,_,_,_,1,_,1,1,_,_,_,_,1,_,1,_,_,_,_,_,_,_,_,_,_,_,_ ),\n",
+        "      ( _,_,_,_,_,_,_,_,_,_,_,1,_,_,_,_,_,1,_,_,_,_,_,_,_,1,_,_,_,_,_,_,_,_,_,_,_,_ ),\n",
+        "      ( _,_,_,_,_,_,_,_,_,_,_,_,1,_,_,_,1,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n",
+        "      ( _,_,_,_,_,_,_,_,_,_,_,_,_,1,1,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n",
+        "      ( _,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n",
+        "      ( _,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_ ),\n",
+        "  ))\n",
+        "  initial_board = tf.pad(initial_board, ((0, 20), (0, 10)))\n",
+        "  \n",
+        "  tf_gol = ag.to_graph(gol)\n",
+        "  game_ops = tf_gol(initial_board)\n",
+        "  with tf.Session() as sess:\n",
+        "    sess.run(game_ops)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "7NgrSPCZxs3h"
+      },
+      "source": [
+        "#### Generated code"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 2323
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 753,
+          "status": "ok",
+          "timestamp": 1532101593840,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "hIGYeX0Cxs3i",
+        "outputId": "e0b62eb1-3e12-4e53-dc54-8a3fa56d823d"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "from __future__ import print_function\n",
+            "import tensorflow as tf\n",
+            "\n",
+            "def tf__gol_episode(board):\n",
+            "  try:\n",
+            "    with tf.name_scope('gol_episode'):\n",
+            "      directions = tf.constant(((-1, -1), (-1, 0), (-1, 1), (0, -1), (0, 1),\n",
+            "          (1, -1), (1, 0), (1, 1)))\n",
+            "      new_board = ag__.new_list([])\n",
+            "\n",
+            "      def extra_test_2(new_board_2):\n",
+            "        with tf.name_scope('extra_test_2'):\n",
+            "          return True\n",
+            "\n",
+            "      def loop_body_2(i, new_board_2):\n",
+            "        with tf.name_scope('loop_body_2'):\n",
+            "\n",
+            "          def extra_test_1(new_board_1):\n",
+            "            with tf.name_scope('extra_test_1'):\n",
+            "              return True\n",
+            "\n",
+            "          def loop_body_1(j, new_board_1):\n",
+            "            with tf.name_scope('loop_body_1'):\n",
+            "              num_neighbors = 0\n",
+            "\n",
+            "              def extra_test(num_neighbors_2):\n",
+            "                with tf.name_scope('extra_test'):\n",
+            "                  return True\n",
+            "\n",
+            "              def loop_body(d, num_neighbors_2):\n",
+            "                with tf.name_scope('loop_body'):\n",
+            "                  ni = i + ag__.get_item(d, (0), opts=ag__.GetItemOpts(\n",
+            "                      element_dtype=None))\n",
+            "                  nj = j + ag__.get_item(d, (1), opts=ag__.GetItemOpts(\n",
+            "                      element_dtype=None))\n",
+            "\n",
+            "                  def if_true():\n",
+            "                    with tf.name_scope('if_true'):\n",
+            "                      num_neighbors_1, = num_neighbors_2,\n",
+            "                      num_neighbors_1 += ag__.get_item(ag__.get_item(board,\n",
+            "                          (ni), opts=ag__.GetItemOpts(element_dtype=None)),\n",
+            "                          (nj), opts=ag__.GetItemOpts(element_dtype=None))\n",
+            "                      return num_neighbors_1,\n",
+            "\n",
+            "                  def if_false():\n",
+            "                    with tf.name_scope('if_false'):\n",
+            "                      return num_neighbors_2,\n",
+            "                  num_neighbors_2 = ag__.utils.run_cond(tf.logical_and(tf.\n",
+            "                      greater_equal(ni, 0), tf.logical_and(tf.greater_equal\n",
+            "                      (nj, 0), tf.logical_and(tf.less(ni, ag__.utils.\n",
+            "                      dynamic_builtin(len, board)), tf.less(nj, ag__.utils.\n",
+            "                      dynamic_builtin(len, ag__.get_item(board, (i), opts=\n",
+            "                      ag__.GetItemOpts(element_dtype=None))))))), if_true,\n",
+            "                      if_false)\n",
+            "                  return num_neighbors_2,\n",
+            "              num_neighbors = ag__.for_stmt(directions, extra_test,\n",
+            "                  loop_body, (num_neighbors,))\n",
+            "              new_cell = 0\n",
+            "\n",
+            "              def if_true_2():\n",
+            "                with tf.name_scope('if_true_2'):\n",
+            "                  new_cell_2, = new_cell,\n",
+            "                  new_cell_2 = ag__.get_item(ag__.get_item(board, (i), opts\n",
+            "                      =ag__.GetItemOpts(element_dtype=None)), (j), opts=\n",
+            "                      ag__.GetItemOpts(element_dtype=None))\n",
+            "                  return new_cell_2,\n",
+            "\n",
+            "              def if_false_2():\n",
+            "                with tf.name_scope('if_false_2'):\n",
+            "                  new_cell_3, = new_cell,\n",
+            "\n",
+            "                  def if_true_1():\n",
+            "                    with tf.name_scope('if_true_1'):\n",
+            "                      new_cell_1, = new_cell_3,\n",
+            "                      new_cell_1 = 1\n",
+            "                      return new_cell_1,\n",
+            "\n",
+            "                  def if_false_1():\n",
+            "                    with tf.name_scope('if_false_1'):\n",
+            "                      return new_cell_3,\n",
+            "                  new_cell_3 = ag__.utils.run_cond(tf.equal(num_neighbors, \n",
+            "                      3), if_true_1, if_false_1)\n",
+            "                  return new_cell_3,\n",
+            "              new_cell = ag__.utils.run_cond(tf.equal(num_neighbors, 2),\n",
+            "                  if_true_2, if_false_2)\n",
+            "              new_board_1 = ag__.list_append(new_board_1, new_cell)\n",
+            "              return new_board_1,\n",
+            "          new_board_2 = ag__.for_stmt(ag__.utils.dynamic_builtin(range,\n",
+            "              ag__.utils.dynamic_builtin(len, ag__.get_item(board, (i),\n",
+            "              opts=ag__.GetItemOpts(element_dtype=None)))), extra_test_1,\n",
+            "              loop_body_1, (new_board_2,))\n",
+            "          return new_board_2,\n",
+            "      new_board = ag__.for_stmt(ag__.utils.dynamic_builtin(range, ag__.\n",
+            "          utils.dynamic_builtin(len, board)), extra_test_2, loop_body_2, (\n",
+            "          new_board,))\n",
+            "      final_board = ag__.list_stack(new_board, opts=ag__.ListStackOpts(\n",
+            "          element_dtype=tf.int32, original_call=ag.stack))\n",
+            "      final_board = tf.reshape(final_board, board.shape)\n",
+            "      return final_board\n",
+            "  except:\n",
+            "    ag__.rewrite_graph_construction_error(ag_source_map__)\n",
+            "\n",
+            "def tf__gol(initial_board):\n",
+            "  try:\n",
+            "    with tf.name_scope('gol'):\n",
+            "      board = initial_board\n",
+            "      boards = ag__.new_list([])\n",
+            "\n",
+            "      def extra_test(board_1, boards_1):\n",
+            "        with tf.name_scope('extra_test'):\n",
+            "          return True\n",
+            "\n",
+            "      def loop_body(i, board_1, boards_1):\n",
+            "        with tf.name_scope('loop_body'):\n",
+            "          board_1 = tf__gol_episode(board_1)\n",
+            "          boards_1 = ag__.list_append(boards_1, board_1)\n",
+            "          return board_1, boards_1\n",
+            "      board, boards = ag__.for_stmt(ag__.utils.dynamic_builtin(range, tf.\n",
+            "          constant(NUM_STEPS)), extra_test, loop_body, (board, boards))\n",
+            "      boards = ag__.list_stack(boards, opts=ag__.ListStackOpts(\n",
+            "          element_dtype=tf.int32, original_call=ag.stack))\n",
+            "      with ag__.utils.control_dependency_on_returns(render(boards)):\n",
+            "        boards_2 = ag__.utils.alias_tensors(boards)\n",
+            "        return tf.no_op()\n",
+            "  except:\n",
+            "    ag__.rewrite_graph_construction_error(ag_source_map__)\n",
+            "\n"
+          ]
+        }
+      ],
+      "source": [
+        "print(ag.to_code(gol))"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "p8zZyj-tq4K3",
+        "Lkq3DBGOv3fA",
+        "r8_0ioEuAI-a",
+        "7NgrSPCZxs3h"
+      ],
+      "default_view": {},
+      "last_runtime": {
+        "build_target": "",
+        "kind": "local"
+      },
+      "name": "Simple algorithms using AutoGraph",
+      "provenance": [
+        {
+          "file_id": "19q8KdVF8Cb_fDd13i-WDOG_6n_QGNW5-",
+          "timestamp": 1528465909719
+        }
+      ],
+      "version": "0.3.2",
+      "views": {}
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
-- 
cgit v1.2.3


From 7fc43521ee5dd12525afbc2ad766f562b24f0043 Mon Sep 17 00:00:00 2001
From: Billy Lamberta 
Date: Fri, 20 Jul 2018 10:03:44 -0700
Subject: Add CSS style sheet to preprocessed documentation and landing pages.
 Remove first button from rendered notebooks (already on page). Quarantine the
 home page a bit.

PiperOrigin-RevId: 205413200
---
 tensorflow/docs_src/tutorials/_index.yaml | 52 +------------------------------
 1 file changed, 1 insertion(+), 51 deletions(-)

diff --git a/tensorflow/docs_src/tutorials/_index.yaml b/tensorflow/docs_src/tutorials/_index.yaml
index c74fe58089..9534114689 100644
--- a/tensorflow/docs_src/tutorials/_index.yaml
+++ b/tensorflow/docs_src/tutorials/_index.yaml
@@ -2,6 +2,7 @@ project_path: /_project.yaml
 book_path: /_book.yaml
 description: 
 landing_page:
+  custom_css_path: /site-assets/css/style.css
   show_side_navs: True
   rows:
   - description: >
@@ -14,57 +15,6 @@ landing_page:
       
    
     items:
     - custom_html: >
-        
         
    
           
    Learn and use ML
    
           
    
-- 
cgit v1.2.3


From c023f46956f8a867d0dc77f1ee742564a3622e68 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 10:08:23 -0700
Subject: Modify AllocatorRegistry to be an AllocatorFactoryRegistry in
 preparation for using NUMA node specific allocators.

Also, add stub NUMA functions on the platform interface to
platform/windows/port.cc.

PiperOrigin-RevId: 205413998
---
 tensorflow/contrib/gdr/gdr_memory_manager.cc       |  32 +++---
 tensorflow/contrib/verbs/rdma_mgr.cc               |  30 +++---
 .../core/common_runtime/threadpool_device.cc       |  16 ++-
 tensorflow/core/framework/allocator.cc             |  41 ++++++-
 tensorflow/core/framework/allocator.h              |  12 ++-
 tensorflow/core/framework/allocator_registry.cc    | 120 +++++++++++++++------
 tensorflow/core/framework/allocator_registry.h     | 111 +++++++++++++------
 tensorflow/core/platform/windows/port.cc           |  20 ++++
 8 files changed, 269 insertions(+), 113 deletions(-)

diff --git a/tensorflow/contrib/gdr/gdr_memory_manager.cc b/tensorflow/contrib/gdr/gdr_memory_manager.cc
index 1435e19109..f3bbf6b4d7 100644
--- a/tensorflow/contrib/gdr/gdr_memory_manager.cc
+++ b/tensorflow/contrib/gdr/gdr_memory_manager.cc
@@ -33,10 +33,11 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/bfc_allocator.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/dma_helper.h"
+#include "tensorflow/core/common_runtime/pool_allocator.h"
+#include "tensorflow/core/common_runtime/process_state.h"
 #if GOOGLE_CUDA
 #include "tensorflow/core/common_runtime/gpu/gpu_process_state.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_util.h"
-#include "tensorflow/core/common_runtime/process_state.h"
 #endif  // GOOGLE_CUDA
 #include "tensorflow/core/framework/allocator_registry.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -182,28 +183,25 @@ class GdrMemoryManager : public RemoteMemoryManager {
   TF_DISALLOW_COPY_AND_ASSIGN(GdrMemoryManager);
 };
 
-// TODO(byronyi): remove this class duplicated from the one in
-// common/runtime/gpu/pool_allocator.h when it is available in common_runtime
-class BasicCPUAllocator : public SubAllocator {
- public:
-  ~BasicCPUAllocator() override {}
-
-  void* Alloc(size_t alignment, size_t num_bytes) override {
-    return port::AlignedMalloc(num_bytes, alignment);
-  }
-  void Free(void* ptr, size_t) override { port::AlignedFree(ptr); }
-};
-
 // TODO(byronyi): remove this class and its registration when the default
-// cpu_allocator() returns visitable allocator
+// cpu_allocator() returns visitable allocator, or cpu_allocator() is no
+// longer in use.
 class BFCRdmaAllocator : public BFCAllocator {
  public:
   BFCRdmaAllocator()
-      : BFCAllocator(new BasicCPUAllocator(), 1LL << 36, true, "cpu_rdma_bfc") {
+      : BFCAllocator(new BasicCPUAllocator(port::kNUMANoAffinity), 1LL << 36,
+                     true, "cpu_rdma_bfc") {}
+};
+class BFCRdmaAllocatorFactory : public AllocatorFactory {
+ public:
+  Allocator* CreateAllocator() override { return new BFCRdmaAllocator; }
+
+  virtual SubAllocator* CreateSubAllocator(int numa_node) {
+    return new BasicCPUAllocator(numa_node);
   }
 };
 
-REGISTER_MEM_ALLOCATOR("BFCRdmaAllocator", 101, BFCRdmaAllocator);
+REGISTER_MEM_ALLOCATOR("BFCRdmaAllocator", 101, BFCRdmaAllocatorFactory);
 
 GdrMemoryManager::GdrMemoryManager(const string& host, const string& port)
     : host_(host),
@@ -276,8 +274,8 @@ Status GdrMemoryManager::Init() {
   Allocator* allocators[] = {
 #if GOOGLE_CUDA
     GPUProcessState::singleton()->GetCUDAHostAllocator(0),
-    ProcessState::singleton()->GetCPUAllocator(0),
 #endif  // GOOGLE_CUDA
+    ProcessState::singleton()->GetCPUAllocator(0),
     cpu_allocator(),
   };
 
diff --git a/tensorflow/contrib/verbs/rdma_mgr.cc b/tensorflow/contrib/verbs/rdma_mgr.cc
index 9cb3d1fbbf..3cb5e61fac 100644
--- a/tensorflow/contrib/verbs/rdma_mgr.cc
+++ b/tensorflow/contrib/verbs/rdma_mgr.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/bfc_allocator.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_process_state.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_util.h"
+#include "tensorflow/core/common_runtime/pool_allocator.h"
 #include "tensorflow/core/common_runtime/process_state.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_worker_cache.h"
 #include "tensorflow/core/distributed_runtime/session_mgr.h"
@@ -255,28 +256,25 @@ void MRDeleter(ibv_mr* mr) {
   }
 }
 
-// TODO(byronyi): remove this class duplicated from the one in
-// common/runtime/gpu/pool_allocator.h when it is available in common_runtime
-class BasicCPUAllocator : public SubAllocator {
- public:
-  ~BasicCPUAllocator() override {}
-
-  void* Alloc(size_t alignment, size_t num_bytes) override {
-    return port::AlignedMalloc(num_bytes, alignment);
-  }
-  void Free(void* ptr, size_t) override { port::AlignedFree(ptr); }
-};
-
 // TODO(byronyi): remove this class and its registration when the default
-// cpu_allocator() returns visitable allocator
+// cpu_allocator() returns visitable allocator, or cpu_allocator() is no
+// longer in use.
 class BFCRdmaAllocator : public BFCAllocator {
  public:
   BFCRdmaAllocator()
-      : BFCAllocator(new BasicCPUAllocator(), 1LL << 36, true, "cpu_rdma_bfc") {
+      : BFCAllocator(new BasicCPUAllocator(port::kNUMANoAffinity), 1LL << 36,
+                     true, "cpu_rdma_bfc") {}
+};
+class BFCRdmaAllocatorFactory : public AllocatorFactory {
+ public:
+  Allocator* CreateAllocator() { return new BFCRdmaAllocator; }
+
+  SubAllocator* CreateSubAllocator(int numa_node) {
+    return new BasicCPUAllocator(numa_node);
   }
 };
 
-REGISTER_MEM_ALLOCATOR("BFCRdmaAllocator", 101, BFCRdmaAllocator);
+REGISTER_MEM_ALLOCATOR("BFCRdmaAllocator", 101, BFCRdmaAllocatorFactory);
 
 void RdmaMgr::InitAllocators() {
   RdmaMemoryMgr::Singleton().pd_ = rdma_adapter_->pd_;
@@ -284,8 +282,8 @@ void RdmaMgr::InitAllocators() {
   Allocator* allocators[] = {
 #if GOOGLE_CUDA
     GPUProcessState::singleton()->GetCUDAHostAllocator(0),
-    ProcessState::singleton()->GetCPUAllocator(0),
 #endif  // GOOGLE_CUDA
+    ProcessState::singleton()->GetCPUAllocator(0),
     cpu_allocator(),
   };
 
diff --git a/tensorflow/core/common_runtime/threadpool_device.cc b/tensorflow/core/common_runtime/threadpool_device.cc
index 74a87215e1..7406ecf4f8 100644
--- a/tensorflow/core/common_runtime/threadpool_device.cc
+++ b/tensorflow/core/common_runtime/threadpool_device.cc
@@ -111,7 +111,21 @@ Status ThreadPoolDevice::MakeTensorFromProto(
 }
 
 #ifdef INTEL_MKL
-REGISTER_MEM_ALLOCATOR("MklCPUAllocator", 200, MklCPUAllocator);
+namespace {
+class MklCPUAllocatorFactory : public AllocatorFactory {
+ public:
+  bool NumaEnabled() override { return false; }
+
+  Allocator* CreateAllocator() override { return new MklCPUAllocator; }
+
+  // Note: Ignores numa_node, for now.
+  virtual SubAllocator* CreateSubAllocator(int numa_node) {
+    return new MklSubAllocator;
+  }
+};
+
+REGISTER_MEM_ALLOCATOR("MklCPUAllocator", 200, MklCPUAllocatorFactory);
+}  // namespace
 #endif
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/framework/allocator.cc b/tensorflow/core/framework/allocator.cc
index 1c62d37955..888ed0c57b 100644
--- a/tensorflow/core/framework/allocator.cc
+++ b/tensorflow/core/framework/allocator.cc
@@ -91,6 +91,11 @@ void EnableCPUAllocatorFullStats(bool enable) {
   cpu_allocator_collect_full_stats = enable;
 }
 
+namespace {
+// A default Allocator for CPU devices.  ProcessState::GetCPUAllocator() will
+// return a different version that may perform better, but may also lack the
+// optional stats triggered by the functions above.  TODO(tucker): migrate all
+// uses of cpu_allocator() except tests to use ProcessState instead.
 class CPUAllocator : public Allocator {
  public:
   CPUAllocator()
@@ -170,14 +175,42 @@ class CPUAllocator : public Allocator {
   TF_DISALLOW_COPY_AND_ASSIGN(CPUAllocator);
 };
 
+class CPUAllocatorFactory : public AllocatorFactory {
+ public:
+  Allocator* CreateAllocator() override { return new CPUAllocator; }
+
+  SubAllocator* CreateSubAllocator(int numa_node) override {
+    return new CPUSubAllocator(new CPUAllocator);
+  }
+
+ private:
+  class CPUSubAllocator : public SubAllocator {
+   public:
+    explicit CPUSubAllocator(CPUAllocator* cpu_allocator)
+        : cpu_allocator_(cpu_allocator) {}
+
+    void* Alloc(size_t alignment, size_t num_bytes) override {
+      return cpu_allocator_->AllocateRaw(alignment, num_bytes);
+    }
+
+    void Free(void* ptr, size_t num_bytes) override {
+      cpu_allocator_->DeallocateRaw(ptr);
+    }
+
+   private:
+    CPUAllocator* cpu_allocator_;
+  };
+};
+
+REGISTER_MEM_ALLOCATOR("DefaultCPUAllocator", 100, CPUAllocatorFactory);
+}  // namespace
+
 Allocator* cpu_allocator() {
-  static Allocator* cpu_alloc = AllocatorRegistry::Global()->GetAllocator();
+  static Allocator* cpu_alloc =
+      AllocatorFactoryRegistry::singleton()->GetAllocator();
   if (cpu_allocator_collect_full_stats && !cpu_alloc->TracksAllocationSizes()) {
     cpu_alloc = new TrackingAllocator(cpu_alloc, true);
   }
   return cpu_alloc;
 }
-
-REGISTER_MEM_ALLOCATOR("DefaultCPUAllocator", 100, CPUAllocator);
-
 }  // namespace tensorflow
diff --git a/tensorflow/core/framework/allocator.h b/tensorflow/core/framework/allocator.h
index 2bb4d32d57..774b1fe137 100644
--- a/tensorflow/core/framework/allocator.h
+++ b/tensorflow/core/framework/allocator.h
@@ -376,16 +376,18 @@ struct AllocatorAttributes {
   int32 scope_id = 0;
 };
 
-// Returns a trivial implementation of Allocator which uses the system
-// default malloc. The returned allocator is a process singleton.
+// Returns a trivial implementation of Allocator, which is a process singleton.
+// Access through this function is only intended for use in tests and auxiliary
+// processing.  Performance sensitive uses should always obtain allocators from
+// ProcessState.
 Allocator* cpu_allocator();
 
-// If 'enable' is true, the process-wide cpu allocator collects
+// If 'enable' is true, the default CPU allocator implementation will collect
 // AllocatorStats. By default, it's disabled.
 void EnableCPUAllocatorStats(bool enable);
 
-// If 'enable' is true, the process-wide cpu allocator collects full
-// statistics. By default, it's disabled.
+// If 'enable' is true, the default CPU allocator implementation will collect
+// full statistics. By default, it's disabled.
 void EnableCPUAllocatorFullStats(bool enable);
 
 // Abstract interface of an object that does the underlying suballoc/free of
diff --git a/tensorflow/core/framework/allocator_registry.cc b/tensorflow/core/framework/allocator_registry.cc
index 486be39ae3..099c4bacc8 100644
--- a/tensorflow/core/framework/allocator_registry.cc
+++ b/tensorflow/core/framework/allocator_registry.cc
@@ -21,60 +21,110 @@ limitations under the License.
 namespace tensorflow {
 
 // static
-AllocatorRegistry* AllocatorRegistry::Global() {
-  static AllocatorRegistry* global_allocator_registry = new AllocatorRegistry;
-  return global_allocator_registry;
+AllocatorFactoryRegistry* AllocatorFactoryRegistry::singleton() {
+  static AllocatorFactoryRegistry* singleton = new AllocatorFactoryRegistry;
+  return singleton;
 }
 
-Allocator* AllocatorRegistry::GetRegisteredAllocator(const string& name,
-                                                     int priority) {
-  for (auto entry : allocators_) {
+const AllocatorFactoryRegistry::FactoryEntry*
+AllocatorFactoryRegistry::FindEntry(const string& name, int priority) const {
+  for (auto& entry : factories_) {
     if (!name.compare(entry.name) && priority == entry.priority) {
-      return entry.allocator;
+      return &entry;
     }
   }
   return nullptr;
 }
 
-void AllocatorRegistry::Register(const string& name, int priority,
-                                 Allocator* allocator) {
+void AllocatorFactoryRegistry::Register(const char* source_file,
+                                        int source_line, const string& name,
+                                        int priority,
+                                        AllocatorFactory* factory) {
+  mutex_lock l(mu_);
+  CHECK(!first_alloc_made_) << "Attempt to register an AllocatorFactory "
+                            << "after call to GetAllocator()";
   CHECK(!name.empty()) << "Need a valid name for Allocator";
   CHECK_GE(priority, 0) << "Priority needs to be non-negative";
 
-  Allocator* existing = GetRegisteredAllocator(name, priority);
+  const FactoryEntry* existing = FindEntry(name, priority);
   if (existing != nullptr) {
-    // A duplicate is if the registration name and priority match
-    // but the Allocator::Name()'s don't match.
-    CHECK_EQ(existing->Name(), allocator->Name())
-        << "Allocator with name: [" << name << "], type [" << existing->Name()
-        << "], priority: [" << priority
-        << "] already registered.  Choose a different name to register "
-        << "an allocator of type " << allocator->Name();
-
-    // The allocator names match, so we can just return.
-    // It should be safe to delete the allocator since the caller
-    // gives up ownership of it.
-    delete allocator;
-    return;
+    // Duplicate registration is a hard failure.
+    LOG(FATAL) << "New registration for AllocatorFactory with name=" << name
+               << " priority=" << priority << " at location " << source_file
+               << ":" << source_line
+               << " conflicts with previous registration at location "
+               << existing->source_file << ":" << existing->source_line;
   }
 
-  AllocatorRegistryEntry tmp_entry;
-  tmp_entry.name = name;
-  tmp_entry.priority = priority;
-  tmp_entry.allocator = allocator;
+  FactoryEntry entry;
+  entry.source_file = source_file;
+  entry.source_line = source_line;
+  entry.name = name;
+  entry.priority = priority;
+  entry.factory.reset(factory);
+  factories_.push_back(std::move(entry));
+}
 
-  allocators_.push_back(tmp_entry);
-  int high_pri = -1;
-  for (auto entry : allocators_) {
-    if (high_pri < entry.priority) {
-      m_curr_allocator_ = entry.allocator;
-      high_pri = entry.priority;
+Allocator* AllocatorFactoryRegistry::GetAllocator() {
+  mutex_lock l(mu_);
+  first_alloc_made_ = true;
+  FactoryEntry* best_entry = nullptr;
+  for (auto& entry : factories_) {
+    if (best_entry == nullptr) {
+      best_entry = &entry;
+    } else if (entry.priority > best_entry->priority) {
+      best_entry = &entry;
     }
   }
+  if (best_entry) {
+    if (!best_entry->allocator) {
+      best_entry->allocator.reset(best_entry->factory->CreateAllocator());
+    }
+    return best_entry->allocator.get();
+  } else {
+    LOG(FATAL) << "No registered CPU AllocatorFactory";
+    return nullptr;
+  }
 }
 
-Allocator* AllocatorRegistry::GetAllocator() {
-  return CHECK_NOTNULL(m_curr_allocator_);
+SubAllocator* AllocatorFactoryRegistry::GetSubAllocator(int numa_node) {
+  mutex_lock l(mu_);
+  first_alloc_made_ = true;
+  FactoryEntry* best_entry = nullptr;
+  for (auto& entry : factories_) {
+    if (best_entry == nullptr) {
+      best_entry = &entry;
+    } else if (best_entry->factory->NumaEnabled()) {
+      if (entry.factory->NumaEnabled() &&
+          (entry.priority > best_entry->priority)) {
+        best_entry = &entry;
+      }
+    } else {
+      DCHECK(!best_entry->factory->NumaEnabled());
+      if (entry.factory->NumaEnabled() ||
+          (entry.priority > best_entry->priority)) {
+        best_entry = &entry;
+      }
+    }
+  }
+  if (best_entry) {
+    int index = 0;
+    if (numa_node != port::kNUMANoAffinity) {
+      CHECK_LE(numa_node, port::NUMANumNodes());
+      index = 1 + numa_node;
+    }
+    if (best_entry->sub_allocators.size() < (index + 1)) {
+      best_entry->sub_allocators.resize(index + 1);
+    }
+    if (!best_entry->sub_allocators[index].get()) {
+      best_entry->sub_allocators[index].reset(
+          best_entry->factory->CreateSubAllocator(numa_node));
+    }
+    return best_entry->sub_allocators[index].get();
+  } else {
+    LOG(FATAL) << "No registered CPU AllocatorFactory";
+    return nullptr;
+  }
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/framework/allocator_registry.h b/tensorflow/core/framework/allocator_registry.h
index b26e79ac3b..24f282ce84 100644
--- a/tensorflow/core/framework/allocator_registry.h
+++ b/tensorflow/core/framework/allocator_registry.h
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// Classes to maintain a static registry of memory allocators
+// Classes to maintain a static registry of memory allocator factories.
 #ifndef TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_REGISTRY_H_
 #define TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_REGISTRY_H_
 
@@ -21,59 +21,100 @@ limitations under the License.
 #include 
 
 #include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/platform/numa.h"
 
 namespace tensorflow {
 
-// A global AllocatorRegistry is used to hold allocators for CPU backends
-class AllocatorRegistry {
+class AllocatorFactory {
  public:
-  // Add an allocator to the registry.  Caller releases ownership of
-  // 'allocator'.
-  void Register(const string& name, int priority, Allocator* allocator);
+  virtual ~AllocatorFactory() {}
 
-  // Return allocator with highest priority
-  // If multiple allocators have the same high priority, return one of them
+  // Returns true if the factory will create a functionally different
+  // SubAllocator for different (legal) values of numa_node.
+  virtual bool NumaEnabled() { return false; }
+
+  // Create an Allocator.
+  virtual Allocator* CreateAllocator() = 0;
+
+  // Create a SubAllocator. If NumaEnabled() is true, then returned SubAllocator
+  // will allocate memory local to numa_node.  If numa_node == kNUMANoAffinity
+  // then allocated memory is not specific to any NUMA node.
+  virtual SubAllocator* CreateSubAllocator(int numa_node) = 0;
+};
+
+// A singleton registry of AllocatorFactories.
+//
+// Allocators should be obtained through ProcessState or cpu_allocator()
+// (deprecated), not directly through this interface.  The purpose of this
+// registry is to allow link-time discovery of multiple AllocatorFactories among
+// which ProcessState will obtain the best fit at startup.
+class AllocatorFactoryRegistry {
+ public:
+  AllocatorFactoryRegistry() {}
+  ~AllocatorFactoryRegistry() {}
+
+  void Register(const char* source_file, int source_line, const string& name,
+                int priority, AllocatorFactory* factory);
+
+  // Returns 'best fit' Allocator.  Find the factory with the highest priority
+  // and return an allocator constructed by it.  If multiple factories have
+  // been registered with the same priority, picks one by unspecified criteria.
   Allocator* GetAllocator();
 
-  // Returns the global registry of allocators.
-  static AllocatorRegistry* Global();
+  // Returns 'best fit' SubAllocator.  First look for the highest priority
+  // factory that is NUMA-enabled.  If none is registered, fall back to the
+  // highest priority non-NUMA-enabled factory.  If NUMA-enabled, return a
+  // SubAllocator specific to numa_node, otherwise return a NUMA-insensitive
+  // SubAllocator.
+  SubAllocator* GetSubAllocator(int numa_node);
+
+  // Returns the singleton value.
+  static AllocatorFactoryRegistry* singleton();
 
  private:
-  typedef struct {
+  mutex mu_;
+  bool first_alloc_made_ = false;
+  struct FactoryEntry {
+    const char* source_file;
+    int source_line;
     string name;
     int priority;
-    Allocator* allocator;  // not owned
-  } AllocatorRegistryEntry;
-
-  // Returns the Allocator registered for 'name' and 'priority',
-  // or 'nullptr' if not found.
-  Allocator* GetRegisteredAllocator(const string& name, int priority);
-
-  std::vector allocators_;
-  Allocator* m_curr_allocator_;  // not owned
+    std::unique_ptr factory;
+    std::unique_ptr allocator;
+    // Index 0 corresponds to kNUMANoAffinity, other indices are (numa_node +
+    // 1).
+    std::vector> sub_allocators;
+  };
+  std::vector factories_ GUARDED_BY(mu_);
+
+  // Returns any FactoryEntry registered under 'name' and 'priority',
+  // or 'nullptr' if none found.
+  const FactoryEntry* FindEntry(const string& name, int priority) const
+      EXCLUSIVE_LOCKS_REQUIRED(mu_);
+
+  TF_DISALLOW_COPY_AND_ASSIGN(AllocatorFactoryRegistry);
 };
 
-namespace allocator_registration {
-
-class AllocatorRegistration {
+class AllocatorFactoryRegistration {
  public:
-  AllocatorRegistration(const string& name, int priority,
-                        Allocator* allocator) {
-    AllocatorRegistry::Global()->Register(name, priority, allocator);
+  AllocatorFactoryRegistration(const char* file, int line, const string& name,
+                               int priority, AllocatorFactory* factory) {
+    AllocatorFactoryRegistry::singleton()->Register(file, line, name, priority,
+                                                    factory);
   }
 };
 
-}  // namespace allocator_registration
-
-#define REGISTER_MEM_ALLOCATOR(name, priority, allocator) \
-  REGISTER_MEM_ALLOCATOR_UNIQ_HELPER(__COUNTER__, name, priority, allocator)
+#define REGISTER_MEM_ALLOCATOR(name, priority, factory)                     \
+  REGISTER_MEM_ALLOCATOR_UNIQ_HELPER(__COUNTER__, __FILE__, __LINE__, name, \
+                                     priority, factory)
 
-#define REGISTER_MEM_ALLOCATOR_UNIQ_HELPER(ctr, name, priority, allocator) \
-  REGISTER_MEM_ALLOCATOR_UNIQ(ctr, name, priority, allocator)
+#define REGISTER_MEM_ALLOCATOR_UNIQ_HELPER(ctr, file, line, name, priority, \
+                                           factory)                         \
+  REGISTER_MEM_ALLOCATOR_UNIQ(ctr, file, line, name, priority, factory)
 
-#define REGISTER_MEM_ALLOCATOR_UNIQ(ctr, name, priority, allocator) \
-  static allocator_registration::AllocatorRegistration              \
-      register_allocator_##ctr(name, priority, new allocator)
+#define REGISTER_MEM_ALLOCATOR_UNIQ(ctr, file, line, name, priority, factory) \
+  static AllocatorFactoryRegistration allocator_factory_reg_##ctr(            \
+      file, line, name, priority, new factory)
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/platform/windows/port.cc b/tensorflow/core/platform/windows/port.cc
index f2aaf13bec..5375f56372 100644
--- a/tensorflow/core/platform/windows/port.cc
+++ b/tensorflow/core/platform/windows/port.cc
@@ -33,6 +33,7 @@ limitations under the License.
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/mem.h"
+#include "tensorflow/core/platform/numa.h"
 #include "tensorflow/core/platform/snappy.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -57,6 +58,17 @@ int NumSchedulableCPUs() {
   return system_info.dwNumberOfProcessors;
 }
 
+bool NUMAEnabled() {
+  // Not yet implemented: coming soon.
+  return false;
+}
+
+int NUMANumNodes() { return 1; }
+
+void NUMASetThreadNodeAffinity(int node) {}
+
+int NUMAGetThreadNodeAffinity() { return kNUMANoAffinity; }
+
 void* AlignedMalloc(size_t size, int minimum_alignment) {
 #ifdef TENSORFLOW_USE_JEMALLOC
   void* ptr = NULL;
@@ -108,6 +120,14 @@ void Free(void* ptr) {
 #endif
 }
 
+void* NUMAMalloc(int node, size_t size, int minimum_alignment) {
+  return AlignedMalloc(size, minimum_alignment);
+}
+
+void NUMAFree(void* ptr, size_t size) { Free(ptr); }
+
+int NUMAGetMemAffinity(const void* addr) { return kNUMANoAffinity; }
+
 void MallocExtension_ReleaseToSystem(std::size_t num_bytes) {
   // No-op.
 }
-- 
cgit v1.2.3


From 4921064dd535d84aa031f8116e583b151dd46e97 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 10:23:10 -0700
Subject: Update Keras TensorBoard callback to log metrics at the batch-level

PiperOrigin-RevId: 205416192
---
 tensorflow/python/keras/callbacks.py      | 46 ++++++++++++++++-----
 tensorflow/python/keras/callbacks_test.py | 68 +++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py
index 0857a3279f..d1b9dc27bd 100644
--- a/tensorflow/python/keras/callbacks.py
+++ b/tensorflow/python/keras/callbacks.py
@@ -740,6 +740,7 @@ class TensorBoard(Callback):
     self.write_images = write_images
     self.batch_size = batch_size
     self._current_batch = 0
+    self._total_batches_seen = 0
     # abstracted writer class to be able to stub for testing
     self._writer_class = tf_summary.FileWriter
     self.embeddings_freq = embeddings_freq
@@ -883,6 +884,24 @@ class TensorBoard(Callback):
         self._epoch + self._current_val_batch / self._validation_batches)
     self._current_val_batch += 1
 
+  def _write_custom_summaries(self, step, logs=None):
+    """Writes metrics out as custom scalar summaries.
+
+    Arguments:
+        step: the global step to use for Tensorboard.
+        logs: dict. Keys are scalar summary names, values are
+            NumPy scalars.
+
+    """
+    logs = logs or {}
+    for name, value in logs.items():
+      summary = tf_summary.Summary()
+      summary_value = summary.value.add()
+      summary_value.simple_value = value.item()
+      summary_value.tag = name
+      self.writer.add_summary(summary, step)
+    self.writer.flush()
+
   def on_train_begin(self, logs=None):
     """Checks if histogram summaries can be run."""
 
@@ -899,6 +918,16 @@ class TensorBoard(Callback):
         raise ValueError(
             'If printing histograms, validation data must have length > 0.')
 
+  def on_batch_end(self, batch, logs=None):
+    """Writes scalar summaries for metrics on every training batch."""
+    # Don't output batch_size and batch number as Tensorboard summaries
+    logs = logs or {}
+    batch_logs = {('batch_' + k): v
+                  for k, v in logs.items()
+                  if k not in ['batch', 'size']}
+    self._write_custom_summaries(self._total_batches_seen, batch_logs)
+    self._total_batches_seen += 1
+
   def on_epoch_begin(self, epoch, logs=None):
     """Add histogram op to Model test_function callbacks, reset batch count."""
 
@@ -915,7 +944,12 @@ class TensorBoard(Callback):
   def on_epoch_end(self, epoch, logs=None):
     """Checks if summary ops should run next epoch, logs scalar summaries."""
 
-    logs = logs or {}
+    # don't output batch_size and
+    # batch number as Tensorboard summaries
+    logs = {('epoch_' + k): v
+            for k, v in logs.items()
+            if k not in ['batch', 'size']}
+    self._write_custom_summaries(epoch, logs)
 
     # pop the histogram summary op after each epoch
     if self.histogram_freq:
@@ -964,16 +998,6 @@ class TensorBoard(Callback):
 
           i += self.batch_size
 
-    for name, value in logs.items():
-      if name in ['batch', 'size']:
-        continue
-      summary = tf_summary.Summary()
-      summary_value = summary.value.add()
-      summary_value.simple_value = value.item()
-      summary_value.tag = name
-      self.writer.add_summary(summary, epoch)
-    self.writer.flush()
-
   def on_train_end(self, logs=None):
     self.writer.close()
 
diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py
index 45598cafd3..7d830078ce 100644
--- a/tensorflow/python/keras/callbacks_test.py
+++ b/tensorflow/python/keras/callbacks_test.py
@@ -1096,6 +1096,74 @@ class KerasCallbacksTest(test.TestCase):
 
       assert os.path.exists(temp_dir)
 
+  def test_Tensorboard_batch_logging(self):
+
+    class FileWriterStub(object):
+
+      def __init__(self, logdir, graph=None):
+        self.logdir = logdir
+        self.graph = graph
+        self.batches_logged = []
+        self.summary_values = []
+        self.summary_tags = []
+
+      def add_summary(self, summary, step):
+        self.summary_values.append(summary.value[0].simple_value)
+        self.summary_tags.append(summary.value[0].tag)
+        self.batches_logged.append(step)
+
+      def flush(self):
+        pass
+
+      def close(self):
+        pass
+
+    logdir = 'fake_dir'
+
+    # log every batch
+    tb_cbk = keras.callbacks.TensorBoard(logdir)
+    tb_cbk.writer = FileWriterStub(logdir)
+
+    for batch in range(5):
+      tb_cbk.on_batch_end(batch, {'acc': np.float32(batch)})
+    self.assertEqual(tb_cbk.writer.batches_logged, [0, 1, 2, 3, 4])
+    self.assertEqual(tb_cbk.writer.summary_values, [0., 1., 2., 3., 4.])
+    self.assertEqual(tb_cbk.writer.summary_tags, ['batch_acc'] * 5)
+
+  def test_Tensorboard_epoch_and_batch_logging(self):
+
+    class FileWriterStub(object):
+
+      def __init__(self, logdir, graph=None):
+        self.logdir = logdir
+        self.graph = graph
+
+      def add_summary(self, summary, step):
+        if 'batch_' in summary.value[0].tag:
+          self.batch_summary = (step, summary)
+        elif 'epoch_' in summary.value[0].tag:
+          self.epoch_summary = (step, summary)
+
+      def flush(self):
+        pass
+
+      def close(self):
+        pass
+
+    logdir = 'fake_dir'
+
+    tb_cbk = keras.callbacks.TensorBoard(logdir)
+    tb_cbk.writer = FileWriterStub(logdir)
+
+    tb_cbk.on_batch_end(0, {'acc': np.float32(5.0)})
+    tb_cbk.on_epoch_end(0, {'acc': np.float32(10.0)})
+    batch_step, batch_summary = tb_cbk.writer.batch_summary
+    self.assertEqual(batch_step, 0)
+    self.assertEqual(batch_summary.value[0].simple_value, 5.0)
+    epoch_step, epoch_summary = tb_cbk.writer.epoch_summary
+    self.assertEqual(epoch_step, 0)
+    self.assertEqual(epoch_summary.value[0].simple_value, 10.0)
+
   def test_RemoteMonitorWithJsonPayload(self):
     if requests is None:
       self.skipTest('`requests` required to run this test')
-- 
cgit v1.2.3


From 3b6bceb87f91fc2d0e0c7d31d3583c39f2d3ca8d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 10:27:51 -0700
Subject: fixing some nits

PiperOrigin-RevId: 205416917
---
 .../python/examples/generative_examples/dcgan.ipynb  | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
index 232f9a8ef0..54cc4dc5da 100644
--- a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
@@ -27,9 +27,9 @@
         "id": "ITZuApL56Mny"
       },
       "source": [
-        "This notebook demonstrates how to generate images of handwritten digits using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). To do this, we use Deep Convolutional Generative Adverserial Networks ([DCGAN](https://arxiv.org/pdf/1511.06434.pdf)).\n",
+        "This notebook demonstrates how to generate images of handwritten digits using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). To do so, we use Deep Convolutional Generative Adverserial Networks ([DCGAN](https://arxiv.org/pdf/1511.06434.pdf)).\n",
         "\n",
-        "On a colab GPU(Tesla K80), the model takes around 40 seconds per epoch to train.\n",
+        "This model takes about 40 seconds per epoch to train on a single Tesla K80 on Colab, as of July 2018.\n",
         "\n",
         "Below is the output generated after training the generator and discriminator models for 150 epochs.\n",
         "\n",
@@ -80,6 +80,8 @@
       },
       "outputs": [],
       "source": [
+        "from __future__ import absolute_import, division, print_function\n",
+        "\n",
         "# Import TensorFlow \u003e= 1.9 and enable eager execution\n",
         "import tensorflow as tf\n",
         "tf.enable_eager_execution()\n",
@@ -202,12 +204,12 @@
         "\n",
         "* **Generator** \n",
         "  * It is responsible for **creating the convincing images good enough to fool the discriminator**.\n",
-        "  * It consists of Conv2DTranspose(Upsampling) layers. We start with a fully connected layer and upsample the image 2 times so as to reach the desired image size(mnist image size) which is (28, 28, 1). \n",
+        "  * It consists of Conv2DTranspose (Upsampling) layers. We start with a fully connected layer and upsample the image 2 times so as to reach the desired image size (mnist image size) which is (28, 28, 1). \n",
         "  * We use **leaky relu** activation except for the **last layer** which uses **tanh** activation.\n",
         "  \n",
         "* **Discriminator**\n",
         "  * **The discriminator is responsible for classifying the fake images from the real images.**\n",
-        "  * In other words, the discriminator is given generated images(from the generator) and the real MNIST images. The job of the discriminator is to classify these images into fake(generated) and real(MNIST images).\n",
+        "  * In other words, the discriminator is given generated images (from the generator) and the real MNIST images. The job of the discriminator is to classify these images into fake (generated) and real (MNIST images).\n",
         "  * **Basically the generator should be good enough to fool the discriminator that the generated images are real**."
       ]
     },
@@ -323,8 +325,8 @@
         "\n",
         "* **Discriminator loss**\n",
         "  * The discriminator loss function takes 2 inputs; **real images, generated images**\n",
-        "  * real_loss is a sigmoid cross entropy loss of the **real images** and an **array of ones(since these are the real images)**\n",
-        "  * generated_loss is a sigmoid cross entropy loss of the **generated images** and an **array of zeros(since these are the fake images)**\n",
+        "  * real_loss is a sigmoid cross entropy loss of the **real images** and an **array of ones (since these are the real images)**\n",
+        "  * generated_loss is a sigmoid cross entropy loss of the **generated images** and an **array of zeros (since these are the fake images)**\n",
         "  * Then the total_loss is the sum of real_loss and the generated_loss\n",
         "  \n",
         "* **Generator loss**\n",
@@ -411,9 +413,9 @@
         "\n",
         "* We start by iterating over the dataset\n",
         "* The generator is given **noise as an input** which when passed through the generator model will output a image looking like a handwritten digit\n",
-        "* The discriminator is given the **real MNIST images as well as the generated images(from the generator)**.\n",
+        "* The discriminator is given the **real MNIST images as well as the generated images (from the generator)**.\n",
         "* Next, we calculate the generator and the discriminator loss.\n",
-        "* Then, we calculate the gradients of loss with respect to both the generator and the discriminator variables(inputs) and apply those to the optimizer.\n",
+        "* Then, we calculate the gradients of loss with respect to both the generator and the discriminator variables (inputs) and apply those to the optimizer.\n",
         "\n",
         "## Generate Images\n",
         "\n",
@@ -442,7 +444,7 @@
         "noise_dim = 100\n",
         "num_examples_to_generate = 100\n",
         "\n",
-        "# keeping the random vector constant for generation(prediction) so\n",
+        "# keeping the random vector constant for generation (prediction) so\n",
         "# it will be easier to see the improvement of the gan.\n",
         "random_vector_for_generation = tf.random_normal([num_examples_to_generate,\n",
         "                                                 noise_dim])"
-- 
cgit v1.2.3


From f965075fca32a41b544f8101a94641f6d322f1ec Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 10:31:03 -0700
Subject: Internal change.

PiperOrigin-RevId: 205417414
---
 tensorflow/contrib/lite/kernels/fully_connected.cc | 2 +-
 tensorflow/contrib/lite/kernels/lstm.cc            | 4 ++--
 tensorflow/contrib/lite/kernels/svdf.cc            | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/fully_connected.cc b/tensorflow/contrib/lite/kernels/fully_connected.cc
index 3b203dd480..d6e297a66a 100644
--- a/tensorflow/contrib/lite/kernels/fully_connected.cc
+++ b/tensorflow/contrib/lite/kernels/fully_connected.cc
@@ -71,7 +71,7 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
   // Instead, we allocate a new object to carry information from Prepare() to
   // Eval().
   gemm_support::IncrementUsageCounter(context);
-  auto* op_data = new OpData;
+  auto* op_data = new OpData();
   context->AddTensors(context, 1, &op_data->input_quantized_index);
   return op_data;
 }
diff --git a/tensorflow/contrib/lite/kernels/lstm.cc b/tensorflow/contrib/lite/kernels/lstm.cc
index 4dfc891548..50487a8d59 100644
--- a/tensorflow/contrib/lite/kernels/lstm.cc
+++ b/tensorflow/contrib/lite/kernels/lstm.cc
@@ -97,7 +97,7 @@ constexpr int kCellStateTensor = 1;
 constexpr int kOutputTensor = 2;
 
 void* Init(TfLiteContext* context, const char* buffer, size_t length) {
-  auto* op_data = new OpData;
+  auto* op_data = new OpData();
   op_data->kernel_type = kTfLiteLSTMFullKernel;
   context->AddTensors(context, /*tensors_to_add=*/7,
                       &op_data->scratch_tensor_index);
@@ -847,7 +847,7 @@ enum OutputTensor {
 };
 
 void* Init(TfLiteContext* context, const char* buffer, size_t length) {
-  auto* op_data = new OpData;
+  auto* op_data = new OpData();
   op_data->kernel_type = kTfLiteLSTMBasicKernel;
   // `scratch_tensor_index` is unused in this kernel.
   op_data->scratch_tensor_index = -1;
diff --git a/tensorflow/contrib/lite/kernels/svdf.cc b/tensorflow/contrib/lite/kernels/svdf.cc
index 22eebdd4ce..179c2dc266 100644
--- a/tensorflow/contrib/lite/kernels/svdf.cc
+++ b/tensorflow/contrib/lite/kernels/svdf.cc
@@ -105,7 +105,7 @@ constexpr int kStateTensor = 0;
 constexpr int kOutputTensor = 1;
 
 void* Init(TfLiteContext* context, const char* buffer, size_t length) {
-  auto* op_data = new OpData;
+  auto* op_data = new OpData();
   op_data->float_weights_time_initialized = false;
   context->AddTensors(context, /*tensors_to_add=*/4,
                       &op_data->scratch_tensor_index);
-- 
cgit v1.2.3


From 8ed40cdd3ea7e9aea996339678efba2b2b04e1ad Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 10:55:31 -0700
Subject: Fix grammar in configure.py

PiperOrigin-RevId: 205421605
---
 configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index 251bebc2e1..1df7bc736f 100644
--- a/configure.py
+++ b/configure.py
@@ -1422,7 +1422,7 @@ def set_windows_build_flags(environ_cp):
       environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline',
       True,
       ('Would you like to override eigen strong inline for some C++ '
-       'compilation to reduce the compiling time?'),
+       'compilation to reduce the compilation time?'),
       'Eigen strong inline overridden.',
       'Not overriding eigen strong inline, '
       'some compilations could take more than 20 mins.'):
-- 
cgit v1.2.3


From da929851e2b5446a5aaee29a869428037a72f2b7 Mon Sep 17 00:00:00 2001
From: Anjali Sridhar 
Date: Fri, 20 Jul 2018 11:11:42 -0700
Subject: Refactor properties and functions common to Mirrored and TowerLocal
 Variables.

PiperOrigin-RevId: 205424692
---
 tensorflow/contrib/distribute/python/values.py | 89 ++++++++++----------------
 1 file changed, 33 insertions(+), 56 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index 1761a43251..3162aebf5b 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -196,10 +196,43 @@ class DistributedVariable(DistributedDelegate):
     # to the container without introducing a reference cycle.
     for v in six.itervalues(index):
       v._distributed_container = weakref.ref(self)  # pylint: disable=protected-access
+    # tf.keras keeps track of variables initialized using this attribute. When
+    # tf.keras gets the default session, it initializes all uninitialized vars.
+    # We need to make _keras_initialized a member of DistributedVariable because
+    # without this it will use `__getattr__` which will delegate to a component
+    # variable.
+    self._keras_initialized = False
     super(DistributedVariable, self).__init__(index)
 
+  def is_initialized(self, name=None):
+    """Identifies if all the component variables are initialized.
+
+    Args:
+      name: Name of the final `logical_and` op.
+
+    Returns:
+      The op that evaluates to True or False depending on if all the
+      component variables are initialized.
+    """
+    # We have to cast the self._index.values() to a `list` because when we
+    # use `model_to_estimator` to run tf.keras models, self._index.values() is
+    # of type `dict_values` and not `list`.
+    values_list = list(self._index.values())
+    result = values_list[0].is_initialized()
+    # We iterate through the list of values except the last one to allow us to
+    # name the final `logical_and` op the same name that is passed by the user
+    # to the `is_initialized` op. For distributed variables, the
+    # `is_initialized` op is a `logical_and` op.
+    for v in values_list[1:-1]:
+      result = math_ops.logical_and(result, v.is_initialized())
+    result = math_ops.logical_and(result, values_list[-1].is_initialized(),
+                                  name=name)
+    return result
+
   @property
   def initializer(self):
+    # return grouped ops of all the var initializations of component values of
+    # the mirrored variable
     return control_flow_ops.group([v.initializer for v in self._index.values()])
 
   @property
@@ -296,12 +329,6 @@ class MirroredVariable(DistributedVariable, Mirrored,
     for v in six.itervalues(index):
       v._mirrored_container = weakref.ref(self)  # pylint: disable=protected-access
     self._primary_var = primary_var
-    # tf.keras keeps track of variables initialized using this attribute. When
-    # tf.keras gets the default session, it initializes all uninitialized vars.
-    # We need to make _keras_initialized a member of MirroredVariable because
-    # without this it will use `__getattr__` which will delegate to a component
-    # variable.
-    self._keras_initialized = False
     self._aggregation = aggregation
     super(MirroredVariable, self).__init__(index)
 
@@ -357,28 +384,6 @@ class MirroredVariable(DistributedVariable, Mirrored,
     assign_fn = lambda var, *a, **kw: var.assign(*a, **kw)
     return self._assign_func(f=assign_fn, *args, **kwargs)
 
-  def is_initialized(self, name=None):
-    # We have to cast the self._index.values() to a `list` because when we
-    # use `model_to_estimator` to run tf.keras models, self._index.values() is
-    # of type `dict_values` and not `list`.
-    values_list = list(self._index.values())
-    result = values_list[0].is_initialized()
-    # We iterate through the list of values except the last one to allow us to
-    # name the final `logical_and` op the same name that is passed by the user
-    # to the `is_initialized` op. For mirrored variables, the `is_initialized`
-    # op is a `logical_and` op.
-    for v in values_list[1:-1]:
-      result = math_ops.logical_and(result, v.is_initialized())
-    result = math_ops.logical_and(result, values_list[-1].is_initialized(),
-                                  name=name)
-    return result
-
-  @property
-  def initializer(self):
-    # return grouped ops of all the var initializations of component values of
-    # the mirrored variable
-    return control_flow_ops.group([v.initializer for v in self._index.values()])
-
   @property
   def aggregation(self):
     return self._aggregation
@@ -466,12 +471,6 @@ class TowerLocalVariable(DistributedVariable, PerDevice,
   def __init__(self, index, primary_var, aggregation):
     self._primary_var = primary_var
     self._aggregation = aggregation
-    # tf.keras keeps track of variables initialized using this attribute. When
-    # tf.keras gets the default session, it initializes all uninitialized vars.
-    # We need to make _keras_initialized a member of TowerLocalVariable because
-    # without this it will use `__getattr__` which will delegate to a component
-    # variable.
-    self._keras_initialized = False
     super(TowerLocalVariable, self).__init__(index)
 
   def assign_sub(self, *args, **kwargs):
@@ -486,28 +485,6 @@ class TowerLocalVariable(DistributedVariable, PerDevice,
     _assert_tower_context()
     return self.get().assign(*args, **kwargs)
 
-  def is_initialized(self, name=None):
-    # We have to cast the self._index.values() to a `list` because when we
-    # use `model_to_estimator` to run tf.keras models, self._index.values() is
-    # of type `dict_values` and not `list`.
-    values_list = list(self._index.values())
-    result = values_list[0].is_initialized()
-    # We iterate through the list of values except the last one to allow us to
-    # name the final `logical_and` op the same name that is passed by the user
-    # to the `is_initialized` op. For tower local variables, the
-    # `is_initialized` op is a `logical_and` op.
-    for v in values_list[1:-1]:
-      result = math_ops.logical_and(result, v.is_initialized())
-    result = math_ops.logical_and(result, values_list[-1].is_initialized(),
-                                  name=name)
-    return result
-
-  @property
-  def initializer(self):
-    # return grouped ops of all the var initializations of component values of
-    # the tower local variable
-    return control_flow_ops.group([v.initializer for v in self._index.values()])
-
   @property
   def aggregation(self):
     return self._aggregation
-- 
cgit v1.2.3


From b47e08b1580033ff63fd7b9a2661e30049afb43d Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Fri, 20 Jul 2018 11:21:44 -0700
Subject: Re-enable the deadness-analysis.

The compile time problem issue has been fixed.

PiperOrigin-RevId: 205426325
---
 tensorflow/compiler/jit/mark_for_compilation_pass.cc | 15 +++++++++++++++
 tensorflow/compiler/jit/xla_fusion_optimizer.cc      | 12 ++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 73db0d5952..a3949bc14b 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include 
 #include 
 
+#include "tensorflow/compiler/jit/deadness_analysis.h"
 #include "tensorflow/compiler/jit/defs.h"
 #include "tensorflow/compiler/jit/graphcycles/graphcycles.h"
 #include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h"
@@ -463,6 +464,12 @@ Status MarkForCompilationPass::Run(
   VLOG(1) << "flags->tf_xla_fusion_only = " << flags->tf_xla_fusion_only;
   const FunctionLibraryDefinition* fld = options.flib_def;
 
+  std::unique_ptr deadness;
+  {
+    XLA_SCOPED_LOGGING_TIMER_LEVEL("DeadnessAnalysis", 1);
+    TF_RETURN_IF_ERROR(DeadnessAnalysis::Run(**options.graph, &deadness));
+  }
+
   auto is_compilable = [&](const Node* node, const DeviceType& device_type) {
     const XlaOpRegistry::DeviceRegistration* registration;
     if (!XlaOpRegistry::GetCompilationDevice(device_type.type(),
@@ -490,6 +497,14 @@ Status MarkForCompilationPass::Run(
     status = fld->GetAttr(*node, kXlaCompileAttr, &compile);
     if (status.ok()) return compile;
 
+    // If inputs to `node` can have conflicting deadness (i.e. some are alive
+    // and some are dead) then don't compile it.  XLA cannot represent the
+    // deadness semantics of these nodes correctly and auto-clustering these
+    // nodes can cause deadness to propagate to nodes that should be live.
+    if (node->IsMerge() || deadness->HasInputsWithMismatchingDeadness(*node)) {
+      return false;
+    }
+
     // Check for fusable ops only if requested.
     if (global_jit_level > 0 && fusion_only && !IsXlaFusable(node->def())) {
       return false;
diff --git a/tensorflow/compiler/jit/xla_fusion_optimizer.cc b/tensorflow/compiler/jit/xla_fusion_optimizer.cc
index 74257b09a8..4b499b1613 100644
--- a/tensorflow/compiler/jit/xla_fusion_optimizer.cc
+++ b/tensorflow/compiler/jit/xla_fusion_optimizer.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include 
 #include 
 
+#include "tensorflow/compiler/jit/deadness_analysis.h"
 #include "tensorflow/compiler/jit/defs.h"
 #include "tensorflow/compiler/jit/graphcycles/graphcycles.h"
 #include "tensorflow/compiler/jit/union_find.h"
@@ -146,6 +147,9 @@ Status XlaFusionOptimizer::Optimize(grappler::Cluster* cluster,
   TF_RETURN_IF_ERROR(
       ImportGraphDef(options, item.graph, &graph, &shape_refiner));
 
+  std::unique_ptr deadness;
+  TF_RETURN_IF_ERROR(DeadnessAnalysis::Run(graph, &deadness));
+
   // Collect nodes that can be fused via XLA, while ignoring those that
   // explicitly ask for XLA: (*) nodes that are marked to be compiled
   // explicitly. (*) nodes assigned to XLA device.
@@ -185,6 +189,14 @@ Status XlaFusionOptimizer::Optimize(grappler::Cluster* cluster,
       continue;
     }
 
+    // If inputs to `node` can have conflicting deadness (i.e. some are alive
+    // and some are dead) then don't compile it.  XLA cannot represent the
+    // deadness semantics of these nodes correctly and auto-clustering these
+    // nodes can cause deadness to propagate to nodes that should be live.
+    if (node->IsMerge() || deadness->HasInputsWithMismatchingDeadness(*node)) {
+      continue;
+    }
+
     compilation_candidates.insert(node);
   }
 
-- 
cgit v1.2.3


From 41b93403ac8148bd880c749165c40840ddb95b44 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 11:26:39 -0700
Subject: Create a config option to not link LGPL

PiperOrigin-RevId: 205427089
---
 tensorflow/BUILD          | 8 ++++++++
 tensorflow/tensorflow.bzl | 8 ++++++++
 2 files changed, 16 insertions(+)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 518c2b0489..388ca3f293 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -24,6 +24,14 @@ load(
     "gen_api_init_files",  # @unused
 )
 
+# Config setting used when building for products
+# which requires restricted licenses to be avoided.
+config_setting(
+    name = "no_lgpl_deps",
+    values = {"define": "__TENSORFLOW_NO_LGPL_DEPS__=1"},
+    visibility = ["//visibility:public"],
+)
+
 # Config setting for determining if we are building for Android.
 config_setting(
     name = "android",
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 955b53f691..954940642b 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -137,6 +137,14 @@ def if_not_mobile(a):
       "//conditions:default": a,
   })
 
+# Config setting selector used when building for products
+# which requires restricted licenses to be avoided.
+def if_not_lgpl_restricted(a):
+  _ = (a,)
+  return select({
+      "//conditions:default": [],
+  })
+
 def if_not_windows(a):
   return select({
       clean_dep("//tensorflow:windows"): [],
-- 
cgit v1.2.3


From 1d3440dea001d7466f6e9f40ddd3afdf94ed8dc4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 12:15:16 -0700
Subject: Add a method for getting in-memory profiles from ProfileContexts

PiperOrigin-RevId: 205434648
---
 tensorflow/python/profiler/profile_context.py      | 33 +++++++++++++++++++---
 tensorflow/python/profiler/profile_context_test.py |  2 ++
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/profiler/profile_context.py b/tensorflow/python/profiler/profile_context.py
index 18eb66ef98..fa4260a712 100644
--- a/tensorflow/python/profiler/profile_context.py
+++ b/tensorflow/python/profiler/profile_context.py
@@ -88,16 +88,19 @@ def _profiled_run(self,
       to_profiles = self.profile_context._profile_candidates()
       for to_prof in to_profiles:
         cmd, opts, _ = to_prof
+        saved_views = self.profile_context._views.setdefault(cmd, {})
         if self.profile_context._debug:
           sys.stderr.write('debug: profiling %s step: %d\n' % (cmd, step))
         if cmd == 'graph':
-          self.profile_context.profiler.profile_graph(opts)
+          saved_views[step] = self.profile_context.profiler.profile_graph(opts)
         elif cmd == 'scope':
-          self.profile_context.profiler.profile_name_scope(opts)
+          saved_views[step] = self.profile_context.profiler.profile_name_scope(
+              opts)
         elif cmd == 'op':
-          self.profile_context.profiler.profile_operations(opts)
+          saved_views[step] = self.profile_context.profiler.profile_operations(
+              opts)
         elif cmd == 'code':
-          self.profile_context.profiler.profile_python(opts)
+          saved_views[step] = self.profile_context.profiler.profile_python(opts)
         else:
           raise ValueError('Unknown cmd: %s\n' % cmd)
       return ret
@@ -185,8 +188,30 @@ class ProfileContext(object):
     self._traced_steps = 0
     self._auto_profiles = []
     self._profiler = None
+    self._views = {}
     self._lock = threading.Lock()
 
+  def get_profiles(self, cmd):
+    """Returns profiling results for each step at which `cmd` was run.
+
+    Args:
+      cmd: string, profiling command used in an `add_auto_profiling` call.
+
+    Returns:
+      dict[int: (MultiGraphNodeProto | GraphNodeProto)]. Keys are steps at which
+      the profiling command was run. Values are the outputs of profiling.
+      For "code" and "op" commands this will be a `MultiGraphNodeProto`, for
+      "scope" and "graph" commands this will be a `GraphNodeProto.
+
+    Raises:
+      ValueError: if `cmd` was never run (either because no session.run call was
+      made or because there was no `add_auto_profiling` call with the specified
+      `cmd`.
+    """
+    if cmd not in self._views:
+      raise ValueError('No autoprofiler for command: {}, was run'.format(cmd))
+    return self._views[cmd]
+
   def add_auto_profiling(self, cmd, options, profile_steps):
     """Traces and profiles at some session run steps.
 
diff --git a/tensorflow/python/profiler/profile_context_test.py b/tensorflow/python/profiler/profile_context_test.py
index a623beee23..107ad443c3 100644
--- a/tensorflow/python/profiler/profile_context_test.py
+++ b/tensorflow/python/profiler/profile_context_test.py
@@ -61,6 +61,8 @@ class ProfilerContextTest(test.TestCase):
               profile_str = f.read()
             gfile.Remove(outfile)
 
+      self.assertEqual(set([15, 50, 100]), set(pctx.get_profiles("op").keys()))
+
     with lib.ProfilerFromFile(
         os.path.join(test.get_temp_dir(), "profile_100")) as profiler:
       profiler.profile_operations(options=opts)
-- 
cgit v1.2.3


From dc9e568f7d4c01636747b13b9f7d12078aa52c24 Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Fri, 20 Jul 2018 12:26:53 -0700
Subject: When in graph mode read will force a tape recording.

This can bypass an incorrect recording which might be generated
in the presence of loop contexts.

PiperOrigin-RevId: 205436238
---
 tensorflow/python/eager/backprop_test.py       | 13 +++++++++++++
 tensorflow/python/ops/resource_variable_ops.py | 10 ++++++++--
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index bdda200ff6..95a3a8b629 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -96,6 +96,19 @@ class BackpropTest(test.TestCase):
     self.assertAllEqual(grads_and_vars[0][0], 1.0)
     self.assertAllEqual(id(grads_and_vars[0][1]), id(x))
 
+  def testGradientInsideLoop(self):
+    with ops.Graph().as_default():
+      v = resource_variable_ops.ResourceVariable(1.0)
+
+      def body(_):
+        _ = v + 1.0  # This reads the variable inside the loop context
+        with backprop.GradientTape() as t:
+          result = v * 2
+        self.assertTrue(t.gradient(result, v) is not None)
+        return 1.0
+
+      control_flow_ops.while_loop(lambda i: False, body, [1.0])
+
   def testWhereGradient(self):
     # Note: where is special because only some of its arguments are of
     # differentiable dtypes.
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 1f56ad25bf..db071e3974 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -742,8 +742,14 @@ class ResourceVariable(variables.RefVariable):
   def _read_variable_op(self):
     if self.trainable:
       tape.watch_variable(self)
-    return gen_resource_variable_ops.read_variable_op(self._handle,
-                                                      self._dtype)
+    result = gen_resource_variable_ops.read_variable_op(self._handle,
+                                                        self._dtype)
+    if not context.executing_eagerly():
+      # Note that if a control flow context is active the input of the read op
+      # might not actually be the handle. This line bypasses it.
+      tape.record_operation(
+          "ReadVariableOp", [result], [self._handle], lambda x: [x])
+    return result
 
   def read_value(self):
     """Constructs an op which reads the value of this variable.
-- 
cgit v1.2.3


From 9f8256a61fcd44eeef7c0bf41c9bb4fddc505ae0 Mon Sep 17 00:00:00 2001
From: Bixia Zheng 
Date: Fri, 20 Jul 2018 12:28:30 -0700
Subject: [XLA] Increase the sample size for PrngTest.Uniformity256.

The original test samples 256 numbers with a range of 256 and performs a
ChiSquare test with 0.05 level of significance. Using the Philox RNG algorithm
on GPU, this test produces error like this:
Expected: (UniformChiSquared(256, 256)) < (293.248), actual: 300.086 vs 293.248

This change increases the sample size for the test to make it pass on the GPU.

PiperOrigin-RevId: 205436438
---
 tensorflow/compiler/xla/tests/prng_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/tests/prng_test.cc b/tensorflow/compiler/xla/tests/prng_test.cc
index 5ebf8344d2..3f98099be6 100644
--- a/tensorflow/compiler/xla/tests/prng_test.cc
+++ b/tensorflow/compiler/xla/tests/prng_test.cc
@@ -177,7 +177,7 @@ XLA_TEST_F(PrngTest, Uniformity108) {
   EXPECT_LT(UniformChiSquared(108, 256), 132.144);
 }
 XLA_TEST_F(PrngTest, Uniformity256) {
-  EXPECT_LT(UniformChiSquared(256, 256), 293.248);
+  EXPECT_LT(UniformChiSquared(256, 512), 293.248);
 }
 
 XLA_TEST_F(PrngTest, MapUsingRng) {
-- 
cgit v1.2.3


From 4efac3283749cb858b7c1bf859f1bf022268932d Mon Sep 17 00:00:00 2001
From: Sunitha Kambhampati 
Date: Fri, 20 Jul 2018 12:53:00 -0700
Subject: Remove dependency on pathlib and workaround it by using
 six.moves.urllib_parse

---
 tensorflow/contrib/summary/summary_ops_test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py
index 77b1c93ff2..4d1807130c 100644
--- a/tensorflow/contrib/summary/summary_ops_test.py
+++ b/tensorflow/contrib/summary/summary_ops_test.py
@@ -17,7 +17,6 @@ from __future__ import division
 from __future__ import print_function
 
 import os
-import pathlib
 import tempfile
 import time
 
@@ -280,7 +279,7 @@ class EagerDbTest(summary_test_util.SummaryDbTest):
 
   def testDbURIOpen(self):
     tmpdb_path = os.path.join(self.get_temp_dir(), 'tmpDbURITest.sqlite')
-    tmpdb_uri = pathlib.Path(tmpdb_path).as_uri()
+    tmpdb_uri = six.moves.urllib_parse.urljoin("file:", tmpdb_path)
     tmpdb_writer = summary_ops.create_db_writer(
         tmpdb_uri,
         "experimentA",
-- 
cgit v1.2.3


From 770e5884629397a19b061eef2b925e0aec23dc14 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 13:16:28 -0700
Subject: Fix minor formatting problem.

PiperOrigin-RevId: 205442775
---
 tensorflow/contrib/lite/g3doc/apis.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/lite/g3doc/apis.md b/tensorflow/contrib/lite/g3doc/apis.md
index a591a353dd..e94a2cc44e 100644
--- a/tensorflow/contrib/lite/g3doc/apis.md
+++ b/tensorflow/contrib/lite/g3doc/apis.md
@@ -53,6 +53,7 @@ typedef enum {
 ```
 
 Failures can be easily verified with:
+
 ```c++
 if (status != kTfLiteOk) {
   // ... error handling here ...
-- 
cgit v1.2.3


From 924a8f24a9b8b8a3b1a561123f3e4cf9ebe91708 Mon Sep 17 00:00:00 2001
From: Amit Patankar 
Date: Fri, 20 Jul 2018 13:19:12 -0700
Subject: Fix the version string in setup.py. The PR seemed to miss it.

PiperOrigin-RevId: 205443195
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index c630ca04b8..2e278aa60b 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -45,7 +45,7 @@ DOCLINES = __doc__.split('\n')
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.9.0-rc0'
+_VERSION = '1.9.0'
 
 REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
-- 
cgit v1.2.3


From 4638518bd3821fb887c59cb82326a77384ad4b69 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 13:23:31 -0700
Subject: Add leaf index modes as an argument, which specifies when output leaf
 indices.

PiperOrigin-RevId: 205443722
---
 .../python/training/functions/gbdt_batch.py            | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
index 1ee7f2395e..643d8d2498 100644
--- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
+++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
@@ -287,7 +287,8 @@ class GradientBoostedDecisionTreeModel(object):
                loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS,
                feature_columns=None,
                use_core_columns=False,
-               output_leaf_index=False):
+               output_leaf_index=False,
+               output_leaf_index_modes=None):
     """Construct a new GradientBoostedDecisionTreeModel function.
 
     Args:
@@ -307,6 +308,9 @@ class GradientBoostedDecisionTreeModel(object):
         used.
       output_leaf_index: A boolean variable indicating whether to output leaf
         index into predictions dictionary.
+      output_leaf_index_modes: A list of modes from (TRAIN, EVAL, INFER) which
+        dictates when leaf indices will be outputted. By default, leaf indices
+        are only outputted in INFER mode.
 
     Raises:
       ValueError: if inputs are not valid.
@@ -404,7 +408,16 @@ class GradientBoostedDecisionTreeModel(object):
         self._learner_config.multi_class_strategy ==
         learner_pb2.LearnerConfig.TREE_PER_CLASS and
         learner_config.num_classes == 2)
+
+    if output_leaf_index_modes is None:
+      output_leaf_index_modes = [learn.ModeKeys.INFER]
+    elif not all(
+        mode in (learn.ModeKeys.TRAIN, learn.ModeKeys.EVAL,
+                 learn.ModeKeys.INFER) for mode in output_leaf_index_modes):
+      raise ValueError("output_leaf_index_modes should only contain ModeKeys.")
+
     self._output_leaf_index = output_leaf_index
+    self._output_leaf_index_modes = output_leaf_index_modes
 
   def _predict_and_return_dict(self, ensemble_handle, ensemble_stamp, mode):
     """Runs prediction and returns a dictionary of the prediction results.
@@ -435,8 +448,7 @@ class GradientBoostedDecisionTreeModel(object):
     # the right stamp.
     with ops.control_dependencies(ensemble_stats):
       leaf_index = None
-      # Only used in infer (predict), not used in train and eval.
-      if self._output_leaf_index and mode == learn.ModeKeys.INFER:
+      if self._output_leaf_index and mode in self._output_leaf_index_modes:
         predictions, _, leaf_index = (
             prediction_ops).gradient_trees_prediction_verbose(
                 ensemble_handle,
-- 
cgit v1.2.3


From 85d2a214fb93921b60383db17b8dbbf013034157 Mon Sep 17 00:00:00 2001
From: Jared Duke 
Date: Fri, 20 Jul 2018 13:32:58 -0700
Subject: Support scalar inputs for reduce ops

PiperOrigin-RevId: 205445091
---
 .../kernels/internal/reference/reference_ops.h     |  5 +++-
 tensorflow/contrib/lite/kernels/internal/types.h   |  8 +++--
 tensorflow/contrib/lite/kernels/reduce.cc          |  4 +++
 tensorflow/contrib/lite/kernels/reduce_test.cc     | 35 ++++++++++++++++++++++
 4 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index 10e23f0b41..ef39be3f91 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -3517,7 +3517,6 @@ inline bool Reduce(const In* input_data, const int* input_dims,
                    Out reducer(const Out current, const In in),
                    Out* output_data) {
   // Reset input iterator.
-  TFLITE_DCHECK(input_num_dims > 0);
   for (int idx = 0; idx < input_num_dims; ++idx) {
     input_iter[idx] = 0;
   }
@@ -3537,6 +3536,10 @@ inline bool ResolveAxis(const int num_dims, const int* axis,
                         const int64_t num_axis, int* out_axis,
                         int* out_num_axis) {
   *out_num_axis = 0;  // Just in case.
+  // Short-circuit axis resolution for scalars; the axis will go unused.
+  if (num_dims == 0) {
+    return true;
+  }
   // o(n^2) is fine since out_num_axis should be really small, mostly <= 4
   for (int64_t idx = 0; idx < num_axis; ++idx) {
     // Handle negative index.
diff --git a/tensorflow/contrib/lite/kernels/internal/types.h b/tensorflow/contrib/lite/kernels/internal/types.h
index fe113dfdd3..c44698b677 100644
--- a/tensorflow/contrib/lite/kernels/internal/types.h
+++ b/tensorflow/contrib/lite/kernels/internal/types.h
@@ -278,7 +278,9 @@ inline tflite::Dims<4> ToRuntimeDims(const tflite::RuntimeShape& array_shape) {
 
 // Gets next index to iterate through a multidimensional array.
 inline bool NextIndex(const int num_dims, const int* dims, int* current) {
-  TFLITE_DCHECK_GT(num_dims, 0);
+  if (num_dims == 0) {
+    return false;
+  }
   TFLITE_DCHECK(dims != nullptr);
   TFLITE_DCHECK(current != nullptr);
   int carry = 1;
@@ -305,7 +307,9 @@ inline bool NextIndex(const int num_dims, const int* dims, int* current) {
 inline size_t ReducedOutputOffset(const int num_dims, const int* dims,
                                   const int* index, const int num_axis,
                                   const int* axis) {
-  TFLITE_DCHECK_GT(num_dims, 0);
+  if (num_dims == 0) {
+    return 0;
+  }
   TFLITE_DCHECK(dims != nullptr);
   TFLITE_DCHECK(index != nullptr);
   size_t offset = 0;
diff --git a/tensorflow/contrib/lite/kernels/reduce.cc b/tensorflow/contrib/lite/kernels/reduce.cc
index 52e4084ff8..e99f67c725 100644
--- a/tensorflow/contrib/lite/kernels/reduce.cc
+++ b/tensorflow/contrib/lite/kernels/reduce.cc
@@ -78,6 +78,10 @@ TfLiteStatus ResizeOutputTensor(TfLiteContext* context, OpContext* op_context) {
   size_t num_axis = NumElements(op_context->axis);
   const TfLiteIntArray* input_dims = op_context->input->dims;
   int input_num_dims = NumDimensions(op_context->input);
+  if (input_num_dims == 0) {
+    return context->ResizeTensor(context, op_context->output,
+                                 TfLiteIntArrayCreate(0));
+  }
   const int* axis = GetTensorData(op_context->axis);
   if (op_context->params->keep_dims) {
     TfLiteIntArray* output_dims = TfLiteIntArrayCreate(input_num_dims);
diff --git a/tensorflow/contrib/lite/kernels/reduce_test.cc b/tensorflow/contrib/lite/kernels/reduce_test.cc
index 7d28931ecd..5d432d34ef 100644
--- a/tensorflow/contrib/lite/kernels/reduce_test.cc
+++ b/tensorflow/contrib/lite/kernels/reduce_test.cc
@@ -22,6 +22,7 @@ namespace tflite {
 namespace {
 
 using ::testing::ElementsAreArray;
+using ::testing::IsEmpty;
 
 class BaseOpModel : public SingleOpModel {
  public:
@@ -197,6 +198,16 @@ TEST(ConstFloatMeanOpTest, KeepDims) {
               ElementsAreArray(ArrayFloatNear({10.5, 12.5, 14.5})));
 }
 
+TEST(ConstFloatMeanOpTest, Scalar) {
+  std::vector data = {3.27};
+  MeanOpConstModel m({TensorType_FLOAT32, {}}, {TensorType_FLOAT32, {}}, {},
+                     {0}, true);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), IsEmpty());
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear({3.27})));
+}
+
 TEST(DynamicFloatMeanOpTest, NotKeepDims) {
   std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
                              9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
@@ -342,6 +353,16 @@ TEST(DynamicFloatSumOpTest, NotKeepDims) {
               ElementsAreArray(ArrayFloatNear({144, 156})));
 }
 
+TEST(ConstFloatSumOpTest, Scalar) {
+  std::vector data = {17.};
+  SumOpConstModel m({TensorType_FLOAT32, {}}, {TensorType_FLOAT32, {}}, {}, {0},
+                    false);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), IsEmpty());
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear({17.})));
+}
+
 TEST(DynamicFloatSumOpTest, KeepDims) {
   std::vector data = {1.0,  2.0,  3.0,  4.0,  5.0,  6.0,  7.0,  8.0,
                              9.0,  10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
@@ -630,6 +651,20 @@ TEST(DynamicUint8MaxOpTest, KeepDims) {
                   ArrayFloatNear({11.1294, 0.862745}, kQuantizedTolerance)));
 }
 
+TEST(DynamicUint8MaxOpTest, Scalar) {
+  float kQuantizedTolerance = GetTolerance(-10.0, 12.0);
+  std::vector data = {11.14};
+  MaxOpDynamicModel m({TensorType_UINT8, {}, -10.0, 12.0},
+                      {TensorType_UINT8, {}, -10.0, 12.0},
+                      {TensorType_INT32, {1}}, true);
+  std::vector axis = {0};
+  m.QuantizeAndPopulate(m.Input(), data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), IsEmpty());
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(ArrayFloatNear({11.1294}, kQuantizedTolerance)));
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
cgit v1.2.3


From 265292420de30f24805d28886d403dc42d3685b3 Mon Sep 17 00:00:00 2001
From: Xuechen Li 
Date: Fri, 20 Jul 2018 13:44:39 -0700
Subject: Add estimator and TPU estimator training script.

PiperOrigin-RevId: 205446896
---
 .../contrib/eager/python/examples/revnet/BUILD     |  36 ++
 .../contrib/eager/python/examples/revnet/blocks.py | 374 ++++++++++++++-------
 .../eager/python/examples/revnet/cifar_input.py    |   2 +-
 .../contrib/eager/python/examples/revnet/config.py |  16 +-
 .../contrib/eager/python/examples/revnet/main.py   |  82 ++---
 .../eager/python/examples/revnet/main_estimator.py | 200 +++++++++++
 .../python/examples/revnet/main_estimator_tpu.py   | 328 ++++++++++++++++++
 .../contrib/eager/python/examples/revnet/revnet.py | 110 ++----
 .../eager/python/examples/revnet/revnet_test.py    |  25 +-
 9 files changed, 905 insertions(+), 268 deletions(-)
 create mode 100644 tensorflow/contrib/eager/python/examples/revnet/main_estimator.py
 create mode 100644 tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py

diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index 0c0e4c0eb9..3316dc1114 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -113,3 +113,39 @@ py_binary(
         "//tensorflow:tensorflow_py",
     ],
 )
+
+py_binary(
+    name = "main_estimator",
+    srcs = ["main_estimator.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":cifar_input",
+        ":main",
+        ":revnet",
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_library(
+    name = "main_estimator_lib",
+    srcs = ["main_estimator.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":cifar_input",
+        ":main",
+        ":revnet",
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_library(
+    name = "main_estimator_tpu_lib",
+    srcs = ["main_estimator_tpu.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":cifar_input",
+        ":main",
+        ":revnet",
+        "//tensorflow:tensorflow_py",
+    ],
+)
diff --git a/tensorflow/contrib/eager/python/examples/revnet/blocks.py b/tensorflow/contrib/eager/python/examples/revnet/blocks.py
index 306096e9f8..639bb06a34 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/blocks.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/blocks.py
@@ -24,6 +24,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+import operator
+
 import tensorflow as tf
 from tensorflow.contrib.eager.python.examples.revnet import ops
 
@@ -45,7 +48,7 @@ class RevBlock(tf.keras.Model):
                bottleneck=False,
                fused=True,
                dtype=tf.float32):
-    """Initialize RevBlock.
+    """Initialization.
 
     Args:
       n_res: number of residual blocks
@@ -99,7 +102,6 @@ class RevBlock(tf.keras.Model):
       if i == 0:
         # First block usually contains downsampling that can't be reversed
         with tf.GradientTape() as tape:
-          x = tf.identity(x)
           tape.watch(x)
           y = block(x, training=training)
 
@@ -121,16 +123,6 @@ class _Residual(tf.keras.Model):
   """Single residual block contained in a _RevBlock. Each `_Residual` object has
   two _ResidualInner objects, corresponding to the `F` and `G` functions in the
   paper.
-
-  Args:
-    filters: output filter size
-    strides: length 2 list/tuple of integers for height and width strides
-    input_shape: length 3 list/tuple of integers
-    batch_norm_first: whether to apply activation and batch norm before conv
-    data_format: tensor data format, "NCHW"/"NHWC",
-    bottleneck: use bottleneck residual if True
-    fused: use fused batch normalization if True
-    dtype: float16, float32, or float64
   """
 
   def __init__(self,
@@ -142,6 +134,18 @@ class _Residual(tf.keras.Model):
                bottleneck=False,
                fused=True,
                dtype=tf.float32):
+    """Initialization.
+
+    Args:
+      filters: output filter size
+      strides: length 2 list/tuple of integers for height and width strides
+      input_shape: length 3 list/tuple of integers
+      batch_norm_first: whether to apply activation and batch norm before conv
+      data_format: tensor data format, "NCHW"/"NHWC",
+      bottleneck: use bottleneck residual if True
+      fused: use fused batch normalization if True
+      dtype: float16, float32, or float64
+    """
     super(_Residual, self).__init__()
 
     self.filters = filters
@@ -196,7 +200,6 @@ class _Residual(tf.keras.Model):
     dy1, dy2 = tf.split(dy, num_or_size_splits=2, axis=self.axis)
 
     with tf.GradientTape(persistent=True) as tape:
-      y = tf.identity(y)
       tape.watch(y)
       y1, y2 = tf.split(y, num_or_size_splits=2, axis=self.axis)
       z1 = y1
@@ -227,131 +230,252 @@ class _Residual(tf.keras.Model):
     return x, dx, grads, vars_
 
 
-def _BottleneckResidualInner(filters,
-                             strides,
-                             input_shape,
-                             batch_norm_first=True,
-                             data_format="channels_first",
-                             fused=True,
-                             dtype=tf.float32):
+# Ideally, the following should be wrapped in `tf.keras.Sequential`, however
+# there are subtle issues with its placeholder insertion policy and batch norm
+class _BottleneckResidualInner(tf.keras.Model):
   """Single bottleneck residual inner function contained in _Resdual.
 
   Corresponds to the `F`/`G` functions in the paper.
   Suitable for training on ImageNet dataset.
-
-  Args:
-    filters: output filter size
-    strides: length 2 list/tuple of integers for height and width strides
-    input_shape: length 3 list/tuple of integers
-    batch_norm_first: whether to apply activation and batch norm before conv
-    data_format: tensor data format, "NCHW"/"NHWC"
-    fused: use fused batch normalization if True
-    dtype: float16, float32, or float64
-
-  Returns:
-    A keras model
   """
 
-  axis = 1 if data_format == "channels_first" else 3
-  model = tf.keras.Sequential()
-  if batch_norm_first:
-    model.add(
-        tf.keras.layers.BatchNormalization(
-            axis=axis, input_shape=input_shape, fused=fused, dtype=dtype))
-    model.add(tf.keras.layers.Activation("relu"))
-  model.add(
-      tf.keras.layers.Conv2D(
-          filters=filters // 4,
-          kernel_size=1,
-          strides=strides,
-          input_shape=input_shape,
-          data_format=data_format,
-          use_bias=False,
-          padding="SAME",
-          dtype=dtype))
-
-  model.add(
-      tf.keras.layers.BatchNormalization(axis=axis, fused=fused, dtype=dtype))
-  model.add(tf.keras.layers.Activation("relu"))
-  model.add(
-      tf.keras.layers.Conv2D(
-          filters=filters // 4,
-          kernel_size=3,
-          strides=(1, 1),
-          data_format=data_format,
-          use_bias=False,
-          padding="SAME",
-          dtype=dtype))
-
-  model.add(
-      tf.keras.layers.BatchNormalization(axis=axis, fused=fused, dtype=dtype))
-  model.add(tf.keras.layers.Activation("relu"))
-  model.add(
-      tf.keras.layers.Conv2D(
-          filters=filters,
-          kernel_size=1,
-          strides=(1, 1),
-          data_format=data_format,
-          use_bias=False,
-          padding="SAME",
-          dtype=dtype))
+  def __init__(self,
+               filters,
+               strides,
+               input_shape,
+               batch_norm_first=True,
+               data_format="channels_first",
+               fused=True,
+               dtype=tf.float32):
+    """Initialization.
+
+    Args:
+      filters: output filter size
+      strides: length 2 list/tuple of integers for height and width strides
+      input_shape: length 3 list/tuple of integers
+      batch_norm_first: whether to apply activation and batch norm before conv
+      data_format: tensor data format, "NCHW"/"NHWC"
+      fused: use fused batch normalization if True
+      dtype: float16, float32, or float64
+    """
+    super(_BottleneckResidualInner, self).__init__()
+    axis = 1 if data_format == "channels_first" else 3
+    if batch_norm_first:
+      self.batch_norm_0 = tf.keras.layers.BatchNormalization(
+          axis=axis, input_shape=input_shape, fused=fused, dtype=dtype)
+
+    self.conv2d_1 = tf.keras.layers.Conv2D(
+        filters=filters // 4,
+        kernel_size=1,
+        strides=strides,
+        input_shape=input_shape,
+        data_format=data_format,
+        use_bias=False,
+        padding="SAME",
+        dtype=dtype)
+    self.batch_norm_1 = tf.keras.layers.BatchNormalization(
+        axis=axis, fused=fused, dtype=dtype)
+
+    self.conv2d_2 = tf.keras.layers.Conv2D(
+        filters=filters // 4,
+        kernel_size=3,
+        strides=(1, 1),
+        data_format=data_format,
+        use_bias=False,
+        padding="SAME",
+        dtype=dtype)
+
+    self.batch_norm_2 = tf.keras.layers.BatchNormalization(
+        axis=axis, fused=fused, dtype=dtype)
+    self.conv2d_3 = tf.keras.layers.Conv2D(
+        filters=filters,
+        kernel_size=1,
+        strides=(1, 1),
+        data_format=data_format,
+        use_bias=False,
+        padding="SAME",
+        dtype=dtype)
+
+    self.batch_norm_first = batch_norm_first
+
+  def call(self, x, training=True):
+    net = x
+    if self.batch_norm_first:
+      net = self.batch_norm_0(net, training=training)
+      net = tf.nn.relu(net)
+
+    net = self.conv2d_1(net)
+    net = self.batch_norm_1(net, training=training)
+    net = tf.nn.relu(net)
+
+    net = self.conv2d_2(net)
+    net = self.batch_norm_2(net, training=training)
+    net = tf.nn.relu(net)
 
-  return model
+    net = self.conv2d_3(net)
 
+    return net
 
-def _ResidualInner(filters,
-                   strides,
-                   input_shape,
-                   batch_norm_first=True,
-                   data_format="channels_first",
-                   fused=True,
-                   dtype=tf.float32):
+
+class _ResidualInner(tf.keras.Model):
   """Single residual inner function contained in _ResdualBlock.
 
   Corresponds to the `F`/`G` functions in the paper.
-
-  Args:
-    filters: output filter size
-    strides: length 2 list/tuple of integers for height and width strides
-    input_shape: length 3 list/tuple of integers
-    batch_norm_first: whether to apply activation and batch norm before conv
-    data_format: tensor data format, "NCHW"/"NHWC"
-    fused: use fused batch normalization if True
-    dtype: float16, float32, or float64
-
-  Returns:
-    A keras model
   """
 
-  axis = 1 if data_format == "channels_first" else 3
-  model = tf.keras.Sequential()
-  if batch_norm_first:
-    model.add(
-        tf.keras.layers.BatchNormalization(
-            axis=axis, input_shape=input_shape, fused=fused, dtype=dtype))
-    model.add(tf.keras.layers.Activation("relu"))
-  model.add(
-      tf.keras.layers.Conv2D(
-          filters=filters,
-          kernel_size=3,
-          strides=strides,
-          input_shape=input_shape,
-          data_format=data_format,
-          use_bias=False,
-          padding="SAME",
-          dtype=dtype))
-
-  model.add(
-      tf.keras.layers.BatchNormalization(axis=axis, fused=fused, dtype=dtype))
-  model.add(tf.keras.layers.Activation("relu"))
-  model.add(
-      tf.keras.layers.Conv2D(
-          filters=filters,
-          kernel_size=3,
-          strides=(1, 1),
-          data_format=data_format,
-          use_bias=False,
+  def __init__(self,
+               filters,
+               strides,
+               input_shape,
+               batch_norm_first=True,
+               data_format="channels_first",
+               fused=True,
+               dtype=tf.float32):
+    """Initialization.
+
+    Args:
+      filters: output filter size
+      strides: length 2 list/tuple of integers for height and width strides
+      input_shape: length 3 list/tuple of integers
+      batch_norm_first: whether to apply activation and batch norm before conv
+      data_format: tensor data format, "NCHW"/"NHWC"
+      fused: use fused batch normalization if True
+      dtype: float16, float32, or float64
+    """
+    super(_ResidualInner, self).__init__()
+    axis = 1 if data_format == "channels_first" else 3
+    if batch_norm_first:
+      self.batch_norm_0 = tf.keras.layers.BatchNormalization(
+          axis=axis, input_shape=input_shape, fused=fused, dtype=dtype)
+    self.conv2d_1 = tf.keras.layers.Conv2D(
+        filters=filters,
+        kernel_size=3,
+        strides=strides,
+        input_shape=input_shape,
+        data_format=data_format,
+        use_bias=False,
+        padding="SAME",
+        dtype=dtype)
+    self.batch_norm_1 = tf.keras.layers.BatchNormalization(
+        axis=axis, fused=fused, dtype=dtype)
+
+    self.conv2d_2 = tf.keras.layers.Conv2D(
+        filters=filters,
+        kernel_size=3,
+        strides=(1, 1),
+        data_format=data_format,
+        use_bias=False,
+        padding="SAME",
+        dtype=dtype)
+
+    self.batch_norm_first = batch_norm_first
+
+  def call(self, x, training=True):
+    net = x
+    if self.batch_norm_first:
+      net = self.batch_norm_0(net, training=training)
+      net = tf.nn.relu(net)
+
+    net = self.conv2d_1(net)
+    net = self.batch_norm_1(net, training=training)
+
+    net = self.conv2d_2(net)
+
+    return net
+
+
+class InitBlock(tf.keras.Model):
+  """Initial block of RevNet."""
+
+  def __init__(self, config):
+    """Initialization.
+
+    Args:
+      config: tf.contrib.training.HParams object; specifies hyperparameters
+    """
+    super(InitBlock, self).__init__()
+    self.config = config
+    self.axis = 1 if self.config.data_format == "channels_first" else 3
+    self.conv2d = tf.keras.layers.Conv2D(
+        filters=self.config.init_filters,
+        kernel_size=self.config.init_kernel,
+        strides=(self.config.init_stride, self.config.init_stride),
+        data_format=self.config.data_format,
+        use_bias=False,
+        padding="SAME",
+        input_shape=self.config.input_shape,
+        dtype=self.config.dtype)
+    self.batch_norm = tf.keras.layers.BatchNormalization(
+        axis=self.axis, fused=self.config.fused, dtype=self.config.dtype)
+    self.activation = tf.keras.layers.Activation("relu")
+
+    if self.config.init_max_pool:
+      self.max_pool = tf.keras.layers.MaxPooling2D(
+          pool_size=(3, 3),
+          strides=(2, 2),
           padding="SAME",
-          dtype=dtype))
+          data_format=self.config.data_format,
+          dtype=self.config.dtype)
+
+  def call(self, x, training=True):
+    net = x
+    net = self.conv2d(net)
+    net = self.batch_norm(net, training=training)
+    net = self.activation(net)
+
+    if self.config.init_max_pool:
+      net = self.max_pool(net)
+
+    return net
+
 
-  return model
+class FinalBlock(tf.keras.Model):
+  """Final block of RevNet."""
+
+  def __init__(self, config):
+    """Initialization.
+
+    Args:
+      config: tf.contrib.training.HParams object; specifies hyperparameters
+
+    Raises:
+      ValueError: Unsupported data format
+    """
+    super(FinalBlock, self).__init__()
+    self.config = config
+    self.axis = 1 if self.config.data_format == "channels_first" else 3
+
+    f = self.config.filters[-1]  # Number of filters
+    r = functools.reduce(operator.mul, self.config.strides, 1)  # Reduce ratio
+    r *= self.config.init_stride
+    if self.config.init_max_pool:
+      r *= 2
+
+    if self.config.data_format == "channels_first":
+      w, h = self.config.input_shape[1], self.config.input_shape[2]
+      input_shape = (f, w // r, h // r)
+    elif self.config.data_format == "channels_last":
+      w, h = self.config.input_shape[0], self.config.input_shape[1]
+      input_shape = (w // r, h // r, f)
+    else:
+      raise ValueError("Data format should be either `channels_first`"
+                       " or `channels_last`")
+    self.batch_norm = tf.keras.layers.BatchNormalization(
+        axis=self.axis,
+        input_shape=input_shape,
+        fused=self.config.fused,
+        dtype=self.config.dtype)
+    self.activation = tf.keras.layers.Activation("relu")
+    self.global_avg_pool = tf.keras.layers.GlobalAveragePooling2D(
+        data_format=self.config.data_format, dtype=self.config.dtype)
+    self.dense = tf.keras.layers.Dense(
+        self.config.n_classes, dtype=self.config.dtype)
+
+  def call(self, x, training=True):
+    net = x
+    net = self.batch_norm(net, training=training)
+    net = self.activation(net)
+    net = self.global_avg_pool(net)
+    net = self.dense(net)
+
+    return net
diff --git a/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py b/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py
index b6d4c35bfd..e9672f13e1 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py
@@ -111,6 +111,6 @@ def get_ds_from_tfrecords(data_dir,
     }[split]
     dataset = dataset.shuffle(size)
 
-  dataset = dataset.batch(batch_size)
+  dataset = dataset.batch(batch_size, drop_remainder=True)
 
   return dataset
diff --git a/tensorflow/contrib/eager/python/examples/revnet/config.py b/tensorflow/contrib/eager/python/examples/revnet/config.py
index 3d93fa955a..1532c7b67b 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/config.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/config.py
@@ -27,17 +27,16 @@ from __future__ import division
 from __future__ import print_function
 
 import tensorflow as tf
-tfe = tf.contrib.eager
 
 
 def get_hparams_cifar_38():
   """RevNet-38 configurations for CIFAR-10/CIFAR-100."""
 
   config = tf.contrib.training.HParams()
+  # Hyperparameters from the RevNet paper
   config.add_hparam("init_filters", 32)
   config.add_hparam("init_kernel", 3)
   config.add_hparam("init_stride", 1)
-  config.add_hparam("n_classes", 10)
   config.add_hparam("n_rev_blocks", 3)
   config.add_hparam("n_res", [3, 3, 3])
   config.add_hparam("filters", [32, 64, 112])
@@ -46,7 +45,7 @@ def get_hparams_cifar_38():
   config.add_hparam("bottleneck", False)
   config.add_hparam("fused", True)
   config.add_hparam("init_max_pool", False)
-  if tfe.num_gpus() > 0:
+  if tf.test.is_gpu_available() > 0:
     config.add_hparam("input_shape", (3, 32, 32))
     config.add_hparam("data_format", "channels_first")
   else:
@@ -71,6 +70,16 @@ def get_hparams_cifar_38():
   config.add_hparam("iters_per_epoch", 50000 // config.batch_size)
   config.add_hparam("epochs", config.max_train_iter // config.iters_per_epoch)
 
+  # Customized TPU hyperparameters due to differing batch size caused by
+  # TPU architecture specifics
+  # Suggested batch sizes to reduce overhead from excessive tensor padding
+  # https://cloud.google.com/tpu/docs/troubleshooting
+  config.add_hparam("tpu_batch_size", 128)
+  config.add_hparam("tpu_eval_batch_size", 1024)
+  config.add_hparam("tpu_iters_per_epoch", 50000 // config.tpu_batch_size)
+  config.add_hparam("tpu_epochs",
+                    config.max_train_iter // config.tpu_iters_per_epoch)
+
   return config
 
 
@@ -101,7 +110,6 @@ def get_hparams_imagenet_56():
   config.add_hparam("init_filters", 128)
   config.add_hparam("init_kernel", 7)
   config.add_hparam("init_stride", 2)
-  config.add_hparam("n_classes", 1000)
   config.add_hparam("n_rev_blocks", 4)
   config.add_hparam("n_res", [2, 2, 2, 2])
   config.add_hparam("filters", [128, 256, 512, 832])
diff --git a/tensorflow/contrib/eager/python/examples/revnet/main.py b/tensorflow/contrib/eager/python/examples/revnet/main.py
index e2f43b03f9..1a4fd45c8b 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/main.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/main.py
@@ -31,8 +31,11 @@ tfe = tf.contrib.eager
 
 def main(_):
   """Eager execution workflow with RevNet trained on CIFAR-10."""
-  config = get_config()
-  ds_train, ds_train_one_shot, ds_validation, ds_test = get_datasets(config)
+  tf.enable_eager_execution()
+
+  config = get_config(config_name=FLAGS.config, dataset=FLAGS.dataset)
+  ds_train, ds_train_one_shot, ds_validation, ds_test = get_datasets(
+      data_dir=FLAGS.data_dir, config=config)
   model = revnet.RevNet(config=config)
   global_step = tf.train.get_or_create_global_step()  # Ensure correct summary
   global_step.assign(1)
@@ -52,23 +55,17 @@ def main(_):
             "with global_step: {}".format(latest_path, global_step.numpy()))
       sys.stdout.flush()
 
-  if FLAGS.manual_grad:
-    print("Using manual gradients.")
-  else:
-    print("Not using manual gradients.")
-  sys.stdout.flush()
-
   for x, y in ds_train:
     train_one_iter(model, x, y, optimizer, global_step=global_step)
 
     if global_step.numpy() % config.log_every == 0:
-      it_train = ds_train_one_shot.make_one_shot_iterator()
       it_test = ds_test.make_one_shot_iterator()
-      acc_train, loss_train = evaluate(model, it_train)
       acc_test, loss_test = evaluate(model, it_test)
 
       if FLAGS.validate:
+        it_train = ds_train_one_shot.make_one_shot_iterator()
         it_validation = ds_validation.make_one_shot_iterator()
+        acc_train, loss_train = evaluate(model, it_train)
         acc_validation, loss_validation = evaluate(model, it_validation)
         print("Iter {}, "
               "training set accuracy {:.4f}, loss {:.4f}; "
@@ -77,11 +74,8 @@ def main(_):
                   global_step.numpy(), acc_train, loss_train, acc_validation,
                   loss_validation, acc_test, loss_test))
       else:
-        print("Iter {}, "
-              "training set accuracy {:.4f}, loss {:.4f}; "
-              "test accuracy {:.4f}, loss {:.4f}".format(
-                  global_step.numpy(), acc_train, loss_train, acc_test,
-                  loss_test))
+        print("Iter {}, test accuracy {:.4f}, loss {:.4f}".format(
+            global_step.numpy(), acc_test, loss_test))
       sys.stdout.flush()
 
       if FLAGS.train_dir:
@@ -103,34 +97,38 @@ def main(_):
       sys.stdout.flush()
 
 
-def get_config():
+def get_config(config_name="revnet-38", dataset="cifar-10"):
   """Return configuration."""
-  print("Config: {}".format(FLAGS.config))
+  print("Config: {}".format(config_name))
   sys.stdout.flush()
   config = {
       "revnet-38": config_.get_hparams_cifar_38(),
       "revnet-110": config_.get_hparams_cifar_110(),
       "revnet-164": config_.get_hparams_cifar_164(),
-  }[FLAGS.config]
+  }[config_name]
 
-  if FLAGS.dataset == "cifar-100":
-    config.n_classes = 100
+  if dataset == "cifar-10":
+    config.add_hparam("n_classes", 10)
+    config.add_hparam("dataset", "cifar-10")
+  else:
+    config.add_hparam("n_classes", 100)
+    config.add_hparam("dataset", "cifar-100")
 
   return config
 
 
-def get_datasets(config):
+def get_datasets(data_dir, config):
   """Return dataset."""
-  if FLAGS.data_dir is None:
+  if data_dir is None:
     raise ValueError("No supplied data directory")
-  if not os.path.exists(FLAGS.data_dir):
-    raise ValueError("Data directory {} does not exist".format(FLAGS.data_dir))
-  if FLAGS.dataset not in ["cifar-10", "cifar-100"]:
-    raise ValueError("Unknown dataset {}".format(FLAGS.dataset))
+  if not os.path.exists(data_dir):
+    raise ValueError("Data directory {} does not exist".format(data_dir))
+  if config.dataset not in ["cifar-10", "cifar-100"]:
+    raise ValueError("Unknown dataset {}".format(config.dataset))
 
-  print("Training on {} dataset.".format(FLAGS.dataset))
+  print("Training on {} dataset.".format(config.dataset))
   sys.stdout.flush()
-  data_dir = os.path.join(FLAGS.data_dir, FLAGS.dataset)
+  data_dir = os.path.join(data_dir, config.dataset)
   if FLAGS.validate:
     # 40k Training set
     ds_train = cifar_input.get_ds_from_tfrecords(
@@ -168,7 +166,7 @@ def get_datasets(config):
         prefetch=config.batch_size)
     ds_validation = None
 
-  # Always compute loss and accuracy on whole training and test set
+  # Always compute loss and accuracy on whole test set
   ds_train_one_shot = cifar_input.get_ds_from_tfrecords(
       data_dir=data_dir,
       split="train_all",
@@ -196,19 +194,11 @@ def get_datasets(config):
 
 def train_one_iter(model, inputs, labels, optimizer, global_step=None):
   """Train for one iteration."""
-  if FLAGS.manual_grad:
-    grads, vars_, loss = model.compute_gradients(inputs, labels, training=True)
-    optimizer.apply_gradients(zip(grads, vars_), global_step=global_step)
-  else:  # For correctness validation
-    with tf.GradientTape() as tape:
-      logits, _ = model(inputs, training=True)
-      loss = model.compute_loss(logits=logits, labels=labels)
-      tf.logging.info("Logits are placed on device: {}".format(logits.device))
-    grads = tape.gradient(loss, model.trainable_variables)
-    optimizer.apply_gradients(
-        zip(grads, model.trainable_variables), global_step=global_step)
+  grads, vars_, logits, loss = model.compute_gradients(
+      inputs, labels, training=True)
+  optimizer.apply_gradients(zip(grads, vars_), global_step=global_step)
 
-  return loss.numpy()
+  return logits, loss
 
 
 def evaluate(model, iterator):
@@ -241,16 +231,14 @@ if __name__ == "__main__":
       "validate",
       default=False,
       help="[Optional] Use the validation set or not for hyperparameter search")
-  flags.DEFINE_boolean(
-      "manual_grad",
-      default=False,
-      help="[Optional] Use manual gradient graph to save memory")
   flags.DEFINE_string(
       "dataset",
       default="cifar-10",
       help="[Optional] The dataset used; either `cifar-10` or `cifar-100`")
   flags.DEFINE_string(
-      "config", default="revnet-38", help="[Optional] Architecture of network.")
+      "config",
+      default="revnet-38",
+      help="[Optional] Architecture of network. "
+      "Other options include `revnet-110` and `revnet-164`")
   FLAGS = flags.FLAGS
-  tf.enable_eager_execution()
   tf.app.run(main)
diff --git a/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py b/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py
new file mode 100644
index 0000000000..c875e8da6d
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py
@@ -0,0 +1,200 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Estimator workflow with RevNet train on CIFAR-10."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from absl import flags
+import tensorflow as tf
+from tensorflow.contrib.eager.python.examples.revnet import cifar_input
+from tensorflow.contrib.eager.python.examples.revnet import main as main_
+from tensorflow.contrib.eager.python.examples.revnet import revnet
+
+
+def model_fn(features, labels, mode, params):
+  """Function specifying the model that is required by the `tf.estimator` API.
+
+  Args:
+    features: Input images
+    labels: Labels of images
+    mode: One of `ModeKeys.TRAIN`, `ModeKeys.EVAL` or 'ModeKeys.PREDICT'
+    params: A dictionary of extra parameter that might be passed
+
+  Returns:
+    An instance of `tf.estimator.EstimatorSpec`
+  """
+
+  inputs = features
+  if isinstance(inputs, dict):
+    inputs = features["image"]
+
+  config = params["config"]
+  model = revnet.RevNet(config=config)
+
+  if mode == tf.estimator.ModeKeys.TRAIN:
+    global_step = tf.train.get_or_create_global_step()
+    learning_rate = tf.train.piecewise_constant(
+        global_step, config.lr_decay_steps, config.lr_list)
+    optimizer = tf.train.MomentumOptimizer(
+        learning_rate, momentum=config.momentum)
+    grads, vars_, logits, loss = model.compute_gradients(
+        inputs, labels, training=True)
+    train_op = optimizer.apply_gradients(
+        zip(grads, vars_), global_step=global_step)
+
+    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
+  else:
+    logits, _ = model(inputs, training=False)
+    predictions = tf.argmax(logits, axis=1)
+    probabilities = tf.nn.softmax(logits)
+    loss = model.compute_loss(labels=labels, logits=logits)
+
+    if mode == tf.estimator.ModeKeys.EVAL:
+      return tf.estimator.EstimatorSpec(
+          mode=mode,
+          loss=loss,
+          eval_metric_ops={
+              "accuracy":
+                  tf.metrics.accuracy(labels=labels, predictions=predictions)
+          })
+
+    else:  # mode == tf.estimator.ModeKeys.PREDICT
+      result = {
+          "classes": predictions,
+          "probabilities": probabilities,
+      }
+
+      return tf.estimator.EstimatorSpec(
+          mode=mode,
+          predictions=predictions,
+          export_outputs={
+              "classify": tf.estimator.export.PredictOutput(result)
+          })
+
+
+def get_input_fn(config, data_dir, split):
+  """Get the input function that is required by the `tf.estimator` API.
+
+  Args:
+    config: Customized hyperparameters
+    data_dir: Directory where the data is stored
+    split: One of `train`, `validation`, `train_all`, and `test`
+
+  Returns:
+    Input function required by the `tf.estimator` API
+  """
+
+  data_dir = os.path.join(data_dir, config.dataset)
+  # Fix split-dependent hyperparameters
+  if split == "train_all" or split == "train":
+    data_aug = True
+    batch_size = config.batch_size
+    epochs = config.epochs
+    shuffle = True
+    prefetch = config.batch_size
+  else:
+    data_aug = False
+    batch_size = config.eval_batch_size
+    epochs = 1
+    shuffle = False
+    prefetch = config.eval_batch_size
+
+  def input_fn():
+    """Input function required by the `tf.estimator.Estimator` API."""
+    return cifar_input.get_ds_from_tfrecords(
+        data_dir=data_dir,
+        split=split,
+        data_aug=data_aug,
+        batch_size=batch_size,
+        epochs=epochs,
+        shuffle=shuffle,
+        prefetch=prefetch,
+        data_format=config.data_format)
+
+  return input_fn
+
+
+def main(argv):
+  FLAGS = argv[0]  # pylint:disable=invalid-name,redefined-outer-name
+  tf.logging.set_verbosity(tf.logging.INFO)
+
+  # RevNet specific configuration
+  config = main_.get_config(config_name=FLAGS.config, dataset=FLAGS.dataset)
+
+  # Estimator specific configuration
+  run_config = tf.estimator.RunConfig(
+      model_dir=FLAGS.train_dir,  # Directory for storing checkpoints
+      tf_random_seed=config.seed,
+      save_summary_steps=config.log_every,
+      save_checkpoints_steps=config.log_every,
+      session_config=None,  # Using default
+      keep_checkpoint_max=100,
+      keep_checkpoint_every_n_hours=10000,  # Using default
+      log_step_count_steps=config.log_every,
+      train_distribute=None  # Default not use distribution strategy
+  )
+
+  # Construct estimator
+  revnet_estimator = tf.estimator.Estimator(
+      model_fn=model_fn,
+      model_dir=FLAGS.train_dir,
+      config=run_config,
+      params={"config": config})
+
+  # Construct input functions
+  train_input_fn = get_input_fn(
+      config=config, data_dir=FLAGS.data_dir, split="train_all")
+  eval_input_fn = get_input_fn(
+      config=config, data_dir=FLAGS.data_dir, split="test")
+
+  # Train and evaluate estimator
+  revnet_estimator.train(input_fn=train_input_fn)
+  revnet_estimator.evaluate(input_fn=eval_input_fn)
+
+  if FLAGS.export:
+    input_shape = (None,) + config.input_shape
+    inputs = tf.placeholder(tf.float32, shape=input_shape)
+    input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn({
+        "image": inputs
+    })
+    revnet_estimator.export_savedmodel(FLAGS.train_dir, input_fn)
+
+
+if __name__ == "__main__":
+  flags.DEFINE_string(
+      "data_dir", default=None, help="Directory to load tfrecords")
+  flags.DEFINE_string(
+      "train_dir",
+      default=None,
+      help="[Optional] Directory to store the training information")
+  flags.DEFINE_string(
+      "dataset",
+      default="cifar-10",
+      help="[Optional] The dataset used; either `cifar-10` or `cifar-100`")
+  flags.DEFINE_boolean(
+      "export",
+      default=False,
+      help="[Optional] Export the model for serving if True")
+  flags.DEFINE_string(
+      "config",
+      default="revnet-38",
+      help="[Optional] Architecture of network. "
+      "Other options include `revnet-110` and `revnet-164`")
+  FLAGS = flags.FLAGS
+  tf.app.run(main=main, argv=[FLAGS])
diff --git a/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py b/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py
new file mode 100644
index 0000000000..f1e1e530df
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py
@@ -0,0 +1,328 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Cloud TPU Estimator workflow with RevNet train on CIFAR-10."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import time
+
+from absl import flags
+import tensorflow as tf
+from tensorflow.contrib.eager.python.examples.revnet import cifar_input
+from tensorflow.contrib.eager.python.examples.revnet import main as main_
+from tensorflow.contrib.eager.python.examples.revnet import revnet
+from tensorflow.contrib.training.python.training import evaluation
+from tensorflow.python.estimator import estimator as estimator_
+
+
+def model_fn(features, labels, mode, params):
+  """Model function required by the `tf.contrib.tpu.TPUEstimator` API.
+
+  Args:
+    features: Input images
+    labels: Labels of images
+    mode: One of `ModeKeys.TRAIN`, `ModeKeys.EVAL` or 'ModeKeys.PREDICT'
+    params: A dictionary of extra parameter that might be passed
+
+  Returns:
+    An instance of `tf.contrib.tpu.TPUEstimatorSpec`
+  """
+
+  inputs = features
+  if isinstance(inputs, dict):
+    inputs = features["image"]
+
+  FLAGS = params["FLAGS"]  # pylint:disable=invalid-name,redefined-outer-name
+  config = params["config"]
+  model = revnet.RevNet(config=config)
+
+  if mode == tf.estimator.ModeKeys.TRAIN:
+    global_step = tf.train.get_or_create_global_step()
+    learning_rate = tf.train.piecewise_constant(
+        global_step, config.lr_decay_steps, config.lr_list)
+    optimizer = tf.train.MomentumOptimizer(
+        learning_rate, momentum=config.momentum)
+
+    if FLAGS.use_tpu:
+      optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)
+
+    # Define gradients
+    grads, vars_, logits, loss = model.compute_gradients(
+        inputs, labels, training=True)
+    train_op = optimizer.apply_gradients(
+        zip(grads, vars_), global_step=global_step)
+
+    names = [v.name for v in model.variables]
+    tf.logging.warn("{}".format(names))
+
+    return tf.contrib.tpu.TPUEstimatorSpec(
+        mode=tf.estimator.ModeKeys.TRAIN, loss=loss, train_op=train_op)
+
+  if mode == tf.estimator.ModeKeys.EVAL:
+    logits, _ = model(inputs, training=False)
+    loss = model.compute_loss(labels=labels, logits=logits)
+
+    def metric_fn(labels, logits):
+      predictions = tf.argmax(logits, axis=1)
+      accuracy = tf.metrics.accuracy(labels=labels, predictions=predictions)
+      return {
+          "accuracy": accuracy,
+      }
+
+    return tf.contrib.tpu.TPUEstimatorSpec(
+        mode=mode, loss=loss, eval_metrics=(metric_fn, [labels, logits]))
+
+  if mode == tf.estimator.ModeKeys.PREDICT:
+    logits, _ = model(inputs, training=False)
+    predictions = {
+        "classes": tf.argmax(logits, axis=1),
+        "probabilities": tf.nn.softmax(logits),
+    }
+
+    return tf.contrib.tpu.TPUEstimatorSpec(
+        mode=mode,
+        predictions=predictions,
+        export_outputs={
+            "classify": tf.estimator.export.PredictOutput(predictions)
+        })
+
+
+def get_input_fn(config, data_dir, split):
+  """Get the input function required by the `tf.contrib.tpu.TPUEstimator` API.
+
+  Args:
+    config: Customized hyperparameters
+    data_dir: Directory where the data is stored
+    split: One of `train`, `validation`, `train_all`, and `test`
+
+  Returns:
+    Input function required by the `tf.contrib.tpu.TPUEstimator` API
+  """
+
+  data_dir = os.path.join(data_dir, config.dataset)
+  # Fix split-dependent hyperparameters
+  if split == "train_all" or split == "train":
+    data_aug = True
+    epochs = config.tpu_epochs
+    shuffle = True
+  else:
+    data_aug = False
+    epochs = 1
+    shuffle = False
+
+  def input_fn(params):
+    """Input function required by the `tf.contrib.tpu.TPUEstimator` API."""
+    batch_size = params["batch_size"]
+    return cifar_input.get_ds_from_tfrecords(
+        data_dir=data_dir,
+        split=split,
+        data_aug=data_aug,
+        batch_size=batch_size,  # per-shard batch size
+        epochs=epochs,
+        shuffle=shuffle,
+        prefetch=batch_size,  # per-shard batch size
+        data_format=config.data_format)
+
+  return input_fn
+
+
+def main(argv):
+  FLAGS = argv[0]  # pylint:disable=invalid-name,redefined-outer-name
+  tf.logging.set_verbosity(tf.logging.INFO)
+
+  # RevNet specific configuration
+  config = main_.get_config(config_name=FLAGS.config, dataset=FLAGS.dataset)
+
+  if FLAGS.use_tpu:
+    tf.logging.info("Using TPU.")
+    tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
+        FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
+  else:
+    tpu_cluster_resolver = None
+
+  # TPU specific configuration
+  tpu_config = tf.contrib.tpu.TPUConfig(
+      # Recommended to be set as number of global steps for next checkpoint
+      iterations_per_loop=FLAGS.iterations_per_loop,
+      num_shards=FLAGS.num_shards)
+
+  # Estimator specific configuration
+  run_config = tf.contrib.tpu.RunConfig(
+      cluster=tpu_cluster_resolver,
+      model_dir=FLAGS.model_dir,
+      session_config=tf.ConfigProto(
+          allow_soft_placement=True, log_device_placement=False),
+      tpu_config=tpu_config,
+  )
+
+  # Construct TPU Estimator
+  estimator = tf.contrib.tpu.TPUEstimator(
+      model_fn=model_fn,
+      use_tpu=FLAGS.use_tpu,
+      train_batch_size=config.tpu_batch_size,
+      eval_batch_size=config.tpu_eval_batch_size,
+      config=run_config,
+      params={
+          "FLAGS": FLAGS,
+          "config": config,
+      })
+
+  # Construct input functions
+  train_input_fn = get_input_fn(
+      config=config, data_dir=FLAGS.data_dir, split="train_all")
+  eval_input_fn = get_input_fn(
+      config=config, data_dir=FLAGS.data_dir, split="test")
+
+  # Disabling a range within an else block currently doesn't work
+  # due to https://github.com/PyCQA/pylint/issues/872
+  # pylint: disable=protected-access
+  if FLAGS.mode == "eval":
+    # TPUEstimator.evaluate *requires* a steps argument.
+    # Note that the number of examples used during evaluation is
+    # --eval_steps * --batch_size.
+    # So if you change --batch_size then change --eval_steps too.
+    eval_steps = 10000 // config.tpu_eval_batch_size
+
+    # Run evaluation when there's a new checkpoint
+    for ckpt in evaluation.checkpoints_iterator(
+        FLAGS.model_dir, timeout=FLAGS.eval_timeout):
+      tf.logging.info("Starting to evaluate.")
+      try:
+        start_timestamp = time.time()  # This time will include compilation time
+        eval_results = estimator.evaluate(
+            input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=ckpt)
+        elapsed_time = int(time.time() - start_timestamp)
+        tf.logging.info("Eval results: %s. Elapsed seconds: %d" %
+                        (eval_results, elapsed_time))
+
+        # Terminate eval job when final checkpoint is reached
+        current_step = int(os.path.basename(ckpt).split("-")[1])
+        if current_step >= config.max_train_iter:
+          tf.logging.info(
+              "Evaluation finished after training step %d" % current_step)
+          break
+
+      except tf.errors.NotFoundError:
+        # Since the coordinator is on a different job than the TPU worker,
+        # sometimes the TPU worker does not finish initializing until long after
+        # the CPU job tells it to start evaluating. In this case, the checkpoint
+        # file could have been deleted already.
+        tf.logging.info(
+            "Checkpoint %s no longer exists, skipping checkpoint" % ckpt)
+
+  else:  # FLAGS.mode == 'train' or FLAGS.mode == 'train_and_eval'
+    current_step = estimator_._load_global_step_from_checkpoint_dir(
+        FLAGS.model_dir)
+    tf.logging.info("Training for %d steps . Current"
+                    " step %d." % (config.max_train_iter, current_step))
+
+    start_timestamp = time.time()  # This time will include compilation time
+    if FLAGS.mode == "train":
+      estimator.train(input_fn=train_input_fn, max_steps=config.max_train_iter)
+    else:
+      eval_steps = 10000 // config.tpu_eval_batch_size
+      assert FLAGS.mode == "train_and_eval"
+      while current_step < config.max_train_iter:
+        # Train for up to steps_per_eval number of steps.
+        # At the end of training, a checkpoint will be written to --model_dir.
+        next_checkpoint = min(current_step + FLAGS.steps_per_eval,
+                              config.max_train_iter)
+        estimator.train(input_fn=train_input_fn, max_steps=next_checkpoint)
+        current_step = next_checkpoint
+
+        # Evaluate the model on the most recent model in --model_dir.
+        # Since evaluation happens in batches of --eval_batch_size, some images
+        # may be consistently excluded modulo the batch size.
+        tf.logging.info("Starting to evaluate.")
+        eval_results = estimator.evaluate(
+            input_fn=eval_input_fn, steps=eval_steps)
+        tf.logging.info("Eval results: %s" % eval_results)
+
+    elapsed_time = int(time.time() - start_timestamp)
+    tf.logging.info("Finished training up to step %d. Elapsed seconds %d." %
+                    (config.max_train_iter, elapsed_time))
+  # pylint: enable=protected-access
+
+
+if __name__ == "__main__":
+  # Cloud TPU Cluster Resolver flags
+  flags.DEFINE_string(
+      "tpu",
+      default=None,
+      help="The Cloud TPU to use for training. This should be either the name "
+      "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 "
+      "url.")
+  flags.DEFINE_string(
+      "tpu_zone",
+      default=None,
+      help="[Optional] GCE zone where the Cloud TPU is located in. If not "
+      "specified, we will attempt to automatically detect the GCE project from "
+      "metadata.")
+  flags.DEFINE_string(
+      "gcp_project",
+      default=None,
+      help="[Optional] Project name for the Cloud TPU-enabled project. If not "
+      "specified, we will attempt to automatically detect the GCE project from "
+      "metadata.")
+
+  # Model specific parameters
+  flags.DEFINE_string(
+      "data_dir", default=None, help="Directory to load tfrecords")
+  flags.DEFINE_string(
+      "model_dir",
+      default=None,
+      help="[Optional] Directory to store the model information")
+  flags.DEFINE_string(
+      "dataset",
+      default="cifar-10",
+      help="[Optional] The dataset used; either `cifar-10` or `cifar-100`")
+  flags.DEFINE_string(
+      "config",
+      default="revnet-38",
+      help="[Optional] Architecture of network. "
+      "Other options include `revnet-110` and `revnet-164`")
+  flags.DEFINE_boolean(
+      "use_tpu", default=True, help="[Optional] Whether to use TPU")
+  flags.DEFINE_integer(
+      "num_shards", default=8, help="Number of shards (TPU chips).")
+  flags.DEFINE_integer(
+      "iterations_per_loop",
+      default=100,
+      help=(
+          "Number of steps to run on TPU before feeding metrics to the CPU."
+          " If the number of iterations in the loop would exceed the number of"
+          " train steps, the loop will exit before reaching"
+          " --iterations_per_loop. The larger this value is, the higher the"
+          " utilization on the TPU."))
+  flags.DEFINE_string(
+      "mode",
+      default="train_and_eval",
+      help="[Optional] Mode to run: train, eval, train_and_eval")
+  flags.DEFINE_integer(
+      "eval_timeout", 60 * 60 * 24,
+      "Maximum seconds between checkpoints before evaluation terminates.")
+  flags.DEFINE_integer(
+      "steps_per_eval",
+      default=1000,
+      help=(
+          "Controls how often evaluation is performed. Since evaluation is"
+          " fairly expensive, it is advised to evaluate as infrequently as"
+          " possible (i.e. up to --train_steps, which evaluates the model only"
+          " after finishing the entire training regime)."))
+  FLAGS = flags.FLAGS
+  tf.app.run(main=main, argv=[FLAGS])
diff --git a/tensorflow/contrib/eager/python/examples/revnet/revnet.py b/tensorflow/contrib/eager/python/examples/revnet/revnet.py
index af0d20fa72..a3c2f7dbec 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/revnet.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/revnet.py
@@ -24,9 +24,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import functools
-import operator
-
 import six
 import tensorflow as tf
 from tensorflow.contrib.eager.python.examples.revnet import blocks
@@ -45,71 +42,9 @@ class RevNet(tf.keras.Model):
     self.axis = 1 if config.data_format == "channels_first" else 3
     self.config = config
 
-    self._init_block = self._construct_init_block()
+    self._init_block = blocks.InitBlock(config=self.config)
+    self._final_block = blocks.FinalBlock(config=self.config)
     self._block_list = self._construct_intermediate_blocks()
-    self._final_block = self._construct_final_block()
-
-  def _construct_init_block(self):
-    init_block = tf.keras.Sequential(
-        [
-            tf.keras.layers.Conv2D(
-                filters=self.config.init_filters,
-                kernel_size=self.config.init_kernel,
-                strides=(self.config.init_stride, self.config.init_stride),
-                data_format=self.config.data_format,
-                use_bias=False,
-                padding="SAME",
-                input_shape=self.config.input_shape,
-                dtype=self.config.dtype),
-            tf.keras.layers.BatchNormalization(
-                axis=self.axis,
-                fused=self.config.fused,
-                dtype=self.config.dtype),
-            tf.keras.layers.Activation("relu"),
-        ],
-        name="init")
-    if self.config.init_max_pool:
-      init_block.add(
-          tf.keras.layers.MaxPooling2D(
-              pool_size=(3, 3),
-              strides=(2, 2),
-              padding="SAME",
-              data_format=self.config.data_format,
-              dtype=self.config.dtype))
-    return init_block
-
-  def _construct_final_block(self):
-    f = self.config.filters[-1]  # Number of filters
-    r = functools.reduce(operator.mul, self.config.strides, 1)  # Reduce ratio
-    r *= self.config.init_stride
-    if self.config.init_max_pool:
-      r *= 2
-
-    if self.config.data_format == "channels_first":
-      w, h = self.config.input_shape[1], self.config.input_shape[2]
-      input_shape = (f, w // r, h // r)
-    elif self.config.data_format == "channels_last":
-      w, h = self.config.input_shape[0], self.config.input_shape[1]
-      input_shape = (w // r, h // r, f)
-    else:
-      raise ValueError("Data format should be either `channels_first`"
-                       " or `channels_last`")
-
-    final_block = tf.keras.Sequential(
-        [
-            tf.keras.layers.BatchNormalization(
-                axis=self.axis,
-                input_shape=input_shape,
-                fused=self.config.fused,
-                dtype=self.config.dtype),
-            tf.keras.layers.Activation("relu"),
-            tf.keras.layers.GlobalAveragePooling2D(
-                data_format=self.config.data_format, dtype=self.config.dtype),
-            tf.keras.layers.Dense(
-                self.config.n_classes, dtype=self.config.dtype)
-        ],
-        name="final")
-    return final_block
 
   def _construct_intermediate_blocks(self):
     # Precompute input shape after initial block
@@ -206,13 +141,20 @@ class RevNet(tf.keras.Model):
       l2_reg: Apply l2 regularization
 
     Returns:
-      list of tuples each being (grad, var) for optimizer to use
+      A tuple with the first entry being a list of all gradients, the second
+      entry being a list of respective variables, the third being the logits,
+      and the forth being the loss
     """
 
-    # Run forward pass to record hidden states; avoid updating running averages
+    # Run forward pass to record hidden states
     vars_and_vals = self.get_moving_stats()
     _, saved_hidden = self.call(inputs, training=training)
-    self.restore_moving_stats(vars_and_vals)
+    if tf.executing_eagerly():
+      # Restore moving averages when executing eagerly to avoid updating twice
+      self.restore_moving_stats(vars_and_vals)
+    else:
+      # Fetch batch norm updates in graph mode
+      updates = self.get_updates_for(inputs)
 
     grads_all = []
     vars_all = []
@@ -220,9 +162,8 @@ class RevNet(tf.keras.Model):
     # Manually backprop through last block
     x = saved_hidden[-1]
     with tf.GradientTape() as tape:
-      x = tf.identity(x)
       tape.watch(x)
-      # Running stats updated below
+      # Running stats updated here
       logits = self._final_block(x, training=training)
       loss = self.compute_loss(logits, labels)
 
@@ -236,6 +177,7 @@ class RevNet(tf.keras.Model):
     for block in reversed(self._block_list):
       y = saved_hidden.pop()
       x = saved_hidden[-1]
+      # Running stats updated here
       dy, grads, vars_ = block.backward_grads_and_vars(
           x, y, dy, training=training)
       grads_all += grads
@@ -247,8 +189,7 @@ class RevNet(tf.keras.Model):
     assert not saved_hidden  # Cleared after backprop
 
     with tf.GradientTape() as tape:
-      x = tf.identity(x)
-      # Running stats updated below
+      # Running stats updated here
       y = self._init_block(x, training=training)
 
     grads_all += tape.gradient(
@@ -259,7 +200,13 @@ class RevNet(tf.keras.Model):
     if l2_reg:
       grads_all = self._apply_weight_decay(grads_all, vars_all)
 
-    return grads_all, vars_all, loss
+    if not tf.executing_eagerly():
+      # Force updates to be executed before gradient computation in graph mode
+      # This does nothing when the function is wrapped in defun
+      with tf.control_dependencies(updates):
+        grads_all[0] = tf.identity(grads_all[0])
+
+    return grads_all, vars_all, logits, loss
 
   def _apply_weight_decay(self, grads, vars_):
     """Update gradients to reflect weight decay."""
@@ -284,8 +231,10 @@ class RevNet(tf.keras.Model):
       n = v.name
       return n.endswith("moving_mean:0") or n.endswith("moving_variance:0")
 
-    for v in filter(_is_moving_var, self.variables):
-      vars_and_vals[v] = v.read_value()
+    device = "/gpu:0" if tf.test.is_gpu_available() else "/cpu:0"
+    with tf.device(device):
+      for v in filter(_is_moving_var, self.variables):
+        vars_and_vals[v] = v.read_value()
 
     return vars_and_vals
 
@@ -297,5 +246,8 @@ class RevNet(tf.keras.Model):
     Args:
       vars_and_vals: The dictionary mapping variables to their previous values.
     """
-    for var_, val in six.iteritems(vars_and_vals):
-      var_.assign(val)
+    device = "/gpu:0" if tf.test.is_gpu_available() else "/cpu:0"
+    with tf.device(device):
+      for var_, val in six.iteritems(vars_and_vals):
+        # `assign` causes a copy to GPU (if variable is already on GPU)
+        var_.assign(val)
diff --git a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
index b0d0a5486d..26b0847523 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
@@ -31,10 +31,11 @@ tfe = tf.contrib.eager
 
 def train_one_iter(model, inputs, labels, optimizer, global_step=None):
   """Train for one iteration."""
-  grads, vars_, loss = model.compute_gradients(inputs, labels, training=True)
+  grads, vars_, logits, loss = model.compute_gradients(
+      inputs, labels, training=True)
   optimizer.apply_gradients(zip(grads, vars_), global_step=global_step)
 
-  return loss
+  return logits, loss
 
 
 class RevNetTest(tf.test.TestCase):
@@ -42,6 +43,8 @@ class RevNetTest(tf.test.TestCase):
   def setUp(self):
     super(RevNetTest, self).setUp()
     config = config_.get_hparams_cifar_38()
+    config.add_hparam("n_classes", 10)
+    config.add_hparam("dataset", "cifar-10")
     # Reconstruction could cause numerical error, use double precision for tests
     config.dtype = tf.float64
     config.fused = False  # Fused batch norm does not support tf.float64
@@ -94,7 +97,7 @@ class RevNetTest(tf.test.TestCase):
   def test_compute_gradients(self):
     """Test `compute_gradients` function."""
     self.model(self.x, training=False)  # Initialize model
-    grads, vars_, loss = self.model.compute_gradients(
+    grads, vars_, logits, loss = self.model.compute_gradients(
         inputs=self.x, labels=self.t, training=True, l2_reg=True)
     self.assertTrue(isinstance(grads, list))
     self.assertTrue(isinstance(vars_, list))
@@ -119,7 +122,7 @@ class RevNetTest(tf.test.TestCase):
   def test_compute_gradients_defun(self):
     """Test `compute_gradients` function with defun."""
     compute_gradients = tfe.defun(self.model.compute_gradients)
-    grads, vars_, _ = compute_gradients(self.x, self.t, training=True)
+    grads, vars_, _, _ = compute_gradients(self.x, self.t, training=True)
     self.assertTrue(isinstance(grads, list))
     self.assertTrue(isinstance(vars_, list))
     self.assertEqual(len(grads), len(vars_))
@@ -131,6 +134,9 @@ class RevNetTest(tf.test.TestCase):
     """Test model training in graph mode."""
     with tf.Graph().as_default():
       config = config_.get_hparams_cifar_38()
+      config.add_hparam("n_classes", 10)
+      config.add_hparam("dataset", "cifar-10")
+
       x = tf.random_normal(
           shape=(self.config.batch_size,) + self.config.input_shape)
       t = tf.random_uniform(
@@ -140,15 +146,10 @@ class RevNetTest(tf.test.TestCase):
           dtype=tf.int32)
       global_step = tf.Variable(0., trainable=False)
       model = revnet.RevNet(config=config)
-      model(x)
-      updates = model.get_updates_for(x)
-
-      x_ = tf.identity(x)
-      grads_all, vars_all, _ = model.compute_gradients(x_, t, training=True)
+      grads_all, vars_all, _, _ = model.compute_gradients(x, t, training=True)
       optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
-      with tf.control_dependencies(updates):
-        train_op = optimizer.apply_gradients(
-            zip(grads_all, vars_all), global_step=global_step)
+      train_op = optimizer.apply_gradients(
+          zip(grads_all, vars_all), global_step=global_step)
 
       with tf.Session() as sess:
         sess.run(tf.global_variables_initializer())
-- 
cgit v1.2.3


From e542062aa1613dc01b82b6378675563160fe0abf Mon Sep 17 00:00:00 2001
From: Nick Desaulniers 
Date: Fri, 20 Jul 2018 13:50:37 -0700
Subject: Start implementation of Iota HLO.

PiperOrigin-RevId: 205447892
---
 tensorflow/compiler/xla/client/lib/numeric_test.cc |  3 ++
 .../compiler/xla/client/xla_client/xla_builder.cc  |  7 +++
 .../compiler/xla/client/xla_client/xla_builder.h   |  3 ++
 tensorflow/compiler/xla/literal.cc                 |  4 +-
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc  |  5 ++
 tensorflow/compiler/xla/service/cpu/ir_emitter.h   |  1 +
 tensorflow/compiler/xla/service/dfs_hlo_visitor.h  |  1 +
 .../xla/service/dfs_hlo_visitor_with_default.h     |  3 ++
 tensorflow/compiler/xla/service/gpu/ir_emitter.cc  |  5 ++
 tensorflow/compiler/xla/service/gpu/ir_emitter.h   |  1 +
 .../compiler/xla/service/hlo_cost_analysis.cc      |  4 ++
 .../compiler/xla/service/hlo_cost_analysis.h       |  1 +
 .../xla/service/hlo_evaluator_typed_visitor.h      | 24 +++++++++
 .../compiler/xla/service/hlo_graph_dumper.cc       |  1 +
 tensorflow/compiler/xla/service/hlo_instruction.cc | 10 ++++
 tensorflow/compiler/xla/service/hlo_instruction.h  |  3 ++
 tensorflow/compiler/xla/service/hlo_opcode.h       |  1 +
 tensorflow/compiler/xla/service/hlo_parser.cc      |  8 +++
 tensorflow/compiler/xla/service/hlo_parser_test.cc | 11 ++++
 tensorflow/compiler/xla/service/hlo_verifier.cc    |  6 +++
 tensorflow/compiler/xla/service/hlo_verifier.h     |  1 +
 .../compiler/xla/service/instruction_fusion.cc     |  1 +
 tensorflow/compiler/xla/tests/BUILD                | 20 +++++++
 tensorflow/compiler/xla/tests/iota_test.cc         | 61 ++++++++++++++++++++++
 .../performance/xla/operation_semantics.md         | 13 +++++
 25 files changed, 197 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/compiler/xla/tests/iota_test.cc

diff --git a/tensorflow/compiler/xla/client/lib/numeric_test.cc b/tensorflow/compiler/xla/client/lib/numeric_test.cc
index bfea3f539d..113d359197 100644
--- a/tensorflow/compiler/xla/client/lib/numeric_test.cc
+++ b/tensorflow/compiler/xla/client/lib/numeric_test.cc
@@ -30,6 +30,9 @@ class NumericTest : public ClientLibraryTestBase {
   void TestMatrixDiagonal();
 };
 
+// TODO(b/64798317): Delete this test case once xla::IotaGen is converted to
+// xla::Iota. This test is already implemented for xla::IotaGen in
+// xla/tests/iota_test.cc.
 XLA_TEST_F(NumericTest, Iota) {
   XlaBuilder builder(TestName());
   Iota(&builder, S32, 10);
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index ced26fc2ed..a9a4b3bc5d 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -2873,4 +2873,11 @@ XlaOp BatchNormGrad(const XlaOp& operand, const XlaOp& scale,
                                           grad_output, epsilon, feature_index);
 }
 
+XlaOp IotaGen(XlaBuilder* builder, PrimitiveType type, int64 size) {
+  HloInstructionProto instr;
+  *instr.mutable_shape() = ShapeUtil::MakeShape(type, {size});
+  return builder->ReportErrorOrReturn(
+      builder->AddInstruction(std::move(instr), HloOpcode::kIota));
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index 445c1e0d77..3c016ebe8f 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -1253,6 +1253,9 @@ class XlaBuilder {
   friend XlaOp Pow(const XlaOp& lhs, const XlaOp& rhs,
                    tensorflow::gtl::ArraySlice broadcast_dimensions);
   friend XlaOp IsFinite(const XlaOp& operand);
+  // TODO(b/64798317): Finish CPU & GPU implementation, then replace xla::Iota
+  // in xla/client/lib/numeric.h with this (renamed to xla::Iota).
+  friend XlaOp IotaGen(XlaBuilder* builder, PrimitiveType type, int64 size);
   friend XlaOp ConvertElementType(const XlaOp& operand,
                                   PrimitiveType new_element_type);
   friend XlaOp BitcastConvertType(const XlaOp& operand,
diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc
index 5db124b5a2..0545deb096 100644
--- a/tensorflow/compiler/xla/literal.cc
+++ b/tensorflow/compiler/xla/literal.cc
@@ -1775,7 +1775,9 @@ void LiteralBase::Piece::WriteToProto(LiteralProto* proto) const {
       // Nothing to do but assign the shape which is done above.
       return;
     default:
-      LOG(FATAL) << "Unhandled primitive type " << subshape().element_type();
+      // TODO(b/111551621): Support serializing more PrimitiveTypes.
+      LOG(FATAL) << "Unhandled primitive type "
+                 << PrimitiveType_Name(subshape().element_type());
   }
 }
 
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 05f431642c..aeab5d8957 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -2546,6 +2546,11 @@ Status IrEmitter::HandleAfterAll(HloInstruction* gen_token) {
   return Status::OK();
 }
 
+Status IrEmitter::HandleIota(HloInstruction* iota) {
+  // TODO(b/64798317): implement iota on CPU.
+  return Unimplemented("Iota is not implemented on CPU.");
+}
+
 Status IrEmitter::FinishVisit(HloInstruction* root) {
   // When this method is called, we should have already emitted an IR value for
   // the root (return) op. The IR value holds the address of the buffer holding
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index 419f19c24d..2840c14303 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -148,6 +148,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status HandleConcatenate(HloInstruction* concatenate) override;
   Status HandleConditional(HloInstruction* conditional) override;
   Status HandleAfterAll(HloInstruction* gen_token) override;
+  Status HandleIota(HloInstruction* iota) override;
   Status FinishVisit(HloInstruction* root) override;
 
   Status Preprocess(HloInstruction* hlo) override;
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
index 51f16bdc94..097fa23027 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
@@ -212,6 +212,7 @@ class DfsHloVisitorBase {
   virtual Status HandleReverse(HloInstructionPtr hlo) = 0;
   virtual Status HandleSort(HloInstructionPtr hlo) = 0;
   virtual Status HandleConstant(HloInstructionPtr hlo) = 0;
+  virtual Status HandleIota(HloInstructionPtr hlo) = 0;
   virtual Status HandleGetTupleElement(HloInstructionPtr hlo) = 0;
   virtual Status HandleReduce(HloInstructionPtr hlo) = 0;
   virtual Status HandleBitcast(HloInstructionPtr hlo) = 0;
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
index 0686ca74af..f4316e0fb7 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
@@ -115,6 +115,9 @@ class DfsHloVisitorWithDefaultBase
   Status HandleConstant(HloInstructionPtr constant) override {
     return DefaultAction(constant);
   }
+  Status HandleIota(HloInstructionPtr iota) override {
+    return DefaultAction(iota);
+  }
   Status HandleGetTupleElement(HloInstructionPtr get_tuple_element) override {
     return DefaultAction(get_tuple_element);
   }
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
index 449a18e710..d7e8be1cf8 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
@@ -784,6 +784,11 @@ Status IrEmitter::HandleBatchNormGrad(HloInstruction*) {
       "to a cudnn CustomCall using CudnnBatchNormRewriter.");
 }
 
+Status IrEmitter::HandleIota(HloInstruction*) {
+  // TODO(b/64798317): implement iota on GPU.
+  return Unimplemented("Iota is not implemented on GPU.");
+}
+
 StatusOr IrEmitter::ComputeNestedElement(
     const HloComputation& computation,
     tensorflow::gtl::ArraySlice parameter_elements) {
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
index 77e48d729c..da03ef831b 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
@@ -96,6 +96,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status HandleBatchNormInference(HloInstruction* batch_norm) override;
   Status HandleBatchNormTraining(HloInstruction* batch_norm) override;
   Status HandleBatchNormGrad(HloInstruction* batch_norm) override;
+  Status HandleIota(HloInstruction* iota) override;
 
   Status FinishVisit(HloInstruction* root) override { return Status::OK(); }
 
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index c49cf7f5db..1f672502f7 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -155,6 +155,10 @@ Status HloCostAnalysis::HandleConstant(const HloInstruction*) {
   return Status::OK();
 }
 
+Status HloCostAnalysis::HandleIota(const HloInstruction*) {
+  return Status::OK();
+}
+
 Status HloCostAnalysis::HandleGetTupleElement(const HloInstruction*) {
   // GetTupleElement forwards a pointer and does not touch each element in the
   // output.
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.h b/tensorflow/compiler/xla/service/hlo_cost_analysis.h
index 0181138a6d..82d650dc7b 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.h
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.h
@@ -52,6 +52,7 @@ class HloCostAnalysis : public ConstDfsHloVisitor {
   Status HandleElementwiseUnary(const HloInstruction* hlo) override;
   Status HandleElementwiseBinary(const HloInstruction* hlo) override;
   Status HandleConstant(const HloInstruction* constant) override;
+  Status HandleIota(const HloInstruction* iota) override;
   Status HandleGetTupleElement(
       const HloInstruction* get_tuple_element) override;
   Status HandleSelect(const HloInstruction* hlo) override;
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
index c0a8ea8bcb..f5e477e115 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_typed_visitor.h
@@ -1997,6 +1997,30 @@ class HloEvaluatorTypedVisitor : public DfsHloVisitorWithDefault {
     return HandleReducePrecision(reduce_precision);
   }
 
+  template ::value ||
+                std::is_same::value ||
+                std::is_same::value>::type* = nullptr>
+  Status HandleIota(HloInstruction* iota) {
+    auto result = MakeUnique(iota->shape());
+    auto data = result->data();
+    std::iota(data.begin(), data.end(), 0);
+    parent_->evaluated_[iota] = std::move(result);
+    return Status::OK();
+  }
+  template ::value ||
+                  std::is_same::value ||
+                  std::is_same::value)>::type* = nullptr>
+  Status HandleIota(HloInstruction* iota) {
+    return InvalidArgument("Unsupported type for iota");
+  }
+  Status HandleIota(HloInstruction* iota) override {
+    return HandleIota(iota);
+  }
+
  private:
   // Creates a vector of multipliers which can be used to create a linear index
   // into shape.
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 57cf34d7de..fd5085bed2 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -948,6 +948,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kGe:
     case HloOpcode::kGt:
     case HloOpcode::kImag:
+    case HloOpcode::kIota:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLe:
     case HloOpcode::kLog:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index ae30d2ad8d..7685c822f4 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -463,6 +463,11 @@ StatusOr> HloInstruction::CreateFromProto(
   return MakeUnique(std::move(literal));
 }
 
+/* static */ std::unique_ptr HloInstruction::CreateIota(
+    const Shape& shape) {
+  return WrapUnique(new HloInstruction(HloOpcode::kIota, shape));
+}
+
 /* static */ std::unique_ptr
 HloInstruction::CreateGetTupleElement(const Shape& shape,
                                       HloInstruction* operand, int64 index) {
@@ -1119,6 +1124,7 @@ std::unique_ptr HloInstruction::CloneWithNewOperands(
     case HloOpcode::kDynamicSlice:
     case HloOpcode::kSort:
     case HloOpcode::kGather:
+    case HloOpcode::kIota:
       clone = CloneWithNewOperandsImpl(shape, new_operands, context);
       break;
     // Unary ops.
@@ -1556,6 +1562,7 @@ bool HloInstruction::IdenticalSlowPath(
     case HloOpcode::kMap:
     case HloOpcode::kSlice:
     case HloOpcode::kConstant:
+    case HloOpcode::kIota:
     case HloOpcode::kTrace:
     case HloOpcode::kFusion:
     case HloOpcode::kRng:
@@ -1576,6 +1583,7 @@ bool HloInstruction::IdenticalSlowPath(
       LOG(FATAL) << "Base class impl called for opcode with subclass: "
                  << opcode();
   }
+  return false;
 }
 
 void HloInstruction::RemoveUser(HloInstruction* user) {
@@ -2300,6 +2308,8 @@ Status HloInstruction::Visit(DfsHloVisitorBase* visitor) {
       return visitor->HandleDomain(this);
     case HloOpcode::kAfterAll:
       return visitor->HandleAfterAll(this);
+    case HloOpcode::kIota:
+      return visitor->HandleIota(this);
 
     // These opcodes are not handled here.
     case HloOpcode::kTrace:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index c6faa69a78..30bff286c2 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -346,6 +346,9 @@ class HloInstruction {
   static std::unique_ptr CreateConstant(
       std::unique_ptr literal);
 
+  // Creates an Iota instruction.
+  static std::unique_ptr CreateIota(const Shape& shape);
+
   // Creates a get tuple element instruction.
   static std::unique_ptr CreateGetTupleElement(
       const Shape& shape, HloInstruction* operand, int64 index);
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h
index 39e12c4815..59e9a5a94a 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.h
+++ b/tensorflow/compiler/xla/service/hlo_opcode.h
@@ -87,6 +87,7 @@ namespace xla {
   V(kHostCompute, "host-compute")                            \
   V(kImag, "imag")                                           \
   V(kInfeed, "infeed")                                       \
+  V(kIota, "iota")                                           \
   V(kIsFinite, "is-finite")                                  \
   V(kLe, "less-than-or-equal-to", kHloOpcodeIsComparison)    \
   V(kLog, "log")                                             \
diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc
index 496eca0739..e8eaf54949 100644
--- a/tensorflow/compiler/xla/service/hlo_parser.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser.cc
@@ -492,6 +492,14 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder,
           HloInstruction::CreateConstant(std::move(literal)));
       break;
     }
+    case HloOpcode::kIota: {
+      if (!ParseOperands(&operands, /*expected_size=*/0) ||
+          !ParseAttributes(attrs)) {
+        return false;
+      }
+      instruction = builder->AddInstruction(HloInstruction::CreateIota(shape));
+      break;
+    }
     // Unary ops.
     case HloOpcode::kAbs:
     case HloOpcode::kRoundNearestAfz:
diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc
index 6ba34cf22a..1f0572c576 100644
--- a/tensorflow/compiler/xla/service/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc
@@ -1004,6 +1004,17 @@ ENTRY CrossReplicaSumWithSubgroups {
   ROOT cross-replica-sum = f32[128,32]{0,1} cross-replica-sum(input), replica_group_ids={0,0,1,1}, barrier="abc", to_apply=add
 }
 
+)"
+},
+// Iota
+{
+"Iota",
+R"(HloModule iota
+
+ENTRY Iota {
+  ROOT iota = f32[100]{0} iota()
+}
+
 )"
 }
   });
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 6a32093b6e..c80c1e0e7d 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -210,6 +210,12 @@ Status ShapeVerifier::HandleConstant(HloInstruction* constant) {
   return CheckShape(constant, constant->literal().shape());
 }
 
+Status ShapeVerifier::HandleIota(HloInstruction* iota) {
+  return ShapeUtil::Rank(iota->shape()) == 1
+             ? Status::OK()
+             : InternalError("Iota only supports arrays of rank 1.");
+}
+
 Status ShapeVerifier::HandleGetTupleElement(HloInstruction* get_tuple_element) {
   return CheckShape(get_tuple_element,
                     ShapeInference::InferGetTupleElementShape(
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.h b/tensorflow/compiler/xla/service/hlo_verifier.h
index 810c66cf02..79f7aa9f4c 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.h
+++ b/tensorflow/compiler/xla/service/hlo_verifier.h
@@ -37,6 +37,7 @@ class ShapeVerifier : public DfsHloVisitor {
   Status HandleSelect(HloInstruction* select) override;
   Status HandleTupleSelect(HloInstruction* tuple_select) override;
   Status HandleConcatenate(HloInstruction* concatenate) override;
+  Status HandleIota(HloInstruction* iota) override;
   Status HandleConvert(HloInstruction* convert) override;
   Status HandleBitcastConvert(HloInstruction* convert) override;
   Status HandleCopy(HloInstruction* copy) override;
diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index da91262130..af07370135 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -73,6 +73,7 @@ bool IsAlwaysDuplicable(const HloInstruction& instruction) {
     case HloOpcode::kGt:
     case HloOpcode::kImag:
     case HloOpcode::kInfeed:
+    case HloOpcode::kIota:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLe:
     case HloOpcode::kLt:
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 6a75aa6794..e840067056 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -2060,3 +2060,23 @@ xla_test(
         "//tensorflow/core:test",
     ],
 )
+
+xla_test(
+    name = "iota_test",
+    srcs = ["iota_test.cc"],
+    blacklisted_backends = [
+        "cpu",
+        "gpu",
+    ],
+    tags = [
+        "enable_for_xla_interpreter",
+    ],
+    deps = [
+        ":client_library_test_base",
+        ":literal_test_util",
+        ":xla_internal_test_main",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+    ],
+)
diff --git a/tensorflow/compiler/xla/tests/iota_test.cc b/tensorflow/compiler/xla/tests/iota_test.cc
new file mode 100644
index 0000000000..f950aa1e8f
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/iota_test.cc
@@ -0,0 +1,61 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include 
+#include 
+
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/core/lib/core/errors.h"
+
+namespace xla {
+namespace {
+
+class IotaTest : public ClientLibraryTestBase {
+ public:
+  explicit IotaTest(se::Platform* platform = nullptr)
+      : ClientLibraryTestBase(platform) {}
+  template 
+  std::vector GetExpected(const int64 num_elements) {
+    std::vector result(num_elements);
+    std::iota(result.begin(), result.end(), 0);
+    return result;
+  }
+};
+
+TEST_F(IotaTest, SimpleR1) {
+  for (int num_elements = 1; num_elements < 10000001; num_elements *= 10) {
+    {
+      XlaBuilder builder(TestName() + "_f32");
+      IotaGen(&builder, F32, num_elements);
+      ComputeAndCompareR1(&builder, GetExpected(num_elements), {},
+                                 ErrorSpec{0.0001});
+    }
+    {
+      XlaBuilder builder(TestName() + "_u32");
+      IotaGen(&builder, U32, num_elements);
+      ComputeAndCompareR1(&builder, GetExpected(num_elements),
+                                  {});
+    }
+    {
+      XlaBuilder builder(TestName() + "_s32");
+      IotaGen(&builder, S32, num_elements);
+      ComputeAndCompareR1(&builder, GetExpected(num_elements),
+                                 {});
+    }
+  }
+}
+
+}  // namespace
+}  // namespace xla
diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md
index 68c427a316..d6fa8ab5f9 100644
--- a/tensorflow/docs_src/performance/xla/operation_semantics.md
+++ b/tensorflow/docs_src/performance/xla/operation_semantics.md
@@ -1293,6 +1293,19 @@ Infeed of the device.
 > which case the compiler will provide information about how the Infeed
 > operations are serialized in the compiled program.
 
+## Iota
+
+ `Iota()` 
+
+Builds a constant literal on device rather than a potentially large host
+transfer.  Creates a rank 1 tensor of values starting at zero and incrementing
+by one.
+
+Arguments          | Type            | Semantics
+------------------ | --------------- | ---------------------------
+`type`             | `PrimitiveType` | type U
+`size`             | `int64`         | The number of elements in the tensor.
+
 ## Map
 
 See also
-- 
cgit v1.2.3


From 8257891f378027a1a7c0403ba6ba0aeb313496a0 Mon Sep 17 00:00:00 2001
From: Katherine Wu 
Date: Fri, 20 Jul 2018 13:59:59 -0700
Subject: Add estimator in contrib that loads its model function from a
 SavedModel.

PiperOrigin-RevId: 205449314
---
 tensorflow/contrib/estimator/BUILD                 |  41 ++
 tensorflow/contrib/estimator/__init__.py           |   5 +
 .../python/estimator/saved_model_estimator.py      | 445 +++++++++++++++++++++
 .../python/estimator/saved_model_estimator_test.py | 369 +++++++++++++++++
 tensorflow/python/estimator/estimator.py           |  62 ++-
 tensorflow/python/framework/importer.py            |   2 +-
 tensorflow/python/framework/meta_graph.py          |  68 +++-
 tensorflow/python/saved_model/loader_impl.py       |  13 +-
 tensorflow/python/saved_model/loader_test.py       |  19 +-
 tensorflow/python/training/saver.py                |  28 +-
 10 files changed, 1017 insertions(+), 35 deletions(-)
 create mode 100644 tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py
 create mode 100644 tensorflow/contrib/estimator/python/estimator/saved_model_estimator_test.py

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 1aa3df8d8d..349f48f7f7 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -28,6 +28,7 @@ py_library(
         ":multi_head",
         ":replicate_model_fn",
         ":rnn",
+        ":saved_model_estimator",
         "//tensorflow:tensorflow_py_no_contrib",
     ],
 )
@@ -465,3 +466,43 @@ py_test(
         "@absl_py//absl/testing:parameterized",
     ],
 )
+
+py_library(
+    name = "saved_model_estimator",
+    srcs = ["python/estimator/saved_model_estimator.py"],
+    deps = [
+        ":export",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:training",
+        "//tensorflow/python/estimator",
+        "//tensorflow/python/estimator:export",
+        "//tensorflow/python/estimator:model_fn",
+        "//tensorflow/python/saved_model",
+    ],
+)
+
+py_test(
+    name = "saved_model_estimator_test",
+    size = "medium",
+    srcs = ["python/estimator/saved_model_estimator_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":export",
+        ":saved_model_estimator",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:metrics",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/estimator",
+        "//tensorflow/python/estimator:export_export",
+        "//tensorflow/python/estimator:export_output",
+        "//tensorflow/python/estimator:model_fn",
+    ],
+)
diff --git a/tensorflow/contrib/estimator/__init__.py b/tensorflow/contrib/estimator/__init__.py
index 09fcfd66a1..e1453ae1d0 100644
--- a/tensorflow/contrib/estimator/__init__.py
+++ b/tensorflow/contrib/estimator/__init__.py
@@ -33,6 +33,8 @@ from tensorflow.contrib.estimator.python.estimator.logit_fns import *
 from tensorflow.contrib.estimator.python.estimator.multi_head import *
 from tensorflow.contrib.estimator.python.estimator.replicate_model_fn import *
 from tensorflow.contrib.estimator.python.estimator.rnn import *
+from tensorflow.contrib.estimator.python.estimator.saved_model_estimator import *
+from tensorflow.python.estimator.export.export import *
 
 from tensorflow.python.util.all_util import remove_undocumented
 # pylint: enable=unused-import,line-too-long,wildcard-import
@@ -70,6 +72,9 @@ _allowed_symbols = [
     'stop_if_higher_hook',
     'stop_if_no_increase_hook',
     'stop_if_no_decrease_hook',
+    'build_raw_supervised_input_receiver_fn',
+    'build_supervised_input_receiver_fn_from_input_fn',
+    'SavedModelEstimator'
 ]
 
 remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)
diff --git a/tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py b/tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py
new file mode 100644
index 0000000000..22188fe663
--- /dev/null
+++ b/tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py
@@ -0,0 +1,445 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Class that creates an Estimator from a SavedModel."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import six
+
+from tensorflow.python.estimator import estimator as estimator_lib
+from tensorflow.python.estimator import model_fn as model_fn_lib
+from tensorflow.python.estimator.export import export as export_lib
+from tensorflow.python.estimator.export import export_output
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.saved_model import constants
+from tensorflow.python.saved_model import loader_impl
+from tensorflow.python.saved_model import signature_constants
+from tensorflow.python.training import checkpoint_utils
+from tensorflow.python.training import monitored_session
+from tensorflow.python.training import training_util
+
+
+class SavedModelEstimator(estimator_lib.Estimator):
+  """Create an Estimator from a SavedModel.
+
+  Only SavedModels exported with
+  `tf.contrib.estimator.export_all_saved_models()` or
+  `tf.estimator.Estimator.export_savedmodel()` are supported for this class.
+
+  Example with `tf.estimator.DNNClassifier`:
+
+  **Step 1: Create and train DNNClassifier.**
+  ```python
+  feature1 = tf.feature_column.embedding_column(
+      tf.feature_column.categorical_column_with_vocabulary_list(
+          key='feature1', vocabulary_list=('green', 'yellow')), dimension=1)
+  feature2 = tf.feature_column.numeric_column(key='feature2', default_value=0.0)
+
+  classifier = tf.estimator.DNNClassifier(
+      hidden_units=[4,2], feature_columns=[feature1, feature2])
+
+  def input_fn():
+    features = {'feature1': tf.constant(['green', 'green', 'yellow']),
+                'feature2': tf.constant([3.5, 4.2, 6.1])}
+    label = tf.constant([1., 0., 0.])
+    return tf.data.Dataset.from_tensors((features, label)).repeat()
+
+  classifier.train(input_fn=input_fn, steps=10)
+  ```
+
+  **Step 2: Export classifier.**
+  First, build functions that specify the expected inputs.
+  ```python
+  # During train and evaluation, both the features and labels should be defined.
+  supervised_input_receiver_fn = (
+      tf.contrib.estimator.build_raw_supervised_input_receiver_fn(
+        {'feature1': tf.placeholder(dtype=tf.string, shape=[None]),
+         'feature2': tf.placeholder(dtype=tf.float32, shape=[None])},
+        tf.placeholder(dtype=tf.float32, shape=[None])))
+
+  # During predict mode, expect to receive a `tf.Example` proto, so a parsing
+  # function is used.
+  serving_input_receiver_fn = (
+      tf.estimator.export.build_parsing_serving_input_receiver_fn(
+          tf.feature_column.make_parse_example_spec([feature1, feature2])))
+  ```
+
+  Next, export the model as a SavedModel. A timestamped directory will be
+  created (for example `/tmp/export_all/1234567890`).
+  ```python
+  # Option 1: Save all modes (train, eval, predict)
+  export_dir = tf.contrib.estimator.export_all_saved_models(
+      classifier, '/tmp/export_all',
+      {tf.estimator.ModeKeys.TRAIN: supervised_input_receiver_fn,
+       tf.estimator.ModeKeys.EVAL: supervised_input_receiver_fn,
+       tf.estimator.ModeKeys.PREDICT: serving_input_receiver_fn})
+
+  # Option 2: Only export predict mode
+  export_dir = classifier.export_savedmodel(
+    '/tmp/export_predict', serving_input_receiver_fn)
+  ```
+
+  **Step 3: Create a SavedModelEstimator from the exported SavedModel.**
+  ```python
+  est = tf.contrib.estimator.SavedModelEstimator(export_dir)
+
+  # If all modes were exported, you can immediately evaluate and predict, or
+  # continue training. Otherwise only predict is available.
+  eval_results = est.evaluate(input_fn=input_fn, steps=1)
+  print(eval_results)
+
+  est.train(input_fn=input_fn, steps=20)
+
+  def predict_input_fn():
+    example = example_pb2.Example()
+    example.features.feature['feature1'].bytes_list.value.extend(['yellow'])
+    example.features.feature['feature2'].float_list.value.extend([1.])
+    return {'inputs':tf.constant([example.SerializeToString()])}
+
+  predictions = est.predict(predict_input_fn)
+  print(next(predictions))
+  ```
+  """
+
+  def __init__(self, saved_model_dir, model_dir=None):
+    """Initialize a SavedModelEstimator.
+
+    The SavedModelEstimator loads its model function and variable values from
+    the graphs defined in the SavedModel. There is no option to pass in
+    `RunConfig` or `params` arguments, because the model function graph is
+    defined statically in the SavedModel.
+
+    Args:
+      saved_model_dir: Directory containing SavedModel protobuf and subfolders.
+      model_dir: Directory to save new checkpoints during training.
+
+    Raises:
+      NotImplementedError: If a DistributionStrategy is defined in the config.
+        Unless the SavedModelEstimator is subclassed, this shouldn't happen.
+    """
+    checkpoint = estimator_lib._get_saved_model_ckpt(saved_model_dir)  # pylint: disable=protected-access
+    vars_to_warm_start = [name for name, _ in
+                          checkpoint_utils.list_variables(checkpoint)]
+    warm_start_settings = estimator_lib.WarmStartSettings(
+        ckpt_to_initialize_from=checkpoint,
+        vars_to_warm_start=vars_to_warm_start)
+
+    super(SavedModelEstimator, self).__init__(
+        model_fn=self._model_fn_from_saved_model, model_dir=model_dir,
+        warm_start_from=warm_start_settings)
+    if self._distribution is not None:
+      raise NotImplementedError(
+          'SavedModelEstimator currently does not support '
+          'DistributionStrategy.')
+    self.saved_model_dir = saved_model_dir
+    self.saved_model_loader = loader_impl.SavedModelLoader(saved_model_dir)
+    self._available_modes = self._extract_available_modes()
+
+  def _extract_available_modes(self):
+    """Return list of modes found in SavedModel."""
+    available_modes = []
+    logging.info('Checking available modes for SavedModelEstimator.')
+    for mode in [model_fn_lib.ModeKeys.TRAIN, model_fn_lib.ModeKeys.EVAL,
+                 model_fn_lib.ModeKeys.PREDICT]:
+      try:
+        self._get_meta_graph_def_for_mode(mode)
+      except RuntimeError:
+        logging.warning('%s mode not found in SavedModel.' % mode)
+        continue
+
+      if self._get_signature_def_for_mode(mode) is not None:
+        available_modes.append(mode)
+
+    logging.info('Available modes for Estimator: %s' % available_modes)
+    return available_modes
+
+  def _validate_mode(self, mode):
+    """Make sure that mode can be run using the SavedModel."""
+    if mode not in self._available_modes:
+      raise RuntimeError('%s mode is not available in the SavedModel. Use '
+                         'saved_model_cli to check that the Metagraph for this '
+                         'mode has been exported.' % mode)
+
+  def _get_meta_graph_def_for_mode(self, mode):
+    tags = model_fn_lib.EXPORT_TAG_MAP[mode]
+    return self.saved_model_loader.get_meta_graph_def_from_tags(tags)
+
+  def _get_signature_def_for_mode(self, mode):
+    meta_graph_def = self._get_meta_graph_def_for_mode(mode)
+    sig_def_key = (signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
+                   if mode == model_fn_lib.ModeKeys.PREDICT else mode)
+    if sig_def_key not in meta_graph_def.signature_def:
+      logging.warning('Metagraph for mode %s was found, but SignatureDef with'
+                      ' key \"%s\" is missing.' % (mode, sig_def_key))
+      return None
+    return meta_graph_def.signature_def[sig_def_key]
+
+  def _create_and_assert_global_step(self, graph):
+    # Do nothing here. The global step variable will be created/loaded from the
+    # SavedModel. If a global step variable were created here, the result
+    # will be two duplicate global step variables, causing issues during
+    # the warm-start phase.
+    # Due to the global variable being created in the model function, this may
+    # cause issues when running DistributionStrategy. Thus, DistributionStrategy
+    # is not yet supported with SavedModelEstimator.
+    pass
+
+  def _model_fn_from_saved_model(self, features, labels, mode):
+    """Load a SavedModel graph and return an EstimatorSpec."""
+    # TODO(kathywu): Model function loads placeholders from the graph. Calling
+    # export_all_saved_models creates another placeholder for the inputs, on top
+    # of the original placeholders. There should be a way to avoid this.
+    self._validate_mode(mode)
+
+    g = ops.get_default_graph()
+    if  training_util.get_global_step(g) is not None:
+      raise RuntimeError(
+          'Graph must not contain a global step tensor before the SavedModel is'
+          ' loaded. Please make sure that the input function does not create a '
+          'global step.')
+
+    # Extract SignatureDef for information about the input and output tensors.
+    signature_def = self._get_signature_def_for_mode(mode)
+
+    # Generate input map for replacing the inputs in the SavedModel graph with
+    # the provided features and labels.
+    input_map = _generate_input_map(signature_def, features, labels)
+
+    # Create a list of the names of output tensors. When the graph is loaded,
+    # names of the output tensors may be remapped. This ensures that the correct
+    # tensors are returned in the EstimatorSpec.
+    output_tensor_names = [
+        value.name for value in six.itervalues(signature_def.outputs)]
+
+    # Load the graph. `output_tensors` contains output `Tensors` in the same
+    # same order as the `output_tensor_names` list.
+    tags = model_fn_lib.EXPORT_TAG_MAP[mode]
+    _, output_tensors = self.saved_model_loader.load_graph(
+        g, tags, input_map=input_map, return_elements=output_tensor_names)
+
+    # Create a scaffold from the MetaGraphDef that contains ops to initialize
+    # the graph. This should mirror the steps from _add_meta_graph_for_mode(),
+    # which creates a MetaGraphDef from the EstimatorSpec's scaffold.
+    scaffold = monitored_session.Scaffold(
+        local_init_op=loader_impl._get_legacy_init_op_tensor(  # pylint: disable=protected-access
+            self._get_meta_graph_def_for_mode(mode)))
+
+    # Ensure that a global step tensor has been created.
+    global_step_tensor = training_util.get_global_step(g)
+    training_util.assert_global_step(global_step_tensor)
+
+    # Extract values to return in the EstimatorSpec.
+    output_map = dict(zip(output_tensor_names, output_tensors))
+    outputs = {key: output_map[value.name]
+               for key, value in six.iteritems(signature_def.outputs)}
+
+    loss, predictions, metrics = _validate_and_extract_outputs(
+        mode, outputs, signature_def.method_name)
+
+    train_op = ops.get_collection(constants.TRAIN_OP_KEY)
+    if len(train_op) > 1:
+      raise RuntimeError('Multiple ops found in the train_op collection.')
+    train_op = None if not train_op else train_op[0]
+
+    _clear_saved_model_collections()
+    return model_fn_lib.EstimatorSpec(
+        scaffold=scaffold,
+        mode=mode,
+        loss=loss,
+        train_op=train_op,
+        predictions=predictions,
+        eval_metric_ops=metrics)
+
+
+def _clear_saved_model_collections():
+  """Clear collections that are expected empty when exporting a SavedModel.
+
+  The SavedModel builder uses these collections to track ops necessary to
+  restore the graph state. These collections are expected to be empty before
+  MetaGraphs are added to the builder.
+  """
+  del ops.get_collection_ref(constants.ASSETS_KEY)[:]
+  del ops.get_collection_ref(constants.LEGACY_INIT_OP_KEY)[:]
+  del ops.get_collection_ref(constants.MAIN_OP_KEY)[:]
+  del ops.get_collection_ref(constants.TRAIN_OP_KEY)[:]
+
+
+def _generate_input_map(signature_def, features, labels):
+  """Return dict mapping an input tensor name to a feature or label tensor.
+
+  Args:
+    signature_def: SignatureDef loaded from SavedModel
+    features: A `Tensor`, `SparseTensor`, or dict of string to `Tensor` or
+      `SparseTensor`, specifying the features to be passed to the model.
+    labels: A `Tensor`, `SparseTensor`, or dict of string to `Tensor` or
+      `SparseTensor`, specifying the labels to be passed to the model. May be
+      `None`.
+
+  Returns:
+    dict mapping string names of inputs to features or labels tensors
+
+  Raises:
+    ValueError: if SignatureDef inputs are not completely mapped by the input
+      features and labels.
+  """
+  # pylint: disable=protected-access
+  if not isinstance(features, dict):
+    features = {export_lib._SINGLE_FEATURE_DEFAULT_NAME: features}
+  if labels is not None and not isinstance(labels, dict):
+    labels = {export_lib._SINGLE_LABEL_DEFAULT_NAME: labels}
+  # pylint: enable=protected-access
+
+  inputs = signature_def.inputs
+  input_map = {}
+  for key, tensor_info in six.iteritems(inputs):
+    input_name = tensor_info.name
+    if ':' in input_name:
+      input_name = input_name[:input_name.find(':')]
+
+    # When tensors are used as control inputs for operations, their names are
+    # prepended with a '^' character in the GraphDef. To handle possible control
+    # flow edge cases, control input names must be included in the input map.
+    control_dependency_name = '^' + input_name
+
+    if key in features:
+      _check_same_dtype_and_shape(features[key], tensor_info, key)
+      input_map[input_name] = input_map[control_dependency_name] = features[key]
+    elif labels is not None and key in labels:
+      _check_same_dtype_and_shape(labels[key], tensor_info, key)
+      input_map[input_name] = input_map[control_dependency_name] = labels[key]
+    else:
+      raise ValueError(
+          'Key \"%s\" not found in features or labels passed in to the model '
+          'function. All required keys: %s' % (key, inputs.keys()))
+
+  return input_map
+
+
+def _check_same_dtype_and_shape(tensor, tensor_info, name):
+  """Validate that tensor has the same properties as the TensorInfo proto.
+
+  Args:
+    tensor: a `Tensor` object.
+    tensor_info: a `TensorInfo` proto.
+    name: Name of the input (to identify Tensor if an error is raised).
+
+  Raises:
+    ValueError: If the tensor shape or dtype don't match the TensorInfo
+  """
+  dtype_error = (tensor.dtype != dtypes.DType(tensor_info.dtype))
+  shape_error = not tensor.shape.is_compatible_with(tensor_info.tensor_shape)
+
+  if dtype_error or shape_error:
+    msg = 'Tensor shape and/or dtype validation failed for input %s:' % name
+    if dtype_error:
+      msg += ('\n\tExpected dtype: %s, Got: %s'
+              % (dtypes.DType(tensor_info.dtype), tensor.dtype))
+    if shape_error:
+      msg += ('\n\tExpected shape: %s, Got: %s'
+              % (tensor_shape.TensorShape(tensor_info.tensor_shape),
+                 tensor.shape))
+
+    raise ValueError(msg)
+
+
+def _extract_eval_metrics(output_dict):
+  """Return a eval metric dict extracted from the output_dict.
+
+  Eval metrics consist of a value tensor and an update op. Both must be in the
+  passed-in tensor dictionary for an eval metric to be added to the returned
+  dictionary.
+
+  Args:
+    output_dict: a dict that maps strings to tensors.
+
+  Returns:
+    dict mapping strings to (value, update_op) tuples.
+  """
+  # pylint: disable=protected-access
+  metric_ops = {}
+  separator_char = export_output._SupervisedOutput._SEPARATOR_CHAR
+
+  for key, tensor in six.iteritems(output_dict):
+    split_key = key.split(separator_char)
+
+    # The metric name may contain the separator character, so recreate its name.
+    metric_name = separator_char.join(split_key[:-1])
+
+    if split_key[0] == export_output._SupervisedOutput.METRICS_NAME:
+      # If the key ends with the value suffix, and there is a corresponding
+      # key ending with the update_op suffix, then add tensors to metrics dict.
+      if split_key[-1] == export_output._SupervisedOutput.METRIC_VALUE_SUFFIX:
+        update_op = ''.join(
+            [metric_name, separator_char,
+             export_output._SupervisedOutput.METRIC_UPDATE_SUFFIX])
+        if update_op in output_dict:
+          update_op_tensor = output_dict[update_op]
+          metric_ops[metric_name] = (tensor, update_op_tensor)
+
+  # pylint: enable=protected-access
+  return metric_ops
+
+
+def _validate_and_extract_outputs(mode, output_dict, method_name):
+  """Extract values from SignatureDef output dictionary.
+
+  Args:
+    mode: One of the modes enumerated in `tf.estimator.ModeKeys`.
+    output_dict: dict of string SignatureDef keys to `Tensor`.
+    method_name: Method name of the SignatureDef as a string.
+
+  Returns:
+    Tuple of (
+      loss: `Tensor` object,
+      predictions: dictionary mapping string keys to `Tensor` objects,
+      metrics: dictionary mapping string keys to a tuple of two `Tensor` objects
+    )
+
+  Raises:
+    RuntimeError: raised if SignatureDef has an invalid method name for the mode
+  """
+  # pylint: disable=protected-access
+  loss, predictions, metrics = None, None, None
+
+  if mode == model_fn_lib.ModeKeys.PREDICT:
+    predictions = output_dict
+  else:
+    # Validate that the SignatureDef's method name matches the expected name for
+    # the given mode.
+    expected_method_name = signature_constants.SUPERVISED_TRAIN_METHOD_NAME
+    if mode == model_fn_lib.ModeKeys.EVAL:
+      expected_method_name = signature_constants.SUPERVISED_EVAL_METHOD_NAME
+    if method_name != expected_method_name:
+      raise RuntimeError(
+          'Invalid SignatureDef method name for mode %s.\n\tExpected: %s\n\t'
+          'Got: %s\nPlease ensure that the SavedModel was exported with '
+          '`tf.contrib.estimator.export_all_saved_models()`.' %
+          (mode, expected_method_name, method_name))
+
+    # Extract loss, metrics and predictions from the output dict.
+    loss = output_dict[export_output._SupervisedOutput.LOSS_NAME]
+    metrics = _extract_eval_metrics(output_dict)
+    predictions = {
+        key: value for key, value in six.iteritems(output_dict)
+        if key.split(export_output._SupervisedOutput._SEPARATOR_CHAR)[0] == (
+            export_output._SupervisedOutput.PREDICTIONS_NAME)}
+
+  # pylint: enable=protected-access
+  return loss, predictions, metrics
diff --git a/tensorflow/contrib/estimator/python/estimator/saved_model_estimator_test.py b/tensorflow/contrib/estimator/python/estimator/saved_model_estimator_test.py
new file mode 100644
index 0000000000..718da1367c
--- /dev/null
+++ b/tensorflow/contrib/estimator/python/estimator/saved_model_estimator_test.py
@@ -0,0 +1,369 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for SavedModelEstimator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import shutil
+import tempfile
+
+from tensorflow.contrib.estimator.python.estimator import export as contrib_export
+from tensorflow.contrib.estimator.python.estimator import saved_model_estimator
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.estimator import estimator
+from tensorflow.python.estimator import model_fn as model_fn_lib
+from tensorflow.python.estimator.export import export
+from tensorflow.python.estimator.export import export_output
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import metrics as metrics_lib
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+from tensorflow.python.training import monitored_session
+from tensorflow.python.training import training
+
+
+def dummy_input_fn():
+  return dataset_ops.Dataset.from_tensors((
+      {'x': constant_op.constant([[1], [-2]], dtype=dtypes.int64)},
+      constant_op.constant([[4], [-3]], dtype=dtypes.float32))).repeat()
+
+
+def dummy_input_fn_features_only():
+  return dataset_ops.Dataset.from_tensors(
+      {'x': constant_op.constant([[5], [6]], dtype=dtypes.int64)}).repeat()
+
+
+def dummy_supervised_receiver_fn():
+  feature_spec = {
+      'x': array_ops.placeholder(
+          dtype=dtypes.int64, shape=(2, 1), name='feature_x'),
+      }
+  label_spec = array_ops.placeholder(
+      dtype=dtypes.float32, shape=[2, 1], name='truth')
+  return export.build_raw_supervised_input_receiver_fn(
+      feature_spec, label_spec)
+
+
+def dummy_serving_receiver_fn():
+  feature_spec = {'x': array_ops.placeholder(
+      dtype=dtypes.int64, shape=(2, 1), name='feature_x'),}
+  return export.build_raw_serving_input_receiver_fn(feature_spec)
+
+
+def model_fn_diff_modes(features, labels, mode):
+  _, _ = features, labels
+  v = variables.Variable(21, name='some_var')
+  train_op = None
+  loss = constant_op.constant(104)
+  if mode == model_fn_lib.ModeKeys.TRAIN:
+    loss = constant_op.constant(105)
+    predictions = constant_op.constant([501])
+    train_op = control_flow_ops.group(
+        state_ops.assign_add(training.get_global_step(), 1),
+        state_ops.assign_add(v, 3))
+  elif mode == model_fn_lib.ModeKeys.EVAL:
+    loss = constant_op.constant(106)
+    predictions = constant_op.constant([502])
+  else:
+    loss = constant_op.constant(107)
+    predictions = constant_op.constant([503])
+  return model_fn_lib.EstimatorSpec(
+      mode,
+      loss=loss,
+      train_op=train_op,
+      eval_metric_ops={
+          'abs_err': metrics_lib.mean_absolute_error(
+              constant_op.constant(0), predictions)},
+      predictions=predictions)
+
+
+class SavedModelEstimatorTest(test.TestCase):
+
+  def setUp(self):
+    self.tmpdirs = []
+
+  def tearDown(self):
+    for tmpdir in self.tmpdirs:
+      # gfile.DeleteRecursively fails in the windows cmake test, so use shutil.
+      shutil.rmtree(tmpdir, ignore_errors=True)
+    self.tmpdirs = []
+
+  def _get_tmp_dir(self):
+    tmpdir = tempfile.mkdtemp()
+    self.tmpdirs.append(tmpdir)
+    return tmpdir
+
+  def _export_estimator(self, train=True, evaluate=True, predict=True,
+                        model_fn=model_fn_diff_modes):
+    est = estimator.Estimator(model_fn, self._get_tmp_dir())
+    est.train(input_fn=dummy_input_fn, steps=10)
+
+    input_receiver_fn_map = {}
+    if train:
+      input_receiver_fn_map[model_fn_lib.ModeKeys.TRAIN] = (
+          dummy_supervised_receiver_fn())
+    if evaluate:
+      input_receiver_fn_map[model_fn_lib.ModeKeys.EVAL] = (
+          dummy_supervised_receiver_fn())
+    if predict:
+      input_receiver_fn_map[model_fn_lib.ModeKeys.PREDICT] = (
+          dummy_serving_receiver_fn())
+
+    export_base_path = self._get_tmp_dir()
+    export_dir = contrib_export.export_all_saved_models(
+        est, export_base_path, input_receiver_fn_map)
+    return export_dir
+
+  def test_load_all_modes(self):
+    sme = saved_model_estimator.SavedModelEstimator(
+        self._export_estimator(), self._get_tmp_dir())
+    sme.train(input_fn=dummy_input_fn, steps=1)
+    sme.train(input_fn=dummy_input_fn, steps=2)
+    self.assertEqual(13, sme.get_variable_value('global_step'))
+    self.assertEqual(60, sme.get_variable_value('some_var'))
+
+    eval_results = sme.evaluate(dummy_input_fn, steps=5)
+
+    self.assertEqual(13, eval_results['global_step'])
+    self.assertEqual(106, eval_results['loss'])
+    self.assertEqual(502, eval_results['metrics/abs_err'])
+
+    predictions = next(sme.predict(dummy_input_fn_features_only))
+    self.assertDictEqual({'output': 503}, predictions)
+
+  def test_load_all_modes_no_train(self):
+    """Ensure that all functions can be used without requiring a ckpt."""
+    sme = saved_model_estimator.SavedModelEstimator(
+        self._export_estimator(), self._get_tmp_dir())
+    eval_results = sme.evaluate(dummy_input_fn, steps=5)
+    self.assertEqual(10, eval_results['global_step'])
+    self.assertEqual(106, eval_results['loss'])
+    self.assertEqual(502, eval_results['metrics/abs_err'])
+
+    predictions = next(sme.predict(dummy_input_fn_features_only))
+    self.assertDictEqual({'output': 503}, predictions)
+
+  def test_partial_exported_estimator(self):
+    sme1 = saved_model_estimator.SavedModelEstimator(
+        self._export_estimator(train=False, predict=False), self._get_tmp_dir())
+    sme1.evaluate(dummy_input_fn, steps=5)
+    with self.assertRaisesRegexp(RuntimeError, 'train mode is not available'):
+      sme1.train(input_fn=dummy_input_fn, steps=1)
+    with self.assertRaisesRegexp(RuntimeError, 'infer mode is not available'):
+      next(sme1.predict(dummy_input_fn_features_only))
+
+    sme2 = saved_model_estimator.SavedModelEstimator(
+        self._export_estimator(evaluate=False), self._get_tmp_dir())
+    sme2.train(input_fn=dummy_input_fn, steps=1)
+    next(sme2.predict(dummy_input_fn_features_only))
+    with self.assertRaisesRegexp(RuntimeError, 'eval mode is not available'):
+      sme2.evaluate(dummy_input_fn, steps=5)
+
+  def test_with_incorrect_input(self):
+    sme = saved_model_estimator.SavedModelEstimator(
+        self._export_estimator(), self._get_tmp_dir())
+
+    def bad_shape_input_fn():
+      return dataset_ops.Dataset.from_tensors((
+          {'x': constant_op.constant([1, 2], dtype=dtypes.int64)},
+          constant_op.constant([1, 2], dtype=dtypes.float32)))
+
+    with self.assertRaisesRegexp(ValueError, 'Expected shape'):
+      sme.train(bad_shape_input_fn, steps=1)
+
+    def bad_dtype_input_fn():
+      return dataset_ops.Dataset.from_tensors((
+          {'x': constant_op.constant([[1], [1]], dtype=dtypes.int32)},
+          constant_op.constant([[1], [1]], dtype=dtypes.int64)))
+
+    with self.assertRaisesRegexp(ValueError, 'Expected dtype'):
+      sme.train(bad_dtype_input_fn, steps=1)
+
+  def test_input_fn_with_global_step(self):
+    sme = saved_model_estimator.SavedModelEstimator(
+        self._export_estimator(), self._get_tmp_dir())
+
+    def bad_input_fn():
+      training.get_or_create_global_step()
+      return dataset_ops.Dataset.from_tensors((
+          {'x': constant_op.constant([[1], [1]], dtype=dtypes.int64)},
+          constant_op.constant([[1], [1]], dtype=dtypes.float32)))
+
+    with self.assertRaisesRegexp(RuntimeError,
+                                 'Graph must not contain a global step tensor'):
+      sme.train(bad_input_fn, steps=1)
+
+  def test_re_export_saved_model_serving_only(self):
+    sme = saved_model_estimator.SavedModelEstimator(
+        self._export_estimator(), self._get_tmp_dir())
+    sme.train(dummy_input_fn, steps=3)
+    self.assertEqual(13, sme.get_variable_value('global_step'))
+    self.assertEqual(60, sme.get_variable_value('some_var'))
+
+    predictions = next(sme.predict(dummy_input_fn_features_only))
+    self.assertDictEqual({'output': 503}, predictions)
+
+    # Export SavedModel, and test that the variable and prediction values are
+    # the same.
+    sme_export_dir = sme.export_savedmodel(
+        self._get_tmp_dir(), dummy_serving_receiver_fn())
+
+    sme2 = saved_model_estimator.SavedModelEstimator(
+        sme_export_dir, self._get_tmp_dir())
+    self.assertEqual(60, sme.get_variable_value('some_var'))
+    self.assertEqual(13, sme.get_variable_value('global_step'))
+
+    predictions = next(sme2.predict(dummy_input_fn_features_only))
+    self.assertDictEqual({'output': 503}, predictions)
+
+  def test_re_export_saved_model(self):
+    sme = saved_model_estimator.SavedModelEstimator(
+        self._export_estimator(), self._get_tmp_dir())
+    self.assertDictEqual(
+        {'loss': 106, 'metrics/abs_err': 502, 'global_step': 10},
+        sme.evaluate(dummy_input_fn, steps=1))
+
+    sme.train(dummy_input_fn, steps=3)
+    self.assertDictEqual(
+        {'loss': 106, 'metrics/abs_err': 502, 'global_step': 13},
+        sme.evaluate(dummy_input_fn, steps=1))
+    self.assertEqual(60, sme.get_variable_value('some_var'))
+
+    predictions = next(sme.predict(dummy_input_fn_features_only))
+    self.assertDictEqual({'output': 503}, predictions)
+
+    # Export SavedModel for all modes
+    input_receiver_fn_map = {
+        model_fn_lib.ModeKeys.TRAIN: dummy_supervised_receiver_fn(),
+        model_fn_lib.ModeKeys.EVAL: dummy_supervised_receiver_fn(),
+        model_fn_lib.ModeKeys.PREDICT: dummy_serving_receiver_fn()}
+    sme_export_dir = contrib_export.export_all_saved_models(
+        sme, self._get_tmp_dir(), input_receiver_fn_map)
+
+    sme2 = saved_model_estimator.SavedModelEstimator(
+        sme_export_dir, self._get_tmp_dir())
+    self.assertDictEqual(
+        {'loss': 106, 'metrics/abs_err': 502, 'global_step': 13},
+        sme.evaluate(dummy_input_fn, steps=1))
+    self.assertEqual(60, sme.get_variable_value('some_var'))
+
+    sme.train(dummy_input_fn, steps=7)
+    self.assertEqual(20, sme.get_variable_value('global_step'))
+
+    predictions = next(sme2.predict(dummy_input_fn_features_only))
+    self.assertDictEqual({'output': 503}, predictions)
+
+  def test_load_saved_model_from_serving_only(self):
+    def model_fn(features, labels, mode):
+      _, _ = features, labels
+      return model_fn_lib.EstimatorSpec(
+          mode,
+          loss=constant_op.constant([103]),
+          train_op=state_ops.assign_add(training.get_global_step(), 1),
+          predictions=constant_op.constant([502]),
+          export_outputs={'test': export_output.ClassificationOutput(
+              constant_op.constant([[32.]]))})
+
+    est = estimator.Estimator(model_fn, self._get_tmp_dir())
+    est.train(input_fn=dummy_input_fn, steps=10)
+
+    def serving_input_receiver_fn():
+      return export.ServingInputReceiver(
+          {'test-features': constant_op.constant([[1], [1]])},
+          array_ops.placeholder(dtype=dtypes.string))
+
+    export_dir = est.export_savedmodel(
+        self._get_tmp_dir(), serving_input_receiver_fn)
+
+    sme = saved_model_estimator.SavedModelEstimator(
+        export_dir, self._get_tmp_dir())
+
+    def input_fn():
+      return {'inputs': constant_op.constant('someinputstr')}
+
+    prediction = next(sme.predict(input_fn))
+    self.assertDictEqual({'scores': 32}, prediction)
+
+  def test_with_local_init_op(self):
+    def model_fn(features, labels, mode):
+      _, _ = features, labels
+      v = variables.Variable(21, name='some_var')
+      scaffold = monitored_session.Scaffold(
+          local_init_op=state_ops.assign_add(v, -3).op
+      )
+      return model_fn_lib.EstimatorSpec(
+          mode,
+          scaffold=scaffold,
+          train_op=state_ops.assign_add(training.get_global_step(), 1),
+          loss=array_ops.identity(v))
+    export_dir = self._export_estimator(predict=False, model_fn=model_fn)
+    sme = saved_model_estimator.SavedModelEstimator(
+        export_dir, self._get_tmp_dir())
+
+    eval_results1 = sme.evaluate(dummy_input_fn, steps=2)
+    self.assertEqual(15, eval_results1['loss'])
+
+    sme.train(dummy_input_fn, steps=1)
+    self.assertEqual(15, sme.get_variable_value('some_var'))
+
+    eval_results2 = sme.evaluate(dummy_input_fn, steps=5)
+    self.assertEqual(12, eval_results2['loss'])
+
+  def test_with_working_input_fn(self):
+    def model_fn(features, labels, mode):
+      loss = None
+      if labels is not None:
+        loss = labels[0][0] + labels[1][0]
+      return model_fn_lib.EstimatorSpec(
+          mode,
+          loss=loss,
+          train_op=state_ops.assign_add(training.get_global_step(), 1),
+          predictions={'features_0': array_ops.identity([features['x'][0][0]]),
+                       'features_1': array_ops.identity([features['x'][1][0]])})
+
+    sme = saved_model_estimator.SavedModelEstimator(
+        self._export_estimator(model_fn=model_fn), self._get_tmp_dir())
+    eval_results = sme.evaluate(dummy_input_fn, steps=1)
+    self.assertEqual(1, eval_results['loss'])
+
+    predictions = next(sme.predict(dummy_input_fn_features_only))
+    self.assertDictEqual({'features_0': 5, 'features_1': 6}, predictions)
+
+  def test_control_dependency(self):
+    # Control dependencies are saved with "^" appended to the start of the input
+    # name. The input map must include control dependencies as well.
+    def model_fn(features, labels, mode):
+      _ = labels
+      with ops.control_dependencies([features['x']]):
+        loss = features['x'][1][0]
+      return model_fn_lib.EstimatorSpec(
+          mode,
+          loss=loss,
+          train_op=state_ops.assign_add(training.get_global_step(), 1))
+    sme = saved_model_estimator.SavedModelEstimator(
+        self._export_estimator(train=False, predict=False, model_fn=model_fn),
+        self._get_tmp_dir())
+    sme.evaluate(dummy_input_fn, steps=1)  # Should run without error
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 2fd6f6fab9..148fcf61fa 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -568,13 +568,14 @@ class Estimator(object):
   def _assert_members_are_not_overridden(self):
     """Asserts members of `Estimator` are not overridden."""
     allowed_overrides = set([
-        '_call_input_fn', '_create_global_step',
+        '_call_input_fn', '_call_model_fn',
         '_convert_train_steps_to_hooks', '_convert_eval_steps_to_hooks',
+        '_create_global_step', '_create_and_assert_global_step',
         '_tf_api_names', '_tf_api_names_v1', '_estimator_api_names',
         '_estimator_api_names_v1', '_estimator_api_constants',
         '_estimator_api_constants_v1',
         '_validate_features_in_predict_input',
-        '_call_model_fn', '_add_meta_graph_for_mode'
+        '_add_meta_graph_for_mode'
     ])
     estimator_members = set([m for m in Estimator.__dict__.keys()
                              if not m.startswith('__')])
@@ -901,9 +902,10 @@ class Estimator(object):
 
       with tf_session.Session(config=self._session_config) as session:
 
-        local_init_op = (
-            estimator_spec.scaffold.local_init_op or
-            monitored_session.Scaffold.default_local_init_op())
+        if estimator_spec.scaffold.local_init_op is not None:
+          local_init_op = estimator_spec.scaffold.local_init_op
+        else:
+          local_init_op = monitored_session.Scaffold.default_local_init_op()
 
         # This saver will be used both for restoring variables now,
         # and in saving out the metagraph below. This ensures that any
@@ -1154,14 +1156,15 @@ class Estimator(object):
     worker_hooks = []
     with ops.Graph().as_default() as g, g.device(self._device_fn):
       random_seed.set_random_seed(self._config.tf_random_seed)
-      global_step_tensor = self._create_and_assert_global_step(g)
-      training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
+      self._create_and_assert_global_step(g)
       features, labels, input_hooks = (
           self._get_features_and_labels_from_input_fn(
               input_fn, model_fn_lib.ModeKeys.TRAIN))
       worker_hooks.extend(input_hooks)
       estimator_spec = self._call_model_fn(
           features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
+      global_step_tensor = training_util.get_global_step(g)
+      training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
       return self._train_with_estimator_spec(estimator_spec, worker_hooks,
                                              hooks, global_step_tensor,
                                              saving_listeners)
@@ -1364,10 +1367,8 @@ class Estimator(object):
   def _train_with_estimator_spec(self, estimator_spec, worker_hooks, hooks,
                                  global_step_tensor, saving_listeners):
     """Train a model with the given Estimator Spec."""
-    if self._warm_start_settings:
-      logging.info('Warm-starting with WarmStartSettings: %s' %
-                   (self._warm_start_settings,))
-      warm_starting_util.warm_start(*self._warm_start_settings)
+    self._maybe_warm_start(self.latest_checkpoint())
+
     # Check if the user created a loss summary, and add one if they didn't.
     # We assume here that the summary is called 'loss'. If it is not, we will
     # make another one with the name 'loss' to ensure it shows up in the right
@@ -1448,13 +1449,13 @@ class Estimator(object):
   def _evaluate_build_graph(self, input_fn, hooks=None, checkpoint_path=None):
     """Builds the graph and related hooks to run evaluation."""
     random_seed.set_random_seed(self._config.tf_random_seed)
-    global_step_tensor = self._create_and_assert_global_step(
-        ops.get_default_graph())
+    self._create_and_assert_global_step(ops.get_default_graph())
     features, labels, input_hooks = (
         self._get_features_and_labels_from_input_fn(input_fn,
                                                     model_fn_lib.ModeKeys.EVAL))
     estimator_spec = self._call_model_fn(
         features, labels, model_fn_lib.ModeKeys.EVAL, self.config)
+    global_step_tensor = training_util.get_global_step(ops.get_default_graph())
 
     # Call to warm_start has to be after model_fn is called.
     self._maybe_warm_start(checkpoint_path)
@@ -1480,7 +1481,21 @@ class Estimator(object):
     all_hooks.extend(hooks)
     all_hooks.extend(list(estimator_spec.evaluation_hooks or []))
 
-    return estimator_spec.scaffold, update_op, eval_dict, all_hooks
+    # New local variables have been added, so update the estimator spec's
+    # local init op if it was defined.
+    scaffold = estimator_spec.scaffold
+    if estimator_spec.scaffold and estimator_spec.scaffold.local_init_op:
+      # Ensure that eval step has been created before updating local init op.
+      evaluation._get_or_create_eval_step()  # pylint: disable=protected-access
+
+      scaffold = monitored_session.Scaffold(
+          local_init_op=control_flow_ops.group(
+              estimator_spec.scaffold.local_init_op,
+              monitored_session.Scaffold.default_local_init_op()),
+          copy_from_scaffold=scaffold
+      )
+
+    return scaffold, update_op, eval_dict, all_hooks
 
   def _evaluate_run(self, checkpoint_path, scaffold, update_op, eval_dict,
                     all_hooks, output_dir):
@@ -1911,6 +1926,19 @@ class WarmStartSettings(
     )
 
 
+def _get_saved_model_ckpt(saved_model_dir):
+  """Return path to variables checkpoint in a SavedModel directory."""
+  if not gfile.Exists(
+      os.path.join(compat.as_bytes(saved_model_dir),
+                   compat.as_bytes('variables/variables.index'))):
+    raise ValueError('Directory provided has an invalid SavedModel format: %s'
+                     % saved_model_dir)
+  return os.path.join(
+      compat.as_bytes(saved_model_dir),
+      compat.as_bytes('{}/{}'.format(constants.VARIABLES_DIRECTORY,
+                                     constants.VARIABLES_FILENAME)))
+
+
 def _get_default_warm_start_settings(warm_start_from):
   """Returns default WarmStartSettings.
 
@@ -1934,10 +1962,8 @@ def _get_default_warm_start_settings(warm_start_from):
     if gfile.Exists(os.path.join(compat.as_bytes(warm_start_from),
                                  compat.as_bytes('variables/variables.index'))):
       logging.info('Warm-starting from a SavedModel')
-      return WarmStartSettings(ckpt_to_initialize_from=os.path.join(
-          compat.as_bytes(warm_start_from),
-          compat.as_bytes('{}/{}'.format(constants.VARIABLES_DIRECTORY,
-                                         constants.VARIABLES_FILENAME))))
+      return WarmStartSettings(
+          ckpt_to_initialize_from=_get_saved_model_ckpt(warm_start_from))
     return WarmStartSettings(ckpt_to_initialize_from=warm_start_from)
   elif isinstance(warm_start_from, WarmStartSettings):
     return warm_start_from
diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py
index 699d2b70d1..687bfebd43 100644
--- a/tensorflow/python/framework/importer.py
+++ b/tensorflow/python/framework/importer.py
@@ -205,7 +205,7 @@ def _PopulateTFImportGraphDefOptions(options, prefix, input_map,
   for input_src, input_dst in input_map.items():
     input_src = compat.as_str(input_src)
     if input_src.startswith('^'):
-      src_name = compat.as_bytes(input_src[1:])
+      src_name = compat.as_str(input_src[1:])
       dst_op = input_dst._as_tf_output().oper  # pylint: disable=protected-access
       c_api.TF_ImportGraphDefOptionsRemapControlDependency(
           options, src_name, dst_op)
diff --git a/tensorflow/python/framework/meta_graph.py b/tensorflow/python/framework/meta_graph.py
index 923e76fc9c..33631282bd 100644
--- a/tensorflow/python/framework/meta_graph.py
+++ b/tensorflow/python/framework/meta_graph.py
@@ -696,6 +696,67 @@ def import_scoped_meta_graph(meta_graph_or_file,
   Raises:
     ValueError: If the graph_def contains unbound inputs.
   """
+  return import_scoped_meta_graph_with_return_elements(
+      meta_graph_or_file, clear_devices, graph, import_scope, input_map,
+      unbound_inputs_col_name, restore_collections_predicate)[0]
+
+
+def import_scoped_meta_graph_with_return_elements(
+    meta_graph_or_file,
+    clear_devices=False,
+    graph=None,
+    import_scope=None,
+    input_map=None,
+    unbound_inputs_col_name="unbound_inputs",
+    restore_collections_predicate=(lambda key: True),
+    return_elements=None):
+  """Imports graph from `MetaGraphDef` and returns vars and return elements.
+
+  This function takes a `MetaGraphDef` protocol buffer as input. If
+  the argument is a file containing a `MetaGraphDef` protocol buffer ,
+  it constructs a protocol buffer from the file content. The function
+  then adds all the nodes from the `graph_def` field to the
+  current graph, recreates the desired collections, and returns a dictionary of
+  all the Variables imported into the name scope.
+
+  In combination with `export_scoped_meta_graph()`, this function can be used to
+
+  * Serialize a graph along with other Python objects such as `QueueRunner`,
+    `Variable` into a `MetaGraphDef`.
+
+  * Restart training from a saved graph and checkpoints.
+
+  * Run inference from a saved graph and checkpoints.
+
+  Args:
+    meta_graph_or_file: `MetaGraphDef` protocol buffer or filename (including
+      the path) containing a `MetaGraphDef`.
+    clear_devices: Boolean which controls whether to clear device information
+      from graph_def. Default false.
+    graph: The `Graph` to import into. If `None`, use the default graph.
+    import_scope: Optional `string`. Name scope into which to import the
+      subgraph. If `None`, the graph is imported to the root name scope.
+    input_map: A dictionary mapping input names (as strings) in `graph_def` to
+      `Tensor` objects. The values of the named input tensors in the imported
+      graph will be re-mapped to the respective `Tensor` values.
+    unbound_inputs_col_name: Collection name for looking up unbound inputs.
+    restore_collections_predicate: a predicate on collection names. A collection
+      named c (i.e whose key is c) will be restored iff
+      1) `restore_collections_predicate(c)` is True, and
+      2) `c != unbound_inputs_col_name`.
+    return_elements:  A list of strings containing operation names in the
+      `MetaGraphDef` that will be returned as `Operation` objects; and/or
+      tensor names in `MetaGraphDef` that will be returned as `Tensor` objects.
+
+  Returns:
+    A tuple of (
+      dictionary of all the `Variables` imported into the name scope,
+      list of `Operation` or `Tensor` objects from the `return_elements` list).
+
+  Raises:
+    ValueError: If the graph_def contains unbound inputs.
+
+  """
   if context.executing_eagerly():
     raise ValueError("Exporting/importing meta graphs is not supported when "
                      "eager execution is enabled.")
@@ -737,11 +798,12 @@ def import_scoped_meta_graph(meta_graph_or_file,
     scope_to_prepend_to_names = graph.unique_name(
         import_scope or "", mark_as_used=False)
 
-    importer.import_graph_def(
+    imported_return_elements = importer.import_graph_def(
         input_graph_def,
         name=(import_scope or scope_to_prepend_to_names),
         input_map=input_map,
-        producer_op_list=producer_op_list)
+        producer_op_list=producer_op_list,
+        return_elements=return_elements)
 
     # Restores all the other collections.
     variable_objects = {}
@@ -806,7 +868,7 @@ def import_scoped_meta_graph(meta_graph_or_file,
     for v in variables:
       var_list[ops.strip_name_scope(v.name, scope_to_prepend_to_names)] = v
 
-  return var_list
+  return var_list, imported_return_elements
 
 
 def export_scoped_meta_graph(filename=None,
diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py
index e5f649fdab..685a913f9c 100644
--- a/tensorflow/python/saved_model/loader_impl.py
+++ b/tensorflow/python/saved_model/loader_impl.py
@@ -284,12 +284,15 @@ class SavedModelLoader(object):
       **saver_kwargs: keyword arguments to pass to tf.train.import_meta_graph.
 
     Returns:
-      Saver defined by the MetaGraph, which can be used to restore the variable
-      values.
+      A tuple of
+        * Saver defined by the MetaGraph, which can be used to restore the
+          variable values.
+        * List of `Operation`/`Tensor` objects returned from
+          `tf.import_graph_def` (may be `None`).
     """
     meta_graph_def = self.get_meta_graph_def_from_tags(tags)
     with graph.as_default():
-      return tf_saver.import_meta_graph(
+      return tf_saver._import_meta_graph_with_return_elements(  # pylint: disable=protected-access
           meta_graph_def, import_scope=import_scope, **saver_kwargs)
 
   def restore_variables(self, sess, saver, import_scope=None):
@@ -361,8 +364,8 @@ class SavedModelLoader(object):
       `MetagraphDef` proto of the graph that was loaded.
     """
     with sess.graph.as_default():
-      saver = self.load_graph(sess.graph, tags, import_scope,
-                              **saver_kwargs)
+      saver, _ = self.load_graph(sess.graph, tags, import_scope,
+                                 **saver_kwargs)
       self.restore_variables(sess, saver, import_scope)
       self.run_init_ops(sess, tags, import_scope)
     return self.get_meta_graph_def_from_tags(tags)
diff --git a/tensorflow/python/saved_model/loader_test.py b/tensorflow/python/saved_model/loader_test.py
index ce18859f6b..9a0b276a4b 100644
--- a/tensorflow/python/saved_model/loader_test.py
+++ b/tensorflow/python/saved_model/loader_test.py
@@ -111,7 +111,8 @@ class SavedModelLoaderTest(test.TestCase):
   def test_load_with_import_scope(self):
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
     with self.test_session(graph=ops.Graph()) as sess:
-      saver = loader.load_graph(sess.graph, ["foo_graph"], import_scope="baz")
+      saver, _ = loader.load_graph(
+          sess.graph, ["foo_graph"], import_scope="baz")
 
       # The default saver should not work when the import scope is set.
       with self.assertRaises(errors.NotFoundError):
@@ -149,7 +150,7 @@ class SavedModelLoaderTest(test.TestCase):
   def test_run_init_op(self):
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
     graph = ops.Graph()
-    saver = loader.load_graph(graph, ["foo_graph"])
+    saver, _ = loader.load_graph(graph, ["foo_graph"])
     with self.test_session(graph=graph) as sess:
       loader.restore_variables(sess, saver)
       self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
@@ -203,7 +204,7 @@ class SavedModelLoaderTest(test.TestCase):
 
     loader = loader_impl.SavedModelLoader(path)
     with self.test_session(graph=ops.Graph()) as sess:
-      saver = loader.load_graph(sess.graph, ["foo_graph"])
+      saver, _ = loader.load_graph(sess.graph, ["foo_graph"])
       self.assertFalse(variables._all_saveable_objects())
       self.assertIsNotNone(saver)
 
@@ -212,6 +213,18 @@ class SavedModelLoaderTest(test.TestCase):
       self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
       self.assertEqual(11, sess.graph.get_tensor_by_name("y:0").eval())
 
+  def test_load_saved_model_graph_with_return_elements(self):
+    """Ensure that the correct elements are returned."""
+    loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
+    graph = ops.Graph()
+    _, ret = loader.load_graph(graph, ["foo_graph"],
+                               return_elements=["y:0", "x:0"])
+
+    self.assertEqual(graph.get_tensor_by_name("y:0"), ret[0])
+    self.assertEqual(graph.get_tensor_by_name("x:0"), ret[1])
+
+    with self.assertRaisesRegexp(ValueError, "not found in graph"):
+      loader.load_graph(graph, ["foo_graph"], return_elements=["z:0"])
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 11510d9928..3a06a52812 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1928,6 +1928,14 @@ def import_meta_graph(meta_graph_or_file, clear_devices=False,
   execution is enabled.
   @end_compatibility
   """  # pylint: disable=g-doc-exception
+  return _import_meta_graph_with_return_elements(
+      meta_graph_or_file, clear_devices, import_scope, **kwargs)[0]
+
+
+def _import_meta_graph_with_return_elements(
+    meta_graph_or_file, clear_devices=False, import_scope=None,
+    return_elements=None, **kwargs):
+  """Import MetaGraph, and return both a saver and returned elements."""
   if context.executing_eagerly():
     raise RuntimeError("Exporting/importing meta graphs is not supported when "
                        "eager execution is enabled. No graph exists when eager "
@@ -1937,12 +1945,22 @@ def import_meta_graph(meta_graph_or_file, clear_devices=False,
   else:
     meta_graph_def = meta_graph_or_file
 
-  imported_vars = meta_graph.import_scoped_meta_graph(
-      meta_graph_def,
-      clear_devices=clear_devices,
-      import_scope=import_scope,
-      **kwargs)
+  imported_vars, imported_return_elements = (
+      meta_graph.import_scoped_meta_graph_with_return_elements(
+          meta_graph_def,
+          clear_devices=clear_devices,
+          import_scope=import_scope,
+          return_elements=return_elements,
+          **kwargs))
+
+  saver = _create_saver_from_imported_meta_graph(
+      meta_graph_def, import_scope, imported_vars)
+  return saver, imported_return_elements
+
 
+def _create_saver_from_imported_meta_graph(
+    meta_graph_def, import_scope, imported_vars):
+  """Return a saver for restoring variable values to an imported MetaGraph."""
   if meta_graph_def.HasField("saver_def"):
     # Infer the scope that is prepended by `import_scoped_meta_graph`.
     scope = import_scope
-- 
cgit v1.2.3


From c92f6011009392bfea14d5673b67e724018286b7 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne 
Date: Fri, 20 Jul 2018 14:03:16 -0700
Subject: Add lookup table size ops to checkpointing whitelist.

Reading the size doesn't actually alter the state of the lookup table,
so it's safe to checkpoint.

PiperOrigin-RevId: 205449858
---
 tensorflow/core/ops/lookup_ops.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/ops/lookup_ops.cc b/tensorflow/core/ops/lookup_ops.cc
index 444aa8b954..2059741da9 100644
--- a/tensorflow/core/ops/lookup_ops.cc
+++ b/tensorflow/core/ops/lookup_ops.cc
@@ -140,11 +140,13 @@ REGISTER_OP("LookupTableSize")
     .Input("table_handle: Ref(string)")
     .Output("size: int64")
     .SetShapeFn(TwoElementVectorInputsAndScalarOutputs);
+WHITELIST_STATEFUL_OP_FOR_DATASET_FUNCTIONS("LookupTableSize");
 
 REGISTER_OP("LookupTableSizeV2")
     .Input("table_handle: resource")
     .Output("size: int64")
     .SetShapeFn(ScalarAndTwoElementVectorInputsAndScalarOutputs);
+WHITELIST_STATEFUL_OP_FOR_DATASET_FUNCTIONS("LookupTableSizeV2");
 
 REGISTER_OP("LookupTableExport")
     .Input("table_handle: Ref(string)")
-- 
cgit v1.2.3


From 2c1f2847c2afbc6f23ef7040d49c71ffaa8b669c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 14:14:34 -0700
Subject: Place infeed_enqueue and infeed_enqueue_tuple ops on the host device
 rather than the accelerator device in TPUEstimator.

This removes the potential for extra accidental transfers of tensors to and from device.

PiperOrigin-RevId: 205451735
---
 tensorflow/contrib/tpu/python/tpu/keras_support.py | 32 ++++++++-------
 tensorflow/contrib/tpu/python/tpu/tpu_context.py   | 46 +++++++++++++---------
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py |  8 ++--
 3 files changed, 51 insertions(+), 35 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/keras_support.py b/tensorflow/contrib/tpu/python/tpu/keras_support.py
index 8292c920fc..81798ee423 100644
--- a/tensorflow/contrib/tpu/python/tpu/keras_support.py
+++ b/tensorflow/contrib/tpu/python/tpu/keras_support.py
@@ -441,21 +441,23 @@ class TPUNumpyInfeedManager(TPUInfeedManager):
     shard_infeed_tensors = []
 
     for shard_id in range(self._strategy.num_towers):
-      with ops.device('/device:TPU:%d' % shard_id):
+      with ops.device('/device:CPU:0'):
         infeed_tensors = []
-        for spec in input_specs:
-          # Construct placeholders for each of the inputs.
-          infeed_tensors.append(
-              array_ops.placeholder(
-                  dtype=spec.dtype,
-                  shape=spec.shape,
-                  name='infeed-enqueue-%s-%d' % (spec.name, shard_id)))
+        with ops.device('/device:TPU:%d' % shard_id):
+          for spec in input_specs:
+            # Construct placeholders for each of the inputs.
+            infeed_tensors.append(
+                array_ops.placeholder(
+                    dtype=spec.dtype,
+                    shape=spec.shape,
+                    name='infeed-enqueue-%s-%d' % (spec.name, shard_id)))
         shard_infeed_tensors.append(infeed_tensors)
 
         infeed_op.append(
             tpu_ops.infeed_enqueue_tuple(
                 infeed_tensors, [spec.shape for spec in input_specs],
-                name='infeed-enqueue-%s-%d' % (execution_mode, shard_id)))
+                name='infeed-enqueue-%s-%d' % (execution_mode, shard_id),
+                device_ordinal=shard_id))
     return SizedInfeed(infeed_ops=infeed_op,
                        sharded_infeed_tensors=shard_infeed_tensors)
 
@@ -584,12 +586,13 @@ class TPUDatasetInfeedManager(TPUInfeedManager):
     assert len(shard_infeed_tensors) == self._strategy.num_towers
     infeed_ops = []
     for shard_id in range(self._strategy.num_towers):
-      with ops.device('/device:TPU:%d' % shard_id):
+      with ops.device('/device:CPU:0'):
         infeed_ops.append(
             tpu_ops.infeed_enqueue_tuple(
                 shard_infeed_tensors[shard_id],
                 [spec.shape for spec in input_specs],
-                name='infeed-enqueue-%s-%d' % (execution_mode, shard_id)))
+                name='infeed-enqueue-%s-%d' % (execution_mode, shard_id),
+                device_ordinal=shard_id))
     return SizedInfeed(infeed_ops=infeed_ops,
                        sharded_infeed_tensors=shard_infeed_tensors)
 
@@ -740,12 +743,13 @@ class TPUFunction(object):
     # Build output ops.
     outfeed_op = []
     for shard_id in range(self._strategy.num_towers):
-      with ops.device('/device:TPU:%d' % shard_id):
+      with ops.device('/device:CPU:0'):
         outfeed_op.extend(
             tpu_ops.outfeed_dequeue_tuple(
                 dtypes=[spec.dtype for spec in self._outfeed_spec],
                 shapes=[spec.shape for spec in self._outfeed_spec],
-                name='outfeed-dequeue-%s-%d' % (self.execution_mode, shard_id)))
+                name='outfeed-dequeue-%s-%d' % (self.execution_mode, shard_id),
+                device_ordinal=shard_id))
 
     return TPUModelOp(
         compile_op,
@@ -1126,7 +1130,7 @@ Output shape: %(output_shape)s
       'layer': layer,
       'input_shape': layer.input_shape,
       'output_shape': layer.output_shape
-      })
+  })
 
 
 @experimental
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
index 750e677263..2cb68f74a0 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
@@ -146,24 +146,7 @@ class TPUContext(object):
     # Note that: For the non-model parallelism, the mapping could be
     # a random permutation. The order should not matter in most cases
     # as far as model is replicated to all cores in the system.
-
-    # If the precise replica_id to device mapping is required, please
-    # set the num_cores_per_replica to 1 in TPUConfig to enable the
-    # model parallelism.
-    if self._internal_ctx.model_parallelism_enabled:
-      return RuntimeError(
-          'device_for_replica is not yet implemented for model parallelism. '
-          'b/79689078.')
-
-    master = self._internal_ctx.master_job
-    job_device = '' if master is None else ('/job:%s' % master)
-
-    num_of_replicas_per_host = self._internal_ctx.num_of_replicas_per_host
-    host_id = replica_id / num_of_replicas_per_host
-    ordinal_id = replica_id % num_of_replicas_per_host
-
-    host_device = '%s/task:%d/device:CPU:0' % (job_device, host_id)
-    return (host_device, ordinal_id)
+    return self._internal_ctx.device_for_replica(replica_id)
 
 
 class _InternalTPUContext(object):
@@ -634,6 +617,33 @@ class _InternalTPUContext(object):
     # Record the state "validated" into lazy dictionary.
     self._lazy_validation_dict[mode] = True
 
+  def device_for_replica(self, replica_id):
+    """Returns the tuple of (CPU device and device ordinal) for replica.
+
+    This should be used for full replicate for non-model-parallelism.
+
+    Args:
+       replica_id: Int, the replica index.
+
+    Returns:
+       A tuple of device spec for CPU device and int device ordinal.
+    """
+    master = self.master_job
+
+    if self.model_parallelism_enabled:
+      return (self.device_assignment.host_device(
+          replica=replica_id, job=master),
+              self.device_assignment.tpu_ordinal(replica=replica_id))
+
+    job_device = '' if master is None else ('/job:%s' % master)
+
+    num_of_replicas_per_host = self.num_of_replicas_per_host
+    host_id = replica_id / num_of_replicas_per_host
+    ordinal_id = replica_id % num_of_replicas_per_host
+
+    host_device = '%s/task:%d/device:CPU:0' % (job_device, host_id)
+    return (host_device, ordinal_id)
+
 
 class _OneCoreTPUContext(_InternalTPUContext):
   """Special _InternalTPUContext for one core usage."""
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 73dfefd19c..8ae0a31b6a 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -1660,11 +1660,13 @@ class _OutfeedHostCall(object):
     # Outfeed ops execute on each replica's first logical core. Note: we must
     # constraint it such that we have at most one outfeed dequeue and enqueue
     # per replica.
-    tpu_device_placement_fn = self._ctx.tpu_device_placement_function
     for i in xrange(self._ctx.num_replicas):
-      with ops.device(tpu_device_placement_fn(i)):
+      host_device, ordinal_id = self._ctx.device_for_replica(i)
+      with ops.device(host_device):
         outfeed_tensors = tpu_ops.outfeed_dequeue_tuple(
-            dtypes=tensor_dtypes, shapes=tensor_shapes)
+            dtypes=tensor_dtypes,
+            shapes=tensor_shapes,
+            device_ordinal=ordinal_id)
         for j, item in enumerate(outfeed_tensors):
           dequeue_ops[j].append(item)
 
-- 
cgit v1.2.3


From 67869d1266f17b4502391a13eac9180bda5bce0b Mon Sep 17 00:00:00 2001
From: Justin Lebar 
Date: Fri, 20 Jul 2018 14:33:29 -0700
Subject: [XLA] s/ir_builder/b/

Brevity.

PiperOrigin-RevId: 205454869
---
 .../compiler/xla/service/cpu/dot_op_emitter.cc     |  265 +++--
 .../compiler/xla/service/cpu/dot_op_emitter.h      |    7 +-
 .../xla/service/cpu/elemental_ir_emitter.cc        |   15 +-
 .../xla/service/cpu/elemental_ir_emitter.h         |    2 +-
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc  |  791 +++++++--------
 tensorflow/compiler/xla/service/cpu/ir_emitter.h   |   17 +-
 tensorflow/compiler/xla/service/cpu/ir_function.cc |   71 +-
 tensorflow/compiler/xla/service/cpu/ir_function.h  |   11 +-
 .../compiler/xla/service/cpu/llvm_ir_runtime.cc    |   53 +-
 .../xla/service/cpu/parallel_loop_emitter.cc       |    9 +-
 .../xla/service/cpu/parallel_loop_emitter.h        |    2 +-
 .../xla/service/cpu/tests/cpu_noalias_test.cc      |    8 +-
 .../xla/service/cpu/vector_support_library.cc      |  163 ++--
 .../xla/service/cpu/vector_support_library.h       |   36 +-
 .../compiler/xla/service/elemental_ir_emitter.cc   | 1013 +++++++++-----------
 .../compiler/xla/service/elemental_ir_emitter.h    |   14 +-
 .../xla/service/gpu/elemental_ir_emitter.cc        |   90 +-
 .../xla/service/gpu/elemental_ir_emitter.h         |    2 +-
 .../compiler/xla/service/gpu/hlo_to_ir_bindings.cc |   20 +-
 .../compiler/xla/service/gpu/hlo_to_ir_bindings.h  |    9 +-
 tensorflow/compiler/xla/service/gpu/ir_emitter.cc  |  240 +++--
 tensorflow/compiler/xla/service/gpu/ir_emitter.h   |    2 +-
 .../compiler/xla/service/gpu/ir_emitter_nested.cc  |   16 +-
 .../xla/service/gpu/ir_emitter_unnested.cc         |  560 +++++------
 .../xla/service/gpu/parallel_loop_emitter.cc       |   60 +-
 .../xla/service/gpu/parallel_loop_emitter.h        |    6 +-
 .../service/llvm_ir/dynamic_update_slice_util.cc   |   66 +-
 .../service/llvm_ir/dynamic_update_slice_util.h    |    7 +-
 .../xla/service/llvm_ir/fused_ir_emitter.cc        |   30 +-
 .../xla/service/llvm_ir/fused_ir_emitter.h         |    4 +-
 .../compiler/xla/service/llvm_ir/ir_array.cc       |   54 +-
 tensorflow/compiler/xla/service/llvm_ir/ir_array.h |   22 +-
 .../xla/service/llvm_ir/kernel_support_library.cc  |   45 +-
 .../xla/service/llvm_ir/kernel_support_library.h   |   56 +-
 .../compiler/xla/service/llvm_ir/kernel_tiling.cc  |   18 +-
 .../compiler/xla/service/llvm_ir/kernel_tiling.h   |    2 +-
 .../compiler/xla/service/llvm_ir/llvm_loop.cc      |   71 +-
 .../compiler/xla/service/llvm_ir/llvm_loop.h       |   21 +-
 .../compiler/xla/service/llvm_ir/llvm_util.cc      |  150 ++-
 .../compiler/xla/service/llvm_ir/llvm_util.h       |   28 +-
 .../compiler/xla/service/llvm_ir/loop_emitter.cc   |   36 +-
 .../compiler/xla/service/llvm_ir/loop_emitter.h    |   11 +-
 .../compiler/xla/service/llvm_ir/sort_util.cc      |   82 +-
 .../compiler/xla/service/llvm_ir/sort_util.h       |    3 +-
 .../compiler/xla/service/llvm_ir/tuple_ops.cc      |   49 +-
 .../compiler/xla/service/llvm_ir/tuple_ops.h       |    7 +-
 46 files changed, 1974 insertions(+), 2270 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index 58228180ca..1fdeceb860 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -49,15 +49,15 @@ class MemoryTile {
   // `tile_size_along_major_dim` vectors from the matrix `matrix`, starting at
   // `major_dim_offset` in the major dimension.  The tile size along the minor
   // dimension is the vector size, and that is implicitly determined by `vsl`.
-  MemoryTile(VectorSupportLibrary* vsl, llvm::IRBuilder<>* ir_builder,
+  MemoryTile(VectorSupportLibrary* vsl, llvm::IRBuilder<>* b,
              llvm::Value* matrix, int64 matrix_size_along_minor_dim,
              llvm::Value* major_dim_offset, int64 tile_size_along_major_dim)
-      : vsl_(vsl), ir_builder_(ir_builder) {
+      : vsl_(vsl), b_(b) {
     pointers_.reserve(tile_size_along_major_dim);
     for (int64 i = 0; i < tile_size_along_major_dim; i++) {
-      llvm::Value* total_offset = ir_builder->CreateMul(
-          ir_builder->getInt64(matrix_size_along_minor_dim),
-          ir_builder->CreateAdd(ir_builder->getInt64(i), major_dim_offset));
+      llvm::Value* total_offset =
+          b->CreateMul(b->getInt64(matrix_size_along_minor_dim),
+                       b->CreateAdd(b->getInt64(i), major_dim_offset));
       pointers_.push_back(vsl_->ComputeOffsetPointer(matrix, total_offset));
     }
   }
@@ -101,8 +101,7 @@ class MemoryTile {
     for (int64 i = 0; i < pointers_.size(); i++) {
       for (int64 j = 0; j < tile_size_along_middle_dim; j++) {
         result[i].push_back(vsl_->LoadBroadcast(
-            pointers_[i], ir_builder_->CreateAdd(minor_dim_offset,
-                                                 ir_builder_->getInt64(j))));
+            pointers_[i], b_->CreateAdd(minor_dim_offset, b_->getInt64(j))));
       }
     }
     return result;
@@ -110,7 +109,7 @@ class MemoryTile {
 
  private:
   VectorSupportLibrary* vsl_;
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
   std::vector pointers_;
 };
 
@@ -249,16 +248,15 @@ class ColumnMajorMatrixVectorProductEmitter
   ColumnMajorMatrixVectorProductEmitter(const Config& config, llvm::Value* lhs,
                                         llvm::Value* rhs, llvm::Value* addend,
                                         llvm::Value* result,
-                                        llvm::IRBuilder<>* ir_builder)
+                                        llvm::IRBuilder<>* b)
       : config_(config),
         lhs_(lhs),
         rhs_(rhs),
         addend_(addend),
         result_(result),
-        ir_builder_(ir_builder),
-        ksl_(ir_builder_),
-        vsl_(config.scalar_type(), /*vector_size=*/config.tile_rows(),
-             ir_builder_, "") {
+        b_(b),
+        ksl_(b_),
+        vsl_(config.scalar_type(), /*vector_size=*/config.tile_rows(), b_, "") {
     CHECK(tile_rows() > 0 && IsPowerOfTwo(static_cast(tile_rows())));
     CHECK(!has_addend() || addend != nullptr);
   }
@@ -272,7 +270,7 @@ class ColumnMajorMatrixVectorProductEmitter
                          bool is_first_column);
 
   MemoryTile GetLhsMemoryTile(llvm::Value* column_start, int64 column_count) {
-    return MemoryTile(&vsl_, ir_builder_, /*matrix=*/lhs_,
+    return MemoryTile(&vsl_, b_, /*matrix=*/lhs_,
                       /*matrix_size_along_minor_dim=*/m(),
                       /*major_dim_offset=*/column_start,
                       /*tile_size_along_major_dim=*/column_count);
@@ -302,7 +300,7 @@ class ColumnMajorMatrixVectorProductEmitter
   llvm::Value* rhs_;
   llvm::Value* addend_;
   llvm::Value* result_;
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
   KernelSupportLibrary ksl_;
   VectorSupportLibrary vsl_;
 };
@@ -331,7 +329,7 @@ void ColumnMajorMatrixVectorProductEmitter::Emit() {
                      });
 
   if (column_remainder != 0) {
-    EmitOuterLoopBody(ir_builder_->getInt64(column_limit), column_remainder,
+    EmitOuterLoopBody(b_->getInt64(column_limit), column_remainder,
                       column_limit == 0);
   }
 }
@@ -364,7 +362,7 @@ void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
     return;
   }
 
-  llvm::Value* columns_llvm = ir_builder_->getInt64(columns);
+  llvm::Value* columns_llvm = b_->getInt64(columns);
 
   // for (col = current_tile_col; col < (columns + current_tile_col); col++)
   //   for (row = row_start, row < m_; row++) {
@@ -375,12 +373,11 @@ void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
 
   ksl_.ForReturnVoid(
       "dot.inner.epilg.outer", /*start=*/current_tile_col,
-      /*end=*/ir_builder_->CreateAdd(columns_llvm, current_tile_col),
+      /*end=*/b_->CreateAdd(columns_llvm, current_tile_col),
       /*step=*/1, /*peel_first_iteration=*/false,
       [&](llvm::Value* col, llvm::Value* is_first_scalar_col) {
         llvm::Value* rhs_element = vsl_.LoadScalar(rhs_, col);
-        llvm::Value* total_offset =
-            ir_builder_->CreateMul(col, ir_builder_->getInt64(m()));
+        llvm::Value* total_offset = b_->CreateMul(col, b_->getInt64(m()));
         llvm::Value* lhs_base_pointer =
             vsl_.ComputeOffsetPointer(lhs_, total_offset);
         ksl_.ForReturnVoid(
@@ -388,9 +385,8 @@ void ColumnMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
             /*step=*/1, [&](llvm::Value* scalar_row) {
               llvm::Value* product = vsl_.Mul(
                   vsl_.LoadScalar(lhs_base_pointer, scalar_row), rhs_element);
-              llvm::Value* setting_result_first_time = ir_builder_->CreateAnd(
-                  is_first_scalar_col,
-                  ir_builder_->getInt1(is_first_tiled_column));
+              llvm::Value* setting_result_first_time = b_->CreateAnd(
+                  is_first_scalar_col, b_->getInt1(is_first_tiled_column));
               ksl_.IfReturnVoid(
                   setting_result_first_time,
                   /*true_block_generator=*/
@@ -478,16 +474,15 @@ class RowMajorMatrixVectorProductEmitter
 
   RowMajorMatrixVectorProductEmitter(const Config& config, llvm::Value* lhs,
                                      llvm::Value* rhs, llvm::Value* addend,
-                                     llvm::Value* result,
-                                     llvm::IRBuilder<>* ir_builder)
+                                     llvm::Value* result, llvm::IRBuilder<>* b)
       : config_(config),
         lhs_(lhs),
         rhs_(rhs),
         addend_(addend),
         result_(result),
-        ir_builder_(ir_builder),
-        ksl_(ir_builder_),
-        vsl_(scalar_type(), /*vector_size=*/tile_cols(), ir_builder_, "") {
+        b_(b),
+        ksl_(b_),
+        vsl_(scalar_type(), /*vector_size=*/tile_cols(), b_, "") {
     CHECK(tile_cols() > 0 && IsPowerOfTwo(static_cast(tile_cols())));
     CHECK(!has_addend() || addend != nullptr);
   }
@@ -498,7 +493,7 @@ class RowMajorMatrixVectorProductEmitter
 
  private:
   MemoryTile GetLhsMemoryTile(llvm::Value* row_start, int64 row_count) {
-    return MemoryTile(&vsl_, ir_builder_, /*matrix=*/lhs_,
+    return MemoryTile(&vsl_, b_, /*matrix=*/lhs_,
                       /*matrix_size_along_minor_dim=*/k(),
                       /*major_dim_offset=*/row_start,
                       /*tile_size_along_major_dim=*/row_count);
@@ -517,7 +512,7 @@ class RowMajorMatrixVectorProductEmitter
   llvm::Value* rhs_;
   llvm::Value* addend_;
   llvm::Value* result_;
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
   KernelSupportLibrary ksl_;
   VectorSupportLibrary vsl_;
 };
@@ -559,7 +554,7 @@ void RowMajorMatrixVectorProductEmitter::EmitOuterLoopBody(llvm::Value* row,
   for (int i = 0; i < row_count; i++) {
     llvm::Value* result_value =
         vsl_.Add(horizontal_sums[i], scalar_accumulators[i].Get());
-    llvm::Value* offset = ir_builder_->CreateAdd(ir_builder_->getInt64(i), row);
+    llvm::Value* offset = b_->CreateAdd(b_->getInt64(i), row);
     if (addend_ && row_count != vsl_.vector_size()) {
       result_value = vsl_.Add(vsl_.LoadScalar(addend_, offset), result_value);
     }
@@ -578,7 +573,7 @@ void RowMajorMatrixVectorProductEmitter::Emit() {
       [&](llvm::Value* row) { EmitOuterLoopBody(row, tile_rows()); });
 
   if (row_remainder != 0) {
-    EmitOuterLoopBody(ir_builder_->getInt64(row_limit), row_remainder);
+    EmitOuterLoopBody(b_->getInt64(row_limit), row_remainder);
   }
 }
 
@@ -609,9 +604,8 @@ void RowMajorMatrixVectorProductEmitter::EmitInnerLoopEpilogue(
   }
 
   for (int r = 0; r < rows; r++) {
-    llvm::Value* total_offset = ir_builder_->CreateMul(
-        ir_builder_->CreateAdd(ir_builder_->getInt64(r), current_tile_row),
-        ir_builder_->getInt64(k()));
+    llvm::Value* total_offset = b_->CreateMul(
+        b_->CreateAdd(b_->getInt64(r), current_tile_row), b_->getInt64(k()));
     llvm::Value* lhs_base_pointer =
         vsl_.ComputeOffsetPointer(lhs_, total_offset);
     ksl_.ForReturnVoid(
@@ -722,13 +716,13 @@ class MatrixMatrixBlockPanelEmitter {
   // `lhs` with `rhs` and stores the result in `result`.
   explicit MatrixMatrixBlockPanelEmitter(Config config, llvm::Value* lhs,
                                          llvm::Value* rhs, llvm::Value* result,
-                                         llvm::IRBuilder<>* ir_builder)
+                                         llvm::IRBuilder<>* b)
       : lhs_(lhs),
         rhs_(rhs),
         result_(result),
         config_(config),
-        ir_builder_(ir_builder),
-        ksl_(ir_builder_) {
+        b_(b),
+        ksl_(b_) {
     CHECK(max_vectorization_width() > 0 &&
           IsPowerOfTwo(static_cast(max_vectorization_width())));
     CHECK_GT(max_vector_count(), 0);
@@ -761,7 +755,7 @@ class MatrixMatrixBlockPanelEmitter {
                      int64 tile_size_m, llvm::Value* m_start,
                      llvm::Value* m_end);
 
-  llvm::Value* GetInt64(int64 value) { return ir_builder_->getInt64(value); }
+  llvm::Value* GetInt64(int64 value) { return b_->getInt64(value); }
 
   Config config() const { return config_; }
   Dimensions dims() const { return config().dims(); }
@@ -782,7 +776,7 @@ class MatrixMatrixBlockPanelEmitter {
   llvm::Value* result_;
   Config config_;
 
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
   KernelSupportLibrary ksl_;
 };
 
@@ -804,8 +798,8 @@ void MatrixMatrixBlockPanelEmitter::HandleResiduesOnN() {
          current_vectorization_width >= min_vectorization_width()) {
     int64 n_end = dims().n() - (dims().n() % current_vectorization_width);
     if (n_start != n_end) {
-      VectorSupportLibrary vsl(scalar_type(), current_vectorization_width,
-                               ir_builder_, "gebp");
+      VectorSupportLibrary vsl(scalar_type(), current_vectorization_width, b_,
+                               "gebp");
       HandleResiduesOnK(&vsl, GetInt64(n_start), GetInt64(n_end));
       n_start = n_end;
     }
@@ -819,10 +813,9 @@ void MatrixMatrixBlockPanelEmitter::HandleResiduesOnN() {
   }
 
   if (n_start != dims().n()) {
-    VectorSupportLibrary vsl(scalar_type(), 1, ir_builder_, "gebp");
+    VectorSupportLibrary vsl(scalar_type(), 1, b_, "gebp");
     ksl_.ForReturnVoid("epi.n", n_start, dims().n(), 1, [&](llvm::Value* n_i) {
-      llvm::Value* n_i_next =
-          ir_builder_->CreateAdd(n_i, ir_builder_->getInt64(1));
+      llvm::Value* n_i_next = b_->CreateAdd(n_i, b_->getInt64(1));
       HandleResiduesOnK(&vsl, n_i, n_i_next);
     });
   }
@@ -935,11 +928,11 @@ void MatrixMatrixBlockPanelEmitter::EmitTiledGemm(
   ksl_.ForReturnVoid(
       "dot.m", m_start, m_end, tile_size_m, [&](llvm::Value* m_i) {
         MemoryTile result_memory_tile(
-            vsl, ir_builder_, /*matrix=*/result_,
+            vsl, b_, /*matrix=*/result_,
             /*matrix_size_along_minor_dim=*/dims().n(),
             /*major_dim_offset=*/m_i,
             /*tile_size_along_major_dim=*/tile_size_m);
-        MemoryTile lhs_memory_tile(vsl, ir_builder_, /*matrix=*/lhs_,
+        MemoryTile lhs_memory_tile(vsl, b_, /*matrix=*/lhs_,
                                    /*matrix_size_along_minor_dim=*/dims().k(),
                                    /*major_dim_offset=*/m_i,
                                    /*tile_size_along_major_dim=*/tile_size_m);
@@ -949,8 +942,8 @@ void MatrixMatrixBlockPanelEmitter::EmitTiledGemm(
                                            result_memory_tile.LoadTile(n_i));
               ksl_.ForReturnVoid(
                   "dot.k", k_start, k_end, tile_size_k, [&](llvm::Value* k_i) {
-                    MemoryTile rhs_memory_tile(vsl, ir_builder_, rhs_,
-                                               dims().n(), k_i, tile_size_k);
+                    MemoryTile rhs_memory_tile(vsl, b_, rhs_, dims().n(), k_i,
+                                               tile_size_k);
                     std::vector> lhs_tile =
                         lhs_memory_tile.LoadBroadcastTile(k_i, tile_size_k);
                     std::vector rhs_tile =
@@ -980,7 +973,7 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
                            const llvm_ir::IrArray& rhs_array,
                            const llvm_ir::IrArray* addend_array,
                            llvm::Value* executable_run_options_value,
-                           llvm::IRBuilder<>* ir_builder,
+                           llvm::IRBuilder<>* b,
                            const HloModuleConfig& hlo_module_config,
                            const TargetMachineFeatures& target_machine_features)
     : dot_(dot),
@@ -989,7 +982,7 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
       rhs_array_(rhs_array),
       addend_array_(addend_array),
       executable_run_options_value_(executable_run_options_value),
-      ir_builder_(ir_builder),
+      b_(b),
       hlo_module_config_(hlo_module_config),
       target_machine_features_(target_machine_features) {}
 
@@ -997,15 +990,14 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot,
     const HloInstruction& dot, const llvm_ir::IrArray& target_array,
     const llvm_ir::IrArray& lhs_array, const llvm_ir::IrArray& rhs_array,
     const llvm_ir::IrArray* addend_array,
-    llvm::Value* executable_run_options_value, llvm::IRBuilder<>* ir_builder,
+    llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
     const HloModuleConfig& hlo_module_config,
     const TargetMachineFeatures& target_machine_features) {
   PrimitiveType type = target_array.GetShape().element_type();
   TF_RET_CHECK(F16 == type || F32 == type || F64 == type || C64 == type);
   DotOpEmitter dot_emitter(dot, target_array, lhs_array, rhs_array,
-                           addend_array, executable_run_options_value,
-                           ir_builder, hlo_module_config,
-                           target_machine_features);
+                           addend_array, executable_run_options_value, b,
+                           hlo_module_config, target_machine_features);
   return dot_emitter.Emit();
 }
 
@@ -1050,13 +1042,13 @@ bool DotOpEmitter::EmitExperimentalGebpDotIfEnabled(
   }
 
   int64 size_bytes = m * n * ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
-  ir_builder_->CreateMemSet(
-      target, ir_builder_->getInt8(0), size_bytes,
+  b_->CreateMemSet(
+      target, b_->getInt8(0), size_bytes,
       target_machine_features_.minimum_alignment_for_allocation(size_bytes));
 
   int64 max_target_vector_width =
       target_machine_features_.vector_register_num_elements(
-          *ir_builder_->GetInsertBlock()->getParent(), primitive_type);
+          *b_->GetInsertBlock()->getParent(), primitive_type);
 
   int64 tile_size_m, tile_size_k, tile_size_n_in_vector_width;
   std::tie(tile_size_m, tile_size_k, tile_size_n_in_vector_width) =
@@ -1080,12 +1072,12 @@ bool DotOpEmitter::EmitExperimentalGebpDotIfEnabled(
 
   KernelSupportLibrary::EmitAndCallOutlinedKernel(
       /*enable_fast_math=*/enable_fast_math,
-      /*optimize_for_size=*/optimize_for_size, ir_builder_,
-      config.GetCacheKey(), lhs, rhs, target,
+      /*optimize_for_size=*/optimize_for_size, b_, config.GetCacheKey(), lhs,
+      rhs, target,
       [this, config](llvm::Value* lhs, llvm::Value* rhs, llvm::Value* target) {
-        MatrixMatrixBlockPanelEmitter gebp_emitter(
-            config, /*lhs=*/lhs, /*rhs=*/rhs,
-            /*result=*/target, ir_builder_);
+        MatrixMatrixBlockPanelEmitter gebp_emitter(config, /*lhs=*/lhs,
+                                                   /*rhs=*/rhs,
+                                                   /*result=*/target, b_);
         gebp_emitter.Emit();
       });
 
@@ -1163,7 +1155,7 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
 
   const int target_vector_register_element_size =
       target_machine_features_.vector_register_num_elements(
-          *ir_builder_->GetInsertBlock()->getParent(), primitive_type);
+          *b_->GetInsertBlock()->getParent(), primitive_type);
 
   // We may not always know the vector register size for the target we're
   // compiling against, in which case target_vector_register_element_size is 0.
@@ -1184,13 +1176,13 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
 
     KernelSupportLibrary::EmitAndCallOutlinedKernel(
         /*enable_fast_math=*/enable_fast_math,
-        /*optimize_for_size=*/optimize_for_size, ir_builder_,
-        config.GetCacheKey(), lhs_op, rhs_op,
+        /*optimize_for_size=*/optimize_for_size, b_, config.GetCacheKey(),
+        lhs_op, rhs_op,
         addend_array_ ? addend_array_->GetBasePointer() : nullptr, result_op,
         [this, config](llvm::Value* lhs_op, llvm::Value* rhs_op,
                        llvm::Value* addend_op, llvm::Value* result_op) {
           ColumnMajorMatrixVectorProductEmitter emitter(
-              config, lhs_op, rhs_op, addend_op, result_op, ir_builder_);
+              config, lhs_op, rhs_op, addend_op, result_op, b_);
           emitter.Emit();
         });
   } else {
@@ -1203,13 +1195,13 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() {
 
     KernelSupportLibrary::EmitAndCallOutlinedKernel(
         /*enable_fast_math=*/enable_fast_math,
-        /*optimize_for_size=*/optimize_for_size, ir_builder_,
-        config.GetCacheKey(), lhs_op, rhs_op,
+        /*optimize_for_size=*/optimize_for_size, b_, config.GetCacheKey(),
+        lhs_op, rhs_op,
         addend_array_ ? addend_array_->GetBasePointer() : nullptr, result_op,
         [this, config](llvm::Value* lhs_op, llvm::Value* rhs_op,
                        llvm::Value* addend_op, llvm::Value* result_op) {
-          RowMajorMatrixVectorProductEmitter emitter(
-              config, lhs_op, rhs_op, addend_op, result_op, ir_builder_);
+          RowMajorMatrixVectorProductEmitter emitter(config, lhs_op, rhs_op,
+                                                     addend_op, result_op, b_);
           emitter.Emit();
         });
   }
@@ -1285,7 +1277,7 @@ Status DotOpEmitter::Emit() {
   // Create loop nests which loop through the LHS operand dimensions and the RHS
   // operand dimensions. The reduction dimension of the LHS and RHS are handled
   // in a separate innermost loop which performs the sum of products.
-  llvm_ir::ForLoopNest loop_nest(llvm_ir::IrName(&dot_), ir_builder_);
+  llvm_ir::ForLoopNest loop_nest(llvm_ir::IrName(&dot_), b_);
   llvm_ir::IrArray::Index lhs_index = EmitOperandArrayLoopNest(
       &loop_nest, lhs_array_, lhs_reduction_dimension, "lhs");
   llvm_ir::IrArray::Index rhs_index = EmitOperandArrayLoopNest(
@@ -1319,62 +1311,55 @@ Status DotOpEmitter::Emit() {
   // Function entry basic block.
   // - Emit alloca for accumulator
   llvm::Function* func = reduction_loop->GetPreheaderBasicBlock()->getParent();
-  SetToFirstInsertPoint(&func->getEntryBlock(), ir_builder_);
+  SetToFirstInsertPoint(&func->getEntryBlock(), b_);
   llvm::Type* accum_type = target_array_.GetElementLlvmType();
-  llvm::Value* accum_address = ir_builder_->CreateAlloca(
-      accum_type, /*ArraySize=*/nullptr, "accum_address");
+  llvm::Value* accum_address =
+      b_->CreateAlloca(accum_type, /*ArraySize=*/nullptr, "accum_address");
 
   // Preheader basic block of reduction loop:
   // - Initialize accumulator to zero.
   llvm::BasicBlock* preheader_bb = reduction_loop->GetPreheaderBasicBlock();
-  ir_builder_->SetInsertPoint(preheader_bb->getTerminator());
+  b_->SetInsertPoint(preheader_bb->getTerminator());
 
-  ir_builder_->CreateStore(llvm::Constant::getNullValue(accum_type),
-                           accum_address);
+  b_->CreateStore(llvm::Constant::getNullValue(accum_type), accum_address);
 
   // Body basic block of reduction loop:
   // - Load elements from lhs and rhs array.
   // - Multiply lhs-element and rhs-element.
   // - Load accumulator and add to product.
   // - Store sum back into accumulator.
-  SetToFirstInsertPoint(reduction_loop->GetBodyBasicBlock(), ir_builder_);
+  SetToFirstInsertPoint(reduction_loop->GetBodyBasicBlock(), b_);
 
-  llvm::Value* lhs_element =
-      lhs_array_.EmitReadArrayElement(lhs_index, ir_builder_);
-  llvm::Value* rhs_element =
-      rhs_array_.EmitReadArrayElement(rhs_index, ir_builder_);
+  llvm::Value* lhs_element = lhs_array_.EmitReadArrayElement(lhs_index, b_);
+  llvm::Value* rhs_element = rhs_array_.EmitReadArrayElement(rhs_index, b_);
 
-  llvm::Value* accum = ir_builder_->CreateLoad(accum_address);
+  llvm::Value* accum = b_->CreateLoad(accum_address);
   llvm::Value* updated_accum;
   if (ShapeUtil::ElementIsComplex(lhs_shape)) {
-    auto real = [&](llvm::Value* x) {
-      return ir_builder_->CreateExtractValue(x, {0});
-    };
-    auto imag = [&](llvm::Value* x) {
-      return ir_builder_->CreateExtractValue(x, {1});
-    };
-    llvm::Value* product_real = ir_builder_->CreateFSub(
-        ir_builder_->CreateFMul(real(lhs_element), real(rhs_element)),
-        ir_builder_->CreateFMul(imag(lhs_element), imag(rhs_element)));
-    llvm::Value* product_imag = ir_builder_->CreateFAdd(
-        ir_builder_->CreateFMul(real(lhs_element), imag(rhs_element)),
-        ir_builder_->CreateFMul(imag(lhs_element), real(rhs_element)));
-    updated_accum = ir_builder_->CreateInsertValue(
-        accum, ir_builder_->CreateFAdd(real(accum), product_real), {0});
-    updated_accum = ir_builder_->CreateInsertValue(
-        updated_accum, ir_builder_->CreateFAdd(imag(accum), product_imag), {1});
+    auto real = [&](llvm::Value* x) { return b_->CreateExtractValue(x, {0}); };
+    auto imag = [&](llvm::Value* x) { return b_->CreateExtractValue(x, {1}); };
+    llvm::Value* product_real =
+        b_->CreateFSub(b_->CreateFMul(real(lhs_element), real(rhs_element)),
+                       b_->CreateFMul(imag(lhs_element), imag(rhs_element)));
+    llvm::Value* product_imag =
+        b_->CreateFAdd(b_->CreateFMul(real(lhs_element), imag(rhs_element)),
+                       b_->CreateFMul(imag(lhs_element), real(rhs_element)));
+    updated_accum = b_->CreateInsertValue(
+        accum, b_->CreateFAdd(real(accum), product_real), {0});
+    updated_accum = b_->CreateInsertValue(
+        updated_accum, b_->CreateFAdd(imag(accum), product_imag), {1});
   } else {
-    llvm::Value* product = ir_builder_->CreateFMul(lhs_element, rhs_element);
-    updated_accum = ir_builder_->CreateFAdd(accum, product);
+    llvm::Value* product = b_->CreateFMul(lhs_element, rhs_element);
+    updated_accum = b_->CreateFAdd(accum, product);
   }
-  ir_builder_->CreateStore(updated_accum, accum_address);
+  b_->CreateStore(updated_accum, accum_address);
 
   // Exit basic block of reduction loop.
   // - Load accumulator value (the result).
   // - Store into output array.
-  SetToFirstInsertPoint(reduction_loop->GetExitBasicBlock(), ir_builder_);
+  SetToFirstInsertPoint(reduction_loop->GetExitBasicBlock(), b_);
 
-  llvm::Value* result = ir_builder_->CreateLoad(accum_address);
+  llvm::Value* result = b_->CreateLoad(accum_address);
 
   // Create index into target address. The target index is the concatenation of
   // the rhs and lhs indexes with the reduction dimensions removed. The terms
@@ -1392,11 +1377,11 @@ Status DotOpEmitter::Emit() {
     }
   }
 
-  target_array_.EmitWriteArrayElement(target_index, result, ir_builder_);
+  target_array_.EmitWriteArrayElement(target_index, result, b_);
 
   // Set the IR builder insert point to the exit basic block of the outer most
   // loop.
-  ir_builder_->SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
+  b_->SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
 
   return Status::OK();
 }
@@ -1405,31 +1390,30 @@ Status DotOpEmitter::EmitScalarDot() {
   // A scalar dot is just a scalar multiply.
   llvm::Value* result;
   // Use the same index_type for all tensor accesses in the same kernel.
-  llvm::Type* index_type = ir_builder_->getInt64Ty();
+  llvm::Type* index_type = b_->getInt64Ty();
   llvm_ir::IrArray::Index element_index(index_type);
   llvm::Value* lhs_value =
-      lhs_array_.EmitReadArrayElement(/*index=*/element_index, ir_builder_);
+      lhs_array_.EmitReadArrayElement(/*index=*/element_index, b_);
   llvm::Value* rhs_value =
-      rhs_array_.EmitReadArrayElement(/*index=*/element_index, ir_builder_);
+      rhs_array_.EmitReadArrayElement(/*index=*/element_index, b_);
   if (ShapeUtil::ElementIsComplex(lhs_array_.GetShape())) {
-#define REAL(x) ir_builder_->CreateExtractValue(x, {0})
-#define IMAG(x) ir_builder_->CreateExtractValue(x, {1})
-    llvm::Value* real = ir_builder_->CreateFSub(
-        ir_builder_->CreateFMul(REAL(lhs_value), REAL(rhs_value)),
-        ir_builder_->CreateFMul(IMAG(lhs_value), IMAG(rhs_value)));
-    llvm::Value* imag = ir_builder_->CreateFAdd(
-        ir_builder_->CreateFMul(REAL(lhs_value), IMAG(rhs_value)),
-        ir_builder_->CreateFMul(IMAG(lhs_value), REAL(rhs_value)));
+#define REAL(x) b_->CreateExtractValue(x, {0})
+#define IMAG(x) b_->CreateExtractValue(x, {1})
+    llvm::Value* real =
+        b_->CreateFSub(b_->CreateFMul(REAL(lhs_value), REAL(rhs_value)),
+                       b_->CreateFMul(IMAG(lhs_value), IMAG(rhs_value)));
+    llvm::Value* imag =
+        b_->CreateFAdd(b_->CreateFMul(REAL(lhs_value), IMAG(rhs_value)),
+                       b_->CreateFMul(IMAG(lhs_value), REAL(rhs_value)));
 #undef IMAG
 #undef REAL
     result = llvm::ConstantAggregateZero::get(lhs_array_.GetElementLlvmType());
-    result = ir_builder_->CreateInsertValue(result, real, {0});
-    result = ir_builder_->CreateInsertValue(result, imag, {1});
+    result = b_->CreateInsertValue(result, real, {0});
+    result = b_->CreateInsertValue(result, imag, {1});
   } else {
-    result = ir_builder_->CreateFMul(lhs_value, rhs_value);
+    result = b_->CreateFMul(lhs_value, rhs_value);
   }
-  target_array_.EmitWriteArrayElement(/*index=*/element_index, result,
-                                      ir_builder_);
+  target_array_.EmitWriteArrayElement(/*index=*/element_index, result, b_);
   return Status::OK();
 }
 
@@ -1452,7 +1436,7 @@ Status DotOpEmitter::EmitCallToRuntime() {
       fn_name = multi_threaded
                     ? runtime::kEigenMatMulF16SymbolName
                     : runtime::kEigenSingleThreadedMatMulF16SymbolName;
-      float_type = ir_builder_->getHalfTy();
+      float_type = b_->getHalfTy();
       break;
     case F32:
       fn_name = multi_threaded
@@ -1461,7 +1445,7 @@ Status DotOpEmitter::EmitCallToRuntime() {
                     : (use_mkl_dnn
                            ? runtime::kMKLSingleThreadedMatMulF32SymbolName
                            : runtime::kEigenSingleThreadedMatMulF32SymbolName);
-      float_type = ir_builder_->getFloatTy();
+      float_type = b_->getFloatTy();
       break;
     case F64:
       fn_name = multi_threaded
@@ -1470,7 +1454,7 @@ Status DotOpEmitter::EmitCallToRuntime() {
                     : (use_mkl_dnn
                            ? runtime::kMKLSingleThreadedMatMulF64SymbolName
                            : runtime::kEigenSingleThreadedMatMulF64SymbolName);
-      float_type = ir_builder_->getDoubleTy();
+      float_type = b_->getDoubleTy();
       break;
     default:
       return Unimplemented("Invalid type %s for dot operation",
@@ -1478,16 +1462,16 @@ Status DotOpEmitter::EmitCallToRuntime() {
   }
 
   llvm::Type* float_ptr_type = float_type->getPointerTo();
-  llvm::Type* int64_type = ir_builder_->getInt64Ty();
-  llvm::Type* int32_type = ir_builder_->getInt32Ty();
-  llvm::Type* int8_ptr_type = ir_builder_->getInt8Ty()->getPointerTo();
+  llvm::Type* int64_type = b_->getInt64Ty();
+  llvm::Type* int32_type = b_->getInt32Ty();
+  llvm::Type* int8_ptr_type = b_->getInt8Ty()->getPointerTo();
   llvm::FunctionType* matmul_type = llvm::FunctionType::get(
-      ir_builder_->getVoidTy(),
+      b_->getVoidTy(),
       {int8_ptr_type, float_ptr_type, float_ptr_type, float_ptr_type,
        int64_type, int64_type, int64_type, int32_type, int32_type},
       /*isVarArg=*/false);
 
-  llvm::Function* function = ir_builder_->GetInsertBlock()->getParent();
+  llvm::Function* function = b_->GetInsertBlock()->getParent();
   llvm::Module* module = function->getParent();
 
   llvm::Function* matmul_func = llvm::cast(
@@ -1522,18 +1506,15 @@ Status DotOpEmitter::EmitCallToRuntime() {
     std::swap(transpose_lhs, transpose_rhs);
   }
 
-  ir_builder_->CreateCall(
+  b_->CreateCall(
       matmul_func,
-      {ir_builder_->CreateBitCast(executable_run_options_value_, int8_ptr_type),
-       ir_builder_->CreateBitCast(target_array_.GetBasePointer(),
-                                  float_ptr_type),
-       ir_builder_->CreateBitCast(lhs->GetBasePointer(), float_ptr_type),
-       ir_builder_->CreateBitCast(rhs->GetBasePointer(), float_ptr_type),
-       ir_builder_->getInt64(mat_mult_dims.m),
-       ir_builder_->getInt64(mat_mult_dims.n),
-       ir_builder_->getInt64(mat_mult_dims.k),
-       ir_builder_->getInt32(transpose_lhs),
-       ir_builder_->getInt32(transpose_rhs)});
+      {b_->CreateBitCast(executable_run_options_value_, int8_ptr_type),
+       b_->CreateBitCast(target_array_.GetBasePointer(), float_ptr_type),
+       b_->CreateBitCast(lhs->GetBasePointer(), float_ptr_type),
+       b_->CreateBitCast(rhs->GetBasePointer(), float_ptr_type),
+       b_->getInt64(mat_mult_dims.m), b_->getInt64(mat_mult_dims.n),
+       b_->getInt64(mat_mult_dims.k), b_->getInt32(transpose_lhs),
+       b_->getInt32(transpose_rhs)});
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
index ed2a18976a..c2eeb0a1f9 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h
@@ -61,7 +61,7 @@ class DotOpEmitter {
       const HloInstruction& dot, const llvm_ir::IrArray& target_array,
       const llvm_ir::IrArray& lhs_array, const llvm_ir::IrArray& rhs_array,
       const llvm_ir::IrArray* addend_array,
-      llvm::Value* executable_run_options_value, llvm::IRBuilder<>* ir_builder,
+      llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
       const HloModuleConfig& hlo_module_config,
       const TargetMachineFeatures& target_machine_features);
 
@@ -70,8 +70,7 @@ class DotOpEmitter {
                const llvm_ir::IrArray& lhs_array,
                const llvm_ir::IrArray& rhs_array,
                const llvm_ir::IrArray* addend_array,
-               llvm::Value* executable_run_options_value,
-               llvm::IRBuilder<>* ir_builder,
+               llvm::Value* executable_run_options_value, llvm::IRBuilder<>* b,
                const HloModuleConfig& hlo_module_config,
                const TargetMachineFeatures& target_machine_features);
 
@@ -171,7 +170,7 @@ class DotOpEmitter {
   const llvm_ir::IrArray& rhs_array_;
   const llvm_ir::IrArray* addend_array_;
   llvm::Value* executable_run_options_value_;
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
   const HloModuleConfig& hlo_module_config_;
   const TargetMachineFeatures& target_machine_features_;
 };
diff --git a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc
index e97113dfa0..cf955a8add 100644
--- a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc
@@ -38,8 +38,7 @@ StatusOr CpuElementalIrEmitter::EmitFloatUnaryOp(
       switch (element_type) {
         case F16:
           cast_result_to_fp16 = true;
-          operand_value = ir_builder_->CreateFPCast(operand_value,
-                                                    ir_builder_->getFloatTy());
+          operand_value = b_->CreateFPCast(operand_value, b_->getFloatTy());
           TF_FALLTHROUGH_INTENDED;
         case F32:
           function_name = "tanhf";
@@ -59,9 +58,9 @@ StatusOr CpuElementalIrEmitter::EmitFloatUnaryOp(
       function->setDoesNotThrow();
       function->setDoesNotAccessMemory();
       // Create an instruction to call the function.
-      llvm::Value* result = ir_builder_->CreateCall(function, operand_value);
+      llvm::Value* result = b_->CreateCall(function, operand_value);
       if (cast_result_to_fp16) {
-        result = ir_builder_->CreateFPCast(result, ir_builder_->getHalfTy());
+        result = b_->CreateFPCast(result, b_->getHalfTy());
       }
       return result;
     }
@@ -77,8 +76,8 @@ StatusOr CpuElementalIrEmitter::EmitAtan2(
   switch (prim_type) {
     case F16:
       cast_result_to_fp16 = true;
-      lhs = ir_builder_->CreateFPCast(lhs, ir_builder_->getFloatTy());
-      rhs = ir_builder_->CreateFPCast(rhs, ir_builder_->getFloatTy());
+      lhs = b_->CreateFPCast(lhs, b_->getFloatTy());
+      rhs = b_->CreateFPCast(rhs, b_->getFloatTy());
       TF_FALLTHROUGH_INTENDED;
     case F32:
       function_name = "atan2f";
@@ -98,9 +97,9 @@ StatusOr CpuElementalIrEmitter::EmitAtan2(
   function->setDoesNotThrow();
   function->setDoesNotAccessMemory();
   // Create an instruction to call the function.
-  llvm::Value* result = ir_builder_->CreateCall(function, {lhs, rhs});
+  llvm::Value* result = b_->CreateCall(function, {lhs, rhs});
   if (cast_result_to_fp16) {
-    result = ir_builder_->CreateFPCast(result, ir_builder_->getHalfTy());
+    result = b_->CreateFPCast(result, b_->getHalfTy());
   }
   return result;
 }
diff --git a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h
index 4446dfd282..9598a886ab 100644
--- a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h
@@ -31,7 +31,7 @@ class CpuElementalIrEmitter : public ElementalIrEmitter {
  public:
   CpuElementalIrEmitter(const HloModuleConfig& module_config,
                         IrEmitter* ir_emitter, llvm::Module* module)
-      : ElementalIrEmitter(module_config, module, ir_emitter->ir_builder()),
+      : ElementalIrEmitter(module_config, module, ir_emitter->b()),
         ir_emitter_(ir_emitter) {}
 
   llvm_ir::ElementGenerator MakeElementGenerator(
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index aeab5d8957..d4ac35a604 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -89,14 +89,14 @@ IrEmitter::IrEmitter(
     : assignment_(assignment),
       module_(llvm_module),
       arch_type_(llvm::Triple(llvm_module->getTargetTriple()).getArch()),
-      ir_builder_(llvm_module->getContext()),
+      b_(llvm_module->getContext()),
       instruction_to_profile_idx_(std::move(instruction_to_profile_idx)),
       computation_to_profile_idx_(std::move(computation_to_profile_idx)),
       alias_analysis_(hlo_module, assignment, &llvm_module->getContext()),
       hlo_module_config_(hlo_module.config()),
       is_top_level_computation_(false),
       target_machine_features_(*target_machine_features) {
-  ir_builder_.setFastMathFlags(llvm_ir::GetFastMathFlags(
+  b_.setFastMathFlags(llvm_ir::GetFastMathFlags(
       /*fast_math_enabled=*/hlo_module_config_.debug_options()
           .xla_enable_fast_math()));
 }
@@ -146,7 +146,7 @@ void IrEmitter::InitializeIrFunction(const string& function_name) {
       new IrFunction(function_name, linkage,
                      options::OptimizeForSizeRequested(hlo_module_config_),
                      hlo_module_config_.debug_options().xla_enable_fast_math(),
-                     module_, &ir_builder_, num_dynamic_loop_bounds_));
+                     module_, &b_, num_dynamic_loop_bounds_));
 }
 
 IrEmitter::~IrEmitter() {}
@@ -154,9 +154,9 @@ IrEmitter::~IrEmitter() {}
 Status IrEmitter::HandleBitcast(HloInstruction* bitcast) {
   VLOG(2) << "HandleBitcast: " << bitcast->ToString();
   emitted_value_[bitcast] =
-      ir_builder_.CreateBitCast(GetEmittedValueFor(bitcast->operand(0)),
-                                IrShapeType(bitcast->shape())->getPointerTo(),
-                                AsStringRef(IrName(bitcast)));
+      b_.CreateBitCast(GetEmittedValueFor(bitcast->operand(0)),
+                       IrShapeType(bitcast->shape())->getPointerTo(),
+                       AsStringRef(IrName(bitcast)));
   return Status::OK();
 }
 
@@ -273,7 +273,7 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) {
   const Shape& shape = get_tuple_element->shape();
   emitted_value_[get_tuple_element] = llvm_ir::EmitGetTupleElement(
       shape, get_tuple_element->tuple_index(), MinimumAlignmentForShape(shape),
-      GetEmittedValueFor(operand), &ir_builder_, module_);
+      GetEmittedValueFor(operand), &b_, module_);
   return Status::OK();
 }
 
@@ -293,7 +293,7 @@ Status IrEmitter::HandleTupleSelect(HloInstruction* tuple_select) {
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(tuple_select));
   llvm_ir::EmitTupleSelect(GetIrArrayFor(tuple_select), GetIrArrayFor(pred),
                            GetEmittedValueFor(on_true),
-                           GetEmittedValueFor(on_false), &ir_builder_, module_);
+                           GetEmittedValueFor(on_false), &b_, module_);
   return Status::OK();
 }
 
@@ -316,8 +316,8 @@ Status IrEmitter::HandleInfeed(HloInstruction* instruction) {
                       assignment_.GetUniqueSlice(infeed, {1}));
   llvm::Value* token_address = EmitTempBufferPointer(
       token_slice, ShapeUtil::GetTupleElementShape(infeed->shape(), 1));
-  llvm_ir::EmitTuple(GetIrArrayFor(infeed), {data_address, token_address},
-                     &ir_builder_, module_);
+  llvm_ir::EmitTuple(GetIrArrayFor(infeed), {data_address, token_address}, &b_,
+                     module_);
 
   if (ShapeUtil::IsTuple(data_shape)) {
     TF_RET_CHECK(!ShapeUtil::IsNestedTuple(data_shape));
@@ -348,7 +348,7 @@ Status IrEmitter::HandleInfeed(HloInstruction* instruction) {
     }
 
     llvm_ir::EmitTuple(llvm_ir::IrArray(data_address, data_shape),
-                       tuple_element_addresses, &ir_builder_, module_);
+                       tuple_element_addresses, &b_, module_);
   } else {
     TF_RETURN_IF_ERROR(
         EmitXfeedTransfer(XfeedKind::kInfeed, data_shape, data_address));
@@ -369,14 +369,14 @@ Status IrEmitter::EmitXfeedTransfer(XfeedKind kind, const Shape& shape,
   int32 length_32 = static_cast(length);
 
   int32 shape_length;
-  TF_ASSIGN_OR_RETURN(llvm::Value * shape_ptr,
-                      llvm_ir::EncodeSelfDescribingShapeConstant(
-                          shape, &shape_length, &ir_builder_));
+  TF_ASSIGN_OR_RETURN(
+      llvm::Value * shape_ptr,
+      llvm_ir::EncodeSelfDescribingShapeConstant(shape, &shape_length, &b_));
 
   // The signature of the acquire infeed buffer function is:
   //
   //   (void*)(int32 length);
-  llvm::Type* int32_type = ir_builder_.getInt32Ty();
+  llvm::Type* int32_type = b_.getInt32Ty();
   llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext());
   llvm::FunctionType* acquire_type = llvm::FunctionType::get(
       i8_ptr_type, {int32_type, i8_ptr_type, int32_type},
@@ -396,8 +396,7 @@ Status IrEmitter::EmitXfeedTransfer(XfeedKind kind, const Shape& shape,
   //
   //   (void)(int32 length, void* buffer);
   llvm::FunctionType* release_type = llvm::FunctionType::get(
-      ir_builder_.getVoidTy(),
-      {int32_type, i8_ptr_type, i8_ptr_type, int32_type},
+      b_.getVoidTy(), {int32_type, i8_ptr_type, i8_ptr_type, int32_type},
       /*isVarArg=*/false);
 
   llvm::Function* release_func;
@@ -414,25 +413,22 @@ Status IrEmitter::EmitXfeedTransfer(XfeedKind kind, const Shape& shape,
   // of size exactly 'length_32', and the runtime is responsible for
   // check-failing the process if there is a mismatch, versus passing us back a
   // buffer that we might overrun.
-  llvm::Value* acquired_pointer = ir_builder_.CreateCall(
-      acquire_func, {ir_builder_.getInt32(length_32), shape_ptr,
-                     ir_builder_.getInt32(shape_length)});
+  llvm::Value* acquired_pointer = b_.CreateCall(
+      acquire_func,
+      {b_.getInt32(length_32), shape_ptr, b_.getInt32(shape_length)});
 
   if (kind == XfeedKind::kInfeed) {
     // Copy to the program buffer address from the acquired buffer.
-    ir_builder_.CreateMemCpy(program_buffer_address, /*DstAlign=*/1,
-                             acquired_pointer,
-                             /*SrcAlign=*/1, length_32);
+    b_.CreateMemCpy(program_buffer_address, /*DstAlign=*/1, acquired_pointer,
+                    /*SrcAlign=*/1, length_32);
   } else {
     // Outfeed -- copy from the in-program address to the acquired buffer.
-    ir_builder_.CreateMemCpy(acquired_pointer, /*DstAlign=*/1,
-                             program_buffer_address,
-                             /*SrcAlign=*/1, length_32);
+    b_.CreateMemCpy(acquired_pointer, /*DstAlign=*/1, program_buffer_address,
+                    /*SrcAlign=*/1, length_32);
   }
 
-  ir_builder_.CreateCall(release_func,
-                         {ir_builder_.getInt32(length_32), acquired_pointer,
-                          shape_ptr, ir_builder_.getInt32(shape_length)});
+  b_.CreateCall(release_func, {b_.getInt32(length_32), acquired_pointer,
+                               shape_ptr, b_.getInt32(shape_length)});
 
   return Status::OK();
 }
@@ -453,7 +449,7 @@ Status IrEmitter::HandleOutfeed(HloInstruction* outfeed) {
         ShapeUtil::GetTupleElementShape(operand_shape, i);
     llvm::Value* tuple_element = llvm_ir::EmitGetTupleElement(
         tuple_element_shape, i, MinimumAlignmentForShape(tuple_element_shape),
-        value, &ir_builder_, module_);
+        value, &b_, module_);
     TF_RETURN_IF_ERROR(EmitXfeedTransfer(XfeedKind::kOutfeed,
                                          tuple_element_shape, tuple_element));
   }
@@ -472,7 +468,7 @@ Status IrEmitter::HandleTuple(HloInstruction* tuple) {
   for (auto operand : tuple->operands()) {
     base_ptrs.push_back(GetEmittedValueFor(operand));
   }
-  llvm_ir::EmitTuple(GetIrArrayFor(tuple), base_ptrs, &ir_builder_, module_);
+  llvm_ir::EmitTuple(GetIrArrayFor(tuple), base_ptrs, &b_, module_);
   return Status::OK();
 }
 
@@ -483,8 +479,7 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForMap(
   std::vector parameter_addresses;
   for (const HloInstruction* operand : map->operands()) {
     const llvm_ir::IrArray& array = GetIrArrayFor(operand);
-    parameter_addresses.push_back(
-        array.EmitArrayElementAddress(index, &ir_builder_));
+    parameter_addresses.push_back(array.EmitArrayElementAddress(index, &b_));
   }
   return EmitElementFunctionCall(mapped_ir_function, map->shape(),
                                  parameter_addresses, "map_function");
@@ -510,13 +505,12 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForReduceWindow(
   PrimitiveType operand_element_type = operand->shape().element_type();
   llvm::Value* accumulator_address = llvm_ir::EmitAllocaAtFunctionEntry(
       llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_),
-      "reduce_window_accumulator_address", &ir_builder_,
+      "reduce_window_accumulator_address", &b_,
       MinimumAlignmentForPrimitiveType(operand_element_type));
-  ir_builder_.CreateStore(
-      ir_builder_.CreateLoad(GetEmittedValueFor(reduce_window->operand(1))),
-      accumulator_address);
+  b_.CreateStore(b_.CreateLoad(GetEmittedValueFor(reduce_window->operand(1))),
+                 accumulator_address);
 
-  llvm_ir::ForLoopNest loops(IrName(reduce_window, "inner"), &ir_builder_);
+  llvm_ir::ForLoopNest loops(IrName(reduce_window, "inner"), &b_);
   std::vector window_size;
   for (const auto& dim : window.dimensions()) {
     window_size.push_back(dim.size());
@@ -525,48 +519,47 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForReduceWindow(
       ShapeUtil::MakeShape(operand_element_type, window_size), "window");
   CHECK_EQ(window_index.size(), index.size());
 
-  SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
+  SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &b_);
 
-  llvm_ir::IrArray::Index input_index(ir_builder_.getInt64Ty(), index.size());
+  llvm_ir::IrArray::Index input_index(b_.getInt64Ty(), index.size());
   llvm::Value* in_bounds_condition = nullptr;
   for (size_t i = 0; i < index.size(); ++i) {
-    llvm::Value* strided_index = ir_builder_.CreateNSWMul(
-        index[i], ir_builder_.getInt64(window.dimensions(i).stride()));
-    input_index[i] = ir_builder_.CreateNSWSub(
-        ir_builder_.CreateNSWAdd(strided_index, window_index[i]),
-        ir_builder_.getInt64(window.dimensions(i).padding_low()));
+    llvm::Value* strided_index =
+        b_.CreateNSWMul(index[i], b_.getInt64(window.dimensions(i).stride()));
+    input_index[i] =
+        b_.CreateNSWSub(b_.CreateNSWAdd(strided_index, window_index[i]),
+                        b_.getInt64(window.dimensions(i).padding_low()));
 
     // We need to check if 0 <= input_index[i] < bound, as otherwise we are in
     // the padding so that we can skip the computation. That is equivalent to
     // input_index[i] < bound as an *unsigned* comparison, since a negative
     // value will wrap to a large positive value.
-    llvm::Value* index_condition = ir_builder_.CreateICmpULT(
+    llvm::Value* index_condition = b_.CreateICmpULT(
         input_index[i],
-        ir_builder_.getInt64(ShapeUtil::GetDimension(operand->shape(), i)));
+        b_.getInt64(ShapeUtil::GetDimension(operand->shape(), i)));
     if (in_bounds_condition == nullptr) {
       in_bounds_condition = index_condition;
     } else {
-      in_bounds_condition =
-          ir_builder_.CreateAnd(in_bounds_condition, index_condition);
+      in_bounds_condition = b_.CreateAnd(in_bounds_condition, index_condition);
     }
   }
   CHECK(in_bounds_condition != nullptr);
 
   llvm_ir::LlvmIfData if_data =
-      llvm_ir::EmitIfThenElse(in_bounds_condition, "in-bounds", &ir_builder_);
-  SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
+      llvm_ir::EmitIfThenElse(in_bounds_condition, "in-bounds", &b_);
+  SetToFirstInsertPoint(if_data.true_block, &b_);
 
   // We are not in the padding, so carry out the computation.
   llvm_ir::IrArray input_array(GetIrArrayFor(operand));
   llvm::Value* input_value_address =
-      input_array.EmitArrayElementAddress(input_index, &ir_builder_);
+      input_array.EmitArrayElementAddress(input_index, &b_);
   llvm::Value* result = EmitElementFunctionCall(
       reducer_function, reduce_window->shape(),
       {accumulator_address, input_value_address}, "reducer_function");
-  ir_builder_.CreateStore(result, accumulator_address);
+  b_.CreateStore(result, accumulator_address);
 
-  SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
-  return ir_builder_.CreateLoad(accumulator_address);
+  SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &b_);
+  return b_.CreateLoad(accumulator_address);
 }
 
 Status IrEmitter::HandleReduceWindow(HloInstruction* reduce_window) {
@@ -649,141 +642,127 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) {
       select_and_scatter, /*desc=*/IrName(select_and_scatter, "init"),
       [this, init_value](const llvm_ir::IrArray::Index& target_index) {
         llvm::Value* init_value_addr = GetEmittedValueFor(init_value);
-        return ir_builder_.CreateLoad(init_value_addr);
+        return b_.CreateLoad(init_value_addr);
       }));
 
   // Create a loop to iterate over the source array to scatter to the output.
-  llvm_ir::ForLoopNest source_loops(IrName(select_and_scatter), &ir_builder_);
+  llvm_ir::ForLoopNest source_loops(IrName(select_and_scatter), &b_);
   const llvm_ir::IrArray::Index source_index =
       source_loops.AddLoopsForShape(source->shape(), "source");
-  SetToFirstInsertPoint(source_loops.GetInnerLoopBodyBasicBlock(),
-                        &ir_builder_);
+  SetToFirstInsertPoint(source_loops.GetInnerLoopBodyBasicBlock(), &b_);
 
   // Allocate space to keep the currently selected value, its index, and
   // the boolean initialized_flag, which is initially set to false.
   llvm::Value* selected_value_address = llvm_ir::EmitAllocaAtFunctionEntry(
       llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_),
-      "selected_value_address", &ir_builder_,
+      "selected_value_address", &b_,
       MinimumAlignmentForPrimitiveType(operand_element_type));
   llvm::Value* selected_index_address =
       llvm_ir::EmitAllocaAtFunctionEntryWithCount(
-          ir_builder_.getInt64Ty(), ir_builder_.getInt32(rank),
-          "selected_index_address", &ir_builder_);
+          b_.getInt64Ty(), b_.getInt32(rank), "selected_index_address", &b_);
   llvm::Value* initialized_flag_address = llvm_ir::EmitAllocaAtFunctionEntry(
-      ir_builder_.getInt1Ty(), "initialized_flag_address", &ir_builder_);
-  ir_builder_.CreateStore(ir_builder_.getInt1(false), initialized_flag_address);
+      b_.getInt1Ty(), "initialized_flag_address", &b_);
+  b_.CreateStore(b_.getInt1(false), initialized_flag_address);
 
   // Create the inner loop to iterate over the window.
-  llvm_ir::ForLoopNest window_loops(IrName(select_and_scatter, "window"),
-                                    &ir_builder_);
+  llvm_ir::ForLoopNest window_loops(IrName(select_and_scatter, "window"), &b_);
   std::vector window_size;
   for (const auto& dim : window.dimensions()) {
     window_size.push_back(dim.size());
   }
   const llvm_ir::IrArray::Index window_index = window_loops.AddLoopsForShape(
       ShapeUtil::MakeShape(operand_element_type, window_size), "window");
-  SetToFirstInsertPoint(window_loops.GetInnerLoopBodyBasicBlock(),
-                        &ir_builder_);
+  SetToFirstInsertPoint(window_loops.GetInnerLoopBodyBasicBlock(), &b_);
 
   // Compute the operand index to visit and evaluate the condition whether the
   // operand index is within the bounds. The unsigned comparison includes
   // checking whether the operand index >= 0.
-  llvm_ir::IrArray::Index operand_index(ir_builder_.getInt64Ty(),
-                                        source_index.size());
-  llvm::Value* in_bounds_condition = ir_builder_.getTrue();
+  llvm_ir::IrArray::Index operand_index(b_.getInt64Ty(), source_index.size());
+  llvm::Value* in_bounds_condition = b_.getTrue();
   for (int64 i = 0; i < rank; ++i) {
-    llvm::Value* strided_index = ir_builder_.CreateNSWMul(
-        source_index[i], ir_builder_.getInt64(window.dimensions(i).stride()));
-    operand_index[i] = ir_builder_.CreateNSWSub(
-        ir_builder_.CreateNSWAdd(strided_index, window_index[i]),
-        ir_builder_.getInt64(window.dimensions(i).padding_low()));
-    llvm::Value* index_condition = ir_builder_.CreateICmpULT(
+    llvm::Value* strided_index = b_.CreateNSWMul(
+        source_index[i], b_.getInt64(window.dimensions(i).stride()));
+    operand_index[i] =
+        b_.CreateNSWSub(b_.CreateNSWAdd(strided_index, window_index[i]),
+                        b_.getInt64(window.dimensions(i).padding_low()));
+    llvm::Value* index_condition = b_.CreateICmpULT(
         operand_index[i],
-        ir_builder_.getInt64(ShapeUtil::GetDimension(operand->shape(), i)));
-    in_bounds_condition =
-        ir_builder_.CreateAnd(in_bounds_condition, index_condition);
+        b_.getInt64(ShapeUtil::GetDimension(operand->shape(), i)));
+    in_bounds_condition = b_.CreateAnd(in_bounds_condition, index_condition);
   }
   CHECK(in_bounds_condition != nullptr);
 
   // Only need to do something if the operand index is within the bounds. First
   // check if the initialized_flag is set.
   llvm_ir::LlvmIfData if_in_bounds =
-      llvm_ir::EmitIfThenElse(in_bounds_condition, "in-bounds", &ir_builder_);
-  SetToFirstInsertPoint(if_in_bounds.true_block, &ir_builder_);
-  llvm_ir::LlvmIfData if_initialized =
-      llvm_ir::EmitIfThenElse(ir_builder_.CreateLoad(initialized_flag_address),
-                              "initialized", &ir_builder_);
+      llvm_ir::EmitIfThenElse(in_bounds_condition, "in-bounds", &b_);
+  SetToFirstInsertPoint(if_in_bounds.true_block, &b_);
+  llvm_ir::LlvmIfData if_initialized = llvm_ir::EmitIfThenElse(
+      b_.CreateLoad(initialized_flag_address), "initialized", &b_);
 
   // If the initialized_flag is false, initialize the selected value and index
   // with the currently visiting operand.
-  SetToFirstInsertPoint(if_initialized.false_block, &ir_builder_);
+  SetToFirstInsertPoint(if_initialized.false_block, &b_);
   const auto save_operand_index =
       [&](const llvm_ir::IrArray::Index& operand_index) {
         for (int64 i = 0; i < rank; ++i) {
           llvm::Value* selected_index_address_slot =
-              ir_builder_.CreateInBoundsGEP(selected_index_address,
-                                            {ir_builder_.getInt32(i)});
-          ir_builder_.CreateStore(operand_index[i],
-                                  selected_index_address_slot);
+              b_.CreateInBoundsGEP(selected_index_address, {b_.getInt32(i)});
+          b_.CreateStore(operand_index[i], selected_index_address_slot);
         }
       };
   llvm_ir::IrArray operand_array(GetIrArrayFor(operand));
   llvm::Value* operand_data =
-      operand_array.EmitReadArrayElement(operand_index, &ir_builder_);
-  ir_builder_.CreateStore(operand_data, selected_value_address);
+      operand_array.EmitReadArrayElement(operand_index, &b_);
+  b_.CreateStore(operand_data, selected_value_address);
   save_operand_index(operand_index);
-  ir_builder_.CreateStore(ir_builder_.getInt1(true), initialized_flag_address);
+  b_.CreateStore(b_.getInt1(true), initialized_flag_address);
 
   // If the initialized_flag is true, call the `select` function to potentially
   // update the selected value and index with the currently visiting operand.
-  SetToFirstInsertPoint(if_initialized.true_block, &ir_builder_);
+  SetToFirstInsertPoint(if_initialized.true_block, &b_);
   const Shape output_shape = ShapeUtil::MakeShape(PRED, {});
   llvm::Value* operand_address =
-      operand_array.EmitArrayElementAddress(operand_index, &ir_builder_);
+      operand_array.EmitArrayElementAddress(operand_index, &b_);
   llvm::Value* result = EmitElementFunctionCall(
       select_function, output_shape, {selected_value_address, operand_address},
       "select_function");
 
   // If the 'select' function returns false, update the selected value and the
   // index to the currently visiting operand.
-  llvm::Value* cond = ir_builder_.CreateICmpNE(
+  llvm::Value* cond = b_.CreateICmpNE(
       result,
       llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, module_), 0),
       "boolean_predicate");
   llvm_ir::LlvmIfData if_select_lhs =
-      llvm_ir::EmitIfThenElse(cond, "if-select-lhs", &ir_builder_);
-  SetToFirstInsertPoint(if_select_lhs.false_block, &ir_builder_);
-  ir_builder_.CreateStore(ir_builder_.CreateLoad(operand_address),
-                          selected_value_address);
+      llvm_ir::EmitIfThenElse(cond, "if-select-lhs", &b_);
+  SetToFirstInsertPoint(if_select_lhs.false_block, &b_);
+  b_.CreateStore(b_.CreateLoad(operand_address), selected_value_address);
   save_operand_index(operand_index);
 
   // After iterating over the window elements, scatter the source element to
   // the selected index of the output. The value we store at the output
   // location is computed by calling the `scatter` function with the source
   // value and the current output value.
-  SetToFirstInsertPoint(window_loops.GetOuterLoopExitBasicBlock(),
-                        &ir_builder_);
+  SetToFirstInsertPoint(window_loops.GetOuterLoopExitBasicBlock(), &b_);
   llvm_ir::IrArray::Index selected_index(source_index.GetType());
   for (int64 i = 0; i < rank; ++i) {
-    llvm::Value* selected_index_address_slot = ir_builder_.CreateInBoundsGEP(
-        selected_index_address, {ir_builder_.getInt32(i)});
-    selected_index.push_back(
-        ir_builder_.CreateLoad(selected_index_address_slot));
+    llvm::Value* selected_index_address_slot =
+        b_.CreateInBoundsGEP(selected_index_address, {b_.getInt32(i)});
+    selected_index.push_back(b_.CreateLoad(selected_index_address_slot));
   }
   llvm_ir::IrArray source_array(GetIrArrayFor(source));
   llvm::Value* source_value_address =
-      source_array.EmitArrayElementAddress(source_index, &ir_builder_);
+      source_array.EmitArrayElementAddress(source_index, &b_);
   llvm_ir::IrArray output_array(GetIrArrayFor(select_and_scatter));
   llvm::Value* output_value_address =
-      output_array.EmitArrayElementAddress(selected_index, &ir_builder_);
+      output_array.EmitArrayElementAddress(selected_index, &b_);
   llvm::Value* scatter_value = EmitElementFunctionCall(
       scatter_function, source->shape(),
       {output_value_address, source_value_address}, "scatter_function");
-  output_array.EmitWriteArrayElement(selected_index, scatter_value,
-                                     &ir_builder_);
+  output_array.EmitWriteArrayElement(selected_index, scatter_value, &b_);
 
-  SetToFirstInsertPoint(source_loops.GetOuterLoopExitBasicBlock(),
-                        &ir_builder_);
+  SetToFirstInsertPoint(source_loops.GetOuterLoopExitBasicBlock(), &b_);
   return Status::OK();
 }
 
@@ -822,7 +801,7 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
   // Dot operation is complicated so we delegate to a helper class.
   return DotOpEmitter::EmitDotOperation(
       *dot, target_array, lhs_array, rhs_array, /*addend_array=*/nullptr,
-      GetExecutableRunOptionsArgument(), &ir_builder_, hlo_module_config_,
+      GetExecutableRunOptionsArgument(), &b_, hlo_module_config_,
       target_machine_features_);
 }
 
@@ -849,12 +828,12 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForConvolution(
   llvm::Type* lhs_llvm_type =
       llvm_ir::PrimitiveTypeToIrType(lhs_element_type, module_);
   llvm::Value* sum_address = llvm_ir::EmitAllocaAtFunctionEntry(
-      lhs_llvm_type, "convolution_sum_address", &ir_builder_,
+      lhs_llvm_type, "convolution_sum_address", &b_,
       MinimumAlignmentForPrimitiveType(lhs_element_type));
   llvm::Value* constant_zero = llvm::Constant::getNullValue(lhs_llvm_type);
-  ir_builder_.CreateStore(constant_zero, sum_address);
+  b_.CreateStore(constant_zero, sum_address);
 
-  llvm_ir::ForLoopNest loops(IrName(convolution, "inner"), &ir_builder_);
+  llvm_ir::ForLoopNest loops(IrName(convolution, "inner"), &b_);
   std::vector kernel_spatial(num_spatial_dims);
   for (int i = 0; i < num_spatial_dims; ++i) {
     kernel_spatial[i] =
@@ -870,7 +849,7 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForConvolution(
                    "iz")
           ->GetIndVarValue();
 
-  SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
+  SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &b_);
 
   // Calculate the spatial index in the input array, taking striding, dilation
   // and padding into account. An index in the padding will be out of the bounds
@@ -878,13 +857,12 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForConvolution(
   const auto calculate_input_index = [this](llvm::Value* output_index,
                                             llvm::Value* kernel_index,
                                             const WindowDimension& window_dim) {
-    llvm::Value* strided_index = ir_builder_.CreateNSWMul(
-        output_index, ir_builder_.getInt64(window_dim.stride()));
-    llvm::Value* dilated_kernel_index = ir_builder_.CreateNSWMul(
-        kernel_index, ir_builder_.getInt64(window_dim.window_dilation()));
-    return ir_builder_.CreateNSWSub(
-        ir_builder_.CreateNSWAdd(strided_index, dilated_kernel_index),
-        ir_builder_.getInt64(window_dim.padding_low()));
+    llvm::Value* strided_index =
+        b_.CreateNSWMul(output_index, b_.getInt64(window_dim.stride()));
+    llvm::Value* dilated_kernel_index = b_.CreateNSWMul(
+        kernel_index, b_.getInt64(window_dim.window_dilation()));
+    return b_.CreateNSWSub(b_.CreateNSWAdd(strided_index, dilated_kernel_index),
+                           b_.getInt64(window_dim.padding_low()));
   };
   std::vector input_spatial(num_spatial_dims);
   for (int i = 0; i < num_spatial_dims; ++i) {
@@ -901,30 +879,27 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForConvolution(
   // Also need to check that the input coordinates are not in one of the
   // holes created by base dilation.
   const auto not_in_hole = [&](llvm::Value* input_index, int64 base_dilation) {
-    llvm::Value* remainder = ir_builder_.CreateSRem(
-        input_index, ir_builder_.getInt64(base_dilation));
-    return ir_builder_.CreateICmpEQ(remainder, ir_builder_.getInt64(0));
+    llvm::Value* remainder =
+        b_.CreateSRem(input_index, b_.getInt64(base_dilation));
+    return b_.CreateICmpEQ(remainder, b_.getInt64(0));
   };
 
-  llvm::Value* in_bounds_condition = ir_builder_.getInt1(true);
+  llvm::Value* in_bounds_condition = b_.getInt1(true);
   for (int i = 0; i < num_spatial_dims; ++i) {
-    llvm::ConstantInt* input_bound =
-        ir_builder_.getInt64(window_util::DilatedBound(
-            lhs->shape().dimensions(dnums.input_spatial_dimensions(i)),
-            window.dimensions(i).base_dilation()));
-    llvm::Value* dim_in_bound =
-        ir_builder_.CreateICmpULT(input_spatial[i], input_bound);
+    llvm::ConstantInt* input_bound = b_.getInt64(window_util::DilatedBound(
+        lhs->shape().dimensions(dnums.input_spatial_dimensions(i)),
+        window.dimensions(i).base_dilation()));
+    llvm::Value* dim_in_bound = b_.CreateICmpULT(input_spatial[i], input_bound);
     llvm::Value* dim_not_in_hole =
         not_in_hole(input_spatial[i], window.dimensions(i).base_dilation());
-    llvm::Value* dim_ok = ir_builder_.CreateAnd(dim_in_bound, dim_not_in_hole);
-    in_bounds_condition = ir_builder_.CreateAnd(in_bounds_condition, dim_ok);
+    llvm::Value* dim_ok = b_.CreateAnd(dim_in_bound, dim_not_in_hole);
+    in_bounds_condition = b_.CreateAnd(in_bounds_condition, dim_ok);
   }
 
   // Now we need to map the dilated base coordinates back to the actual
   // data indices on the lhs.
   const auto undilate = [&](llvm::Value* input_index, int64 base_dilation) {
-    return ir_builder_.CreateSDiv(input_index,
-                                  ir_builder_.getInt64(base_dilation));
+    return b_.CreateSDiv(input_index, b_.getInt64(base_dilation));
   };
   for (int i = 0; i < num_spatial_dims; ++i) {
     input_spatial[i] =
@@ -932,12 +907,12 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForConvolution(
   }
 
   llvm_ir::LlvmIfData if_data =
-      llvm_ir::EmitIfThenElse(in_bounds_condition, "in-bounds", &ir_builder_);
-  SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
+      llvm_ir::EmitIfThenElse(in_bounds_condition, "in-bounds", &b_);
+  SetToFirstInsertPoint(if_data.true_block, &b_);
 
   // We are not in the padding, so carry out the computation.
   int num_dims = num_spatial_dims + 2;
-  llvm_ir::IrArray::Index input_index(ir_builder_.getInt64Ty(), num_dims);
+  llvm_ir::IrArray::Index input_index(b_.getInt64Ty(), num_dims);
   for (int i = 0; i < num_spatial_dims; ++i) {
     input_index[dnums.input_spatial_dimensions(i)] = input_spatial[i];
   }
@@ -945,13 +920,12 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForConvolution(
   input_index[dnums.input_batch_dimension()] = batch;
 
   llvm_ir::IrArray kernel_array(GetIrArrayFor(rhs));
-  llvm_ir::IrArray::Index kernel_index(ir_builder_.getInt64Ty(), num_dims);
+  llvm_ir::IrArray::Index kernel_index(b_.getInt64Ty(), num_dims);
   for (int i = 0; i < num_spatial_dims; ++i) {
     kernel_index[dnums.kernel_spatial_dimensions(i)] =
         window.dimensions(i).window_reversal()
-            ? ir_builder_.CreateNSWSub(
-                  ir_builder_.getInt64(window.dimensions(i).size() - 1),
-                  kernel_spatial[i])
+            ? b_.CreateNSWSub(b_.getInt64(window.dimensions(i).size() - 1),
+                              kernel_spatial[i])
             : kernel_spatial[i];
   }
 
@@ -959,15 +933,14 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForConvolution(
   kernel_index[dnums.kernel_output_feature_dimension()] = output_feature;
 
   llvm_ir::IrArray input_array(GetIrArrayFor(lhs));
-  llvm::Value* product = ir_builder_.CreateFMul(
-      input_array.EmitReadArrayElement(input_index, &ir_builder_),
-      kernel_array.EmitReadArrayElement(kernel_index, &ir_builder_));
-  llvm::Value* sum =
-      ir_builder_.CreateFAdd(ir_builder_.CreateLoad(sum_address), product);
-  ir_builder_.CreateStore(sum, sum_address);
+  llvm::Value* product =
+      b_.CreateFMul(input_array.EmitReadArrayElement(input_index, &b_),
+                    kernel_array.EmitReadArrayElement(kernel_index, &b_));
+  llvm::Value* sum = b_.CreateFAdd(b_.CreateLoad(sum_address), product);
+  b_.CreateStore(sum, sum_address);
 
-  SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
-  return ir_builder_.CreateLoad(sum_address);
+  SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &b_);
+  return b_.CreateLoad(sum_address);
 }
 
 Status IrEmitter::HandleConvolution(HloInstruction* convolution) {
@@ -1056,12 +1029,12 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution) {
 
       PrimitiveType primitive_type = lhs->shape().element_type();
       llvm::Type* ir_ptr_type = primitive_type == F16
-                                    ? ir_builder_.getHalfTy()->getPointerTo()
-                                    : ir_builder_.getFloatTy()->getPointerTo();
-      llvm::Type* int64_type = ir_builder_.getInt64Ty();
-      llvm::Type* int8_ptr_type = ir_builder_.getInt8Ty()->getPointerTo();
+                                    ? b_.getHalfTy()->getPointerTo()
+                                    : b_.getFloatTy()->getPointerTo();
+      llvm::Type* int64_type = b_.getInt64Ty();
+      llvm::Type* int8_ptr_type = b_.getInt8Ty()->getPointerTo();
       llvm::FunctionType* conv_type = llvm::FunctionType::get(
-          ir_builder_.getVoidTy(),
+          b_.getVoidTy(),
           {int8_ptr_type, ir_ptr_type, ir_ptr_type, ir_ptr_type, int64_type,
            int64_type,    int64_type,  int64_type,  int64_type,  int64_type,
            int64_type,    int64_type,  int64_type,  int64_type,  int64_type,
@@ -1093,34 +1066,34 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution) {
       conv_func->setCallingConv(llvm::CallingConv::C);
       conv_func->setDoesNotThrow();
       conv_func->setOnlyAccessesArgMemory();
-      ir_builder_.CreateCall(
-          conv_func, {
-                         GetExecutableRunOptionsArgument(),
-                         ir_builder_.CreateBitCast(
-                             GetEmittedValueFor(convolution), ir_ptr_type),
-                         ir_builder_.CreateBitCast(lhs_address, ir_ptr_type),
-                         ir_builder_.CreateBitCast(rhs_address, ir_ptr_type),
-                         ir_builder_.getInt64(input_batch),
-                         ir_builder_.getInt64(input_rows),
-                         ir_builder_.getInt64(input_cols),
-                         ir_builder_.getInt64(input_channels),
-                         ir_builder_.getInt64(kernel_rows),
-                         ir_builder_.getInt64(kernel_cols),
-                         ir_builder_.getInt64(kernel_channels),
-                         ir_builder_.getInt64(kernel_filters),
-                         ir_builder_.getInt64(output_rows),
-                         ir_builder_.getInt64(output_cols),
-                         ir_builder_.getInt64(row_stride),
-                         ir_builder_.getInt64(col_stride),
-                         ir_builder_.getInt64(padding_top),
-                         ir_builder_.getInt64(padding_bottom),
-                         ir_builder_.getInt64(padding_left),
-                         ir_builder_.getInt64(padding_right),
-                         ir_builder_.getInt64(lhs_row_dilation),
-                         ir_builder_.getInt64(lhs_col_dilation),
-                         ir_builder_.getInt64(rhs_row_dilation),
-                         ir_builder_.getInt64(rhs_col_dilation),
-                     });
+      b_.CreateCall(
+          conv_func,
+          {
+              GetExecutableRunOptionsArgument(),
+              b_.CreateBitCast(GetEmittedValueFor(convolution), ir_ptr_type),
+              b_.CreateBitCast(lhs_address, ir_ptr_type),
+              b_.CreateBitCast(rhs_address, ir_ptr_type),
+              b_.getInt64(input_batch),
+              b_.getInt64(input_rows),
+              b_.getInt64(input_cols),
+              b_.getInt64(input_channels),
+              b_.getInt64(kernel_rows),
+              b_.getInt64(kernel_cols),
+              b_.getInt64(kernel_channels),
+              b_.getInt64(kernel_filters),
+              b_.getInt64(output_rows),
+              b_.getInt64(output_cols),
+              b_.getInt64(row_stride),
+              b_.getInt64(col_stride),
+              b_.getInt64(padding_top),
+              b_.getInt64(padding_bottom),
+              b_.getInt64(padding_left),
+              b_.getInt64(padding_right),
+              b_.getInt64(lhs_row_dilation),
+              b_.getInt64(lhs_col_dilation),
+              b_.getInt64(rhs_row_dilation),
+              b_.getInt64(rhs_col_dilation),
+          });
 
       return Status::OK();
     }
@@ -1159,11 +1132,11 @@ Status IrEmitter::HandleFft(HloInstruction* fft) {
   }
 
   // Args have been computed, make the call.
-  llvm::Type* int8_ptr_type = ir_builder_.getInt8Ty()->getPointerTo();
-  llvm::Type* int32_type = ir_builder_.getInt32Ty();
-  llvm::Type* int64_type = ir_builder_.getInt64Ty();
+  llvm::Type* int8_ptr_type = b_.getInt8Ty()->getPointerTo();
+  llvm::Type* int32_type = b_.getInt32Ty();
+  llvm::Type* int64_type = b_.getInt64Ty();
   llvm::FunctionType* fft_type = llvm::FunctionType::get(
-      ir_builder_.getVoidTy(),
+      b_.getVoidTy(),
       {int8_ptr_type, int8_ptr_type, int8_ptr_type, int32_type, int32_type,
        int64_type, int64_type, int64_type, int64_type},
       /*isVarArg=*/false);
@@ -1180,16 +1153,15 @@ Status IrEmitter::HandleFft(HloInstruction* fft) {
   fft_func->setDoesNotThrow();
   fft_func->setOnlyAccessesInaccessibleMemOrArgMem();
   const int fft_rank = fft_length.size();
-  ir_builder_.CreateCall(
+  b_.CreateCall(
       fft_func,
       {GetExecutableRunOptionsArgument(),
-       ir_builder_.CreateBitCast(GetEmittedValueFor(fft), int8_ptr_type),
-       ir_builder_.CreateBitCast(operand_address, int8_ptr_type),
-       ir_builder_.getInt32(fft->fft_type()), ir_builder_.getInt32(fft_rank),
-       ir_builder_.getInt64(input_batch),
-       ir_builder_.getInt64(fft_rank > 0 ? fft_length[0] : 0),
-       ir_builder_.getInt64(fft_rank > 1 ? fft_length[1] : 0),
-       ir_builder_.getInt64(fft_rank > 2 ? fft_length[2] : 0)});
+       b_.CreateBitCast(GetEmittedValueFor(fft), int8_ptr_type),
+       b_.CreateBitCast(operand_address, int8_ptr_type),
+       b_.getInt32(fft->fft_type()), b_.getInt32(fft_rank),
+       b_.getInt64(input_batch), b_.getInt64(fft_rank > 0 ? fft_length[0] : 0),
+       b_.getInt64(fft_rank > 1 ? fft_length[1] : 0),
+       b_.getInt64(fft_rank > 2 ? fft_length[2] : 0)});
 
   return Status::OK();
 }
@@ -1228,11 +1200,10 @@ Status IrEmitter::HandleCrossReplicaSum(HloInstruction* crs) {
     operand_ptrs.push_back(EmitTempBufferPointer(out_slice, operand_shape));
 
     // TODO(b/63762267): Be more aggressive about specifying alignment.
-    ir_builder_.CreateMemCpy(operand_ptrs.back(), /*DstAlign=*/1, in_ptr,
-                             /*SrcAlign=*/1,
-                             ShapeUtil::ByteSizeOf(operand_shape));
+    b_.CreateMemCpy(operand_ptrs.back(), /*DstAlign=*/1, in_ptr,
+                    /*SrcAlign=*/1, ShapeUtil::ByteSizeOf(operand_shape));
   }
-  llvm_ir::EmitTuple(GetIrArrayFor(crs), operand_ptrs, &ir_builder_, module_);
+  llvm_ir::EmitTuple(GetIrArrayFor(crs), operand_ptrs, &b_, module_);
   return Status::OK();
 }
 
@@ -1278,9 +1249,8 @@ Status IrEmitter::HandleParameter(HloInstruction* parameter) {
   // example, float for an XLA F32 element type).
   llvm::Value* params = compute_function_->parameters_arg();
   llvm::Value* param_address_offset =
-      llvm_ir::EmitBufferIndexingGEP(params, param_number, &ir_builder_);
-  llvm::LoadInst* param_address_untyped =
-      ir_builder_.CreateLoad(param_address_offset);
+      llvm_ir::EmitBufferIndexingGEP(params, param_number, &b_);
+  llvm::LoadInst* param_address_untyped = b_.CreateLoad(param_address_offset);
   param_address_untyped->setName(AsStringRef(IrName(parameter, "untyped")));
   if (is_top_level_computation_ &&
       hlo_module_config_.debug_options()
@@ -1295,7 +1265,7 @@ Status IrEmitter::HandleParameter(HloInstruction* parameter) {
         llvm::MDNode::get(param_address_untyped->getContext(), /*MDs=*/{}));
   }
 
-  llvm::Value* param_address_typed = ir_builder_.CreateBitCast(
+  llvm::Value* param_address_typed = b_.CreateBitCast(
       param_address_untyped, IrShapeType(param_shape)->getPointerTo());
   emitted_value_[parameter] = param_address_typed;
 
@@ -1403,62 +1373,61 @@ IrEmitter::ReductionGenerator IrEmitter::MatchReductionGenerator(
       return nullptr;
 
     case HloOpcode::kAdd:
-      return [root_is_integral](llvm::IRBuilder<>* ir_builder, llvm::Value* lhs,
+      return [root_is_integral](llvm::IRBuilder<>* b, llvm::Value* lhs,
                                 llvm::Value* rhs) {
-        return root_is_integral ? ir_builder->CreateAdd(lhs, rhs)
-                                : ir_builder->CreateFAdd(lhs, rhs);
+        return root_is_integral ? b->CreateAdd(lhs, rhs)
+                                : b->CreateFAdd(lhs, rhs);
       };
 
     case HloOpcode::kMultiply:
-      return [root_is_integral](llvm::IRBuilder<>* ir_builder, llvm::Value* lhs,
+      return [root_is_integral](llvm::IRBuilder<>* b, llvm::Value* lhs,
                                 llvm::Value* rhs) {
-        return root_is_integral ? ir_builder->CreateMul(lhs, rhs)
-                                : ir_builder->CreateFMul(lhs, rhs);
+        return root_is_integral ? b->CreateMul(lhs, rhs)
+                                : b->CreateFMul(lhs, rhs);
       };
 
     case HloOpcode::kAnd:
-      return [](llvm::IRBuilder<>* ir_builder, llvm::Value* lhs,
-                llvm::Value* rhs) { return ir_builder->CreateAnd(lhs, rhs); };
+      return [](llvm::IRBuilder<>* b, llvm::Value* lhs, llvm::Value* rhs) {
+        return b->CreateAnd(lhs, rhs);
+      };
 
     case HloOpcode::kOr:
-      return [](llvm::IRBuilder<>* ir_builder, llvm::Value* lhs,
-                llvm::Value* rhs) { return ir_builder->CreateOr(lhs, rhs); };
+      return [](llvm::IRBuilder<>* b, llvm::Value* lhs, llvm::Value* rhs) {
+        return b->CreateOr(lhs, rhs);
+      };
 
     case HloOpcode::kXor:
-      return [](llvm::IRBuilder<>* ir_builder, llvm::Value* lhs,
-                llvm::Value* rhs) { return ir_builder->CreateXor(lhs, rhs); };
+      return [](llvm::IRBuilder<>* b, llvm::Value* lhs, llvm::Value* rhs) {
+        return b->CreateXor(lhs, rhs);
+      };
 
     case HloOpcode::kMaximum:
       return [root_is_floating_point, root_is_signed](
-                 llvm::IRBuilder<>* ir_builder, llvm::Value* lhs,
-                 llvm::Value* rhs) {
+                 llvm::IRBuilder<>* b, llvm::Value* lhs, llvm::Value* rhs) {
         if (root_is_floating_point) {
           return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::maxnum,
-                                              {lhs, rhs}, {lhs->getType()},
-                                              ir_builder);
+                                              {lhs, rhs}, {lhs->getType()}, b);
         }
 
-        return ir_builder->CreateSelect(
-            ir_builder->CreateICmp(root_is_signed ? llvm::ICmpInst::ICMP_SGE
-                                                  : llvm::ICmpInst::ICMP_UGE,
-                                   lhs, rhs),
+        return b->CreateSelect(
+            b->CreateICmp(root_is_signed ? llvm::ICmpInst::ICMP_SGE
+                                         : llvm::ICmpInst::ICMP_UGE,
+                          lhs, rhs),
             lhs, rhs);
       };
 
     case HloOpcode::kMinimum:
       return [root_is_floating_point, root_is_signed](
-                 llvm::IRBuilder<>* ir_builder, llvm::Value* lhs,
-                 llvm::Value* rhs) {
+                 llvm::IRBuilder<>* b, llvm::Value* lhs, llvm::Value* rhs) {
         if (root_is_floating_point) {
           return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::minnum,
-                                              {lhs, rhs}, {lhs->getType()},
-                                              ir_builder);
+                                              {lhs, rhs}, {lhs->getType()}, b);
         }
 
-        return ir_builder->CreateSelect(
-            ir_builder->CreateICmp(root_is_signed ? llvm::ICmpInst::ICMP_SLE
-                                                  : llvm::ICmpInst::ICMP_ULE,
-                                   lhs, rhs),
+        return b->CreateSelect(
+            b->CreateICmp(root_is_signed ? llvm::ICmpInst::ICMP_SLE
+                                         : llvm::ICmpInst::ICMP_ULE,
+                          lhs, rhs),
             lhs, rhs);
       };
   }
@@ -1527,34 +1496,31 @@ IrEmitter::EmitInnerLoopForVectorizedReduction(
   accumulator.reserve(accumulator_type.size());
   for (auto accumulator_shard_type : accumulator_type) {
     accumulator.push_back(llvm_ir::EmitAllocaAtFunctionEntry(
-        accumulator_shard_type, "accumulator", &ir_builder_, 0));
+        accumulator_shard_type, "accumulator", &b_, 0));
   }
 
-  llvm::Value* init_value_ssa =
-      ir_builder_.CreateLoad(GetEmittedValueFor(init_value));
+  llvm::Value* init_value_ssa = b_.CreateLoad(GetEmittedValueFor(init_value));
 
   for (llvm::Value* accumulator_shard : accumulator) {
     llvm::Value* initial_value;
     auto shard_type = accumulator_shard->getType()->getPointerElementType();
     if (auto vector_type = llvm::dyn_cast(shard_type)) {
-      initial_value = ir_builder_.CreateVectorSplat(
-          vector_type->getNumElements(), init_value_ssa);
+      initial_value =
+          b_.CreateVectorSplat(vector_type->getNumElements(), init_value_ssa);
     } else {
       initial_value = init_value_ssa;
     }
 
-    ir_builder_.CreateAlignedStore(initial_value, accumulator_shard,
-                                   element_alignment);
+    b_.CreateAlignedStore(initial_value, accumulator_shard, element_alignment);
   }
 
   llvm_ir::ForLoopNest reduction_loop_nest(IrName(arg, "vectorized_inner"),
-                                           &ir_builder_);
+                                           &b_);
   llvm_ir::IrArray::Index reduced_dims_index =
       reduction_loop_nest.AddLoopsForShapeOnDimensions(arg->shape(), dimensions,
                                                        "reduction_dim");
 
-  SetToFirstInsertPoint(reduction_loop_nest.GetInnerLoopBodyBasicBlock(),
-                        &ir_builder_);
+  SetToFirstInsertPoint(reduction_loop_nest.GetInnerLoopBodyBasicBlock(), &b_);
 
   llvm_ir::IrArray arg_array(GetIrArrayFor(arg));
   llvm_ir::IrArray::Index input_index = reduced_dims_index;
@@ -1567,38 +1533,34 @@ IrEmitter::EmitInnerLoopForVectorizedReduction(
   }
   CHECK(output_index.end() == it);
 
-  llvm::Value* input_address = ir_builder_.CreateBitCast(
-      arg_array.EmitArrayElementAddress(input_index, &ir_builder_),
-      ir_builder_.getInt8PtrTy());
+  llvm::Value* input_address = b_.CreateBitCast(
+      arg_array.EmitArrayElementAddress(input_index, &b_), b_.getInt8PtrTy());
 
   for (int i = 0; i < accumulator.size(); i++) {
     auto input_address_typed =
-        ir_builder_.CreateBitCast(input_address, accumulator[i]->getType());
+        b_.CreateBitCast(input_address, accumulator[i]->getType());
     auto current_accumulator_value =
-        ir_builder_.CreateAlignedLoad(accumulator[i], element_alignment);
-    auto addend =
-        ir_builder_.CreateAlignedLoad(input_address_typed, element_alignment);
+        b_.CreateAlignedLoad(accumulator[i], element_alignment);
+    auto addend = b_.CreateAlignedLoad(input_address_typed, element_alignment);
     arg_array.AnnotateLoadStoreInstructionWithMetadata(addend);
 
     auto reduced_result =
-        reduction_generator(&ir_builder_, current_accumulator_value, addend);
-    ir_builder_.CreateAlignedStore(reduced_result, accumulator[i],
-                                   element_alignment);
+        reduction_generator(&b_, current_accumulator_value, addend);
+    b_.CreateAlignedStore(reduced_result, accumulator[i], element_alignment);
 
     if (i != (accumulator.size() - 1)) {
-      input_address = ir_builder_.CreateConstInBoundsGEP1_32(
-          reduced_result->getType(), input_address_typed, 1);
+      input_address = b_.CreateConstInBoundsGEP1_32(reduced_result->getType(),
+                                                    input_address_typed, 1);
     }
   }
 
-  SetToFirstInsertPoint(reduction_loop_nest.GetOuterLoopExitBasicBlock(),
-                        &ir_builder_);
+  SetToFirstInsertPoint(reduction_loop_nest.GetOuterLoopExitBasicBlock(), &b_);
 
   ShardedVector result_ssa;
   result_ssa.reserve(accumulator.size());
   for (auto accumulator_shard : accumulator) {
     result_ssa.push_back(
-        ir_builder_.CreateAlignedLoad(accumulator_shard, element_alignment));
+        b_.CreateAlignedLoad(accumulator_shard, element_alignment));
   }
   return result_ssa;
 }
@@ -1607,17 +1569,17 @@ void IrEmitter::EmitShardedVectorStore(
     llvm::Value* store_address, const std::vector& value_to_store,
     const int alignment, const llvm_ir::IrArray& containing_array) {
   for (int i = 0; i < value_to_store.size(); i++) {
-    auto store_address_typed = ir_builder_.CreateBitCast(
+    auto store_address_typed = b_.CreateBitCast(
         store_address,
         llvm::PointerType::getUnqual(value_to_store[i]->getType()));
 
-    auto store_instruction = ir_builder_.CreateAlignedStore(
+    auto store_instruction = b_.CreateAlignedStore(
         value_to_store[i], store_address_typed, alignment);
     containing_array.AnnotateLoadStoreInstructionWithMetadata(
         store_instruction);
 
     if (i != (value_to_store.size() - 1)) {
-      store_address = ir_builder_.CreateConstInBoundsGEP1_32(
+      store_address = b_.CreateConstInBoundsGEP1_32(
           value_to_store[i]->getType(), store_address_typed, 1);
     }
   }
@@ -1683,8 +1645,8 @@ StatusOr IrEmitter::EmitVectorizedReduce(
   //    }
   //  }
 
-  llvm_ir::ForLoopNest loop_nest(IrName(reduce), &ir_builder_);
-  llvm_ir::IrArray::Index array_index(ir_builder_.getInt64Ty(),
+  llvm_ir::ForLoopNest loop_nest(IrName(reduce), &b_);
+  llvm_ir::IrArray::Index array_index(b_.getInt64Ty(),
                                       reduce->shape().dimensions_size());
   for (int i = LayoutUtil::MinorToMajor(reduce->shape()).size() - 1; i > 0;
        --i) {
@@ -1703,7 +1665,7 @@ StatusOr IrEmitter::EmitVectorizedReduce(
 
   if (llvm::BasicBlock* innermost_body_bb =
           loop_nest.GetInnerLoopBodyBasicBlock()) {
-    SetToFirstInsertPoint(innermost_body_bb, &ir_builder_);
+    SetToFirstInsertPoint(innermost_body_bb, &b_);
   }
 
   auto outermost_loop_exit_block = loop_nest.GetOuterLoopExitBasicBlock();
@@ -1717,7 +1679,7 @@ StatusOr IrEmitter::EmitVectorizedReduce(
         tensorflow::strings::Printf("dim.%lld", innermost_dimension));
     array_index[innermost_dimension] = loop->GetIndVarValue();
 
-    SetToFirstInsertPoint(loop->GetBodyBasicBlock(), &ir_builder_);
+    SetToFirstInsertPoint(loop->GetBodyBasicBlock(), &b_);
 
     ShardedVectorType vector_type = CreateShardedVectorType(
         reduce->shape().element_type(), vectorization_factor);
@@ -1728,16 +1690,16 @@ StatusOr IrEmitter::EmitVectorizedReduce(
 
     llvm_ir::IrArray target_array = GetIrArrayFor(reduce);
     llvm::Value* output_address =
-        target_array.EmitArrayElementAddress(array_index, &ir_builder_);
+        target_array.EmitArrayElementAddress(array_index, &b_);
     EmitShardedVectorStore(output_address, accumulator, element_alignment,
                            target_array);
 
     if (auto exit_terminator = loop->GetExitBasicBlock()->getTerminator()) {
       CHECK_GT(LayoutUtil::MinorToMajor(reduce->shape()).size(), 1);
-      ir_builder_.SetInsertPoint(exit_terminator);
+      b_.SetInsertPoint(exit_terminator);
     } else {
       CHECK_EQ(LayoutUtil::MinorToMajor(reduce->shape()).size(), 1);
-      ir_builder_.SetInsertPoint(loop->GetExitBasicBlock());
+      b_.SetInsertPoint(loop->GetExitBasicBlock());
     }
   }
 
@@ -1747,8 +1709,8 @@ StatusOr IrEmitter::EmitVectorizedReduce(
   if (innermost_dimension_size % vectorization_factor) {
     // TODO(b/63775531): Consider using a scalar loop here to save on code size.
     array_index[innermost_dimension] =
-        ir_builder_.getInt64(innermost_dimension_size -
-                             (innermost_dimension_size % vectorization_factor));
+        b_.getInt64(innermost_dimension_size -
+                    (innermost_dimension_size % vectorization_factor));
 
     ShardedVectorType vector_type = CreateShardedVectorType(
         reduce->shape().element_type(),
@@ -1760,13 +1722,13 @@ StatusOr IrEmitter::EmitVectorizedReduce(
 
     llvm_ir::IrArray target_array = GetIrArrayFor(reduce);
     llvm::Value* output_address =
-        target_array.EmitArrayElementAddress(array_index, &ir_builder_);
+        target_array.EmitArrayElementAddress(array_index, &b_);
     EmitShardedVectorStore(output_address, accumulator, element_alignment,
                            target_array);
   }
 
   if (outermost_loop_exit_block) {
-    ir_builder_.SetInsertPoint(outermost_loop_exit_block);
+    b_.SetInsertPoint(outermost_loop_exit_block);
   }
 
   return true;
@@ -1785,22 +1747,22 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForReduce(
   PrimitiveType accumulator_type = reduce->shape().element_type();
   llvm::AllocaInst* accumulator_addr = llvm_ir::EmitAllocaAtFunctionEntry(
       llvm_ir::PrimitiveTypeToIrType(accumulator_type, module_), "accumulator",
-      &ir_builder_, MinimumAlignmentForPrimitiveType(accumulator_type));
+      &b_, MinimumAlignmentForPrimitiveType(accumulator_type));
   llvm::Value* init_value_addr = GetEmittedValueFor(init_value);
-  llvm::Value* load_init_value = ir_builder_.CreateLoad(init_value_addr);
-  ir_builder_.CreateStore(load_init_value, accumulator_addr);
+  llvm::Value* load_init_value = b_.CreateLoad(init_value_addr);
+  b_.CreateStore(load_init_value, accumulator_addr);
 
   // The enclosing loops go over all the target elements. Now we have to compute
   // the actual target element. For this, we build a new loop nest to iterate
   // over all the reduction dimensions in the argument.
   // AddLoopsForShapeOnDimensions will return an Index where induction Value*s
   // are placed for each dimension in dimensions, and all the rest are nullptrs.
-  llvm_ir::ForLoopNest loops(IrName(reduce, "inner"), &ir_builder_);
+  llvm_ir::ForLoopNest loops(IrName(reduce, "inner"), &b_);
   const llvm_ir::IrArray::Index reduced_dims_index =
       loops.AddLoopsForShapeOnDimensions(arg->shape(), dimensions,
                                          "reduction_dim");
 
-  SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
+  SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &b_);
 
   // Build a full index for the input argument, using reduced_dims_index as the
   // base. In reduced_dims_index only the reduction dimensions are filled in. We
@@ -1820,14 +1782,14 @@ StatusOr IrEmitter::EmitTargetElementLoopBodyForReduce(
 
   // Apply the reduction function to the loaded value.
   llvm::Value* input_address =
-      arg_array.EmitArrayElementAddress(input_index, &ir_builder_);
+      arg_array.EmitArrayElementAddress(input_index, &b_);
   llvm::Value* result = EmitElementFunctionCall(
       reducer_function, reduce->shape(), {accumulator_addr, input_address},
       "reduce_function");
-  ir_builder_.CreateStore(result, accumulator_addr);
+  b_.CreateStore(result, accumulator_addr);
 
-  SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
-  return ir_builder_.CreateLoad(accumulator_addr);
+  SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &b_);
+  return b_.CreateLoad(accumulator_addr);
 }
 
 Status IrEmitter::HandleReduce(HloInstruction* reduce) {
@@ -1957,7 +1919,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice) {
   llvm_ir::IrArray target_array = GetIrArrayFor(slice);
 
   const int64 num_outer_loops = outer_dims.size();
-  llvm_ir::ForLoopNest loops(IrName(slice), &ir_builder_);
+  llvm_ir::ForLoopNest loops(IrName(slice), &b_);
   llvm_ir::IrArray::Index target_index =
       loops.AddLoopsForShapeOnDimensions(slice->shape(), outer_dims, "slice");
 
@@ -1966,21 +1928,21 @@ Status IrEmitter::HandleSlice(HloInstruction* slice) {
   // for the rest of the dimensions the copy writes to the full dimension.
   std::replace(target_index.begin(), target_index.end(),
                static_cast(nullptr),
-               static_cast(ir_builder_.getInt64(0)));
+               static_cast(b_.getInt64(0)));
 
   if (num_outer_loops > 0) {
-    SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
+    SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &b_);
   }
 
   llvm_ir::IrArray source_array = GetIrArrayFor(operand);
   const llvm_ir::IrArray::Index source_index = target_index.SourceIndexOfSlice(
       /*shape=*/slice->shape(), /*starts=*/slice->slice_starts(),
-      /*strides=*/slice->slice_strides(), /*builder=*/&ir_builder_);
+      /*strides=*/slice->slice_strides(), /*builder=*/&b_);
 
-  llvm::Value* memcpy_dest = target_array.EmitArrayElementAddress(
-      target_index, &ir_builder_, "slice.dest");
-  llvm::Value* memcpy_source = source_array.EmitArrayElementAddress(
-      source_index, &ir_builder_, "slice.source");
+  llvm::Value* memcpy_dest =
+      target_array.EmitArrayElementAddress(target_index, &b_, "slice.dest");
+  llvm::Value* memcpy_source =
+      source_array.EmitArrayElementAddress(source_index, &b_, "slice.source");
 
   const int64 memcpy_elements =
       primitive_elements_per_logical_element * memcpy_logical_elements;
@@ -1997,7 +1959,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice) {
   }
 
   if (num_outer_loops > 0) {
-    SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
+    SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &b_);
   }
 
   return Status::OK();
@@ -2023,7 +1985,7 @@ Status IrEmitter::HandleDynamicUpdateSlice(
     auto operands = GetIrArraysForOperandsOf(dynamic_update_slice);
     return llvm_ir::EmitDynamicUpdateSliceInPlace(
         operands, GetIrArrayFor(dynamic_update_slice),
-        IrName(dynamic_update_slice, "in_place"), &ir_builder_);
+        IrName(dynamic_update_slice, "in_place"), &b_);
   }
   return DefaultAction(dynamic_update_slice);
 }
@@ -2057,43 +2019,41 @@ Status IrEmitter::HandlePad(HloInstruction* pad) {
       [this, pad](const llvm_ir::IrArray::Index& target_index) {
         const HloInstruction* padding_value = pad->operand(1);
         llvm::Value* padding_value_addr = GetEmittedValueFor(padding_value);
-        return ir_builder_.CreateLoad(padding_value_addr);
+        return b_.CreateLoad(padding_value_addr);
       }));
 
   // Create a loop to iterate over the operand elements and update the output
   // locations where the operand elements should be stored.
-  llvm_ir::ForLoopNest loops(IrName(pad, "assign"), &ir_builder_);
+  llvm_ir::ForLoopNest loops(IrName(pad, "assign"), &b_);
   const HloInstruction* operand = pad->operand(0);
   const llvm_ir::IrArray::Index operand_index =
       loops.AddLoopsForShape(operand->shape(), "operand");
 
-  SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
+  SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &b_);
 
   // Load an element from the operand.
   llvm_ir::IrArray operand_array(GetIrArrayFor(operand));
   llvm::Value* operand_data =
-      operand_array.EmitReadArrayElement(operand_index, &ir_builder_);
+      operand_array.EmitReadArrayElement(operand_index, &b_);
 
   // Compute the output index the operand element should be assigned to.
   // output_index := edge_padding_low + operand_index * (interior_padding + 1)
   const PaddingConfig& padding_config = pad->padding_config();
   llvm_ir::IrArray::Index output_index(operand_index.GetType());
   for (size_t i = 0; i < operand_index.size(); ++i) {
-    llvm::Value* offset = ir_builder_.CreateMul(
+    llvm::Value* offset = b_.CreateMul(
         operand_index[i],
-        ir_builder_.getInt64(padding_config.dimensions(i).interior_padding() +
-                             1));
-    llvm::Value* index = ir_builder_.CreateAdd(
-        offset,
-        ir_builder_.getInt64(padding_config.dimensions(i).edge_padding_low()));
+        b_.getInt64(padding_config.dimensions(i).interior_padding() + 1));
+    llvm::Value* index = b_.CreateAdd(
+        offset, b_.getInt64(padding_config.dimensions(i).edge_padding_low()));
     output_index.push_back(index);
   }
 
   // Store the operand element to the computed output location.
   llvm_ir::IrArray output_array(GetIrArrayFor(pad));
-  output_array.EmitWriteArrayElement(output_index, operand_data, &ir_builder_);
+  output_array.EmitWriteArrayElement(output_index, operand_data, &b_);
 
-  SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
+  SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &b_);
   return Status::OK();
 }
 
@@ -2115,8 +2075,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
     // Delegate to common implementation of fused in-place dynamic-update-slice.
     auto operands = GetIrArraysForOperandsOf(fusion);
     return llvm_ir::EmitFusedDynamicUpdateSliceInPlace(
-        fusion, operands, GetIrArrayFor(fusion), &elemental_emitter,
-        &ir_builder_);
+        fusion, operands, GetIrArrayFor(fusion), &elemental_emitter, &b_);
   } else if (fusion->fusion_kind() == HloInstruction::FusionKind::kLoop) {
     VLOG(3) << "HandleFusion kLoop";
     CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_);
@@ -2151,7 +2110,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
 
     TF_RETURN_IF_ERROR(DotOpEmitter::EmitDotOperation(
         *dot, target_array, lhs_array, rhs_array, &addend_array,
-        GetExecutableRunOptionsArgument(), &ir_builder_, hlo_module_config_,
+        GetExecutableRunOptionsArgument(), &b_, hlo_module_config_,
         target_machine_features_));
     return Status::OK();
   } else {
@@ -2174,7 +2133,7 @@ Status IrEmitter::HandleCall(HloInstruction* call) {
     // ParallelTaskAssignment assigned partitions, emit call to
     // ParallelForkJoin.
     std::vector call_args = GetArrayFunctionCallArguments(
-        parameter_addresses, &ir_builder_, computation->name(),
+        parameter_addresses, &b_, computation->name(),
         /*return_value_buffer=*/emitted_value_[call],
         /*exec_run_options_arg=*/GetExecutableRunOptionsArgument(),
         /*temp_buffers_arg=*/GetTempBuffersArgument(),
@@ -2182,8 +2141,8 @@ Status IrEmitter::HandleCall(HloInstruction* call) {
 
     HloInstruction* root = computation->root_instruction();
     TF_RETURN_IF_ERROR(EmitCallToParallelForkJoin(
-        call_args, root->shape(), root->outer_dimension_partitions(),
-        &ir_builder_, call_ir_function, computation->name()));
+        call_args, root->shape(), root->outer_dimension_partitions(), &b_,
+        call_ir_function, computation->name()));
   } else {
     EmitArrayFunctionCallInto(call_ir_function, parameter_addresses,
                               emitted_value_[call], computation->name());
@@ -2195,33 +2154,31 @@ Status IrEmitter::HandleCall(HloInstruction* call) {
 Status IrEmitter::HandleCustomCall(HloInstruction* custom_call) {
   gtl::ArraySlice operands(custom_call->operands());
   tensorflow::StringPiece custom_call_target(custom_call->custom_call_target());
-  llvm::Type* i8_ptr_type = ir_builder_.getInt8PtrTy();
+  llvm::Type* i8_ptr_type = b_.getInt8PtrTy();
   llvm::AllocaInst* operands_alloca =
       llvm_ir::EmitAllocaAtFunctionEntryWithCount(
-          i8_ptr_type, ir_builder_.getInt32(operands.size()),
-          "cc_operands_alloca", &ir_builder_);
+          i8_ptr_type, b_.getInt32(operands.size()), "cc_operands_alloca", &b_);
   for (size_t i = 0; i < operands.size(); ++i) {
     const HloInstruction* operand = operands[i];
     llvm::Value* operand_as_i8ptr =
-        ir_builder_.CreatePointerCast(GetEmittedValueFor(operand), i8_ptr_type);
-    llvm::Value* slot_in_operands_alloca = ir_builder_.CreateInBoundsGEP(
-        operands_alloca, {ir_builder_.getInt64(i)});
-    ir_builder_.CreateStore(operand_as_i8ptr, slot_in_operands_alloca);
+        b_.CreatePointerCast(GetEmittedValueFor(operand), i8_ptr_type);
+    llvm::Value* slot_in_operands_alloca =
+        b_.CreateInBoundsGEP(operands_alloca, {b_.getInt64(i)});
+    b_.CreateStore(operand_as_i8ptr, slot_in_operands_alloca);
   }
   auto* custom_call_ir_function =
       llvm::cast(module_->getOrInsertFunction(
           AsStringRef(custom_call_target),
           llvm::FunctionType::get(
-              /*Result=*/ir_builder_.getVoidTy(),
+              /*Result=*/b_.getVoidTy(),
               /*Params=*/{i8_ptr_type, operands_alloca->getType()},
               /*isVarArg=*/false)));
 
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(custom_call));
-  auto* output_address_arg = ir_builder_.CreatePointerCast(
-      GetEmittedValueFor(custom_call), i8_ptr_type);
+  auto* output_address_arg =
+      b_.CreatePointerCast(GetEmittedValueFor(custom_call), i8_ptr_type);
 
-  ir_builder_.CreateCall(custom_call_ir_function,
-                         {output_address_arg, operands_alloca});
+  b_.CreateCall(custom_call_ir_function, {output_address_arg, operands_alloca});
 
   return Status::OK();
 }
@@ -2286,8 +2243,8 @@ Status IrEmitter::HandleWhile(HloInstruction* xla_while) {
   llvm::BasicBlock* header_bb = llvm::BasicBlock::Create(
       module_->getContext(), AsStringRef(IrName(xla_while, "header")),
       compute_function_->function());
-  ir_builder_.CreateBr(header_bb);
-  ir_builder_.SetInsertPoint(header_bb);
+  b_.CreateBr(header_bb);
+  b_.SetInsertPoint(header_bb);
 
   // Calls the condition function to determine whether to proceed with the
   // body.  It must return a bool, so use the scalar call form.
@@ -2295,7 +2252,7 @@ Status IrEmitter::HandleWhile(HloInstruction* xla_while) {
   llvm::Value* while_condition = EmitElementFunctionCall(
       condition_ir_function, condition->root_instruction()->shape(),
       {while_result}, IrName(xla_while, "cond"));
-  llvm::Value* while_predicate = ir_builder_.CreateICmpNE(
+  llvm::Value* while_predicate = b_.CreateICmpNE(
       while_condition,
       llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, module_), 0));
 
@@ -2305,20 +2262,20 @@ Status IrEmitter::HandleWhile(HloInstruction* xla_while) {
       compute_function_->function());
   llvm::BasicBlock* exit_bb = llvm::BasicBlock::Create(
       module_->getContext(), AsStringRef(IrName(xla_while, "exit")));
-  ir_builder_.CreateCondBr(while_predicate, body_bb, exit_bb);
+  b_.CreateCondBr(while_predicate, body_bb, exit_bb);
 
   // Calls the body function from the body block.
-  ir_builder_.SetInsertPoint(body_bb);
+  b_.SetInsertPoint(body_bb);
 
   // Calls the body function.
   EmitArrayFunctionCallInto(body_ir_function, {while_result}, while_result,
                             IrName(xla_while, "body"));
   // Finishes with a branch back to the header.
-  ir_builder_.CreateBr(header_bb);
+  b_.CreateBr(header_bb);
 
   // Adds the exit block to the function and sets the insert point there.
   compute_function_->function()->getBasicBlockList().push_back(exit_bb);
-  ir_builder_.SetInsertPoint(exit_bb);
+  b_.SetInsertPoint(exit_bb);
 
   return Status::OK();
 }
@@ -2360,21 +2317,21 @@ StatusOr IrEmitter::EmitFastConcatenate(
   std::vector outer_dims(std::next(concat_dim_layout_itr),
                                 output_min2maj.end());
 
-  llvm::Type* i8_ptr_type = ir_builder_.getInt8PtrTy();
-  llvm::Type* i8_type = ir_builder_.getInt8Ty();
+  llvm::Type* i8_ptr_type = b_.getInt8PtrTy();
+  llvm::Type* i8_type = b_.getInt8Ty();
 
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(concatenate));
   llvm_ir::IrArray target_array = GetIrArrayFor(concatenate);
 
-  llvm_ir::ForLoopNest loops(IrName(concatenate), &ir_builder_);
+  llvm_ir::ForLoopNest loops(IrName(concatenate), &b_);
   llvm_ir::IrArray::Index outer_dims_index =
       loops.AddLoopsForShapeOnDimensions(output_shape, outer_dims, "concat");
   std::replace(outer_dims_index.begin(), outer_dims_index.end(),
                static_cast(nullptr),
-               static_cast(ir_builder_.getInt64(0)));
+               static_cast(b_.getInt64(0)));
 
   if (!outer_dims.empty()) {
-    SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
+    SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &b_);
   }
 
   PrimitiveType primitive_type = output_shape.element_type();
@@ -2383,10 +2340,10 @@ StatusOr IrEmitter::EmitFastConcatenate(
 
   // Contiguous subregions from each operand to the concatenate contribute to a
   // contiguous subregion in the target buffer starting at target_region_begin.
-  llvm::Value* target_region_begin = ir_builder_.CreateBitCast(
-      target_array.EmitArrayElementAddress(outer_dims_index, &ir_builder_,
-                                           "target_region"),
-      i8_ptr_type);
+  llvm::Value* target_region_begin =
+      b_.CreateBitCast(target_array.EmitArrayElementAddress(
+                           outer_dims_index, &b_, "target_region"),
+                       i8_ptr_type);
   int64 byte_offset_into_target_region = 0;
 
   int64 inner_dims_product =
@@ -2400,14 +2357,13 @@ StatusOr IrEmitter::EmitFastConcatenate(
   for (HloInstruction* operand : operands) {
     const Shape& input_shape = operand->shape();
     llvm_ir::IrArray source_array = GetIrArrayFor(operand);
-    llvm::Value* copy_source_address = ir_builder_.CreateBitCast(
-        source_array.EmitArrayElementAddress(outer_dims_index, &ir_builder_,
-                                             "src_addr"),
+    llvm::Value* copy_source_address = b_.CreateBitCast(
+        source_array.EmitArrayElementAddress(outer_dims_index, &b_, "src_addr"),
         i8_ptr_type);
 
-    llvm::Value* copy_target_address = ir_builder_.CreateGEP(
-        i8_type, target_region_begin,
-        ir_builder_.getInt64(byte_offset_into_target_region));
+    llvm::Value* copy_target_address =
+        b_.CreateGEP(i8_type, target_region_begin,
+                     b_.getInt64(byte_offset_into_target_region));
 
     EmitTransferElements(
         copy_target_address, copy_source_address,
@@ -2420,7 +2376,7 @@ StatusOr IrEmitter::EmitFastConcatenate(
   }
 
   if (!outer_dims.empty()) {
-    SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
+    SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &b_);
   }
 
   return true;
@@ -2439,16 +2395,15 @@ void IrEmitter::EmitTransferElements(llvm::Value* target, llvm::Value* source,
       llvm_ir::PrimitiveTypeToIrType(primitive_type, module_));
 
   if (element_count == 1) {
-    auto* load_instruction = ir_builder_.CreateAlignedLoad(
-        ir_builder_.CreateBitCast(source, primitive_ptr_type),
-        element_alignment);
+    auto* load_instruction = b_.CreateAlignedLoad(
+        b_.CreateBitCast(source, primitive_ptr_type), element_alignment);
     source_array.AnnotateLoadStoreInstructionWithMetadata(load_instruction);
-    auto* store_instruction = ir_builder_.CreateAlignedStore(
-        load_instruction, ir_builder_.CreateBitCast(target, primitive_ptr_type),
+    auto* store_instruction = b_.CreateAlignedStore(
+        load_instruction, b_.CreateBitCast(target, primitive_ptr_type),
         element_alignment);
     target_array.AnnotateLoadStoreInstructionWithMetadata(store_instruction);
   } else {
-    auto* memcpy_instruction = ir_builder_.CreateMemCpy(
+    auto* memcpy_instruction = b_.CreateMemCpy(
         target, /*DstAlign=*/element_alignment, source,
         /*SrcAlign=*/element_alignment, element_count * primitive_type_size);
 
@@ -2518,24 +2473,24 @@ Status IrEmitter::HandleConditional(HloInstruction* conditional) {
   //     cond_result = true_computation(true_operand)
   //   else
   //     cond_result = false_computation(false_operand)
-  llvm::LoadInst* pred_value = ir_builder_.CreateLoad(
+  llvm::LoadInst* pred_value = b_.CreateLoad(
       GetIrArrayFor(pred).GetBasePointer(), "load_predicate_value");
-  llvm::Value* pred_cond = ir_builder_.CreateICmpNE(
+  llvm::Value* pred_cond = b_.CreateICmpNE(
       pred_value,
       llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, module_), 0),
       "boolean_predicate");
   llvm_ir::LlvmIfData if_data =
-      llvm_ir::EmitIfThenElse(pred_cond, "conditional", &ir_builder_);
+      llvm_ir::EmitIfThenElse(pred_cond, "conditional", &b_);
 
-  SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
+  SetToFirstInsertPoint(if_data.true_block, &b_);
   EmitArrayFunctionCallInto(true_function, {GetEmittedValueFor(true_arg)},
                             conditional_result, IrName(conditional, "_true"));
 
-  SetToFirstInsertPoint(if_data.false_block, &ir_builder_);
+  SetToFirstInsertPoint(if_data.false_block, &b_);
   EmitArrayFunctionCallInto(false_function, {GetEmittedValueFor(false_arg)},
                             conditional_result, IrName(conditional, "_false"));
 
-  SetToFirstInsertPoint(if_data.after_block, &ir_builder_);
+  SetToFirstInsertPoint(if_data.after_block, &b_);
   return Status::OK();
 }
 
@@ -2568,7 +2523,7 @@ Status IrEmitter::FinishVisit(HloInstruction* root) {
 
   auto record_complete_computation = [&](llvm::Value* prof_counter) {
     if (prof_counter) {
-      profiling_state_.RecordCompleteComputation(&ir_builder_, prof_counter);
+      profiling_state_.RecordCompleteComputation(&b_, prof_counter);
     }
   };
 
@@ -2590,54 +2545,51 @@ llvm::Value* IrEmitter::GetProfileCounterCommon(
 
   int64 prof_counter_idx = it->second;
   string counter_name = IrName("prof_counter", hlo.name());
-  return ir_builder_.CreateGEP(GetProfileCountersArgument(),
-                               ir_builder_.getInt64(prof_counter_idx),
-                               AsStringRef(counter_name));
+  return b_.CreateGEP(GetProfileCountersArgument(),
+                      b_.getInt64(prof_counter_idx), AsStringRef(counter_name));
 }
 
-void IrEmitter::ProfilingState::UpdateProfileCounter(
-    llvm::IRBuilder<>* ir_builder, llvm::Value* prof_counter,
-    llvm::Value* cycle_end, llvm::Value* cycle_start) {
-  auto* cycle_diff = ir_builder->CreateSub(cycle_end, cycle_start);
+void IrEmitter::ProfilingState::UpdateProfileCounter(llvm::IRBuilder<>* b,
+                                                     llvm::Value* prof_counter,
+                                                     llvm::Value* cycle_end,
+                                                     llvm::Value* cycle_start) {
+  auto* cycle_diff = b->CreateSub(cycle_end, cycle_start);
   llvm::LoadInst* old_cycle_count =
-      ir_builder->CreateLoad(prof_counter, "old_cycle_count");
+      b->CreateLoad(prof_counter, "old_cycle_count");
   auto* new_cycle_count =
-      ir_builder->CreateAdd(cycle_diff, old_cycle_count, "new_cycle_count");
-  ir_builder->CreateStore(new_cycle_count, prof_counter);
+      b->CreateAdd(cycle_diff, old_cycle_count, "new_cycle_count");
+  b->CreateStore(new_cycle_count, prof_counter);
 }
 
-llvm::Value* IrEmitter::ProfilingState::ReadCycleCounter(
-    llvm::IRBuilder<>* ir_builder) {
-  llvm::Module* module = ir_builder->GetInsertBlock()->getModule();
+llvm::Value* IrEmitter::ProfilingState::ReadCycleCounter(llvm::IRBuilder<>* b) {
+  llvm::Module* module = b->GetInsertBlock()->getModule();
   if (use_rdtscp_) {
     llvm::Function* func_llvm_readcyclecounter =
         llvm::Intrinsic::getDeclaration(module,
                                         llvm::Intrinsic::readcyclecounter);
-    return ir_builder->CreateCall(func_llvm_readcyclecounter);
+    return b->CreateCall(func_llvm_readcyclecounter);
   }
   llvm::Function* func_llvm_x86_rdtscp =
       llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::x86_rdtscp);
   if (!aux_i8ptr_) {
-    llvm::AllocaInst* rdtscp_aux = llvm_ir::EmitAllocaAtFunctionEntry(
-        ir_builder->getInt32Ty(), "rdtscp_aux", ir_builder);
-    aux_i8ptr_ =
-        ir_builder->CreateBitCast(rdtscp_aux, ir_builder->getInt8PtrTy());
+    llvm::AllocaInst* rdtscp_aux =
+        llvm_ir::EmitAllocaAtFunctionEntry(b->getInt32Ty(), "rdtscp_aux", b);
+    aux_i8ptr_ = b->CreateBitCast(rdtscp_aux, b->getInt8PtrTy());
   }
-  llvm::ConstantInt* alloca_size = ir_builder->getInt64(4);
+  llvm::ConstantInt* alloca_size = b->getInt64(4);
   llvm::Function* func_llvm_lifetime_start =
       llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::lifetime_start);
-  ir_builder->CreateCall(func_llvm_lifetime_start, {alloca_size, aux_i8ptr_});
-  llvm::Value* rdtscp_call =
-      ir_builder->CreateCall(func_llvm_x86_rdtscp, aux_i8ptr_);
+  b->CreateCall(func_llvm_lifetime_start, {alloca_size, aux_i8ptr_});
+  llvm::Value* rdtscp_call = b->CreateCall(func_llvm_x86_rdtscp, aux_i8ptr_);
   llvm::Function* func_llvm_lifetime_end =
       llvm::Intrinsic::getDeclaration(module, llvm::Intrinsic::lifetime_end);
-  ir_builder->CreateCall(func_llvm_lifetime_end, {alloca_size, aux_i8ptr_});
+  b->CreateCall(func_llvm_lifetime_end, {alloca_size, aux_i8ptr_});
   return rdtscp_call;
 }
 
-void IrEmitter::ProfilingState::RecordCycleStart(llvm::IRBuilder<>* ir_builder,
+void IrEmitter::ProfilingState::RecordCycleStart(llvm::IRBuilder<>* b,
                                                  HloInstruction* hlo) {
-  auto* cycle_start = ReadCycleCounter(ir_builder);
+  auto* cycle_start = ReadCycleCounter(b);
   cycle_start->setName(AsStringRef(IrName(hlo, "cycle_start")));
   cycle_starts_[hlo] = cycle_start;
   if (first_read_cycle_start_ == nullptr) {
@@ -2645,20 +2597,20 @@ void IrEmitter::ProfilingState::RecordCycleStart(llvm::IRBuilder<>* ir_builder,
   }
 }
 
-void IrEmitter::ProfilingState::RecordCycleDelta(llvm::IRBuilder<>* ir_builder,
+void IrEmitter::ProfilingState::RecordCycleDelta(llvm::IRBuilder<>* b,
                                                  HloInstruction* hlo,
                                                  llvm::Value* prof_counter) {
-  auto* cycle_end = ReadCycleCounter(ir_builder);
+  auto* cycle_end = ReadCycleCounter(b);
   cycle_end->setName(AsStringRef(IrName(hlo, "cycle_end")));
   auto* cycle_start = cycle_starts_[hlo];
-  UpdateProfileCounter(ir_builder, prof_counter, cycle_end, cycle_start);
+  UpdateProfileCounter(b, prof_counter, cycle_end, cycle_start);
   last_read_cycle_end_ = cycle_end;
 }
 
 void IrEmitter::ProfilingState::RecordCompleteComputation(
-    llvm::IRBuilder<>* ir_builder, llvm::Value* prof_counter) {
+    llvm::IRBuilder<>* b, llvm::Value* prof_counter) {
   if (last_read_cycle_end_ && first_read_cycle_start_) {
-    UpdateProfileCounter(ir_builder, prof_counter, last_read_cycle_end_,
+    UpdateProfileCounter(b, prof_counter, last_read_cycle_end_,
                          first_read_cycle_start_);
   }
 }
@@ -2666,14 +2618,14 @@ void IrEmitter::ProfilingState::RecordCompleteComputation(
 Status IrEmitter::Preprocess(HloInstruction* hlo) {
   VLOG(3) << "Visiting: " << hlo->ToString();
   if (instruction_to_profile_idx_.count(hlo)) {
-    profiling_state_.RecordCycleStart(&ir_builder_, hlo);
+    profiling_state_.RecordCycleStart(&b_, hlo);
   }
   return Status::OK();
 }
 
 Status IrEmitter::Postprocess(HloInstruction* hlo) {
   if (auto* prof_counter = GetProfileCounterFor(*hlo)) {
-    profiling_state_.RecordCycleDelta(&ir_builder_, hlo, prof_counter);
+    profiling_state_.RecordCycleDelta(&b_, hlo, prof_counter);
   }
   return Status::OK();
 }
@@ -2732,22 +2684,20 @@ llvm::Value* IrEmitter::EmitTempBufferPointer(
     CHECK_EQ(1, assigned_buffers.size());
     const Shape& shape = assigned_buffers.begin()->first->shape();
 
-    llvm::AllocaInst*& tempbuf_address = thread_local_buffers_[{
-        ir_builder_.GetInsertBlock()->getParent(), slice}];
+    llvm::AllocaInst*& tempbuf_address =
+        thread_local_buffers_[{b_.GetInsertBlock()->getParent(), slice}];
     if (tempbuf_address == nullptr) {
       tempbuf_address = llvm_ir::EmitAllocaAtFunctionEntry(
           IrShapeType(shape),
-          tensorflow::strings::StrCat("thread_local", slice.ToString()),
-          &ir_builder_, MinimumAlignmentForShape(target_shape));
+          tensorflow::strings::StrCat("thread_local", slice.ToString()), &b_,
+          MinimumAlignmentForShape(target_shape));
     }
-    return ir_builder_.CreateBitCast(tempbuf_address,
-                                     element_type->getPointerTo());
+    return b_.CreateBitCast(tempbuf_address, element_type->getPointerTo());
   }
 
   llvm::Value* tempbuf_address_ptr = llvm_ir::EmitBufferIndexingGEP(
-      GetTempBuffersArgument(), slice.index(), &ir_builder_);
-  llvm::LoadInst* tempbuf_address_base =
-      ir_builder_.CreateLoad(tempbuf_address_ptr);
+      GetTempBuffersArgument(), slice.index(), &b_);
+  llvm::LoadInst* tempbuf_address_base = b_.CreateLoad(tempbuf_address_ptr);
   if (is_top_level_computation_ &&
       hlo_module_config_.debug_options()
           .xla_llvm_enable_invariant_load_metadata()) {
@@ -2766,11 +2716,11 @@ llvm::Value* IrEmitter::EmitTempBufferPointer(
   llvm::Value* tempbuf_address_untyped = tempbuf_address_base;
   if (slice.offset() > 0) {
     // Adjust the address to account for the slice offset.
-    tempbuf_address_untyped = ir_builder_.CreateInBoundsGEP(
-        tempbuf_address_base, ir_builder_.getInt64(slice.offset()));
+    tempbuf_address_untyped =
+        b_.CreateInBoundsGEP(tempbuf_address_base, b_.getInt64(slice.offset()));
   }
-  return ir_builder_.CreateBitCast(tempbuf_address_untyped,
-                                   element_type->getPointerTo());
+  return b_.CreateBitCast(tempbuf_address_untyped,
+                          element_type->getPointerTo());
 }
 
 // Emits a function call returning a single array element.  Allocates space
@@ -2781,7 +2731,7 @@ llvm::Value* IrEmitter::EmitElementFunctionCall(
     tensorflow::StringPiece name) {
   llvm::Value* return_value_buffer = EmitArrayFunctionCall(
       function, return_shape, 1, parameter_addresses, name);
-  return ir_builder_.CreateLoad(
+  return b_.CreateLoad(
       return_value_buffer,
       AsStringRef(tensorflow::strings::StrCat(name, "_return_value")));
 }
@@ -2799,9 +2749,9 @@ llvm::Value* IrEmitter::EmitElementFunctionCall(
 void IrEmitter::EmitArrayFunctionCallInto(
     llvm::Function* function, gtl::ArraySlice parameter_addresses,
     llvm::Value* return_value_buffer, tensorflow::StringPiece name) {
-  ir_builder_.CreateCall(
-      function, GetArrayFunctionCallArguments(
-                    parameter_addresses, &ir_builder_, name,
+  b_.CreateCall(function,
+                GetArrayFunctionCallArguments(
+                    parameter_addresses, &b_, name,
                     /*return_value_buffer=*/return_value_buffer,
                     /*exec_run_options_arg=*/GetExecutableRunOptionsArgument(),
                     /*temp_buffers_arg=*/GetTempBuffersArgument(),
@@ -2813,13 +2763,13 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall(
     gtl::ArraySlice parameter_addresses,
     tensorflow::StringPiece name) {
   llvm::Value* elements =
-      llvm::ConstantInt::get(ir_builder_.getInt64Ty(), element_count);
+      llvm::ConstantInt::get(b_.getInt64Ty(), element_count);
   PrimitiveType return_type = return_shape.element_type();
   llvm::Value* return_value_buffer =
       llvm_ir::EmitAllocaAtFunctionEntryWithCount(
           llvm_ir::PrimitiveTypeToIrType(return_type, module_), elements,
-          tensorflow::strings::StrCat(name, "_return_value_address"),
-          &ir_builder_, MinimumAlignmentForPrimitiveType(return_type));
+          tensorflow::strings::StrCat(name, "_return_value_address"), &b_,
+          MinimumAlignmentForPrimitiveType(return_type));
   EmitArrayFunctionCallInto(function, parameter_addresses, return_value_buffer,
                             name);
   return return_value_buffer;
@@ -2841,8 +2791,7 @@ Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) {
       attr_builder.addDereferenceableAttr(ByteSizeOf(target_shape));
       retval->addAttrs(attr_builder);
     }
-    addr = ir_builder_.CreateBitCast(retval,
-                                     IrShapeType(target_shape)->getPointerTo());
+    addr = b_.CreateBitCast(retval, IrShapeType(target_shape)->getPointerTo());
   } else {
     // For other nodes, we need the temporary buffer allocated for this node to
     // write the result into.
@@ -2884,14 +2833,14 @@ Status IrEmitter::EmitTargetElementLoop(
           llvm_ir::IrArray(op_target_address, element_shape));
     }
     TF_RETURN_IF_ERROR(
-        llvm_ir::LoopEmitter(element_generator, output_arrays, &ir_builder_)
+        llvm_ir::LoopEmitter(element_generator, output_arrays, &b_)
             .EmitLoop(IrName(target_op)));
 
     std::vector tuple_operand_ptrs;
     for (int64 i = 0; i < output_arrays.size(); ++i) {
       tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer());
     }
-    llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &ir_builder_, module_);
+    llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &b_, module_);
 
   } else {
     if (ShouldEmitParallelLoopFor(*target_op)) {
@@ -2900,11 +2849,11 @@ Status IrEmitter::EmitTargetElementLoop(
           compute_function_->GetDynamicLoopBounds();
       // Emit parallel loop with dynamic loop bounds for most-major dimensions.
       TF_RETURN_IF_ERROR(ParallelLoopEmitter(element_generator, target_array,
-                                             &dynamic_loop_bounds, &ir_builder_)
+                                             &dynamic_loop_bounds, &b_)
                              .EmitLoop(IrName(target_op)));
     } else {
       TF_RETURN_IF_ERROR(
-          llvm_ir::LoopEmitter(element_generator, target_array, &ir_builder_)
+          llvm_ir::LoopEmitter(element_generator, target_array, &b_)
               .EmitLoop(IrName(target_op)));
     }
   }
@@ -2917,8 +2866,8 @@ Status IrEmitter::EmitMemcpy(const HloInstruction& source,
   llvm::Value* destination_value = GetEmittedValueFor(&destination);
   int64 source_size = ByteSizeOf(source.shape());
   // TODO(b/63762267): Be more aggressive about specifying alignment.
-  ir_builder_.CreateMemCpy(destination_value, /*DstAlign=*/1, source_value,
-                           /*SrcAlign=*/1, source_size);
+  b_.CreateMemCpy(destination_value, /*DstAlign=*/1, source_value,
+                  /*SrcAlign=*/1, source_size);
   return Status::OK();
 }
 
@@ -2946,7 +2895,7 @@ Status IrEmitter::DefaultAction(HloInstruction* hlo) {
   ElementalIrEmitter::HloToElementGeneratorMap operand_to_generator;
   for (const HloInstruction* operand : hlo->operands()) {
     operand_to_generator[operand] = [=](const llvm_ir::IrArray::Index& index) {
-      return GetIrArrayFor(operand).EmitReadArrayElement(index, &ir_builder_);
+      return GetIrArrayFor(operand).EmitReadArrayElement(index, &b_);
     };
   }
   CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_);
@@ -2961,8 +2910,8 @@ StatusOr IrEmitter::EmitScalarCall(
   std::vector argument_addrs;
   for (auto argument : arguments) {
     llvm::Value* argument_addr = llvm_ir::EmitAllocaAtFunctionEntry(
-        argument->getType(), "arg_addr", &ir_builder_);
-    ir_builder_.CreateStore(argument, argument_addr);
+        argument->getType(), "arg_addr", &b_);
+    b_.CreateStore(argument, argument_addr);
     argument_addrs.push_back(argument_addr);
   }
   return EmitElementFunctionCall(llvm_function,
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index 2840c14303..4e928ffadc 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -98,7 +98,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
       bool is_top_level_computation,
       std::vector* instruction_order);
 
-  llvm::IRBuilder<>* ir_builder() { return &ir_builder_; }
+  llvm::IRBuilder<>* b() { return &b_; }
 
   // Emits a call to `computation` with scalar arguments `arguments`.
   StatusOr EmitScalarCall(
@@ -416,7 +416,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   // creates the encapsulated llvm::Function s.t. it is added to the llvm
   // module's function list).
   std::unique_ptr compute_function_;
-  llvm::IRBuilder<> ir_builder_;
+  llvm::IRBuilder<> b_;
 
   // Maps HLO instructions to their index into the profile counter array.
   const std::unordered_map
@@ -452,23 +452,22 @@ class IrEmitter : public DfsHloVisitorWithDefault {
         : use_rdtscp_(use_rdtscp), prof_counters_(prof_counters) {}
 
     // Record the cycle counter before an HLO executes.
-    void RecordCycleStart(llvm::IRBuilder<>* ir_builder, HloInstruction* hlo);
+    void RecordCycleStart(llvm::IRBuilder<>* b, HloInstruction* hlo);
     // Record the number of cycles it took for an HLO to execute.
-    void RecordCycleDelta(llvm::IRBuilder<>* ir_builder, HloInstruction* hlo,
+    void RecordCycleDelta(llvm::IRBuilder<>* b, HloInstruction* hlo,
                           llvm::Value* prof_counter);
     // Record the number of cycles it took for the entire computation to
     // execute.
-    void RecordCompleteComputation(llvm::IRBuilder<>* ir_builder,
+    void RecordCompleteComputation(llvm::IRBuilder<>* b,
                                    llvm::Value* prof_counter);
 
     // Convenience function to generate a call to an intrinsic which reads the
     // CPU cycle counter.
-    llvm::Value* ReadCycleCounter(llvm::IRBuilder<>* ir_builder);
+    llvm::Value* ReadCycleCounter(llvm::IRBuilder<>* b);
 
     // Store the cycle counter delta to the per-HLO profile counter.
-    void UpdateProfileCounter(llvm::IRBuilder<>* ir_builder,
-                              llvm::Value* prof_counter, llvm::Value* cycle_end,
-                              llvm::Value* cycle_start);
+    void UpdateProfileCounter(llvm::IRBuilder<>* b, llvm::Value* prof_counter,
+                              llvm::Value* cycle_end, llvm::Value* cycle_start);
 
    private:
     // Should we use the x86-specific rdtscp or the generic readcyclecounter
diff --git a/tensorflow/compiler/xla/service/cpu/ir_function.cc b/tensorflow/compiler/xla/service/cpu/ir_function.cc
index 2d6f2f3818..6aff838462 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_function.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_function.cc
@@ -49,11 +49,10 @@ IrFunction::IrFunction(const string& function_name,
                        llvm::Function::LinkageTypes linkage,
                        const bool optimize_for_size_requested,
                        const bool enable_fast_math, llvm::Module* llvm_module,
-                       llvm::IRBuilder<>* ir_builder,
-                       int64 num_dynamic_loop_bounds)
-    : ir_builder_(ir_builder),
+                       llvm::IRBuilder<>* b, int64 num_dynamic_loop_bounds)
+    : b_(b),
       llvm_module_(llvm_module),
-      caller_insert_point_guard_(*ir_builder),
+      caller_insert_point_guard_(*b),
       num_dynamic_loop_bounds_(num_dynamic_loop_bounds) {
   Initialize(function_name, linkage, optimize_for_size_requested,
              enable_fast_math);
@@ -61,7 +60,7 @@ IrFunction::IrFunction(const string& function_name,
 
 IrFunction::~IrFunction() {
   // Emit function return value.
-  ir_builder_->CreateRetVoid();
+  b_->CreateRetVoid();
 }
 
 DynamicLoopBounds IrFunction::GetDynamicLoopBounds() {
@@ -174,7 +173,7 @@ void IrFunction::Initialize(const string& function_name,
     function_->addAttribute(argument.getArgNo() + 1, llvm::Attribute::NoAlias);
   }
 
-  ir_builder_->SetInsertPoint(llvm::BasicBlock::Create(
+  b_->SetInsertPoint(llvm::BasicBlock::Create(
       /*Context=*/llvm_module_->getContext(),
       /*Name=*/"entry",
       /*Parent=*/function_));
@@ -184,9 +183,8 @@ llvm::Value* IrFunction::GetDynamicLoopBound(const int64 offset) {
   CHECK_GT(num_dynamic_loop_bounds_, 0);
   CHECK_LT(offset, num_dynamic_loop_bounds_ * 2);
   string name = tensorflow::strings::StrCat("dynamic_loop_bound_", offset);
-  return ir_builder_->CreateLoad(
-      ir_builder_->CreateGEP(CHECK_NOTNULL(dynamic_loop_bounds_arg_),
-                             ir_builder_->getInt64(offset), AsStringRef(name)));
+  return b_->CreateLoad(b_->CreateGEP(CHECK_NOTNULL(dynamic_loop_bounds_arg_),
+                                      b_->getInt64(offset), AsStringRef(name)));
 }
 
 // Emits code to allocate an array of parameter address pointers, and store
@@ -195,27 +193,25 @@ llvm::Value* IrFunction::GetDynamicLoopBound(const int64 offset) {
 // address buffer).
 std::vector GetArrayFunctionCallArguments(
     tensorflow::gtl::ArraySlice parameter_addresses,
-    llvm::IRBuilder<>* ir_builder, tensorflow::StringPiece name,
+    llvm::IRBuilder<>* b, tensorflow::StringPiece name,
     llvm::Value* return_value_buffer, llvm::Value* exec_run_options_arg,
     llvm::Value* temp_buffers_arg, llvm::Value* profile_counters_arg) {
   llvm::Value* parameter_addresses_buffer =
       llvm_ir::EmitAllocaAtFunctionEntryWithCount(
-          ir_builder->getInt8PtrTy(),
-          ir_builder->getInt32(parameter_addresses.size()),
-          tensorflow::strings::StrCat(name, "_parameter_addresses"),
-          ir_builder);
+          b->getInt8PtrTy(), b->getInt32(parameter_addresses.size()),
+          tensorflow::strings::StrCat(name, "_parameter_addresses"), b);
   for (size_t i = 0; i < parameter_addresses.size(); ++i) {
-    llvm::Value* parameter_as_i8ptr = ir_builder->CreateBitCast(
-        parameter_addresses[i], ir_builder->getInt8PtrTy(),
-        AsStringRef(tensorflow::strings::StrCat(name, "_parameter_", i,
-                                                "_address_as_i8ptr")));
-    llvm::Value* slot_in_param_addresses = ir_builder->CreateInBoundsGEP(
-        parameter_addresses_buffer, {ir_builder->getInt64(i)});
-    ir_builder->CreateStore(parameter_as_i8ptr, slot_in_param_addresses);
+    llvm::Value* parameter_as_i8ptr =
+        b->CreateBitCast(parameter_addresses[i], b->getInt8PtrTy(),
+                         AsStringRef(tensorflow::strings::StrCat(
+                             name, "_parameter_", i, "_address_as_i8ptr")));
+    llvm::Value* slot_in_param_addresses =
+        b->CreateInBoundsGEP(parameter_addresses_buffer, {b->getInt64(i)});
+    b->CreateStore(parameter_as_i8ptr, slot_in_param_addresses);
   }
 
   const auto to_int8_ptr = [=](llvm::Value* ptr) {
-    return ir_builder->CreatePointerCast(ptr, ir_builder->getInt8PtrTy());
+    return b->CreatePointerCast(ptr, b->getInt8PtrTy());
   };
   std::vector arguments{
       to_int8_ptr(return_value_buffer), to_int8_ptr(exec_run_options_arg),
@@ -230,22 +226,21 @@ std::vector GetArrayFunctionCallArguments(
 // calls to 'parallel_function' (and joins threads before returning).
 Status EmitCallToParallelForkJoin(
     const std::vector& arguments, const Shape& shape,
-    const std::vector& dimension_partition_counts,
-    llvm::IRBuilder<>* ir_builder, llvm::Function* parallel_function,
-    const string& name) {
-  llvm::Module* module = ir_builder->GetInsertBlock()->getModule();
+    const std::vector& dimension_partition_counts, llvm::IRBuilder<>* b,
+    llvm::Function* parallel_function, const string& name) {
+  llvm::Module* module = b->GetInsertBlock()->getModule();
 
   // Build ParallelForkJoin function type.
   std::vector compute_function_params =
       GetComputeFunctionParams(module, /*num_dynamic_loop_bounds=*/0);
   // Number of parallel compute functions.
-  compute_function_params.push_back(ir_builder->getInt32Ty());
+  compute_function_params.push_back(b->getInt32Ty());
   // Array of partitions. There is an array element for each
   // partition x partition_dim x 2 (for dimension start and limit).
   compute_function_params.push_back(
       llvm::Type::getInt64PtrTy(module->getContext()));
   // Number of partitioned most-major dimensions in 'shape'.
-  compute_function_params.push_back(ir_builder->getInt32Ty());
+  compute_function_params.push_back(b->getInt32Ty());
   // Function pointer for compute function to be dispatched in parallel.
   compute_function_params.push_back(
       llvm::Type::getInt8PtrTy(module->getContext()));
@@ -268,7 +263,7 @@ Status EmitCallToParallelForkJoin(
   ShapePartitionIterator partition_iterator(shape, dimension_partition_counts);
   const int64 num_partitions = partition_iterator.GetTotalPartitionCount();
   // Add argument specifying the number of parallel partitions.
-  fork_join_arguments.push_back(ir_builder->getInt32(num_partitions));
+  fork_join_arguments.push_back(b->getInt32(num_partitions));
 
   // The number of partitioned most-major dimensions in 'shape'.
   const int32 num_partitioned_dims = dimension_partition_counts.size();
@@ -293,15 +288,15 @@ Status EmitCallToParallelForkJoin(
       const std::pair& dim_partition = dim_partitions[j];
       const int32 index = partition_index + j * dim_partition_size;
       // Store partition [dim_start, dim_limit) intervals for each dimension.
-      partitions[index] = ir_builder->getInt64(dim_partition.first);
+      partitions[index] = b->getInt64(dim_partition.first);
       partitions[index + 1] =
-          ir_builder->getInt64(dim_partition.first + dim_partition.second);
+          b->getInt64(dim_partition.first + dim_partition.second);
     }
   }
 
   // Create global variable out of dimension partitions in 'partitions'.
   llvm::ArrayType* partitions_array_type =
-      llvm::ArrayType::get(ir_builder->getInt64Ty(), partition_array_size);
+      llvm::ArrayType::get(b->getInt64Ty(), partition_array_size);
   llvm::Constant* partitions_array =
       llvm::ConstantArray::get(partitions_array_type, partitions);
   llvm::GlobalVariable* global_partitions_array = new llvm::GlobalVariable(
@@ -315,16 +310,16 @@ Status EmitCallToParallelForkJoin(
           tensorflow::strings::StrCat(name, "_parallel_dimension_partitions")));
 
   // Add argument specifying parallel dimension partitions.
-  fork_join_arguments.push_back(ir_builder->CreateBitCast(
-      global_partitions_array,
-      llvm::Type::getInt64PtrTy(module->getContext())));
+  fork_join_arguments.push_back(
+      b->CreateBitCast(global_partitions_array,
+                       llvm::Type::getInt64PtrTy(module->getContext())));
   // Add argument specifying the number of partitioned most-major dimensions.
-  fork_join_arguments.push_back(ir_builder->getInt32(num_partitioned_dims));
+  fork_join_arguments.push_back(b->getInt32(num_partitioned_dims));
   // Add argument for parallel compute function pointer.
   fork_join_arguments.push_back(
-      ir_builder->CreateBitCast(parallel_function, ir_builder->getInt8PtrTy()));
+      b->CreateBitCast(parallel_function, b->getInt8PtrTy()));
   // Emit call to parallel fork/join.
-  ir_builder->CreateCall(fork_join_func, fork_join_arguments);
+  b->CreateCall(fork_join_func, fork_join_arguments);
 
   return Status::OK();
 }
diff --git a/tensorflow/compiler/xla/service/cpu/ir_function.h b/tensorflow/compiler/xla/service/cpu/ir_function.h
index 2e55181eed..a41cbb64cd 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_function.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_function.h
@@ -54,7 +54,7 @@ class IrFunction {
   IrFunction(const string& function_name, llvm::Function::LinkageTypes linkage,
              const bool optimize_for_size_requested,
              const bool enable_fast_math, llvm::Module* llvm_module,
-             llvm::IRBuilder<>* ir_builder, int64 num_dynamic_loop_bounds);
+             llvm::IRBuilder<>* b, int64 num_dynamic_loop_bounds);
   ~IrFunction();
 
   // Emit ir to read and return the set of ir values representing the dynamic
@@ -97,7 +97,7 @@ class IrFunction {
   // 'offset' from the "dynamic_loop_bounds" argument of this function.
   llvm::Value* GetDynamicLoopBound(int64 offset);
 
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
   llvm::Module* llvm_module_;
   llvm::IRBuilder<>::InsertPointGuard caller_insert_point_guard_;
 
@@ -116,7 +116,7 @@ class IrFunction {
 // Returns an array of compute function call argument ir values.
 std::vector GetArrayFunctionCallArguments(
     tensorflow::gtl::ArraySlice parameter_addresses,
-    llvm::IRBuilder<>* ir_builder, tensorflow::StringPiece name,
+    llvm::IRBuilder<>* b, tensorflow::StringPiece name,
     llvm::Value* return_value_buffer, llvm::Value* exec_run_options_arg,
     llvm::Value* temp_buffers_arg, llvm::Value* profile_counters_arg);
 
@@ -124,9 +124,8 @@ std::vector GetArrayFunctionCallArguments(
 // calls to 'parallel_function' (and joins threads before returning).
 Status EmitCallToParallelForkJoin(
     const std::vector& arguments, const Shape& shape,
-    const std::vector& dimension_partition_counts,
-    llvm::IRBuilder<>* ir_builder, llvm::Function* parallel_function,
-    const string& name);
+    const std::vector& dimension_partition_counts, llvm::IRBuilder<>* b,
+    llvm::Function* parallel_function, const string& name);
 
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc b/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
index 2e5cc96098..ec0498e04e 100644
--- a/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/llvm_ir_runtime.cc
@@ -52,12 +52,12 @@ llvm::Function* EmitVectorF32TanhIfNeeded(llvm::Module* module,
   llvm::BasicBlock* vector_tanh_body =
       llvm::BasicBlock::Create(*context, "body", vector_tanh_function);
 
-  llvm::IRBuilder<> ir_builder(vector_tanh_body);
+  llvm::IRBuilder<> b(vector_tanh_body);
   llvm::FastMathFlags fast_math_flags;
   fast_math_flags.setFast();
-  ir_builder.setFastMathFlags(fast_math_flags);
+  b.setFastMathFlags(fast_math_flags);
 
-  VectorSupportLibrary vsl(F32, vector_width, &ir_builder, "tanh_f32");
+  VectorSupportLibrary vsl(F32, vector_width, &b, "tanh_f32");
 
   llvm::Value* input = &*vector_tanh_function->arg_begin();
   CHECK_EQ(input->getType(), vsl.vector_type());
@@ -91,7 +91,7 @@ llvm::Function* EmitVectorF32TanhIfNeeded(llvm::Module* module,
   }
 
   llvm::Value* result = vsl.Div(numerator, denominator);
-  ir_builder.CreateRet(result);
+  b.CreateRet(result);
 
   DCHECK(!llvm::verifyFunction(*vector_tanh_function));
   return vector_tanh_function;
@@ -113,12 +113,12 @@ llvm::Function* EmitVectorF32ExpIfNeeded(llvm::Module* module,
   llvm::BasicBlock* vector_exp_body =
       llvm::BasicBlock::Create(*context, "body", vector_exp_function);
 
-  llvm::IRBuilder<> ir_builder(vector_exp_body);
+  llvm::IRBuilder<> b(vector_exp_body);
   llvm::FastMathFlags fast_math_flags;
   fast_math_flags.setFast();
-  ir_builder.setFastMathFlags(fast_math_flags);
+  b.setFastMathFlags(fast_math_flags);
 
-  VectorSupportLibrary vsl(F32, vector_width, &ir_builder, "exp_f32");
+  VectorSupportLibrary vsl(F32, vector_width, &b, "exp_f32");
 
   // This implements the same polynomial approximation as implemented in Eigen3.
 
@@ -160,21 +160,21 @@ llvm::Function* EmitVectorF32ExpIfNeeded(llvm::Module* module,
   // VectorSupportLibrary (intentionally) can't juggle more than one type at a
   // time so drop down to IRBuilder for this bit.
   llvm::Value* vector_constant_0x7f =
-      ir_builder.CreateVectorSplat(vector_width, ir_builder.getInt32(0x7f));
+      b.CreateVectorSplat(vector_width, b.getInt32(0x7f));
   llvm::Value* vector_constant_23 =
-      ir_builder.CreateVectorSplat(vector_width, ir_builder.getInt32(23));
+      b.CreateVectorSplat(vector_width, b.getInt32(23));
   llvm::Type* i32_vector_type =
-      llvm::VectorType::get(ir_builder.getInt32Ty(), vector_width);
+      llvm::VectorType::get(b.getInt32Ty(), vector_width);
   // fx is clamped so we don't have to worry about it being out of range for
   // i32.
-  llvm::Value* emm0 = ir_builder.CreateFPToSI(fx, i32_vector_type);
-  emm0 = ir_builder.CreateAdd(emm0, vector_constant_0x7f);
-  emm0 = ir_builder.CreateShl(emm0, vector_constant_23);
-  llvm::Value* emm0_f32 = ir_builder.CreateBitCast(emm0, vsl.vector_type());
+  llvm::Value* emm0 = b.CreateFPToSI(fx, i32_vector_type);
+  emm0 = b.CreateAdd(emm0, vector_constant_0x7f);
+  emm0 = b.CreateShl(emm0, vector_constant_23);
+  llvm::Value* emm0_f32 = b.CreateBitCast(emm0, vsl.vector_type());
 
   llvm::Value* result = vsl.Max(vsl.Mul(y, emm0_f32), input);
 
-  ir_builder.CreateRet(result);
+  b.CreateRet(result);
 
   DCHECK(!llvm::verifyFunction(*vector_exp_function));
   return vector_exp_function;
@@ -196,13 +196,13 @@ llvm::Function* EmitVectorF32LogIfNeeded(llvm::Module* module,
   llvm::BasicBlock* vector_log_body =
       llvm::BasicBlock::Create(*context, "body", vector_log_function);
 
-  llvm::IRBuilder<> ir_builder(vector_log_body);
+  llvm::IRBuilder<> b(vector_log_body);
   llvm::FastMathFlags fast_math_flags;
   fast_math_flags.setFast();
-  ir_builder.setFastMathFlags(fast_math_flags);
+  b.setFastMathFlags(fast_math_flags);
 
   llvm::Value* input = &*vector_log_function->arg_begin();
-  VectorSupportLibrary vsl(F32, vector_width, &ir_builder, "log_f32");
+  VectorSupportLibrary vsl(F32, vector_width, &b, "log_f32");
 
   const llvm::APFloat half = GetIeeeF32(0.5);
   const llvm::APFloat one = GetIeeeF32(1.0);
@@ -238,22 +238,21 @@ llvm::Function* EmitVectorF32LogIfNeeded(llvm::Module* module,
   // VectorSupportLibrary (intentionally) can't juggle more than one type at a
   // time so drop down to IRBuilder for this bit.
   llvm::Value* vector_constant_0x7f =
-      ir_builder.CreateVectorSplat(vector_width, ir_builder.getInt32(0x7f));
+      b.CreateVectorSplat(vector_width, b.getInt32(0x7f));
   llvm::Value* vector_constant_23 =
-      ir_builder.CreateVectorSplat(vector_width, ir_builder.getInt32(23));
+      b.CreateVectorSplat(vector_width, b.getInt32(23));
   llvm::Type* i32_vector_type =
-      llvm::VectorType::get(ir_builder.getInt32Ty(), vector_width);
+      llvm::VectorType::get(b.getInt32Ty(), vector_width);
 
-  llvm::Value* emm0 = ir_builder.CreateLShr(
-      ir_builder.CreateBitCast(input, i32_vector_type), vector_constant_23);
+  llvm::Value* emm0 =
+      b.CreateLShr(b.CreateBitCast(input, i32_vector_type), vector_constant_23);
 
   // Keep only the fractional part.
   input = vsl.FloatAnd(input, inv_mant_mask);
   input = vsl.FloatOr(input, half);
 
-  emm0 = ir_builder.CreateSub(emm0, vector_constant_0x7f);
-  llvm::Value* e =
-      vsl.Add(one, ir_builder.CreateSIToFP(emm0, vsl.vector_type()));
+  emm0 = b.CreateSub(emm0, vector_constant_0x7f);
+  llvm::Value* e = vsl.Add(one, b.CreateSIToFP(emm0, vsl.vector_type()));
 
   // part2:
   //   if( x < SQRTHF ) {
@@ -294,7 +293,7 @@ llvm::Function* EmitVectorF32LogIfNeeded(llvm::Module* module,
   llvm::Value* or_rhs = vsl.FloatAnd(iszero_mask, minus_inf);
   llvm::Value* result = vsl.FloatOr(or_lhs, or_rhs);
 
-  ir_builder.CreateRet(result);
+  b.CreateRet(result);
 
   DCHECK(!llvm::verifyFunction(*vector_log_function));
   return vector_log_function;
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc
index 59ae5acd8b..8560e4296a 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc
@@ -25,8 +25,8 @@ namespace cpu {
 ParallelLoopEmitter::ParallelLoopEmitter(
     const llvm_ir::ElementGenerator& target_element_generator,
     const llvm_ir::IrArray& target_array,
-    const DynamicLoopBounds* dynamic_loop_bounds, llvm::IRBuilder<>* ir_builder)
-    : LoopEmitter(target_element_generator, target_array, ir_builder),
+    const DynamicLoopBounds* dynamic_loop_bounds, llvm::IRBuilder<>* b)
+    : LoopEmitter(target_element_generator, target_array, b),
       dynamic_loop_bounds_(dynamic_loop_bounds) {}
 
 std::vector
@@ -37,7 +37,7 @@ ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock(
   CHECK(!ShapeUtil::IsTuple(shape_));
   CHECK(!ShapeUtil::IsScalar(shape_));
 
-  llvm_ir::ForLoopNest loop_nest(loop_name, ir_builder_);
+  llvm_ir::ForLoopNest loop_nest(loop_name, b_);
   const int64 num_dims = shape_.dimensions_size();
   llvm_ir::IrArray::Index array_index(index_type, num_dims);
 
@@ -65,8 +65,7 @@ ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock(
     }
   }
   // Point IR builder at inner loop BB.
-  llvm_ir::SetToFirstInsertPoint(loop_nest.GetInnerLoopBodyBasicBlock(),
-                                 ir_builder_);
+  llvm_ir::SetToFirstInsertPoint(loop_nest.GetInnerLoopBodyBasicBlock(), b_);
 
   // Set exit_bb_ to the exit block of the loop nest.
   exit_bb_ = loop_nest.GetOuterLoopExitBasicBlock();
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h
index 25e182a26d..076c683ca5 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h
@@ -54,7 +54,7 @@ class ParallelLoopEmitter : public llvm_ir::LoopEmitter {
   ParallelLoopEmitter(const llvm_ir::ElementGenerator& target_element_generator,
                       const llvm_ir::IrArray& target_array,
                       const DynamicLoopBounds* dynamic_loop_bounds,
-                      llvm::IRBuilder<>* ir_builder);
+                      llvm::IRBuilder<>* b);
 
   ParallelLoopEmitter(const ParallelLoopEmitter&) = delete;
   ParallelLoopEmitter& operator=(const ParallelLoopEmitter&) = delete;
diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc
index ccb61740f6..01daed4bcd 100644
--- a/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_noalias_test.cc
@@ -78,7 +78,7 @@ TEST_F(CpuNoAliasTest, Concat) {
   llvm::Function* func = llvm::cast(
       ir_module.getOrInsertFunction("test_fn", llvm::Type::getVoidTy(context)));
   llvm::BasicBlock* bb = llvm::BasicBlock::Create(context, "body", func);
-  llvm::IRBuilder<> ir_builder(bb);
+  llvm::IRBuilder<> b(bb);
   auto* zero = llvm::ConstantInt::get(llvm::Type::getInt32Ty(context), 0);
   llvm_ir::IrArray::Index zero2D({zero, zero});
 
@@ -90,7 +90,7 @@ TEST_F(CpuNoAliasTest, Concat) {
         ir_module.getOrInsertGlobal("param_x", array2d_type);
     llvm_ir::IrArray param_x_array(param_x_val, param_shape);
     aa.AddAliasingInformationToIrArray(*param_x, ¶m_x_array);
-    param_x_array.EmitReadArrayElement(zero2D, &ir_builder)
+    param_x_array.EmitReadArrayElement(zero2D, &b)
         ->setName("read_param_x_array");
   }
 
@@ -100,7 +100,7 @@ TEST_F(CpuNoAliasTest, Concat) {
     auto shape = ShapeUtil::MakeShape(F32, {2, 4});
     llvm_ir::IrArray concat1_array(concat1_val, shape);
     aa.AddAliasingInformationToIrArray(*concat1, &concat1_array);
-    concat1_array.EmitReadArrayElement(zero2D, &ir_builder)
+    concat1_array.EmitReadArrayElement(zero2D, &b)
         ->setName("read_concat1_array");
   }
 
@@ -110,7 +110,7 @@ TEST_F(CpuNoAliasTest, Concat) {
     auto shape = ShapeUtil::MakeShape(F32, {2, 6});
     llvm_ir::IrArray concat2_array(concat2_val, shape);
     aa.AddAliasingInformationToIrArray(*concat2, &concat2_array);
-    concat2_array.EmitReadArrayElement(zero2D, &ir_builder)
+    concat2_array.EmitReadArrayElement(zero2D, &b)
         ->setName("read_concat2_array");
   }
 
diff --git a/tensorflow/compiler/xla/service/cpu/vector_support_library.cc b/tensorflow/compiler/xla/service/cpu/vector_support_library.cc
index c444d15185..3274be8d9d 100644
--- a/tensorflow/compiler/xla/service/cpu/vector_support_library.cc
+++ b/tensorflow/compiler/xla/service/cpu/vector_support_library.cc
@@ -23,14 +23,14 @@ namespace xla {
 namespace cpu {
 VectorSupportLibrary::VectorSupportLibrary(PrimitiveType primitive_type,
                                            int64 vector_size,
-                                           llvm::IRBuilder<>* ir_builder,
+                                           llvm::IRBuilder<>* b,
                                            std::string name)
     : vector_size_(vector_size),
       primitive_type_(primitive_type),
-      ir_builder_(ir_builder),
+      b_(b),
       name_(std::move(name)) {
   scalar_type_ = llvm_ir::PrimitiveTypeToIrType(
-      primitive_type, ir_builder_->GetInsertBlock()->getModule());
+      primitive_type, b_->GetInsertBlock()->getModule());
   scalar_pointer_type_ = llvm::PointerType::getUnqual(scalar_type_);
   vector_type_ = llvm::VectorType::get(scalar_type_, vector_size);
   vector_pointer_type_ = llvm::PointerType::getUnqual(vector_type_);
@@ -63,9 +63,9 @@ llvm::Value* VectorSupportLibrary::Mul(llvm::Value* lhs, llvm::Value* rhs) {
 llvm::Value* VectorSupportLibrary::MulInternal(llvm::Value* lhs,
                                                llvm::Value* rhs) {
   if (scalar_type_->isFloatingPointTy()) {
-    return ir_builder()->CreateFMul(lhs, rhs, name());
+    return b()->CreateFMul(lhs, rhs, name());
   } else {
-    return ir_builder()->CreateMul(lhs, rhs, name());
+    return b()->CreateMul(lhs, rhs, name());
   }
 }
 
@@ -76,13 +76,13 @@ llvm::Value* VectorSupportLibrary::Add(llvm::Value* lhs, llvm::Value* rhs) {
 
 llvm::Value* VectorSupportLibrary::Sub(llvm::Value* lhs, llvm::Value* rhs) {
   AssertCorrectTypes({lhs, rhs});
-  return ir_builder()->CreateFSub(lhs, rhs);
+  return b()->CreateFSub(lhs, rhs);
 }
 
 llvm::Value* VectorSupportLibrary::Max(llvm::Value* lhs, llvm::Value* rhs) {
   AssertCorrectTypes({lhs, rhs});
   if (scalar_type_->isFloatingPointTy()) {
-    return llvm_ir::EmitFloatMax(lhs, rhs, ir_builder_);
+    return llvm_ir::EmitFloatMax(lhs, rhs, b_);
   } else {
     LOG(FATAL) << "Max for integers is unimplemented";
   }
@@ -91,13 +91,13 @@ llvm::Value* VectorSupportLibrary::Max(llvm::Value* lhs, llvm::Value* rhs) {
 llvm::Value* VectorSupportLibrary::Floor(llvm::Value* a) {
   AssertCorrectTypes({a});
   return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::floor, {a},
-                                      {a->getType()}, ir_builder());
+                                      {a->getType()}, b());
 }
 
 llvm::Value* VectorSupportLibrary::Div(llvm::Value* lhs, llvm::Value* rhs) {
   AssertCorrectTypes({lhs, rhs});
   if (scalar_type_->isFloatingPointTy()) {
-    return ir_builder()->CreateFDiv(lhs, rhs, name());
+    return b()->CreateFDiv(lhs, rhs, name());
   } else {
     LOG(FATAL) << "Division for integers is unimplemented";
   }
@@ -111,42 +111,41 @@ llvm::Value* VectorSupportLibrary::Clamp(llvm::Value* a,
   CHECK(low.compare(high) == llvm::APFloat::cmpLessThan);
   CHECK(scalar_type_->isFloatingPointTy());
   return llvm_ir::EmitFloatMin(
-      llvm_ir::EmitFloatMax(a, GetConstantFloat(type, low), ir_builder_),
-      GetConstantFloat(type, high), ir_builder_);
+      llvm_ir::EmitFloatMax(a, GetConstantFloat(type, low), b_),
+      GetConstantFloat(type, high), b_);
 }
 
 llvm::Value* VectorSupportLibrary::FCmpEQMask(llvm::Value* lhs,
                                               llvm::Value* rhs) {
   AssertCorrectTypes({lhs, rhs});
-  return I1ToFloat(ir_builder()->CreateFCmpOEQ(lhs, rhs, name()));
+  return I1ToFloat(b()->CreateFCmpOEQ(lhs, rhs, name()));
 }
 
 llvm::Value* VectorSupportLibrary::FCmpOLTMask(llvm::Value* lhs,
                                                llvm::Value* rhs) {
   AssertCorrectTypes({lhs, rhs});
-  return I1ToFloat(ir_builder()->CreateFCmpOLT(lhs, rhs, name()));
+  return I1ToFloat(b()->CreateFCmpOLT(lhs, rhs, name()));
 }
 
 llvm::Value* VectorSupportLibrary::FCmpULEMask(llvm::Value* lhs,
                                                llvm::Value* rhs) {
   AssertCorrectTypes({lhs, rhs});
-  return I1ToFloat(ir_builder()->CreateFCmpULE(lhs, rhs, name()));
+  return I1ToFloat(b()->CreateFCmpULE(lhs, rhs, name()));
 }
 
 llvm::Value* VectorSupportLibrary::I1ToFloat(llvm::Value* i1) {
   bool is_vector = llvm::isa(i1->getType());
   llvm::Type* integer_type = IntegerTypeForFloatSize(is_vector);
-  return ir_builder()->CreateBitCast(
-      ir_builder()->CreateSExt(i1, integer_type, name()),
-      is_vector ? vector_type() : scalar_type(), name());
+  return b()->CreateBitCast(b()->CreateSExt(i1, integer_type, name()),
+                            is_vector ? vector_type() : scalar_type(), name());
 }
 
 llvm::Type* VectorSupportLibrary::IntegerTypeForFloatSize(bool vector) {
   CHECK(scalar_type()->isFloatingPointTy());
   const llvm::DataLayout& data_layout =
-      ir_builder()->GetInsertBlock()->getModule()->getDataLayout();
+      b()->GetInsertBlock()->getModule()->getDataLayout();
   int64 float_size_bits = data_layout.getTypeSizeInBits(scalar_type());
-  llvm::Type* scalar_int_type = ir_builder()->getIntNTy(float_size_bits);
+  llvm::Type* scalar_int_type = b()->getIntNTy(float_size_bits);
   if (vector) {
     return llvm::VectorType::get(scalar_int_type, vector_size());
   } else {
@@ -156,7 +155,7 @@ llvm::Type* VectorSupportLibrary::IntegerTypeForFloatSize(bool vector) {
 
 llvm::Value* VectorSupportLibrary::BroadcastScalar(llvm::Value* x) {
   CHECK_EQ(x->getType(), scalar_type());
-  return ir_builder()->CreateVectorSplat(vector_size(), x, name());
+  return b()->CreateVectorSplat(vector_size(), x, name());
 }
 
 llvm::Value* VectorSupportLibrary::FloatAnd(llvm::Value* lhs,
@@ -164,10 +163,9 @@ llvm::Value* VectorSupportLibrary::FloatAnd(llvm::Value* lhs,
   AssertCorrectTypes({lhs, rhs});
   llvm::Type* int_type =
       IntegerTypeForFloatSize(lhs->getType() == vector_type());
-  return ir_builder()->CreateBitCast(
-      ir_builder()->CreateAnd(
-          ir_builder()->CreateBitCast(lhs, int_type, name()),
-          ir_builder()->CreateBitCast(rhs, int_type, name()), name()),
+  return b()->CreateBitCast(
+      b()->CreateAnd(b()->CreateBitCast(lhs, int_type, name()),
+                     b()->CreateBitCast(rhs, int_type, name()), name()),
       vector_type());
 }
 
@@ -175,9 +173,8 @@ llvm::Value* VectorSupportLibrary::FloatNot(llvm::Value* lhs) {
   AssertCorrectTypes({lhs});
   llvm::Type* int_type =
       IntegerTypeForFloatSize(lhs->getType() == vector_type());
-  return ir_builder()->CreateBitCast(
-      ir_builder()->CreateNot(
-          ir_builder()->CreateBitCast(lhs, int_type, name()), name()),
+  return b()->CreateBitCast(
+      b()->CreateNot(b()->CreateBitCast(lhs, int_type, name()), name()),
       vector_type());
 }
 
@@ -185,47 +182,43 @@ llvm::Value* VectorSupportLibrary::FloatOr(llvm::Value* lhs, llvm::Value* rhs) {
   AssertCorrectTypes({lhs, rhs});
   llvm::Type* int_type =
       IntegerTypeForFloatSize(lhs->getType() == vector_type());
-  return ir_builder()->CreateBitCast(
-      ir_builder()->CreateOr(ir_builder()->CreateBitCast(lhs, int_type, name()),
-                             ir_builder()->CreateBitCast(rhs, int_type, name()),
-                             name()),
+  return b()->CreateBitCast(
+      b()->CreateOr(b()->CreateBitCast(lhs, int_type, name()),
+                    b()->CreateBitCast(rhs, int_type, name()), name()),
       vector_type(), name());
 }
 
 llvm::Value* VectorSupportLibrary::AddInternal(llvm::Value* lhs,
                                                llvm::Value* rhs) {
   if (scalar_type_->isFloatingPointTy()) {
-    return ir_builder()->CreateFAdd(lhs, rhs, name());
+    return b()->CreateFAdd(lhs, rhs, name());
   } else {
-    return ir_builder()->CreateAdd(lhs, rhs, name());
+    return b()->CreateAdd(lhs, rhs, name());
   }
 }
 
 llvm::Value* VectorSupportLibrary::ComputeOffsetPointer(
     llvm::Value* base_pointer, llvm::Value* offset_elements) {
   if (base_pointer->getType() != scalar_pointer_type()) {
-    base_pointer = ir_builder()->CreateBitCast(base_pointer,
-                                               scalar_pointer_type(), name());
+    base_pointer =
+        b()->CreateBitCast(base_pointer, scalar_pointer_type(), name());
   }
-  return ir_builder()->CreateInBoundsGEP(base_pointer, {offset_elements},
-                                         name());
+  return b()->CreateInBoundsGEP(base_pointer, {offset_elements}, name());
 }
 
 llvm::Value* VectorSupportLibrary::LoadVector(llvm::Value* pointer) {
   if (pointer->getType() != vector_pointer_type()) {
-    pointer =
-        ir_builder()->CreateBitCast(pointer, vector_pointer_type(), name());
+    pointer = b()->CreateBitCast(pointer, vector_pointer_type(), name());
   }
-  return ir_builder()->CreateAlignedLoad(
+  return b()->CreateAlignedLoad(
       pointer, ShapeUtil::ByteSizeOfPrimitiveType(primitive_type_), name());
 }
 
 llvm::Value* VectorSupportLibrary::LoadScalar(llvm::Value* pointer) {
   if (pointer->getType() != scalar_pointer_type()) {
-    pointer =
-        ir_builder()->CreateBitCast(pointer, scalar_pointer_type(), name());
+    pointer = b()->CreateBitCast(pointer, scalar_pointer_type(), name());
   }
-  return ir_builder()->CreateAlignedLoad(
+  return b()->CreateAlignedLoad(
       pointer, ShapeUtil::ByteSizeOfPrimitiveType(primitive_type_), name());
 }
 
@@ -233,30 +226,28 @@ void VectorSupportLibrary::StoreVector(llvm::Value* value,
                                        llvm::Value* pointer) {
   AssertCorrectTypes({value});
   if (pointer->getType() != vector_pointer_type()) {
-    pointer = ir_builder()->CreateBitCast(pointer, vector_pointer_type());
+    pointer = b()->CreateBitCast(pointer, vector_pointer_type());
   }
-  ir_builder()->CreateAlignedStore(
-      value, pointer, ShapeUtil::ByteSizeOfPrimitiveType(primitive_type_));
+  b()->CreateAlignedStore(value, pointer,
+                          ShapeUtil::ByteSizeOfPrimitiveType(primitive_type_));
 }
 
 void VectorSupportLibrary::StoreScalar(llvm::Value* value,
                                        llvm::Value* pointer) {
   AssertCorrectTypes({value});
   if (pointer->getType() != scalar_pointer_type()) {
-    pointer =
-        ir_builder()->CreateBitCast(pointer, scalar_pointer_type(), name());
+    pointer = b()->CreateBitCast(pointer, scalar_pointer_type(), name());
   }
-  ir_builder()->CreateAlignedStore(
-      value, pointer, ShapeUtil::ByteSizeOfPrimitiveType(primitive_type_));
+  b()->CreateAlignedStore(value, pointer,
+                          ShapeUtil::ByteSizeOfPrimitiveType(primitive_type_));
 }
 
 llvm::Value* VectorSupportLibrary::LoadBroadcast(llvm::Value* pointer) {
   if (pointer->getType() != scalar_pointer_type()) {
-    pointer =
-        ir_builder()->CreateBitCast(pointer, scalar_pointer_type(), name());
+    pointer = b()->CreateBitCast(pointer, scalar_pointer_type(), name());
   }
-  return ir_builder()->CreateVectorSplat(
-      vector_size(), ir_builder()->CreateLoad(pointer), name());
+  return b()->CreateVectorSplat(vector_size(), b()->CreateLoad(pointer),
+                                name());
 }
 
 llvm::Value* VectorSupportLibrary::AddReduce(llvm::Value* vector) {
@@ -267,20 +258,19 @@ llvm::Value* VectorSupportLibrary::AddReduce(llvm::Value* vector) {
 
     for (unsigned j = 0; j < vector_size(); ++j) {
       if (j < (i / 2)) {
-        mask[j] = ir_builder()->getInt32(i / 2 + j);
+        mask[j] = b()->getInt32(i / 2 + j);
       } else {
-        mask[j] = llvm::UndefValue::get(ir_builder()->getInt32Ty());
+        mask[j] = llvm::UndefValue::get(b()->getInt32Ty());
       }
     }
 
-    llvm::Value* half_remaining_lanes = ir_builder()->CreateShuffleVector(
-        vector, llvm::UndefValue::get(vector_type()),
-        llvm::ConstantVector::get(mask), "");
+    llvm::Value* half_remaining_lanes =
+        b()->CreateShuffleVector(vector, llvm::UndefValue::get(vector_type()),
+                                 llvm::ConstantVector::get(mask), "");
     vector = Add(vector, half_remaining_lanes);
   }
 
-  return ir_builder()->CreateExtractElement(vector, ir_builder()->getInt32(0),
-                                            name());
+  return b()->CreateExtractElement(vector, b()->getInt32(0), name());
 }
 
 llvm::Value* VectorSupportLibrary::AvxStyleHorizontalAdd(llvm::Value* lhs,
@@ -307,19 +297,19 @@ llvm::Value* VectorSupportLibrary::AvxStyleHorizontalAdd(llvm::Value* lhs,
   // vector, which are the lanes 2 and 3 in the rhs vector.
   for (int i = 0; i < vector_size(); i += 2) {
     int increment = i < vector_size() / 2 ? 0 : (vector_size() / 2);
-    mask_a.push_back(ir_builder()->getInt32(increment + i));
-    mask_b.push_back(ir_builder()->getInt32(increment + i + 1));
+    mask_a.push_back(b()->getInt32(increment + i));
+    mask_b.push_back(b()->getInt32(increment + i + 1));
   }
   for (int i = 0; i < vector_size(); i += 2) {
     int increment = i < vector_size() / 2 ? (vector_size() / 2) : vector_size();
-    mask_a.push_back(ir_builder()->getInt32(increment + i));
-    mask_b.push_back(ir_builder()->getInt32(increment + i + 1));
+    mask_a.push_back(b()->getInt32(increment + i));
+    mask_b.push_back(b()->getInt32(increment + i + 1));
   }
 
-  llvm::Value* shuffle_0 = ir_builder()->CreateShuffleVector(
-      lhs, rhs, llvm::ConstantVector::get(mask_a));
-  llvm::Value* shuffle_1 = ir_builder()->CreateShuffleVector(
-      lhs, rhs, llvm::ConstantVector::get(mask_b));
+  llvm::Value* shuffle_0 =
+      b()->CreateShuffleVector(lhs, rhs, llvm::ConstantVector::get(mask_a));
+  llvm::Value* shuffle_1 =
+      b()->CreateShuffleVector(lhs, rhs, llvm::ConstantVector::get(mask_b));
 
   return Add(shuffle_0, shuffle_1);
 }
@@ -327,23 +317,21 @@ llvm::Value* VectorSupportLibrary::AvxStyleHorizontalAdd(llvm::Value* lhs,
 llvm::Value* VectorSupportLibrary::ExtractLowHalf(llvm::Value* vector) {
   llvm::SmallVector mask;
   for (int i = 0; i < vector_size() / 2; i++) {
-    mask.push_back(ir_builder()->getInt32(i));
+    mask.push_back(b()->getInt32(i));
   }
 
-  return ir_builder()->CreateShuffleVector(vector,
-                                           llvm::UndefValue::get(vector_type()),
-                                           llvm::ConstantVector::get(mask));
+  return b()->CreateShuffleVector(vector, llvm::UndefValue::get(vector_type()),
+                                  llvm::ConstantVector::get(mask));
 }
 
 llvm::Value* VectorSupportLibrary::ExtractHighHalf(llvm::Value* vector) {
   llvm::SmallVector mask;
   for (int i = 0; i < vector_size() / 2; i++) {
-    mask.push_back(ir_builder()->getInt32(i + vector_size() / 2));
+    mask.push_back(b()->getInt32(i + vector_size() / 2));
   }
 
-  return ir_builder()->CreateShuffleVector(vector,
-                                           llvm::UndefValue::get(vector_type()),
-                                           llvm::ConstantVector::get(mask));
+  return b()->CreateShuffleVector(vector, llvm::UndefValue::get(vector_type()),
+                                  llvm::ConstantVector::get(mask));
 }
 
 std::vector VectorSupportLibrary::ComputeHorizontalSums(
@@ -360,8 +348,8 @@ std::vector VectorSupportLibrary::ComputeHorizontalSums(
                  [this](llvm::Value* vector) { return AddReduce(vector); });
   if (init_values) {
     for (int64 i = 0, e = result.size(); i < e; i++) {
-      result[i] = Add(result[i], ir_builder()->CreateExtractElement(
-                                     init_values, ir_builder()->getInt32(i)));
+      result[i] = Add(result[i],
+                      b()->CreateExtractElement(init_values, b()->getInt32(i)));
     }
   }
   return result;
@@ -398,9 +386,9 @@ VectorSupportLibrary::ComputeAvxOptimizedHorizontalSums(
 
   std::vector results;
   for (int i = 0; i < lane_width; i++) {
-    llvm::Value* scalar_result = ir_builder()->CreateExtractElement(
-        i < (lane_width / 2) ? low : high,
-        ir_builder()->getInt32(i % (lane_width / 2)), name());
+    llvm::Value* scalar_result =
+        b()->CreateExtractElement(i < (lane_width / 2) ? low : high,
+                                  b()->getInt32(i % (lane_width / 2)), name());
     results.push_back(scalar_result);
   }
 
@@ -415,17 +403,14 @@ llvm::Value* VectorSupportLibrary::GetZeroScalar() {
   return llvm::Constant::getNullValue(scalar_type());
 }
 
-LlvmVariable::LlvmVariable(llvm::Type* type, llvm::IRBuilder<>* ir_builder)
-    : ir_builder_(ir_builder) {
-  alloca_ = llvm_ir::EmitAllocaAtFunctionEntry(type, "", ir_builder_);
+LlvmVariable::LlvmVariable(llvm::Type* type, llvm::IRBuilder<>* b) : b_(b) {
+  alloca_ = llvm_ir::EmitAllocaAtFunctionEntry(type, "", b_);
 }
 
-llvm::Value* LlvmVariable::Get() const {
-  return ir_builder_->CreateLoad(alloca_);
-}
+llvm::Value* LlvmVariable::Get() const { return b_->CreateLoad(alloca_); }
 
 void LlvmVariable::Set(llvm::Value* new_value) {
-  ir_builder_->CreateStore(new_value, alloca_);
+  b_->CreateStore(new_value, alloca_);
 }
 
 TileVariable::TileVariable(VectorSupportLibrary* vector_support,
diff --git a/tensorflow/compiler/xla/service/cpu/vector_support_library.h b/tensorflow/compiler/xla/service/cpu/vector_support_library.h
index 49c2a4e2f4..c728f6df0a 100644
--- a/tensorflow/compiler/xla/service/cpu/vector_support_library.h
+++ b/tensorflow/compiler/xla/service/cpu/vector_support_library.h
@@ -46,11 +46,11 @@ class VectorSupportLibrary {
   // instance (i.e. LoadVector will load a vector of type <`vector_size` x
   // `primitive_type`>).
   VectorSupportLibrary(PrimitiveType primitive_type, int64 vector_size,
-                       llvm::IRBuilder<>* ir_builder, std::string name);
+                       llvm::IRBuilder<>* b, std::string name);
 
   llvm::Value* Mul(llvm::Value* lhs, llvm::Value* rhs);
   llvm::Value* Mul(int64 lhs, llvm::Value* rhs) {
-    return Mul(ir_builder()->getInt64(lhs), rhs);
+    return Mul(b()->getInt64(lhs), rhs);
   }
   llvm::Value* Mul(const llvm::APFloat& lhs, llvm::Value* rhs) {
     return Mul(GetConstantFloat(rhs->getType(), lhs), rhs);
@@ -63,7 +63,7 @@ class VectorSupportLibrary {
 
   llvm::Value* Add(llvm::Value* lhs, llvm::Value* rhs);
   llvm::Value* Add(int64 lhs, llvm::Value* rhs) {
-    return Add(ir_builder()->getInt64(lhs), rhs);
+    return Add(b()->getInt64(lhs), rhs);
   }
   llvm::Value* Add(const llvm::APFloat& lhs, llvm::Value* rhs) {
     return Add(GetConstantFloat(rhs->getType(), lhs), rhs);
@@ -147,13 +147,11 @@ class VectorSupportLibrary {
   llvm::Value* ComputeOffsetPointer(llvm::Value* base_pointer,
                                     llvm::Value* offset_elements, int64 scale) {
     return ComputeOffsetPointer(
-        base_pointer,
-        ir_builder_->CreateMul(ir_builder_->getInt64(scale), offset_elements));
+        base_pointer, b_->CreateMul(b_->getInt64(scale), offset_elements));
   }
   llvm::Value* ComputeOffsetPointer(llvm::Value* base_pointer,
                                     int64 offset_elements) {
-    return ComputeOffsetPointer(base_pointer,
-                                ir_builder()->getInt64(offset_elements));
+    return ComputeOffsetPointer(base_pointer, b()->getInt64(offset_elements));
   }
 
   llvm::Value* LoadVector(llvm::Value* pointer);
@@ -164,7 +162,7 @@ class VectorSupportLibrary {
   }
 
   llvm::Value* LoadVector(llvm::Value* base_pointer, int64 offset_elements) {
-    return LoadVector(base_pointer, ir_builder()->getInt64(offset_elements));
+    return LoadVector(base_pointer, b()->getInt64(offset_elements));
   }
 
   llvm::Value* LoadScalar(llvm::Value* pointer);
@@ -175,7 +173,7 @@ class VectorSupportLibrary {
   }
 
   llvm::Value* LoadScalar(llvm::Value* base_pointer, int64 offset_elements) {
-    return LoadScalar(base_pointer, ir_builder()->getInt64(offset_elements));
+    return LoadScalar(base_pointer, b()->getInt64(offset_elements));
   }
 
   void StoreVector(llvm::Value* value, llvm::Value* pointer);
@@ -187,7 +185,7 @@ class VectorSupportLibrary {
 
   void StoreVector(llvm::Value* value, llvm::Value* base_pointer,
                    int64 offset_elements) {
-    StoreVector(value, base_pointer, ir_builder()->getInt64(offset_elements));
+    StoreVector(value, base_pointer, b()->getInt64(offset_elements));
   }
 
   void StoreScalar(llvm::Value* value, llvm::Value* pointer);
@@ -198,7 +196,7 @@ class VectorSupportLibrary {
 
   void StoreScalar(llvm::Value* value, llvm::Value* base_pointer,
                    int64 offset_elements) {
-    StoreScalar(base_pointer, ir_builder()->getInt64(offset_elements));
+    StoreScalar(base_pointer, b()->getInt64(offset_elements));
   }
 
   llvm::Value* LoadBroadcast(llvm::Value* pointer);
@@ -207,7 +205,7 @@ class VectorSupportLibrary {
     return LoadBroadcast(ComputeOffsetPointer(base_pointer, offset_elements));
   }
   llvm::Value* LoadBroadcast(llvm::Value* base_pointer, int64 offset_elements) {
-    return LoadBroadcast(base_pointer, ir_builder()->getInt64(offset_elements));
+    return LoadBroadcast(base_pointer, b()->getInt64(offset_elements));
   }
 
   // Compute the horizontal sum of each vector in `vectors`.  The i'th element
@@ -220,7 +218,7 @@ class VectorSupportLibrary {
   llvm::Value* GetZeroVector();
   llvm::Value* GetZeroScalar();
 
-  llvm::IRBuilder<>* ir_builder() const { return ir_builder_; }
+  llvm::IRBuilder<>* b() const { return b_; }
   int64 vector_size() const { return vector_size_; }
   llvm::Type* vector_type() const { return vector_type_; }
   llvm::Type* vector_pointer_type() const { return vector_pointer_type_; }
@@ -277,7 +275,7 @@ class VectorSupportLibrary {
 
   int64 vector_size_;
   PrimitiveType primitive_type_;
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
   llvm::Type* vector_type_;
   llvm::Type* vector_pointer_type_;
   llvm::Type* scalar_type_;
@@ -289,22 +287,21 @@ class VectorSupportLibrary {
 // can later convert to a SSA value.
 class LlvmVariable {
  public:
-  LlvmVariable(llvm::Type*, llvm::IRBuilder<>* ir_builder);
+  LlvmVariable(llvm::Type*, llvm::IRBuilder<>* b);
 
   llvm::Value* Get() const;
   void Set(llvm::Value* new_value);
 
  private:
   llvm::AllocaInst* alloca_;
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
 };
 
 class VectorVariable : public LlvmVariable {
  public:
   VectorVariable(VectorSupportLibrary* vector_support,
                  llvm::Value* initial_value)
-      : LlvmVariable(vector_support->vector_type(),
-                     vector_support->ir_builder()) {
+      : LlvmVariable(vector_support->vector_type(), vector_support->b()) {
     Set(initial_value);
   }
 };
@@ -313,8 +310,7 @@ class ScalarVariable : public LlvmVariable {
  public:
   ScalarVariable(VectorSupportLibrary* vector_support,
                  llvm::Value* initial_value)
-      : LlvmVariable(vector_support->scalar_type(),
-                     vector_support->ir_builder()) {
+      : LlvmVariable(vector_support->scalar_type(), vector_support->b()) {
     Set(initial_value);
   }
 };
diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index c51632597a..1eedd85363 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -61,13 +61,13 @@ int64 GlobalRandomValue() {
 
 llvm::Value* EmitReducePrecisionFloat(llvm::Value* x, int64 exponent_bits,
                                       int64 mantissa_bits,
-                                      llvm::IRBuilder<>* ir_builder) {
+                                      llvm::IRBuilder<>* b) {
   // Integer and float types for casting and constant generation.
   llvm::Type* float_type = x->getType();
-  llvm::IntegerType* int_type = ir_builder->getInt32Ty();
+  llvm::IntegerType* int_type = b->getInt32Ty();
 
   // Cast the input value to an integer for bitwise manipulation.
-  llvm::Value* x_as_int = ir_builder->CreateBitCast(x, int_type);
+  llvm::Value* x_as_int = b->CreateBitCast(x, int_type);
 
   if (mantissa_bits < 23) {
     // Last remaining mantissa bit.
@@ -77,22 +77,22 @@ llvm::Value* EmitReducePrecisionFloat(llvm::Value* x, int64 exponent_bits,
     // equal to a base value of 0111... plus one bit if the last remaining
     // mantissa bit is 1.
     const uint32_t base_rounding_bias = (last_mantissa_bit_mask >> 1) - 1;
-    llvm::Value* x_last_mantissa_bit = ir_builder->CreateLShr(
-        ir_builder->CreateAnd(
-            x_as_int, llvm::ConstantInt::get(int_type, last_mantissa_bit_mask)),
+    llvm::Value* x_last_mantissa_bit = b->CreateLShr(
+        b->CreateAnd(x_as_int,
+                     llvm::ConstantInt::get(int_type, last_mantissa_bit_mask)),
         (23 - mantissa_bits));
-    llvm::Value* x_rounding_bias = ir_builder->CreateAdd(
-        x_last_mantissa_bit,
-        llvm::ConstantInt::get(int_type, base_rounding_bias));
+    llvm::Value* x_rounding_bias =
+        b->CreateAdd(x_last_mantissa_bit,
+                     llvm::ConstantInt::get(int_type, base_rounding_bias));
 
     // Add rounding bias, and mask out truncated bits.  Note that the case
     // where adding the rounding bias overflows into the exponent bits is
     // correct; the non-masked mantissa bits will all be zero, and the
     // exponent will be incremented by one.
     const uint32_t truncation_mask = ~(last_mantissa_bit_mask - 1);
-    x_as_int = ir_builder->CreateAdd(x_as_int, x_rounding_bias);
-    x_as_int = ir_builder->CreateAnd(
-        x_as_int, llvm::ConstantInt::get(int_type, truncation_mask));
+    x_as_int = b->CreateAdd(x_as_int, x_rounding_bias);
+    x_as_int = b->CreateAnd(x_as_int,
+                            llvm::ConstantInt::get(int_type, truncation_mask));
   }
 
   if (exponent_bits < 8) {
@@ -120,29 +120,29 @@ llvm::Value* EmitReducePrecisionFloat(llvm::Value* x, int64 exponent_bits,
         f32_exponent_bias - reduced_exponent_bias;
 
     // Do we overflow or underflow?
-    llvm::Value* x_exponent = ir_builder->CreateAnd(
+    llvm::Value* x_exponent = b->CreateAnd(
         x_as_int, llvm::ConstantInt::get(int_type, f32_exp_bits_mask));
-    llvm::Value* x_overflows = ir_builder->CreateICmpUGT(
+    llvm::Value* x_overflows = b->CreateICmpUGT(
         x_exponent,
         llvm::ConstantInt::get(int_type, reduced_max_exponent << 23));
-    llvm::Value* x_underflows = ir_builder->CreateICmpULE(
+    llvm::Value* x_underflows = b->CreateICmpULE(
         x_exponent,
         llvm::ConstantInt::get(int_type, reduced_min_exponent << 23));
 
     // Compute appropriately-signed values of zero and infinity.
-    llvm::Value* x_signed_zero = ir_builder->CreateAnd(
+    llvm::Value* x_signed_zero = b->CreateAnd(
         x_as_int, llvm::ConstantInt::get(int_type, f32_sign_bit_mask));
-    llvm::Value* x_signed_inf = ir_builder->CreateOr(
+    llvm::Value* x_signed_inf = b->CreateOr(
         x_signed_zero, llvm::ConstantInt::get(int_type, f32_exp_bits_mask));
 
     // Force to zero or infinity if overflow or underflow.  (Note that this
     // truncates all denormal values to zero, rather than rounding them.)
-    x_as_int = ir_builder->CreateSelect(x_overflows, x_signed_inf, x_as_int);
-    x_as_int = ir_builder->CreateSelect(x_underflows, x_signed_zero, x_as_int);
+    x_as_int = b->CreateSelect(x_overflows, x_signed_inf, x_as_int);
+    x_as_int = b->CreateSelect(x_underflows, x_signed_zero, x_as_int);
   }
 
   // Cast the result back to a floating-point type.
-  llvm::Value* result = ir_builder->CreateBitCast(x_as_int, float_type);
+  llvm::Value* result = b->CreateBitCast(x_as_int, float_type);
 
   // Correct result for NaN inputs.
   //
@@ -154,53 +154,49 @@ llvm::Value* EmitReducePrecisionFloat(llvm::Value* x, int64 exponent_bits,
   //
   // If the fast-math flags are set to assume no NaNs, the comparison is likely
   // to be optimized away, so there's no point in even emitting it.
-  if (!ir_builder->getFastMathFlags().noNaNs()) {
-    llvm::Value* x_is_nan = ir_builder->CreateFCmpUNO(x, x);
+  if (!b->getFastMathFlags().noNaNs()) {
+    llvm::Value* x_is_nan = b->CreateFCmpUNO(x, x);
 
     if (mantissa_bits > 0) {
-      result = ir_builder->CreateSelect(x_is_nan, x, result);
+      result = b->CreateSelect(x_is_nan, x, result);
     } else {
-      result = ir_builder->CreateSelect(
+      result = b->CreateSelect(
           x_is_nan, llvm::ConstantFP::getInfinity(float_type), result);
     }
   }
   return result;
 }
 
-llvm::Value* EmitF32ToBF16(llvm::Value* f32_value,
-                           llvm::IRBuilder<>* ir_builder) {
+llvm::Value* EmitF32ToBF16(llvm::Value* f32_value, llvm::IRBuilder<>* b) {
   auto reduced_precision = EmitReducePrecisionFloat(
       f32_value,
       /*exponent_bits=*/primitive_util::kBFloat16ExponentBits,
-      /*mantissa_bits=*/primitive_util::kBFloat16MantissaBits, ir_builder);
-  auto as_int32 =
-      ir_builder->CreateBitCast(reduced_precision, ir_builder->getInt32Ty());
-  auto shifted = ir_builder->CreateLShr(as_int32, 16);
-  auto truncated = ir_builder->CreateTrunc(shifted, ir_builder->getInt16Ty());
-  return ir_builder->CreateBitCast(truncated, ir_builder->getInt16Ty());
+      /*mantissa_bits=*/primitive_util::kBFloat16MantissaBits, b);
+  auto as_int32 = b->CreateBitCast(reduced_precision, b->getInt32Ty());
+  auto shifted = b->CreateLShr(as_int32, 16);
+  auto truncated = b->CreateTrunc(shifted, b->getInt16Ty());
+  return b->CreateBitCast(truncated, b->getInt16Ty());
 }
 
-llvm::Value* EmitBF16ToF32(llvm::Value* bf16_value,
-                           llvm::IRBuilder<>* ir_builder) {
-  auto as_int16 =
-      ir_builder->CreateBitCast(bf16_value, ir_builder->getInt16Ty());
-  auto as_int32 = ir_builder->CreateZExt(as_int16, ir_builder->getInt32Ty());
-  auto shifted = ir_builder->CreateShl(as_int32, 16);
-  return ir_builder->CreateBitCast(shifted, ir_builder->getFloatTy());
+llvm::Value* EmitBF16ToF32(llvm::Value* bf16_value, llvm::IRBuilder<>* b) {
+  auto as_int16 = b->CreateBitCast(bf16_value, b->getInt16Ty());
+  auto as_int32 = b->CreateZExt(as_int16, b->getInt32Ty());
+  auto shifted = b->CreateShl(as_int32, 16);
+  return b->CreateBitCast(shifted, b->getFloatTy());
 }
 
 llvm::Value* EmitIntegralToFloating(llvm::Value* integer_value,
                                     PrimitiveType from_type,
                                     PrimitiveType to_type, llvm::Module* module,
-                                    llvm::IRBuilder<>* ir_builder) {
+                                    llvm::IRBuilder<>* b) {
   if (primitive_util::IsSignedIntegralType(from_type)) {
-    return ir_builder->CreateSIToFP(
-        integer_value, llvm_ir::PrimitiveTypeToIrType(to_type, module));
+    return b->CreateSIToFP(integer_value,
+                           llvm_ir::PrimitiveTypeToIrType(to_type, module));
   } else {
     CHECK(primitive_util::IsUnsignedIntegralType(from_type) ||
           from_type == PRED);
-    return ir_builder->CreateUIToFP(
-        integer_value, llvm_ir::PrimitiveTypeToIrType(to_type, module));
+    return b->CreateUIToFP(integer_value,
+                           llvm_ir::PrimitiveTypeToIrType(to_type, module));
   }
 }
 
@@ -231,34 +227,31 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp(
         return operand_value;
       }
       if (primitive_util::IsIntegralType(to_type)) {
-        return ir_builder_->CreateIntCast(
+        return b_->CreateIntCast(
             operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_),
             primitive_util::IsSignedIntegralType(from_type));
       }
       if (primitive_util::IsFloatingPointType(to_type)) {
         if (to_type == BF16) {
-          return EmitF32ToBF16(
-              EmitIntegralToFloating(operand_value, from_type, F32, module_,
-                                     ir_builder_),
-              ir_builder_);
+          return EmitF32ToBF16(EmitIntegralToFloating(operand_value, from_type,
+                                                      F32, module_, b_),
+                               b_);
         }
         return EmitIntegralToFloating(operand_value, from_type, to_type,
-                                      module_, ir_builder_);
+                                      module_, b_);
       }
       if (primitive_util::IsComplexType(to_type)) {
         auto to_ir_component_type = llvm_ir::PrimitiveTypeToIrType(
             primitive_util::ComplexComponentType(to_type), module_);
         if (primitive_util::IsSignedIntegralType(from_type)) {
           return EmitComposeComplex(
-              op,
-              ir_builder_->CreateSIToFP(operand_value, to_ir_component_type),
+              op, b_->CreateSIToFP(operand_value, to_ir_component_type),
               nullptr);
         }
         if (primitive_util::IsUnsignedIntegralType(from_type) ||
             from_type == PRED) {
           return EmitComposeComplex(
-              op,
-              ir_builder_->CreateUIToFP(operand_value, to_ir_component_type),
+              op, b_->CreateUIToFP(operand_value, to_ir_component_type),
               nullptr);
         }
       }
@@ -275,7 +268,7 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp(
       }
       if (primitive_util::BitWidth(from_type) ==
           primitive_util::BitWidth(to_type)) {
-        return ir_builder_->CreateBitCast(
+        return b_->CreateBitCast(
             operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
       }
       return InvalidArgument(
@@ -293,18 +286,18 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp(
         auto type =
             llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_);
         auto zero = llvm::ConstantInt::get(type, 0);
-        auto cmp = ir_builder_->CreateICmpSGE(operand_value, zero);
-        return ir_builder_->CreateSelect(cmp, operand_value,
-                                         ir_builder_->CreateNeg(operand_value));
+        auto cmp = b_->CreateICmpSGE(operand_value, zero);
+        return b_->CreateSelect(cmp, operand_value,
+                                b_->CreateNeg(operand_value));
       } else {
         return operand_value;
       }
     }
     case HloOpcode::kClz: {
-      auto is_zero_undef = ir_builder_->getFalse();
-      return llvm_ir::EmitCallToIntrinsic(
-          llvm::Intrinsic::ctlz, {operand_value, is_zero_undef},
-          {operand_value->getType()}, ir_builder_);
+      auto is_zero_undef = b_->getFalse();
+      return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::ctlz,
+                                          {operand_value, is_zero_undef},
+                                          {operand_value->getType()}, b_);
     }
     case HloOpcode::kSign: {
       bool is_signed =
@@ -312,31 +305,28 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp(
       auto type =
           llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_);
       auto zero = llvm::ConstantInt::get(type, 0);
-      auto cmp = ir_builder_->CreateICmpEQ(operand_value, zero);
+      auto cmp = b_->CreateICmpEQ(operand_value, zero);
       if (is_signed) {
-        auto ashr = ir_builder_->CreateAShr(operand_value,
-                                            type->getIntegerBitWidth() - 1);
-        return ir_builder_->CreateSelect(cmp, zero,
-                                         ir_builder_->CreateOr(ashr, 1));
+        auto ashr =
+            b_->CreateAShr(operand_value, type->getIntegerBitWidth() - 1);
+        return b_->CreateSelect(cmp, zero, b_->CreateOr(ashr, 1));
       } else {
-        return ir_builder_->CreateSelect(cmp, zero,
-                                         llvm::ConstantInt::get(type, 1));
+        return b_->CreateSelect(cmp, zero, llvm::ConstantInt::get(type, 1));
       }
     }
     case HloOpcode::kNegate:
-      return ir_builder_->CreateNeg(operand_value);
+      return b_->CreateNeg(operand_value);
     case HloOpcode::kNot: {
       auto type = op->shape().element_type();
       if (type == PRED) {
         // It is not sufficient to just call CreateNot() here because a PRED
         // is represented as an i8 and the truth value is stored only in the
         // bottom bit.
-        return ir_builder_->CreateZExt(
-            ir_builder_->CreateNot(ir_builder_->CreateTrunc(
-                operand_value, ir_builder_->getInt1Ty())),
+        return b_->CreateZExt(
+            b_->CreateNot(b_->CreateTrunc(operand_value, b_->getInt1Ty())),
             llvm_ir::PrimitiveTypeToIrType(PRED, module_));
       } else if (primitive_util::IsIntegralType(type)) {
-        return ir_builder_->CreateNot(operand_value);
+        return b_->CreateNot(operand_value);
       }
       return Unimplemented("unary op Not is not defined for type '%d'", type);
     }
@@ -364,32 +354,31 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp(
         }
         return EmitComposeComplex(
             op,
-            ir_builder_->CreateFPCast(
-                operand_value,
-                llvm_ir::PrimitiveTypeToIrType(to_component_type, module_)),
+            b_->CreateFPCast(operand_value, llvm_ir::PrimitiveTypeToIrType(
+                                                to_component_type, module_)),
             nullptr);
       }
       if (from_type == BF16) {
         TF_RET_CHECK(to_type != BF16);
-        operand_value = EmitBF16ToF32(operand_value, ir_builder_);
+        operand_value = EmitBF16ToF32(operand_value, b_);
         from_type = F32;
         if (from_type == to_type) {
           return operand_value;
         }
       }
       if (from_type == F32 && to_type == BF16) {
-        return EmitF32ToBF16(operand_value, ir_builder_);
+        return EmitF32ToBF16(operand_value, b_);
       }
       if (primitive_util::IsFloatingPointType(to_type)) {
-        return ir_builder_->CreateFPCast(
+        return b_->CreateFPCast(
             operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
       }
       if (primitive_util::IsSignedIntegralType(to_type)) {
-        return ir_builder_->CreateFPToSI(
+        return b_->CreateFPToSI(
             operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
       }
       if (primitive_util::IsUnsignedIntegralType(to_type)) {
-        return ir_builder_->CreateFPToUI(
+        return b_->CreateFPToUI(
             operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
       }
       return Unimplemented("unhandled conversion operation: %s => %s",
@@ -405,7 +394,7 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp(
       }
       if (primitive_util::BitWidth(from_type) ==
           primitive_util::BitWidth(to_type)) {
-        return ir_builder_->CreateBitCast(
+        return b_->CreateBitCast(
             operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
       }
       return InvalidArgument(
@@ -429,45 +418,45 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp(
     case HloOpcode::kSin:
       return EmitSin(op->shape().element_type(), operand_value);
     case HloOpcode::kFloor:
-      return llvm_ir::EmitCallToIntrinsic(
-          llvm::Intrinsic::floor, {operand_value}, {operand_value->getType()},
-          ir_builder_);
+      return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::floor,
+                                          {operand_value},
+                                          {operand_value->getType()}, b_);
     case HloOpcode::kCeil:
-      return llvm_ir::EmitCallToIntrinsic(
-          llvm::Intrinsic::ceil, {operand_value}, {operand_value->getType()},
-          ir_builder_);
+      return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::ceil,
+                                          {operand_value},
+                                          {operand_value->getType()}, b_);
     case HloOpcode::kAbs:
-      return llvm_ir::EmitCallToIntrinsic(
-          llvm::Intrinsic::fabs, {operand_value}, {operand_value->getType()},
-          ir_builder_);
+      return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::fabs,
+                                          {operand_value},
+                                          {operand_value->getType()}, b_);
     case HloOpcode::kRoundNearestAfz:
-      return llvm_ir::EmitCallToIntrinsic(
-          llvm::Intrinsic::round, {operand_value}, {operand_value->getType()},
-          ir_builder_);
+      return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::round,
+                                          {operand_value},
+                                          {operand_value->getType()}, b_);
     case HloOpcode::kSign: {
       // TODO(b/32151903): Ensure consistent sign behavior for -0.0.
       auto type = operand_value->getType();
       auto zero = llvm::ConstantFP::get(type, 0.0);
-      auto oeq = ir_builder_->CreateFCmpOEQ(operand_value, zero);
-      auto olt = ir_builder_->CreateFCmpOLT(operand_value, zero);
-      return ir_builder_->CreateSelect(
+      auto oeq = b_->CreateFCmpOEQ(operand_value, zero);
+      auto olt = b_->CreateFCmpOLT(operand_value, zero);
+      return b_->CreateSelect(
           oeq, zero,
-          ir_builder_->CreateSelect(olt, llvm::ConstantFP::get(type, -1.0),
-                                    llvm::ConstantFP::get(type, 1.0)));
+          b_->CreateSelect(olt, llvm::ConstantFP::get(type, -1.0),
+                           llvm::ConstantFP::get(type, 1.0)));
     }
     case HloOpcode::kIsFinite: {
       // abs(x) o!= inf, this works because the comparison returns false if
       // either operand is NaN.
       auto type = operand_value->getType();
       auto abs_value = llvm_ir::EmitCallToIntrinsic(
-          llvm::Intrinsic::fabs, {operand_value}, {type}, ir_builder_);
+          llvm::Intrinsic::fabs, {operand_value}, {type}, b_);
       auto infinity = llvm::ConstantFP::getInfinity(type);
-      auto not_infinite = ir_builder_->CreateFCmpONE(abs_value, infinity);
-      return ir_builder_->CreateZExt(
-          not_infinite, llvm_ir::PrimitiveTypeToIrType(PRED, module_));
+      auto not_infinite = b_->CreateFCmpONE(abs_value, infinity);
+      return b_->CreateZExt(not_infinite,
+                            llvm_ir::PrimitiveTypeToIrType(PRED, module_));
     }
     case HloOpcode::kNegate:
-      return ir_builder_->CreateFNeg(operand_value);
+      return b_->CreateFNeg(operand_value);
     case HloOpcode::kReal:
       return operand_value;
     case HloOpcode::kImag:
@@ -491,13 +480,12 @@ StatusOr ElementalIrEmitter::EmitComplexUnaryOp(
       auto a = EmitExtractReal(operand_value);
       auto b = EmitExtractImag(operand_value);
       llvm::Type* llvm_ty = a->getType();
-      auto sum_sq = ir_builder_->CreateFAdd(ir_builder_->CreateFMul(a, a),
-                                            ir_builder_->CreateFMul(b, b));
+      auto sum_sq = b_->CreateFAdd(b_->CreateFMul(a, a), b_->CreateFMul(b, b));
       TF_ASSIGN_OR_RETURN(auto log_sum_sq, EmitLog(component_type, sum_sq));
       TF_ASSIGN_OR_RETURN(auto angle, EmitAtan2(component_type, b, a));
       auto one_half = llvm::ConstantFP::get(llvm_ty, 0.5);
-      return EmitComposeComplex(
-          op, ir_builder_->CreateFMul(one_half, log_sum_sq), angle);
+      return EmitComposeComplex(op, b_->CreateFMul(one_half, log_sum_sq),
+                                angle);
     }
     case HloOpcode::kLog1p: {
       // log1p(a+bi) = .5*log((a+1)^2+b^2) + i*atan2(b, a + 1)
@@ -505,15 +493,14 @@ StatusOr ElementalIrEmitter::EmitComplexUnaryOp(
       auto b = EmitExtractImag(operand_value);
       llvm::Type* llvm_ty = a->getType();
       auto one = llvm::ConstantFP::get(llvm_ty, 1.0);
-      auto a_plus_one = ir_builder_->CreateFAdd(a, one);
-      auto sum_sq = ir_builder_->CreateFAdd(
-          ir_builder_->CreateFMul(a_plus_one, a_plus_one),
-          ir_builder_->CreateFMul(b, b));
+      auto a_plus_one = b_->CreateFAdd(a, one);
+      auto sum_sq = b_->CreateFAdd(b_->CreateFMul(a_plus_one, a_plus_one),
+                                   b_->CreateFMul(b, b));
       TF_ASSIGN_OR_RETURN(auto log_sum_sq, EmitLog(component_type, sum_sq));
       TF_ASSIGN_OR_RETURN(auto angle, EmitAtan2(component_type, b, a_plus_one));
       auto one_half = llvm::ConstantFP::get(llvm_ty, 0.5);
-      return EmitComposeComplex(
-          op, ir_builder_->CreateFMul(one_half, log_sum_sq), angle);
+      return EmitComposeComplex(op, b_->CreateFMul(one_half, log_sum_sq),
+                                angle);
     }
     case HloOpcode::kConvert: {
       PrimitiveType from_type = op->operand(0)->shape().element_type();
@@ -527,12 +514,11 @@ StatusOr ElementalIrEmitter::EmitComplexUnaryOp(
           primitive_util::ComplexComponentType(to_type);
       auto to_ir_component_type =
           llvm_ir::PrimitiveTypeToIrType(to_component_type, module_);
-      return EmitComposeComplex(
-          op,
-          ir_builder_->CreateFPCast(EmitExtractReal(operand_value),
-                                    to_ir_component_type),
-          ir_builder_->CreateFPCast(EmitExtractImag(operand_value),
-                                    to_ir_component_type));
+      return EmitComposeComplex(op,
+                                b_->CreateFPCast(EmitExtractReal(operand_value),
+                                                 to_ir_component_type),
+                                b_->CreateFPCast(EmitExtractImag(operand_value),
+                                                 to_ir_component_type));
     }
     case HloOpcode::kExp: {
       // e^(a+bi) = e^a*(cos(b)+sin(b)i)
@@ -542,8 +528,8 @@ StatusOr ElementalIrEmitter::EmitComplexUnaryOp(
           auto cos_b, EmitCos(component_type, EmitExtractImag(operand_value)));
       TF_ASSIGN_OR_RETURN(
           auto sin_b, EmitSin(component_type, EmitExtractImag(operand_value)));
-      return EmitComposeComplex(op, ir_builder_->CreateFMul(exp_a, cos_b),
-                                ir_builder_->CreateFMul(exp_a, sin_b));
+      return EmitComposeComplex(op, b_->CreateFMul(exp_a, cos_b),
+                                b_->CreateFMul(exp_a, sin_b));
     }
     case HloOpcode::kExpm1: {
       // e^(a+bi)-1 = (e^a*cos(b)-1)+e^a*sin(b)i
@@ -554,9 +540,8 @@ StatusOr ElementalIrEmitter::EmitComplexUnaryOp(
       TF_ASSIGN_OR_RETURN(
           auto sin_b, EmitSin(component_type, EmitExtractImag(operand_value)));
       auto one = llvm::ConstantFP::get(exp_a->getType(), 1.0);
-      auto real_result =
-          ir_builder_->CreateFSub(ir_builder_->CreateFMul(exp_a, cos_b), one);
-      auto imag_result = ir_builder_->CreateFMul(exp_a, sin_b);
+      auto real_result = b_->CreateFSub(b_->CreateFMul(exp_a, cos_b), one);
+      auto imag_result = b_->CreateFMul(exp_a, sin_b);
       return EmitComposeComplex(op, real_result, imag_result);
     }
     case HloOpcode::kCos: {
@@ -571,18 +556,14 @@ StatusOr ElementalIrEmitter::EmitComplexUnaryOp(
       auto b = EmitExtractImag(operand_value);
       auto type = a->getType();
       TF_ASSIGN_OR_RETURN(auto exp_b, EmitExp(component_type, b));
-      auto half_exp_b =
-          ir_builder_->CreateFMul(llvm::ConstantFP::get(type, 0.5), exp_b);
+      auto half_exp_b = b_->CreateFMul(llvm::ConstantFP::get(type, 0.5), exp_b);
       auto half_exp_neg_b =
-          ir_builder_->CreateFDiv(llvm::ConstantFP::get(type, 0.5), exp_b);
+          b_->CreateFDiv(llvm::ConstantFP::get(type, 0.5), exp_b);
       TF_ASSIGN_OR_RETURN(auto cos_a, EmitCos(component_type, a));
       TF_ASSIGN_OR_RETURN(auto sin_a, EmitSin(component_type, a));
       return EmitComposeComplex(
-          op,
-          ir_builder_->CreateFMul(
-              cos_a, ir_builder_->CreateFAdd(half_exp_neg_b, half_exp_b)),
-          ir_builder_->CreateFMul(
-              sin_a, ir_builder_->CreateFSub(half_exp_neg_b, half_exp_b)));
+          op, b_->CreateFMul(cos_a, b_->CreateFAdd(half_exp_neg_b, half_exp_b)),
+          b_->CreateFMul(sin_a, b_->CreateFSub(half_exp_neg_b, half_exp_b)));
     }
     case HloOpcode::kSin: {
       // sin(z) = .5i(e^(-iz) - e^(iz))
@@ -598,18 +579,14 @@ StatusOr ElementalIrEmitter::EmitComplexUnaryOp(
       auto b = EmitExtractImag(operand_value);
       auto type = a->getType();
       TF_ASSIGN_OR_RETURN(auto exp_b, EmitExp(component_type, b));
-      auto half_exp_b =
-          ir_builder_->CreateFMul(llvm::ConstantFP::get(type, 0.5), exp_b);
+      auto half_exp_b = b_->CreateFMul(llvm::ConstantFP::get(type, 0.5), exp_b);
       auto half_exp_neg_b =
-          ir_builder_->CreateFDiv(llvm::ConstantFP::get(type, 0.5), exp_b);
+          b_->CreateFDiv(llvm::ConstantFP::get(type, 0.5), exp_b);
       TF_ASSIGN_OR_RETURN(auto cos_a, EmitCos(component_type, a));
       TF_ASSIGN_OR_RETURN(auto sin_a, EmitSin(component_type, a));
       return EmitComposeComplex(
-          op,
-          ir_builder_->CreateFMul(
-              sin_a, ir_builder_->CreateFAdd(half_exp_b, half_exp_neg_b)),
-          ir_builder_->CreateFMul(
-              cos_a, ir_builder_->CreateFSub(half_exp_b, half_exp_neg_b)));
+          op, b_->CreateFMul(sin_a, b_->CreateFAdd(half_exp_b, half_exp_neg_b)),
+          b_->CreateFMul(cos_a, b_->CreateFSub(half_exp_b, half_exp_neg_b)));
     }
     case HloOpcode::kTanh: {
       /*
@@ -637,64 +614,61 @@ StatusOr ElementalIrEmitter::EmitComplexUnaryOp(
       TF_ASSIGN_OR_RETURN(auto exp_a, EmitExp(component_type, a));
       TF_ASSIGN_OR_RETURN(auto cos_b, EmitCos(component_type, b));
       TF_ASSIGN_OR_RETURN(auto sin_b, EmitSin(component_type, b));
-      auto exp_neg_a = ir_builder_->CreateFDiv(
-          llvm::ConstantFP::get(exp_a->getType(), 1), exp_a);
-      auto exp_2a_minus_exp_neg_2a = ir_builder_->CreateFSub(
-          ir_builder_->CreateFMul(exp_a, exp_a),
-          ir_builder_->CreateFMul(exp_neg_a, exp_neg_a));
-      auto cos_b_sq = ir_builder_->CreateFMul(cos_b, cos_b);
-      auto sin_b_sq = ir_builder_->CreateFMul(sin_b, sin_b);
-      auto real_num = ir_builder_->CreateFAdd(
-          ir_builder_->CreateFMul(cos_b_sq, exp_2a_minus_exp_neg_2a),
-          ir_builder_->CreateFMul(sin_b_sq, exp_2a_minus_exp_neg_2a));
-      auto cos_b_sin_b = ir_builder_->CreateFMul(cos_b, sin_b);
-      auto exp_a_plus_exp_neg_a = ir_builder_->CreateFAdd(exp_a, exp_neg_a);
+      auto exp_neg_a =
+          b_->CreateFDiv(llvm::ConstantFP::get(exp_a->getType(), 1), exp_a);
+      auto exp_2a_minus_exp_neg_2a = b_->CreateFSub(
+          b_->CreateFMul(exp_a, exp_a), b_->CreateFMul(exp_neg_a, exp_neg_a));
+      auto cos_b_sq = b_->CreateFMul(cos_b, cos_b);
+      auto sin_b_sq = b_->CreateFMul(sin_b, sin_b);
+      auto real_num =
+          b_->CreateFAdd(b_->CreateFMul(cos_b_sq, exp_2a_minus_exp_neg_2a),
+                         b_->CreateFMul(sin_b_sq, exp_2a_minus_exp_neg_2a));
+      auto cos_b_sin_b = b_->CreateFMul(cos_b, sin_b);
+      auto exp_a_plus_exp_neg_a = b_->CreateFAdd(exp_a, exp_neg_a);
       auto exp_a_plus_exp_neg_a_sq =
-          ir_builder_->CreateFMul(exp_a_plus_exp_neg_a, exp_a_plus_exp_neg_a);
-      auto exp_a_minus_exp_neg_a = ir_builder_->CreateFSub(exp_a, exp_neg_a);
+          b_->CreateFMul(exp_a_plus_exp_neg_a, exp_a_plus_exp_neg_a);
+      auto exp_a_minus_exp_neg_a = b_->CreateFSub(exp_a, exp_neg_a);
       auto exp_a_minus_exp_neg_a_sq =
-          ir_builder_->CreateFMul(exp_a_minus_exp_neg_a, exp_a_minus_exp_neg_a);
-      auto imag_num = ir_builder_->CreateFMul(
-          cos_b_sin_b, ir_builder_->CreateFSub(exp_a_plus_exp_neg_a_sq,
-                                               exp_a_minus_exp_neg_a_sq));
-      auto denom = ir_builder_->CreateFAdd(
-          ir_builder_->CreateFMul(cos_b_sq, exp_a_plus_exp_neg_a_sq),
-          ir_builder_->CreateFMul(sin_b_sq, exp_a_minus_exp_neg_a_sq));
-      return EmitComposeComplex(op, ir_builder_->CreateFDiv(real_num, denom),
-                                ir_builder_->CreateFDiv(imag_num, denom));
+          b_->CreateFMul(exp_a_minus_exp_neg_a, exp_a_minus_exp_neg_a);
+      auto imag_num = b_->CreateFMul(
+          cos_b_sin_b,
+          b_->CreateFSub(exp_a_plus_exp_neg_a_sq, exp_a_minus_exp_neg_a_sq));
+      auto denom =
+          b_->CreateFAdd(b_->CreateFMul(cos_b_sq, exp_a_plus_exp_neg_a_sq),
+                         b_->CreateFMul(sin_b_sq, exp_a_minus_exp_neg_a_sq));
+      return EmitComposeComplex(op, b_->CreateFDiv(real_num, denom),
+                                b_->CreateFDiv(imag_num, denom));
     }
     case HloOpcode::kAbs: {
-      auto sum_sq = ir_builder_->CreateFAdd(
-          ir_builder_->CreateFMul(EmitExtractReal(operand_value),
-                                  EmitExtractReal(operand_value)),
-          ir_builder_->CreateFMul(EmitExtractImag(operand_value),
-                                  EmitExtractImag(operand_value)));
+      auto sum_sq =
+          b_->CreateFAdd(b_->CreateFMul(EmitExtractReal(operand_value),
+                                        EmitExtractReal(operand_value)),
+                         b_->CreateFMul(EmitExtractImag(operand_value),
+                                        EmitExtractImag(operand_value)));
       return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sqrt, {sum_sq},
-                                          {sum_sq->getType()}, ir_builder_);
+                                          {sum_sq->getType()}, b_);
     }
     case HloOpcode::kSign: {  // Sign(c) = c / |c|
-      auto sum_sq = ir_builder_->CreateFAdd(
-          ir_builder_->CreateFMul(EmitExtractReal(operand_value),
-                                  EmitExtractReal(operand_value)),
-          ir_builder_->CreateFMul(EmitExtractImag(operand_value),
-                                  EmitExtractImag(operand_value)));
+      auto sum_sq =
+          b_->CreateFAdd(b_->CreateFMul(EmitExtractReal(operand_value),
+                                        EmitExtractReal(operand_value)),
+                         b_->CreateFMul(EmitExtractImag(operand_value),
+                                        EmitExtractImag(operand_value)));
       auto cplx_abs = llvm_ir::EmitCallToIntrinsic(
-          llvm::Intrinsic::sqrt, {sum_sq}, {sum_sq->getType()}, ir_builder_);
+          llvm::Intrinsic::sqrt, {sum_sq}, {sum_sq->getType()}, b_);
       auto type = cplx_abs->getType();
       auto zero = llvm::ConstantFP::get(type, 0.0);
-      auto oeq = ir_builder_->CreateFCmpOEQ(cplx_abs, zero);
-      return ir_builder_->CreateSelect(
+      auto oeq = b_->CreateFCmpOEQ(cplx_abs, zero);
+      return b_->CreateSelect(
           oeq, EmitComposeComplex(op, zero, zero),
           EmitComposeComplex(
-              op,
-              ir_builder_->CreateFDiv(EmitExtractReal(operand_value), cplx_abs),
-              ir_builder_->CreateFDiv(EmitExtractImag(operand_value),
-                                      cplx_abs)));
+              op, b_->CreateFDiv(EmitExtractReal(operand_value), cplx_abs),
+              b_->CreateFDiv(EmitExtractImag(operand_value), cplx_abs)));
     }
     case HloOpcode::kNegate:
-      return EmitComposeComplex(
-          op, ir_builder_->CreateFNeg(EmitExtractReal(operand_value)),
-          ir_builder_->CreateFNeg(EmitExtractImag(operand_value)));
+      return EmitComposeComplex(op,
+                                b_->CreateFNeg(EmitExtractReal(operand_value)),
+                                b_->CreateFNeg(EmitExtractImag(operand_value)));
     case HloOpcode::kReal:
       return EmitExtractReal(operand_value);
     case HloOpcode::kImag:
@@ -728,15 +702,15 @@ StatusOr ElementalIrEmitter::EmitFloatBinaryOp(
     case HloOpcode::kComplex:
       return EmitComposeComplex(op, lhs_value, rhs_value);
     case HloOpcode::kAdd:
-      return ir_builder_->CreateFAdd(lhs_value, rhs_value);
+      return b_->CreateFAdd(lhs_value, rhs_value);
     case HloOpcode::kSubtract:
-      return ir_builder_->CreateFSub(lhs_value, rhs_value);
+      return b_->CreateFSub(lhs_value, rhs_value);
     case HloOpcode::kMultiply:
-      return ir_builder_->CreateFMul(lhs_value, rhs_value);
+      return b_->CreateFMul(lhs_value, rhs_value);
     case HloOpcode::kDivide:
-      return ir_builder_->CreateFDiv(lhs_value, rhs_value);
+      return b_->CreateFDiv(lhs_value, rhs_value);
     case HloOpcode::kRemainder:
-      return ir_builder_->CreateFRem(lhs_value, rhs_value);
+      return b_->CreateFRem(lhs_value, rhs_value);
     // LLVM comparisons can be "unordered" (U) or "ordered" (O) -- ordered
     // comparisons always return false when one of the operands is NaN, whereas
     // unordered comparisons return true.
@@ -746,22 +720,22 @@ StatusOr ElementalIrEmitter::EmitFloatBinaryOp(
     // matches C++'s semantics.
     case HloOpcode::kEq:
       return llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OEQ, lhs_value,
-                                     rhs_value, ir_builder_);
+                                     rhs_value, b_);
     case HloOpcode::kNe:
       return llvm_ir::EmitComparison(llvm::CmpInst::FCMP_UNE, lhs_value,
-                                     rhs_value, ir_builder_);
+                                     rhs_value, b_);
     case HloOpcode::kLt:
       return llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OLT, lhs_value,
-                                     rhs_value, ir_builder_);
+                                     rhs_value, b_);
     case HloOpcode::kGt:
       return llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OGT, lhs_value,
-                                     rhs_value, ir_builder_);
+                                     rhs_value, b_);
     case HloOpcode::kLe:
       return llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OLE, lhs_value,
-                                     rhs_value, ir_builder_);
+                                     rhs_value, b_);
     case HloOpcode::kGe:
       return llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OGE, lhs_value,
-                                     rhs_value, ir_builder_);
+                                     rhs_value, b_);
 
     case HloOpcode::kMaximum:
       return EmitFloatMax(lhs_value, rhs_value);
@@ -782,64 +756,56 @@ StatusOr ElementalIrEmitter::EmitComplexBinaryOp(
     llvm::Value* rhs_value) const {
   switch (op->opcode()) {
     case HloOpcode::kAdd:
-      return EmitComposeComplex(
-          op,
-          ir_builder_->CreateFAdd(EmitExtractReal(lhs_value),
-                                  EmitExtractReal(rhs_value)),
-          ir_builder_->CreateFAdd(EmitExtractImag(lhs_value),
-                                  EmitExtractImag(rhs_value)));
+      return EmitComposeComplex(op,
+                                b_->CreateFAdd(EmitExtractReal(lhs_value),
+                                               EmitExtractReal(rhs_value)),
+                                b_->CreateFAdd(EmitExtractImag(lhs_value),
+                                               EmitExtractImag(rhs_value)));
     case HloOpcode::kSubtract:
-      return EmitComposeComplex(
-          op,
-          ir_builder_->CreateFSub(EmitExtractReal(lhs_value),
-                                  EmitExtractReal(rhs_value)),
-          ir_builder_->CreateFSub(EmitExtractImag(lhs_value),
-                                  EmitExtractImag(rhs_value)));
+      return EmitComposeComplex(op,
+                                b_->CreateFSub(EmitExtractReal(lhs_value),
+                                               EmitExtractReal(rhs_value)),
+                                b_->CreateFSub(EmitExtractImag(lhs_value),
+                                               EmitExtractImag(rhs_value)));
     case HloOpcode::kMultiply:
       return EmitComposeComplex(
           op,
-          ir_builder_->CreateFSub(
-              ir_builder_->CreateFMul(EmitExtractReal(lhs_value),
-                                      EmitExtractReal(rhs_value)),
-              ir_builder_->CreateFMul(EmitExtractImag(lhs_value),
-                                      EmitExtractImag(rhs_value))),
-          ir_builder_->CreateFAdd(
-              ir_builder_->CreateFMul(EmitExtractReal(lhs_value),
-                                      EmitExtractImag(rhs_value)),
-              ir_builder_->CreateFMul(EmitExtractImag(lhs_value),
-                                      EmitExtractReal(rhs_value))));
+          b_->CreateFSub(b_->CreateFMul(EmitExtractReal(lhs_value),
+                                        EmitExtractReal(rhs_value)),
+                         b_->CreateFMul(EmitExtractImag(lhs_value),
+                                        EmitExtractImag(rhs_value))),
+          b_->CreateFAdd(b_->CreateFMul(EmitExtractReal(lhs_value),
+                                        EmitExtractImag(rhs_value)),
+                         b_->CreateFMul(EmitExtractImag(lhs_value),
+                                        EmitExtractReal(rhs_value))));
     case HloOpcode::kDivide: {
       // (a+bi) / (c+di) = ((a+bi)(c-di)) / ((c+di)(c-di))
       // = ((ac + bd) + (bc - ad)i) / (c^2 + d^2)
-      auto rhs_sum_sq = ir_builder_->CreateFAdd(
-          ir_builder_->CreateFMul(EmitExtractReal(rhs_value),
-                                  EmitExtractReal(rhs_value)),
-          ir_builder_->CreateFMul(EmitExtractImag(rhs_value),
-                                  EmitExtractImag(rhs_value)));
+      auto rhs_sum_sq =
+          b_->CreateFAdd(b_->CreateFMul(EmitExtractReal(rhs_value),
+                                        EmitExtractReal(rhs_value)),
+                         b_->CreateFMul(EmitExtractImag(rhs_value),
+                                        EmitExtractImag(rhs_value)));
       auto type = rhs_sum_sq->getType();
       auto zero = llvm::ConstantFP::get(type, 0.0);
-      auto oeq = ir_builder_->CreateFCmpOEQ(rhs_sum_sq, zero);
-      auto real_inf_or_nan =
-          ir_builder_->CreateFDiv(EmitExtractReal(lhs_value), zero);
-      auto imag_inf_or_nan =
-          ir_builder_->CreateFDiv(EmitExtractImag(lhs_value), zero);
-      return ir_builder_->CreateSelect(
+      auto oeq = b_->CreateFCmpOEQ(rhs_sum_sq, zero);
+      auto real_inf_or_nan = b_->CreateFDiv(EmitExtractReal(lhs_value), zero);
+      auto imag_inf_or_nan = b_->CreateFDiv(EmitExtractImag(lhs_value), zero);
+      return b_->CreateSelect(
           oeq, EmitComposeComplex(op, real_inf_or_nan, imag_inf_or_nan),
           EmitComposeComplex(
               op,
-              ir_builder_->CreateFDiv(
-                  ir_builder_->CreateFAdd(
-                      ir_builder_->CreateFMul(EmitExtractReal(lhs_value),
-                                              EmitExtractReal(rhs_value)),
-                      ir_builder_->CreateFMul(EmitExtractImag(lhs_value),
-                                              EmitExtractImag(rhs_value))),
+              b_->CreateFDiv(
+                  b_->CreateFAdd(b_->CreateFMul(EmitExtractReal(lhs_value),
+                                                EmitExtractReal(rhs_value)),
+                                 b_->CreateFMul(EmitExtractImag(lhs_value),
+                                                EmitExtractImag(rhs_value))),
                   rhs_sum_sq),
-              ir_builder_->CreateFDiv(
-                  ir_builder_->CreateFSub(
-                      ir_builder_->CreateFMul(EmitExtractImag(lhs_value),
-                                              EmitExtractReal(rhs_value)),
-                      ir_builder_->CreateFMul(EmitExtractReal(lhs_value),
-                                              EmitExtractImag(rhs_value))),
+              b_->CreateFDiv(
+                  b_->CreateFSub(b_->CreateFMul(EmitExtractImag(lhs_value),
+                                                EmitExtractReal(rhs_value)),
+                                 b_->CreateFMul(EmitExtractReal(lhs_value),
+                                                EmitExtractImag(rhs_value))),
                   rhs_sum_sq)));
     }
     // LLVM comparisons can be "unordered" (U) or "ordered" (O) -- ordered
@@ -850,21 +816,21 @@ StatusOr ElementalIrEmitter::EmitComplexBinaryOp(
     // unordered comparison.  This makes x != y equivalent to !(x == y), and
     // matches C++'s semantics.
     case HloOpcode::kEq:
-      return ir_builder_->CreateAnd(
+      return b_->CreateAnd(
           llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OEQ,
                                   EmitExtractReal(lhs_value),
-                                  EmitExtractReal(rhs_value), ir_builder_),
+                                  EmitExtractReal(rhs_value), b_),
           llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OEQ,
                                   EmitExtractImag(lhs_value),
-                                  EmitExtractImag(rhs_value), ir_builder_));
+                                  EmitExtractImag(rhs_value), b_));
     case HloOpcode::kNe:
-      return ir_builder_->CreateOr(
+      return b_->CreateOr(
           llvm_ir::EmitComparison(llvm::CmpInst::FCMP_UNE,
                                   EmitExtractReal(lhs_value),
-                                  EmitExtractReal(rhs_value), ir_builder_),
+                                  EmitExtractReal(rhs_value), b_),
           llvm_ir::EmitComparison(llvm::CmpInst::FCMP_UNE,
                                   EmitExtractImag(lhs_value),
-                                  EmitExtractImag(rhs_value), ir_builder_));
+                                  EmitExtractImag(rhs_value), b_));
 
     case HloOpcode::kPower: {
       // (a+bi)^(c+di) =
@@ -876,29 +842,26 @@ StatusOr ElementalIrEmitter::EmitComplexBinaryOp(
       auto b = EmitExtractImag(lhs_value);
       auto c = EmitExtractReal(rhs_value);
       auto d = EmitExtractImag(rhs_value);
-      auto aa_p_bb = ir_builder_->CreateFAdd(ir_builder_->CreateFMul(a, a),
-                                             ir_builder_->CreateFMul(b, b));
+      auto aa_p_bb = b_->CreateFAdd(b_->CreateFMul(a, a), b_->CreateFMul(b, b));
       auto one_half = llvm::ConstantFP::get(a->getType(), 0.5);
-      auto half_c = ir_builder_->CreateFMul(one_half, c);
+      auto half_c = b_->CreateFMul(one_half, c);
 
       TF_ASSIGN_OR_RETURN(auto aa_p_bb_to_half_c,
                           EmitPow(component_type, aa_p_bb, half_c));
-      auto neg_d = ir_builder_->CreateFNeg(d);
+      auto neg_d = b_->CreateFNeg(d);
       TF_ASSIGN_OR_RETURN(auto arg_lhs, EmitAtan2(component_type, b, a));
-      auto neg_d_arg_lhs = ir_builder_->CreateFMul(neg_d, arg_lhs);
+      auto neg_d_arg_lhs = b_->CreateFMul(neg_d, arg_lhs);
       TF_ASSIGN_OR_RETURN(auto e_to_neg_d_arg_lhs,
                           EmitExp(component_type, neg_d_arg_lhs));
-      auto coeff =
-          ir_builder_->CreateFMul(aa_p_bb_to_half_c, e_to_neg_d_arg_lhs);
+      auto coeff = b_->CreateFMul(aa_p_bb_to_half_c, e_to_neg_d_arg_lhs);
       TF_ASSIGN_OR_RETURN(auto ln_aa_p_bb, EmitLog(component_type, aa_p_bb));
-      auto half_d = ir_builder_->CreateFMul(one_half, d);
-      auto q =
-          ir_builder_->CreateFAdd(ir_builder_->CreateFMul(c, arg_lhs),
-                                  ir_builder_->CreateFMul(half_d, ln_aa_p_bb));
+      auto half_d = b_->CreateFMul(one_half, d);
+      auto q = b_->CreateFAdd(b_->CreateFMul(c, arg_lhs),
+                              b_->CreateFMul(half_d, ln_aa_p_bb));
       TF_ASSIGN_OR_RETURN(auto cos_q, EmitCos(component_type, q));
       TF_ASSIGN_OR_RETURN(auto sin_q, EmitSin(component_type, q));
-      return EmitComposeComplex(op, ir_builder_->CreateFMul(coeff, cos_q),
-                                ir_builder_->CreateFMul(coeff, sin_q));
+      return EmitComposeComplex(op, b_->CreateFMul(coeff, cos_q),
+                                b_->CreateFMul(coeff, sin_q));
     }
     default:
       return Unimplemented("binary complex op '%s'",
@@ -908,12 +871,12 @@ StatusOr ElementalIrEmitter::EmitComplexBinaryOp(
 
 llvm::Value* ElementalIrEmitter::EmitFloatMax(llvm::Value* lhs_value,
                                               llvm::Value* rhs_value) const {
-  return llvm_ir::EmitFloatMax(lhs_value, rhs_value, ir_builder_);
+  return llvm_ir::EmitFloatMax(lhs_value, rhs_value, b_);
 }
 
 llvm::Value* ElementalIrEmitter::EmitFloatMin(llvm::Value* lhs_value,
                                               llvm::Value* rhs_value) const {
-  return llvm_ir::EmitFloatMin(lhs_value, rhs_value, ir_builder_);
+  return llvm_ir::EmitFloatMin(lhs_value, rhs_value, b_);
 }
 
 StatusOr ElementalIrEmitter::EmitErfInv(PrimitiveType prim_type,
@@ -925,15 +888,14 @@ StatusOr ElementalIrEmitter::EmitErfInv(PrimitiveType prim_type,
         "type F32.");
   }
   auto getFloat = [&](const float f) {
-    return llvm::ConstantFP::get(ir_builder_->getFloatTy(), f);
+    return llvm::ConstantFP::get(b_->getFloatTy(), f);
   };
   auto multiply_add = [&](tensorflow::gtl::ArraySlice coefficients,
                           llvm::Value* w) {
     llvm::Value* p = getFloat(coefficients.front());
     coefficients.pop_front();
     for (float coefficient : coefficients) {
-      p = ir_builder_->CreateFAdd(ir_builder_->CreateFMul(p, w),
-                                  getFloat(coefficient));
+      p = b_->CreateFAdd(b_->CreateFMul(p, w), getFloat(coefficient));
     }
     return p;
   };
@@ -951,50 +913,48 @@ StatusOr ElementalIrEmitter::EmitErfInv(PrimitiveType prim_type,
   //   }
   //   return p*x
   llvm::Function* logf_fn = llvm::Intrinsic::getDeclaration(
-      module_, llvm::Intrinsic::log, {ir_builder_->getFloatTy()});
+      module_, llvm::Intrinsic::log, {b_->getFloatTy()});
 
-  llvm::Value* w = ir_builder_->CreateFNeg(ir_builder_->CreateCall(
-      logf_fn,
-      {ir_builder_->CreateFMul(ir_builder_->CreateFSub(getFloat(1.0f), x),
-                               ir_builder_->CreateFAdd(getFloat(1.0f), x))}));
+  llvm::Value* w = b_->CreateFNeg(b_->CreateCall(
+      logf_fn, {b_->CreateFMul(b_->CreateFSub(getFloat(1.0f), x),
+                               b_->CreateFAdd(getFloat(1.0f), x))}));
 
-  llvm::Value* p_addr = llvm_ir::EmitAllocaAtFunctionEntry(
-      ir_builder_->getFloatTy(), "p.addr", ir_builder_);
+  llvm::Value* p_addr =
+      llvm_ir::EmitAllocaAtFunctionEntry(b_->getFloatTy(), "p.addr", b_);
 
-  llvm_ir::LlvmIfData if_data =
-      llvm_ir::EmitIfThenElse(ir_builder_->CreateFCmpOLT(w, getFloat(5.0f)),
-                              "w_less_than_five", ir_builder_);
+  llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
+      b_->CreateFCmpOLT(w, getFloat(5.0f)), "w_less_than_five", b_);
   // Handle true BB.
-  SetToFirstInsertPoint(if_data.true_block, ir_builder_);
+  SetToFirstInsertPoint(if_data.true_block, b_);
   {
-    llvm::Value* lw = ir_builder_->CreateFSub(w, getFloat(2.5f));
+    llvm::Value* lw = b_->CreateFSub(w, getFloat(2.5f));
     tensorflow::gtl::ArraySlice lq{
         2.81022636e-08f,  3.43273939e-07f, -3.5233877e-06f,
         -4.39150654e-06f, 0.00021858087f,  -0.00125372503f,
         -0.00417768164f,  0.246640727f,    1.50140941f};
     llvm::Value* p = multiply_add(lq, lw);
-    ir_builder_->CreateStore(p, p_addr);
+    b_->CreateStore(p, p_addr);
   }
 
   // Handle false BB.
-  SetToFirstInsertPoint(if_data.false_block, ir_builder_);
+  SetToFirstInsertPoint(if_data.false_block, b_);
   {
     llvm::Function* sqrtf_fn = llvm::Intrinsic::getDeclaration(
-        module_, llvm::Intrinsic::sqrt, {ir_builder_->getFloatTy()});
+        module_, llvm::Intrinsic::sqrt, {b_->getFloatTy()});
 
-    llvm::Value* gw = ir_builder_->CreateFSub(
-        ir_builder_->CreateCall(sqrtf_fn, {w}), getFloat(3.0f));
+    llvm::Value* gw =
+        b_->CreateFSub(b_->CreateCall(sqrtf_fn, {w}), getFloat(3.0f));
     tensorflow::gtl::ArraySlice gq{
         -0.000200214257f, 0.000100950558f, 0.00134934322f,
         -0.00367342844f,  0.00573950773f,  -0.0076224613f,
         0.00943887047f,   1.00167406f,     2.83297682f};
     llvm::Value* p = multiply_add(gq, gw);
-    ir_builder_->CreateStore(p, p_addr);
+    b_->CreateStore(p, p_addr);
   }
 
-  SetToFirstInsertPoint(if_data.after_block, ir_builder_);
-  llvm::Value* p = ir_builder_->CreateLoad(p_addr);
-  return ir_builder_->CreateFMul(p, x);
+  SetToFirstInsertPoint(if_data.after_block, b_);
+  llvm::Value* p = b_->CreateLoad(p_addr);
+  return b_->CreateFMul(p, x);
 }
 
 StatusOr ElementalIrEmitter::EmitErfcInv(
@@ -1002,13 +962,13 @@ StatusOr ElementalIrEmitter::EmitErfcInv(
   // Compute erfcinv(value) by calculating erfinv(1.0 - value).
   auto type = llvm_ir::PrimitiveTypeToIrType(prim_type, module_);
   auto one = llvm::ConstantFP::get(type, 1.0);
-  return EmitErfInv(prim_type, ir_builder_->CreateFSub(one, value));
+  return EmitErfInv(prim_type, b_->CreateFSub(one, value));
 }
 
 StatusOr ElementalIrEmitter::EmitLog(PrimitiveType prim_type,
                                                    llvm::Value* value) const {
   return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::log, {value},
-                                      {value->getType()}, ir_builder_);
+                                      {value->getType()}, b_);
 }
 
 StatusOr ElementalIrEmitter::EmitLog1p(PrimitiveType prim_type,
@@ -1020,35 +980,34 @@ StatusOr ElementalIrEmitter::EmitLog1p(PrimitiveType prim_type,
   // When x is large, the naive evaluation of ln(x + 1) is more
   // accurate than the Taylor series.
   TF_ASSIGN_OR_RETURN(auto for_large_x,
-                      EmitLog(prim_type, ir_builder_->CreateFAdd(x, one)));
+                      EmitLog(prim_type, b_->CreateFAdd(x, one)));
   // The Taylor series for ln(x+1) is x - x^2/2 - x^3/3 + ….
-  auto for_small_x = ir_builder_->CreateFMul(
-      ir_builder_->CreateFAdd(ir_builder_->CreateFMul(negative_half, x), one),
-      x);
+  auto for_small_x =
+      b_->CreateFMul(b_->CreateFAdd(b_->CreateFMul(negative_half, x), one), x);
   const auto kAntilogarithmIsSmallThreshold = 1e-4;
-  auto abs_x = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::fabs, {value},
-                                            {type}, ir_builder_);
-  auto x_is_small = ir_builder_->CreateFCmpOLT(
+  auto abs_x =
+      llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::fabs, {value}, {type}, b_);
+  auto x_is_small = b_->CreateFCmpOLT(
       abs_x, llvm::ConstantFP::get(type, kAntilogarithmIsSmallThreshold));
-  return ir_builder_->CreateSelect(x_is_small, for_small_x, for_large_x);
+  return b_->CreateSelect(x_is_small, for_small_x, for_large_x);
 }
 
 StatusOr ElementalIrEmitter::EmitSin(PrimitiveType prim_type,
                                                    llvm::Value* value) const {
   return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sin, {value},
-                                      {value->getType()}, ir_builder_);
+                                      {value->getType()}, b_);
 }
 
 StatusOr ElementalIrEmitter::EmitCos(PrimitiveType prim_type,
                                                    llvm::Value* value) const {
   return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::cos, {value},
-                                      {value->getType()}, ir_builder_);
+                                      {value->getType()}, b_);
 }
 
 StatusOr ElementalIrEmitter::EmitExp(PrimitiveType prim_type,
                                                    llvm::Value* value) const {
   return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::exp, {value},
-                                      {value->getType()}, ir_builder_);
+                                      {value->getType()}, b_);
 }
 
 StatusOr ElementalIrEmitter::EmitExpm1(PrimitiveType prim_type,
@@ -1060,25 +1019,25 @@ StatusOr ElementalIrEmitter::EmitExpm1(PrimitiveType prim_type,
   // When the exponent is large, the naive evaluation of e^(x) - 1 is more
   // accurate than the Taylor series.
   TF_ASSIGN_OR_RETURN(auto exp_x, EmitExp(prim_type, value));
-  auto for_large_x = ir_builder_->CreateFSub(exp_x, one);
+  auto for_large_x = b_->CreateFSub(exp_x, one);
   // The Taylor series for exp(x) is 1 + x + x^2/2 + x^3/6 + ….
   // We want exp(x)-1 which is x + x^2/2 + x^3/6 + ….
-  auto x_squared = ir_builder_->CreateFAdd(x, x);
-  auto x_squared_over_two = ir_builder_->CreateFMul(x_squared, half);
-  auto for_small_x = ir_builder_->CreateFAdd(x, x_squared_over_two);
+  auto x_squared = b_->CreateFAdd(x, x);
+  auto x_squared_over_two = b_->CreateFMul(x_squared, half);
+  auto for_small_x = b_->CreateFAdd(x, x_squared_over_two);
   const auto kExponentIsSmallThreshold = 1e-5;
-  auto abs_x = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::fabs, {value},
-                                            {type}, ir_builder_);
-  auto x_is_small = ir_builder_->CreateFCmpOLT(
+  auto abs_x =
+      llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::fabs, {value}, {type}, b_);
+  auto x_is_small = b_->CreateFCmpOLT(
       abs_x, llvm::ConstantFP::get(type, kExponentIsSmallThreshold));
-  return ir_builder_->CreateSelect(x_is_small, for_small_x, for_large_x);
+  return b_->CreateSelect(x_is_small, for_small_x, for_large_x);
 }
 
 StatusOr ElementalIrEmitter::EmitPow(PrimitiveType prim_type,
                                                    llvm::Value* lhs,
                                                    llvm::Value* rhs) const {
   return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::pow, {lhs, rhs},
-                                      {lhs->getType()}, ir_builder_);
+                                      {lhs->getType()}, b_);
 }
 
 StatusOr ElementalIrEmitter::EmitAtan2(PrimitiveType prim_type,
@@ -1093,11 +1052,10 @@ StatusOr ElementalIrEmitter::EmitReducePrecision(
     return Unimplemented("reduce-precision only implemented for F32");
   }
   return EmitReducePrecisionFloat(x, /*exponent_bits=*/hlo->exponent_bits(),
-                                  /*mantissa_bits=*/hlo->mantissa_bits(),
-                                  ir_builder_);
+                                  /*mantissa_bits=*/hlo->mantissa_bits(), b_);
 }
 
-static llvm::Value* SaturateShiftIfNecessary(llvm::IRBuilder<>* ir_builder,
+static llvm::Value* SaturateShiftIfNecessary(llvm::IRBuilder<>* b,
                                              llvm::Value* lhs, llvm::Value* rhs,
                                              llvm::Value* shift_result,
                                              bool saturate_to_sign_bit) {
@@ -1110,15 +1068,14 @@ static llvm::Value* SaturateShiftIfNecessary(llvm::IRBuilder<>* ir_builder,
   llvm::ConstantInt* minus_one = llvm::ConstantInt::get(integer_type, -1);
   llvm::Value* saturated_value;
   if (saturate_to_sign_bit) {
-    saturated_value = ir_builder->CreateSelect(
-        ir_builder->CreateICmpSLT(lhs, zero), minus_one, zero);
+    saturated_value =
+        b->CreateSelect(b->CreateICmpSLT(lhs, zero), minus_one, zero);
   } else {
     saturated_value = zero;
   }
   llvm::Value* shift_amt_in_range =
-      ir_builder->CreateICmpULT(rhs, integer_bitsize_constant, "shft.chk");
-  return ir_builder->CreateSelect(shift_amt_in_range, shift_result,
-                                  saturated_value);
+      b->CreateICmpULT(rhs, integer_bitsize_constant, "shft.chk");
+  return b->CreateSelect(shift_amt_in_range, shift_result, saturated_value);
 }
 
 StatusOr ElementalIrEmitter::EmitIntegerBinaryOp(
@@ -1127,49 +1084,49 @@ StatusOr ElementalIrEmitter::EmitIntegerBinaryOp(
   switch (op->opcode()) {
     // TODO(jingyue): add the "nsw" attribute for signed types.
     case HloOpcode::kAdd:
-      return ir_builder_->CreateAdd(lhs_value, rhs_value);
+      return b_->CreateAdd(lhs_value, rhs_value);
     case HloOpcode::kSubtract:
-      return ir_builder_->CreateSub(lhs_value, rhs_value);
+      return b_->CreateSub(lhs_value, rhs_value);
     case HloOpcode::kMultiply:
-      return ir_builder_->CreateMul(lhs_value, rhs_value);
+      return b_->CreateMul(lhs_value, rhs_value);
     case HloOpcode::kDivide:
-      return is_signed ? ir_builder_->CreateSDiv(lhs_value, rhs_value)
-                       : ir_builder_->CreateUDiv(lhs_value, rhs_value);
+      return is_signed ? b_->CreateSDiv(lhs_value, rhs_value)
+                       : b_->CreateUDiv(lhs_value, rhs_value);
     case HloOpcode::kRemainder:
-      return is_signed ? ir_builder_->CreateSRem(lhs_value, rhs_value)
-                       : ir_builder_->CreateURem(lhs_value, rhs_value);
+      return is_signed ? b_->CreateSRem(lhs_value, rhs_value)
+                       : b_->CreateURem(lhs_value, rhs_value);
     case HloOpcode::kEq:
       return llvm_ir::EmitComparison(llvm::CmpInst::ICMP_EQ, lhs_value,
-                                     rhs_value, ir_builder_);
+                                     rhs_value, b_);
     case HloOpcode::kNe:
       return llvm_ir::EmitComparison(llvm::CmpInst::ICMP_NE, lhs_value,
-                                     rhs_value, ir_builder_);
+                                     rhs_value, b_);
     case HloOpcode::kLt:
       return llvm_ir::EmitComparison(
           is_signed ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT,
-          lhs_value, rhs_value, ir_builder_);
+          lhs_value, rhs_value, b_);
     case HloOpcode::kGt:
       return llvm_ir::EmitComparison(
           is_signed ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT,
-          lhs_value, rhs_value, ir_builder_);
+          lhs_value, rhs_value, b_);
     case HloOpcode::kLe:
       return llvm_ir::EmitComparison(
           is_signed ? llvm::CmpInst::ICMP_SLE : llvm::CmpInst::ICMP_ULE,
-          lhs_value, rhs_value, ir_builder_);
+          lhs_value, rhs_value, b_);
     case HloOpcode::kGe:
       return llvm_ir::EmitComparison(
           is_signed ? llvm::CmpInst::ICMP_SGE : llvm::CmpInst::ICMP_UGE,
-          lhs_value, rhs_value, ir_builder_);
+          lhs_value, rhs_value, b_);
     case HloOpcode::kMinimum:
       return EmitIntegralMin(lhs_value, rhs_value, is_signed);
     case HloOpcode::kMaximum:
       return EmitIntegralMax(lhs_value, rhs_value, is_signed);
     case HloOpcode::kAnd:
-      return ir_builder_->CreateAnd(lhs_value, rhs_value);
+      return b_->CreateAnd(lhs_value, rhs_value);
     case HloOpcode::kOr:
-      return ir_builder_->CreateOr(lhs_value, rhs_value);
+      return b_->CreateOr(lhs_value, rhs_value);
     case HloOpcode::kXor:
-      return ir_builder_->CreateXor(lhs_value, rhs_value);
+      return b_->CreateXor(lhs_value, rhs_value);
 
     // Shifting out bits >= the number of bits in the type being shifted
     // produces a poison value in LLVM which is basically "deferred undefined
@@ -1177,20 +1134,17 @@ StatusOr ElementalIrEmitter::EmitIntegerBinaryOp(
     // UB.  We replace the poison value with a constant to avoid this deferred
     // UB.
     case HloOpcode::kShiftRightArithmetic:
-      return SaturateShiftIfNecessary(
-          ir_builder_, lhs_value, rhs_value,
-          ir_builder_->CreateAShr(lhs_value, rhs_value),
-          /*saturate_to_sign_bit=*/true);
+      return SaturateShiftIfNecessary(b_, lhs_value, rhs_value,
+                                      b_->CreateAShr(lhs_value, rhs_value),
+                                      /*saturate_to_sign_bit=*/true);
     case HloOpcode::kShiftLeft:
-      return SaturateShiftIfNecessary(
-          ir_builder_, lhs_value, rhs_value,
-          ir_builder_->CreateShl(lhs_value, rhs_value),
-          /*saturate_to_sign_bit=*/false);
+      return SaturateShiftIfNecessary(b_, lhs_value, rhs_value,
+                                      b_->CreateShl(lhs_value, rhs_value),
+                                      /*saturate_to_sign_bit=*/false);
     case HloOpcode::kShiftRightLogical:
-      return SaturateShiftIfNecessary(
-          ir_builder_, lhs_value, rhs_value,
-          ir_builder_->CreateLShr(lhs_value, rhs_value),
-          /*saturate_to_sign_bit=*/false);
+      return SaturateShiftIfNecessary(b_, lhs_value, rhs_value,
+                                      b_->CreateLShr(lhs_value, rhs_value),
+                                      /*saturate_to_sign_bit=*/false);
     default:
       return Unimplemented("binary integer op '%s'",
                            HloOpcodeString(op->opcode()).c_str());
@@ -1200,21 +1154,19 @@ StatusOr ElementalIrEmitter::EmitIntegerBinaryOp(
 llvm::Value* ElementalIrEmitter::EmitIntegralMax(llvm::Value* lhs_value,
                                                  llvm::Value* rhs_value,
                                                  bool is_signed) const {
-  return ir_builder_->CreateSelect(
-      ir_builder_->CreateICmp(
-          is_signed ? llvm::ICmpInst::ICMP_SGE : llvm::ICmpInst::ICMP_UGE,
-          lhs_value, rhs_value),
-      lhs_value, rhs_value);
+  return b_->CreateSelect(b_->CreateICmp(is_signed ? llvm::ICmpInst::ICMP_SGE
+                                                   : llvm::ICmpInst::ICMP_UGE,
+                                         lhs_value, rhs_value),
+                          lhs_value, rhs_value);
 }
 
 llvm::Value* ElementalIrEmitter::EmitIntegralMin(llvm::Value* lhs_value,
                                                  llvm::Value* rhs_value,
                                                  bool is_signed) const {
-  return ir_builder_->CreateSelect(
-      ir_builder_->CreateICmp(
-          is_signed ? llvm::ICmpInst::ICMP_SLE : llvm::ICmpInst::ICMP_ULE,
-          lhs_value, rhs_value),
-      lhs_value, rhs_value);
+  return b_->CreateSelect(b_->CreateICmp(is_signed ? llvm::ICmpInst::ICMP_SLE
+                                                   : llvm::ICmpInst::ICMP_ULE,
+                                         lhs_value, rhs_value),
+                          lhs_value, rhs_value);
 }
 
 llvm_ir::IrArray::Index ElementalIrEmitter::ElementwiseSourceIndex(
@@ -1267,10 +1219,10 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator(
 
   // Same values as PCG library
   // https://github.com/imneme/pcg-c/blob/master/include/pcg_variants.h
-  llvm::Value* multiplier = ir_builder_->getInt(
-      llvm::APInt(128, {0x4385DF649FCCF645, 0x2360ED051FC65DA4}));
-  llvm::Value* increment = ir_builder_->getInt(
-      llvm::APInt(128, {0x14057B7EF767814F, 0x5851F42D4C957F2D}));
+  llvm::Value* multiplier =
+      b_->getInt(llvm::APInt(128, {0x4385DF649FCCF645, 0x2360ED051FC65DA4}));
+  llvm::Value* increment =
+      b_->getInt(llvm::APInt(128, {0x14057B7EF767814F, 0x5851F42D4C957F2D}));
 
   auto random_value_from_hlo = [hlo]() {
     const HloModule* module =
@@ -1291,10 +1243,10 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator(
   // values.
   llvm::GlobalVariable* state_ptr0 = new llvm::GlobalVariable(
       /*M=*/*module_,
-      /*Ty=*/ir_builder_->getInt64Ty(),
+      /*Ty=*/b_->getInt64Ty(),
       /*isConstant=*/false,
       /*Linkage=*/llvm::GlobalValue::PrivateLinkage,
-      /*Initializer=*/ir_builder_->getInt64(random_value_from_hlo()),
+      /*Initializer=*/b_->getInt64(random_value_from_hlo()),
       /*Name=*/"state_ptr0");
 
   // When the module config seed is 0, the expected result of a prng is a random
@@ -1305,17 +1257,16 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator(
                                                      : GlobalRandomValue();
   llvm::GlobalVariable* state_ptr1 = new llvm::GlobalVariable(
       /*M=*/*module_,
-      /*Ty=*/ir_builder_->getInt64Ty(),
+      /*Ty=*/b_->getInt64Ty(),
       /*isConstant=*/false,
       /*Linkage=*/llvm::GlobalValue::PrivateLinkage,
-      /*Initializer=*/ir_builder_->getInt64(graph_seed),
+      /*Initializer=*/b_->getInt64(graph_seed),
       /*Name=*/"state_ptr1");
 
   // We want each thread to use its own stream, so we modify the increment per
   // thread. We want the increment to remain odd, so we shift the thread id left
   // 1 and add it to the increment.
-  increment = ir_builder_->CreateAdd(increment,
-                                     ir_builder_->CreateShl(EmitThreadId(), 1));
+  increment = b_->CreateAdd(increment, b_->CreateShl(EmitThreadId(), 1));
 
   // PCG-XSL-RR algorithm
   // http://www.pcg-random.org/pdf/toms-oneill-pcg-family-v1.02.pdf
@@ -1323,38 +1274,29 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator(
   //   return uint64_t(state ^ (state >> 64))) >>> (state >> 122)
   // where ">>>" is bitwise rotation
   auto get_next_i64 = [=]() {
-    llvm::Value* state0 = ir_builder_->CreateZExtOrTrunc(
-        ir_builder_->CreateLoad(state_ptr0, "state0"),
-        ir_builder_->getInt128Ty());
-    llvm::Value* state1 = ir_builder_->CreateShl(
-        ir_builder_->CreateZExtOrTrunc(
-            ir_builder_->CreateLoad(state_ptr1, "state1"),
-            ir_builder_->getInt128Ty()),
+    llvm::Value* state0 = b_->CreateZExtOrTrunc(
+        b_->CreateLoad(state_ptr0, "state0"), b_->getInt128Ty());
+    llvm::Value* state1 = b_->CreateShl(
+        b_->CreateZExtOrTrunc(b_->CreateLoad(state_ptr1, "state1"),
+                              b_->getInt128Ty()),
         64);
-    llvm::Value* state = ir_builder_->CreateOr(state0, state1);
-    llvm::Value* updated = ir_builder_->CreateAdd(
-        ir_builder_->CreateMul(state, multiplier), increment);
-    ir_builder_->CreateStore(
-        ir_builder_->CreateTrunc(updated, ir_builder_->getInt64Ty()),
-        state_ptr0);
-    ir_builder_->CreateStore(
-        ir_builder_->CreateTrunc(ir_builder_->CreateLShr(updated, 64),
-                                 ir_builder_->getInt64Ty()),
+    llvm::Value* state = b_->CreateOr(state0, state1);
+    llvm::Value* updated =
+        b_->CreateAdd(b_->CreateMul(state, multiplier), increment);
+    b_->CreateStore(b_->CreateTrunc(updated, b_->getInt64Ty()), state_ptr0);
+    b_->CreateStore(
+        b_->CreateTrunc(b_->CreateLShr(updated, 64), b_->getInt64Ty()),
         state_ptr1);
 
     return llvm_ir::CreateRor(
-        ir_builder_->CreateTrunc(
-            ir_builder_->CreateXor(state, ir_builder_->CreateLShr(state, 64)),
-            ir_builder_->getInt64Ty()),
-        ir_builder_->CreateTrunc(ir_builder_->CreateLShr(state, 122),
-                                 ir_builder_->getInt64Ty()),
-        ir_builder_);
+        b_->CreateTrunc(b_->CreateXor(state, b_->CreateLShr(state, 64)),
+                        b_->getInt64Ty()),
+        b_->CreateTrunc(b_->CreateLShr(state, 122), b_->getInt64Ty()), b_);
   };
 
   auto get_next_uniform_float = [=]() {
-    return ir_builder_->CreateFDiv(
-        ir_builder_->CreateUIToFP(get_next_i64(), param_ir_type),
-        llvm::ConstantFP::get(param_ir_type, 0x1p64));
+    return b_->CreateFDiv(b_->CreateUIToFP(get_next_i64(), param_ir_type),
+                          llvm::ConstantFP::get(param_ir_type, 0x1p64));
   };
 
   return [=](const llvm_ir::IrArray::Index& index) -> StatusOr {
@@ -1365,52 +1307,50 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator(
         TF_ASSIGN_OR_RETURN(llvm::Value * q,
                             operand_to_generator.at(hlo->operand(1))(index));
         if (primitive_util::IsFloatingPointType(param_prim_type)) {
-          return ir_builder_->CreateFAdd(
-              ir_builder_->CreateFMul(ir_builder_->CreateFSub(q, p),
-                                      get_next_uniform_float()),
+          return b_->CreateFAdd(
+              b_->CreateFMul(b_->CreateFSub(q, p), get_next_uniform_float()),
               p);
         } else {
-          auto r = ir_builder_->CreateSub(q, p);
+          auto r = b_->CreateSub(q, p);
           auto leading_zeros = llvm_ir::EmitCallToIntrinsic(
-              llvm::Intrinsic::ctlz, {r, ir_builder_->getInt1(true)},
-              {param_ir_type}, ir_builder_);
-          auto in_block = ir_builder_->GetInsertBlock();
+              llvm::Intrinsic::ctlz, {r, b_->getInt1(true)}, {param_ir_type},
+              b_);
+          auto in_block = b_->GetInsertBlock();
 
           // A terminator should be present iff we're emitting code
           // into the middle (as opposed to the end) of a basic block.
-          CHECK_EQ(ir_builder_->GetInsertPoint() == in_block->end(),
+          CHECK_EQ(b_->GetInsertPoint() == in_block->end(),
                    in_block->getTerminator() == nullptr);
 
           llvm::BasicBlock* body_block;
           llvm::BasicBlock* out_block;
 
-          if (ir_builder_->GetInsertPoint() == in_block->end()) {
-            body_block = llvm_ir::CreateBasicBlock(
-                nullptr, IrName(hlo, "rng_body"), ir_builder_);
-            out_block = llvm_ir::CreateBasicBlock(
-                nullptr, IrName(hlo, "rng_out"), ir_builder_);
+          if (b_->GetInsertPoint() == in_block->end()) {
+            body_block =
+                llvm_ir::CreateBasicBlock(nullptr, IrName(hlo, "rng_body"), b_);
+            out_block =
+                llvm_ir::CreateBasicBlock(nullptr, IrName(hlo, "rng_out"), b_);
             llvm::BranchInst::Create(body_block, in_block);
           } else {
-            body_block = in_block->splitBasicBlock(
-                ir_builder_->GetInsertPoint(), "rng_body");
-            out_block = body_block->splitBasicBlock(
-                ir_builder_->GetInsertPoint(), "rng_out");
+            body_block =
+                in_block->splitBasicBlock(b_->GetInsertPoint(), "rng_body");
+            out_block =
+                body_block->splitBasicBlock(b_->GetInsertPoint(), "rng_out");
             body_block->getTerminator()->eraseFromParent();
           }
 
-          SetToFirstInsertPoint(body_block, ir_builder_);
-          auto random = ir_builder_->CreateAnd(
-              ir_builder_->CreateZExtOrTrunc(get_next_i64(), param_ir_type),
-              ir_builder_->CreateLShr(llvm::ConstantInt::get(param_ir_type, ~0),
-                                      leading_zeros));
+          SetToFirstInsertPoint(body_block, b_);
+          auto random = b_->CreateAnd(
+              b_->CreateZExtOrTrunc(get_next_i64(), param_ir_type),
+              b_->CreateLShr(llvm::ConstantInt::get(param_ir_type, ~0),
+                             leading_zeros));
           llvm::BranchInst::Create(out_block, body_block,
-                                   ir_builder_->CreateICmpULT(random, r),
-                                   body_block);
-          SetToFirstInsertPoint(out_block, ir_builder_);
-          return ir_builder_->CreateAdd(
-              p, ir_builder_->CreateSelect(
-                     ir_builder_->CreateICmpEQ(p, q),
-                     llvm::ConstantInt::get(param_ir_type, 0), random));
+                                   b_->CreateICmpULT(random, r), body_block);
+          SetToFirstInsertPoint(out_block, b_);
+          return b_->CreateAdd(
+              p, b_->CreateSelect(b_->CreateICmpEQ(p, q),
+                                  llvm::ConstantInt::get(param_ir_type, 0),
+                                  random));
         }
       }
       case RNG_NORMAL: {
@@ -1420,11 +1360,11 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator(
                             operand_to_generator.at(hlo->operand(1))(index));
         TF_ASSIGN_OR_RETURN(
             llvm::Value * r,
-            EmitErfcInv(param_prim_type,
-                        ir_builder_->CreateFMul(
-                            llvm::ConstantFP::get(param_ir_type, 2.0),
-                            get_next_uniform_float())));
-        return ir_builder_->CreateFAdd(ir_builder_->CreateFMul(r, s), m);
+            EmitErfcInv(
+                param_prim_type,
+                b_->CreateFMul(llvm::ConstantFP::get(param_ir_type, 2.0),
+                               get_next_uniform_float())));
+        return b_->CreateFAdd(b_->CreateFMul(r, s), m);
       }
       default:
         return InvalidArgument(
@@ -1447,9 +1387,8 @@ StatusOr ElementalIrEmitter::EmitElementalSelect(
   TF_ASSIGN_OR_RETURN(llvm::Value * on_false_value,
                       operand_to_generator.at(hlo->operand(2))(
                           ElementwiseSourceIndex(index, *hlo, 2)));
-  return ir_builder_->CreateSelect(
-      ir_builder_->CreateTrunc(pred_value, ir_builder_->getInt1Ty()),
-      on_true_value, on_false_value);
+  return b_->CreateSelect(b_->CreateTrunc(pred_value, b_->getInt1Ty()),
+                          on_true_value, on_false_value);
 }
 
 StatusOr ElementalIrEmitter::EmitElementalClamp(
@@ -1485,64 +1424,62 @@ StatusOr ElementalIrEmitter::EmitElementalConcatenate(
   const int64 concat_dim = hlo->dimensions(0);
   auto source_index = target_index;
 
-  llvm::BasicBlock* init_block = ir_builder_->GetInsertBlock();
+  llvm::BasicBlock* init_block = b_->GetInsertBlock();
 
   // A terminator should be present iff we're emitting code
   // into the middle (as opposed to the end) of a basic block.
-  CHECK_EQ(ir_builder_->GetInsertPoint() == init_block->end(),
+  CHECK_EQ(b_->GetInsertPoint() == init_block->end(),
            init_block->getTerminator() == nullptr);
 
   llvm::BasicBlock* exit_block;
-  if (ir_builder_->GetInsertPoint() == init_block->end()) {
+  if (b_->GetInsertPoint() == init_block->end()) {
     exit_block = llvm_ir::CreateBasicBlock(
-        /*insert_before=*/nullptr, IrName(hlo, "merge"), ir_builder_);
+        /*insert_before=*/nullptr, IrName(hlo, "merge"), b_);
   } else {
-    exit_block = init_block->splitBasicBlock(ir_builder_->GetInsertPoint(),
+    exit_block = init_block->splitBasicBlock(b_->GetInsertPoint(),
                                              AsStringRef(IrName(hlo, "merge")));
     init_block->getTerminator()->eraseFromParent();
   }
 
-  llvm_ir::SetToFirstInsertPoint(exit_block, ir_builder_);
-  llvm::PHINode* output = ir_builder_->CreatePHI(
+  llvm_ir::SetToFirstInsertPoint(exit_block, b_);
+  llvm::PHINode* output = b_->CreatePHI(
       llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), module_),
       hlo->operands().size());
-  auto prior_insert_point = ir_builder_->GetInsertPoint();
+  auto prior_insert_point = b_->GetInsertPoint();
 
-  ir_builder_->SetInsertPoint(init_block);
+  b_->SetInsertPoint(init_block);
 
   for (int64 operand_idx = 0; operand_idx < hlo->operand_count();
        ++operand_idx) {
     const HloInstruction* operand = hlo->operand(operand_idx);
     auto true_block = llvm_ir::CreateBasicBlock(
-        exit_block, StrCat("concat_index_from_operand", operand_idx),
-        ir_builder_);
+        exit_block, StrCat("concat_index_from_operand", operand_idx), b_);
     auto false_block = llvm_ir::CreateBasicBlock(
-        exit_block, StrCat("concat_index_not_from_operand", operand_idx),
-        ir_builder_);
+        exit_block, StrCat("concat_index_not_from_operand", operand_idx), b_);
     auto concat_dim_size =
         llvm::ConstantInt::get(source_index[concat_dim]->getType(),
                                operand->shape().dimensions(concat_dim));
-    ir_builder_->CreateCondBr(
-        ir_builder_->CreateICmpULT(source_index[concat_dim], concat_dim_size),
+    b_->CreateCondBr(
+        b_->CreateICmpULT(source_index[concat_dim], concat_dim_size),
         true_block, false_block);
 
     // Create the terminator of the true block before calling operand
     // generators, because they require non-degenerate basic blocks.
-    ir_builder_->SetInsertPoint(
+    b_->SetInsertPoint(
         llvm::BranchInst::Create(exit_block, /*InsertAtEnd=*/true_block));
     TF_ASSIGN_OR_RETURN(llvm::Value * value,
                         operand_to_generator.at(operand)(source_index));
-    output->addIncoming(value, ir_builder_->GetInsertBlock());
+    output->addIncoming(value, b_->GetInsertBlock());
 
     // Subtract the size of the concat dimension of the current operand
     // from the source index.
-    ir_builder_->SetInsertPoint(false_block);
+    b_->SetInsertPoint(false_block);
     source_index[concat_dim] =
-        ir_builder_->CreateSub(source_index[concat_dim], concat_dim_size);
+        b_->CreateSub(source_index[concat_dim], concat_dim_size);
   }
 
-  ir_builder_->CreateUnreachable();
-  ir_builder_->SetInsertPoint(exit_block, prior_insert_point);
+  b_->CreateUnreachable();
+  b_->SetInsertPoint(exit_block, prior_insert_point);
   return output;
 }
 
@@ -1570,8 +1507,7 @@ StatusOr ElementalIrEmitter::EmitElementalDynamicSlice(
     // TODO(b/74360564): This is implementation defined behavior, but is
     // currently respected by all implementations. Change this if we ever decide
     // to officially document different behavior.
-    start_index_value =
-        ir_builder_->CreateSExtOrTrunc(start_index_value, index_type);
+    start_index_value = b_->CreateSExtOrTrunc(start_index_value, index_type);
     int64 largest_valid_start_index =
         input_hlo->shape().dimensions(i) - hlo->shape().dimensions(i);
     CHECK_GE(largest_valid_start_index, 0);
@@ -1591,7 +1527,7 @@ StatusOr ElementalIrEmitter::EmitElementalDynamicSlice(
   for (int64 i = 0; i < rank; ++i) {
     // Emit IR which computes:
     //   input_index = start_index + offset_index
-    input_index[i] = ir_builder_->CreateAdd(slice_start_index[i], index[i]);
+    input_index[i] = b_->CreateAdd(slice_start_index[i], index[i]);
   }
   return operand_to_generator.at(input_hlo)(input_index);
 }
@@ -1649,7 +1585,7 @@ StatusOr ElementalIrEmitter::EmitElementalGather(
 
   auto add_to_operand_index = [&](llvm::Value* index_component, int64 dim) {
     llvm::Value* gather_dim_component_extended =
-        ir_builder_->CreateSExtOrTrunc(index_component, index_type);
+        b_->CreateSExtOrTrunc(index_component, index_type);
     int64 operand_dim = dim_numbers.gather_dims_to_operand_dims(dim);
     int64 output_dim = operand_to_output_dim[operand_dim];
     // If 'output_dim' is -1, it means 'operand_dim' is an elided window dim.
@@ -1673,7 +1609,7 @@ StatusOr ElementalIrEmitter::EmitElementalGather(
                         gather_dim_component_extended, is_signed),
         is_signed);
 
-    operand_index[operand_dim] = ir_builder_->CreateAdd(
+    operand_index[operand_dim] = b_->CreateAdd(
         operand_index[operand_dim], gather_dim_component_extended_inbound);
   };
 
@@ -1708,7 +1644,7 @@ StatusOr ElementalIrEmitter::EmitElementalDynamicUpdateSlice(
   llvm_ir::IrArray::Index slice_limit_index(index.GetType(), rank);
   // Slice intersection gathers (ANDs) conditions on all ranks for which
   // 'input' is set to 'update'
-  llvm::Value* slice_intersection = ir_builder_->getTrue();
+  llvm::Value* slice_intersection = b_->getTrue();
 
   for (int64 i = 0; i < rank; ++i) {
     llvm::Type* index_type = index[0]->getType();
@@ -1725,8 +1661,7 @@ StatusOr ElementalIrEmitter::EmitElementalDynamicUpdateSlice(
     // TODO(b/74360564): This is implementation defined behavior, but is
     // currently respected by all implementations. Change this if we ever decide
     // to officially document different behavior.
-    start_index_value =
-        ir_builder_->CreateSExtOrTrunc(start_index_value, index_type);
+    start_index_value = b_->CreateSExtOrTrunc(start_index_value, index_type);
     llvm::Value* update_dim_size =
         index_typed_const(update_hlo->shape().dimensions(i));
     int64 largest_valid_start_index =
@@ -1742,16 +1677,13 @@ StatusOr ElementalIrEmitter::EmitElementalDynamicUpdateSlice(
     start_index_value->setName(
         AsStringRef(IrName(hlo, StrCat("start_idx", i))));
     slice_start_index[i] = start_index_value;
-    slice_limit_index[i] =
-        ir_builder_->CreateAdd(slice_start_index[i], update_dim_size);
+    slice_limit_index[i] = b_->CreateAdd(slice_start_index[i], update_dim_size);
 
-    slice_intersection = ir_builder_->CreateAnd(
-        slice_intersection,
-        ir_builder_->CreateICmpSGE(index[i], slice_start_index[i]),
+    slice_intersection = b_->CreateAnd(
+        slice_intersection, b_->CreateICmpSGE(index[i], slice_start_index[i]),
         "slice_intersection");
-    slice_intersection = ir_builder_->CreateAnd(
-        slice_intersection,
-        ir_builder_->CreateICmpSLT(index[i], slice_limit_index[i]),
+    slice_intersection = b_->CreateAnd(
+        slice_intersection, b_->CreateICmpSLT(index[i], slice_limit_index[i]),
         "slice_intersection");
   }
 
@@ -1760,29 +1692,29 @@ StatusOr ElementalIrEmitter::EmitElementalDynamicUpdateSlice(
   // else                    -> return data from 'input'.
   llvm::Value* ret_value_addr = llvm_ir::EmitAllocaAtFunctionEntry(
       llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), module_),
-      "ret_value_addr", ir_builder_);
-  llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
-      slice_intersection, "slice_intersection", ir_builder_);
+      "ret_value_addr", b_);
+  llvm_ir::LlvmIfData if_data =
+      llvm_ir::EmitIfThenElse(slice_intersection, "slice_intersection", b_);
 
   // Handle true BB (return data from 'update')
-  SetToFirstInsertPoint(if_data.true_block, ir_builder_);
+  SetToFirstInsertPoint(if_data.true_block, b_);
   // Compute update index for intersection case.
   llvm_ir::IrArray::Index update_index(index.GetType(), rank);
   for (int64 i = 0; i < rank; ++i) {
-    update_index[i] = ir_builder_->CreateSub(index[i], slice_start_index[i]);
+    update_index[i] = b_->CreateSub(index[i], slice_start_index[i]);
   }
   TF_ASSIGN_OR_RETURN(llvm::Value * true_value,
                       operand_to_generator.at(update_hlo)(update_index));
-  ir_builder_->CreateStore(true_value, ret_value_addr);
+  b_->CreateStore(true_value, ret_value_addr);
 
   // Handle false BB (return data from 'input')
-  SetToFirstInsertPoint(if_data.false_block, ir_builder_);
+  SetToFirstInsertPoint(if_data.false_block, b_);
   TF_ASSIGN_OR_RETURN(llvm::Value * false_value,
                       operand_to_generator.at(input_hlo)(index));
-  ir_builder_->CreateStore(false_value, ret_value_addr);
+  b_->CreateStore(false_value, ret_value_addr);
 
-  SetToFirstInsertPoint(if_data.after_block, ir_builder_);
-  return ir_builder_->CreateLoad(ret_value_addr);
+  SetToFirstInsertPoint(if_data.after_block, b_);
+  return b_->CreateLoad(ret_value_addr);
 }
 
 StatusOr ElementalIrEmitter::EmitElementalPad(
@@ -1790,29 +1722,29 @@ StatusOr ElementalIrEmitter::EmitElementalPad(
     const ElementalIrEmitter::HloToElementGeneratorMap& operand_to_generator,
     const llvm_ir::IrArray::Index& padded_index) const {
   auto index = padded_index;
-  llvm::Value* in_bounds = ir_builder_->getTrue();
+  llvm::Value* in_bounds = b_->getTrue();
   for (size_t i = 0; i < index.size(); ++i) {
     auto index_typed_const = [=](int64 n) {
       return llvm::ConstantInt::get(index[i]->getType(), n);
     };
     const auto& pad_dim = hlo->padding_config().dimensions(i);
-    index[i] = ir_builder_->CreateSub(
-        index[i], index_typed_const(pad_dim.edge_padding_low()));
-    in_bounds = ir_builder_->CreateAnd(
-        in_bounds, ir_builder_->CreateICmpSGE(index[i], index_typed_const(0)),
-        "in_bounds");
-    in_bounds = ir_builder_->CreateAnd(
+    index[i] =
+        b_->CreateSub(index[i], index_typed_const(pad_dim.edge_padding_low()));
+    in_bounds = b_->CreateAnd(in_bounds,
+                              b_->CreateICmpSGE(index[i], index_typed_const(0)),
+                              "in_bounds");
+    in_bounds = b_->CreateAnd(
         in_bounds,
-        ir_builder_->CreateICmpEQ(
+        b_->CreateICmpEQ(
             index_typed_const(0),
-            ir_builder_->CreateURem(
-                index[i], index_typed_const(pad_dim.interior_padding() + 1))),
+            b_->CreateURem(index[i],
+                           index_typed_const(pad_dim.interior_padding() + 1))),
         "in_bounds");
-    index[i] = ir_builder_->CreateSDiv(
+    index[i] = b_->CreateSDiv(
         index[i], index_typed_const(pad_dim.interior_padding() + 1));
-    in_bounds = ir_builder_->CreateAnd(
+    in_bounds = b_->CreateAnd(
         in_bounds,
-        ir_builder_->CreateICmpSLT(
+        b_->CreateICmpSLT(
             index[i],
             index_typed_const(hlo->operand(0)->shape().dimensions(i))),
         "in_bounds");
@@ -1825,26 +1757,26 @@ StatusOr ElementalIrEmitter::EmitElementalPad(
   // }
   llvm::Value* ret_value_addr = llvm_ir::EmitAllocaAtFunctionEntry(
       llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), module_),
-      "pad_result_addr", ir_builder_);
+      "pad_result_addr", b_);
   llvm_ir::LlvmIfData if_data =
-      llvm_ir::EmitIfThenElse(in_bounds, "in_bounds", ir_builder_);
-  SetToFirstInsertPoint(if_data.true_block, ir_builder_);
+      llvm_ir::EmitIfThenElse(in_bounds, "in_bounds", b_);
+  SetToFirstInsertPoint(if_data.true_block, b_);
   TF_ASSIGN_OR_RETURN(llvm::Value * operand_value,
                       operand_to_generator.at(hlo->operand(0))(index));
-  ir_builder_->CreateStore(operand_value, ret_value_addr);
+  b_->CreateStore(operand_value, ret_value_addr);
 
-  SetToFirstInsertPoint(if_data.false_block, ir_builder_);
+  SetToFirstInsertPoint(if_data.false_block, b_);
   TF_ASSIGN_OR_RETURN(llvm::Value * padding_value,
                       operand_to_generator.at(hlo->operand(1))(
                           IrArray::Index(index.GetType())));
-  ir_builder_->CreateStore(padding_value, ret_value_addr);
+  b_->CreateStore(padding_value, ret_value_addr);
 
-  SetToFirstInsertPoint(if_data.after_block, ir_builder_);
+  SetToFirstInsertPoint(if_data.after_block, b_);
   // Don't create phi(operand_value, padding_value) here, because invoking
   // operand_to_generator may create new basic blocks, making the parent
   // of operand_value or padding_value no longer a predecessor of
   // if_data.after_block.
-  return ir_builder_->CreateLoad(ret_value_addr);
+  return b_->CreateLoad(ret_value_addr);
 }
 
 StatusOr ElementalIrEmitter::EmitElementalDot(
@@ -1868,21 +1800,20 @@ StatusOr ElementalIrEmitter::EmitElementalDot(
     return llvm::ConstantInt::get(index_type, c);
   };
 
-  std::unique_ptr inner_loop =
-      llvm_ir::ForLoop::EmitForLoop(IrName(hlo, "inner"), index_typed_const(0),
-                                    index_typed_const(contracted_dim_size),
-                                    index_typed_const(1), ir_builder_);
+  std::unique_ptr inner_loop = llvm_ir::ForLoop::EmitForLoop(
+      IrName(hlo, "inner"), index_typed_const(0),
+      index_typed_const(contracted_dim_size), index_typed_const(1), b_);
 
-  SetToFirstInsertPoint(inner_loop->GetPreheaderBasicBlock(), ir_builder_);
+  SetToFirstInsertPoint(inner_loop->GetPreheaderBasicBlock(), b_);
   PrimitiveType primitive_type = hlo->shape().element_type();
   llvm::Type* primitive_type_llvm =
       llvm_ir::PrimitiveTypeToIrType(primitive_type, module_);
-  llvm::Value* accumulator_alloca = llvm_ir::EmitAllocaAtFunctionEntry(
-      primitive_type_llvm, "dot_acc", ir_builder_);
-  ir_builder_->CreateStore(llvm::Constant::getNullValue(primitive_type_llvm),
-                           accumulator_alloca);
+  llvm::Value* accumulator_alloca =
+      llvm_ir::EmitAllocaAtFunctionEntry(primitive_type_llvm, "dot_acc", b_);
+  b_->CreateStore(llvm::Constant::getNullValue(primitive_type_llvm),
+                  accumulator_alloca);
 
-  SetToFirstInsertPoint(inner_loop->GetBodyBasicBlock(), ir_builder_);
+  SetToFirstInsertPoint(inner_loop->GetBodyBasicBlock(), b_);
 
   // This is the inner reduction loop for a dot operation that produces
   // one element in the output.  If the operands to the dot operation have
@@ -1902,43 +1833,36 @@ StatusOr ElementalIrEmitter::EmitElementalDot(
   }
   rhs_index.InsertAt(rhs_contracting_dim, inner_loop->GetIndVarValue());
 
-  llvm::Value* current_accumulator =
-      ir_builder_->CreateLoad(accumulator_alloca);
+  llvm::Value* current_accumulator = b_->CreateLoad(accumulator_alloca);
   TF_ASSIGN_OR_RETURN(llvm::Value * lhs_value, lhs_generator(lhs_index));
   TF_ASSIGN_OR_RETURN(llvm::Value * rhs_value, rhs_generator(rhs_index));
   llvm::Value* next_accumulator;
   if (primitive_util::IsComplexType(primitive_type)) {
-    llvm::Value* product_real = ir_builder_->CreateFSub(
-        ir_builder_->CreateFMul(EmitExtractReal(lhs_value),
-                                EmitExtractReal(rhs_value)),
-        ir_builder_->CreateFMul(EmitExtractImag(lhs_value),
-                                EmitExtractImag(rhs_value)));
-    llvm::Value* product_imag = ir_builder_->CreateFAdd(
-        ir_builder_->CreateFMul(EmitExtractReal(lhs_value),
-                                EmitExtractImag(rhs_value)),
-        ir_builder_->CreateFMul(EmitExtractImag(lhs_value),
-                                EmitExtractReal(rhs_value)));
-    next_accumulator = ir_builder_->CreateInsertValue(
+    llvm::Value* product_real = b_->CreateFSub(
+        b_->CreateFMul(EmitExtractReal(lhs_value), EmitExtractReal(rhs_value)),
+        b_->CreateFMul(EmitExtractImag(lhs_value), EmitExtractImag(rhs_value)));
+    llvm::Value* product_imag = b_->CreateFAdd(
+        b_->CreateFMul(EmitExtractReal(lhs_value), EmitExtractImag(rhs_value)),
+        b_->CreateFMul(EmitExtractImag(lhs_value), EmitExtractReal(rhs_value)));
+    next_accumulator = b_->CreateInsertValue(
         current_accumulator,
-        ir_builder_->CreateFAdd(EmitExtractReal(current_accumulator),
-                                product_real),
+        b_->CreateFAdd(EmitExtractReal(current_accumulator), product_real),
         {0});
-    next_accumulator = ir_builder_->CreateInsertValue(
+    next_accumulator = b_->CreateInsertValue(
         next_accumulator,
-        ir_builder_->CreateFAdd(EmitExtractImag(current_accumulator),
-                                product_imag),
+        b_->CreateFAdd(EmitExtractImag(current_accumulator), product_imag),
         {1});
   } else if (primitive_util::IsFloatingPointType(primitive_type)) {
-    next_accumulator = ir_builder_->CreateFAdd(
-        current_accumulator, ir_builder_->CreateFMul(lhs_value, rhs_value));
+    next_accumulator = b_->CreateFAdd(current_accumulator,
+                                      b_->CreateFMul(lhs_value, rhs_value));
   } else {
-    next_accumulator = ir_builder_->CreateAdd(
-        current_accumulator, ir_builder_->CreateMul(lhs_value, rhs_value));
+    next_accumulator =
+        b_->CreateAdd(current_accumulator, b_->CreateMul(lhs_value, rhs_value));
   }
-  ir_builder_->CreateStore(next_accumulator, accumulator_alloca);
+  b_->CreateStore(next_accumulator, accumulator_alloca);
 
-  SetToFirstInsertPoint(inner_loop->GetExitBasicBlock(), ir_builder_);
-  return ir_builder_->CreateLoad(accumulator_alloca);
+  SetToFirstInsertPoint(inner_loop->GetExitBasicBlock(), b_);
+  return b_->CreateLoad(accumulator_alloca);
 }
 
 llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
@@ -2038,7 +1962,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
         const HloInstruction* operand = hlo->operand(0);
         auto source_index = target_index;
         for (int64 dim : hlo->dimensions()) {
-          source_index[dim] = ir_builder_->CreateSub(
+          source_index[dim] = b_->CreateSub(
               llvm::ConstantInt::get(target_index[dim]->getType(),
                                      hlo->shape().dimensions(dim) - 1),
               target_index[dim]);
@@ -2051,16 +1975,16 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
         const HloInstruction* operand = hlo->operand(0);
         // The `dimensions` member of the broadcast instruction maps from
         // input dimensions to output dimensions.
-        return operand_to_generator.at(
-            operand)(target_index.SourceIndexOfBroadcast(
-            hlo->shape(), operand->shape(), hlo->dimensions(), ir_builder_));
+        return operand_to_generator.at(operand)(
+            target_index.SourceIndexOfBroadcast(hlo->shape(), operand->shape(),
+                                                hlo->dimensions(), b_));
       };
     case HloOpcode::kSlice:
       return [this, hlo, &operand_to_generator](
                  const IrArray::Index& index) -> StatusOr {
         IrArray::Index sliced_index = index.SourceIndexOfSlice(
             /*shape=*/hlo->shape(), /*starts=*/hlo->slice_starts(),
-            /*strides=*/hlo->slice_strides(), /*builder=*/ir_builder_);
+            /*strides=*/hlo->slice_strides(), /*builder=*/b_);
         return operand_to_generator.at(hlo->operand(0))(sliced_index);
       };
     case HloOpcode::kDynamicSlice:
@@ -2085,24 +2009,23 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
                ShapeUtil::ElementsIn(hlo->operand(0)->shape()));
       return [this, hlo, &operand_to_generator](const IrArray::Index& index) {
         const HloInstruction* operand = hlo->operand(0);
-        return operand_to_generator.at(operand)(index.SourceIndexOfBitcast(
-            hlo->shape(), operand->shape(), ir_builder_));
+        return operand_to_generator.at(operand)(
+            index.SourceIndexOfBitcast(hlo->shape(), operand->shape(), b_));
       };
     case HloOpcode::kReshape:
       CHECK_EQ(ShapeUtil::ElementsIn(hlo->shape()),
                ShapeUtil::ElementsIn(hlo->operand(0)->shape()));
       return [this, hlo, &operand_to_generator](const IrArray::Index& index) {
         const HloInstruction* operand = hlo->operand(0);
-        return operand_to_generator.at(operand)(index.SourceIndexOfReshape(
-            hlo->shape(), operand->shape(), ir_builder_));
+        return operand_to_generator.at(operand)(
+            index.SourceIndexOfReshape(hlo->shape(), operand->shape(), b_));
       };
     case HloOpcode::kTranspose:
       return [this, hlo,
               &operand_to_generator](const IrArray::Index& target_index) {
         return operand_to_generator.at(hlo->operand(0))(
             target_index.SourceIndexOfTranspose(
-                hlo->shape(), hlo->operand(0)->shape(), hlo->dimensions(),
-                ir_builder_));
+                hlo->shape(), hlo->operand(0)->shape(), hlo->dimensions(), b_));
       };
     case HloOpcode::kRng:
       return MakeRngElementGenerator(hlo, operand_to_generator);
@@ -2127,11 +2050,11 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
 }
 
 llvm::Value* ElementalIrEmitter::EmitExtractReal(llvm::Value* value) const {
-  return ir_builder_->CreateExtractValue(value, {0});
+  return b_->CreateExtractValue(value, {0});
 }
 
 llvm::Value* ElementalIrEmitter::EmitExtractImag(llvm::Value* value) const {
-  return ir_builder_->CreateExtractValue(value, {1});
+  return b_->CreateExtractValue(value, {1});
 }
 
 llvm::Value* ElementalIrEmitter::EmitComposeComplex(const HloInstruction* op,
@@ -2139,10 +2062,10 @@ llvm::Value* ElementalIrEmitter::EmitComposeComplex(const HloInstruction* op,
                                                     llvm::Value* imag) const {
   auto cplx_type =
       llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_);
-  auto complex = ir_builder_->CreateInsertValue(
+  auto complex = b_->CreateInsertValue(
       llvm::ConstantAggregateZero::get(cplx_type), real, {0});
   if (imag != nullptr) {
-    complex = ir_builder_->CreateInsertValue(complex, imag, {1});
+    complex = b_->CreateInsertValue(complex, imag, {1});
   }
   return complex;
 }
diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.h b/tensorflow/compiler/xla/service/elemental_ir_emitter.h
index d199473374..deba6bea0a 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.h
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.h
@@ -34,10 +34,8 @@ class ElementalIrEmitter {
       std::unordered_map;
 
   ElementalIrEmitter(const HloModuleConfig& hlo_module_config,
-                     llvm::Module* module, llvm::IRBuilder<>* ir_builder)
-      : ir_builder_(ir_builder),
-        module_(module),
-        hlo_module_config_(hlo_module_config) {}
+                     llvm::Module* module, llvm::IRBuilder<>* b)
+      : b_(b), module_(module), hlo_module_config_(hlo_module_config) {}
 
   virtual ~ElementalIrEmitter() = default;
 
@@ -54,7 +52,7 @@ class ElementalIrEmitter {
       const HloInstruction* hlo,
       const HloToElementGeneratorMap& operand_to_generator) const;
 
-  llvm::IRBuilder<>* ir_builder() const { return ir_builder_; }
+  llvm::IRBuilder<>* b() const { return b_; }
   llvm::Module* module() const { return module_; }
 
  protected:
@@ -144,9 +142,7 @@ class ElementalIrEmitter {
       int64 operand_no) const;
 
   // Identifier of the thread unique among all threads on the device
-  virtual llvm::Value* EmitThreadId() const {
-    return ir_builder_->getIntN(128, 0);
-  }
+  virtual llvm::Value* EmitThreadId() const { return b_->getIntN(128, 0); }
 
   StatusOr EmitElementalSelect(
       const HloInstruction* hlo,
@@ -188,7 +184,7 @@ class ElementalIrEmitter {
       const HloToElementGeneratorMap& operand_to_generator,
       const llvm_ir::IrArray::Index& dot_result_index) const;
 
-  llvm::IRBuilder<>* const ir_builder_;
+  llvm::IRBuilder<>* const b_;
 
   llvm::Module* module_;
 
diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
index e594cec2f8..b97a627d9b 100644
--- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
@@ -67,8 +67,8 @@ bool IsFPLiteralWithValue(const HloInstruction* operand, float value) {
 
 GpuElementalIrEmitter::GpuElementalIrEmitter(
     const HloModuleConfig& hlo_module_config, llvm::Module* module,
-    llvm::IRBuilder<>* ir_builder, NestedComputer compute_nested)
-    : ElementalIrEmitter(hlo_module_config, module, ir_builder),
+    llvm::IRBuilder<>* b, NestedComputer compute_nested)
+    : ElementalIrEmitter(hlo_module_config, module, b),
       hlo_module_config_(hlo_module_config),
       compute_nested_(std::move(compute_nested)) {}
 
@@ -92,8 +92,8 @@ StatusOr GpuElementalIrEmitter::EmitLibdeviceMathCall(
       cast_result_to_fp16 = true;
       for (int64 i = 0; i < operands.size(); ++i) {
         if (input_types[i] == F16) {
-          converted_operands[i] = ir_builder_->CreateFPCast(
-              converted_operands[i], ir_builder_->getFloatTy());
+          converted_operands[i] =
+              b_->CreateFPCast(converted_operands[i], b_->getFloatTy());
           converted_input_types[i] = F32;
         }
       }
@@ -112,7 +112,7 @@ StatusOr GpuElementalIrEmitter::EmitLibdeviceMathCall(
                                      converted_input_types, output_type)
                             .ValueOrDie();
   if (cast_result_to_fp16) {
-    result = ir_builder_->CreateFPCast(result, ir_builder_->getHalfTy());
+    result = b_->CreateFPCast(result, b_->getHalfTy());
   }
   return result;
 }
@@ -215,7 +215,7 @@ StatusOr GpuElementalIrEmitter::EmitPowerOp(
     // LLVM's NVPTX backend knows how to transform 1/sqrt(A) into the NVPTX
     // rsqrt.approx instruction.
     TF_ASSIGN_OR_RETURN(auto* sqrt, make_sqrt());
-    return ir_builder_->CreateFDiv(llvm::ConstantFP::get(llvm_ty, 1), sqrt);
+    return b_->CreateFDiv(llvm::ConstantFP::get(llvm_ty, 1), sqrt);
   }
 
   VLOG(10) << "emitting pow as regular call to pow(): " << op->ToString();
@@ -302,32 +302,31 @@ llvm::Value* GpuElementalIrEmitter::EmitDeviceFunctionCall(
 
   // Declares the callee if it is not declared already.
   llvm::Function* callee = llvm::cast(
-      ir_builder_->GetInsertBlock()->getModule()->getOrInsertFunction(
+      b_->GetInsertBlock()->getModule()->getOrInsertFunction(
           llvm_ir::AsStringRef(callee_name), callee_type));
 
   for (auto attribute : attributes) {
     callee->addFnAttr(attribute);
   }
 
-  return ir_builder_->CreateCall(callee, llvm_ir::AsArrayRef(operands));
+  return b_->CreateCall(callee, llvm_ir::AsArrayRef(operands));
 }
 
 llvm::Value* GpuElementalIrEmitter::EmitThreadId() const {
-  llvm::Value* block_id = ir_builder_->CreateIntCast(
+  llvm::Value* block_id = b_->CreateIntCast(
       llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_read_ptx_sreg_ctaid_x,
-                                   {}, {}, ir_builder_),
-      ir_builder_->getIntNTy(128), /*isSigned=*/true, "block.id");
-  llvm::Value* thread_id_in_block = ir_builder_->CreateIntCast(
+                                   {}, {}, b_),
+      b_->getIntNTy(128), /*isSigned=*/true, "block.id");
+  llvm::Value* thread_id_in_block = b_->CreateIntCast(
       llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x,
-                                   {}, {}, ir_builder_),
-      ir_builder_->getIntNTy(128), /*isSigned=*/true, "thread.id");
-  llvm::Value* threads_per_block = ir_builder_->CreateIntCast(
+                                   {}, {}, b_),
+      b_->getIntNTy(128), /*isSigned=*/true, "thread.id");
+  llvm::Value* threads_per_block = b_->CreateIntCast(
       llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_read_ptx_sreg_ntid_x,
-                                   {}, {}, ir_builder_),
-      ir_builder_->getIntNTy(128), /*isSigned=*/true, "threads_per_block");
-  return ir_builder_->CreateNSWAdd(
-      ir_builder_->CreateNSWMul(block_id, threads_per_block),
-      thread_id_in_block);
+                                   {}, {}, b_),
+      b_->getIntNTy(128), /*isSigned=*/true, "threads_per_block");
+  return b_->CreateNSWAdd(b_->CreateNSWMul(block_id, threads_per_block),
+                          thread_id_in_block);
 }
 
 llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator(
@@ -373,12 +372,12 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator(
         PrimitiveType operand_element_type = operand->shape().element_type();
         llvm::Value* accum_ptr = llvm_ir::EmitAllocaAtFunctionEntry(
             llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_),
-            "reduce_window_accum_ptr", ir_builder_);
+            "reduce_window_accum_ptr", b_);
         {
           TF_ASSIGN_OR_RETURN(llvm::Value * init_value,
                               operand_to_generator.at(hlo->operand(1))(
                                   IrArray::Index(index.GetType())));
-          ir_builder_->CreateStore(init_value, accum_ptr);
+          b_->CreateStore(init_value, accum_ptr);
         }
 
         llvm::Type* index_type = index.GetType();
@@ -386,7 +385,7 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator(
           return index.GetConstantWithIndexType(c);
         };
 
-        llvm_ir::ForLoopNest loops(IrName(hlo), ir_builder_, index_type);
+        llvm_ir::ForLoopNest loops(IrName(hlo), b_, index_type);
         std::vector window_size;
         for (const auto& dim : window.dimensions()) {
           window_size.push_back(dim.size());
@@ -395,15 +394,15 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator(
             ShapeUtil::MakeShape(operand_element_type, window_size), "window");
         CHECK_EQ(window_index.size(), index.size());
 
-        SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), ir_builder_);
+        SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), b_);
 
         IrArray::Index input_index(index_type, index.size());
-        llvm::Value* in_bounds = ir_builder_->getInt1(true);
+        llvm::Value* in_bounds = b_->getInt1(true);
         for (size_t i = 0; i < index.size(); ++i) {
-          llvm::Value* stridden_index = ir_builder_->CreateNSWMul(
+          llvm::Value* stridden_index = b_->CreateNSWMul(
               index[i], index_typed_const(window.dimensions(i).stride()));
-          input_index[i] = ir_builder_->CreateNSWSub(
-              ir_builder_->CreateNSWAdd(stridden_index, window_index[i]),
+          input_index[i] = b_->CreateNSWSub(
+              b_->CreateNSWAdd(stridden_index, window_index[i]),
               index_typed_const(window.dimensions(i).padding_low()));
 
           // We must check whether 0 ≤ input_index[i] < bound, as otherwise
@@ -411,16 +410,16 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator(
           // comparison is equivalent to the unsigned comparison
           // input_index[i] < bound, as a negative value wraps to a large
           // positive value.
-          in_bounds = ir_builder_->CreateAnd(
+          in_bounds = b_->CreateAnd(
               in_bounds,
-              ir_builder_->CreateICmpULT(
+              b_->CreateICmpULT(
                   input_index[i],
                   index_typed_const(operand->shape().dimensions(i))));
         }
 
         llvm_ir::LlvmIfData if_data =
-            llvm_ir::EmitIfThenElse(in_bounds, "in_bounds", ir_builder_);
-        SetToFirstInsertPoint(if_data.true_block, ir_builder_);
+            llvm_ir::EmitIfThenElse(in_bounds, "in_bounds", b_);
+        SetToFirstInsertPoint(if_data.true_block, b_);
 
         // We are not in pad, so do the computation.
         TF_ASSIGN_OR_RETURN(llvm::Value * input_value,
@@ -428,26 +427,26 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator(
         TF_ASSIGN_OR_RETURN(
             llvm::Value * accum_value,
             compute_nested_(*hlo->to_apply(),
-                            {ir_builder_->CreateLoad(accum_ptr), input_value}));
-        ir_builder_->CreateStore(accum_value, accum_ptr);
+                            {b_->CreateLoad(accum_ptr), input_value}));
+        b_->CreateStore(accum_value, accum_ptr);
 
-        SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), ir_builder_);
-        return ir_builder_->CreateLoad(accum_ptr);
+        SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), b_);
+        return b_->CreateLoad(accum_ptr);
       };
     case HloOpcode::kReduce:
       return [=, &operand_to_generator](
                  const IrArray::Index& output_index) -> StatusOr {
         const HloInstruction* operand = hlo->operand(0);
         llvm::Value* accum_ptr =
-            ir_builder()->CreateAlloca(llvm_ir::PrimitiveTypeToIrType(
+            b()->CreateAlloca(llvm_ir::PrimitiveTypeToIrType(
                 hlo->shape().element_type(), module_));
         llvm::Type* index_type = output_index.GetType();
         TF_ASSIGN_OR_RETURN(llvm::Value * init_value,
                             operand_to_generator.at(hlo->operand(1))(
                                 IrArray::Index(index_type)));
-        ir_builder()->CreateStore(init_value, accum_ptr);
+        b()->CreateStore(init_value, accum_ptr);
 
-        llvm_ir::ForLoopNest loops(IrName(hlo), ir_builder_, index_type);
+        llvm_ir::ForLoopNest loops(IrName(hlo), b_, index_type);
         IrArray::Index input_index = loops.AddLoopsForShapeOnDimensions(
             operand->shape(), hlo->dimensions(), "reduction_dim");
         if (!ShapeUtil::IsScalar(hlo->shape())) {
@@ -462,18 +461,17 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator(
           CHECK_EQ(output_index.size(), j);
         }
 
-        SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), ir_builder());
+        SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), b());
         TF_ASSIGN_OR_RETURN(
             llvm::Value * input_value,
             operand_to_generator.at(hlo->operand(0))(input_index));
         TF_ASSIGN_OR_RETURN(
             llvm::Value * accum_value,
-            compute_nested_(
-                *hlo->to_apply(),
-                {ir_builder()->CreateLoad(accum_ptr), input_value}));
-        ir_builder()->CreateStore(accum_value, accum_ptr);
-        SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), ir_builder());
-        return ir_builder()->CreateLoad(accum_ptr);
+            compute_nested_(*hlo->to_apply(),
+                            {b()->CreateLoad(accum_ptr), input_value}));
+        b()->CreateStore(accum_value, accum_ptr);
+        SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), b());
+        return b()->CreateLoad(accum_ptr);
       };
     default:
       return ElementalIrEmitter::MakeElementGenerator(hlo,
diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h
index 91f4d960aa..e3eacef133 100644
--- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h
@@ -43,7 +43,7 @@ class GpuElementalIrEmitter : public ElementalIrEmitter {
       const HloComputation&, tensorflow::gtl::ArraySlice)>;
 
   GpuElementalIrEmitter(const HloModuleConfig& hlo_module_config,
-                        llvm::Module* module, llvm::IRBuilder<>* ir_builder,
+                        llvm::Module* module, llvm::IRBuilder<>* b,
                         NestedComputer compute_nested);
 
   llvm_ir::ElementGenerator MakeElementGenerator(
diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
index 6f2a7e1850..1b6315ec03 100644
--- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
+++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
@@ -39,7 +39,7 @@ void HloToIrBindings::EmitBasePointersForHlos(
   // I/O HLOs are bound to the arguments of the current IR function. I.e.,
   //
   // void IrFunction(io_0, io_1, ..., io_{m-1}, temp_buffer_base) {
-  llvm::Function* function = ir_builder_->GetInsertBlock()->getParent();
+  llvm::Function* function = b_->GetInsertBlock()->getParent();
   CHECK_EQ(io_hlos.size() + 1, function->arg_size());
 
   // An HLO can have duplicated operands. This data structure remembers which
@@ -79,8 +79,8 @@ void HloToIrBindings::EmitBasePointersForHlos(
         const int64 offset = slice.offset();
         CHECK_NE(nullptr, temp_buffer_base_);
         // Emit IR for GetTupleElement instruction and bind to emitted value.
-        llvm::Value* base_ptr = ir_builder_->CreateInBoundsGEP(
-            temp_buffer_base_, ir_builder_->getInt64(offset));
+        llvm::Value* base_ptr =
+            b_->CreateInBoundsGEP(temp_buffer_base_, b_->getInt64(offset));
         BindHloToIrValue(*non_io_hlo,
                          EmitGetTupleElement(non_io_hlo, base_ptr));
       }
@@ -108,15 +108,14 @@ void HloToIrBindings::EmitBasePointersForHlos(
           if (slice.allocation()->is_thread_local()) {
             llvm::Type* pointee_type =
                 llvm_ir::ShapeToIrType(non_io_hlo->shape(), module_);
-            BindHloToIrValue(*non_io_hlo,
-                             ir_builder_->CreateAlloca(pointee_type), index);
+            BindHloToIrValue(*non_io_hlo, b_->CreateAlloca(pointee_type),
+                             index);
           } else {
             const int64 offset = slice.offset();
             CHECK_NE(nullptr, temp_buffer_base_);
             BindHloToIrValue(
                 *non_io_hlo,
-                ir_builder_->CreateInBoundsGEP(temp_buffer_base_,
-                                               ir_builder_->getInt64(offset)),
+                b_->CreateInBoundsGEP(temp_buffer_base_, b_->getInt64(offset)),
                 index);
           }
         });
@@ -129,11 +128,11 @@ llvm::Value* HloToIrBindings::EmitGetTupleElement(const HloInstruction* gte,
   if (gte->operand(0)->opcode() != HloOpcode::kGetTupleElement) {
     return llvm_ir::EmitGetTupleElement(
         gte->shape(), gte->tuple_index(), /*alignment=*/1,
-        GetTypedIrValue(*gte->operand(0), {}, base_ptr), ir_builder_, module_);
+        GetTypedIrValue(*gte->operand(0), {}, base_ptr), b_, module_);
   }
   return llvm_ir::EmitGetTupleElement(
       gte->shape(), gte->tuple_index(), /*alignment=*/1,
-      EmitGetTupleElement(gte->operand(0), base_ptr), ir_builder_, module_);
+      EmitGetTupleElement(gte->operand(0), base_ptr), b_, module_);
 }
 
 llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo,
@@ -148,8 +147,7 @@ llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo,
     typed_ir_value = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
         llvm::cast(ir_value), dest_type);
   } else {
-    typed_ir_value =
-        ir_builder_->CreateBitCast(ir_value, pointee_type->getPointerTo());
+    typed_ir_value = b_->CreateBitCast(ir_value, pointee_type->getPointerTo());
   }
   ir_value->setName(llvm_ir::AsStringRef(llvm_ir::IrName(&hlo, "raw")));
   typed_ir_value->setName(llvm_ir::AsStringRef(llvm_ir::IrName(&hlo, "typed")));
diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
index a86e6e78c6..eee40b0e91 100644
--- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
+++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
@@ -36,14 +36,13 @@ class HloToIrBindings {
  public:
   HloToIrBindings(const HloModule& module,
                   const BufferAssignment* buffer_assignment,
-                  llvm::IRBuilder<>* ir_builder, llvm::Module* llvm_module,
+                  llvm::IRBuilder<>* b, llvm::Module* llvm_module,
                   bool is_nested)
       : buffer_assignment_(buffer_assignment),
         is_nested_(is_nested),
-        ir_builder_(ir_builder),
+        b_(b),
         module_(llvm_module),
-        alias_analysis_(module, *buffer_assignment_,
-                        &ir_builder_->getContext()) {}
+        alias_analysis_(module, *buffer_assignment_, &b_->getContext()) {}
 
   void EmitBasePointersForHlos(
       tensorflow::gtl::ArraySlice io_hlos,
@@ -104,7 +103,7 @@ class HloToIrBindings {
 
   const bool is_nested_;
 
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
   llvm::Module* module_;
 
   // Stores the underlying llvm::IrArray for each HloInstruction.
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
index d7e8be1cf8..76180cf486 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
@@ -58,12 +58,12 @@ IrEmitter::IrEmitter(const HloModuleConfig& hlo_module_config,
                      IrEmitterContext* ir_emitter_context, bool is_nested)
     : ir_emitter_context_(ir_emitter_context),
       module_(ir_emitter_context->llvm_module()),
-      ir_builder_(module_->getContext()),
+      b_(module_->getContext()),
       bindings_(ir_emitter_context->hlo_module(),
-                &ir_emitter_context->buffer_assignment(), &ir_builder_, module_,
+                &ir_emitter_context->buffer_assignment(), &b_, module_,
                 is_nested),
       hlo_module_config_(hlo_module_config) {
-  ir_builder_.setFastMathFlags(llvm_ir::GetFastMathFlags(
+  b_.setFastMathFlags(llvm_ir::GetFastMathFlags(
       /*fast_math_enabled=*/hlo_module_config.debug_options()
           .xla_enable_fast_math()));
 }
@@ -72,12 +72,11 @@ Status IrEmitter::DefaultAction(HloInstruction* hlo) {
   ElementalIrEmitter::HloToElementGeneratorMap operand_to_generator;
   for (const HloInstruction* operand : hlo->operands()) {
     operand_to_generator[operand] = [=](const llvm_ir::IrArray::Index& index) {
-      return GetIrArray(*operand, *hlo)
-          .EmitReadArrayElement(index, &ir_builder_);
+      return GetIrArray(*operand, *hlo).EmitReadArrayElement(index, &b_);
     };
   }
   return EmitTargetElementLoop(
-      *hlo, GpuElementalIrEmitter(hlo_module_config_, module_, &ir_builder_,
+      *hlo, GpuElementalIrEmitter(hlo_module_config_, module_, &b_,
                                   GetNestedComputer())
                 .MakeElementGenerator(hlo, operand_to_generator));
 }
@@ -120,7 +119,7 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) {
           get_tuple_element->shape(), get_tuple_element->tuple_index(),
           // TODO(b/26344050): tighten the alignment here
           // based on the real element type.
-          /*alignment=*/1, GetBasePointer(*operand), &ir_builder_, module_));
+          /*alignment=*/1, GetBasePointer(*operand), &b_, module_));
   return Status::OK();
 }
 
@@ -132,7 +131,7 @@ Status IrEmitter::HandleSort(HloInstruction* sort) {
   }
   int dimension_to_sort = sort->dimensions(0);
   return llvm_ir::EmitSortInPlace(dimension_to_sort, GetIrArray(*sort, *sort),
-                                  IrName(sort), &ir_builder_);
+                                  IrName(sort), &b_);
 }
 
 Status IrEmitter::HandleSend(HloInstruction*) {
@@ -156,8 +155,7 @@ Status IrEmitter::HandleTuple(HloInstruction* tuple) {
   for (const HloInstruction* operand : tuple->operands()) {
     base_ptrs.push_back(GetBasePointer(*operand));
   }
-  llvm_ir::EmitTuple(GetIrArray(*tuple, *tuple), base_ptrs, &ir_builder_,
-                     module_);
+  llvm_ir::EmitTuple(GetIrArray(*tuple, *tuple), base_ptrs, &b_, module_);
   return Status::OK();
 }
 
@@ -178,7 +176,7 @@ Status IrEmitter::EmitCallToNestedComputation(
   std::vector arguments(operands.begin(), operands.end());
   arguments.push_back(output);
   arguments.push_back(bindings_.GetTempBufferBase());
-  ir_builder_.CreateCall(emitted_function, arguments);
+  b_.CreateCall(emitted_function, arguments);
 
   return Status::OK();
 }
@@ -200,21 +198,20 @@ bool IrEmitter::MaybeEmitDirectAtomicOperation(
       computation.root_instruction()->shape().element_type();
   bool is_atomic_integral = element_type == S32 || element_type == U32 ||
                             element_type == S64 || element_type == U64;
-  llvm::Value* source = ir_builder_.CreateLoad(source_address, "source");
+  llvm::Value* source = b_.CreateLoad(source_address, "source");
   if (root_opcode == HloOpcode::kAdd) {
     // NVPTX supports atomicAdd on F32 and integer types.
     if (element_type == F32) {
       // F32 + F32
       llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_atomic_load_add_f32,
                                    {output_address, source},
-                                   {output_address->getType()}, &ir_builder_);
+                                   {output_address->getType()}, &b_);
       return true;
     }
     if (is_atomic_integral) {
       // integral + integral
-      ir_builder_.CreateAtomicRMW(llvm::AtomicRMWInst::Add, output_address,
-                                  source,
-                                  llvm::AtomicOrdering::SequentiallyConsistent);
+      b_.CreateAtomicRMW(llvm::AtomicRMWInst::Add, output_address, source,
+                         llvm::AtomicOrdering::SequentiallyConsistent);
       return true;
     }
   }
@@ -225,8 +222,8 @@ bool IrEmitter::MaybeEmitDirectAtomicOperation(
     auto opcode = primitive_util::IsSignedIntegralType(element_type)
                       ? llvm::AtomicRMWInst::Max
                       : llvm::AtomicRMWInst::UMax;
-    ir_builder_.CreateAtomicRMW(opcode, output_address, source,
-                                llvm::AtomicOrdering::SequentiallyConsistent);
+    b_.CreateAtomicRMW(opcode, output_address, source,
+                       llvm::AtomicOrdering::SequentiallyConsistent);
     return true;
   }
 
@@ -235,8 +232,8 @@ bool IrEmitter::MaybeEmitDirectAtomicOperation(
     auto opcode = primitive_util::IsSignedIntegralType(element_type)
                       ? llvm::AtomicRMWInst::Min
                       : llvm::AtomicRMWInst::UMin;
-    ir_builder_.CreateAtomicRMW(opcode, output_address, source,
-                                llvm::AtomicOrdering::SequentiallyConsistent);
+    b_.CreateAtomicRMW(opcode, output_address, source,
+                       llvm::AtomicOrdering::SequentiallyConsistent);
     return true;
   }
 
@@ -308,20 +305,20 @@ Status IrEmitter::EmitAtomicOperationUsingCAS(const HloComputation& computation,
   llvm::Type* element_address_type = element_type->getPointerTo();
 
   int atomic_size = (element_size < 32) ? 32 : element_size;
-  llvm::Type* atomic_type = ir_builder_.getIntNTy(atomic_size);
+  llvm::Type* atomic_type = b_.getIntNTy(atomic_size);
   llvm::Type* atomic_address_type =
       atomic_type->getPointerTo(output_address_type->getPointerAddressSpace());
 
   // cas_old_output_address and cas_new_output_address point to the scratch
   // memory where we store the old and new values for the repeated atomicCAS
   // operations.
-  llvm::Value* cas_old_output_address = ir_builder_.CreateAlloca(
+  llvm::Value* cas_old_output_address = b_.CreateAlloca(
       atomic_type, /*ArraySize=*/nullptr, "cas_old_output_address");
-  llvm::Value* cas_new_output_address = ir_builder_.CreateAlloca(
+  llvm::Value* cas_new_output_address = b_.CreateAlloca(
       atomic_type, /*ArraySize=*/nullptr, "cas_new_output_address");
 
   // Emit preparation code to the preheader.
-  llvm::BasicBlock* loop_preheader_bb = ir_builder_.GetInsertBlock();
+  llvm::BasicBlock* loop_preheader_bb = b_.GetInsertBlock();
 
   llvm::Value* atomic_memory_address;
   // binop_output_address points to the scratch memory that stores the
@@ -332,77 +329,71 @@ Status IrEmitter::EmitAtomicOperationUsingCAS(const HloComputation& computation,
     CHECK_EQ((element_size % sizeof(char)), 0);
     llvm::Type* address_int_type =
         module_->getDataLayout().getIntPtrType(output_address_type);
-    atomic_memory_address =
-        ir_builder_.CreatePtrToInt(output_address, address_int_type);
+    atomic_memory_address = b_.CreatePtrToInt(output_address, address_int_type);
     llvm::Value* mask = llvm::ConstantInt::get(address_int_type, 3);
-    llvm::Value* offset = ir_builder_.CreateAnd(atomic_memory_address, mask);
+    llvm::Value* offset = b_.CreateAnd(atomic_memory_address, mask);
     mask = llvm::ConstantInt::get(address_int_type, -4);
-    atomic_memory_address = ir_builder_.CreateAnd(atomic_memory_address, mask);
+    atomic_memory_address = b_.CreateAnd(atomic_memory_address, mask);
     atomic_memory_address =
-        ir_builder_.CreateIntToPtr(atomic_memory_address, atomic_address_type);
-    binop_output_address = ir_builder_.CreateAdd(
-        ir_builder_.CreatePtrToInt(cas_new_output_address, address_int_type),
-        offset);
+        b_.CreateIntToPtr(atomic_memory_address, atomic_address_type);
+    binop_output_address = b_.CreateAdd(
+        b_.CreatePtrToInt(cas_new_output_address, address_int_type), offset);
     binop_output_address =
-        ir_builder_.CreateIntToPtr(binop_output_address, element_address_type);
+        b_.CreateIntToPtr(binop_output_address, element_address_type);
   } else {
     atomic_memory_address =
-        ir_builder_.CreateBitCast(output_address, atomic_address_type);
+        b_.CreateBitCast(output_address, atomic_address_type);
     binop_output_address =
-        ir_builder_.CreateBitCast(cas_new_output_address, element_address_type);
+        b_.CreateBitCast(cas_new_output_address, element_address_type);
   }
 
   // Use the value from the memory that atomicCAS operates on to initialize
   // cas_old_output.
   llvm::Value* cas_old_output =
-      ir_builder_.CreateLoad(atomic_memory_address, "cas_old_output");
-  ir_builder_.CreateStore(cas_old_output, cas_old_output_address);
+      b_.CreateLoad(atomic_memory_address, "cas_old_output");
+  b_.CreateStore(cas_old_output, cas_old_output_address);
 
   llvm::BasicBlock* loop_exit_bb = loop_preheader_bb->splitBasicBlock(
-      ir_builder_.GetInsertPoint(), "atomic_op_loop_exit");
-  llvm::BasicBlock* loop_body_bb =
-      llvm::BasicBlock::Create(ir_builder_.getContext(), "atomic_op_loop_body",
-                               ir_builder_.GetInsertBlock()->getParent());
-  ir_builder_.SetInsertPoint(loop_body_bb);
+      b_.GetInsertPoint(), "atomic_op_loop_exit");
+  llvm::BasicBlock* loop_body_bb = llvm::BasicBlock::Create(
+      b_.getContext(), "atomic_op_loop_body", b_.GetInsertBlock()->getParent());
+  b_.SetInsertPoint(loop_body_bb);
   // Change preheader's successor from loop_exit_bb to loop_body_bb.
   loop_preheader_bb->getTerminator()->setSuccessor(0, loop_body_bb);
 
   // Emit the body of the loop that repeatedly invokes atomicCAS.
   //
   // Use cas_old_output to initialize cas_new_output.
-  cas_old_output =
-      ir_builder_.CreateLoad(cas_old_output_address, "cas_old_output");
-  ir_builder_.CreateStore(cas_old_output, cas_new_output_address);
+  cas_old_output = b_.CreateLoad(cas_old_output_address, "cas_old_output");
+  b_.CreateStore(cas_old_output, cas_new_output_address);
   // Emits code to calculate new_output = operation(old_output, source);
   TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
       computation, {binop_output_address, source_address},
       binop_output_address));
 
   llvm::Value* cas_new_output =
-      ir_builder_.CreateLoad(cas_new_output_address, "cas_new_output");
+      b_.CreateLoad(cas_new_output_address, "cas_new_output");
 
   // Emit code to perform the atomicCAS operation
   // (cas_old_output, success) = atomicCAS(memory_address, cas_old_output,
   //                                       cas_new_output);
-  llvm::Value* ret_value = ir_builder_.CreateAtomicCmpXchg(
+  llvm::Value* ret_value = b_.CreateAtomicCmpXchg(
       atomic_memory_address, cas_old_output, cas_new_output,
       llvm::AtomicOrdering::SequentiallyConsistent,
       llvm::AtomicOrdering::SequentiallyConsistent);
 
   // Extract the memory value returned from atomicCAS and store it as
   // cas_old_output.
-  ir_builder_.CreateStore(
-      ir_builder_.CreateExtractValue(ret_value, 0, "cas_old_output"),
-      cas_old_output_address);
+  b_.CreateStore(b_.CreateExtractValue(ret_value, 0, "cas_old_output"),
+                 cas_old_output_address);
   // Extract the success bit returned from atomicCAS and generate a
   // conditional branch on the success bit.
-  ir_builder_.CreateCondBr(
-      ir_builder_.CreateExtractValue(ret_value, 1, "success"), loop_exit_bb,
-      loop_body_bb);
+  b_.CreateCondBr(b_.CreateExtractValue(ret_value, 1, "success"), loop_exit_bb,
+                  loop_body_bb);
 
   // Set the insertion point to the exit basic block so that the caller of
   // this method can continue emitting code to the right place.
-  SetToFirstInsertPoint(loop_exit_bb, &ir_builder_);
+  SetToFirstInsertPoint(loop_exit_bb, &b_);
   return Status::OK();
 }
 
@@ -445,32 +436,32 @@ Status IrEmitter::HandleTupleSelect(HloInstruction* tuple_select) {
   llvm_ir::EmitTupleSelect(GetIrArray(*tuple_select, *tuple_select),
                            GetIrArray(*pred, *tuple_select),
                            GetBasePointer(*on_true), GetBasePointer(*on_false),
-                           &ir_builder_, module_);
+                           &b_, module_);
   return Status::OK();
 }
 
 namespace {
-llvm::Value* Real(llvm::Value* x, llvm::IRBuilder<>* ir_builder) {
-  return ir_builder->CreateExtractValue(x, {0});
-}
-
-llvm::Value* Imag(llvm::Value* x, llvm::IRBuilder<>* ir_builder) {
-  return ir_builder->CreateExtractValue(x, {1});
-}
-
-std::pair MultiplyComplex(
-    llvm::Value* lhs_value, llvm::Value* rhs_value,
-    llvm::IRBuilder<>* ir_builder) {
-  llvm::Value* lhs_real = Real(lhs_value, ir_builder);
-  llvm::Value* lhs_imag = Imag(lhs_value, ir_builder);
-  llvm::Value* rhs_real = Real(rhs_value, ir_builder);
-  llvm::Value* rhs_imag = Imag(rhs_value, ir_builder);
-  llvm::Value* real_result1 = ir_builder->CreateFMul(lhs_real, rhs_real);
-  llvm::Value* real_result2 = ir_builder->CreateFMul(lhs_imag, rhs_imag);
-  llvm::Value* real_result = ir_builder->CreateFSub(real_result1, real_result2);
-  llvm::Value* imag_result1 = ir_builder->CreateFMul(lhs_real, rhs_imag);
-  llvm::Value* imag_result2 = ir_builder->CreateFMul(lhs_imag, rhs_real);
-  llvm::Value* imag_result = ir_builder->CreateFAdd(imag_result1, imag_result2);
+llvm::Value* Real(llvm::Value* x, llvm::IRBuilder<>* b) {
+  return b->CreateExtractValue(x, {0});
+}
+
+llvm::Value* Imag(llvm::Value* x, llvm::IRBuilder<>* b) {
+  return b->CreateExtractValue(x, {1});
+}
+
+std::pair MultiplyComplex(llvm::Value* lhs_value,
+                                                      llvm::Value* rhs_value,
+                                                      llvm::IRBuilder<>* b) {
+  llvm::Value* lhs_real = Real(lhs_value, b);
+  llvm::Value* lhs_imag = Imag(lhs_value, b);
+  llvm::Value* rhs_real = Real(rhs_value, b);
+  llvm::Value* rhs_imag = Imag(rhs_value, b);
+  llvm::Value* real_result1 = b->CreateFMul(lhs_real, rhs_real);
+  llvm::Value* real_result2 = b->CreateFMul(lhs_imag, rhs_imag);
+  llvm::Value* real_result = b->CreateFSub(real_result1, real_result2);
+  llvm::Value* imag_result1 = b->CreateFMul(lhs_real, rhs_imag);
+  llvm::Value* imag_result2 = b->CreateFMul(lhs_imag, rhs_real);
+  llvm::Value* imag_result = b->CreateFAdd(imag_result1, imag_result2);
   return {real_result, imag_result};
 }
 }  // namespace
@@ -486,25 +477,24 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
   const Shape& rhs_shape = rhs_instruction->shape();
 
   // TODO(b/110211620): Convert to use i32 index_type when it is possible.
-  llvm::Type* index_type = ir_builder_.getInt64Ty();
+  llvm::Type* index_type = b_.getInt64Ty();
   llvm_ir::IrArray::Index element_index(index_type);
   if (ShapeUtil::IsScalar(lhs_shape) && ShapeUtil::IsScalar(rhs_shape)) {
     // If the operands are scalar, don't emit any loops.
     llvm::Value* lhs_value =
-        lhs_array.EmitReadArrayElement(/*index=*/element_index, &ir_builder_);
+        lhs_array.EmitReadArrayElement(/*index=*/element_index, &b_);
     llvm::Value* rhs_value =
-        rhs_array.EmitReadArrayElement(/*index=*/element_index, &ir_builder_);
+        rhs_array.EmitReadArrayElement(/*index=*/element_index, &b_);
     llvm::Value* result;
     if (ShapeUtil::ElementIsComplex(lhs_shape)) {
-      auto value = MultiplyComplex(lhs_value, rhs_value, &ir_builder_);
+      auto value = MultiplyComplex(lhs_value, rhs_value, &b_);
       result = llvm::ConstantAggregateZero::get(lhs_array.GetElementLlvmType());
-      result = ir_builder_.CreateInsertValue(result, value.first, {0});
-      result = ir_builder_.CreateInsertValue(result, value.second, {1});
+      result = b_.CreateInsertValue(result, value.first, {0});
+      result = b_.CreateInsertValue(result, value.second, {1});
     } else {
-      result = ir_builder_.CreateFMul(lhs_value, rhs_value);
+      result = b_.CreateFMul(lhs_value, rhs_value);
     }
-    target_array.EmitWriteArrayElement(/*index=*/element_index, result,
-                                       &ir_builder_);
+    target_array.EmitWriteArrayElement(/*index=*/element_index, result, &b_);
     return Status::OK();
   }
 
@@ -531,7 +521,7 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
   // Create loop nests which loop through the LHS operand dimensions and the RHS
   // operand dimensions. The reduction dimension of the LHS and RHS are handled
   // in a separate innermost loop which performs the sum of products.
-  llvm_ir::ForLoopNest loop_nest(IrName(dot), &ir_builder_);
+  llvm_ir::ForLoopNest loop_nest(IrName(dot), &b_);
   llvm_ir::IrArray::Index lhs_index = loop_nest.EmitOperandArrayLoopNest(
       lhs_array, /*dimension_to_skip=*/lhs_reduction_dimension, "lhs");
   llvm_ir::IrArray::Index rhs_index = loop_nest.EmitOperandArrayLoopNest(
@@ -555,7 +545,7 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
   llvm::Value* accum_address = llvm_ir::EmitAllocaAtFunctionEntry(
       accum_type,       // The pointee type of the alloca instruction.
       "accum_address",  // The name of the alloca instruction.
-      &ir_builder_);
+      &b_);
 
   // Initialize the accumulator in the preheader to zero.
   new llvm::StoreInst(
@@ -569,27 +559,25 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
   //   updated_accum = accum + lhs_element * rhs_element
   //   *accum_address = updated_accum
   TF_RET_CHECK(!reduction_loop->GetBodyBasicBlock()->empty());
-  ir_builder_.SetInsertPoint(
+  b_.SetInsertPoint(
       &*reduction_loop->GetBodyBasicBlock()->getFirstInsertionPt());
-  llvm::Value* lhs_element =
-      lhs_array.EmitReadArrayElement(lhs_index, &ir_builder_);
-  llvm::Value* rhs_element =
-      rhs_array.EmitReadArrayElement(rhs_index, &ir_builder_);
-  llvm::Value* accum = ir_builder_.CreateLoad(accum_address);
+  llvm::Value* lhs_element = lhs_array.EmitReadArrayElement(lhs_index, &b_);
+  llvm::Value* rhs_element = rhs_array.EmitReadArrayElement(rhs_index, &b_);
+  llvm::Value* accum = b_.CreateLoad(accum_address);
   llvm::Value* updated_accum;
   if (ShapeUtil::ElementIsComplex(lhs_shape)) {
-    auto value = MultiplyComplex(lhs_element, rhs_element, &ir_builder_);
-    llvm::Value* accum_real = Real(accum, &ir_builder_);
-    llvm::Value* real_sum = ir_builder_.CreateFAdd(accum_real, value.first);
-    updated_accum = ir_builder_.CreateInsertValue(accum, real_sum, {0});
-    llvm::Value* accum_imag = Imag(accum, &ir_builder_);
-    llvm::Value* imag_sum = ir_builder_.CreateFAdd(accum_imag, value.second);
-    updated_accum = ir_builder_.CreateInsertValue(updated_accum, imag_sum, {1});
+    auto value = MultiplyComplex(lhs_element, rhs_element, &b_);
+    llvm::Value* accum_real = Real(accum, &b_);
+    llvm::Value* real_sum = b_.CreateFAdd(accum_real, value.first);
+    updated_accum = b_.CreateInsertValue(accum, real_sum, {0});
+    llvm::Value* accum_imag = Imag(accum, &b_);
+    llvm::Value* imag_sum = b_.CreateFAdd(accum_imag, value.second);
+    updated_accum = b_.CreateInsertValue(updated_accum, imag_sum, {1});
   } else {
-    llvm::Value* product = ir_builder_.CreateFMul(lhs_element, rhs_element);
-    updated_accum = ir_builder_.CreateFAdd(accum, product);
+    llvm::Value* product = b_.CreateFMul(lhs_element, rhs_element);
+    updated_accum = b_.CreateFAdd(accum, product);
   }
-  ir_builder_.CreateStore(updated_accum, accum_address);
+  b_.CreateStore(updated_accum, accum_address);
 
   // After the reduction loop exits, store the accumulator into the target
   // address. The index into the target address is the concatenation of the rhs
@@ -606,16 +594,15 @@ Status IrEmitter::HandleDot(HloInstruction* dot) {
       target_index.push_back(rhs_index[dimension]);
     }
   }
-  SetToFirstInsertPoint(reduction_loop->GetExitBasicBlock(), &ir_builder_);
+  SetToFirstInsertPoint(reduction_loop->GetExitBasicBlock(), &b_);
   target_array.EmitWriteArrayElement(
       target_index,
-      ir_builder_.CreateLoad(
-          accum_address),  // The value written to the target array.
-      &ir_builder_);
+      b_.CreateLoad(accum_address),  // The value written to the target array.
+      &b_);
 
   // Set the IR builder insert point to the exit basic block of the outer most
   // loop. This ensures later instructions are inserted after this loop nest.
-  ir_builder_.SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
+  b_.SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
 
   return Status::OK();
 }
@@ -657,11 +644,10 @@ Status IrEmitter::HandleReduce(HloInstruction* reduce) {
       [=](const llvm_ir::IrArray::Index& index) -> StatusOr {
         // Initialize an accumulator with init_value.
         llvm::AllocaInst* accumulator_addr =
-            ir_builder_.CreateAlloca(llvm_ir::PrimitiveTypeToIrType(
+            b_.CreateAlloca(llvm_ir::PrimitiveTypeToIrType(
                 reduce->shape().element_type(), module_));
-        ir_builder_.CreateStore(
-            ir_builder_.CreateLoad(GetBasePointer(*init_value)),
-            accumulator_addr);
+        b_.CreateStore(b_.CreateLoad(GetBasePointer(*init_value)),
+                       accumulator_addr);
 
         // The enclosing loops go over all the target elements. Now we have to
         // compute the actual target element. For this, we build a new loop nest
@@ -669,12 +655,12 @@ Status IrEmitter::HandleReduce(HloInstruction* reduce) {
         // AddLoopsForShapeOnDimensions will return an Index where induction
         // Value*s are placed for each dimension in dimensions, and all the rest
         // are nullptrs.
-        llvm_ir::ForLoopNest loops(IrName(reduce, "inner"), &ir_builder_);
+        llvm_ir::ForLoopNest loops(IrName(reduce, "inner"), &b_);
         const llvm_ir::IrArray::Index reduced_dims_index =
             loops.AddLoopsForShapeOnDimensions(arg->shape(), dimensions,
                                                "reduction_dim");
 
-        SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
+        SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &b_);
 
         // Build a full index for the input argument, using reduced_dims_index
         // as the base. In reduced_dims_index only the reduction dimensions are
@@ -693,13 +679,12 @@ Status IrEmitter::HandleReduce(HloInstruction* reduce) {
 
         // Apply the reduction function to the loaded value.
         llvm::Value* input_address =
-            GetIrArray(*arg, *reduce)
-                .EmitArrayElementAddress(input_index, &ir_builder_);
+            GetIrArray(*arg, *reduce).EmitArrayElementAddress(input_index, &b_);
         TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
             *function, {accumulator_addr, input_address}, accumulator_addr));
 
-        SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
-        return ir_builder_.CreateLoad(accumulator_addr);
+        SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &b_);
+        return b_.CreateLoad(accumulator_addr);
       });
 }
 
@@ -712,8 +697,8 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
   for (HloInstruction* operand : fusion->operands()) {
     parameter_arrays.push_back(GetIrArray(*operand, *fusion));
   }
-  GpuElementalIrEmitter elemental_emitter(hlo_module_config_, module_,
-                                          &ir_builder_, GetNestedComputer());
+  GpuElementalIrEmitter elemental_emitter(hlo_module_config_, module_, &b_,
+                                          GetNestedComputer());
   FusedIrEmitter fused_emitter(parameter_arrays, &elemental_emitter);
   TF_RETURN_IF_ERROR(fusion->fused_expression_root()->Accept(&fused_emitter));
 
@@ -747,17 +732,16 @@ Status IrEmitter::HandleRng(HloInstruction* random) {
   ElementalIrEmitter::HloToElementGeneratorMap operand_to_generator;
   for (const HloInstruction* operand : random->operands()) {
     operand_to_generator[operand] = [=](const llvm_ir::IrArray::Index& index) {
-      return GetIrArray(*operand, *random)
-          .EmitReadArrayElement(index, &ir_builder_);
+      return GetIrArray(*operand, *random).EmitReadArrayElement(index, &b_);
     };
   }
   // Emits a single-threaded loop because the loop body generated by the element
   // generator for Rng can't be parallelized (b/32333178).
   return llvm_ir::LoopEmitter(
-             GpuElementalIrEmitter(hlo_module_config_, module_, &ir_builder_,
+             GpuElementalIrEmitter(hlo_module_config_, module_, &b_,
                                    GetNestedComputer())
                  .MakeElementGenerator(random, operand_to_generator),
-             GetIrArray(*random, *random), &ir_builder_)
+             GetIrArray(*random, *random), &b_)
       .EmitLoop(IrName(random));
 }
 
@@ -795,16 +779,16 @@ StatusOr IrEmitter::ComputeNestedElement(
   llvm::Value* return_buffer = llvm_ir::EmitAllocaAtFunctionEntry(
       llvm_ir::PrimitiveTypeToIrType(
           computation.root_instruction()->shape().element_type(), module_),
-      "return_buffer", &ir_builder_);
+      "return_buffer", &b_);
   std::vector parameter_buffers;
   for (llvm::Value* parameter_element : parameter_elements) {
     parameter_buffers.push_back(llvm_ir::EmitAllocaAtFunctionEntry(
-        parameter_element->getType(), "parameter_buffer", &ir_builder_));
-    ir_builder_.CreateStore(parameter_element, parameter_buffers.back());
+        parameter_element->getType(), "parameter_buffer", &b_));
+    b_.CreateStore(parameter_element, parameter_buffers.back());
   }
   TF_RETURN_IF_ERROR(EmitCallToNestedComputation(computation, parameter_buffers,
                                                  return_buffer));
-  return ir_builder_.CreateLoad(return_buffer);
+  return b_.CreateLoad(return_buffer);
 }
 
 }  // namespace gpu
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
index da03ef831b..172d4a4e29 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
@@ -163,7 +163,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
 
   // The following fields track the IR emission state. According to LLVM memory
   // management rules, their memory is owned by the module.
-  llvm::IRBuilder<> ir_builder_;
+  llvm::IRBuilder<> b_;
 
   // Mapping from HLO to its underlying LLVM value.
   HloToIrBindings bindings_;
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc
index c9574c87a3..5c827e5f9c 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc
@@ -70,10 +70,10 @@ llvm::Function* IrEmitterNested::EmitBasePointersForNestedComputation(
     argument_dereferenceable_bytes.push_back(root_size);
   }
   // The base pointer of the memory block for all pre-allocated temp buffers.
-  argument_types.push_back(ir_builder_.getInt8PtrTy());
+  argument_types.push_back(b_.getInt8PtrTy());
 
   llvm::FunctionType* function_type =
-      llvm::FunctionType::get(ir_builder_.getVoidTy(), argument_types, false);
+      llvm::FunctionType::get(b_.getVoidTy(), argument_types, false);
   llvm::Function* function = llvm::Function::Create(
       function_type,                       // The function type.
       llvm::GlobalValue::InternalLinkage,  // The linkage type.
@@ -96,8 +96,7 @@ llvm::Function* IrEmitterNested::EmitBasePointersForNestedComputation(
       llvm::BasicBlock::Create(function->getContext(), "entry", function);
   // Emit a "return void" at entry_bb's end, and sets the insert point before
   // that return instruction.
-  ir_builder_.SetInsertPoint(
-      llvm::ReturnInst::Create(function->getContext(), entry_bb));
+  b_.SetInsertPoint(llvm::ReturnInst::Create(function->getContext(), entry_bb));
 
   std::vector non_io_hlos;
   for (const auto* hlo : nested_computation.instructions()) {
@@ -127,20 +126,17 @@ Status IrEmitterNested::EmitTargetElementLoop(
       target_arrays.push_back(GetIrArray(hlo, hlo, {i}));
     }
     TF_RETURN_IF_ERROR(
-        llvm_ir::LoopEmitter(element_generator, target_arrays, &ir_builder_)
-            .EmitLoop());
+        llvm_ir::LoopEmitter(element_generator, target_arrays, &b_).EmitLoop());
 
     std::vector tuple_operand_ptrs;
     tuple_operand_ptrs.reserve(num_elems);
     for (const llvm_ir::IrArray& array : target_arrays) {
       tuple_operand_ptrs.push_back(array.GetBasePointer());
     }
-    llvm_ir::EmitTuple(GetIrArray(hlo, hlo), tuple_operand_ptrs, &ir_builder_,
-                       module_);
+    llvm_ir::EmitTuple(GetIrArray(hlo, hlo), tuple_operand_ptrs, &b_, module_);
     return Status::OK();
   }
-  return llvm_ir::LoopEmitter(element_generator, GetIrArray(hlo, hlo),
-                              &ir_builder_)
+  return llvm_ir::LoopEmitter(element_generator, GetIrArray(hlo, hlo), &b_)
       .EmitLoop();
 }
 
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 1caf10a6c1..7100c9a08a 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -213,7 +213,7 @@ llvm::Function* IrEmitterUnnested::BuildKernelPrototype(
   llvm::LLVMContext& context = module->getContext();
   llvm::FunctionType* kernel_type = llvm::FunctionType::get(
       /*Result=*/llvm::Type::getVoidTy(context),
-      std::vector(args.size(), ir_builder_.getInt8PtrTy()),
+      std::vector(args.size(), b_.getInt8PtrTy()),
       /*isVarArg=*/false);
   llvm::Function* kernel =
       llvm::Function::Create(kernel_type, llvm::GlobalValue::ExternalLinkage,
@@ -249,7 +249,7 @@ llvm::Function* IrEmitterUnnested::BuildKernelPrototype(
   nvvm_annotations_node->addOperand(llvm::MDNode::get(
       context, {llvm::ConstantAsMetadata::get(kernel),
                 llvm::MDString::get(context, "kernel"),
-                llvm::ConstantAsMetadata::get(ir_builder_.getInt32(1))}));
+                llvm::ConstantAsMetadata::get(b_.getInt32(1))}));
 
   // Update the insert point to the entry basic block.
   llvm::BasicBlock* entry_bb =
@@ -257,7 +257,7 @@ llvm::Function* IrEmitterUnnested::BuildKernelPrototype(
 
   // Emit a "return void" at entry_bb's end, and set the insert point before
   // that return instruction.
-  ir_builder_.SetInsertPoint(llvm::ReturnInst::Create(context, entry_bb));
+  b_.SetInsertPoint(llvm::ReturnInst::Create(context, entry_bb));
 
   return kernel;
 }
@@ -295,7 +295,7 @@ int ComputeMaxUnrollFactor(const HloInstruction* hlo) {
 //    range of i32.
 // Otherwise, the return type is i64.
 llvm::Type* GetIndexTypeForKernel(const HloInstruction* hlo, int64 launch_size,
-                                  llvm::IRBuilder<>* ir_builder) {
+                                  llvm::IRBuilder<>* b) {
   // Find the unnested hlo instructon for which the kernel is generated for.
   const HloInstruction* unnested_hlo = hlo;
   const HloComputation* computation = hlo->parent();
@@ -316,7 +316,7 @@ llvm::Type* GetIndexTypeForKernel(const HloInstruction* hlo, int64 launch_size,
     return in_range;
   };
 
-  llvm::Type* i64_ty = ir_builder->getInt64Ty();
+  llvm::Type* i64_ty = b->getInt64Ty();
   // Check launch dimension
   if (!IsInt32(launch_size)) {
     return i64_ty;
@@ -345,7 +345,7 @@ llvm::Type* GetIndexTypeForKernel(const HloInstruction* hlo, int64 launch_size,
     }
   }
 
-  return ir_builder->getInt32Ty();
+  return b->getInt32Ty();
 }
 
 }  // namespace
@@ -600,8 +600,8 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
           parameter_arrays.push_back(GetIrArray(*operand, *fusion));
         }
         GpuElementalIrEmitter elemental_emitter(
-            hlo_module_config_, ir_emitter_context_->llvm_module(),
-            &ir_builder_, GetNestedComputer());
+            hlo_module_config_, ir_emitter_context_->llvm_module(), &b_,
+            GetNestedComputer());
         FusedIrEmitter fused_emitter(parameter_arrays, &elemental_emitter);
         TF_RETURN_IF_ERROR(root->Accept(&fused_emitter));
 
@@ -674,7 +674,7 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
     }
     GpuElementalIrEmitter elemental_emitter(hlo_module_config_,
                                             ir_emitter_context_->llvm_module(),
-                                            &ir_builder_, GetNestedComputer());
+                                            &b_, GetNestedComputer());
 
     // Shape of the dynamic-update-slice's "update" operand.
     Shape update_shape = root->operand(1)->shape();
@@ -692,7 +692,7 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
 
     return llvm_ir::EmitParallelFusedDynamicUpdateSliceInPlace(
         fusion, operand_arrays, output_array, &elemental_emitter,
-        launch_dimensions, &ir_builder_);
+        launch_dimensions, &b_);
   }
 
   if (ImplementedAsGemm(*fusion)) {
@@ -740,11 +740,11 @@ Status IrEmitterUnnested::EmitExtraOutputsForReduce(
     const HloInstruction* output = reduce->parent()->FusionInstruction();
     llvm::Value* extra_output_address =
         GetIrArray(*output, *output, extra_output_gens[i].second)
-            .EmitArrayElementAddress(index, &ir_builder_,
+            .EmitArrayElementAddress(index, &b_,
                                      "extra_output_element_address");
     TF_ASSIGN_OR_RETURN(llvm::Value* const extra_output_ir_value,
                         extra_output_gens[i].first(index));
-    ir_builder_.CreateStore(extra_output_ir_value, extra_output_address);
+    b_.CreateStore(extra_output_ir_value, extra_output_address);
   }
   return Status::OK();
 }
@@ -774,8 +774,8 @@ Status IrEmitterUnnested::EmitReductionToScalar(
   LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
       tiled_input_shape, ir_emitter_context_->device_description());
 
-  llvm::Type* index_ty = GetIndexTypeForKernel(
-      reduce, launch_dimensions.launch_bound(), &ir_builder_);
+  llvm::Type* index_ty =
+      GetIndexTypeForKernel(reduce, launch_dimensions.launch_bound(), &b_);
 
   auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
     return llvm::ConstantInt::get(index_ty, c);
@@ -825,52 +825,51 @@ Status IrEmitterUnnested::EmitReductionToScalar(
         llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_);
     std::vector partial_reduction_result_addresses;
     for (int i = 0; i != num_reduces; ++i) {
-      llvm::Value* partial_reduction_result_address = ir_builder_.CreateAlloca(
-          element_ir_type, /*ArraySize=*/nullptr,
-          "partial_reduction_result." + llvm::Twine(i));
+      llvm::Value* partial_reduction_result_address =
+          b_.CreateAlloca(element_ir_type, /*ArraySize=*/nullptr,
+                          "partial_reduction_result." + llvm::Twine(i));
       TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
                           init_value_gens[i](IrArray::Index(index_ty)));
-      ir_builder_.CreateStore(init_ir_value, partial_reduction_result_address);
+      b_.CreateStore(init_ir_value, partial_reduction_result_address);
       partial_reduction_result_addresses.push_back(
           partial_reduction_result_address);
     }
 
     llvm::Value* x_in_tiles = tile_index[0];
-    x_in_tiles = ir_builder_.CreateZExtOrTrunc(x_in_tiles, index_ty);
+    x_in_tiles = b_.CreateZExtOrTrunc(x_in_tiles, index_ty);
 
     // Emit an inner for-loop that reduces the elements in the tile.
     auto emit_tile_element_loop = [=](bool tile_in_bounds) -> Status {
       std::unique_ptr tile_element_loop =
-          llvm_ir::ForLoop::EmitForLoop("element_id_in_tile",
-                                        index_typed_constant(0),
-                                        index_typed_constant(kTileSize),
-                                        index_typed_constant(1), &ir_builder_);
+          llvm_ir::ForLoop::EmitForLoop(
+              "element_id_in_tile", index_typed_constant(0),
+              index_typed_constant(kTileSize), index_typed_constant(1), &b_);
 
       // Emit the body of the partial reduction loop.
       llvm_ir::SetToFirstInsertPoint(tile_element_loop->GetBodyBasicBlock(),
-                                     &ir_builder_);
-      llvm::Value* x = ir_builder_.CreateNSWAdd(
-          ir_builder_.CreateNSWMul(x_in_tiles, index_typed_constant(kTileSize)),
+                                     &b_);
+      llvm::Value* x = b_.CreateNSWAdd(
+          b_.CreateNSWMul(x_in_tiles, index_typed_constant(kTileSize)),
           tile_element_loop->GetIndVarValue());
       // Unless we know the tile is entirely in bounds, we have to emit a
       // x-in-bounds check before reading from the input.
       if (!tile_in_bounds) {
         llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
-            ir_builder_.CreateICmpULT(x, index_typed_constant(num_elems)),
-            "x_in_bounds", &ir_builder_);
+            b_.CreateICmpULT(x, index_typed_constant(num_elems)), "x_in_bounds",
+            &b_);
 
         // Emit code that reads the input element and accumulates it to
         // the partial reduction result.
-        llvm_ir::SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
+        llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_);
       }
 
       IrArray::Index input_index(
-          /*linear=*/x, input_shape, &ir_builder_);
-      llvm::Value* input_address = ir_builder_.CreateAlloca(element_ir_type);
+          /*linear=*/x, input_shape, &b_);
+      llvm::Value* input_address = b_.CreateAlloca(element_ir_type);
       for (int i = 0; i != num_reduces; ++i) {
         TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value,
                             input_gens[i](input_index));
-        ir_builder_.CreateStore(input_ir_value, input_address);
+        b_.CreateStore(input_ir_value, input_address);
         TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
             *reducers[i],
             {partial_reduction_result_addresses[i], input_address},
@@ -881,52 +880,48 @@ Status IrEmitterUnnested::EmitReductionToScalar(
 
     // x_end = kTileSize + x_in_tiles * kTileSize, i.e., the location that's
     // immediately beyond the tile.
-    llvm::Value* x_end = ir_builder_.CreateNSWAdd(
+    llvm::Value* x_end = b_.CreateNSWAdd(
         index_typed_constant(kTileSize),
-        ir_builder_.CreateNSWMul(x_in_tiles, index_typed_constant(kTileSize)));
+        b_.CreateNSWMul(x_in_tiles, index_typed_constant(kTileSize)));
     // The tile is entirely in bound if all_threads_in_bounds or
     // x_end <= num_elems.
-    llvm::Value* tile_in_bounds = ir_builder_.CreateOr(
-        ir_builder_.CreateICmpULE(x_end, index_typed_constant(num_elems)),
-        ir_builder_.getInt1(all_threads_in_bounds));
+    llvm::Value* tile_in_bounds =
+        b_.CreateOr(b_.CreateICmpULE(x_end, index_typed_constant(num_elems)),
+                    b_.getInt1(all_threads_in_bounds));
     llvm_ir::LlvmIfData if_tile_in_bounds_data =
-        llvm_ir::EmitIfThenElse(tile_in_bounds, "tile_in_bounds", &ir_builder_);
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.true_block,
-                                   &ir_builder_);
+        llvm_ir::EmitIfThenElse(tile_in_bounds, "tile_in_bounds", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.true_block, &b_);
     TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_bounds=*/true));
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.false_block,
-                                   &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.false_block, &b_);
     TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_bounds=*/false));
 
     // After the if-then-else statement on tile_in_bounds, emit calls to
     // shfl_down that accumulate the partial reduction results of all threads
     // from the warp.
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.after_block,
-                                   &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_bounds_data.after_block, &b_);
     int bit_width = llvm_ir::GetSizeInBits(element_ir_type);
     // bitcast cannot be applied to aggregate types (even packed ones), so we
     // instead bitcast addresses of load/store to intN* of the same bit-width.
     llvm::Type* shuffle_ir_type = element_ir_type->isStructTy()
-                                      ? ir_builder_.getIntNTy(bit_width)
+                                      ? b_.getIntNTy(bit_width)
                                       : element_ir_type;
     for (int shuffle_distance = kWarpSize / 2; shuffle_distance >= 1;
          shuffle_distance /= 2) {
-      llvm::Value* result_from_other_lane = ir_builder_.CreateAlloca(
-          element_ir_type, nullptr, "result_from_other_lane");
+      llvm::Value* result_from_other_lane =
+          b_.CreateAlloca(element_ir_type, nullptr, "result_from_other_lane");
       for (int i = 0; i != num_reduces; ++i) {
-        llvm::Value* partial_reduction_result = ir_builder_.CreateLoad(
-            ir_builder_.CreateBitCast(partial_reduction_result_addresses[i],
-                                      shuffle_ir_type->getPointerTo()),
+        llvm::Value* partial_reduction_result = b_.CreateLoad(
+            b_.CreateBitCast(partial_reduction_result_addresses[i],
+                             shuffle_ir_type->getPointerTo()),
             "partial_reduction_result");
         CHECK_EQ(launch_dimensions.threads_per_block() % kWarpSize, 0)
             << "Requires block size a multiple of the warp size, otherwise we "
                "will read undefined elements.";
-        ir_builder_.CreateStore(
+        b_.CreateStore(
             EmitFullWarpShuffleDown(partial_reduction_result,
-                                    ir_builder_.getInt32(shuffle_distance),
-                                    &ir_builder_),
-            ir_builder_.CreateBitCast(result_from_other_lane,
-                                      shuffle_ir_type->getPointerTo()));
+                                    b_.getInt32(shuffle_distance), &b_),
+            b_.CreateBitCast(result_from_other_lane,
+                             shuffle_ir_type->getPointerTo()));
         TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
             *reducers[i],
             {partial_reduction_result_addresses[i], result_from_other_lane},
@@ -940,24 +935,23 @@ Status IrEmitterUnnested::EmitReductionToScalar(
     // Emit an atomic operation that accumulates the partial reduction result of
     // lane 0 (which holds the partially accumulated result for its warp) to the
     // output element.
-    llvm::Value* lane_id = ir_builder_.CreateURem(
-        x_in_tiles, index_typed_constant(kWarpSize), "lane_id");
+    llvm::Value* lane_id =
+        b_.CreateURem(x_in_tiles, index_typed_constant(kWarpSize), "lane_id");
     llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse(
-        ir_builder_.CreateICmpEQ(lane_id, index_typed_constant(0)),
-        "lane_id_is_zero", &ir_builder_);
-    llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block,
-                                   &ir_builder_);
+        b_.CreateICmpEQ(lane_id, index_typed_constant(0)), "lane_id_is_zero",
+        &b_);
+    llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_);
 
     for (int i = 0; i != num_reduces; ++i) {
       llvm::Value* output_address =
           GetIrArray(*output, *output, reduce_output_shapes[i])
               .EmitArrayElementAddress(
                   IrArray::Index(
-                      /*linear=*/ir_builder_.getInt64(0),
+                      /*linear=*/b_.getInt64(0),
                       ShapeUtil::GetSubshape(output->shape(),
                                              reduce_output_shapes[i]),
-                      &ir_builder_),
-                  &ir_builder_, "output_element_address");
+                      &b_),
+                  &b_, "output_element_address");
       TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation(
           *reducers[i], output_address, partial_reduction_result_addresses[i]));
     }
@@ -971,7 +965,7 @@ Status IrEmitterUnnested::EmitReductionToScalar(
       static_cast(LastThunk())->thunks().back().get(),
       ir_emitter_context_->llvm_module());
   return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape,
-                             launch_dimensions, &ir_builder_)
+                             launch_dimensions, &b_)
       .EmitLoop(IrName(reduce), index_ty);
 }
 
@@ -1015,7 +1009,7 @@ Status IrEmitterUnnested::EmitColumnReduction(
       tiled_input_shape, ir_emitter_context_->device_description());
 
   // TODO(b/110211620): Convert to use i32 index_type when it is possible.
-  llvm::Type* index_ty = ir_builder_.getInt64Ty();
+  llvm::Type* index_ty = b_.getInt64Ty();
 
   auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
     return llvm::ConstantInt::get(index_ty, c);
@@ -1065,14 +1059,12 @@ Status IrEmitterUnnested::EmitColumnReduction(
     for (int i = 0; i != num_reduces; ++i) {
       for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
         llvm::Value* partial_reduction_result_address =
-            ir_builder_.CreateAlloca(
-                element_ir_type, /*ArraySize=*/nullptr,
-                "partial_reduction_result." +
-                    llvm::Twine(i * kTileWidth + x_offset));
+            b_.CreateAlloca(element_ir_type, /*ArraySize=*/nullptr,
+                            "partial_reduction_result." +
+                                llvm::Twine(i * kTileWidth + x_offset));
         TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
                             init_value_gens[i](IrArray::Index(index_ty)));
-        ir_builder_.CreateStore(init_ir_value,
-                                partial_reduction_result_address);
+        b_.CreateStore(init_ir_value, partial_reduction_result_address);
         partial_reduction_result_addresses.push_back(
             partial_reduction_result_address);
       }
@@ -1083,50 +1075,47 @@ Status IrEmitterUnnested::EmitColumnReduction(
     llvm::Value* y_in_tiles = tile_index[0];
     llvm::Value* x_in_tiles = tile_index[1];
 
-    y_in_tiles = ir_builder_.CreateZExtOrTrunc(y_in_tiles, index_ty);
-    x_in_tiles = ir_builder_.CreateZExtOrTrunc(x_in_tiles, index_ty);
+    y_in_tiles = b_.CreateZExtOrTrunc(y_in_tiles, index_ty);
+    x_in_tiles = b_.CreateZExtOrTrunc(x_in_tiles, index_ty);
 
     auto emit_tile_element_loop = [=](bool tile_in_y_bounds,
                                       bool tile_in_x_bounds) -> Status {
       std::unique_ptr tile_element_loop =
-          llvm_ir::ForLoop::EmitForLoop("element_id_in_tile",
-                                        index_typed_constant(0),
-                                        index_typed_constant(kTileHeight),
-                                        index_typed_constant(1), &ir_builder_);
+          llvm_ir::ForLoop::EmitForLoop(
+              "element_id_in_tile", index_typed_constant(0),
+              index_typed_constant(kTileHeight), index_typed_constant(1), &b_);
 
       // Emit the body of the partial reduction loop.
       llvm_ir::SetToFirstInsertPoint(tile_element_loop->GetBodyBasicBlock(),
-                                     &ir_builder_);
-      llvm::Value* y = ir_builder_.CreateNSWAdd(
-          ir_builder_.CreateNSWMul(y_in_tiles,
-                                   index_typed_constant(kTileHeight)),
+                                     &b_);
+      llvm::Value* y = b_.CreateNSWAdd(
+          b_.CreateNSWMul(y_in_tiles, index_typed_constant(kTileHeight)),
           tile_element_loop->GetIndVarValue());
 
       // Unless we know that y is in bounds, we have to emit a check before
       // reading from the input.
       if (!tile_in_y_bounds) {
         llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
-            ir_builder_.CreateICmpULT(y, index_typed_constant(height)),
-            "y_in_bounds", &ir_builder_);
+            b_.CreateICmpULT(y, index_typed_constant(height)), "y_in_bounds",
+            &b_);
 
         // Emit code that reads the input element and accumulates it to
         // the partial reduction result.
-        llvm_ir::SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
+        llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_);
       }
       for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
-        llvm::Value* x = ir_builder_.CreateNSWAdd(
-            ir_builder_.CreateNSWMul(x_in_tiles,
-                                     index_typed_constant(kTileWidth)),
+        llvm::Value* x = b_.CreateNSWAdd(
+            b_.CreateNSWMul(x_in_tiles, index_typed_constant(kTileWidth)),
             index_typed_constant(x_offset));
         // Unless we know that x is in bounds, we have to emit a check before
         // reading from the input.
         if (!tile_in_x_bounds) {
           llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
-              ir_builder_.CreateICmpULT(x, index_typed_constant(width)),
-              "x_in_bounds", &ir_builder_);
-          llvm_ir::SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
+              b_.CreateICmpULT(x, index_typed_constant(width)), "x_in_bounds",
+              &b_);
+          llvm_ir::SetToFirstInsertPoint(if_data.true_block, &b_);
         }
-        llvm::Value* input_address = ir_builder_.CreateAlloca(element_ir_type);
+        llvm::Value* input_address = b_.CreateAlloca(element_ir_type);
         // {y,x} is an index to input_matrix_shape [height,width]. We need to
         // convert that to an index to input_shape (the shape of the operand of
         // "reduce"). This conversion is composed of a transposition from
@@ -1143,18 +1132,17 @@ Status IrEmitterUnnested::EmitColumnReduction(
             ShapeUtil::MakeShapeWithDescendingLayout(input_shape.element_type(),
                                                      {height, width});
         const IrArray::Index input_matrix_index({y, x}, input_matrix_shape,
-                                                &ir_builder_);
+                                                &b_);
         const IrArray::Index input_index =
             input_matrix_index
                 .SourceIndexOfReshape(input_matrix_shape,
-                                      normalized_input_shape, &ir_builder_)
+                                      normalized_input_shape, &b_)
                 .SourceIndexOfTranspose(normalized_input_shape, input_shape,
-                                        transpose_dimension_mapping,
-                                        &ir_builder_);
+                                        transpose_dimension_mapping, &b_);
         for (int i = 0; i != num_reduces; ++i) {
           TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value,
                               input_gens[i](input_index));
-          ir_builder_.CreateStore(input_ir_value, input_address);
+          b_.CreateStore(input_ir_value, input_address);
           TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
               *reducers[i],
               {partial_reduction_result_addresses[i * kTileWidth + x_offset],
@@ -1169,64 +1157,55 @@ Status IrEmitterUnnested::EmitColumnReduction(
 
     // y_end = kTileHeight + y_in_tiles * kTileHeight, i.e., the y location
     // that's immediately beyond the tile.
-    llvm::Value* y_end = ir_builder_.CreateNSWAdd(
+    llvm::Value* y_end = b_.CreateNSWAdd(
         index_typed_constant(kTileHeight),
-        ir_builder_.CreateNSWMul(y_in_tiles,
-                                 index_typed_constant(kTileHeight)));
+        b_.CreateNSWMul(y_in_tiles, index_typed_constant(kTileHeight)));
     // x_end = kTileWidth + x_in_tiles * kTileWidth, i.e., the x location
     // that's immediately beyond the tile.
-    llvm::Value* x_end = ir_builder_.CreateNSWAdd(
+    llvm::Value* x_end = b_.CreateNSWAdd(
         index_typed_constant(kTileWidth),
-        ir_builder_.CreateNSWMul(x_in_tiles, index_typed_constant(kTileWidth)));
-    llvm::Value* tile_in_y_bounds = ir_builder_.CreateOr(
-        ir_builder_.CreateICmpULE(y_end, index_typed_constant(height)),
-        ir_builder_.getInt1(height % kTileHeight == 0));
-    llvm::Value* tile_in_x_bounds = ir_builder_.CreateOr(
-        ir_builder_.CreateICmpULE(x_end, index_typed_constant(width)),
-        ir_builder_.getInt1(width % kTileWidth == 0));
+        b_.CreateNSWMul(x_in_tiles, index_typed_constant(kTileWidth)));
+    llvm::Value* tile_in_y_bounds =
+        b_.CreateOr(b_.CreateICmpULE(y_end, index_typed_constant(height)),
+                    b_.getInt1(height % kTileHeight == 0));
+    llvm::Value* tile_in_x_bounds =
+        b_.CreateOr(b_.CreateICmpULE(x_end, index_typed_constant(width)),
+                    b_.getInt1(width % kTileWidth == 0));
     // The tile is in y bounds if "height" is a multiple of kTileHeight or
     // y_end <= height.
-    llvm_ir::LlvmIfData if_tile_in_y_bounds_data = llvm_ir::EmitIfThenElse(
-        tile_in_y_bounds, "tile_in_y_bounds", &ir_builder_);
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.true_block,
-                                   &ir_builder_);
+    llvm_ir::LlvmIfData if_tile_in_y_bounds_data =
+        llvm_ir::EmitIfThenElse(tile_in_y_bounds, "tile_in_y_bounds", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.true_block, &b_);
     // The tile is in x bounds if "width" is a multiple of kTileWidth or
     // x_end <= width.
-    llvm_ir::LlvmIfData if_tile_in_x_bounds_data = llvm_ir::EmitIfThenElse(
-        tile_in_x_bounds, "tile_in_x_bounds", &ir_builder_);
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block,
-                                   &ir_builder_);
+    llvm_ir::LlvmIfData if_tile_in_x_bounds_data =
+        llvm_ir::EmitIfThenElse(tile_in_x_bounds, "tile_in_x_bounds", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block, &b_);
     TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/true,
                                               /*tile_in_x_bounds=*/true));
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block,
-                                   &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block, &b_);
     TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/true,
                                               /*tile_in_x_bounds=*/false));
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.false_block,
-                                   &ir_builder_);
-    if_tile_in_x_bounds_data = llvm_ir::EmitIfThenElse(
-        tile_in_x_bounds, "tile_in_x_bounds", &ir_builder_);
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block,
-                                   &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.false_block, &b_);
+    if_tile_in_x_bounds_data =
+        llvm_ir::EmitIfThenElse(tile_in_x_bounds, "tile_in_x_bounds", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.true_block, &b_);
     TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/false,
                                               /*tile_in_x_bounds=*/true));
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block,
-                                   &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_x_bounds_data.false_block, &b_);
     TF_RETURN_IF_ERROR(emit_tile_element_loop(/*tile_in_y_bounds=*/false,
                                               /*tile_in_x_bounds=*/false));
 
     // After the nested if-then-else statement on tile_in_y_bounds and
     // tile_in_x_bounds, emit atomic operations to accumulate the partial
     // reduction result to the output element.
-    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.after_block,
-                                   &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_tile_in_y_bounds_data.after_block, &b_);
     const HloInstruction* output =
         reduce->IsFused() ? reduce->parent()->FusionInstruction() : reduce;
     for (int i = 0; i != num_reduces; ++i) {
       for (int x_offset = 0; x_offset < kTileWidth; ++x_offset) {
-        llvm::Value* x = ir_builder_.CreateNSWAdd(
-            ir_builder_.CreateNSWMul(x_in_tiles,
-                                     index_typed_constant(kTileWidth)),
+        llvm::Value* x = b_.CreateNSWAdd(
+            b_.CreateNSWMul(x_in_tiles, index_typed_constant(kTileWidth)),
             index_typed_constant(x_offset));
         llvm::Value* output_address =
             GetIrArray(*output, *output, reduce_output_shapes[i])
@@ -1235,8 +1214,8 @@ Status IrEmitterUnnested::EmitColumnReduction(
                         x,
                         ShapeUtil::GetSubshape(output->shape(),
                                                reduce_output_shapes[i]),
-                        &ir_builder_),
-                    &ir_builder_, "output_element_address");
+                        &b_),
+                    &b_, "output_element_address");
         TF_RETURN_IF_ERROR(EmitAtomicOperationForNestedComputation(
             *reducers[i], output_address,
             partial_reduction_result_addresses[i * kTileWidth + x_offset]));
@@ -1252,7 +1231,7 @@ Status IrEmitterUnnested::EmitColumnReduction(
       static_cast(LastThunk())->thunks().back().get(),
       ir_emitter_context_->llvm_module());
   return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape,
-                             launch_dimensions, &ir_builder_)
+                             launch_dimensions, &b_)
       .EmitLoop(IrName(reduce), index_ty);
 }
 
@@ -1402,8 +1381,8 @@ Status IrEmitterUnnested::EmitRowReduction(
       {depth / z_tile_size, height, width_in_tiles}, {2, 1, 0});
   LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
       tiled_input_shape, ir_emitter_context_->device_description());
-  llvm::Type* index_ty = GetIndexTypeForKernel(
-      reduce, launch_dimensions.launch_bound(), &ir_builder_);
+  llvm::Type* index_ty =
+      GetIndexTypeForKernel(reduce, launch_dimensions.launch_bound(), &b_);
 
   auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
     return llvm::ConstantInt::get(index_ty, c);
@@ -1415,12 +1394,12 @@ Status IrEmitterUnnested::EmitRowReduction(
         input_shape.element_type(), ir_emitter_context_->llvm_module());
     std::vector partial_reduction_result_addresses;
     for (int i = 0; i != num_reduces; ++i) {
-      llvm::Value* partial_reduction_result_address = ir_builder_.CreateAlloca(
-          element_ir_type, /*ArraySize=*/nullptr,
-          "partial_reduction_result." + llvm::Twine(i));
+      llvm::Value* partial_reduction_result_address =
+          b_.CreateAlloca(element_ir_type, /*ArraySize=*/nullptr,
+                          "partial_reduction_result." + llvm::Twine(i));
       TF_ASSIGN_OR_RETURN(llvm::Value* const init_ir_value,
                           init_value_gens[i](IrArray::Index(index_ty)));
-      ir_builder_.CreateStore(init_ir_value, partial_reduction_result_address);
+      b_.CreateStore(init_ir_value, partial_reduction_result_address);
       partial_reduction_result_addresses.push_back(
           partial_reduction_result_address);
     }
@@ -1429,25 +1408,25 @@ Status IrEmitterUnnested::EmitRowReduction(
     llvm::Value* y = tile_index[1];
     llvm::Value* x_tile = tile_index[2];
 
-    x_tile = ir_builder_.CreateZExtOrTrunc(x_tile, index_ty);
+    x_tile = b_.CreateZExtOrTrunc(x_tile, index_ty);
 
-    llvm::Value* warp_id = ir_builder_.CreateUDiv(
-        x_tile, index_typed_constant(kWarpSize), "warp_id");
-    llvm::Value* lane_id = ir_builder_.CreateURem(
-        x_tile, index_typed_constant(kWarpSize), "lane_id");
+    llvm::Value* warp_id =
+        b_.CreateUDiv(x_tile, index_typed_constant(kWarpSize), "warp_id");
+    llvm::Value* lane_id =
+        b_.CreateURem(x_tile, index_typed_constant(kWarpSize), "lane_id");
 
     // The x-location of the last element in this z-x-tile.
     // last_x = lane_id + warpSize * (x_tile_size - 1 + warp_id * x_tile_size);
-    llvm::Value* last_x = ir_builder_.CreateNSWAdd(
-        lane_id, ir_builder_.CreateNSWMul(
-                     index_typed_constant(kWarpSize),
-                     ir_builder_.CreateNSWAdd(
-                         index_typed_constant(x_tile_size - 1),
-                         ir_builder_.CreateNSWMul(
-                             warp_id, index_typed_constant(x_tile_size)))));
+    llvm::Value* last_x = b_.CreateNSWAdd(
+        lane_id,
+        b_.CreateNSWMul(
+            index_typed_constant(kWarpSize),
+            b_.CreateNSWAdd(
+                index_typed_constant(x_tile_size - 1),
+                b_.CreateNSWMul(warp_id, index_typed_constant(x_tile_size)))));
 
     KernelSupportLibrary ksl(
-        &ir_builder_,
+        &b_,
         /*unroll_mode=*/xla::llvm_ir::UnrollMode::kFullyUnroll,
         /*prevent_vectorization=*/false);
 
@@ -1456,9 +1435,9 @@ Status IrEmitterUnnested::EmitRowReduction(
     auto emit_z_x_tile_element_loop = [&](bool x_tile_in_bounds,
                                           int64 x_tile_loop_bound) -> Status {
       auto emit_z_tile_element_loop = [&](llvm::Value* z_indvar) -> Status {
-        llvm::Value* z = ir_builder_.CreateNSWAdd(
-            z_indvar, ir_builder_.CreateNSWMul(
-                          index_typed_constant(z_tile_size), z_tile));
+        llvm::Value* z = b_.CreateNSWAdd(
+            z_indvar,
+            b_.CreateNSWMul(index_typed_constant(z_tile_size), z_tile));
         TF_RETURN_IF_ERROR(ksl.For(
             "x_tile",
             /*start=*/index_typed_constant(0),
@@ -1466,12 +1445,12 @@ Status IrEmitterUnnested::EmitRowReduction(
             /*step=*/1, [&](llvm::Value* x_indvar) -> Status {
               // x = lane_id +
               //     warpSize * (element_id_in_x_tile + warp_id * x_tile_size);
-              llvm::Value* x = ir_builder_.CreateNSWAdd(
+              llvm::Value* x = b_.CreateNSWAdd(
                   lane_id,
-                  ir_builder_.CreateNSWMul(
+                  b_.CreateNSWMul(
                       index_typed_constant(kWarpSize),
-                      ir_builder_.CreateNSWAdd(
-                          x_indvar, ir_builder_.CreateNSWMul(
+                      b_.CreateNSWAdd(
+                          x_indvar, b_.CreateNSWMul(
                                         warp_id, llvm::ConstantInt::get(
                                                      index_ty, x_tile_size)))));
 
@@ -1479,18 +1458,17 @@ Status IrEmitterUnnested::EmitRowReduction(
               // emit a x-in-bounds check before reading from the input.
               if (!x_tile_in_bounds) {
                 llvm_ir::LlvmIfData if_x_in_bounds_data =
-                    llvm_ir::EmitIfThenElse(ir_builder_.CreateICmpULT(
-                                                x, index_typed_constant(width)),
-                                            "x_in_bounds", &ir_builder_);
-                // Points ir_builder_ to the then-block.
+                    llvm_ir::EmitIfThenElse(
+                        b_.CreateICmpULT(x, index_typed_constant(width)),
+                        "x_in_bounds", &b_);
+                // Points b_ to the then-block.
                 llvm_ir::SetToFirstInsertPoint(if_x_in_bounds_data.true_block,
-                                               &ir_builder_);
+                                               &b_);
               }
 
               // Emit code that reads the input element and accumulates it
               // to the partial reduction result.
-              llvm::Value* input_address =
-                  ir_builder_.CreateAlloca(element_ir_type);
+              llvm::Value* input_address = b_.CreateAlloca(element_ir_type);
               {
                 // {z,y,x} is an index to input_3d_tensor_shape
                 // [depth,height,width]. We need to convert that to an index
@@ -1509,20 +1487,19 @@ Status IrEmitterUnnested::EmitRowReduction(
                     ShapeUtil::MakeShapeWithDescendingLayout(
                         input_shape.element_type(), {depth, height, width});
                 const IrArray::Index input_3d_tensor_index(
-                    {z, y, x}, input_3d_tensor_shape, &ir_builder_);
+                    {z, y, x}, input_3d_tensor_shape, &b_);
                 const IrArray::Index input_index =
                     input_3d_tensor_index
                         .SourceIndexOfReshape(input_3d_tensor_shape,
-                                              normalized_input_shape,
-                                              &ir_builder_)
+                                              normalized_input_shape, &b_)
                         .SourceIndexOfTranspose(
                             normalized_input_shape, input_shape,
-                            transpose_dimension_mapping, &ir_builder_);
+                            transpose_dimension_mapping, &b_);
 
                 for (int i = 0; i != num_reduces; ++i) {
                   TF_ASSIGN_OR_RETURN(llvm::Value* const input_ir_value,
                                       input_gens[i](input_index));
-                  ir_builder_.CreateStore(input_ir_value, input_address);
+                  b_.CreateStore(input_ir_value, input_address);
                   TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
                       *reducers[i],
                       {partial_reduction_result_addresses[i], input_address},
@@ -1541,9 +1518,9 @@ Status IrEmitterUnnested::EmitRowReduction(
                      /*step=*/1, emit_z_tile_element_loop);
     };
 
-    llvm::Value* tile_in_bounds = ir_builder_.CreateOr(
-        ir_builder_.getInt1(width % (x_tile_size * kWarpSize) == 0),
-        ir_builder_.CreateICmpULT(last_x, index_typed_constant(width)));
+    llvm::Value* tile_in_bounds =
+        b_.CreateOr(b_.getInt1(width % (x_tile_size * kWarpSize) == 0),
+                    b_.CreateICmpULT(last_x, index_typed_constant(width)));
 
     TF_RETURN_IF_ERROR(
         ksl.If(tile_in_bounds,
@@ -1566,26 +1543,25 @@ Status IrEmitterUnnested::EmitRowReduction(
     // bitcast cannot be applied to aggregate types (even packed ones), so we
     // instead bitcast addresses of load/store to intN* of the same bit-width.
     llvm::Type* shuffle_ir_type = element_ir_type->isStructTy()
-                                      ? ir_builder_.getIntNTy(bit_width)
+                                      ? b_.getIntNTy(bit_width)
                                       : element_ir_type;
     for (int shuffle_distance = 16; shuffle_distance >= 1;
          shuffle_distance /= 2) {
-      llvm::Value* result_from_other_lane = ir_builder_.CreateAlloca(
-          element_ir_type, nullptr, "result_from_other_lane");
+      llvm::Value* result_from_other_lane =
+          b_.CreateAlloca(element_ir_type, nullptr, "result_from_other_lane");
       for (int i = 0; i != num_reduces; ++i) {
-        llvm::Value* partial_reduction_result = ir_builder_.CreateLoad(
-            ir_builder_.CreateBitCast(partial_reduction_result_addresses[i],
-                                      shuffle_ir_type->getPointerTo()),
+        llvm::Value* partial_reduction_result = b_.CreateLoad(
+            b_.CreateBitCast(partial_reduction_result_addresses[i],
+                             shuffle_ir_type->getPointerTo()),
             "partial_reduction_result");
         CHECK_EQ(launch_dimensions.threads_per_block() % kWarpSize, 0)
             << "Requires block size a multiple of the warp size, otherwise we "
                "will read undefined elements.";
-        ir_builder_.CreateStore(
+        b_.CreateStore(
             EmitFullWarpShuffleDown(partial_reduction_result,
-                                    ir_builder_.getInt32(shuffle_distance),
-                                    &ir_builder_),
-            ir_builder_.CreateBitCast(result_from_other_lane,
-                                      shuffle_ir_type->getPointerTo()));
+                                    b_.getInt32(shuffle_distance), &b_),
+            b_.CreateBitCast(result_from_other_lane,
+                             shuffle_ir_type->getPointerTo()));
         TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
             *reducers[i],
             {partial_reduction_result_addresses[i], result_from_other_lane},
@@ -1600,10 +1576,9 @@ Status IrEmitterUnnested::EmitRowReduction(
     // lane 0 (which holds the partially accumulated result for its warp) to the
     // output element.
     llvm_ir::LlvmIfData if_lane_id_is_zero_data = llvm_ir::EmitIfThenElse(
-        ir_builder_.CreateICmpEQ(lane_id, index_typed_constant(0)),
-        "lane_id_is_zero", &ir_builder_);
-    llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block,
-                                   &ir_builder_);
+        b_.CreateICmpEQ(lane_id, index_typed_constant(0)), "lane_id_is_zero",
+        &b_);
+    llvm_ir::SetToFirstInsertPoint(if_lane_id_is_zero_data.true_block, &b_);
     for (int i = 0; i != num_reduces; ++i) {
       llvm::Value* output_address =
           GetIrArray(*output, *output, reduce_output_shapes[i])
@@ -1611,8 +1586,8 @@ Status IrEmitterUnnested::EmitRowReduction(
                   IrArray::Index(y,
                                  ShapeUtil::GetSubshape(
                                      output->shape(), reduce_output_shapes[i]),
-                                 &ir_builder_),
-                  &ir_builder_, "output_element_address");
+                                 &b_),
+                  &b_, "output_element_address");
       // We don't need to emit atomic operations if there is only one tile of
       // results. 'depth' is the z dimension, 'width' is the x dimension.
       if (z_tile_size >= depth && x_tile_size >= width) {
@@ -1636,7 +1611,7 @@ Status IrEmitterUnnested::EmitRowReduction(
       static_cast(LastThunk())->thunks().back().get(),
       ir_emitter_context_->llvm_module());
   return ParallelLoopEmitter(loop_body_emitter, tiled_input_shape,
-                             launch_dimensions, &ir_builder_)
+                             launch_dimensions, &b_)
       .EmitLoop(IrName(reduce), index_ty);
 }
 
@@ -1762,12 +1737,11 @@ Status IrEmitterUnnested::HandleReduce(HloInstruction* reduce) {
 
     return EmitReductionToVector(
         reduce, input->shape(), {[&](const IrArray::Index& index) {
-          return GetIrArray(*input, *reduce)
-              .EmitReadArrayElement(index, &ir_builder_);
+          return GetIrArray(*input, *reduce).EmitReadArrayElement(index, &b_);
         }},
         {[&](const IrArray::Index& index) {
           return GetIrArray(*init_value, *reduce)
-              .EmitReadArrayElement(index, &ir_builder_);
+              .EmitReadArrayElement(index, &b_);
         }},
         dimensions_to_reduce, {reducer}, {{}}, {});
   }
@@ -1842,7 +1816,7 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
   LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
       source->shape(), ir_emitter_context_->device_description());
   llvm::Type* index_type = GetIndexTypeForKernel(
-      select_and_scatter, launch_dimensions.launch_bound(), &ir_builder_);
+      select_and_scatter, launch_dimensions.launch_bound(), &b_);
   auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
     return llvm::ConstantInt::get(index_type, c);
   };
@@ -1873,19 +1847,18 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
     llvm::Value* selected_value_address = llvm_ir::EmitAllocaAtFunctionEntry(
         llvm_ir::PrimitiveTypeToIrType(operand_element_type,
                                        ir_emitter_context_->llvm_module()),
-        "selected_value_address", &ir_builder_);
+        "selected_value_address", &b_);
     llvm::Value* selected_index_address =
         llvm_ir::EmitAllocaAtFunctionEntryWithCount(
             index_type, index_typed_constant(rank), "selected_index_address",
-            &ir_builder_);
+            &b_);
     llvm::Value* initialized_flag_address = llvm_ir::EmitAllocaAtFunctionEntry(
-        ir_builder_.getInt1Ty(), "initialized_flag_address", &ir_builder_);
-    ir_builder_.CreateStore(ir_builder_.getInt1(false),
-                            initialized_flag_address);
+        b_.getInt1Ty(), "initialized_flag_address", &b_);
+    b_.CreateStore(b_.getInt1(false), initialized_flag_address);
 
     // Create the inner loop to iterate over the window.
-    llvm_ir::ForLoopNest window_loops(IrName(select_and_scatter, "inner"),
-                                      &ir_builder_, index_type);
+    llvm_ir::ForLoopNest window_loops(IrName(select_and_scatter, "inner"), &b_,
+                                      index_type);
     std::vector window_size;
     for (const auto& dim : window.dimensions()) {
       window_size.push_back(dim.size());
@@ -1894,84 +1867,79 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
     const IrArray::Index window_index = window_loops.AddLoopsForShape(
         ShapeUtil::MakeShape(operand_element_type, window_size), "window");
     llvm_ir::SetToFirstInsertPoint(window_loops.GetInnerLoopBodyBasicBlock(),
-                                   &ir_builder_);
+                                   &b_);
 
     // Compute the operand index to visit and evaluate the condition whether the
     // operand index is within the bounds. The unsigned comparison includes
     // checking whether the operand index >= 0.
     IrArray::Index operand_index(index_type, source_index.size());
-    llvm::Value* in_bounds_condition = ir_builder_.getInt1(true);
+    llvm::Value* in_bounds_condition = b_.getInt1(true);
     for (int64 i = 0; i < rank; ++i) {
-      llvm::Value* strided_index = ir_builder_.CreateNSWMul(
+      llvm::Value* strided_index = b_.CreateNSWMul(
           source_index[i], index_typed_constant(window.dimensions(i).stride()));
-      operand_index[i] = ir_builder_.CreateNSWSub(
-          ir_builder_.CreateNSWAdd(strided_index, window_index[i]),
+      operand_index[i] = b_.CreateNSWSub(
+          b_.CreateNSWAdd(strided_index, window_index[i]),
           index_typed_constant(window.dimensions(i).padding_low()));
-      llvm::Value* index_condition = ir_builder_.CreateICmpULT(
+      llvm::Value* index_condition = b_.CreateICmpULT(
           operand_index[i],
           index_typed_constant(ShapeUtil::GetDimension(operand->shape(), i)));
-      in_bounds_condition =
-          ir_builder_.CreateAnd(in_bounds_condition, index_condition);
+      in_bounds_condition = b_.CreateAnd(in_bounds_condition, index_condition);
     }
     CHECK(in_bounds_condition != nullptr);
 
     // Only need to do something if the operand index is within the bounds.
     // First check if the initialized_flag is set.
     llvm_ir::LlvmIfData if_in_bounds =
-        llvm_ir::EmitIfThenElse(in_bounds_condition, "in-bounds", &ir_builder_);
-    llvm_ir::SetToFirstInsertPoint(if_in_bounds.true_block, &ir_builder_);
+        llvm_ir::EmitIfThenElse(in_bounds_condition, "in-bounds", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_in_bounds.true_block, &b_);
     llvm_ir::LlvmIfData if_initialized = llvm_ir::EmitIfThenElse(
-        ir_builder_.CreateLoad(initialized_flag_address), "initialized",
-        &ir_builder_);
+        b_.CreateLoad(initialized_flag_address), "initialized", &b_);
 
     // If the initialized_flag is false, initialize the selected value and index
     // with the currently visiting operand.
-    llvm_ir::SetToFirstInsertPoint(if_initialized.false_block, &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_initialized.false_block, &b_);
     const auto save_operand_index = [&](const IrArray::Index& operand_index) {
       for (int64 i = 0; i < rank; ++i) {
         llvm::Value* selected_index_address_slot =
-            ir_builder_.CreateInBoundsGEP(selected_index_address,
-                                          {ir_builder_.getInt32(i)});
-        ir_builder_.CreateStore(operand_index[i], selected_index_address_slot);
+            b_.CreateInBoundsGEP(selected_index_address, {b_.getInt32(i)});
+        b_.CreateStore(operand_index[i], selected_index_address_slot);
       }
     };
     IrArray operand_array = GetIrArray(*operand, *select_and_scatter);
     llvm::Value* operand_data =
-        operand_array.EmitReadArrayElement(operand_index, &ir_builder_);
-    ir_builder_.CreateStore(operand_data, selected_value_address);
+        operand_array.EmitReadArrayElement(operand_index, &b_);
+    b_.CreateStore(operand_data, selected_value_address);
     save_operand_index(operand_index);
-    ir_builder_.CreateStore(ir_builder_.getInt1(true),
-                            initialized_flag_address);
+    b_.CreateStore(b_.getInt1(true), initialized_flag_address);
 
     // If the initialized_flag is true, call the `select` function to
     // potentially update the selected value and index with the currently
     // visiting operand.
-    llvm_ir::SetToFirstInsertPoint(if_initialized.true_block, &ir_builder_);
+    llvm_ir::SetToFirstInsertPoint(if_initialized.true_block, &b_);
     const Shape output_shape = ShapeUtil::MakeShape(PRED, {});
     llvm::Value* operand_address =
-        operand_array.EmitArrayElementAddress(operand_index, &ir_builder_);
+        operand_array.EmitArrayElementAddress(operand_index, &b_);
     llvm::Value* select_return_buffer = llvm_ir::EmitAllocaAtFunctionEntry(
         llvm_ir::PrimitiveTypeToIrType(PRED,
                                        ir_emitter_context_->llvm_module()),
-        "select_return_buffer", &ir_builder_);
+        "select_return_buffer", &b_);
     TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
         *select_and_scatter->select(),
         {selected_value_address, operand_address}, select_return_buffer));
-    llvm::Value* result = ir_builder_.CreateLoad(select_return_buffer);
+    llvm::Value* result = b_.CreateLoad(select_return_buffer);
 
     // If the 'select' function returns false, update the selected value and the
     // index to the currently visiting operand.
-    llvm::Value* cond = ir_builder_.CreateICmpNE(
+    llvm::Value* cond = b_.CreateICmpNE(
         result,
         llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(
                                    PRED, ir_emitter_context_->llvm_module()),
                                0),
         "boolean_predicate");
     llvm_ir::LlvmIfData if_select_lhs =
-        llvm_ir::EmitIfThenElse(cond, "if-select-lhs", &ir_builder_);
-    llvm_ir::SetToFirstInsertPoint(if_select_lhs.false_block, &ir_builder_);
-    ir_builder_.CreateStore(ir_builder_.CreateLoad(operand_address),
-                            selected_value_address);
+        llvm_ir::EmitIfThenElse(cond, "if-select-lhs", &b_);
+    llvm_ir::SetToFirstInsertPoint(if_select_lhs.false_block, &b_);
+    b_.CreateStore(b_.CreateLoad(operand_address), selected_value_address);
     save_operand_index(operand_index);
 
     // After iterating over the window elements, scatter the source element to
@@ -1979,20 +1947,19 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
     // location is computed by calling the `scatter` function with the source
     // value and the current output value.
     llvm_ir::SetToFirstInsertPoint(window_loops.GetOuterLoopExitBasicBlock(),
-                                   &ir_builder_);
+                                   &b_);
     IrArray::Index selected_index(operand_index.GetType());
     for (int64 i = 0; i < rank; ++i) {
-      llvm::Value* selected_index_address_slot = ir_builder_.CreateInBoundsGEP(
-          selected_index_address, {ir_builder_.getInt32(i)});
-      selected_index.push_back(
-          ir_builder_.CreateLoad(selected_index_address_slot));
+      llvm::Value* selected_index_address_slot =
+          b_.CreateInBoundsGEP(selected_index_address, {b_.getInt32(i)});
+      selected_index.push_back(b_.CreateLoad(selected_index_address_slot));
     }
     llvm::Value* source_value_address =
         GetIrArray(*source, *select_and_scatter)
-            .EmitArrayElementAddress(source_index, &ir_builder_);
+            .EmitArrayElementAddress(source_index, &b_);
     llvm::Value* output_value_address =
         GetIrArray(*select_and_scatter, *select_and_scatter)
-            .EmitArrayElementAddress(selected_index, &ir_builder_);
+            .EmitArrayElementAddress(selected_index, &b_);
     return EmitAtomicOperationForNestedComputation(
         *select_and_scatter->scatter(), output_value_address,
         source_value_address);
@@ -2007,7 +1974,7 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
       static_cast(LastThunk())->thunks().back().get(),
       ir_emitter_context_->llvm_module());
   return ParallelLoopEmitter(loop_body_emitter, source->shape(),
-                             launch_dimensions, &ir_builder_)
+                             launch_dimensions, &b_)
       .EmitLoop(IrName(select_and_scatter), index_type);
 }
 
@@ -2326,18 +2293,16 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk(
             << " is found in slice " << slice.ToString() << " at GTE index "
             << gte_index.ToString();
 
-    llvm::Value* loc =
-        ir_builder_.CreateInBoundsGEP(kernel_args.at(slice.allocation()),
-                                      {ir_builder_.getInt64(slice.offset())});
+    llvm::Value* loc = b_.CreateInBoundsGEP(kernel_args.at(slice.allocation()),
+                                            {b_.getInt64(slice.offset())});
 
     // If gte_index is nonempty, we have to dereference `loc` to get to the
     // value we're ultimately interested in.
     llvm::Type* int8_double_pointer =
-        llvm::PointerType::get(ir_builder_.getInt8PtrTy(), /*AddressSpace=*/0);
+        llvm::PointerType::get(b_.getInt8PtrTy(), /*AddressSpace=*/0);
     for (int64 idx : gte_index) {
-      loc = ir_builder_.CreateBitCast(loc, int8_double_pointer);
-      loc = ir_builder_.CreateLoad(
-          ir_builder_.CreateInBoundsGEP(loc, {ir_builder_.getInt64(idx)}));
+      loc = b_.CreateBitCast(loc, int8_double_pointer);
+      loc = b_.CreateLoad(b_.CreateInBoundsGEP(loc, {b_.getInt64(idx)}));
     }
 
     bindings_.BindHloToIrValue(*instr, loc, index);
@@ -2349,7 +2314,7 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk(
     bindings_.SetTempBufferBase(kernel_args.at(*temp_buffer));
   } else {
     bindings_.SetTempBufferBase(
-        llvm::ConstantPointerNull::get(ir_builder_.getInt8PtrTy()));
+        llvm::ConstantPointerNull::get(b_.getInt8PtrTy()));
   }
 
   return MakeUnique(buffers, llvm_ir::AsString(kernel->getName()),
@@ -2596,10 +2561,9 @@ StatusOr> IrEmitterUnnested::BuildInitializerThunk(
   TF_RETURN_IF_ERROR(ParallelLoopEmitter(
                          [=](const IrArray::Index& index) {
                            return GetIrArray(*init_value, *hlo)
-                               .EmitReadArrayElement(index, &ir_builder_);
+                               .EmitReadArrayElement(index, &b_);
                          },
-                         GetIrArray(*hlo, *hlo, index), launch_dimensions,
-                         &ir_builder_)
+                         GetIrArray(*hlo, *hlo, index), launch_dimensions, &b_)
                          .EmitLoop(IrName(hlo)));
 
   // Clean up state left behind by emitting the loop above.  (This is normally
@@ -2783,10 +2747,10 @@ Status IrEmitterUnnested::EmitTargetElementLoopInThunk(
                          ir_emitter_context_->llvm_module());
   if (!hlo.IsMultiOutputFusion()) {
     return ParallelLoopEmitter(element_generator, GetIrArray(hlo, hlo),
-                               launch_dimensions, &ir_builder_, unroll_factor)
-        .EmitLoop(IrName(&hlo),
-                  GetIndexTypeForKernel(&hlo, launch_dimensions.launch_bound(),
-                                        &ir_builder_));
+                               launch_dimensions, &b_, unroll_factor)
+        .EmitLoop(
+            IrName(&hlo),
+            GetIndexTypeForKernel(&hlo, launch_dimensions.launch_bound(), &b_));
   }
 
   // For multioutput fusion, we need to emit each operand and the root.
@@ -2796,18 +2760,17 @@ Status IrEmitterUnnested::EmitTargetElementLoopInThunk(
   }
   TF_RETURN_IF_ERROR(
       ParallelLoopEmitter(element_generator, output_arrays, launch_dimensions,
-                          &ir_builder_, unroll_factor)
+                          &b_, unroll_factor)
           .EmitLoop(IrName(&hlo),
                     GetIndexTypeForKernel(
-                        &hlo, launch_dimensions.launch_bound(), &ir_builder_)));
+                        &hlo, launch_dimensions.launch_bound(), &b_)));
 
   std::vector tuple_operand_ptrs;
   for (int64 i = 0; i < output_arrays.size(); ++i) {
     tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer());
   }
-  ir_builder_.SetInsertPoint(ir_builder_.GetInsertBlock()->getTerminator());
-  llvm_ir::EmitTuple(GetIrArray(hlo, hlo), tuple_operand_ptrs, &ir_builder_,
-                     module_);
+  b_.SetInsertPoint(b_.GetInsertBlock()->getTerminator());
+  llvm_ir::EmitTuple(GetIrArray(hlo, hlo), tuple_operand_ptrs, &b_, module_);
   return Status::OK();
 }
 
@@ -2858,14 +2821,14 @@ int IrEmitterUnnested::ConstructOutputReducedShapeAndCastOutputIrArrayToShape(
       output_reduced_shapes->push_back(ShapeUtil::MakeShapeWithDescendingLayout(
           ShapeUtil::GetSubshape(hlo.shape(), {i}).element_type(),
           reduced_output_dims));
-      output_in_reduced_shape_arrays->push_back(output_arrays[i].CastToShape(
-          (*output_reduced_shapes)[i], &ir_builder_));
+      output_in_reduced_shape_arrays->push_back(
+          output_arrays[i].CastToShape((*output_reduced_shapes)[i], &b_));
     }
   } else {
     output_reduced_shapes->push_back(ShapeUtil::MakeShapeWithDescendingLayout(
         hlo.shape().element_type(), reduced_output_dims));
-    output_in_reduced_shape_arrays->push_back(output_arrays[0].CastToShape(
-        (*output_reduced_shapes)[0], &ir_builder_));
+    output_in_reduced_shape_arrays->push_back(
+        output_arrays[0].CastToShape((*output_reduced_shapes)[0], &b_));
   }
   return num_outputs;
 }
@@ -2889,8 +2852,8 @@ int IrEmitterUnnested::ConstructInputReducedShapeAndCastInputIrArrayToShape(
     param_reduced_shapes->push_back(ShapeUtil::MakeShapeWithDescendingLayout(
         param->shape().element_type(),
         Permute({0, 2, 1}, reduced_output_dims)));
-    param_in_reduced_shape_arrays->push_back(param_arrays[id].CastToShape(
-        (*param_reduced_shapes)[id], &ir_builder_));
+    param_in_reduced_shape_arrays->push_back(
+        param_arrays[id].CastToShape((*param_reduced_shapes)[id], &b_));
   }
   return num_params;
 }
@@ -3039,7 +3002,7 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
         kTileSize);
     const int kNVPTXSharedMemoryAddrSpace = 3;
     auto* tile_base_ptr = new llvm::GlobalVariable(
-        *ir_builder_.GetInsertBlock()->getParent()->getParent(), tile_type,
+        *b_.GetInsertBlock()->getParent()->getParent(), tile_type,
         /*isConstant=*/false, llvm::GlobalValue::PrivateLinkage,
         llvm::UndefValue::get(tile_type),
         llvm_ir::AsStringRef(IrName(hlo, StrCat("tile", id))), nullptr,
@@ -3063,8 +3026,8 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
       c_accumulate(output_dims_in_tiles, 1, std::multiplies());
   LaunchDimensions launch_dimensions(num_tiles, kThreadsPerTile);
 
-  llvm::Type* index_ty = GetIndexTypeForKernel(
-      hlo, launch_dimensions.launch_bound(), &ir_builder_);
+  llvm::Type* index_ty =
+      GetIndexTypeForKernel(hlo, launch_dimensions.launch_bound(), &b_);
   auto index_typed_constant = [&](uint64 c) -> llvm::Constant* {
     return llvm::ConstantInt::get(index_ty, c);
   };
@@ -3092,23 +3055,23 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
   llvm::Value* x;
   llvm::Value* y;
   std::tie(y, x) = CalculateYXCoordinateWithinTile(
-      &ir_builder_, index_typed_constant(kTileSize), kThreadsPerTile);
+      &b_, index_typed_constant(kTileSize), kThreadsPerTile);
 
   // Calculate the index for the current output tile from block_id.
   const IrArray::Index output_tile_index(
-      GetBlockIdx(&ir_builder_, index_ty, num_tiles),
+      GetBlockIdx(&b_, index_ty, num_tiles),
       ShapeUtil::MakeShapeWithDescendingLayout(PRED /*arbitrary*/,
                                                output_dims_in_tiles),
-      &ir_builder_);
+      &b_);
 
   // Output tile origin is the index for the first element of the current output
   // tile.
   const IrArray::Index output_tile_origin = [&] {
     IrArray::Index index = output_tile_index;
     for (int i = 1; i < 3; ++i) {
-      index[i] = ir_builder_.CreateMul(output_tile_index[i],
-                                       index_typed_constant(kTileSize),
-                                       "tile_origin." + std::to_string(i));
+      index[i] =
+          b_.CreateMul(output_tile_index[i], index_typed_constant(kTileSize),
+                       "tile_origin." + std::to_string(i));
     }
     return index;
   }();
@@ -3121,16 +3084,15 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
   std::vector output_tile_bounds(3);
   for (int i = 1; i < 3; ++i) {
     // Only last row or column may not have full size.
-    output_tile_bounds[i] = ir_builder_.CreateSelect(
-        ir_builder_.CreateICmpEQ(
-            output_tile_index[i],
-            index_typed_constant(output_dims_in_tiles[i] - 1)),
+    output_tile_bounds[i] = b_.CreateSelect(
+        b_.CreateICmpEQ(output_tile_index[i],
+                        index_typed_constant(output_dims_in_tiles[i] - 1)),
         index_typed_constant(reduced_output_dims[i] -
                              (output_dims_in_tiles[i] - 1) * kTileSize),
         index_typed_constant(kTileSize), "kTileSize");
   }
 
-  KernelSupportLibrary ksl(&ir_builder_, llvm_ir::UnrollMode::kDefaultUnroll);
+  KernelSupportLibrary ksl(&b_, llvm_ir::UnrollMode::kDefaultUnroll);
 
   // Curry a few parameters to EmitTiledElementalCodeWithBoundsCheck.
   auto emit_tiled_elemental_code_with_bounds_check =
@@ -3139,13 +3101,13 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
           const std::function&
               emit_elem_function) {
         EmitTiledElementalCodeWithBoundsCheck(
-            kTileSize, kNumRows, index, loop_name, &ksl, &ir_builder_, y, x,
-            tile_width, tile_height, emit_elem_function);
+            kTileSize, kNumRows, index, loop_name, &ksl, &b_, y, x, tile_width,
+            tile_height, emit_elem_function);
       };
 
   // Adds `addend` to the given `dim` of `index`.
   auto offset_dim = [&](IrArray::Index index, llvm::Value* addend, int64 dim) {
-    index[dim] = ir_builder_.CreateAdd(index[dim], addend);
+    index[dim] = b_.CreateAdd(index[dim], addend);
     return index;
   };
   const IrArray::Index input_index =
@@ -3161,19 +3123,17 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
           llvm::Value* shmem_buffer = param_shmem_buffers[id];
           // TODO(jlebar): Add AA metadata to this store.  Tile buffers are
           // global variables, so LLVM can't infer much about it.
-          ir_builder_.CreateStore(
-              input_in_logical_shape.EmitReadArrayElement(index, &ir_builder_,
+          b_.CreateStore(
+              input_in_logical_shape.EmitReadArrayElement(index, &b_,
                                                           "input_element"),
-              ir_builder_.CreateGEP(shmem_buffer,
-                                    {index_typed_constant(0), y_loc, x}));
+              b_.CreateGEP(shmem_buffer, {index_typed_constant(0), y_loc, x}));
         }
       });
 
   // Wait for all threads to reach this point, lest we copy a value from tile to
   // output before the other thread copies it from input to tile.
   // This is `__syncthreads` in CUDA.
-  llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {},
-                               &ir_builder_);
+  llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_barrier0, {}, {}, &b_);
 
   llvm_ir::TiledParameterInfo tiled_param_info(param_shmem_buffers, y, x);
 
@@ -3187,27 +3147,26 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
         output_index, "output", output_tile_bounds[2], output_tile_bounds[1],
         [&](const IrArray::Index& index, llvm::Value* y_loc) {
           // TODO(jlebar): Add AA metadata to this load.
-          llvm::Instruction* load_from_shmem_buffer = ir_builder_.CreateLoad(
-              ir_builder_.CreateGEP(param_shmem_buffers[0],
-                                    {ir_builder_.getInt64(0), x, y_loc}),
+          llvm::Instruction* load_from_shmem_buffer = b_.CreateLoad(
+              b_.CreateGEP(param_shmem_buffers[0], {b_.getInt64(0), x, y_loc}),
               "output_element");
           output_in_reduced_shape_arrays[0].EmitWriteArrayElement(
-              index, load_from_shmem_buffer, &ir_builder_);
+              index, load_from_shmem_buffer, &b_);
         });
   } else {
     CHECK_EQ(hlo->opcode(), HloOpcode::kFusion);
     emit_tiled_elemental_code_with_bounds_check(
         output_index, "output", output_tile_bounds[2], output_tile_bounds[1],
         [&](const IrArray::Index& index, llvm::Value* y_loc) {
-          GpuElementalIrEmitter elem_emitter(hlo_module_config_, module_,
-                                             &ir_builder_, GetNestedComputer());
+          GpuElementalIrEmitter elem_emitter(hlo_module_config_, module_, &b_,
+                                             GetNestedComputer());
           FusedIrEmitter fused_emitter(param_arrays, &elem_emitter);
           tiled_param_info.set_y(y_loc);
           fused_emitter.SetTiledParameterInfo(&tiled_param_info);
           TF_CHECK_OK(hlo->fused_expression_root()->Accept(&fused_emitter));
           IrArray::Index untiled_index = llvm_ir::GetUnreducedOutputIndex(
               index, output_reduced_shapes[0], output_arrays[0].GetShape(),
-              &ir_builder_);
+              &b_);
           const llvm_ir::ElementGenerator& output_generator =
               fused_emitter.GetRootGenerator();
           llvm::Value* output_value =
@@ -3218,12 +3177,11 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
                      output_in_reduced_shape_arrays.size());
             for (int64 i = 0; i < output_in_reduced_shape_arrays.size(); ++i) {
               output_in_reduced_shape_arrays[i].EmitWriteArrayElement(
-                  index, ir_builder_.CreateExtractValue(output_value, i),
-                  &ir_builder_);
+                  index, b_.CreateExtractValue(output_value, i), &b_);
             }
           } else {
             output_in_reduced_shape_arrays[0].EmitWriteArrayElement(
-                index, output_value, &ir_builder_);
+                index, output_value, &b_);
           }
         });
   }
@@ -3234,7 +3192,7 @@ LaunchDimensions IrEmitterUnnested::EmitHlo021Tile(
     for (int64 i = 0; i < output_arrays.size(); ++i) {
       tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer());
     }
-    llvm_ir::EmitTuple(GetIrArray(*hlo, *hlo), tuple_operand_ptrs, &ir_builder_,
+    llvm_ir::EmitTuple(GetIrArray(*hlo, *hlo), tuple_operand_ptrs, &b_,
                        module_);
   }
 
diff --git a/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc b/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc
index cd833ec7bd..3838fee674 100644
--- a/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.cc
@@ -32,27 +32,27 @@ namespace gpu {
 
 ParallelLoopEmitter::ParallelLoopEmitter(
     BodyEmitter body_emitter, const Shape& shape,
-    const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder,
+    const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* b,
     int unroll_factor)
-    : LoopEmitter(body_emitter, shape, ir_builder),
+    : LoopEmitter(body_emitter, shape, b),
       launch_dimensions_(launch_dimensions),
       unroll_factor_(unroll_factor) {}
 
 ParallelLoopEmitter::ParallelLoopEmitter(
     const llvm_ir::ElementGenerator& target_element_generator,
     tensorflow::gtl::ArraySlice target_arrays,
-    const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder,
+    const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* b,
     int unroll_factor)
-    : LoopEmitter(target_element_generator, target_arrays, ir_builder),
+    : LoopEmitter(target_element_generator, target_arrays, b),
       launch_dimensions_(launch_dimensions),
       unroll_factor_(unroll_factor) {}
 
 ParallelLoopEmitter::ParallelLoopEmitter(
     const llvm_ir::ElementGenerator& target_element_generator,
     const llvm_ir::IrArray& target_array,
-    const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder,
+    const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* b,
     int unroll_factor)
-    : LoopEmitter(target_element_generator, target_array, ir_builder),
+    : LoopEmitter(target_element_generator, target_array, b),
       launch_dimensions_(launch_dimensions),
       unroll_factor_(unroll_factor) {}
 
@@ -74,29 +74,27 @@ ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock(
   CHECK_NE(index_type, nullptr);
   std::vector array_indices;
   llvm::Value* block_id = llvm_ir::EmitCallToIntrinsic(
-      llvm::Intrinsic::nvvm_read_ptx_sreg_ctaid_x, {}, {}, ir_builder_);
+      llvm::Intrinsic::nvvm_read_ptx_sreg_ctaid_x, {}, {}, b_);
   llvm_ir::AddRangeMetadata(0, launch_dimensions_.block_count(),
                             static_cast(block_id));
-  block_id = ir_builder_->CreateZExtOrTrunc(block_id, index_type, "block_id");
+  block_id = b_->CreateZExtOrTrunc(block_id, index_type, "block_id");
 
   // Per the PTX documentation:
   //   "It is guaranteed that [...] 0  <=  %tid.x <  %ntid.x"
   //
   // %ntid.x is currently specified as 1024.
   llvm::Value* thread_id = llvm_ir::EmitCallToIntrinsic(
-      llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x, {}, {}, ir_builder_);
+      llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x, {}, {}, b_);
   llvm_ir::AddRangeMetadata(0, launch_dimensions_.threads_per_block(),
                             static_cast(thread_id));
-  thread_id =
-      ir_builder_->CreateZExtOrTrunc(thread_id, index_type, "thread_id");
-
-  llvm::Value* linear_index_base = ir_builder_->CreateAdd(
-      ir_builder_->CreateMul(
-          block_id,
-          llvm::ConstantInt::get(index_type,
-                                 launch_dimensions_.threads_per_block()),
-          "",
-          /*HasNUW=*/true, /*HasNSW=*/true),
+  thread_id = b_->CreateZExtOrTrunc(thread_id, index_type, "thread_id");
+
+  llvm::Value* linear_index_base = b_->CreateAdd(
+      b_->CreateMul(block_id,
+                    llvm::ConstantInt::get(
+                        index_type, launch_dimensions_.threads_per_block()),
+                    "",
+                    /*HasNUW=*/true, /*HasNSW=*/true),
       thread_id, "linear_index", /*HasNUW=*/true, /*HasNSW=*/true);
 
   // Add an @llvm.assume(linear_index < threads_per_block * num_blocks).
@@ -109,41 +107,41 @@ ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock(
   // conditions in the same basic block as their operands.
   llvm_ir::EmitCallToIntrinsic(
       llvm::Intrinsic::assume,
-      {ir_builder_->CreateICmpULT(
+      {b_->CreateICmpULT(
           linear_index_base,
           llvm::ConstantInt::get(index_type,
                                  launch_dimensions_.threads_per_block() *
                                      launch_dimensions_.block_count()),
           "linear_index_in_range")},
-      {}, ir_builder_);
+      {}, b_);
 
   if (unroll_factor_ > 1) {
-    linear_index_base = ir_builder_->CreateMul(
+    linear_index_base = b_->CreateMul(
         linear_index_base, llvm::ConstantInt::get(index_type, unroll_factor_),
         "linear_index_base", /*HasNUW=*/true, /*HasNSW=*/true);
   }
 
-  array_indices.emplace_back(linear_index_base, shape_, ir_builder_);
+  array_indices.emplace_back(linear_index_base, shape_, b_);
   for (int i = 1; i < unroll_factor_; ++i) {
-    llvm::Value* linear_index = ir_builder_->CreateAdd(
-        linear_index_base, llvm::ConstantInt::get(index_type, i),
-        "linear_index",
-        /*HasNUW=*/true, /*HasNSW=*/true);
-    array_indices.emplace_back(linear_index, shape_, ir_builder_);
+    llvm::Value* linear_index =
+        b_->CreateAdd(linear_index_base, llvm::ConstantInt::get(index_type, i),
+                      "linear_index",
+                      /*HasNUW=*/true, /*HasNSW=*/true);
+    array_indices.emplace_back(linear_index, shape_, b_);
   }
 
   auto if_in_bounds = llvm_ir::EmitIfThenElse(
-      ir_builder_->CreateICmpULT(
+      b_->CreateICmpULT(
           linear_index_base,
           llvm::ConstantInt::get(index_type, ShapeUtil::ElementsIn(shape_))),
-      llvm_ir::IrName(loop_name, "in_bounds"), ir_builder_, false);
+      llvm_ir::IrName(loop_name, "in_bounds"), b_, false);
 
   // Set exit_bb_ to the exit block of the if structure.
   exit_bb_ = if_in_bounds.after_block;
   CHECK_NE(nullptr, exit_bb_);
 
   // Set IR builder insertion point to the body of the if structure.
-  llvm_ir::SetToFirstInsertPoint(if_in_bounds.true_block, ir_builder_);
+  llvm_ir::SetToFirstInsertPoint(if_in_bounds.true_block, b_);
 
   return array_indices;
 }
diff --git a/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h b/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h
index 302e1bf1bc..b82a23419d 100644
--- a/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h
@@ -34,13 +34,13 @@ class ParallelLoopEmitter : public llvm_ir::LoopEmitter {
   // The meanings of other parameters are the same as LoopEmitter.
   ParallelLoopEmitter(BodyEmitter body_emitter, const Shape& shape,
                       const LaunchDimensions& launch_dimensions,
-                      llvm::IRBuilder<>* ir_builder, int unroll_factor = 1);
+                      llvm::IRBuilder<>* b, int unroll_factor = 1);
   // Constructs a ParallelLoopEmitter from an element generator that generates
   // each element of the given target array.
   ParallelLoopEmitter(const llvm_ir::ElementGenerator& target_element_generator,
                       const llvm_ir::IrArray& target_array,
                       const LaunchDimensions& launch_dimensions,
-                      llvm::IRBuilder<>* ir_builder, int unroll_factor = 1);
+                      llvm::IRBuilder<>* b, int unroll_factor = 1);
 
   // Constructs a loop emitter for a loop that generates on element of each of N
   // arrays on each iteration.
@@ -50,7 +50,7 @@ class ParallelLoopEmitter : public llvm_ir::LoopEmitter {
   ParallelLoopEmitter(
       const llvm_ir::ElementGenerator& target_element_generator,
       tensorflow::gtl::ArraySlice target_arrays,
-      const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* ir_builder,
+      const LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* b,
       int unroll_factor = 1);
 
   ParallelLoopEmitter(const ParallelLoopEmitter&) = delete;
diff --git a/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
index 7048fcfdc9..1bd73fc793 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.cc
@@ -40,14 +40,14 @@ static Status EmitDynamicUpdateSliceInPlaceImpl(
     const Shape& update_shape, const ElementGenerator& start_indices_generator,
     bool is_signed, ElementGenerator update_array_generator,
     const IrArray& output_array, const gpu::LaunchDimensions* launch_dimensions,
-    tensorflow::StringPiece name, llvm::IRBuilder<>* ir_builder) {
+    tensorflow::StringPiece name, llvm::IRBuilder<>* b) {
   const Shape& output_shape = output_array.GetShape();
 
   // Read start indices from start_indices_generator.
   const int64 rank = ShapeUtil::Rank(output_shape);
-  IrArray::Index start_index(ir_builder->getInt64Ty(), rank);
+  IrArray::Index start_index(b->getInt64Ty(), rank);
   for (int64 i = 0; i < rank; ++i) {
-    IrArray::Index dim_index({ir_builder->getInt64(i)});
+    IrArray::Index dim_index({b->getInt64(i)});
     TF_ASSIGN_OR_RETURN(start_index[i], start_indices_generator(dim_index));
     llvm::Value* output_dim_size = llvm::ConstantInt::get(
         start_index[i]->getType(), output_shape.dimensions(i));
@@ -60,20 +60,19 @@ static Status EmitDynamicUpdateSliceInPlaceImpl(
     // TODO(b/74360564): This is implementation defined behavior, but is
     // currently respected by all implementations. Change this if we ever decide
     // to officially document different behavior.
-    llvm::Value* max_bound =
-        ir_builder->CreateSub(output_dim_size, update_dim_size);
+    llvm::Value* max_bound = b->CreateSub(output_dim_size, update_dim_size);
     llvm::Value* zero = llvm::ConstantInt::get(start_index[i]->getType(), 0);
-    start_index[i] = ir_builder->CreateSelect(
-        ir_builder->CreateICmp(
-            is_signed ? llvm::ICmpInst::ICMP_SGE : llvm::ICmpInst::ICMP_UGE,
-            zero, start_index[i]),
-        zero, start_index[i]);
-
-    start_index[i] = ir_builder->CreateSelect(
-        ir_builder->CreateICmp(
-            is_signed ? llvm::ICmpInst::ICMP_SLE : llvm::ICmpInst::ICMP_ULE,
-            max_bound, start_index[i]),
-        max_bound, start_index[i]);
+    start_index[i] =
+        b->CreateSelect(b->CreateICmp(is_signed ? llvm::ICmpInst::ICMP_SGE
+                                                : llvm::ICmpInst::ICMP_UGE,
+                                      zero, start_index[i]),
+                        zero, start_index[i]);
+
+    start_index[i] =
+        b->CreateSelect(b->CreateICmp(is_signed ? llvm::ICmpInst::ICMP_SLE
+                                                : llvm::ICmpInst::ICMP_ULE,
+                                      max_bound, start_index[i]),
+                        max_bound, start_index[i]);
   }
 
   auto loop_body_emitter = [&](const IrArray::Index& update_index) -> Status {
@@ -84,31 +83,30 @@ static Status EmitDynamicUpdateSliceInPlaceImpl(
     //
     IrArray::Index output_index(start_index.GetType(), rank);
     for (int64 i = 0; i < rank; ++i) {
-      llvm::Value* start_index0 = ir_builder->CreateSExtOrBitCast(
-          start_index[i], update_index[i]->getType());
-      output_index[i] = ir_builder->CreateAdd(start_index0, update_index[i]);
+      llvm::Value* start_index0 =
+          b->CreateSExtOrBitCast(start_index[i], update_index[i]->getType());
+      output_index[i] = b->CreateAdd(start_index0, update_index[i]);
     }
 
     // Do output[output_index] = update[update_index].
     TF_ASSIGN_OR_RETURN(llvm::Value * update_data,
                         update_array_generator(update_index));
-    output_array.EmitWriteArrayElement(output_index, update_data, ir_builder);
+    output_array.EmitWriteArrayElement(output_index, update_data, b);
     return Status::OK();
   };
 
   if (launch_dimensions != nullptr) {
     return gpu::ParallelLoopEmitter(loop_body_emitter, update_shape,
-                                    *launch_dimensions, ir_builder)
+                                    *launch_dimensions, b)
         .EmitLoop(name);
   }
-  return LoopEmitter(loop_body_emitter, update_shape, ir_builder)
-      .EmitLoop(name);
+  return LoopEmitter(loop_body_emitter, update_shape, b).EmitLoop(name);
 }
 
 Status EmitDynamicUpdateSliceInPlace(
     tensorflow::gtl::ArraySlice operand_arrays,
     const IrArray& output_array, tensorflow::StringPiece name,
-    llvm::IRBuilder<>* ir_builder) {
+    llvm::IRBuilder<>* b) {
   VLOG(2) << "EmitDynamicUpdateSliceInPlace for " << name;
 
   // No need to use operand_arrays[0], the input array of the
@@ -119,16 +117,16 @@ Status EmitDynamicUpdateSliceInPlace(
   Shape update_shape = update_array.GetShape();
 
   ElementGenerator start_indices_generator = [&](const IrArray::Index& index) {
-    return start_indices_array.EmitReadArrayElement(index, ir_builder);
+    return start_indices_array.EmitReadArrayElement(index, b);
   };
   ElementGenerator update_array_generator = [&](const IrArray::Index& index) {
-    return update_array.EmitReadArrayElement(index, ir_builder);
+    return update_array.EmitReadArrayElement(index, b);
   };
 
   bool is_signed = ShapeUtil::ElementIsSigned(start_indices_array.GetShape());
   return EmitDynamicUpdateSliceInPlaceImpl(
       update_shape, start_indices_generator, is_signed, update_array_generator,
-      output_array, /*launch_dimensions=*/nullptr, name, ir_builder);
+      output_array, /*launch_dimensions=*/nullptr, name, b);
 }
 
 // Shared implementation for EmitFusedDynamicUpdateSliceInPlace and
@@ -139,8 +137,7 @@ static Status EmitFusedDynamicUpdateSliceInPlaceImpl(
     HloInstruction* fusion,
     tensorflow::gtl::ArraySlice fusion_operand_arrays,
     const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
-    const gpu::LaunchDimensions* launch_dimensions,
-    llvm::IRBuilder<>* ir_builder) {
+    const gpu::LaunchDimensions* launch_dimensions, llvm::IRBuilder<>* b) {
   CHECK_EQ(fusion->opcode(), HloOpcode::kFusion);
   VLOG(2) << "EmitFusedDynamicUpdateSliceInPlace for "
           << fusion->ToShortString();
@@ -177,28 +174,27 @@ static Status EmitFusedDynamicUpdateSliceInPlaceImpl(
   bool is_signed = ShapeUtil::ElementIsSigned(start_indices->shape());
   return EmitDynamicUpdateSliceInPlaceImpl(
       update_shape, start_indices_generator, is_signed, update_array_generator,
-      fusion_output_array, launch_dimensions, IrName(fusion), ir_builder);
+      fusion_output_array, launch_dimensions, IrName(fusion), b);
 }
 
 Status EmitFusedDynamicUpdateSliceInPlace(
     HloInstruction* fusion,
     tensorflow::gtl::ArraySlice fusion_operand_arrays,
     const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
-    llvm::IRBuilder<>* ir_builder) {
+    llvm::IRBuilder<>* b) {
   return EmitFusedDynamicUpdateSliceInPlaceImpl(
       fusion, fusion_operand_arrays, fusion_output_array, elemental_emitter,
-      /*launch_dimensions=*/nullptr, ir_builder);
+      /*launch_dimensions=*/nullptr, b);
 }
 
 Status EmitParallelFusedDynamicUpdateSliceInPlace(
     HloInstruction* fusion,
     tensorflow::gtl::ArraySlice fusion_operand_arrays,
     const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
-    const gpu::LaunchDimensions& launch_dimensions,
-    llvm::IRBuilder<>* ir_builder) {
+    const gpu::LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* b) {
   return EmitFusedDynamicUpdateSliceInPlaceImpl(
       fusion, fusion_operand_arrays, fusion_output_array, elemental_emitter,
-      &launch_dimensions, ir_builder);
+      &launch_dimensions, b);
 }
 
 }  // namespace llvm_ir
diff --git a/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h
index 7f73fb6b29..3502577d23 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/dynamic_update_slice_util.h
@@ -66,7 +66,7 @@ inline bool CanEmitFusedDynamicUpdateSliceInPlace(
 Status EmitDynamicUpdateSliceInPlace(
     tensorflow::gtl::ArraySlice operand_arrays,
     const IrArray& output_array, tensorflow::StringPiece name,
-    llvm::IRBuilder<>* ir_builder);
+    llvm::IRBuilder<>* b);
 
 // Given a loop-fusion node whose root is a dynamic-update-slice op whose
 // array-to-be-updated and output share the same buffer slice, emits
@@ -76,7 +76,7 @@ Status EmitFusedDynamicUpdateSliceInPlace(
     HloInstruction* fusion,
     tensorflow::gtl::ArraySlice fusion_operand_arrays,
     const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
-    llvm::IRBuilder<>* ir_builder);
+    llvm::IRBuilder<>* b);
 
 // Same as EmitFusedDynamicUpdateSliceInPlace, except emits a parallel loop with
 // the given launch dimensions.
@@ -84,8 +84,7 @@ Status EmitParallelFusedDynamicUpdateSliceInPlace(
     HloInstruction* fusion,
     tensorflow::gtl::ArraySlice fusion_operand_arrays,
     const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
-    const gpu::LaunchDimensions& launch_dimensions,
-    llvm::IRBuilder<>* ir_builder);
+    const gpu::LaunchDimensions& launch_dimensions, llvm::IRBuilder<>* b);
 
 }  // namespace llvm_ir
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
index b12ce97e28..72ede377e1 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
@@ -52,7 +52,7 @@ Status FusedIrEmitter::DefaultAction(HloInstruction* hlo) {
       // that would be regenerated without caching. But this might increase the
       // JIT compilation time.
       if (generated_value_bb == nullptr ||
-          generated_value_bb == ir_builder_->GetInsertBlock()) {
+          generated_value_bb == b_->GetInsertBlock()) {
         VLOG(3) << "The cached generated value is reused.";
         return generated_value;
       }
@@ -60,8 +60,7 @@ Status FusedIrEmitter::DefaultAction(HloInstruction* hlo) {
                  "a different BB ("
               << llvm_ir::AsString(generated_value_bb->getName())
               << ") from the current insertion block ("
-              << llvm_ir::AsString(ir_builder_->GetInsertBlock()->getName())
-              << ").";
+              << llvm_ir::AsString(b_->GetInsertBlock()->getName()) << ").";
     }
 
     TF_ASSIGN_OR_RETURN(
@@ -77,14 +76,14 @@ Status FusedIrEmitter::HandleConstant(HloInstruction* constant) {
   llvm::Constant* initializer =
       llvm_ir::ConvertLiteralToIrConstant(literal, module_);
   llvm::GlobalVariable* global = new llvm::GlobalVariable(
-      *ir_builder_->GetInsertBlock()->getModule(), initializer->getType(),
+      *b_->GetInsertBlock()->getModule(), initializer->getType(),
       /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, initializer,
       /*Name=*/"");
   llvm::Constant* shape_constant = llvm::ConstantExpr::getBitCast(
       global, llvm_ir::ShapeToIrType(literal.shape(), module_)->getPointerTo());
   generators_[constant] = [=](const IrArray::Index& index) {
     return IrArray(shape_constant, constant->shape())
-        .EmitReadArrayElement(index, ir_builder_);
+        .EmitReadArrayElement(index, b_);
   };
 
   return Status::OK();
@@ -104,7 +103,7 @@ Status FusedIrEmitter::HandleGetTupleElement(
   // Emit code to lookup tuple element pointer, and store it in 'gte_values_'.
   llvm::Value* tuple_element_ptr = llvm_ir::EmitGetTupleElement(
       get_tuple_element->shape(), get_tuple_element->tuple_index(),
-      /*alignment=*/1, it->second, ir_builder_, module_);
+      /*alignment=*/1, it->second, b_, module_);
   gte_values_.insert(std::make_pair(get_tuple_element, tuple_element_ptr));
   // Emit code to read base tuple element array (if non-tuple shaped).
   if (!ShapeUtil::IsTuple(get_tuple_element->shape())) {
@@ -112,7 +111,7 @@ Status FusedIrEmitter::HandleGetTupleElement(
         [=](const IrArray::Index& index) -> StatusOr {
       // TODO(b/34080002) Add aliasing information to tuple element IrArray.
       return IrArray(tuple_element_ptr, get_tuple_element->shape())
-          .EmitReadArrayElement(index, ir_builder_);
+          .EmitReadArrayElement(index, b_);
     };
   }
   return Status::OK();
@@ -129,16 +128,15 @@ Status FusedIrEmitter::HandleParameter(HloInstruction* parameter) {
         // want the AA info to be present before address spaces are inferred
         // (which is pretty late in the pipeline), so even if we had
         // address-space-based AA in LLVM, it wouldn't help us much here.
-        return ir_builder_->CreateLoad(
-            ir_builder_->CreateGEP(
-                param_tile_buffer,
-                {index.GetConstantWithIndexType(0), tiled_parameter_info_->x(),
-                 tiled_parameter_info_->y()}),
+        return b_->CreateLoad(
+            b_->CreateGEP(param_tile_buffer, {index.GetConstantWithIndexType(0),
+                                              tiled_parameter_info_->x(),
+                                              tiled_parameter_info_->y()}),
             "tiled_buffer");
       }
     }
     return parameter_arrays_[parameter->parameter_number()]
-        .EmitReadArrayElement(index, ir_builder_);
+        .EmitReadArrayElement(index, b_);
   };
   // Store ir value for fusion operand associated with fusion parameter to be
   // accessed by subsequent fused GetTupleElement instructions.
@@ -157,11 +155,11 @@ Status FusedIrEmitter::HandleTuple(HloInstruction* tuple) {
   }
   generators_[tuple] =
       [=](const IrArray::Index& index) -> StatusOr {
-    llvm::Value* ret = llvm::UndefValue::get(llvm::StructType::get(
-        ir_builder_->getContext(), operand_elemental_ir_types));
+    llvm::Value* ret = llvm::UndefValue::get(
+        llvm::StructType::get(b_->getContext(), operand_elemental_ir_types));
     for (size_t i = 0; i < ShapeUtil::TupleElementCount(tuple->shape()); ++i) {
       TF_ASSIGN_OR_RETURN(llvm::Value * val_i, generators_[operands[i]](index));
-      ret = ir_builder_->CreateInsertValue(ret, val_i, i);
+      ret = b_->CreateInsertValue(ret, val_i, i);
     }
     return ret;
   };
diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
index a6ceec7b23..30471480c4 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
@@ -59,7 +59,7 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault {
       : parameter_arrays_(parameter_arrays),
         tiled_parameter_info_(nullptr),
         elemental_emitter_(elemental_emitter),
-        ir_builder_(elemental_emitter->ir_builder()),
+        b_(elemental_emitter->b()),
         module_(elemental_emitter->module()) {}
 
   Status DefaultAction(HloInstruction* hlo) override;
@@ -103,7 +103,7 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault {
   const HloInstruction* fused_root_ = nullptr;
 
   // Borrowed
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
   llvm::Module* module_;
 
   // Map from instruction pointers to functions to generate elements of their
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
index dcf9838d80..7a9170f379 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
@@ -31,7 +31,7 @@ namespace llvm_ir {
 
 void IrArray::Index::Delinearize(std::vector* multidim,
                                  llvm::Value* linear, const Shape& shape,
-                                 llvm::IRBuilder<>* ir_builder) const {
+                                 llvm::IRBuilder<>* b) const {
   int64 divisor = 1;
   const Layout& layout = shape.layout();
   for (int64 i = 0; i < layout.minor_to_major_size(); ++i) {
@@ -48,10 +48,9 @@ void IrArray::Index::Delinearize(std::vector* multidim,
     // useful because cuda-memcheck can't help us much in XLA: Most of our
     // memory lives in one big allocation, so cuda-memcheck can't detect
     // out-of-bounds accesses.
-    auto* quot =
-        ir_builder->CreateUDiv(linear, GetConstantWithIndexType(divisor));
+    auto* quot = b->CreateUDiv(linear, GetConstantWithIndexType(divisor));
     if (i < layout.minor_to_major_size() - 1) {
-      (*multidim)[dimension] = ir_builder->CreateURem(
+      (*multidim)[dimension] = b->CreateURem(
           quot, GetConstantWithIndexType(size_of_current_dimension));
     } else {
       (*multidim)[dimension] = quot;
@@ -61,7 +60,7 @@ void IrArray::Index::Delinearize(std::vector* multidim,
 }
 
 IrArray::Index::Index(llvm::Value* linear, const Shape& shape,
-                      llvm::IRBuilder<>* ir_builder)
+                      llvm::IRBuilder<>* b)
     : multidim_(ShapeUtil::Rank(shape)),
       linear_(linear),
       layout_(shape.layout()),
@@ -71,7 +70,7 @@ IrArray::Index::Index(llvm::Value* linear, const Shape& shape,
   CHECK(LayoutUtil::HasLayout(shape))
       << "Shape " << ShapeUtil::HumanStringWithLayout(shape)
       << " should have a layout.";
-  Delinearize(&multidim_, linear, shape, ir_builder);
+  Delinearize(&multidim_, linear, shape, b);
 }
 
 IrArray::Index::Index(tensorflow::gtl::ArraySlice multidim,
@@ -94,7 +93,7 @@ IrArray::Index::Index(tensorflow::gtl::ArraySlice multidim,
 }
 
 IrArray::Index::Index(tensorflow::gtl::ArraySlice multidim,
-                      const Shape& shape, llvm::IRBuilder<>* ir_builder)
+                      const Shape& shape, llvm::IRBuilder<>* b)
     : multidim_(multidim.begin(), multidim.end()),
       layout_(shape.layout()),
       dims_(shape.dimensions().begin(), shape.dimensions().end()) {
@@ -343,7 +342,7 @@ llvm::Value* IrArray::Index::Linearize(
 }
 
 llvm::Value* IrArray::EmitArrayElementAddress(
-    const IrArray::Index& index, llvm::IRBuilder<>* ir_builder,
+    const IrArray::Index& index, llvm::IRBuilder<>* b,
     tensorflow::StringPiece name) const {
   if (ShapeUtil::IsScalar(*shape_)) {
     // Special handling of scalars: a scalar pretends to have the same value for
@@ -354,12 +353,11 @@ llvm::Value* IrArray::EmitArrayElementAddress(
   CHECK_EQ(index.size(), ShapeUtil::Rank(*shape_));
 
   if (index.LinearValidOnShape(*shape_)) {
-    llvm::Module* module =
-        ir_builder->GetInsertBlock()->getParent()->getParent();
-    return ir_builder->CreateInBoundsGEP(
-        ir_builder->CreateBitCast(
-            base_ptr_, PrimitiveTypeToIrType(shape_->element_type(), module)
-                           ->getPointerTo()),
+    llvm::Module* module = b->GetInsertBlock()->getParent()->getParent();
+    return b->CreateInBoundsGEP(
+        b->CreateBitCast(base_ptr_,
+                         PrimitiveTypeToIrType(shape_->element_type(), module)
+                             ->getPointerTo()),
         {index.linear()}, llvm_ir::AsStringRef(name));
   }
 
@@ -385,8 +383,8 @@ llvm::Value* IrArray::EmitArrayElementAddress(
     int64 dimension = LayoutUtil::Major(shape_->layout(), i);
     gep_indices.push_back(actual_index[dimension]);
   }
-  return ir_builder->CreateInBoundsGEP(base_ptr_, gep_indices,
-                                       llvm_ir::AsStringRef(name));
+  return b->CreateInBoundsGEP(base_ptr_, gep_indices,
+                              llvm_ir::AsStringRef(name));
 }
 
 void IrArray::AnnotateLoadStoreInstructionWithMetadata(
@@ -402,29 +400,27 @@ void IrArray::AnnotateLoadStoreInstructionWithMetadata(
 }
 
 llvm::Value* IrArray::EmitReadArrayElement(const Index& index,
-                                           llvm::IRBuilder<>* ir_builder,
+                                           llvm::IRBuilder<>* b,
                                            tensorflow::StringPiece name) const {
-  llvm::Value* element_address =
-      EmitArrayElementAddress(index, ir_builder, name);
-  llvm::LoadInst* load = ir_builder->CreateLoad(element_address);
+  llvm::Value* element_address = EmitArrayElementAddress(index, b, name);
+  llvm::LoadInst* load = b->CreateLoad(element_address);
   AnnotateLoadStoreInstructionWithMetadata(load);
   return load;
 }
 
 void IrArray::EmitWriteArrayElement(const Index& index, llvm::Value* value,
-                                    llvm::IRBuilder<>* ir_builder) const {
-  llvm::Value* element_address = EmitArrayElementAddress(index, ir_builder);
-  llvm::StoreInst* store = ir_builder->CreateStore(value, element_address);
+                                    llvm::IRBuilder<>* b) const {
+  llvm::Value* element_address = EmitArrayElementAddress(index, b);
+  llvm::StoreInst* store = b->CreateStore(value, element_address);
   AnnotateLoadStoreInstructionWithMetadata(store);
 }
 
 IrArray IrArray::CastToShape(const Shape& new_shape,
-                             llvm::IRBuilder<>* ir_builder) const {
-  llvm::Module* module = ir_builder->GetInsertBlock()->getParent()->getParent();
+                             llvm::IRBuilder<>* b) const {
+  llvm::Module* module = b->GetInsertBlock()->getParent()->getParent();
   llvm::Type* new_ir_type = llvm_ir::ShapeToIrType(new_shape, module);
   IrArray new_irarray(
-      ir_builder->CreatePointerCast(base_ptr_, new_ir_type->getPointerTo()),
-      new_shape);
+      b->CreatePointerCast(base_ptr_, new_ir_type->getPointerTo()), new_shape);
   new_irarray.metadata_ = metadata_;
   return new_irarray;
 }
@@ -432,9 +428,9 @@ IrArray IrArray::CastToShape(const Shape& new_shape,
 /* static */ IrArray::Index IrArray::BumpIndex(const Index& index,
                                                int64 which_dimension,
                                                int64 addend,
-                                               llvm::IRBuilder<>* ir_builder) {
+                                               llvm::IRBuilder<>* b) {
   Index new_index = index;
-  new_index[which_dimension] = ir_builder->CreateAdd(
+  new_index[which_dimension] = b->CreateAdd(
       index[which_dimension],
       llvm::ConstantInt::get(index[which_dimension]->getType(), addend), "",
       /*HasNUW=*/true,
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
index 0777c49923..28ca793e3e 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.h
@@ -87,20 +87,19 @@ class IrArray {
     }
 
     // Constructs an index from linear index "linear" and computes the
-    // multi-dimensional index from "linear" and "shape". "ir_builder" is the IR
+    // multi-dimensional index from "linear" and "shape". "b" is the IR
     // builder to emit the index of each dimension in the multi-dimensional
     // index.
     //
     // Precondition: "shape" has a layout.
-    Index(llvm::Value* linear, const Shape& shape,
-          llvm::IRBuilder<>* ir_builder);
+    Index(llvm::Value* linear, const Shape& shape, llvm::IRBuilder<>* b);
 
     // Constructs an index from the given multi-dimensional index and the shape
     // that it indexes into.
     //
     // Precondition: "shape" has a layout.
     Index(tensorflow::gtl::ArraySlice multidim,
-          const Shape& shape, llvm::IRBuilder<>* ir_builder);
+          const Shape& shape, llvm::IRBuilder<>* b);
 
     // Constructs an index from both a multi-dimensional index and a linear
     // index. "shape" has the same meaning as that in the constructor that takes
@@ -191,7 +190,7 @@ class IrArray {
     }
 
     void Delinearize(std::vector* multidim, llvm::Value* linear,
-                     const Shape& shape, llvm::IRBuilder<>* ir_builder) const;
+                     const Shape& shape, llvm::IRBuilder<>* b) const;
 
     std::vector multidim_;
 
@@ -240,8 +239,7 @@ class IrArray {
   //
   // The optional name is useful for debugging when looking at
   // the emitted LLVM IR.
-  llvm::Value* EmitArrayElementAddress(const Index& index,
-                                       llvm::IRBuilder<>* ir_builder,
+  llvm::Value* EmitArrayElementAddress(const Index& index, llvm::IRBuilder<>* b,
                                        tensorflow::StringPiece name = "") const;
 
   // Attach metadata this IrArray instance knows about to "instruction".
@@ -255,18 +253,16 @@ class IrArray {
   //
   // The optional name is useful for debugging when looking at
   // the emitted LLVM IR.
-  llvm::Value* EmitReadArrayElement(const Index& index,
-                                    llvm::IRBuilder<>* ir_builder,
+  llvm::Value* EmitReadArrayElement(const Index& index, llvm::IRBuilder<>* b,
                                     tensorflow::StringPiece name = "") const;
 
   // Emit IR to write the given value to the array element at the given index.
   void EmitWriteArrayElement(const Index& index, llvm::Value* value,
-                             llvm::IRBuilder<>* ir_builder) const;
+                             llvm::IRBuilder<>* b) const;
 
   // Returns a new IrArray whose shape is "new_shape" and base pointer is a
   // bitcast of the base pointer of "this" IrArray.
-  IrArray CastToShape(const Shape& new_shape,
-                      llvm::IRBuilder<>* ir_builder) const;
+  IrArray CastToShape(const Shape& new_shape, llvm::IRBuilder<>* b) const;
 
   void AddAliasScopeMetadata(llvm::MDNode* alias_scope) {
     CHECK_NE(alias_scope, nullptr);
@@ -312,7 +308,7 @@ class IrArray {
   // Bumps the "which_dimension" value within the provided index by the provided
   // addend.
   static Index BumpIndex(const Index& index, int64 which_dimension,
-                         int64 addend, llvm::IRBuilder<>* ir_builder);
+                         int64 addend, llvm::IRBuilder<>* b);
 
  private:
   // Add the specified LLVM IR metadata to loads/stores associated with this
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.cc b/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.cc
index 98d0ceb3e2..b79567369a 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.cc
@@ -22,9 +22,9 @@ Status KernelSupportLibrary::For(
     tensorflow::StringPiece name, llvm::Value* start, llvm::Value* end,
     llvm::Value* step,
     const std::function& for_body_generator) {
-  return If(ir_builder_->CreateICmpSLT(start, end), [&]() -> Status {
+  return If(b_->CreateICmpSLT(start, end), [&]() -> Status {
     TF_RETURN_IF_ERROR(for_body_generator(start, /*is_first_iteration=*/true));
-    return For(name, ir_builder_->CreateAdd(start, step), end, step,
+    return For(name, b_->CreateAdd(start, step), end, step,
                [&](llvm::Value* iv) { return for_body_generator(iv, false); });
   });
 }
@@ -37,20 +37,20 @@ Status KernelSupportLibrary::For(
   if (peel_first_iteration) {
     return For(name, start, end, step, true,
                [&](llvm::Value* indvar, bool is_first_iteration) -> Status {
-                 return for_body_generator(
-                     indvar, ir_builder_->getInt1(is_first_iteration));
+                 return for_body_generator(indvar,
+                                           b_->getInt1(is_first_iteration));
                });
   } else {
     std::unique_ptr loop = llvm_ir::ForLoop::EmitForLoop(
-        name, start, end, step, ir_builder_,
+        name, start, end, step, b_,
         /*unroll_mode=*/unroll_mode_,
         /*prevent_vectorization=*/prevent_vectorization_);
-    ir_builder_->SetInsertPoint(&loop->GetBodyBasicBlock()->back());
+    b_->SetInsertPoint(&loop->GetBodyBasicBlock()->back());
     TF_RETURN_IF_ERROR(
         for_body_generator(loop->GetIndVarValue(),
-                           /*is_first_iteration=*/ir_builder_->CreateICmpEQ(
+                           /*is_first_iteration=*/b_->CreateICmpEQ(
                                loop->GetIndVarValue(), start)));
-    llvm_ir::SetToLastInsertPoint(loop->GetExitBasicBlock(), ir_builder_);
+    llvm_ir::SetToLastInsertPoint(loop->GetExitBasicBlock(), b_);
     return Status::OK();
   }
 }
@@ -59,23 +59,22 @@ Status KernelSupportLibrary::If(
     tensorflow::StringPiece name, llvm::Value* condition,
     const std::function& true_block_generator,
     const std::function& false_block_generator) {
-  llvm_ir::LlvmIfData if_data =
-      llvm_ir::EmitIfThenElse(condition, name, ir_builder_);
-  ir_builder_->SetInsertPoint(&if_data.true_block->back());
+  llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(condition, name, b_);
+  b_->SetInsertPoint(&if_data.true_block->back());
   TF_RETURN_IF_ERROR(true_block_generator());
-  ir_builder_->SetInsertPoint(&if_data.false_block->back());
+  b_->SetInsertPoint(&if_data.false_block->back());
   TF_RETURN_IF_ERROR(false_block_generator());
-  llvm_ir::SetToLastInsertPoint(if_data.after_block, ir_builder_);
+  llvm_ir::SetToLastInsertPoint(if_data.after_block, b_);
   return Status::OK();
 }
 
 void KernelSupportLibrary::EmitAndCallOutlinedKernel(
-    bool enable_fast_math, bool optimize_for_size,
-    llvm::IRBuilder<>* ir_builder, tensorflow::StringPiece kernel_name,
+    bool enable_fast_math, bool optimize_for_size, llvm::IRBuilder<>* b,
+    tensorflow::StringPiece kernel_name,
     KernelSupportLibrary::ArgumentVector arguments,
     const std::function&
         kernel_body_generator) {
-  llvm::Module* module = ir_builder->GetInsertBlock()->getModule();
+  llvm::Module* module = b->GetInsertBlock()->getModule();
   llvm::Function* function =
       module->getFunction(llvm_ir::AsStringRef(kernel_name));
 
@@ -98,22 +97,22 @@ void KernelSupportLibrary::EmitAndCallOutlinedKernel(
                    std::back_inserter(arg_types),
                    [](llvm::Value* arg) { return arg->getType(); });
 
-    auto* function_type = llvm::FunctionType::get(
-        ir_builder->getVoidTy(), arg_types, /*isVarArg=*/false);
+    auto* function_type =
+        llvm::FunctionType::get(b->getVoidTy(), arg_types, /*isVarArg=*/false);
 
     function = llvm_ir::CreateFunction(
         function_type, llvm::GlobalValue::InternalLinkage,
         /*enable_fast_math=*/enable_fast_math,
         /*optimize_for_size=*/optimize_for_size, kernel_name, module);
 
-    llvm::IRBuilder<>::InsertPointGuard guard(*ir_builder);
+    llvm::IRBuilder<>::InsertPointGuard guard(*b);
 
     auto* entry_bb =
-        llvm::BasicBlock::Create(ir_builder->getContext(), "entry", function);
-    auto* return_inst = llvm::ReturnInst::Create(ir_builder->getContext(),
+        llvm::BasicBlock::Create(b->getContext(), "entry", function);
+    auto* return_inst = llvm::ReturnInst::Create(b->getContext(),
                                                  /*retVal=*/nullptr, entry_bb);
     // Set the insert point to before return_inst.
-    ir_builder->SetInsertPoint(return_inst);
+    b->SetInsertPoint(return_inst);
 
     std::vector arg_values;
     /*
@@ -133,7 +132,7 @@ void KernelSupportLibrary::EmitAndCallOutlinedKernel(
     VLOG(3) << "Re-using kernel for " << kernel_name;
   }
 
-  ir_builder->CreateCall(function, llvm_ir::AsArrayRef(sanitized_args));
+  b->CreateCall(function, llvm_ir::AsArrayRef(sanitized_args));
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h
index 9d770cc4c3..b00f903d56 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h
@@ -30,14 +30,14 @@ namespace xla {
 // flow more readable.
 class KernelSupportLibrary {
  public:
-  // `ir_builder` is the llvm::IRBuilder instance used to generate LLVM IR.
+  // `b` is the llvm::IRBuilder instance used to generate LLVM IR.
   // `unroll_mode` specifies the desired LLVM unrolling behavior for every loop
   // generated by this instance of KernelSupportLibrary.
   explicit KernelSupportLibrary(
-      llvm::IRBuilder<>* ir_builder,
+      llvm::IRBuilder<>* b,
       llvm_ir::UnrollMode unroll_mode = llvm_ir::UnrollMode::kNoUnroll,
       bool prevent_vectorization = true)
-      : ir_builder_(ir_builder),
+      : b_(b),
         unroll_mode_(unroll_mode),
         prevent_vectorization_(prevent_vectorization) {}
 
@@ -71,18 +71,18 @@ class KernelSupportLibrary {
              const std::function&
                  for_body_generator) {
-    return For(name, /*start=*/ir_builder_->getInt64(start),
-               /*end=*/ir_builder_->getInt64(end),
-               /*step=*/ir_builder_->getInt64(step), for_body_generator);
+    return For(name, /*start=*/b_->getInt64(start),
+               /*end=*/b_->getInt64(end),
+               /*step=*/b_->getInt64(step), for_body_generator);
   }
 
   void ForReturnVoid(
       tensorflow::StringPiece name, int64 start, int64 end, int64 step,
       const std::function&
           for_body_generator) {
-    ForReturnVoid(name, /*start=*/ir_builder_->getInt64(start),
-                  /*end=*/ir_builder_->getInt64(end),
-                  /*step=*/ir_builder_->getInt64(step), for_body_generator);
+    ForReturnVoid(name, /*start=*/b_->getInt64(start),
+                  /*end=*/b_->getInt64(end),
+                  /*step=*/b_->getInt64(step), for_body_generator);
   }
 
   // Generates the following control flow structure if `peel_first_iteration` is
@@ -184,17 +184,17 @@ class KernelSupportLibrary {
   Status For(
       tensorflow::StringPiece name, int64 start, int64 end, int64 step,
       const std::function& for_body_generator) {
-    return For(name, /*start=*/ir_builder_->getInt64(start),
-               /*end=*/ir_builder_->getInt64(end),
-               /*step=*/ir_builder_->getInt64(step), for_body_generator);
+    return For(name, /*start=*/b_->getInt64(start),
+               /*end=*/b_->getInt64(end),
+               /*step=*/b_->getInt64(step), for_body_generator);
   }
 
   void ForReturnVoid(
       tensorflow::StringPiece name, int64 start, int64 end, int64 step,
       const std::function& for_body_generator) {
-    ForReturnVoid(name, /*start=*/ir_builder_->getInt64(start),
-                  /*end=*/ir_builder_->getInt64(end),
-                  /*step=*/ir_builder_->getInt64(step), for_body_generator);
+    ForReturnVoid(name, /*start=*/b_->getInt64(start),
+                  /*end=*/b_->getInt64(end),
+                  /*step=*/b_->getInt64(step), for_body_generator);
   }
 
   // Generates the following control flow structure:
@@ -258,41 +258,39 @@ class KernelSupportLibrary {
   // in a nullptr llvm::Value* in its position to `kernel_body_generator`.
   // Currently we only support at most one nullptr value in `arguments`.
   static void EmitAndCallOutlinedKernel(
-      bool enable_fast_math, bool optimize_for_size,
-      llvm::IRBuilder<>* ir_builder, tensorflow::StringPiece kernel_name,
-      ArgumentVector arguments,
+      bool enable_fast_math, bool optimize_for_size, llvm::IRBuilder<>* b,
+      tensorflow::StringPiece kernel_name, ArgumentVector arguments,
       const std::function& kernel_body_generator);
 
   // Thin wrappers around the more general EmitAndCallOutlinedKernel above.
   static void EmitAndCallOutlinedKernel(
-      bool enable_fast_math, bool optimize_for_size,
-      llvm::IRBuilder<>* ir_builder, tensorflow::StringPiece kernel_name,
-      llvm::Value* arg0, llvm::Value* arg1, llvm::Value* arg2,
+      bool enable_fast_math, bool optimize_for_size, llvm::IRBuilder<>* b,
+      tensorflow::StringPiece kernel_name, llvm::Value* arg0, llvm::Value* arg1,
+      llvm::Value* arg2,
       const std::function&
           kernel_body_generator) {
     EmitAndCallOutlinedKernel(
-        enable_fast_math, optimize_for_size, ir_builder, kernel_name,
-        {arg0, arg1, arg2}, [&](ArgumentVector args) {
+        enable_fast_math, optimize_for_size, b, kernel_name, {arg0, arg1, arg2},
+        [&](ArgumentVector args) {
           kernel_body_generator(args[0], args[1], args[2]);
         });
   }
 
   static void EmitAndCallOutlinedKernel(
-      bool enable_fast_math, bool optimize_for_size,
-      llvm::IRBuilder<>* ir_builder, tensorflow::StringPiece kernel_name,
-      llvm::Value* arg0, llvm::Value* arg1, llvm::Value* arg2,
-      llvm::Value* arg3,
+      bool enable_fast_math, bool optimize_for_size, llvm::IRBuilder<>* b,
+      tensorflow::StringPiece kernel_name, llvm::Value* arg0, llvm::Value* arg1,
+      llvm::Value* arg2, llvm::Value* arg3,
       const std::function& kernel_body_generator) {
     EmitAndCallOutlinedKernel(
-        enable_fast_math, optimize_for_size, ir_builder, kernel_name,
+        enable_fast_math, optimize_for_size, b, kernel_name,
         {arg0, arg1, arg2, arg3}, [&](ArgumentVector args) {
           kernel_body_generator(args[0], args[1], args[2], args[3]);
         });
   }
 
  private:
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
   llvm_ir::UnrollMode unroll_mode_;
   bool prevent_vectorization_;
 };
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
index 533b75cdae..35b3941272 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.cc
@@ -94,24 +94,24 @@ tensorflow::gtl::optional > FindTranspose021(
 IrArray::Index GetUnreducedOutputIndex(
     const IrArray::Index& reduced_output_index,
     const Shape& reduced_output_shape, const Shape& unreduced_output_shape,
-    llvm::IRBuilder<>* ir_builder) {
+    llvm::IRBuilder<>* b) {
   auto bounds = reduced_output_shape.dimensions();
   auto minor_to_major = reduced_output_shape.layout().minor_to_major();
   llvm::Value* linear_index = reduced_output_index.GetConstantWithIndexType(0);
   int64 multiplier = 1;
   for (int i = 0; i < reduced_output_index.size(); ++i) {
     int64 dim = minor_to_major[i];
-    llvm::Value* addend = ir_builder->CreateMul(
-        reduced_output_index[dim],
-        reduced_output_index.GetConstantWithIndexType(multiplier),
-        "linearizing",
-        /*HasNUW=*/true, /*HasNSW=*/true);
-    linear_index = ir_builder->CreateAdd(linear_index, addend, "",
-                                         /*HasNUW=*/true, /*HasNSW=*/true);
+    llvm::Value* addend =
+        b->CreateMul(reduced_output_index[dim],
+                     reduced_output_index.GetConstantWithIndexType(multiplier),
+                     "linearizing",
+                     /*HasNUW=*/true, /*HasNSW=*/true);
+    linear_index = b->CreateAdd(linear_index, addend, "",
+                                /*HasNUW=*/true, /*HasNSW=*/true);
     multiplier *= bounds[dim];
   }
 
-  return IrArray::Index(linear_index, unreduced_output_shape, ir_builder);
+  return IrArray::Index(linear_index, unreduced_output_shape, b);
 }
 
 }  // namespace llvm_ir
diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
index 6f1268fffb..ccb9b8ba3e 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_tiling.h
@@ -44,7 +44,7 @@ tensorflow::gtl::optional > FindTranspose021(const Shape& a,
 IrArray::Index GetUnreducedOutputIndex(
     const IrArray::Index& reduced_output_index,
     const Shape& reduced_output_shape, const Shape& unreduced_output_shape,
-    llvm::IRBuilder<>* ir_builder);
+    llvm::IRBuilder<>* b);
 
 // A class to represent information for tiled parameters to support IR emission
 // for 021 transpose.
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc
index 1227534779..ba7f94834c 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.cc
@@ -47,27 +47,27 @@ ForLoop::ForLoop(tensorflow::StringPiece prefix, tensorflow::StringPiece suffix,
 
 /* static */ std::unique_ptr ForLoop::EmitForLoop(
     tensorflow::StringPiece prefix, llvm::Value* start_index,
-    llvm::Value* end_index, llvm::Value* step, llvm::IRBuilder<>* ir_builder,
+    llvm::Value* end_index, llvm::Value* step, llvm::IRBuilder<>* b,
     UnrollMode unroll_mode, bool prevent_vectorization) {
   std::unique_ptr loop(new ForLoop(prefix, /*suffix=*/"", start_index,
                                             end_index, step, unroll_mode,
                                             prevent_vectorization));
-  loop->Emit(ir_builder);
+  loop->Emit(b);
   return loop;
 }
 
-void ForLoop::Emit(llvm::IRBuilder<>* ir_builder) {
+void ForLoop::Emit(llvm::IRBuilder<>* b) {
   // The preheader block is the block the builder is currently emitting
   // code into.
-  preheader_bb_ = ir_builder->GetInsertBlock();
+  preheader_bb_ = b->GetInsertBlock();
 
-  llvm::BasicBlock::iterator insert_point = ir_builder->GetInsertPoint();
+  llvm::BasicBlock::iterator insert_point = b->GetInsertPoint();
   if (insert_point == preheader_bb_->end()) {
     // We're emitting the loop at the end of a basic block. Verify there is no
     // terminator (eg, branch) in the basic block.
     CHECK_EQ(nullptr, preheader_bb_->getTerminator());
 
-    exit_bb_ = CreateLoopBB("loop_exit", ir_builder);
+    exit_bb_ = CreateLoopBB("loop_exit", b);
   } else {
     // We're emitting the loop into the middle of a basic block. splitBasicBlock
     // requires that this basic block be well-formed (have a terminator).
@@ -86,51 +86,50 @@ void ForLoop::Emit(llvm::IRBuilder<>* ir_builder) {
   insert_before_bb_ = exit_bb_;
 
   // Create remaining basic block which form the inside of the loop.
-  header_bb_ = CreateLoopBB("loop_header", ir_builder);
-  body_bb_ = CreateLoopBB("loop_body", ir_builder);
+  header_bb_ = CreateLoopBB("loop_header", b);
+  body_bb_ = CreateLoopBB("loop_body", b);
 
   // Function entry basic block.
   // Emit alloca for the induction variable. We do this at the entry to the
   // basic block to ensure the alloc only executes once per function (we could
   // be emitting a nested loop).
   llvm::Function* func = preheader_bb_->getParent();
-  ir_builder->SetInsertPoint(&func->getEntryBlock(),
-                             func->getEntryBlock().getFirstInsertionPt());
+  b->SetInsertPoint(&func->getEntryBlock(),
+                    func->getEntryBlock().getFirstInsertionPt());
   llvm::Value* indvar_address =
-      ir_builder->CreateAlloca(start_index_->getType(), nullptr,
-                               AsStringRef(GetQualifiedName("invar_address")));
+      b->CreateAlloca(start_index_->getType(), nullptr,
+                      AsStringRef(GetQualifiedName("invar_address")));
 
   // Preheader basic block.
   // Initialize induction variable starting index. Create branch to the header.
-  ir_builder->SetInsertPoint(preheader_bb_);
-  ir_builder->CreateStore(start_index_, indvar_address);
+  b->SetInsertPoint(preheader_bb_);
+  b->CreateStore(start_index_, indvar_address);
   // The preheader should not have a branch yet.
   CHECK_EQ(preheader_bb_->getTerminator(), nullptr);
-  ir_builder->CreateBr(header_bb_);
+  b->CreateBr(header_bb_);
 
   // Header basic block.
   // Emit the loop conditional branch. Load and compare indvar with ending
   // index and jump to loop exit if equal. Jump to body otherwise.
-  ir_builder->SetInsertPoint(header_bb_);
-  indvar_ = ir_builder->CreateLoad(indvar_address,
-                                   AsStringRef(GetQualifiedName("indvar")));
-  llvm::Value* exit_cond = ir_builder->CreateICmpUGE(indvar_, end_index_);
-  ir_builder->CreateCondBr(/*Cond=*/exit_cond,
-                           /*True=*/exit_bb_, /*False=*/body_bb_);
+  b->SetInsertPoint(header_bb_);
+  indvar_ =
+      b->CreateLoad(indvar_address, AsStringRef(GetQualifiedName("indvar")));
+  llvm::Value* exit_cond = b->CreateICmpUGE(indvar_, end_index_);
+  b->CreateCondBr(/*Cond=*/exit_cond,
+                  /*True=*/exit_bb_, /*False=*/body_bb_);
 
   // Body basic block.
   // Increment indvar, store indvar, and jump to header.
-  ir_builder->SetInsertPoint(body_bb_);
+  b->SetInsertPoint(body_bb_);
   llvm::Value* step = step_;
   llvm::Value* indvar = indvar_;
 
-  llvm::Value* indvar_inc =
-      ir_builder->CreateAdd(indvar, step, "invar.inc",
-                            /*HasNUW=*/true, /*HasNSW=*/true);
-  ir_builder->CreateStore(indvar_inc, indvar_address);
-  llvm::BranchInst* back_branch = ir_builder->CreateBr(header_bb_);
+  llvm::Value* indvar_inc = b->CreateAdd(indvar, step, "invar.inc",
+                                         /*HasNUW=*/true, /*HasNSW=*/true);
+  b->CreateStore(indvar_inc, indvar_address);
+  llvm::BranchInst* back_branch = b->CreateBr(header_bb_);
 
-  std::vector loop_metadata = GetLoopMetadata(ir_builder);
+  std::vector loop_metadata = GetLoopMetadata(b);
   if (!loop_metadata.empty()) {
     llvm::LLVMContext* ctx = &start_index_->getContext();
     auto temp_node = llvm::MDNode::getTemporary(*ctx, llvm::None);
@@ -141,11 +140,10 @@ void ForLoop::Emit(llvm::IRBuilder<>* ir_builder) {
   }
 
   // Re-point the IR builder to the loop exit block.
-  ir_builder->SetInsertPoint(exit_bb_);
+  b->SetInsertPoint(exit_bb_);
 }
 
-std::vector ForLoop::GetLoopMetadata(
-    llvm::IRBuilder<>* ir_builder) {
+std::vector ForLoop::GetLoopMetadata(llvm::IRBuilder<>* b) {
   const char* const kLlvmLoopUnrollDisableMDName = "llvm.loop.unroll.disable";
   const char* const kLlvmLoopUnrollFullMDName = "llvm.loop.unroll.full";
   const char* const kLlvmLoopVectorizeMDName = "llvm.loop.vectorize.enable";
@@ -160,7 +158,7 @@ std::vector ForLoop::GetLoopMetadata(
   if (prevent_vectorization_) {
     result.push_back(llvm::MDNode::get(
         *ctx, {llvm::MDString::get(*ctx, kLlvmLoopVectorizeMDName),
-               llvm::ConstantAsMetadata::get(ir_builder->getFalse())}));
+               llvm::ConstantAsMetadata::get(b->getFalse())}));
   }
 
   if (unroll_mode_ == xla::llvm_ir::UnrollMode::kFullyUnroll) {
@@ -175,9 +173,8 @@ string ForLoop::GetQualifiedName(tensorflow::StringPiece name) {
 }
 
 llvm::BasicBlock* ForLoop::CreateLoopBB(tensorflow::StringPiece name,
-                                        llvm::IRBuilder<>* ir_builder) {
-  return CreateBasicBlock(insert_before_bb_, GetQualifiedName(name),
-                          ir_builder);
+                                        llvm::IRBuilder<>* b) {
+  return CreateBasicBlock(insert_before_bb_, GetQualifiedName(name), b);
 }
 
 std::unique_ptr ForLoopNest::AddLoop(tensorflow::StringPiece suffix,
@@ -197,12 +194,12 @@ std::unique_ptr ForLoopNest::AddLoop(tensorflow::StringPiece suffix,
                                               bool prevent_vectorization) {
   if (inner_loop_body_bb_ != nullptr) {
     // Create this loop inside the previous one.
-    ir_builder_->SetInsertPoint(&*inner_loop_body_bb_->getFirstInsertionPt());
+    b_->SetInsertPoint(&*inner_loop_body_bb_->getFirstInsertionPt());
   }
   std::unique_ptr loop(new ForLoop(
       /*prefix=*/name_, suffix, start_index, end_index, stride, unroll_mode,
       prevent_vectorization));
-  loop->Emit(ir_builder_);
+  loop->Emit(b_);
 
   if (outer_loop_preheader_bb_ == nullptr) {
     outer_loop_preheader_bb_ = loop->GetPreheaderBasicBlock();
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h
index b3266022db..a4fed5c8dc 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h
@@ -79,7 +79,7 @@ class ForLoop {
   //  loop.
   static std::unique_ptr EmitForLoop(
       tensorflow::StringPiece prefix, llvm::Value* start_index,
-      llvm::Value* end_index, llvm::Value* step, llvm::IRBuilder<>* ir_builder,
+      llvm::Value* end_index, llvm::Value* step, llvm::IRBuilder<>* b,
       UnrollMode unroll_mode = llvm_ir::UnrollMode::kDefaultUnroll,
       bool prevent_vectorization = false);
 
@@ -138,10 +138,10 @@ class ForLoop {
           UnrollMode unroll_mode, bool prevent_vectorization);
 
   // Emit the loop at the insert point of the builder.
-  void Emit(llvm::IRBuilder<>* ir_builder);
+  void Emit(llvm::IRBuilder<>* b);
 
   llvm::BasicBlock* CreateLoopBB(tensorflow::StringPiece name,
-                                 llvm::IRBuilder<>* ir_builder);
+                                 llvm::IRBuilder<>* b);
 
   // Creates a name for an LLVM construct, appending prefix_ and suffix_, if
   // they are set.
@@ -149,7 +149,7 @@ class ForLoop {
 
   // Return a list of metadata nodes that should be associated with the
   // llvm::Loop for this `ForLoop`.
-  std::vector GetLoopMetadata(llvm::IRBuilder<>* ir_builder);
+  std::vector GetLoopMetadata(llvm::IRBuilder<>* b);
 
   string prefix_;
   string suffix_;
@@ -177,19 +177,18 @@ class ForLoop {
 // A simple class for constructing nested for-loops.
 class ForLoopNest {
  public:
-  explicit ForLoopNest(llvm::IRBuilder<>* ir_builder,
-                       llvm::Type* index_ty = nullptr)
-      : ForLoopNest(/*name=*/"", ir_builder) {
+  explicit ForLoopNest(llvm::IRBuilder<>* b, llvm::Type* index_ty = nullptr)
+      : ForLoopNest(/*name=*/"", b) {
     SetIndexType(index_ty);
   }
 
-  ForLoopNest(tensorflow::StringPiece name, llvm::IRBuilder<>* ir_builder,
+  ForLoopNest(tensorflow::StringPiece name, llvm::IRBuilder<>* b,
               llvm::Type* index_ty = nullptr)
       : name_(std::string(name)),
         outer_loop_preheader_bb_(nullptr),
         outer_loop_exit_bb_(nullptr),
         inner_loop_body_bb_(nullptr),
-        ir_builder_(ir_builder) {
+        b_(b) {
     SetIndexType(index_ty);
   }
 
@@ -270,7 +269,7 @@ class ForLoopNest {
 
  private:
   void SetIndexType(llvm::Type* index_ty) {
-    index_type_ = index_ty == nullptr ? ir_builder_->getInt64Ty() : index_ty;
+    index_type_ = index_ty == nullptr ? b_->getInt64Ty() : index_ty;
   }
 
   llvm::Constant* GetConstantWithIndexType(int64 c) const {
@@ -289,7 +288,7 @@ class ForLoopNest {
   // has been added yet.
   llvm::BasicBlock* inner_loop_body_bb_;
 
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
 
   llvm::Type* index_type_;
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
index 6c55361b44..e4f65bd427 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
@@ -48,8 +48,8 @@ namespace {
 
 // Note, this function is only useful in an insertion context; in a global
 // (e.g. constants) context it will CHECK fail.
-llvm::Module* ModuleFromIRBuilder(llvm::IRBuilder<>* ir_builder) {
-  auto block = CHECK_NOTNULL(ir_builder->GetInsertBlock());
+llvm::Module* ModuleFromIRBuilder(llvm::IRBuilder<>* b) {
+  auto block = CHECK_NOTNULL(b->GetInsertBlock());
   auto fn = CHECK_NOTNULL(block->getParent());
   auto module = CHECK_NOTNULL(fn->getParent());
   return module;
@@ -87,41 +87,41 @@ llvm::Value* EmitCallToIntrinsic(
     llvm::Intrinsic::ID intrinsic_id,
     tensorflow::gtl::ArraySlice operands,
     tensorflow::gtl::ArraySlice overloaded_types,
-    llvm::IRBuilder<>* ir_builder) {
-  llvm::Module* module = ModuleFromIRBuilder(ir_builder);
+    llvm::IRBuilder<>* b) {
+  llvm::Module* module = ModuleFromIRBuilder(b);
   llvm::Function* intrinsic = llvm::Intrinsic::getDeclaration(
       module, intrinsic_id, AsArrayRef(overloaded_types));
-  return ir_builder->CreateCall(intrinsic, AsArrayRef(operands));
+  return b->CreateCall(intrinsic, AsArrayRef(operands));
 }
 
 llvm::Value* EmitFloatMax(llvm::Value* lhs_value, llvm::Value* rhs_value,
-                          llvm::IRBuilder<>* ir_builder) {
-  if (ir_builder->getFastMathFlags().noNaNs()) {
-    auto cmp = ir_builder->CreateFCmpUGE(lhs_value, rhs_value);
-    return ir_builder->CreateSelect(cmp, lhs_value, rhs_value);
+                          llvm::IRBuilder<>* b) {
+  if (b->getFastMathFlags().noNaNs()) {
+    auto cmp = b->CreateFCmpUGE(lhs_value, rhs_value);
+    return b->CreateSelect(cmp, lhs_value, rhs_value);
   } else {
-    auto cmp_ge = ir_builder->CreateFCmpOGE(lhs_value, rhs_value);
-    auto lhs_is_nan = ir_builder->CreateFCmpUNE(lhs_value, lhs_value);
-    auto sel_lhs = ir_builder->CreateOr(cmp_ge, lhs_is_nan);
-    return ir_builder->CreateSelect(sel_lhs, lhs_value, rhs_value);
+    auto cmp_ge = b->CreateFCmpOGE(lhs_value, rhs_value);
+    auto lhs_is_nan = b->CreateFCmpUNE(lhs_value, lhs_value);
+    auto sel_lhs = b->CreateOr(cmp_ge, lhs_is_nan);
+    return b->CreateSelect(sel_lhs, lhs_value, rhs_value);
   }
 }
 
 llvm::Value* EmitFloatMin(llvm::Value* lhs_value, llvm::Value* rhs_value,
-                          llvm::IRBuilder<>* ir_builder) {
-  if (ir_builder->getFastMathFlags().noNaNs()) {
-    auto cmp = ir_builder->CreateFCmpULE(lhs_value, rhs_value);
-    return ir_builder->CreateSelect(cmp, lhs_value, rhs_value);
+                          llvm::IRBuilder<>* b) {
+  if (b->getFastMathFlags().noNaNs()) {
+    auto cmp = b->CreateFCmpULE(lhs_value, rhs_value);
+    return b->CreateSelect(cmp, lhs_value, rhs_value);
   } else {
-    auto cmp_le = ir_builder->CreateFCmpOLE(lhs_value, rhs_value);
-    auto lhs_is_nan = ir_builder->CreateFCmpUNE(lhs_value, lhs_value);
-    auto sel_lhs = ir_builder->CreateOr(cmp_le, lhs_is_nan);
-    return ir_builder->CreateSelect(sel_lhs, lhs_value, rhs_value);
+    auto cmp_le = b->CreateFCmpOLE(lhs_value, rhs_value);
+    auto lhs_is_nan = b->CreateFCmpUNE(lhs_value, lhs_value);
+    auto sel_lhs = b->CreateOr(cmp_le, lhs_is_nan);
+    return b->CreateSelect(sel_lhs, lhs_value, rhs_value);
   }
 }
 
 llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, llvm::Value* index,
-                                   llvm::IRBuilder<>* ir_builder) {
+                                   llvm::IRBuilder<>* b) {
   llvm::Type* array_type = array->getType();
   CHECK(array_type->isPointerTy());
   llvm::PointerType* array_type_as_pointer =
@@ -131,16 +131,16 @@ llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, llvm::Value* index,
           << " array=" << llvm_ir::DumpToString(*array)
           << " index=" << llvm_ir::DumpToString(*index);
 
-  return ir_builder->CreateInBoundsGEP(
+  return b->CreateInBoundsGEP(
       array_type_as_pointer->getElementType(), array,
       llvm::isa(array)
-          ? llvm::ArrayRef({ir_builder->getInt64(0), index})
+          ? llvm::ArrayRef({b->getInt64(0), index})
           : index);
 }
 
 llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, int64 index,
-                                   llvm::IRBuilder<>* ir_builder) {
-  return EmitBufferIndexingGEP(array, ir_builder->getInt64(index), ir_builder);
+                                   llvm::IRBuilder<>* b) {
+  return EmitBufferIndexingGEP(array, b->getInt64(index), b);
 }
 
 llvm::Type* PrimitiveTypeToIrType(PrimitiveType element_type,
@@ -232,14 +232,15 @@ llvm::Type* ShapeToIrType(const Shape& shape, llvm::Module* module) {
   return result_type;
 }
 
-StatusOr EncodeSelfDescribingShapeConstant(
-    const Shape& shape, int32* shape_size, llvm::IRBuilder<>* ir_builder) {
+StatusOr EncodeSelfDescribingShapeConstant(const Shape& shape,
+                                                         int32* shape_size,
+                                                         llvm::IRBuilder<>* b) {
   string encoded_shape = shape.SerializeAsString();
   if (encoded_shape.size() > std::numeric_limits::max()) {
     return InternalError("Encoded shape size exceeded int32 size limit.");
   }
   *shape_size = static_cast(encoded_shape.size());
-  return ir_builder->CreateGlobalStringPtr(llvm_ir::AsStringRef(encoded_shape));
+  return b->CreateGlobalStringPtr(llvm_ir::AsStringRef(encoded_shape));
 }
 
 StatusOr DecodeSelfDescribingShapeConstant(const void* shape_ptr,
@@ -262,59 +263,57 @@ llvm::Constant* ConvertLiteralToIrConstant(const Literal& literal,
 
 llvm::AllocaInst* EmitAllocaAtFunctionEntry(llvm::Type* type,
                                             tensorflow::StringPiece name,
-                                            llvm::IRBuilder<>* ir_builder,
+                                            llvm::IRBuilder<>* b,
                                             int alignment) {
-  return EmitAllocaAtFunctionEntryWithCount(type, nullptr, name, ir_builder,
-                                            alignment);
+  return EmitAllocaAtFunctionEntryWithCount(type, nullptr, name, b, alignment);
 }
 
 llvm::AllocaInst* EmitAllocaAtFunctionEntryWithCount(
     llvm::Type* type, llvm::Value* element_count, tensorflow::StringPiece name,
-    llvm::IRBuilder<>* ir_builder, int alignment) {
-  llvm::IRBuilder<>::InsertPoint insert_point = ir_builder->saveIP();
-  llvm::Function* function = ir_builder->GetInsertBlock()->getParent();
-  ir_builder->SetInsertPoint(&function->getEntryBlock(),
-                             function->getEntryBlock().getFirstInsertionPt());
+    llvm::IRBuilder<>* b, int alignment) {
+  llvm::IRBuilder<>::InsertPoint insert_point = b->saveIP();
+  llvm::Function* function = b->GetInsertBlock()->getParent();
+  b->SetInsertPoint(&function->getEntryBlock(),
+                    function->getEntryBlock().getFirstInsertionPt());
   llvm::AllocaInst* alloca =
-      ir_builder->CreateAlloca(type, element_count, AsStringRef(name));
+      b->CreateAlloca(type, element_count, AsStringRef(name));
   if (alignment != 0) {
     alloca->setAlignment(alignment);
   }
-  ir_builder->restoreIP(insert_point);
+  b->restoreIP(insert_point);
   return alloca;
 }
 
 llvm::BasicBlock* CreateBasicBlock(llvm::BasicBlock* insert_before,
                                    tensorflow::StringPiece name,
-                                   llvm::IRBuilder<>* ir_builder) {
+                                   llvm::IRBuilder<>* b) {
   return llvm::BasicBlock::Create(
-      /*Context=*/ir_builder->getContext(),
+      /*Context=*/b->getContext(),
       /*Name=*/AsStringRef(name),
-      /*Parent=*/ir_builder->GetInsertBlock()->getParent(),
+      /*Parent=*/b->GetInsertBlock()->getParent(),
       /*InsertBefore*/ insert_before);
 }
 
 LlvmIfData EmitIfThenElse(llvm::Value* condition, tensorflow::StringPiece name,
-                          llvm::IRBuilder<>* ir_builder, bool emit_else) {
+                          llvm::IRBuilder<>* b, bool emit_else) {
   llvm_ir::LlvmIfData if_data;
-  if_data.if_block = ir_builder->GetInsertBlock();
-  if_data.true_block = CreateBasicBlock(
-      nullptr, tensorflow::strings::StrCat(name, "-true"), ir_builder);
+  if_data.if_block = b->GetInsertBlock();
+  if_data.true_block =
+      CreateBasicBlock(nullptr, tensorflow::strings::StrCat(name, "-true"), b);
   if_data.false_block =
-      emit_else ? CreateBasicBlock(nullptr,
-                                   tensorflow::strings::StrCat(name, "-false"),
-                                   ir_builder)
+      emit_else ? CreateBasicBlock(
+                      nullptr, tensorflow::strings::StrCat(name, "-false"), b)
                 : nullptr;
 
   // Add a terminator to the if block, if necessary.
   if (if_data.if_block->getTerminator() == nullptr) {
-    ir_builder->SetInsertPoint(if_data.if_block);
+    b->SetInsertPoint(if_data.if_block);
     if_data.after_block = CreateBasicBlock(
-        nullptr, tensorflow::strings::StrCat(name, "-after"), ir_builder);
-    ir_builder->CreateBr(if_data.after_block);
+        nullptr, tensorflow::strings::StrCat(name, "-after"), b);
+    b->CreateBr(if_data.after_block);
   } else {
     if_data.after_block = if_data.if_block->splitBasicBlock(
-        ir_builder->GetInsertPoint(),
+        b->GetInsertPoint(),
         AsStringRef(tensorflow::strings::StrCat(name, "-after")));
   }
 
@@ -322,39 +321,37 @@ LlvmIfData EmitIfThenElse(llvm::Value* condition, tensorflow::StringPiece name,
   // we're going to replace it with a conditional branch.
   if_data.if_block->getTerminator()->eraseFromParent();
 
-  ir_builder->SetInsertPoint(if_data.if_block);
-  ir_builder->CreateCondBr(
-      condition, if_data.true_block,
-      emit_else ? if_data.false_block : if_data.after_block);
+  b->SetInsertPoint(if_data.if_block);
+  b->CreateCondBr(condition, if_data.true_block,
+                  emit_else ? if_data.false_block : if_data.after_block);
 
-  ir_builder->SetInsertPoint(if_data.true_block);
-  ir_builder->CreateBr(if_data.after_block);
+  b->SetInsertPoint(if_data.true_block);
+  b->CreateBr(if_data.after_block);
 
   if (emit_else) {
-    ir_builder->SetInsertPoint(if_data.false_block);
-    ir_builder->CreateBr(if_data.after_block);
+    b->SetInsertPoint(if_data.false_block);
+    b->CreateBr(if_data.after_block);
   }
 
-  ir_builder->SetInsertPoint(if_data.after_block,
-                             if_data.after_block->getFirstInsertionPt());
+  b->SetInsertPoint(if_data.after_block,
+                    if_data.after_block->getFirstInsertionPt());
 
   return if_data;
 }
 
 llvm::Value* EmitComparison(llvm::CmpInst::Predicate predicate,
                             llvm::Value* lhs_value, llvm::Value* rhs_value,
-                            llvm::IRBuilder<>* ir_builder) {
+                            llvm::IRBuilder<>* b) {
   llvm::Value* comparison_result;
   if (lhs_value->getType()->isIntegerTy()) {
-    comparison_result = ir_builder->CreateICmp(predicate, lhs_value, rhs_value);
+    comparison_result = b->CreateICmp(predicate, lhs_value, rhs_value);
   } else {
-    comparison_result = ir_builder->CreateFCmp(predicate, lhs_value, rhs_value);
+    comparison_result = b->CreateFCmp(predicate, lhs_value, rhs_value);
   }
   // comparison_result is i1, but the NVPTX codegen incorrectly lowers i1
   // arrays. So we extend it to i8 so that it's addressable.
-  return ir_builder->CreateZExt(
-      comparison_result,
-      llvm_ir::PrimitiveTypeToIrType(PRED, ModuleFromIRBuilder(ir_builder)));
+  return b->CreateZExt(comparison_result, llvm_ir::PrimitiveTypeToIrType(
+                                              PRED, ModuleFromIRBuilder(b)));
 }
 
 // Internal helper that is called from emitted code to log an int64 value with a
@@ -363,17 +360,14 @@ static void LogS64(const char* tag, int64 value) {
   LOG(INFO) << tag << " (int64): " << value;
 }
 
-void EmitLogging(const char* tag, llvm::Value* value,
-                 llvm::IRBuilder<>* ir_builder) {
+void EmitLogging(const char* tag, llvm::Value* value, llvm::IRBuilder<>* b) {
   llvm::FunctionType* log_function_type = llvm::FunctionType::get(
-      ir_builder->getVoidTy(),
-      {ir_builder->getInt64Ty(), ir_builder->getInt64Ty()}, /*isVarArg=*/false);
-  ir_builder->CreateCall(
+      b->getVoidTy(), {b->getInt64Ty(), b->getInt64Ty()}, /*isVarArg=*/false);
+  b->CreateCall(
       log_function_type,
-      ir_builder->CreateIntToPtr(
-          ir_builder->getInt64(tensorflow::bit_cast(&LogS64)),
-          log_function_type->getPointerTo()),
-      {ir_builder->getInt64(tensorflow::bit_cast(tag)), value});
+      b->CreateIntToPtr(b->getInt64(tensorflow::bit_cast(&LogS64)),
+                        log_function_type->getPointerTo()),
+      {b->getInt64(tensorflow::bit_cast(tag)), value});
 }
 
 void SetAlignmentMetadataForLoad(llvm::LoadInst* load, uint64_t alignment) {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h
index 9c51861eac..d8746ffe01 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h
@@ -105,26 +105,26 @@ llvm::Value* EmitCallToIntrinsic(
     llvm::Intrinsic::ID intrinsic_id,
     tensorflow::gtl::ArraySlice operands,
     tensorflow::gtl::ArraySlice overloaded_types,
-    llvm::IRBuilder<>* ir_builder);
+    llvm::IRBuilder<>* b);
 
 // Emit float max. Emit maxnum intrinsic is fast math is disabled, or
 // fcmp+select otherwise
 llvm::Value* EmitFloatMax(llvm::Value* lhs_value, llvm::Value* rhs_value,
-                          llvm::IRBuilder<>* ir_builder);
+                          llvm::IRBuilder<>* b);
 
 // Emit float min. Emit minnum intrinsic is fast math is disabled, or
 // fcmp+select otherwise
 llvm::Value* EmitFloatMin(llvm::Value* lhs_value, llvm::Value* rhs_value,
-                          llvm::IRBuilder<>* ir_builder);
+                          llvm::IRBuilder<>* b);
 
 // Convenience methods for emitting a GEP instruction that indexes into a buffer
 // (1-dimensional array), equivalent to array[index]. The type is automatically
 // determined from the element type of the array.  The int64 index overload
 // wraps the index in a i64 llvm::Value.
 llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, llvm::Value* index,
-                                   llvm::IRBuilder<>* ir_builder);
+                                   llvm::IRBuilder<>* b);
 llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, int64 index,
-                                   llvm::IRBuilder<>* ir_builder);
+                                   llvm::IRBuilder<>* b);
 
 // Returns the LLVM type which represents the given XLA primitive type.
 llvm::Type* PrimitiveTypeToIrType(PrimitiveType element_type,
@@ -139,8 +139,9 @@ llvm::Type* ShapeToIrType(const Shape& shape, llvm::Module* module);
 
 // Returns a value that represents a pointer to a global string constant that
 // encodes the shape as a serialized protobuf.
-StatusOr EncodeSelfDescribingShapeConstant(
-    const Shape& shape, int32* shape_size, llvm::IRBuilder<>* ir_builder);
+StatusOr EncodeSelfDescribingShapeConstant(const Shape& shape,
+                                                         int32* shape_size,
+                                                         llvm::IRBuilder<>* b);
 
 // Inverses the encoding of a Shape protobuf into an LLVM global variable.
 //
@@ -164,21 +165,21 @@ llvm::Constant* ConvertLiteralToIrConstant(const Literal& literal,
 // through a loop.
 llvm::AllocaInst* EmitAllocaAtFunctionEntry(llvm::Type* type,
                                             tensorflow::StringPiece name,
-                                            llvm::IRBuilder<>* ir_builder,
+                                            llvm::IRBuilder<>* b,
                                             int alignment = 0);
 
 // As EmitAllocaAtFunctionEntry, but allocates element_count entries
 // instead of a single element.
 llvm::AllocaInst* EmitAllocaAtFunctionEntryWithCount(
     llvm::Type* type, llvm::Value* element_count, tensorflow::StringPiece name,
-    llvm::IRBuilder<>* ir_builder, int alignment = 0);
+    llvm::IRBuilder<>* b, int alignment = 0);
 
 // Creates a basic block with the same context and function as for the
 // builder. Inserts at the end of the function if insert_before is
 // null.
 llvm::BasicBlock* CreateBasicBlock(llvm::BasicBlock* insert_before,
                                    tensorflow::StringPiece name,
-                                   llvm::IRBuilder<>* ir_builder);
+                                   llvm::IRBuilder<>* b);
 
 // Struct with data on a conditional branch in a diamond shape created
 // via EmitIfThenElse.
@@ -210,13 +211,13 @@ struct LlvmIfData {
 // block with a terminator. If you need to use this for a
 // non-terminated block, just make the function able to do that too.
 LlvmIfData EmitIfThenElse(llvm::Value* condition, tensorflow::StringPiece name,
-                          llvm::IRBuilder<>* ir_builder, bool emit_else = true);
+                          llvm::IRBuilder<>* b, bool emit_else = true);
 
 // Emits a compare operation between "lhs" and "rhs" with the given predicate,
 // and then converts the result to i8 so that it is addressable.
 llvm::Value* EmitComparison(llvm::CmpInst::Predicate predicate,
                             llvm::Value* lhs, llvm::Value* rhs,
-                            llvm::IRBuilder<>* ir_builder);
+                            llvm::IRBuilder<>* b);
 
 // Emits a call that logs the given value with the given tag as a prefix.
 // The provided tag and value are passed to a runtime logging call that is
@@ -228,8 +229,7 @@ llvm::Value* EmitComparison(llvm::CmpInst::Predicate predicate,
 // Precondition: value must be an int64.
 // Precondition: tag must be a stable pointer for the lifetime of the generated
 // program (the constant pointer is burned in to the program).
-void EmitLogging(const char* tag, llvm::Value* value,
-                 llvm::IRBuilder<>* ir_builder);
+void EmitLogging(const char* tag, llvm::Value* value, llvm::IRBuilder<>* b);
 
 // Adds alignment metadata to a load instruction using the given alignment.
 // The alignment refers to the result of the load, not the load itself.
diff --git a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc
index e8b0605b9d..36f5fa1952 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc
@@ -33,26 +33,24 @@ namespace xla {
 namespace llvm_ir {
 
 LoopEmitter::LoopEmitter(const BodyEmitter& body_emitter, const Shape& shape,
-                         llvm::IRBuilder<>* ir_builder)
-    : body_emitter_(body_emitter), shape_(shape), ir_builder_(ir_builder) {}
+                         llvm::IRBuilder<>* b)
+    : body_emitter_(body_emitter), shape_(shape), b_(b) {}
 
 LoopEmitter::LoopEmitter(const ElementGenerator& target_element_generator,
-                         const IrArray& target_array,
-                         llvm::IRBuilder<>* ir_builder)
+                         const IrArray& target_array, llvm::IRBuilder<>* b)
     : body_emitter_([=](const llvm_ir::IrArray::Index array_index) -> Status {
         // Convert target_element_generator to a BodyEmitter.
         TF_ASSIGN_OR_RETURN(llvm::Value * target_element,
                             target_element_generator(array_index));
-        target_array.EmitWriteArrayElement(array_index, target_element,
-                                           ir_builder);
+        target_array.EmitWriteArrayElement(array_index, target_element, b);
         return Status::OK();
       }),
       shape_(target_array.GetShape()),
-      ir_builder_(ir_builder) {}
+      b_(b) {}
 
 static LoopEmitter::BodyEmitter MakeBodyEmitterForMultiOutputFusion(
     const ElementGenerator& target_element_generator,
-    const std::vector& target_arrays, llvm::IRBuilder<>* ir_builder) {
+    const std::vector& target_arrays, llvm::IRBuilder<>* b) {
   return [=](const llvm_ir::IrArray::Index array_index) {
     TF_ASSIGN_OR_RETURN(llvm::Value * target_element,
                         target_element_generator(array_index));
@@ -64,8 +62,7 @@ static LoopEmitter::BodyEmitter MakeBodyEmitterForMultiOutputFusion(
 
     for (int64 i = 0; i < target_arrays.size(); ++i) {
       target_arrays[i].EmitWriteArrayElement(
-          array_index, ir_builder->CreateExtractValue(target_element, i),
-          ir_builder);
+          array_index, b->CreateExtractValue(target_element, i), b);
     }
     return Status::OK();
   };
@@ -73,13 +70,12 @@ static LoopEmitter::BodyEmitter MakeBodyEmitterForMultiOutputFusion(
 
 LoopEmitter::LoopEmitter(const ElementGenerator& target_element_generator,
                          tensorflow::gtl::ArraySlice target_arrays,
-                         llvm::IRBuilder<>* ir_builder)
+                         llvm::IRBuilder<>* b)
     : body_emitter_(MakeBodyEmitterForMultiOutputFusion(
           target_element_generator,
-          std::vector(target_arrays.begin(), target_arrays.end()),
-          ir_builder)),
+          std::vector(target_arrays.begin(), target_arrays.end()), b)),
       shape_(target_arrays[0].GetShape()),
-      ir_builder_(ir_builder) {
+      b_(b) {
   // Sanity check: In multi-output fusion, all shapes produced must have the
   // same dimensions.
   for (const IrArray& array : target_arrays) {
@@ -102,7 +98,7 @@ std::vector LoopEmitter::EmitIndexAndSetExitBasicBlock(
   // Loops are added from outermost to innermost order with the ForLoopNest
   // class so emit loops in order from most-major dimension down to most-minor
   // dimension (of the target shape).
-  ForLoopNest loop_nest(loop_name, ir_builder_);
+  ForLoopNest loop_nest(loop_name, b_);
   IrArray::Index array_index(index_type, shape_.dimensions_size());
   for (int i = 0; i < LayoutUtil::MinorToMajor(shape_).size(); ++i) {
     int64 dimension = LayoutUtil::Major(shape_.layout(), i);
@@ -116,8 +112,8 @@ std::vector LoopEmitter::EmitIndexAndSetExitBasicBlock(
   // Set IR builder insertion point to the loop body basic block of the
   // innermost loop.
   llvm::BasicBlock* innermost_body_bb = loop_nest.GetInnerLoopBodyBasicBlock();
-  ir_builder_->SetInsertPoint(innermost_body_bb,
-                              innermost_body_bb->getFirstInsertionPt());
+  b_->SetInsertPoint(innermost_body_bb,
+                     innermost_body_bb->getFirstInsertionPt());
 
   // Set exit_bb_ to the exit block of the loop nest.
   exit_bb_ = loop_nest.GetOuterLoopExitBasicBlock();
@@ -129,7 +125,7 @@ std::vector LoopEmitter::EmitIndexAndSetExitBasicBlock(
 Status LoopEmitter::EmitLoop(tensorflow::StringPiece loop_name,
                              llvm::Type* index_type) {
   if (index_type == nullptr) {
-    index_type = ir_builder_->getInt64Ty();
+    index_type = b_->getInt64Ty();
   }
 
   for (const IrArray::Index& array_index :
@@ -137,10 +133,10 @@ Status LoopEmitter::EmitLoop(tensorflow::StringPiece loop_name,
     TF_RETURN_IF_ERROR(body_emitter_(array_index));
   }
 
-  // Set the insertion point of ir_builder_ to the loop exit, so that
+  // Set the insertion point of b_ to the loop exit, so that
   // code emitted for later instructions will be correctly placed.
   if (exit_bb_ != nullptr) {
-    ir_builder_->SetInsertPoint(exit_bb_);
+    b_->SetInsertPoint(exit_bb_);
   }
   return Status::OK();
 }
diff --git a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h
index 6be1c2fba2..c4f5c82086 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h
@@ -41,11 +41,11 @@ class LoopEmitter {
   using BodyEmitter = std::function;
 
   LoopEmitter(const BodyEmitter& body_emitter, const Shape& shape,
-              llvm::IRBuilder<>* ir_builder);
+              llvm::IRBuilder<>* b);
   // Constructs a LoopEmitter from an element generator that generates each
   // element of the given target array.
   LoopEmitter(const ElementGenerator& target_element_generator,
-              const IrArray& target_array, llvm::IRBuilder<>* ir_builder);
+              const IrArray& target_array, llvm::IRBuilder<>* b);
 
   // Constructs a LoopEmitter that emits one element into each of N separate
   // arrays on each iteration of the loop.
@@ -54,7 +54,7 @@ class LoopEmitter {
   // produce an LLVM struct with N elements.
   LoopEmitter(const ElementGenerator& target_element_generator,
               tensorflow::gtl::ArraySlice target_arrays,
-              llvm::IRBuilder<>* ir_builder);
+              llvm::IRBuilder<>* b);
 
   LoopEmitter(const LoopEmitter&) = delete;
   LoopEmitter& operator=(const LoopEmitter&) = delete;
@@ -65,8 +65,7 @@ class LoopEmitter {
   // specifies the element, will return multiple indices if the loop is
   // unrolled.
   std::vector EmitIndexAndSetExitBasicBlock() {
-    return EmitIndexAndSetExitBasicBlock(/*loop_name=*/"",
-                                         ir_builder_->getInt64Ty());
+    return EmitIndexAndSetExitBasicBlock(/*loop_name=*/"", b_->getInt64Ty());
   }
 
   virtual std::vector EmitIndexAndSetExitBasicBlock(
@@ -87,7 +86,7 @@ class LoopEmitter {
   // scalar, no loops are emitted and exit_bb_ is nullptr in that case.
   llvm::BasicBlock* exit_bb_;
 
-  llvm::IRBuilder<>* ir_builder_;
+  llvm::IRBuilder<>* b_;
 };
 
 }  // namespace llvm_ir
diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
index 16a9a5aaeb..585364458a 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
@@ -38,46 +38,42 @@ namespace {
 void EmitCompareLoop(int64 dimension_to_sort,
                      const llvm_ir::IrArray::Index& keys_index,
                      const llvm_ir::IrArray::Index& compare_keys_index,
-                     const llvm_ir::IrArray& keys_array,
-                     llvm::IRBuilder<>* ir_builder) {
+                     const llvm_ir::IrArray& keys_array, llvm::IRBuilder<>* b) {
   // TODO(b/26783907): parallelize this loop.
 
   // if (is_smaller_index &&
   //     compare_keys[dimension_to_sort] < dimension_to_sort_bound)
-  llvm::Value* is_smaller_index = ir_builder->CreateICmpSLT(
+  llvm::Value* is_smaller_index = b->CreateICmpSLT(
       keys_index[dimension_to_sort], compare_keys_index[dimension_to_sort]);
   int64 dimension_to_sort_bound =
       keys_array.GetShape().dimensions(dimension_to_sort);
   auto if_data = llvm_ir::EmitIfThenElse(
-      ir_builder->CreateAnd(
-          is_smaller_index,
-          ir_builder->CreateICmpSLT(
-              compare_keys_index[dimension_to_sort],
-              keys_index.GetConstantWithIndexType(dimension_to_sort_bound))),
-      "smaller_comparison_index", ir_builder, /*emit_else=*/false);
-  SetToFirstInsertPoint(if_data.true_block, ir_builder);
-  auto key1 = keys_array.EmitReadArrayElement(keys_index, ir_builder);
-  auto key2 = keys_array.EmitReadArrayElement(compare_keys_index, ir_builder);
+      b->CreateAnd(is_smaller_index,
+                   b->CreateICmpSLT(compare_keys_index[dimension_to_sort],
+                                    keys_index.GetConstantWithIndexType(
+                                        dimension_to_sort_bound))),
+      "smaller_comparison_index", b, /*emit_else=*/false);
+  SetToFirstInsertPoint(if_data.true_block, b);
+  auto key1 = keys_array.EmitReadArrayElement(keys_index, b);
+  auto key2 = keys_array.EmitReadArrayElement(compare_keys_index, b);
   auto key_type = keys_array.GetShape().element_type();
   auto comparison =
       primitive_util::IsFloatingPointType(key_type)
           // TODO(b/26783907): Figure out how to handle NaNs.
-          ? ir_builder->CreateFCmp(llvm::FCmpInst::FCMP_ULT, key1, key2)
-          : ir_builder->CreateICmp(
-                primitive_util::IsSignedIntegralType(key_type)
-                    ? llvm::ICmpInst::ICMP_SLT
-                    : llvm::ICmpInst::ICMP_ULT,
-                key1, key2);
-  auto min_key = ir_builder->CreateSelect(comparison, key1, key2);
-  auto max_key = ir_builder->CreateSelect(comparison, key2, key1);
-  keys_array.EmitWriteArrayElement(keys_index, min_key, ir_builder);
-  keys_array.EmitWriteArrayElement(compare_keys_index, max_key, ir_builder);
+          ? b->CreateFCmp(llvm::FCmpInst::FCMP_ULT, key1, key2)
+          : b->CreateICmp(primitive_util::IsSignedIntegralType(key_type)
+                              ? llvm::ICmpInst::ICMP_SLT
+                              : llvm::ICmpInst::ICMP_ULT,
+                          key1, key2);
+  auto min_key = b->CreateSelect(comparison, key1, key2);
+  auto max_key = b->CreateSelect(comparison, key2, key1);
+  keys_array.EmitWriteArrayElement(keys_index, min_key, b);
+  keys_array.EmitWriteArrayElement(compare_keys_index, max_key, b);
 }
 }  // namespace
 
 Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
-                       tensorflow::StringPiece name,
-                       llvm::IRBuilder<>* ir_builder) {
+                       tensorflow::StringPiece name, llvm::IRBuilder<>* b) {
   const Shape& keys_shape = keys_array.GetShape();
 
   // TODO(b/26783907): This case can probably be avoided with the Algebraic
@@ -89,7 +85,7 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
   // Create loop nests which loop through the operand dimensions. The sort
   // dimension is handled in three separate innermost loops which perform the
   // sorting.
-  ForLoopNest loop_nest(name, ir_builder);
+  ForLoopNest loop_nest(name, b);
   IrArray::Index keys_index =
       loop_nest.EmitOperandArrayLoopNest(keys_array, dimension_to_sort, "keys");
 
@@ -149,12 +145,11 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
   // This follows the algorithm described on Wikipedia:
   // https://en.wikipedia.org/wiki/Bitonic_sorter
 
-  SetToFirstInsertPoint(stages_loop->GetBodyBasicBlock(), ir_builder);
+  SetToFirstInsertPoint(stages_loop->GetBodyBasicBlock(), b);
   // The first xor mask of a stage is 2^(stage + 1) - 1.
-  auto first_xor_mask = ir_builder->CreateSub(
-      ir_builder->CreateShl(
-          keys_index.GetConstantWithIndexType(1),
-          ir_builder->CreateAdd(stages_loop->GetIndVarValue(),
+  auto first_xor_mask = b->CreateSub(
+      b->CreateShl(keys_index.GetConstantWithIndexType(1),
+                   b->CreateAdd(stages_loop->GetIndVarValue(),
                                 keys_index.GetConstantWithIndexType(1))),
       keys_index.GetConstantWithIndexType(1));
   std::unique_ptr first_compare_loop = ForLoop::EmitForLoop(
@@ -163,36 +158,35 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
       /*end_index=*/
       keys_index.GetConstantWithIndexType(dimension_to_sort_bound),
       /*step=*/keys_index.GetConstantWithIndexType(1),
-      /*ir_builder=*/ir_builder);
+      /*b=*/b);
 
-  SetToFirstInsertPoint(first_compare_loop->GetBodyBasicBlock(), ir_builder);
+  SetToFirstInsertPoint(first_compare_loop->GetBodyBasicBlock(), b);
   // 'first_compare_loop' iterates through the 'dimension_to_sort'.
   keys_index[dimension_to_sort] = first_compare_loop->GetIndVarValue();
-  compare_keys_index[dimension_to_sort] = ir_builder->CreateXor(
-      first_compare_loop->GetIndVarValue(), first_xor_mask);
+  compare_keys_index[dimension_to_sort] =
+      b->CreateXor(first_compare_loop->GetIndVarValue(), first_xor_mask);
   EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index, keys_array,
-                  ir_builder);
+                  b);
 
-  SetToFirstInsertPoint(compare_loop->GetPreheaderBasicBlock(), ir_builder);
+  SetToFirstInsertPoint(compare_loop->GetPreheaderBasicBlock(), b);
   // The later masks of a stage are 2^(stage - (mask_loop_ind_var + 1)).
-  auto later_xor_mask = ir_builder->CreateShl(
+  auto later_xor_mask = b->CreateShl(
       keys_index.GetConstantWithIndexType(1),
-      ir_builder->CreateSub(
-          stages_loop->GetIndVarValue(),
-          ir_builder->CreateAdd(mask_loop->GetIndVarValue(),
+      b->CreateSub(stages_loop->GetIndVarValue(),
+                   b->CreateAdd(mask_loop->GetIndVarValue(),
                                 keys_index.GetConstantWithIndexType(1))));
 
-  SetToFirstInsertPoint(compare_loop->GetBodyBasicBlock(), ir_builder);
+  SetToFirstInsertPoint(compare_loop->GetBodyBasicBlock(), b);
   // 'compare_loop' iterates through the 'dimension_to_sort'.
   keys_index[dimension_to_sort] = compare_loop->GetIndVarValue();
   compare_keys_index[dimension_to_sort] =
-      ir_builder->CreateXor(compare_loop->GetIndVarValue(), later_xor_mask);
+      b->CreateXor(compare_loop->GetIndVarValue(), later_xor_mask);
   EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index, keys_array,
-                  ir_builder);
+                  b);
 
   // Set the IR builder insert point to the exit basic block of the outer most
   // loop. This ensures later instructions are inserted after this loop nest.
-  ir_builder->SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
+  b->SetInsertPoint(loop_nest.GetOuterLoopExitBasicBlock());
 
   return Status::OK();
 }
diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
index fc45bfab12..d0f185e70b 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
@@ -26,8 +26,7 @@ namespace llvm_ir {
 // Emits llvm IR to sort the 'dimension_to_sort' dimension of 'keys_array' into
 // ascending order.
 Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
-                       tensorflow::StringPiece name,
-                       llvm::IRBuilder<>* ir_builder);
+                       tensorflow::StringPiece name, llvm::IRBuilder<>* b);
 }  // namespace llvm_ir
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc
index 5fc08aab91..11ed6ee59f 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc
@@ -31,12 +31,12 @@ namespace llvm_ir {
 
 void EmitTupleSelect(const IrArray& select, const IrArray& pred,
                      llvm::Value* on_true, llvm::Value* on_false,
-                     llvm::IRBuilder<>* ir_builder, llvm::Module* module) {
+                     llvm::IRBuilder<>* b, llvm::Module* module) {
   CHECK(ShapeUtil::IsScalar(pred.GetShape()));
 
   llvm::LoadInst* pred_value =
-      ir_builder->CreateLoad(pred.GetBasePointer(), "load_predicate_value");
-  llvm::Value* pred_cond = ir_builder->CreateICmpNE(
+      b->CreateLoad(pred.GetBasePointer(), "load_predicate_value");
+  llvm::Value* pred_cond = b->CreateICmpNE(
       pred_value,
       llvm::ConstantInt::get(PrimitiveTypeToIrType(PRED, module), 0),
       "boolean_predicate");
@@ -46,47 +46,42 @@ void EmitTupleSelect(const IrArray& select, const IrArray& pred,
   VLOG(2) << "  pred_cond: " << DumpToString(*pred_cond);
 
   for (int i = 0; i < ShapeUtil::TupleElementCount(select.GetShape()); ++i) {
-    llvm::Value* const element_index[] = {ir_builder->getInt64(0),
-                                          ir_builder->getInt64(i)};
+    llvm::Value* const element_index[] = {b->getInt64(0), b->getInt64(i)};
     llvm::Value* on_true_element_address =
-        ir_builder->CreateInBoundsGEP(on_true, element_index);
-    llvm::Value* on_true_element = ir_builder->CreateLoad(
+        b->CreateInBoundsGEP(on_true, element_index);
+    llvm::Value* on_true_element = b->CreateLoad(
         on_true_element_address, "on_true_element_" + llvm::Twine(i));
     llvm::Value* on_false_element_address =
-        ir_builder->CreateInBoundsGEP(on_false, element_index);
-    llvm::Value* on_false_element = ir_builder->CreateLoad(
+        b->CreateInBoundsGEP(on_false, element_index);
+    llvm::Value* on_false_element = b->CreateLoad(
         on_false_element_address, "on_false_element_" + llvm::Twine(i));
 
     llvm::Value* output_element_address =
-        ir_builder->CreateInBoundsGEP(select.GetBasePointer(), element_index);
-    ir_builder->CreateStore(
-        ir_builder->CreateSelect(pred_cond, on_true_element, on_false_element,
-                                 "select_output_element_" + llvm::Twine(i)),
-        output_element_address);
+        b->CreateInBoundsGEP(select.GetBasePointer(), element_index);
+    b->CreateStore(b->CreateSelect(pred_cond, on_true_element, on_false_element,
+                                   "select_output_element_" + llvm::Twine(i)),
+                   output_element_address);
   }
 }
 
 void EmitTuple(const IrArray& tuple,
                tensorflow::gtl::ArraySlice operands,
-               llvm::IRBuilder<>* ir_builder, llvm::Module* module) {
+               llvm::IRBuilder<>* b, llvm::Module* module) {
   for (size_t i = 0; i < operands.size(); ++i) {
-    auto* store = ir_builder->CreateStore(
-        ir_builder->CreatePointerCast(operands[i],
-                                      PrimitiveTypeToIrType(TUPLE, module)),
-        ir_builder->CreateInBoundsGEP(
-            tuple.GetBasePointer(),
-            {ir_builder->getInt64(0), ir_builder->getInt64(i)}));
+    auto* store = b->CreateStore(
+        b->CreatePointerCast(operands[i], PrimitiveTypeToIrType(TUPLE, module)),
+        b->CreateInBoundsGEP(tuple.GetBasePointer(),
+                             {b->getInt64(0), b->getInt64(i)}));
     tuple.AnnotateLoadStoreInstructionWithMetadata(store);
   }
 }
 
 llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index,
                                  int alignment, llvm::Value* operand,
-                                 llvm::IRBuilder<>* ir_builder,
-                                 llvm::Module* module) {
-  llvm::Value* element_ptr = ir_builder->CreateInBoundsGEP(
-      operand, {ir_builder->getInt64(0), ir_builder->getInt64(index)});
-  llvm::LoadInst* src_buffer = ir_builder->CreateLoad(element_ptr);
+                                 llvm::IRBuilder<>* b, llvm::Module* module) {
+  llvm::Value* element_ptr =
+      b->CreateInBoundsGEP(operand, {b->getInt64(0), b->getInt64(index)});
+  llvm::LoadInst* src_buffer = b->CreateLoad(element_ptr);
 
   // Mark the loaded pointer as dereferenceable if we know its shape.
   if (!ShapeUtil::IsOpaque(target_shape)) {
@@ -98,7 +93,7 @@ llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index,
 
   llvm::Type* element_type = ShapeToIrType(target_shape, module);
   llvm::Value* ret_val =
-      ir_builder->CreateBitCast(src_buffer, element_type->getPointerTo());
+      b->CreateBitCast(src_buffer, element_type->getPointerTo());
   return ret_val;
 }
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h
index 352d34ebf8..cf6bf5d0b1 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h
@@ -61,13 +61,13 @@ namespace llvm_ir {
 //   output[i] = pred ? tuple_on_true[i] : tuple_on_false[i]
 void EmitTupleSelect(const IrArray& select, const IrArray& pred,
                      llvm::Value* on_true, llvm::Value* on_false,
-                     llvm::IRBuilder<>* ir_builder, llvm::Module* module);
+                     llvm::IRBuilder<>* b, llvm::Module* module);
 
 // A tuple is an array of pointers, one for each operand. Each pointer points to
 // the output buffer of its corresponding operand.
 void EmitTuple(const IrArray& tuple,
                tensorflow::gtl::ArraySlice operands,
-               llvm::IRBuilder<>* ir_builder, llvm::Module* module);
+               llvm::IRBuilder<>* b, llvm::Module* module);
 
 // A tuple is an array of pointers, one for each operand. Each pointer points to
 // the output buffer of its corresponding operand. A GetTupleElement instruction
@@ -75,8 +75,7 @@ void EmitTuple(const IrArray& tuple,
 // Returns an llvm value representing a pointer to the tuple element buffer.
 llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index,
                                  int alignment, llvm::Value* operand,
-                                 llvm::IRBuilder<>* ir_builder,
-                                 llvm::Module* module);
+                                 llvm::IRBuilder<>* b, llvm::Module* module);
 }  // namespace llvm_ir
 }  // namespace xla
 
-- 
cgit v1.2.3


From 7e8a83543b7eb36647894453129f15eeec60b3ba Mon Sep 17 00:00:00 2001
From: Justin Lebar 
Date: Fri, 20 Jul 2018 14:45:30 -0700
Subject: [XLA] Make it illegal to call XlaOp::builder() if the op is
 uninitialized.

It's very common to do foo.builder()->bar().  Without this precondition,
if foo.builder() is null, the call to bar will segfault at some point
possibly deep in the callstack when we finally dereference `this`.  The
precondition lets us avoid this tricky-to-debug problem.

PiperOrigin-RevId: 205456769
---
 tensorflow/compiler/xla/client/xla_client/xla_builder.h | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index 3c016ebe8f..8359d936b7 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -54,7 +54,16 @@ class XlaOp {
   }
   ~XlaOp() = default;
 
-  XlaBuilder* builder() const { return builder_; }
+  // Precondition: !IsUninitialized().
+  //
+  // It's very common to do foo.builder()->bar().  Without this precondition, if
+  // foo.builder() is null, the call to bar will segfault at some point possibly
+  // deep in the callstack when we finally dereference `this`.  The precondition
+  // lets us avoid this tricky-to-debug problem.
+  XlaBuilder* builder() const {
+    CHECK(builder_ != nullptr);
+    return builder_;
+  }
 
   // Returns true if the XlaOp represents valid, non-erroneous value.
   bool valid() const { return handle_ >= 0; }
-- 
cgit v1.2.3


From 1711a9a08ce29029c66924f880fa1e619aed10aa Mon Sep 17 00:00:00 2001
From: RJ Ryan 
Date: Fri, 20 Jul 2018 14:47:04 -0700
Subject: Remove float64 math in linear_to_mel_weight_matrix.

This was causing portability problems for platforms that do not support float64. Callers who want higher precision can simply pass tf.float64 as the dtype.

PiperOrigin-RevId: 205457007
---
 .../signal/python/kernel_tests/mel_ops_test.py     | 13 +++++++-----
 tensorflow/contrib/signal/python/ops/mel_ops.py    | 24 +++++++++-------------
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/signal/python/kernel_tests/mel_ops_test.py b/tensorflow/contrib/signal/python/kernel_tests/mel_ops_test.py
index 345eb6cfaa..f4348e80ea 100644
--- a/tensorflow/contrib/signal/python/kernel_tests/mel_ops_test.py
+++ b/tensorflow/contrib/signal/python/kernel_tests/mel_ops_test.py
@@ -53,7 +53,8 @@ def spectrogram_to_mel_matrix(num_mel_bins=20,
                               num_spectrogram_bins=129,
                               audio_sample_rate=8000,
                               lower_edge_hertz=125.0,
-                              upper_edge_hertz=3800.0):
+                              upper_edge_hertz=3800.0,
+                              unused_dtype=None):
   """Return a matrix that can post-multiply spectrogram rows to make mel.
 
   Copied from
@@ -132,9 +133,9 @@ class LinearToMelTest(test.TestCase):
     # lower_edge_hertz, upper_edge_hertz) to test.
     configs = [
         # Defaults.
-        (20, 129, 8000.0, 125.0, 3800.0),
+        (20, 129, 8000.0, 125.0, 3800.0, dtypes.float64),
         # Settings used by Tacotron (https://arxiv.org/abs/1703.10135).
-        (80, 1025, 24000.0, 80.0, 12000.0)
+        (80, 1025, 24000.0, 80.0, 12000.0, dtypes.float64)
     ]
     with self.test_session(use_gpu=True):
       for config in configs:
@@ -143,7 +144,8 @@ class LinearToMelTest(test.TestCase):
         self.assertAllClose(mel_matrix_np, mel_matrix.eval(), atol=3e-6)
 
   def test_dtypes(self):
-    for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
+    # LinSpace is not supported for tf.float16.
+    for dtype in (dtypes.bfloat16, dtypes.float32, dtypes.float64):
       self.assertEqual(dtype,
                        mel_ops.linear_to_mel_weight_matrix(dtype=dtype).dtype)
 
@@ -167,7 +169,8 @@ class LinearToMelTest(test.TestCase):
 
   def test_constant_folding(self):
     """Mel functions should be constant foldable."""
-    for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
+    # TODO(rjryan): tf.bloat16 cannot be constant folded by Grappler.
+    for dtype in (dtypes.float32, dtypes.float64):
       g = ops.Graph()
       with g.as_default():
         mel_matrix = mel_ops.linear_to_mel_weight_matrix(dtype=dtype)
diff --git a/tensorflow/contrib/signal/python/ops/mel_ops.py b/tensorflow/contrib/signal/python/ops/mel_ops.py
index 1e84006116..062d84aea1 100644
--- a/tensorflow/contrib/signal/python/ops/mel_ops.py
+++ b/tensorflow/contrib/signal/python/ops/mel_ops.py
@@ -151,22 +151,21 @@ def linear_to_mel_weight_matrix(num_mel_bins=20,
     _validate_arguments(num_mel_bins, sample_rate,
                         lower_edge_hertz, upper_edge_hertz, dtype)
 
-    # To preserve accuracy, we compute the matrix at float64 precision and then
-    # cast to `dtype` at the end. This function can be constant folded by graph
-    # optimization since there are no Tensor inputs.
+    # This function can be constant folded by graph optimization since there are
+    # no Tensor inputs.
     sample_rate = ops.convert_to_tensor(
-        sample_rate, dtypes.float64, name='sample_rate')
+        sample_rate, dtype, name='sample_rate')
     lower_edge_hertz = ops.convert_to_tensor(
-        lower_edge_hertz, dtypes.float64, name='lower_edge_hertz')
+        lower_edge_hertz, dtype, name='lower_edge_hertz')
     upper_edge_hertz = ops.convert_to_tensor(
-        upper_edge_hertz, dtypes.float64, name='upper_edge_hertz')
-    zero_float64 = ops.convert_to_tensor(0.0, dtypes.float64)
+        upper_edge_hertz, dtype, name='upper_edge_hertz')
+    zero = ops.convert_to_tensor(0.0, dtype)
 
     # HTK excludes the spectrogram DC bin.
     bands_to_zero = 1
     nyquist_hertz = sample_rate / 2.0
     linear_frequencies = math_ops.linspace(
-        zero_float64, nyquist_hertz, num_spectrogram_bins)[bands_to_zero:]
+        zero, nyquist_hertz, num_spectrogram_bins)[bands_to_zero:]
     spectrogram_bins_mel = array_ops.expand_dims(
         _hertz_to_mel(linear_frequencies), 1)
 
@@ -193,11 +192,8 @@ def linear_to_mel_weight_matrix(num_mel_bins=20,
 
     # Intersect the line segments with each other and zero.
     mel_weights_matrix = math_ops.maximum(
-        zero_float64, math_ops.minimum(lower_slopes, upper_slopes))
+        zero, math_ops.minimum(lower_slopes, upper_slopes))
 
     # Re-add the zeroed lower bins we sliced out above.
-    mel_weights_matrix = array_ops.pad(
-        mel_weights_matrix, [[bands_to_zero, 0], [0, 0]])
-
-    # Cast to the desired type.
-    return math_ops.cast(mel_weights_matrix, dtype, name=name)
+    return array_ops.pad(
+        mel_weights_matrix, [[bands_to_zero, 0], [0, 0]], name=name)
-- 
cgit v1.2.3


From 248980e6422f97aa44d6bbac942389f2e9de75ad Mon Sep 17 00:00:00 2001
From: Shashi Shekhar 
Date: Fri, 20 Jul 2018 14:56:08 -0700
Subject: Fix a bug in stats updation and add a test.

PiperOrigin-RevId: 205458337
---
 .../contrib/lite/profiling/profile_summarizer.cc   | 19 +-----
 tensorflow/core/BUILD                              | 10 +++
 tensorflow/core/util/stat_summarizer.cc            | 22 ++-----
 tensorflow/core/util/stats_calculator.cc           | 21 +++++-
 tensorflow/core/util/stats_calculator.h            |  5 +-
 tensorflow/core/util/stats_calculator_test.cc      | 76 ++++++++++++++++++++++
 6 files changed, 117 insertions(+), 36 deletions(-)
 create mode 100644 tensorflow/core/util/stats_calculator_test.cc

diff --git a/tensorflow/contrib/lite/profiling/profile_summarizer.cc b/tensorflow/contrib/lite/profiling/profile_summarizer.cc
index 36e87b666a..720bd717b9 100644
--- a/tensorflow/contrib/lite/profiling/profile_summarizer.cc
+++ b/tensorflow/contrib/lite/profiling/profile_summarizer.cc
@@ -23,8 +23,6 @@ namespace tflite {
 namespace profiling {
 namespace {
 
-using Detail = tensorflow::StatsCalculator::Detail;
-
 struct OperatorDetails {
   std::string name;
   std::vector inputs;
@@ -125,28 +123,17 @@ void ProfileSummarizer::ProcessProfiles(
   int64_t base_start_us = events[0]->begin_timestamp_us;
   int node_num = 0;
   int64_t curr_total_us = 0;
-  std::map details;
   for (auto event : events) {
     auto op_details = GetOperatorDetails(interpreter, event->event_metadata);
     auto node_name = ToString(op_details.outputs);
-    auto result = details.emplace(node_name, Detail());
-    Detail* detail = &(result.first->second);
-    detail->start_us.UpdateStat(event->begin_timestamp_us - base_start_us);
+    int64_t start_us = event->begin_timestamp_us - base_start_us;
     int64_t node_exec_time =
         event->end_timestamp_us - event->begin_timestamp_us;
-    detail->rel_end_us.UpdateStat(node_exec_time);
+    stats_calculator_->AddNodeStats(node_name, op_details.name, node_num,
+                                    start_us, node_exec_time, 0 /*memory */);
     curr_total_us += node_exec_time;
     ++node_num;
-
-    if (result.second) {
-      detail->name = node_name;
-      detail->type = op_details.name;
-      detail->run_order = node_num;
-      detail->times_called = 0;
-    }
-    ++detail->times_called;
   }
-  stats_calculator_->UpdateDetails(details);
   stats_calculator_->UpdateRunTotalUs(curr_total_us);
 }
 }  // namespace profiling
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index fc12027291..870bde7bc8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -883,6 +883,16 @@ cc_library(
     copts = tf_copts(),
 )
 
+tf_cc_test(
+    name = "stats_calculator_test",
+    srcs = ["util/stats_calculator_test.cc"],
+    deps = [
+        ":stats_calculator_portable",
+        ":test",
+        ":test_main",
+    ],
+)
+
 cc_library(
     name = "overflow",
     hdrs = ["util/overflow.h"],
diff --git a/tensorflow/core/util/stat_summarizer.cc b/tensorflow/core/util/stat_summarizer.cc
index a5c1fda102..2117042034 100644
--- a/tensorflow/core/util/stat_summarizer.cc
+++ b/tensorflow/core/util/stat_summarizer.cc
@@ -133,7 +133,6 @@ void StatSummarizer::ProcessStepStats(const StepStats& step_stats) {
 
   int64 first_node_start_us =
       step_stats.dev_stats(0).node_stats(0).all_start_micros();
-  std::map details;
 
   int node_num = 0;
   for (const auto& ds : step_stats.dev_stats()) {
@@ -177,22 +176,15 @@ void StatSummarizer::ProcessStepStats(const StepStats& step_stats) {
       ++node_num;
       const int64 curr_time = ns.all_end_rel_micros();
       curr_total_us += curr_time;
-      auto result = details.emplace(name, Detail());
       auto output_result =
           outputs_.emplace(name, std::vector());
       std::vector* outputs = &(output_result.first->second);
-      Detail* detail = &(result.first->second);
 
-      detail->start_us.UpdateStat(ns.all_start_micros() - first_node_start_us);
-      detail->rel_end_us.UpdateStat(curr_time);
+      int64_t start_us = (ns.all_start_micros() - first_node_start_us);
+      int64_t rel_end_us = curr_time;
 
       // If this is the first pass, initialize some values.
-      if (result.second) {
-        detail->name = name;
-        detail->type = op_type;
-
-        detail->run_order = node_num;
-
+      if (output_result.second) {
         outputs->resize(ns.output_size());
         for (const auto& output : ns.output()) {
           const int32 slot = output.slot();
@@ -202,7 +194,6 @@ void StatSummarizer::ProcessStepStats(const StepStats& step_stats) {
           }
           (*outputs)[slot] = output.tensor_description();
         }
-        detail->times_called = 0;
       }
 
       int64 curr_node_mem = 0;
@@ -210,11 +201,10 @@ void StatSummarizer::ProcessStepStats(const StepStats& step_stats) {
         const int64 mem_usage = mem.total_bytes();
         curr_node_mem += mem_usage;
       }
-      detail->mem_used.UpdateStat(curr_node_mem);
-      mem_total += curr_node_mem;
+      stats_calculator_->AddNodeStats(name, op_type, node_num, start_us,
+                                      rel_end_us, curr_node_mem);
 
-      ++detail->times_called;
-      stats_calculator_->UpdateDetails(details);
+      mem_total += curr_node_mem;
 
       Validate(outputs, ns);
     }
diff --git a/tensorflow/core/util/stats_calculator.cc b/tensorflow/core/util/stats_calculator.cc
index c4befbdb84..eb07754650 100644
--- a/tensorflow/core/util/stats_calculator.cc
+++ b/tensorflow/core/util/stats_calculator.cc
@@ -272,9 +272,24 @@ std::string StatsCalculator::GetOutputString() const {
   return stream.str();
 }
 
-void StatsCalculator::UpdateDetails(
-    const std::map& details) {
-  details_.insert(details.begin(), details.end());
+void StatsCalculator::AddNodeStats(const std::string& name,
+                                   const std::string& type, int64_t run_order,
+                                   int64_t start_us, int64_t rel_end_us,
+                                   int64_t mem_used) {
+  Detail* detail = nullptr;
+  if (details_.find(name) == details_.end()) {
+    details_.insert({name, {}});
+    detail = &details_.at(name);
+    detail->type = type;
+    detail->name = name;
+    detail->run_order = run_order;
+  } else {
+    detail = &details_.at(name);
+  }
+  detail->start_us.UpdateStat(start_us);
+  detail->rel_end_us.UpdateStat(rel_end_us);
+  detail->mem_used.UpdateStat(mem_used);
+  detail->times_called++;
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/util/stats_calculator.h b/tensorflow/core/util/stats_calculator.h
index 39cef816f1..e191737bb2 100644
--- a/tensorflow/core/util/stats_calculator.h
+++ b/tensorflow/core/util/stats_calculator.h
@@ -163,7 +163,10 @@ class StatsCalculator {
   };
 
   const std::map& GetDetails() const { return details_; }
-  void UpdateDetails(const std::map& details);
+
+  void AddNodeStats(const std::string& name, const std::string& type,
+                    int64_t run_order, int64_t start_us, int64_t rel_end_us,
+                    int64_t mem_used);
 
  private:
   void OrderNodesByMetric(SortingMetric sorting_metric,
diff --git a/tensorflow/core/util/stats_calculator_test.cc b/tensorflow/core/util/stats_calculator_test.cc
new file mode 100644
index 0000000000..00d7bfc2f9
--- /dev/null
+++ b/tensorflow/core/util/stats_calculator_test.cc
@@ -0,0 +1,76 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/util/stats_calculator.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace {
+
+using Detail = StatsCalculator::Detail;
+
+TEST(StatsCalculatorTest, TotalTimeMs) {
+  auto options = StatSummarizerOptions();
+  StatsCalculator calc(options);
+
+  EXPECT_EQ(0, calc.num_runs());
+  calc.UpdateRunTotalUs(1);
+
+  EXPECT_EQ(1, calc.num_runs());
+  calc.UpdateRunTotalUs(2);
+
+  EXPECT_EQ(2, calc.num_runs());
+  auto run_time_us = calc.run_total_us();
+  EXPECT_EQ(1, run_time_us.min());
+  EXPECT_FLOAT_EQ(1.5, run_time_us.avg());
+}
+
+TEST(StatsCalculatorTest, AddNodeStatsUpdate) {
+  auto options = StatSummarizerOptions();
+  StatsCalculator calc(options);
+  EXPECT_TRUE(calc.GetDetails().empty());
+
+  const int64_t node1_run_order = 1;
+  const int64_t run1_start_us = 1;
+  const int64_t run1_end_us = 2;
+  const int64_t run1_mem_used = 45;
+  calc.AddNodeStats("node1", "type_1", node1_run_order, run1_start_us,
+                    run1_end_us, run1_mem_used);
+  ASSERT_EQ(1, calc.GetDetails().size());
+  const Detail& detail = calc.GetDetails().at("node1");
+  EXPECT_EQ(1, detail.times_called);
+  EXPECT_EQ("node1", detail.name);
+  EXPECT_EQ("type_1", detail.type);
+  EXPECT_EQ(node1_run_order, detail.run_order);
+
+  const int64_t run2_start_us = 3;
+  const int64_t run2_end_us = 5;
+  const int64_t run2_mem_used = 145;
+  calc.AddNodeStats("node1", "type_1", node1_run_order, run2_start_us,
+                    run2_end_us, run2_mem_used);
+  EXPECT_EQ(1, calc.GetDetails().size());
+
+  EXPECT_EQ(2, detail.times_called);
+  EXPECT_EQ("node1", detail.name);
+  EXPECT_EQ("type_1", detail.type);
+  EXPECT_EQ(node1_run_order, detail.run_order);
+
+  EXPECT_EQ(run1_start_us + run2_start_us, detail.start_us.sum());
+  EXPECT_EQ(run1_end_us + run2_end_us, detail.rel_end_us.sum());
+  EXPECT_EQ(run1_mem_used + run2_mem_used, detail.mem_used.sum());
+}
+
+}  // namespace
+}  // namespace tensorflow
-- 
cgit v1.2.3


From 62a10974897c3cdc929a079f389f6770c767377a Mon Sep 17 00:00:00 2001
From: Justin Lebar 
Date: Fri, 20 Jul 2018 15:15:44 -0700
Subject: [XLA] Make ClientLibraryTestBase::AddParam work with the reference
 backend.

Previously, AddParam only worked with the "real" backend -- we'd never
pass the parameters to the reference backend, so it would always fail.

PiperOrigin-RevId: 205461805
---
 .../compiler/xla/tests/client_library_test_base.cc | 64 +++++++++++++++++-----
 .../compiler/xla/tests/client_library_test_base.h  |  6 +-
 2 files changed, 55 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc
index ef784da457..7a2e70d39f 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.cc
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc
@@ -273,10 +273,16 @@ Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus(
     const Shape* shape_with_layout) {
   std::vector arguments(arguments_passed_in.begin(),
                                      arguments_passed_in.end());
+
+  // Transfer and use elements of arguments_, if the AddParam() API was used.
+  std::vector> owning_arguments;
   if (!arguments_.empty()) {
     CHECK(arguments.empty());
     for (const auto& argument : arguments_) {
-      arguments.push_back(argument.get());
+      owning_arguments.push_back(
+          client_->TransferToServer(MaybeConvertLiteralToBfloat16(argument))
+              .ValueOrDie());
+      arguments.push_back(owning_arguments.back().get());
     }
   }
 
@@ -331,10 +337,16 @@ Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus(
     ErrorSpec error, const Shape* shape_with_layout) {
   std::vector arguments(arguments_passed_in.begin(),
                                      arguments_passed_in.end());
+
+  // Transfer and use elements of arguments_, if the AddParam() API was used.
+  std::vector> owning_arguments;
   if (!arguments_.empty()) {
     CHECK(arguments.empty());
     for (const auto& argument : arguments_) {
-      arguments.push_back(argument.get());
+      owning_arguments.push_back(
+          client_->TransferToServer(MaybeConvertLiteralToBfloat16(argument))
+              .ValueOrDie());
+      arguments.push_back(owning_arguments.back().get());
     }
   }
 
@@ -454,6 +466,14 @@ ClientLibraryTestBase::ComputeValueAndReference(
   // function.
   std::vector> argument_data;
   std::vector> ref_argument_data;
+
+  // Use `arguments_` if the AddParam() API was used.  Otherwise, use
+  // plain `arguments`.
+  if (!arguments_.empty()) {
+    CHECK_EQ(arguments.size(), 0);
+    arguments = arguments_;
+  }
+
   for (const auto& arg : arguments) {
     TF_ASSIGN_OR_RETURN(auto data, client_->TransferToServer(arg.Clone()));
     TF_ASSIGN_OR_RETURN(auto ref_data, ref_client_->TransferToServer(arg));
@@ -552,10 +572,9 @@ ClientLibraryTestBase::CreatePatternedMatrixWithZeroPadding(int rows, int cols,
 
 XlaOp ClientLibraryTestBase::AddParam(const Literal& argument,
                                       XlaBuilder* builder) {
-  XlaOp data_handle;
-  arguments_.push_back(CreateParameterAndTransferLiteral(
-      arguments_.size(), argument, "", builder, &data_handle));
-  return data_handle;
+  arguments_.push_back(argument.Clone());
+  return Parameter(builder, /*parameter_number=*/arguments_.size() - 1,
+                   MaybeConvertShapeToBfloat16(argument.shape()), "");
 }
 
 XlaOp ClientLibraryTestBase::CreateConstantFromLiteral(const Literal& literal,
@@ -575,22 +594,39 @@ ClientLibraryTestBase::CreateParameterAndTransferLiteral(int64 parameter_number,
                                            nullptr, builder, data_handle);
 }
 
+Shape ClientLibraryTestBase::MaybeConvertShapeToBfloat16(const Shape& shape) {
+  if (!use_bfloat16_) {
+    return shape;
+  }
+  Shape new_shape = shape;
+  ShapeUtil::ForEachMutableSubshape(&new_shape,
+                                    [](Shape* subshape, const ShapeIndex&) {
+                                      if (subshape->element_type() == F32) {
+                                        subshape->set_element_type(BF16);
+                                      }
+                                    });
+  return new_shape;
+}
+
+Literal ClientLibraryTestBase::MaybeConvertLiteralToBfloat16(
+    const Literal& literal) {
+  if (use_bfloat16_) {
+    return std::move(*LiteralUtil::ConvertF32ToBF16(literal));
+  }
+  return literal.Clone();
+}
+
 std::unique_ptr
 ClientLibraryTestBase::CreateParameterAndTransferLiteral(
     int64 parameter_number, const Literal& literal, const string& name,
     const DeviceHandle* device_handle, XlaBuilder* builder,
     XlaOp* data_handle) {
-  const Literal* param_literal = &literal;
-  std::unique_ptr converted_literal;
-  if (use_bfloat16_) {
-    converted_literal = LiteralUtil::ConvertF32ToBF16(literal);
-    param_literal = converted_literal.get();
-  }
+  Literal param_literal = MaybeConvertLiteralToBfloat16(literal);
   std::unique_ptr data =
-      client_->TransferToServer(*param_literal, device_handle)
+      client_->TransferToServer(param_literal, device_handle)
           .ConsumeValueOrDie();
   *data_handle =
-      Parameter(builder, parameter_number, param_literal->shape(), name);
+      Parameter(builder, parameter_number, param_literal.shape(), name);
   return data;
 }
 
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index fcc9347db5..f0f7ff1ea0 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -399,12 +399,16 @@ class ClientLibraryTestBase : public ::testing::Test {
                                const string& error_message)>& verify_output,
       const Shape* output_with_layout = nullptr);
 
+  // Converts an f32 shape/literal to bf16 if use_bfloat16_ is true.
+  Literal MaybeConvertLiteralToBfloat16(const Literal& literal);
+  Shape MaybeConvertShapeToBfloat16(const Shape& shape);
+
   // Whether to run tests with all float-type input/output converted to
   // bfloat16.
   bool use_bfloat16_ = false;
 
   // Arguments to be passed to the computation when it runs.
-  std::vector> arguments_;
+  std::vector arguments_;
 };
 
 template 
-- 
cgit v1.2.3


From ee851755a687ec126280bb19e9c9b892b36e58a3 Mon Sep 17 00:00:00 2001
From: Jiri Simsa 
Date: Fri, 20 Jul 2018 15:26:00 -0700
Subject: Adding descriptor source test.

PiperOrigin-RevId: 205463246
---
 tensorflow/contrib/proto/BUILD                     |  12 --
 tensorflow/contrib/proto/python/kernel_tests/BUILD |  27 ++++
 .../python/kernel_tests/descriptor_source_test.py  |  36 +++++
 .../kernel_tests/descriptor_source_test_base.py    | 176 +++++++++++++++++++++
 tensorflow/tools/pip_package/BUILD                 |   2 +-
 5 files changed, 240 insertions(+), 13 deletions(-)
 create mode 100644 tensorflow/contrib/proto/python/kernel_tests/descriptor_source_test.py
 create mode 100644 tensorflow/contrib/proto/python/kernel_tests/descriptor_source_test_base.py

diff --git a/tensorflow/contrib/proto/BUILD b/tensorflow/contrib/proto/BUILD
index d45622174f..b27142cf4a 100644
--- a/tensorflow/contrib/proto/BUILD
+++ b/tensorflow/contrib/proto/BUILD
@@ -16,15 +16,3 @@ py_library(
         "//tensorflow/contrib/proto/python/ops:encode_proto_op_py",
     ],
 )
-
-py_library(
-    name = "proto_pip",
-    data = if_static(
-        [],
-        otherwise = ["//tensorflow/contrib/proto/python/kernel_tests:libtestexample.so"],
-    ),
-    deps = [
-        ":proto",
-        "//tensorflow/contrib/proto/python/kernel_tests:py_test_deps",
-    ],
-)
diff --git a/tensorflow/contrib/proto/python/kernel_tests/BUILD b/tensorflow/contrib/proto/python/kernel_tests/BUILD
index 3c6fde23d2..125c1cee29 100644
--- a/tensorflow/contrib/proto/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/proto/python/kernel_tests/BUILD
@@ -100,3 +100,30 @@ tf_cc_shared_object(
         ":test_example_proto_cc",
     ],
 )
+
+py_library(
+    name = "descriptor_source_test_base",
+    testonly = 1,
+    srcs = ["descriptor_source_test_base.py"],
+    deps = [
+        ":proto_op_test_base",
+        "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
+        "@protobuf_archive//:protobuf_python",
+    ],
+)
+
+tf_py_test(
+    name = "descriptor_source_test",
+    size = "small",
+    srcs = ["descriptor_source_test.py"],
+    additional_deps = [
+        ":descriptor_source_test_base",
+        "//tensorflow/contrib/proto/python/ops:decode_proto_op_py",
+        "//tensorflow/contrib/proto/python/ops:encode_proto_op_py",
+        "//tensorflow/python:client_testlib",
+    ],
+    tags = [
+        "no_pip",
+    ],
+)
diff --git a/tensorflow/contrib/proto/python/kernel_tests/descriptor_source_test.py b/tensorflow/contrib/proto/python/kernel_tests/descriptor_source_test.py
new file mode 100644
index 0000000000..32ca318f73
--- /dev/null
+++ b/tensorflow/contrib/proto/python/kernel_tests/descriptor_source_test.py
@@ -0,0 +1,36 @@
+# =============================================================================
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+"""Tests for proto ops reading descriptors from other sources."""
+# Python3 preparedness imports.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.proto.python.kernel_tests import descriptor_source_test_base as test_base
+from tensorflow.contrib.proto.python.ops import decode_proto_op
+from tensorflow.contrib.proto.python.ops import encode_proto_op
+from tensorflow.python.platform import test
+
+
+class DescriptorSourceTest(test_base.DescriptorSourceTestBase):
+
+  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
+    super(DescriptorSourceTest, self).__init__(decode_proto_op, encode_proto_op,
+                                               methodName)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/proto/python/kernel_tests/descriptor_source_test_base.py b/tensorflow/contrib/proto/python/kernel_tests/descriptor_source_test_base.py
new file mode 100644
index 0000000000..9a1c04af32
--- /dev/null
+++ b/tensorflow/contrib/proto/python/kernel_tests/descriptor_source_test_base.py
@@ -0,0 +1,176 @@
+# =============================================================================
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+"""Tests for proto ops reading descriptors from other sources."""
+# Python3 preparedness imports.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+import numpy as np
+
+from google.protobuf.descriptor_pb2 import FieldDescriptorProto
+from google.protobuf.descriptor_pb2 import FileDescriptorSet
+from tensorflow.contrib.proto.python.kernel_tests import proto_op_test_base as test_base
+from tensorflow.python.framework import dtypes
+from tensorflow.python.platform import test
+
+
+class DescriptorSourceTestBase(test.TestCase):
+  """Base class for testing descriptor sources."""
+
+  def __init__(self, decode_module, encode_module, methodName='runTest'):  # pylint: disable=invalid-name
+    """DescriptorSourceTestBase initializer.
+
+    Args:
+      decode_module: a module containing the `decode_proto_op` method
+      encode_module: a module containing the `encode_proto_op` method
+      methodName: the name of the test method (same as for test.TestCase)
+    """
+
+    super(DescriptorSourceTestBase, self).__init__(methodName)
+    self._decode_module = decode_module
+    self._encode_module = encode_module
+
+  # NOTE: We generate the descriptor programmatically instead of via a compiler
+  # because of differences between different versions of the compiler.
+  #
+  # The generated descriptor should capture the subset of `test_example.proto`
+  # used in `test_base.simple_test_case()`.
+  def _createDescriptorFile(self):
+    set_proto = FileDescriptorSet()
+
+    file_proto = set_proto.file.add(
+        name='types.proto',
+        package='tensorflow',
+        syntax='proto3')
+    enum_proto = file_proto.enum_type.add(name='DataType')
+    enum_proto.value.add(name='DT_DOUBLE', number=0)
+    enum_proto.value.add(name='DT_BOOL', number=1)
+
+    file_proto = set_proto.file.add(
+        name='test_example.proto',
+        package='tensorflow.contrib.proto',
+        dependency=['types.proto'])
+    message_proto = file_proto.message_type.add(name='TestCase')
+    message_proto.field.add(
+        name='values',
+        number=1,
+        type=FieldDescriptorProto.TYPE_MESSAGE,
+        type_name='.tensorflow.contrib.proto.TestValue',
+        label=FieldDescriptorProto.LABEL_REPEATED)
+    message_proto.field.add(
+        name='shapes',
+        number=2,
+        type=FieldDescriptorProto.TYPE_INT32,
+        label=FieldDescriptorProto.LABEL_REPEATED)
+    message_proto.field.add(
+        name='sizes',
+        number=3,
+        type=FieldDescriptorProto.TYPE_INT32,
+        label=FieldDescriptorProto.LABEL_REPEATED)
+    message_proto.field.add(
+        name='fields',
+        number=4,
+        type=FieldDescriptorProto.TYPE_MESSAGE,
+        type_name='.tensorflow.contrib.proto.FieldSpec',
+        label=FieldDescriptorProto.LABEL_REPEATED)
+
+    message_proto = file_proto.message_type.add(
+        name='TestValue')
+    message_proto.field.add(
+        name='double_value',
+        number=1,
+        type=FieldDescriptorProto.TYPE_DOUBLE,
+        label=FieldDescriptorProto.LABEL_REPEATED)
+    message_proto.field.add(
+        name='bool_value',
+        number=2,
+        type=FieldDescriptorProto.TYPE_BOOL,
+        label=FieldDescriptorProto.LABEL_REPEATED)
+
+    message_proto = file_proto.message_type.add(
+        name='FieldSpec')
+    message_proto.field.add(
+        name='name',
+        number=1,
+        type=FieldDescriptorProto.TYPE_STRING,
+        label=FieldDescriptorProto.LABEL_OPTIONAL)
+    message_proto.field.add(
+        name='dtype',
+        number=2,
+        type=FieldDescriptorProto.TYPE_ENUM,
+        type_name='.tensorflow.DataType',
+        label=FieldDescriptorProto.LABEL_OPTIONAL)
+    message_proto.field.add(
+        name='value',
+        number=3,
+        type=FieldDescriptorProto.TYPE_MESSAGE,
+        type_name='.tensorflow.contrib.proto.TestValue',
+        label=FieldDescriptorProto.LABEL_OPTIONAL)
+
+    fn = os.path.join(self.get_temp_dir(), 'descriptor.pb')
+    with open(fn, 'wb') as f:
+      f.write(set_proto.SerializeToString())
+    return fn
+
+  def _testRoundtrip(self, descriptor_source):
+    # Numpy silently truncates the strings if you don't specify dtype=object.
+    in_bufs = np.array(
+        [test_base.ProtoOpTestBase.simple_test_case().SerializeToString()],
+        dtype=object)
+    message_type = 'tensorflow.contrib.proto.TestCase'
+    field_names = ['values', 'shapes', 'sizes', 'fields']
+    tensor_types = [dtypes.string, dtypes.int32, dtypes.int32, dtypes.string]
+
+    with self.test_session() as sess:
+      sizes, field_tensors = self._decode_module.decode_proto(
+          in_bufs,
+          message_type=message_type,
+          field_names=field_names,
+          output_types=tensor_types,
+          descriptor_source=descriptor_source)
+
+      out_tensors = self._encode_module.encode_proto(
+          sizes,
+          field_tensors,
+          message_type=message_type,
+          field_names=field_names,
+          descriptor_source=descriptor_source)
+
+      out_bufs, = sess.run([out_tensors])
+
+      # Check that the re-encoded tensor has the same shape.
+      self.assertEqual(in_bufs.shape, out_bufs.shape)
+
+      # Compare the input and output.
+      for in_buf, out_buf in zip(in_bufs.flat, out_bufs.flat):
+        # Check that the input and output serialized messages are identical.
+        # If we fail here, there is a difference in the serialized
+        # representation but the new serialization still parses. This could
+        # be harmless (a change in map ordering?) or it could be bad (e.g.
+        # loss of packing in the encoding).
+        self.assertEqual(in_buf, out_buf)
+
+  def testWithFileDescriptorSet(self):
+    # First try parsing with a local proto db, which should fail.
+    with self.assertRaisesOpError('No descriptor found for message type'):
+      self._testRoundtrip('local://')
+
+    # Now try parsing with a FileDescriptorSet which contains the test proto.
+    descriptor_file = self._createDescriptorFile()
+    self._testRoundtrip(descriptor_file)
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index e661fb1adc..ab39ed8d69 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -78,7 +78,7 @@ COMMON_PIP_DEPS = [
     "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip",
     "//tensorflow/contrib/nn:nn_py",
     "//tensorflow/contrib/predictor:predictor_pip",
-    "//tensorflow/contrib/proto:proto_pip",
+    "//tensorflow/contrib/proto:proto",
     "//tensorflow/contrib/receptive_field:receptive_field_pip",
     "//tensorflow/contrib/rpc:rpc_pip",
     "//tensorflow/contrib/session_bundle:session_bundle_pip",
-- 
cgit v1.2.3


From 8be889a3034b4dd5ea46330ffad185fc91901723 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling 
Date: Fri, 20 Jul 2018 15:30:19 -0700
Subject: Comment about TfLiteDelegateParams

PiperOrigin-RevId: 205463881
---
 tensorflow/contrib/lite/context.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/contrib/lite/context.h b/tensorflow/contrib/lite/context.h
index 1ff8843fa7..cbfce12d7e 100644
--- a/tensorflow/contrib/lite/context.h
+++ b/tensorflow/contrib/lite/context.h
@@ -464,6 +464,12 @@ typedef struct _TfLiteDelegate {
 } TfLiteDelegate;
 
 // WARNING: This is an experimental interface that is subject to change.
+//
+// Currently, TfLiteDelegateParams has to be allocated in a way that it's
+// trivially destructable. It will be stored as `builtin_data` field in
+// `TfLiteNode` of the delegate node.
+//
+// See also the `CreateDelegateParams` function in `interpreter.cc` details.
 typedef struct {
   TfLiteDelegate* delegate;
   TfLiteIntArray* nodes_to_replace;
-- 
cgit v1.2.3


From f4f37efdc95adc4b2c6235479b89ddfbaf4b3eed Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 15:41:04 -0700
Subject: Update Grappler to use existing functions for retrieving a node's
 name and position.

PiperOrigin-RevId: 205465354
---
 tensorflow/core/grappler/costs/graph_properties.cc |  18 +-
 .../core/grappler/costs/graph_properties_test.cc   |  38 ++++
 .../function_functional_while.pbtxt                | 239 +++++++++++++++++++++
 3 files changed, 285 insertions(+), 10 deletions(-)
 create mode 100644 tensorflow/core/grappler/costs/graph_properties_testdata/function_functional_while.pbtxt

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 83a8326e79..231c7c63be 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -496,18 +496,11 @@ class SymbolicShapeRefiner {
             "supported.");
       }
 
+      // It is guaranteed that output_tensors does not contain any control
+      // inputs, so port_id >= 0.
       string out_tensor = out_arg.output_tensors[0];
-      auto out_tensor_pieces = str_util::Split(out_tensor, ",");
-      string node_name = out_tensor_pieces[0];
       int port_id;
-
-      // Check if port_id was included in out_tensor
-      if (out_tensor_pieces.size() <= 1) {
-        port_id = 0;
-      } else if (!strings::safe_strto32(out_tensor_pieces[1], &port_id)) {
-        return errors::FailedPrecondition(
-            "Failed string to integer conversion for ", out_tensor_pieces[1]);
-      }
+      string node_name = ParseNodeName(out_tensor, &port_id);
 
       const NodeDef* retnode = gv.GetNode(node_name);
       if (retnode == nullptr) {
@@ -516,6 +509,11 @@ class SymbolicShapeRefiner {
       }
 
       auto output_properties = gp.GetOutputProperties(retnode->name());
+      if (port_id >= output_properties.size()) {
+        return errors::InvalidArgument(
+            out_tensor, " has invalid position ", port_id,
+            " (output_properties.size() = ", output_properties.size(), ").");
+      }
       auto const& outprop = output_properties[port_id];
       const TensorShapeProto& shape = outprop.shape();
       ShapeHandle out;
diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc
index 1be19d291a..5acfb56b05 100644
--- a/tensorflow/core/grappler/costs/graph_properties_test.cc
+++ b/tensorflow/core/grappler/costs/graph_properties_test.cc
@@ -887,6 +887,44 @@ TEST_F(GraphPropertiesTest, LargeFunctionStaticShapeInference) {
   EXPECT_EQ(8, in_prop3.shape().dim(3).size());
 }
 
+TEST_F(GraphPropertiesTest, LargeFunctionWithMultipleOutputs) {
+  // Test graph produced in python using:
+  /*
+    @function.Defun(noinline=True)
+    def MyFunc():
+      @function.Defun(*[tf.float32] * 2)
+      def Cond(n, unused_x):
+        return n > 0
+
+      @function.Defun(*[tf.float32] * 2)
+      def Body(n, x):
+        return n - 1, x + n
+
+      i = tf.constant(10)
+      return functional_ops.While([i, 0.], Cond, Body)
+
+    with tf.Graph().as_default():
+      z = MyFunc()
+  */
+  GrapplerItem item;
+  string filename = io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataPath,
+                                 "function_functional_while.pbtxt");
+  TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph));
+  GraphProperties properties(item);
+  TF_CHECK_OK(properties.InferStatically(false));
+
+  const auto out_props = properties.GetOutputProperties("MyFunc_AenMyWWx1Us");
+  EXPECT_EQ(2, out_props.size());
+
+  const OpInfo::TensorProperties& out_prop0 = out_props[0];
+  EXPECT_EQ(DT_INT32, out_prop0.dtype());
+  EXPECT_FALSE(out_prop0.shape().unknown_rank());
+
+  const OpInfo::TensorProperties& out_prop1 = out_props[1];
+  EXPECT_EQ(DT_FLOAT, out_prop1.dtype());
+  EXPECT_FALSE(out_prop1.shape().unknown_rank());
+}
+
 TEST_F(GraphPropertiesTest, FunctionWithErrorStaticShapeInference) {
   GrapplerItem item;
   string filename = io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataPath,
diff --git a/tensorflow/core/grappler/costs/graph_properties_testdata/function_functional_while.pbtxt b/tensorflow/core/grappler/costs/graph_properties_testdata/function_functional_while.pbtxt
new file mode 100644
index 0000000000..c94ee2f227
--- /dev/null
+++ b/tensorflow/core/grappler/costs/graph_properties_testdata/function_functional_while.pbtxt
@@ -0,0 +1,239 @@
+node {
+  name: "MyFunc_AenMyWWx1Us"
+  op: "MyFunc_AenMyWWx1Us"
+}
+library {
+  function {
+    signature {
+      name: "MyFunc_AenMyWWx1Us"
+      output_arg {
+        name: "while"
+        type: DT_INT32
+      }
+      output_arg {
+        name: "while_0"
+        type: DT_FLOAT
+      }
+      is_stateful: true
+    }
+    node_def {
+      name: "Const"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 10
+          }
+        }
+      }
+    }
+    node_def {
+      name: "While/input_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 0.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "While"
+      op: "While"
+      input: "Const:output:0"
+      input: "While/input_1:output:0"
+      attr {
+        key: "T"
+        value {
+          list {
+            type: DT_INT32
+            type: DT_FLOAT
+          }
+        }
+      }
+      attr {
+        key: "body"
+        value {
+          func {
+            name: "Body_8GOMGeZeK5c"
+          }
+        }
+      }
+      attr {
+        key: "cond"
+        value {
+          func {
+            name: "Cond_Xf5ttAHgUCg"
+          }
+        }
+      }
+    }
+    ret {
+      key: "while"
+      value: "While:output:0"
+    }
+    ret {
+      key: "while_0"
+      value: "While:output:1"
+    }
+    attr {
+      key: "_noinline"
+      value {
+        b: true
+      }
+    }
+  }
+  function {
+    signature {
+      name: "Body_8GOMGeZeK5c"
+      input_arg {
+        name: "n"
+        type: DT_FLOAT
+      }
+      input_arg {
+        name: "x"
+        type: DT_FLOAT
+      }
+      output_arg {
+        name: "sub"
+        type: DT_FLOAT
+      }
+      output_arg {
+        name: "add"
+        type: DT_FLOAT
+      }
+    }
+    node_def {
+      name: "sub/y"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 1.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "sub_0"
+      op: "Sub"
+      input: "n"
+      input: "sub/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "add_0"
+      op: "Add"
+      input: "x"
+      input: "n"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    ret {
+      key: "add"
+      value: "add_0:z:0"
+    }
+    ret {
+      key: "sub"
+      value: "sub_0:z:0"
+    }
+  }
+  function {
+    signature {
+      name: "Cond_Xf5ttAHgUCg"
+      input_arg {
+        name: "n"
+        type: DT_FLOAT
+      }
+      input_arg {
+        name: "unused_x"
+        type: DT_FLOAT
+      }
+      output_arg {
+        name: "greater"
+        type: DT_BOOL
+      }
+    }
+    node_def {
+      name: "Greater/y"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 0.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "Greater"
+      op: "Greater"
+      input: "n"
+      input: "Greater/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    ret {
+      key: "greater"
+      value: "Greater:z:0"
+    }
+  }
+}
+versions {
+  producer: 26
+  min_consumer: 12
+}
-- 
cgit v1.2.3


From 5e876a8c25819070d78aa96595943afa207a6671 Mon Sep 17 00:00:00 2001
From: Justin Lebar 
Date: Fri, 20 Jul 2018 15:41:36 -0700
Subject: [XLA:GPU] Limit the number of shmem tiles XLA:GPU will use for 021
 transposes.

There's a limit to how much shared memory we can use.

PiperOrigin-RevId: 205465441
---
 .../xla/service/gpu/ir_emitter_unnested.cc         | 34 ++++++++++++++++++
 tensorflow/compiler/xla/tests/fusion_test.cc       | 40 ++++++++++++++++++++++
 2 files changed, 74 insertions(+)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 7100c9a08a..b3229303df 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -3243,6 +3243,40 @@ bool IrEmitterUnnested::CheckAndEmitHloWithTile021(HloInstruction* hlo) {
     return false;
   }
 
+  // Each of our shared memory tiles has 32*33 elements (so ~4kb, if the
+  // elements are of size 4 bytes), and CUDA has an architectural limit of 48kb
+  // shared memory per SM.  (This is increased to 96kb in Volta, but we don't
+  // use this, in part because it eats into our L1 cache space.)
+  //
+  // For correctness we need to ensure that we don't make more than 48kb worth
+  // of shmem tiles per block.  And for performance, we'd probably like to use
+  // significantly less, so that we can fit more than one block at a time on a
+  // gpu core.
+  //
+  // We say without benchmarks that we want at least 3 threads/block,
+  // corresponding to 3 shmem tiles if the elements are 32 bits wide.  We choose
+  // which params get the shmem transpose treatment arbitrarily; it's not clear
+  // if there's a Right Choice.
+  //
+  // This is only sound if tiled transposes are the only place where we use
+  // shared memory in fusions.  If in the future other fusile ops use shared
+  // memory, we'll have to adjust this heuristic.
+  constexpr int kMinBlocksPerCore = 3;
+  constexpr int64 kShmemPerCore = 48 * 1024;
+  int64 shmem_used = 0;
+  for (int64 i = 0; i < params_012.size(); ++i) {
+    const HloInstruction* operand = hlo->operand(params_012[i]);
+    shmem_used +=
+        32 * 33 *
+        ShapeUtil::ByteSizeOfPrimitiveType(operand->shape().element_type());
+
+    if (kMinBlocksPerCore * shmem_used > kShmemPerCore) {
+      // Erase this element and everything after it from params_012.
+      params_012.resize(i);
+      break;
+    }
+  }
+
   VLOG(3) << "EmitHlo021Tile Emitting hlo tile 0-2-1" << hlo->ToString();
   thunk_sequence_->emplace_back(
       BuildKernelThunk(hlo, /*implements_whole_instruction=*/true));
diff --git a/tensorflow/compiler/xla/tests/fusion_test.cc b/tensorflow/compiler/xla/tests/fusion_test.cc
index dc64477935..607bcdd51e 100644
--- a/tensorflow/compiler/xla/tests/fusion_test.cc
+++ b/tensorflow/compiler/xla/tests/fusion_test.cc
@@ -799,6 +799,46 @@ ENTRY main {
       *result));
 }
 
+class FusionClientLibraryTest : public ClientLibraryTestBase {};
+
+XLA_TEST_F(FusionClientLibraryTest, ManyLayoutTransformations) {
+  // On the GPU backend, it's possible to have too many transposes within one
+  // fusion, causing the kernel to run out shared memory and thus not compile.
+  // We want to check that doesn't happen.
+  //
+  // To do this, we create a computation that computes
+  //
+  //   P0 + P0*P1*P1 + P0*P2*P2 ...
+  //
+  // where even parameters have layout 1 and odd parameters have layout 2.
+  //
+  // Our goal is to tempt the backend into creating one giant multi-output
+  // fusion for the whole computation, including the transposes.  Currently
+  // multi-output fusion only fuses fusions, so each of the terms in the sum
+  // needs to be a fusion itself, thus the contortions above.
+  constexpr int kNumParams = 25;
+  XlaBuilder b("ManyLayoutTransformations");
+
+  // This test produces values that overflow int32, which is UB, so use uint32,
+  // where overflow is OK.
+  Array2D arr(32, 32);
+  arr.FillUnique();
+  std::unique_ptr l1 = LiteralUtil::CreateR2FromArray2D(arr)->Relayout(
+      LayoutUtil::MakeLayout({0, 1}));
+
+  std::unique_ptr l2 = LiteralUtil::CreateR2FromArray2D(arr)->Relayout(
+      LayoutUtil::MakeLayout({1, 0}));
+
+  XlaOp p0 = AddParam(*l1, &b);
+  XlaOp sum = p0;
+  for (int i = 1; i < kNumParams; ++i) {
+    auto pN = AddParam((i % 2 == 0 ? *l1 : *l2), &b);
+    sum = sum + p0 * pN * pN;
+  }
+
+  ComputeAndCompare(&b, {});
+}
+
 void BM_ParallelFusion(int num_iters) {
   // Simple element-wise computation to benchmark parallel task partitioning.
   tensorflow::testing::StopTiming();
-- 
cgit v1.2.3


From 6c528feaf820bdde820833ad24e05167adb5daa7 Mon Sep 17 00:00:00 2001
From: Katherine Wu 
Date: Fri, 20 Jul 2018 15:45:15 -0700
Subject: Automated rollback of commit 8257891f378027a1a7c0403ba6ba0aeb313496a0

PiperOrigin-RevId: 205466000
---
 tensorflow/contrib/estimator/BUILD                 |  41 --
 tensorflow/contrib/estimator/__init__.py           |   5 -
 .../python/estimator/saved_model_estimator.py      | 445 ---------------------
 .../python/estimator/saved_model_estimator_test.py | 369 -----------------
 tensorflow/python/estimator/estimator.py           |  62 +--
 tensorflow/python/framework/importer.py            |   2 +-
 tensorflow/python/framework/meta_graph.py          |  68 +---
 tensorflow/python/saved_model/loader_impl.py       |  13 +-
 tensorflow/python/saved_model/loader_test.py       |  19 +-
 tensorflow/python/training/saver.py                |  28 +-
 10 files changed, 35 insertions(+), 1017 deletions(-)
 delete mode 100644 tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py
 delete mode 100644 tensorflow/contrib/estimator/python/estimator/saved_model_estimator_test.py

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 349f48f7f7..1aa3df8d8d 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -28,7 +28,6 @@ py_library(
         ":multi_head",
         ":replicate_model_fn",
         ":rnn",
-        ":saved_model_estimator",
         "//tensorflow:tensorflow_py_no_contrib",
     ],
 )
@@ -466,43 +465,3 @@ py_test(
         "@absl_py//absl/testing:parameterized",
     ],
 )
-
-py_library(
-    name = "saved_model_estimator",
-    srcs = ["python/estimator/saved_model_estimator.py"],
-    deps = [
-        ":export",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:training",
-        "//tensorflow/python/estimator",
-        "//tensorflow/python/estimator:export",
-        "//tensorflow/python/estimator:model_fn",
-        "//tensorflow/python/saved_model",
-    ],
-)
-
-py_test(
-    name = "saved_model_estimator_test",
-    size = "medium",
-    srcs = ["python/estimator/saved_model_estimator_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":export",
-        ":saved_model_estimator",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:metrics",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:state_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python:variables",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/estimator",
-        "//tensorflow/python/estimator:export_export",
-        "//tensorflow/python/estimator:export_output",
-        "//tensorflow/python/estimator:model_fn",
-    ],
-)
diff --git a/tensorflow/contrib/estimator/__init__.py b/tensorflow/contrib/estimator/__init__.py
index e1453ae1d0..09fcfd66a1 100644
--- a/tensorflow/contrib/estimator/__init__.py
+++ b/tensorflow/contrib/estimator/__init__.py
@@ -33,8 +33,6 @@ from tensorflow.contrib.estimator.python.estimator.logit_fns import *
 from tensorflow.contrib.estimator.python.estimator.multi_head import *
 from tensorflow.contrib.estimator.python.estimator.replicate_model_fn import *
 from tensorflow.contrib.estimator.python.estimator.rnn import *
-from tensorflow.contrib.estimator.python.estimator.saved_model_estimator import *
-from tensorflow.python.estimator.export.export import *
 
 from tensorflow.python.util.all_util import remove_undocumented
 # pylint: enable=unused-import,line-too-long,wildcard-import
@@ -72,9 +70,6 @@ _allowed_symbols = [
     'stop_if_higher_hook',
     'stop_if_no_increase_hook',
     'stop_if_no_decrease_hook',
-    'build_raw_supervised_input_receiver_fn',
-    'build_supervised_input_receiver_fn_from_input_fn',
-    'SavedModelEstimator'
 ]
 
 remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)
diff --git a/tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py b/tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py
deleted file mode 100644
index 22188fe663..0000000000
--- a/tensorflow/contrib/estimator/python/estimator/saved_model_estimator.py
+++ /dev/null
@@ -1,445 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Class that creates an Estimator from a SavedModel."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import six
-
-from tensorflow.python.estimator import estimator as estimator_lib
-from tensorflow.python.estimator import model_fn as model_fn_lib
-from tensorflow.python.estimator.export import export as export_lib
-from tensorflow.python.estimator.export import export_output
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
-from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.saved_model import constants
-from tensorflow.python.saved_model import loader_impl
-from tensorflow.python.saved_model import signature_constants
-from tensorflow.python.training import checkpoint_utils
-from tensorflow.python.training import monitored_session
-from tensorflow.python.training import training_util
-
-
-class SavedModelEstimator(estimator_lib.Estimator):
-  """Create an Estimator from a SavedModel.
-
-  Only SavedModels exported with
-  `tf.contrib.estimator.export_all_saved_models()` or
-  `tf.estimator.Estimator.export_savedmodel()` are supported for this class.
-
-  Example with `tf.estimator.DNNClassifier`:
-
-  **Step 1: Create and train DNNClassifier.**
-  ```python
-  feature1 = tf.feature_column.embedding_column(
-      tf.feature_column.categorical_column_with_vocabulary_list(
-          key='feature1', vocabulary_list=('green', 'yellow')), dimension=1)
-  feature2 = tf.feature_column.numeric_column(key='feature2', default_value=0.0)
-
-  classifier = tf.estimator.DNNClassifier(
-      hidden_units=[4,2], feature_columns=[feature1, feature2])
-
-  def input_fn():
-    features = {'feature1': tf.constant(['green', 'green', 'yellow']),
-                'feature2': tf.constant([3.5, 4.2, 6.1])}
-    label = tf.constant([1., 0., 0.])
-    return tf.data.Dataset.from_tensors((features, label)).repeat()
-
-  classifier.train(input_fn=input_fn, steps=10)
-  ```
-
-  **Step 2: Export classifier.**
-  First, build functions that specify the expected inputs.
-  ```python
-  # During train and evaluation, both the features and labels should be defined.
-  supervised_input_receiver_fn = (
-      tf.contrib.estimator.build_raw_supervised_input_receiver_fn(
-        {'feature1': tf.placeholder(dtype=tf.string, shape=[None]),
-         'feature2': tf.placeholder(dtype=tf.float32, shape=[None])},
-        tf.placeholder(dtype=tf.float32, shape=[None])))
-
-  # During predict mode, expect to receive a `tf.Example` proto, so a parsing
-  # function is used.
-  serving_input_receiver_fn = (
-      tf.estimator.export.build_parsing_serving_input_receiver_fn(
-          tf.feature_column.make_parse_example_spec([feature1, feature2])))
-  ```
-
-  Next, export the model as a SavedModel. A timestamped directory will be
-  created (for example `/tmp/export_all/1234567890`).
-  ```python
-  # Option 1: Save all modes (train, eval, predict)
-  export_dir = tf.contrib.estimator.export_all_saved_models(
-      classifier, '/tmp/export_all',
-      {tf.estimator.ModeKeys.TRAIN: supervised_input_receiver_fn,
-       tf.estimator.ModeKeys.EVAL: supervised_input_receiver_fn,
-       tf.estimator.ModeKeys.PREDICT: serving_input_receiver_fn})
-
-  # Option 2: Only export predict mode
-  export_dir = classifier.export_savedmodel(
-    '/tmp/export_predict', serving_input_receiver_fn)
-  ```
-
-  **Step 3: Create a SavedModelEstimator from the exported SavedModel.**
-  ```python
-  est = tf.contrib.estimator.SavedModelEstimator(export_dir)
-
-  # If all modes were exported, you can immediately evaluate and predict, or
-  # continue training. Otherwise only predict is available.
-  eval_results = est.evaluate(input_fn=input_fn, steps=1)
-  print(eval_results)
-
-  est.train(input_fn=input_fn, steps=20)
-
-  def predict_input_fn():
-    example = example_pb2.Example()
-    example.features.feature['feature1'].bytes_list.value.extend(['yellow'])
-    example.features.feature['feature2'].float_list.value.extend([1.])
-    return {'inputs':tf.constant([example.SerializeToString()])}
-
-  predictions = est.predict(predict_input_fn)
-  print(next(predictions))
-  ```
-  """
-
-  def __init__(self, saved_model_dir, model_dir=None):
-    """Initialize a SavedModelEstimator.
-
-    The SavedModelEstimator loads its model function and variable values from
-    the graphs defined in the SavedModel. There is no option to pass in
-    `RunConfig` or `params` arguments, because the model function graph is
-    defined statically in the SavedModel.
-
-    Args:
-      saved_model_dir: Directory containing SavedModel protobuf and subfolders.
-      model_dir: Directory to save new checkpoints during training.
-
-    Raises:
-      NotImplementedError: If a DistributionStrategy is defined in the config.
-        Unless the SavedModelEstimator is subclassed, this shouldn't happen.
-    """
-    checkpoint = estimator_lib._get_saved_model_ckpt(saved_model_dir)  # pylint: disable=protected-access
-    vars_to_warm_start = [name for name, _ in
-                          checkpoint_utils.list_variables(checkpoint)]
-    warm_start_settings = estimator_lib.WarmStartSettings(
-        ckpt_to_initialize_from=checkpoint,
-        vars_to_warm_start=vars_to_warm_start)
-
-    super(SavedModelEstimator, self).__init__(
-        model_fn=self._model_fn_from_saved_model, model_dir=model_dir,
-        warm_start_from=warm_start_settings)
-    if self._distribution is not None:
-      raise NotImplementedError(
-          'SavedModelEstimator currently does not support '
-          'DistributionStrategy.')
-    self.saved_model_dir = saved_model_dir
-    self.saved_model_loader = loader_impl.SavedModelLoader(saved_model_dir)
-    self._available_modes = self._extract_available_modes()
-
-  def _extract_available_modes(self):
-    """Return list of modes found in SavedModel."""
-    available_modes = []
-    logging.info('Checking available modes for SavedModelEstimator.')
-    for mode in [model_fn_lib.ModeKeys.TRAIN, model_fn_lib.ModeKeys.EVAL,
-                 model_fn_lib.ModeKeys.PREDICT]:
-      try:
-        self._get_meta_graph_def_for_mode(mode)
-      except RuntimeError:
-        logging.warning('%s mode not found in SavedModel.' % mode)
-        continue
-
-      if self._get_signature_def_for_mode(mode) is not None:
-        available_modes.append(mode)
-
-    logging.info('Available modes for Estimator: %s' % available_modes)
-    return available_modes
-
-  def _validate_mode(self, mode):
-    """Make sure that mode can be run using the SavedModel."""
-    if mode not in self._available_modes:
-      raise RuntimeError('%s mode is not available in the SavedModel. Use '
-                         'saved_model_cli to check that the Metagraph for this '
-                         'mode has been exported.' % mode)
-
-  def _get_meta_graph_def_for_mode(self, mode):
-    tags = model_fn_lib.EXPORT_TAG_MAP[mode]
-    return self.saved_model_loader.get_meta_graph_def_from_tags(tags)
-
-  def _get_signature_def_for_mode(self, mode):
-    meta_graph_def = self._get_meta_graph_def_for_mode(mode)
-    sig_def_key = (signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
-                   if mode == model_fn_lib.ModeKeys.PREDICT else mode)
-    if sig_def_key not in meta_graph_def.signature_def:
-      logging.warning('Metagraph for mode %s was found, but SignatureDef with'
-                      ' key \"%s\" is missing.' % (mode, sig_def_key))
-      return None
-    return meta_graph_def.signature_def[sig_def_key]
-
-  def _create_and_assert_global_step(self, graph):
-    # Do nothing here. The global step variable will be created/loaded from the
-    # SavedModel. If a global step variable were created here, the result
-    # will be two duplicate global step variables, causing issues during
-    # the warm-start phase.
-    # Due to the global variable being created in the model function, this may
-    # cause issues when running DistributionStrategy. Thus, DistributionStrategy
-    # is not yet supported with SavedModelEstimator.
-    pass
-
-  def _model_fn_from_saved_model(self, features, labels, mode):
-    """Load a SavedModel graph and return an EstimatorSpec."""
-    # TODO(kathywu): Model function loads placeholders from the graph. Calling
-    # export_all_saved_models creates another placeholder for the inputs, on top
-    # of the original placeholders. There should be a way to avoid this.
-    self._validate_mode(mode)
-
-    g = ops.get_default_graph()
-    if  training_util.get_global_step(g) is not None:
-      raise RuntimeError(
-          'Graph must not contain a global step tensor before the SavedModel is'
-          ' loaded. Please make sure that the input function does not create a '
-          'global step.')
-
-    # Extract SignatureDef for information about the input and output tensors.
-    signature_def = self._get_signature_def_for_mode(mode)
-
-    # Generate input map for replacing the inputs in the SavedModel graph with
-    # the provided features and labels.
-    input_map = _generate_input_map(signature_def, features, labels)
-
-    # Create a list of the names of output tensors. When the graph is loaded,
-    # names of the output tensors may be remapped. This ensures that the correct
-    # tensors are returned in the EstimatorSpec.
-    output_tensor_names = [
-        value.name for value in six.itervalues(signature_def.outputs)]
-
-    # Load the graph. `output_tensors` contains output `Tensors` in the same
-    # same order as the `output_tensor_names` list.
-    tags = model_fn_lib.EXPORT_TAG_MAP[mode]
-    _, output_tensors = self.saved_model_loader.load_graph(
-        g, tags, input_map=input_map, return_elements=output_tensor_names)
-
-    # Create a scaffold from the MetaGraphDef that contains ops to initialize
-    # the graph. This should mirror the steps from _add_meta_graph_for_mode(),
-    # which creates a MetaGraphDef from the EstimatorSpec's scaffold.
-    scaffold = monitored_session.Scaffold(
-        local_init_op=loader_impl._get_legacy_init_op_tensor(  # pylint: disable=protected-access
-            self._get_meta_graph_def_for_mode(mode)))
-
-    # Ensure that a global step tensor has been created.
-    global_step_tensor = training_util.get_global_step(g)
-    training_util.assert_global_step(global_step_tensor)
-
-    # Extract values to return in the EstimatorSpec.
-    output_map = dict(zip(output_tensor_names, output_tensors))
-    outputs = {key: output_map[value.name]
-               for key, value in six.iteritems(signature_def.outputs)}
-
-    loss, predictions, metrics = _validate_and_extract_outputs(
-        mode, outputs, signature_def.method_name)
-
-    train_op = ops.get_collection(constants.TRAIN_OP_KEY)
-    if len(train_op) > 1:
-      raise RuntimeError('Multiple ops found in the train_op collection.')
-    train_op = None if not train_op else train_op[0]
-
-    _clear_saved_model_collections()
-    return model_fn_lib.EstimatorSpec(
-        scaffold=scaffold,
-        mode=mode,
-        loss=loss,
-        train_op=train_op,
-        predictions=predictions,
-        eval_metric_ops=metrics)
-
-
-def _clear_saved_model_collections():
-  """Clear collections that are expected empty when exporting a SavedModel.
-
-  The SavedModel builder uses these collections to track ops necessary to
-  restore the graph state. These collections are expected to be empty before
-  MetaGraphs are added to the builder.
-  """
-  del ops.get_collection_ref(constants.ASSETS_KEY)[:]
-  del ops.get_collection_ref(constants.LEGACY_INIT_OP_KEY)[:]
-  del ops.get_collection_ref(constants.MAIN_OP_KEY)[:]
-  del ops.get_collection_ref(constants.TRAIN_OP_KEY)[:]
-
-
-def _generate_input_map(signature_def, features, labels):
-  """Return dict mapping an input tensor name to a feature or label tensor.
-
-  Args:
-    signature_def: SignatureDef loaded from SavedModel
-    features: A `Tensor`, `SparseTensor`, or dict of string to `Tensor` or
-      `SparseTensor`, specifying the features to be passed to the model.
-    labels: A `Tensor`, `SparseTensor`, or dict of string to `Tensor` or
-      `SparseTensor`, specifying the labels to be passed to the model. May be
-      `None`.
-
-  Returns:
-    dict mapping string names of inputs to features or labels tensors
-
-  Raises:
-    ValueError: if SignatureDef inputs are not completely mapped by the input
-      features and labels.
-  """
-  # pylint: disable=protected-access
-  if not isinstance(features, dict):
-    features = {export_lib._SINGLE_FEATURE_DEFAULT_NAME: features}
-  if labels is not None and not isinstance(labels, dict):
-    labels = {export_lib._SINGLE_LABEL_DEFAULT_NAME: labels}
-  # pylint: enable=protected-access
-
-  inputs = signature_def.inputs
-  input_map = {}
-  for key, tensor_info in six.iteritems(inputs):
-    input_name = tensor_info.name
-    if ':' in input_name:
-      input_name = input_name[:input_name.find(':')]
-
-    # When tensors are used as control inputs for operations, their names are
-    # prepended with a '^' character in the GraphDef. To handle possible control
-    # flow edge cases, control input names must be included in the input map.
-    control_dependency_name = '^' + input_name
-
-    if key in features:
-      _check_same_dtype_and_shape(features[key], tensor_info, key)
-      input_map[input_name] = input_map[control_dependency_name] = features[key]
-    elif labels is not None and key in labels:
-      _check_same_dtype_and_shape(labels[key], tensor_info, key)
-      input_map[input_name] = input_map[control_dependency_name] = labels[key]
-    else:
-      raise ValueError(
-          'Key \"%s\" not found in features or labels passed in to the model '
-          'function. All required keys: %s' % (key, inputs.keys()))
-
-  return input_map
-
-
-def _check_same_dtype_and_shape(tensor, tensor_info, name):
-  """Validate that tensor has the same properties as the TensorInfo proto.
-
-  Args:
-    tensor: a `Tensor` object.
-    tensor_info: a `TensorInfo` proto.
-    name: Name of the input (to identify Tensor if an error is raised).
-
-  Raises:
-    ValueError: If the tensor shape or dtype don't match the TensorInfo
-  """
-  dtype_error = (tensor.dtype != dtypes.DType(tensor_info.dtype))
-  shape_error = not tensor.shape.is_compatible_with(tensor_info.tensor_shape)
-
-  if dtype_error or shape_error:
-    msg = 'Tensor shape and/or dtype validation failed for input %s:' % name
-    if dtype_error:
-      msg += ('\n\tExpected dtype: %s, Got: %s'
-              % (dtypes.DType(tensor_info.dtype), tensor.dtype))
-    if shape_error:
-      msg += ('\n\tExpected shape: %s, Got: %s'
-              % (tensor_shape.TensorShape(tensor_info.tensor_shape),
-                 tensor.shape))
-
-    raise ValueError(msg)
-
-
-def _extract_eval_metrics(output_dict):
-  """Return a eval metric dict extracted from the output_dict.
-
-  Eval metrics consist of a value tensor and an update op. Both must be in the
-  passed-in tensor dictionary for an eval metric to be added to the returned
-  dictionary.
-
-  Args:
-    output_dict: a dict that maps strings to tensors.
-
-  Returns:
-    dict mapping strings to (value, update_op) tuples.
-  """
-  # pylint: disable=protected-access
-  metric_ops = {}
-  separator_char = export_output._SupervisedOutput._SEPARATOR_CHAR
-
-  for key, tensor in six.iteritems(output_dict):
-    split_key = key.split(separator_char)
-
-    # The metric name may contain the separator character, so recreate its name.
-    metric_name = separator_char.join(split_key[:-1])
-
-    if split_key[0] == export_output._SupervisedOutput.METRICS_NAME:
-      # If the key ends with the value suffix, and there is a corresponding
-      # key ending with the update_op suffix, then add tensors to metrics dict.
-      if split_key[-1] == export_output._SupervisedOutput.METRIC_VALUE_SUFFIX:
-        update_op = ''.join(
-            [metric_name, separator_char,
-             export_output._SupervisedOutput.METRIC_UPDATE_SUFFIX])
-        if update_op in output_dict:
-          update_op_tensor = output_dict[update_op]
-          metric_ops[metric_name] = (tensor, update_op_tensor)
-
-  # pylint: enable=protected-access
-  return metric_ops
-
-
-def _validate_and_extract_outputs(mode, output_dict, method_name):
-  """Extract values from SignatureDef output dictionary.
-
-  Args:
-    mode: One of the modes enumerated in `tf.estimator.ModeKeys`.
-    output_dict: dict of string SignatureDef keys to `Tensor`.
-    method_name: Method name of the SignatureDef as a string.
-
-  Returns:
-    Tuple of (
-      loss: `Tensor` object,
-      predictions: dictionary mapping string keys to `Tensor` objects,
-      metrics: dictionary mapping string keys to a tuple of two `Tensor` objects
-    )
-
-  Raises:
-    RuntimeError: raised if SignatureDef has an invalid method name for the mode
-  """
-  # pylint: disable=protected-access
-  loss, predictions, metrics = None, None, None
-
-  if mode == model_fn_lib.ModeKeys.PREDICT:
-    predictions = output_dict
-  else:
-    # Validate that the SignatureDef's method name matches the expected name for
-    # the given mode.
-    expected_method_name = signature_constants.SUPERVISED_TRAIN_METHOD_NAME
-    if mode == model_fn_lib.ModeKeys.EVAL:
-      expected_method_name = signature_constants.SUPERVISED_EVAL_METHOD_NAME
-    if method_name != expected_method_name:
-      raise RuntimeError(
-          'Invalid SignatureDef method name for mode %s.\n\tExpected: %s\n\t'
-          'Got: %s\nPlease ensure that the SavedModel was exported with '
-          '`tf.contrib.estimator.export_all_saved_models()`.' %
-          (mode, expected_method_name, method_name))
-
-    # Extract loss, metrics and predictions from the output dict.
-    loss = output_dict[export_output._SupervisedOutput.LOSS_NAME]
-    metrics = _extract_eval_metrics(output_dict)
-    predictions = {
-        key: value for key, value in six.iteritems(output_dict)
-        if key.split(export_output._SupervisedOutput._SEPARATOR_CHAR)[0] == (
-            export_output._SupervisedOutput.PREDICTIONS_NAME)}
-
-  # pylint: enable=protected-access
-  return loss, predictions, metrics
diff --git a/tensorflow/contrib/estimator/python/estimator/saved_model_estimator_test.py b/tensorflow/contrib/estimator/python/estimator/saved_model_estimator_test.py
deleted file mode 100644
index 718da1367c..0000000000
--- a/tensorflow/contrib/estimator/python/estimator/saved_model_estimator_test.py
+++ /dev/null
@@ -1,369 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for SavedModelEstimator."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import shutil
-import tempfile
-
-from tensorflow.contrib.estimator.python.estimator import export as contrib_export
-from tensorflow.contrib.estimator.python.estimator import saved_model_estimator
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.estimator import estimator
-from tensorflow.python.estimator import model_fn as model_fn_lib
-from tensorflow.python.estimator.export import export
-from tensorflow.python.estimator.export import export_output
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import metrics as metrics_lib
-from tensorflow.python.ops import state_ops
-from tensorflow.python.ops import variables
-from tensorflow.python.platform import test
-from tensorflow.python.training import monitored_session
-from tensorflow.python.training import training
-
-
-def dummy_input_fn():
-  return dataset_ops.Dataset.from_tensors((
-      {'x': constant_op.constant([[1], [-2]], dtype=dtypes.int64)},
-      constant_op.constant([[4], [-3]], dtype=dtypes.float32))).repeat()
-
-
-def dummy_input_fn_features_only():
-  return dataset_ops.Dataset.from_tensors(
-      {'x': constant_op.constant([[5], [6]], dtype=dtypes.int64)}).repeat()
-
-
-def dummy_supervised_receiver_fn():
-  feature_spec = {
-      'x': array_ops.placeholder(
-          dtype=dtypes.int64, shape=(2, 1), name='feature_x'),
-      }
-  label_spec = array_ops.placeholder(
-      dtype=dtypes.float32, shape=[2, 1], name='truth')
-  return export.build_raw_supervised_input_receiver_fn(
-      feature_spec, label_spec)
-
-
-def dummy_serving_receiver_fn():
-  feature_spec = {'x': array_ops.placeholder(
-      dtype=dtypes.int64, shape=(2, 1), name='feature_x'),}
-  return export.build_raw_serving_input_receiver_fn(feature_spec)
-
-
-def model_fn_diff_modes(features, labels, mode):
-  _, _ = features, labels
-  v = variables.Variable(21, name='some_var')
-  train_op = None
-  loss = constant_op.constant(104)
-  if mode == model_fn_lib.ModeKeys.TRAIN:
-    loss = constant_op.constant(105)
-    predictions = constant_op.constant([501])
-    train_op = control_flow_ops.group(
-        state_ops.assign_add(training.get_global_step(), 1),
-        state_ops.assign_add(v, 3))
-  elif mode == model_fn_lib.ModeKeys.EVAL:
-    loss = constant_op.constant(106)
-    predictions = constant_op.constant([502])
-  else:
-    loss = constant_op.constant(107)
-    predictions = constant_op.constant([503])
-  return model_fn_lib.EstimatorSpec(
-      mode,
-      loss=loss,
-      train_op=train_op,
-      eval_metric_ops={
-          'abs_err': metrics_lib.mean_absolute_error(
-              constant_op.constant(0), predictions)},
-      predictions=predictions)
-
-
-class SavedModelEstimatorTest(test.TestCase):
-
-  def setUp(self):
-    self.tmpdirs = []
-
-  def tearDown(self):
-    for tmpdir in self.tmpdirs:
-      # gfile.DeleteRecursively fails in the windows cmake test, so use shutil.
-      shutil.rmtree(tmpdir, ignore_errors=True)
-    self.tmpdirs = []
-
-  def _get_tmp_dir(self):
-    tmpdir = tempfile.mkdtemp()
-    self.tmpdirs.append(tmpdir)
-    return tmpdir
-
-  def _export_estimator(self, train=True, evaluate=True, predict=True,
-                        model_fn=model_fn_diff_modes):
-    est = estimator.Estimator(model_fn, self._get_tmp_dir())
-    est.train(input_fn=dummy_input_fn, steps=10)
-
-    input_receiver_fn_map = {}
-    if train:
-      input_receiver_fn_map[model_fn_lib.ModeKeys.TRAIN] = (
-          dummy_supervised_receiver_fn())
-    if evaluate:
-      input_receiver_fn_map[model_fn_lib.ModeKeys.EVAL] = (
-          dummy_supervised_receiver_fn())
-    if predict:
-      input_receiver_fn_map[model_fn_lib.ModeKeys.PREDICT] = (
-          dummy_serving_receiver_fn())
-
-    export_base_path = self._get_tmp_dir()
-    export_dir = contrib_export.export_all_saved_models(
-        est, export_base_path, input_receiver_fn_map)
-    return export_dir
-
-  def test_load_all_modes(self):
-    sme = saved_model_estimator.SavedModelEstimator(
-        self._export_estimator(), self._get_tmp_dir())
-    sme.train(input_fn=dummy_input_fn, steps=1)
-    sme.train(input_fn=dummy_input_fn, steps=2)
-    self.assertEqual(13, sme.get_variable_value('global_step'))
-    self.assertEqual(60, sme.get_variable_value('some_var'))
-
-    eval_results = sme.evaluate(dummy_input_fn, steps=5)
-
-    self.assertEqual(13, eval_results['global_step'])
-    self.assertEqual(106, eval_results['loss'])
-    self.assertEqual(502, eval_results['metrics/abs_err'])
-
-    predictions = next(sme.predict(dummy_input_fn_features_only))
-    self.assertDictEqual({'output': 503}, predictions)
-
-  def test_load_all_modes_no_train(self):
-    """Ensure that all functions can be used without requiring a ckpt."""
-    sme = saved_model_estimator.SavedModelEstimator(
-        self._export_estimator(), self._get_tmp_dir())
-    eval_results = sme.evaluate(dummy_input_fn, steps=5)
-    self.assertEqual(10, eval_results['global_step'])
-    self.assertEqual(106, eval_results['loss'])
-    self.assertEqual(502, eval_results['metrics/abs_err'])
-
-    predictions = next(sme.predict(dummy_input_fn_features_only))
-    self.assertDictEqual({'output': 503}, predictions)
-
-  def test_partial_exported_estimator(self):
-    sme1 = saved_model_estimator.SavedModelEstimator(
-        self._export_estimator(train=False, predict=False), self._get_tmp_dir())
-    sme1.evaluate(dummy_input_fn, steps=5)
-    with self.assertRaisesRegexp(RuntimeError, 'train mode is not available'):
-      sme1.train(input_fn=dummy_input_fn, steps=1)
-    with self.assertRaisesRegexp(RuntimeError, 'infer mode is not available'):
-      next(sme1.predict(dummy_input_fn_features_only))
-
-    sme2 = saved_model_estimator.SavedModelEstimator(
-        self._export_estimator(evaluate=False), self._get_tmp_dir())
-    sme2.train(input_fn=dummy_input_fn, steps=1)
-    next(sme2.predict(dummy_input_fn_features_only))
-    with self.assertRaisesRegexp(RuntimeError, 'eval mode is not available'):
-      sme2.evaluate(dummy_input_fn, steps=5)
-
-  def test_with_incorrect_input(self):
-    sme = saved_model_estimator.SavedModelEstimator(
-        self._export_estimator(), self._get_tmp_dir())
-
-    def bad_shape_input_fn():
-      return dataset_ops.Dataset.from_tensors((
-          {'x': constant_op.constant([1, 2], dtype=dtypes.int64)},
-          constant_op.constant([1, 2], dtype=dtypes.float32)))
-
-    with self.assertRaisesRegexp(ValueError, 'Expected shape'):
-      sme.train(bad_shape_input_fn, steps=1)
-
-    def bad_dtype_input_fn():
-      return dataset_ops.Dataset.from_tensors((
-          {'x': constant_op.constant([[1], [1]], dtype=dtypes.int32)},
-          constant_op.constant([[1], [1]], dtype=dtypes.int64)))
-
-    with self.assertRaisesRegexp(ValueError, 'Expected dtype'):
-      sme.train(bad_dtype_input_fn, steps=1)
-
-  def test_input_fn_with_global_step(self):
-    sme = saved_model_estimator.SavedModelEstimator(
-        self._export_estimator(), self._get_tmp_dir())
-
-    def bad_input_fn():
-      training.get_or_create_global_step()
-      return dataset_ops.Dataset.from_tensors((
-          {'x': constant_op.constant([[1], [1]], dtype=dtypes.int64)},
-          constant_op.constant([[1], [1]], dtype=dtypes.float32)))
-
-    with self.assertRaisesRegexp(RuntimeError,
-                                 'Graph must not contain a global step tensor'):
-      sme.train(bad_input_fn, steps=1)
-
-  def test_re_export_saved_model_serving_only(self):
-    sme = saved_model_estimator.SavedModelEstimator(
-        self._export_estimator(), self._get_tmp_dir())
-    sme.train(dummy_input_fn, steps=3)
-    self.assertEqual(13, sme.get_variable_value('global_step'))
-    self.assertEqual(60, sme.get_variable_value('some_var'))
-
-    predictions = next(sme.predict(dummy_input_fn_features_only))
-    self.assertDictEqual({'output': 503}, predictions)
-
-    # Export SavedModel, and test that the variable and prediction values are
-    # the same.
-    sme_export_dir = sme.export_savedmodel(
-        self._get_tmp_dir(), dummy_serving_receiver_fn())
-
-    sme2 = saved_model_estimator.SavedModelEstimator(
-        sme_export_dir, self._get_tmp_dir())
-    self.assertEqual(60, sme.get_variable_value('some_var'))
-    self.assertEqual(13, sme.get_variable_value('global_step'))
-
-    predictions = next(sme2.predict(dummy_input_fn_features_only))
-    self.assertDictEqual({'output': 503}, predictions)
-
-  def test_re_export_saved_model(self):
-    sme = saved_model_estimator.SavedModelEstimator(
-        self._export_estimator(), self._get_tmp_dir())
-    self.assertDictEqual(
-        {'loss': 106, 'metrics/abs_err': 502, 'global_step': 10},
-        sme.evaluate(dummy_input_fn, steps=1))
-
-    sme.train(dummy_input_fn, steps=3)
-    self.assertDictEqual(
-        {'loss': 106, 'metrics/abs_err': 502, 'global_step': 13},
-        sme.evaluate(dummy_input_fn, steps=1))
-    self.assertEqual(60, sme.get_variable_value('some_var'))
-
-    predictions = next(sme.predict(dummy_input_fn_features_only))
-    self.assertDictEqual({'output': 503}, predictions)
-
-    # Export SavedModel for all modes
-    input_receiver_fn_map = {
-        model_fn_lib.ModeKeys.TRAIN: dummy_supervised_receiver_fn(),
-        model_fn_lib.ModeKeys.EVAL: dummy_supervised_receiver_fn(),
-        model_fn_lib.ModeKeys.PREDICT: dummy_serving_receiver_fn()}
-    sme_export_dir = contrib_export.export_all_saved_models(
-        sme, self._get_tmp_dir(), input_receiver_fn_map)
-
-    sme2 = saved_model_estimator.SavedModelEstimator(
-        sme_export_dir, self._get_tmp_dir())
-    self.assertDictEqual(
-        {'loss': 106, 'metrics/abs_err': 502, 'global_step': 13},
-        sme.evaluate(dummy_input_fn, steps=1))
-    self.assertEqual(60, sme.get_variable_value('some_var'))
-
-    sme.train(dummy_input_fn, steps=7)
-    self.assertEqual(20, sme.get_variable_value('global_step'))
-
-    predictions = next(sme2.predict(dummy_input_fn_features_only))
-    self.assertDictEqual({'output': 503}, predictions)
-
-  def test_load_saved_model_from_serving_only(self):
-    def model_fn(features, labels, mode):
-      _, _ = features, labels
-      return model_fn_lib.EstimatorSpec(
-          mode,
-          loss=constant_op.constant([103]),
-          train_op=state_ops.assign_add(training.get_global_step(), 1),
-          predictions=constant_op.constant([502]),
-          export_outputs={'test': export_output.ClassificationOutput(
-              constant_op.constant([[32.]]))})
-
-    est = estimator.Estimator(model_fn, self._get_tmp_dir())
-    est.train(input_fn=dummy_input_fn, steps=10)
-
-    def serving_input_receiver_fn():
-      return export.ServingInputReceiver(
-          {'test-features': constant_op.constant([[1], [1]])},
-          array_ops.placeholder(dtype=dtypes.string))
-
-    export_dir = est.export_savedmodel(
-        self._get_tmp_dir(), serving_input_receiver_fn)
-
-    sme = saved_model_estimator.SavedModelEstimator(
-        export_dir, self._get_tmp_dir())
-
-    def input_fn():
-      return {'inputs': constant_op.constant('someinputstr')}
-
-    prediction = next(sme.predict(input_fn))
-    self.assertDictEqual({'scores': 32}, prediction)
-
-  def test_with_local_init_op(self):
-    def model_fn(features, labels, mode):
-      _, _ = features, labels
-      v = variables.Variable(21, name='some_var')
-      scaffold = monitored_session.Scaffold(
-          local_init_op=state_ops.assign_add(v, -3).op
-      )
-      return model_fn_lib.EstimatorSpec(
-          mode,
-          scaffold=scaffold,
-          train_op=state_ops.assign_add(training.get_global_step(), 1),
-          loss=array_ops.identity(v))
-    export_dir = self._export_estimator(predict=False, model_fn=model_fn)
-    sme = saved_model_estimator.SavedModelEstimator(
-        export_dir, self._get_tmp_dir())
-
-    eval_results1 = sme.evaluate(dummy_input_fn, steps=2)
-    self.assertEqual(15, eval_results1['loss'])
-
-    sme.train(dummy_input_fn, steps=1)
-    self.assertEqual(15, sme.get_variable_value('some_var'))
-
-    eval_results2 = sme.evaluate(dummy_input_fn, steps=5)
-    self.assertEqual(12, eval_results2['loss'])
-
-  def test_with_working_input_fn(self):
-    def model_fn(features, labels, mode):
-      loss = None
-      if labels is not None:
-        loss = labels[0][0] + labels[1][0]
-      return model_fn_lib.EstimatorSpec(
-          mode,
-          loss=loss,
-          train_op=state_ops.assign_add(training.get_global_step(), 1),
-          predictions={'features_0': array_ops.identity([features['x'][0][0]]),
-                       'features_1': array_ops.identity([features['x'][1][0]])})
-
-    sme = saved_model_estimator.SavedModelEstimator(
-        self._export_estimator(model_fn=model_fn), self._get_tmp_dir())
-    eval_results = sme.evaluate(dummy_input_fn, steps=1)
-    self.assertEqual(1, eval_results['loss'])
-
-    predictions = next(sme.predict(dummy_input_fn_features_only))
-    self.assertDictEqual({'features_0': 5, 'features_1': 6}, predictions)
-
-  def test_control_dependency(self):
-    # Control dependencies are saved with "^" appended to the start of the input
-    # name. The input map must include control dependencies as well.
-    def model_fn(features, labels, mode):
-      _ = labels
-      with ops.control_dependencies([features['x']]):
-        loss = features['x'][1][0]
-      return model_fn_lib.EstimatorSpec(
-          mode,
-          loss=loss,
-          train_op=state_ops.assign_add(training.get_global_step(), 1))
-    sme = saved_model_estimator.SavedModelEstimator(
-        self._export_estimator(train=False, predict=False, model_fn=model_fn),
-        self._get_tmp_dir())
-    sme.evaluate(dummy_input_fn, steps=1)  # Should run without error
-
-
-if __name__ == '__main__':
-  test.main()
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 148fcf61fa..2fd6f6fab9 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -568,14 +568,13 @@ class Estimator(object):
   def _assert_members_are_not_overridden(self):
     """Asserts members of `Estimator` are not overridden."""
     allowed_overrides = set([
-        '_call_input_fn', '_call_model_fn',
+        '_call_input_fn', '_create_global_step',
         '_convert_train_steps_to_hooks', '_convert_eval_steps_to_hooks',
-        '_create_global_step', '_create_and_assert_global_step',
         '_tf_api_names', '_tf_api_names_v1', '_estimator_api_names',
         '_estimator_api_names_v1', '_estimator_api_constants',
         '_estimator_api_constants_v1',
         '_validate_features_in_predict_input',
-        '_add_meta_graph_for_mode'
+        '_call_model_fn', '_add_meta_graph_for_mode'
     ])
     estimator_members = set([m for m in Estimator.__dict__.keys()
                              if not m.startswith('__')])
@@ -902,10 +901,9 @@ class Estimator(object):
 
       with tf_session.Session(config=self._session_config) as session:
 
-        if estimator_spec.scaffold.local_init_op is not None:
-          local_init_op = estimator_spec.scaffold.local_init_op
-        else:
-          local_init_op = monitored_session.Scaffold.default_local_init_op()
+        local_init_op = (
+            estimator_spec.scaffold.local_init_op or
+            monitored_session.Scaffold.default_local_init_op())
 
         # This saver will be used both for restoring variables now,
         # and in saving out the metagraph below. This ensures that any
@@ -1156,15 +1154,14 @@ class Estimator(object):
     worker_hooks = []
     with ops.Graph().as_default() as g, g.device(self._device_fn):
       random_seed.set_random_seed(self._config.tf_random_seed)
-      self._create_and_assert_global_step(g)
+      global_step_tensor = self._create_and_assert_global_step(g)
+      training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
       features, labels, input_hooks = (
           self._get_features_and_labels_from_input_fn(
               input_fn, model_fn_lib.ModeKeys.TRAIN))
       worker_hooks.extend(input_hooks)
       estimator_spec = self._call_model_fn(
           features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
-      global_step_tensor = training_util.get_global_step(g)
-      training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
       return self._train_with_estimator_spec(estimator_spec, worker_hooks,
                                              hooks, global_step_tensor,
                                              saving_listeners)
@@ -1367,8 +1364,10 @@ class Estimator(object):
   def _train_with_estimator_spec(self, estimator_spec, worker_hooks, hooks,
                                  global_step_tensor, saving_listeners):
     """Train a model with the given Estimator Spec."""
-    self._maybe_warm_start(self.latest_checkpoint())
-
+    if self._warm_start_settings:
+      logging.info('Warm-starting with WarmStartSettings: %s' %
+                   (self._warm_start_settings,))
+      warm_starting_util.warm_start(*self._warm_start_settings)
     # Check if the user created a loss summary, and add one if they didn't.
     # We assume here that the summary is called 'loss'. If it is not, we will
     # make another one with the name 'loss' to ensure it shows up in the right
@@ -1449,13 +1448,13 @@ class Estimator(object):
   def _evaluate_build_graph(self, input_fn, hooks=None, checkpoint_path=None):
     """Builds the graph and related hooks to run evaluation."""
     random_seed.set_random_seed(self._config.tf_random_seed)
-    self._create_and_assert_global_step(ops.get_default_graph())
+    global_step_tensor = self._create_and_assert_global_step(
+        ops.get_default_graph())
     features, labels, input_hooks = (
         self._get_features_and_labels_from_input_fn(input_fn,
                                                     model_fn_lib.ModeKeys.EVAL))
     estimator_spec = self._call_model_fn(
         features, labels, model_fn_lib.ModeKeys.EVAL, self.config)
-    global_step_tensor = training_util.get_global_step(ops.get_default_graph())
 
     # Call to warm_start has to be after model_fn is called.
     self._maybe_warm_start(checkpoint_path)
@@ -1481,21 +1480,7 @@ class Estimator(object):
     all_hooks.extend(hooks)
     all_hooks.extend(list(estimator_spec.evaluation_hooks or []))
 
-    # New local variables have been added, so update the estimator spec's
-    # local init op if it was defined.
-    scaffold = estimator_spec.scaffold
-    if estimator_spec.scaffold and estimator_spec.scaffold.local_init_op:
-      # Ensure that eval step has been created before updating local init op.
-      evaluation._get_or_create_eval_step()  # pylint: disable=protected-access
-
-      scaffold = monitored_session.Scaffold(
-          local_init_op=control_flow_ops.group(
-              estimator_spec.scaffold.local_init_op,
-              monitored_session.Scaffold.default_local_init_op()),
-          copy_from_scaffold=scaffold
-      )
-
-    return scaffold, update_op, eval_dict, all_hooks
+    return estimator_spec.scaffold, update_op, eval_dict, all_hooks
 
   def _evaluate_run(self, checkpoint_path, scaffold, update_op, eval_dict,
                     all_hooks, output_dir):
@@ -1926,19 +1911,6 @@ class WarmStartSettings(
     )
 
 
-def _get_saved_model_ckpt(saved_model_dir):
-  """Return path to variables checkpoint in a SavedModel directory."""
-  if not gfile.Exists(
-      os.path.join(compat.as_bytes(saved_model_dir),
-                   compat.as_bytes('variables/variables.index'))):
-    raise ValueError('Directory provided has an invalid SavedModel format: %s'
-                     % saved_model_dir)
-  return os.path.join(
-      compat.as_bytes(saved_model_dir),
-      compat.as_bytes('{}/{}'.format(constants.VARIABLES_DIRECTORY,
-                                     constants.VARIABLES_FILENAME)))
-
-
 def _get_default_warm_start_settings(warm_start_from):
   """Returns default WarmStartSettings.
 
@@ -1962,8 +1934,10 @@ def _get_default_warm_start_settings(warm_start_from):
     if gfile.Exists(os.path.join(compat.as_bytes(warm_start_from),
                                  compat.as_bytes('variables/variables.index'))):
       logging.info('Warm-starting from a SavedModel')
-      return WarmStartSettings(
-          ckpt_to_initialize_from=_get_saved_model_ckpt(warm_start_from))
+      return WarmStartSettings(ckpt_to_initialize_from=os.path.join(
+          compat.as_bytes(warm_start_from),
+          compat.as_bytes('{}/{}'.format(constants.VARIABLES_DIRECTORY,
+                                         constants.VARIABLES_FILENAME))))
     return WarmStartSettings(ckpt_to_initialize_from=warm_start_from)
   elif isinstance(warm_start_from, WarmStartSettings):
     return warm_start_from
diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py
index 687bfebd43..699d2b70d1 100644
--- a/tensorflow/python/framework/importer.py
+++ b/tensorflow/python/framework/importer.py
@@ -205,7 +205,7 @@ def _PopulateTFImportGraphDefOptions(options, prefix, input_map,
   for input_src, input_dst in input_map.items():
     input_src = compat.as_str(input_src)
     if input_src.startswith('^'):
-      src_name = compat.as_str(input_src[1:])
+      src_name = compat.as_bytes(input_src[1:])
       dst_op = input_dst._as_tf_output().oper  # pylint: disable=protected-access
       c_api.TF_ImportGraphDefOptionsRemapControlDependency(
           options, src_name, dst_op)
diff --git a/tensorflow/python/framework/meta_graph.py b/tensorflow/python/framework/meta_graph.py
index 33631282bd..923e76fc9c 100644
--- a/tensorflow/python/framework/meta_graph.py
+++ b/tensorflow/python/framework/meta_graph.py
@@ -696,67 +696,6 @@ def import_scoped_meta_graph(meta_graph_or_file,
   Raises:
     ValueError: If the graph_def contains unbound inputs.
   """
-  return import_scoped_meta_graph_with_return_elements(
-      meta_graph_or_file, clear_devices, graph, import_scope, input_map,
-      unbound_inputs_col_name, restore_collections_predicate)[0]
-
-
-def import_scoped_meta_graph_with_return_elements(
-    meta_graph_or_file,
-    clear_devices=False,
-    graph=None,
-    import_scope=None,
-    input_map=None,
-    unbound_inputs_col_name="unbound_inputs",
-    restore_collections_predicate=(lambda key: True),
-    return_elements=None):
-  """Imports graph from `MetaGraphDef` and returns vars and return elements.
-
-  This function takes a `MetaGraphDef` protocol buffer as input. If
-  the argument is a file containing a `MetaGraphDef` protocol buffer ,
-  it constructs a protocol buffer from the file content. The function
-  then adds all the nodes from the `graph_def` field to the
-  current graph, recreates the desired collections, and returns a dictionary of
-  all the Variables imported into the name scope.
-
-  In combination with `export_scoped_meta_graph()`, this function can be used to
-
-  * Serialize a graph along with other Python objects such as `QueueRunner`,
-    `Variable` into a `MetaGraphDef`.
-
-  * Restart training from a saved graph and checkpoints.
-
-  * Run inference from a saved graph and checkpoints.
-
-  Args:
-    meta_graph_or_file: `MetaGraphDef` protocol buffer or filename (including
-      the path) containing a `MetaGraphDef`.
-    clear_devices: Boolean which controls whether to clear device information
-      from graph_def. Default false.
-    graph: The `Graph` to import into. If `None`, use the default graph.
-    import_scope: Optional `string`. Name scope into which to import the
-      subgraph. If `None`, the graph is imported to the root name scope.
-    input_map: A dictionary mapping input names (as strings) in `graph_def` to
-      `Tensor` objects. The values of the named input tensors in the imported
-      graph will be re-mapped to the respective `Tensor` values.
-    unbound_inputs_col_name: Collection name for looking up unbound inputs.
-    restore_collections_predicate: a predicate on collection names. A collection
-      named c (i.e whose key is c) will be restored iff
-      1) `restore_collections_predicate(c)` is True, and
-      2) `c != unbound_inputs_col_name`.
-    return_elements:  A list of strings containing operation names in the
-      `MetaGraphDef` that will be returned as `Operation` objects; and/or
-      tensor names in `MetaGraphDef` that will be returned as `Tensor` objects.
-
-  Returns:
-    A tuple of (
-      dictionary of all the `Variables` imported into the name scope,
-      list of `Operation` or `Tensor` objects from the `return_elements` list).
-
-  Raises:
-    ValueError: If the graph_def contains unbound inputs.
-
-  """
   if context.executing_eagerly():
     raise ValueError("Exporting/importing meta graphs is not supported when "
                      "eager execution is enabled.")
@@ -798,12 +737,11 @@ def import_scoped_meta_graph_with_return_elements(
     scope_to_prepend_to_names = graph.unique_name(
         import_scope or "", mark_as_used=False)
 
-    imported_return_elements = importer.import_graph_def(
+    importer.import_graph_def(
         input_graph_def,
         name=(import_scope or scope_to_prepend_to_names),
         input_map=input_map,
-        producer_op_list=producer_op_list,
-        return_elements=return_elements)
+        producer_op_list=producer_op_list)
 
     # Restores all the other collections.
     variable_objects = {}
@@ -868,7 +806,7 @@ def import_scoped_meta_graph_with_return_elements(
     for v in variables:
       var_list[ops.strip_name_scope(v.name, scope_to_prepend_to_names)] = v
 
-  return var_list, imported_return_elements
+  return var_list
 
 
 def export_scoped_meta_graph(filename=None,
diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py
index 685a913f9c..e5f649fdab 100644
--- a/tensorflow/python/saved_model/loader_impl.py
+++ b/tensorflow/python/saved_model/loader_impl.py
@@ -284,15 +284,12 @@ class SavedModelLoader(object):
       **saver_kwargs: keyword arguments to pass to tf.train.import_meta_graph.
 
     Returns:
-      A tuple of
-        * Saver defined by the MetaGraph, which can be used to restore the
-          variable values.
-        * List of `Operation`/`Tensor` objects returned from
-          `tf.import_graph_def` (may be `None`).
+      Saver defined by the MetaGraph, which can be used to restore the variable
+      values.
     """
     meta_graph_def = self.get_meta_graph_def_from_tags(tags)
     with graph.as_default():
-      return tf_saver._import_meta_graph_with_return_elements(  # pylint: disable=protected-access
+      return tf_saver.import_meta_graph(
           meta_graph_def, import_scope=import_scope, **saver_kwargs)
 
   def restore_variables(self, sess, saver, import_scope=None):
@@ -364,8 +361,8 @@ class SavedModelLoader(object):
       `MetagraphDef` proto of the graph that was loaded.
     """
     with sess.graph.as_default():
-      saver, _ = self.load_graph(sess.graph, tags, import_scope,
-                                 **saver_kwargs)
+      saver = self.load_graph(sess.graph, tags, import_scope,
+                              **saver_kwargs)
       self.restore_variables(sess, saver, import_scope)
       self.run_init_ops(sess, tags, import_scope)
     return self.get_meta_graph_def_from_tags(tags)
diff --git a/tensorflow/python/saved_model/loader_test.py b/tensorflow/python/saved_model/loader_test.py
index 9a0b276a4b..ce18859f6b 100644
--- a/tensorflow/python/saved_model/loader_test.py
+++ b/tensorflow/python/saved_model/loader_test.py
@@ -111,8 +111,7 @@ class SavedModelLoaderTest(test.TestCase):
   def test_load_with_import_scope(self):
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
     with self.test_session(graph=ops.Graph()) as sess:
-      saver, _ = loader.load_graph(
-          sess.graph, ["foo_graph"], import_scope="baz")
+      saver = loader.load_graph(sess.graph, ["foo_graph"], import_scope="baz")
 
       # The default saver should not work when the import scope is set.
       with self.assertRaises(errors.NotFoundError):
@@ -150,7 +149,7 @@ class SavedModelLoaderTest(test.TestCase):
   def test_run_init_op(self):
     loader = loader_impl.SavedModelLoader(SAVED_MODEL_WITH_MAIN_OP)
     graph = ops.Graph()
-    saver, _ = loader.load_graph(graph, ["foo_graph"])
+    saver = loader.load_graph(graph, ["foo_graph"])
     with self.test_session(graph=graph) as sess:
       loader.restore_variables(sess, saver)
       self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
@@ -204,7 +203,7 @@ class SavedModelLoaderTest(test.TestCase):
 
     loader = loader_impl.SavedModelLoader(path)
     with self.test_session(graph=ops.Graph()) as sess:
-      saver, _ = loader.load_graph(sess.graph, ["foo_graph"])
+      saver = loader.load_graph(sess.graph, ["foo_graph"])
       self.assertFalse(variables._all_saveable_objects())
       self.assertIsNotNone(saver)
 
@@ -213,18 +212,6 @@ class SavedModelLoaderTest(test.TestCase):
       self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
       self.assertEqual(11, sess.graph.get_tensor_by_name("y:0").eval())
 
-  def test_load_saved_model_graph_with_return_elements(self):
-    """Ensure that the correct elements are returned."""
-    loader = loader_impl.SavedModelLoader(SIMPLE_ADD_SAVED_MODEL)
-    graph = ops.Graph()
-    _, ret = loader.load_graph(graph, ["foo_graph"],
-                               return_elements=["y:0", "x:0"])
-
-    self.assertEqual(graph.get_tensor_by_name("y:0"), ret[0])
-    self.assertEqual(graph.get_tensor_by_name("x:0"), ret[1])
-
-    with self.assertRaisesRegexp(ValueError, "not found in graph"):
-      loader.load_graph(graph, ["foo_graph"], return_elements=["z:0"])
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 3a06a52812..11510d9928 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1928,14 +1928,6 @@ def import_meta_graph(meta_graph_or_file, clear_devices=False,
   execution is enabled.
   @end_compatibility
   """  # pylint: disable=g-doc-exception
-  return _import_meta_graph_with_return_elements(
-      meta_graph_or_file, clear_devices, import_scope, **kwargs)[0]
-
-
-def _import_meta_graph_with_return_elements(
-    meta_graph_or_file, clear_devices=False, import_scope=None,
-    return_elements=None, **kwargs):
-  """Import MetaGraph, and return both a saver and returned elements."""
   if context.executing_eagerly():
     raise RuntimeError("Exporting/importing meta graphs is not supported when "
                        "eager execution is enabled. No graph exists when eager "
@@ -1945,22 +1937,12 @@ def _import_meta_graph_with_return_elements(
   else:
     meta_graph_def = meta_graph_or_file
 
-  imported_vars, imported_return_elements = (
-      meta_graph.import_scoped_meta_graph_with_return_elements(
-          meta_graph_def,
-          clear_devices=clear_devices,
-          import_scope=import_scope,
-          return_elements=return_elements,
-          **kwargs))
-
-  saver = _create_saver_from_imported_meta_graph(
-      meta_graph_def, import_scope, imported_vars)
-  return saver, imported_return_elements
-
+  imported_vars = meta_graph.import_scoped_meta_graph(
+      meta_graph_def,
+      clear_devices=clear_devices,
+      import_scope=import_scope,
+      **kwargs)
 
-def _create_saver_from_imported_meta_graph(
-    meta_graph_def, import_scope, imported_vars):
-  """Return a saver for restoring variable values to an imported MetaGraph."""
   if meta_graph_def.HasField("saver_def"):
     # Infer the scope that is prepended by `import_scoped_meta_graph`.
     scope = import_scope
-- 
cgit v1.2.3


From 41781bad97698c29cd74203cef465d2adb2f04e8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 16:23:13 -0700
Subject: Add support for computing Softmax activation over tensors of rank 1.

PiperOrigin-RevId: 205470922
---
 tensorflow/contrib/lite/kernels/activations.cc     | 53 +++++++++++++++++-----
 .../contrib/lite/kernels/activations_test.cc       | 23 ++++++++++
 2 files changed, 65 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/activations.cc b/tensorflow/contrib/lite/kernels/activations.cc
index 99f81c4a8a..d5ac2a7814 100644
--- a/tensorflow/contrib/lite/kernels/activations.cc
+++ b/tensorflow/contrib/lite/kernels/activations.cc
@@ -186,8 +186,8 @@ TfLiteStatus SoftmaxPrepare(TfLiteContext* context, TfLiteNode* node) {
   TfLiteTensor* output = GetOutput(context, node, 0);
   TF_LITE_ENSURE_EQ(context, input->type, output->type);
 
-  TF_LITE_ENSURE(context,
-                 NumDimensions(input) == 2 || NumDimensions(input) == 4);
+  const int num_dims = NumDimensions(input);
+  TF_LITE_ENSURE(context, num_dims == 1 || num_dims == 2 || num_dims == 4);
 
   if (input->type == kTfLiteUInt8) {
     TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0);
@@ -365,13 +365,9 @@ TfLiteStatus SigmoidEval(TfLiteContext* context, TfLiteNode* node) {
   return kTfLiteOk;
 }
 
-// Takes a 2D tensor and perform softmax along the second dimension.
-void Softmax2DFloat(const TfLiteTensor* input, TfLiteTensor* output,
-                    TfLiteSoftmaxParams* params) {
-  const int batch_size = input->dims->data[0];
-  const int input_size = input->dims->data[1];
-  float* in = input->data.f;
-  float* out = output->data.f;
+// Performs softmax along the input of size (input_size * batch_size).
+void Softmax(const float* in, const int input_size, const int batch_size,
+             const float beta, float* out) {
   TF_LITE_ASSERT(input_size > 0);
 
   // For each batch
@@ -385,7 +381,7 @@ void Softmax2DFloat(const TfLiteTensor* input, TfLiteTensor* output,
     // Compute the normalized sum of exps.
     float exp_sum = 0.0;
     for (int i = 0; i < input_size; i++) {
-      out[i] = std::exp((in[i] - max_coeff) * params->beta);
+      out[i] = std::exp((in[i] - max_coeff) * beta);
       exp_sum += out[i];
     }
 
@@ -401,6 +397,33 @@ void Softmax2DFloat(const TfLiteTensor* input, TfLiteTensor* output,
   }
 }
 
+// Takes a 1D tensor and performs softmax along it.
+void Softmax1DFloat(const TfLiteTensor* input, TfLiteTensor* output,
+                    TfLiteSoftmaxParams* params) {
+  const int input_size = input->dims->data[0];
+  Softmax(input->data.f, input_size, 1, params->beta, output->data.f);
+}
+
+// Takes a 2D tensor and perform softmax along the last dimension.
+void Softmax2DFloat(const TfLiteTensor* input, TfLiteTensor* output,
+                    TfLiteSoftmaxParams* params) {
+  const int batch_size = input->dims->data[0];
+  const int input_size = input->dims->data[1];
+  Softmax(input->data.f, input_size, batch_size, params->beta, output->data.f);
+}
+
+void Softmax1DQuantized(const TfLiteTensor* input, TfLiteTensor* output,
+                        TfLiteSoftmaxParams* params, OpData* data) {
+  // TODO(ahentz): this is arguably a dirty trick. Since the implementation
+  // always traverses the last dimension of a 4D tensor, we will pretend our 1D
+  // tensor is 4D in a special way. We will convert a (Y) shape into a (1,
+  // 1, 1, Y) shape.
+  const int input_size = input->dims->data[0];
+  optimized_ops::Softmax(
+      GetTensorData(input), GetTensorShape({1, 1, 1, input_size}),
+      data->input_multiplier, data->input_left_shift, data->diff_min,
+      GetTensorData(output), GetTensorShape({1, 1, 1, input_size}));
+}
 void Softmax2DQuantized(const TfLiteTensor* input, TfLiteTensor* output,
                         TfLiteSoftmaxParams* params, OpData* data) {
   // TODO(ahentz): this is arguably a dirty trick. Since the implementation
@@ -443,6 +466,10 @@ TfLiteStatus SoftmaxEval(TfLiteContext* context, TfLiteNode* node) {
   // dimensions.
   switch (input->type) {
     case kTfLiteFloat32: {
+      if (NumDimensions(input) == 1) {
+        Softmax1DFloat(input, output, params);
+        return kTfLiteOk;
+      }
       if (NumDimensions(input) == 2) {
         Softmax2DFloat(input, output, params);
         return kTfLiteOk;
@@ -452,11 +479,15 @@ TfLiteStatus SoftmaxEval(TfLiteContext* context, TfLiteNode* node) {
         return kTfLiteOk;
       }
       context->ReportError(
-          context, "Only 2D and 4D tensors supported currently, got %dD.",
+          context, "Only 1D, 2D and 4D tensors supported currently, got %dD.",
           NumDimensions(input));
       return kTfLiteError;
     }
     case kTfLiteUInt8: {
+      if (NumDimensions(input) == 1) {
+        Softmax1DQuantized(input, output, params, data);
+        return kTfLiteOk;
+      }
       if (NumDimensions(input) == 2) {
         Softmax2DQuantized(input, output, params, data);
         return kTfLiteOk;
diff --git a/tensorflow/contrib/lite/kernels/activations_test.cc b/tensorflow/contrib/lite/kernels/activations_test.cc
index 587e1303da..083cdf78d7 100644
--- a/tensorflow/contrib/lite/kernels/activations_test.cc
+++ b/tensorflow/contrib/lite/kernels/activations_test.cc
@@ -339,6 +339,29 @@ TEST(QuantizedActivationsOpTest, Softmax4D) {
                   kQuantizedTolerance)));
 }
 
+TEST(FloatActivationsOpTest, Softmax1D) {
+  FloatActivationsOpModel m(0.1,
+                            /*input=*/{TensorType_FLOAT32, {8}});
+  m.SetInput({0, -6, 2, 4, 3, -2, 10, 1});
+  m.Invoke();
+  EXPECT_THAT(
+      m.GetOutput(),
+      ElementsAreArray(ArrayFloatNear(
+          {.09752, .05352, .11911, .14548, .13164, .07984, .26509, .10778})));
+}
+
+TEST(QuantizedActivationsOpTest, Softmax1D) {
+  QuantizedActivationsOpModel m(0.1,
+                                /*input=*/{TensorType_UINT8, {8}, -10, 10});
+  m.SetInput({0, -6, 2, 4, 3, -2, 10, 1});
+  m.Invoke();
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear({0.09766, 0.05469, 0.12109, 0.14453,
+                                       0.13281, 0.07813, 0.26563, 0.10938},
+                                      kQuantizedTolerance)));
+}
+
 TEST(FloatActivationsOpTest, Softmax2D) {
   FloatActivationsOpModel m(0.1,
                             /*input=*/{TensorType_FLOAT32, {2, 4}});
-- 
cgit v1.2.3


From a4bab4517eddef07236529c0141e85bcae06ad74 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling 
Date: Fri, 20 Jul 2018 16:23:51 -0700
Subject: Align TFLite tensors to 64 bytes for EIGEN_DONT_ALIGN

PiperOrigin-RevId: 205471025
---
 tensorflow/contrib/lite/arena_planner.cc         | 25 +++++++++---------------
 tensorflow/contrib/lite/arena_planner.h          | 10 +++++++++-
 tensorflow/contrib/lite/arena_planner_test.cc    |  8 +++++---
 tensorflow/contrib/lite/kernels/BUILD            |  1 +
 tensorflow/contrib/lite/kernels/eigen_support.cc | 11 +++++++++++
 tensorflow/contrib/lite/simple_memory_arena.cc   |  2 +-
 6 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/tensorflow/contrib/lite/arena_planner.cc b/tensorflow/contrib/lite/arena_planner.cc
index 16a0e71624..02442575b3 100644
--- a/tensorflow/contrib/lite/arena_planner.cc
+++ b/tensorflow/contrib/lite/arena_planner.cc
@@ -17,14 +17,6 @@ limitations under the License.
 
 namespace tflite {
 
-namespace {
-
-// Memory allocation tuning
-constexpr const int kDefaultArenaAlignment = 64;
-constexpr const int kDefaultTensorAlignment = 4;
-
-}  // namespace
-
 struct AllocationInfo {
   // The node index requesting this allocation.
   int node;
@@ -36,13 +28,16 @@ struct AllocationInfo {
 
 ArenaPlanner::ArenaPlanner(TfLiteContext* context,
                            std::unique_ptr graph_info,
-                           bool preserve_inputs, bool preserve_intermediates)
+                           bool preserve_inputs, bool preserve_intermediates,
+                           int tensor_alignment)
     : context_(context),
       graph_info_(std::move(graph_info)),
       arena_(kDefaultArenaAlignment),
       persistent_arena_(kDefaultArenaAlignment),
       preserve_inputs_(preserve_inputs),
-      preserve_intermediates_(preserve_intermediates) {}
+      preserve_intermediates_(preserve_intermediates),
+      tensor_alignment_(tensor_alignment) {}
+
 ArenaPlanner::~ArenaPlanner() {}
 
 int64_t ArenaPlanner::BasePointer(TfLiteAllocationType type) {
@@ -264,14 +259,12 @@ TfLiteStatus ArenaPlanner::ResolveTensorAllocation(int tensor_index) {
 TfLiteStatus ArenaPlanner::CalculateTensorAllocation(int tensor_index) {
   TfLiteTensor& tensor = *graph_info_->tensor(tensor_index);
   if (tensor.allocation_type == kTfLiteArenaRw) {
-    TF_LITE_ENSURE_STATUS(arena_.Allocate(context_, kDefaultTensorAlignment,
-                                          tensor.bytes,
-                                          &allocs_[tensor_index]));
+    TF_LITE_ENSURE_STATUS(arena_.Allocate(
+        context_, tensor_alignment_, tensor.bytes, &allocs_[tensor_index]));
   }
   if (tensor.allocation_type == kTfLiteArenaRwPersistent) {
-    TF_LITE_ENSURE_STATUS(
-        persistent_arena_.Allocate(context_, kDefaultTensorAlignment,
-                                   tensor.bytes, &allocs_[tensor_index]));
+    TF_LITE_ENSURE_STATUS(persistent_arena_.Allocate(
+        context_, tensor_alignment_, tensor.bytes, &allocs_[tensor_index]));
   }
   return kTfLiteOk;
 }
diff --git a/tensorflow/contrib/lite/arena_planner.h b/tensorflow/contrib/lite/arena_planner.h
index 82c866734f..55003cf4e9 100644
--- a/tensorflow/contrib/lite/arena_planner.h
+++ b/tensorflow/contrib/lite/arena_planner.h
@@ -25,6 +25,10 @@ limitations under the License.
 
 namespace tflite {
 
+// Memory allocation tuning
+constexpr const int kDefaultArenaAlignment = 64;
+constexpr const int kDefaultTensorAlignment = 64;
+
 struct AllocationInfo;
 
 // A memory planner that makes all the allocations using arenas.
@@ -47,7 +51,8 @@ class ArenaPlanner : public MemoryPlanner {
   // graph will not share memory with any other tensor, effectively preserving
   // them until the end of inference.
   ArenaPlanner(TfLiteContext* context, std::unique_ptr graph_info,
-               bool preserve_inputs, bool preserve_intermediates);
+               bool preserve_inputs, bool preserve_intermediates,
+               int tensor_alignment = kDefaultTensorAlignment);
   ~ArenaPlanner() override;
   ArenaPlanner(const ArenaPlanner&) = delete;
   ArenaPlanner& operator=(const ArenaPlanner&) = delete;
@@ -112,6 +117,9 @@ class ArenaPlanner : public MemoryPlanner {
   // If true, then no overlapping of memory areas is done, meaning intermediates
   // results can be queried after running (modulo running delegates).
   bool preserve_intermediates_;
+
+  // Number of bytes that tensor buffers should be aligned to.
+  int tensor_alignment_;
 };
 
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/arena_planner_test.cc b/tensorflow/contrib/lite/arena_planner_test.cc
index 1adb426d58..7d7c41289c 100644
--- a/tensorflow/contrib/lite/arena_planner_test.cc
+++ b/tensorflow/contrib/lite/arena_planner_test.cc
@@ -24,6 +24,8 @@ limitations under the License.
 namespace tflite {
 namespace {
 
+constexpr const int kTensorAlignment = 4;
+
 // A simple op to be used in tests, as syntactic sugar.
 class TestOp {
  public:
@@ -156,7 +158,7 @@ class ArenaPlannerTest : public ::testing::Test {
     context_.ReportError = ReportError;
     planner_.reset(new ArenaPlanner(
         &context_, std::unique_ptr(new TestGraphInfo(graph)),
-        preserve_inputs, /*preserve intermediates*/ false));
+        preserve_inputs, /*preserve intermediates*/ false, kTensorAlignment));
     CHECK(planner_->ResetAllocations() == kTfLiteOk);
     CHECK(planner_->PlanAllocations() == kTfLiteOk);
   }
@@ -178,8 +180,8 @@ class ArenaPlannerTest : public ::testing::Test {
     const TfLiteTensor& tensor = (*graph_->tensors())[tensor_index];
     int64_t offset = GetOffset(tensor_index) + tensor.bytes;
     // We must make sure the offset is aligned to kDefaultArenaAlignment.
-    if (offset % 4 != 0) {
-      offset += 4 - offset % 4;
+    if (offset % kTensorAlignment != 0) {
+      offset += kTensorAlignment - offset % kTensorAlignment;
     }
     return offset;
   };
diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index ad30624f40..9549b4445d 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -58,6 +58,7 @@ cc_library(
     }),
     deps = [
         ":op_macros",
+        "//tensorflow/contrib/lite:arena_planner",
         "//tensorflow/contrib/lite:context",
         "//tensorflow/contrib/lite/kernels/internal:optimized",
     ],
diff --git a/tensorflow/contrib/lite/kernels/eigen_support.cc b/tensorflow/contrib/lite/kernels/eigen_support.cc
index 4f0d020793..e542ad0765 100644
--- a/tensorflow/contrib/lite/kernels/eigen_support.cc
+++ b/tensorflow/contrib/lite/kernels/eigen_support.cc
@@ -16,6 +16,7 @@ limitations under the License.
 
 #include 
 
+#include "tensorflow/contrib/lite/arena_planner.h"
 #include "tensorflow/contrib/lite/kernels/internal/optimized/eigen_spatial_convolutions.h"
 #include "tensorflow/contrib/lite/kernels/op_macros.h"
 
@@ -23,6 +24,16 @@ namespace tflite {
 namespace eigen_support {
 namespace {
 
+#ifndef EIGEN_DONT_ALIGN
+// Eigen may require buffers to be algiend to 16, 32 or 64 bytes depending on
+// hardware architecture and build configurations.
+// If the static assertion fails, try to increase `kDefaultTensorAlignment` to
+// in `arena_planner.h` to 32 or 64.
+static_assert(
+    kDefaultTensorAlignment % EIGEN_MAX_ALIGN_BYTES == 0,
+    "kDefaultArenaAlignment doesn't comply with Eigen alignment requirement.");
+#endif  // EIGEN_DONT_ALIGN
+
 // We have a single global threadpool for all convolution operations. This means
 // that inferences started from different threads may block each other, but
 // since the underlying resource of CPU cores should be consumed by the
diff --git a/tensorflow/contrib/lite/simple_memory_arena.cc b/tensorflow/contrib/lite/simple_memory_arena.cc
index 4eaf6f1bfe..24593d2a67 100644
--- a/tensorflow/contrib/lite/simple_memory_arena.cc
+++ b/tensorflow/contrib/lite/simple_memory_arena.cc
@@ -34,7 +34,7 @@ namespace tflite {
 TfLiteStatus SimpleMemoryArena::Allocate(TfLiteContext* context,
                                          size_t alignment, size_t size,
                                          ArenaAlloc* new_alloc) {
-  TF_LITE_ENSURE(context, alignment < arena_alignment_);
+  TF_LITE_ENSURE(context, alignment <= arena_alignment_);
 
   if (size == 0) {
     new_alloc->offset = 0;
-- 
cgit v1.2.3


From 9e61678787d329322dd729db92e833c874bdf835 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling 
Date: Fri, 20 Jul 2018 16:26:54 -0700
Subject: TFLite Reshape - Uses shape input tensor

Now it:
(1) Use shape (2nd input tensor) if it exists. Mark the output
    as a dynamic tensor if shape tensor isn't constant.
(2) Fallback to `TfLiteReshapeParams.shape` if the shape input
    tensor doesn't exist.

PiperOrigin-RevId: 205471380
---
 tensorflow/contrib/lite/kernels/reshape.cc         | 69 +++++++++++++++++-----
 .../contrib/lite/testing/generate_examples.py      | 27 +++++++--
 2 files changed, 75 insertions(+), 21 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/reshape.cc b/tensorflow/contrib/lite/kernels/reshape.cc
index 3287040695..99ecc16093 100644
--- a/tensorflow/contrib/lite/kernels/reshape.cc
+++ b/tensorflow/contrib/lite/kernels/reshape.cc
@@ -25,16 +25,11 @@ namespace builtin {
 namespace reshape {
 
 constexpr int kInputTensor = 0;
+constexpr int kShapeTensor = 1;
 constexpr int kOutputTensor = 0;
 
-TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
-  auto* params = reinterpret_cast(node->builtin_data);
-
-  // TODO(ahentz): we are often given a tensor with the shape but we only pay
-  // attention to what the shape specified in 'params'.
-  TF_LITE_ENSURE(context, NumInputs(node) == 1 || NumInputs(node) == 2);
-  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
-
+TfLiteStatus ResizeOutput(TfLiteContext* context, TfLiteNode* node,
+                          TfLiteIntArray* output_shape) {
   const TfLiteTensor* input = GetInput(context, node, kInputTensor);
   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
 
@@ -47,32 +42,76 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
     num_input_elements *= SizeOfDimension(input, i);
   }
 
-  TfLiteIntArray* output_size = TfLiteIntArrayCreate(params->num_dimensions);
   int num_output_elements = 1;
   int stretch_dim = -1;
-  for (int i = 0; i < params->num_dimensions; ++i) {
-    int value = params->shape[i];
+  for (int i = 0; i < output_shape->size; ++i) {
+    int value = output_shape->data[i];
     if (value == -1) {
       TF_LITE_ENSURE_EQ(context, stretch_dim, -1);
       stretch_dim = i;
     } else {
       num_output_elements *= value;
-      output_size->data[i] = value;
     }
   }
   if (stretch_dim != -1) {
-    output_size->data[stretch_dim] = num_input_elements / num_output_elements;
-    num_output_elements *= output_size->data[stretch_dim];
+    output_shape->data[stretch_dim] = num_input_elements / num_output_elements;
+    num_output_elements *= output_shape->data[stretch_dim];
   }
 
   TF_LITE_ENSURE_EQ(context, num_input_elements, num_output_elements);
-  return context->ResizeTensor(context, output, output_size);
+  return context->ResizeTensor(context, output, output_shape);
+}
+
+TfLiteStatus ResizeOutputWithShapeTensor(TfLiteContext* context,
+                                         TfLiteNode* node) {
+  const TfLiteTensor* shape = GetInput(context, node, kShapeTensor);
+
+  TfLiteIntArray* output_shape = TfLiteIntArrayCreate(shape->dims->data[0]);
+  for (int i = 0; i < output_shape->size; ++i) {
+    output_shape->data[i] = shape->data.i32[i];
+  }
+  return ResizeOutput(context, node, output_shape);
+}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  auto* params = reinterpret_cast(node->builtin_data);
+
+  TF_LITE_ENSURE(context, NumInputs(node) == 1 || NumInputs(node) == 2);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  // Attempt to use shape tensor if it exists.
+  if (NumInputs(node) == 2) {
+    const TfLiteTensor* shape = GetInput(context, node, kShapeTensor);
+    // Check if the shape tensor is valid.
+    if (shape->dims->size == 1 && shape->type == kTfLiteInt32) {
+      // Set the output tensor as dynamic if the shape isn't constnat.
+      if (!IsConstantTensor(shape)) {
+        TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+        SetTensorToDynamic(output);
+        return kTfLiteOk;
+      }
+      // Shape is constant. Resize now.
+      return ResizeOutputWithShapeTensor(context, node);
+    }
+  }
+  // The function is returned above this line if the shape tensor is usable.
+  // Now fallback to the shape parameter in `TfLiteReshapeParams`.
+
+  TfLiteIntArray* output_shape = TfLiteIntArrayCreate(params->num_dimensions);
+  for (int i = 0; i < params->num_dimensions; ++i) {
+    output_shape->data[i] = params->shape[i];
+  }
+  return ResizeOutput(context, node, output_shape);
 }
 
 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const TfLiteTensor* input = GetInput(context, node, kInputTensor);
   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
 
+  if (IsDynamicTensor(output)) {
+    TF_LITE_ENSURE_OK(context, ResizeOutputWithShapeTensor(context, node));
+  }
+
   memcpy(output->data.raw, input->data.raw, input->bytes);
 
   return kTfLiteOk;
diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index 32d04c0717..a91ff8626a 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -1595,19 +1595,34 @@ def make_reshape_tests(zip_path):
       "dtype": [tf.float32, tf.int32],
       "input_shape": [[3, 4, 5, 7], [4, 105], [21, 5, 2, 2], [420]],
       "output_shape": [[15, 28], [420], [1, -1, 5, 7], [-1]],
+      "constant_shape": [True, False],
   }]
 
   def build_graph(parameters):
     input_tensor = tf.placeholder(dtype=parameters["dtype"], name="input",
                                   shape=parameters["input_shape"])
-    out = tf.reshape(input_tensor, shape=parameters["output_shape"])
-    return [input_tensor], [out]
+
+    # Get shape as either a placeholder or constants.
+    if parameters["constant_shape"]:
+      output_shape = parameters["output_shape"]
+      input_tensors = [input_tensor]
+    else:
+      # The shape of the shape tensor.
+      shape_tensor_shape = [len(parameters["output_shape"])]
+      output_shape = tf.placeholder(
+          dtype=tf.int32, name="output_shape", shape=shape_tensor_shape)
+      input_tensors = [input_tensor, output_shape]
+    out = tf.reshape(input_tensor, shape=output_shape)
+    return input_tensors, [out]
 
   def build_inputs(parameters, sess, inputs, outputs):
-    input_values = create_tensor_data(parameters["dtype"],
-                                      parameters["input_shape"])
-    return [input_values], sess.run(
-        outputs, feed_dict=dict(zip(inputs, [input_values])))
+    values = [
+        create_tensor_data(parameters["dtype"], parameters["input_shape"])
+    ]
+    if not parameters["constant_shape"]:
+      values.append(np.array(parameters["output_shape"]))
+
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
-- 
cgit v1.2.3


From a4b95884f870a040038e530c978239999933acd9 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling 
Date: Fri, 20 Jul 2018 16:27:29 -0700
Subject: TFLite Python: Make resize_input_tensor accept list/tuple sizes.

PiperOrigin-RevId: 205471451
---
 tensorflow/contrib/lite/python/BUILD               | 1 +
 tensorflow/contrib/lite/python/interpreter.py      | 6 +++++-
 tensorflow/contrib/lite/python/interpreter_test.py | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lite/python/BUILD b/tensorflow/contrib/lite/python/BUILD
index 727fbff38e..860aff9e7e 100644
--- a/tensorflow/contrib/lite/python/BUILD
+++ b/tensorflow/contrib/lite/python/BUILD
@@ -20,6 +20,7 @@ py_library(
     deps = [
         "//tensorflow/contrib/lite/python/interpreter_wrapper:tensorflow_wrap_interpreter_wrapper",
         "//tensorflow/python:util",
+        "//third_party/py/numpy",
     ],
 )
 
diff --git a/tensorflow/contrib/lite/python/interpreter.py b/tensorflow/contrib/lite/python/interpreter.py
index e1981ceae2..3243bddac8 100644
--- a/tensorflow/contrib/lite/python/interpreter.py
+++ b/tensorflow/contrib/lite/python/interpreter.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 import sys
+import numpy as np
 from tensorflow.python.util.lazy_loader import LazyLoader
 
 # Lazy load since some of the performance benchmark skylark rules
@@ -162,6 +163,9 @@ class Interpreter(object):
       ValueError: If the interpreter could not resize the input tensor.
     """
     self._ensure_safe()
+    # `ResizeInputTensor` now only accepts int32 numpy array as `tensor_size
+    # parameter.
+    tensor_size = np.array(tensor_size, dtype=np.int32)
     self._interpreter.ResizeInputTensor(input_index, tensor_size)
 
   def get_output_details(self):
@@ -204,7 +208,7 @@ class Interpreter(object):
     for i in range(10):
       input().fill(3.)
       interpreter.invoke()
-      print("inference %s" % output)
+      print("inference %s" % output())
 
     Notice how this function avoids making a numpy array directly. This is
     because it is important to not hold actual numpy views to the data longer
diff --git a/tensorflow/contrib/lite/python/interpreter_test.py b/tensorflow/contrib/lite/python/interpreter_test.py
index 95fa4b8584..e77d52ca99 100644
--- a/tensorflow/contrib/lite/python/interpreter_test.py
+++ b/tensorflow/contrib/lite/python/interpreter_test.py
@@ -83,7 +83,7 @@ class InterpreterTest(test_util.TensorFlowTestCase):
     test_input = np.array([[1, 2, 3, 4]], dtype=np.uint8)
     expected_output = np.array([[4, 3, 2, 1]], dtype=np.uint8)
     interpreter.resize_tensor_input(input_details[0]['index'],
-                                    np.array(test_input.shape, dtype=np.int32))
+                                    test_input.shape)
     interpreter.allocate_tensors()
     interpreter.set_tensor(input_details[0]['index'], test_input)
     interpreter.invoke()
-- 
cgit v1.2.3


From 0cc0166a97f95499f0af673f3004d6bb748dc7e4 Mon Sep 17 00:00:00 2001
From: Allen Lavoie 
Date: Fri, 20 Jul 2018 16:34:46 -0700
Subject: Relax dependency checking for custom op libraries

These checks were necessary when we used RTLD_GLOBAL to expose TF symbols to custom ops, since :framework and :lib pulled in implementations. They're now header-only. Ideally we'd switch the checks to framework_internal_impl and lib_internal_impl, but that would require visibility for those rules (thus making it more likely they'd get included in silly places). So this change disables the check for dynamic builds, on the theory that accidentally relying on implementation rules is much more difficult than it was with a static build.

Should allow tf_custom_op_libraries to depend on GPU kernels (which depend on core:gpu_lib which depends on :framework).

PiperOrigin-RevId: 205472434
---
 tensorflow/tensorflow.bzl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 954940642b..26970c8cb0 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1359,7 +1359,7 @@ def tf_custom_op_library(name, srcs=[], gpu_srcs=[], deps=[], linkopts=[]):
       name=name,
       srcs=srcs,
       deps=deps + if_cuda(cuda_deps),
-      data=[name + "_check_deps"],
+      data=if_static([name + "_check_deps"]),
       copts=tf_copts(is_external=True),
       features = ["windows_export_all_symbols"],
       linkopts=linkopts + select({
-- 
cgit v1.2.3


From 8741006018326350467fe86785d98963ff9e983e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 16:39:46 -0700
Subject: Automated rollback of commit 265292420de30f24805d28886d403dc42d3685b3

PiperOrigin-RevId: 205472990
---
 .../contrib/eager/python/examples/revnet/BUILD     |  36 --
 .../contrib/eager/python/examples/revnet/blocks.py | 374 +++++++--------------
 .../eager/python/examples/revnet/cifar_input.py    |   2 +-
 .../contrib/eager/python/examples/revnet/config.py |  16 +-
 .../contrib/eager/python/examples/revnet/main.py   |  82 +++--
 .../eager/python/examples/revnet/main_estimator.py | 200 -----------
 .../python/examples/revnet/main_estimator_tpu.py   | 328 ------------------
 .../contrib/eager/python/examples/revnet/revnet.py | 110 ++++--
 .../eager/python/examples/revnet/revnet_test.py    |  25 +-
 9 files changed, 268 insertions(+), 905 deletions(-)
 delete mode 100644 tensorflow/contrib/eager/python/examples/revnet/main_estimator.py
 delete mode 100644 tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py

diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index 3316dc1114..0c0e4c0eb9 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -113,39 +113,3 @@ py_binary(
         "//tensorflow:tensorflow_py",
     ],
 )
-
-py_binary(
-    name = "main_estimator",
-    srcs = ["main_estimator.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":cifar_input",
-        ":main",
-        ":revnet",
-        "//tensorflow:tensorflow_py",
-    ],
-)
-
-py_library(
-    name = "main_estimator_lib",
-    srcs = ["main_estimator.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":cifar_input",
-        ":main",
-        ":revnet",
-        "//tensorflow:tensorflow_py",
-    ],
-)
-
-py_library(
-    name = "main_estimator_tpu_lib",
-    srcs = ["main_estimator_tpu.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":cifar_input",
-        ":main",
-        ":revnet",
-        "//tensorflow:tensorflow_py",
-    ],
-)
diff --git a/tensorflow/contrib/eager/python/examples/revnet/blocks.py b/tensorflow/contrib/eager/python/examples/revnet/blocks.py
index 639bb06a34..306096e9f8 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/blocks.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/blocks.py
@@ -24,9 +24,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import functools
-import operator
-
 import tensorflow as tf
 from tensorflow.contrib.eager.python.examples.revnet import ops
 
@@ -48,7 +45,7 @@ class RevBlock(tf.keras.Model):
                bottleneck=False,
                fused=True,
                dtype=tf.float32):
-    """Initialization.
+    """Initialize RevBlock.
 
     Args:
       n_res: number of residual blocks
@@ -102,6 +99,7 @@ class RevBlock(tf.keras.Model):
       if i == 0:
         # First block usually contains downsampling that can't be reversed
         with tf.GradientTape() as tape:
+          x = tf.identity(x)
           tape.watch(x)
           y = block(x, training=training)
 
@@ -123,6 +121,16 @@ class _Residual(tf.keras.Model):
   """Single residual block contained in a _RevBlock. Each `_Residual` object has
   two _ResidualInner objects, corresponding to the `F` and `G` functions in the
   paper.
+
+  Args:
+    filters: output filter size
+    strides: length 2 list/tuple of integers for height and width strides
+    input_shape: length 3 list/tuple of integers
+    batch_norm_first: whether to apply activation and batch norm before conv
+    data_format: tensor data format, "NCHW"/"NHWC",
+    bottleneck: use bottleneck residual if True
+    fused: use fused batch normalization if True
+    dtype: float16, float32, or float64
   """
 
   def __init__(self,
@@ -134,18 +142,6 @@ class _Residual(tf.keras.Model):
                bottleneck=False,
                fused=True,
                dtype=tf.float32):
-    """Initialization.
-
-    Args:
-      filters: output filter size
-      strides: length 2 list/tuple of integers for height and width strides
-      input_shape: length 3 list/tuple of integers
-      batch_norm_first: whether to apply activation and batch norm before conv
-      data_format: tensor data format, "NCHW"/"NHWC",
-      bottleneck: use bottleneck residual if True
-      fused: use fused batch normalization if True
-      dtype: float16, float32, or float64
-    """
     super(_Residual, self).__init__()
 
     self.filters = filters
@@ -200,6 +196,7 @@ class _Residual(tf.keras.Model):
     dy1, dy2 = tf.split(dy, num_or_size_splits=2, axis=self.axis)
 
     with tf.GradientTape(persistent=True) as tape:
+      y = tf.identity(y)
       tape.watch(y)
       y1, y2 = tf.split(y, num_or_size_splits=2, axis=self.axis)
       z1 = y1
@@ -230,252 +227,131 @@ class _Residual(tf.keras.Model):
     return x, dx, grads, vars_
 
 
-# Ideally, the following should be wrapped in `tf.keras.Sequential`, however
-# there are subtle issues with its placeholder insertion policy and batch norm
-class _BottleneckResidualInner(tf.keras.Model):
+def _BottleneckResidualInner(filters,
+                             strides,
+                             input_shape,
+                             batch_norm_first=True,
+                             data_format="channels_first",
+                             fused=True,
+                             dtype=tf.float32):
   """Single bottleneck residual inner function contained in _Resdual.
 
   Corresponds to the `F`/`G` functions in the paper.
   Suitable for training on ImageNet dataset.
-  """
-
-  def __init__(self,
-               filters,
-               strides,
-               input_shape,
-               batch_norm_first=True,
-               data_format="channels_first",
-               fused=True,
-               dtype=tf.float32):
-    """Initialization.
-
-    Args:
-      filters: output filter size
-      strides: length 2 list/tuple of integers for height and width strides
-      input_shape: length 3 list/tuple of integers
-      batch_norm_first: whether to apply activation and batch norm before conv
-      data_format: tensor data format, "NCHW"/"NHWC"
-      fused: use fused batch normalization if True
-      dtype: float16, float32, or float64
-    """
-    super(_BottleneckResidualInner, self).__init__()
-    axis = 1 if data_format == "channels_first" else 3
-    if batch_norm_first:
-      self.batch_norm_0 = tf.keras.layers.BatchNormalization(
-          axis=axis, input_shape=input_shape, fused=fused, dtype=dtype)
-
-    self.conv2d_1 = tf.keras.layers.Conv2D(
-        filters=filters // 4,
-        kernel_size=1,
-        strides=strides,
-        input_shape=input_shape,
-        data_format=data_format,
-        use_bias=False,
-        padding="SAME",
-        dtype=dtype)
-    self.batch_norm_1 = tf.keras.layers.BatchNormalization(
-        axis=axis, fused=fused, dtype=dtype)
-
-    self.conv2d_2 = tf.keras.layers.Conv2D(
-        filters=filters // 4,
-        kernel_size=3,
-        strides=(1, 1),
-        data_format=data_format,
-        use_bias=False,
-        padding="SAME",
-        dtype=dtype)
 
-    self.batch_norm_2 = tf.keras.layers.BatchNormalization(
-        axis=axis, fused=fused, dtype=dtype)
-    self.conv2d_3 = tf.keras.layers.Conv2D(
-        filters=filters,
-        kernel_size=1,
-        strides=(1, 1),
-        data_format=data_format,
-        use_bias=False,
-        padding="SAME",
-        dtype=dtype)
-
-    self.batch_norm_first = batch_norm_first
-
-  def call(self, x, training=True):
-    net = x
-    if self.batch_norm_first:
-      net = self.batch_norm_0(net, training=training)
-      net = tf.nn.relu(net)
-
-    net = self.conv2d_1(net)
-    net = self.batch_norm_1(net, training=training)
-    net = tf.nn.relu(net)
-
-    net = self.conv2d_2(net)
-    net = self.batch_norm_2(net, training=training)
-    net = tf.nn.relu(net)
+  Args:
+    filters: output filter size
+    strides: length 2 list/tuple of integers for height and width strides
+    input_shape: length 3 list/tuple of integers
+    batch_norm_first: whether to apply activation and batch norm before conv
+    data_format: tensor data format, "NCHW"/"NHWC"
+    fused: use fused batch normalization if True
+    dtype: float16, float32, or float64
+
+  Returns:
+    A keras model
+  """
 
-    net = self.conv2d_3(net)
+  axis = 1 if data_format == "channels_first" else 3
+  model = tf.keras.Sequential()
+  if batch_norm_first:
+    model.add(
+        tf.keras.layers.BatchNormalization(
+            axis=axis, input_shape=input_shape, fused=fused, dtype=dtype))
+    model.add(tf.keras.layers.Activation("relu"))
+  model.add(
+      tf.keras.layers.Conv2D(
+          filters=filters // 4,
+          kernel_size=1,
+          strides=strides,
+          input_shape=input_shape,
+          data_format=data_format,
+          use_bias=False,
+          padding="SAME",
+          dtype=dtype))
+
+  model.add(
+      tf.keras.layers.BatchNormalization(axis=axis, fused=fused, dtype=dtype))
+  model.add(tf.keras.layers.Activation("relu"))
+  model.add(
+      tf.keras.layers.Conv2D(
+          filters=filters // 4,
+          kernel_size=3,
+          strides=(1, 1),
+          data_format=data_format,
+          use_bias=False,
+          padding="SAME",
+          dtype=dtype))
+
+  model.add(
+      tf.keras.layers.BatchNormalization(axis=axis, fused=fused, dtype=dtype))
+  model.add(tf.keras.layers.Activation("relu"))
+  model.add(
+      tf.keras.layers.Conv2D(
+          filters=filters,
+          kernel_size=1,
+          strides=(1, 1),
+          data_format=data_format,
+          use_bias=False,
+          padding="SAME",
+          dtype=dtype))
 
-    return net
+  return model
 
 
-class _ResidualInner(tf.keras.Model):
+def _ResidualInner(filters,
+                   strides,
+                   input_shape,
+                   batch_norm_first=True,
+                   data_format="channels_first",
+                   fused=True,
+                   dtype=tf.float32):
   """Single residual inner function contained in _ResdualBlock.
 
   Corresponds to the `F`/`G` functions in the paper.
-  """
-
-  def __init__(self,
-               filters,
-               strides,
-               input_shape,
-               batch_norm_first=True,
-               data_format="channels_first",
-               fused=True,
-               dtype=tf.float32):
-    """Initialization.
-
-    Args:
-      filters: output filter size
-      strides: length 2 list/tuple of integers for height and width strides
-      input_shape: length 3 list/tuple of integers
-      batch_norm_first: whether to apply activation and batch norm before conv
-      data_format: tensor data format, "NCHW"/"NHWC"
-      fused: use fused batch normalization if True
-      dtype: float16, float32, or float64
-    """
-    super(_ResidualInner, self).__init__()
-    axis = 1 if data_format == "channels_first" else 3
-    if batch_norm_first:
-      self.batch_norm_0 = tf.keras.layers.BatchNormalization(
-          axis=axis, input_shape=input_shape, fused=fused, dtype=dtype)
-    self.conv2d_1 = tf.keras.layers.Conv2D(
-        filters=filters,
-        kernel_size=3,
-        strides=strides,
-        input_shape=input_shape,
-        data_format=data_format,
-        use_bias=False,
-        padding="SAME",
-        dtype=dtype)
-    self.batch_norm_1 = tf.keras.layers.BatchNormalization(
-        axis=axis, fused=fused, dtype=dtype)
-
-    self.conv2d_2 = tf.keras.layers.Conv2D(
-        filters=filters,
-        kernel_size=3,
-        strides=(1, 1),
-        data_format=data_format,
-        use_bias=False,
-        padding="SAME",
-        dtype=dtype)
-
-    self.batch_norm_first = batch_norm_first
-
-  def call(self, x, training=True):
-    net = x
-    if self.batch_norm_first:
-      net = self.batch_norm_0(net, training=training)
-      net = tf.nn.relu(net)
-
-    net = self.conv2d_1(net)
-    net = self.batch_norm_1(net, training=training)
-
-    net = self.conv2d_2(net)
-
-    return net
 
+  Args:
+    filters: output filter size
+    strides: length 2 list/tuple of integers for height and width strides
+    input_shape: length 3 list/tuple of integers
+    batch_norm_first: whether to apply activation and batch norm before conv
+    data_format: tensor data format, "NCHW"/"NHWC"
+    fused: use fused batch normalization if True
+    dtype: float16, float32, or float64
+
+  Returns:
+    A keras model
+  """
 
-class InitBlock(tf.keras.Model):
-  """Initial block of RevNet."""
-
-  def __init__(self, config):
-    """Initialization.
-
-    Args:
-      config: tf.contrib.training.HParams object; specifies hyperparameters
-    """
-    super(InitBlock, self).__init__()
-    self.config = config
-    self.axis = 1 if self.config.data_format == "channels_first" else 3
-    self.conv2d = tf.keras.layers.Conv2D(
-        filters=self.config.init_filters,
-        kernel_size=self.config.init_kernel,
-        strides=(self.config.init_stride, self.config.init_stride),
-        data_format=self.config.data_format,
-        use_bias=False,
-        padding="SAME",
-        input_shape=self.config.input_shape,
-        dtype=self.config.dtype)
-    self.batch_norm = tf.keras.layers.BatchNormalization(
-        axis=self.axis, fused=self.config.fused, dtype=self.config.dtype)
-    self.activation = tf.keras.layers.Activation("relu")
-
-    if self.config.init_max_pool:
-      self.max_pool = tf.keras.layers.MaxPooling2D(
-          pool_size=(3, 3),
-          strides=(2, 2),
+  axis = 1 if data_format == "channels_first" else 3
+  model = tf.keras.Sequential()
+  if batch_norm_first:
+    model.add(
+        tf.keras.layers.BatchNormalization(
+            axis=axis, input_shape=input_shape, fused=fused, dtype=dtype))
+    model.add(tf.keras.layers.Activation("relu"))
+  model.add(
+      tf.keras.layers.Conv2D(
+          filters=filters,
+          kernel_size=3,
+          strides=strides,
+          input_shape=input_shape,
+          data_format=data_format,
+          use_bias=False,
           padding="SAME",
-          data_format=self.config.data_format,
-          dtype=self.config.dtype)
-
-  def call(self, x, training=True):
-    net = x
-    net = self.conv2d(net)
-    net = self.batch_norm(net, training=training)
-    net = self.activation(net)
-
-    if self.config.init_max_pool:
-      net = self.max_pool(net)
-
-    return net
-
-
-class FinalBlock(tf.keras.Model):
-  """Final block of RevNet."""
-
-  def __init__(self, config):
-    """Initialization.
-
-    Args:
-      config: tf.contrib.training.HParams object; specifies hyperparameters
+          dtype=dtype))
+
+  model.add(
+      tf.keras.layers.BatchNormalization(axis=axis, fused=fused, dtype=dtype))
+  model.add(tf.keras.layers.Activation("relu"))
+  model.add(
+      tf.keras.layers.Conv2D(
+          filters=filters,
+          kernel_size=3,
+          strides=(1, 1),
+          data_format=data_format,
+          use_bias=False,
+          padding="SAME",
+          dtype=dtype))
 
-    Raises:
-      ValueError: Unsupported data format
-    """
-    super(FinalBlock, self).__init__()
-    self.config = config
-    self.axis = 1 if self.config.data_format == "channels_first" else 3
-
-    f = self.config.filters[-1]  # Number of filters
-    r = functools.reduce(operator.mul, self.config.strides, 1)  # Reduce ratio
-    r *= self.config.init_stride
-    if self.config.init_max_pool:
-      r *= 2
-
-    if self.config.data_format == "channels_first":
-      w, h = self.config.input_shape[1], self.config.input_shape[2]
-      input_shape = (f, w // r, h // r)
-    elif self.config.data_format == "channels_last":
-      w, h = self.config.input_shape[0], self.config.input_shape[1]
-      input_shape = (w // r, h // r, f)
-    else:
-      raise ValueError("Data format should be either `channels_first`"
-                       " or `channels_last`")
-    self.batch_norm = tf.keras.layers.BatchNormalization(
-        axis=self.axis,
-        input_shape=input_shape,
-        fused=self.config.fused,
-        dtype=self.config.dtype)
-    self.activation = tf.keras.layers.Activation("relu")
-    self.global_avg_pool = tf.keras.layers.GlobalAveragePooling2D(
-        data_format=self.config.data_format, dtype=self.config.dtype)
-    self.dense = tf.keras.layers.Dense(
-        self.config.n_classes, dtype=self.config.dtype)
-
-  def call(self, x, training=True):
-    net = x
-    net = self.batch_norm(net, training=training)
-    net = self.activation(net)
-    net = self.global_avg_pool(net)
-    net = self.dense(net)
-
-    return net
+  return model
diff --git a/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py b/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py
index e9672f13e1..b6d4c35bfd 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py
@@ -111,6 +111,6 @@ def get_ds_from_tfrecords(data_dir,
     }[split]
     dataset = dataset.shuffle(size)
 
-  dataset = dataset.batch(batch_size, drop_remainder=True)
+  dataset = dataset.batch(batch_size)
 
   return dataset
diff --git a/tensorflow/contrib/eager/python/examples/revnet/config.py b/tensorflow/contrib/eager/python/examples/revnet/config.py
index 1532c7b67b..3d93fa955a 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/config.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/config.py
@@ -27,16 +27,17 @@ from __future__ import division
 from __future__ import print_function
 
 import tensorflow as tf
+tfe = tf.contrib.eager
 
 
 def get_hparams_cifar_38():
   """RevNet-38 configurations for CIFAR-10/CIFAR-100."""
 
   config = tf.contrib.training.HParams()
-  # Hyperparameters from the RevNet paper
   config.add_hparam("init_filters", 32)
   config.add_hparam("init_kernel", 3)
   config.add_hparam("init_stride", 1)
+  config.add_hparam("n_classes", 10)
   config.add_hparam("n_rev_blocks", 3)
   config.add_hparam("n_res", [3, 3, 3])
   config.add_hparam("filters", [32, 64, 112])
@@ -45,7 +46,7 @@ def get_hparams_cifar_38():
   config.add_hparam("bottleneck", False)
   config.add_hparam("fused", True)
   config.add_hparam("init_max_pool", False)
-  if tf.test.is_gpu_available() > 0:
+  if tfe.num_gpus() > 0:
     config.add_hparam("input_shape", (3, 32, 32))
     config.add_hparam("data_format", "channels_first")
   else:
@@ -70,16 +71,6 @@ def get_hparams_cifar_38():
   config.add_hparam("iters_per_epoch", 50000 // config.batch_size)
   config.add_hparam("epochs", config.max_train_iter // config.iters_per_epoch)
 
-  # Customized TPU hyperparameters due to differing batch size caused by
-  # TPU architecture specifics
-  # Suggested batch sizes to reduce overhead from excessive tensor padding
-  # https://cloud.google.com/tpu/docs/troubleshooting
-  config.add_hparam("tpu_batch_size", 128)
-  config.add_hparam("tpu_eval_batch_size", 1024)
-  config.add_hparam("tpu_iters_per_epoch", 50000 // config.tpu_batch_size)
-  config.add_hparam("tpu_epochs",
-                    config.max_train_iter // config.tpu_iters_per_epoch)
-
   return config
 
 
@@ -110,6 +101,7 @@ def get_hparams_imagenet_56():
   config.add_hparam("init_filters", 128)
   config.add_hparam("init_kernel", 7)
   config.add_hparam("init_stride", 2)
+  config.add_hparam("n_classes", 1000)
   config.add_hparam("n_rev_blocks", 4)
   config.add_hparam("n_res", [2, 2, 2, 2])
   config.add_hparam("filters", [128, 256, 512, 832])
diff --git a/tensorflow/contrib/eager/python/examples/revnet/main.py b/tensorflow/contrib/eager/python/examples/revnet/main.py
index 1a4fd45c8b..e2f43b03f9 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/main.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/main.py
@@ -31,11 +31,8 @@ tfe = tf.contrib.eager
 
 def main(_):
   """Eager execution workflow with RevNet trained on CIFAR-10."""
-  tf.enable_eager_execution()
-
-  config = get_config(config_name=FLAGS.config, dataset=FLAGS.dataset)
-  ds_train, ds_train_one_shot, ds_validation, ds_test = get_datasets(
-      data_dir=FLAGS.data_dir, config=config)
+  config = get_config()
+  ds_train, ds_train_one_shot, ds_validation, ds_test = get_datasets(config)
   model = revnet.RevNet(config=config)
   global_step = tf.train.get_or_create_global_step()  # Ensure correct summary
   global_step.assign(1)
@@ -55,17 +52,23 @@ def main(_):
             "with global_step: {}".format(latest_path, global_step.numpy()))
       sys.stdout.flush()
 
+  if FLAGS.manual_grad:
+    print("Using manual gradients.")
+  else:
+    print("Not using manual gradients.")
+  sys.stdout.flush()
+
   for x, y in ds_train:
     train_one_iter(model, x, y, optimizer, global_step=global_step)
 
     if global_step.numpy() % config.log_every == 0:
+      it_train = ds_train_one_shot.make_one_shot_iterator()
       it_test = ds_test.make_one_shot_iterator()
+      acc_train, loss_train = evaluate(model, it_train)
       acc_test, loss_test = evaluate(model, it_test)
 
       if FLAGS.validate:
-        it_train = ds_train_one_shot.make_one_shot_iterator()
         it_validation = ds_validation.make_one_shot_iterator()
-        acc_train, loss_train = evaluate(model, it_train)
         acc_validation, loss_validation = evaluate(model, it_validation)
         print("Iter {}, "
               "training set accuracy {:.4f}, loss {:.4f}; "
@@ -74,8 +77,11 @@ def main(_):
                   global_step.numpy(), acc_train, loss_train, acc_validation,
                   loss_validation, acc_test, loss_test))
       else:
-        print("Iter {}, test accuracy {:.4f}, loss {:.4f}".format(
-            global_step.numpy(), acc_test, loss_test))
+        print("Iter {}, "
+              "training set accuracy {:.4f}, loss {:.4f}; "
+              "test accuracy {:.4f}, loss {:.4f}".format(
+                  global_step.numpy(), acc_train, loss_train, acc_test,
+                  loss_test))
       sys.stdout.flush()
 
       if FLAGS.train_dir:
@@ -97,38 +103,34 @@ def main(_):
       sys.stdout.flush()
 
 
-def get_config(config_name="revnet-38", dataset="cifar-10"):
+def get_config():
   """Return configuration."""
-  print("Config: {}".format(config_name))
+  print("Config: {}".format(FLAGS.config))
   sys.stdout.flush()
   config = {
       "revnet-38": config_.get_hparams_cifar_38(),
       "revnet-110": config_.get_hparams_cifar_110(),
       "revnet-164": config_.get_hparams_cifar_164(),
-  }[config_name]
+  }[FLAGS.config]
 
-  if dataset == "cifar-10":
-    config.add_hparam("n_classes", 10)
-    config.add_hparam("dataset", "cifar-10")
-  else:
-    config.add_hparam("n_classes", 100)
-    config.add_hparam("dataset", "cifar-100")
+  if FLAGS.dataset == "cifar-100":
+    config.n_classes = 100
 
   return config
 
 
-def get_datasets(data_dir, config):
+def get_datasets(config):
   """Return dataset."""
-  if data_dir is None:
+  if FLAGS.data_dir is None:
     raise ValueError("No supplied data directory")
-  if not os.path.exists(data_dir):
-    raise ValueError("Data directory {} does not exist".format(data_dir))
-  if config.dataset not in ["cifar-10", "cifar-100"]:
-    raise ValueError("Unknown dataset {}".format(config.dataset))
+  if not os.path.exists(FLAGS.data_dir):
+    raise ValueError("Data directory {} does not exist".format(FLAGS.data_dir))
+  if FLAGS.dataset not in ["cifar-10", "cifar-100"]:
+    raise ValueError("Unknown dataset {}".format(FLAGS.dataset))
 
-  print("Training on {} dataset.".format(config.dataset))
+  print("Training on {} dataset.".format(FLAGS.dataset))
   sys.stdout.flush()
-  data_dir = os.path.join(data_dir, config.dataset)
+  data_dir = os.path.join(FLAGS.data_dir, FLAGS.dataset)
   if FLAGS.validate:
     # 40k Training set
     ds_train = cifar_input.get_ds_from_tfrecords(
@@ -166,7 +168,7 @@ def get_datasets(data_dir, config):
         prefetch=config.batch_size)
     ds_validation = None
 
-  # Always compute loss and accuracy on whole test set
+  # Always compute loss and accuracy on whole training and test set
   ds_train_one_shot = cifar_input.get_ds_from_tfrecords(
       data_dir=data_dir,
       split="train_all",
@@ -194,11 +196,19 @@ def get_datasets(data_dir, config):
 
 def train_one_iter(model, inputs, labels, optimizer, global_step=None):
   """Train for one iteration."""
-  grads, vars_, logits, loss = model.compute_gradients(
-      inputs, labels, training=True)
-  optimizer.apply_gradients(zip(grads, vars_), global_step=global_step)
+  if FLAGS.manual_grad:
+    grads, vars_, loss = model.compute_gradients(inputs, labels, training=True)
+    optimizer.apply_gradients(zip(grads, vars_), global_step=global_step)
+  else:  # For correctness validation
+    with tf.GradientTape() as tape:
+      logits, _ = model(inputs, training=True)
+      loss = model.compute_loss(logits=logits, labels=labels)
+      tf.logging.info("Logits are placed on device: {}".format(logits.device))
+    grads = tape.gradient(loss, model.trainable_variables)
+    optimizer.apply_gradients(
+        zip(grads, model.trainable_variables), global_step=global_step)
 
-  return logits, loss
+  return loss.numpy()
 
 
 def evaluate(model, iterator):
@@ -231,14 +241,16 @@ if __name__ == "__main__":
       "validate",
       default=False,
       help="[Optional] Use the validation set or not for hyperparameter search")
+  flags.DEFINE_boolean(
+      "manual_grad",
+      default=False,
+      help="[Optional] Use manual gradient graph to save memory")
   flags.DEFINE_string(
       "dataset",
       default="cifar-10",
       help="[Optional] The dataset used; either `cifar-10` or `cifar-100`")
   flags.DEFINE_string(
-      "config",
-      default="revnet-38",
-      help="[Optional] Architecture of network. "
-      "Other options include `revnet-110` and `revnet-164`")
+      "config", default="revnet-38", help="[Optional] Architecture of network.")
   FLAGS = flags.FLAGS
+  tf.enable_eager_execution()
   tf.app.run(main)
diff --git a/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py b/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py
deleted file mode 100644
index c875e8da6d..0000000000
--- a/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py
+++ /dev/null
@@ -1,200 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Estimator workflow with RevNet train on CIFAR-10."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-from absl import flags
-import tensorflow as tf
-from tensorflow.contrib.eager.python.examples.revnet import cifar_input
-from tensorflow.contrib.eager.python.examples.revnet import main as main_
-from tensorflow.contrib.eager.python.examples.revnet import revnet
-
-
-def model_fn(features, labels, mode, params):
-  """Function specifying the model that is required by the `tf.estimator` API.
-
-  Args:
-    features: Input images
-    labels: Labels of images
-    mode: One of `ModeKeys.TRAIN`, `ModeKeys.EVAL` or 'ModeKeys.PREDICT'
-    params: A dictionary of extra parameter that might be passed
-
-  Returns:
-    An instance of `tf.estimator.EstimatorSpec`
-  """
-
-  inputs = features
-  if isinstance(inputs, dict):
-    inputs = features["image"]
-
-  config = params["config"]
-  model = revnet.RevNet(config=config)
-
-  if mode == tf.estimator.ModeKeys.TRAIN:
-    global_step = tf.train.get_or_create_global_step()
-    learning_rate = tf.train.piecewise_constant(
-        global_step, config.lr_decay_steps, config.lr_list)
-    optimizer = tf.train.MomentumOptimizer(
-        learning_rate, momentum=config.momentum)
-    grads, vars_, logits, loss = model.compute_gradients(
-        inputs, labels, training=True)
-    train_op = optimizer.apply_gradients(
-        zip(grads, vars_), global_step=global_step)
-
-    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
-  else:
-    logits, _ = model(inputs, training=False)
-    predictions = tf.argmax(logits, axis=1)
-    probabilities = tf.nn.softmax(logits)
-    loss = model.compute_loss(labels=labels, logits=logits)
-
-    if mode == tf.estimator.ModeKeys.EVAL:
-      return tf.estimator.EstimatorSpec(
-          mode=mode,
-          loss=loss,
-          eval_metric_ops={
-              "accuracy":
-                  tf.metrics.accuracy(labels=labels, predictions=predictions)
-          })
-
-    else:  # mode == tf.estimator.ModeKeys.PREDICT
-      result = {
-          "classes": predictions,
-          "probabilities": probabilities,
-      }
-
-      return tf.estimator.EstimatorSpec(
-          mode=mode,
-          predictions=predictions,
-          export_outputs={
-              "classify": tf.estimator.export.PredictOutput(result)
-          })
-
-
-def get_input_fn(config, data_dir, split):
-  """Get the input function that is required by the `tf.estimator` API.
-
-  Args:
-    config: Customized hyperparameters
-    data_dir: Directory where the data is stored
-    split: One of `train`, `validation`, `train_all`, and `test`
-
-  Returns:
-    Input function required by the `tf.estimator` API
-  """
-
-  data_dir = os.path.join(data_dir, config.dataset)
-  # Fix split-dependent hyperparameters
-  if split == "train_all" or split == "train":
-    data_aug = True
-    batch_size = config.batch_size
-    epochs = config.epochs
-    shuffle = True
-    prefetch = config.batch_size
-  else:
-    data_aug = False
-    batch_size = config.eval_batch_size
-    epochs = 1
-    shuffle = False
-    prefetch = config.eval_batch_size
-
-  def input_fn():
-    """Input function required by the `tf.estimator.Estimator` API."""
-    return cifar_input.get_ds_from_tfrecords(
-        data_dir=data_dir,
-        split=split,
-        data_aug=data_aug,
-        batch_size=batch_size,
-        epochs=epochs,
-        shuffle=shuffle,
-        prefetch=prefetch,
-        data_format=config.data_format)
-
-  return input_fn
-
-
-def main(argv):
-  FLAGS = argv[0]  # pylint:disable=invalid-name,redefined-outer-name
-  tf.logging.set_verbosity(tf.logging.INFO)
-
-  # RevNet specific configuration
-  config = main_.get_config(config_name=FLAGS.config, dataset=FLAGS.dataset)
-
-  # Estimator specific configuration
-  run_config = tf.estimator.RunConfig(
-      model_dir=FLAGS.train_dir,  # Directory for storing checkpoints
-      tf_random_seed=config.seed,
-      save_summary_steps=config.log_every,
-      save_checkpoints_steps=config.log_every,
-      session_config=None,  # Using default
-      keep_checkpoint_max=100,
-      keep_checkpoint_every_n_hours=10000,  # Using default
-      log_step_count_steps=config.log_every,
-      train_distribute=None  # Default not use distribution strategy
-  )
-
-  # Construct estimator
-  revnet_estimator = tf.estimator.Estimator(
-      model_fn=model_fn,
-      model_dir=FLAGS.train_dir,
-      config=run_config,
-      params={"config": config})
-
-  # Construct input functions
-  train_input_fn = get_input_fn(
-      config=config, data_dir=FLAGS.data_dir, split="train_all")
-  eval_input_fn = get_input_fn(
-      config=config, data_dir=FLAGS.data_dir, split="test")
-
-  # Train and evaluate estimator
-  revnet_estimator.train(input_fn=train_input_fn)
-  revnet_estimator.evaluate(input_fn=eval_input_fn)
-
-  if FLAGS.export:
-    input_shape = (None,) + config.input_shape
-    inputs = tf.placeholder(tf.float32, shape=input_shape)
-    input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn({
-        "image": inputs
-    })
-    revnet_estimator.export_savedmodel(FLAGS.train_dir, input_fn)
-
-
-if __name__ == "__main__":
-  flags.DEFINE_string(
-      "data_dir", default=None, help="Directory to load tfrecords")
-  flags.DEFINE_string(
-      "train_dir",
-      default=None,
-      help="[Optional] Directory to store the training information")
-  flags.DEFINE_string(
-      "dataset",
-      default="cifar-10",
-      help="[Optional] The dataset used; either `cifar-10` or `cifar-100`")
-  flags.DEFINE_boolean(
-      "export",
-      default=False,
-      help="[Optional] Export the model for serving if True")
-  flags.DEFINE_string(
-      "config",
-      default="revnet-38",
-      help="[Optional] Architecture of network. "
-      "Other options include `revnet-110` and `revnet-164`")
-  FLAGS = flags.FLAGS
-  tf.app.run(main=main, argv=[FLAGS])
diff --git a/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py b/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py
deleted file mode 100644
index f1e1e530df..0000000000
--- a/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py
+++ /dev/null
@@ -1,328 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Cloud TPU Estimator workflow with RevNet train on CIFAR-10."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import time
-
-from absl import flags
-import tensorflow as tf
-from tensorflow.contrib.eager.python.examples.revnet import cifar_input
-from tensorflow.contrib.eager.python.examples.revnet import main as main_
-from tensorflow.contrib.eager.python.examples.revnet import revnet
-from tensorflow.contrib.training.python.training import evaluation
-from tensorflow.python.estimator import estimator as estimator_
-
-
-def model_fn(features, labels, mode, params):
-  """Model function required by the `tf.contrib.tpu.TPUEstimator` API.
-
-  Args:
-    features: Input images
-    labels: Labels of images
-    mode: One of `ModeKeys.TRAIN`, `ModeKeys.EVAL` or 'ModeKeys.PREDICT'
-    params: A dictionary of extra parameter that might be passed
-
-  Returns:
-    An instance of `tf.contrib.tpu.TPUEstimatorSpec`
-  """
-
-  inputs = features
-  if isinstance(inputs, dict):
-    inputs = features["image"]
-
-  FLAGS = params["FLAGS"]  # pylint:disable=invalid-name,redefined-outer-name
-  config = params["config"]
-  model = revnet.RevNet(config=config)
-
-  if mode == tf.estimator.ModeKeys.TRAIN:
-    global_step = tf.train.get_or_create_global_step()
-    learning_rate = tf.train.piecewise_constant(
-        global_step, config.lr_decay_steps, config.lr_list)
-    optimizer = tf.train.MomentumOptimizer(
-        learning_rate, momentum=config.momentum)
-
-    if FLAGS.use_tpu:
-      optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)
-
-    # Define gradients
-    grads, vars_, logits, loss = model.compute_gradients(
-        inputs, labels, training=True)
-    train_op = optimizer.apply_gradients(
-        zip(grads, vars_), global_step=global_step)
-
-    names = [v.name for v in model.variables]
-    tf.logging.warn("{}".format(names))
-
-    return tf.contrib.tpu.TPUEstimatorSpec(
-        mode=tf.estimator.ModeKeys.TRAIN, loss=loss, train_op=train_op)
-
-  if mode == tf.estimator.ModeKeys.EVAL:
-    logits, _ = model(inputs, training=False)
-    loss = model.compute_loss(labels=labels, logits=logits)
-
-    def metric_fn(labels, logits):
-      predictions = tf.argmax(logits, axis=1)
-      accuracy = tf.metrics.accuracy(labels=labels, predictions=predictions)
-      return {
-          "accuracy": accuracy,
-      }
-
-    return tf.contrib.tpu.TPUEstimatorSpec(
-        mode=mode, loss=loss, eval_metrics=(metric_fn, [labels, logits]))
-
-  if mode == tf.estimator.ModeKeys.PREDICT:
-    logits, _ = model(inputs, training=False)
-    predictions = {
-        "classes": tf.argmax(logits, axis=1),
-        "probabilities": tf.nn.softmax(logits),
-    }
-
-    return tf.contrib.tpu.TPUEstimatorSpec(
-        mode=mode,
-        predictions=predictions,
-        export_outputs={
-            "classify": tf.estimator.export.PredictOutput(predictions)
-        })
-
-
-def get_input_fn(config, data_dir, split):
-  """Get the input function required by the `tf.contrib.tpu.TPUEstimator` API.
-
-  Args:
-    config: Customized hyperparameters
-    data_dir: Directory where the data is stored
-    split: One of `train`, `validation`, `train_all`, and `test`
-
-  Returns:
-    Input function required by the `tf.contrib.tpu.TPUEstimator` API
-  """
-
-  data_dir = os.path.join(data_dir, config.dataset)
-  # Fix split-dependent hyperparameters
-  if split == "train_all" or split == "train":
-    data_aug = True
-    epochs = config.tpu_epochs
-    shuffle = True
-  else:
-    data_aug = False
-    epochs = 1
-    shuffle = False
-
-  def input_fn(params):
-    """Input function required by the `tf.contrib.tpu.TPUEstimator` API."""
-    batch_size = params["batch_size"]
-    return cifar_input.get_ds_from_tfrecords(
-        data_dir=data_dir,
-        split=split,
-        data_aug=data_aug,
-        batch_size=batch_size,  # per-shard batch size
-        epochs=epochs,
-        shuffle=shuffle,
-        prefetch=batch_size,  # per-shard batch size
-        data_format=config.data_format)
-
-  return input_fn
-
-
-def main(argv):
-  FLAGS = argv[0]  # pylint:disable=invalid-name,redefined-outer-name
-  tf.logging.set_verbosity(tf.logging.INFO)
-
-  # RevNet specific configuration
-  config = main_.get_config(config_name=FLAGS.config, dataset=FLAGS.dataset)
-
-  if FLAGS.use_tpu:
-    tf.logging.info("Using TPU.")
-    tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
-        FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
-  else:
-    tpu_cluster_resolver = None
-
-  # TPU specific configuration
-  tpu_config = tf.contrib.tpu.TPUConfig(
-      # Recommended to be set as number of global steps for next checkpoint
-      iterations_per_loop=FLAGS.iterations_per_loop,
-      num_shards=FLAGS.num_shards)
-
-  # Estimator specific configuration
-  run_config = tf.contrib.tpu.RunConfig(
-      cluster=tpu_cluster_resolver,
-      model_dir=FLAGS.model_dir,
-      session_config=tf.ConfigProto(
-          allow_soft_placement=True, log_device_placement=False),
-      tpu_config=tpu_config,
-  )
-
-  # Construct TPU Estimator
-  estimator = tf.contrib.tpu.TPUEstimator(
-      model_fn=model_fn,
-      use_tpu=FLAGS.use_tpu,
-      train_batch_size=config.tpu_batch_size,
-      eval_batch_size=config.tpu_eval_batch_size,
-      config=run_config,
-      params={
-          "FLAGS": FLAGS,
-          "config": config,
-      })
-
-  # Construct input functions
-  train_input_fn = get_input_fn(
-      config=config, data_dir=FLAGS.data_dir, split="train_all")
-  eval_input_fn = get_input_fn(
-      config=config, data_dir=FLAGS.data_dir, split="test")
-
-  # Disabling a range within an else block currently doesn't work
-  # due to https://github.com/PyCQA/pylint/issues/872
-  # pylint: disable=protected-access
-  if FLAGS.mode == "eval":
-    # TPUEstimator.evaluate *requires* a steps argument.
-    # Note that the number of examples used during evaluation is
-    # --eval_steps * --batch_size.
-    # So if you change --batch_size then change --eval_steps too.
-    eval_steps = 10000 // config.tpu_eval_batch_size
-
-    # Run evaluation when there's a new checkpoint
-    for ckpt in evaluation.checkpoints_iterator(
-        FLAGS.model_dir, timeout=FLAGS.eval_timeout):
-      tf.logging.info("Starting to evaluate.")
-      try:
-        start_timestamp = time.time()  # This time will include compilation time
-        eval_results = estimator.evaluate(
-            input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=ckpt)
-        elapsed_time = int(time.time() - start_timestamp)
-        tf.logging.info("Eval results: %s. Elapsed seconds: %d" %
-                        (eval_results, elapsed_time))
-
-        # Terminate eval job when final checkpoint is reached
-        current_step = int(os.path.basename(ckpt).split("-")[1])
-        if current_step >= config.max_train_iter:
-          tf.logging.info(
-              "Evaluation finished after training step %d" % current_step)
-          break
-
-      except tf.errors.NotFoundError:
-        # Since the coordinator is on a different job than the TPU worker,
-        # sometimes the TPU worker does not finish initializing until long after
-        # the CPU job tells it to start evaluating. In this case, the checkpoint
-        # file could have been deleted already.
-        tf.logging.info(
-            "Checkpoint %s no longer exists, skipping checkpoint" % ckpt)
-
-  else:  # FLAGS.mode == 'train' or FLAGS.mode == 'train_and_eval'
-    current_step = estimator_._load_global_step_from_checkpoint_dir(
-        FLAGS.model_dir)
-    tf.logging.info("Training for %d steps . Current"
-                    " step %d." % (config.max_train_iter, current_step))
-
-    start_timestamp = time.time()  # This time will include compilation time
-    if FLAGS.mode == "train":
-      estimator.train(input_fn=train_input_fn, max_steps=config.max_train_iter)
-    else:
-      eval_steps = 10000 // config.tpu_eval_batch_size
-      assert FLAGS.mode == "train_and_eval"
-      while current_step < config.max_train_iter:
-        # Train for up to steps_per_eval number of steps.
-        # At the end of training, a checkpoint will be written to --model_dir.
-        next_checkpoint = min(current_step + FLAGS.steps_per_eval,
-                              config.max_train_iter)
-        estimator.train(input_fn=train_input_fn, max_steps=next_checkpoint)
-        current_step = next_checkpoint
-
-        # Evaluate the model on the most recent model in --model_dir.
-        # Since evaluation happens in batches of --eval_batch_size, some images
-        # may be consistently excluded modulo the batch size.
-        tf.logging.info("Starting to evaluate.")
-        eval_results = estimator.evaluate(
-            input_fn=eval_input_fn, steps=eval_steps)
-        tf.logging.info("Eval results: %s" % eval_results)
-
-    elapsed_time = int(time.time() - start_timestamp)
-    tf.logging.info("Finished training up to step %d. Elapsed seconds %d." %
-                    (config.max_train_iter, elapsed_time))
-  # pylint: enable=protected-access
-
-
-if __name__ == "__main__":
-  # Cloud TPU Cluster Resolver flags
-  flags.DEFINE_string(
-      "tpu",
-      default=None,
-      help="The Cloud TPU to use for training. This should be either the name "
-      "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 "
-      "url.")
-  flags.DEFINE_string(
-      "tpu_zone",
-      default=None,
-      help="[Optional] GCE zone where the Cloud TPU is located in. If not "
-      "specified, we will attempt to automatically detect the GCE project from "
-      "metadata.")
-  flags.DEFINE_string(
-      "gcp_project",
-      default=None,
-      help="[Optional] Project name for the Cloud TPU-enabled project. If not "
-      "specified, we will attempt to automatically detect the GCE project from "
-      "metadata.")
-
-  # Model specific parameters
-  flags.DEFINE_string(
-      "data_dir", default=None, help="Directory to load tfrecords")
-  flags.DEFINE_string(
-      "model_dir",
-      default=None,
-      help="[Optional] Directory to store the model information")
-  flags.DEFINE_string(
-      "dataset",
-      default="cifar-10",
-      help="[Optional] The dataset used; either `cifar-10` or `cifar-100`")
-  flags.DEFINE_string(
-      "config",
-      default="revnet-38",
-      help="[Optional] Architecture of network. "
-      "Other options include `revnet-110` and `revnet-164`")
-  flags.DEFINE_boolean(
-      "use_tpu", default=True, help="[Optional] Whether to use TPU")
-  flags.DEFINE_integer(
-      "num_shards", default=8, help="Number of shards (TPU chips).")
-  flags.DEFINE_integer(
-      "iterations_per_loop",
-      default=100,
-      help=(
-          "Number of steps to run on TPU before feeding metrics to the CPU."
-          " If the number of iterations in the loop would exceed the number of"
-          " train steps, the loop will exit before reaching"
-          " --iterations_per_loop. The larger this value is, the higher the"
-          " utilization on the TPU."))
-  flags.DEFINE_string(
-      "mode",
-      default="train_and_eval",
-      help="[Optional] Mode to run: train, eval, train_and_eval")
-  flags.DEFINE_integer(
-      "eval_timeout", 60 * 60 * 24,
-      "Maximum seconds between checkpoints before evaluation terminates.")
-  flags.DEFINE_integer(
-      "steps_per_eval",
-      default=1000,
-      help=(
-          "Controls how often evaluation is performed. Since evaluation is"
-          " fairly expensive, it is advised to evaluate as infrequently as"
-          " possible (i.e. up to --train_steps, which evaluates the model only"
-          " after finishing the entire training regime)."))
-  FLAGS = flags.FLAGS
-  tf.app.run(main=main, argv=[FLAGS])
diff --git a/tensorflow/contrib/eager/python/examples/revnet/revnet.py b/tensorflow/contrib/eager/python/examples/revnet/revnet.py
index a3c2f7dbec..af0d20fa72 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/revnet.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/revnet.py
@@ -24,6 +24,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+import operator
+
 import six
 import tensorflow as tf
 from tensorflow.contrib.eager.python.examples.revnet import blocks
@@ -42,9 +45,71 @@ class RevNet(tf.keras.Model):
     self.axis = 1 if config.data_format == "channels_first" else 3
     self.config = config
 
-    self._init_block = blocks.InitBlock(config=self.config)
-    self._final_block = blocks.FinalBlock(config=self.config)
+    self._init_block = self._construct_init_block()
     self._block_list = self._construct_intermediate_blocks()
+    self._final_block = self._construct_final_block()
+
+  def _construct_init_block(self):
+    init_block = tf.keras.Sequential(
+        [
+            tf.keras.layers.Conv2D(
+                filters=self.config.init_filters,
+                kernel_size=self.config.init_kernel,
+                strides=(self.config.init_stride, self.config.init_stride),
+                data_format=self.config.data_format,
+                use_bias=False,
+                padding="SAME",
+                input_shape=self.config.input_shape,
+                dtype=self.config.dtype),
+            tf.keras.layers.BatchNormalization(
+                axis=self.axis,
+                fused=self.config.fused,
+                dtype=self.config.dtype),
+            tf.keras.layers.Activation("relu"),
+        ],
+        name="init")
+    if self.config.init_max_pool:
+      init_block.add(
+          tf.keras.layers.MaxPooling2D(
+              pool_size=(3, 3),
+              strides=(2, 2),
+              padding="SAME",
+              data_format=self.config.data_format,
+              dtype=self.config.dtype))
+    return init_block
+
+  def _construct_final_block(self):
+    f = self.config.filters[-1]  # Number of filters
+    r = functools.reduce(operator.mul, self.config.strides, 1)  # Reduce ratio
+    r *= self.config.init_stride
+    if self.config.init_max_pool:
+      r *= 2
+
+    if self.config.data_format == "channels_first":
+      w, h = self.config.input_shape[1], self.config.input_shape[2]
+      input_shape = (f, w // r, h // r)
+    elif self.config.data_format == "channels_last":
+      w, h = self.config.input_shape[0], self.config.input_shape[1]
+      input_shape = (w // r, h // r, f)
+    else:
+      raise ValueError("Data format should be either `channels_first`"
+                       " or `channels_last`")
+
+    final_block = tf.keras.Sequential(
+        [
+            tf.keras.layers.BatchNormalization(
+                axis=self.axis,
+                input_shape=input_shape,
+                fused=self.config.fused,
+                dtype=self.config.dtype),
+            tf.keras.layers.Activation("relu"),
+            tf.keras.layers.GlobalAveragePooling2D(
+                data_format=self.config.data_format, dtype=self.config.dtype),
+            tf.keras.layers.Dense(
+                self.config.n_classes, dtype=self.config.dtype)
+        ],
+        name="final")
+    return final_block
 
   def _construct_intermediate_blocks(self):
     # Precompute input shape after initial block
@@ -141,20 +206,13 @@ class RevNet(tf.keras.Model):
       l2_reg: Apply l2 regularization
 
     Returns:
-      A tuple with the first entry being a list of all gradients, the second
-      entry being a list of respective variables, the third being the logits,
-      and the forth being the loss
+      list of tuples each being (grad, var) for optimizer to use
     """
 
-    # Run forward pass to record hidden states
+    # Run forward pass to record hidden states; avoid updating running averages
     vars_and_vals = self.get_moving_stats()
     _, saved_hidden = self.call(inputs, training=training)
-    if tf.executing_eagerly():
-      # Restore moving averages when executing eagerly to avoid updating twice
-      self.restore_moving_stats(vars_and_vals)
-    else:
-      # Fetch batch norm updates in graph mode
-      updates = self.get_updates_for(inputs)
+    self.restore_moving_stats(vars_and_vals)
 
     grads_all = []
     vars_all = []
@@ -162,8 +220,9 @@ class RevNet(tf.keras.Model):
     # Manually backprop through last block
     x = saved_hidden[-1]
     with tf.GradientTape() as tape:
+      x = tf.identity(x)
       tape.watch(x)
-      # Running stats updated here
+      # Running stats updated below
       logits = self._final_block(x, training=training)
       loss = self.compute_loss(logits, labels)
 
@@ -177,7 +236,6 @@ class RevNet(tf.keras.Model):
     for block in reversed(self._block_list):
       y = saved_hidden.pop()
       x = saved_hidden[-1]
-      # Running stats updated here
       dy, grads, vars_ = block.backward_grads_and_vars(
           x, y, dy, training=training)
       grads_all += grads
@@ -189,7 +247,8 @@ class RevNet(tf.keras.Model):
     assert not saved_hidden  # Cleared after backprop
 
     with tf.GradientTape() as tape:
-      # Running stats updated here
+      x = tf.identity(x)
+      # Running stats updated below
       y = self._init_block(x, training=training)
 
     grads_all += tape.gradient(
@@ -200,13 +259,7 @@ class RevNet(tf.keras.Model):
     if l2_reg:
       grads_all = self._apply_weight_decay(grads_all, vars_all)
 
-    if not tf.executing_eagerly():
-      # Force updates to be executed before gradient computation in graph mode
-      # This does nothing when the function is wrapped in defun
-      with tf.control_dependencies(updates):
-        grads_all[0] = tf.identity(grads_all[0])
-
-    return grads_all, vars_all, logits, loss
+    return grads_all, vars_all, loss
 
   def _apply_weight_decay(self, grads, vars_):
     """Update gradients to reflect weight decay."""
@@ -231,10 +284,8 @@ class RevNet(tf.keras.Model):
       n = v.name
       return n.endswith("moving_mean:0") or n.endswith("moving_variance:0")
 
-    device = "/gpu:0" if tf.test.is_gpu_available() else "/cpu:0"
-    with tf.device(device):
-      for v in filter(_is_moving_var, self.variables):
-        vars_and_vals[v] = v.read_value()
+    for v in filter(_is_moving_var, self.variables):
+      vars_and_vals[v] = v.read_value()
 
     return vars_and_vals
 
@@ -246,8 +297,5 @@ class RevNet(tf.keras.Model):
     Args:
       vars_and_vals: The dictionary mapping variables to their previous values.
     """
-    device = "/gpu:0" if tf.test.is_gpu_available() else "/cpu:0"
-    with tf.device(device):
-      for var_, val in six.iteritems(vars_and_vals):
-        # `assign` causes a copy to GPU (if variable is already on GPU)
-        var_.assign(val)
+    for var_, val in six.iteritems(vars_and_vals):
+      var_.assign(val)
diff --git a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
index 26b0847523..b0d0a5486d 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
@@ -31,11 +31,10 @@ tfe = tf.contrib.eager
 
 def train_one_iter(model, inputs, labels, optimizer, global_step=None):
   """Train for one iteration."""
-  grads, vars_, logits, loss = model.compute_gradients(
-      inputs, labels, training=True)
+  grads, vars_, loss = model.compute_gradients(inputs, labels, training=True)
   optimizer.apply_gradients(zip(grads, vars_), global_step=global_step)
 
-  return logits, loss
+  return loss
 
 
 class RevNetTest(tf.test.TestCase):
@@ -43,8 +42,6 @@ class RevNetTest(tf.test.TestCase):
   def setUp(self):
     super(RevNetTest, self).setUp()
     config = config_.get_hparams_cifar_38()
-    config.add_hparam("n_classes", 10)
-    config.add_hparam("dataset", "cifar-10")
     # Reconstruction could cause numerical error, use double precision for tests
     config.dtype = tf.float64
     config.fused = False  # Fused batch norm does not support tf.float64
@@ -97,7 +94,7 @@ class RevNetTest(tf.test.TestCase):
   def test_compute_gradients(self):
     """Test `compute_gradients` function."""
     self.model(self.x, training=False)  # Initialize model
-    grads, vars_, logits, loss = self.model.compute_gradients(
+    grads, vars_, loss = self.model.compute_gradients(
         inputs=self.x, labels=self.t, training=True, l2_reg=True)
     self.assertTrue(isinstance(grads, list))
     self.assertTrue(isinstance(vars_, list))
@@ -122,7 +119,7 @@ class RevNetTest(tf.test.TestCase):
   def test_compute_gradients_defun(self):
     """Test `compute_gradients` function with defun."""
     compute_gradients = tfe.defun(self.model.compute_gradients)
-    grads, vars_, _, _ = compute_gradients(self.x, self.t, training=True)
+    grads, vars_, _ = compute_gradients(self.x, self.t, training=True)
     self.assertTrue(isinstance(grads, list))
     self.assertTrue(isinstance(vars_, list))
     self.assertEqual(len(grads), len(vars_))
@@ -134,9 +131,6 @@ class RevNetTest(tf.test.TestCase):
     """Test model training in graph mode."""
     with tf.Graph().as_default():
       config = config_.get_hparams_cifar_38()
-      config.add_hparam("n_classes", 10)
-      config.add_hparam("dataset", "cifar-10")
-
       x = tf.random_normal(
           shape=(self.config.batch_size,) + self.config.input_shape)
       t = tf.random_uniform(
@@ -146,10 +140,15 @@ class RevNetTest(tf.test.TestCase):
           dtype=tf.int32)
       global_step = tf.Variable(0., trainable=False)
       model = revnet.RevNet(config=config)
-      grads_all, vars_all, _, _ = model.compute_gradients(x, t, training=True)
+      model(x)
+      updates = model.get_updates_for(x)
+
+      x_ = tf.identity(x)
+      grads_all, vars_all, _ = model.compute_gradients(x_, t, training=True)
       optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
-      train_op = optimizer.apply_gradients(
-          zip(grads_all, vars_all), global_step=global_step)
+      with tf.control_dependencies(updates):
+        train_op = optimizer.apply_gradients(
+            zip(grads_all, vars_all), global_step=global_step)
 
       with tf.Session() as sess:
         sess.run(tf.global_variables_initializer())
-- 
cgit v1.2.3


From 9dbbea7a4327cd694ccff1f9edf6ae1af4329362 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 16:59:11 -0700
Subject: Add gpu_event_mgr::WarnIfInCallback() analysis function, included by
 compiler option only.  Useful for tracking down performance problems caused
 by misuse of EventMgr::ThenExecute callbacks.

PiperOrigin-RevId: 205475177
---
 tensorflow/core/BUILD                              | 18 +++++-
 .../core/common_runtime/gpu/gpu_event_mgr.cc       | 74 +++++++++++++++++++++-
 tensorflow/core/common_runtime/gpu/gpu_event_mgr.h | 22 +++++++
 .../core/common_runtime/gpu/gpu_event_mgr_test.cc  | 23 +++++++
 4 files changed, 133 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 870bde7bc8..17e6ccda14 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3746,7 +3746,6 @@ tf_cc_tests_gpu(
         "common_runtime/gpu/gpu_bfc_allocator_test.cc",
         "common_runtime/gpu/gpu_device_test.cc",
         "common_runtime/gpu/gpu_id_manager_test.cc",
-        "common_runtime/gpu/gpu_event_mgr_test.cc",
         "common_runtime/gpu/pool_allocator_test.cc",
     ],
     linkstatic = tf_kernel_tests_linkstatic(),
@@ -3770,6 +3769,23 @@ tf_cc_tests_gpu(
     ],
 )
 
+tf_cc_test_gpu(
+    name = "gpu_event_mgr_test",
+    srcs = ["common_runtime/gpu/gpu_event_mgr_test.cc"],
+    linkstatic = tf_kernel_tests_linkstatic(),
+    tags = tf_cuda_tests_tags(),
+    deps = [
+        ":framework",
+        ":framework_internal",
+        ":lib",
+        ":lib_internal",
+        ":protos_all_cc",
+        ":test",
+        ":test_main",
+        ":testlib",
+    ],
+)
+
 tf_cuda_cc_test(
     name = "gpu_device_unified_memory_test",
     size = "small",
diff --git a/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc b/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc
index 4898448476..3c1c31aa73 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc
@@ -15,11 +15,80 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 
+#include "tensorflow/core/platform/stacktrace.h"
 #include "tensorflow/core/platform/stream_executor.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
 namespace tensorflow {
 
+namespace {
+// The EventMgr has 1 thread for the polling loop and one to execute
+// event callback functions. Issues for reconsideration:
+//  - Is this the right number of threads?
+//  - Should EventMgrs be shared between GPUDevices on a multi-GPU machine?
+static const int kNumThreads = 2;
+}  // namespace
+
+namespace gpu_event_mgr {
+class ThreadLabel {
+ public:
+  static const char* GetValue() { return value_; }
+
+  // v must be a static const because value_ will capture and use its value
+  // until reset or thread terminates.
+  static void SetValue(const char* v) { value_ = v; }
+
+ private:
+  static thread_local const char* value_;
+};
+thread_local const char* ThreadLabel::value_ = "";
+
+void WarnIfInCallback(std::function f) {
+  const char* label = ThreadLabel::GetValue();
+  if (label && !strcmp(label, "gpu_event_mgr")) {
+    if (f) {
+      f();
+    } else {
+      LOG(WARNING) << "Executing inside EventMgr callback thread: "
+                   << CurrentStackTrace();
+    }
+  }
+}
+
+void InitThreadpoolLabels(thread::ThreadPool* threadpool) {
+  static const char* label = "gpu_event_mgr";
+  mutex mu;
+  int init_count = 0;
+  condition_variable all_initialized;
+  int exit_count = 0;
+  condition_variable ready_to_exit;
+  const int num_threads = threadpool->NumThreads();
+  for (int i = 0; i < num_threads; ++i) {
+    threadpool->Schedule([num_threads, &mu, &init_count, &all_initialized,
+                          &exit_count, &ready_to_exit]() {
+      gpu_event_mgr::ThreadLabel::SetValue(label);
+      mutex_lock l(mu);
+      ++init_count;
+      if (init_count == num_threads) {
+        all_initialized.notify_all();
+      }
+      while (init_count < num_threads) {
+        all_initialized.wait(l);
+      }
+      if (++exit_count == num_threads) {
+        ready_to_exit.notify_all();
+      }
+    });
+  }
+  {
+    mutex_lock l(mu);
+    while (exit_count < num_threads) {
+      ready_to_exit.wait(l);
+    }
+  }
+}
+}  // namespace gpu_event_mgr
+
 EventMgr::EventMgr(se::StreamExecutor* se, const GPUOptions& gpu_options)
     : exec_(se),
       deferred_bytes_threshold_(gpu_options.deferred_deletion_bytes()
@@ -31,9 +100,8 @@ EventMgr::EventMgr(se::StreamExecutor* se, const GPUOptions& gpu_options)
       accumulated_stream_(nullptr),
       accumulated_tensors_(new TensorReferenceVector),
       accumulated_tensor_bytes_(0),
-      // threadpool_ has 1 thread for the polling loop, and one to execute
-      // event callback functions. Maybe we should have more?
-      threadpool_(Env::Default(), "GPU_Event_Manager", 2) {
+      threadpool_(Env::Default(), "GPU_Event_Manager", kNumThreads) {
+  gpu_event_mgr::InitThreadpoolLabels(&threadpool_);
   StartPollingLoop();
 }
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_event_mgr.h b/tensorflow/core/common_runtime/gpu/gpu_event_mgr.h
index b26f88a201..f0a109cc10 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_event_mgr.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_event_mgr.h
@@ -39,6 +39,25 @@ namespace tensorflow {
 
 class GPUOptions;
 
+// The callback provided to EventMgr::ThenExecute must not block or take a long
+// time.  If it does, performance may be impacted and GPU memory may be
+// exhausted.  This macro is for checking that an EventMgr thread is not
+// accidentally entering blocking parts of the code, e.g. the RPC subsystem.
+//
+// Intended use is something like
+//
+//   void RespondToAnRPC(Params* params) {
+//      WARN_IF_IN_EVENT_MGR_THREAD;
+//      if (params->status.ok()) { ...
+//
+namespace gpu_event_mgr {
+// Logs a stack trace if current execution thread belongs to this EventMgr
+// object.  If f is not nullptr, executes instead of  logging the stack trace.
+// trace.
+void WarnIfInCallback(std::function f);
+}  // namespace gpu_event_mgr
+#define WARN_IF_IN_EVENT_MGR_THREAD gpu_event_mgr::WarnIfInCallback(nullptr)
+
 // An object to keep track of pending Events in the StreamExecutor streams
 // and associated Tensors that cannot safely be deleted until the associated
 // Events are recorded.
@@ -74,6 +93,9 @@ class EventMgr {
     FreeMemory(to_free);
   }
 
+  // Execute func when all pending stream actions have completed.
+  // func must be brief and non-blocking since it executes in the one
+  // thread used for all such callbacks and also buffer deletions.
   inline void ThenExecute(se::Stream* stream, std::function func) {
     ToFreeVector to_free;
     {
diff --git a/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc b/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
index c5ff6c97a1..d2adf699f5 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_event_mgr_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include 
 #include "tensorflow/core/common_runtime/gpu/gpu_init.h"
+#include "tensorflow/core/lib/core/notification.h"
 #include "tensorflow/core/platform/stream_executor.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/protobuf/config.pb.h"
@@ -243,6 +244,28 @@ TEST(EventMgr, NonEmptyShutdown) {
   }
 }
 
+// Tests that WarnIfInCallback() triggers correctly.
+TEST(EventMgr, WarnIfInCallback) {
+  auto stream_exec = GPUMachineManager()->ExecutorForDevice(0).ValueOrDie();
+  EventMgr em(stream_exec, GPUOptions());
+  TEST_EventMgrHelper th(&em);
+  std::unique_ptr stream(new se::Stream(stream_exec));
+  CHECK(stream);
+  stream->Init();
+  bool hit = false;
+  gpu_event_mgr::WarnIfInCallback([&hit] { hit = true; });
+  EXPECT_FALSE(hit);
+  Notification note;
+  em.ThenExecute(stream.get(), [&hit, ¬e]() {
+    gpu_event_mgr::WarnIfInCallback([&hit, ¬e] {
+      hit = true;
+      note.Notify();
+    });
+  });
+  note.WaitForNotification();
+  EXPECT_TRUE(hit);
+}
+
 }  // namespace
 }  // namespace tensorflow
 
-- 
cgit v1.2.3


From c66cc3cf0d6a752430f37b8798a8adaa973875d0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 17:10:14 -0700
Subject: build_tf_windows.sh: Add --test_target option to control which tests
 to run

PiperOrigin-RevId: 205476590
---
 .../ci_build/windows/cpu/pip/build_tf_windows.sh   | 37 ++++++++++++++--------
 .../ci_build/windows/gpu/pip/build_tf_windows.sh   | 37 ++++++++++++++--------
 2 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 42f58deb42..22d389fd44 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -53,20 +53,31 @@ function cleanup {
 }
 trap cleanup EXIT
 
-skip_test=0
-release_build=0
+PY_TEST_DIR="py_test_dir"
 
+SKIP_TEST=0
+RELEASE_BUILD=0
+TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/... "
+          + "//${PY_TEST_DIR}tensorflow/contrib/... "
+
+# --skip_test            Skip running tests
+# --enable_remote_cache  Add options to enable remote cache for build and test
+# --release_build        Build for release, compilation time will be longer to
+#                        ensure performance
+# --test_core_only       Use tensorflow/python/... as test target
+# --test_contrib_only    Use tensorflow/contrib/... as test target
 for ARG in "$@"; do
-  if [[ "$ARG" == --skip_test ]]; then
-    skip_test=1
-  elif [[ "$ARG" == --enable_remote_cache ]]; then
-    set_remote_cache_options
-  elif [[ "$ARG" == --release_build ]]; then
-    release_build=1
-  fi
+  case "$ARG" in
+    --skip_test) SKIP_TEST=1 ;;
+    --enable_remote_cache) set_remote_cache_options ;;
+    --release_build) RELEASE_BUILD=1 ;;
+    --test_core_only) TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/..." ;;
+    --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}tensorflow/contrib/..." ;;
+    *)
+  esac
 done
 
-if [[ "$release_build" == 1 ]]; then
+if [[ "$RELEASE_BUILD" == 1 ]]; then
   # Overriding eigen strong inline speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc
   # by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521
   # Because this hurts the performance of TF, we don't override it in release build.
@@ -86,12 +97,11 @@ run_configure_for_cpu_build
 
 bazel build --announce_rc --config=opt tensorflow/tools/pip_package:build_pip_package || exit $?
 
-if [[ "$skip_test" == 1 ]]; then
+if [[ "$SKIP_TEST" == 1 ]]; then
   exit 0
 fi
 
 # Create a python test directory to avoid package name conflict
-PY_TEST_DIR="py_test_dir"
 create_python_test_dir "${PY_TEST_DIR}"
 
 ./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}"
@@ -112,5 +122,4 @@ bazel test --announce_rc --config=opt -k --test_output=errors \
   --test_size_filters=small,medium \
   --jobs="${N_JOBS}" --test_timeout="300,450,1200,3600" \
   --flaky_test_attempts=3 \
-  //${PY_TEST_DIR}/tensorflow/python/... \
-  //${PY_TEST_DIR}/tensorflow/contrib/...
+  ${TEST_TARGET}
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index 2a8c2d9167..682a396d10 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -53,20 +53,31 @@ function cleanup {
 }
 trap cleanup EXIT
 
-skip_test=0
-release_build=0
+PY_TEST_DIR="py_test_dir"
 
+SKIP_TEST=0
+RELEASE_BUILD=0
+TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/... "
+          + "//${PY_TEST_DIR}tensorflow/contrib/... "
+
+# --skip_test            Skip running tests
+# --enable_remote_cache  Add options to enable remote cache for build and test
+# --release_build        Build for release, compilation time will be longer to
+#                        ensure performance
+# --test_core_only       Use tensorflow/python/... as test target
+# --test_contrib_only    Use tensorflow/contrib/... as test target
 for ARG in "$@"; do
-  if [[ "$ARG" == --skip_test ]]; then
-    skip_test=1
-  elif [[ "$ARG" == --enable_remote_cache ]]; then
-    set_remote_cache_options
-  elif [[ "$ARG" == --release_build ]]; then
-    release_build=1
-  fi
+  case "$ARG" in
+    --skip_test) SKIP_TEST=1 ;;
+    --enable_remote_cache) set_remote_cache_options ;;
+    --release_build) RELEASE_BUILD=1 ;;
+    --test_core_only) TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/..." ;;
+    --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}tensorflow/contrib/..." ;;
+    *)
+  esac
 done
 
-if [[ "$release_build" == 1 ]]; then
+if [[ "$RELEASE_BUILD" == 1 ]]; then
   # Overriding eigen strong inline speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc
   # by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521
   # Because this hurts the performance of TF, we don't override it in release build.
@@ -89,12 +100,11 @@ run_configure_for_gpu_build
 
 bazel build --announce_rc --config=opt tensorflow/tools/pip_package:build_pip_package || exit $?
 
-if [[ "$skip_test" == 1 ]]; then
+if [[ "$SKIP_TEST" == 1 ]]; then
   exit 0
 fi
 
 # Create a python test directory to avoid package name conflict
-PY_TEST_DIR="py_test_dir"
 create_python_test_dir "${PY_TEST_DIR}"
 
 ./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}"
@@ -117,5 +127,4 @@ bazel test --announce_rc --config=opt -k --test_output=errors \
   --test_size_filters=small,medium \
   --local_test_jobs=$TF_GPU_COUNT --test_timeout="300,450,1200,3600" \
   --flaky_test_attempts=3 \
-  //${PY_TEST_DIR}/tensorflow/python/... \
-  //${PY_TEST_DIR}/tensorflow/contrib/...
+  ${TEST_TARGET}
-- 
cgit v1.2.3


From 57e5dfa76a32ff0ee6ec4b72a2461487b7969a3e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 17:10:38 -0700
Subject: Avoid argmax/argmin divide by 0 when the output tensor is an empty
 tensor.

PiperOrigin-RevId: 205476629
---
 tensorflow/core/kernels/argmax_op.cc             | 4 ++++
 tensorflow/python/kernel_tests/argmax_op_test.py | 7 +++++++
 2 files changed, 11 insertions(+)

diff --git a/tensorflow/core/kernels/argmax_op.cc b/tensorflow/core/kernels/argmax_op.cc
index adc573e40c..c731b64993 100644
--- a/tensorflow/core/kernels/argmax_op.cc
+++ b/tensorflow/core/kernels/argmax_op.cc
@@ -76,6 +76,10 @@ class ArgOp : public OpKernel {
     Tensor* output = nullptr;
     OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output));
 
+    if (output_shape.num_elements() == 0) {
+      return;
+    }
+
 #define HANDLE_DIM(NDIM)                                        \
   case NDIM:                                                    \
     ArgFunctor::Reduce##NDIM(context->eigen_device(),   \
diff --git a/tensorflow/python/kernel_tests/argmax_op_test.py b/tensorflow/python/kernel_tests/argmax_op_test.py
index ce06769902..1202c463e8 100644
--- a/tensorflow/python/kernel_tests/argmax_op_test.py
+++ b/tensorflow/python/kernel_tests/argmax_op_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -115,6 +116,12 @@ class ArgMaxTest(test.TestCase):
         ans = op([1]).eval()
         self.assertAllEqual(ans, 0)
 
+  def testOutputEmpty(self):
+    with self.test_session():
+      for op in math_ops.argmin, math_ops.argmax:
+        ret = op(array_ops.zeros(shape=[1, 0, 2]), axis=-1).eval()
+        self.assertEqual(ret.shape, (1, 0))
+
 
 if __name__ == "__main__":
   test.main()
-- 
cgit v1.2.3


From b840e5ac84319e6e091a0f9351b7691390275f2f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 17:44:31 -0700
Subject: [XLA] add BitcastConvertType to local Python client

PiperOrigin-RevId: 205479860
---
 .../xla/python/local_computation_builder.cc        |  5 ++++
 .../xla/python/local_computation_builder.h         |  3 +++
 .../xla/python/local_computation_builder.i         |  1 +
 tensorflow/compiler/xla/python/xla_client.py       | 12 ++++++++++
 tensorflow/compiler/xla/python/xla_client_test.py  | 28 ++++++++++++++++++++++
 5 files changed, 49 insertions(+)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index f25348e735..8aefc4cd5e 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -486,6 +486,11 @@ LocalOp LocalComputationBuilder::ConvertElementType(
   return xla::ConvertElementType(operand.op(), new_element_type);
 }
 
+LocalOp LocalComputationBuilder::BitcastConvertType(
+    const LocalOp& operand, PrimitiveType new_element_type) {
+  return xla::BitcastConvertType(operand.op(), new_element_type);
+}
+
 LocalOp LocalComputationBuilder::Call(
     const LocalComputation& local_computation,
     tensorflow::gtl::ArraySlice operands) {
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index 0e0d8ac29a..dd9e2fbe72 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -259,6 +259,9 @@ class LocalComputationBuilder {
   LocalOp ConvertElementType(const LocalOp& operand,
                              PrimitiveType new_element_type);
 
+  LocalOp BitcastConvertType(const LocalOp& operand,
+                             PrimitiveType new_element_type);
+
   LocalOp Call(const LocalComputation& local_computation,
                tensorflow::gtl::ArraySlice operands);
 
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index eeccbd7cfa..9b8b0aa7f2 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -957,6 +957,7 @@ tensorflow::ImportNumpy();
 %unignore xla::swig::LocalComputationBuilder::Tuple;
 %unignore xla::swig::LocalComputationBuilder::GetTupleElement;
 %unignore xla::swig::LocalComputationBuilder::ConvertElementType;
+%unignore xla::swig::LocalComputationBuilder::BitcastConvertType;
 %unignore xla::swig::LocalComputationBuilder::Call;
 %unignore xla::swig::LocalComputationBuilder::Transpose;
 %unignore xla::swig::LocalComputationBuilder::Rev;
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index ef043e4ca0..c0105b385b 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -721,6 +721,18 @@ class ComputationBuilder(object):
     """
     return self._client.ConvertElementType(operand, new_element_type)
 
+  def BitcastConvertType(self, operand, new_element_type):
+    """Enqueues a bitcast type conversion operation onto the computation.
+
+    Args:
+      operand: the operand to convert.
+      new_element_type: the target primitive type.
+
+    Returns:
+      A LocalOp representing the added conversion op.
+    """
+    return self._client.BitcastConvertType(operand, new_element_type)
+
   def GetShape(self, operand):
     return _wrap_shape(self._client.GetShape(operand))
 
diff --git a/tensorflow/compiler/xla/python/xla_client_test.py b/tensorflow/compiler/xla/python/xla_client_test.py
index 93177aa647..fd98e19457 100644
--- a/tensorflow/compiler/xla/python/xla_client_test.py
+++ b/tensorflow/compiler/xla/python/xla_client_test.py
@@ -489,6 +489,34 @@ class SingleOpTest(LocalComputationTest):
     for src_dtype, dst_dtype in itertools.product(xla_types, xla_types):
       _ConvertAndTest(x, src_dtype, dst_dtype)
 
+  def testBitcastConvertType(self):
+    xla_x32_types = {
+        np.int32: xla_client.xla_data_pb2.S32,
+        np.float32: xla_client.xla_data_pb2.F32,
+    }
+
+    xla_x64_types = {
+        np.int64: xla_client.xla_data_pb2.S64,
+        np.float64: xla_client.xla_data_pb2.F64,
+    }
+
+    def _ConvertAndTest(template, src_dtype, dst_dtype, dst_etype):
+      c = self._NewComputation()
+      x = c.Constant(np.array(template, dtype=src_dtype))
+      c.BitcastConvertType(x, dst_etype)
+
+      result = c.Build().Compile().Execute()
+      expected = np.array(template, src_dtype).view(dst_dtype)
+
+      self.assertEqual(result.shape, expected.shape)
+      self.assertEqual(result.dtype, expected.dtype)
+      np.testing.assert_equal(result, expected)
+
+    x = [0, 1, 0, 0, 1]
+    for xla_types in [xla_x32_types, xla_x64_types]:
+      for src_dtype, dst_dtype in itertools.product(xla_types, xla_types):
+        _ConvertAndTest(x, src_dtype, dst_dtype, xla_types[dst_dtype])
+
   def testCrossReplicaSumOneReplica(self):
     samples = [
         NumpyArrayF32(42.0),
-- 
cgit v1.2.3


From 5c15427443506d4beafc8223fbe665024191464a Mon Sep 17 00:00:00 2001
From: Xuechen Li 
Date: Fri, 20 Jul 2018 17:57:34 -0700
Subject: BEGIN_PUBLIC Fix broken benchmark. END_PUBLIC Automated rollback of
 commit 8741006018326350467fe86785d98963ff9e983e

PiperOrigin-RevId: 205480787
---
 .../contrib/eager/python/examples/revnet/BUILD     |  36 ++
 .../contrib/eager/python/examples/revnet/blocks.py | 374 ++++++++++++++-------
 .../eager/python/examples/revnet/cifar_input.py    |   2 +-
 .../contrib/eager/python/examples/revnet/config.py |  16 +-
 .../contrib/eager/python/examples/revnet/main.py   |  82 ++---
 .../eager/python/examples/revnet/main_estimator.py | 200 +++++++++++
 .../python/examples/revnet/main_estimator_tpu.py   | 328 ++++++++++++++++++
 .../contrib/eager/python/examples/revnet/revnet.py | 112 ++----
 .../eager/python/examples/revnet/revnet_test.py    |  29 +-
 9 files changed, 910 insertions(+), 269 deletions(-)
 create mode 100644 tensorflow/contrib/eager/python/examples/revnet/main_estimator.py
 create mode 100644 tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py

diff --git a/tensorflow/contrib/eager/python/examples/revnet/BUILD b/tensorflow/contrib/eager/python/examples/revnet/BUILD
index 0c0e4c0eb9..3316dc1114 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/BUILD
+++ b/tensorflow/contrib/eager/python/examples/revnet/BUILD
@@ -113,3 +113,39 @@ py_binary(
         "//tensorflow:tensorflow_py",
     ],
 )
+
+py_binary(
+    name = "main_estimator",
+    srcs = ["main_estimator.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":cifar_input",
+        ":main",
+        ":revnet",
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_library(
+    name = "main_estimator_lib",
+    srcs = ["main_estimator.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":cifar_input",
+        ":main",
+        ":revnet",
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_library(
+    name = "main_estimator_tpu_lib",
+    srcs = ["main_estimator_tpu.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":cifar_input",
+        ":main",
+        ":revnet",
+        "//tensorflow:tensorflow_py",
+    ],
+)
diff --git a/tensorflow/contrib/eager/python/examples/revnet/blocks.py b/tensorflow/contrib/eager/python/examples/revnet/blocks.py
index 306096e9f8..639bb06a34 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/blocks.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/blocks.py
@@ -24,6 +24,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+import operator
+
 import tensorflow as tf
 from tensorflow.contrib.eager.python.examples.revnet import ops
 
@@ -45,7 +48,7 @@ class RevBlock(tf.keras.Model):
                bottleneck=False,
                fused=True,
                dtype=tf.float32):
-    """Initialize RevBlock.
+    """Initialization.
 
     Args:
       n_res: number of residual blocks
@@ -99,7 +102,6 @@ class RevBlock(tf.keras.Model):
       if i == 0:
         # First block usually contains downsampling that can't be reversed
         with tf.GradientTape() as tape:
-          x = tf.identity(x)
           tape.watch(x)
           y = block(x, training=training)
 
@@ -121,16 +123,6 @@ class _Residual(tf.keras.Model):
   """Single residual block contained in a _RevBlock. Each `_Residual` object has
   two _ResidualInner objects, corresponding to the `F` and `G` functions in the
   paper.
-
-  Args:
-    filters: output filter size
-    strides: length 2 list/tuple of integers for height and width strides
-    input_shape: length 3 list/tuple of integers
-    batch_norm_first: whether to apply activation and batch norm before conv
-    data_format: tensor data format, "NCHW"/"NHWC",
-    bottleneck: use bottleneck residual if True
-    fused: use fused batch normalization if True
-    dtype: float16, float32, or float64
   """
 
   def __init__(self,
@@ -142,6 +134,18 @@ class _Residual(tf.keras.Model):
                bottleneck=False,
                fused=True,
                dtype=tf.float32):
+    """Initialization.
+
+    Args:
+      filters: output filter size
+      strides: length 2 list/tuple of integers for height and width strides
+      input_shape: length 3 list/tuple of integers
+      batch_norm_first: whether to apply activation and batch norm before conv
+      data_format: tensor data format, "NCHW"/"NHWC",
+      bottleneck: use bottleneck residual if True
+      fused: use fused batch normalization if True
+      dtype: float16, float32, or float64
+    """
     super(_Residual, self).__init__()
 
     self.filters = filters
@@ -196,7 +200,6 @@ class _Residual(tf.keras.Model):
     dy1, dy2 = tf.split(dy, num_or_size_splits=2, axis=self.axis)
 
     with tf.GradientTape(persistent=True) as tape:
-      y = tf.identity(y)
       tape.watch(y)
       y1, y2 = tf.split(y, num_or_size_splits=2, axis=self.axis)
       z1 = y1
@@ -227,131 +230,252 @@ class _Residual(tf.keras.Model):
     return x, dx, grads, vars_
 
 
-def _BottleneckResidualInner(filters,
-                             strides,
-                             input_shape,
-                             batch_norm_first=True,
-                             data_format="channels_first",
-                             fused=True,
-                             dtype=tf.float32):
+# Ideally, the following should be wrapped in `tf.keras.Sequential`, however
+# there are subtle issues with its placeholder insertion policy and batch norm
+class _BottleneckResidualInner(tf.keras.Model):
   """Single bottleneck residual inner function contained in _Resdual.
 
   Corresponds to the `F`/`G` functions in the paper.
   Suitable for training on ImageNet dataset.
-
-  Args:
-    filters: output filter size
-    strides: length 2 list/tuple of integers for height and width strides
-    input_shape: length 3 list/tuple of integers
-    batch_norm_first: whether to apply activation and batch norm before conv
-    data_format: tensor data format, "NCHW"/"NHWC"
-    fused: use fused batch normalization if True
-    dtype: float16, float32, or float64
-
-  Returns:
-    A keras model
   """
 
-  axis = 1 if data_format == "channels_first" else 3
-  model = tf.keras.Sequential()
-  if batch_norm_first:
-    model.add(
-        tf.keras.layers.BatchNormalization(
-            axis=axis, input_shape=input_shape, fused=fused, dtype=dtype))
-    model.add(tf.keras.layers.Activation("relu"))
-  model.add(
-      tf.keras.layers.Conv2D(
-          filters=filters // 4,
-          kernel_size=1,
-          strides=strides,
-          input_shape=input_shape,
-          data_format=data_format,
-          use_bias=False,
-          padding="SAME",
-          dtype=dtype))
-
-  model.add(
-      tf.keras.layers.BatchNormalization(axis=axis, fused=fused, dtype=dtype))
-  model.add(tf.keras.layers.Activation("relu"))
-  model.add(
-      tf.keras.layers.Conv2D(
-          filters=filters // 4,
-          kernel_size=3,
-          strides=(1, 1),
-          data_format=data_format,
-          use_bias=False,
-          padding="SAME",
-          dtype=dtype))
-
-  model.add(
-      tf.keras.layers.BatchNormalization(axis=axis, fused=fused, dtype=dtype))
-  model.add(tf.keras.layers.Activation("relu"))
-  model.add(
-      tf.keras.layers.Conv2D(
-          filters=filters,
-          kernel_size=1,
-          strides=(1, 1),
-          data_format=data_format,
-          use_bias=False,
-          padding="SAME",
-          dtype=dtype))
+  def __init__(self,
+               filters,
+               strides,
+               input_shape,
+               batch_norm_first=True,
+               data_format="channels_first",
+               fused=True,
+               dtype=tf.float32):
+    """Initialization.
+
+    Args:
+      filters: output filter size
+      strides: length 2 list/tuple of integers for height and width strides
+      input_shape: length 3 list/tuple of integers
+      batch_norm_first: whether to apply activation and batch norm before conv
+      data_format: tensor data format, "NCHW"/"NHWC"
+      fused: use fused batch normalization if True
+      dtype: float16, float32, or float64
+    """
+    super(_BottleneckResidualInner, self).__init__()
+    axis = 1 if data_format == "channels_first" else 3
+    if batch_norm_first:
+      self.batch_norm_0 = tf.keras.layers.BatchNormalization(
+          axis=axis, input_shape=input_shape, fused=fused, dtype=dtype)
+
+    self.conv2d_1 = tf.keras.layers.Conv2D(
+        filters=filters // 4,
+        kernel_size=1,
+        strides=strides,
+        input_shape=input_shape,
+        data_format=data_format,
+        use_bias=False,
+        padding="SAME",
+        dtype=dtype)
+    self.batch_norm_1 = tf.keras.layers.BatchNormalization(
+        axis=axis, fused=fused, dtype=dtype)
+
+    self.conv2d_2 = tf.keras.layers.Conv2D(
+        filters=filters // 4,
+        kernel_size=3,
+        strides=(1, 1),
+        data_format=data_format,
+        use_bias=False,
+        padding="SAME",
+        dtype=dtype)
+
+    self.batch_norm_2 = tf.keras.layers.BatchNormalization(
+        axis=axis, fused=fused, dtype=dtype)
+    self.conv2d_3 = tf.keras.layers.Conv2D(
+        filters=filters,
+        kernel_size=1,
+        strides=(1, 1),
+        data_format=data_format,
+        use_bias=False,
+        padding="SAME",
+        dtype=dtype)
+
+    self.batch_norm_first = batch_norm_first
+
+  def call(self, x, training=True):
+    net = x
+    if self.batch_norm_first:
+      net = self.batch_norm_0(net, training=training)
+      net = tf.nn.relu(net)
+
+    net = self.conv2d_1(net)
+    net = self.batch_norm_1(net, training=training)
+    net = tf.nn.relu(net)
+
+    net = self.conv2d_2(net)
+    net = self.batch_norm_2(net, training=training)
+    net = tf.nn.relu(net)
 
-  return model
+    net = self.conv2d_3(net)
 
+    return net
 
-def _ResidualInner(filters,
-                   strides,
-                   input_shape,
-                   batch_norm_first=True,
-                   data_format="channels_first",
-                   fused=True,
-                   dtype=tf.float32):
+
+class _ResidualInner(tf.keras.Model):
   """Single residual inner function contained in _ResdualBlock.
 
   Corresponds to the `F`/`G` functions in the paper.
-
-  Args:
-    filters: output filter size
-    strides: length 2 list/tuple of integers for height and width strides
-    input_shape: length 3 list/tuple of integers
-    batch_norm_first: whether to apply activation and batch norm before conv
-    data_format: tensor data format, "NCHW"/"NHWC"
-    fused: use fused batch normalization if True
-    dtype: float16, float32, or float64
-
-  Returns:
-    A keras model
   """
 
-  axis = 1 if data_format == "channels_first" else 3
-  model = tf.keras.Sequential()
-  if batch_norm_first:
-    model.add(
-        tf.keras.layers.BatchNormalization(
-            axis=axis, input_shape=input_shape, fused=fused, dtype=dtype))
-    model.add(tf.keras.layers.Activation("relu"))
-  model.add(
-      tf.keras.layers.Conv2D(
-          filters=filters,
-          kernel_size=3,
-          strides=strides,
-          input_shape=input_shape,
-          data_format=data_format,
-          use_bias=False,
-          padding="SAME",
-          dtype=dtype))
-
-  model.add(
-      tf.keras.layers.BatchNormalization(axis=axis, fused=fused, dtype=dtype))
-  model.add(tf.keras.layers.Activation("relu"))
-  model.add(
-      tf.keras.layers.Conv2D(
-          filters=filters,
-          kernel_size=3,
-          strides=(1, 1),
-          data_format=data_format,
-          use_bias=False,
+  def __init__(self,
+               filters,
+               strides,
+               input_shape,
+               batch_norm_first=True,
+               data_format="channels_first",
+               fused=True,
+               dtype=tf.float32):
+    """Initialization.
+
+    Args:
+      filters: output filter size
+      strides: length 2 list/tuple of integers for height and width strides
+      input_shape: length 3 list/tuple of integers
+      batch_norm_first: whether to apply activation and batch norm before conv
+      data_format: tensor data format, "NCHW"/"NHWC"
+      fused: use fused batch normalization if True
+      dtype: float16, float32, or float64
+    """
+    super(_ResidualInner, self).__init__()
+    axis = 1 if data_format == "channels_first" else 3
+    if batch_norm_first:
+      self.batch_norm_0 = tf.keras.layers.BatchNormalization(
+          axis=axis, input_shape=input_shape, fused=fused, dtype=dtype)
+    self.conv2d_1 = tf.keras.layers.Conv2D(
+        filters=filters,
+        kernel_size=3,
+        strides=strides,
+        input_shape=input_shape,
+        data_format=data_format,
+        use_bias=False,
+        padding="SAME",
+        dtype=dtype)
+    self.batch_norm_1 = tf.keras.layers.BatchNormalization(
+        axis=axis, fused=fused, dtype=dtype)
+
+    self.conv2d_2 = tf.keras.layers.Conv2D(
+        filters=filters,
+        kernel_size=3,
+        strides=(1, 1),
+        data_format=data_format,
+        use_bias=False,
+        padding="SAME",
+        dtype=dtype)
+
+    self.batch_norm_first = batch_norm_first
+
+  def call(self, x, training=True):
+    net = x
+    if self.batch_norm_first:
+      net = self.batch_norm_0(net, training=training)
+      net = tf.nn.relu(net)
+
+    net = self.conv2d_1(net)
+    net = self.batch_norm_1(net, training=training)
+
+    net = self.conv2d_2(net)
+
+    return net
+
+
+class InitBlock(tf.keras.Model):
+  """Initial block of RevNet."""
+
+  def __init__(self, config):
+    """Initialization.
+
+    Args:
+      config: tf.contrib.training.HParams object; specifies hyperparameters
+    """
+    super(InitBlock, self).__init__()
+    self.config = config
+    self.axis = 1 if self.config.data_format == "channels_first" else 3
+    self.conv2d = tf.keras.layers.Conv2D(
+        filters=self.config.init_filters,
+        kernel_size=self.config.init_kernel,
+        strides=(self.config.init_stride, self.config.init_stride),
+        data_format=self.config.data_format,
+        use_bias=False,
+        padding="SAME",
+        input_shape=self.config.input_shape,
+        dtype=self.config.dtype)
+    self.batch_norm = tf.keras.layers.BatchNormalization(
+        axis=self.axis, fused=self.config.fused, dtype=self.config.dtype)
+    self.activation = tf.keras.layers.Activation("relu")
+
+    if self.config.init_max_pool:
+      self.max_pool = tf.keras.layers.MaxPooling2D(
+          pool_size=(3, 3),
+          strides=(2, 2),
           padding="SAME",
-          dtype=dtype))
+          data_format=self.config.data_format,
+          dtype=self.config.dtype)
+
+  def call(self, x, training=True):
+    net = x
+    net = self.conv2d(net)
+    net = self.batch_norm(net, training=training)
+    net = self.activation(net)
+
+    if self.config.init_max_pool:
+      net = self.max_pool(net)
+
+    return net
+
 
-  return model
+class FinalBlock(tf.keras.Model):
+  """Final block of RevNet."""
+
+  def __init__(self, config):
+    """Initialization.
+
+    Args:
+      config: tf.contrib.training.HParams object; specifies hyperparameters
+
+    Raises:
+      ValueError: Unsupported data format
+    """
+    super(FinalBlock, self).__init__()
+    self.config = config
+    self.axis = 1 if self.config.data_format == "channels_first" else 3
+
+    f = self.config.filters[-1]  # Number of filters
+    r = functools.reduce(operator.mul, self.config.strides, 1)  # Reduce ratio
+    r *= self.config.init_stride
+    if self.config.init_max_pool:
+      r *= 2
+
+    if self.config.data_format == "channels_first":
+      w, h = self.config.input_shape[1], self.config.input_shape[2]
+      input_shape = (f, w // r, h // r)
+    elif self.config.data_format == "channels_last":
+      w, h = self.config.input_shape[0], self.config.input_shape[1]
+      input_shape = (w // r, h // r, f)
+    else:
+      raise ValueError("Data format should be either `channels_first`"
+                       " or `channels_last`")
+    self.batch_norm = tf.keras.layers.BatchNormalization(
+        axis=self.axis,
+        input_shape=input_shape,
+        fused=self.config.fused,
+        dtype=self.config.dtype)
+    self.activation = tf.keras.layers.Activation("relu")
+    self.global_avg_pool = tf.keras.layers.GlobalAveragePooling2D(
+        data_format=self.config.data_format, dtype=self.config.dtype)
+    self.dense = tf.keras.layers.Dense(
+        self.config.n_classes, dtype=self.config.dtype)
+
+  def call(self, x, training=True):
+    net = x
+    net = self.batch_norm(net, training=training)
+    net = self.activation(net)
+    net = self.global_avg_pool(net)
+    net = self.dense(net)
+
+    return net
diff --git a/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py b/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py
index b6d4c35bfd..e9672f13e1 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/cifar_input.py
@@ -111,6 +111,6 @@ def get_ds_from_tfrecords(data_dir,
     }[split]
     dataset = dataset.shuffle(size)
 
-  dataset = dataset.batch(batch_size)
+  dataset = dataset.batch(batch_size, drop_remainder=True)
 
   return dataset
diff --git a/tensorflow/contrib/eager/python/examples/revnet/config.py b/tensorflow/contrib/eager/python/examples/revnet/config.py
index 3d93fa955a..1532c7b67b 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/config.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/config.py
@@ -27,17 +27,16 @@ from __future__ import division
 from __future__ import print_function
 
 import tensorflow as tf
-tfe = tf.contrib.eager
 
 
 def get_hparams_cifar_38():
   """RevNet-38 configurations for CIFAR-10/CIFAR-100."""
 
   config = tf.contrib.training.HParams()
+  # Hyperparameters from the RevNet paper
   config.add_hparam("init_filters", 32)
   config.add_hparam("init_kernel", 3)
   config.add_hparam("init_stride", 1)
-  config.add_hparam("n_classes", 10)
   config.add_hparam("n_rev_blocks", 3)
   config.add_hparam("n_res", [3, 3, 3])
   config.add_hparam("filters", [32, 64, 112])
@@ -46,7 +45,7 @@ def get_hparams_cifar_38():
   config.add_hparam("bottleneck", False)
   config.add_hparam("fused", True)
   config.add_hparam("init_max_pool", False)
-  if tfe.num_gpus() > 0:
+  if tf.test.is_gpu_available() > 0:
     config.add_hparam("input_shape", (3, 32, 32))
     config.add_hparam("data_format", "channels_first")
   else:
@@ -71,6 +70,16 @@ def get_hparams_cifar_38():
   config.add_hparam("iters_per_epoch", 50000 // config.batch_size)
   config.add_hparam("epochs", config.max_train_iter // config.iters_per_epoch)
 
+  # Customized TPU hyperparameters due to differing batch size caused by
+  # TPU architecture specifics
+  # Suggested batch sizes to reduce overhead from excessive tensor padding
+  # https://cloud.google.com/tpu/docs/troubleshooting
+  config.add_hparam("tpu_batch_size", 128)
+  config.add_hparam("tpu_eval_batch_size", 1024)
+  config.add_hparam("tpu_iters_per_epoch", 50000 // config.tpu_batch_size)
+  config.add_hparam("tpu_epochs",
+                    config.max_train_iter // config.tpu_iters_per_epoch)
+
   return config
 
 
@@ -101,7 +110,6 @@ def get_hparams_imagenet_56():
   config.add_hparam("init_filters", 128)
   config.add_hparam("init_kernel", 7)
   config.add_hparam("init_stride", 2)
-  config.add_hparam("n_classes", 1000)
   config.add_hparam("n_rev_blocks", 4)
   config.add_hparam("n_res", [2, 2, 2, 2])
   config.add_hparam("filters", [128, 256, 512, 832])
diff --git a/tensorflow/contrib/eager/python/examples/revnet/main.py b/tensorflow/contrib/eager/python/examples/revnet/main.py
index e2f43b03f9..1a4fd45c8b 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/main.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/main.py
@@ -31,8 +31,11 @@ tfe = tf.contrib.eager
 
 def main(_):
   """Eager execution workflow with RevNet trained on CIFAR-10."""
-  config = get_config()
-  ds_train, ds_train_one_shot, ds_validation, ds_test = get_datasets(config)
+  tf.enable_eager_execution()
+
+  config = get_config(config_name=FLAGS.config, dataset=FLAGS.dataset)
+  ds_train, ds_train_one_shot, ds_validation, ds_test = get_datasets(
+      data_dir=FLAGS.data_dir, config=config)
   model = revnet.RevNet(config=config)
   global_step = tf.train.get_or_create_global_step()  # Ensure correct summary
   global_step.assign(1)
@@ -52,23 +55,17 @@ def main(_):
             "with global_step: {}".format(latest_path, global_step.numpy()))
       sys.stdout.flush()
 
-  if FLAGS.manual_grad:
-    print("Using manual gradients.")
-  else:
-    print("Not using manual gradients.")
-  sys.stdout.flush()
-
   for x, y in ds_train:
     train_one_iter(model, x, y, optimizer, global_step=global_step)
 
     if global_step.numpy() % config.log_every == 0:
-      it_train = ds_train_one_shot.make_one_shot_iterator()
       it_test = ds_test.make_one_shot_iterator()
-      acc_train, loss_train = evaluate(model, it_train)
       acc_test, loss_test = evaluate(model, it_test)
 
       if FLAGS.validate:
+        it_train = ds_train_one_shot.make_one_shot_iterator()
         it_validation = ds_validation.make_one_shot_iterator()
+        acc_train, loss_train = evaluate(model, it_train)
         acc_validation, loss_validation = evaluate(model, it_validation)
         print("Iter {}, "
               "training set accuracy {:.4f}, loss {:.4f}; "
@@ -77,11 +74,8 @@ def main(_):
                   global_step.numpy(), acc_train, loss_train, acc_validation,
                   loss_validation, acc_test, loss_test))
       else:
-        print("Iter {}, "
-              "training set accuracy {:.4f}, loss {:.4f}; "
-              "test accuracy {:.4f}, loss {:.4f}".format(
-                  global_step.numpy(), acc_train, loss_train, acc_test,
-                  loss_test))
+        print("Iter {}, test accuracy {:.4f}, loss {:.4f}".format(
+            global_step.numpy(), acc_test, loss_test))
       sys.stdout.flush()
 
       if FLAGS.train_dir:
@@ -103,34 +97,38 @@ def main(_):
       sys.stdout.flush()
 
 
-def get_config():
+def get_config(config_name="revnet-38", dataset="cifar-10"):
   """Return configuration."""
-  print("Config: {}".format(FLAGS.config))
+  print("Config: {}".format(config_name))
   sys.stdout.flush()
   config = {
       "revnet-38": config_.get_hparams_cifar_38(),
       "revnet-110": config_.get_hparams_cifar_110(),
       "revnet-164": config_.get_hparams_cifar_164(),
-  }[FLAGS.config]
+  }[config_name]
 
-  if FLAGS.dataset == "cifar-100":
-    config.n_classes = 100
+  if dataset == "cifar-10":
+    config.add_hparam("n_classes", 10)
+    config.add_hparam("dataset", "cifar-10")
+  else:
+    config.add_hparam("n_classes", 100)
+    config.add_hparam("dataset", "cifar-100")
 
   return config
 
 
-def get_datasets(config):
+def get_datasets(data_dir, config):
   """Return dataset."""
-  if FLAGS.data_dir is None:
+  if data_dir is None:
     raise ValueError("No supplied data directory")
-  if not os.path.exists(FLAGS.data_dir):
-    raise ValueError("Data directory {} does not exist".format(FLAGS.data_dir))
-  if FLAGS.dataset not in ["cifar-10", "cifar-100"]:
-    raise ValueError("Unknown dataset {}".format(FLAGS.dataset))
+  if not os.path.exists(data_dir):
+    raise ValueError("Data directory {} does not exist".format(data_dir))
+  if config.dataset not in ["cifar-10", "cifar-100"]:
+    raise ValueError("Unknown dataset {}".format(config.dataset))
 
-  print("Training on {} dataset.".format(FLAGS.dataset))
+  print("Training on {} dataset.".format(config.dataset))
   sys.stdout.flush()
-  data_dir = os.path.join(FLAGS.data_dir, FLAGS.dataset)
+  data_dir = os.path.join(data_dir, config.dataset)
   if FLAGS.validate:
     # 40k Training set
     ds_train = cifar_input.get_ds_from_tfrecords(
@@ -168,7 +166,7 @@ def get_datasets(config):
         prefetch=config.batch_size)
     ds_validation = None
 
-  # Always compute loss and accuracy on whole training and test set
+  # Always compute loss and accuracy on whole test set
   ds_train_one_shot = cifar_input.get_ds_from_tfrecords(
       data_dir=data_dir,
       split="train_all",
@@ -196,19 +194,11 @@ def get_datasets(config):
 
 def train_one_iter(model, inputs, labels, optimizer, global_step=None):
   """Train for one iteration."""
-  if FLAGS.manual_grad:
-    grads, vars_, loss = model.compute_gradients(inputs, labels, training=True)
-    optimizer.apply_gradients(zip(grads, vars_), global_step=global_step)
-  else:  # For correctness validation
-    with tf.GradientTape() as tape:
-      logits, _ = model(inputs, training=True)
-      loss = model.compute_loss(logits=logits, labels=labels)
-      tf.logging.info("Logits are placed on device: {}".format(logits.device))
-    grads = tape.gradient(loss, model.trainable_variables)
-    optimizer.apply_gradients(
-        zip(grads, model.trainable_variables), global_step=global_step)
+  grads, vars_, logits, loss = model.compute_gradients(
+      inputs, labels, training=True)
+  optimizer.apply_gradients(zip(grads, vars_), global_step=global_step)
 
-  return loss.numpy()
+  return logits, loss
 
 
 def evaluate(model, iterator):
@@ -241,16 +231,14 @@ if __name__ == "__main__":
       "validate",
       default=False,
       help="[Optional] Use the validation set or not for hyperparameter search")
-  flags.DEFINE_boolean(
-      "manual_grad",
-      default=False,
-      help="[Optional] Use manual gradient graph to save memory")
   flags.DEFINE_string(
       "dataset",
       default="cifar-10",
       help="[Optional] The dataset used; either `cifar-10` or `cifar-100`")
   flags.DEFINE_string(
-      "config", default="revnet-38", help="[Optional] Architecture of network.")
+      "config",
+      default="revnet-38",
+      help="[Optional] Architecture of network. "
+      "Other options include `revnet-110` and `revnet-164`")
   FLAGS = flags.FLAGS
-  tf.enable_eager_execution()
   tf.app.run(main)
diff --git a/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py b/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py
new file mode 100644
index 0000000000..4868f1931f
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/revnet/main_estimator.py
@@ -0,0 +1,200 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Estimator workflow with RevNet train on CIFAR-10."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from absl import flags
+import tensorflow as tf
+from tensorflow.contrib.eager.python.examples.revnet import cifar_input
+from tensorflow.contrib.eager.python.examples.revnet import main as main_
+from tensorflow.contrib.eager.python.examples.revnet import revnet
+
+
+def model_fn(features, labels, mode, params):
+  """Function specifying the model that is required by the `tf.estimator` API.
+
+  Args:
+    features: Input images
+    labels: Labels of images
+    mode: One of `ModeKeys.TRAIN`, `ModeKeys.EVAL` or 'ModeKeys.PREDICT'
+    params: A dictionary of extra parameter that might be passed
+
+  Returns:
+    An instance of `tf.estimator.EstimatorSpec`
+  """
+
+  inputs = features
+  if isinstance(inputs, dict):
+    inputs = features["image"]
+
+  config = params["config"]
+  model = revnet.RevNet(config=config)
+
+  if mode == tf.estimator.ModeKeys.TRAIN:
+    global_step = tf.train.get_or_create_global_step()
+    learning_rate = tf.train.piecewise_constant(
+        global_step, config.lr_decay_steps, config.lr_list)
+    optimizer = tf.train.MomentumOptimizer(
+        learning_rate, momentum=config.momentum)
+    grads, vars_, logits, loss = model.compute_gradients(
+        inputs, labels, training=True)
+    train_op = optimizer.apply_gradients(
+        zip(grads, vars_), global_step=global_step)
+
+    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
+  else:
+    logits, _ = model(inputs, training=False)
+    predictions = tf.argmax(logits, axis=1)
+    probabilities = tf.nn.softmax(logits)
+
+    if mode == tf.estimator.ModeKeys.EVAL:
+      loss = model.compute_loss(labels=labels, logits=logits)
+      return tf.estimator.EstimatorSpec(
+          mode=mode,
+          loss=loss,
+          eval_metric_ops={
+              "accuracy":
+                  tf.metrics.accuracy(labels=labels, predictions=predictions)
+          })
+
+    else:  # mode == tf.estimator.ModeKeys.PREDICT
+      result = {
+          "classes": predictions,
+          "probabilities": probabilities,
+      }
+
+      return tf.estimator.EstimatorSpec(
+          mode=mode,
+          predictions=predictions,
+          export_outputs={
+              "classify": tf.estimator.export.PredictOutput(result)
+          })
+
+
+def get_input_fn(config, data_dir, split):
+  """Get the input function that is required by the `tf.estimator` API.
+
+  Args:
+    config: Customized hyperparameters
+    data_dir: Directory where the data is stored
+    split: One of `train`, `validation`, `train_all`, and `test`
+
+  Returns:
+    Input function required by the `tf.estimator` API
+  """
+
+  data_dir = os.path.join(data_dir, config.dataset)
+  # Fix split-dependent hyperparameters
+  if split == "train_all" or split == "train":
+    data_aug = True
+    batch_size = config.batch_size
+    epochs = config.epochs
+    shuffle = True
+    prefetch = config.batch_size
+  else:
+    data_aug = False
+    batch_size = config.eval_batch_size
+    epochs = 1
+    shuffle = False
+    prefetch = config.eval_batch_size
+
+  def input_fn():
+    """Input function required by the `tf.estimator.Estimator` API."""
+    return cifar_input.get_ds_from_tfrecords(
+        data_dir=data_dir,
+        split=split,
+        data_aug=data_aug,
+        batch_size=batch_size,
+        epochs=epochs,
+        shuffle=shuffle,
+        prefetch=prefetch,
+        data_format=config.data_format)
+
+  return input_fn
+
+
+def main(argv):
+  FLAGS = argv[0]  # pylint:disable=invalid-name,redefined-outer-name
+  tf.logging.set_verbosity(tf.logging.INFO)
+
+  # RevNet specific configuration
+  config = main_.get_config(config_name=FLAGS.config, dataset=FLAGS.dataset)
+
+  # Estimator specific configuration
+  run_config = tf.estimator.RunConfig(
+      model_dir=FLAGS.train_dir,  # Directory for storing checkpoints
+      tf_random_seed=config.seed,
+      save_summary_steps=config.log_every,
+      save_checkpoints_steps=config.log_every,
+      session_config=None,  # Using default
+      keep_checkpoint_max=100,
+      keep_checkpoint_every_n_hours=10000,  # Using default
+      log_step_count_steps=config.log_every,
+      train_distribute=None  # Default not use distribution strategy
+  )
+
+  # Construct estimator
+  revnet_estimator = tf.estimator.Estimator(
+      model_fn=model_fn,
+      model_dir=FLAGS.train_dir,
+      config=run_config,
+      params={"config": config})
+
+  # Construct input functions
+  train_input_fn = get_input_fn(
+      config=config, data_dir=FLAGS.data_dir, split="train_all")
+  eval_input_fn = get_input_fn(
+      config=config, data_dir=FLAGS.data_dir, split="test")
+
+  # Train and evaluate estimator
+  revnet_estimator.train(input_fn=train_input_fn)
+  revnet_estimator.evaluate(input_fn=eval_input_fn)
+
+  if FLAGS.export:
+    input_shape = (None,) + config.input_shape
+    inputs = tf.placeholder(tf.float32, shape=input_shape)
+    input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn({
+        "image": inputs
+    })
+    revnet_estimator.export_savedmodel(FLAGS.train_dir, input_fn)
+
+
+if __name__ == "__main__":
+  flags.DEFINE_string(
+      "data_dir", default=None, help="Directory to load tfrecords")
+  flags.DEFINE_string(
+      "train_dir",
+      default=None,
+      help="[Optional] Directory to store the training information")
+  flags.DEFINE_string(
+      "dataset",
+      default="cifar-10",
+      help="[Optional] The dataset used; either `cifar-10` or `cifar-100`")
+  flags.DEFINE_boolean(
+      "export",
+      default=False,
+      help="[Optional] Export the model for serving if True")
+  flags.DEFINE_string(
+      "config",
+      default="revnet-38",
+      help="[Optional] Architecture of network. "
+      "Other options include `revnet-110` and `revnet-164`")
+  FLAGS = flags.FLAGS
+  tf.app.run(main=main, argv=[FLAGS])
diff --git a/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py b/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py
new file mode 100644
index 0000000000..d809bcd287
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/revnet/main_estimator_tpu.py
@@ -0,0 +1,328 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Cloud TPU Estimator workflow with RevNet train on CIFAR-10."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import time
+
+from absl import flags
+import tensorflow as tf
+from tensorflow.contrib.eager.python.examples.revnet import cifar_input
+from tensorflow.contrib.eager.python.examples.revnet import main as main_
+from tensorflow.contrib.eager.python.examples.revnet import revnet
+from tensorflow.contrib.training.python.training import evaluation
+from tensorflow.python.estimator import estimator as estimator_
+
+
+def model_fn(features, labels, mode, params):
+  """Model function required by the `tf.contrib.tpu.TPUEstimator` API.
+
+  Args:
+    features: Input images
+    labels: Labels of images
+    mode: One of `ModeKeys.TRAIN`, `ModeKeys.EVAL` or 'ModeKeys.PREDICT'
+    params: A dictionary of extra parameter that might be passed
+
+  Returns:
+    An instance of `tf.contrib.tpu.TPUEstimatorSpec`
+  """
+
+  inputs = features
+  if isinstance(inputs, dict):
+    inputs = features["image"]
+
+  FLAGS = params["FLAGS"]  # pylint:disable=invalid-name,redefined-outer-name
+  config = params["config"]
+  model = revnet.RevNet(config=config)
+
+  if mode == tf.estimator.ModeKeys.TRAIN:
+    global_step = tf.train.get_or_create_global_step()
+    learning_rate = tf.train.piecewise_constant(
+        global_step, config.lr_decay_steps, config.lr_list)
+    optimizer = tf.train.MomentumOptimizer(
+        learning_rate, momentum=config.momentum)
+
+    if FLAGS.use_tpu:
+      optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)
+
+    # Define gradients
+    grads, vars_, logits, loss = model.compute_gradients(
+        inputs, labels, training=True)
+    train_op = optimizer.apply_gradients(
+        zip(grads, vars_), global_step=global_step)
+
+    names = [v.name for v in model.variables]
+    tf.logging.warn("{}".format(names))
+
+    return tf.contrib.tpu.TPUEstimatorSpec(
+        mode=tf.estimator.ModeKeys.TRAIN, loss=loss, train_op=train_op)
+
+  elif mode == tf.estimator.ModeKeys.EVAL:
+    logits, _ = model(inputs, training=False)
+    loss = model.compute_loss(labels=labels, logits=logits)
+
+    def metric_fn(labels, logits):
+      predictions = tf.argmax(logits, axis=1)
+      accuracy = tf.metrics.accuracy(labels=labels, predictions=predictions)
+      return {
+          "accuracy": accuracy,
+      }
+
+    return tf.contrib.tpu.TPUEstimatorSpec(
+        mode=mode, loss=loss, eval_metrics=(metric_fn, [labels, logits]))
+
+  else:  # Predict or export
+    logits, _ = model(inputs, training=False)
+    predictions = {
+        "classes": tf.argmax(logits, axis=1),
+        "probabilities": tf.nn.softmax(logits),
+    }
+
+    return tf.contrib.tpu.TPUEstimatorSpec(
+        mode=mode,
+        predictions=predictions,
+        export_outputs={
+            "classify": tf.estimator.export.PredictOutput(predictions)
+        })
+
+
+def get_input_fn(config, data_dir, split):
+  """Get the input function required by the `tf.contrib.tpu.TPUEstimator` API.
+
+  Args:
+    config: Customized hyperparameters
+    data_dir: Directory where the data is stored
+    split: One of `train`, `validation`, `train_all`, and `test`
+
+  Returns:
+    Input function required by the `tf.contrib.tpu.TPUEstimator` API
+  """
+
+  data_dir = os.path.join(data_dir, config.dataset)
+  # Fix split-dependent hyperparameters
+  if split == "train_all" or split == "train":
+    data_aug = True
+    epochs = config.tpu_epochs
+    shuffle = True
+  else:
+    data_aug = False
+    epochs = 1
+    shuffle = False
+
+  def input_fn(params):
+    """Input function required by the `tf.contrib.tpu.TPUEstimator` API."""
+    batch_size = params["batch_size"]
+    return cifar_input.get_ds_from_tfrecords(
+        data_dir=data_dir,
+        split=split,
+        data_aug=data_aug,
+        batch_size=batch_size,  # per-shard batch size
+        epochs=epochs,
+        shuffle=shuffle,
+        prefetch=batch_size,  # per-shard batch size
+        data_format=config.data_format)
+
+  return input_fn
+
+
+def main(argv):
+  FLAGS = argv[0]  # pylint:disable=invalid-name,redefined-outer-name
+  tf.logging.set_verbosity(tf.logging.INFO)
+
+  # RevNet specific configuration
+  config = main_.get_config(config_name=FLAGS.config, dataset=FLAGS.dataset)
+
+  if FLAGS.use_tpu:
+    tf.logging.info("Using TPU.")
+    tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
+        FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
+  else:
+    tpu_cluster_resolver = None
+
+  # TPU specific configuration
+  tpu_config = tf.contrib.tpu.TPUConfig(
+      # Recommended to be set as number of global steps for next checkpoint
+      iterations_per_loop=FLAGS.iterations_per_loop,
+      num_shards=FLAGS.num_shards)
+
+  # Estimator specific configuration
+  run_config = tf.contrib.tpu.RunConfig(
+      cluster=tpu_cluster_resolver,
+      model_dir=FLAGS.model_dir,
+      session_config=tf.ConfigProto(
+          allow_soft_placement=True, log_device_placement=False),
+      tpu_config=tpu_config,
+  )
+
+  # Construct TPU Estimator
+  estimator = tf.contrib.tpu.TPUEstimator(
+      model_fn=model_fn,
+      use_tpu=FLAGS.use_tpu,
+      train_batch_size=config.tpu_batch_size,
+      eval_batch_size=config.tpu_eval_batch_size,
+      config=run_config,
+      params={
+          "FLAGS": FLAGS,
+          "config": config,
+      })
+
+  # Construct input functions
+  train_input_fn = get_input_fn(
+      config=config, data_dir=FLAGS.data_dir, split="train_all")
+  eval_input_fn = get_input_fn(
+      config=config, data_dir=FLAGS.data_dir, split="test")
+
+  # Disabling a range within an else block currently doesn't work
+  # due to https://github.com/PyCQA/pylint/issues/872
+  # pylint: disable=protected-access
+  if FLAGS.mode == "eval":
+    # TPUEstimator.evaluate *requires* a steps argument.
+    # Note that the number of examples used during evaluation is
+    # --eval_steps * --batch_size.
+    # So if you change --batch_size then change --eval_steps too.
+    eval_steps = 10000 // config.tpu_eval_batch_size
+
+    # Run evaluation when there's a new checkpoint
+    for ckpt in evaluation.checkpoints_iterator(
+        FLAGS.model_dir, timeout=FLAGS.eval_timeout):
+      tf.logging.info("Starting to evaluate.")
+      try:
+        start_timestamp = time.time()  # This time will include compilation time
+        eval_results = estimator.evaluate(
+            input_fn=eval_input_fn, steps=eval_steps, checkpoint_path=ckpt)
+        elapsed_time = int(time.time() - start_timestamp)
+        tf.logging.info("Eval results: %s. Elapsed seconds: %d" %
+                        (eval_results, elapsed_time))
+
+        # Terminate eval job when final checkpoint is reached
+        current_step = int(os.path.basename(ckpt).split("-")[1])
+        if current_step >= config.max_train_iter:
+          tf.logging.info(
+              "Evaluation finished after training step %d" % current_step)
+          break
+
+      except tf.errors.NotFoundError:
+        # Since the coordinator is on a different job than the TPU worker,
+        # sometimes the TPU worker does not finish initializing until long after
+        # the CPU job tells it to start evaluating. In this case, the checkpoint
+        # file could have been deleted already.
+        tf.logging.info(
+            "Checkpoint %s no longer exists, skipping checkpoint" % ckpt)
+
+  else:  # FLAGS.mode == 'train' or FLAGS.mode == 'train_and_eval'
+    current_step = estimator_._load_global_step_from_checkpoint_dir(
+        FLAGS.model_dir)
+    tf.logging.info("Training for %d steps . Current"
+                    " step %d." % (config.max_train_iter, current_step))
+
+    start_timestamp = time.time()  # This time will include compilation time
+    if FLAGS.mode == "train":
+      estimator.train(input_fn=train_input_fn, max_steps=config.max_train_iter)
+    else:
+      eval_steps = 10000 // config.tpu_eval_batch_size
+      assert FLAGS.mode == "train_and_eval"
+      while current_step < config.max_train_iter:
+        # Train for up to steps_per_eval number of steps.
+        # At the end of training, a checkpoint will be written to --model_dir.
+        next_checkpoint = min(current_step + FLAGS.steps_per_eval,
+                              config.max_train_iter)
+        estimator.train(input_fn=train_input_fn, max_steps=next_checkpoint)
+        current_step = next_checkpoint
+
+        # Evaluate the model on the most recent model in --model_dir.
+        # Since evaluation happens in batches of --eval_batch_size, some images
+        # may be consistently excluded modulo the batch size.
+        tf.logging.info("Starting to evaluate.")
+        eval_results = estimator.evaluate(
+            input_fn=eval_input_fn, steps=eval_steps)
+        tf.logging.info("Eval results: %s" % eval_results)
+
+    elapsed_time = int(time.time() - start_timestamp)
+    tf.logging.info("Finished training up to step %d. Elapsed seconds %d." %
+                    (config.max_train_iter, elapsed_time))
+  # pylint: enable=protected-access
+
+
+if __name__ == "__main__":
+  # Cloud TPU Cluster Resolver flags
+  flags.DEFINE_string(
+      "tpu",
+      default=None,
+      help="The Cloud TPU to use for training. This should be either the name "
+      "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 "
+      "url.")
+  flags.DEFINE_string(
+      "tpu_zone",
+      default=None,
+      help="[Optional] GCE zone where the Cloud TPU is located in. If not "
+      "specified, we will attempt to automatically detect the GCE project from "
+      "metadata.")
+  flags.DEFINE_string(
+      "gcp_project",
+      default=None,
+      help="[Optional] Project name for the Cloud TPU-enabled project. If not "
+      "specified, we will attempt to automatically detect the GCE project from "
+      "metadata.")
+
+  # Model specific parameters
+  flags.DEFINE_string(
+      "data_dir", default=None, help="Directory to load tfrecords")
+  flags.DEFINE_string(
+      "model_dir",
+      default=None,
+      help="[Optional] Directory to store the model information")
+  flags.DEFINE_string(
+      "dataset",
+      default="cifar-10",
+      help="[Optional] The dataset used; either `cifar-10` or `cifar-100`")
+  flags.DEFINE_string(
+      "config",
+      default="revnet-38",
+      help="[Optional] Architecture of network. "
+      "Other options include `revnet-110` and `revnet-164`")
+  flags.DEFINE_boolean(
+      "use_tpu", default=True, help="[Optional] Whether to use TPU")
+  flags.DEFINE_integer(
+      "num_shards", default=8, help="Number of shards (TPU chips).")
+  flags.DEFINE_integer(
+      "iterations_per_loop",
+      default=100,
+      help=(
+          "Number of steps to run on TPU before feeding metrics to the CPU."
+          " If the number of iterations in the loop would exceed the number of"
+          " train steps, the loop will exit before reaching"
+          " --iterations_per_loop. The larger this value is, the higher the"
+          " utilization on the TPU."))
+  flags.DEFINE_string(
+      "mode",
+      default="train_and_eval",
+      help="[Optional] Mode to run: train, eval, train_and_eval")
+  flags.DEFINE_integer(
+      "eval_timeout", 60 * 60 * 24,
+      "Maximum seconds between checkpoints before evaluation terminates.")
+  flags.DEFINE_integer(
+      "steps_per_eval",
+      default=1000,
+      help=(
+          "Controls how often evaluation is performed. Since evaluation is"
+          " fairly expensive, it is advised to evaluate as infrequently as"
+          " possible (i.e. up to --train_steps, which evaluates the model only"
+          " after finishing the entire training regime)."))
+  FLAGS = flags.FLAGS
+  tf.app.run(main=main, argv=[FLAGS])
diff --git a/tensorflow/contrib/eager/python/examples/revnet/revnet.py b/tensorflow/contrib/eager/python/examples/revnet/revnet.py
index af0d20fa72..b1cb312b74 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/revnet.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/revnet.py
@@ -24,9 +24,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import functools
-import operator
-
 import six
 import tensorflow as tf
 from tensorflow.contrib.eager.python.examples.revnet import blocks
@@ -45,71 +42,9 @@ class RevNet(tf.keras.Model):
     self.axis = 1 if config.data_format == "channels_first" else 3
     self.config = config
 
-    self._init_block = self._construct_init_block()
+    self._init_block = blocks.InitBlock(config=self.config)
+    self._final_block = blocks.FinalBlock(config=self.config)
     self._block_list = self._construct_intermediate_blocks()
-    self._final_block = self._construct_final_block()
-
-  def _construct_init_block(self):
-    init_block = tf.keras.Sequential(
-        [
-            tf.keras.layers.Conv2D(
-                filters=self.config.init_filters,
-                kernel_size=self.config.init_kernel,
-                strides=(self.config.init_stride, self.config.init_stride),
-                data_format=self.config.data_format,
-                use_bias=False,
-                padding="SAME",
-                input_shape=self.config.input_shape,
-                dtype=self.config.dtype),
-            tf.keras.layers.BatchNormalization(
-                axis=self.axis,
-                fused=self.config.fused,
-                dtype=self.config.dtype),
-            tf.keras.layers.Activation("relu"),
-        ],
-        name="init")
-    if self.config.init_max_pool:
-      init_block.add(
-          tf.keras.layers.MaxPooling2D(
-              pool_size=(3, 3),
-              strides=(2, 2),
-              padding="SAME",
-              data_format=self.config.data_format,
-              dtype=self.config.dtype))
-    return init_block
-
-  def _construct_final_block(self):
-    f = self.config.filters[-1]  # Number of filters
-    r = functools.reduce(operator.mul, self.config.strides, 1)  # Reduce ratio
-    r *= self.config.init_stride
-    if self.config.init_max_pool:
-      r *= 2
-
-    if self.config.data_format == "channels_first":
-      w, h = self.config.input_shape[1], self.config.input_shape[2]
-      input_shape = (f, w // r, h // r)
-    elif self.config.data_format == "channels_last":
-      w, h = self.config.input_shape[0], self.config.input_shape[1]
-      input_shape = (w // r, h // r, f)
-    else:
-      raise ValueError("Data format should be either `channels_first`"
-                       " or `channels_last`")
-
-    final_block = tf.keras.Sequential(
-        [
-            tf.keras.layers.BatchNormalization(
-                axis=self.axis,
-                input_shape=input_shape,
-                fused=self.config.fused,
-                dtype=self.config.dtype),
-            tf.keras.layers.Activation("relu"),
-            tf.keras.layers.GlobalAveragePooling2D(
-                data_format=self.config.data_format, dtype=self.config.dtype),
-            tf.keras.layers.Dense(
-                self.config.n_classes, dtype=self.config.dtype)
-        ],
-        name="final")
-    return final_block
 
   def _construct_intermediate_blocks(self):
     # Precompute input shape after initial block
@@ -206,13 +141,20 @@ class RevNet(tf.keras.Model):
       l2_reg: Apply l2 regularization
 
     Returns:
-      list of tuples each being (grad, var) for optimizer to use
+      A tuple with the first entry being a list of all gradients, the second
+      entry being a list of respective variables, the third being the logits,
+      and the forth being the loss
     """
 
-    # Run forward pass to record hidden states; avoid updating running averages
+    # Run forward pass to record hidden states
     vars_and_vals = self.get_moving_stats()
-    _, saved_hidden = self.call(inputs, training=training)
-    self.restore_moving_stats(vars_and_vals)
+    _, saved_hidden = self(inputs, training=training)  # pylint:disable=not-callable
+    if tf.executing_eagerly():
+      # Restore moving averages when executing eagerly to avoid updating twice
+      self.restore_moving_stats(vars_and_vals)
+    else:
+      # Fetch batch norm updates in graph mode
+      updates = self.get_updates_for(inputs)
 
     grads_all = []
     vars_all = []
@@ -220,9 +162,8 @@ class RevNet(tf.keras.Model):
     # Manually backprop through last block
     x = saved_hidden[-1]
     with tf.GradientTape() as tape:
-      x = tf.identity(x)
       tape.watch(x)
-      # Running stats updated below
+      # Running stats updated here
       logits = self._final_block(x, training=training)
       loss = self.compute_loss(logits, labels)
 
@@ -236,6 +177,7 @@ class RevNet(tf.keras.Model):
     for block in reversed(self._block_list):
       y = saved_hidden.pop()
       x = saved_hidden[-1]
+      # Running stats updated here
       dy, grads, vars_ = block.backward_grads_and_vars(
           x, y, dy, training=training)
       grads_all += grads
@@ -247,8 +189,7 @@ class RevNet(tf.keras.Model):
     assert not saved_hidden  # Cleared after backprop
 
     with tf.GradientTape() as tape:
-      x = tf.identity(x)
-      # Running stats updated below
+      # Running stats updated here
       y = self._init_block(x, training=training)
 
     grads_all += tape.gradient(
@@ -259,7 +200,13 @@ class RevNet(tf.keras.Model):
     if l2_reg:
       grads_all = self._apply_weight_decay(grads_all, vars_all)
 
-    return grads_all, vars_all, loss
+    if not tf.executing_eagerly():
+      # Force updates to be executed before gradient computation in graph mode
+      # This does nothing when the function is wrapped in defun
+      with tf.control_dependencies(updates):
+        grads_all[0] = tf.identity(grads_all[0])
+
+    return grads_all, vars_all, logits, loss
 
   def _apply_weight_decay(self, grads, vars_):
     """Update gradients to reflect weight decay."""
@@ -284,8 +231,10 @@ class RevNet(tf.keras.Model):
       n = v.name
       return n.endswith("moving_mean:0") or n.endswith("moving_variance:0")
 
-    for v in filter(_is_moving_var, self.variables):
-      vars_and_vals[v] = v.read_value()
+    device = "/gpu:0" if tf.test.is_gpu_available() else "/cpu:0"
+    with tf.device(device):
+      for v in filter(_is_moving_var, self.variables):
+        vars_and_vals[v] = v.read_value()
 
     return vars_and_vals
 
@@ -297,5 +246,8 @@ class RevNet(tf.keras.Model):
     Args:
       vars_and_vals: The dictionary mapping variables to their previous values.
     """
-    for var_, val in six.iteritems(vars_and_vals):
-      var_.assign(val)
+    device = "/gpu:0" if tf.test.is_gpu_available() else "/cpu:0"
+    with tf.device(device):
+      for var_, val in six.iteritems(vars_and_vals):
+        # `assign` causes a copy to GPU (if variable is already on GPU)
+        var_.assign(val)
diff --git a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
index b0d0a5486d..2dc7b9fd70 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
@@ -31,10 +31,11 @@ tfe = tf.contrib.eager
 
 def train_one_iter(model, inputs, labels, optimizer, global_step=None):
   """Train for one iteration."""
-  grads, vars_, loss = model.compute_gradients(inputs, labels, training=True)
+  grads, vars_, logits, loss = model.compute_gradients(
+      inputs, labels, training=True)
   optimizer.apply_gradients(zip(grads, vars_), global_step=global_step)
 
-  return loss
+  return logits, loss
 
 
 class RevNetTest(tf.test.TestCase):
@@ -42,6 +43,8 @@ class RevNetTest(tf.test.TestCase):
   def setUp(self):
     super(RevNetTest, self).setUp()
     config = config_.get_hparams_cifar_38()
+    config.add_hparam("n_classes", 10)
+    config.add_hparam("dataset", "cifar-10")
     # Reconstruction could cause numerical error, use double precision for tests
     config.dtype = tf.float64
     config.fused = False  # Fused batch norm does not support tf.float64
@@ -94,7 +97,7 @@ class RevNetTest(tf.test.TestCase):
   def test_compute_gradients(self):
     """Test `compute_gradients` function."""
     self.model(self.x, training=False)  # Initialize model
-    grads, vars_, loss = self.model.compute_gradients(
+    grads, vars_, logits, loss = self.model.compute_gradients(
         inputs=self.x, labels=self.t, training=True, l2_reg=True)
     self.assertTrue(isinstance(grads, list))
     self.assertTrue(isinstance(vars_, list))
@@ -119,7 +122,7 @@ class RevNetTest(tf.test.TestCase):
   def test_compute_gradients_defun(self):
     """Test `compute_gradients` function with defun."""
     compute_gradients = tfe.defun(self.model.compute_gradients)
-    grads, vars_, _ = compute_gradients(self.x, self.t, training=True)
+    grads, vars_, _, _ = compute_gradients(self.x, self.t, training=True)
     self.assertTrue(isinstance(grads, list))
     self.assertTrue(isinstance(vars_, list))
     self.assertEqual(len(grads), len(vars_))
@@ -131,6 +134,9 @@ class RevNetTest(tf.test.TestCase):
     """Test model training in graph mode."""
     with tf.Graph().as_default():
       config = config_.get_hparams_cifar_38()
+      config.add_hparam("n_classes", 10)
+      config.add_hparam("dataset", "cifar-10")
+
       x = tf.random_normal(
           shape=(self.config.batch_size,) + self.config.input_shape)
       t = tf.random_uniform(
@@ -140,15 +146,10 @@ class RevNetTest(tf.test.TestCase):
           dtype=tf.int32)
       global_step = tf.Variable(0., trainable=False)
       model = revnet.RevNet(config=config)
-      model(x)
-      updates = model.get_updates_for(x)
-
-      x_ = tf.identity(x)
-      grads_all, vars_all, _ = model.compute_gradients(x_, t, training=True)
+      grads_all, vars_all, _, _ = model.compute_gradients(x, t, training=True)
       optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
-      with tf.control_dependencies(updates):
-        train_op = optimizer.apply_gradients(
-            zip(grads_all, vars_all), global_step=global_step)
+      train_op = optimizer.apply_gradients(
+          zip(grads_all, vars_all), global_step=global_step)
 
       with tf.Session() as sess:
         sess.run(tf.global_variables_initializer())
@@ -222,6 +223,8 @@ class RevNetBenchmark(tf.test.Benchmark):
                              execution_mode=None,
                              compiled=False):
     config = config_.get_hparams_imagenet_56()
+    config.add_hparam("n_classes", 1000)
+    config.add_hparam("dataset", "ImageNet")
     with tfe.execution_mode(execution_mode):
       device, data_format = device_and_format
       model = revnet.RevNet(config=config)
@@ -267,6 +270,8 @@ class RevNetBenchmark(tf.test.Benchmark):
                              execution_mode=None,
                              compiled=False):
     config = config_.get_hparams_imagenet_56()
+    config.add_hparam("n_classes", 1000)
+    config.add_hparam("dataset", "ImageNet")
     with tfe.execution_mode(execution_mode):
       device, data_format = device_and_format
       for batch_size in self._train_batch_sizes():
-- 
cgit v1.2.3


From a8f3646377af20101b180a5f64df3f0066a9653b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Fri, 20 Jul 2018 18:01:30 -0700
Subject: [TF:XLA] Enable better fusion-operand sharing in copy elision.

Make use of the back-end specific fusion-operand sharing functions in copy insertion.

This allows potentially better copy insertion/elision.

PiperOrigin-RevId: 205481101
---
 tensorflow/compiler/xla/service/copy_insertion.cc  | 77 +++++++++++-----------
 tensorflow/compiler/xla/service/copy_insertion.h   | 22 ++++---
 .../compiler/xla/service/hlo_alias_analysis.h      |  2 +-
 .../xla/service/hlo_alias_analysis_test.cc         |  4 +-
 .../compiler/xla/service/hlo_rematerialization.cc  | 11 ++--
 .../compiler/xla/service/hlo_rematerialization.h   | 10 +--
 .../xla/service/hlo_rematerialization_test.cc      |  2 +-
 7 files changed, 69 insertions(+), 59 deletions(-)

diff --git a/tensorflow/compiler/xla/service/copy_insertion.cc b/tensorflow/compiler/xla/service/copy_insertion.cc
index ca2a78da67..36fb9b43aa 100644
--- a/tensorflow/compiler/xla/service/copy_insertion.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion.cc
@@ -351,26 +351,6 @@ Status StripControlDependenciesFrom(HloInstruction* instruction) {
   return Status::OK();
 }
 
-// Add kCopy instructions to the given module to guarantee there is no
-// live-range interference. Generally interference can only occur around kWhile
-// instructions which have update-in-place semantics.
-Status AddCopiesToResolveInterference(HloModule* module) {
-  TF_ASSIGN_OR_RETURN(std::unique_ptr alias_analysis,
-                      HloAliasAnalysis::Run(module));
-
-  for (HloComputation* computation : module->computations()) {
-    for (HloInstruction* instruction : computation->instructions()) {
-      if (instruction->opcode() == HloOpcode::kWhile) {
-        TF_RETURN_IF_ERROR(AddCopiesForWhile(*alias_analysis, instruction));
-      } else if (instruction->opcode() == HloOpcode::kConditional) {
-        TF_RETURN_IF_ERROR(
-            AddCopiesForConditional(*alias_analysis, instruction));
-      }
-    }
-  }
-  return Status::OK();
-}
-
 // Class for removing unnecessary copies from the module.
 //
 // kCopy instructions are added conservatively to guarantee no live range
@@ -945,6 +925,36 @@ class CopyRemover {
   BufferValueTracker buffer_value_tracker_;
 };
 
+void MaybeDumpModule(const string& message, const HloModule& module) {
+  if (VLOG_IS_ON(3)) {
+    VLOG(3) << message;
+    XLA_VLOG_LINES(3, module.ToString());
+    hlo_graph_dumper::MaybeDumpHloModule(module, message);
+  }
+}
+
+}  // namespace
+
+// Add kCopy instructions to the given module to guarantee there is no
+// live-range interference. Generally interference can only occur around kWhile
+// instructions which have update-in-place semantics.
+Status CopyInsertion::AddCopiesToResolveInterference(HloModule* module) {
+  TF_ASSIGN_OR_RETURN(std::unique_ptr alias_analysis,
+                      HloAliasAnalysis::Run(module, fusion_can_share_buffer_));
+
+  for (HloComputation* computation : module->computations()) {
+    for (HloInstruction* instruction : computation->instructions()) {
+      if (instruction->opcode() == HloOpcode::kWhile) {
+        TF_RETURN_IF_ERROR(AddCopiesForWhile(*alias_analysis, instruction));
+      } else if (instruction->opcode() == HloOpcode::kConditional) {
+        TF_RETURN_IF_ERROR(
+            AddCopiesForConditional(*alias_analysis, instruction));
+      }
+    }
+  }
+  return Status::OK();
+}
+
 // Add copies to address special constraints on the roots of computations not
 // related to live range interference:
 //
@@ -955,9 +965,10 @@ class CopyRemover {
 //
 //    (3) Constants and parameters cannot be live out of the entry computation
 //
-Status AddSpecialCaseCopies(const CallGraph& call_graph, HloModule* module) {
+Status CopyInsertion::AddSpecialCaseCopies(const CallGraph& call_graph,
+                                           HloModule* module) {
   TF_ASSIGN_OR_RETURN(std::unique_ptr alias_analysis,
-                      HloAliasAnalysis::Run(module));
+                      HloAliasAnalysis::Run(module, fusion_can_share_buffer_));
 
   // Identify which shape indices of which instructions need to be copied. Store
   // these results in 'instructions_to_copy'.
@@ -1065,32 +1076,20 @@ Status AddSpecialCaseCopies(const CallGraph& call_graph, HloModule* module) {
   return Status::OK();
 }
 
-Status VerifyNoLiveRangeInterference(HloModule* module) {
+Status CopyInsertion::VerifyNoLiveRangeInterference(HloModule* module) {
   TF_ASSIGN_OR_RETURN(std::unique_ptr alias_analysis,
-                      HloAliasAnalysis::Run(module));
+                      HloAliasAnalysis::Run(module, fusion_can_share_buffer_));
   DependencyHloOrdering ordering(module);
   TF_RET_CHECK(!alias_analysis->HasLiveRangeInterference(ordering));
   return Status::OK();
 }
 
-void MaybeDumpModule(const string& message, const HloModule& module) {
-  if (VLOG_IS_ON(3)) {
-    VLOG(3) << message;
-    XLA_VLOG_LINES(3, module.ToString());
-    hlo_graph_dumper::MaybeDumpHloModule(module, message);
-  }
-}
-
-}  // namespace
-
-Status RemoveUnnecessaryCopies(
-    const HloOrdering& ordering, HloModule* module,
-    const HloDataflowAnalysis::FusionCanShareBufferFunction&
-        fusion_can_share_buffer) {
+Status CopyInsertion::RemoveUnnecessaryCopies(const HloOrdering& ordering,
+                                              HloModule* module) {
   MaybeDumpModule("after adding copies to resolve interference", *module);
 
   TF_ASSIGN_OR_RETURN(std::unique_ptr alias_analysis,
-                      HloAliasAnalysis::Run(module, fusion_can_share_buffer));
+                      HloAliasAnalysis::Run(module, fusion_can_share_buffer_));
   CopyRemover copy_remover(*alias_analysis, ordering, module);
   XLA_VLOG_LINES(3, copy_remover.ToString());
 
diff --git a/tensorflow/compiler/xla/service/copy_insertion.h b/tensorflow/compiler/xla/service/copy_insertion.h
index e1973db928..5ba64b78a3 100644
--- a/tensorflow/compiler/xla/service/copy_insertion.h
+++ b/tensorflow/compiler/xla/service/copy_insertion.h
@@ -71,20 +71,26 @@ class CopyInsertion : public HloPassInterface {
   // TODO(b/62548313): Remove this when buffer assignment is module-scoped.
   static StatusOr AddCopiesForBufferAssignment(HloModule* module);
 
+  // Try to remove as many copies from the module as possible without
+  // introducing live range interference. Only copy instructions that are
+  // eligible for copy elision are considered for removal.
+  Status RemoveUnnecessaryCopies(const HloOrdering& ordering,
+                                 HloModule* module);
+
  private:
+  // Verifies that no HLO values have interfering live ranged assuming the
+  // ordering used by copy insertion.
+  Status VerifyNoLiveRangeInterference(HloModule* module);
+
+  Status AddCopiesToResolveInterference(HloModule* module);
+
+  Status AddSpecialCaseCopies(const CallGraph& call_graph, HloModule* module);
+
   // Backend specific function that decides whether a fusion can share buffer
   // with its operand.
   HloDataflowAnalysis::FusionCanShareBufferFunction fusion_can_share_buffer_;
 };
 
-// Try to remove as many copies from the module as possible without introducing
-// live range interference. Only copy instructions that are eligible for
-// copy elision are considered for removal.
-Status RemoveUnnecessaryCopies(
-    const HloOrdering& ordering, HloModule* module,
-    const HloDataflowAnalysis::FusionCanShareBufferFunction&
-        fusion_can_share_buffer = nullptr);
-
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_SERVICE_COPY_INSERTION_H_
diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis.h b/tensorflow/compiler/xla/service/hlo_alias_analysis.h
index afb0c20f0c..1fea544730 100644
--- a/tensorflow/compiler/xla/service/hlo_alias_analysis.h
+++ b/tensorflow/compiler/xla/service/hlo_alias_analysis.h
@@ -42,7 +42,7 @@ class HloAliasAnalysis {
   static StatusOr> Run(
       HloModule* module,
       const HloDataflowAnalysis::FusionCanShareBufferFunction&
-          fusion_can_share_buffer = nullptr);
+          fusion_can_share_buffer);
 
   string ToString() const;
 
diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc
index 403d4df6b5..da94ab5346 100644
--- a/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc
@@ -47,7 +47,9 @@ class HloAliasAnalysisTest : public HloTestBase {
   // reference to the generated analysis stored in analysis_.
   HloAliasAnalysis& RunAnalysis() {
     hlo_graph_dumper::MaybeDumpHloModule(*module_, "Before alias analysis");
-    analysis_ = HloAliasAnalysis::Run(module_.get()).ConsumeValueOrDie();
+    analysis_ = HloAliasAnalysis::Run(module_.get(),
+                                      /*fusion_can_share_buffer=*/nullptr)
+                    .ConsumeValueOrDie();
     return *analysis_;
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
index 59a8800a7d..cf0be30c7a 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
@@ -1203,7 +1203,7 @@ StatusOr HloRematerialization::RematerializeComputation(
 StatusOr HloRematerialization::Run(
     HloModule* module, SequentialHloOrdering::HloModuleSequence* sequence,
     int64 memory_limit_bytes, RematerializationSizes* sizes,
-    bool run_copy_elision) {
+    CopyInsertion* copy_insertion) {
   // The sequence is constructed entirely by this method.
   TF_RET_CHECK(sequence->empty());
 
@@ -1238,13 +1238,14 @@ StatusOr HloRematerialization::Run(
                                        return size_function_(buffer.shape());
                                      },
                                      scheduler_algorithm_));
-  if (run_copy_elision) {
+  if (copy_insertion) {
     // We run a separate pass of copy elision here because the sequential
     // ordering from the HLO schedule allows for more copies to be eliminated.
     // TODO(b/80249101): Instead of a separate copy elision pass, use the
     // ordering from the HLO schedule directly for copy insertion.
     SequentialHloOrdering ordering(module, *sequence);
-    TF_RETURN_IF_ERROR(RemoveUnnecessaryCopies(ordering, module));
+    TF_RETURN_IF_ERROR(
+        copy_insertion->RemoveUnnecessaryCopies(ordering, module));
   }
 
   // Compute peak memory usage of all computations in the module called in a
@@ -1349,10 +1350,10 @@ StatusOr HloRematerialization::Run(
     int64 memory_limit_bytes, HloModule* hlo_module,
     MemorySchedulerAlgorithm scheduler_algorithm,
     SequentialHloOrdering::HloModuleSequence* sequence,
-    RematerializationSizes* sizes, bool run_copy_elision) {
+    RematerializationSizes* sizes, CopyInsertion* copy_insertion) {
   HloRematerialization remat(scheduler_algorithm, size_function);
   return remat.Run(hlo_module, sequence, memory_limit_bytes, sizes,
-                   run_copy_elision);
+                   copy_insertion);
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.h b/tensorflow/compiler/xla/service/hlo_rematerialization.h
index 59b4cf5dcc..2ec004350a 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.h
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.h
@@ -17,6 +17,7 @@
 
 #include "tensorflow/compiler/xla/service/buffer_liveness.h"
 #include "tensorflow/compiler/xla/service/call_graph.h"
+#include "tensorflow/compiler/xla/service/copy_insertion.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -57,8 +58,9 @@ class HloRematerialization {
   //   sizes: Optional outparam that indicates the peak memory usage of the HLO
   //     module before/after rematerialization.
   //
-  //   run_copy_elision: Enable copy elision. This pass is used to eliminate
-  //     copies that were inserted before HLO scheduling.
+  //   copy_insertion: If non-null, run copy elision after scheduling. This
+  //     pass is used to eliminate copies that were inserted by copy insertion
+  //     before HLO scheduling.
   //
   // TODO(b/80249101): Remove the 'run_copy_elision' parameter when copy
   // insertion is integrated with HLO scheduling.
@@ -74,7 +76,7 @@ class HloRematerialization {
       const ShapeSizeFunction& size_function, int64 memory_limit_bytes,
       HloModule* hlo_module, MemorySchedulerAlgorithm scheduler_algorithm,
       SequentialHloOrdering::HloModuleSequence* sequence,
-      RematerializationSizes* sizes, bool run_copy_elision = true);
+      RematerializationSizes* sizes, CopyInsertion* copy_insertion = nullptr);
 
  protected:
   HloRematerialization(MemorySchedulerAlgorithm scheduler_algorithm,
@@ -90,7 +92,7 @@ class HloRematerialization {
   StatusOr Run(HloModule* module,
                      SequentialHloOrdering::HloModuleSequence* sequence,
                      int64 memory_limit, RematerializationSizes* sizes,
-                     bool run_copy_elision);
+                     CopyInsertion* copy_insertion);
 
   // Rematerializes instructions within the given computation. 'order' is the
   // order in which the computation's instructions will be emitted in the
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
index cd131147e6..ac8c97d380 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
@@ -147,7 +147,7 @@ class HloRematerializationTest : public HloTestBase {
     TF_EXPECT_OK(verifier().Run(module).status());
     return HloRematerialization::RematerializeAndSchedule(
         ByteSizeOf, memory_limit_bytes, module, DefaultMemoryScheduler,
-        sequence, /*sizes=*/nullptr, /*run_copy_elision=*/false);
+        sequence, /*sizes=*/nullptr);
   }
 
   // Various shapes used in the canned computations.
-- 
cgit v1.2.3


From 2a9c19d44c5e3ec80a859bf65219c0ff2fc451f0 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy 
Date: Fri, 20 Jul 2018 21:59:00 -0700
Subject: Fix typo in windows build scripts.

---
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 4 ++--
 tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 22d389fd44..9883bb622c 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -57,8 +57,8 @@ PY_TEST_DIR="py_test_dir"
 
 SKIP_TEST=0
 RELEASE_BUILD=0
-TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/... "
-          + "//${PY_TEST_DIR}tensorflow/contrib/... "
+TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/... \
+    //${PY_TEST_DIR}tensorflow/contrib/... "
 
 # --skip_test            Skip running tests
 # --enable_remote_cache  Add options to enable remote cache for build and test
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index 682a396d10..57463e3366 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -57,8 +57,8 @@ PY_TEST_DIR="py_test_dir"
 
 SKIP_TEST=0
 RELEASE_BUILD=0
-TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/... "
-          + "//${PY_TEST_DIR}tensorflow/contrib/... "
+TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/... \
+  //${PY_TEST_DIR}tensorflow/contrib/... "
 
 # --skip_test            Skip running tests
 # --enable_remote_cache  Add options to enable remote cache for build and test
-- 
cgit v1.2.3


From 3153ebe72a5dd5c5848122ddc3c883aab14b5cf1 Mon Sep 17 00:00:00 2001
From: Guangda Lai 
Date: Fri, 20 Jul 2018 22:03:40 -0700
Subject: Exclude (some) tensorrt tests from mac build and cuda_on_cpu build.

PiperOrigin-RevId: 205494168
---
 tensorflow/contrib/tensorrt/BUILD | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 7999f718e3..08b267c11a 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -32,7 +32,10 @@ tf_cuda_cc_test(
     name = "tensorrt_test_cc",
     size = "small",
     srcs = ["tensorrt_test.cc"],
-    tags = ["no_windows"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
     deps = [
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
@@ -234,7 +237,10 @@ tf_cc_test(
     name = "trt_allocator_test",
     size = "small",
     srcs = ["resources/trt_allocator_test.cc"],
-    tags = ["no_windows"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
     deps = [
         ":trt_allocator",
         "//tensorflow/core:test",
@@ -302,7 +308,10 @@ tf_cc_test(
     name = "segment_test",
     size = "small",
     srcs = ["segment/segment_test.cc"],
-    tags = ["no_windows"],
+    tags = [
+        "no_windows",
+        "nomac",
+    ],
     deps = [
         ":segment",
         "//tensorflow/c:c_api",
@@ -338,7 +347,11 @@ tf_cuda_cc_test(
     name = "trt_plugin_factory_test",
     size = "small",
     srcs = ["plugin/trt_plugin_factory_test.cc"],
-    tags = ["no_windows"],
+    tags = [
+        "no_cuda_on_cpu_tap",
+        "no_windows",
+        "nomac",
+    ],
     deps = [
         ":trt_plugins",
         "//tensorflow/core:lib",
@@ -382,6 +395,7 @@ cuda_py_tests(
         "//tensorflow/python:framework_test_lib",
     ],
     tags = [
+        "no_cuda_on_cpu_tap",
         "no_windows",
         "nomac",
     ],
-- 
cgit v1.2.3


From 7eb2bce1eb94ecc24aab02af90f51700500308cd Mon Sep 17 00:00:00 2001
From: Shashi Shekhar 
Date: Fri, 20 Jul 2018 22:44:17 -0700
Subject: Enable a few oss profiler tests.

PiperOrigin-RevId: 205495906
---
 tensorflow/contrib/lite/profiling/BUILD | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/lite/profiling/BUILD b/tensorflow/contrib/lite/profiling/BUILD
index b29ca330dc..1172722f7a 100644
--- a/tensorflow/contrib/lite/profiling/BUILD
+++ b/tensorflow/contrib/lite/profiling/BUILD
@@ -20,7 +20,6 @@ cc_test(
     srcs = ["profiler_test.cc"],
     copts = ["-DTFLITE_PROFILING_ENABLED"],
     defines = ["TFLITE_PROFILING_ENABLED"],
-    tags = ["no_oss"],
     deps = [
         ":profiler",
         "//tensorflow/contrib/lite/testing:util",
@@ -77,7 +76,6 @@ cc_test(
     srcs = ["profile_buffer_test.cc"],
     copts = ["-DTFLITE_PROFILING_ENABLED"],
     defines = ["TFLITE_PROFILING_ENABLED"],
-    tags = ["no_oss"],
     deps = [
         ":profile_buffer",
         "//tensorflow/contrib/lite/testing:util",
-- 
cgit v1.2.3


From f0b0f4d485a60e703212a3f14d84d3e479504bd0 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy 
Date: Fri, 20 Jul 2018 22:56:57 -0700
Subject: Fix more typos.

---
 tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh | 8 ++++----
 tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 9883bb622c..47e0e5dd59 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -57,8 +57,8 @@ PY_TEST_DIR="py_test_dir"
 
 SKIP_TEST=0
 RELEASE_BUILD=0
-TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/... \
-    //${PY_TEST_DIR}tensorflow/contrib/... "
+TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/... \
+    //${PY_TEST_DIR}/tensorflow/contrib/... "
 
 # --skip_test            Skip running tests
 # --enable_remote_cache  Add options to enable remote cache for build and test
@@ -71,8 +71,8 @@ for ARG in "$@"; do
     --skip_test) SKIP_TEST=1 ;;
     --enable_remote_cache) set_remote_cache_options ;;
     --release_build) RELEASE_BUILD=1 ;;
-    --test_core_only) TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/..." ;;
-    --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}tensorflow/contrib/..." ;;
+    --test_core_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." ;;
+    --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/contrib/..." ;;
     *)
   esac
 done
diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
index 57463e3366..e3eee11080 100644
--- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh
@@ -57,8 +57,8 @@ PY_TEST_DIR="py_test_dir"
 
 SKIP_TEST=0
 RELEASE_BUILD=0
-TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/... \
-  //${PY_TEST_DIR}tensorflow/contrib/... "
+TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/... \
+  //${PY_TEST_DIR}/tensorflow/contrib/... "
 
 # --skip_test            Skip running tests
 # --enable_remote_cache  Add options to enable remote cache for build and test
@@ -71,8 +71,8 @@ for ARG in "$@"; do
     --skip_test) SKIP_TEST=1 ;;
     --enable_remote_cache) set_remote_cache_options ;;
     --release_build) RELEASE_BUILD=1 ;;
-    --test_core_only) TEST_TARGET="//${PY_TEST_DIR}tensorflow/python/..." ;;
-    --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}tensorflow/contrib/..." ;;
+    --test_core_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." ;;
+    --test_contrib_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/contrib/..." ;;
     *)
   esac
 done
-- 
cgit v1.2.3


From 19354d18bb2ce412409f4d79ec4fa23dd9057d8b Mon Sep 17 00:00:00 2001
From: Shashi Shekhar 
Date: Fri, 20 Jul 2018 23:21:37 -0700
Subject: Add a simple test for benchmark.

PiperOrigin-RevId: 205497641
---
 tensorflow/contrib/lite/tools/benchmark/BUILD      | 33 ++++++++--
 .../lite/tools/benchmark/benchmark_model.cc        | 12 +++-
 .../contrib/lite/tools/benchmark/benchmark_model.h |  7 +-
 .../contrib/lite/tools/benchmark/benchmark_test.cc | 74 ++++++++++++++++++++++
 .../lite/tools/benchmark/benchmark_tflite_model.cc |  6 +-
 .../lite/tools/benchmark/benchmark_tflite_model.h  |  4 +-
 6 files changed, 121 insertions(+), 15 deletions(-)
 create mode 100644 tensorflow/contrib/lite/tools/benchmark/benchmark_test.cc

diff --git a/tensorflow/contrib/lite/tools/benchmark/BUILD b/tensorflow/contrib/lite/tools/benchmark/BUILD
index 810e25961f..2cb07eb6ec 100644
--- a/tensorflow/contrib/lite/tools/benchmark/BUILD
+++ b/tensorflow/contrib/lite/tools/benchmark/BUILD
@@ -10,11 +10,16 @@ load("//tensorflow/contrib/lite:build_def.bzl", "tflite_copts")
 
 common_copts = ["-Wall"] + tflite_copts()
 
+cc_library(
+    name = "logging",
+    hdrs = ["logging.h"],
+    copts = common_copts,
+)
+
 cc_binary(
     name = "benchmark_model",
     srcs = [
         "benchmark_main.cc",
-        "logging.h",
     ],
     copts = common_copts,
     linkopts = tflite_linkopts() + select({
@@ -26,6 +31,26 @@ cc_binary(
     }),
     deps = [
         ":benchmark_tflite_model_lib",
+        ":logging",
+    ],
+)
+
+cc_test(
+    name = "benchmark_test",
+    srcs = ["benchmark_test.cc"],
+    args = [
+        "--graph=$(location //tensorflow/contrib/lite:testdata/multi_add.bin)",
+    ],
+    data = ["//tensorflow/contrib/lite:testdata/multi_add.bin"],
+    tags = [
+        "tflite_not_portable_android",
+        "tflite_not_portable_ios",
+    ],
+    deps = [
+        ":benchmark_tflite_model_lib",
+        ":command_line_flags",
+        "//tensorflow/contrib/lite/testing:util",
+        "@com_google_googletest//:gtest",
     ],
 )
 
@@ -40,7 +65,6 @@ cc_test(
     name = "command_line_flags_test",
     srcs = ["command_line_flags_test.cc"],
     copts = common_copts,
-    tags = ["no_oss"],
     visibility = ["//visibility:private"],
     deps = [
         ":command_line_flags",
@@ -59,6 +83,7 @@ cc_library(
     copts = common_copts,
     deps = [
         ":benchmark_model_lib",
+        ":logging",
         "//tensorflow/contrib/lite:framework",
         "//tensorflow/contrib/lite:string_util",
         "//tensorflow/contrib/lite/kernels:builtin_ops",
@@ -71,23 +96,23 @@ cc_library(
     name = "benchmark_params",
     srcs = [
         "benchmark_params.cc",
-        "logging.h",
     ],
     hdrs = ["benchmark_params.h"],
     copts = common_copts,
+    deps = [":logging"],
 )
 
 cc_library(
     name = "benchmark_model_lib",
     srcs = [
         "benchmark_model.cc",
-        "logging.h",
     ],
     hdrs = ["benchmark_model.h"],
     copts = common_copts,
     deps = [
         ":benchmark_params",
         ":command_line_flags",
+        ":logging",
         "//tensorflow/contrib/lite:framework",
         "//tensorflow/contrib/lite:string_util",
         "//tensorflow/contrib/lite/kernels:builtin_ops",
diff --git a/tensorflow/contrib/lite/tools/benchmark/benchmark_model.cc b/tensorflow/contrib/lite/tools/benchmark/benchmark_model.cc
index 19b9a9c7ba..f86c0445b0 100644
--- a/tensorflow/contrib/lite/tools/benchmark/benchmark_model.cc
+++ b/tensorflow/contrib/lite/tools/benchmark/benchmark_model.cc
@@ -84,7 +84,7 @@ std::vector BenchmarkModel::GetFlags() {
   };
 }
 
-void BenchmarkModel::LogFlags() {
+void BenchmarkModel::LogParams() {
   TFLITE_LOG(INFO) << "Num runs: [" << params_.Get("num_runs") << "]";
   TFLITE_LOG(INFO) << "Inter-run delay (seconds): ["
                    << params_.Get("run_delay") << "]";
@@ -122,12 +122,18 @@ Stat BenchmarkModel::Run(int num_times, RunType run_type) {
   return run_stats;
 }
 
+bool BenchmarkModel::ValidateParams() { return true; }
+
 void BenchmarkModel::Run(int argc, char **argv) {
   if (!ParseFlags(argc, argv)) {
     return;
   }
+  Run();
+}
 
-  LogFlags();
+void BenchmarkModel::Run() {
+  ValidateParams();
+  LogParams();
 
   listeners_.OnBenchmarkStart(params_);
   int64_t initialization_start_us = profiling::time::NowMicros();
@@ -155,7 +161,7 @@ bool BenchmarkModel::ParseFlags(int argc, char **argv) {
     TFLITE_LOG(ERROR) << usage;
     return false;
   }
-  return ValidateFlags();
+  return true;
 }
 
 }  // namespace benchmark
diff --git a/tensorflow/contrib/lite/tools/benchmark/benchmark_model.h b/tensorflow/contrib/lite/tools/benchmark/benchmark_model.h
index 3c7063b2d4..677a1ee68c 100644
--- a/tensorflow/contrib/lite/tools/benchmark/benchmark_model.h
+++ b/tensorflow/contrib/lite/tools/benchmark/benchmark_model.h
@@ -137,16 +137,17 @@ class BenchmarkModel {
   BenchmarkModel();
   BenchmarkModel(BenchmarkParams params) : params_(std::move(params)) {}
   virtual ~BenchmarkModel() {}
-  bool ParseFlags(int argc, char** argv);
   virtual void Init() = 0;
   void Run(int argc, char** argv);
+  virtual void Run();
   void AddListener(BenchmarkListener* listener) {
     listeners_.AddListener(listener);
   }
 
  protected:
-  virtual void LogFlags();
-  virtual bool ValidateFlags() { return true; }
+  virtual void LogParams();
+  virtual bool ValidateParams();
+  bool ParseFlags(int argc, char** argv);
   virtual std::vector GetFlags();
   virtual uint64_t ComputeInputBytes() = 0;
   virtual tensorflow::Stat Run(int num_times, RunType run_type);
diff --git a/tensorflow/contrib/lite/tools/benchmark/benchmark_test.cc b/tensorflow/contrib/lite/tools/benchmark/benchmark_test.cc
new file mode 100644
index 0000000000..b697bb394d
--- /dev/null
+++ b/tensorflow/contrib/lite/tools/benchmark/benchmark_test.cc
@@ -0,0 +1,74 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include "tensorflow/contrib/lite/testing/util.h"
+#include "tensorflow/contrib/lite/tools/benchmark/benchmark_tflite_model.h"
+#include "tensorflow/contrib/lite/tools/benchmark/command_line_flags.h"
+
+namespace {
+const std::string* g_model_path = nullptr;
+}
+
+namespace tflite {
+namespace benchmark {
+namespace {
+
+BenchmarkParams CreateParams() {
+  BenchmarkParams params;
+  params.AddParam("num_runs", BenchmarkParam::Create(2));
+  params.AddParam("run_delay", BenchmarkParam::Create(-1.0f));
+  params.AddParam("num_threads", BenchmarkParam::Create(1));
+  params.AddParam("benchmark_name", BenchmarkParam::Create(""));
+  params.AddParam("output_prefix", BenchmarkParam::Create(""));
+  params.AddParam("warmup_runs", BenchmarkParam::Create(1));
+  params.AddParam("graph", BenchmarkParam::Create(*g_model_path));
+  params.AddParam("input_layer", BenchmarkParam::Create(""));
+  params.AddParam("input_layer_shape", BenchmarkParam::Create(""));
+  params.AddParam("use_nnapi", BenchmarkParam::Create(false));
+  return params;
+}
+
+TEST(BenchmarkTest, DoesntCrash) {
+  ASSERT_THAT(g_model_path, testing::NotNull());
+
+  BenchmarkTfLiteModel benchmark(CreateParams());
+  benchmark.Run();
+}
+
+}  // namespace
+}  // namespace benchmark
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  std::string model_path;
+  std::vector flags = {
+      tflite::Flag::CreateFlag("graph", &model_path, "Path to model file.")};
+  g_model_path = &model_path;
+  const bool parse_result =
+      tflite::Flags::Parse(&argc, const_cast(argv), flags);
+  if (!parse_result) {
+    std::cerr << tflite::Flags::Usage(argv[0], flags);
+    return 1;
+  }
+
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/contrib/lite/tools/benchmark/benchmark_tflite_model.cc
index 73affc26b0..7f97f5d0cd 100644
--- a/tensorflow/contrib/lite/tools/benchmark/benchmark_tflite_model.cc
+++ b/tensorflow/contrib/lite/tools/benchmark/benchmark_tflite_model.cc
@@ -198,8 +198,8 @@ std::vector BenchmarkTfLiteModel::GetFlags() {
   return flags;
 }
 
-void BenchmarkTfLiteModel::LogFlags() {
-  BenchmarkModel::LogFlags();
+void BenchmarkTfLiteModel::LogParams() {
+  BenchmarkModel::LogParams();
   TFLITE_LOG(INFO) << "Graph: [" << params_.Get("graph") << "]";
   TFLITE_LOG(INFO) << "Input layers: ["
                    << params_.Get("input_layer") << "]";
@@ -208,7 +208,7 @@ void BenchmarkTfLiteModel::LogFlags() {
   TFLITE_LOG(INFO) << "Use nnapi : [" << params_.Get("use_nnapi") << "]";
 }
 
-bool BenchmarkTfLiteModel::ValidateFlags() {
+bool BenchmarkTfLiteModel::ValidateParams() {
   if (params_.Get("graph").empty()) {
     TFLITE_LOG(ERROR)
         << "Please specify the name of your TF Lite input file with --graph";
diff --git a/tensorflow/contrib/lite/tools/benchmark/benchmark_tflite_model.h b/tensorflow/contrib/lite/tools/benchmark/benchmark_tflite_model.h
index 50cc3f24b3..9931dcbafe 100644
--- a/tensorflow/contrib/lite/tools/benchmark/benchmark_tflite_model.h
+++ b/tensorflow/contrib/lite/tools/benchmark/benchmark_tflite_model.h
@@ -54,8 +54,8 @@ class BenchmarkTfLiteModel : public BenchmarkModel {
   BenchmarkTfLiteModel(BenchmarkParams params);
 
   std::vector GetFlags() override;
-  void LogFlags() override;
-  bool ValidateFlags() override;
+  void LogParams() override;
+  bool ValidateParams() override;
   uint64_t ComputeInputBytes() override;
   void Init() override;
   void RunImpl() override;
-- 
cgit v1.2.3


From db866b6fb02a3b7855f521fdb4175b2d1b217259 Mon Sep 17 00:00:00 2001
From: qwertWZ 
Date: Sat, 21 Jul 2018 15:22:29 +0800
Subject: Add a missing left bracket in comments

---
 tensorflow/contrib/rnn/python/ops/rnn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/rnn/python/ops/rnn.py b/tensorflow/contrib/rnn/python/ops/rnn.py
index 2f0caadda3..0266b72dcb 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn.py
@@ -175,7 +175,7 @@ def stack_bidirectional_dynamic_rnn(cells_fw,
   Returns:
     A tuple (outputs, output_state_fw, output_state_bw) where:
       outputs: Output `Tensor` shaped:
-        `batch_size, max_time, layers_output]`. Where layers_output
+        `[batch_size, max_time, layers_output]`. Where layers_output
         are depth-concatenated forward and backward outputs.
       output_states_fw is the final states, one tensor per layer,
         of the forward rnn.
-- 
cgit v1.2.3


From fb8d1ca4eaefe58d42c27b6fc676f64f137f4675 Mon Sep 17 00:00:00 2001
From: Ray Kim 
Date: Sat, 21 Jul 2018 21:42:15 +0900
Subject: fixed build error on gcc-7

---
 tensorflow/compiler/xla/service/gpu/xfeed_queue.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/service/gpu/xfeed_queue.h b/tensorflow/compiler/xla/service/gpu/xfeed_queue.h
index 737c7eb025..dd46ff433b 100644
--- a/tensorflow/compiler/xla/service/gpu/xfeed_queue.h
+++ b/tensorflow/compiler/xla/service/gpu/xfeed_queue.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_XFEED_QUEUE_H_
 
 #include 
+#include 
 #include 
 
 #include "tensorflow/core/platform/mutex.h"
-- 
cgit v1.2.3


From aa5d3126ced57f8117678bb1cb5cc41e2a72eb9a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Sat, 21 Jul 2018 10:11:49 -0700
Subject: Support while ops to be CSEd in HLO. Do so if they have same bodies,
 conditions, and init conditions.

PiperOrigin-RevId: 205524367
---
 tensorflow/compiler/xla/service/hlo_cse_test.cc    | 179 ++++++++++++++++++++-
 tensorflow/compiler/xla/service/hlo_instruction.cc |  11 +-
 2 files changed, 187 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_cse_test.cc b/tensorflow/compiler/xla/service/hlo_cse_test.cc
index 76b9c66651..90fbaa37c5 100644
--- a/tensorflow/compiler/xla/service/hlo_cse_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_cse_test.cc
@@ -239,7 +239,7 @@ TEST_F(HloCseTest, IdenticalInstructions) {
   EXPECT_EQ(5, computation->instruction_count());
   EXPECT_THAT(tuple, op::Tuple(exp1, exp2, exp3));
 
-  HloCSE cse(/*is_layout_sensitive=*/false);
+  HloCSE cse(/*is_layout_sensitive=*/true);
   EXPECT_TRUE(cse.Run(module.get()).ValueOrDie());
 
   EXPECT_EQ(3, computation->instruction_count());
@@ -248,6 +248,183 @@ TEST_F(HloCseTest, IdenticalInstructions) {
   EXPECT_THAT(tuple, op::Tuple(first_operand, first_operand, first_operand));
 }
 
+// Test two identical while loops with same inputs
+TEST_F(HloCseTest, WhileLoopsIdenticalConditionsAndBodiesSameInput) {
+  auto module = ParseHloString(R"(
+    HloModule WhileLoopsIdenticalConditionsAndBodiesSameInput
+
+    %body (param: (f32[], f32[])) -> (f32[], f32[]) {
+      %param = (f32[], f32[]) parameter(0)
+      %get-tuple-element = f32[] get-tuple-element((f32[], f32[]) %param),
+index=0 %get-tuple-element.1 = f32[] get-tuple-element((f32[], f32[]) %param),
+index=1 %add = f32[] add(f32[] %get-tuple-element, f32[] %get-tuple-element.1)
+      ROOT %tuple = (f32[], f32[]) tuple(f32[] %get-tuple-element, f32[] %add)
+    }
+
+    %condition (param.1: (f32[], f32[])) -> pred[] {
+      %param.1 = (f32[], f32[]) parameter(0)
+      ROOT %constant = pred[] constant(false)
+    }
+
+    %condition.1 (param.2: (f32[], f32[])) -> pred[] {
+      %param.2 = (f32[], f32[]) parameter(0)
+      ROOT %constant.1 = pred[] constant(false)
+    }
+
+    ENTRY %WhileLoopsIdenticalConditionsAndBodiesSameInput () -> (f32[], f32[])
+{ %constant.2 = f32[] constant(1) %constant.3 = f32[] constant(2) %tuple.1 =
+(f32[], f32[]) tuple(f32[] %constant.2, f32[] %constant.3) %while = (f32[],
+f32[]) while((f32[], f32[]) %tuple.1), condition=%condition, body=%body ROOT
+%while.1 = (f32[], f32[]) while((f32[], f32[]) %tuple.1),
+condition=%condition.1, body=%body
+    }
+    )")
+                    .ValueOrDie();
+
+  auto computation = module->entry_computation();
+
+  EXPECT_EQ(5, computation->instruction_count());
+  HloCSE cse(true);
+  EXPECT_TRUE(cse.Run(module.get()).ValueOrDie());
+  EXPECT_EQ(4, computation->instruction_count());
+}
+
+// Test two while loops with same conditions, same inputs, but different
+// bodies
+TEST_F(HloCseTest, WhileLoopsIdenticalConditionsSameInputAndDifferentBodies) {
+  auto module = ParseHloString(R"(
+    HloModule WhileLoopsIdenticalConditionsSameInputAndDifferentBodies
+
+    %body (param: (f32[], f32[])) -> (f32[], f32[]) {
+      %param = (f32[], f32[]) parameter(0)
+      %get-tuple-element = f32[] get-tuple-element((f32[], f32[]) %param),
+index=0 %get-tuple-element.1 = f32[] get-tuple-element((f32[], f32[]) %param),
+index=1 %add = f32[] add(f32[] %get-tuple-element, f32[] %get-tuple-element.1)
+      ROOT %tuple = (f32[], f32[]) tuple(f32[] %get-tuple-element, f32[] %add)
+    }
+
+    %body2 (param.1: (f32[], f32[])) -> (f32[], f32[]) {
+      %param.1 = (f32[], f32[]) parameter(0)
+      %get-tuple-element.2 = f32[] get-tuple-element((f32[], f32[]) %param.1),
+index=0 %get-tuple-element.3 = f32[] get-tuple-element((f32[], f32[]) %param.1),
+index=1 %sub = f32[] subtract(f32[] %get-tuple-element.2, f32[]
+%get-tuple-element.3) ROOT %tuple.2 = (f32[], f32[]) tuple(f32[]
+%get-tuple-element.2, f32[] %sub)
+    }
+
+    %condition (param.2: (f32[], f32[])) -> pred[] {
+      %param.2 = (f32[], f32[]) parameter(0)
+      ROOT %constant = pred[] constant(false)
+    }
+
+    %condition.1 (param.3: (f32[], f32[])) -> pred[] {
+      %param.3 = (f32[], f32[]) parameter(0)
+      ROOT %constant.1 = pred[] constant(false)
+    }
+
+    ENTRY %WhileLoopsIdenticalConditionsSameInputAndDifferentBodies () ->
+(f32[], f32[]) { %constant.2 = f32[] constant(1) %constant.3 = f32[] constant(2)
+      %tuple.1 = (f32[], f32[]) tuple(f32[] %constant.2, f32[] %constant.3)
+      %while = (f32[], f32[]) while((f32[], f32[]) %tuple.1),
+condition=%condition, body=%body ROOT %while.1 = (f32[], f32[]) while((f32[],
+f32[]) %tuple.1), condition=%condition.1, body=%body2
+    }
+    )")
+                    .ValueOrDie();
+
+  auto computation = module->entry_computation();
+
+  EXPECT_EQ(5, computation->instruction_count());
+  HloCSE cse(true);
+  EXPECT_FALSE(cse.Run(module.get()).ValueOrDie());
+  EXPECT_EQ(5, computation->instruction_count());
+}
+
+// Test two identical while loops with different inputs
+TEST_F(HloCseTest, WhileLoopsIdenticalConditionsAndBodiesDifferentInput) {
+  auto module = ParseHloString(R"(
+    HloModule WhileLoopsIdenticalConditionsAndBodiesDifferentInput
+
+    %body (param: (f32[], f32[])) -> (f32[], f32[]) {
+      %param = (f32[], f32[]) parameter(0)
+      %get-tuple-element = f32[] get-tuple-element((f32[], f32[]) %param),
+index=0 %get-tuple-element.1 = f32[] get-tuple-element((f32[], f32[]) %param),
+index=1 %add = f32[] add(f32[] %get-tuple-element, f32[] %get-tuple-element.1)
+      ROOT %tuple = (f32[], f32[]) tuple(f32[] %get-tuple-element, f32[] %add)
+    }
+
+    %condition (param.1: (f32[], f32[])) -> pred[] {
+      %param.1 = (f32[], f32[]) parameter(0)
+      ROOT %constant = pred[] constant(false)
+    }
+
+    %condition.1 (param.2: (f32[], f32[])) -> pred[] {
+      %param.2 = (f32[], f32[]) parameter(0)
+      ROOT %constant.1 = pred[] constant(false)
+    }
+
+    ENTRY %WhileLoopsIdenticalConditionsAndBodiesDifferentInput () -> (f32[],
+f32[]) { %constant.2 = f32[] constant(1) %constant.3 = f32[] constant(2)
+      %tuple.1 = (f32[], f32[]) tuple(f32[] %constant.2, f32[] %constant.3)
+      %while = (f32[], f32[]) while((f32[], f32[]) %tuple.1),
+condition=%condition, body=%body %constant.4 = f32[] constant(1) %constant.5 =
+f32[] constant(2) %tuple.2 = (f32[], f32[]) tuple(f32[] %constant.4, f32[]
+%constant.5) ROOT %while.1 = (f32[], f32[]) while((f32[], f32[]) %tuple.2),
+condition=%condition.1, body=%body
+    }
+
+    )")
+                    .ValueOrDie();
+
+  auto computation = module->entry_computation();
+
+  EXPECT_EQ(8, computation->instruction_count());
+  HloCSE cse(true);
+  EXPECT_FALSE(cse.Run(module.get()).ValueOrDie());
+  EXPECT_EQ(8, computation->instruction_count());
+}
+
+// Test two while loops with identical bodies and same inputs, but different
+// conditions
+TEST_F(HloCseTest, WhileLoopsIdenticalBodiesAndInputDifferntConditions) {
+  auto module = ParseHloString(R"(
+    HloModule WhileLoopsIdenticalBodiesAndInputDifferntConditions
+
+    %body (param: (f32[], f32[])) -> (f32[], f32[]) {
+      %param = (f32[], f32[]) parameter(0)
+      %get-tuple-element = f32[] get-tuple-element((f32[], f32[]) %param),
+index=0 %get-tuple-element.1 = f32[] get-tuple-element((f32[], f32[]) %param),
+index=1 %add = f32[] add(f32[] %get-tuple-element, f32[] %get-tuple-element.1)
+      ROOT %tuple = (f32[], f32[]) tuple(f32[] %get-tuple-element, f32[] %add)
+    }
+
+    %condition (param.1: (f32[], f32[])) -> pred[] {
+      %param.1 = (f32[], f32[]) parameter(0)
+      ROOT %constant = pred[] constant(false)
+    }
+
+    %condition.1 (param.2: (f32[], f32[])) -> pred[] {
+      %param.2 = (f32[], f32[]) parameter(0)
+      ROOT %constant.1 = pred[] constant(true)
+    }
+
+    ENTRY %WhileLoopsIdenticalBodiesAndInputDifferntConditions () -> (f32[],
+f32[]) { %constant.2 = f32[] constant(1) %constant.3 = f32[] constant(2)
+      %tuple.1 = (f32[], f32[]) tuple(f32[] %constant.2, f32[] %constant.3)
+      %while = (f32[], f32[]) while((f32[], f32[]) %tuple.1),
+condition=%condition, body=%body ROOT %while.1 = (f32[], f32[]) while((f32[],
+f32[]) %tuple.1), condition=%condition.1, body=%body
+    })")
+                    .ValueOrDie();
+
+  auto computation = module->entry_computation();
+
+  EXPECT_EQ(5, computation->instruction_count());
+  HloCSE cse(true);
+  EXPECT_FALSE(cse.Run(module.get()).ValueOrDie());
+  EXPECT_EQ(5, computation->instruction_count());
+}
+
 TEST_F(HloCseTest, IdenticalInstructionsDifferentLayoutsSensitive) {
   // Test that two identical instructions with different layouts are *not*
   // commoned if the pass is layout sensitive.
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 7685c822f4..8b9bdd2f46 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -1522,8 +1522,7 @@ bool HloInstruction::IdenticalSlowPath(
     case HloOpcode::kTupleSelect:
       return true;
 
-    // These opcodes have complex or special behavior so just return false.
-    case HloOpcode::kWhile:
+    // This opcode has complex or special behavior so just return false.
     case HloOpcode::kAfterAll:
       return false;
 
@@ -1539,6 +1538,14 @@ bool HloInstruction::IdenticalSlowPath(
       return eq_computations(true_computation(), other.true_computation()) &&
              eq_computations(false_computation(), other.false_computation());
 
+    case HloOpcode::kWhile: {
+      if (eq_computations(while_body(), other.while_body()) &&
+          eq_computations(while_condition(), other.while_condition())) {
+        return true;
+      }
+      return false;
+    }
+
     case HloOpcode::kDomain:
       return operand_side_metadata().Matches(other.operand_side_metadata()) &&
              user_side_metadata().Matches(other.user_side_metadata());
-- 
cgit v1.2.3


From 2279279fd15369e361a02fb09a1df41e08a34aae Mon Sep 17 00:00:00 2001
From: Brennan Saeta 
Date: Sat, 21 Jul 2018 12:55:03 -0700
Subject: [tf.data / Bigtable] Document use of the Cloud Bigtable API

PiperOrigin-RevId: 205530581
---
 tensorflow/contrib/bigtable/README.md | 344 +++++++++++++++++++++++++++++++++-
 1 file changed, 342 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/bigtable/README.md b/tensorflow/contrib/bigtable/README.md
index ef3c60069e..b2c0460f04 100644
--- a/tensorflow/contrib/bigtable/README.md
+++ b/tensorflow/contrib/bigtable/README.md
@@ -1,10 +1,350 @@
 # Bigtable #
 
-[Google Cloud Bigtable](https://cloud.google.com/bigtable/) is a high
+[Cloud Bigtable](https://cloud.google.com/bigtable/) is a high
 performance storage system that can store and serve training data. This contrib
 package contains an experimental integration with TensorFlow.
 
 > **Status: Highly experimental.** The current implementation is very much in
 > flux. Please use at your own risk! :-)
 
-
+The TensorFlow integration with Cloud Bigtable is optimized for common
+TensorFlow usage and workloads. It is currently optimized for reading from Cloud
+Bigtable at high speed, in particular to feed modern accelerators. For
+general-purpose Cloud Bigtable
+APIs, see the [official Cloud Bigtable client library documentation][clientdoc].
+
+[clientdoc]:  https://cloud.google.com/bigtable/docs/reference/libraries
+
+## Sample Use
+
+There are three main reading styles supported by the `BigTable` class:
+
+ 1. **Reading keys**: Read only the row keys in a table. Keys are returned in
+    sorted order from the table. Most key reading operations retrieve all keys
+    in a contiguous range, however the `sample_keys` operation skips keys, and
+    operates on the whole table (and not a contiguous subset).
+ 2. **Retrieving a row's values**: Given a row key, look up the data associated
+    with a defined set of columns. This operation takes advantage of Cloud
+    Bigtable's low-latency and excellent support for random access.
+ 3. **Scanning ranges**: Given a contiguous range of rows retrieve both the row
+    key and the data associated with a fixed set of columns. This operation
+    takes advantage of Cloud Bigtable's high throughput scans, and is the most
+    efficient way to read data.
+
+When using the Cloud Bigtable API, the workflow is:
+
+ 1. Create a `BigtableClient` object.
+ 2. Use the `BigtableClient` to create `BigTable` objects corresponding to each
+    table in the Bigtable instance you would like to access.
+ 3. Call methods on the `BigTable` object to create `tf.data.Dataset`s to
+    retrieve data.
+
+The following is an example for how to read all row keys with the prefix
+`train-`.
+
+```python
+import tensorflow as tf
+
+GCP_PROJECT_ID = ''
+BIGTABLE_INSTANCE_ID = ''
+BIGTABLE_TABLE_NAME = ''
+PREFIX = 'train-'
+
+def main():
+  client = tf.contrib.cloud.BigtableClient(GCP_PROJECT_ID, BIGTABLE_INSTANCE_ID)
+  table = client.table(BIGTABLE_TABLE_NAME)
+  dataset = table.keys_by_prefix_dataset(PREFIX)
+  iterator = dataset.make_initializable_iterator()
+  get_next_op = iterator.get_next()
+
+  with tf.Session() as sess:
+    print('Initializing the iterator.')
+    sess.run(iterator.initializer)
+    print('Retrieving rows:')
+    row_index = 0
+    while True:
+      try:
+        row_key = sess.run(get_next_op)
+        print('Row key %d: %s' % (row_index, row_key))
+        row_index += 1
+      except tf.errors.OutOfRangeError:
+        print('Finished reading data!')
+        break
+
+if __name__ == '__main__':
+  main()
+
+```
+
+### Reading row keys
+
+Read only the row keys in a table. Keys are returned in sorted order from the
+table. Most key reading operations retrieve all keys in a contiguous range,
+however the `sample_keys` operation skips keys, and operates on the whole table
+(and not a contiguous subset).
+
+There are 3 methods to retrieve row keys:
+
+ - `table.keys_by_range_dataset(start, end)`: Retrieve row keys starting with
+   `start`, and ending with `end`. The range is "half-open", and thus it
+   includes `start` if `start` is present in the table. It does not include
+   `end`.
+ - `table.keys_by_prefix_dataset(prefix)`: Retrieves all row keys that start
+   with `prefix`. It includes the row key `prefix` if present in the table.
+ - `table.sample_keys()`: Retrieves a sampling of keys from the underlying
+   table. This is often useful in conjunction with parallel scans.
+
+### Reading cell values given a row key
+
+Given a dataset producing row keys, you can use the `table.lookup_columns`
+transformation to retrieve values. Example:
+
+```python
+key_dataset = tf.data.Dataset.from_tensor_slices([
+    'row_key_1',
+    'other_row_key',
+    'final_row_key',
+])
+values_dataset = key_dataset.apply(
+  table.lookup_columns(('my_column_family', 'column_name'),
+                       ('other_cf', 'col')))
+training_data = values_dataset.map(my_parsing_function)  # ...
+```
+
+### Scanning ranges
+Given a contiguous range of rows retrieve both the row key and the data
+associated with a fixed set of columns. Scanning is the most efficient way to
+retrieve data from Cloud Bigtable and is thus a very common API for high
+performance data pipelines. To construct a scanning `tf.data.Dataset` from a
+`BigTable` object, call one of the following methods:
+
+ - `table.scan_prefix(prefix, ...)`
+ - `table.scan_range(start, end, ...)`
+ - `table.parallel_scan_prefix(prefix, ...)`
+ - `table.parallel_scan_range(start, end, ...)`
+
+Aside from the specification of the contiguous range of rows, they all take the
+following arguments:
+
+ - `probability`: (Optional.) A float between 0 (exclusive) and 1 (inclusive).
+      A non-1 value indicates to probabilistically sample rows with the
+      provided probability.
+ - `columns`: The columns to read. (See below.)
+ - `**kwargs`: The columns to read. (See below.)
+
+In addition the two parallel operations accept the following optional argument:
+`num_parallel_scans` which configures the number of parallel Cloud Bigtable scan
+operations to run. A reasonable default is automatically chosen for small
+Cloud Bigtable clusters. If you have a large cluster, or an extremely demanding
+workload, you can tune this value to optimize performance.
+
+#### Specifying columns to read when scanning
+
+All of the scan operations allow you to specify the column family and columns
+in the same ways.
+
+##### Using `columns`
+
+The first way to specify the data to read is via the `columns` parameter. The
+value should be a tuple (or list of tuples) of strings. The first string in the
+tuple is the column family, and the second string in the tuple is the column
+qualifier.
+
+##### Using `**kwargs`
+
+The second way to specify the data to read is via the `**kwargs` parameter,
+which you can use to specify keyword arguments corresponding to the columns that
+you want to read. The keyword to use is the column family name, and the argument
+value should be either a string, or a tuple of strings, specifying the column
+qualifiers (column names).
+
+Although using `**kwargs` has the advantage of requiring less typing, it is not
+future-proof in all cases. (If we add a new parameter to the scan functions that
+has the same name as your column family, your code will break.)
+
+##### Examples
+
+Below are two equivalent snippets for how to specify which columns to read:
+
+```python
+ds1 = table.scan_range("row_start", "row_end", columns=[("cfa", "c1"),
+                                                        ("cfa", "c2"),
+                                                        ("cfb", "c3")])
+ds2 = table.scan_range("row_start", "row_end", cfa=["c1", "c2"], cfb="c3")
+```
+
+In this example, we are reading 3 columns from a total of 2 column families.
+From the `cfa` column family, we are reading columns `c1`, and `c2`. From the
+second column family (`cfb`), we are reading `c3`. Both `ds1` and `ds2` will
+output elements of the following types (`tf.string`, `tf.string`, `tf.string`,
+`tf.string`). The first `tf.string` is the row key, the second `tf.string` is
+the latest data in cell `cfa:c1`, the third corresponds to `cfa:c2`, and the
+final one is `cfb:c3`.
+
+#### Determinism when scanning
+
+While the non-parallel scan operations are fully deterministic, the parallel
+scan operations are not. If you would like to scan in parallel without losing
+determinism, you can build up the `parallel_interleave` yourself. As an example,
+say we wanted to scan all rows between `training_data_00000`, and
+`training_data_90000`, we can use the following code snippet:
+
+```python
+table = # ...
+columns = [('cf1', 'col1'), ('cf1', 'col2')]
+NUM_PARALLEL_READS = # ...
+ds = tf.data.Dataset.range(9).shuffle(10)
+def interleave_fn(index):
+  # Given a starting index, create 2 strings to be the start and end
+  start_idx = index
+  end_idx = index + 1
+  start_idx_str = tf.as_string(start_idx * 10000, width=5, fill='0')
+  end_idx_str = tf.as_string(end_idx * 10000, width=5, fill='0')
+  start = tf.string_join(['training_data_', start_idx_str])
+  end = tf.string_join(['training_data_', end_idx_str])
+  return table.scan_range(start_idx, end_idx, columns=columns)
+ds = ds.apply(tf.contrib.data.parallel_interleave(
+    interleave_fn, cycle_length=NUM_PARALLEL_READS, prefetch_input_elements=1))
+```
+
+> Note: you should divide up the key range into more sub-ranges for increased
+> parallelism.
+
+## Writing to Cloud Bigtable
+
+In order to simplify getting started, this package provides basic support for
+writing data into Cloud Bigtable.
+
+> Note: The implementation is not optimized for performance! Please consider
+> using alternative frameworks such as Apache Beam / Cloud Dataflow for
+> production workloads.
+
+Below is an example for how to write a trivial dataset into Cloud Bigtable.
+
+```python
+import tensorflow as tf
+
+GCP_PROJECT_ID = ''
+BIGTABLE_INSTANCE_ID = ''
+BIGTABLE_TABLE_NAME = ''
+COLUMN_FAMILY = ''
+COLUMN_QUALIFIER = ''
+
+def make_dataset():
+  """Makes a dataset to write to Cloud Bigtable."""
+  return tf.data.Dataset.from_tensor_slices([
+      'training_data_1',
+      'training_data_2',
+      'training_data_3',
+  ])
+
+def make_row_key_dataset():
+  """Makes a dataset of strings used for row keys.
+
+  The strings are of the form: `fake-data-` followed by a sequential counter.
+  For example, this dataset would contain the following elements:
+
+   - fake-data-00000001
+   - fake-data-00000002
+   - ...
+   - fake-data-23498103
+  """
+  counter_dataset = tf.contrib.data.Counter()
+  width = 8
+  row_key_prefix = 'fake-data-'
+  ds = counter_dataset.map(lambda index: tf.as_string(index,
+                                                      width=width,
+                                                      fill='0'))
+  ds = ds.map(lambda idx_str: tf.string_join([row_key_prefix, idx_str]))
+  return ds
+
+
+def main():
+  client = tf.contrib.cloud.BigtableClient(GCP_PROJECT_ID, BIGTABLE_INSTANCE_ID)
+  table = client.table(BIGTABLE_TABLE_NAME)
+  dataset = make_dataset()
+  index_dataset = make_row_key_dataset()
+  aggregate_dataset = tf.data.Dataset.zip((index_dataset, dataset))
+  write_op = table.write(aggregate_dataset, column_families=[COLUMN_FAMILY],
+                         columns=[COLUMN_QUALIFIER])
+
+  with tf.Session() as sess:
+    print('Starting transfer.')
+    sess.run(write_op)
+    print('Transfer complete.')
+
+if __name__ == '__main__':
+  main()
+```
+
+## Sample applications and architectures
+
+While most machine learning applications are well suited by a high performance
+distributed file system, there are certain applications where using Cloud
+Bigtable works extremely well.
+
+### Perfect Shuffling
+
+Normally, training data is stored in flat files, and a combination of
+(1) `tf.data.Dataset.interleave` (or `parallel_interleave`), (2)
+`tf.data.Dataset.shuffle`, and (3) writing the data in an unsorted order in the
+data files in the first place, provides enough randomization to ensure models
+train efficiently. However, if you would like perfect shuffling, you can use
+Cloud Bigtable's low-latency random access capabilities. Create a
+`tf.data.Dataset` that generates the keys in a perfectly random order (or read
+all the keys into memory and use a shuffle buffer sized to fit all of them for a
+perfect random shuffle using `tf.data.Dataset.shuffle`), and then use
+`lookup_columns` to retrieve the training data.
+
+### Distributed Reinforcement Learning
+
+Sophisticated reinforcement learning algorithms are commonly trained across a
+distributed cluster. (See [IMPALA by DeepMind][impala].) One part of the cluster
+runs self-play, while the other part of the cluster learns a new version of the
+model based on the training data generated by self-play. The new model version
+is then distributed to the self-play half of the cluster, and new training data
+is generated to continue the cycle.
+
+In such a configuration, because there is value in training on the freshest
+examples, a storage service like Cloud Bigtable can be used to store and
+serve the generated training data. When using Cloud Bigtable, there is no need
+to aggregate the examples into large batch files, but the examples can instead
+be written as soon as they are generated, and then retrieved at high speed.
+
+[impala]: https://arxiv.org/abs/1802.01561
+
+## Common Gotchas!
+
+### gRPC Certificates
+
+If you encounter a log line that includes the following:
+
+```
+"description":"Failed to load file", [...],
+"filename":"/usr/share/grpc/roots.pem"
+```
+
+you likely need to copy the [gRPC roots.pem file][grpcPem] to
+`/usr/share/grpc/roots.pem` on your local machine.
+
+[grpcPem]: https://github.com/grpc/grpc/blob/master/etc/roots.pem
+
+### Permission denied errors
+
+The TensorFlow Cloud Bigtable client will search for credentials to use in the
+process's environment. It will use the first credentials it finds if multiple
+are available.
+
+ - **Compute Engine**: When running on Compute Engine, the client will often use
+   the service account from the virtual machine's metadata service. Be sure to
+   authorize your Compute Engine VM to have access to the Cloud Bigtable service
+   when creating your VM.
+ - **Cloud TPU**: Your Cloud TPUs run with the designated Cloud TPU service
+   account dedicated to your GCP project. Ensure the service account has been
+   authorized via the Cloud Console to access your Cloud Bigtable instances.
+
+### `BigTable` vs Bigtable?
+
+Cloud Bigtable is spelled with a lower-case (aka common) `t`. The Python class
+`BigTable`, however is short for `BigtableTable`, and thus uses an upper-case
+(aka capital) `T`.
-- 
cgit v1.2.3


From b7bbe64b5fde8a909d4410f758244a6703f84780 Mon Sep 17 00:00:00 2001
From: Miguel Mota 
Date: Sat, 21 Jul 2018 19:50:58 -0700
Subject: Update readme with working bazel config flag

---
 tensorflow/go/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/go/README.md b/tensorflow/go/README.md
index e251356ec8..288a32530a 100644
--- a/tensorflow/go/README.md
+++ b/tensorflow/go/README.md
@@ -46,7 +46,7 @@ from source.
     ```sh
     cd ${GOPATH}/src/github.com/tensorflow/tensorflow
     ./configure
-    bazel build --config opt //tensorflow:libtensorflow.so
+    bazel build -c opt //tensorflow:libtensorflow.so
     ```
 
     This can take a while (tens of minutes, more if also building for GPU).
-- 
cgit v1.2.3


From e578eb00f2eedae6ce25eabdbf349581f22c4df9 Mon Sep 17 00:00:00 2001
From: Anirudh Koul 
Date: Sat, 21 Jul 2018 20:23:15 -0700
Subject: Fixed typo in TOCO command line doc

---
 tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md b/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md
index 18b7848db8..4bf47aa3c4 100644
--- a/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md
+++ b/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md
@@ -36,7 +36,7 @@ There are two approaches to running TOCO via command line.
 *   `tflite_convert`: Starting from TensorFlow 1.9, the command-line tool
     `tflite_convert` will be installed as part of the Python package. All of the
     examples below use `tflite_convert` for simplicity.
-    *   Example: `tflite --output_file=...`
+    *   Example: `tflite_convert --output_file=...`
 *   `bazel`: In order to run the latest version of TOCO, [clone the TensorFlow
     repository](https://www.tensorflow.org/install/install_sources#clone_the_tensorflow_repository)
     and use `bazel`. This is the recommended approach for converting models that
-- 
cgit v1.2.3


From f31939d24e3c544933b98ef48fac9ccac5679e05 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Sat, 21 Jul 2018 22:04:15 -0700
Subject: Add function crf_multitag_sequence_score which enables calculating
 scores with more than one tag at each index.

PiperOrigin-RevId: 205551004
---
 tensorflow/contrib/crf/__init__.py                 |  2 +
 .../contrib/crf/python/kernel_tests/crf_test.py    | 62 +++++++++++++++++++---
 tensorflow/contrib/crf/python/ops/crf.py           | 52 +++++++++++++++++-
 3 files changed, 109 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/crf/__init__.py b/tensorflow/contrib/crf/__init__.py
index 046c509626..615e62b16f 100644
--- a/tensorflow/contrib/crf/__init__.py
+++ b/tensorflow/contrib/crf/__init__.py
@@ -20,6 +20,7 @@ See the @{$python/contrib.crf} guide.
 @@crf_decode
 @@crf_log_likelihood
 @@crf_log_norm
+@@crf_multitag_sequence_score
 @@crf_sequence_score
 @@crf_unary_score
 @@CrfDecodeBackwardRnnCell
@@ -36,6 +37,7 @@ from tensorflow.contrib.crf.python.ops.crf import crf_binary_score
 from tensorflow.contrib.crf.python.ops.crf import crf_decode
 from tensorflow.contrib.crf.python.ops.crf import crf_log_likelihood
 from tensorflow.contrib.crf.python.ops.crf import crf_log_norm
+from tensorflow.contrib.crf.python.ops.crf import crf_multitag_sequence_score
 from tensorflow.contrib.crf.python.ops.crf import crf_sequence_score
 from tensorflow.contrib.crf.python.ops.crf import crf_unary_score
 from tensorflow.contrib.crf.python.ops.crf import CrfDecodeBackwardRnnCell
diff --git a/tensorflow/contrib/crf/python/kernel_tests/crf_test.py b/tensorflow/contrib/crf/python/kernel_tests/crf_test.py
index 74f2ec22ff..f56a973f6f 100644
--- a/tensorflow/contrib/crf/python/kernel_tests/crf_test.py
+++ b/tensorflow/contrib/crf/python/kernel_tests/crf_test.py
@@ -31,6 +31,15 @@ from tensorflow.python.platform import test
 
 class CrfTest(test.TestCase):
 
+  def calculateSequenceScore(self, inputs, transition_params, tag_indices,
+                             sequence_lengths):
+    expected_unary_score = sum(
+        inputs[i][tag_indices[i]] for i in range(sequence_lengths))
+    expected_binary_score = sum(
+        transition_params[tag_indices[i], tag_indices[i + 1]]
+        for i in range(sequence_lengths - 1))
+    return expected_unary_score + expected_binary_score
+
   def testCrfSequenceScore(self):
     transition_params = np.array(
         [[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32)
@@ -60,14 +69,55 @@ class CrfTest(test.TestCase):
             transition_params=constant_op.constant(transition_params))
         sequence_score = array_ops.squeeze(sequence_score, [0])
         tf_sequence_score = sess.run(sequence_score)
-        expected_unary_score = sum(inputs[i][tag_indices[i]]
-                                   for i in range(sequence_lengths))
-        expected_binary_score = sum(
-            transition_params[tag_indices[i], tag_indices[i + 1]]
-            for i in range(sequence_lengths - 1))
-        expected_sequence_score = expected_unary_score + expected_binary_score
+        expected_sequence_score = self.calculateSequenceScore(
+            inputs, transition_params, tag_indices, sequence_lengths)
         self.assertAllClose(tf_sequence_score, expected_sequence_score)
 
+  def testCrfMultiTagSequenceScore(self):
+    transition_params = np.array(
+        [[-3, 5, -2], [3, 4, 1], [1, 2, 1]], dtype=np.float32)
+    # Test both the length-1 and regular cases.
+    sequence_lengths_list = [
+        np.array(3, dtype=np.int32),
+        np.array(1, dtype=np.int32)
+    ]
+    inputs_list = [
+        np.array([[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]],
+                 dtype=np.float32),
+        np.array([[4, 5, -3]],
+                 dtype=np.float32),
+    ]
+    tag_bitmap_list = [
+        np.array(
+            [[True, True, False], [True, False, True], [False, True, True],
+             [True, False, True]],
+            dtype=np.bool),
+        np.array([[True, True, False]], dtype=np.bool)
+    ]
+    for sequence_lengths, inputs, tag_bitmap in zip(
+        sequence_lengths_list, inputs_list, tag_bitmap_list):
+      with self.test_session() as sess:
+        sequence_score = crf.crf_multitag_sequence_score(
+            inputs=array_ops.expand_dims(inputs, 0),
+            tag_bitmap=array_ops.expand_dims(tag_bitmap, 0),
+            sequence_lengths=array_ops.expand_dims(sequence_lengths, 0),
+            transition_params=constant_op.constant(transition_params))
+        sequence_score = array_ops.squeeze(sequence_score, [0])
+        tf_sum_sequence_score = sess.run(sequence_score)
+        all_indices_list = [
+            single_index_bitmap.nonzero()[0]
+            for single_index_bitmap in tag_bitmap[:sequence_lengths]
+        ]
+        expected_sequence_scores = [
+            self.calculateSequenceScore(inputs, transition_params, indices,
+                                        sequence_lengths)
+            for indices in itertools.product(*all_indices_list)
+        ]
+        expected_log_sum_exp_sequence_scores = np.logaddexp.reduce(
+            expected_sequence_scores)
+        self.assertAllClose(tf_sum_sequence_score,
+                            expected_log_sum_exp_sequence_scores)
+
   def testCrfUnaryScore(self):
     inputs = np.array(
         [[4, 5, -3], [3, -1, 3], [-1, 2, 1], [0, 0, 0]], dtype=np.float32)
diff --git a/tensorflow/contrib/crf/python/ops/crf.py b/tensorflow/contrib/crf/python/ops/crf.py
index 2d2cbdc199..8a7ff61bc8 100644
--- a/tensorflow/contrib/crf/python/ops/crf.py
+++ b/tensorflow/contrib/crf/python/ops/crf.py
@@ -67,7 +67,7 @@ __all__ = [
     "crf_sequence_score", "crf_log_norm", "crf_log_likelihood",
     "crf_unary_score", "crf_binary_score", "CrfForwardRnnCell",
     "viterbi_decode", "crf_decode", "CrfDecodeForwardRnnCell",
-    "CrfDecodeBackwardRnnCell"
+    "CrfDecodeBackwardRnnCell", "crf_multitag_sequence_score"
 ]
 
 
@@ -114,6 +114,56 @@ def crf_sequence_score(inputs, tag_indices, sequence_lengths,
       false_fn=_multi_seq_fn)
 
 
+def crf_multitag_sequence_score(inputs, tag_bitmap, sequence_lengths,
+                                transition_params):
+  """Computes the unnormalized score of all tag sequences matching tag_bitmap.
+
+  tag_bitmap enables more than one tag to be considered correct at each time
+  step. This is useful when an observed output at a given time step is
+  consistent with more than one tag, and thus the log likelihood of that
+  observation must take into account all possible consistent tags.
+
+  Using one-hot vectors in tag_bitmap gives results identical to
+  crf_sequence_score.
+
+  Args:
+    inputs: A [batch_size, max_seq_len, num_tags] tensor of unary potentials
+        to use as input to the CRF layer.
+    tag_bitmap: A [batch_size, max_seq_len, num_tags] boolean tensor
+        representing all active tags at each index for which to calculate the
+        unnormalized score.
+    sequence_lengths: A [batch_size] vector of true sequence lengths.
+    transition_params: A [num_tags, num_tags] transition matrix.
+  Returns:
+    sequence_scores: A [batch_size] vector of unnormalized sequence scores.
+  """
+
+  # If max_seq_len is 1, we skip the score calculation and simply gather the
+  # unary potentials of all active tags.
+  def _single_seq_fn():
+    filtered_inputs = array_ops.where(
+        tag_bitmap, inputs,
+        array_ops.fill(array_ops.shape(inputs), float("-inf")))
+    return math_ops.reduce_logsumexp(
+        filtered_inputs, axis=[1, 2], keepdims=False)
+
+  def _multi_seq_fn():
+    # Compute the logsumexp of all scores of sequences matching the given tags.
+    filtered_inputs = array_ops.where(
+        tag_bitmap, inputs,
+        array_ops.fill(array_ops.shape(inputs), float("-inf")))
+    return crf_log_norm(
+        inputs=filtered_inputs,
+        sequence_lengths=sequence_lengths,
+        transition_params=transition_params)
+
+  return utils.smart_cond(
+      pred=math_ops.equal(inputs.shape[1].value or array_ops.shape(inputs)[1],
+                          1),
+      true_fn=_single_seq_fn,
+      false_fn=_multi_seq_fn)
+
+
 def crf_log_norm(inputs, sequence_lengths, transition_params):
   """Computes the normalization for a CRF.
 
-- 
cgit v1.2.3


From 571f7a2488f653ed0647d55ed62dd51473d3eaa3 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Sat, 21 Jul 2018 23:05:38 -0700
Subject: Fix formatting issues

---
 tensorflow/contrib/tensorrt/BUILD              |  5 ++++-
 tensorflow/contrib/tensorrt/segment/segment.cc | 16 ++++++++--------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index cb2daa7b12..e3248699dd 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -278,11 +278,14 @@ tf_cc_test(
     tags = ["no_windows"],
     deps = [
         ":segment",
-        "//tensorflow/c:c_api",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:scope",
+        "//tensorflow/core:core_cpu",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
     ],
 )
 
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 92807bed14..008fffc954 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -562,8 +562,8 @@ tensorflow::Status SegmentGraph(
       // input, for output nodes remove all its output. In this way, for common
       // cases the number of removed nodes should be minimum.
       auto remove_nodes = [&segment_nodes](
-          bool is_input_nodes,
-          std::deque* que) {
+                              bool is_input_nodes,
+                              std::deque* que) {
         // Run a BFS on the queue to find all the input/output nodes.
         std::set visited;
         while (!que->empty()) {
@@ -571,13 +571,14 @@ tensorflow::Status SegmentGraph(
           que->pop_front();
           if (!visited.insert(node).second) continue;
           segment_nodes.erase(node);
-          for (auto in : is_input_nodes ? node->in_nodes() : node->out_nodes()) {
+          for (auto in :
+               is_input_nodes ? node->in_nodes() : node->out_nodes()) {
             if (segment_nodes.count(in)) {
               que->push_back(in);
               VLOG(2) << "Need to remove node " << in->name()
-                         << " because one of its "
-                         << (is_input_nodes ? "output" : "input")
-                         << " nodes in the graph was removed: " << node->name();
+                      << " because one of its "
+                      << (is_input_nodes ? "output" : "input")
+                      << " nodes in the graph was removed: " << node->name();
             }
           }
         }
@@ -599,8 +600,7 @@ tensorflow::Status SegmentGraph(
     }
 
     // Don't use small segments.
-    if (static_cast(segment_nodes.size()) <
-        options.minimum_segment_size) {
+    if (static_cast(segment_nodes.size()) < options.minimum_segment_size) {
       VLOG(1) << "Segment " << segments->size() << " has only "
               << segment_nodes.size() << " nodes, dropping";
       continue;
-- 
cgit v1.2.3


From 88e560d6fadc1cf23519b00a9de5ed7c973536fd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Sun, 22 Jul 2018 00:16:05 -0700
Subject: Use paramaterized tests in `train_test.py`.

PiperOrigin-RevId: 205555784
---
 tensorflow/contrib/gan/BUILD                |   2 +
 tensorflow/contrib/gan/python/train_test.py | 571 +++++++++++-----------------
 2 files changed, 219 insertions(+), 354 deletions(-)

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index c8c2af49d4..781e4ae4d7 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -57,6 +57,7 @@ py_library(
 py_test(
     name = "train_test",
     srcs = ["python/train_test.py"],
+    shard_count = 50,
     srcs_version = "PY2AND3",
     tags = ["notsan"],
     deps = [
@@ -80,6 +81,7 @@ py_test(
         "//tensorflow/python:variables",
         "//tensorflow/python/ops/distributions",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/contrib/gan/python/train_test.py b/tensorflow/contrib/gan/python/train_test.py
index 93a12af944..cd99a33c03 100644
--- a/tensorflow/contrib/gan/python/train_test.py
+++ b/tensorflow/contrib/gan/python/train_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.contrib import layers
@@ -296,38 +297,24 @@ def get_tensor_pool_fn_for_infogan(pool_size):
   return tensor_pool_fn_impl
 
 
-class GANModelTest(test.TestCase):
+class GANModelTest(test.TestCase, parameterized.TestCase):
   """Tests for `gan_model`."""
 
-  def _test_output_type_helper(self, create_fn, tuple_type):
-    self.assertTrue(isinstance(create_fn(), tuple_type))
-
-  def test_output_type_gan(self):
-    self._test_output_type_helper(get_gan_model, namedtuples.GANModel)
-
-  def test_output_type_callable_gan(self):
-    self._test_output_type_helper(get_callable_gan_model, namedtuples.GANModel)
-
-  def test_output_type_infogan(self):
-    self._test_output_type_helper(get_infogan_model, namedtuples.InfoGANModel)
-
-  def test_output_type_callable_infogan(self):
-    self._test_output_type_helper(get_callable_infogan_model,
-                                  namedtuples.InfoGANModel)
-
-  def test_output_type_acgan(self):
-    self._test_output_type_helper(get_acgan_model, namedtuples.ACGANModel)
-
-  def test_output_type_callable_acgan(self):
-    self._test_output_type_helper(get_callable_acgan_model,
-                                  namedtuples.ACGANModel)
-
-  def test_output_type_cyclegan(self):
-    self._test_output_type_helper(get_cyclegan_model, namedtuples.CycleGANModel)
-
-  def test_output_type_callable_cyclegan(self):
-    self._test_output_type_helper(get_callable_cyclegan_model,
-                                  namedtuples.CycleGANModel)
+  @parameterized.named_parameters(
+      ('gan', get_gan_model, namedtuples.GANModel),
+      ('callable_gan', get_callable_gan_model, namedtuples.GANModel),
+      ('infogan', get_infogan_model, namedtuples.InfoGANModel),
+      ('callable_infogan', get_callable_infogan_model,
+       namedtuples.InfoGANModel),
+      ('acgan', get_acgan_model, namedtuples.ACGANModel),
+      ('callable_acgan', get_callable_acgan_model, namedtuples.ACGANModel),
+      ('cyclegan', get_cyclegan_model, namedtuples.CycleGANModel),
+      ('callable_cyclegan', get_callable_cyclegan_model,
+       namedtuples.CycleGANModel),
+  )
+  def test_output_type(self, create_fn, expected_tuple_type):
+    """Test that output type is as expected."""
+    self.assertIsInstance(create_fn(), expected_tuple_type)
 
   def test_no_shape_check(self):
 
@@ -484,53 +471,55 @@ class StarGANModelTest(test.TestCase):
                             disc_gen_label.shape)
 
 
-class GANLossTest(test.TestCase):
+class GANLossTest(test.TestCase, parameterized.TestCase):
   """Tests for `gan_loss`."""
 
-  # Test output type.
-  def _test_output_type_helper(self, get_gan_model_fn):
+  @parameterized.named_parameters(
+      ('gan', get_gan_model),
+      ('callable_gan', get_callable_gan_model),
+      ('infogan', get_infogan_model),
+      ('callable_infogan', get_callable_infogan_model),
+      ('acgan', get_acgan_model),
+      ('callable_acgan', get_callable_acgan_model),
+  )
+  def test_output_type(self, get_gan_model_fn):
+    """Test output type."""
     loss = train.gan_loss(get_gan_model_fn(), add_summaries=True)
-    self.assertTrue(isinstance(loss, namedtuples.GANLoss))
-    self.assertGreater(len(ops.get_collection(ops.GraphKeys.SUMMARIES)), 0)
-
-  def test_output_type_gan(self):
-    self._test_output_type_helper(get_gan_model)
-
-  def test_output_type_callable_gan(self):
-    self._test_output_type_helper(get_callable_gan_model)
-
-  def test_output_type_infogan(self):
-    self._test_output_type_helper(get_infogan_model)
-
-  def test_output_type_callable_infogan(self):
-    self._test_output_type_helper(get_callable_infogan_model)
-
-  def test_output_type_acgan(self):
-    self._test_output_type_helper(get_acgan_model)
-
-  def test_output_type_callable_acgan(self):
-    self._test_output_type_helper(get_callable_acgan_model)
-
-  def test_output_type_cyclegan(self):
-    loss = train.cyclegan_loss(create_cyclegan_model(), add_summaries=True)
-    self.assertIsInstance(loss, namedtuples.CycleGANLoss)
+    self.assertIsInstance(loss, namedtuples.GANLoss)
     self.assertGreater(len(ops.get_collection(ops.GraphKeys.SUMMARIES)), 0)
 
-  def test_output_type_callable_cyclegan(self):
-    loss = train.cyclegan_loss(
-        create_callable_cyclegan_model(), add_summaries=True)
+  @parameterized.named_parameters(
+      ('cyclegan', create_cyclegan_model),
+      ('callable_cyclegan', create_callable_cyclegan_model),
+  )
+  def test_cyclegan_output_type(self, get_gan_model_fn):
+    loss = train.cyclegan_loss(get_gan_model_fn(), add_summaries=True)
     self.assertIsInstance(loss, namedtuples.CycleGANLoss)
     self.assertGreater(len(ops.get_collection(ops.GraphKeys.SUMMARIES)), 0)
 
-  # Test gradient penalty option.
-  def _test_grad_penalty_helper(self, create_gan_model_fn, one_sided=False):
+  @parameterized.named_parameters(
+      ('gan', create_gan_model, False),
+      ('gan_one_sided', create_gan_model, True),
+      ('callable_gan', create_callable_gan_model, False),
+      ('callable_gan_one_sided', create_callable_gan_model, True),
+      ('infogan', create_infogan_model, False),
+      ('infogan_one_sided', create_infogan_model, True),
+      ('callable_infogan', create_callable_infogan_model, False),
+      ('callable_infogan_one_sided', create_callable_infogan_model, True),
+      ('acgan', create_acgan_model, False),
+      ('acgan_one_sided', create_acgan_model, True),
+      ('callable_acgan', create_callable_acgan_model, False),
+      ('callable_acgan_one_sided', create_callable_acgan_model, True),
+  )
+  def test_grad_penalty(self, create_gan_model_fn, one_sided):
+    """Test gradient penalty option."""
     model = create_gan_model_fn()
     loss = train.gan_loss(model)
     loss_gp = train.gan_loss(
         model,
         gradient_penalty_weight=1.0,
         gradient_penalty_one_sided=one_sided)
-    self.assertTrue(isinstance(loss_gp, namedtuples.GANLoss))
+    self.assertIsInstance(loss_gp, namedtuples.GANLoss)
 
     # Check values.
     with self.test_session(use_gpu=True) as sess:
@@ -541,59 +530,28 @@ class GANLossTest(test.TestCase):
           [loss.discriminator_loss, loss_gp.discriminator_loss])
 
     self.assertEqual(loss_gen_np, loss_gen_gp_np)
-    self.assertTrue(loss_dis_np < loss_dis_gp_np)
-
-  def test_grad_penalty_gan(self):
-    self._test_grad_penalty_helper(create_gan_model)
-
-  def test_grad_penalty_callable_gan(self):
-    self._test_grad_penalty_helper(create_callable_gan_model)
-
-  def test_grad_penalty_infogan(self):
-    self._test_grad_penalty_helper(create_infogan_model)
-
-  def test_grad_penalty_callable_infogan(self):
-    self._test_grad_penalty_helper(create_callable_infogan_model)
-
-  def test_grad_penalty_acgan(self):
-    self._test_grad_penalty_helper(create_acgan_model)
-
-  def test_grad_penalty_callable_acgan(self):
-    self._test_grad_penalty_helper(create_callable_acgan_model)
-
-  def test_grad_penalty_one_sided_gan(self):
-    self._test_grad_penalty_helper(create_gan_model, one_sided=True)
-
-  def test_grad_penalty_one_sided_callable_gan(self):
-    self._test_grad_penalty_helper(create_callable_gan_model, one_sided=True)
-
-  def test_grad_penalty_one_sided_infogan(self):
-    self._test_grad_penalty_helper(create_infogan_model, one_sided=True)
-
-  def test_grad_penalty_one_sided_callable_infogan(self):
-    self._test_grad_penalty_helper(
-        create_callable_infogan_model, one_sided=True)
-
-  def test_grad_penalty_one_sided_acgan(self):
-    self._test_grad_penalty_helper(create_acgan_model, one_sided=True)
-
-  def test_grad_penalty_one_sided_callable_acgan(self):
-    self._test_grad_penalty_helper(create_callable_acgan_model, one_sided=True)
-
-  # Test mutual information penalty option.
-  def _test_mutual_info_penalty_helper(self, create_gan_model_fn):
+    self.assertLess(loss_dis_np, loss_dis_gp_np)
+
+  @parameterized.named_parameters(
+      ('infogan', get_infogan_model),
+      ('callable_infogan', get_callable_infogan_model),
+  )
+  def test_mutual_info_penalty(self, create_gan_model_fn):
+    """Test mutual information penalty option."""
     train.gan_loss(
         create_gan_model_fn(),
         mutual_information_penalty_weight=constant_op.constant(1.0))
 
-  def test_mutual_info_penalty_infogan(self):
-    self._test_mutual_info_penalty_helper(get_infogan_model)
-
-  def test_mutual_info_penalty_callable_infogan(self):
-    self._test_mutual_info_penalty_helper(get_callable_infogan_model)
-
-  # Test regularization loss.
-  def _test_regularization_helper(self, get_gan_model_fn):
+  @parameterized.named_parameters(
+      ('gan', get_gan_model),
+      ('callable_gan', get_callable_gan_model),
+      ('infogan', get_infogan_model),
+      ('callable_infogan', get_callable_infogan_model),
+      ('acgan', get_acgan_model),
+      ('callable_acgan', get_callable_acgan_model),
+  )
+  def test_regularization_helper(self, get_gan_model_fn):
+    """Test regularization loss."""
     # Evaluate losses without regularization.
     no_reg_loss = train.gan_loss(get_gan_model_fn())
     with self.test_session(use_gpu=True):
@@ -616,33 +574,19 @@ class GANLossTest(test.TestCase):
     self.assertEqual(3.0, reg_loss_gen_np - no_reg_loss_gen_np)
     self.assertEqual(2.0, reg_loss_dis_np - no_reg_loss_dis_np)
 
-  def test_regularization_gan(self):
-    self._test_regularization_helper(get_gan_model)
-
-  def test_regularization_callable_gan(self):
-    self._test_regularization_helper(get_callable_gan_model)
-
-  def test_regularization_infogan(self):
-    self._test_regularization_helper(get_infogan_model)
-
-  def test_regularization_callable_infogan(self):
-    self._test_regularization_helper(get_callable_infogan_model)
-
-  def test_regularization_acgan(self):
-    self._test_regularization_helper(get_acgan_model)
-
-  def test_regularization_callable_acgan(self):
-    self._test_regularization_helper(get_callable_acgan_model)
-
-  # Test that ACGan models work.
-  def _test_acgan_helper(self, create_gan_model_fn):
+  @parameterized.named_parameters(
+      ('notcallable', create_acgan_model),
+      ('callable', create_callable_acgan_model),
+  )
+  def test_acgan(self, create_gan_model_fn):
+    """Test that ACGAN models work."""
     model = create_gan_model_fn()
     loss = train.gan_loss(model)
     loss_ac_gen = train.gan_loss(model, aux_cond_generator_weight=1.0)
     loss_ac_dis = train.gan_loss(model, aux_cond_discriminator_weight=1.0)
-    self.assertTrue(isinstance(loss, namedtuples.GANLoss))
-    self.assertTrue(isinstance(loss_ac_gen, namedtuples.GANLoss))
-    self.assertTrue(isinstance(loss_ac_dis, namedtuples.GANLoss))
+    self.assertIsInstance(loss, namedtuples.GANLoss)
+    self.assertIsInstance(loss_ac_gen, namedtuples.GANLoss)
+    self.assertIsInstance(loss_ac_dis, namedtuples.GANLoss)
 
     # Check values.
     with self.test_session(use_gpu=True) as sess:
@@ -656,20 +600,18 @@ class GANLossTest(test.TestCase):
           loss_ac_dis.discriminator_loss
       ])
 
-    self.assertTrue(loss_gen_np < loss_dis_np)
+    self.assertLess(loss_gen_np, loss_dis_np)
     self.assertTrue(np.isscalar(loss_ac_gen_gen_np))
     self.assertTrue(np.isscalar(loss_ac_dis_gen_np))
     self.assertTrue(np.isscalar(loss_ac_gen_dis_np))
     self.assertTrue(np.isscalar(loss_ac_dis_dis_np))
 
-  def test_acgan(self):
-    self._test_acgan_helper(create_acgan_model)
-
-  def test_callable_acgan(self):
-    self._test_acgan_helper(create_callable_acgan_model)
-
-  # Test that CycleGan models work.
-  def _test_cyclegan_helper(self, create_gan_model_fn):
+  @parameterized.named_parameters(
+      ('notcallable', create_cyclegan_model),
+      ('callable', create_callable_cyclegan_model),
+  )
+  def test_cyclegan(self, create_gan_model_fn):
+    """Test that CycleGan models work."""
     model = create_gan_model_fn()
     loss = train.cyclegan_loss(model)
     self.assertIsInstance(loss, namedtuples.CycleGANLoss)
@@ -690,11 +632,46 @@ class GANLossTest(test.TestCase):
     self.assertTrue(np.isscalar(loss_y2x_gen_np))
     self.assertTrue(np.isscalar(loss_y2x_dis_np))
 
-  def test_cyclegan(self):
-    self._test_cyclegan_helper(create_cyclegan_model)
+  @parameterized.named_parameters(
+      ('gan', create_gan_model),
+      ('callable_gan', create_callable_gan_model),
+      ('infogan', create_infogan_model),
+      ('callable_infogan', create_callable_infogan_model),
+      ('acgan', create_acgan_model),
+      ('callable_acgan', create_callable_acgan_model),
+  )
+  def test_tensor_pool(self, create_gan_model_fn):
+    """Test tensor pool option."""
+    model = create_gan_model_fn()
+    if isinstance(model, namedtuples.InfoGANModel):
+      tensor_pool_fn = get_tensor_pool_fn_for_infogan(pool_size=5)
+    else:
+      tensor_pool_fn = get_tensor_pool_fn(pool_size=5)
+    loss = train.gan_loss(model, tensor_pool_fn=tensor_pool_fn)
+    self.assertIsInstance(loss, namedtuples.GANLoss)
+
+    # Check values.
+    with self.test_session(use_gpu=True) as sess:
+      variables.global_variables_initializer().run()
+      for _ in range(10):
+        sess.run([loss.generator_loss, loss.discriminator_loss])
+
+  def test_doesnt_crash_when_in_nested_scope(self):
+    with variable_scope.variable_scope('outer_scope'):
+      gan_model = train.gan_model(
+          generator_model,
+          discriminator_model,
+          real_data=array_ops.zeros([1, 2]),
+          generator_inputs=random_ops.random_normal([1, 2]))
+
+      # This should work inside a scope.
+      train.gan_loss(gan_model, gradient_penalty_weight=1.0)
+
+    # This should also work outside a scope.
+    train.gan_loss(gan_model, gradient_penalty_weight=1.0)
+
 
-  def test_callable_cyclegan(self):
-    self._test_cyclegan_helper(create_callable_cyclegan_model)
+class TensorPoolAdjusteModelTest(test.TestCase):
 
   def _check_tensor_pool_adjusted_model_outputs(self, tensor1, tensor2,
                                                 pool_size):
@@ -714,115 +691,77 @@ class GANLossTest(test.TestCase):
           # pool).
           self.assertTrue(any([(v == t2).all() for v in history_values]))
 
-  # Test `_tensor_pool_adjusted_model` for gan model.
-  def test_tensor_pool_adjusted_model_gan(self):
-    model = create_gan_model()
-
-    new_model = train._tensor_pool_adjusted_model(model, None)
+  def _make_new_model_and_check(self, model, pool_size,
+                                pool_fn=get_tensor_pool_fn):
+    new_model = train._tensor_pool_adjusted_model(
+        model, pool_fn(pool_size=pool_size))
     # 'Generator/dummy_g:0' and 'Discriminator/dummy_d:0'
     self.assertEqual(2, len(ops.get_collection(ops.GraphKeys.VARIABLES)))
+    self.assertIsNot(new_model.discriminator_gen_outputs,
+                     model.discriminator_gen_outputs)
+
+    return new_model
+
+  def test_tensor_pool_adjusted_model_no_pool(self):
+    """Test `_tensor_pool_adjusted_model` for no pool size."""
+    model = create_gan_model()
+    new_model = train._tensor_pool_adjusted_model(model, None)
+
+    # Check values.
     self.assertIs(new_model.discriminator_gen_outputs,
                   model.discriminator_gen_outputs)
 
+  def test_tensor_pool_adjusted_model_gan(self):
+    """Test `_tensor_pool_adjusted_model` for gan model."""
     pool_size = 5
-    new_model = train._tensor_pool_adjusted_model(
-        model, get_tensor_pool_fn(pool_size=pool_size))
-    self.assertIsNot(new_model.discriminator_gen_outputs,
-                     model.discriminator_gen_outputs)
+    model = create_gan_model()
+    new_model = self._make_new_model_and_check(model, pool_size)
+
     # Check values.
     self._check_tensor_pool_adjusted_model_outputs(
         model.discriminator_gen_outputs, new_model.discriminator_gen_outputs,
         pool_size)
 
-  # Test _tensor_pool_adjusted_model for infogan model.
   def test_tensor_pool_adjusted_model_infogan(self):
+    """Test _tensor_pool_adjusted_model for infogan model."""
+    pool_size = 5
     model = create_infogan_model()
+    new_model = self._make_new_model_and_check(
+        model, pool_size, pool_fn=get_tensor_pool_fn_for_infogan)
 
-    pool_size = 5
-    new_model = train._tensor_pool_adjusted_model(
-        model, get_tensor_pool_fn_for_infogan(pool_size=pool_size))
-    # 'Generator/dummy_g:0' and 'Discriminator/dummy_d:0'
-    self.assertEqual(2, len(ops.get_collection(ops.GraphKeys.VARIABLES)))
-    self.assertIsNot(new_model.discriminator_gen_outputs,
-                     model.discriminator_gen_outputs)
+    # Check values.
     self.assertIsNot(new_model.predicted_distributions,
                      model.predicted_distributions)
-    # Check values.
     self._check_tensor_pool_adjusted_model_outputs(
         model.discriminator_gen_outputs, new_model.discriminator_gen_outputs,
         pool_size)
 
-  # Test _tensor_pool_adjusted_model for acgan model.
   def test_tensor_pool_adjusted_model_acgan(self):
+    """Test _tensor_pool_adjusted_model for acgan model."""
+    pool_size = 5
     model = create_acgan_model()
+    new_model = self._make_new_model_and_check(model, pool_size)
 
-    pool_size = 5
-    new_model = train._tensor_pool_adjusted_model(
-        model, get_tensor_pool_fn(pool_size=pool_size))
-    # 'Generator/dummy_g:0' and 'Discriminator/dummy_d:0'
-    self.assertEqual(2, len(ops.get_collection(ops.GraphKeys.VARIABLES)))
-    self.assertIsNot(new_model.discriminator_gen_outputs,
-                     model.discriminator_gen_outputs)
+    # Check values.
     self.assertIsNot(new_model.discriminator_gen_classification_logits,
                      model.discriminator_gen_classification_logits)
-    # Check values.
     self._check_tensor_pool_adjusted_model_outputs(
         model.discriminator_gen_outputs, new_model.discriminator_gen_outputs,
         pool_size)
 
-  # Test tensor pool.
-  def _test_tensor_pool_helper(self, create_gan_model_fn):
-    model = create_gan_model_fn()
-    if isinstance(model, namedtuples.InfoGANModel):
-      tensor_pool_fn = get_tensor_pool_fn_for_infogan(pool_size=5)
-    else:
-      tensor_pool_fn = get_tensor_pool_fn(pool_size=5)
-    loss = train.gan_loss(model, tensor_pool_fn=tensor_pool_fn)
-    self.assertTrue(isinstance(loss, namedtuples.GANLoss))
-
-    # Check values.
-    with self.test_session(use_gpu=True) as sess:
-      variables.global_variables_initializer().run()
-      for _ in range(10):
-        sess.run([loss.generator_loss, loss.discriminator_loss])
-
-  def test_tensor_pool_gan(self):
-    self._test_tensor_pool_helper(create_gan_model)
-
-  def test_tensor_pool_callable_gan(self):
-    self._test_tensor_pool_helper(create_callable_gan_model)
-
-  def test_tensor_pool_infogan(self):
-    self._test_tensor_pool_helper(create_infogan_model)
-
-  def test_tensor_pool_callable_infogan(self):
-    self._test_tensor_pool_helper(create_callable_infogan_model)
-
-  def test_tensor_pool_acgan(self):
-    self._test_tensor_pool_helper(create_acgan_model)
-
-  def test_tensor_pool_callable_acgan(self):
-    self._test_tensor_pool_helper(create_callable_acgan_model)
-
-  def test_doesnt_crash_when_in_nested_scope(self):
-    with variable_scope.variable_scope('outer_scope'):
-      gan_model = train.gan_model(
-          generator_model,
-          discriminator_model,
-          real_data=array_ops.zeros([1, 2]),
-          generator_inputs=random_ops.random_normal([1, 2]))
-
-      # This should work inside a scope.
-      train.gan_loss(gan_model, gradient_penalty_weight=1.0)
-
-    # This should also work outside a scope.
-    train.gan_loss(gan_model, gradient_penalty_weight=1.0)
-
 
-class GANTrainOpsTest(test.TestCase):
+class GANTrainOpsTest(test.TestCase, parameterized.TestCase):
   """Tests for `gan_train_ops`."""
 
-  def _test_output_type_helper(self, create_gan_model_fn):
+  @parameterized.named_parameters(
+      ('gan', create_gan_model),
+      ('callable_gan', create_callable_gan_model),
+      ('infogan', create_infogan_model),
+      ('callable_infogan', create_callable_infogan_model),
+      ('acgan', create_acgan_model),
+      ('callable_acgan', create_callable_acgan_model),
+  )
+  def test_output_type(self, create_gan_model_fn):
     model = create_gan_model_fn()
     loss = train.gan_loss(model)
 
@@ -836,28 +775,24 @@ class GANTrainOpsTest(test.TestCase):
         summarize_gradients=True,
         colocate_gradients_with_ops=True)
 
-    self.assertTrue(isinstance(train_ops, namedtuples.GANTrainOps))
-
-  def test_output_type_gan(self):
-    self._test_output_type_helper(create_gan_model)
-
-  def test_output_type_callable_gan(self):
-    self._test_output_type_helper(create_callable_gan_model)
-
-  def test_output_type_infogan(self):
-    self._test_output_type_helper(create_infogan_model)
-
-  def test_output_type_callable_infogan(self):
-    self._test_output_type_helper(create_callable_infogan_model)
-
-  def test_output_type_acgan(self):
-    self._test_output_type_helper(create_acgan_model)
-
-  def test_output_type_callable_acgan(self):
-    self._test_output_type_helper(create_callable_acgan_model)
+    self.assertIsInstance(train_ops, namedtuples.GANTrainOps)
 
   # TODO(joelshor): Add a test to check that custom update op is run.
-  def _test_unused_update_ops(self, create_gan_model_fn, provide_update_ops):
+  @parameterized.named_parameters(
+      ('gan', create_gan_model, False),
+      ('gan_provideupdates', create_gan_model, True),
+      ('callable_gan', create_callable_gan_model, False),
+      ('callable_gan_provideupdates', create_callable_gan_model, True),
+      ('infogan', create_infogan_model, False),
+      ('infogan_provideupdates', create_infogan_model, True),
+      ('callable_infogan', create_callable_infogan_model, False),
+      ('callable_infogan_provideupdates', create_callable_infogan_model, True),
+      ('acgan', create_acgan_model, False),
+      ('acgan_provideupdates', create_acgan_model, True),
+      ('callable_acgan', create_callable_acgan_model, False),
+      ('callable_acgan_provideupdates', create_callable_acgan_model, True),
+  )
+  def test_unused_update_ops(self, create_gan_model_fn, provide_update_ops):
     model = create_gan_model_fn()
     loss = train.gan_loss(model)
 
@@ -904,45 +839,16 @@ class GANTrainOpsTest(test.TestCase):
       self.assertEqual(1, gen_update_count.eval())
       self.assertEqual(1, dis_update_count.eval())
 
-  def test_unused_update_ops_gan(self):
-    self._test_unused_update_ops(create_gan_model, False)
-
-  def test_unused_update_ops_gan_provideupdates(self):
-    self._test_unused_update_ops(create_gan_model, True)
-
-  def test_unused_update_ops_callable_gan(self):
-    self._test_unused_update_ops(create_callable_gan_model, False)
-
-  def test_unused_update_ops_callable_gan_provideupdates(self):
-    self._test_unused_update_ops(create_callable_gan_model, True)
-
-  def test_unused_update_ops_infogan(self):
-    self._test_unused_update_ops(create_infogan_model, False)
-
-  def test_unused_update_ops_infogan_provideupdates(self):
-    self._test_unused_update_ops(create_infogan_model, True)
-
-  def test_unused_update_ops_callable_infogan(self):
-    self._test_unused_update_ops(create_callable_infogan_model, False)
-
-  def test_unused_update_ops_callable_infogan_provideupdates(self):
-    self._test_unused_update_ops(create_callable_infogan_model, True)
-
-  def test_unused_update_ops_acgan(self):
-    self._test_unused_update_ops(create_acgan_model, False)
-
-  def test_unused_update_ops_acgan_provideupdates(self):
-    self._test_unused_update_ops(create_acgan_model, True)
-
-  def test_unused_update_ops_callable_acgan(self):
-    self._test_unused_update_ops(create_callable_acgan_model, False)
-
-  def test_unused_update_ops_callable_acgan_provideupdates(self):
-    self._test_unused_update_ops(create_callable_acgan_model, True)
-
-  def _test_sync_replicas_helper(self,
-                                 create_gan_model_fn,
-                                 create_global_step=False):
+  @parameterized.named_parameters(
+      ('gan', create_gan_model, False),
+      ('callable_gan', create_callable_gan_model, False),
+      ('infogan', create_infogan_model, False),
+      ('callable_infogan', create_callable_infogan_model, False),
+      ('acgan', create_acgan_model, False),
+      ('callable_acgan', create_callable_acgan_model, False),
+      ('gan_canbeint32', create_gan_model, True),
+  )
+  def test_sync_replicas(self, create_gan_model_fn, create_global_step):
     model = create_gan_model_fn()
     loss = train.gan_loss(model)
     num_trainable_vars = len(variables_lib.get_trainable_variables())
@@ -956,7 +862,7 @@ class GANTrainOpsTest(test.TestCase):
     d_opt = get_sync_optimizer()
     train_ops = train.gan_train_ops(
         model, loss, generator_optimizer=g_opt, discriminator_optimizer=d_opt)
-    self.assertTrue(isinstance(train_ops, namedtuples.GANTrainOps))
+    self.assertIsInstance(train_ops, namedtuples.GANTrainOps)
     # No new trainable variables should have been added.
     self.assertEqual(num_trainable_vars,
                      len(variables_lib.get_trainable_variables()))
@@ -994,29 +900,8 @@ class GANTrainOpsTest(test.TestCase):
       coord.request_stop()
       coord.join(g_threads + d_threads)
 
-  def test_sync_replicas_gan(self):
-    self._test_sync_replicas_helper(create_gan_model)
-
-  def test_sync_replicas_callable_gan(self):
-    self._test_sync_replicas_helper(create_callable_gan_model)
-
-  def test_sync_replicas_infogan(self):
-    self._test_sync_replicas_helper(create_infogan_model)
-
-  def test_sync_replicas_callable_infogan(self):
-    self._test_sync_replicas_helper(create_callable_infogan_model)
-
-  def test_sync_replicas_acgan(self):
-    self._test_sync_replicas_helper(create_acgan_model)
-
-  def test_sync_replicas_callable_acgan(self):
-    self._test_sync_replicas_helper(create_callable_acgan_model)
 
-  def test_global_step_can_be_int32(self):
-    self._test_sync_replicas_helper(create_gan_model, create_global_step=True)
-
-
-class GANTrainTest(test.TestCase):
+class GANTrainTest(test.TestCase, parameterized.TestCase):
   """Tests for `gan_train`."""
 
   def _gan_train_ops(self, generator_add, discriminator_add):
@@ -1032,7 +917,15 @@ class GANTrainTest(test.TestCase):
         global_step_inc_op=step.assign_add(1))
     return train_ops
 
-  def _test_run_helper(self, create_gan_model_fn):
+  @parameterized.named_parameters(
+      ('gan', create_gan_model),
+      ('callable_gan', create_callable_gan_model),
+      ('infogan', create_infogan_model),
+      ('callable_infogan', create_callable_infogan_model),
+      ('acgan', create_acgan_model),
+      ('callable_acgan', create_callable_acgan_model),
+  )
+  def test_run_helper(self, create_gan_model_fn):
     random_seed.set_random_seed(1234)
     model = create_gan_model_fn()
     loss = train.gan_loss(model)
@@ -1048,26 +941,12 @@ class GANTrainTest(test.TestCase):
     self.assertTrue(np.isscalar(final_step))
     self.assertEqual(2, final_step)
 
-  def test_run_gan(self):
-    self._test_run_helper(create_gan_model)
-
-  def test_run_callable_gan(self):
-    self._test_run_helper(create_callable_gan_model)
-
-  def test_run_infogan(self):
-    self._test_run_helper(create_infogan_model)
-
-  def test_run_callable_infogan(self):
-    self._test_run_helper(create_callable_infogan_model)
-
-  def test_run_acgan(self):
-    self._test_run_helper(create_acgan_model)
-
-  def test_run_callable_acgan(self):
-    self._test_run_helper(create_callable_acgan_model)
-
-  # Test multiple train steps.
-  def _test_multiple_steps_helper(self, get_hooks_fn_fn):
+  @parameterized.named_parameters(
+      ('seq_train_steps', train.get_sequential_train_hooks),
+      ('efficient_seq_train_steps', train.get_joint_train_hooks),
+  )
+  def test_multiple_steps(self, get_hooks_fn_fn):
+    """Test multiple train steps."""
     train_ops = self._gan_train_ops(generator_add=10, discriminator_add=100)
     train_steps = namedtuples.GANTrainSteps(
         generator_train_steps=3, discriminator_train_steps=4)
@@ -1080,12 +959,6 @@ class GANTrainTest(test.TestCase):
     self.assertTrue(np.isscalar(final_step))
     self.assertEqual(1 + 3 * 10 + 4 * 100, final_step)
 
-  def test_multiple_steps_seq_train_steps(self):
-    self._test_multiple_steps_helper(train.get_sequential_train_hooks)
-
-  def test_multiple_steps_efficient_seq_train_steps(self):
-    self._test_multiple_steps_helper(train.get_joint_train_hooks)
-
   def test_supervisor_run_gan_model_train_ops_multiple_steps(self):
     step = training_util.create_global_step()
     train_ops = namedtuples.GANTrainOps(
@@ -1105,10 +978,18 @@ class GANTrainTest(test.TestCase):
     self.assertEqual(17.0, final_loss)
 
 
-class PatchGANTest(test.TestCase):
+class PatchGANTest(test.TestCase, parameterized.TestCase):
   """Tests that functions work on PatchGAN style output."""
 
-  def _test_patchgan_helper(self, create_gan_model_fn):
+  @parameterized.named_parameters(
+      ('gan', create_gan_model),
+      ('callable_gan', create_callable_gan_model),
+      ('infogan', create_infogan_model),
+      ('callable_infogan', create_callable_infogan_model),
+      ('acgan', create_acgan_model),
+      ('callable_acgan', create_callable_acgan_model),
+  )
+  def test_patchgan(self, create_gan_model_fn):
     """Ensure that patch-based discriminators work end-to-end."""
     random_seed.set_random_seed(1234)
     model = create_gan_model_fn()
@@ -1125,24 +1006,6 @@ class PatchGANTest(test.TestCase):
     self.assertTrue(np.isscalar(final_step))
     self.assertEqual(2, final_step)
 
-  def test_patchgan_gan(self):
-    self._test_patchgan_helper(create_gan_model)
-
-  def test_patchgan_callable_gan(self):
-    self._test_patchgan_helper(create_callable_gan_model)
-
-  def test_patchgan_infogan(self):
-    self._test_patchgan_helper(create_infogan_model)
-
-  def test_patchgan_callable_infogan(self):
-    self._test_patchgan_helper(create_callable_infogan_model)
-
-  def test_patchgan_acgan(self):
-    self._test_patchgan_helper(create_acgan_model)
-
-  def test_patchgan_callable_acgan(self):
-    self._test_patchgan_helper(create_callable_acgan_model)
-
 
 if __name__ == '__main__':
   test.main()
-- 
cgit v1.2.3


From 162304f9da4114f5ed3f0e4c27929413e7abc965 Mon Sep 17 00:00:00 2001
From: Misha Brukman 
Date: Sun, 22 Jul 2018 12:48:00 -0700
Subject: [tf.data / Bigtable] Renamed BigTable class to BigtableTable for
 clarity

This removes the confusion between BigTable and Bigtable naming. Also cleaned
up all other uses of BigTable in error messages.

PiperOrigin-RevId: 205586899
---
 tensorflow/contrib/bigtable/README.md                   | 16 +++++-----------
 tensorflow/contrib/bigtable/__init__.py                 |  6 +++---
 tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc |  2 +-
 tensorflow/contrib/bigtable/kernels/bigtable_lib.cc     |  8 ++++----
 .../bigtable/python/kernel_tests/bigtable_ops_test.py   |  2 +-
 tensorflow/contrib/bigtable/python/ops/bigtable_api.py  | 17 +++++++++--------
 tensorflow/contrib/cloud/README.md                      |  4 ++--
 tensorflow/contrib/cloud/__init__.py                    |  4 ++--
 8 files changed, 27 insertions(+), 32 deletions(-)

diff --git a/tensorflow/contrib/bigtable/README.md b/tensorflow/contrib/bigtable/README.md
index b2c0460f04..d7c71a20ed 100644
--- a/tensorflow/contrib/bigtable/README.md
+++ b/tensorflow/contrib/bigtable/README.md
@@ -17,7 +17,7 @@ APIs, see the [official Cloud Bigtable client library documentation][clientdoc].
 
 ## Sample Use
 
-There are three main reading styles supported by the `BigTable` class:
+There are three main reading styles supported by the `BigtableTable` class:
 
  1. **Reading keys**: Read only the row keys in a table. Keys are returned in
     sorted order from the table. Most key reading operations retrieve all keys
@@ -34,9 +34,9 @@ There are three main reading styles supported by the `BigTable` class:
 When using the Cloud Bigtable API, the workflow is:
 
  1. Create a `BigtableClient` object.
- 2. Use the `BigtableClient` to create `BigTable` objects corresponding to each
-    table in the Bigtable instance you would like to access.
- 3. Call methods on the `BigTable` object to create `tf.data.Dataset`s to
+ 2. Use the `BigtableClient` to create `BigtableTable` objects corresponding to
+    each table in the Cloud Bigtable instance you would like to access.
+ 3. Call methods on the `BigtableTable` object to create `tf.data.Dataset`s to
     retrieve data.
 
 The following is an example for how to read all row keys with the prefix
@@ -116,7 +116,7 @@ Given a contiguous range of rows retrieve both the row key and the data
 associated with a fixed set of columns. Scanning is the most efficient way to
 retrieve data from Cloud Bigtable and is thus a very common API for high
 performance data pipelines. To construct a scanning `tf.data.Dataset` from a
-`BigTable` object, call one of the following methods:
+`BigtableTable` object, call one of the following methods:
 
  - `table.scan_prefix(prefix, ...)`
  - `table.scan_range(start, end, ...)`
@@ -342,9 +342,3 @@ are available.
  - **Cloud TPU**: Your Cloud TPUs run with the designated Cloud TPU service
    account dedicated to your GCP project. Ensure the service account has been
    authorized via the Cloud Console to access your Cloud Bigtable instances.
-
-### `BigTable` vs Bigtable?
-
-Cloud Bigtable is spelled with a lower-case (aka common) `t`. The Python class
-`BigTable`, however is short for `BigtableTable`, and thus uses an upper-case
-(aka capital) `T`.
diff --git a/tensorflow/contrib/bigtable/__init__.py b/tensorflow/contrib/bigtable/__init__.py
index 7df054637c..b7d89c9842 100644
--- a/tensorflow/contrib/bigtable/__init__.py
+++ b/tensorflow/contrib/bigtable/__init__.py
@@ -18,7 +18,7 @@ This contrib package allows TensorFlow to interface directly with Cloud Bigtable
 for high-speed data loading.
 
 @@BigtableClient
-@@BigTable
+@@BigtableTable
 
 """
 
@@ -26,14 +26,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.bigtable.python.ops.bigtable_api import BigTable
 from tensorflow.contrib.bigtable.python.ops.bigtable_api import BigtableClient
+from tensorflow.contrib.bigtable.python.ops.bigtable_api import BigtableTable
 
 from tensorflow.python.util.all_util import remove_undocumented
 
 _allowed_symbols = [
-    'BigTable',
     'BigtableClient',
+    'BigtableTable',
 ]
 
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc b/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc
index 70923e6287..a6755a3496 100644
--- a/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc
+++ b/tensorflow/contrib/bigtable/kernels/bigtable_kernels.cc
@@ -276,7 +276,7 @@ class ToBigtableOp : public AsyncOpKernel {
         }
         OP_REQUIRES_ASYNC(
             ctx, failures.empty() && mutation_status.ok(),
-            errors::Unknown("Failure while writing to BigTable: ",
+            errors::Unknown("Failure while writing to Cloud Bigtable: ",
                             mutation_status.error_code(), " - ",
                             mutation_status.error_message(), " (",
                             mutation_status.error_details(),
diff --git a/tensorflow/contrib/bigtable/kernels/bigtable_lib.cc b/tensorflow/contrib/bigtable/kernels/bigtable_lib.cc
index 2514575f30..67bf14c176 100644
--- a/tensorflow/contrib/bigtable/kernels/bigtable_lib.cc
+++ b/tensorflow/contrib/bigtable/kernels/bigtable_lib.cc
@@ -27,10 +27,10 @@ Status GrpcStatusToTfStatus(const ::grpc::Status& status) {
       status.error_code() == ::grpc::StatusCode::OUT_OF_RANGE) {
     grpc_code = ::grpc::StatusCode::INTERNAL;
   }
-  return Status(
-      static_cast<::tensorflow::error::Code>(status.error_code()),
-      strings::StrCat("Error reading from BigTable: ", status.error_message(),
-                      " (Details: ", status.error_details(), ")"));
+  return Status(static_cast<::tensorflow::error::Code>(status.error_code()),
+                strings::StrCat("Error reading from Cloud Bigtable: ",
+                                status.error_message(),
+                                " (Details: ", status.error_details(), ")"));
 }
 
 string RegexFromStringSet(const std::vector& strs) {
diff --git a/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py b/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py
index 2f20064619..e36f7f32c6 100644
--- a/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py
+++ b/tensorflow/contrib/bigtable/python/kernel_tests/bigtable_ops_test.py
@@ -44,7 +44,7 @@ class BigtableOpsTest(test.TestCase):
   def setUp(self):
     self._client = gen_bigtable_test_ops.bigtable_test_client()
     table = gen_bigtable_ops.bigtable_table(self._client, "testtable")
-    self._table = bigtable.BigTable("testtable", None, table)
+    self._table = bigtable.BigtableTable("testtable", None, table)
 
   def _makeSimpleDataset(self):
     output_rows = dataset_ops.Dataset.from_tensor_slices(self.COMMON_ROW_KEYS)
diff --git a/tensorflow/contrib/bigtable/python/ops/bigtable_api.py b/tensorflow/contrib/bigtable/python/ops/bigtable_api.py
index 9f73b7223c..fd30aa8bbb 100644
--- a/tensorflow/contrib/bigtable/python/ops/bigtable_api.py
+++ b/tensorflow/contrib/bigtable/python/ops/bigtable_api.py
@@ -94,7 +94,7 @@ class BigtableClient(object):
         project_id, instance_id, connection_pool_size, max_receive_message_size)
 
   def table(self, name, snapshot=None):
-    """Opens a table and returns a `BigTable` object.
+    """Opens a table and returns a `BigtableTable` object.
 
     Args:
       name: A `tf.string` `tf.Tensor` name of the table to open.
@@ -102,19 +102,20 @@ class BigtableClient(object):
         request the creation of a snapshot. (Note: currently unimplemented.)
 
     Returns:
-      A `BigTable` python object representing the operations available on the
-      table.
+      A `BigtableTable` python object representing the operations available on
+      the table.
     """
     # TODO(saeta): Implement snapshot functionality.
     table = gen_bigtable_ops.bigtable_table(self._resource, name)
-    return BigTable(name, snapshot, table)
+    return BigtableTable(name, snapshot, table)
 
 
-class BigTable(object):
-  """BigTable is the entrypoint for reading and writing data in Cloud Bigtable.
+class BigtableTable(object):
+  """BigtableTable is the entrypoint for reading and writing data in Cloud
+  Bigtable.
 
-  This BigTable class is the python representation of the Cloud Bigtable table
-  within TensorFlow. Methods on this class allow data to be read from and
+  This BigtableTable class is the Python representation of the Cloud Bigtable
+  table within TensorFlow. Methods on this class allow data to be read from and
   written to the Cloud Bigtable service in flexible and high performance
   manners.
   """
diff --git a/tensorflow/contrib/cloud/README.md b/tensorflow/contrib/cloud/README.md
index 134ce057f4..a80d8965f3 100644
--- a/tensorflow/contrib/cloud/README.md
+++ b/tensorflow/contrib/cloud/README.md
@@ -1,8 +1,8 @@
 # Cloud #
 
-## BigTable ##
+## Cloud Bigtable ##
 
-[Google Cloud BigTable](https://cloud.google.com/bigtable/) is a high
+[Google Cloud Bigtable](https://cloud.google.com/bigtable/) is a high
 performance storage system that can store and serve training data. This contrib
 package contains an experimental integration with TensorFlow.
 
diff --git a/tensorflow/contrib/cloud/__init__.py b/tensorflow/contrib/cloud/__init__.py
index af81106a68..8efd259946 100644
--- a/tensorflow/contrib/cloud/__init__.py
+++ b/tensorflow/contrib/cloud/__init__.py
@@ -25,8 +25,8 @@ from tensorflow.contrib.cloud.python.ops.bigquery_reader_ops import *
 from tensorflow.contrib.cloud.python.ops.gcs_config_ops import *
 
 if os.name != 'nt':
-  from tensorflow.contrib.bigtable.python.ops.bigtable_api import BigTable
   from tensorflow.contrib.bigtable.python.ops.bigtable_api import BigtableClient
+  from tensorflow.contrib.bigtable.python.ops.bigtable_api import BigtableTable
 
 del os
 
@@ -34,8 +34,8 @@ from tensorflow.python.util.all_util import remove_undocumented
 
 _allowed_symbols = [
     'BigQueryReader',
-    'BigTable',
     'BigtableClient',
+    'BigtableTable',
     'BlockCacheParams',
     'configure_colab_session',
     'configure_gcs',
-- 
cgit v1.2.3


From 012f97121441f936b5262b98e2ca488c0c92422f Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou 
Date: Sun, 22 Jul 2018 13:41:51 -0700
Subject: Add synchronization and aggregation arguments to variable creation
 methods in contrib/layers.

PiperOrigin-RevId: 205588849
---
 .../contrib/framework/python/ops/variables.py      | 97 +++++++++++++++++-----
 tensorflow/contrib/layers/python/layers/layers.py  | 33 ++++----
 2 files changed, 94 insertions(+), 36 deletions(-)

diff --git a/tensorflow/contrib/framework/python/ops/variables.py b/tensorflow/contrib/framework/python/ops/variables.py
index e8e3180019..322d5c335e 100644
--- a/tensorflow/contrib/framework/python/ops/variables.py
+++ b/tensorflow/contrib/framework/python/ops/variables.py
@@ -34,6 +34,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
 from tensorflow.python.platform import resource_loader
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import saver as tf_saver
@@ -199,10 +200,20 @@ def global_variable(initial_value,
 
 
 @contrib_add_arg_scope
-def variable(name, shape=None, dtype=None, initializer=None,
-             regularizer=None, trainable=True, collections=None,
-             caching_device=None, device=None,
-             partitioner=None, custom_getter=None, use_resource=None):
+def variable(name,
+             shape=None,
+             dtype=None,
+             initializer=None,
+             regularizer=None,
+             trainable=True,
+             collections=None,
+             caching_device=None,
+             device=None,
+             partitioner=None,
+             custom_getter=None,
+             use_resource=None,
+             synchronization=variables.VariableSynchronization.AUTO,
+             aggregation=variables.VariableAggregation.NONE):
   """Gets an existing variable with these parameters or creates a new one.
 
   Args:
@@ -228,6 +239,15 @@ def variable(name, shape=None, dtype=None, initializer=None,
     custom_getter: Callable that allows overwriting the internal
       get_variable method and has to have the same signature.
     use_resource: If `True` use a ResourceVariable instead of a Variable.
+    synchronization: Indicates when a distributed a variable will be
+      aggregated. Accepted values are constants defined in the class
+      @{tf.VariableSynchronization}. By default the synchronization is set to
+      `AUTO` and the current `DistributionStrategy` chooses
+      when to synchronize. If `synchronization` is set to `ON_READ`,
+      `trainable` must not be set to `True`.
+    aggregation: Indicates how a distributed variable will be aggregated.
+      Accepted values are constants defined in the class
+      @{tf.VariableAggregation}.
 
   Returns:
     The created or existing variable.
@@ -242,21 +262,36 @@ def variable(name, shape=None, dtype=None, initializer=None,
     getter = functools.partial(custom_getter,
                                reuse=variable_scope.get_variable_scope().reuse)
   with ops.device(device or ''):
-    return getter(name, shape=shape, dtype=dtype,
-                  initializer=initializer,
-                  regularizer=regularizer,
-                  trainable=trainable,
-                  collections=collections,
-                  caching_device=caching_device,
-                  partitioner=partitioner,
-                  use_resource=use_resource)
+    return getter(
+        name,
+        shape=shape,
+        dtype=dtype,
+        initializer=initializer,
+        regularizer=regularizer,
+        trainable=trainable,
+        collections=collections,
+        caching_device=caching_device,
+        partitioner=partitioner,
+        use_resource=use_resource,
+        synchronization=synchronization,
+        aggregation=aggregation)
 
 
 @contrib_add_arg_scope
-def model_variable(name, shape=None, dtype=dtypes.float32, initializer=None,
-                   regularizer=None, trainable=True, collections=None,
-                   caching_device=None, device=None, partitioner=None,
-                   custom_getter=None, use_resource=None):
+def model_variable(name,
+                   shape=None,
+                   dtype=dtypes.float32,
+                   initializer=None,
+                   regularizer=None,
+                   trainable=True,
+                   collections=None,
+                   caching_device=None,
+                   device=None,
+                   partitioner=None,
+                   custom_getter=None,
+                   use_resource=None,
+                   synchronization=variables.VariableSynchronization.AUTO,
+                   aggregation=variables.VariableAggregation.NONE):
   """Gets an existing model variable with these parameters or creates a new one.
 
   Args:
@@ -283,18 +318,36 @@ def model_variable(name, shape=None, dtype=dtypes.float32, initializer=None,
     custom_getter: Callable that allows overwriting the internal
       get_variable method and has to have the same signature.
     use_resource: If `True` use a ResourceVariable instead of a Variable.
+    synchronization: Indicates when a distributed a variable will be
+      aggregated. Accepted values are constants defined in the class
+      @{tf.VariableSynchronization}. By default the synchronization is set to
+      `AUTO` and the current `DistributionStrategy` chooses
+      when to synchronize. If `synchronization` is set to `ON_READ`,
+      `trainable` must not be set to `True`.
+    aggregation: Indicates how a distributed variable will be aggregated.
+      Accepted values are constants defined in the class
+      @{tf.VariableAggregation}.
 
   Returns:
     The created or existing variable.
   """
   collections = list(collections or [])
   collections += [ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.MODEL_VARIABLES]
-  var = variable(name, shape=shape, dtype=dtype,
-                 initializer=initializer, regularizer=regularizer,
-                 trainable=trainable, collections=collections,
-                 caching_device=caching_device, device=device,
-                 partitioner=partitioner, custom_getter=custom_getter,
-                 use_resource=use_resource)
+  var = variable(
+      name,
+      shape=shape,
+      dtype=dtype,
+      initializer=initializer,
+      regularizer=regularizer,
+      trainable=trainable,
+      collections=collections,
+      caching_device=caching_device,
+      device=device,
+      partitioner=partitioner,
+      custom_getter=custom_getter,
+      use_resource=use_resource,
+      synchronization=synchronization,
+      aggregation=aggregation)
   return var
 
 
diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py
index beeabd6b65..dd602cf3a9 100644
--- a/tensorflow/contrib/layers/python/layers/layers.py
+++ b/tensorflow/contrib/layers/python/layers/layers.py
@@ -1702,19 +1702,22 @@ def _inner_flatten(inputs, new_rank, output_collections=None, scope=None):
   return utils.collect_named_outputs(output_collections, sc, flattened)
 
 
-def _model_variable_getter(getter,
-                           name,
-                           shape=None,
-                           dtype=None,
-                           initializer=None,
-                           regularizer=None,
-                           trainable=True,
-                           collections=None,
-                           caching_device=None,
-                           partitioner=None,
-                           rename=None,
-                           use_resource=None,
-                           **_):
+def _model_variable_getter(
+    getter,
+    name,
+    shape=None,
+    dtype=None,
+    initializer=None,
+    regularizer=None,
+    trainable=True,
+    collections=None,
+    caching_device=None,
+    partitioner=None,
+    rename=None,
+    use_resource=None,
+    synchronization=tf_variables.VariableSynchronization.AUTO,
+    aggregation=tf_variables.VariableAggregation.NONE,
+    **_):
   """Getter that uses model_variable for compatibility with core layers."""
   short_name = name.split('/')[-1]
   if rename and short_name in rename:
@@ -1732,7 +1735,9 @@ def _model_variable_getter(getter,
       caching_device=caching_device,
       partitioner=partitioner,
       custom_getter=getter,
-      use_resource=use_resource)
+      use_resource=use_resource,
+      synchronization=synchronization,
+      aggregation=aggregation)
 
 
 def _build_variable_getter(rename=None):
-- 
cgit v1.2.3


From 89e06304aad35bfb019a8c10f39fc1ead83e0f99 Mon Sep 17 00:00:00 2001
From: Anjali Sridhar 
Date: Sun, 22 Jul 2018 16:32:41 -0700
Subject: Add support for `is_tensor_like` property to DistributedValues and
 add support for calling `assign` on TowerLocalVariables.

PiperOrigin-RevId: 205595323
---
 .../python/mirrored_strategy_multigpu_test.py      | 69 ++++++++++++++++++++++
 tensorflow/contrib/distribute/python/values.py     | 33 +++++++----
 .../contrib/distribute/python/values_test.py       | 25 ++++++++
 tensorflow/python/framework/tensor_util.py         |  9 ++-
 4 files changed, 123 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
index 6a14b833d2..9807ce4351 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -967,5 +967,74 @@ class MirroredAndTowerLocalVariableInitializerTest(test.TestCase):
         self.evaluate(tower_local_var.initializer)
         self.assertTrue(self.evaluate(tower_local_var.is_initialized()))
 
+
+class TowerLocalVariableAssignTest(test.TestCase):
+  config = config_pb2.ConfigProto()
+  config.allow_soft_placement = True
+
+  def _skip_eager_if_gpus_less_than(self, num_gpus):
+    if context.num_gpus() < num_gpus and context.executing_eagerly():
+      self.skipTest("Enough GPUs not available for this test in eager mode.")
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testAssignTowerLocalVarSumAggregation(self):
+    self._skip_eager_if_gpus_less_than(1)
+    def model_fn():
+      v_sum = variable_scope.variable(
+          1.0,
+          synchronization=variable_scope.VariableSynchronization.ON_READ,
+          aggregation=variable_scope.VariableAggregation.SUM)
+      return v_sum
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with dist.scope():
+      tower_local_var = dist.call_for_each_tower(model_fn,
+                                                 run_concurrently=False)
+      self.assertTrue(isinstance(tower_local_var, values.TowerLocalVariable))
+      self.evaluate(variables.global_variables_initializer())
+      # Each tower has a value of 1.0 assigned to it in tower context.
+      # When we read the value using `read_var` we should see the SUM of each of
+      # values on each of the towers.
+      self.assertEqual(2.0, self.evaluate(dist.read_var(tower_local_var)))
+      # Assigning 6.0 in cross tower context will assign a value of
+      # 6.0/num_towers to each tower.
+      tlv_ops = tower_local_var.assign(6.0)
+      self.evaluate(tlv_ops)
+      # On reading the tower local var we should get the assigned value back.
+      # The value on all the towers are added before being returned by
+      # `read_var`.
+      self.assertEqual(6.0, self.evaluate(dist.read_var(tower_local_var)))
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testAssignTowerLocalVarMeanAggregation(self):
+    self._skip_eager_if_gpus_less_than(1)
+    def model_fn():
+      v_sum = variable_scope.variable(
+          1.0,
+          synchronization=variable_scope.VariableSynchronization.ON_READ,
+          aggregation=variable_scope.VariableAggregation.MEAN)
+      return v_sum
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with dist.scope():
+      tower_local_var = dist.call_for_each_tower(model_fn,
+                                                 run_concurrently=False)
+      self.assertTrue(isinstance(tower_local_var, values.TowerLocalVariable))
+      self.evaluate(variables.global_variables_initializer())
+      # Each tower has a value of 1.0 assigned to it in tower context.
+      # When we read the value using `read_var` we should see the MEAN of values
+      # on all towers which is the value assigned in tower context.
+      self.assertEqual(1.0, self.evaluate(dist.read_var(tower_local_var)))
+      tlv_ops = tower_local_var.assign(6.0)
+      self.evaluate(tlv_ops)
+      # On reading the tower local var we should get the MEAN of all values
+      # which is equal to the value assigned.
+      self.assertEqual(6.0, self.evaluate(dist.read_var(tower_local_var)))
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index 3162aebf5b..47dcf679c2 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -30,6 +30,7 @@ from tensorflow.contrib.distribute.python import prefetching_ops_v2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import device as tf_device
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
@@ -77,6 +78,13 @@ class DistributedValues(object):
   def devices(self):
     return list(self._index.keys())
 
+  @property
+  def is_tensor_like(self):
+    for v in self._index.values():
+      if not tensor_util.is_tensor(v):
+        return False
+    return True
+
   def __str__(self):
     return "%s:%s" % (self.__class__.__name__, self._index)
 
@@ -352,6 +360,7 @@ class MirroredVariable(DistributedVariable, Mirrored,
       return distribute_lib.get_distribution_strategy().update(
           self, f, *args, **kwargs)
     else:
+      _assert_tower_context()
       # We are calling an assign function on the mirrored variable in tower
       # context.
       # We reduce the value we want to assign/add/sub. More details about how we
@@ -448,14 +457,7 @@ class _TowerLocalSaveable(saver.BaseSaverBuilder.SaveableObject):
   def restore(self, restored_tensors, restored_shapes):
     """Restore the same value into all variables."""
     tensor, = restored_tensors
-    # To preserve the sum across save and restore, we have to divide the
-    # total across all devices when restoring a variable that was summed
-    # when saving.
-    if self._tower_local_variable.aggregation == vs.VariableAggregation.SUM:
-      tensor *= 1. / len(self._tower_local_variable.devices)
-    return control_flow_ops.group([
-        _assign_on_device(d, v, tensor)
-        for d, v in six.iteritems(self._tower_local_variable._index)])  # pylint: disable=protected-access
+    return self._tower_local_variable.assign(tensor)
 
 
 def _assert_tower_context():
@@ -482,8 +484,19 @@ class TowerLocalVariable(DistributedVariable, PerDevice,
     return self.get().assign_add(*args, **kwargs)
 
   def assign(self, *args, **kwargs):
-    _assert_tower_context()
-    return self.get().assign(*args, **kwargs)
+    if distribute_lib.get_cross_tower_context():
+      # To preserve the sum across save and restore, we have to divide the
+      # total across all devices when restoring a variable that was summed
+      # when saving.
+      tensor = args[0]
+      if self._aggregation == vs.VariableAggregation.SUM:
+        tensor *= 1. / len(self.devices)
+      return control_flow_ops.group(
+          [_assign_on_device(d, v, tensor)
+           for d, v in six.iteritems(self._index)])
+    else:
+      _assert_tower_context()
+      return self.get().assign(*args, **kwargs)
 
   @property
   def aggregation(self):
diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
index 8e44f2fea1..91a43d4999 100644
--- a/tensorflow/contrib/distribute/python/values_test.py
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
@@ -79,6 +80,30 @@ class DistributedValuesTest(test.TestCase):
     with self.assertRaises(AssertionError):
       v = values.DistributedValues({"/device:cpu:0": 42})
 
+  def testIsTensorLike(self):
+    with context.graph_mode(), \
+         ops.Graph().as_default(), \
+         ops.device("/device:CPU:0"):
+      one = constant_op.constant(1)
+      two = constant_op.constant(2)
+      v = values.DistributedValues({"/device:CPU:0": one, "/device:GPU:0": two})
+      self.assertEqual(two, v.get("/device:GPU:0"))
+      self.assertEqual(one, v.get())
+      self.assertTrue(v.is_tensor_like)
+      self.assertTrue(tensor_util.is_tensor(v))
+
+  def testIsTensorLikeWithAConstant(self):
+    with context.graph_mode(), \
+         ops.Graph().as_default(), \
+         ops.device("/device:CPU:0"):
+      one = constant_op.constant(1)
+      two = 2.0
+      v = values.DistributedValues({"/device:CPU:0": one, "/device:GPU:0": two})
+      self.assertEqual(two, v.get("/device:GPU:0"))
+      self.assertEqual(one, v.get())
+      self.assertFalse(v.is_tensor_like)
+      self.assertFalse(tensor_util.is_tensor(v))
+
 
 class DistributedDelegateTest(test.TestCase):
 
diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index ca63efbc84..8c9dfce7cc 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -935,8 +935,10 @@ def constant_value_as_shape(tensor):  # pylint: disable=invalid-name
 def is_tensor(x):  # pylint: disable=invalid-name
   """Check whether `x` is of tensor type.
 
-  Check whether an object is a tensor. Equivalent to
-  `isinstance(x, [tf.Tensor, tf.SparseTensor, tf.Variable])`.
+  Check whether an object is a tensor. This check is equivalent to calling
+  `isinstance(x, [tf.Tensor, tf.SparseTensor, tf.Variable])` and also checks
+  if all the component variables of a MirroredVariable or a TowerLocalVariable
+  are tensors.
 
   Args:
     x: A python object to check.
@@ -944,4 +946,5 @@ def is_tensor(x):  # pylint: disable=invalid-name
   Returns:
     `True` if `x` is a tensor, `False` if not.
   """
-  return isinstance(x, ops._TensorLike) or ops.is_dense_tensor_like(x)  # pylint: disable=protected-access
+  return (isinstance(x, ops._TensorLike) or ops.is_dense_tensor_like(x) or  # pylint: disable=protected-access
+          (hasattr(x, "is_tensor_like") and x.is_tensor_like))
-- 
cgit v1.2.3


From 21d0205916eded7e2bf2f26e43dd41b2f86cba3f Mon Sep 17 00:00:00 2001
From: David Majnemer 
Date: Mon, 23 Jul 2018 06:15:33 -0700
Subject: [XLA:CPU,GPU] Implement more cases of convert

This adds support for {S32,U32,F32} -> PRED and adds test for several other
cases as well.

PiperOrigin-RevId: 205650630
---
 .../compiler/xla/service/elemental_ir_emitter.cc   | 18 +++++-
 tensorflow/compiler/xla/tests/convert_test.cc      | 70 +++++++++++++++++++++-
 2 files changed, 83 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index 1eedd85363..b58b87a978 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -222,10 +222,17 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp(
     case HloOpcode::kConvert: {
       PrimitiveType from_type = op->operand(0)->shape().element_type();
       PrimitiveType to_type = op->shape().element_type();
-      CHECK(primitive_util::IsIntegralType(from_type) || from_type == PRED);
+      CHECK(primitive_util::IsIntegralType(from_type) || from_type == PRED)
+          << from_type;
       if (from_type == to_type) {
         return operand_value;
       }
+      if (to_type == PRED) {
+        return b_->CreateZExt(
+            b_->CreateICmpNE(operand_value, llvm::ConstantInt::get(
+                                                operand_value->getType(), 0)),
+            llvm_ir::PrimitiveTypeToIrType(PRED, module_));
+      }
       if (primitive_util::IsIntegralType(to_type)) {
         return b_->CreateIntCast(
             operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_),
@@ -342,7 +349,7 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp(
     case HloOpcode::kConvert: {
       PrimitiveType from_type = op->operand(0)->shape().element_type();
       PrimitiveType to_type = op->shape().element_type();
-      CHECK(primitive_util::IsFloatingPointType(from_type));
+      CHECK(primitive_util::IsFloatingPointType(from_type)) << from_type;
       if (from_type == to_type) {
         return operand_value;
       }
@@ -369,6 +376,13 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp(
       if (from_type == F32 && to_type == BF16) {
         return EmitF32ToBF16(operand_value, b_);
       }
+      if (to_type == PRED) {
+        return b_->CreateZExt(
+            b_->CreateFCmpUNE(
+                operand_value,
+                llvm::ConstantFP::get(operand_value->getType(), 0.0)),
+            llvm_ir::PrimitiveTypeToIrType(PRED, module_));
+      }
       if (primitive_util::IsFloatingPointType(to_type)) {
         return b_->CreateFPCast(
             operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
diff --git a/tensorflow/compiler/xla/tests/convert_test.cc b/tensorflow/compiler/xla/tests/convert_test.cc
index dca57fd1c7..0fb6853e3f 100644
--- a/tensorflow/compiler/xla/tests/convert_test.cc
+++ b/tensorflow/compiler/xla/tests/convert_test.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include 
 #include 
 #include 
 #include 
@@ -52,13 +53,67 @@ TEST_F(ConvertTest, ConvertR1S32ToR1S32) {
   ComputeAndCompareR1(&builder, expected, {});
 }
 
+TEST_F(ConvertTest, ConvertR1S32ToR1U32) {
+  XlaBuilder builder(TestName());
+  auto a = ConstantR1(&builder, {42, 64});
+  ConvertElementType(a, U32);
+
+  std::vector expected = {42, 64};
+  ComputeAndCompareR1(&builder, expected, {});
+}
+
+TEST_F(ConvertTest, ConvertR1S32ToR1PRED) {
+  XlaBuilder builder(TestName());
+  auto a = ConstantR1(&builder, {42, 0, -64});
+  ConvertElementType(a, PRED);
+
+  std::array expected = {true, false, true};
+  ComputeAndCompareR1(&builder, expected, {});
+}
+
+TEST_F(ConvertTest, ConvertR1U32ToR1U32) {
+  XlaBuilder builder(TestName());
+  auto a = ConstantR1(&builder, {42, 64});
+  ConvertElementType(a, U32);
+
+  std::vector expected = {42, 64};
+  ComputeAndCompareR1(&builder, expected, {});
+}
+
+TEST_F(ConvertTest, ConvertR1U32ToR1S32) {
+  XlaBuilder builder(TestName());
+  auto a = ConstantR1(&builder, {42, 64});
+  ConvertElementType(a, S32);
+
+  std::vector expected = {42, 64};
+  ComputeAndCompareR1(&builder, expected, {});
+}
+
+TEST_F(ConvertTest, ConvertR1U32ToR1PRED) {
+  XlaBuilder builder(TestName());
+  auto a = ConstantR1(&builder, {42, 0, 64});
+  ConvertElementType(a, PRED);
+
+  std::array expected = {true, false, true};
+  ComputeAndCompareR1(&builder, expected, {});
+}
+
 TEST_F(ConvertTest, ConvertR1F32ToR1F32) {
   XlaBuilder builder(TestName());
   auto a = ConstantR1(&builder, {42.0f, 64.0f});
   ConvertElementType(a, F32);
 
   std::vector expected = {42.0f, 64.0f};
-  ComputeAndCompareR1(&builder, expected, {}, ErrorSpec(0.0001));
+  ComputeAndCompareR1(&builder, expected, {});
+}
+
+TEST_F(ConvertTest, ConvertR1F32ToR1PRED) {
+  XlaBuilder builder(TestName());
+  auto a = ConstantR1(&builder, {42.0f, 0.0f, 64.0f});
+  ConvertElementType(a, PRED);
+
+  std::array expected = {true, false, true};
+  ComputeAndCompareR1(&builder, expected, {});
 }
 
 TEST_F(ConvertTest, ConvertR1S32ToR1F32) {
@@ -67,7 +122,7 @@ TEST_F(ConvertTest, ConvertR1S32ToR1F32) {
   ConvertElementType(a, F32);
 
   std::vector expected = {42.0f, 64.0f};
-  ComputeAndCompareR1(&builder, expected, {}, ErrorSpec(0.0001));
+  ComputeAndCompareR1(&builder, expected, {});
 }
 
 TEST_F(ConvertTest, ConvertR1PREDToR1S32) {
@@ -79,6 +134,15 @@ TEST_F(ConvertTest, ConvertR1PREDToR1S32) {
   ComputeAndCompareR1(&builder, expected, {});
 }
 
+TEST_F(ConvertTest, ConvertR1PREDToR1U32) {
+  XlaBuilder builder(TestName());
+  auto a = ConstantR1(&builder, {true, false, true});
+  ConvertElementType(a, U32);
+
+  std::vector expected = {1, 0, 1};
+  ComputeAndCompareR1(&builder, expected, {});
+}
+
 TEST_F(ConvertTest, ConvertR1PREDToR1F32) {
   XlaBuilder builder(TestName());
   auto a = ConstantR1(&builder, {true, false, true});
@@ -94,7 +158,7 @@ XLA_TEST_F(ConvertTest, ConvertR1S0S32ToR1S0F32) {
   ConvertElementType(a, F32);
 
   std::vector expected = {};
-  ComputeAndCompareR1(&builder, expected, {}, ErrorSpec(0.0001));
+  ComputeAndCompareR1(&builder, expected, {});
 }
 
 TEST_F(ConvertTest, ConvertR1F32ToR1S32) {
-- 
cgit v1.2.3


From 7aef462279657517377e0da15b90b3f3f5be16e1 Mon Sep 17 00:00:00 2001
From: James Keeling 
Date: Mon, 23 Jul 2018 07:12:57 -0700
Subject: Add GetFilteredRegisteredKernels and refactor

GetFilteredRegisteredKernels makes it easier for users to query at runtime which kernels are available which match some predicate. The most common usage will be querying which kernels are available for a given op, so we add the specialized GetRegisteredKernelsForOp.

This is part of the work to make available kernels possible to query, to support Swift For TensorFlow. There are also a number of github issues asking for the functionality.

I will add C API and Python API support in upcoming changes.

PiperOrigin-RevId: 205656251
---
 tensorflow/core/framework/op_kernel.cc      | 45 ++++++++++++++++++-----------
 tensorflow/core/framework/op_kernel.h       |  7 +++++
 tensorflow/core/framework/op_kernel_test.cc | 18 +++++++++++-
 3 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc
index 58feec90f0..507aa9e447 100644
--- a/tensorflow/core/framework/op_kernel.cc
+++ b/tensorflow/core/framework/op_kernel.cc
@@ -1061,40 +1061,51 @@ Status SupportedDeviceTypesForNode(
 }
 
 void LogAllRegisteredKernels() {
-  for (const auto& key_registration : *GlobalKernelRegistryTyped()) {
-    const KernelDef& kernel_def(key_registration.second.def);
+  KernelList kernel_list = GetAllRegisteredKernels();
+  for (const auto& kernel_def : kernel_list.kernel()) {
     LOG(INFO) << "OpKernel ('" << ProtoShortDebugString(kernel_def) << "')";
   }
 }
 
 KernelList GetAllRegisteredKernels() {
+  return GetFilteredRegisteredKernels([](const KernelDef& k) { return true; });
+}
+
+KernelList GetFilteredRegisteredKernels(
+    const std::function& predicate) {
   const KernelRegistry* const typed_registry = GlobalKernelRegistryTyped();
   KernelList kernel_list;
   kernel_list.mutable_kernel()->Reserve(typed_registry->size());
   for (const auto& p : *typed_registry) {
-    *kernel_list.add_kernel() = p.second.def;
+    const KernelDef& kernel_def = p.second.def;
+    if (predicate(kernel_def)) {
+      *kernel_list.add_kernel() = kernel_def;
+    }
   }
   return kernel_list;
 }
 
+KernelList GetRegisteredKernelsForOp(StringPiece op_name) {
+  auto op_pred = [op_name](const KernelDef& k) { return k.op() == op_name; };
+  return GetFilteredRegisteredKernels(op_pred);
+}
+
 string KernelsRegisteredForOp(StringPiece op_name) {
+  KernelList kernel_list = GetRegisteredKernelsForOp(op_name);
+  if (kernel_list.kernel_size() == 0) return "  \n";
   string ret;
-  for (const auto& key_registration : *GlobalKernelRegistryTyped()) {
-    const KernelDef& kernel_def(key_registration.second.def);
-    if (kernel_def.op() == op_name) {
-      strings::StrAppend(&ret, "  device='", kernel_def.device_type(), "'");
-      if (!kernel_def.label().empty()) {
-        strings::StrAppend(&ret, "; label='", kernel_def.label(), "'");
-      }
-      for (int i = 0; i < kernel_def.constraint_size(); ++i) {
-        strings::StrAppend(
-            &ret, "; ", kernel_def.constraint(i).name(), " in ",
-            SummarizeAttrValue(kernel_def.constraint(i).allowed_values()));
-      }
-      strings::StrAppend(&ret, "\n");
+  for (const auto& kernel_def : kernel_list.kernel()) {
+    strings::StrAppend(&ret, "  device='", kernel_def.device_type(), "'");
+    if (!kernel_def.label().empty()) {
+      strings::StrAppend(&ret, "; label='", kernel_def.label(), "'");
+    }
+    for (int i = 0; i < kernel_def.constraint_size(); ++i) {
+      strings::StrAppend(
+          &ret, "; ", kernel_def.constraint(i).name(), " in ",
+          SummarizeAttrValue(kernel_def.constraint(i).allowed_values()));
     }
+    strings::StrAppend(&ret, "\n");
   }
-  if (ret.empty()) return "  \n";
   return ret;
 }
 
diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h
index d9fe42fcbb..1fc5e9908e 100644
--- a/tensorflow/core/framework/op_kernel.h
+++ b/tensorflow/core/framework/op_kernel.h
@@ -1304,6 +1304,13 @@ void LogAllRegisteredKernels();
 // Gets a list of all registered kernels.
 KernelList GetAllRegisteredKernels();
 
+// Gets a list of all registered kernels for which predicate returns true
+KernelList GetFilteredRegisteredKernels(
+    const std::function& predicate);
+
+// Gets a list of all registered kernels for a given op
+KernelList GetRegisteredKernelsForOp(StringPiece op_name);
+
 namespace kernel_factory {
 
 class OpKernelRegistrar {
diff --git a/tensorflow/core/framework/op_kernel_test.cc b/tensorflow/core/framework/op_kernel_test.cc
index b76a3400a8..83dda6579b 100644
--- a/tensorflow/core/framework/op_kernel_test.cc
+++ b/tensorflow/core/framework/op_kernel_test.cc
@@ -965,7 +965,8 @@ BENCHMARK(BM_ConcatInputRange);
 BENCHMARK(BM_SelectInputRange);
 
 TEST(RegisteredKernels, CanCallGetAllRegisteredKernels) {
-  auto all_registered_kernels = GetAllRegisteredKernels().kernel();
+  auto kernel_list = GetAllRegisteredKernels();
+  auto all_registered_kernels = kernel_list.kernel();
   auto has_name_test1 = [](const KernelDef& k) { return k.op() == "Test1"; };
 
   // Verify we can find the "Test1" op registered above
@@ -986,5 +987,20 @@ TEST(RegisteredKernels, CanLogAllRegisteredKernels) {
   tensorflow::LogAllRegisteredKernels();
 }
 
+TEST(RegisteredKernels, GetFilteredRegisteredKernels) {
+  auto has_name_test1 = [](const KernelDef& k) { return k.op() == "Test1"; };
+  auto kernel_list = GetFilteredRegisteredKernels(has_name_test1);
+  ASSERT_EQ(kernel_list.kernel_size(), 1);
+  EXPECT_EQ(kernel_list.kernel(0).op(), "Test1");
+  EXPECT_EQ(kernel_list.kernel(0).device_type(), "CPU");
+}
+
+TEST(RegisteredKernels, GetRegisteredKernelsForOp) {
+  auto kernel_list = GetRegisteredKernelsForOp("Test1");
+  ASSERT_EQ(kernel_list.kernel_size(), 1);
+  EXPECT_EQ(kernel_list.kernel(0).op(), "Test1");
+  EXPECT_EQ(kernel_list.kernel(0).device_type(), "CPU");
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
cgit v1.2.3


From 8647db865ce41361413a2eb4c3b4d0ba404dd4e0 Mon Sep 17 00:00:00 2001
From: James Keeling 
Date: Mon, 23 Jul 2018 07:56:46 -0700
Subject: Add C API for kernel info

This is part of the work to make available kernels possible to query, to support Swift For TensorFlow. There are also a number of github issues asking for the functionality.

PiperOrigin-RevId: 205660862
---
 tensorflow/c/c_api.cc      | 24 +++++++++++++++++++++
 tensorflow/c/c_api.h       | 12 +++++++++++
 tensorflow/c/c_api_test.cc | 53 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 89 insertions(+)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index a3003953a3..1b937883c8 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -33,6 +33,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eval_const_tensor.h"
 #include "tensorflow/core/common_runtime/shape_refiner.h"
 #include "tensorflow/core/framework/allocation_description.pb.h"
+#include "tensorflow/core/framework/kernel_def.pb.h"
 #include "tensorflow/core/framework/log_memory.h"
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -2729,4 +2730,27 @@ TF_Buffer* TF_ApiDefMapGet(TF_ApiDefMap* api_def_map, const char* name,
   return ret;
 #endif  // __ANDROID__
 }
+
+TF_Buffer* TF_GetAllRegisteredKernels(TF_Status* status) {
+  tensorflow::KernelList kernel_list = tensorflow::GetAllRegisteredKernels();
+  TF_Buffer* ret = TF_NewBuffer();
+  status->status = MessageToBuffer(kernel_list, ret);
+  if (!status->status.ok()) {
+    TF_DeleteBuffer(ret);
+    return nullptr;
+  }
+  return ret;
+}
+
+TF_Buffer* TF_GetRegisteredKernelsForOp(const char* name, TF_Status* status) {
+  tensorflow::KernelList kernel_list =
+      tensorflow::GetRegisteredKernelsForOp(name);
+  TF_Buffer* ret = TF_NewBuffer();
+  status->status = MessageToBuffer(kernel_list, ret);
+  if (!status->status.ok()) {
+    TF_DeleteBuffer(ret);
+    return nullptr;
+  }
+  return ret;
+}
 }  // end extern "C"
diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index fddc09d45e..c5035e0e41 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -1610,6 +1610,18 @@ TF_CAPI_EXPORT extern TF_Buffer* TF_ApiDefMapGet(TF_ApiDefMap* api_def_map,
                                                  size_t name_len,
                                                  TF_Status* status);
 
+// --------------------------------------------------------------------------
+// Kernel definition information.
+
+// Returns a serialized KernelList protocol buffer containing KernelDefs for all
+// registered kernels.
+TF_CAPI_EXPORT extern TF_Buffer* TF_GetAllRegisteredKernels(TF_Status* status);
+
+// Returns a serialized KernelList protocol buffer containing KernelDefs for all
+// kernels registered for the operation named `name`.
+TF_CAPI_EXPORT extern TF_Buffer* TF_GetRegisteredKernelsForOp(
+    const char* name, TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc
index bc04b53fbb..c470ab5649 100644
--- a/tensorflow/c/c_api_test.cc
+++ b/tensorflow/c/c_api_test.cc
@@ -29,9 +29,11 @@ limitations under the License.
 #include "tensorflow/core/framework/api_def.pb.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/graph.pb_text.h"
+#include "tensorflow/core/framework/kernel_def.pb.h"
 #include "tensorflow/core/framework/node_def.pb_text.h"
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
@@ -2312,6 +2314,57 @@ TEST(TestApiDef, TestCreateApiDefWithOverwrites) {
   TF_DeleteLibraryHandle(lib);
 }
 
+class DummyKernel : public tensorflow::OpKernel {
+ public:
+  explicit DummyKernel(tensorflow::OpKernelConstruction* context)
+      : OpKernel(context) {}
+  void Compute(tensorflow::OpKernelContext* context) override {}
+};
+
+// Test we can query kernels
+REGISTER_OP("TestOpWithSingleKernel")
+    .Input("a: float")
+    .Input("b: float")
+    .Output("o: float");
+REGISTER_KERNEL_BUILDER(
+    Name("TestOpWithSingleKernel").Device(tensorflow::DEVICE_CPU), DummyKernel);
+
+TEST(TestKernel, TestGetAllRegisteredKernels) {
+  TF_Status* status = TF_NewStatus();
+  TF_Buffer* kernel_list_buf = TF_GetAllRegisteredKernels(status);
+  EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  KernelList kernel_list;
+  kernel_list.ParseFromArray(kernel_list_buf->data, kernel_list_buf->length);
+  ASSERT_GT(kernel_list.kernel_size(), 0);
+  TF_DeleteBuffer(kernel_list_buf);
+  TF_DeleteStatus(status);
+}
+
+TEST(TestKernel, TestGetRegisteredKernelsForOp) {
+  TF_Status* status = TF_NewStatus();
+  TF_Buffer* kernel_list_buf =
+      TF_GetRegisteredKernelsForOp("TestOpWithSingleKernel", status);
+  EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  KernelList kernel_list;
+  kernel_list.ParseFromArray(kernel_list_buf->data, kernel_list_buf->length);
+  ASSERT_EQ(kernel_list.kernel_size(), 1);
+  EXPECT_EQ(kernel_list.kernel(0).op(), "TestOpWithSingleKernel");
+  EXPECT_EQ(kernel_list.kernel(0).device_type(), "CPU");
+  TF_DeleteBuffer(kernel_list_buf);
+  TF_DeleteStatus(status);
+}
+
+TEST(TestKernel, TestGetRegisteredKernelsForOpNoKernels) {
+  TF_Status* status = TF_NewStatus();
+  TF_Buffer* kernel_list_buf = TF_GetRegisteredKernelsForOp("Unknown", status);
+  EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  KernelList kernel_list;
+  kernel_list.ParseFromArray(kernel_list_buf->data, kernel_list_buf->length);
+  ASSERT_EQ(kernel_list.kernel_size(), 0);
+  TF_DeleteBuffer(kernel_list_buf);
+  TF_DeleteStatus(status);
+}
+
 #undef EXPECT_TF_META
 
 }  // namespace
-- 
cgit v1.2.3


From 32fe0302b0cf02d6cc3ae6cf67b233ad65c74bfe Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 08:03:04 -0700
Subject: Delegate L2Norm to nnapi.

PiperOrigin-RevId: 205661557
---
 .../contrib/lite/delegates/nnapi/nnapi_delegate.cc | 12 ++++++++
 .../lite/delegates/nnapi/nnapi_delegate_test.cc    | 35 ++++++++++++++++++++++
 tensorflow/contrib/lite/nnapi_delegate.cc          |  9 +++++-
 3 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/contrib/lite/delegates/nnapi/nnapi_delegate.cc
index f0d16575ec..0c7f6d3125 100644
--- a/tensorflow/contrib/lite/delegates/nnapi/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/delegates/nnapi/nnapi_delegate.cc
@@ -452,6 +452,18 @@ class NNAPIDelegateKernel {
         } else {
           return nullptr;
         }
+      case kTfLiteBuiltinL2Normalization: {
+        auto builtin =
+            reinterpret_cast(node->builtin_data);
+        if (builtin->activation != kTfLiteActNone) {
+          // NNAPI does not support activations
+          return nullptr;
+        }
+        return [](TfLiteContext* context, NNAPIOpBuilder* builder,
+                  TfLiteNode* node) -> ANeuralNetworksOperationType {
+          return ANEURALNETWORKS_L2_NORMALIZATION;
+        };
+      }
       case kTfLiteBuiltinTranspose:
         // Transpose requires NNAPI1.1. Also note that the permutation input
         // tensor value dictates the output dimensions.
diff --git a/tensorflow/contrib/lite/delegates/nnapi/nnapi_delegate_test.cc b/tensorflow/contrib/lite/delegates/nnapi/nnapi_delegate_test.cc
index ab2181e8ff..baf8046f9b 100644
--- a/tensorflow/contrib/lite/delegates/nnapi/nnapi_delegate_test.cc
+++ b/tensorflow/contrib/lite/delegates/nnapi/nnapi_delegate_test.cc
@@ -641,6 +641,41 @@ TEST(NNAPIDelegate, SqueezeWithAxisTest) {
                         17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0}));
 }
 
+class L2NormOpModel : public SingleOpModelWithNNAPI {
+ public:
+  L2NormOpModel(const TensorData& input, const TensorData& output,
+                ActivationFunctionType activation_type) {
+    input_ = AddInput(input);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_L2_NORMALIZATION, BuiltinOptions_L2NormOptions,
+                 CreateL2NormOptions(builder_, activation_type).Union());
+    BuildInterpreter({GetShape(input_)});
+  }
+
+  void SetInput(std::initializer_list data) {
+    PopulateTensor(input_, data);
+  }
+  std::vector GetOutput() { return ExtractVector(output_); }
+  std::vector GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int input_;
+  int new_shape_;
+  int output_;
+};
+
+TEST(NNAPIDelegate, L2NormSimpleTest) {
+  std::initializer_list data = {-1.1, 0.6, 0.7, 1.2, -0.7, 0.1};
+  L2NormOpModel m({TensorType_FLOAT32, {1, 1, 1, 6}},
+                  {TensorType_FLOAT32, {1, 1, 1, 6}},
+                  ActivationFunctionType_NONE);
+  m.SetInput(data);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 1, 1, 6}));
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray({-0.55, 0.3, 0.35, 0.6, -0.35, 0.05}));
+}
+
 class TransposeSimpleModel : public SingleOpModelWithNNAPI {
  public:
   TransposeSimpleModel(std::initializer_list input_shape,
diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc
index 710ce1632e..659230e033 100644
--- a/tensorflow/contrib/lite/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/nnapi_delegate.cc
@@ -560,6 +560,14 @@ TfLiteStatus AddOpsAndParams(
         nnapi_version = 11;  // require NNAPI 1.1
         nn_op_type = ANEURALNETWORKS_TRANSPOSE;
         break;
+      case tflite::BuiltinOperator_L2_NORMALIZATION:
+        nn_op_type = ANEURALNETWORKS_L2_NORMALIZATION;
+        if (reinterpret_cast(node.builtin_data)
+                ->activation != kTfLiteActNone) {
+          FATAL(
+              "NNAPI does not support L2Normalization with fused activations");
+        }
+        break;
       case tflite::BuiltinOperator_CONCAT_EMBEDDINGS:
       case tflite::BuiltinOperator_LSH_PROJECTION:
       case tflite::BuiltinOperator_HASHTABLE_LOOKUP:
@@ -568,7 +576,6 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_EMBEDDING_LOOKUP_SPARSE:
       case tflite::BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM:
       case tflite::BuiltinOperator_UNIDIRECTIONAL_SEQUENCE_LSTM:
-      case tflite::BuiltinOperator_L2_NORMALIZATION:
       case tflite::BuiltinOperator_LOCAL_RESPONSE_NORMALIZATION:
       case tflite::BuiltinOperator_PADV2:
       case tflite::BuiltinOperator_RESIZE_BILINEAR:
-- 
cgit v1.2.3


From 68d48f52bc00a09e9d2458d0f267616030b27bf4 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer 
Date: Mon, 23 Jul 2018 08:31:00 -0700
Subject: [XLA:GPU] Make sure that buffers for tuple() have a unique top-level
 allocation

There are edge cases where a top-level allocation exists but it's ambiguous.

PiperOrigin-RevId: 205665320
---
 .../xla/service/gpu/ir_emitter_unnested.cc         |  5 ++-
 tensorflow/compiler/xla/tests/tuple_test.cc        | 47 ++++++++++++++++++++++
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index b3229303df..4844dc92db 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -1754,8 +1754,9 @@ Status IrEmitterUnnested::HandleReduce(HloInstruction* reduce) {
 Status IrEmitterUnnested::HandleTuple(HloInstruction* tuple) {
   bool all_tuple_elements_have_buffer =
       c_all_of(tuple->operands(), [&](HloInstruction* tuple_element) {
-        return ir_emitter_context_->buffer_assignment().HasTopLevelAllocation(
-            tuple_element);
+        return ir_emitter_context_->buffer_assignment()
+            .GetUniqueTopLevelSlice(tuple_element)
+            .ok();
       });
   // Tuples (especially tuples that are the final result of a computation) can
   // be so huge that if we were to emit a kernel that took each tuple element as
diff --git a/tensorflow/compiler/xla/tests/tuple_test.cc b/tensorflow/compiler/xla/tests/tuple_test.cc
index bf86c5dfb6..a517007591 100644
--- a/tensorflow/compiler/xla/tests/tuple_test.cc
+++ b/tensorflow/compiler/xla/tests/tuple_test.cc
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace xla {
@@ -545,5 +546,51 @@ XLA_TEST_F(TupleHloTest, DISABLED_ON_INTERPRETER(BitcastAfterGTE)) {
       *result));
 }
 
+// Disabled on interpreter due to lack of outfeed.
+XLA_TEST_F(TupleHloTest,
+           DISABLED_ON_INTERPRETER(NonAmbiguousTopLevelAllocation)) {
+  const char* testcase = R"(
+    HloModule tuple
+
+    ENTRY main {
+      a = f32[2] parameter(0)
+      b = f32[2] parameter(1)
+      c = f32[2] parameter(2)
+      d = f32[2] parameter(3)
+      cond = pred[] parameter(4)
+
+      tup0 = (f32[2],f32[2]) tuple(a, b)
+      tup1 = (f32[2],f32[2]) tuple(c, d)
+
+      s = (f32[2],f32[2]) tuple-select(cond, tup0, tup1)
+      gte = f32[2] get-tuple-element(s), index=0
+      tuple = (f32[2]) tuple(gte)
+      token = token[] after-all()
+      ROOT outfeed = token[] outfeed(tuple, token)
+    }
+  )";
+  auto module =
+      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
+          .ValueOrDie();
+  auto param0 = LiteralUtil::CreateR1({1, 2});
+  auto param1 = LiteralUtil::CreateR1({2, 3});
+  auto param4 = LiteralUtil::CreateR0(false);
+  // Put execution on a separate thread so we can block on outfeed.
+  std::unique_ptr thread(
+      tensorflow::Env::Default()->StartThread(
+          tensorflow::ThreadOptions(), "execute_thread", [&] {
+            TF_EXPECT_OK(Execute(std::move(module),
+                                 {param0.get(), param1.get(), param1.get(),
+                                  param0.get(), param4.get()})
+                             .status());
+          }));
+  auto expected =
+      LiteralUtil::MakeTupleOwned(LiteralUtil::CreateR1({2, 3}));
+  auto literal = MakeUnique();
+  TF_EXPECT_OK(backend().transfer_manager()->TransferLiteralFromOutfeed(
+      backend().default_stream_executor(), expected->shape(), literal.get()));
+  EXPECT_TRUE(LiteralTestUtil::Equal(*expected, *literal));
+}
+
 }  // namespace
 }  // namespace xla
-- 
cgit v1.2.3


From 482b056d3ba925f52ccad8e7166a81120f43a761 Mon Sep 17 00:00:00 2001
From: gracehoney <31743510+aaroey@users.noreply.github.com>
Date: Mon, 23 Jul 2018 08:40:11 -0700
Subject: Fix segment_test build dependency.

---
 tensorflow/contrib/tensorrt/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index e3248699dd..1ae3376a4f 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -282,6 +282,7 @@ tf_cc_test(
         "//tensorflow/cc:scope",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:lib",
+        "//tensorflow/core:ops",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
-- 
cgit v1.2.3


From efb8a6d2f89bdb307c23ea393df8701e61bedf9a Mon Sep 17 00:00:00 2001
From: Sergio Guadarrama 
Date: Mon, 23 Jul 2018 08:51:19 -0700
Subject: Default .inputs .outputs of Network should be [] otherwise
 uses_learning_phase fails.

PiperOrigin-RevId: 205667812
---
 tensorflow/python/keras/engine/network.py       |  4 ++--
 tensorflow/python/keras/engine/training_test.py | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py
index a4d96de74f..752e9963ca 100644
--- a/tensorflow/python/keras/engine/network.py
+++ b/tensorflow/python/keras/engine/network.py
@@ -318,8 +318,8 @@ class Network(base_layer.Layer):
     else:
       self._expects_training_arg = False
     self._call_convention = self._determine_call_convention(call_argspec)
-    self.outputs = None
-    self.inputs = None
+    self.outputs = []
+    self.inputs = []
     self.built = False
 
   def _determine_call_convention(self, call_argspec):
diff --git a/tensorflow/python/keras/engine/training_test.py b/tensorflow/python/keras/engine/training_test.py
index c621a88fb3..301a6ca866 100644
--- a/tensorflow/python/keras/engine/training_test.py
+++ b/tensorflow/python/keras/engine/training_test.py
@@ -767,6 +767,22 @@ class LossMaskingTest(test.TestCase):
               keras.backend.variable(weights), keras.backend.variable(mask)))
 
 
+class LearningPhaseTest(test.TestCase):
+
+  def test_empty_model_no_learning_phase(self):
+    with self.test_session():
+      model = keras.models.Sequential()
+      self.assertFalse(model.uses_learning_phase)
+
+  def test_dropout_has_learning_phase(self):
+    with self.test_session():
+      model = keras.models.Sequential()
+      model.add(keras.layers.Dense(2, input_dim=3))
+      model.add(keras.layers.Dropout(0.5))
+      model.add(keras.layers.Dense(2))
+      self.assertTrue(model.uses_learning_phase)
+
+
 class TestDynamicTrainability(test.TestCase):
 
   def test_trainable_warning(self):
-- 
cgit v1.2.3


From 8048b3a53d5a919fef74874fcffbec9039e6acd1 Mon Sep 17 00:00:00 2001
From: Russell Power 
Date: Mon, 23 Jul 2018 09:08:21 -0700
Subject: Destroy tf.Session when the session is closed.

Previously session destruction was delayed until the destructor for the Python session object.  If the session ends up requiring the Python cycle collector for deallocation, it could end up persisting for a long, non-deterministic period.  This can tie up resources and lead to out of memory issues.

This change introduces a SessionRef which causes session.close() to block until all outstanding run operations are finished and tears down the underlying session.

PiperOrigin-RevId: 205670577
---
 tensorflow/core/BUILD                         |   8 ++
 tensorflow/core/common_runtime/session_ref.cc | 170 ++++++++++++++++++++++++++
 tensorflow/core/common_runtime/session_ref.h  |  86 +++++++++++++
 tensorflow/python/BUILD                       |   1 +
 tensorflow/python/client/session.py           |   2 +-
 tensorflow/python/client/tf_session.i         |   1 +
 tensorflow/python/client/tf_session_helper.cc |  14 +++
 tensorflow/python/client/tf_session_helper.h  |   3 +
 8 files changed, 284 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/core/common_runtime/session_ref.cc
 create mode 100644 tensorflow/core/common_runtime/session_ref.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 17e6ccda14..d51d9f0295 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2921,6 +2921,14 @@ tf_cuda_library(
     ] + tf_additional_device_tracer_deps(),
 )
 
+cc_library(
+    name = "session_ref",
+    srcs = ["common_runtime/session_ref.cc"],
+    hdrs = ["common_runtime/session_ref.h"],
+    copts = tf_copts(),
+    deps = [":core_cpu_base"],
+)
+
 cc_library(
     name = "gpu_id",
     hdrs = [
diff --git a/tensorflow/core/common_runtime/session_ref.cc b/tensorflow/core/common_runtime/session_ref.cc
new file mode 100644
index 0000000000..b931ef4229
--- /dev/null
+++ b/tensorflow/core/common_runtime/session_ref.cc
@@ -0,0 +1,170 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/session_ref.h"
+
+#include 
+
+namespace tensorflow {
+
+namespace {
+
+// Scope helper to track active calls and manage session lifetime.
+struct RunCounter {
+  std::shared_ptr session;
+  uint64* value;
+  mutex* m;
+  condition_variable* cv;
+
+  explicit RunCounter(std::shared_ptr s, uint64* v, mutex* m,
+                      condition_variable* cv)
+      : session(std::move(s)), value(v), m(m), cv(cv) {
+    mutex_lock l(*m);
+    ++*value;
+  }
+
+  ~RunCounter() {
+    mutex_lock l(*m);
+    if (--*value == 0) {
+      cv->notify_all();
+    }
+  }
+};
+
+}  // namespace
+
+Status SessionRef::CheckNotClosed() {
+  mutex_lock l(run_lock_);
+  if (session_ == nullptr) return errors::Cancelled("Session has been closed.");
+  return ::tensorflow::Status::OK();
+}
+
+Status SessionRef::Run(const RunOptions& run_options,
+                       const std::vector >& inputs,
+                       const std::vector& output_tensor_names,
+                       const std::vector& target_node_names,
+                       std::vector* outputs,
+                       RunMetadata* run_metadata) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->Run(run_options, inputs, output_tensor_names,
+                         target_node_names, outputs, run_metadata);
+}
+
+Status SessionRef::Create(const GraphDef& graph) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->Create(graph);
+}
+
+Status SessionRef::Create(const RunOptions& run_options,
+                          const GraphDef& graph) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->Create(run_options, graph);
+}
+
+Status SessionRef::Extend(const RunOptions& run_options,
+                          const GraphDef& graph) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->Extend(run_options, graph);
+}
+
+Status SessionRef::Extend(const GraphDef& graph) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->Extend(graph);
+}
+
+Status SessionRef::Close(const RunOptions& run_options) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  mutex_lock l(run_lock_);
+  Status status = session_->Close(run_options);
+  session_.reset();
+  while (run_count_ > 0) {
+    run_finished_.wait(l);
+  }
+  return status;
+}
+
+Status SessionRef::Close() {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  mutex_lock l(run_lock_);
+  Status status = session_->Close();
+  session_.reset();
+  while (run_count_ > 0) {
+    run_finished_.wait(l);
+  }
+  return status;
+}
+
+Status SessionRef::Run(const std::vector >& inputs,
+                       const std::vector& output_tensor_names,
+                       const std::vector& target_node_names,
+                       std::vector* outputs) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->Run(inputs, output_tensor_names, target_node_names,
+                         outputs);
+}
+
+Status SessionRef::ListDevices(std::vector* response) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->ListDevices(response);
+}
+
+Status SessionRef::PRunSetup(const std::vector& input_names,
+                             const std::vector& output_names,
+                             const std::vector& target_nodes,
+                             string* handle) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->PRunSetup(input_names, output_names, target_nodes, handle);
+}
+
+Status SessionRef::PRun(const string& handle,
+                        const std::vector >& inputs,
+                        const std::vector& output_names,
+                        std::vector* outputs) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->PRun(handle, inputs, output_names, outputs);
+}
+
+Status SessionRef::MakeCallable(const CallableOptions& callable_options,
+                                CallableHandle* out_handle) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->MakeCallable(callable_options, out_handle);
+}
+
+Status SessionRef::RunCallable(CallableHandle handle,
+                               const std::vector& feed_tensors,
+                               std::vector* fetch_tensors,
+                               RunMetadata* run_metadata) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->RunCallable(handle, feed_tensors, fetch_tensors,
+                                 run_metadata);
+}
+
+Status SessionRef::ReleaseCallable(CallableHandle handle) {
+  TF_RETURN_IF_ERROR(CheckNotClosed());
+  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
+  return rc.session->ReleaseCallable(handle);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/session_ref.h b/tensorflow/core/common_runtime/session_ref.h
new file mode 100644
index 0000000000..6146933326
--- /dev/null
+++ b/tensorflow/core/common_runtime/session_ref.h
@@ -0,0 +1,86 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SESSION_REF_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_SESSION_REF_H_
+
+#include 
+
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/public/session.h"
+
+namespace tensorflow {
+
+// A `SessionRef` manages the lifetime of a wrapped `Session` pointer.
+//
+// SessionRef blocks the return of Close() until all pending operations have
+// been completed or cancelled and underlying session has been freed.  Any
+// subsequent operations on the SessionRef object will return errors::Cancelled.
+class SessionRef : public Session {
+ public:
+  SessionRef(Session* session) : session_(session) {}
+  virtual ~SessionRef() {}
+
+  Status Create(const GraphDef& graph) override;
+  Status Extend(const GraphDef& graph) override;
+  Status Create(const RunOptions& run_options, const GraphDef& graph) override;
+  Status Extend(const RunOptions& run_options, const GraphDef& graph) override;
+  Status Run(const std::vector >& inputs,
+             const std::vector& output_tensor_names,
+             const std::vector& target_node_names,
+             std::vector* outputs) override;
+
+  Status ListDevices(std::vector* response) override;
+
+  Status Close() override;
+  Status Close(const RunOptions& run_options) override;
+
+  Status Run(const RunOptions& run_options,
+             const std::vector >& inputs,
+             const std::vector& output_tensor_names,
+             const std::vector& target_node_names,
+             std::vector* outputs, RunMetadata* run_metadata) override;
+
+  Status PRunSetup(const std::vector& input_names,
+                   const std::vector& output_names,
+                   const std::vector& target_nodes,
+                   string* handle) override;
+
+  Status PRun(const string& handle,
+              const std::vector >& inputs,
+              const std::vector& output_names,
+              std::vector* outputs) override;
+
+  Status MakeCallable(const CallableOptions& callable_options,
+                      CallableHandle* out_handle);
+
+  Status RunCallable(CallableHandle handle,
+                     const std::vector& feed_tensors,
+                     std::vector* fetch_tensors,
+                     RunMetadata* run_metadata);
+
+  Status ReleaseCallable(CallableHandle handle);
+
+ private:
+  mutex run_lock_;
+  condition_variable run_finished_;
+  uint64 run_count_ GUARDED_BY(run_lock_) = {0};
+  std::shared_ptr session_;
+
+  Status CheckNotClosed();
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_SESSION_REF_H_
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 9c7f3b7b25..d45566c55e 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3629,6 +3629,7 @@ tf_cuda_library(
         "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:session_ref",
         "//third_party/py/numpy:headers",
         "//third_party/python_runtime:headers",
     ],
diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 8ede6ab54c..f3aa135fe4 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -629,7 +629,7 @@ class BaseSession(SessionInterface):
     opts = tf_session.TF_NewSessionOptions(target=self._target, config=config)
     try:
       # pylint: disable=protected-access
-      self._session = tf_session.TF_NewSession(self._graph._c_graph, opts)
+      self._session = tf_session.TF_NewSessionRef(self._graph._c_graph, opts)
       # pylint: enable=protected-access
     finally:
       tf_session.TF_DeleteSessionOptions(opts)
diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index 1cdd8e0b6a..39a2922ac0 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -777,6 +777,7 @@ def TF_Reset(target, containers=None, config=None):
   $1 = &types_local;
 }
 
+%unignore TF_NewSessionRef;
 %unignore SetRequireShapeInferenceFns;
 %unignore TF_TryEvaluateConstant_wrapper;
 %noexception TF_TryEvaluateConstant_wrapper;
diff --git a/tensorflow/python/client/tf_session_helper.cc b/tensorflow/python/client/tf_session_helper.cc
index b6481e7e29..bcd4af2912 100644
--- a/tensorflow/python/client/tf_session_helper.cc
+++ b/tensorflow/python/client/tf_session_helper.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/tf_status_helper.h"
+#include "tensorflow/core/common_runtime/session_ref.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/attr_value_util.h"
@@ -42,6 +43,19 @@ static const char* kFeedDictErrorMsg =
     "feed_dict must be a dictionary mapping strings to NumPy arrays.";
 }  // end namespace
 
+TF_Session* TF_NewSessionRef(TF_Graph* graph, const TF_SessionOptions* opts,
+                             TF_Status* status) {
+  TF_Session* tf_session = TF_NewSession(graph, opts, status);
+  if (tf_session == nullptr) {
+    return nullptr;
+  }
+
+  Session* session = reinterpret_cast(tf_session->session);
+  SessionRef* session_ref = new SessionRef(session);
+  tf_session->session = session_ref;
+  return tf_session;
+}
+
 void TF_Run_wrapper_helper(TF_DeprecatedSession* session, const char* handle,
                            const TF_Buffer* run_options, PyObject* feed_dict,
                            const NameVector& output_names,
diff --git a/tensorflow/python/client/tf_session_helper.h b/tensorflow/python/client/tf_session_helper.h
index cfd27c2bee..dab7e71aac 100644
--- a/tensorflow/python/client/tf_session_helper.h
+++ b/tensorflow/python/client/tf_session_helper.h
@@ -40,6 +40,9 @@ typedef tensorflow::gtl::InlinedVector PyObjectVector;
 // A TF_TensorVector is a vector of borrowed pointers to TF_Tensors.
 typedef gtl::InlinedVector TF_TensorVector;
 
+TF_Session* TF_NewSessionRef(TF_Graph* graph, const TF_SessionOptions* opts,
+                             TF_Status* status);
+
 // Run the graph associated with the session starting with the
 // supplied inputs[].  Regardless of success or failure, inputs[] are
 // stolen by the implementation (i.e. the implementation will
-- 
cgit v1.2.3


From 7f2ebb2bc1cfc8f1d7713ec45a973d20b932c9aa Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Mon, 23 Jul 2018 09:20:05 -0700
Subject: [XLA] Don't call into the allocator for 0 byte allocations

This makes it obvious that XLA does not depend on an allocator returning
non-nullptr for zero sized allocations.

PiperOrigin-RevId: 205672174
---
 tensorflow/compiler/jit/xla_launch_util.cc | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_launch_util.cc b/tensorflow/compiler/jit/xla_launch_util.cc
index 616c3ed2a2..6134b8c694 100644
--- a/tensorflow/compiler/jit/xla_launch_util.cc
+++ b/tensorflow/compiler/jit/xla_launch_util.cc
@@ -64,11 +64,13 @@ xla::StatusOr XlaAllocator::Allocate(
     int device_ordinal, uint64 size, bool retry_on_failure) {
   AllocationAttributes attrs;
   attrs.no_retry_on_failure = !retry_on_failure;
-  void* data =
-      wrapped_->AllocateRaw(Allocator::kAllocatorAlignment, size, attrs);
-  if (data == nullptr) {
-    return errors::ResourceExhausted("Out of memory while trying to allocate ",
-                                     size, " bytes.");
+  void* data = nullptr;
+  if (size != 0) {
+    data = wrapped_->AllocateRaw(Allocator::kAllocatorAlignment, size, attrs);
+    if (data == nullptr) {
+      return errors::ResourceExhausted(
+          "Out of memory while trying to allocate ", size, " bytes.");
+    }
   }
   return xla::OwningDeviceMemory(se::DeviceMemoryBase(data, size),
                                  device_ordinal, this);
-- 
cgit v1.2.3


From 3795a79e1db750efb7af1b83834db5a4f0c03b41 Mon Sep 17 00:00:00 2001
From: Russell Power 
Date: Mon, 23 Jul 2018 09:21:45 -0700
Subject: Use a error rendezvous wrapper to capture infeed/outfeed/training
 session errors.

PiperOrigin-RevId: 205672414
---
 tensorflow/contrib/tpu/BUILD                       |   1 +
 .../contrib/tpu/python/tpu/error_handling.py       | 128 +++++++++++++++++
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 159 ++++++++++-----------
 tensorflow/python/estimator/estimator.py           |   4 +
 4 files changed, 211 insertions(+), 81 deletions(-)
 create mode 100644 tensorflow/contrib/tpu/python/tpu/error_handling.py

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 14e4e9cc2b..643a7cc13a 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -36,6 +36,7 @@ cc_library(
 py_library(
     name = "tpu_estimator",
     srcs = [
+        "python/tpu/error_handling.py",
         "python/tpu/tpu_config.py",
         "python/tpu/tpu_context.py",
         "python/tpu/tpu_estimator.py",
diff --git a/tensorflow/contrib/tpu/python/tpu/error_handling.py b/tensorflow/contrib/tpu/python/tpu/error_handling.py
new file mode 100644
index 0000000000..8d6d44b1a1
--- /dev/null
+++ b/tensorflow/contrib/tpu/python/tpu/error_handling.py
@@ -0,0 +1,128 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ===================================================================
+"""ErrorRendezvous handler for collecting errors from multiple threads."""
+
+import contextlib
+import threading
+import time
+import traceback
+
+
+from tensorflow.python.framework import errors
+from tensorflow.python.platform import tf_logging as logging
+
+_UNINTERESTING_ERRORS = (errors.CancelledError,)
+
+
+class ErrorRendezvous(object):
+  """Resolve errors from multiple threads during TPU execution.
+
+  TPU errors can occur on the infeed or outfeed threads as well as the main
+  training thread.
+
+  Depending on which thread "wins" and receives the session error first, we may
+  end up showing users a confusing and non-actionable error message (session
+  cancelled) instead of a root cause (e.g. a bad filename).
+
+  The rendezvous object provides a location to capture these errors until all
+  threads terminate.  At that point we can choose the most informative error
+  to report.
+  """
+
+  def __init__(self, num_sources):
+    # string -> (message, traceback)
+    self._errors = {}
+    self._num_sources = num_sources
+    self._session_cancel_timer = None
+
+  def record_error(self, source, exception, session=None):
+    """Report an exception from the given source.
+
+    If a session is passed, a timer will be registered to close it after a few
+    seconds.  This is necessary to ensure the main training loop does not hang
+    if an infeed/oufeed error occurs.  We sleep a few seconds to allow a more
+    interesting error from another thread to propagate.
+
+    Args:
+      source: string, source of the error
+      exception: Exception being thrown
+      session: Session to close after delay.
+    """
+    logging.info('Error recorded from %s: %s', source, exception)
+    stack_trace = traceback.format_exc()
+    self._errors[source] = (exception, stack_trace)
+
+    if session is not None and self._session_cancel_timer is None:
+
+      def _cancel_session():
+        time.sleep(5)
+        try:
+          session.close()
+        except:  # pylint: disable=bare-except
+          pass
+
+      self._session_cancel_timer = threading.Thread(target=_cancel_session,)
+      self._session_cancel_timer.daemon = True
+      self._session_cancel_timer.start()
+
+  def record_done(self, source):
+    """Mark execution source `source` as done.
+
+    If an error was originally reported from `source` it is left intact.
+
+    Args:
+      source: `str`, source being recorded
+    """
+    logging.info('%s marked as finished', source)
+    if source not in self._errors:
+      self._errors[source] = None
+
+  @contextlib.contextmanager
+  def catch_errors(self, source, session=None):
+    """Context manager to report any errors within a block."""
+    try:
+      yield
+    except Exception as e:  # pylint: disable=broad-except
+      self.record_error(source, e, session)
+
+  def raise_errors(self, timeout_sec=5):
+    """Wait for up to `timeout` seconds for all error sources to finish.
+
+    Preferentially raise "interesting" errors (errors not in the
+    _UNINTERESTING_ERRORS) set.
+
+    Args:
+      timeout_sec: Seconds to wait for other error sources.
+    """
+    for _ in range(timeout_sec):
+      if len(self._errors) == self._num_sources:
+        break
+      time.sleep(1)
+
+    kept_errors = [(k, v) for (k, v) in self._errors.items() if v is not None]
+
+    if not kept_errors:
+      return
+
+    # First check for any interesting errors, then fall back on the session
+    # cancelled errors etc.
+    for k, (exc, _) in kept_errors:
+      if isinstance(exc, _UNINTERESTING_ERRORS):
+        continue
+      else:
+        raise exc
+
+    for k, (exc, _) in kept_errors:
+      raise exc
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 8ae0a31b6a..1208d557e7 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -24,7 +24,6 @@ import os
 import signal
 import threading
 import time
-import traceback
 
 import numpy as np
 import six
@@ -32,6 +31,7 @@ from six.moves import queue as Queue  # pylint: disable=redefined-builtin
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.contrib.tpu.python.ops import tpu_ops
+from tensorflow.contrib.tpu.python.tpu import error_handling
 from tensorflow.contrib.tpu.python.tpu import session_support
 from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_config
@@ -365,17 +365,17 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
                ctx,
                enqueue_ops,
                dequeue_ops,
-               run_infeed_loop_on_coordinator=True):
+               run_infeed_loop_on_coordinator=True,
+               rendezvous=None):
     self._master_job = ctx.master_job
     self._enqueue_ops = enqueue_ops
     self._dequeue_ops = dequeue_ops
+    self._rendezvous = rendezvous
 
     self._run_infeed_loop_on_coordinator = run_infeed_loop_on_coordinator
     self._initial_infeed_sleep_secs = (
         ctx.config.tpu_config.initial_infeed_sleep_secs)
 
-    self._session_cancel_timer = None
-
     self._feed_error = None
     self._finished = False
 
@@ -392,62 +392,6 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
     for op in summary_writer_init_ops:
       self._finalize_ops.append(contrib_summary.flush(writer=op.inputs[0]))
 
-  def _log_error(self, session, error):
-    """Log an infeed or outfeed error.
-
-    This logs a short error message immediately, and schedules a timer to
-    emit the full stack trace and error message after a short period of time.
-    If the main session has terminated by the time the timer triggers, we
-    assume the real source of the error was from the main session and avoid
-    emitting a stack trace for the infeed.
-
-    Args:
-      session: `tf.Session`, session to be terminated error: exception that
-        triggered logging.
-      error: the Exception to log.
-    """
-    logging.warning(
-        '\n\n'
-        'Error occurred during infeed/outfeed.  This may be due to a compile '
-        'error in the main session.  Waiting for a short time for the main '
-        'session to come back.\n\n%s', error)
-
-    self._feed_error = traceback.format_exc()
-
-    # If we've already encountered a feed error, don't schedule another
-    # cancellation op.
-    if self._session_cancel_timer:
-      return
-
-    def _cancel_session():
-      """Close the session to avoid the main thread from hanging.
-
-      If input pipeline triggers any error, the infeed thread dies but the main
-      thread for TPU computation waits for the infeed enqueue forever. Close the
-      Session to cancel the main thread Session.run execution.
-
-      We sleep for a few seconds before closing to give some time for the TPU
-      compilation error, if any, propagating, from TPU to CPU host. Compilation
-      errors should be reported by the main thread so that the program can be
-      interrupted and users can take action.  Due to a race condition, the
-      infeed thread might see an error first.  Closing the session here
-      immediately would result in a session cancellation exception in the main
-      thread, instead of the expected compile error.  User code that depends on
-      having the proper exception type will therefore be confused.
-      """
-      time.sleep(5)
-
-      # If the main session is still running, the infeed/outfeed errors are
-      # legitimate, and should be logged.
-      if not self._finished and self._feed_error:
-        logging.error('Feed error: %s', self._feed_error)
-        logging.error('Closing session.  A RuntimeError should follow.')
-        session.close()
-
-    self._session_cancel_timer = threading.Thread(target=_cancel_session)
-    self._session_cancel_timer.daemon = True
-    self._session_cancel_timer.start()
-
   def _run_infeed(self, queue_ctx, session):
     logging.info('Starting infeed thread controller.')
     if self._initial_infeed_sleep_secs:
@@ -456,7 +400,7 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
       time.sleep(self._initial_infeed_sleep_secs)
       logging.info('%s thread starting after sleep', self._name)
 
-    try:
+    with self._rendezvous.catch_errors(source='infeed', session=session):
       if self._run_infeed_loop_on_coordinator:
         for count, steps in enumerate(queue_ctx.read_iteration_counts()):
           for i in xrange(steps):
@@ -466,19 +410,15 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
         for _ in queue_ctx.read_iteration_counts():
           session.run(self._enqueue_ops)
       logging.info('Infeed thread finished, shutting down.')
-    except Exception as e:  # pylint: disable=broad-except
-      self._log_error(session, e)
 
   def _run_outfeed(self, queue_ctx, session):
     logging.info('Starting outfeed thread controller.')
-    try:
+    with self._rendezvous.catch_errors(source='outfeed', session=session):
       for count, steps in enumerate(queue_ctx.read_iteration_counts()):
         for i in xrange(steps):
           logging.debug('Outfeed dequeue for iteration (%d, %d)', count, i)
           session.run(self._dequeue_ops)
       logging.info('Outfeed thread finished, shutting down.')
-    except Exception as e:  # pylint: disable=broad-except
-      self._log_error(session, e)
 
   def _create_infeed_controller(self, name, target, args):
     return _OpQueueContext(name=name, target=target, args=args)
@@ -497,11 +437,6 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
   def before_run(self, run_context):
     self._feed_error = None
 
-    # Wait for the cancellation timer to complete before continuing.
-    if self._session_cancel_timer:
-      self._session_cancel_timer.join()
-      self._session_cancel_timer = None
-
     iterations = run_context.session.run(self._iterations_per_loop_var)
 
     logging.info('Enqueue next (%d) batch(es) of data to infeed.', iterations)
@@ -512,16 +447,14 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
     self._outfeed_controller.send_next_batch_signal(iterations)
 
   def end(self, session):
-    if self._session_cancel_timer:
-      logging.warning('Feed error occurred; waiting for message.')
-      self._session_cancel_timer.join()
-
     self._finished = True
     logging.info('Stop infeed thread controller')
     self._infeed_controller.join()
+    self._rendezvous.record_done('infeed')
 
     logging.info('Stop output thread controller')
     self._outfeed_controller.join()
+    self._rendezvous.record_done('outfeed')
 
     logging.info('Shutdown TPU system.')
     session.run(self._finalize_ops)
@@ -529,9 +462,10 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
 
 class TPUInfeedOutfeedSessionHookForPrediction(TPUInfeedOutfeedSessionHook):
 
-  def __init__(self, ctx, enqueue_ops, dequeue_ops):
+  def __init__(self, ctx, enqueue_ops, dequeue_ops, rendezvous=None):
     super(TPUInfeedOutfeedSessionHookForPrediction, self).__init__(
-        ctx, enqueue_ops, dequeue_ops, run_infeed_loop_on_coordinator=False)
+        ctx, enqueue_ops, dequeue_ops, run_infeed_loop_on_coordinator=False,
+        rendezvous=rendezvous)
 
   def _create_infeed_controller(self, name, target, args):
     return _OpSignalOnceQueueContext(name=name, target=target, args=args)
@@ -2113,6 +2047,7 @@ class TPUEstimator(estimator_lib.Estimator):
     self._export_to_tpu = export_to_tpu
 
     self._is_input_fn_invoked = None
+    self._rendezvous = {}
 
   def _add_meta_graph_for_mode(self,
                                builder,
@@ -2356,6 +2291,65 @@ class TPUEstimator(estimator_lib.Estimator):
     """
     pass
 
+  def train(self,
+            input_fn,
+            hooks=None,
+            steps=None,
+            max_steps=None,
+            saving_listeners=None):
+    rendezvous = error_handling.ErrorRendezvous(num_sources=3)
+    self._rendezvous[model_fn_lib.ModeKeys.TRAIN] = rendezvous
+    try:
+      return super(TPUEstimator, self).train(
+          input_fn=input_fn, hooks=hooks, steps=steps, max_steps=max_steps,
+          saving_listeners=saving_listeners
+      )
+    except Exception as e:  # pylint: disable=broad-except
+      rendezvous.record_error('training_loop', e)
+    finally:
+      rendezvous.record_done('training_loop')
+      rendezvous.raise_errors()
+
+  def evaluate(self, input_fn, steps=None, hooks=None, checkpoint_path=None,
+               name=None):
+    rendezvous = error_handling.ErrorRendezvous(num_sources=3)
+    self._rendezvous[model_fn_lib.ModeKeys.EVAL] = rendezvous
+    try:
+      return super(TPUEstimator, self).evaluate(
+          input_fn, steps=steps, hooks=hooks, checkpoint_path=checkpoint_path,
+          name=name
+      )
+    except Exception as e:  # pylint: disable=broad-except
+      rendezvous.record_error('evaluation_loop', e)
+    finally:
+      rendezvous.record_done('evaluation_loop')
+      rendezvous.raise_errors()
+
+  def predict(self,
+              input_fn,
+              predict_keys=None,
+              hooks=None,
+              checkpoint_path=None,
+              yield_single_examples=True):
+    rendezvous = error_handling.ErrorRendezvous(num_sources=3)
+    self._rendezvous[model_fn_lib.ModeKeys.PREDICT] = rendezvous
+    try:
+      for result in super(TPUEstimator, self).predict(
+          input_fn=input_fn,
+          predict_keys=predict_keys,
+          hooks=hooks,
+          checkpoint_path=checkpoint_path,
+          yield_single_examples=True):
+        yield result
+    except Exception as e:  # pylint: disable=broad-except
+      rendezvous.record_error('prediction_loop', e)
+    finally:
+      rendezvous.record_done('prediction_loop')
+      rendezvous.raise_errors()
+
+    rendezvous.record_done('prediction_loop')
+    rendezvous.raise_errors()
+
   def _augment_model_fn(self, model_fn, batch_axis):
     """Returns a new model_fn, which wraps the TPU support."""
 
@@ -2450,7 +2444,9 @@ class TPUEstimator(estimator_lib.Estimator):
                   enqueue_ops,
                   host_ops,
                   run_infeed_loop_on_coordinator=(
-                      run_infeed_loop_on_coordinator)),
+                      run_infeed_loop_on_coordinator),
+                  rendezvous=self._rendezvous[mode],
+              ),
               InstallSignalHandlerHook(),
               training.LoggingTensorHook(
                   {
@@ -2533,7 +2529,8 @@ class TPUEstimator(estimator_lib.Estimator):
                   enqueue_ops,
                   eval_update_ops + host_ops,
                   run_infeed_loop_on_coordinator=(
-                      run_infeed_loop_on_coordinator)),
+                      run_infeed_loop_on_coordinator),
+                  rendezvous=self._rendezvous[mode]),
           ] + input_hooks
 
           return model_fn_lib.EstimatorSpec(
@@ -2599,8 +2596,8 @@ class TPUEstimator(estimator_lib.Estimator):
 
         hooks = [
             _StoppingPredictHook(scalar_stopping_signal),
-            TPUInfeedOutfeedSessionHookForPrediction(ctx, enqueue_ops,
-                                                     host_ops),
+            TPUInfeedOutfeedSessionHookForPrediction(
+                ctx, enqueue_ops, host_ops, rendezvous=self._rendezvous[mode]),
         ] + input_hooks
 
         return model_fn_lib.EstimatorSpec(
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 2fd6f6fab9..915ceeb98b 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -567,6 +567,10 @@ class Estimator(object):
 
   def _assert_members_are_not_overridden(self):
     """Asserts members of `Estimator` are not overridden."""
+    # TPUEstimator is special cased (owned by TF).
+    if self.__class__.__name__ == 'TPUEstimator':
+      return
+
     allowed_overrides = set([
         '_call_input_fn', '_create_global_step',
         '_convert_train_steps_to_hooks', '_convert_eval_steps_to_hooks',
-- 
cgit v1.2.3


From b148d228886e53f71401c332b8d48d45467dad47 Mon Sep 17 00:00:00 2001
From: Jared Duke 
Date: Mon, 23 Jul 2018 09:29:21 -0700
Subject: Specify minSdkVersion/targetSdkVersion in library manifests

PiperOrigin-RevId: 205673447
---
 tensorflow/contrib/android/cmake/src/main/AndroidManifest.xml | 4 ++++
 tensorflow/contrib/lite/java/AndroidManifest.xml              | 3 ++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/android/cmake/src/main/AndroidManifest.xml b/tensorflow/contrib/android/cmake/src/main/AndroidManifest.xml
index bced47e046..c17110a78b 100644
--- a/tensorflow/contrib/android/cmake/src/main/AndroidManifest.xml
+++ b/tensorflow/contrib/android/cmake/src/main/AndroidManifest.xml
@@ -1,6 +1,10 @@
 
 
+    
+
     
 
diff --git a/tensorflow/contrib/lite/java/AndroidManifest.xml b/tensorflow/contrib/lite/java/AndroidManifest.xml
index c3849e6868..b91c6d149a 100644
--- a/tensorflow/contrib/lite/java/AndroidManifest.xml
+++ b/tensorflow/contrib/lite/java/AndroidManifest.xml
@@ -7,5 +7,6 @@
         android:targetSdkVersion="19" />
 
     
-    
+
 
+
-- 
cgit v1.2.3


From e2d19b5be97c231a00a895b87fd7ca756395bb35 Mon Sep 17 00:00:00 2001
From: James Keeling 
Date: Mon, 23 Jul 2018 09:31:29 -0700
Subject: Fix meaningless "const" on return type.

Because this file is included in lots of places, this generates a large number of warnings during compilation.

PiperOrigin-RevId: 205673744
---
 tensorflow/core/framework/device_base.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/core/framework/device_base.h b/tensorflow/core/framework/device_base.h
index 922d34fac9..b184fd91e1 100644
--- a/tensorflow/core/framework/device_base.h
+++ b/tensorflow/core/framework/device_base.h
@@ -184,9 +184,7 @@ class DeviceBase {
 
   virtual ScopedAllocatorMgr* GetScopedAllocatorMgr() const { return nullptr; }
 
-  const bool has_eigen_cpu_device() const {
-    return !eigen_cpu_devices_.empty();
-  }
+  bool has_eigen_cpu_device() const { return !eigen_cpu_devices_.empty(); }
 
   virtual const Eigen::ThreadPoolDevice* eigen_cpu_device();
 
-- 
cgit v1.2.3


From bf62c1042cfc79ae9df00bd2ac77de772f350762 Mon Sep 17 00:00:00 2001
From: Jared Duke 
Date: Mon, 23 Jul 2018 09:35:31 -0700
Subject: Allow scalar inputs for toco and TFLite zip tests

PiperOrigin-RevId: 205674334
---
 .../contrib/lite/testing/generate_examples.py      | 41 ++++++++++++++--------
 .../contrib/lite/toco/model_cmdline_flags.cc       |  4 +++
 tensorflow/contrib/lite/toco/tooling_util.cc       |  5 ---
 tensorflow/contrib/lite/toco/tooling_util_test.cc  | 17 +++++++++
 4 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index a91ff8626a..b3ccc65e85 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -772,6 +772,11 @@ def make_binary_op_tests(zip_path, binary_operator):
       "input_shape_1": [[1, 3, 4, 3]],
       "input_shape_2": [[3]],
       "activation": [True]
+  }, {
+      "dtype": [tf.float32],
+      "input_shape_1": [[]],
+      "input_shape_2": [[]],
+      "activation": [False]
   }]
 
   def build_graph(parameters):
@@ -821,7 +826,7 @@ def make_reduce_tests(reduce_op):
         "input_dtype": [tf.float32, tf.int32, tf.int64],
         "input_shape": [[3, 2, 4]],
         "axis": [
-            None, 0, 1, 2, [0, 1], [0, 2], [1, 2], [0, 1, 2], [1, 0], [2, 0],
+            0, 1, 2, [0, 1], [0, 2], [1, 2], [0, 1, 2], [1, 0], [2, 0],
             [2, 1], [2, 1, 0], [2, 0, 1], -1, -2, -3, [1, -1], [0, -1], [-1, 0],
             [-1, -2, -3], [0, 0, 0], [2, 2, 0], [1, 0, -3, -3]
         ],
@@ -831,13 +836,19 @@ def make_reduce_tests(reduce_op):
         "input_dtype": [tf.float32],
         "input_shape": [[1, 8, 8, 3]],
         "axis": [
-            None, 0, 1, 2, 3, [1, 2], [0, 3], [1, 2, 3], [0, 1, 2, 3],
+            0, 1, 2, 3, [1, 2], [0, 3], [1, 2, 3], [0, 1, 2, 3],
             [3, 2, 1, 0], [3, 1, 0, 2], [2, 0], [3, 0], [3, 1], [1, 0], -1, -2,
             -3, -4, [0, -2], [2, 3, -1, 0], [3, 1, 2, -3], [3, -4], [2, 2, 2],
             [2, 2, 3], [-3, -3, -4], [-3, 2, 1]
         ],
         "const_axis": [True, False],
         "keepdims": [True, False],
+    }, {
+        "input_dtype": [tf.float32],
+        "input_shape": [[], [1, 8, 8, 3], [3, 2, 4]],
+        "axis": [None],
+        "const_axis": [True],
+        "keepdims": [True, False],
     }]
 
     def build_graph(parameters):
@@ -855,7 +866,7 @@ def make_reduce_tests(reduce_op):
         if isinstance(parameters["axis"], list):
           shape = [len(parameters["axis"])]
         else:
-          shape = [0]  # shape for None or integers.
+          shape = []  # shape for None or integers.
         axis = tf.placeholder(dtype=tf.int32, name="axis", shape=shape)
         input_tensors = [input_tensor, axis]
 
@@ -866,10 +877,11 @@ def make_reduce_tests(reduce_op):
     def build_inputs(parameters, sess, inputs, outputs):
       values = [
           create_tensor_data(parameters["input_dtype"],
-                             parameters["input_shape"])]
+                             parameters["input_shape"],
+                             min_value=-10,
+                             max_value=10)]
       if not parameters["const_axis"]:
-        if parameters["axis"]:
-          values.append(np.array(parameters["axis"]))
+        values.append(np.array(parameters["axis"]))
       return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
 
     make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
@@ -902,7 +914,7 @@ def make_exp_tests(zip_path):
 
   test_parameters = [{
       "input_dtype": [tf.float32],
-      "input_shape": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+      "input_shape": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
   }]
 
   def build_graph(parameters):
@@ -961,8 +973,8 @@ def make_maximum_tests(zip_path):
 
   test_parameters = [{
       "input_dtype": [tf.float32],
-      "input_shape_1": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
-      "input_shape_2": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+      "input_shape_1": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+      "input_shape_2": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
   }]
 
   def build_graph(parameters):
@@ -996,8 +1008,8 @@ def make_minimum_tests(zip_path):
 
   test_parameters = [{
       "input_dtype": [tf.float32],
-      "input_shape_1": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
-      "input_shape_2": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+      "input_shape_1": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+      "input_shape_2": [[], [3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
   }]
 
   def build_graph(parameters):
@@ -2252,7 +2264,7 @@ def make_arg_min_max_tests(zip_path):
 
   test_parameters = [{
       "input_dtype": [tf.float32, tf.int32],
-      "input_shape": [[1, 1, 1, 3], [2, 3, 4, 5], [2, 3, 3], [5, 5], [10]],
+      "input_shape": [[], [1, 1, 1, 3], [2, 3, 4, 5], [2, 3, 3], [5, 5], [10]],
       "output_type": [tf.int32, tf.int64],
       "axis_is_last_dim": [True, False],
       "is_arg_max": [True],
@@ -2288,7 +2300,8 @@ def make_equal_tests(zip_path):
 
   test_parameters = [{
       "input_dtype": [tf.float32, tf.int32, tf.int64],
-      "input_shape_pair": [([1, 1, 1, 3], [1, 1, 1, 3]),
+      "input_shape_pair": [([], []),
+                           ([1, 1, 1, 3], [1, 1, 1, 3]),
                            ([2, 3, 4, 5], [2, 3, 4, 5]), ([2, 3, 3], [2, 3]),
                            ([5, 5], [1]), ([10], [2, 4, 10])],
   }]
@@ -2545,7 +2558,7 @@ def _make_elementwise_tests(op):
     """Actual function that generates examples."""
     test_parameters = [{
         "input_dtype": [tf.float32],
-        "input_shape": [[1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
+        "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]],
     }]
 
     def build_graph(parameters):
diff --git a/tensorflow/contrib/lite/toco/model_cmdline_flags.cc b/tensorflow/contrib/lite/toco/model_cmdline_flags.cc
index 06072d1fcb..d34da63e43 100644
--- a/tensorflow/contrib/lite/toco/model_cmdline_flags.cc
+++ b/tensorflow/contrib/lite/toco/model_cmdline_flags.cc
@@ -322,6 +322,10 @@ void ReadModelFlagsFromCommandLineFlags(
     for (int i = 0; i < input_shapes.size(); ++i) {
       auto* shape = model_flags->mutable_input_arrays(i)->mutable_shape();
       shape->clear_dims();
+      // Treat an empty input shape as a scalar.
+      if (input_shapes[i].empty()) {
+        continue;
+      }
       for (const auto& dim_str : absl::StrSplit(input_shapes[i], ',')) {
         int size;
         CHECK(absl::SimpleAtoi(dim_str, &size))
diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index 52f8df45a2..98e416b76e 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -1585,11 +1585,6 @@ void ResolveModelFlags(const ModelFlags& model_flags, Model* model) {
                                model);
   }
 
-  for (const auto& input_array : model->flags.input_arrays()) {
-    if (input_array.has_shape()) {
-      CHECK(input_array.shape().dims_size());
-    }
-  }
   model->flags.set_change_concat_input_ranges(
       model_flags.change_concat_input_ranges());
   model->flags.set_allow_nonascii_arrays(model_flags.allow_nonascii_arrays());
diff --git a/tensorflow/contrib/lite/toco/tooling_util_test.cc b/tensorflow/contrib/lite/toco/tooling_util_test.cc
index 8609e5bedd..eb495646a2 100644
--- a/tensorflow/contrib/lite/toco/tooling_util_test.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util_test.cc
@@ -39,6 +39,8 @@ std::vector CreateShapePairs() {
        {Shape({256, 256, 3}), Shape({256, 256, 3}), Agreement::kBroadcast},
        {Shape({256, 256, 3}), Shape({3}), Agreement::kBroadcast},
        {Shape({8, 1, 6, 1}), Shape({7, 1, 5}), Agreement::kBroadcast},
+       {Shape({}), Shape({3}), Agreement::kBroadcast},
+       {Shape({}), Shape({3, 1}), Agreement::kBroadcast},
 
        // These extend (and therefore broadcast).
        {Shape({3}), Shape({3}), Agreement::kExtend},
@@ -54,6 +56,7 @@ std::vector CreateShapePairs() {
        {Shape({15, 3, 5}), Shape({15, 1, 5}), Agreement::kBroadcastNotExtend},
        {Shape({15, 3, 5}), Shape({3, 5}), Agreement::kBroadcastNotExtend},
        {Shape({15, 3, 5}), Shape({3, 1}), Agreement::kBroadcastNotExtend},
+       {Shape({3, 1}), Shape({}), Agreement::kBroadcastNotExtend},
 
        // These do not broadcast (and therefore also do not extend).
        {Shape({3}), Shape({4}), Agreement::kNeither},
@@ -175,6 +178,20 @@ TEST(NumElementsTest, UnsignedInt64) {
   EXPECT_EQ(status.error_message(), kLargeTensorMessage);
 }
 
+TEST(NumElementsTest, Scalar) {
+  tensorflow::Status status = tensorflow::Status::OK();
+
+  int32_t count;
+  status = NumElements(std::vector{}, &count);
+  EXPECT_TRUE(status.ok());
+  EXPECT_EQ(count, 1);
+
+  uint64_t countu64;
+  status = NumElements(std::vector{}, &countu64);
+  EXPECT_TRUE(status.ok());
+  EXPECT_EQ(countu64, 1ULL);
+}
+
 TEST(FusedActivationTest, DefaultsToUnfused) {
   EXPECT_TRUE(OperatorSupportsFusedActivation(OperatorType::kAdd));
   EXPECT_FALSE(OperatorSupportsFusedActivation(OperatorType::kNone));
-- 
cgit v1.2.3


From 6795a8c3a3678fb805b6a8ba806af77ddfe61628 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 09:41:14 -0700
Subject: Automated rollback of commit
 aee128fb46f2721b07e5fa04ed6fac4b67963807. Revert #20746.

PiperOrigin-RevId: 205675194
---
 tensorflow/core/kernels/BUILD                      |  11 +-
 tensorflow/core/kernels/crop_and_resize_op.cc      |  64 +--
 .../core/kernels/crop_resize_bilinear_core.h       | 464 ---------------------
 tensorflow/core/kernels/resize_bilinear_op.cc      | 153 ++++++-
 4 files changed, 170 insertions(+), 522 deletions(-)
 delete mode 100644 tensorflow/core/kernels/crop_resize_bilinear_core.h

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 10cbcdecc8..23f84c46a9 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -557,12 +557,6 @@ cc_header_only_library(
     deps = [":image_resizer_state"],
 )
 
-cc_library(
-    name = "crop_resize_bilinear_core",
-    hdrs = ["crop_resize_bilinear_core.h"],
-    visibility = ["//visibility:private"],
-)
-
 # OpKernel libraries ----------------------------------------------------------
 
 ARRAY_DEPS = [
@@ -2158,7 +2152,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "crop_and_resize_op",
     prefix = "crop_and_resize_op",
-    deps = IMAGE_DEPS + [":crop_resize_bilinear_core"],
+    deps = IMAGE_DEPS,
 )
 
 tf_kernel_library(
@@ -2224,7 +2218,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "resize_bilinear_op",
     prefix = "resize_bilinear_op",
-    deps = IMAGE_DEPS + [":crop_resize_bilinear_core"],
+    deps = IMAGE_DEPS,
 )
 
 tf_kernel_library(
@@ -4864,7 +4858,6 @@ filegroup(
         "concat_op.cc",
         "constant_op.cc",
         "constant_op.h",
-        "crop_resize_bilinear_core.h",
         "cwise_ops.h",
         "cwise_ops_common.cc",
         "cwise_ops_common.h",
diff --git a/tensorflow/core/kernels/crop_and_resize_op.cc b/tensorflow/core/kernels/crop_and_resize_op.cc
index 22524dc1c4..99d01b4db6 100644
--- a/tensorflow/core/kernels/crop_and_resize_op.cc
+++ b/tensorflow/core/kernels/crop_and_resize_op.cc
@@ -28,7 +28,6 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/bounds_check.h"
-#include "tensorflow/core/kernels/crop_resize_bilinear_core.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
@@ -43,10 +42,6 @@ limitations under the License.
 using stream_executor::cuda::ScopedActivateExecutorContext;
 #endif  // GOOGLE_CUDA
 
-using ::tensorflow::internal::CachedInterpolation;
-using ::tensorflow::internal::compute_interpolation_weights;
-using ::tensorflow::internal::crop_resize_single_image;
-
 namespace tensorflow {
 namespace {
 
@@ -254,34 +249,39 @@ struct CropAndResize {
             continue;
           }
           if (method_name == "bilinear") {
-            CachedInterpolation *interp_x = nullptr, *interp_y = nullptr;
-            int min_ix, max_ix, min_iy, max_iy;
-            compute_interpolation_weights(crop_width, image_width, x1, x2,
-                                          min_ix, max_ix, interp_x);
-            compute_interpolation_weights(crop_height, image_height, y1, y2,
-                                          min_iy, max_iy, interp_y);
-
-            // multiply by depth to avoid multiplication in resize_single_image.
-            for (int i = min_ix; i <= max_ix; ++i) {
-              interp_x[i - min_ix].lower *= depth;
-              interp_x[i - min_ix].upper *= depth;
-            }
+            const int top_y_index = floorf(in_y);
+            const int bottom_y_index = ceilf(in_y);
+            const float y_lerp = in_y - top_y_index;
+
+            for (int x = 0; x < crop_width; ++x) {
+              const float in_x = (crop_width > 1)
+                                     ? x1 * (image_width - 1) + x * width_scale
+                                     : 0.5 * (x1 + x2) * (image_width - 1);
+              if (in_x < 0 || in_x > image_width - 1) {
+                for (int d = 0; d < depth; ++d) {
+                  crops(b, y, x, d) = extrapolation_value;
+                }
+                continue;
+              }
+              const int left_x_index = floorf(in_x);
+              const int right_x_index = ceilf(in_x);
+              const float x_lerp = in_x - left_x_index;
 
-            crop_resize_single_image(
-                image.data() + static_cast(b_in) *
-                                   static_cast(image_height) *
-                                   static_cast(image_width) *
-                                   static_cast(depth),
-                image_height, image_width, crop_height, crop_width, depth,
-                min_ix, max_ix, interp_x, min_iy, max_iy, interp_y,
-                extrapolation_value, false, false,
-                crops.data() + static_cast(b) *
-                                   static_cast(crop_height) *
-                                   static_cast(crop_width) *
-                                   static_cast(depth));
-
-            delete[] interp_y;
-            delete[] interp_x;
+              for (int d = 0; d < depth; ++d) {
+                const float top_left(static_cast(
+                    image(b_in, top_y_index, left_x_index, d)));
+                const float top_right(static_cast(
+                    image(b_in, top_y_index, right_x_index, d)));
+                const float bottom_left(static_cast(
+                    image(b_in, bottom_y_index, left_x_index, d)));
+                const float bottom_right(static_cast(
+                    image(b_in, bottom_y_index, right_x_index, d)));
+                const float top = top_left + (top_right - top_left) * x_lerp;
+                const float bottom =
+                    bottom_left + (bottom_right - bottom_left) * x_lerp;
+                crops(b, y, x, d) = top + (bottom - top) * y_lerp;
+              }
+            }
           } else {  // method == "nearest"
             for (int x = 0; x < crop_width; ++x) {
               const float in_x = (crop_width > 1)
diff --git a/tensorflow/core/kernels/crop_resize_bilinear_core.h b/tensorflow/core/kernels/crop_resize_bilinear_core.h
deleted file mode 100644
index 51327aca67..0000000000
--- a/tensorflow/core/kernels/crop_resize_bilinear_core.h
+++ /dev/null
@@ -1,464 +0,0 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_KERNELS_CROP_RESIZE_BILINEAR_CORE_H_
-#define TENSORFLOW_CORE_KERNELS_CROP_RESIZE_BILINEAR_CORE_H_
-
-namespace tensorflow {
-namespace internal {
-// Compute the interpolation indices only once.
-struct CachedInterpolation {
-  int lower;  // Lower source index used in the interpolation
-  int upper;  // Upper source index used in the interpolation
-  // 1-D linear iterpolation scale (see:
-  // https://en.wikipedia.org/wiki/Bilinear_interpolation)
-  float lerp;
-};
-
-inline bool compute_single_interpolation_weight(
-    const int in_size, const float out2in_scale, const float out2in_start,
-    const bool clip, const int i, int* lower, int* upper, float* lerp) {
-  const float in = i * out2in_scale + out2in_start;
-  *lower = (int)floor(in);
-  *upper = (int)ceil(in);
-  *lerp = (float)(in - (float)*lower);
-  if (clip) {
-    if (*lower < 0)
-      *lower = 0;
-    else if (*lower >= in_size)
-      *lower = in_size - 1;
-    if (*upper < 0)
-      *upper = 0;
-    else if (*upper >= in_size)
-      *upper = in_size - 1;
-    return true;
-  } else {
-    return (*lower >= 0 && *upper < in_size) ? true : false;
-  }
-}
-/**
- * Compute interpolation values for output indexes in range
- * [out_start,out_start+out_size-1].
- * Returns true if all output indexes have lower and upper (input) indexes
- * within range [0,in_size-1].
- */
-inline bool compute_interpolation_weights(const int min_i, const int max_i,
-                                          const int in_size,
-                                          const float out2in_scale,
-                                          const float out2in_start,
-                                          const bool clip,
-                                          CachedInterpolation* interpolation) {
-  bool rval = true;
-  int num_i = max_i - min_i + 1;
-  for (int i = 0; i < num_i; ++i) {
-    if (!compute_single_interpolation_weight(
-            in_size, out2in_scale, out2in_start, clip, i + min_i,
-            &interpolation[i].lower, &interpolation[i].upper,
-            &interpolation[i].lerp)) {
-      rval = false;
-    }
-  }
-  return rval;
-}
-/**
- * Compatibility method for resize_bilinear_op.cc
- */
-inline void compute_interpolation_weights(const int out_size, const int in_size,
-                                          const float out2in_scale,
-                                          CachedInterpolation* interpolation) {
-  interpolation[out_size].lower = 0;
-  interpolation[out_size].upper = 0;
-  const bool clip = true;
-  if (!compute_interpolation_weights(0, out_size - 1, in_size, out2in_scale,
-                                     0.0f, clip, interpolation)) {
-    // Should never happen, check for it anyway
-    printf(
-        "Warning! Interpolation values have lower,upper indexes outside of "
-        "range [0,in_size-1]\n");
-  }
-}
-/**
- * Compute minimum and maximum (output) i where both lower and upper (input) is
- * in range [0,in_size-1]
- * If no values of i satisfy condition, min_i = in_size, max_i = -1 and method
- * returns false.
- * Returns true if min_i >= max_i.
- */
-inline bool compute_minmax_indexes(const int out_size, const int in_size,
-                                   const float out2in_scale,
-                                   const float out2in_start, int& min_i,
-                                   int& max_i) {
-  min_i = out_size;
-  max_i = -1;
-  int lower, upper;
-  float lerp;
-  for (int i = 0; i < out_size; ++i) {
-    if (compute_single_interpolation_weight(in_size, out2in_scale, out2in_start,
-                                            false, i, &lower, &upper, &lerp)) {
-      if (i < min_i) min_i = i;
-      if (i > max_i) max_i = i;
-    }
-  }
-  return (min_i <= max_i) ? true : false;
-}
-/**
- * Compute interpolation weights for crop_and_resize_op.cc
- * Also computes extrapolation areas.
- * Returns true if at least one point requires interpolation, false otherwise.
- */
-inline bool compute_interpolation_weights(
-    const int out_size, const int in_size,
-    const float x1,  // lower bounding box, crop region starts at in_size*x1
-    const float x2,  // upper bounding box, crop region ends at in_size*x2
-    int& min_i, int& max_i, CachedInterpolation*& interpolation) {
-  float out2in_start = out_size > 1
-                           ? (float)(in_size - 1) * (float)x1
-                           : (float)(in_size - 1) * (float)(x1 + x2) / 2.0f;
-  float out2in_scale = out_size > 1 ? (float)(x2 - x1) * (float)(in_size - 1) /
-                                          (float)(out_size - 1)
-                                    : 0.0f;
-  if (compute_minmax_indexes(out_size, in_size, out2in_scale, out2in_start,
-                             min_i, max_i)) {
-    interpolation = new CachedInterpolation[max_i - min_i + 1];
-    bool all_inputs_ok =
-        compute_interpolation_weights(min_i, max_i, in_size, out2in_scale,
-                                      out2in_start, false, interpolation);
-    if (!all_inputs_ok) {
-      // should never happen, purpose of compute_minmax_indexes is to ensure
-      // that all inputs are ok.
-      printf(
-          "Error! compute_interpolation_weights returned input indexes outside "
-          "valid range - SEGV will likely ensue.\n");
-    }
-    return true;
-  } else {
-    interpolation = 0l;
-    return false;
-  }
-}
-
-/**
- * Cast float v to type U with range clamping.
- *
- * If vmax_val,
- * return value is clamped to u_max_val.
- */
-template 
-U cast_to(float v, float min_val, float max_val, U u_min_val, U u_max_val);
-template 
-inline U cast_to(float v, float min_val, float max_val, U u_min_val,
-                 U u_max_val) {
-  if (v < min_val)
-    return u_min_val;
-  else if (v > max_val)
-    return u_max_val;
-  else
-    return static_cast(v);
-}
-template <>
-inline float cast_to(float v, float min_val, float max_val,
-                            float u_min_val, float u_max_val) {
-  return v;
-}
-
-inline float compute_lerp(const float top_left, const float top_right,
-                          const float bottom_left, const float bottom_right,
-                          const float x_lerp, const float y_lerp) {
-  const float top = top_left + (top_right - top_left) * x_lerp;
-  const float bottom = bottom_left + (bottom_right - bottom_left) * x_lerp;
-  return top + (bottom - top) * y_lerp;
-}
-
-/**
- * Computes the bilinear interpolation from the appropriate 4 float points
- * and the linear interpolation weights.
- * Accepts input tensors of type T and produces output tensors of type U.
- * Optionally flips horizontal and/or vertical axis.
- */
-template 
-void crop_resize_single_image(const T* image, const int64 in_height,
-                              const int64 in_width, const int64 out_height,
-                              const int64 out_width, const int channels,
-                              const int min_ix, const int max_ix,
-                              const CachedInterpolation* xs, const int min_iy,
-                              const int max_iy, const CachedInterpolation* ys,
-                              const float extrapolated_value, const bool flip_x,
-                              const bool flip_y,
-                              U* output) TF_ATTRIBUTE_NOINLINE;
-template 
-void crop_resize_single_image(const T* image, const int64 in_height,
-                              const int64 in_width, const int64 out_height,
-                              const int64 out_width, const int channels,
-                              const int min_ix, const int max_ix,
-                              const CachedInterpolation* xs, const int min_iy,
-                              const int max_iy, const CachedInterpolation* ys,
-                              const float extrapolated_value, const bool flip_x,
-                              const bool flip_y, U* output) {
-  const int64 in_row_size = in_width * channels;
-  const int64 out_row_size = out_width * channels;
-  U u_min_val = std::numeric_limits::min();
-  U u_max_val = std::numeric_limits::max();
-  float min_val = static_cast(u_min_val);
-  float max_val = static_cast(u_max_val);
-  U uEx =
-      cast_to(extrapolated_value, min_val, max_val, u_min_val, u_max_val);
-  // low y extrapolation zone
-  if (min_iy > 0) {
-    U* p = flip_y ? output + out_row_size * (out_height - min_iy) : output;
-    int64 nn = out_row_size * (int64)min_iy;
-    for (int64 i = 0; i < nn; ++i) p[i] = uEx;
-  }
-  // high y extrapolation zone
-  if (max_iy < out_height - 1) {
-    U* p = flip_y ? output : output + out_row_size * (max_iy + 1);
-    int64 nn = out_row_size * (int64)(out_height - 1 - max_iy);
-    for (int64 i = 0; i < nn; ++i) p[i] = uEx;
-  }
-  // low x extrapolation zone
-  if (min_ix > 0) {
-    for (int iy = min_iy; iy <= max_iy; ++iy) {
-      int xx0 = flip_x ? (out_width - min_ix) * channels : 0;
-      int nxx = min_ix * channels;
-      U* p = output + xx0 +
-             out_row_size * (int64)(flip_y ? out_height - 1 - iy : iy);
-      for (int ix = 0; ix < nxx; ++ix) {
-        p[ix] = uEx;
-      }
-    }
-  }
-  // high x extrapolation zone
-  if (max_ix < out_width - 1) {
-    for (int iy = min_iy; iy <= max_iy; ++iy) {
-      int xx0 = flip_x ? 0 : (max_ix + 1) * channels;
-      int nxx = (out_width - 1 - max_ix) * channels;
-      U* p = output + xx0 +
-             out_row_size * (int64)(flip_y ? out_height - 1 - iy : iy);
-      for (int ix = 0; ix < nxx; ++ix) {
-        p[ix] = uEx;
-      }
-    }
-  }
-  U* output_y_ptr =
-      output +
-      out_row_size * (int64)(flip_y ? out_height - 1 - min_iy : min_iy);
-  // interpolation zone
-  if (channels == 1) {
-    for (int y = min_iy; y <= max_iy; ++y) {
-      const int iy = y - min_iy;
-      const T* ys_input_lower_ptr = image + ys[iy].lower * in_row_size;
-      const T* ys_input_upper_ptr = image + ys[iy].upper * in_row_size;
-      const float ys_lerp = ys[iy].lerp;
-      const int x0 = flip_x ? out_width - 1 - max_ix : min_ix;
-      const int x1 = flip_x ? out_width - 1 - min_ix : max_ix;
-      for (int x = x0; x <= x1; ++x) {
-        const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix;
-        const int64 xs_lower = xs[ix].lower;
-        const int64 xs_upper = xs[ix].upper;
-        const float xs_lerp = xs[ix].lerp;
-
-        // Read channel 0.
-        const float top_left0(ys_input_lower_ptr[xs_lower]);
-        const float top_right0(ys_input_lower_ptr[xs_upper]);
-        const float bottom_left0(ys_input_upper_ptr[xs_lower]);
-        const float bottom_right0(ys_input_upper_ptr[xs_upper]);
-
-        // Compute output.
-        float result0 = compute_lerp(top_left0, top_right0, bottom_left0,
-                                     bottom_right0, xs_lerp, ys_lerp);
-        output_y_ptr[x] =
-            cast_to(result0, min_val, max_val, u_min_val, u_max_val);
-      }
-      output_y_ptr =
-          flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size;
-    }
-  } else if (channels == 2) {
-    for (int y = min_iy; y <= max_iy; ++y) {
-      const int iy = y - min_iy;
-      const T* ys_input_lower_ptr = image + ys[iy].lower * in_row_size;
-      const T* ys_input_upper_ptr = image + ys[iy].upper * in_row_size;
-      const float ys_lerp = ys[iy].lerp;
-      const int x0 = flip_x ? out_width - 1 - max_ix : min_ix;
-      const int x1 = flip_x ? out_width - 1 - min_ix : max_ix;
-      for (int x = x0; x <= x1; ++x) {
-        const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix;
-        const int64 xs_lower = xs[ix].lower;
-        const int64 xs_upper = xs[ix].upper;
-        const float xs_lerp = xs[ix].lerp;
-
-        // Read channel 0.
-        const float top_left0(ys_input_lower_ptr[xs_lower + 0]);
-        const float top_right0(ys_input_lower_ptr[xs_upper + 0]);
-        const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]);
-        const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]);
-
-        // Read channel 1.
-        const float top_left1(ys_input_lower_ptr[xs_lower + 1]);
-        const float top_right1(ys_input_lower_ptr[xs_upper + 1]);
-        const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]);
-        const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]);
-
-        // Compute output.
-        float result0 = compute_lerp(top_left0, top_right0, bottom_left0,
-                                     bottom_right0, xs_lerp, ys_lerp);
-        float result1 = compute_lerp(top_left1, top_right1, bottom_left1,
-                                     bottom_right1, xs_lerp, ys_lerp);
-        output_y_ptr[x * 2 + 0] =
-            cast_to(result0, min_val, max_val, u_min_val, u_max_val);
-        output_y_ptr[x * 2 + 1] =
-            cast_to(result1, min_val, max_val, u_min_val, u_max_val);
-      }
-      output_y_ptr =
-          flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size;
-    }
-  } else if (channels == 3) {
-    for (int y = min_iy; y <= max_iy; ++y) {
-      const int iy = y - min_iy;
-      const T* ys_input_lower_ptr = image + ys[iy].lower * in_row_size;
-      const T* ys_input_upper_ptr = image + ys[iy].upper * in_row_size;
-      const float ys_lerp = ys[iy].lerp;
-      const int x0 = flip_x ? out_width - 1 - max_ix : min_ix;
-      const int x1 = flip_x ? out_width - 1 - min_ix : max_ix;
-      for (int x = x0; x <= x1; ++x) {
-        const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix;
-        const int64 xs_lower = xs[ix].lower;
-        const int64 xs_upper = xs[ix].upper;
-        const float xs_lerp = xs[ix].lerp;
-
-        // Read channel 0.
-        const float top_left0(ys_input_lower_ptr[xs_lower + 0]);
-        const float top_right0(ys_input_lower_ptr[xs_upper + 0]);
-        const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]);
-        const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]);
-
-        // Read channel 1.
-        const float top_left1(ys_input_lower_ptr[xs_lower + 1]);
-        const float top_right1(ys_input_lower_ptr[xs_upper + 1]);
-        const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]);
-        const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]);
-
-        // Read channel 2.
-        const float top_left2(ys_input_lower_ptr[xs_lower + 2]);
-        const float top_right2(ys_input_lower_ptr[xs_upper + 2]);
-        const float bottom_left2(ys_input_upper_ptr[xs_lower + 2]);
-        const float bottom_right2(ys_input_upper_ptr[xs_upper + 2]);
-
-        // Compute output.
-        float result0 = compute_lerp(top_left0, top_right0, bottom_left0,
-                                     bottom_right0, xs_lerp, ys_lerp);
-        float result1 = compute_lerp(top_left1, top_right1, bottom_left1,
-                                     bottom_right1, xs_lerp, ys_lerp);
-        float result2 = compute_lerp(top_left2, top_right2, bottom_left2,
-                                     bottom_right2, xs_lerp, ys_lerp);
-        output_y_ptr[x * 3 + 0] =
-            cast_to(result0, min_val, max_val, u_min_val, u_max_val);
-        output_y_ptr[x * 3 + 1] =
-            cast_to(result1, min_val, max_val, u_min_val, u_max_val);
-        output_y_ptr[x * 3 + 2] =
-            cast_to(result2, min_val, max_val, u_min_val, u_max_val);
-      }
-      output_y_ptr =
-          flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size;
-    }
-  } else if (channels == 4) {
-    for (int y = min_iy; y <= max_iy; ++y) {
-      const int iy = y - min_iy;
-      const T* ys_input_lower_ptr = image + ys[iy].lower * in_row_size;
-      const T* ys_input_upper_ptr = image + ys[iy].upper * in_row_size;
-      const float ys_lerp = ys[iy].lerp;
-      const int x0 = flip_x ? out_width - 1 - max_ix : min_ix;
-      const int x1 = flip_x ? out_width - 1 - min_ix : max_ix;
-      for (int x = x0; x <= x1; ++x) {
-        const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix;
-        const int64 xs_lower = xs[ix].lower;
-        const int64 xs_upper = xs[ix].upper;
-        const float xs_lerp = xs[ix].lerp;
-
-        // Read channel 0.
-        const float top_left0(ys_input_lower_ptr[xs_lower + 0]);
-        const float top_right0(ys_input_lower_ptr[xs_upper + 0]);
-        const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]);
-        const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]);
-
-        // Read channel 1.
-        const float top_left1(ys_input_lower_ptr[xs_lower + 1]);
-        const float top_right1(ys_input_lower_ptr[xs_upper + 1]);
-        const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]);
-        const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]);
-
-        // Read channel 2.
-        const float top_left2(ys_input_lower_ptr[xs_lower + 2]);
-        const float top_right2(ys_input_lower_ptr[xs_upper + 2]);
-        const float bottom_left2(ys_input_upper_ptr[xs_lower + 2]);
-        const float bottom_right2(ys_input_upper_ptr[xs_upper + 2]);
-
-        // Read channel 3.
-        const float top_left3(ys_input_lower_ptr[xs_lower + 3]);
-        const float top_right3(ys_input_lower_ptr[xs_upper + 3]);
-        const float bottom_left3(ys_input_upper_ptr[xs_lower + 3]);
-        const float bottom_right3(ys_input_upper_ptr[xs_upper + 3]);
-
-        // Compute output.
-        float result0 = compute_lerp(top_left0, top_right0, bottom_left0,
-                                     bottom_right0, xs_lerp, ys_lerp);
-        float result1 = compute_lerp(top_left1, top_right1, bottom_left1,
-                                     bottom_right1, xs_lerp, ys_lerp);
-        float result2 = compute_lerp(top_left2, top_right2, bottom_left2,
-                                     bottom_right2, xs_lerp, ys_lerp);
-        float result3 = compute_lerp(top_left3, top_right3, bottom_left3,
-                                     bottom_right3, xs_lerp, ys_lerp);
-        output_y_ptr[x * 4 + 0] =
-            cast_to(result0, min_val, max_val, u_min_val, u_max_val);
-        output_y_ptr[x * 4 + 1] =
-            cast_to(result1, min_val, max_val, u_min_val, u_max_val);
-        output_y_ptr[x * 4 + 2] =
-            cast_to(result2, min_val, max_val, u_min_val, u_max_val);
-        output_y_ptr[x * 4 + 3] =
-            cast_to(result3, min_val, max_val, u_min_val, u_max_val);
-      }
-      output_y_ptr =
-          flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size;
-    }
-  } else {
-    for (int y = min_iy; y <= max_iy; ++y) {
-      const int iy = y - min_iy;
-      const T* ys_input_lower_ptr = image + ys[iy].lower * in_row_size;
-      const T* ys_input_upper_ptr = image + ys[iy].upper * in_row_size;
-      const float ys_lerp = ys[iy].lerp;
-      const int x0 = flip_x ? out_width - 1 - max_ix : min_ix;
-      const int x1 = flip_x ? out_width - 1 - min_ix : max_ix;
-      for (int x = x0; x <= x1; ++x) {
-        const int ix = flip_x ? out_width - 1 - min_ix - x : x - min_ix;
-        const int64 xs_lower = xs[ix].lower;
-        const int64 xs_upper = xs[ix].upper;
-        const float xs_lerp = xs[ix].lerp;
-        for (int ichan = 0; ichan < channels; ++ichan) {
-          const float top_left0(ys_input_lower_ptr[xs_lower + ichan]);
-          const float top_right0(ys_input_lower_ptr[xs_upper + ichan]);
-          const float bottom_left0(ys_input_upper_ptr[xs_lower + ichan]);
-          const float bottom_right0(ys_input_upper_ptr[xs_upper + ichan]);
-          float result0 = compute_lerp(top_left0, top_right0, bottom_left0,
-                                       bottom_right0, xs_lerp, ys_lerp);
-          output_y_ptr[x * channels + ichan] =
-              cast_to(result0, min_val, max_val, u_min_val, u_max_val);
-        }
-      }
-      output_y_ptr =
-          flip_y ? output_y_ptr - out_row_size : output_y_ptr + out_row_size;
-    }
-  }
-}
-}  // namespace internal
-}  // namespace tensorflow
-#endif  // TENSORFLOW_CORE_KERNELS_CROP_RESIZE_BILINEAR_CORE_H_
diff --git a/tensorflow/core/kernels/resize_bilinear_op.cc b/tensorflow/core/kernels/resize_bilinear_op.cc
index bf9e93ef3f..dde59e8e74 100644
--- a/tensorflow/core/kernels/resize_bilinear_op.cc
+++ b/tensorflow/core/kernels/resize_bilinear_op.cc
@@ -25,15 +25,10 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/kernels/crop_resize_bilinear_core.h"
 #include "tensorflow/core/kernels/image_resizer_state.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
 
-using ::tensorflow::internal::CachedInterpolation;
-using ::tensorflow::internal::compute_interpolation_weights;
-using ::tensorflow::internal::crop_resize_single_image;
-
 namespace tensorflow {
 
 typedef Eigen::ThreadPoolDevice CPUDevice;
@@ -68,6 +63,140 @@ class ResizeBilinearOp : public OpKernel {
   bool align_corners_;
 };
 
+namespace {
+// Compute the interpolation indices only once.
+struct CachedInterpolation {
+  int64 lower;  // Lower source index used in the interpolation
+  int64 upper;  // Upper source index used in the interpolation
+  // 1-D linear iterpolation scale (see:
+  // https://en.wikipedia.org/wiki/Bilinear_interpolation)
+  float lerp;
+};
+
+inline void compute_interpolation_weights(const int64 out_size,
+                                          const int64 in_size,
+                                          const float scale,
+                                          CachedInterpolation* interpolation) {
+  interpolation[out_size].lower = 0;
+  interpolation[out_size].upper = 0;
+  for (int64 i = out_size - 1; i >= 0; --i) {
+    const float in = i * scale;
+    interpolation[i].lower = static_cast(in);
+    interpolation[i].upper = std::min(interpolation[i].lower + 1, in_size - 1);
+    interpolation[i].lerp = in - interpolation[i].lower;
+  }
+}
+
+/**
+ * Computes the bilinear interpolation from the appropriate 4 float points
+ * and the linear interpolation weights.
+ */
+inline float compute_lerp(const float top_left, const float top_right,
+                          const float bottom_left, const float bottom_right,
+                          const float x_lerp, const float y_lerp) {
+  const float top = top_left + (top_right - top_left) * x_lerp;
+  const float bottom = bottom_left + (bottom_right - bottom_left) * x_lerp;
+  return top + (bottom - top) * y_lerp;
+}
+
+template 
+void resize_image(
+    typename TTypes::ConstTensor images, const int batch_size,
+    const int64 in_height, const int64 in_width, const int64 out_height,
+    const int64 out_width, const int channels,
+    const std::vector& xs,
+    const std::vector& ys,
+    typename TTypes::Tensor output) TF_ATTRIBUTE_NOINLINE;
+template 
+void resize_image(typename TTypes::ConstTensor images,
+                  const int batch_size, const int64 in_height,
+                  const int64 in_width, const int64 out_height,
+                  const int64 out_width, const int channels,
+                  const std::vector& xs_vec,
+                  const std::vector& ys,
+                  typename TTypes::Tensor output) {
+  const int64 in_row_size = in_width * channels;
+  const int64 in_batch_num_values = in_height * in_row_size;
+  const int64 out_row_size = out_width * channels;
+
+  const T* input_b_ptr = images.data();
+  const CachedInterpolation* xs = xs_vec.data();
+
+  if (channels == 3) {
+    float* output_y_ptr = output.data();
+    for (int b = 0; b < batch_size; ++b) {
+      for (int64 y = 0; y < out_height; ++y) {
+        const T* ys_input_lower_ptr = input_b_ptr + ys[y].lower * in_row_size;
+        const T* ys_input_upper_ptr = input_b_ptr + ys[y].upper * in_row_size;
+        const float ys_lerp = ys[y].lerp;
+        for (int64 x = 0; x < out_width; ++x) {
+          const int64 xs_lower = xs[x].lower;
+          const int64 xs_upper = xs[x].upper;
+          const float xs_lerp = xs[x].lerp;
+
+          // Read channel 0.
+          const float top_left0(ys_input_lower_ptr[xs_lower + 0]);
+          const float top_right0(ys_input_lower_ptr[xs_upper + 0]);
+          const float bottom_left0(ys_input_upper_ptr[xs_lower + 0]);
+          const float bottom_right0(ys_input_upper_ptr[xs_upper + 0]);
+
+          // Read channel 1.
+          const float top_left1(ys_input_lower_ptr[xs_lower + 1]);
+          const float top_right1(ys_input_lower_ptr[xs_upper + 1]);
+          const float bottom_left1(ys_input_upper_ptr[xs_lower + 1]);
+          const float bottom_right1(ys_input_upper_ptr[xs_upper + 1]);
+
+          // Read channel 2.
+          const float top_left2(ys_input_lower_ptr[xs_lower + 2]);
+          const float top_right2(ys_input_lower_ptr[xs_upper + 2]);
+          const float bottom_left2(ys_input_upper_ptr[xs_lower + 2]);
+          const float bottom_right2(ys_input_upper_ptr[xs_upper + 2]);
+
+          // Compute output.
+          output_y_ptr[x * channels + 0] =
+              compute_lerp(top_left0, top_right0, bottom_left0, bottom_right0,
+                           xs_lerp, ys_lerp);
+          output_y_ptr[x * channels + 1] =
+              compute_lerp(top_left1, top_right1, bottom_left1, bottom_right1,
+                           xs_lerp, ys_lerp);
+          output_y_ptr[x * channels + 2] =
+              compute_lerp(top_left2, top_right2, bottom_left2, bottom_right2,
+                           xs_lerp, ys_lerp);
+        }
+        output_y_ptr += out_row_size;
+      }
+      input_b_ptr += in_batch_num_values;
+    }
+  } else {
+    float* output_y_ptr = output.data();
+    for (int b = 0; b < batch_size; ++b) {
+      for (int64 y = 0; y < out_height; ++y) {
+        const T* ys_input_lower_ptr = input_b_ptr + ys[y].lower * in_row_size;
+        const T* ys_input_upper_ptr = input_b_ptr + ys[y].upper * in_row_size;
+        const float ys_lerp = ys[y].lerp;
+        for (int64 x = 0; x < out_width; ++x) {
+          auto xs_lower = xs[x].lower;
+          auto xs_upper = xs[x].upper;
+          auto xs_lerp = xs[x].lerp;
+          for (int c = 0; c < channels; ++c) {
+            const float top_left(ys_input_lower_ptr[xs_lower + c]);
+            const float top_right(ys_input_lower_ptr[xs_upper + c]);
+            const float bottom_left(ys_input_upper_ptr[xs_lower + c]);
+            const float bottom_right(ys_input_upper_ptr[xs_upper + c]);
+            output_y_ptr[x * channels + c] =
+                compute_lerp(top_left, top_right, bottom_left, bottom_right,
+                             xs_lerp, ys_lerp);
+          }
+        }
+        output_y_ptr += out_row_size;
+      }
+      input_b_ptr += in_batch_num_values;
+    }
+  }
+}
+
+}  // namespace
+
 // Partial specialization of ResizeBilinear functor for a CPUDevice.
 namespace functor {
 template 
@@ -83,11 +212,6 @@ struct ResizeBilinear {
     const int64 out_height = output.dimension(1);
     const int64 out_width = output.dimension(2);
 
-    const int64 in_row_size = in_width * channels;
-    const int64 in_batch_num_values = in_height * in_row_size;
-    const int64 out_row_size = out_width * channels;
-    const int64 out_batch_num_values = out_row_size * out_height;
-
     // Handle no-op resizes efficiently.
     if (out_height == in_height && out_width == in_width) {
       output = images.template cast();
@@ -108,13 +232,8 @@ struct ResizeBilinear {
       xs[i].upper *= channels;
     }
 
-    for (int b = 0; b < batch_size; ++b) {
-      crop_resize_single_image(
-          images.data() + static_cast(b) * in_batch_num_values,
-          in_height, in_width, out_height, out_width, channels, 0,
-          out_width - 1, xs.data(), 0, out_height - 1, ys.data(), 0.0f, false,
-          false, output.data() + static_cast(b) * out_batch_num_values);
-    }
+    resize_image(images, batch_size, in_height, in_width, out_height,
+                    out_width, channels, xs, ys, output);
   }
 };
 }  // namespace functor
-- 
cgit v1.2.3


From bbc2c3f1c82fb3987134019e11dbd055a623d395 Mon Sep 17 00:00:00 2001
From: Xuechen Li 
Date: Mon, 23 Jul 2018 09:42:36 -0700
Subject: Refactor imagenet configurations.

PiperOrigin-RevId: 205675401
---
 .../contrib/eager/python/examples/revnet/config.py | 34 +++++++++++++++++-----
 .../eager/python/examples/revnet/revnet_test.py    |  4 ---
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/revnet/config.py b/tensorflow/contrib/eager/python/examples/revnet/config.py
index 1532c7b67b..e108686b66 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/config.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/config.py
@@ -45,7 +45,7 @@ def get_hparams_cifar_38():
   config.add_hparam("bottleneck", False)
   config.add_hparam("fused", True)
   config.add_hparam("init_max_pool", False)
-  if tf.test.is_gpu_available() > 0:
+  if tf.test.is_gpu_available():
     config.add_hparam("input_shape", (3, 32, 32))
     config.add_hparam("data_format", "channels_first")
   else:
@@ -74,7 +74,7 @@ def get_hparams_cifar_38():
   # TPU architecture specifics
   # Suggested batch sizes to reduce overhead from excessive tensor padding
   # https://cloud.google.com/tpu/docs/troubleshooting
-  config.add_hparam("tpu_batch_size", 128)
+  config.add_hparam("tpu_batch_size", 1024)
   config.add_hparam("tpu_eval_batch_size", 1024)
   config.add_hparam("tpu_iters_per_epoch", 50000 // config.tpu_batch_size)
   config.add_hparam("tpu_epochs",
@@ -107,6 +107,8 @@ def get_hparams_imagenet_56():
   """RevNet-56 configurations for ImageNet."""
 
   config = tf.contrib.training.HParams()
+  config.add_hparam("n_classes", 1000)
+  config.add_hparam("dataset", "ImageNet")
   config.add_hparam("init_filters", 128)
   config.add_hparam("init_kernel", 7)
   config.add_hparam("init_stride", 2)
@@ -114,7 +116,7 @@ def get_hparams_imagenet_56():
   config.add_hparam("n_res", [2, 2, 2, 2])
   config.add_hparam("filters", [128, 256, 512, 832])
   config.add_hparam("strides", [1, 2, 2, 2])
-  config.add_hparam("batch_size", 16)
+  config.add_hparam("batch_size", 256)
   config.add_hparam("bottleneck", True)
   config.add_hparam("fused", True)
   config.add_hparam("init_max_pool", True)
@@ -133,16 +135,32 @@ def get_hparams_imagenet_56():
   config.add_hparam("max_train_iter", 600000)
   config.add_hparam("seed", 1234)
   config.add_hparam("shuffle", True)
-  config.add_hparam("log_every", 50)
-  config.add_hparam("save_every", 50)
+  config.add_hparam("log_every", 500)
+  config.add_hparam("save_every", 500)
   config.add_hparam("dtype", tf.float32)
-  config.add_hparam("eval_batch_size", 1000)
+  config.add_hparam("eval_batch_size", 256)
   config.add_hparam("div255", True)
-  # TODO(lxuechen): Update this according to ImageNet data
-  config.add_hparam("iters_per_epoch", 50000 // config.batch_size)
+  config.add_hparam("iters_per_epoch", 1281167 // config.batch_size)
   config.add_hparam("epochs", config.max_train_iter // config.iters_per_epoch)
   # Due to bottleneck residual blocks
   filters = [f * 4 for f in config.filters]
   config.filters = filters
 
+  # Customized TPU hyperparameters due to differing batch size caused by
+  # TPU architecture specifics
+  # Suggested batch sizes to reduce overhead from excessive tensor padding
+  # https://cloud.google.com/tpu/docs/troubleshooting
+  config.add_hparam("tpu_batch_size", 1024)
+  config.add_hparam("tpu_eval_batch_size", 1024)
+  config.add_hparam("tpu_iters_per_epoch", 1281167 // config.tpu_batch_size)
+  config.add_hparam("tpu_epochs",
+                    config.max_train_iter // config.tpu_iters_per_epoch)
+
+  return config
+
+
+def get_hparams_imagenet_104():
+  config = get_hparams_imagenet_56()
+  config.n_res = [2, 2, 11, 2]
+
   return config
diff --git a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
index 2dc7b9fd70..26b0847523 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/revnet_test.py
@@ -223,8 +223,6 @@ class RevNetBenchmark(tf.test.Benchmark):
                              execution_mode=None,
                              compiled=False):
     config = config_.get_hparams_imagenet_56()
-    config.add_hparam("n_classes", 1000)
-    config.add_hparam("dataset", "ImageNet")
     with tfe.execution_mode(execution_mode):
       device, data_format = device_and_format
       model = revnet.RevNet(config=config)
@@ -270,8 +268,6 @@ class RevNetBenchmark(tf.test.Benchmark):
                              execution_mode=None,
                              compiled=False):
     config = config_.get_hparams_imagenet_56()
-    config.add_hparam("n_classes", 1000)
-    config.add_hparam("dataset", "ImageNet")
     with tfe.execution_mode(execution_mode):
       device, data_format = device_and_format
       for batch_size in self._train_batch_sizes():
-- 
cgit v1.2.3


From b8a9d163d9cbb4b581c044d9c4b1b256c801a9c4 Mon Sep 17 00:00:00 2001
From: Russell Power 
Date: Mon, 23 Jul 2018 10:05:16 -0700
Subject: Automated rollback of commit 8048b3a53d5a919fef74874fcffbec9039e6acd1

PiperOrigin-RevId: 205679162
---
 tensorflow/core/BUILD                         |   8 --
 tensorflow/core/common_runtime/session_ref.cc | 170 --------------------------
 tensorflow/core/common_runtime/session_ref.h  |  86 -------------
 tensorflow/python/BUILD                       |   1 -
 tensorflow/python/client/session.py           |   2 +-
 tensorflow/python/client/tf_session.i         |   1 -
 tensorflow/python/client/tf_session_helper.cc |  14 ---
 tensorflow/python/client/tf_session_helper.h  |   3 -
 8 files changed, 1 insertion(+), 284 deletions(-)
 delete mode 100644 tensorflow/core/common_runtime/session_ref.cc
 delete mode 100644 tensorflow/core/common_runtime/session_ref.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index d51d9f0295..17e6ccda14 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2921,14 +2921,6 @@ tf_cuda_library(
     ] + tf_additional_device_tracer_deps(),
 )
 
-cc_library(
-    name = "session_ref",
-    srcs = ["common_runtime/session_ref.cc"],
-    hdrs = ["common_runtime/session_ref.h"],
-    copts = tf_copts(),
-    deps = [":core_cpu_base"],
-)
-
 cc_library(
     name = "gpu_id",
     hdrs = [
diff --git a/tensorflow/core/common_runtime/session_ref.cc b/tensorflow/core/common_runtime/session_ref.cc
deleted file mode 100644
index b931ef4229..0000000000
--- a/tensorflow/core/common_runtime/session_ref.cc
+++ /dev/null
@@ -1,170 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/core/common_runtime/session_ref.h"
-
-#include 
-
-namespace tensorflow {
-
-namespace {
-
-// Scope helper to track active calls and manage session lifetime.
-struct RunCounter {
-  std::shared_ptr session;
-  uint64* value;
-  mutex* m;
-  condition_variable* cv;
-
-  explicit RunCounter(std::shared_ptr s, uint64* v, mutex* m,
-                      condition_variable* cv)
-      : session(std::move(s)), value(v), m(m), cv(cv) {
-    mutex_lock l(*m);
-    ++*value;
-  }
-
-  ~RunCounter() {
-    mutex_lock l(*m);
-    if (--*value == 0) {
-      cv->notify_all();
-    }
-  }
-};
-
-}  // namespace
-
-Status SessionRef::CheckNotClosed() {
-  mutex_lock l(run_lock_);
-  if (session_ == nullptr) return errors::Cancelled("Session has been closed.");
-  return ::tensorflow::Status::OK();
-}
-
-Status SessionRef::Run(const RunOptions& run_options,
-                       const std::vector >& inputs,
-                       const std::vector& output_tensor_names,
-                       const std::vector& target_node_names,
-                       std::vector* outputs,
-                       RunMetadata* run_metadata) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->Run(run_options, inputs, output_tensor_names,
-                         target_node_names, outputs, run_metadata);
-}
-
-Status SessionRef::Create(const GraphDef& graph) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->Create(graph);
-}
-
-Status SessionRef::Create(const RunOptions& run_options,
-                          const GraphDef& graph) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->Create(run_options, graph);
-}
-
-Status SessionRef::Extend(const RunOptions& run_options,
-                          const GraphDef& graph) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->Extend(run_options, graph);
-}
-
-Status SessionRef::Extend(const GraphDef& graph) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->Extend(graph);
-}
-
-Status SessionRef::Close(const RunOptions& run_options) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  mutex_lock l(run_lock_);
-  Status status = session_->Close(run_options);
-  session_.reset();
-  while (run_count_ > 0) {
-    run_finished_.wait(l);
-  }
-  return status;
-}
-
-Status SessionRef::Close() {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  mutex_lock l(run_lock_);
-  Status status = session_->Close();
-  session_.reset();
-  while (run_count_ > 0) {
-    run_finished_.wait(l);
-  }
-  return status;
-}
-
-Status SessionRef::Run(const std::vector >& inputs,
-                       const std::vector& output_tensor_names,
-                       const std::vector& target_node_names,
-                       std::vector* outputs) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->Run(inputs, output_tensor_names, target_node_names,
-                         outputs);
-}
-
-Status SessionRef::ListDevices(std::vector* response) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->ListDevices(response);
-}
-
-Status SessionRef::PRunSetup(const std::vector& input_names,
-                             const std::vector& output_names,
-                             const std::vector& target_nodes,
-                             string* handle) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->PRunSetup(input_names, output_names, target_nodes, handle);
-}
-
-Status SessionRef::PRun(const string& handle,
-                        const std::vector >& inputs,
-                        const std::vector& output_names,
-                        std::vector* outputs) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->PRun(handle, inputs, output_names, outputs);
-}
-
-Status SessionRef::MakeCallable(const CallableOptions& callable_options,
-                                CallableHandle* out_handle) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->MakeCallable(callable_options, out_handle);
-}
-
-Status SessionRef::RunCallable(CallableHandle handle,
-                               const std::vector& feed_tensors,
-                               std::vector* fetch_tensors,
-                               RunMetadata* run_metadata) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->RunCallable(handle, feed_tensors, fetch_tensors,
-                                 run_metadata);
-}
-
-Status SessionRef::ReleaseCallable(CallableHandle handle) {
-  TF_RETURN_IF_ERROR(CheckNotClosed());
-  RunCounter rc(session_, &run_count_, &run_lock_, &run_finished_);
-  return rc.session->ReleaseCallable(handle);
-}
-
-}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/session_ref.h b/tensorflow/core/common_runtime/session_ref.h
deleted file mode 100644
index 6146933326..0000000000
--- a/tensorflow/core/common_runtime/session_ref.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SESSION_REF_H_
-#define TENSORFLOW_CORE_COMMON_RUNTIME_SESSION_REF_H_
-
-#include 
-
-#include "tensorflow/core/platform/mutex.h"
-#include "tensorflow/core/public/session.h"
-
-namespace tensorflow {
-
-// A `SessionRef` manages the lifetime of a wrapped `Session` pointer.
-//
-// SessionRef blocks the return of Close() until all pending operations have
-// been completed or cancelled and underlying session has been freed.  Any
-// subsequent operations on the SessionRef object will return errors::Cancelled.
-class SessionRef : public Session {
- public:
-  SessionRef(Session* session) : session_(session) {}
-  virtual ~SessionRef() {}
-
-  Status Create(const GraphDef& graph) override;
-  Status Extend(const GraphDef& graph) override;
-  Status Create(const RunOptions& run_options, const GraphDef& graph) override;
-  Status Extend(const RunOptions& run_options, const GraphDef& graph) override;
-  Status Run(const std::vector >& inputs,
-             const std::vector& output_tensor_names,
-             const std::vector& target_node_names,
-             std::vector* outputs) override;
-
-  Status ListDevices(std::vector* response) override;
-
-  Status Close() override;
-  Status Close(const RunOptions& run_options) override;
-
-  Status Run(const RunOptions& run_options,
-             const std::vector >& inputs,
-             const std::vector& output_tensor_names,
-             const std::vector& target_node_names,
-             std::vector* outputs, RunMetadata* run_metadata) override;
-
-  Status PRunSetup(const std::vector& input_names,
-                   const std::vector& output_names,
-                   const std::vector& target_nodes,
-                   string* handle) override;
-
-  Status PRun(const string& handle,
-              const std::vector >& inputs,
-              const std::vector& output_names,
-              std::vector* outputs) override;
-
-  Status MakeCallable(const CallableOptions& callable_options,
-                      CallableHandle* out_handle);
-
-  Status RunCallable(CallableHandle handle,
-                     const std::vector& feed_tensors,
-                     std::vector* fetch_tensors,
-                     RunMetadata* run_metadata);
-
-  Status ReleaseCallable(CallableHandle handle);
-
- private:
-  mutex run_lock_;
-  condition_variable run_finished_;
-  uint64 run_count_ GUARDED_BY(run_lock_) = {0};
-  std::shared_ptr session_;
-
-  Status CheckNotClosed();
-};
-
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_SESSION_REF_H_
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index d45566c55e..9c7f3b7b25 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3629,7 +3629,6 @@ tf_cuda_library(
         "//tensorflow/core:graph",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:session_ref",
         "//third_party/py/numpy:headers",
         "//third_party/python_runtime:headers",
     ],
diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index f3aa135fe4..8ede6ab54c 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -629,7 +629,7 @@ class BaseSession(SessionInterface):
     opts = tf_session.TF_NewSessionOptions(target=self._target, config=config)
     try:
       # pylint: disable=protected-access
-      self._session = tf_session.TF_NewSessionRef(self._graph._c_graph, opts)
+      self._session = tf_session.TF_NewSession(self._graph._c_graph, opts)
       # pylint: enable=protected-access
     finally:
       tf_session.TF_DeleteSessionOptions(opts)
diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index 39a2922ac0..1cdd8e0b6a 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -777,7 +777,6 @@ def TF_Reset(target, containers=None, config=None):
   $1 = &types_local;
 }
 
-%unignore TF_NewSessionRef;
 %unignore SetRequireShapeInferenceFns;
 %unignore TF_TryEvaluateConstant_wrapper;
 %noexception TF_TryEvaluateConstant_wrapper;
diff --git a/tensorflow/python/client/tf_session_helper.cc b/tensorflow/python/client/tf_session_helper.cc
index bcd4af2912..b6481e7e29 100644
--- a/tensorflow/python/client/tf_session_helper.cc
+++ b/tensorflow/python/client/tf_session_helper.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/tf_status_helper.h"
-#include "tensorflow/core/common_runtime/session_ref.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/attr_value_util.h"
@@ -43,19 +42,6 @@ static const char* kFeedDictErrorMsg =
     "feed_dict must be a dictionary mapping strings to NumPy arrays.";
 }  // end namespace
 
-TF_Session* TF_NewSessionRef(TF_Graph* graph, const TF_SessionOptions* opts,
-                             TF_Status* status) {
-  TF_Session* tf_session = TF_NewSession(graph, opts, status);
-  if (tf_session == nullptr) {
-    return nullptr;
-  }
-
-  Session* session = reinterpret_cast(tf_session->session);
-  SessionRef* session_ref = new SessionRef(session);
-  tf_session->session = session_ref;
-  return tf_session;
-}
-
 void TF_Run_wrapper_helper(TF_DeprecatedSession* session, const char* handle,
                            const TF_Buffer* run_options, PyObject* feed_dict,
                            const NameVector& output_names,
diff --git a/tensorflow/python/client/tf_session_helper.h b/tensorflow/python/client/tf_session_helper.h
index dab7e71aac..cfd27c2bee 100644
--- a/tensorflow/python/client/tf_session_helper.h
+++ b/tensorflow/python/client/tf_session_helper.h
@@ -40,9 +40,6 @@ typedef tensorflow::gtl::InlinedVector PyObjectVector;
 // A TF_TensorVector is a vector of borrowed pointers to TF_Tensors.
 typedef gtl::InlinedVector TF_TensorVector;
 
-TF_Session* TF_NewSessionRef(TF_Graph* graph, const TF_SessionOptions* opts,
-                             TF_Status* status);
-
 // Run the graph associated with the session starting with the
 // supplied inputs[].  Regardless of success or failure, inputs[] are
 // stolen by the implementation (i.e. the implementation will
-- 
cgit v1.2.3


From 97a0da7cd265d25a1b2caf2e6e344694bb795a1c Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Mon, 23 Jul 2018 10:19:12 -0700
Subject: Allow clustering ControlTrigger nodes

With b/111092066 fixed, we should be able to cluster ControlTrigger since all
inputs to the XLA computation (and thus all inputs to every internal node)
should be live and in that case ControlTrigger is a no-op.

PRESUBMIT=passed
FIXED=111570009
CC=brain-reviews
DELTA=9 (0 added, 9 deleted, 0 changed)
DELTA_BY_EXTENSION=cc=0
PiperOrigin-RevId: 205681775
---
 .../compiler/jit/mark_for_compilation_pass.cc      |  9 ------
 .../compiler/jit/mark_for_compilation_pass_test.cc | 33 ++++++++++++++++++++++
 2 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index a3949bc14b..38eb6d830f 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -477,15 +477,6 @@ Status MarkForCompilationPass::Run(
       return false;
     }
 
-    // TODO(b/111570009): This bailout for ControlTrigger is probably not
-    // needed.
-    //
-    // Don't compile control trigger nodes. We won't preserve their deadness
-    // semantics correctly, so it's safest not to compile them.
-    if (node->IsControlTrigger()) {
-      return false;
-    }
-
     // If this device requires a JIT, we must say yes.
     if (registration->requires_compilation) return true;
 
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
index 772c92d369..2c5f4fb774 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/array_ops.h"
 #include "tensorflow/cc/ops/control_flow_ops_internal.h"
 #include "tensorflow/cc/ops/function_ops.h"
+#include "tensorflow/cc/ops/sendrecv_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/compiler/jit/defs.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
@@ -680,5 +681,37 @@ TEST(XlaCompilationTest, ClusterIdentityWithNonRefInput) {
   EXPECT_EQ(clusters, expected_clusters);
 }
 
+TEST(XlaCompilationTest, ClusterControlTrigger) {
+  Scope root = Scope::NewRootScope().ExitOnError();
+
+  Output recv_a = ops::_Recv(root.WithOpName("recv_a"), DT_BOOL, "tensor_a",
+                             "sender", 0, "receiver");
+  Output recv_b = ops::_Recv(root.WithOpName("recv_b"), DT_BOOL, "tensor_b",
+                             "sender", 0, "receiver");
+  Output const_a = ops::Const(root.WithOpName("const_a"), 42);
+
+  ops::ControlTrigger ctrl_trigger_a(root.WithOpName("ctrl_trigger_a"));
+  ops::ControlTrigger ctrl_trigger_b(root.WithOpName("ctrl_trigger_b"));
+  root.graph()->AddControlEdge(recv_a.node(), ctrl_trigger_a.operation.node());
+  root.graph()->AddControlEdge(recv_b.node(), ctrl_trigger_a.operation.node());
+  root.graph()->AddControlEdge(ctrl_trigger_b.operation.node(), const_a.node());
+
+  std::unique_ptr graph(new Graph(OpRegistry::Global()));
+
+  TF_ASSERT_OK(root.ToGraph(graph.get()));
+  TF_ASSERT_OK(MarkForCompilation(&graph));
+
+  std::unordered_map clusters = GetClusters(*graph);
+
+  ASSERT_FALSE(clusters.empty());
+  string cluster_name = clusters.begin()->second;
+
+  // ctrl_trigger_a has inputs with mismatching deadness so it won't be
+  // clustered.  ctrl_trigger_b is okay to cluster.
+  std::unordered_map expected_clusters(
+      {{"const_a", cluster_name}, {"ctrl_trigger_b", cluster_name}});
+  EXPECT_EQ(clusters, expected_clusters);
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
cgit v1.2.3


From 5f4ebb000057d2c0c76ef856526d374648958d86 Mon Sep 17 00:00:00 2001
From: Russell Power 
Date: Mon, 23 Jul 2018 11:35:07 -0700
Subject: Fix improperly forwarded argument.

PiperOrigin-RevId: 205695945
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 1208d557e7..09eeb6a7f5 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2339,7 +2339,7 @@ class TPUEstimator(estimator_lib.Estimator):
           predict_keys=predict_keys,
           hooks=hooks,
           checkpoint_path=checkpoint_path,
-          yield_single_examples=True):
+          yield_single_examples=yield_single_examples):
         yield result
     except Exception as e:  # pylint: disable=broad-except
       rendezvous.record_error('prediction_loop', e)
-- 
cgit v1.2.3


From 989819a78d59387aea728eb1807fb59f9d101411 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 12:14:01 -0700
Subject: Broaden dropped exception set in error handler in
 `transformer.Base.visit`.

Test with an example that breaks error reporting in said
`transformer.Base.visit` without this change.  (Admittedly, by
directly calling `visit` and committing a type error in the API.)

PiperOrigin-RevId: 205702348
---
 tensorflow/contrib/autograph/pyct/transformer.py   | 17 ++++-
 .../contrib/autograph/pyct/transformer_test.py     | 82 ++++++++++++++++++++++
 2 files changed, 98 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/autograph/pyct/transformer.py b/tensorflow/contrib/autograph/pyct/transformer.py
index bbdfefc50a..3e8906823e 100644
--- a/tensorflow/contrib/autograph/pyct/transformer.py
+++ b/tensorflow/contrib/autograph/pyct/transformer.py
@@ -398,10 +398,25 @@ class Base(gast.NodeTransformer):
     try:
       source, _ = compiler.ast_to_source(node)
       return source
-    except AssertionError:
+    # pylint: disable=broad-except
+    # This function is used for error reporting.  If an exception occurs here,
+    # it should be suppressed, in favor of emitting as informative a message
+    # about the original error as possible.
+    except Exception:
       return ''
 
   def visit(self, node):
+    if not isinstance(node, gast.AST):
+      # This is not that uncommon a mistake: various node bodies are lists, for
+      # example, posing a land mine for transformers that need to recursively
+      # call `visit`.  The error needs to be raised before the exception handler
+      # below is installed, because said handler will mess up if `node` is not,
+      # in fact, a node.
+      msg = (
+          'invalid value for "node": expected "ast.AST", got "{}"; to'
+          ' visit lists of nodes, use "visit_block" instead').format(type(node))
+      raise ValueError(msg)
+
     source_code = self.entity_info.source_code
     source_file = self.entity_info.source_file
     did_enter_function = False
diff --git a/tensorflow/contrib/autograph/pyct/transformer_test.py b/tensorflow/contrib/autograph/pyct/transformer_test.py
index 19b80b09ac..a37e922a1d 100644
--- a/tensorflow/contrib/autograph/pyct/transformer_test.py
+++ b/tensorflow/contrib/autograph/pyct/transformer_test.py
@@ -282,6 +282,88 @@ class TransformerTest(test.TestCase):
     self.assertTrue(isinstance(node.body[1].body[0], gast.Assign))
     self.assertTrue(isinstance(node.body[1].body[1], gast.Return))
 
+  def test_robust_error_on_list_visit(self):
+
+    class BrokenTransformer(transformer.Base):
+
+      def visit_If(self, node):
+        # This is broken because visit expects a single node, not a list, and
+        # the body of an if is a list.
+        # Importantly, the default error handling in visit also expects a single
+        # node.  Therefore, mistakes like this need to trigger a type error
+        # before the visit called here installs its error handler.
+        # That type error can then be caught by the enclosing call to visit,
+        # and correctly blame the If node.
+        self.visit(node.body)
+        return node
+
+    def test_function(x):
+      if x > 0:
+        return x
+
+    tr = BrokenTransformer(self._simple_source_info())
+
+    node, _ = parser.parse_entity(test_function)
+    with self.assertRaises(transformer.AutographParseError) as cm:
+      node = tr.visit(node)
+    obtained_message = str(cm.exception)
+    expected_message = r'expected "ast.AST", got "\<(type|class) \'list\'\>"'
+    self.assertRegexpMatches(obtained_message, expected_message)
+    # The exception should point at the if statement, not any place else.  Could
+    # also check the stack trace.
+    self.assertTrue(
+        'Occurred at node:\nIf' in obtained_message, obtained_message)
+    self.assertTrue(
+        'Occurred at node:\nFunctionDef' not in obtained_message,
+        obtained_message)
+    self.assertTrue(
+        'Occurred at node:\nReturn' not in obtained_message, obtained_message)
+
+  def test_robust_error_on_ast_corruption(self):
+    # A child class should not be able to be so broken that it causes the error
+    # handling in `transformer.Base` to raise an exception.  Why not?  Because
+    # then the original error location is dropped, and an error handler higher
+    # up in the call stack gives misleading information.
+
+    # Here we test that the error handling in `visit` completes, and blames the
+    # correct original exception, even if the AST gets corrupted.
+
+    class NotANode(object):
+      pass
+
+    class BrokenTransformer(transformer.Base):
+
+      def visit_If(self, node):
+        node.body = NotANode()
+        raise ValueError('I blew up')
+
+    def test_function(x):
+      if x > 0:
+        return x
+
+    tr = BrokenTransformer(self._simple_source_info())
+
+    node, _ = parser.parse_entity(test_function)
+    with self.assertRaises(transformer.AutographParseError) as cm:
+      node = tr.visit(node)
+    obtained_message = str(cm.exception)
+    # The message should reference the exception actually raised, not anything
+    # from the exception handler.
+    expected_substring = 'I blew up'
+    self.assertTrue(expected_substring in obtained_message, obtained_message)
+    # Expect the exception to have failed to parse the corrupted AST
+    self.assertTrue(
+        '' in obtained_message,
+        obtained_message)
+    # The exception should point at the if statement, not any place else.  Could
+    # also check the stack trace.
+    self.assertTrue(
+        'Occurred at node:\nIf' in obtained_message, obtained_message)
+    self.assertTrue(
+        'Occurred at node:\nFunctionDef' not in obtained_message,
+        obtained_message)
+    self.assertTrue(
+        'Occurred at node:\nReturn' not in obtained_message, obtained_message)
 
 if __name__ == '__main__':
   test.main()
-- 
cgit v1.2.3


From c26b95e707ed2304e2e50f51f6751c9b9cb87f1c Mon Sep 17 00:00:00 2001
From: Wesley Qian 
Date: Mon, 23 Jul 2018 12:15:03 -0700
Subject: Change partial reconstruction loss in cyclegan_loss to the standard
 abs_diff.

PiperOrigin-RevId: 205702476
---
 .../contrib/gan/python/losses/python/losses_impl.py      | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
index 1ba3a64167..d389748374 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
@@ -949,6 +949,11 @@ def cycle_consistency_loss(data_x,
   * loss = (loss_x2x + loss_y2y) / 2
   where `loss` is the final result.
 
+  For the L1-norm, we follow the original implementation:
+  https://github.com/junyanz/CycleGAN/blob/master/models/cycle_gan_model.lua
+  we use L1-norm of pixel-wise error normalized by data size such that
+  `cycle_loss_weight` can be specified independent of image size.
+
   See https://arxiv.org/abs/1703.10593 for more details.
 
   Args:
@@ -965,19 +970,12 @@ def cycle_consistency_loss(data_x,
     A scalar `Tensor` of cycle consistency loss.
   """
 
-  def _partial_cycle_consistency_loss(data, reconstructed_data):
-    # Following the original implementation
-    # https://github.com/junyanz/CycleGAN/blob/master/models/cycle_gan_model.lua
-    # use L1-norm of pixel-wise error normalized by data size so that
-    # `cycle_loss_weight` can be specified independent of image size.
-    return math_ops.reduce_mean(math_ops.abs(data - reconstructed_data))
-
   with ops.name_scope(
       scope,
       'cycle_consistency_loss',
       values=[data_x, reconstructed_data_x, data_y, reconstructed_data_y]):
-    loss_x2x = _partial_cycle_consistency_loss(data_x, reconstructed_data_x)
-    loss_y2y = _partial_cycle_consistency_loss(data_y, reconstructed_data_y)
+    loss_x2x = losses.absolute_difference(data_x, reconstructed_data_x)
+    loss_y2y = losses.absolute_difference(data_y, reconstructed_data_y)
     loss = (loss_x2x + loss_y2y) / 2.0
     if add_summaries:
       summary.scalar('cycle_consistency_loss_x2x', loss_x2x)
-- 
cgit v1.2.3


From 049fc23966eeef02a0945ddb80ae5f40592b90c1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 12:19:01 -0700
Subject: Extend random pool to work with arbitrarily nested tensor structures.

PiperOrigin-RevId: 205703156
---
 tensorflow/contrib/gan/BUILD                       |  3 ++
 .../features/python/random_tensor_pool_impl.py     | 37 ++++++++++------------
 .../features/python/random_tensor_pool_test.py     | 19 +++++++++++
 3 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 781e4ae4d7..7e6cb72485 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -257,12 +257,15 @@ py_library(
 py_test(
     name = "random_tensor_pool_test",
     srcs = ["python/features/python/random_tensor_pool_test.py"],
+    shard_count = 6,
     srcs_version = "PY2AND3",
     deps = [
         ":random_tensor_pool",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/contrib/gan/python/features/python/random_tensor_pool_impl.py b/tensorflow/contrib/gan/python/features/python/random_tensor_pool_impl.py
index 9e4ec59e70..ca2d724b49 100644
--- a/tensorflow/contrib/gan/python/features/python/random_tensor_pool_impl.py
+++ b/tensorflow/contrib/gan/python/features/python/random_tensor_pool_impl.py
@@ -36,16 +36,15 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.util import nest
 
 __all__ = [
     'tensor_pool',
 ]
 
 
-def _to_tuple(x):
-  if isinstance(x, (list, tuple)):
-    return tuple(x)
-  return (x,)
+def _to_list(x):
+  return [x] if isinstance(x, ops.Tensor) else list(x)
 
 
 def tensor_pool(input_values,
@@ -63,8 +62,8 @@ def tensor_pool(input_values,
   `pool_size` = 0 or `pooling_probability` = 0.
 
   Args:
-    input_values: A `Tensor`, or a list or tuple of `Tensor`s from which to read
-      values to be pooled.
+    input_values: An arbitrarily nested structure of `tf.Tensors`, from which to
+      read values to be pooled.
     pool_size: An integer specifying the maximum size of the pool. Defaults to
       50.
     pooling_probability: A float `Tensor` specifying the probability of getting
@@ -72,9 +71,10 @@ def tensor_pool(input_values,
     name: A string prefix for the name scope for all tensorflow ops.
 
   Returns:
-    A `Tensor`, or a list or tuple of `Tensor`s (according to the type ofx
-    `input_values`) which is with given probability either the `input_values` or
-    a randomly chosen sample that was previously inserted in the pool.
+    A nested structure of `Tensor` objects with the same structure as
+    `input_values`. With the given probability, the Tensor values are either the
+    same as in `input_values` or a randomly chosen sample that was previously
+    inserted in the pool.
 
   Raises:
     ValueError: If `pool_size` is negative.
@@ -86,11 +86,10 @@ def tensor_pool(input_values,
     return input_values
 
   original_input_values = input_values
-  input_values = _to_tuple(input_values)
+  input_values = nest.flatten(input_values)
 
-  with ops.name_scope(
-      '{}_pool_queue'.format(name),
-      values=input_values + (pooling_probability,)):
+  with ops.name_scope('{}_pool_queue'.format(name),
+                      values=input_values + [pooling_probability]):
     pool_queue = data_flow_ops.RandomShuffleQueue(
         capacity=pool_size,
         min_after_dequeue=0,
@@ -112,10 +111,10 @@ def tensor_pool(input_values,
     def _get_input_value_pooled():
       enqueue_op = pool_queue.enqueue(input_values)
       with ops.control_dependencies([enqueue_op]):
-        return tuple(array_ops.identity(v) for v in input_values)
+        return [array_ops.identity(v) for v in input_values]
 
     def _get_random_pool_value_and_enqueue_input():
-      dequeue_values = _to_tuple(pool_queue.dequeue())
+      dequeue_values = _to_list(pool_queue.dequeue())
       with ops.control_dependencies(dequeue_values):
         enqueue_op = pool_queue.enqueue(input_values)
         with ops.control_dependencies([enqueue_op]):
@@ -124,7 +123,7 @@ def tensor_pool(input_values,
           return control_flow_ops.cond(prob, lambda: dequeue_values,
                                        lambda: input_values)
 
-    output_values = _to_tuple(control_flow_ops.cond(
+    output_values = _to_list(control_flow_ops.cond(
         pool_queue.size() < pool_size, _get_input_value_pooled,
         _get_random_pool_value_and_enqueue_input))
 
@@ -132,8 +131,4 @@ def tensor_pool(input_values,
     for input_value, output_value in zip(input_values, output_values):
       output_value.set_shape(input_value.shape)
 
-  if isinstance(original_input_values, list):
-    return list(output_values)
-  elif isinstance(original_input_values, tuple):
-    return output_values
-  return output_values[0]
+  return nest.pack_sequence_as(original_input_values, output_values)
diff --git a/tensorflow/contrib/gan/python/features/python/random_tensor_pool_test.py b/tensorflow/contrib/gan/python/features/python/random_tensor_pool_test.py
index d8cf549cf7..08584dcd65 100644
--- a/tensorflow/contrib/gan/python/features/python/random_tensor_pool_test.py
+++ b/tensorflow/contrib/gan/python/features/python/random_tensor_pool_test.py
@@ -21,7 +21,9 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.contrib.gan.python.features.python.random_tensor_pool_impl import tensor_pool
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
@@ -111,6 +113,23 @@ class TensorPoolTest(test.TestCase):
         self.assertEqual(len(outs), len(input_values))
         self.assertEqual(outs[1] - outs[0], 1)
 
+  def test_pool_preserves_shape(self):
+    t = constant_op.constant(1)
+    input_values = [[t, t, t], (t, t), t]
+    output_values = tensor_pool(input_values, pool_size=5)
+    print('stuff: ', output_values)
+    # Overall shape.
+    self.assertIsInstance(output_values, list)
+    self.assertEqual(3, len(output_values))
+    # Shape of first element.
+    self.assertIsInstance(output_values[0], list)
+    self.assertEqual(3, len(output_values[0]))
+    # Shape of second element.
+    self.assertIsInstance(output_values[1], tuple)
+    self.assertEqual(2, len(output_values[1]))
+    # Shape of third element.
+    self.assertIsInstance(output_values[2], ops.Tensor)
+
 
 if __name__ == '__main__':
   test.main()
-- 
cgit v1.2.3


From 65b611d3cc67d0f12007ba0eb87e2b3d2a074ff3 Mon Sep 17 00:00:00 2001
From: Yifei Feng 
Date: Mon, 23 Jul 2018 12:19:47 -0700
Subject: Fix sanity. Adding future imports.

PiperOrigin-RevId: 205703315
---
 tensorflow/contrib/tpu/python/tpu/error_handling.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/contrib/tpu/python/tpu/error_handling.py b/tensorflow/contrib/tpu/python/tpu/error_handling.py
index 8d6d44b1a1..182cac6f0f 100644
--- a/tensorflow/contrib/tpu/python/tpu/error_handling.py
+++ b/tensorflow/contrib/tpu/python/tpu/error_handling.py
@@ -14,6 +14,10 @@
 # ===================================================================
 """ErrorRendezvous handler for collecting errors from multiple threads."""
 
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
 import contextlib
 import threading
 import time
-- 
cgit v1.2.3


From 54870b3de345a748dc5189f4c7dc759d6ffd6084 Mon Sep 17 00:00:00 2001
From: Dan Ringwalt 
Date: Mon, 23 Jul 2018 12:22:38 -0700
Subject: Use code blocks in the swap_ts and reroute_ts docstrings.

This makes the diagrams readable in the HTML documentation.

PiperOrigin-RevId: 205703761
---
 tensorflow/contrib/graph_editor/reroute.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/graph_editor/reroute.py b/tensorflow/contrib/graph_editor/reroute.py
index 95c02a64d4..d42e0c01f4 100644
--- a/tensorflow/contrib/graph_editor/reroute.py
+++ b/tensorflow/contrib/graph_editor/reroute.py
@@ -208,9 +208,9 @@ def _reroute_ts(ts0, ts1, mode, can_modify=None, cannot_modify=None):
 def swap_ts(ts0, ts1, can_modify=None, cannot_modify=None):
   """For each tensor's pair, swap the end of (t0,t1).
 
-  B0 B1     B0 B1
-  |  |    =>  X
-  A0 A1     A0 A1
+      B0 B1     B0 B1
+      |  |    =>  X
+      A0 A1     A0 A1
 
   Args:
     ts0: an object convertible to a list of `tf.Tensor`.
@@ -233,9 +233,9 @@ def swap_ts(ts0, ts1, can_modify=None, cannot_modify=None):
 def reroute_ts(ts0, ts1, can_modify=None, cannot_modify=None):
   """For each tensor's pair, replace the end of t1 by the end of t0.
 
-  B0 B1     B0 B1
-  |  |    => |/
-  A0 A1     A0 A1
+      B0 B1     B0 B1
+      |  |    => |/
+      A0 A1     A0 A1
 
   The end of the tensors in ts1 are left dangling.
 
-- 
cgit v1.2.3


From 4c9e4ba5d305fdedf7e0bbcd6fd6b84e29dfa5a2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 12:39:07 -0700
Subject: Fix ShapeUtil::CompatibleIgnoringElementType for opaque types

Previously we had an assymetric comparision when comparing an array type
with an opaque type returning false for array vs opaque while true for
opaque vs array.

PiperOrigin-RevId: 205706477
---
 tensorflow/compiler/xla/shape_util.cc      |  4 ++--
 tensorflow/compiler/xla/shape_util_test.cc | 11 +++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 6480148336..ec901af1e2 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -682,7 +682,7 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) {
                            CompatibleIgnoringElementType);
   } else {
     // Opaque, token, etc types are vacuously compatible.
-    return true;
+    return lhs.element_type() == rhs.element_type();
   }
 }
 
@@ -697,7 +697,7 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) {
                            CompatibleIgnoringFpPrecision);
   } else {
     // Opaque, token, etc types are vacuously compatible.
-    return true;
+    return lhs.element_type() == rhs.element_type();
   }
 }
 
diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc
index ed2d16c0e9..e5dd62ae9a 100644
--- a/tensorflow/compiler/xla/shape_util_test.cc
+++ b/tensorflow/compiler/xla/shape_util_test.cc
@@ -334,6 +334,17 @@ TEST(ShapeUtilTest, IncompatibleScalarVsTuple) {
   EXPECT_FALSE(ShapeUtil::CompatibleIgnoringFpPrecision(shape2, shape1));
 }
 
+TEST(ShapeUtilTest, OpaqueVsArray) {
+  Shape shape1 = ShapeUtil::MakeShape(F32, {5, 7});
+  Shape shape2 = ShapeUtil::MakeOpaqueShape();
+  EXPECT_FALSE(ShapeUtil::Compatible(shape1, shape2));
+  EXPECT_FALSE(ShapeUtil::Compatible(shape2, shape1));
+  EXPECT_FALSE(ShapeUtil::CompatibleIgnoringFpPrecision(shape1, shape2));
+  EXPECT_FALSE(ShapeUtil::CompatibleIgnoringFpPrecision(shape2, shape1));
+  EXPECT_FALSE(ShapeUtil::CompatibleIgnoringElementType(shape1, shape2));
+  EXPECT_FALSE(ShapeUtil::CompatibleIgnoringElementType(shape2, shape1));
+}
+
 TEST(ShapeUtilTest, CompareShapesWithPaddedDimensionsMismatch) {
   Shape shape1 = ShapeUtil::MakeShape(F32, {20, 30});
   shape1.mutable_layout()->add_padded_dimensions(10);
-- 
cgit v1.2.3


From f6548bffe577202381fa5893ad7aa452ae4e4931 Mon Sep 17 00:00:00 2001
From: Xuechen Li 
Date: Mon, 23 Jul 2018 12:46:01 -0700
Subject: Add main script training sampler on 2D energy landscapes.

PiperOrigin-RevId: 205707483
---
 .../contrib/eager/python/examples/l2hmc/l2hmc.py   | 143 +++++++------
 .../eager/python/examples/l2hmc/l2hmc_test.py      |  97 +++------
 .../contrib/eager/python/examples/l2hmc/main.py    | 235 +++++++++++++++++++++
 3 files changed, 346 insertions(+), 129 deletions(-)
 create mode 100644 tensorflow/contrib/eager/python/examples/l2hmc/main.py

diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py b/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py
index 275aee5130..14b8324e48 100644
--- a/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py
+++ b/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc.py
@@ -32,20 +32,28 @@ from tensorflow.contrib.eager.python.examples.l2hmc import neural_nets
 
 
 class Dynamics(tf.keras.Model):
-  """Dynamics engine of naive L2HMC sampler.
-
-  Args:
-    x_dim: dimensionality of observed data
-    loglikelihood_fn: log-likelihood function of conditional probability
-    n_steps: number of leapfrog steps within each transition
-    eps: initial value learnable scale of step size
-  """
-
-  def __init__(self, x_dim, loglikelihood_fn, n_steps=25, eps=.1):
+  """Dynamics engine of naive L2HMC sampler."""
+
+  def __init__(self,
+               x_dim,
+               minus_loglikelihood_fn,
+               n_steps=25,
+               eps=.1,
+               np_seed=1):
+    """Initialization.
+
+    Args:
+      x_dim: dimensionality of observed data
+      minus_loglikelihood_fn: log-likelihood function of conditional probability
+      n_steps: number of leapfrog steps within each transition
+      eps: initial value learnable scale of step size
+      np_seed: Random seed for numpy; used to control sampled masks.
+    """
     super(Dynamics, self).__init__()
 
+    npr.seed(np_seed)
     self.x_dim = x_dim
-    self.potential = loglikelihood_fn
+    self.potential = minus_loglikelihood_fn
     self.n_steps = n_steps
 
     self._construct_time()
@@ -68,8 +76,8 @@ class Dynamics(tf.keras.Model):
         position, forward=False)
 
     # Decide direction uniformly
-    forward_mask = tf.cast(
-        tf.random_uniform(shape=[tf.shape(position)[0]]) > .5, tf.float32)
+    batch_size = tf.shape(position)[0]
+    forward_mask = tf.cast(tf.random_uniform((batch_size,)) > .5, tf.float32)
     backward_mask = 1. - forward_mask
 
     # Obtain proposed states
@@ -108,7 +116,6 @@ class Dynamics(tf.keras.Model):
       position_post, momentum_post, logdet = lf_fn(position_post, momentum_post,
                                                    i)
       sumlogdet += logdet
-
     accept_prob = self._compute_accept_prob(position, momentum, position_post,
                                             momentum_post, sumlogdet)
 
@@ -125,17 +132,17 @@ class Dynamics(tf.keras.Model):
     sumlogdet += logdet
 
     position, logdet = self._update_position_forward(position, momentum, t,
-                                                     mask)
+                                                     mask, mask_inv)
     sumlogdet += logdet
 
     position, logdet = self._update_position_forward(position, momentum, t,
-                                                     mask_inv)
+                                                     mask_inv, mask)
     sumlogdet += logdet
 
     momentum, logdet = self._update_momentum_forward(position, momentum, t)
     sumlogdet += logdet
 
-    return position, momentum, tf.reduce_sum(sumlogdet, axis=1)
+    return position, momentum, sumlogdet
 
   def _backward_lf(self, position, momentum, i):
     """One backward augmented leapfrog step. See Appendix A in paper."""
@@ -149,17 +156,17 @@ class Dynamics(tf.keras.Model):
     sumlogdet += logdet
 
     position, logdet = self._update_position_backward(position, momentum, t,
-                                                      mask)
+                                                      mask_inv, mask)
     sumlogdet += logdet
 
     position, logdet = self._update_position_backward(position, momentum, t,
-                                                      mask_inv)
+                                                      mask, mask_inv)
     sumlogdet += logdet
 
     momentum, logdet = self._update_momentum_backward(position, momentum, t)
     sumlogdet += logdet
 
-    return position, momentum, tf.reduce_sum(sumlogdet, axis=1)
+    return position, momentum, sumlogdet
 
   def _update_momentum_forward(self, position, momentum, t):
     """Update v in the forward leapfrog step."""
@@ -172,12 +179,11 @@ class Dynamics(tf.keras.Model):
         momentum * tf.exp(scale) -
         .5 * self.eps * (tf.exp(transformed) * grad - translation))
 
-    return momentum, scale
+    return momentum, tf.reduce_sum(scale, axis=1)
 
-  def _update_position_forward(self, position, momentum, t, mask):
+  def _update_position_forward(self, position, momentum, t, mask, mask_inv):
     """Update x in the forward leapfrog step."""
 
-    mask_inv = 1. - mask
     scale, translation, transformed = self.position_fn(
         [momentum, mask * position, t])
     scale *= self.eps
@@ -186,8 +192,7 @@ class Dynamics(tf.keras.Model):
         mask * position +
         mask_inv * (position * tf.exp(scale) + self.eps *
                     (tf.exp(transformed) * momentum + translation)))
-
-    return position, mask_inv * scale
+    return position, tf.reduce_sum(mask_inv * scale, axis=1)
 
   def _update_momentum_backward(self, position, momentum, t):
     """Update v in the backward leapfrog step. Inverting the forward update."""
@@ -200,21 +205,20 @@ class Dynamics(tf.keras.Model):
         tf.exp(scale) * (momentum + .5 * self.eps *
                          (tf.exp(transformed) * grad - translation)))
 
-    return momentum, scale
+    return momentum, tf.reduce_sum(scale, axis=1)
 
-  def _update_position_backward(self, position, momentum, t, mask):
+  def _update_position_backward(self, position, momentum, t, mask, mask_inv):
     """Update x in the backward leapfrog step. Inverting the forward update."""
 
-    mask_inv = 1. - mask
     scale, translation, transformed = self.position_fn(
-        [momentum, mask_inv * position, t])
+        [momentum, mask * position, t])
     scale *= -self.eps
     transformed *= self.eps
     position = (
-        mask_inv * position + mask * tf.exp(scale) *
-        (position - self.eps * tf.exp(transformed) * momentum + translation))
+        mask * position + mask_inv * tf.exp(scale) *
+        (position - self.eps * (tf.exp(transformed) * momentum + translation)))
 
-    return position, mask * scale
+    return position, tf.reduce_sum(mask_inv * scale, axis=1)
 
   def _compute_accept_prob(self, position, momentum, position_post,
                            momentum_post, sumlogdet):
@@ -222,8 +226,10 @@ class Dynamics(tf.keras.Model):
 
     old_hamil = self.hamiltonian(position, momentum)
     new_hamil = self.hamiltonian(position_post, momentum_post)
+    prob = tf.exp(tf.minimum(old_hamil - new_hamil + sumlogdet, 0.))
 
-    return tf.exp(tf.minimum(old_hamil - new_hamil + sumlogdet, 0.))
+    # Ensure numerical stability as well as correct gradients
+    return tf.where(tf.is_finite(prob), prob, tf.zeros_like(prob))
 
   def _construct_time(self):
     """Convert leapfrog step index into sinusoidal time."""
@@ -248,6 +254,8 @@ class Dynamics(tf.keras.Model):
 
     self.masks = []
     for _ in range(self.n_steps):
+      # Need to use npr here because tf would generated different random
+      # values across different `sess.run`
       idx = npr.permutation(np.arange(self.x_dim))[:self.x_dim // 2]
       mask = np.zeros((self.x_dim,))
       mask[idx] = 1.
@@ -273,19 +281,15 @@ class Dynamics(tf.keras.Model):
   def grad_potential(self, position, check_numerics=True):
     """Get gradient of potential function at current location."""
 
-    if not tf.executing_eagerly():
-      # TODO(lxuechen): Change this to tfe.gradients_function when it works
-      grad = tf.gradients(self.potential(position), position)[0]
-    else:
+    if tf.executing_eagerly():
       grad = tfe.gradients_function(self.potential)(position)[0]
-
-    if check_numerics:
-      return tf.check_numerics(grad, message="gradient of potential")
+    else:
+      grad = tf.gradients(self.potential(position), position)[0]
 
     return grad
 
 
-# Examples of unnormalized log density/probabilities
+# Examples of unnormalized log densities
 def get_scg_energy_fn():
   """Get energy function for 2d strongly correlated Gaussian."""
 
@@ -295,32 +299,53 @@ def get_scg_energy_fn():
   sigma_inv = tf.matrix_inverse(sigma)
 
   def energy(x):
-    """Unnormalized log density/energy of 2d strongly correlated Gaussian."""
+    """Unnormalized minus log density of 2d strongly correlated Gaussian."""
 
     xmmu = x - mu
     return .5 * tf.diag_part(
         tf.matmul(tf.matmul(xmmu, sigma_inv), tf.transpose(xmmu)))
 
-  return energy
+  return energy, mu, sigma
 
 
-def get_multivariate_gaussian_energy_fn(x_dim=2):
-  """Get energy function for 2d strongly correlated Gaussian."""
-
-  mu = tf.random_normal(shape=[x_dim])
-  # Lower triangularize and positive diagonal
-  l = tf.sigmoid(
-      tf.matrix_band_part(tf.random_normal(shape=[x_dim, x_dim]), -1, 0))
-  # Exploit Cholesky decomposition
-  sigma = tf.matmul(l, tf.transpose(l))
-  sigma *= 100.  # Small covariance causes extreme numerical instability
-  sigma_inv = tf.matrix_inverse(sigma)
+def get_rw_energy_fn():
+  """Get energy function for rough well distribution."""
+  # For small eta, the density underlying the rough-well energy is very close to
+  # a unit Gaussian; however, the gradient is greatly affected by the small
+  # cosine perturbations
+  eta = 1e-2
+  mu = tf.constant([0., 0.])
+  sigma = tf.constant([[1., 0.], [0., 1.]])
 
   def energy(x):
-    """Unnormalized log density/energy of 2d strongly correlated Gaussian."""
+    ip = tf.reduce_sum(x**2., axis=1)
+    return .5 * ip + eta * tf.reduce_sum(tf.cos(x / eta), axis=1)
 
-    xmmu = x - mu
-    return .5 * tf.diag_part(
-        tf.matmul(tf.matmul(xmmu, sigma_inv), tf.transpose(xmmu)))
+  return energy, mu, sigma
+
+
+# Loss function
+def compute_loss(dynamics, x, scale=.1, eps=1e-4):
+  """Compute loss defined in equation (8)."""
+
+  z = tf.random_normal(tf.shape(x))  # Auxiliary variable
+  x_, _, x_accept_prob, x_out = dynamics.apply_transition(x)
+  z_, _, z_accept_prob, _ = dynamics.apply_transition(z)
+
+  # Add eps for numerical stability; following released impl
+  x_loss = tf.reduce_sum((x - x_)**2, axis=1) * x_accept_prob + eps
+  z_loss = tf.reduce_sum((z - z_)**2, axis=1) * z_accept_prob + eps
+
+  loss = tf.reduce_mean(
+      (1. / x_loss + 1. / z_loss) * scale - (x_loss + z_loss) / scale, axis=0)
+
+  return loss, x_out, x_accept_prob
+
+
+def loss_and_grads(dynamics, x, loss_fn=compute_loss):
+  """Obtain loss value and gradients."""
+  with tf.GradientTape() as tape:
+    loss_val, out, accept_prob = loss_fn(dynamics, x)
+  grads = tape.gradient(loss_val, dynamics.trainable_variables)
 
-  return energy
+  return loss_val, grads, out, accept_prob
diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc_test.py b/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc_test.py
index e33b4cae4c..9557479885 100644
--- a/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc_test.py
+++ b/tensorflow/contrib/eager/python/examples/l2hmc/l2hmc_test.py
@@ -37,63 +37,37 @@ def get_default_hparams():
       n_warmup_iters=3)
 
 
-# Relevant functions for benchmarking
-def compute_loss(dynamics, x, scale=.1, eps=1e-4):
-  """Compute loss defined in equation (8)."""
-
-  z = tf.random_normal(tf.shape(x))
-  x_, _, x_accept_prob, x_out = dynamics.apply_transition(x)
-  z_, _, z_accept_prob, _ = dynamics.apply_transition(z)
-
-  # Add eps for numerical stability; following released impl
-  x_loss = tf.reduce_sum((x - x_)**2, axis=1) * x_accept_prob + eps
-  z_loss = tf.reduce_sum((z - z_)**2, axis=1) * z_accept_prob + eps
-
-  loss = tf.reduce_mean(
-      (1. / x_loss + 1. / z_loss) * scale - (x_loss + z_loss) / scale, axis=0)
-
-  return loss, x_out
-
-
-def loss_and_grads(dynamics, x, loss_fn=compute_loss):
-  """Obtain loss value and gradients."""
-
-  with tf.GradientTape() as tape:
-    loss_val, x_out = loss_fn(dynamics, x)
-  grads = tape.gradient(loss_val, dynamics.variables)
-
-  return loss_val, grads, x_out
-
-
-def warmup(dynamics, optimizer, n_iters=1, n_samples=200, loss_fn=compute_loss):
+def warmup(dynamics,
+           optimizer,
+           n_iters=1,
+           n_samples=200,
+           loss_fn=l2hmc.compute_loss):
   """Warmup optimization to reduce overhead."""
 
   samples = tf.random_normal(
       shape=[n_samples, dynamics.x_dim], dtype=tf.float32)
 
   for _ in range(n_iters):
-    _, grads, samples = loss_and_grads(dynamics, samples, loss_fn=loss_fn)
+    _, grads, samples, _ = l2hmc.loss_and_grads(
+        dynamics, samples, loss_fn=loss_fn)
     optimizer.apply_gradients(zip(grads, dynamics.variables))
 
 
 def fit(dynamics,
         samples,
         optimizer,
-        loss_fn=compute_loss,
+        loss_fn=l2hmc.compute_loss,
         n_iters=5000,
         verbose=True,
-        logdir=None,
-        decay_lr=True):
+        logdir=None):
   """Fit L2HMC sampler with given log-likelihood function."""
 
   if logdir:
     summary_writer = tf.contrib.summary.create_file_writer(logdir)
 
   for i in range(n_iters):
-    loss, grads, samples = loss_and_grads(dynamics, samples, loss_fn=loss_fn)
-    # TODO(lxuechen): Proper learning rate decay
-    if decay_lr:
-      grads = [grad * .96**(i // 1000) for grad in grads]
+    loss, grads, samples, _ = l2hmc.loss_and_grads(
+        dynamics, samples, loss_fn=loss_fn)
     optimizer.apply_gradients(zip(grads, dynamics.variables))
     if verbose:
       print("Iteration %d: loss %.4f" % (i, loss))
@@ -112,9 +86,10 @@ class L2hmcTest(tf.test.TestCase):
 
     # Eager mode testing
     hparams = get_default_hparams()
+    energy_fn, _, _ = l2hmc.get_scg_energy_fn()
     dynamics = l2hmc.Dynamics(
         x_dim=hparams.x_dim,
-        loglikelihood_fn=l2hmc.get_scg_energy_fn(),
+        minus_loglikelihood_fn=energy_fn,
         n_steps=hparams.n_steps,
         eps=hparams.eps)
     samples = tf.random_normal(shape=[hparams.n_samples, hparams.x_dim])
@@ -127,9 +102,10 @@ class L2hmcTest(tf.test.TestCase):
 
     # Graph mode testing
     with tf.Graph().as_default():
+      energy_fn, _, _ = l2hmc.get_scg_energy_fn()
       dynamics = l2hmc.Dynamics(
           x_dim=hparams.x_dim,
-          loglikelihood_fn=l2hmc.get_scg_energy_fn(),
+          minus_loglikelihood_fn=energy_fn,
           n_steps=hparams.n_steps,
           eps=hparams.eps)
       x = tf.placeholder(tf.float32, shape=[None, hparams.x_dim])
@@ -150,32 +126,20 @@ class L2hmcTest(tf.test.TestCase):
 class L2hmcBenchmark(tf.test.Benchmark):
   """Eager and graph benchmarks for l2hmc."""
 
-  def _get_energy_fn(self):
-    """Get specific energy function according to FLAGS."""
-
-    if FLAGS.energy_fn == "scg":
-      energy_fn = l2hmc.get_scg_energy_fn()
-    elif FLAGS.energy_fn == "multivariate_gaussian":
-      energy_fn = l2hmc.get_multivariate_gaussian_energy_fn(x_dim=FLAGS.x_dim)
-    else:
-      raise ValueError("No such energy function %s" % FLAGS.energy_fn)
-
-    return energy_fn
-
   def benchmark_graph(self):
     """Benchmark Graph performance."""
 
     hparams = get_default_hparams()
     tf.reset_default_graph()
     with tf.Graph().as_default():
-      energy_fn = self._get_energy_fn()
+      energy_fn, _, _ = l2hmc.get_scg_energy_fn()
       dynamics = l2hmc.Dynamics(
           x_dim=hparams.x_dim,
-          loglikelihood_fn=energy_fn,
+          minus_loglikelihood_fn=energy_fn,
           n_steps=hparams.n_steps,
           eps=hparams.eps)
       x = tf.placeholder(tf.float32, shape=[None, hparams.x_dim])
-      loss, x_out = compute_loss(dynamics, x)
+      loss, x_out, _ = l2hmc.compute_loss(dynamics, x)
 
       global_step = tf.Variable(0., name="global_step", trainable=False)
       learning_rate = tf.train.exponential_decay(
@@ -183,7 +147,11 @@ class L2hmcBenchmark(tf.test.Benchmark):
       optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
       train_op = optimizer.minimize(loss, global_step=global_step)
 
-      with tf.Session() as sess:
+      # Single thread; fairer comparison against eager
+      session_conf = tf.ConfigProto(
+          intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
+
+      with tf.Session(config=session_conf) as sess:
         sess.run(tf.global_variables_initializer())
 
         # Warmup to reduce initialization effect when timing
@@ -218,14 +186,14 @@ class L2hmcBenchmark(tf.test.Benchmark):
     """Benchmark Eager performance."""
 
     hparams = get_default_hparams()
-    energy_fn = self._get_energy_fn()
+    energy_fn, _, _ = l2hmc.get_scg_energy_fn()
     dynamics = l2hmc.Dynamics(
         x_dim=hparams.x_dim,
-        loglikelihood_fn=energy_fn,
+        minus_loglikelihood_fn=energy_fn,
         n_steps=hparams.n_steps,
         eps=hparams.eps)
     optimizer = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate)
-    loss_fn = tfe.defun(compute_loss) if defun else compute_loss
+    loss_fn = tfe.defun(l2hmc.compute_loss) if defun else l2hmc.compute_loss
 
     # Warmup to reduce initialization effect when timing
     warmup(dynamics, optimizer, n_iters=hparams.n_warmup_iters, loss_fn=loss_fn)
@@ -234,12 +202,7 @@ class L2hmcBenchmark(tf.test.Benchmark):
     samples = tf.random_normal(
         shape=[hparams.n_samples, hparams.x_dim], dtype=tf.float32)
     start_time = time.time()
-    fit(dynamics,
-        samples,
-        optimizer,
-        loss_fn=loss_fn,
-        n_iters=hparams.n_iters,
-        decay_lr=True)
+    fit(dynamics, samples, optimizer, loss_fn=loss_fn, n_iters=hparams.n_iters)
     wall_time = time.time() - start_time
     examples_per_sec = hparams.n_samples / wall_time
 
@@ -251,14 +214,8 @@ class L2hmcBenchmark(tf.test.Benchmark):
         wall_time=wall_time)
 
     del dynamics
-    del loss_fn
 
 
 if __name__ == "__main__":
-  tf.flags.DEFINE_string("energy_fn", "scg",
-                         ("The energy function/unnormalized log-probability. "
-                          "Either be `scg` or `multivariate_gaussian`"))
-  tf.flags.DEFINE_integer("x_dim", 2, "Dimensionality of observation space.")
-  FLAGS = tf.flags.FLAGS
   tf.enable_eager_execution()
   tf.test.main()
diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/main.py b/tensorflow/contrib/eager/python/examples/l2hmc/main.py
new file mode 100644
index 0000000000..45e1f98429
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/l2hmc/main.py
@@ -0,0 +1,235 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""L2HMC on simple Gaussian mixture model with TensorFlow eager."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+
+from absl import flags
+import numpy as np
+import tensorflow as tf
+from tensorflow.contrib.eager.python.examples.l2hmc import l2hmc
+try:
+  import matplotlib.pyplot as plt  # pylint: disable=g-import-not-at-top
+  HAS_MATPLOTLIB = True
+except ImportError:
+  HAS_MATPLOTLIB = False
+tfe = tf.contrib.eager
+
+
+def main(_):
+  tf.enable_eager_execution()
+  global_step = tf.train.get_or_create_global_step()
+  global_step.assign(1)
+
+  energy_fn, mean, covar = {
+      "scg": l2hmc.get_scg_energy_fn(),
+      "rw": l2hmc.get_rw_energy_fn()
+  }[FLAGS.energy_fn]
+
+  x_dim = 2
+  train_iters = 5000
+  eval_iters = 2000
+  eps = 0.1
+  n_steps = 10  # Chain length
+  n_samples = 200
+  record_loss_every = 100
+
+  dynamics = l2hmc.Dynamics(
+      x_dim=x_dim, minus_loglikelihood_fn=energy_fn, n_steps=n_steps, eps=eps)
+  learning_rate = tf.train.exponential_decay(
+      1e-3, global_step, 1000, 0.96, staircase=True)
+  optimizer = tf.train.AdamOptimizer(learning_rate)
+  checkpointer = tf.train.Checkpoint(
+      optimizer=optimizer, dynamics=dynamics, global_step=global_step)
+
+  if FLAGS.train_dir:
+    summary_writer = tf.contrib.summary.create_file_writer(FLAGS.train_dir)
+    if FLAGS.restore:
+      latest_path = tf.train.latest_checkpoint(FLAGS.train_dir)
+      checkpointer.restore(latest_path)
+      print("Restored latest checkpoint at path:\"{}\" ".format(latest_path))
+      sys.stdout.flush()
+
+  if not FLAGS.restore:
+    # Training
+    if FLAGS.use_defun:
+      # Use `tfe.deun` to boost performance when there are lots of small ops
+      loss_fn = tfe.defun(l2hmc.compute_loss)
+    else:
+      loss_fn = l2hmc.compute_loss
+
+    samples = tf.random_normal(shape=[n_samples, x_dim])
+    for i in range(1, train_iters + 1):
+      loss, samples, accept_prob = train_one_iter(
+          dynamics,
+          samples,
+          optimizer,
+          loss_fn=loss_fn,
+          global_step=global_step)
+
+      if i % record_loss_every == 0:
+        print("Iteration {}, loss {:.4f}, x_accept_prob {:.4f}".format(
+            i, loss.numpy(),
+            accept_prob.numpy().mean()))
+        if FLAGS.train_dir:
+          with summary_writer.as_default():
+            with tf.contrib.summary.always_record_summaries():
+              tf.contrib.summary.scalar("Training loss", loss, step=global_step)
+    print("Training complete.")
+    sys.stdout.flush()
+
+    if FLAGS.train_dir:
+      saved_path = checkpointer.save(
+          file_prefix=os.path.join(FLAGS.train_dir, "ckpt"))
+      print("Saved checkpoint at path: \"{}\" ".format(saved_path))
+      sys.stdout.flush()
+
+  # Evaluation
+  if FLAGS.use_defun:
+    # Use tfe.deun to boost performance when there are lots of small ops
+    apply_transition = tfe.defun(dynamics.apply_transition)
+  else:
+    apply_transition = dynamics.apply_transition
+
+  samples = tf.random_normal(shape=[n_samples, x_dim])
+  samples_history = []
+  for i in range(eval_iters):
+    samples_history.append(samples.numpy())
+    _, _, _, samples = apply_transition(samples)
+  samples_history = np.array(samples_history)
+  print("Sampling complete.")
+  sys.stdout.flush()
+
+  # Mean and covariance of target distribution
+  mean = mean.numpy()
+  covar = covar.numpy()
+  ac_spectrum = compute_ac_spectrum(samples_history, mean, covar)
+  print("First 25 entries of the auto-correlation spectrum: {}".format(
+      ac_spectrum[:25]))
+  ess = compute_ess(ac_spectrum)
+  print("Effective sample size per Metropolis-Hastings step: {}".format(ess))
+  sys.stdout.flush()
+
+  if FLAGS.train_dir:
+    # Plot autocorrelation spectrum in tensorboard
+    plot_step = tfe.Variable(1, trainable=False, dtype=tf.int64)
+
+    for ac in ac_spectrum:
+      with summary_writer.as_default():
+        with tf.contrib.summary.always_record_summaries():
+          tf.contrib.summary.scalar("Autocorrelation", ac, step=plot_step)
+      plot_step.assign(plot_step + n_steps)
+
+    if HAS_MATPLOTLIB:
+      # Choose a single chain and plot the trajectory
+      single_chain = samples_history[:, 0, :]
+      xs = single_chain[:100, 0]
+      ys = single_chain[:100, 1]
+      plt.figure()
+      plt.plot(xs, ys, color="orange", marker="o", alpha=0.6)  # Trained chain
+      plt.savefig(os.path.join(FLAGS.train_dir, "single_chain.png"))
+
+
+def train_one_iter(dynamics,
+                   x,
+                   optimizer,
+                   loss_fn=l2hmc.compute_loss,
+                   global_step=None):
+  """Train the sampler for one iteration."""
+  loss, grads, out, accept_prob = l2hmc.loss_and_grads(
+      dynamics, x, loss_fn=loss_fn)
+  optimizer.apply_gradients(
+      zip(grads, dynamics.trainable_variables), global_step=global_step)
+
+  return loss, out, accept_prob
+
+
+def compute_ac_spectrum(samples_history, target_mean, target_covar):
+  """Compute autocorrelation spectrum.
+
+  Follows equation 15 from the L2HMC paper.
+
+  Args:
+    samples_history: Numpy array of shape [T, B, D], where T is the total
+        number of time steps, B is the batch size, and D is the dimensionality
+        of sample space.
+    target_mean: 1D Numpy array of the mean of target(true) distribution.
+    target_covar: 2D Numpy array representing a symmetric matrix for variance.
+  Returns:
+    Autocorrelation spectrum, Numpy array of shape [T-1].
+  """
+
+  # Using numpy here since eager is a bit slow due to the loop
+  time_steps = samples_history.shape[0]
+  trace = np.trace(target_covar)
+
+  rhos = []
+  for t in range(time_steps - 1):
+    rho_t = 0.
+    for tau in range(time_steps - t):
+      v_tau = samples_history[tau, :, :] - target_mean
+      v_tau_plus_t = samples_history[tau + t, :, :] - target_mean
+      # Take dot product over observation dims and take mean over batch dims
+      rho_t += np.mean(np.sum(v_tau * v_tau_plus_t, axis=1))
+
+    rho_t /= trace * (time_steps - t)
+    rhos.append(rho_t)
+
+  return np.array(rhos)
+
+
+def compute_ess(ac_spectrum):
+  """Compute the effective sample size based on autocorrelation spectrum.
+
+  This follows equation 16 from the L2HMC paper.
+
+  Args:
+    ac_spectrum: Autocorrelation spectrum
+  Returns:
+    The effective sample size
+  """
+  # Cutoff from the first value less than 0.05
+  cutoff = np.argmax(ac_spectrum[1:] < .05)
+  if cutoff == 0:
+    cutoff = len(ac_spectrum)
+  ess = 1. / (1. + 2. * np.sum(ac_spectrum[1:cutoff]))
+  return ess
+
+
+if __name__ == "__main__":
+  flags.DEFINE_string(
+      "train_dir",
+      default=None,
+      help="[Optional] Directory to store the training information")
+  flags.DEFINE_boolean(
+      "restore",
+      default=False,
+      help="[Optional] Restore the latest checkpoint from `train_dir` if True")
+  flags.DEFINE_boolean(
+      "use_defun",
+      default=False,
+      help="[Optional] Use `tfe.defun` to boost performance")
+  flags.DEFINE_string(
+      "energy_fn",
+      default="scg",
+      help="[Optional] The energy function used for experimentation"
+      "Other options include `rw`")
+  FLAGS = flags.FLAGS
+  tf.app.run(main)
-- 
cgit v1.2.3


From bd515dc93105a8f66437a2c2c70847eaa689b2ef Mon Sep 17 00:00:00 2001
From: Dan Moldovan 
Date: Mon, 23 Jul 2018 12:57:56 -0700
Subject: Fix file paths.

PiperOrigin-RevId: 205709394
---
 .../examples/notebooks/dev_summit_2018_demo.ipynb  | 24 +++++++++-------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb
index 86e38c3490..a3109fa5db 100644
--- a/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb
+++ b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb
@@ -130,7 +130,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 4,
+      "execution_count": 0,
       "metadata": {
         "colab": {
           "autoexec": {
@@ -201,7 +201,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 5,
+      "execution_count": 0,
       "metadata": {
         "colab": {
           "autoexec": {
@@ -275,7 +275,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 6,
+      "execution_count": 0,
       "metadata": {
         "colab": {
           "autoexec": {
@@ -362,7 +362,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 7,
+      "execution_count": 0,
       "metadata": {
         "colab": {
           "autoexec": {
@@ -467,7 +467,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 9,
+      "execution_count": 0,
       "metadata": {
         "colab": {
           "autoexec": {
@@ -531,7 +531,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 10,
+      "execution_count": 0,
       "metadata": {
         "cellView": "code",
         "colab": {
@@ -896,7 +896,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 17,
+      "execution_count": 0,
       "metadata": {
         "colab": {
           "autoexec": {
@@ -1070,8 +1070,8 @@
         "  return dataset\n",
         "\n",
         "\n",
-        "train_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/train.csv\"\n",
-        "test_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/test.csv\"\n",
+        "train_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/archive/extras/colorbot/data/train.csv\"\n",
+        "test_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/archive/extras/colorbot/data/test.csv\"\n",
         "data_dir = \"tmp/rnn/data\""
       ]
     },
@@ -1304,7 +1304,7 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 22,
+      "execution_count": 0,
       "metadata": {
         "colab": {
           "autoexec": {
@@ -1905,10 +1905,6 @@
     "colab": {
       "collapsed_sections": [],
       "default_view": {},
-      "last_runtime": {
-        "build_target": "",
-        "kind": "local"
-      },
       "name": "Dev Summit 2018 - Autograph",
       "provenance": [
         {
-- 
cgit v1.2.3


From 0720cced3baa17a5930f5d965cfd9b693954bc04 Mon Sep 17 00:00:00 2001
From: Timon Van Overveldt 
Date: Mon, 23 Jul 2018 13:03:44 -0700
Subject: Add explicit "-lz" linkopt for Android selective registration
 targets.

tensorflow/core/lib/io/zlib_inputstream.cc uses zlib, and hence, if a binary is compiled
that uses any of the symbols in zlib_inputstream.cc, then the binary needs zlib's
symbols to be available. One way to do that is to specify "-lz" as a linkopt, making the
binary use the system's zlib library.

PiperOrigin-RevId: 205710390
---
 tensorflow/core/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 17e6ccda14..b6a990ac7d 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1655,6 +1655,7 @@ cc_library(
     copts = tf_copts(android_optimization_level_override = None) + [
         "-DSUPPORT_SELECTIVE_REGISTRATION",
     ],
+    linkopts = if_android(["-lz"]),
     tags = [
         "manual",
         "notap",
@@ -1678,6 +1679,7 @@ cc_library(
     copts = tf_copts(android_optimization_level_override = None) + tf_opts_nortti_if_android() + [
         "-DSUPPORT_SELECTIVE_REGISTRATION",
     ],
+    linkopts = if_android(["-lz"]),
     tags = [
         "manual",
         "notap",
-- 
cgit v1.2.3


From 31c3d55ec5b509142df00268611438ff52f3794a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 13:16:37 -0700
Subject: Internal change

PiperOrigin-RevId: 205712557
---
 tensorflow/core/common_runtime/direct_session.cc   |  25 +++-
 .../core/common_runtime/direct_session_test.cc     | 139 ++++++++++++++++++++-
 tensorflow/core/protobuf/config.proto              |   5 +
 3 files changed, 161 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 4c670820be..44291b0b20 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -235,7 +235,11 @@ void DirectSession::SchedClosure(thread::ThreadPool* pool,
   // safe given the reasoning above.
   c();
 #else
-  pool->Schedule(std::move(c));
+  if (pool != nullptr) {
+    pool->Schedule(std::move(c));
+  } else {
+    c();
+  }
 #endif  // __ANDROID__
 }
 
@@ -522,8 +526,9 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
     }
   }
 
-  if (run_options.inter_op_thread_pool() < 0 ||
-      run_options.inter_op_thread_pool() >= thread_pools_.size()) {
+  if (run_options.inter_op_thread_pool() < -1 ||
+      run_options.inter_op_thread_pool() >=
+          static_cast(thread_pools_.size())) {
     run_state.executors_done.Notify();
     delete barrier;
     return errors::InvalidArgument("Invalid inter_op_thread_pool: ",
@@ -548,7 +553,19 @@ Status DirectSession::RunInternal(int64 step_id, const RunOptions& run_options,
   }
 
   thread::ThreadPool* pool =
-      thread_pools_[run_options.inter_op_thread_pool()].first;
+      run_options.inter_op_thread_pool() >= 0
+          ? thread_pools_[run_options.inter_op_thread_pool()].first
+          : nullptr;
+
+  if (pool == nullptr) {
+    // We allow using the caller thread only when having a single executor
+    // specified.
+    if (executors_and_keys->items.size() > 1) {
+      pool = thread_pools_[0].first;
+    } else {
+      VLOG(1) << "Executing Session::Run() synchronously!";
+    }
+  }
 
   Executor::Args::Runner default_runner = [this,
                                            pool](Executor::Args::Closure c) {
diff --git a/tensorflow/core/common_runtime/direct_session_test.cc b/tensorflow/core/common_runtime/direct_session_test.cc
index 142d613129..4b51b20bb1 100644
--- a/tensorflow/core/common_runtime/direct_session_test.cc
+++ b/tensorflow/core/common_runtime/direct_session_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 
@@ -896,6 +897,125 @@ TEST(DirectSessionTest, FetchMultipleTimes) {
   }
 }
 
+TEST(DirectSessionTest, MultipleFeedTestSomeSyncRun) {
+  GraphDef def;
+  Graph g(OpRegistry::Global());
+  RunOptions run_options;
+  run_options.set_inter_op_thread_pool(-1);
+
+  Tensor first_value(DT_FLOAT, TensorShape({}));
+  first_value.scalar()() = 1.0;
+  Node* first_const = test::graph::Constant(&g, first_value);
+  Node* first_identity = test::graph::Identity(&g, first_const);
+
+  Tensor second_value(DT_FLOAT, TensorShape({}));
+  second_value.scalar()() = 2.0;
+  Node* second_const = test::graph::Constant(&g, second_value);
+  Node* second_identity = test::graph::Identity(&g, second_const);
+
+  test::graph::ToGraphDef(&g, &def);
+
+  auto session = CreateSession();
+  ASSERT_TRUE(session != nullptr);
+  TF_ASSERT_OK(session->Create(def));
+
+  std::vector outputs;
+
+  // Fetch without feeding.
+  Status s = session->Run(
+      run_options, {},
+      {first_identity->name() + ":0", second_identity->name() + ":0"}, {},
+      &outputs, nullptr);
+  TF_ASSERT_OK(s);
+  ASSERT_EQ(2, outputs.size());
+  ASSERT_EQ(1.0, outputs[0].flat()(0));
+  ASSERT_EQ(2.0, outputs[1].flat()(0));
+
+  s = session->Run(
+      {}, {second_identity->name() + ":0", first_identity->name() + ":0"}, {},
+      &outputs);
+  TF_ASSERT_OK(s);
+  ASSERT_EQ(2, outputs.size());
+  ASSERT_EQ(2.0, outputs[0].flat()(0));
+  ASSERT_EQ(1.0, outputs[1].flat()(0));
+
+  Tensor value_11(DT_FLOAT, TensorShape({}));
+  value_11.scalar()() = 11.0;
+  Tensor value_22(DT_FLOAT, TensorShape({}));
+  value_22.scalar()() = 22.0;
+
+  // Feed [first_const, second_const]
+  s = session->Run(
+      {{first_const->name(), value_11}, {second_const->name(), value_22}},
+      {first_identity->name() + ":0", second_identity->name() + ":0"}, {},
+      &outputs);
+  TF_ASSERT_OK(s);
+  ASSERT_EQ(2, outputs.size());
+  ASSERT_EQ(11.0, outputs[0].flat()(0));
+  ASSERT_EQ(22.0, outputs[1].flat()(0));
+
+  // Feed [second_const, first_const]
+  s = session->Run(
+      {{second_const->name(), value_22}, {first_const->name(), value_11}},
+      {first_identity->name() + ":0", second_identity->name() + ":0"}, {},
+      &outputs);
+  TF_ASSERT_OK(s);
+  ASSERT_EQ(2, outputs.size());
+  ASSERT_EQ(11.0, outputs[0].flat()(0));
+  ASSERT_EQ(22.0, outputs[1].flat()(0));
+
+  // Feed [first_const, first_const]
+  s = session->Run(
+      run_options,
+      {{first_const->name(), value_11}, {first_const->name(), value_22}},
+      {first_identity->name() + ":0", second_identity->name() + ":0"}, {},
+      &outputs, nullptr);
+  EXPECT_TRUE(errors::IsInvalidArgument(s));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "fed more than once"));
+}
+
+REGISTER_OP("ThreadID").Input("x: int64").Output("y: int64").Doc(R"doc(
+ThreadID returns the thread ID that called compute.
+
+x: int64
+y: int64
+)doc");
+
+// The ThreadID kernel returns the thread ID that executed Compute.
+class ThreadIDOp : public OpKernel {
+ public:
+  explicit ThreadIDOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+  void Compute(OpKernelContext* ctx) override {
+    Tensor* out_tensor = nullptr;
+    OP_REQUIRES_OK(ctx,
+                   ctx->allocate_output("y", TensorShape({}), &out_tensor));
+    std::hash hasher;
+    out_tensor->scalar()() =
+        static_cast(hasher(std::this_thread::get_id()));
+  }
+};
+REGISTER_KERNEL_BUILDER(Name("ThreadID").Device(DEVICE_CPU), ThreadIDOp);
+
+TEST(DirectSessionTest, SessionSyncRun) {
+  Graph g(OpRegistry::Global());
+  Tensor vx(DT_INT64, TensorShape({}));
+  vx.scalar()() = 17;
+  Node* x = test::graph::Constant(&g, vx);
+  Node* y = test::graph::Unary(&g, "ThreadID", x);
+  GraphDef def;
+  test::graph::ToGraphDef(&g, &def);
+  auto sess = CreateSession();
+  TF_ASSERT_OK(sess->Create(def));
+  std::vector outputs;
+  RunOptions run_opts;
+  run_opts.set_inter_op_thread_pool(-1);
+  auto s = sess->Run(run_opts, {}, {y->name() + ":0"}, {}, &outputs, nullptr);
+
+  std::hash hasher;
+  EXPECT_EQ(static_cast(hasher(std::this_thread::get_id())),
+            static_cast(outputs[0].scalar()()));
+}
+
 REGISTER_OP("Darth").Input("x: float").Output("y: float").Doc(R"doc(
 Darth promises one return value.
 
@@ -1400,6 +1520,7 @@ static void TestSessionInterOpThreadsImpl(bool use_function_lib,
   p = options.config.add_session_inter_op_thread_pool();
   if (use_global_pools) p->set_global_name("small pool");
   p->set_num_threads(1);
+  const int kSyncPool = -1;
   const int kLargePool = 0;
   const int kSmallPool = 1;
 
@@ -1442,7 +1563,11 @@ static void TestSessionInterOpThreadsImpl(bool use_function_lib,
           EXPECT_FLOAT_EQ(1.2, flat(0));
           num_done.fetch_add(1);
         };
-        tp->Schedule(fn);
+        if (tp != nullptr) {
+          tp->Schedule(fn);
+        } else {
+          fn();
+        }
       };
 
   // For blocking states:
@@ -1463,9 +1588,10 @@ static void TestSessionInterOpThreadsImpl(bool use_function_lib,
 
   tp1 = new thread::ThreadPool(Env::Default(), "tp1", 5);
 
-  // Launch 2 session run calls. Neither will finish until the blocking op is
+  // Launch a session run call. It will not finish until the blocking op is
   // unblocked, because it is using all threads in the small pool.
   add_session_run_call(tp1, y, kSmallPool);
+
   blocking_op_state->AwaitState(1);  // Wait for the blocking op to Compute.
 
   // These will block on .
@@ -1484,10 +1610,15 @@ static void TestSessionInterOpThreadsImpl(bool use_function_lib,
   delete tp2;
   EXPECT_EQ(kUnblockedThreads, num_done.load());
 
+  // Launch a session call using this thread. This will finish as it runs
+  // synchronously in this thread.
+  add_session_run_call(nullptr, x, kSyncPool);
+
   // Unblock the blocked op and wait for the blocked functions to finish.
   blocking_op_state->MoveToState(1, 2);
   delete tp1;
-  EXPECT_EQ(kUnblockedThreads + kBlockedThreads + 1, num_done.load());
+
+  EXPECT_EQ(kUnblockedThreads + kBlockedThreads + 1 + 1, num_done.load());
   delete blocking_op_state;
   blocking_op_state = nullptr;
 }
@@ -1532,7 +1663,7 @@ TEST(DirectSessionTest, TestSessionInterOpThreadsInvalidOptions) {
   {
     std::unique_ptr session(NewSession(options));
     TF_ASSERT_OK(session->Create(def));
-    for (int pool_num = -1; pool_num <= 1; pool_num += 2) {
+    for (int pool_num = -2; pool_num <= 1; pool_num += 3) {
       RunOptions run_options;
       run_options.set_inter_op_thread_pool(pool_num);
       std::vector outputs;
diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto
index 22a2691dcc..d701ce8e12 100644
--- a/tensorflow/core/protobuf/config.proto
+++ b/tensorflow/core/protobuf/config.proto
@@ -416,6 +416,11 @@ message RunOptions {
   int64 timeout_in_ms = 2;
 
   // The thread pool to use, if session_inter_op_thread_pool is configured.
+  // To use the caller thread set this to -1 - this uses the caller thread
+  // to execute Session::Run() and thus avoids a context switch. Using the
+  // caller thread to execute Session::Run() should be done ONLY for simple
+  // graphs, where the overhead of an additional context switch is
+  // comparable with the overhead of Session::Run().
   int32 inter_op_thread_pool = 3;
 
   // Whether the partition graph(s) executed by the executor(s) should be
-- 
cgit v1.2.3


From f1df683d04f828e6e677116bc8632c4c4f3bc4a0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 13:26:43 -0700
Subject: Update description of TPUEstimator.

PiperOrigin-RevId: 205714060
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 09eeb6a7f5..42406db88a 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -1724,6 +1724,9 @@ class InstallSignalHandlerHook(session_run_hook.SessionRunHook):
 class TPUEstimator(estimator_lib.Estimator):
   """Estimator with TPU support.
 
+  TPUEstimator also supports training on CPU and GPU. You don't need to define
+  a separate `tf.estimator.Estimator`.
+
   TPUEstimator handles many of the details of running on TPU devices, such as
   replicating inputs and models for each core, and returning to host
   periodically to run hooks.
-- 
cgit v1.2.3


From 5e7807be1c709f55f5643e7993bba04d2ba72ea6 Mon Sep 17 00:00:00 2001
From: Timon Van Overveldt 
Date: Mon, 23 Jul 2018 13:52:14 -0700
Subject: Split textual headers off of ":android_all_ops" into
 ":android_all_ops_textual_hdrs".

Also add some missing includes.

PiperOrigin-RevId: 205718290
---
 tensorflow/core/kernels/BUILD | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 23f84c46a9..2cb54bd973 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5225,6 +5225,16 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 
+ANDROID_TEXTUAL_HDRS = [
+    "gather_nd_op_cpu_impl.h",
+    "gemm_functors.h",
+    "mirror_pad_op_cpu_impl.h",
+    "scatter_nd_op_cpu_impl.h",
+    "slice_op_cpu_impl.h",
+    "strided_slice_op_impl.h",
+    "tile_ops_cpu_impl.h",
+]
+
 # A file group which contains nearly all available operators which
 # may work on Android. This is intended to be used with selective
 # registration.
@@ -5286,10 +5296,20 @@ filegroup(
             "batch_kernels.*",
             "regex_full_match_op.cc",
             "regex_replace_op.cc",
-        ],
+            # Ops that are inherently incompatible with Android (e.g. tied to x86 platform).
+            "mkl_*",
+            "xsmm_*",
+            "cwise_ops_sycl_common.h",
+        ] + ANDROID_TEXTUAL_HDRS,
     ),
     visibility = ["//visibility:public"],
 )
+
+filegroup(
+    name = "android_all_ops_textual_hdrs",
+    srcs = ANDROID_TEXTUAL_HDRS,
+    visibility = ["//visibility:public"],
+)
 # LINT.ThenChange(//tensorflow/contrib/makefile/tf_op_files.txt)
 
 cc_library(
-- 
cgit v1.2.3


From 69f229a56652f076454ce9f3cb99bba285604ebe Mon Sep 17 00:00:00 2001
From: Goutham Bhat 
Date: Mon, 23 Jul 2018 14:30:36 -0700
Subject: Work around gfile.Glob's divergent behavior in different
 environments.

PiperOrigin-RevId: 205725301
---
 .../contrib/estimator/python/estimator/early_stopping.py    |  9 +++++----
 .../estimator/python/estimator/early_stopping_test.py       | 13 +++++++++++++
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/estimator/python/estimator/early_stopping.py b/tensorflow/contrib/estimator/python/estimator/early_stopping.py
index af4855e91e..3eab21d5ac 100644
--- a/tensorflow/contrib/estimator/python/estimator/early_stopping.py
+++ b/tensorflow/contrib/estimator/python/estimator/early_stopping.py
@@ -394,10 +394,11 @@ def _summaries(eval_dir):
   Yields:
     `tensorflow.Event` object read from the event files.
   """
-  for event_file in gfile.Glob(
-      os.path.join(eval_dir, _EVENT_FILE_GLOB_PATTERN)):
-    for event in summary_iterator.summary_iterator(event_file):
-      yield event
+  if gfile.Exists(eval_dir):
+    for event_file in gfile.Glob(
+        os.path.join(eval_dir, _EVENT_FILE_GLOB_PATTERN)):
+      for event in summary_iterator.summary_iterator(event_file):
+        yield event
 
 
 def _get_or_create_stop_var():
diff --git a/tensorflow/contrib/estimator/python/estimator/early_stopping_test.py b/tensorflow/contrib/estimator/python/estimator/early_stopping_test.py
index b5eee818fa..e4bfd4b446 100644
--- a/tensorflow/contrib/estimator/python/estimator/early_stopping_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/early_stopping_test.py
@@ -92,6 +92,19 @@ class ReadEvalMetricsTest(test.TestCase):
         },
     }, early_stopping.read_eval_metrics(eval_dir))
 
+  def test_read_eval_metrics_when_no_events(self):
+    eval_dir = tempfile.mkdtemp()
+    self.assertTrue(os.path.exists(eval_dir))
+
+    # No error should be raised when eval directory exists with no event files.
+    self.assertEqual({}, early_stopping.read_eval_metrics(eval_dir))
+
+    os.rmdir(eval_dir)
+    self.assertFalse(os.path.exists(eval_dir))
+
+    # No error should be raised when eval directory does not exist.
+    self.assertEqual({}, early_stopping.read_eval_metrics(eval_dir))
+
 
 class EarlyStoppingHooksTest(test.TestCase, parameterized.TestCase):
 
-- 
cgit v1.2.3


From cf94a46c34f8568608d78b77e9a1c4369ebcafa2 Mon Sep 17 00:00:00 2001
From: Karmel Allison 
Date: Mon, 23 Jul 2018 14:47:30 -0700
Subject: The SavedModel legacy_init_op and main_op are functionally
 equivalent. Here, we remove duplicated code paths by mapping legacy_init_op
 into main_op in the SavedModelBuilder, and we deprecate the legacy_init_op
 arg. Note that the loader will still look for both, so old SavedModels will
 still load without trouble.

PiperOrigin-RevId: 205728344
---
 tensorflow/cc/saved_model/loader.cc               | 39 +++---------
 tensorflow/python/saved_model/builder_impl.py     | 76 ++++++++++++-----------
 tensorflow/python/saved_model/loader_impl.py      | 42 ++++---------
 tensorflow/python/saved_model/saved_model_test.py | 28 ++++++---
 4 files changed, 79 insertions(+), 106 deletions(-)

diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index 07807ed2f3..d47b025743 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -86,10 +86,11 @@ bool HasMainOp(const MetaGraphDef& meta_graph_def) {
 Status RunMainOp(const RunOptions& run_options, const string& export_dir,
                  const MetaGraphDef& meta_graph_def,
                  const std::vector& asset_file_defs,
-                 Session* session) {
-  LOG(INFO) << "Running MainOp on SavedModel bundle.";
+                 Session* session, const string& main_op_key) {
+  LOG(INFO) << "Running MainOp with key " << main_op_key
+            << " on SavedModel bundle.";
   const auto& collection_def_map = meta_graph_def.collection_def();
-  const auto main_op_it = collection_def_map.find(kSavedModelMainOpKey);
+  const auto main_op_it = collection_def_map.find(main_op_key);
   if (main_op_it != collection_def_map.end()) {
     if (main_op_it->second.node_list().value_size() != 1) {
       return errors::FailedPrecondition(
@@ -141,30 +142,6 @@ Status RunRestore(const RunOptions& run_options, const string& export_dir,
                       nullptr /* outputs */, &run_metadata);
 }
 
-Status RunLegacyInitOp(const RunOptions& run_options, const string& export_dir,
-                       const MetaGraphDef& meta_graph_def,
-                       const std::vector& asset_file_defs,
-                       Session* session) {
-  LOG(INFO) << "Running LegacyInitOp on SavedModel bundle.";
-  const auto& collection_def_map = meta_graph_def.collection_def();
-  const auto init_op_it = collection_def_map.find(kSavedModelLegacyInitOpKey);
-  if (init_op_it != collection_def_map.end()) {
-    if (init_op_it->second.node_list().value_size() != 1) {
-      return errors::FailedPrecondition(strings::StrCat(
-          "Expected exactly one serving init op in : ", export_dir));
-    }
-    std::vector> inputs;
-    AddAssetsTensorsToInputs(export_dir, asset_file_defs, &inputs);
-    RunMetadata run_metadata;
-    const StringPiece legacy_init_op_name =
-        init_op_it->second.node_list().value(0);
-    return session->Run(run_options, inputs, {},
-                        {legacy_init_op_name.ToString()}, nullptr /* outputs */,
-                        &run_metadata);
-  }
-  return Status::OK();
-}
-
 Status GetAssetFileDefs(const MetaGraphDef& meta_graph_def,
                         std::vector* asset_file_defs) {
   const auto& collection_def_map = meta_graph_def.collection_def();
@@ -204,11 +181,11 @@ Status LoadSavedModelInternal(const SessionOptions& session_options,
   if (HasMainOp(bundle->meta_graph_def)) {
     TF_RETURN_IF_ERROR(RunMainOp(run_options, export_dir,
                                  bundle->meta_graph_def, asset_file_defs,
-                                 bundle->session.get()));
+                                 bundle->session.get(), kSavedModelMainOpKey));
   } else {
-    TF_RETURN_IF_ERROR(RunLegacyInitOp(run_options, export_dir,
-                                       bundle->meta_graph_def, asset_file_defs,
-                                       bundle->session.get()));
+    TF_RETURN_IF_ERROR(RunMainOp(
+        run_options, export_dir, bundle->meta_graph_def, asset_file_defs,
+        bundle->session.get(), kSavedModelLegacyInitOpKey));
   }
   return Status::OK();
 }
diff --git a/tensorflow/python/saved_model/builder_impl.py b/tensorflow/python/saved_model/builder_impl.py
index e58be804c2..8c985a7c2f 100644
--- a/tensorflow/python/saved_model/builder_impl.py
+++ b/tensorflow/python/saved_model/builder_impl.py
@@ -34,6 +34,7 @@ from tensorflow.python.platform import tf_logging
 from tensorflow.python.saved_model import constants
 from tensorflow.python.training import saver as tf_saver
 from tensorflow.python.util import compat
+from tensorflow.python.util.deprecation import deprecated_args
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -133,39 +134,32 @@ class SavedModelBuilder(object):
     tf_logging.info("Assets written to: %s",
                     compat.as_text(assets_destination_dir))
 
-  def _maybe_add_legacy_init_op(self, legacy_init_op=None):
-    """Add legacy init op to the SavedModel.
+  def _maybe_add_main_op(self, main_op):
+    """Adds main op to the SavedModel.
 
     Args:
-      legacy_init_op: Optional legacy init op to support backward compatibility.
+      main_op: Main op to run as part of graph initialization. If None, no
+        main op will be added to the graph.
 
     Raises:
-      TypeError if legacy init op is not of type `Operation`.
-      AssertionError if the graph already contains one or more legacy init ops.
+      TypeError: if main op is provided but is not of type `Operation`.
+      ValueError: if the Graph already contains an init op.
     """
-    if legacy_init_op is not None:
-      if not isinstance(legacy_init_op, ops.Operation):
-        raise TypeError("legacy_init_op needs to be an Operation: %r" %
-                        legacy_init_op)
-      if ops.get_collection(constants.LEGACY_INIT_OP_KEY):
-        raise AssertionError(
-            "graph already contains one or more legacy init ops under the "
-            "collection {}.".format(constants.LEGACY_INIT_OP_KEY))
-      ops.add_to_collection(constants.LEGACY_INIT_OP_KEY, legacy_init_op)
-
-  def _add_main_op(self, main_op):
-    """Add main op to the SavedModel.
+    if main_op is None:
+      return
 
-    Args:
-      main_op: Main op to run as part of graph initialization.
+    if not isinstance(main_op, ops.Operation):
+      raise TypeError("main_op needs to be an Operation: %r" % main_op)
 
-    Raises:
-      TypeError if main op is not of type `Operation`.
-    """
-    if main_op is not None:
-      if not isinstance(main_op, ops.Operation):
-        raise TypeError("main_op needs to be an Operation: %r" % main_op)
-      ops.add_to_collection(constants.MAIN_OP_KEY, main_op)
+    # Validate that no other init ops have been added to this graph already.
+    # We check main_op and legacy_init_op for thoroughness and explicitness.
+    for init_op_key in (constants.MAIN_OP_KEY, constants.LEGACY_INIT_OP_KEY):
+      if ops.get_collection(init_op_key):
+        raise ValueError(
+            "Graph already contains one or more main ops under the "
+            "collection {}.".format(init_op_key))
+
+    ops.add_to_collection(constants.MAIN_OP_KEY, main_op)
 
   def _add_train_op(self, train_op):
     """Add train op to the SavedModel.
@@ -257,16 +251,12 @@ class SavedModelBuilder(object):
           self._validate_tensor_info(outputs[outputs_key])
 
   def _add_collections(
-      self, assets_collection, legacy_init_op, main_op, train_op):
+      self, assets_collection, main_op, train_op):
     """Add asset and op collections to be saved."""
     # Save asset files and write them to disk, if any.
     self._save_and_write_assets(assets_collection)
 
-    if main_op is None:
-      # Add legacy init op to the SavedModel.
-      self._maybe_add_legacy_init_op(legacy_init_op)
-    else:
-      self._add_main_op(main_op)
+    self._maybe_add_main_op(main_op)
 
     self._add_train_op(train_op)
 
@@ -282,6 +272,9 @@ class SavedModelBuilder(object):
           allow_empty=True)
     return saver
 
+  @deprecated_args(None,
+                   "Pass your op to the equivalent parameter main_op instead.",
+                   "legacy_init_op")
   def add_meta_graph(self,
                      tags,
                      signature_def_map=None,
@@ -306,7 +299,7 @@ class SavedModelBuilder(object):
           that this collection should be a subset of the assets saved as part of
           the first meta graph in the SavedModel.
       legacy_init_op: Legacy support for op or group of ops to execute after the
-          restore op upon a load.
+          restore op upon a load. Deprecated; please use main_op instead.
       clear_devices: Set to true if the device info on the default graph should
           be cleared.
       main_op: Op or group of ops to execute when the graph is loaded. Note
@@ -333,8 +326,12 @@ class SavedModelBuilder(object):
     # properly populated.
     self._validate_signature_def_map(signature_def_map)
 
+    # legacy_init_op is deprecated, and going away in TF 2.0.
+    # Re-mapping to main_op, as treatment is identical regardless.
+    main_op = main_op or legacy_init_op
+
     # Add assets and ops
-    self._add_collections(assets_collection, legacy_init_op, main_op, None)
+    self._add_collections(assets_collection, main_op, None)
 
     saver = self._maybe_create_saver(saver)
 
@@ -351,6 +348,9 @@ class SavedModelBuilder(object):
     # Tag the meta graph def and add it to the SavedModel.
     self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
 
+  @deprecated_args(None,
+                   "Pass your op to the equivalent parameter main_op instead.",
+                   "legacy_init_op")
   def add_meta_graph_and_variables(self,
                                    sess,
                                    tags,
@@ -378,7 +378,7 @@ class SavedModelBuilder(object):
         def.
       assets_collection: Assets collection to be saved with SavedModel.
       legacy_init_op: Legacy support for op or group of ops to execute after the
-          restore op upon a load.
+          restore op upon a load. Deprecated; please use main_op instead.
       clear_devices: Set to true if the device info on the default graph should
           be cleared.
       main_op: Op or group of ops to execute when the graph is loaded. Note
@@ -402,8 +402,12 @@ class SavedModelBuilder(object):
     # properly populated.
     self._validate_signature_def_map(signature_def_map)
 
+    # legacy_init_op is deprecated, and going away in TF 2.0.
+    # Re-mapping to main_op, as treatment is identical regardless.
+    main_op = main_op or legacy_init_op
+
     # Add assets and ops
-    self._add_collections(assets_collection, legacy_init_op, main_op, None)
+    self._add_collections(assets_collection, main_op, None)
 
     # Create the variables sub-directory, if it does not exist.
     variables_dir = os.path.join(
diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py
index e5f649fdab..fb70c91c29 100644
--- a/tensorflow/python/saved_model/loader_impl.py
+++ b/tensorflow/python/saved_model/loader_impl.py
@@ -116,11 +116,14 @@ def _get_asset_tensors(export_dir, meta_graph_def_to_load, import_scope=None):
   return asset_tensor_dict
 
 
-def _get_main_op_tensor(meta_graph_def_to_load):
+def _get_main_op_tensor(
+    meta_graph_def_to_load, init_op_key=constants.MAIN_OP_KEY):
   """Gets the main op tensor, if one exists.
 
   Args:
     meta_graph_def_to_load: The meta graph def from the SavedModel to be loaded.
+    init_op_key: name of collection to check; should be one of MAIN_OP_KEY
+      or the deprecated LEGACY_INIT_OP_KEY
 
   Returns:
     The main op tensor, if it exists and `None` otherwise.
@@ -131,38 +134,15 @@ def _get_main_op_tensor(meta_graph_def_to_load):
   """
   collection_def = meta_graph_def_to_load.collection_def
   main_op_tensor = None
-  if constants.MAIN_OP_KEY in collection_def:
-    main_ops = collection_def[constants.MAIN_OP_KEY].node_list.value
+  if init_op_key in collection_def:
+    main_ops = collection_def[init_op_key].node_list.value
     if len(main_ops) != 1:
-      raise RuntimeError("Expected exactly one SavedModel main op.")
-    main_op_tensor = ops.get_collection(constants.MAIN_OP_KEY)[0]
+      raise RuntimeError("Expected exactly one SavedModel main op. "
+                         "Found: {}".format(main_ops))
+    main_op_tensor = ops.get_collection(init_op_key)[0]
   return main_op_tensor
 
 
-def _get_legacy_init_op_tensor(meta_graph_def_to_load):
-  """Gets the legacy init op tensor, if one exists.
-
-  Args:
-    meta_graph_def_to_load: The meta graph def from the SavedModel to be loaded.
-
-  Returns:
-    The legacy init op tensor, if it exists and `None` otherwise.
-
-  Raises:
-    RuntimeError: If the collection def corresponding to the legacy init op key
-        has other than exactly one tensor.
-  """
-  collection_def = meta_graph_def_to_load.collection_def
-  legacy_init_op_tensor = None
-  if constants.LEGACY_INIT_OP_KEY in collection_def:
-    legacy_init_ops = collection_def[
-        constants.LEGACY_INIT_OP_KEY].node_list.value
-    if len(legacy_init_ops) != 1:
-      raise RuntimeError("Expected exactly one legacy serving init op.")
-    legacy_init_op_tensor = ops.get_collection(constants.LEGACY_INIT_OP_KEY)[0]
-  return legacy_init_op_tensor
-
-
 @tf_export("saved_model.loader.maybe_saved_model_directory")
 def maybe_saved_model_directory(export_dir):
   """Checks whether the provided export directory could contain a SavedModel.
@@ -340,8 +320,8 @@ class SavedModelLoader(object):
           self._export_dir, meta_graph_def, import_scope=import_scope)
 
       main_op_tensor = (
-          _get_main_op_tensor(meta_graph_def) or
-          (_get_legacy_init_op_tensor(meta_graph_def)))
+          _get_main_op_tensor(meta_graph_def, constants.MAIN_OP_KEY) or
+          _get_main_op_tensor(meta_graph_def, constants.LEGACY_INIT_OP_KEY))
       if main_op_tensor is not None:
         sess.run(fetches=[main_op_tensor], feed_dict=asset_tensors_dictionary)
 
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index fb4732aca2..00b669fc97 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -846,9 +846,19 @@ class SavedModelTest(test.TestCase):
   def testLegacyInitOpWithNonEmptyCollection(self):
     export_dir = self._get_export_dir(
         "test_legacy_init_op_with_non_empty_collection")
+    self._testInitOpsWithNonEmptyCollection(
+        export_dir, constants.LEGACY_INIT_OP_KEY)
+
+  def testMainOpWithNonEmptyCollection(self):
+    export_dir = self._get_export_dir(
+        "test_main_op_with_non_empty_collection")
+    self._testInitOpsWithNonEmptyCollection(export_dir, constants.MAIN_OP_KEY)
+
+  def _testInitOpsWithNonEmptyCollection(self, export_dir, key):
     builder = saved_model_builder.SavedModelBuilder(export_dir)
 
-    with self.test_session(graph=ops.Graph()) as sess:
+    g = ops.Graph()
+    with self.test_session(graph=g) as sess:
       # Initialize variable `v1` to 1.
       v1 = variables.Variable(1, name="v1")
       ops.add_to_collection("v", v1)
@@ -857,19 +867,21 @@ class SavedModelTest(test.TestCase):
       v2 = variables.Variable(42, name="v2", trainable=False, collections=[])
       ops.add_to_collection("v", v2)
 
-      # Set up an assignment op to be run as part of the legacy_init_op.
+      # Set up an assignment op to be run as part of the init op.
       assign_v2 = state_ops.assign(v2, v1)
-      legacy_init_op = control_flow_ops.group(assign_v2, name="legacy_init_op")
+      init_op = control_flow_ops.group(assign_v2, name="init_op")
 
       sess.run(variables.global_variables_initializer())
 
-      ops.add_to_collection(constants.LEGACY_INIT_OP_KEY,
-                            control_flow_ops.no_op())
-      # AssertionError should be raised since the LEGACY_INIT_OP_KEY collection
+      ops.add_to_collection(key, control_flow_ops.no_op())
+      # ValueError should be raised since the LEGACY_INIT_OP_KEY collection
       # is not empty and we don't support multiple init ops.
-      with self.assertRaises(AssertionError):
+      with self.assertRaisesRegexp(ValueError, "Graph already contains"):
         builder.add_meta_graph_and_variables(
-            sess, ["foo"], legacy_init_op=legacy_init_op)
+            sess, ["foo"], legacy_init_op=init_op)
+      # We shouldn't be able to add as MAIN_OP, either.
+      with self.assertRaisesRegexp(ValueError, "Graph already contains"):
+        builder.add_meta_graph_and_variables(sess, ["foo"], main_op=init_op)
 
   def testTrainOp(self):
     export_dir = self._get_export_dir("test_train_op")
-- 
cgit v1.2.3


From 218bd6facc481da1ed199a8c680427051cb1c6cb Mon Sep 17 00:00:00 2001
From: Jared Duke 
Date: Mon, 23 Jul 2018 14:49:59 -0700
Subject: Add initial experimental C API for TFLite

PiperOrigin-RevId: 205728711
---
 tensorflow/contrib/lite/build_def.bzl              |  15 +++
 tensorflow/contrib/lite/experimental/c/BUILD       |  63 +++++++++
 tensorflow/contrib/lite/experimental/c/c_api.cc    | 118 ++++++++++++++++
 tensorflow/contrib/lite/experimental/c/c_api.h     | 149 +++++++++++++++++++++
 .../contrib/lite/experimental/c/c_api_test.cc      |  84 ++++++++++++
 .../lite/experimental/c/exported_symbols.lds       |   1 +
 .../contrib/lite/experimental/c/version_script.lds |   9 ++
 tensorflow/contrib/lite/testdata/add.bin           | Bin 0 -> 476 bytes
 8 files changed, 439 insertions(+)
 create mode 100644 tensorflow/contrib/lite/experimental/c/BUILD
 create mode 100644 tensorflow/contrib/lite/experimental/c/c_api.cc
 create mode 100644 tensorflow/contrib/lite/experimental/c/c_api.h
 create mode 100644 tensorflow/contrib/lite/experimental/c/c_api_test.cc
 create mode 100644 tensorflow/contrib/lite/experimental/c/exported_symbols.lds
 create mode 100644 tensorflow/contrib/lite/experimental/c/version_script.lds
 create mode 100644 tensorflow/contrib/lite/testdata/add.bin

diff --git a/tensorflow/contrib/lite/build_def.bzl b/tensorflow/contrib/lite/build_def.bzl
index bed862454e..79f7455ad8 100644
--- a/tensorflow/contrib/lite/build_def.bzl
+++ b/tensorflow/contrib/lite/build_def.bzl
@@ -2,6 +2,7 @@
 load(
     "//tensorflow:tensorflow.bzl",
     "tf_cc_test",
+    "tf_cc_shared_object",
 )
 
 def tflite_copts():
@@ -118,6 +119,20 @@ def tflite_jni_binary(name,
       deps= deps + [linkscript],
       linkopts=linkopts)
 
+def tflite_cc_shared_object(name,
+                            copts=tflite_copts(),
+                            linkopts=[],
+                            linkstatic=1,
+                            deps=[]):
+  """Builds a shared object for TFLite."""
+  tf_cc_shared_object(
+      name=name,
+      copts=copts,
+      linkstatic=linkstatic,
+      linkopts=linkopts + tflite_jni_linkopts(),
+      framework_so=[],
+      deps=deps)
+
 def tf_to_tflite(name, src, options, out):
   """Convert a frozen tensorflow graphdef to TF Lite's flatbuffer.
 
diff --git a/tensorflow/contrib/lite/experimental/c/BUILD b/tensorflow/contrib/lite/experimental/c/BUILD
new file mode 100644
index 0000000000..b09bb9ea10
--- /dev/null
+++ b/tensorflow/contrib/lite/experimental/c/BUILD
@@ -0,0 +1,63 @@
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow/contrib/lite:build_def.bzl",
+    "tflite_cc_shared_object",
+    "tflite_jni_binary",
+)
+
+tflite_cc_shared_object(
+    name = "libtensorflowlite_c.so",
+    linkopts = select({
+        "//tensorflow:darwin": [
+            "-Wl,-exported_symbols_list",  # This line must be directly followed by the exported_symbols.lds file
+            "$(location //tensorflow/contrib/lite/experimental/c:exported_symbols.lds)",
+            "-Wl,-install_name,@rpath/libtensorflowlite_c.so",
+        ],
+        "//tensorflow:windows": [],
+        "//conditions:default": [
+            "-z defs",
+            "-Wl,--version-script",  #  This line must be directly followed by the version_script.lds file
+            "$(location //tensorflow/contrib/lite/experimental/c:version_script.lds)",
+        ],
+    }),
+    deps = [
+        ":c_api",
+        ":exported_symbols.lds",
+        ":version_script.lds",
+    ],
+)
+
+tflite_jni_binary(
+    name = "libtensorflowlite_c_jni.so",
+    linkscript = ":version_script.lds",
+    deps = [":c_api"],
+)
+
+cc_library(
+    name = "c_api",
+    srcs = ["c_api.cc"],
+    hdrs = ["c_api.h"],
+    deps = [
+        "//tensorflow/contrib/lite:context",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite:schema_fbs_version",
+        "//tensorflow/contrib/lite/kernels:builtin_ops",
+    ],
+)
+
+cc_test(
+    name = "c_api_test",
+    size = "small",
+    srcs = ["c_api_test.cc"],
+    data = ["//tensorflow/contrib/lite:testdata/add.bin"],
+    deps = [
+        ":c_api",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite:kernel_api",
+        "//tensorflow/contrib/lite/testing:util",
+        "@com_google_googletest//:gtest",
+    ],
+)
diff --git a/tensorflow/contrib/lite/experimental/c/c_api.cc b/tensorflow/contrib/lite/experimental/c/c_api.cc
new file mode 100644
index 0000000000..add4c6813d
--- /dev/null
+++ b/tensorflow/contrib/lite/experimental/c/c_api.cc
@@ -0,0 +1,118 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/contrib/lite/experimental/c/c_api.h"
+
+#include "tensorflow/contrib/lite/context.h"
+#include "tensorflow/contrib/lite/interpreter.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/model.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif  // __cplusplus
+
+struct _TFL_Interpreter {
+  std::unique_ptr impl;
+};
+
+TFL_Interpreter* TFL_NewInterpreter(const void* model_data,
+                                    int32_t model_size) {
+  auto model = tflite::FlatBufferModel::BuildFromBuffer(
+      static_cast(model_data), static_cast(model_size));
+  if (!model) {
+    return nullptr;
+  }
+
+  tflite::ops::builtin::BuiltinOpResolver resolver;
+  tflite::InterpreterBuilder builder(*model, resolver);
+  std::unique_ptr interpreter_impl;
+  if (builder(&interpreter_impl) != kTfLiteOk) {
+    return nullptr;
+  }
+
+  return new TFL_Interpreter{std::move(interpreter_impl)};
+}
+
+void TFL_DeleteInterpreter(TFL_Interpreter* interpreter) { delete interpreter; }
+
+int32_t TFL_InterpreterGetInputTensorCount(const TFL_Interpreter* interpreter) {
+  return static_cast(interpreter->impl->inputs().size());
+}
+
+TFL_Tensor* TFL_InterpreterGetInputTensor(const TFL_Interpreter* interpreter,
+                                          int32_t input_index) {
+  return interpreter->impl->tensor(interpreter->impl->inputs()[input_index]);
+}
+
+TFL_Status TFL_InterpreterResizeInputTensor(TFL_Interpreter* interpreter,
+                                            int32_t input_index,
+                                            const int* input_dims,
+                                            int32_t input_dims_size) {
+  std::vector dims{input_dims, input_dims + input_dims_size};
+  return interpreter->impl->ResizeInputTensor(
+      interpreter->impl->inputs()[input_index], dims);
+}
+
+TFL_Status TFL_InterpreterAllocateTensors(TFL_Interpreter* interpreter) {
+  return interpreter->impl->AllocateTensors();
+}
+
+TFL_Status TFL_InterpreterInvoke(TFL_Interpreter* interpreter) {
+  return interpreter->impl->Invoke();
+}
+
+int32_t TFL_InterpreterGetOutputTensorCount(
+    const TFL_Interpreter* interpreter) {
+  return static_cast(interpreter->impl->outputs().size());
+}
+
+const TFL_Tensor* TFL_InterpreterGetOutputTensor(
+    const TFL_Interpreter* interpreter, int32_t output_index) {
+  return interpreter->impl->tensor(interpreter->impl->outputs()[output_index]);
+}
+
+TFL_Type TFL_TensorType(const TFL_Tensor* tensor) { return tensor->type; }
+
+int32_t TFL_TensorNumDims(const TFL_Tensor* tensor) {
+  return tensor->dims->size;
+}
+
+int32_t TFL_TensorDim(const TFL_Tensor* tensor, int32_t dim_index) {
+  return tensor->dims->data[dim_index];
+}
+
+size_t TFL_TensorByteSize(const TFL_Tensor* tensor) { return tensor->bytes; }
+
+TFL_Status TFL_TensorCopyFromBuffer(TFL_Tensor* tensor, const void* input_data,
+                                    int32_t input_data_size) {
+  if (tensor->bytes != static_cast(input_data_size)) {
+    return kTfLiteError;
+  }
+  memcpy(tensor->data.raw, input_data, input_data_size);
+  return kTfLiteOk;
+}
+
+TFL_Status TFL_TensorCopyToBuffer(const TFL_Tensor* tensor, void* output_data,
+                                  int32_t output_data_size) {
+  if (tensor->bytes != static_cast(output_data_size)) {
+    return kTfLiteError;
+  }
+  memcpy(output_data, tensor->data.raw, output_data_size);
+  return kTfLiteOk;
+}
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif  // __cplusplus
diff --git a/tensorflow/contrib/lite/experimental/c/c_api.h b/tensorflow/contrib/lite/experimental/c/c_api.h
new file mode 100644
index 0000000000..070f1add13
--- /dev/null
+++ b/tensorflow/contrib/lite/experimental/c/c_api.h
@@ -0,0 +1,149 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_C_C_API_H_
+#define TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_C_C_API_H_
+
+#include 
+
+// Eventually the various C APIs defined in context.h will be migrated into
+// the appropriate /c/c_api*.h header. For now, we pull in existing definitions
+// for convenience.
+#include "tensorflow/contrib/lite/context.h"
+
+// --------------------------------------------------------------------------
+// Experimental C API for TensorFlowLite.
+//
+// The API leans towards simplicity and uniformity instead of convenience, as
+// most usage will be by language-specific wrappers.
+//
+// Conventions:
+// * We use the prefix TFL_ for everything in the API.
+
+#ifdef SWIG
+#define TFL_CAPI_EXPORT
+#else
+#if defined(_WIN32)
+#ifdef TF_COMPILE_LIBRARY
+#define TFL_CAPI_EXPORT __declspec(dllexport)
+#else
+#define TFL_CAPI_EXPORT __declspec(dllimport)
+#endif  // TF_COMPILE_LIBRARY
+#else
+#define TFL_CAPI_EXPORT __attribute__((visibility("default")))
+#endif  // _WIN32
+#endif  // SWIG
+
+#ifdef __cplusplus
+extern "C" {
+#endif  // __cplusplus
+
+typedef TfLiteTensor TFL_Tensor;
+typedef TfLiteStatus TFL_Status;
+typedef TfLiteType TFL_Type;
+
+// --------------------------------------------------------------------------
+// TFL_Interpreter provides inference from a provided model.
+typedef struct _TFL_Interpreter TFL_Interpreter;
+
+// Returns an interpreter for the provided model, or null on failure.
+//
+// NOTE: The client *must* explicitly allocate tensors before attempting to
+// access input tensor data or invoke the interpreter.
+TFL_CAPI_EXPORT extern TFL_Interpreter* TFL_NewInterpreter(
+    const void* model_data, int32_t model_size);
+
+// Destroys the interpreter.
+TFL_CAPI_EXPORT extern void TFL_DeleteInterpreter(TFL_Interpreter* interpreter);
+
+// Returns the number of input tensors associated with the model.
+TFL_CAPI_EXPORT extern int TFL_InterpreterGetInputTensorCount(
+    const TFL_Interpreter* interpreter);
+
+// Returns the tensor associated with the input index.
+// REQUIRES: 0 <= input_index < TFL_InterpreterGetInputTensorCount(tensor)
+TFL_CAPI_EXPORT extern TFL_Tensor* TFL_InterpreterGetInputTensor(
+    const TFL_Interpreter* interpreter, int32_t input_index);
+
+// Attempts to resize the specified input tensor.
+// NOTE: After a resize, the client *must* explicitly allocate tensors before
+// attempting to access the resized tensor data or invoke the interpreter.
+// REQUIRES: 0 <= input_index < TFL_InterpreterGetInputTensorCount(tensor)
+TFL_CAPI_EXPORT extern TFL_Status TFL_InterpreterResizeInputTensor(
+    TFL_Interpreter* interpreter, int32_t input_index, const int* input_dims,
+    int32_t input_dims_size);
+
+// Updates allocations for all tensors, resizing dependent tensors using the
+// specified input tensor dimensionality.
+//
+// This is a relatively expensive operation, and need only be called after
+// creating the graph and/or resizing any inputs.
+TFL_CAPI_EXPORT extern TFL_Status TFL_InterpreterAllocateTensors(
+    TFL_Interpreter* interpreter);
+
+// Runs inference for the loaded graph.
+//
+// NOTE: It is possible that the interpreter is not in a ready state to
+// evaluate (e.g., if a ResizeInputTensor() has been performed without a call to
+// AllocateTensors()).
+TFL_CAPI_EXPORT extern TFL_Status TFL_InterpreterInvoke(
+    TFL_Interpreter* interpreter);
+
+// Returns the number of output tensors associated with the model.
+TFL_CAPI_EXPORT extern int32_t TFL_InterpreterGetOutputTensorCount(
+    const TFL_Interpreter* interpreter);
+
+// Returns the tensor associated with the output index.
+// REQUIRES: 0 <= input_index < TFL_InterpreterGetOutputTensorCount(tensor)
+TFL_CAPI_EXPORT extern const TFL_Tensor* TFL_InterpreterGetOutputTensor(
+    const TFL_Interpreter* interpreter, int32_t output_index);
+
+// --------------------------------------------------------------------------
+// TFL_Tensor wraps data associated with a graph tensor.
+//
+// Note that, while the TFL_Tensor struct is not currently opaque, and its
+// fields can be accessed directly, these methods are still convenient for
+// language bindings. In the future the tensor struct will likely be made opaque
+// in the public API.
+
+// Returns the type of a tensor element.
+TFL_CAPI_EXPORT extern TFL_Type TFL_TensorType(const TFL_Tensor* tensor);
+
+// Returns the number of dimensions that the tensor has.
+TFL_CAPI_EXPORT extern int32_t TFL_TensorNumDims(const TFL_Tensor* tensor);
+
+// Returns the length of the tensor in the "dim_index" dimension.
+// REQUIRES: 0 <= dim_index < TFLiteTensorNumDims(tensor)
+TFL_CAPI_EXPORT extern int32_t TFL_TensorDim(const TFL_Tensor* tensor,
+                                             int32_t dim_index);
+
+// Returns the size of the underlying data in bytes.
+TFL_CAPI_EXPORT extern size_t TFL_TensorByteSize(const TFL_Tensor* tensor);
+
+// Copies from the provided input buffer into the tensor's buffer.
+// REQUIRES: input_data_size == TFL_TensorByteSize(tensor)
+TFL_CAPI_EXPORT extern TFL_Status TFL_TensorCopyFromBuffer(
+    TFL_Tensor* tensor, const void* input_data, int32_t input_data_size);
+
+// Copies to the provided output buffer from the tensor's buffer.
+// REQUIRES: output_data_size == TFL_TensorByteSize(tensor)
+TFL_CAPI_EXPORT extern TFL_Status TFL_TensorCopyToBuffer(
+    const TFL_Tensor* output_tensor, void* output_data,
+    int32_t output_data_size);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif  // __cplusplus
+
+#endif  // TENSORFLOW_CONTRIB_LITE_EXPERIMENTAL_C_C_API_H_
diff --git a/tensorflow/contrib/lite/experimental/c/c_api_test.cc b/tensorflow/contrib/lite/experimental/c/c_api_test.cc
new file mode 100644
index 0000000000..bc925e00a6
--- /dev/null
+++ b/tensorflow/contrib/lite/experimental/c/c_api_test.cc
@@ -0,0 +1,84 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include 
+
+#include "tensorflow/contrib/lite/experimental/c/c_api.h"
+
+#include 
+#include "tensorflow/contrib/lite/allocation.h"
+#include "tensorflow/contrib/lite/context.h"
+#include "tensorflow/contrib/lite/testing/util.h"
+
+namespace {
+
+TEST(CApiSimple, Smoke) {
+  tflite::FileCopyAllocation model_file(
+      "tensorflow/contrib/lite/testdata/add.bin",
+      tflite::DefaultErrorReporter());
+
+  TFL_Interpreter* interpreter =
+      TFL_NewInterpreter(model_file.base(), model_file.bytes());
+  ASSERT_NE(interpreter, nullptr);
+  ASSERT_EQ(TFL_InterpreterAllocateTensors(interpreter), kTfLiteOk);
+
+  ASSERT_EQ(TFL_InterpreterGetInputTensorCount(interpreter), 1);
+  ASSERT_EQ(TFL_InterpreterGetOutputTensorCount(interpreter), 1);
+
+  std::array input_dims = {2};
+  ASSERT_EQ(TFL_InterpreterResizeInputTensor(interpreter, 0, input_dims.data(),
+                                             input_dims.size()),
+            kTfLiteOk);
+  ASSERT_EQ(TFL_InterpreterAllocateTensors(interpreter), kTfLiteOk);
+
+  TFL_Tensor* input_tensor = TFL_InterpreterGetInputTensor(interpreter, 0);
+  ASSERT_NE(input_tensor, nullptr);
+  EXPECT_EQ(TFL_TensorType(input_tensor), kTfLiteFloat32);
+  EXPECT_EQ(TFL_TensorNumDims(input_tensor), 1);
+  EXPECT_EQ(TFL_TensorDim(input_tensor, 0), 2);
+  EXPECT_EQ(TFL_TensorByteSize(input_tensor), sizeof(float) * 2);
+
+  std::array input = {1.f, 3.f};
+  ASSERT_EQ(TFL_TensorCopyFromBuffer(input_tensor, input.data(),
+                                     input.size() * sizeof(float)),
+            kTfLiteOk);
+
+  ASSERT_EQ(TFL_InterpreterInvoke(interpreter), kTfLiteOk);
+
+  const TFL_Tensor* output_tensor =
+      TFL_InterpreterGetOutputTensor(interpreter, 0);
+  ASSERT_NE(output_tensor, nullptr);
+  EXPECT_EQ(TFL_TensorType(output_tensor), kTfLiteFloat32);
+  EXPECT_EQ(TFL_TensorNumDims(output_tensor), 1);
+  EXPECT_EQ(TFL_TensorDim(output_tensor, 0), 2);
+  EXPECT_EQ(TFL_TensorByteSize(output_tensor), sizeof(float) * 2);
+
+  std::array output;
+  ASSERT_EQ(TFL_TensorCopyToBuffer(output_tensor, output.data(),
+                                   output.size() * sizeof(float)),
+            kTfLiteOk);
+  EXPECT_EQ(output[0], 3.f);
+  EXPECT_EQ(output[1], 9.f);
+
+  TFL_DeleteInterpreter(interpreter);
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/experimental/c/exported_symbols.lds b/tensorflow/contrib/lite/experimental/c/exported_symbols.lds
new file mode 100644
index 0000000000..a3ddc6bc8d
--- /dev/null
+++ b/tensorflow/contrib/lite/experimental/c/exported_symbols.lds
@@ -0,0 +1 @@
+_TFL_*
diff --git a/tensorflow/contrib/lite/experimental/c/version_script.lds b/tensorflow/contrib/lite/experimental/c/version_script.lds
new file mode 100644
index 0000000000..c0c8a2bca1
--- /dev/null
+++ b/tensorflow/contrib/lite/experimental/c/version_script.lds
@@ -0,0 +1,9 @@
+VERS_1.0 {
+  # Export symbols in c_api.h.
+  global:
+    *TFL_*;
+
+  # Hide everything else.
+  local:
+    *;
+};
diff --git a/tensorflow/contrib/lite/testdata/add.bin b/tensorflow/contrib/lite/testdata/add.bin
new file mode 100644
index 0000000000..aef0fe3d82
Binary files /dev/null and b/tensorflow/contrib/lite/testdata/add.bin differ
-- 
cgit v1.2.3


From 3baa7b63edf7890b5489cf2085a79598f13af2c6 Mon Sep 17 00:00:00 2001
From: Xuechen Li 
Date: Mon, 23 Jul 2018 14:50:48 -0700
Subject: Further simplify backward pass.

PiperOrigin-RevId: 205728836
---
 .../contrib/eager/python/examples/revnet/blocks.py | 28 ++++++++++------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/revnet/blocks.py b/tensorflow/contrib/eager/python/examples/revnet/blocks.py
index 639bb06a34..8a530b0d71 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/blocks.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/blocks.py
@@ -202,20 +202,18 @@ class _Residual(tf.keras.Model):
     with tf.GradientTape(persistent=True) as tape:
       tape.watch(y)
       y1, y2 = tf.split(y, num_or_size_splits=2, axis=self.axis)
-      z1 = y1
-      gz1 = self.g(z1, training=training)
-      x2 = y2 - gz1
+      gy1 = self.g(y1, training=training)
+      x2 = y2 - gy1
       fx2 = self.f(x2, training=training)
-      x1 = z1 - fx2
+      x1 = y1 - fx2
 
     grads_combined = tape.gradient(
-        gz1, [z1] + self.g.trainable_variables, output_gradients=dy2)
-    dz1 = dy1 + grads_combined[0]
+        gy1, [y1] + self.g.trainable_variables, output_gradients=dy2)
     dg = grads_combined[1:]
-    dx1 = dz1
+    dx1 = dy1 + grads_combined[0]
 
     grads_combined = tape.gradient(
-        fx2, [x2] + self.f.trainable_variables, output_gradients=dz1)
+        fx2, [x2] + self.f.trainable_variables, output_gradients=dx1)
     dx2 = dy2 + grads_combined[0]
     df = grads_combined[1:]
 
@@ -263,7 +261,6 @@ class _BottleneckResidualInner(tf.keras.Model):
     if batch_norm_first:
       self.batch_norm_0 = tf.keras.layers.BatchNormalization(
           axis=axis, input_shape=input_shape, fused=fused, dtype=dtype)
-
     self.conv2d_1 = tf.keras.layers.Conv2D(
         filters=filters // 4,
         kernel_size=1,
@@ -273,9 +270,9 @@ class _BottleneckResidualInner(tf.keras.Model):
         use_bias=False,
         padding="SAME",
         dtype=dtype)
+
     self.batch_norm_1 = tf.keras.layers.BatchNormalization(
         axis=axis, fused=fused, dtype=dtype)
-
     self.conv2d_2 = tf.keras.layers.Conv2D(
         filters=filters // 4,
         kernel_size=3,
@@ -303,15 +300,14 @@ class _BottleneckResidualInner(tf.keras.Model):
     if self.batch_norm_first:
       net = self.batch_norm_0(net, training=training)
       net = tf.nn.relu(net)
-
     net = self.conv2d_1(net)
+
     net = self.batch_norm_1(net, training=training)
     net = tf.nn.relu(net)
-
     net = self.conv2d_2(net)
+
     net = self.batch_norm_2(net, training=training)
     net = tf.nn.relu(net)
-
     net = self.conv2d_3(net)
 
     return net
@@ -356,9 +352,9 @@ class _ResidualInner(tf.keras.Model):
         use_bias=False,
         padding="SAME",
         dtype=dtype)
+
     self.batch_norm_1 = tf.keras.layers.BatchNormalization(
         axis=axis, fused=fused, dtype=dtype)
-
     self.conv2d_2 = tf.keras.layers.Conv2D(
         filters=filters,
         kernel_size=3,
@@ -375,10 +371,10 @@ class _ResidualInner(tf.keras.Model):
     if self.batch_norm_first:
       net = self.batch_norm_0(net, training=training)
       net = tf.nn.relu(net)
-
     net = self.conv2d_1(net)
-    net = self.batch_norm_1(net, training=training)
 
+    net = self.batch_norm_1(net, training=training)
+    net = tf.nn.relu(net)
     net = self.conv2d_2(net)
 
     return net
-- 
cgit v1.2.3


From 32d121be1b105ef44fd5d4b421b78eb74dc94870 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 14:51:53 -0700
Subject: Adding core interface to a contrib version

PiperOrigin-RevId: 205728990
---
 .../boosted_trees/estimator_batch/estimator.py     | 43 +++++++++++++
 .../estimator_batch/estimator_test.py              | 29 ++++++++-
 .../contrib/boosted_trees/estimator_batch/model.py | 74 ++++++++++++++++------
 3 files changed, 124 insertions(+), 22 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
index 59a78515c6..38fa8c3834 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
@@ -22,6 +22,7 @@ from tensorflow.contrib.boosted_trees.estimator_batch import model
 from tensorflow.contrib.boosted_trees.python.utils import losses
 from tensorflow.contrib.learn.python.learn.estimators import estimator
 from tensorflow.contrib.learn.python.learn.estimators import head as head_lib
+from tensorflow.python.estimator import estimator as core_estimator
 from tensorflow.python.ops import math_ops
 
 
@@ -354,3 +355,45 @@ class GradientBoostedDecisionTreeRanker(estimator.Estimator):
         model_dir=model_dir,
         config=config,
         feature_engineering_fn=feature_engineering_fn)
+
+
+class CoreGradientBoostedDecisionTreeEstimator(core_estimator.Estimator):
+  """An estimator using gradient boosted decision trees."""
+
+  def __init__(self,
+               learner_config,
+               examples_per_layer,
+               head,
+               num_trees=None,
+               feature_columns=None,
+               weight_column_name=None,
+               model_dir=None,
+               config=None,
+               label_keys=None,
+               feature_engineering_fn=None,
+               logits_modifier_function=None,
+               center_bias=True,
+               output_leaf_index=False):
+
+    def _model_fn(features, labels, mode, config):
+      return model.model_builder(
+          features=features,
+          labels=labels,
+          mode=mode,
+          config=config,
+          params={
+              'head': head,
+              'feature_columns': feature_columns,
+              'learner_config': learner_config,
+              'num_trees': num_trees,
+              'weight_column_name': weight_column_name,
+              'examples_per_layer': examples_per_layer,
+              'center_bias': center_bias,
+              'logits_modifier_function': logits_modifier_function,
+              'use_core_libs': True,
+              'output_leaf_index': output_leaf_index,
+          },
+          output_type=model.ModelBuilderOutputType.ESTIMATOR_SPEC)
+
+    super(CoreGradientBoostedDecisionTreeEstimator, self).__init__(
+        model_fn=_model_fn, model_dir=model_dir, config=config)
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
index 2c2dcb039d..f787d3cdb8 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator_test.py
@@ -182,7 +182,7 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
     config = run_config.RunConfig()
 
     head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss(
-        loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE)
+        loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)
 
     model = estimator.GradientBoostedDecisionTreeRanker(
         head=head_fn,
@@ -203,5 +203,32 @@ class BoostedTreeEstimatorTest(test_util.TensorFlowTestCase):
     model.predict(input_fn=_infer_ranking_train_input_fn)
 
 
+class CoreGradientBoostedDecisionTreeEstimator(test_util.TensorFlowTestCase):
+
+  def testTrainEvaluateInferDoesNotThrowError(self):
+    head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss(
+        loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)
+
+    learner_config = learner_pb2.LearnerConfig()
+    learner_config.num_classes = 2
+    learner_config.constraints.max_tree_depth = 1
+    model_dir = tempfile.mkdtemp()
+    config = run_config.RunConfig()
+
+    est = estimator.CoreGradientBoostedDecisionTreeEstimator(
+        head=head_fn,
+        learner_config=learner_config,
+        num_trees=1,
+        examples_per_layer=3,
+        model_dir=model_dir,
+        config=config,
+        feature_columns=[core_feature_column.numeric_column("x")])
+
+    # Train for a few steps.
+    est.train(input_fn=_train_input_fn, steps=1000)
+    est.evaluate(input_fn=_eval_input_fn, steps=1)
+    est.predict(input_fn=_eval_input_fn)
+
+
 if __name__ == "__main__":
   googletest.main()
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/model.py b/tensorflow/contrib/boosted_trees/estimator_batch/model.py
index 0e8a56e6e9..2fbe72951a 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/model.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/model.py
@@ -29,7 +29,17 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.training import training_util
 
-def model_builder(features, labels, mode, params, config):
+class ModelBuilderOutputType(object):
+  MODEL_FN_OPS = 0
+  ESTIMATOR_SPEC = 1
+
+
+def model_builder(features,
+                  labels,
+                  mode,
+                  params,
+                  config,
+                  output_type=ModelBuilderOutputType.MODEL_FN_OPS):
   """Multi-machine batch gradient descent tree model.
 
   Args:
@@ -115,31 +125,53 @@ def model_builder(features, labels, mode, params, config):
         return update_op
 
   create_estimator_spec_op = getattr(head, "create_estimator_spec", None)
-  if use_core_libs and callable(create_estimator_spec_op):
-    model_fn_ops = head.create_estimator_spec(
-        features=features,
-        mode=mode,
-        labels=labels,
-        train_op_fn=_train_op_fn,
-        logits=logits)
-    model_fn_ops = estimator_utils.estimator_spec_to_model_fn_ops(model_fn_ops)
-  else:
-    model_fn_ops = head.create_model_fn_ops(
-        features=features,
-        mode=mode,
-        labels=labels,
-        train_op_fn=_train_op_fn,
-        logits=logits)
-  if output_leaf_index and gbdt_batch.LEAF_INDEX in predictions_dict:
-    model_fn_ops.predictions[gbdt_batch.LEAF_INDEX] = predictions_dict[
-        gbdt_batch.LEAF_INDEX]
+
   if num_trees:
     if center_bias:
       num_trees += 1
     finalized_trees, attempted_trees = gbdt_model.get_number_of_trees_tensor()
-    model_fn_ops.training_hooks.append(
+    training_hooks = [
         trainer_hooks.StopAfterNTrees(num_trees, attempted_trees,
-                                      finalized_trees))
+                                      finalized_trees)
+    ]
+
+  if output_type == ModelBuilderOutputType.MODEL_FN_OPS:
+    if use_core_libs and callable(create_estimator_spec_op):
+      model_fn_ops = head.create_estimator_spec(
+          features=features,
+          mode=mode,
+          labels=labels,
+          train_op_fn=_train_op_fn,
+          logits=logits)
+      model_fn_ops = estimator_utils.estimator_spec_to_model_fn_ops(
+          model_fn_ops)
+    else:
+      model_fn_ops = head.create_model_fn_ops(
+          features=features,
+          mode=mode,
+          labels=labels,
+          train_op_fn=_train_op_fn,
+          logits=logits)
+
+    if output_leaf_index and gbdt_batch.LEAF_INDEX in predictions_dict:
+      model_fn_ops.predictions[gbdt_batch.LEAF_INDEX] = predictions_dict[
+          gbdt_batch.LEAF_INDEX]
+
+    model_fn_ops.training_hooks.extend(training_hooks)
+    return model_fn_ops
+  elif output_type == ModelBuilderOutputType.ESTIMATOR_SPEC:
+    assert callable(create_estimator_spec_op)
+    estimator_spec = head.create_estimator_spec(
+        features=features,
+        mode=mode,
+        labels=labels,
+        train_op_fn=_train_op_fn,
+        logits=logits)
+
+    estimator_spec = estimator_spec._replace(
+        training_hooks=training_hooks + list(estimator_spec.training_hooks))
+    return estimator_spec
+
   return model_fn_ops
 
 
-- 
cgit v1.2.3


From f8e8c0c6f7746d3f2b5820e76c9e382149090034 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 14:53:58 -0700
Subject: Fix for conditional attributes crashing in static analysis

PiperOrigin-RevId: 205729321
---
 .../examples/integration_tests/keras_test.py       | 25 ++++++++++++++++++++++
 .../autograph/pyct/static_analysis/live_values.py  | 16 ++++++++++----
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/autograph/examples/integration_tests/keras_test.py b/tensorflow/contrib/autograph/examples/integration_tests/keras_test.py
index a2fc7c550e..73125eb452 100644
--- a/tensorflow/contrib/autograph/examples/integration_tests/keras_test.py
+++ b/tensorflow/contrib/autograph/examples/integration_tests/keras_test.py
@@ -20,6 +20,8 @@ from __future__ import print_function
 
 import tensorflow as tf
 
+from tensorflow.contrib import autograph
+
 
 class MinimalKeras(tf.keras.Model):
 
@@ -27,11 +29,34 @@ class MinimalKeras(tf.keras.Model):
     return x * 3
 
 
+class ModelWithStaticConditional(object):
+
+  def __init__(self, initial):
+    self.initial = initial
+    if self.initial:
+      self.h = 15
+
+  @autograph.convert()
+  def call(self):
+    x = 10
+    if self.initial:
+      x += self.h
+    return x
+
+
 class KerasTest(tf.test.TestCase):
 
   def test_basic(self):
     MinimalKeras()
 
+  def test_conditional_attributes_False(self):
+    model = ModelWithStaticConditional(False)
+    self.assertEqual(model.call(), 10)
+
+  def test_conditional_attributes_True(self):
+    model = ModelWithStaticConditional(True)
+    self.assertEqual(model.call(), 25)
+
 
 if __name__ == '__main__':
   tf.test.main()
diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py b/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py
index 32802069ba..2d8f922a45 100644
--- a/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py
@@ -91,12 +91,20 @@ class LiveValueResolver(transformer.Base):
     if anno.hasanno(node.value, 'live_val'):
       assert anno.hasanno(node.value, 'fqn')
       parent_object = anno.getanno(node.value, 'live_val')
-      if not hasattr(parent_object, node.attr):
-        raise AttributeError('%s has no attribute %s' % (parent_object,
-                                                         node.attr))
+
       anno.setanno(node, 'parent_type', type(parent_object))
-      anno.setanno(node, 'live_val', getattr(parent_object, node.attr))
       anno.setanno(node, 'fqn', anno.getanno(node.value, 'fqn') + (node.attr,))
+      if hasattr(parent_object, node.attr):
+        # This can happen when the attribute's creation and use depend on the
+        # same static condition, for example:
+        #
+        #  if cond:
+        #    foo.bar = baz
+        #  if cond:
+        #    x = foo.bar
+        #
+        anno.setanno(node, 'live_val', getattr(parent_object, node.attr))
+
     # TODO(mdan): Investigate the role built-in annotations can play here.
     elif anno.hasanno(node.value, 'type'):
       parent_type = anno.getanno(node.value, 'type')
-- 
cgit v1.2.3


From 09c4c387913c86247121589caa7fb2e85351fa58 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne 
Date: Mon, 23 Jul 2018 15:01:42 -0700
Subject: Add check at GPU initialization to see if GPU kernels can be run.

PiperOrigin-RevId: 205730535
---
 tensorflow/core/BUILD                              | 11 +++++
 tensorflow/core/common_runtime/gpu/gpu_device.cc   | 51 +++++++++++++++++++++-
 tensorflow/core/common_runtime/gpu/gpu_device.h    | 11 +++++
 .../gpu/gpu_device_kernel_check.cu.cc              | 37 ++++++++++++++++
 .../common_runtime/gpu/gpu_device_kernel_check.h   | 32 ++++++++++++++
 5 files changed, 141 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
 create mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index b6a990ac7d..13e1b643d1 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -90,6 +90,7 @@ load(
     "tf_genrule_cmd_append_to_srcs",
     "tf_opts_nortti_if_android",
     "tf_features_nomodules_if_android",
+    "tf_gpu_kernel_library",
 )
 load("//tensorflow:tensorflow.bzl", "tf_cc_test_mkl")
 load("//tensorflow:tensorflow.bzl", "tf_cc_test_gpu")
@@ -2948,6 +2949,15 @@ cc_library(
     ],
 )
 
+tf_gpu_kernel_library(
+    name = "gpu_device_kernel_check",
+    srcs = ["common_runtime/gpu/gpu_device_kernel_check.cu.cc"],
+    hdrs = ["common_runtime/gpu/gpu_device_kernel_check.h"],
+    deps = [
+        "//tensorflow/core:stream_executor",
+    ],
+)
+
 GPU_RUNTIME_HEADERS = [
     "common_runtime/gpu/cuda_host_allocator.h",
     "common_runtime/gpu/gpu_bfc_allocator.h",
@@ -2986,6 +2996,7 @@ tf_cuda_library(
         ":core_cpu_lib",
         ":framework",
         ":framework_internal",
+        ":gpu_device_kernel_check",
         ":gpu_id_impl",
         ":gpu_init_impl",
         ":gpu_lib",
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 3292ef2f62..fbe158c777 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -31,6 +31,7 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id_manager.h"
@@ -377,7 +378,7 @@ Status BaseGPUDevice::Init(const SessionOptions& options) {
     }
   }
 
-  return Status::OK();
+  return CheckGPU();
 }
 
 bool BaseGPUDevice::RequiresRecordingAccessedTensors() const {
@@ -894,6 +895,54 @@ Allocator* BaseGPUDevice::GetScopedAllocator(AllocatorAttributes attr,
   return gpu_allocator_;
 }
 
+Status BaseGPUDevice::CheckGPU() {
+  se::Stream* stream = tensorflow_gpu_device_info()->stream;
+  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
+  Tensor device_tensor(gpu_allocator_, DT_FLOAT, {});
+  if (!device_tensor.IsInitialized()) {
+    return errors::ResourceExhausted("Failed to allocate ", sizeof(float),
+                                     " bytes on the GPU for initialization "
+                                     "checks");
+  }
+  float* val_dev = device_tensor.scalar().data();
+  const cudaStream_t cu_stream = *reinterpret_cast(
+      stream->implementation()->GpuStreamMemberHack());
+  {
+    se::cuda::ScopedActivateExecutorContext scoped_activation{stream->parent()};
+    run_test_kernel(val_dev, cu_stream);
+    // We have to use the CUDA runtime function cudaPeekAtLastError here,
+    // because 'stream' does not provide a way to check if a kernel launch
+    // succeeds. Calling 'stream->BlockHostUntilDone()', which internally calls
+    // 'cuCtxSynchronize()', does not catch all kernel launch errors.
+    cudaError_t cuda_error = cudaPeekAtLastError();
+    if (cuda_error == cudaSuccess) {
+      cuda_error = cudaDeviceSynchronize();
+    }
+    TF_RETURN_IF_ERROR(CudaErrorToStatus(cuda_error, *stream));
+  }
+
+  float val_host = 0.;
+  stream->ThenMemcpy(&val_host, se::DeviceMemoryBase(val_dev, sizeof(float)),
+                     sizeof(float));
+  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
+  if (val_host != 12345.) {
+    return errors::Internal(
+        "GPU kernel for initialization returned wrong value: ", val_host);
+  }
+  return Status::OK();
+}
+
+Status BaseGPUDevice::CudaErrorToStatus(cudaError_t cuda_error,
+                                        const se::Stream& stream) {
+  if (cuda_error != cudaSuccess) {
+    return errors::Internal(
+        "Failed to run GPU kernel for the initialization check. Received "
+        "error ",
+        cudaGetErrorName(cuda_error), " after running GPU kernel.");
+  }
+  return Status::OK();
+}
+
 const int BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength = 1000;
 const int BaseGPUDeviceFactory::InterconnectMap::kStreamExecutorStrength = 1;
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index 56d03d7a8c..d02901a7ae 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -26,6 +26,7 @@ limitations under the License.
 #include 
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "cuda/include/cuda_runtime_api.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
@@ -115,6 +116,12 @@ class BaseGPUDevice : public LocalDevice {
   se::StreamExecutor* executor_;  // not owned
   std::unique_ptr scoped_allocator_mgr_;
 
+  // Returns a Status corresponding to a cudaError_t. The CUDA error must have
+  // been obtained from a CUDA kernel launch used to check if the GPU is
+  // initialized properly.
+  virtual Status CudaErrorToStatus(cudaError_t cuda_error,
+                                   const se::Stream& stream);
+
  private:
   struct StreamGroup {
     se::Stream* compute = nullptr;
@@ -151,6 +158,10 @@ class BaseGPUDevice : public LocalDevice {
   Status MaybeCopyTensorToGPU(const AllocatorAttributes& alloc_attrs,
                               const Tensor& from, Tensor* to,
                               StatusCallback done);
+
+  // Checks that the GPU is capable of doing work, by running a test kernel on
+  // it.
+  Status CheckGPU();
 };
 
 class BaseGPUDeviceFactory : public DeviceFactory {
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
new file mode 100644
index 0000000000..017565195b
--- /dev/null
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
@@ -0,0 +1,37 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
+#include "tensorflow/stream_executor/cuda/cuda_activation.h"
+
+namespace {
+__global__ void test_kernel(float* val) {
+  if (blockIdx.x == 0 && threadIdx.x == 0) {
+    (*val) = 12345.;
+  }
+}
+}  // namespace
+
+namespace tensorflow {
+
+void run_test_kernel(float* val, cudaStream_t cu_stream) {
+  test_kernel<<<1, 1, 0, cu_stream>>>(val);
+}
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
new file mode 100644
index 0000000000..064fb7a49f
--- /dev/null
+++ b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
@@ -0,0 +1,32 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/platform/stream_executor.h"
+
+namespace tensorflow {
+
+// Runs a GPU kernel to test that it functions correctly. Sets 'val' to 12345.
+void run_test_kernel(float* val, cudaStream_t cu_stream);
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-- 
cgit v1.2.3


From 931a3054d2c13c3438fc58978b3463a0bd268aee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 15:05:15 -0700
Subject: [tfgan] Issue #18041: Make pooling consistent in `gan_loss`.

PiperOrigin-RevId: 205731279
---
 tensorflow/contrib/gan/python/train.py      | 56 ++++++++++++++---------
 tensorflow/contrib/gan/python/train_test.py | 71 +++++++++++------------------
 2 files changed, 61 insertions(+), 66 deletions(-)

diff --git a/tensorflow/contrib/gan/python/train.py b/tensorflow/contrib/gan/python/train.py
index 49d9327333..df603d1f18 100644
--- a/tensorflow/contrib/gan/python/train.py
+++ b/tensorflow/contrib/gan/python/train.py
@@ -514,33 +514,42 @@ def _tensor_pool_adjusted_model(model, tensor_pool_fn):
   Raises:
     ValueError: If tensor pool does not support the `model`.
   """
-  if tensor_pool_fn is None:
-    return model
-
-  pooled_generated_data, pooled_generator_inputs = tensor_pool_fn(
-      (model.generated_data, model.generator_inputs))
-
   if isinstance(model, namedtuples.GANModel):
+    pooled_generator_inputs, pooled_generated_data = tensor_pool_fn(
+        (model.generator_inputs, model.generated_data))
     with variable_scope.variable_scope(model.discriminator_scope, reuse=True):
       dis_gen_outputs = model.discriminator_fn(pooled_generated_data,
                                                pooled_generator_inputs)
-    return model._replace(discriminator_gen_outputs=dis_gen_outputs)
+    return model._replace(
+        generator_inputs=pooled_generator_inputs,
+        generated_data=pooled_generated_data,
+        discriminator_gen_outputs=dis_gen_outputs)
   elif isinstance(model, namedtuples.ACGANModel):
+    pooled_generator_inputs, pooled_generated_data = tensor_pool_fn(
+        (model.generator_inputs, model.generated_data))
     with variable_scope.variable_scope(model.discriminator_scope, reuse=True):
-      (dis_pooled_gen_outputs,
-       dis_pooled_gen_classification_logits) = model.discriminator_fn(
+      (pooled_discriminator_gen_outputs,
+       pooled_discriminator_gen_classification_logits) = model.discriminator_fn(
            pooled_generated_data, pooled_generator_inputs)
     return model._replace(
-        discriminator_gen_outputs=dis_pooled_gen_outputs,
+        generator_inputs=pooled_generator_inputs,
+        generated_data=pooled_generated_data,
+        discriminator_gen_outputs=pooled_discriminator_gen_outputs,
         discriminator_gen_classification_logits=
-        dis_pooled_gen_classification_logits)
+        pooled_discriminator_gen_classification_logits)
   elif isinstance(model, namedtuples.InfoGANModel):
+    pooled_generator_inputs, pooled_generated_data, pooled_structured_input = (
+        tensor_pool_fn((model.generator_inputs, model.generated_data,
+                        model.structured_generator_inputs)))
     with variable_scope.variable_scope(model.discriminator_scope, reuse=True):
-      (dis_pooled_gen_outputs,
+      (pooled_discriminator_gen_outputs,
        pooled_predicted_distributions) = model.discriminator_and_aux_fn(
            pooled_generated_data, pooled_generator_inputs)
     return model._replace(
-        discriminator_gen_outputs=dis_pooled_gen_outputs,
+        generator_inputs=pooled_generator_inputs,
+        generated_data=pooled_generated_data,
+        structured_generator_inputs=pooled_structured_input,
+        discriminator_gen_outputs=pooled_discriminator_gen_outputs,
         predicted_distributions=pooled_predicted_distributions)
   else:
     raise ValueError('Tensor pool does not support `model`: %s.' % type(model))
@@ -632,33 +641,38 @@ def gan_loss(
         'is provided, `model` must be an `ACGANModel`. Instead, was %s.' %
         type(model))
 
+  # Optionally create pooled model.
+  pooled_model = (_tensor_pool_adjusted_model(model, tensor_pool_fn) if
+                  tensor_pool_fn else model)
+
   # Create standard losses.
   gen_loss = generator_loss_fn(model, add_summaries=add_summaries)
-  dis_loss = discriminator_loss_fn(
-      _tensor_pool_adjusted_model(model, tensor_pool_fn),
-      add_summaries=add_summaries)
+  dis_loss = discriminator_loss_fn(pooled_model, add_summaries=add_summaries)
 
   # Add optional extra losses.
   if _use_aux_loss(gradient_penalty_weight):
     gp_loss = tfgan_losses.wasserstein_gradient_penalty(
-        model,
+        pooled_model,
         epsilon=gradient_penalty_epsilon,
         target=gradient_penalty_target,
         one_sided=gradient_penalty_one_sided,
         add_summaries=add_summaries)
     dis_loss += gradient_penalty_weight * gp_loss
   if _use_aux_loss(mutual_information_penalty_weight):
-    info_loss = tfgan_losses.mutual_information_penalty(
+    gen_info_loss = tfgan_losses.mutual_information_penalty(
         model, add_summaries=add_summaries)
-    dis_loss += mutual_information_penalty_weight * info_loss
-    gen_loss += mutual_information_penalty_weight * info_loss
+    dis_info_loss = (gen_info_loss if tensor_pool_fn is None else
+                     tfgan_losses.mutual_information_penalty(
+                         pooled_model, add_summaries=add_summaries))
+    gen_loss += mutual_information_penalty_weight * gen_info_loss
+    dis_loss += mutual_information_penalty_weight * dis_info_loss
   if _use_aux_loss(aux_cond_generator_weight):
     ac_gen_loss = tfgan_losses.acgan_generator_loss(
         model, add_summaries=add_summaries)
     gen_loss += aux_cond_generator_weight * ac_gen_loss
   if _use_aux_loss(aux_cond_discriminator_weight):
     ac_disc_loss = tfgan_losses.acgan_discriminator_loss(
-        model, add_summaries=add_summaries)
+        pooled_model, add_summaries=add_summaries)
     dis_loss += aux_cond_discriminator_weight * ac_disc_loss
   # Gathers auxiliary losses.
   if model.generator_scope:
diff --git a/tensorflow/contrib/gan/python/train_test.py b/tensorflow/contrib/gan/python/train_test.py
index cd99a33c03..fa52e9cca1 100644
--- a/tensorflow/contrib/gan/python/train_test.py
+++ b/tensorflow/contrib/gan/python/train_test.py
@@ -278,25 +278,6 @@ def get_sync_optimizer():
       replicas_to_aggregate=1)
 
 
-def get_tensor_pool_fn(pool_size):
-
-  def tensor_pool_fn_impl(input_values):
-    return random_tensor_pool.tensor_pool(input_values, pool_size=pool_size)
-
-  return tensor_pool_fn_impl
-
-
-def get_tensor_pool_fn_for_infogan(pool_size):
-
-  def tensor_pool_fn_impl(input_values):
-    generated_data, generator_inputs = input_values
-    output_values = random_tensor_pool.tensor_pool(
-        [generated_data] + generator_inputs, pool_size=pool_size)
-    return output_values[0], output_values[1:]
-
-  return tensor_pool_fn_impl
-
-
 class GANModelTest(test.TestCase, parameterized.TestCase):
   """Tests for `gan_model`."""
 
@@ -344,7 +325,6 @@ class StarGANModelTest(test.TestCase):
 
   @staticmethod
   def create_input_and_label_tensor(batch_size, img_size, c_size, num_domains):
-
     input_tensor_list = []
     label_tensor_list = []
     for _ in range(num_domains):
@@ -356,7 +336,6 @@ class StarGANModelTest(test.TestCase):
     return input_tensor_list, label_tensor_list
 
   def test_generate_stargan_random_domain_target(self):
-
     batch_size = 8
     domain_numbers = 3
 
@@ -371,7 +350,6 @@ class StarGANModelTest(test.TestCase):
         self.assertEqual(1, np.max(target))
 
   def test_stargan_model_output_type(self):
-
     batch_size = 2
     img_size = 16
     c_size = 3
@@ -395,7 +373,6 @@ class StarGANModelTest(test.TestCase):
     self.assertTrue(callable(model.generator_fn))
 
   def test_stargan_model_generator_output(self):
-
     batch_size = 2
     img_size = 16
     c_size = 3
@@ -426,7 +403,6 @@ class StarGANModelTest(test.TestCase):
           reconstructed_data.shape)
 
   def test_stargan_model_discriminator_output(self):
-
     batch_size = 2
     img_size = 16
     c_size = 3
@@ -643,10 +619,7 @@ class GANLossTest(test.TestCase, parameterized.TestCase):
   def test_tensor_pool(self, create_gan_model_fn):
     """Test tensor pool option."""
     model = create_gan_model_fn()
-    if isinstance(model, namedtuples.InfoGANModel):
-      tensor_pool_fn = get_tensor_pool_fn_for_infogan(pool_size=5)
-    else:
-      tensor_pool_fn = get_tensor_pool_fn(pool_size=5)
+    tensor_pool_fn = lambda x: random_tensor_pool.tensor_pool(x, pool_size=5)
     loss = train.gan_loss(model, tensor_pool_fn=tensor_pool_fn)
     self.assertIsInstance(loss, namedtuples.GANLoss)
 
@@ -656,6 +629,25 @@ class GANLossTest(test.TestCase, parameterized.TestCase):
       for _ in range(10):
         sess.run([loss.generator_loss, loss.discriminator_loss])
 
+  def test_discriminator_only_sees_pool(self):
+    """Checks that discriminator only sees pooled values."""
+    def checker_gen_fn(_):
+      return constant_op.constant(0.0)
+    model = train.gan_model(
+        checker_gen_fn,
+        discriminator_model,
+        real_data=array_ops.zeros([]),
+        generator_inputs=random_ops.random_normal([]))
+    def tensor_pool_fn(_):
+      return (random_ops.random_uniform([]), random_ops.random_uniform([]))
+    def checker_dis_fn(inputs, _):
+      """Discriminator that checks that it only sees pooled Tensors."""
+      self.assertFalse(constant_op.is_constant(inputs))
+      return inputs
+    model = model._replace(
+        discriminator_fn=checker_dis_fn)
+    train.gan_loss(model, tensor_pool_fn=tensor_pool_fn)
+
   def test_doesnt_crash_when_in_nested_scope(self):
     with variable_scope.variable_scope('outer_scope'):
       gan_model = train.gan_model(
@@ -673,8 +665,8 @@ class GANLossTest(test.TestCase, parameterized.TestCase):
 
 class TensorPoolAdjusteModelTest(test.TestCase):
 
-  def _check_tensor_pool_adjusted_model_outputs(self, tensor1, tensor2,
-                                                pool_size):
+  def _check_tensor_pool_adjusted_model_outputs(
+      self, tensor1, tensor2, pool_size):
     history_values = []
     with self.test_session(use_gpu=True) as sess:
       variables.global_variables_initializer().run()
@@ -691,10 +683,9 @@ class TensorPoolAdjusteModelTest(test.TestCase):
           # pool).
           self.assertTrue(any([(v == t2).all() for v in history_values]))
 
-  def _make_new_model_and_check(self, model, pool_size,
-                                pool_fn=get_tensor_pool_fn):
-    new_model = train._tensor_pool_adjusted_model(
-        model, pool_fn(pool_size=pool_size))
+  def _make_new_model_and_check(self, model, pool_size):
+    pool_fn = lambda x: random_tensor_pool.tensor_pool(x, pool_size=pool_size)
+    new_model = train._tensor_pool_adjusted_model(model, pool_fn)
     # 'Generator/dummy_g:0' and 'Discriminator/dummy_d:0'
     self.assertEqual(2, len(ops.get_collection(ops.GraphKeys.VARIABLES)))
     self.assertIsNot(new_model.discriminator_gen_outputs,
@@ -702,15 +693,6 @@ class TensorPoolAdjusteModelTest(test.TestCase):
 
     return new_model
 
-  def test_tensor_pool_adjusted_model_no_pool(self):
-    """Test `_tensor_pool_adjusted_model` for no pool size."""
-    model = create_gan_model()
-    new_model = train._tensor_pool_adjusted_model(model, None)
-
-    # Check values.
-    self.assertIs(new_model.discriminator_gen_outputs,
-                  model.discriminator_gen_outputs)
-
   def test_tensor_pool_adjusted_model_gan(self):
     """Test `_tensor_pool_adjusted_model` for gan model."""
     pool_size = 5
@@ -726,8 +708,7 @@ class TensorPoolAdjusteModelTest(test.TestCase):
     """Test _tensor_pool_adjusted_model for infogan model."""
     pool_size = 5
     model = create_infogan_model()
-    new_model = self._make_new_model_and_check(
-        model, pool_size, pool_fn=get_tensor_pool_fn_for_infogan)
+    new_model = self._make_new_model_and_check(model, pool_size)
 
     # Check values.
     self.assertIsNot(new_model.predicted_distributions,
-- 
cgit v1.2.3


From f85d825500357603afb7a02d2c88ad306ee43006 Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Mon, 23 Jul 2018 15:11:22 -0700
Subject: Allow differentiating tfe.defun functions which contain conds.

PiperOrigin-RevId: 205732423
---
 tensorflow/core/common_runtime/direct_session.cc    |  3 ++-
 tensorflow/core/common_runtime/function.cc          | 19 +++++++++++--------
 tensorflow/core/framework/function.cc               |  5 ++++-
 tensorflow/core/framework/function.h                |  8 +++++++-
 tensorflow/core/kernels/partitioned_function_ops.cc |  1 +
 tensorflow/python/eager/function_test.py            | 13 +++++++++++++
 6 files changed, 38 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 44291b0b20..d1fd930d25 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -717,7 +717,8 @@ Status DirectSession::Run(const RunOptions& run_options,
   // Receive outputs.
   if (outputs) {
     std::vector sorted_outputs;
-    const Status s = call_frame.ConsumeRetvals(&sorted_outputs);
+    const Status s = call_frame.ConsumeRetvals(
+        &sorted_outputs, /* allow_dead_tensors = */ false);
     if (errors::IsInternal(s)) {
       return errors::InvalidArgument(s.error_message());
     } else if (!s.ok()) {
diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index a93cfa2ec5..54bbe84b57 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -746,6 +746,8 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle,
     rets_alloc_attrs.push_back(ret_alloc_attrs);
   }
 
+  bool allow_dead_tensors = opts.allow_dead_tensors;
+
   // The ProcFLR sends the arguments to the function from the source_device to
   // the target_device. So here we receive those arguments. Similarly, when the
   // computation is done and stored in *rets, we send the return values back
@@ -756,7 +758,7 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle,
       device_context, args_alloc_attrs, rendezvous, remote_args,
       [frame, remote_args, item, source_device, target_device,
        target_incarnation, rendezvous, device_context, rets, done, exec_args,
-       rets_alloc_attrs](const Status& status) {
+       rets_alloc_attrs, allow_dead_tensors](const Status& status) {
         Status s = status;
         if (s.ok()) {
           s = frame->SetArgs(*remote_args);
@@ -769,13 +771,13 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle,
           return;
         }
         item->exec->RunAsync(
-            *exec_args,
-            [frame, rets, done, source_device, target_device,
-             target_incarnation, rendezvous, device_context, remote_args,
-             exec_args, rets_alloc_attrs](const Status& status) {
+            *exec_args, [frame, rets, done, source_device, target_device,
+                         target_incarnation, rendezvous, device_context,
+                         remote_args, exec_args, rets_alloc_attrs,
+                         allow_dead_tensors](const Status& status) {
               Status s = status;
               if (s.ok()) {
-                s = frame->ConsumeRetvals(rets);
+                s = frame->ConsumeRetvals(rets, allow_dead_tensors);
               }
               delete frame;
               if (!s.ok()) {
@@ -859,14 +861,15 @@ void FunctionLibraryRuntimeImpl::Run(const Options& opts, Handle handle,
     return;
   }
 
+  bool allow_dead_tensors = opts.allow_dead_tensors;
   item->exec->RunAsync(
       // Executor args
       *exec_args,
       // Done callback.
-      [frame, rets, done, exec_args](const Status& status) {
+      [frame, rets, done, exec_args, allow_dead_tensors](const Status& status) {
         Status s = status;
         if (s.ok()) {
-          s = frame->ConsumeRetvals(rets);
+          s = frame->ConsumeRetvals(rets, allow_dead_tensors);
         }
         delete frame;
         delete exec_args;
diff --git a/tensorflow/core/framework/function.cc b/tensorflow/core/framework/function.cc
index 88d9d65f5a..57bcc0f513 100644
--- a/tensorflow/core/framework/function.cc
+++ b/tensorflow/core/framework/function.cc
@@ -865,12 +865,15 @@ Status FunctionCallFrame::GetRetvals(std::vector* rets) const {
   return Status::OK();
 }
 
-Status FunctionCallFrame::ConsumeRetvals(std::vector* rets) {
+Status FunctionCallFrame::ConsumeRetvals(std::vector* rets,
+                                         bool allow_dead_tensors) {
   rets->clear();
   rets->reserve(rets_.size());
   for (size_t i = 0; i < rets_.size(); ++i) {
     if (rets_[i].has_val) {
       rets->emplace_back(std::move(rets_[i].val));
+    } else if (allow_dead_tensors) {
+      rets->emplace_back();
     } else {
       return errors::Internal("Retval[", i, "] does not have value");
     }
diff --git a/tensorflow/core/framework/function.h b/tensorflow/core/framework/function.h
index 8e607b927c..5da9af7db3 100644
--- a/tensorflow/core/framework/function.h
+++ b/tensorflow/core/framework/function.h
@@ -261,7 +261,10 @@ class FunctionCallFrame : public CallFrameInterface {
   // Caller methods.
   Status SetArgs(gtl::ArraySlice args);
   Status GetRetvals(std::vector* rets) const;
-  Status ConsumeRetvals(std::vector* rets);
+
+  // Moves the return values from the frame to rets. If allow_dead_tensors is
+  // false it will fail if any of the retvals do not have a value.
+  Status ConsumeRetvals(std::vector* rets, bool allow_dead_tensors);
 
   size_t num_args() const override { return arg_types_.size(); }
   size_t num_retvals() const override { return ret_types_.size(); }
@@ -510,6 +513,9 @@ class FunctionLibraryRuntime {
     // If true, we create a new IntraProcessRendezvous, else use the existing
     // one.
     bool create_rendezvous = false;
+
+    // If True, allow returning dead tensors.
+    bool allow_dead_tensors = false;
   };
   typedef std::function DoneCallback;
   virtual void Run(const Options& opts, Handle handle,
diff --git a/tensorflow/core/kernels/partitioned_function_ops.cc b/tensorflow/core/kernels/partitioned_function_ops.cc
index b5c6ba1da3..a7a9609c21 100644
--- a/tensorflow/core/kernels/partitioned_function_ops.cc
+++ b/tensorflow/core/kernels/partitioned_function_ops.cc
@@ -330,6 +330,7 @@ class PartitionedCallOp : public AsyncOpKernel {
     // using device-specific threadpools when available.
     opts.runner = ctx->runner();
     opts.source_device = local_device_name_;
+    opts.allow_dead_tensors = true;
     // TODO(akshayka): Accommodate the multiple-worker scenario by adding the
     // constructed rendezvous to a rendezvous manager.
     Rendezvous* rendez = new IntraProcessRendezvous(lib->device_mgr());
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index e6592b2e37..2e86563a7d 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -213,6 +213,19 @@ class FunctionTest(test.TestCase):
     self.assertEqual(fn_op.output_shapes, None)
     self.assertAllEqual(fn_op(x, x), None)
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testDefunCondGradient(self):
+
+    @function.defun
+    def f(x):
+      return control_flow_ops.cond(x > 0.5, lambda: 2 * x, lambda: 3 * x)
+
+    with backprop.GradientTape() as t:
+      x = constant_op.constant(1.0)
+      t.watch(x)
+      y = f(x)
+    self.assertAllEqual(self.evaluate(t.gradient(y, x)), 2.0)
+
   def testDefunCapturedInt32(self):
     x = constant_op.constant(1, dtype=dtypes.int32)
 
-- 
cgit v1.2.3


From ed0f543bb03e684cf8f1fa8da3666876fcb43674 Mon Sep 17 00:00:00 2001
From: Xuechen Li 
Date: Mon, 23 Jul 2018 15:15:07 -0700
Subject: Add defun option.

PiperOrigin-RevId: 205732970
---
 tensorflow/contrib/eager/python/examples/revnet/main.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/revnet/main.py b/tensorflow/contrib/eager/python/examples/revnet/main.py
index 1a4fd45c8b..dcd4e1697f 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/main.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/main.py
@@ -46,6 +46,9 @@ def main(_):
   checkpointer = tf.train.Checkpoint(
       optimizer=optimizer, model=model, optimizer_step=global_step)
 
+  if FLAGS.use_defun:
+    model.call = tfe.defun(model.call)
+
   if FLAGS.train_dir:
     summary_writer = tf.contrib.summary.create_file_writer(FLAGS.train_dir)
     if FLAGS.restore:
@@ -69,7 +72,7 @@ def main(_):
         acc_validation, loss_validation = evaluate(model, it_validation)
         print("Iter {}, "
               "training set accuracy {:.4f}, loss {:.4f}; "
-              "validation set accuracy {:.4f}, loss {:4.f}"
+              "validation set accuracy {:.4f}, loss {:.4f}; "
               "test accuracy {:.4f}, loss {:.4f}".format(
                   global_step.numpy(), acc_train, loss_train, acc_validation,
                   loss_validation, acc_test, loss_test))
@@ -81,11 +84,11 @@ def main(_):
       if FLAGS.train_dir:
         with summary_writer.as_default():
           with tf.contrib.summary.always_record_summaries():
-            tf.contrib.summary.scalar("Training accuracy", acc_train)
             tf.contrib.summary.scalar("Test accuracy", acc_test)
-            tf.contrib.summary.scalar("Training loss", loss_train)
             tf.contrib.summary.scalar("Test loss", loss_test)
             if FLAGS.validate:
+              tf.contrib.summary.scalar("Training accuracy", acc_train)
+              tf.contrib.summary.scalar("Training loss", loss_train)
               tf.contrib.summary.scalar("Validation accuracy", acc_validation)
               tf.contrib.summary.scalar("Validation loss", loss_validation)
 
@@ -240,5 +243,9 @@ if __name__ == "__main__":
       default="revnet-38",
       help="[Optional] Architecture of network. "
       "Other options include `revnet-110` and `revnet-164`")
+  flags.DEFINE_boolean(
+      "use_defun",
+      default=False,
+      help="[Optional] Use `tfe.defun` to boost performance.")
   FLAGS = flags.FLAGS
   tf.app.run(main)
-- 
cgit v1.2.3


From 9affd713580f5f6cf81511c82e1d94bcd3c84e32 Mon Sep 17 00:00:00 2001
From: Xuechen Li 
Date: Mon, 23 Jul 2018 15:17:14 -0700
Subject: Add README for l2hmc.

PiperOrigin-RevId: 205733306
---
 .../contrib/eager/python/examples/l2hmc/README.md  | 54 ++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 tensorflow/contrib/eager/python/examples/l2hmc/README.md

diff --git a/tensorflow/contrib/eager/python/examples/l2hmc/README.md b/tensorflow/contrib/eager/python/examples/l2hmc/README.md
new file mode 100644
index 0000000000..d6a2ff7558
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/l2hmc/README.md
@@ -0,0 +1,54 @@
+# L2HMC with TensorFlow eager execution
+
+This folder contains an implementation of [L2HMC](https://arxiv.org/pdf/1711.09268.pdf)  adapted from the released implementation by the authors. The presented implementation runs in both eager and graph mode.
+With eager execution enabled, longer sample chains can be handled compared to graph mode, since no graph is explicitly stored. Moreover, with eager execution enabled, there is no need to use a `tf.while_loop`.
+
+## What is L2HMC?
+L2HMC is an algorithm that learns a non-volume preserving transformation
+for an HMC-like sampling algorithm. More specifically, the non-volume preserving
+transformation is learned with neural nets instantiated within Normalizing Flows
+(more precisely, real-NVPs).
+
+##  Content
+
+- `l2hmc.py`: Dynamics definitions and example energy functions,
+including the 2D strongly correlated Gaussian, the rough well energy function,
+and a Gaussian mixture model.
+- `l2hmc_test.py`: Unit tests and benchmarks for training a sampler on the energy functions in both eager and graph mode.
+- `neural_nets.py`: The neural net for learning the kernel on the 2D strongly correlated example.
+- `main.py`: Run to train a samplers on 2D energy landscapes.
+
+## To run
+- Make sure you have installed TensorFlow 1.9+ or the latest `tf-nightly` or `tf-nightly-gpu` pip package.
+- Execute the command
+
+```bash
+python main.py --train_dir ${PWD}/dump --use_defun
+```
+
+Specifying the optional argument `train_dir` will store event files for
+tensorboard and a plot of sampled chain from the trained sampler.
+
+Specifying the optional argument `use_defun` will let the program use compiled
+graphs when running specific sections and improve the overall speed.
+
+## Boosting Performance with `defun`
+Currently, some models may experience increased overhead with eager execution enabled.
+To improve performance, we could wrap certain functions with the decorator `@tfe.defun`.
+For example, we could wrap the function that does the sampling step:
+
+```python
+@tfe.defun
+def apply_transition(old_sample):
+  new_sample = ...
+  return new_sample
+```
+
+We could also explicitly wrap the desired function with `tfe.defun`:
+
+```python
+apply_transition = tfe.defun(apply_transition)
+```
+
+## Reference
+Generalizing Hamiltonian Monte Carlo with Neural Networks. Levy, Daniel, Hoffman, Matthew D, and Sohl-Dickstein, Jascha. International Conference on Learning Representations (ICLR), 2018.
-- 
cgit v1.2.3


From 9225bbbe0aaaa14b69176576097bb67bae98e6c5 Mon Sep 17 00:00:00 2001
From: Blake Hechtman 
Date: Mon, 23 Jul 2018 16:02:19 -0700
Subject: [XLA] Simplify slice(slice()) to a single slice and simplify
 Reduce(Concat()) to Accumulate(Reudce(),Reduce(),...Reduce())

PiperOrigin-RevId: 205740411
---
 .../compiler/xla/service/algebraic_simplifier.cc   | 39 ++++++++++
 .../xla/service/algebraic_simplifier_test.cc       | 82 ++++++++++++++++++++++
 2 files changed, 121 insertions(+)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 2205a7ec18..26a8a67601 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1744,6 +1744,25 @@ Status AlgebraicSimplifierVisitor::HandleSlice(HloInstruction* slice) {
   if (ReplaceInstructionIfSameShape(slice, slice->mutable_operand(0))) {
     return Status::OK();
   }
+
+  auto is_unstrided_slice = [](const HloInstruction* hlo) {
+    return c_all_of(hlo->slice_strides(),
+                    [](int64 stride) { return stride == 1; });
+  };
+  if (slice->operand(0)->opcode() == HloOpcode::kSlice &&
+      is_unstrided_slice(slice) && is_unstrided_slice(slice->operand(0))) {
+    HloInstruction* operand_slice = slice->mutable_operand(0);
+    std::vector new_slice_starts = slice->slice_starts();
+    std::vector new_slice_limits = slice->slice_limits();
+    for (int64 i = 0; i < new_slice_starts.size(); ++i) {
+      new_slice_starts[i] += operand_slice->slice_starts(i);
+      new_slice_limits[i] += operand_slice->slice_starts(i);
+    }
+    return ReplaceWithNewInstruction(
+        slice, HloInstruction::CreateSlice(
+                   slice->shape(), operand_slice->mutable_operand(0),
+                   new_slice_starts, new_slice_limits, slice->slice_strides()));
+  }
   return Status::OK();
 }
 
@@ -1904,6 +1923,26 @@ Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) {
                       new_reduce_dimensions, function));
     }
   }
+  // Convert Reduce(concat({a,b,...})) to
+  //  map(reduce(a),map(reduce(b),...,))
+  //
+  // This should make fusion easier or use less memory bandwidth in the unfused
+  // case.
+  if (arg->opcode() == HloOpcode::kConcatenate &&
+      c_linear_search(reduce->dimensions(), arg->concatenate_dimension())) {
+    HloInstruction* old_reduce = nullptr;
+    for (HloInstruction* operand : arg->operands()) {
+      HloInstruction* new_reduce = computation_->AddInstruction(
+          HloInstruction::CreateReduce(reduce->shape(), operand, init_value,
+                                       reduce->dimensions(), function));
+      if (old_reduce != nullptr) {
+        new_reduce = computation_->AddInstruction(HloInstruction::CreateMap(
+            reduce->shape(), {old_reduce, new_reduce}, function));
+      }
+      old_reduce = new_reduce;
+    }
+    return ReplaceInstruction(reduce, old_reduce);
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 3f0f2afadd..ddf0a513c0 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -1250,6 +1250,55 @@ TEST_F(AlgebraicSimplifierTest, RemoveEmptyConcatenateOperands) {
               op::Concatenate(param0, param0, param1));
 }
 
+// Test that reduce of concat is simplified.
+TEST_F(AlgebraicSimplifierTest, SimplifyReduceOfConcat) {
+  const int kParamLength = 100;
+  Shape r3f32 =
+      ShapeUtil::MakeShape(F32, {kParamLength, kParamLength, kParamLength});
+  HloComputation::Builder builder(TestName());
+  HloInstruction* param0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, r3f32, "param0"));
+  HloInstruction* param1 = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, r3f32, "param1"));
+  HloInstruction* param2 = builder.AddInstruction(
+      HloInstruction::CreateParameter(2, r3f32, "param2"));
+  Shape concat_shape =
+      ShapeUtil::MakeShape(F32, {kParamLength, 3 * kParamLength, kParamLength});
+  HloInstruction* Concatenate =
+      builder.AddInstruction(HloInstruction::CreateConcatenate(
+          concat_shape, {param0, param1, param2}, 1));
+  HloComputation* add_computation = nullptr;
+  {
+    HloComputation::Builder builder(TestName() + ".add");
+    const Shape scalar_shape = ShapeUtil::MakeShape(F32, {});
+    HloInstruction* p0 = builder.AddInstruction(
+        HloInstruction::CreateParameter(0, scalar_shape, "p0"));
+    HloInstruction* p1 = builder.AddInstruction(
+        HloInstruction::CreateParameter(1, scalar_shape, "p1"));
+    builder.AddInstruction(
+        HloInstruction::CreateBinary(scalar_shape, HloOpcode::kAdd, p0, p1));
+    add_computation = module().AddEmbeddedComputation(builder.Build());
+  }
+  Shape r4f32 = ShapeUtil::MakeShape(F32, {4, 5, 6, 7});
+  Shape reduce_shape = ShapeUtil::MakeShape(F32, {kParamLength});
+
+  HloInstruction* zero = builder.AddInstruction(
+      HloInstruction::CreateConstant(LiteralUtil::CreateR0(0)));
+  builder.AddInstruction(HloInstruction::CreateReduce(
+      reduce_shape, Concatenate, zero, {1, 2}, add_computation));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(&module()).ValueOrDie());
+
+  EXPECT_THAT(
+      computation->root_instruction(),
+      op::Map(op::Map(op::Reduce(param0, zero), op::Reduce(param1, zero)),
+              op::Reduce(param2, zero)));
+}
+
 // Test a concatenate with only empty operands is removed.
 TEST_F(AlgebraicSimplifierTest, OnlyEmptyConcatenateOperands) {
   const int kParamLength = 100;
@@ -1859,6 +1908,39 @@ TEST_F(AlgebraicSimplifierTest, RemoveNoopSlice) {
   EXPECT_THAT(computation->root_instruction(), param);
 }
 
+TEST_F(AlgebraicSimplifierTest, SliceOfSliceToSlice) {
+  HloComputation::Builder builder(TestName());
+  const int64 dim0 = 11;
+  const int64 dim1 = 12;
+  HloInstruction* param =
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          0, ShapeUtil::MakeShape(F32, {dim0, dim1}), "param"));
+  HloInstruction* original_slice =
+      builder.AddInstruction(HloInstruction::CreateSlice(
+          ShapeUtil::MakeShape(F32, {dim0 - 2, dim1 - 4}), param,
+          /*start_indices=*/{1, 2},
+          /*limit_indices=*/{dim0 - 1, dim1 - 2}, /*strides=*/{1, 1}));
+
+  builder.AddInstruction(HloInstruction::CreateSlice(
+      ShapeUtil::MakeShape(F32, {dim0 - 5, dim1 - 9}), original_slice,
+      /*start_indices=*/{2, 3},
+      /*limit_indices=*/{dim0 - 3, dim1 - 6}, /*strides=*/{1, 1}));
+  auto module = CreateNewModule();
+  HloComputation* computation = module->AddEntryComputation(builder.Build());
+
+  EXPECT_THAT(computation->root_instruction(), op::Slice(op::Slice(param)));
+
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(module).ValueOrDie());
+
+  EXPECT_THAT(computation->root_instruction(), op::Slice(param));
+  EXPECT_EQ(computation->root_instruction()->slice_starts(0), 3);
+  EXPECT_EQ(computation->root_instruction()->slice_starts(1), 5);
+  EXPECT_EQ(computation->root_instruction()->slice_limits(0), dim0 - 2);
+  EXPECT_EQ(computation->root_instruction()->slice_limits(1), dim1 - 4);
+}
+
 TEST_F(AlgebraicSimplifierTest, ConvertConvToMatmul) {
   struct ConvTestOptions {
     int in_batch = 10;
-- 
cgit v1.2.3


From 632e48c27e09b53ab52523149e759f9bc1711e71 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Mon, 23 Jul 2018 16:17:12 -0700
Subject: Teach StreamExecutor to load modules and resolve symbols in them

This will be used in a future CL.

PiperOrigin-RevId: 205742731
---
 .../stream_executor/cuda/cuda_gpu_executor.cc      | 179 ++++++++++++++-------
 .../stream_executor/cuda/cuda_gpu_executor.h       |  16 +-
 tensorflow/stream_executor/module_spec.h           |  65 ++++++++
 .../stream_executor/stream_executor_internal.h     |  32 +++-
 .../stream_executor/stream_executor_pimpl.cc       |  38 ++++-
 tensorflow/stream_executor/stream_executor_pimpl.h |  76 +++++++--
 6 files changed, 331 insertions(+), 75 deletions(-)
 create mode 100644 tensorflow/stream_executor/module_spec.h

diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 259c813c57..73f05b94db 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -206,6 +206,48 @@ static string GetBinaryDir(bool strip_exe) {
   return exe_path;
 }
 
+bool CUDAExecutor::LoadModuleFromCuBin(const char *cubin, CUmodule *module) {
+  uint64_t module_refcount;
+  std::tie(*module, module_refcount) = gpu_binary_to_module_[cubin];
+
+  if (*module == nullptr) {
+    auto load_status = CUDADriver::LoadCubin(context_, cubin, module);
+    if (!load_status.ok()) {
+      LOG(ERROR) << "failed to load CUBIN: " << load_status;
+      return false;
+    }
+    module_refcount = 1;
+    VLOG(3) << "Loaded CUBIN " << static_cast(cubin)
+            << " as module " << *module;
+  } else {
+    ++module_refcount;
+    VLOG(3) << "CUBIN " << static_cast(cubin)
+            << " is already loaded as module " << *module;
+  }
+  gpu_binary_to_module_[cubin] = {*module, module_refcount};
+  return true;
+}
+
+bool CUDAExecutor::LoadModuleFromPtx(const char *ptx, CUmodule *module) {
+  uint64_t module_refcount;
+  std::tie(*module, module_refcount) = gpu_binary_to_module_[ptx];
+
+  if (*module == nullptr) {
+    if (!CUDADriver::LoadPtx(context_, ptx, module)) {
+      return false;
+    }
+    VLOG(3) << "Loaded PTX " << static_cast(ptx) << " as module "
+            << *module;
+    module_refcount = 1;
+  } else {
+    ++module_refcount;
+    VLOG(3) << "PTX " << static_cast(ptx)
+            << " is already loaded as module " << module;
+  }
+  gpu_binary_to_module_[ptx] = {*module, module_refcount};
+  return true;
+}
+
 bool CUDAExecutor::GetKernel(const MultiKernelLoaderSpec &spec,
                              KernelBase *kernel) {
   CUDAKernel *cuda_kernel = AsCUDAKernel(kernel);
@@ -215,28 +257,13 @@ bool CUDAExecutor::GetKernel(const MultiKernelLoaderSpec &spec,
   VLOG(3) << "GetKernel on kernel " << kernel << " : " << kernel->name();
 
   if (spec.has_cuda_cubin_in_memory()) {
+    mutex_lock lock{in_memory_modules_mu_};
     kernelname = &spec.cuda_cubin_in_memory().kernelname();
     const char *cubin = spec.cuda_cubin_in_memory().bytes();
-    mutex_lock lock{in_memory_modules_mu_};
-    uint64_t module_refcount;
-    std::tie(module, module_refcount) = gpu_binary_to_module_[cubin];
-
-    if (module == nullptr) {
-      auto load_status = CUDADriver::LoadCubin(context_, cubin, &module);
-      if (!load_status.ok()) {
-        LOG(ERROR) << "failed to load CUBIN: " << load_status;
-        return false;
-      }
-      module_refcount = 1;
-      VLOG(3) << "Loaded CUBIN " << static_cast(cubin)
-              << " as module " << module;
-    } else {
-      ++module_refcount;
-      VLOG(3) << "CUBIN " << static_cast(cubin)
-              << " is already loaded as module " << module;
+    if (!LoadModuleFromCuBin(cubin, &module)) {
+      return false;
     }
     kernel_to_gpu_binary_[kernel] = cubin;
-    gpu_binary_to_module_[cubin] = {module, module_refcount};
   } else if (spec.has_cuda_ptx_in_memory()) {
     kernelname = &spec.cuda_ptx_in_memory().kernelname();
 
@@ -254,24 +281,10 @@ bool CUDAExecutor::GetKernel(const MultiKernelLoaderSpec &spec,
     }
 
     mutex_lock lock{in_memory_modules_mu_};
-    uint64_t module_refcount;
-    std::tie(module, module_refcount) = gpu_binary_to_module_[ptx];
-
-    if (module == nullptr) {
-      if (!CUDADriver::LoadPtx(context_, ptx, &module)) {
-        LOG(ERROR) << "failed to load PTX for kernel " << *kernelname;
-        return false;
-      }
-      VLOG(3) << "Loaded PTX " << static_cast(ptx)
-              << " as module " << module;
-      module_refcount = 1;
-    } else {
-      ++module_refcount;
-      VLOG(3) << "PTX " << static_cast(ptx)
-              << " is already loaded as module " << module;
+    if (!LoadModuleFromPtx(ptx, &module)) {
+      return false;
     }
     kernel_to_gpu_binary_[kernel] = ptx;
-    gpu_binary_to_module_[ptx] = {module, module_refcount};
   } else {
     LOG(WARNING) << "no method of loading CUDA kernel provided";
     return false;
@@ -295,6 +308,23 @@ bool CUDAExecutor::GetKernel(const MultiKernelLoaderSpec &spec,
   return true;
 }
 
+bool CUDAExecutor::UnloadGpuBinary(const void *gpu_binary) {
+  auto module_it = gpu_binary_to_module_.find(gpu_binary);
+  if (gpu_binary_to_module_.end() == module_it) {
+    VLOG(3) << "No loaded CUDA module for " << gpu_binary;
+    return false;
+  }
+  auto &module = module_it->second.first;
+  auto &refcount = module_it->second.second;
+  VLOG(3) << "Found CUDA module " << module << " with refcount " << refcount;
+  if (--refcount == 0) {
+    VLOG(3) << "Unloading CUDA module " << module;
+    CUDADriver::UnloadModule(context_, module);
+    gpu_binary_to_module_.erase(module_it);
+  }
+  return true;
+}
+
 void CUDAExecutor::UnloadKernel(const KernelBase *kernel) {
   VLOG(3) << "Unloading kernel " << kernel << " : " << kernel->name();
 
@@ -307,25 +337,52 @@ void CUDAExecutor::UnloadKernel(const KernelBase *kernel) {
   }
   VLOG(3) << "Kernel " << kernel << " : " << kernel->name()
           << " has loaded GPU code " << gpu_binary_it->second;
-  auto module_it = gpu_binary_to_module_.find(gpu_binary_it->second);
-  if (gpu_binary_to_module_.end() == module_it) {
-    VLOG(3) << "Kernel " << kernel << " : " << kernel->name()
-            << " has no loaded CUDA module.";
-    return;  // This kernel never loaded any modules
-  }
-  auto &module = module_it->second.first;
-  auto &refcount = module_it->second.second;
-  VLOG(3) << "Kernel " << kernel << " : " << kernel->name()
-          << " has loaded GPU code " << gpu_binary_it->second
-          << " into CUDA module " << module << " with refcount " << refcount;
-  if (--refcount == 0) {
-    VLOG(3) << "Unloading CUDA module " << module;
-    CUDADriver::UnloadModule(context_, module);
-    gpu_binary_to_module_.erase(module_it);
-  }
+  UnloadGpuBinary(gpu_binary_it->second);
   kernel_to_gpu_binary_.erase(gpu_binary_it);
 }
 
+bool CUDAExecutor::LoadModule(const MultiModuleLoaderSpec &spec,
+                              ModuleHandle *module_handle) {
+  // In CUDAExecutor we store the pointer to the GPU binary (PTX or CUBIN) as
+  // ModuleHandle::id().
+  CUmodule cu_module;
+  if (spec.has_cuda_cubin_in_memory()) {
+    mutex_lock lock{in_memory_modules_mu_};
+    if (!LoadModuleFromCuBin(
+            reinterpret_cast(spec.cuda_cubin_in_memory().data()),
+            &cu_module)) {
+      return false;
+    }
+    *module_handle = ModuleHandle(const_cast(
+        static_cast(spec.cuda_cubin_in_memory().data())));
+    return true;
+  } else if (spec.has_cuda_ptx_in_memory()) {
+    if (cc_major_ == 0 && cc_minor_ == 0) {
+      return false;
+    }
+
+    if (!spec.cuda_ptx_in_memory()) {
+      return false;
+    }
+
+    mutex_lock lock{in_memory_modules_mu_};
+    if (!LoadModuleFromPtx(spec.cuda_ptx_in_memory(), &cu_module)) {
+      return false;
+    }
+    *module_handle = ModuleHandle(const_cast(
+        static_cast(spec.cuda_ptx_in_memory())));
+    return true;
+  }
+  LOG(WARNING) << "no method of loading CUDA module provided";
+  return false;
+}
+
+bool CUDAExecutor::UnloadModule(ModuleHandle module_handle) {
+  const char *gpu_binary = reinterpret_cast(module_handle.id());
+  mutex_lock lock{in_memory_modules_mu_};
+  return UnloadGpuBinary(gpu_binary);
+}
+
 bool CUDAExecutor::GetKernelMetadata(CUDAKernel *cuda_kernel,
                                      KernelMetadata *kernel_metadata) {
   int value;
@@ -783,16 +840,26 @@ bool CUDAExecutor::DeviceMemoryUsage(int64 *free, int64 *total) const {
   return CUDADriver::GetDeviceMemoryInfo(context_, free, total);
 }
 
-bool CUDAExecutor::GetSymbol(const string& symbol_name, void **mem,
+bool CUDAExecutor::GetSymbol(const string &symbol_name,
+                             ModuleHandle module_handle, void **mem,
                              size_t *bytes) {
+  auto lookup_in_module = [&](CUmodule module) {
+    CHECK(module != nullptr);
+    return CUDADriver::GetModuleSymbol(context_, module, symbol_name.c_str(),
+                                       reinterpret_cast(mem),
+                                       bytes);
+  };
+
   {  // give limited scope to mutex_lock
     mutex_lock lock{in_memory_modules_mu_};
+    if (static_cast(module_handle)) {
+      auto it = gpu_binary_to_module_.find(module_handle.id());
+      CHECK(it != gpu_binary_to_module_.end());
+      return lookup_in_module(it->second.first);
+    }
+
     for (auto &it : gpu_binary_to_module_) {
-      CUmodule module = it.second.first;
-      CHECK(module != nullptr);
-      if (CUDADriver::GetModuleSymbol(context_, module, symbol_name.c_str(),
-                                      reinterpret_cast(mem),
-                                      bytes)) {
+      if (lookup_in_module(it.second.first)) {
         return true;
       }
     }
diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h
index f7c341c857..8a954d5461 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.h
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.h
@@ -62,6 +62,9 @@ class CUDAExecutor : public internal::StreamExecutorInterface {
   bool GetKernel(const MultiKernelLoaderSpec &spec,
                  KernelBase *kernel) override;
   void UnloadKernel(const KernelBase *kernel) override;
+  bool LoadModule(const MultiModuleLoaderSpec &spec,
+                  ModuleHandle *module_handle) override;
+  bool UnloadModule(ModuleHandle module_handle) override;
 
   bool Launch(Stream *stream, const ThreadDim &thread_dims,
               const BlockDim &block_dims, const KernelBase &k,
@@ -175,7 +178,8 @@ class CUDAExecutor : public internal::StreamExecutorInterface {
 
   // Search for the symbol and returns a device pointer and size.
   // Returns false if symbol does not exist.
-  bool GetSymbol(const string& symbol_name, void **mem, size_t *bytes) override;
+  bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
+                 void **mem, size_t *bytes) override;
 
   DeviceDescription *PopulateDeviceDescription() const override;
 
@@ -239,6 +243,16 @@ class CUDAExecutor : public internal::StreamExecutorInterface {
   void VlogOccupancyInfo(const KernelBase &kernel, const ThreadDim &thread_dims,
                          const BlockDim &block_dims);
 
+  bool LoadModuleFromCuBin(const char *cubin, CUmodule *module)
+      EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
+
+  // Loads the PTX text `ptx` as a CUDA module.  `ptx` must be null terminated.
+  bool LoadModuleFromPtx(const char *ptx, CUmodule *module)
+      EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
+
+  bool UnloadGpuBinary(const void *gpu_binary)
+      EXCLUSIVE_LOCKS_REQUIRED(in_memory_modules_mu_);
+
   // Guards the in-memory-module mapping.
   mutex in_memory_modules_mu_;
 
diff --git a/tensorflow/stream_executor/module_spec.h b/tensorflow/stream_executor/module_spec.h
new file mode 100644
index 0000000000..212ae7ba9c
--- /dev/null
+++ b/tensorflow/stream_executor/module_spec.h
@@ -0,0 +1,65 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_MODULE_SPEC_H_
+#define TENSORFLOW_STREAM_EXECUTOR_MODULE_SPEC_H_
+
+#include "tensorflow/stream_executor/lib/array_slice.h"
+#include "tensorflow/stream_executor/lib/stringpiece.h"
+#include "tensorflow/stream_executor/platform/logging.h"
+#include "tensorflow/stream_executor/platform/port.h"
+
+namespace stream_executor {
+
+// Describes how to load a module on a target platform.
+//
+// The exact meaning of a "module" may differ from platform to platform but
+// loosely speaking a module a collection of kernels and global variables.  It
+// corresponds to CUmodule when running on CUDA.
+class MultiModuleLoaderSpec {
+ public:
+  bool has_cuda_cubin_in_memory() const { return has_cuda_cubin_in_memory_; }
+  port::ArraySlice cuda_cubin_in_memory() const {
+    CHECK(has_cuda_cubin_in_memory());
+    return {cuda_cubin_in_memory_.data(), cuda_cubin_in_memory_.size()};
+  }
+
+  bool has_cuda_ptx_in_memory() const { return has_cuda_ptx_in_memory_; }
+  const char* cuda_ptx_in_memory() const {
+    CHECK(has_cuda_ptx_in_memory());
+    return cuda_ptx_in_memory_;
+  }
+
+  void AddCudaCubinInMemory(port::ArraySlice cubin_bytes) {
+    has_cuda_cubin_in_memory_ = true;
+    cuda_cubin_in_memory_ = cubin_bytes;
+  }
+
+  void AddCudaPtxInMemory(const char* ptx) {
+    has_cuda_ptx_in_memory_ = true;
+    // The CUDA driver does not like getting an empty string as PTX.
+    cuda_ptx_in_memory_ = *ptx ? ptx : nullptr;
+  }
+
+ private:
+  port::ArraySlice cuda_cubin_in_memory_;
+  bool has_cuda_cubin_in_memory_ = false;
+  const char* cuda_ptx_in_memory_;
+  bool has_cuda_ptx_in_memory_ = false;
+};
+
+}  // namespace stream_executor
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_MODULE_SPEC_H_
diff --git a/tensorflow/stream_executor/stream_executor_internal.h b/tensorflow/stream_executor/stream_executor_internal.h
index fb1b92cb84..f34b1fc083 100644
--- a/tensorflow/stream_executor/stream_executor_internal.h
+++ b/tensorflow/stream_executor/stream_executor_internal.h
@@ -36,20 +36,38 @@ limitations under the License.
 #include "tensorflow/stream_executor/kernel_cache_config.h"
 #include "tensorflow/stream_executor/kernel_spec.h"
 #include "tensorflow/stream_executor/launch_dim.h"
+#include "tensorflow/stream_executor/lib/inlined_vector.h"
 #include "tensorflow/stream_executor/lib/status.h"
 #include "tensorflow/stream_executor/lib/statusor.h"
+#include "tensorflow/stream_executor/module_spec.h"
 #include "tensorflow/stream_executor/platform.h"
 #include "tensorflow/stream_executor/platform/port.h"
 #include "tensorflow/stream_executor/plugin_registry.h"
 #include "tensorflow/stream_executor/shared_memory_config.h"
 #include "tensorflow/stream_executor/trace_listener.h"
-#include "tensorflow/stream_executor/lib/inlined_vector.h"
 
 namespace stream_executor {
 
 class Stream;
 class Timer;
 
+// An opaque handle to a loaded module.
+//
+// An instance of this is returned from StreamExecutor::GetModule.
+class ModuleHandle {
+ public:
+  /*implicit*/ ModuleHandle(void *id = nullptr) : id_(id) {}
+
+  // A ModuleHandle with id() == nullptr is an invalid module handle, akin to a
+  // null pointer.
+  void *id() const { return id_; }
+
+  explicit operator bool() const { return id() != nullptr; }
+
+ private:
+  void *id_;
+};
+
 namespace internal {
 
 // Platform-dependent interface class for the generic Events interface, in
@@ -164,6 +182,11 @@ class StreamExecutorInterface {
                          KernelBase *kernel) {
     return false;
   }
+  virtual bool LoadModule(const MultiModuleLoaderSpec &spec,
+                          ModuleHandle *module_handle) {
+    return false;
+  }
+  virtual bool UnloadModule(ModuleHandle module_handle) { return false; }
   virtual bool Launch(Stream *stream, const ThreadDim &thread_dims,
                       const BlockDim &block_dims, const KernelBase &k,
                       const KernelArgsArrayBase &args) {
@@ -247,7 +270,12 @@ class StreamExecutorInterface {
   // null, however, both of them cannot be null at the same time. To use
   // constant memory in CUDA, GetSymbol has to be used. Returns true if symbol
   // is found.
-  virtual bool GetSymbol(const string& symbol_name, void **mem, size_t *bytes) {
+  //
+  // If ModuleHandle is set then we search for `symbol_name` only within the
+  // module corresponding to `module_handle`.  Otherwise all loaded modules are
+  // searched.
+  virtual bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
+                         void **mem, size_t *bytes) {
     return false;
   }
 
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.cc b/tensorflow/stream_executor/stream_executor_pimpl.cc
index 000795ff00..2e0137a485 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.cc
+++ b/tensorflow/stream_executor/stream_executor_pimpl.cc
@@ -220,6 +220,15 @@ void StreamExecutor::UnloadKernel(const KernelBase *kernel) {
   implementation_->UnloadKernel(kernel);
 }
 
+bool StreamExecutor::LoadModule(const MultiModuleLoaderSpec &spec,
+                                ModuleHandle *module_handle) {
+  return implementation_->LoadModule(spec, module_handle);
+}
+
+bool StreamExecutor::UnloadModule(ModuleHandle module_handle) {
+  return implementation_->UnloadModule(module_handle);
+}
+
 void StreamExecutor::Deallocate(DeviceMemoryBase *mem) {
   VLOG(1) << "Called StreamExecutor::Deallocate(mem=" << mem->opaque()
           << ") mem->size()=" << mem->size() << StackTraceIfVLOG10();
@@ -459,9 +468,34 @@ void *StreamExecutor::Allocate(uint64 size) {
   return buf;
 }
 
-bool StreamExecutor::GetSymbol(const string &symbol_name, void **mem,
+port::StatusOr StreamExecutor::GetUntypedSymbol(
+    const string &symbol_name, ModuleHandle module_handle) {
+  // If failed to get the symbol, opaque/bytes are unchanged. Initialize them to
+  // be nullptr/0 for consistency with DeviceMemory semantics.
+  void *opaque = nullptr;
+  size_t bytes = 0;
+  if (GetSymbol(symbol_name, module_handle, &opaque, &bytes)) {
+    return DeviceMemoryBase(opaque, bytes);
+  }
+
+  if (static_cast(module_handle)) {
+    return port::Status(
+        port::error::NOT_FOUND,
+        port::StrCat("Check if module containing symbol ", symbol_name,
+                     " is loaded (module_handle = ",
+                     reinterpret_cast(module_handle.id()), ")"));
+  } else {
+    return port::Status(
+        port::error::NOT_FOUND,
+        port::StrCat("Check if kernel using the symbol is loaded: ",
+                     symbol_name));
+  }
+}
+
+bool StreamExecutor::GetSymbol(const string &symbol_name,
+                               ModuleHandle module_handle, void **mem,
                                size_t *bytes) {
-  return implementation_->GetSymbol(symbol_name, mem, bytes);
+  return implementation_->GetSymbol(symbol_name, module_handle, mem, bytes);
 }
 
 void *StreamExecutor::UnifiedMemoryAllocate(uint64 bytes) {
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.h b/tensorflow/stream_executor/stream_executor_pimpl.h
index ad80a1ba25..47b3a2b030 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.h
+++ b/tensorflow/stream_executor/stream_executor_pimpl.h
@@ -106,6 +106,16 @@ class StreamExecutor {
   // Releases any state associated with the previously loaded kernel.
   void UnloadKernel(const KernelBase *kernel);
 
+  // Loads a module for the platform this StreamExecutor is acting upon.
+  //
+  // `spec` describes the module to be loaded.  On success writes the handle for
+  // the loaded module to `module_handle` and returns true.  Else returns false.
+  bool LoadModule(const MultiModuleLoaderSpec &spec,
+                  ModuleHandle *module_handle);
+
+  // Unloads the module with handle `module_handle`.
+  bool UnloadModule(ModuleHandle module_handle);
+
   // Synchronously allocates an array on the device of type T with element_count
   // elements.
   template 
@@ -169,8 +179,16 @@ class StreamExecutor {
   // type of symbol and T match.
   // - Note: symbol_name should include its namespace as well. For example,
   //         pass "nms0::symbol" if referring to nms0::symbol.
+  //
+  // If `module_handle` is set then searches only within the module
+  // corresponding to `module_handle`.
   template 
-  port::StatusOr> GetSymbol(const string &symbol_name);
+  port::StatusOr> GetSymbol(const string &symbol_name,
+                                            ModuleHandle module_handle = {});
+
+  // An untyped version of GetSymbol.
+  port::StatusOr GetUntypedSymbol(
+      const string &symbol_name, ModuleHandle module_handle = {});
 
   // Deallocate the DeviceMemory previously allocated via this interface.
   // Deallocation of a nullptr-representative value is permitted.
@@ -507,7 +525,8 @@ class StreamExecutor {
 
   // Finds and retrieves device memory for the symbol on the underlying
   // platform.
-  bool GetSymbol(const string& symbol_name, void **mem, size_t *bytes);
+  bool GetSymbol(const string &symbol_name, ModuleHandle module_handle,
+                 void **mem, size_t *bytes);
 
   // Entrains a memcpy operation onto stream, with a host destination location
   // host_dst and a device memory source, with target size size.
@@ -678,6 +697,41 @@ class StreamExecutor {
   SE_DISALLOW_COPY_AND_ASSIGN(StreamExecutor);
 };
 
+// A wrapper around ModuleHandle that uses RAII to manage its lifetime.
+class ScopedModuleHandle {
+ public:
+  explicit ScopedModuleHandle(StreamExecutor *executor,
+                              ModuleHandle module_handle)
+      : executor_(executor), module_handle_(module_handle) {}
+
+  ScopedModuleHandle(ScopedModuleHandle &&other) {
+    executor_ = other.executor_;
+    module_handle_ = other.module_handle_;
+    other.executor_ = nullptr;
+    other.module_handle_ = ModuleHandle();
+  }
+
+  ScopedModuleHandle &operator=(ScopedModuleHandle &&other) {
+    executor_ = other.executor_;
+    module_handle_ = other.module_handle_;
+    other.executor_ = nullptr;
+    other.module_handle_ = ModuleHandle();
+    return *this;
+  }
+
+  ~ScopedModuleHandle() {
+    if (static_cast(module_handle_)) {
+      CHECK(executor_->UnloadModule(module_handle_));
+    }
+  }
+
+ private:
+  StreamExecutor *executor_;
+  ModuleHandle module_handle_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(ScopedModuleHandle);
+};
+
 ////////////
 // Inlines
 
@@ -690,19 +744,13 @@ inline DeviceMemory StreamExecutor::AllocateArray(uint64 element_count) {
 
 template 
 inline port::StatusOr> StreamExecutor::GetSymbol(
-    const string &symbol_name) {
-  // If failed to get the symbol, opaque/bytes are unchanged. Initialize them to
-  // be nullptr/0 for consistency with DeviceMemory semantics.
-  void *opaque = nullptr;
-  size_t bytes = 0;
-  if (GetSymbol(symbol_name, &opaque, &bytes)) {
-    CHECK_EQ(bytes % sizeof(T), 0);
-    return DeviceMemory::MakeFromByteSize(opaque, bytes);
+    const string &symbol_name, ModuleHandle module_handle) {
+  port::StatusOr untyped_symbol =
+      GetUntypedSymbol(symbol_name, module_handle);
+  if (!untyped_symbol.ok()) {
+    return untyped_symbol.status();
   }
-  return port::Status(
-      port::error::NOT_FOUND,
-      port::StrCat("Check if kernel using the symbol is loaded: ",
-                   symbol_name));
+  return DeviceMemory(untyped_symbol.ValueOrDie());
 }
 
 template 
-- 
cgit v1.2.3


From 806105c2f5c43cee58ab997b1822286bc3f15ad7 Mon Sep 17 00:00:00 2001
From: Jared Duke 
Date: Mon, 23 Jul 2018 16:29:59 -0700
Subject: Remove unnecessary  includes

PiperOrigin-RevId: 205744600
---
 tensorflow/contrib/lite/kernels/activations.cc                  | 1 -
 tensorflow/contrib/lite/kernels/bidirectional_sequence_lstm.cc  | 1 -
 tensorflow/contrib/lite/kernels/bidirectional_sequence_rnn.cc   | 1 -
 tensorflow/contrib/lite/kernels/concatenation.cc                | 1 -
 tensorflow/contrib/lite/kernels/conv.cc                         | 1 -
 tensorflow/contrib/lite/kernels/depthwise_conv.cc               | 1 -
 tensorflow/contrib/lite/kernels/embedding_lookup.cc             | 1 -
 tensorflow/contrib/lite/kernels/fully_connected.cc              | 1 -
 tensorflow/contrib/lite/kernels/hashtable_lookup.cc             | 1 -
 tensorflow/contrib/lite/kernels/lsh_projection.cc               | 1 -
 tensorflow/contrib/lite/kernels/lstm.cc                         | 1 -
 tensorflow/contrib/lite/kernels/pooling.cc                      | 1 -
 tensorflow/contrib/lite/kernels/sparse_to_dense.cc              | 1 -
 tensorflow/contrib/lite/kernels/svdf.cc                         | 1 -
 tensorflow/contrib/lite/kernels/transpose_conv.cc               | 1 -
 tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc | 1 -
 tensorflow/contrib/lite/kernels/unidirectional_sequence_rnn.cc  | 1 -
 tensorflow/contrib/lite/model.cc                                | 1 -
 tensorflow/contrib/lite/model_test.cc                           | 1 -
 19 files changed, 19 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/activations.cc b/tensorflow/contrib/lite/kernels/activations.cc
index d5ac2a7814..6e13b8c667 100644
--- a/tensorflow/contrib/lite/kernels/activations.cc
+++ b/tensorflow/contrib/lite/kernels/activations.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/bidirectional_sequence_lstm.cc b/tensorflow/contrib/lite/kernels/bidirectional_sequence_lstm.cc
index 14a19aeef3..a11a59aa05 100644
--- a/tensorflow/contrib/lite/kernels/bidirectional_sequence_lstm.cc
+++ b/tensorflow/contrib/lite/kernels/bidirectional_sequence_lstm.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/bidirectional_sequence_rnn.cc b/tensorflow/contrib/lite/kernels/bidirectional_sequence_rnn.cc
index aa24c1f34c..517309a226 100644
--- a/tensorflow/contrib/lite/kernels/bidirectional_sequence_rnn.cc
+++ b/tensorflow/contrib/lite/kernels/bidirectional_sequence_rnn.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/concatenation.cc b/tensorflow/contrib/lite/kernels/concatenation.cc
index 45ea8d0049..ad211e9c67 100644
--- a/tensorflow/contrib/lite/kernels/concatenation.cc
+++ b/tensorflow/contrib/lite/kernels/concatenation.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/conv.cc b/tensorflow/contrib/lite/kernels/conv.cc
index a4fe9e5550..6f174763df 100644
--- a/tensorflow/contrib/lite/kernels/conv.cc
+++ b/tensorflow/contrib/lite/kernels/conv.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/depthwise_conv.cc b/tensorflow/contrib/lite/kernels/depthwise_conv.cc
index 16e5f1d065..21518156b8 100644
--- a/tensorflow/contrib/lite/kernels/depthwise_conv.cc
+++ b/tensorflow/contrib/lite/kernels/depthwise_conv.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/embedding_lookup.cc b/tensorflow/contrib/lite/kernels/embedding_lookup.cc
index f550339d03..b2dff87e62 100644
--- a/tensorflow/contrib/lite/kernels/embedding_lookup.cc
+++ b/tensorflow/contrib/lite/kernels/embedding_lookup.cc
@@ -29,7 +29,6 @@ limitations under the License.
 //   When indices are out of bound, the ops will not succeed.
 //
 
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/fully_connected.cc b/tensorflow/contrib/lite/kernels/fully_connected.cc
index d6e297a66a..bc370608c0 100644
--- a/tensorflow/contrib/lite/kernels/fully_connected.cc
+++ b/tensorflow/contrib/lite/kernels/fully_connected.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/hashtable_lookup.cc b/tensorflow/contrib/lite/kernels/hashtable_lookup.cc
index 41211d41aa..f37c66acb3 100644
--- a/tensorflow/contrib/lite/kernels/hashtable_lookup.cc
+++ b/tensorflow/contrib/lite/kernels/hashtable_lookup.cc
@@ -31,7 +31,6 @@ limitations under the License.
 //   Each item indicates whether the corresponding lookup has a returned value.
 //   0 for missing key, 1 for found key.
 
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/lsh_projection.cc b/tensorflow/contrib/lite/kernels/lsh_projection.cc
index 25d2dc2cdd..69523b02cc 100644
--- a/tensorflow/contrib/lite/kernels/lsh_projection.cc
+++ b/tensorflow/contrib/lite/kernels/lsh_projection.cc
@@ -50,7 +50,6 @@ limitations under the License.
 //     Output.Dim == { Tensor[0].Dim[0] * Tensor[0].Dim[1] }
 //     A flattened tensor represents projected bit vectors.
 
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/lstm.cc b/tensorflow/contrib/lite/kernels/lstm.cc
index 50487a8d59..ba251c451e 100644
--- a/tensorflow/contrib/lite/kernels/lstm.cc
+++ b/tensorflow/contrib/lite/kernels/lstm.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/pooling.cc b/tensorflow/contrib/lite/kernels/pooling.cc
index 9b0487ae16..29a5be0683 100644
--- a/tensorflow/contrib/lite/kernels/pooling.cc
+++ b/tensorflow/contrib/lite/kernels/pooling.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/sparse_to_dense.cc b/tensorflow/contrib/lite/kernels/sparse_to_dense.cc
index 404c32ad9c..7be5e66c16 100644
--- a/tensorflow/contrib/lite/kernels/sparse_to_dense.cc
+++ b/tensorflow/contrib/lite/kernels/sparse_to_dense.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/svdf.cc b/tensorflow/contrib/lite/kernels/svdf.cc
index 179c2dc266..6d4912ce3a 100644
--- a/tensorflow/contrib/lite/kernels/svdf.cc
+++ b/tensorflow/contrib/lite/kernels/svdf.cc
@@ -16,7 +16,6 @@ limitations under the License.
 // SVDF op that compresses a fully connected op via low-rank matrix
 // factorization. See https://research.google.com/pubs/archive/43813.pdf for
 // details.
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/transpose_conv.cc b/tensorflow/contrib/lite/kernels/transpose_conv.cc
index 8b9deeed20..a9baa5c698 100644
--- a/tensorflow/contrib/lite/kernels/transpose_conv.cc
+++ b/tensorflow/contrib/lite/kernels/transpose_conv.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc b/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
index c48b470f92..0acd705950 100644
--- a/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
+++ b/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
@@ -13,7 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/kernels/unidirectional_sequence_rnn.cc b/tensorflow/contrib/lite/kernels/unidirectional_sequence_rnn.cc
index 164a0cbd08..0d6d29a171 100644
--- a/tensorflow/contrib/lite/kernels/unidirectional_sequence_rnn.cc
+++ b/tensorflow/contrib/lite/kernels/unidirectional_sequence_rnn.cc
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include 
 #include 
 #include 
 #include 
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index 5e6106a87e..d318591b49 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -19,7 +19,6 @@ limitations under the License.
 #include 
 #include 
 #include 
-#include 
 
 #include "tensorflow/contrib/lite/allocation.h"
 #include "tensorflow/contrib/lite/builtin_op_data.h"
diff --git a/tensorflow/contrib/lite/model_test.cc b/tensorflow/contrib/lite/model_test.cc
index 15bae21a41..edfdec9315 100644
--- a/tensorflow/contrib/lite/model_test.cc
+++ b/tensorflow/contrib/lite/model_test.cc
@@ -19,7 +19,6 @@ limitations under the License.
 #include 
 #include 
 #include 
-#include 
 
 #include "tensorflow/contrib/lite/model.h"
 
-- 
cgit v1.2.3


From 7087243b8594faa92b274b92d586cbb2d3b24bfe Mon Sep 17 00:00:00 2001
From: Karmel Allison 
Date: Mon, 23 Jul 2018 16:40:25 -0700
Subject: Automated rollback of commit cf94a46c34f8568608d78b77e9a1c4369ebcafa2

PiperOrigin-RevId: 205746329
---
 tensorflow/cc/saved_model/loader.cc               | 39 +++++++++---
 tensorflow/python/saved_model/builder_impl.py     | 76 +++++++++++------------
 tensorflow/python/saved_model/loader_impl.py      | 42 +++++++++----
 tensorflow/python/saved_model/saved_model_test.py | 28 +++------
 4 files changed, 106 insertions(+), 79 deletions(-)

diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index d47b025743..07807ed2f3 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -86,11 +86,10 @@ bool HasMainOp(const MetaGraphDef& meta_graph_def) {
 Status RunMainOp(const RunOptions& run_options, const string& export_dir,
                  const MetaGraphDef& meta_graph_def,
                  const std::vector& asset_file_defs,
-                 Session* session, const string& main_op_key) {
-  LOG(INFO) << "Running MainOp with key " << main_op_key
-            << " on SavedModel bundle.";
+                 Session* session) {
+  LOG(INFO) << "Running MainOp on SavedModel bundle.";
   const auto& collection_def_map = meta_graph_def.collection_def();
-  const auto main_op_it = collection_def_map.find(main_op_key);
+  const auto main_op_it = collection_def_map.find(kSavedModelMainOpKey);
   if (main_op_it != collection_def_map.end()) {
     if (main_op_it->second.node_list().value_size() != 1) {
       return errors::FailedPrecondition(
@@ -142,6 +141,30 @@ Status RunRestore(const RunOptions& run_options, const string& export_dir,
                       nullptr /* outputs */, &run_metadata);
 }
 
+Status RunLegacyInitOp(const RunOptions& run_options, const string& export_dir,
+                       const MetaGraphDef& meta_graph_def,
+                       const std::vector& asset_file_defs,
+                       Session* session) {
+  LOG(INFO) << "Running LegacyInitOp on SavedModel bundle.";
+  const auto& collection_def_map = meta_graph_def.collection_def();
+  const auto init_op_it = collection_def_map.find(kSavedModelLegacyInitOpKey);
+  if (init_op_it != collection_def_map.end()) {
+    if (init_op_it->second.node_list().value_size() != 1) {
+      return errors::FailedPrecondition(strings::StrCat(
+          "Expected exactly one serving init op in : ", export_dir));
+    }
+    std::vector> inputs;
+    AddAssetsTensorsToInputs(export_dir, asset_file_defs, &inputs);
+    RunMetadata run_metadata;
+    const StringPiece legacy_init_op_name =
+        init_op_it->second.node_list().value(0);
+    return session->Run(run_options, inputs, {},
+                        {legacy_init_op_name.ToString()}, nullptr /* outputs */,
+                        &run_metadata);
+  }
+  return Status::OK();
+}
+
 Status GetAssetFileDefs(const MetaGraphDef& meta_graph_def,
                         std::vector* asset_file_defs) {
   const auto& collection_def_map = meta_graph_def.collection_def();
@@ -181,11 +204,11 @@ Status LoadSavedModelInternal(const SessionOptions& session_options,
   if (HasMainOp(bundle->meta_graph_def)) {
     TF_RETURN_IF_ERROR(RunMainOp(run_options, export_dir,
                                  bundle->meta_graph_def, asset_file_defs,
-                                 bundle->session.get(), kSavedModelMainOpKey));
+                                 bundle->session.get()));
   } else {
-    TF_RETURN_IF_ERROR(RunMainOp(
-        run_options, export_dir, bundle->meta_graph_def, asset_file_defs,
-        bundle->session.get(), kSavedModelLegacyInitOpKey));
+    TF_RETURN_IF_ERROR(RunLegacyInitOp(run_options, export_dir,
+                                       bundle->meta_graph_def, asset_file_defs,
+                                       bundle->session.get()));
   }
   return Status::OK();
 }
diff --git a/tensorflow/python/saved_model/builder_impl.py b/tensorflow/python/saved_model/builder_impl.py
index 8c985a7c2f..e58be804c2 100644
--- a/tensorflow/python/saved_model/builder_impl.py
+++ b/tensorflow/python/saved_model/builder_impl.py
@@ -34,7 +34,6 @@ from tensorflow.python.platform import tf_logging
 from tensorflow.python.saved_model import constants
 from tensorflow.python.training import saver as tf_saver
 from tensorflow.python.util import compat
-from tensorflow.python.util.deprecation import deprecated_args
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -134,32 +133,39 @@ class SavedModelBuilder(object):
     tf_logging.info("Assets written to: %s",
                     compat.as_text(assets_destination_dir))
 
-  def _maybe_add_main_op(self, main_op):
-    """Adds main op to the SavedModel.
+  def _maybe_add_legacy_init_op(self, legacy_init_op=None):
+    """Add legacy init op to the SavedModel.
 
     Args:
-      main_op: Main op to run as part of graph initialization. If None, no
-        main op will be added to the graph.
+      legacy_init_op: Optional legacy init op to support backward compatibility.
 
     Raises:
-      TypeError: if main op is provided but is not of type `Operation`.
-      ValueError: if the Graph already contains an init op.
+      TypeError if legacy init op is not of type `Operation`.
+      AssertionError if the graph already contains one or more legacy init ops.
     """
-    if main_op is None:
-      return
-
-    if not isinstance(main_op, ops.Operation):
-      raise TypeError("main_op needs to be an Operation: %r" % main_op)
+    if legacy_init_op is not None:
+      if not isinstance(legacy_init_op, ops.Operation):
+        raise TypeError("legacy_init_op needs to be an Operation: %r" %
+                        legacy_init_op)
+      if ops.get_collection(constants.LEGACY_INIT_OP_KEY):
+        raise AssertionError(
+            "graph already contains one or more legacy init ops under the "
+            "collection {}.".format(constants.LEGACY_INIT_OP_KEY))
+      ops.add_to_collection(constants.LEGACY_INIT_OP_KEY, legacy_init_op)
+
+  def _add_main_op(self, main_op):
+    """Add main op to the SavedModel.
 
-    # Validate that no other init ops have been added to this graph already.
-    # We check main_op and legacy_init_op for thoroughness and explicitness.
-    for init_op_key in (constants.MAIN_OP_KEY, constants.LEGACY_INIT_OP_KEY):
-      if ops.get_collection(init_op_key):
-        raise ValueError(
-            "Graph already contains one or more main ops under the "
-            "collection {}.".format(init_op_key))
+    Args:
+      main_op: Main op to run as part of graph initialization.
 
-    ops.add_to_collection(constants.MAIN_OP_KEY, main_op)
+    Raises:
+      TypeError if main op is not of type `Operation`.
+    """
+    if main_op is not None:
+      if not isinstance(main_op, ops.Operation):
+        raise TypeError("main_op needs to be an Operation: %r" % main_op)
+      ops.add_to_collection(constants.MAIN_OP_KEY, main_op)
 
   def _add_train_op(self, train_op):
     """Add train op to the SavedModel.
@@ -251,12 +257,16 @@ class SavedModelBuilder(object):
           self._validate_tensor_info(outputs[outputs_key])
 
   def _add_collections(
-      self, assets_collection, main_op, train_op):
+      self, assets_collection, legacy_init_op, main_op, train_op):
     """Add asset and op collections to be saved."""
     # Save asset files and write them to disk, if any.
     self._save_and_write_assets(assets_collection)
 
-    self._maybe_add_main_op(main_op)
+    if main_op is None:
+      # Add legacy init op to the SavedModel.
+      self._maybe_add_legacy_init_op(legacy_init_op)
+    else:
+      self._add_main_op(main_op)
 
     self._add_train_op(train_op)
 
@@ -272,9 +282,6 @@ class SavedModelBuilder(object):
           allow_empty=True)
     return saver
 
-  @deprecated_args(None,
-                   "Pass your op to the equivalent parameter main_op instead.",
-                   "legacy_init_op")
   def add_meta_graph(self,
                      tags,
                      signature_def_map=None,
@@ -299,7 +306,7 @@ class SavedModelBuilder(object):
           that this collection should be a subset of the assets saved as part of
           the first meta graph in the SavedModel.
       legacy_init_op: Legacy support for op or group of ops to execute after the
-          restore op upon a load. Deprecated; please use main_op instead.
+          restore op upon a load.
       clear_devices: Set to true if the device info on the default graph should
           be cleared.
       main_op: Op or group of ops to execute when the graph is loaded. Note
@@ -326,12 +333,8 @@ class SavedModelBuilder(object):
     # properly populated.
     self._validate_signature_def_map(signature_def_map)
 
-    # legacy_init_op is deprecated, and going away in TF 2.0.
-    # Re-mapping to main_op, as treatment is identical regardless.
-    main_op = main_op or legacy_init_op
-
     # Add assets and ops
-    self._add_collections(assets_collection, main_op, None)
+    self._add_collections(assets_collection, legacy_init_op, main_op, None)
 
     saver = self._maybe_create_saver(saver)
 
@@ -348,9 +351,6 @@ class SavedModelBuilder(object):
     # Tag the meta graph def and add it to the SavedModel.
     self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
 
-  @deprecated_args(None,
-                   "Pass your op to the equivalent parameter main_op instead.",
-                   "legacy_init_op")
   def add_meta_graph_and_variables(self,
                                    sess,
                                    tags,
@@ -378,7 +378,7 @@ class SavedModelBuilder(object):
         def.
       assets_collection: Assets collection to be saved with SavedModel.
       legacy_init_op: Legacy support for op or group of ops to execute after the
-          restore op upon a load. Deprecated; please use main_op instead.
+          restore op upon a load.
       clear_devices: Set to true if the device info on the default graph should
           be cleared.
       main_op: Op or group of ops to execute when the graph is loaded. Note
@@ -402,12 +402,8 @@ class SavedModelBuilder(object):
     # properly populated.
     self._validate_signature_def_map(signature_def_map)
 
-    # legacy_init_op is deprecated, and going away in TF 2.0.
-    # Re-mapping to main_op, as treatment is identical regardless.
-    main_op = main_op or legacy_init_op
-
     # Add assets and ops
-    self._add_collections(assets_collection, main_op, None)
+    self._add_collections(assets_collection, legacy_init_op, main_op, None)
 
     # Create the variables sub-directory, if it does not exist.
     variables_dir = os.path.join(
diff --git a/tensorflow/python/saved_model/loader_impl.py b/tensorflow/python/saved_model/loader_impl.py
index fb70c91c29..e5f649fdab 100644
--- a/tensorflow/python/saved_model/loader_impl.py
+++ b/tensorflow/python/saved_model/loader_impl.py
@@ -116,14 +116,11 @@ def _get_asset_tensors(export_dir, meta_graph_def_to_load, import_scope=None):
   return asset_tensor_dict
 
 
-def _get_main_op_tensor(
-    meta_graph_def_to_load, init_op_key=constants.MAIN_OP_KEY):
+def _get_main_op_tensor(meta_graph_def_to_load):
   """Gets the main op tensor, if one exists.
 
   Args:
     meta_graph_def_to_load: The meta graph def from the SavedModel to be loaded.
-    init_op_key: name of collection to check; should be one of MAIN_OP_KEY
-      or the deprecated LEGACY_INIT_OP_KEY
 
   Returns:
     The main op tensor, if it exists and `None` otherwise.
@@ -134,15 +131,38 @@ def _get_main_op_tensor(
   """
   collection_def = meta_graph_def_to_load.collection_def
   main_op_tensor = None
-  if init_op_key in collection_def:
-    main_ops = collection_def[init_op_key].node_list.value
+  if constants.MAIN_OP_KEY in collection_def:
+    main_ops = collection_def[constants.MAIN_OP_KEY].node_list.value
     if len(main_ops) != 1:
-      raise RuntimeError("Expected exactly one SavedModel main op. "
-                         "Found: {}".format(main_ops))
-    main_op_tensor = ops.get_collection(init_op_key)[0]
+      raise RuntimeError("Expected exactly one SavedModel main op.")
+    main_op_tensor = ops.get_collection(constants.MAIN_OP_KEY)[0]
   return main_op_tensor
 
 
+def _get_legacy_init_op_tensor(meta_graph_def_to_load):
+  """Gets the legacy init op tensor, if one exists.
+
+  Args:
+    meta_graph_def_to_load: The meta graph def from the SavedModel to be loaded.
+
+  Returns:
+    The legacy init op tensor, if it exists and `None` otherwise.
+
+  Raises:
+    RuntimeError: If the collection def corresponding to the legacy init op key
+        has other than exactly one tensor.
+  """
+  collection_def = meta_graph_def_to_load.collection_def
+  legacy_init_op_tensor = None
+  if constants.LEGACY_INIT_OP_KEY in collection_def:
+    legacy_init_ops = collection_def[
+        constants.LEGACY_INIT_OP_KEY].node_list.value
+    if len(legacy_init_ops) != 1:
+      raise RuntimeError("Expected exactly one legacy serving init op.")
+    legacy_init_op_tensor = ops.get_collection(constants.LEGACY_INIT_OP_KEY)[0]
+  return legacy_init_op_tensor
+
+
 @tf_export("saved_model.loader.maybe_saved_model_directory")
 def maybe_saved_model_directory(export_dir):
   """Checks whether the provided export directory could contain a SavedModel.
@@ -320,8 +340,8 @@ class SavedModelLoader(object):
           self._export_dir, meta_graph_def, import_scope=import_scope)
 
       main_op_tensor = (
-          _get_main_op_tensor(meta_graph_def, constants.MAIN_OP_KEY) or
-          _get_main_op_tensor(meta_graph_def, constants.LEGACY_INIT_OP_KEY))
+          _get_main_op_tensor(meta_graph_def) or
+          (_get_legacy_init_op_tensor(meta_graph_def)))
       if main_op_tensor is not None:
         sess.run(fetches=[main_op_tensor], feed_dict=asset_tensors_dictionary)
 
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index 00b669fc97..fb4732aca2 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -846,19 +846,9 @@ class SavedModelTest(test.TestCase):
   def testLegacyInitOpWithNonEmptyCollection(self):
     export_dir = self._get_export_dir(
         "test_legacy_init_op_with_non_empty_collection")
-    self._testInitOpsWithNonEmptyCollection(
-        export_dir, constants.LEGACY_INIT_OP_KEY)
-
-  def testMainOpWithNonEmptyCollection(self):
-    export_dir = self._get_export_dir(
-        "test_main_op_with_non_empty_collection")
-    self._testInitOpsWithNonEmptyCollection(export_dir, constants.MAIN_OP_KEY)
-
-  def _testInitOpsWithNonEmptyCollection(self, export_dir, key):
     builder = saved_model_builder.SavedModelBuilder(export_dir)
 
-    g = ops.Graph()
-    with self.test_session(graph=g) as sess:
+    with self.test_session(graph=ops.Graph()) as sess:
       # Initialize variable `v1` to 1.
       v1 = variables.Variable(1, name="v1")
       ops.add_to_collection("v", v1)
@@ -867,21 +857,19 @@ class SavedModelTest(test.TestCase):
       v2 = variables.Variable(42, name="v2", trainable=False, collections=[])
       ops.add_to_collection("v", v2)
 
-      # Set up an assignment op to be run as part of the init op.
+      # Set up an assignment op to be run as part of the legacy_init_op.
       assign_v2 = state_ops.assign(v2, v1)
-      init_op = control_flow_ops.group(assign_v2, name="init_op")
+      legacy_init_op = control_flow_ops.group(assign_v2, name="legacy_init_op")
 
       sess.run(variables.global_variables_initializer())
 
-      ops.add_to_collection(key, control_flow_ops.no_op())
-      # ValueError should be raised since the LEGACY_INIT_OP_KEY collection
+      ops.add_to_collection(constants.LEGACY_INIT_OP_KEY,
+                            control_flow_ops.no_op())
+      # AssertionError should be raised since the LEGACY_INIT_OP_KEY collection
       # is not empty and we don't support multiple init ops.
-      with self.assertRaisesRegexp(ValueError, "Graph already contains"):
+      with self.assertRaises(AssertionError):
         builder.add_meta_graph_and_variables(
-            sess, ["foo"], legacy_init_op=init_op)
-      # We shouldn't be able to add as MAIN_OP, either.
-      with self.assertRaisesRegexp(ValueError, "Graph already contains"):
-        builder.add_meta_graph_and_variables(sess, ["foo"], main_op=init_op)
+            sess, ["foo"], legacy_init_op=legacy_init_op)
 
   def testTrainOp(self):
     export_dir = self._get_export_dir("test_train_op")
-- 
cgit v1.2.3


From 808db1f8d49618e64170f174998bf1e0db49701f Mon Sep 17 00:00:00 2001
From: Akshay Modi 
Date: Mon, 23 Jul 2018 16:47:57 -0700
Subject: Allow ResourceVariable to be written to TensorArray.

This is similar to how graph behaves.

PiperOrigin-RevId: 205747377
---
 tensorflow/python/ops/tensor_array_ops.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index cc92da4fd7..f86dfb3527 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -554,7 +554,7 @@ class _EagerTensorArray(object):
       self._tensor_array.extend([None for _ in range(index - size + 1)])
 
     if not isinstance(value, ops.EagerTensor):
-      value = constant_op.constant(value)
+      value = ops.convert_to_tensor(value)
 
     if self._infer_shape:
       if self._element_shape is None:
@@ -633,8 +633,8 @@ class _EagerTensorArray(object):
   def split(self, value, lengths, name=None):
     """See TensorArray."""
     # error checking to match graph-mode errors
-    value = constant_op.constant(value)
-    lengths = constant_op.constant(lengths)
+    value = ops.convert_to_tensor(value)
+    lengths = ops.convert_to_tensor(lengths)
     sum_lengths = math_ops.reduce_sum(lengths)
     if lengths.shape.ndims != 1:
       raise errors_impl.InvalidArgumentError(
-- 
cgit v1.2.3


From 340cbbee4ee1b268378ee342bbc19bd52f30e8da Mon Sep 17 00:00:00 2001
From: Toby Boyd 
Date: Mon, 23 Jul 2018 16:51:22 -0700
Subject: Remove linking NCCL license file.

PiperOrigin-RevId: 205747892
---
 tensorflow/tools/docker/Dockerfile.devel-gpu | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 44120bf274..a5560e459c 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -40,10 +40,6 @@ RUN mkdir /usr/local/cuda-9.0/lib &&  \
     ln -s /usr/lib/x86_64-linux-gnu/libnccl.so.2 /usr/local/cuda/lib/libnccl.so.2 && \
     ln -s /usr/include/nccl.h /usr/local/cuda/include/nccl.h
 
-# TODO(tobyboyd): Remove after license is excluded from BUILD file.
-RUN gunzip /usr/share/doc/libnccl2/NCCL-SLA.txt.gz && \
-    cp /usr/share/doc/libnccl2/NCCL-SLA.txt /usr/local/cuda/
-
 RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
     python get-pip.py && \
     rm get-pip.py
-- 
cgit v1.2.3


From 6218750b0616e6b7cf14196b04549b2842f3dd99 Mon Sep 17 00:00:00 2001
From: Akshay Modi 
Date: Mon, 23 Jul 2018 16:51:49 -0700
Subject: op->Device can be a nullptr, so don't dereference directly.

PiperOrigin-RevId: 205747965
---
 tensorflow/core/common_runtime/eager/execute.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 5ea814ed4e..27d0cd611f 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -88,6 +88,8 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
   TF_RETURN_IF_ERROR((*handle)->Device(&handle_device));
   const Device* actual_device =
       handle_device == nullptr ? ctx->HostCPU() : handle_device;
+  const Device* op_device =
+      op->Device() == nullptr ? ctx->HostCPU() : op->Device();
 
   if (expected_device != actual_device) {
     switch (ctx->GetDevicePlacementPolicy()) {
@@ -106,8 +108,8 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
             " cannot compute ",
             op->Name(), " as input #", i, " was expected to be on ",
             expected_device->name(), " but is actually on ",
-            actual_device->name(), " (operation running on ",
-            op->Device()->name(), ")",
+            actual_device->name(), " (operation running on ", op_device->name(),
+            ")",
             " Tensors can be copied explicitly using .gpu() or .cpu() "
             "methods,"
             " or transparently copied by using tf.enable_eager_execution("
@@ -118,7 +120,7 @@ Status MaybeCopyInputToExpectedDevice(EagerOperation* op, int i,
         LOG(WARNING) << "before computing " << op->Name() << " input #" << i
                      << " was expected to be on " << expected_device->name()
                      << " but is actually on " << actual_device->name()
-                     << " (operation running on " << op->Device()->name()
+                     << " (operation running on " << op_device->name()
                      << "). This triggers a copy which can be a performance "
                         "bottleneck.";
         break;
-- 
cgit v1.2.3


From 04c6e7de8494835aa2df295899e66e9f5ae7c3a0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 17:21:38 -0700
Subject: Improve the documentation for the Android app for TensorFlow Lite.

PiperOrigin-RevId: 205752218
---
 tensorflow/contrib/lite/examples/android/app/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lite/examples/android/app/README.md b/tensorflow/contrib/lite/examples/android/app/README.md
index 8e12bd04dd..cbdeeac879 100644
--- a/tensorflow/contrib/lite/examples/android/app/README.md
+++ b/tensorflow/contrib/lite/examples/android/app/README.md
@@ -2,9 +2,9 @@
 
 ## Building from Source with Bazel
 
-1. Follow the [Bazel steps for the TF Demo App](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#bazel).
+1. Install [Bazel](https://docs.bazel.build/versions/master/install.html), the Android NDK and SDK. The recommended versions are specified on this [webpage](https://www.tensorflow.org/mobile/tflite/demo_android#build_tensorflow_lite_and_the_demo_app_from_source).
 
-2. Build the app with Bazel. The demo needs C++11. We configure the fat_apk_cpu flag to package support for 4 hardware variants. You may replace it with --config=android_arm64 on a 64-bit device and --config=android_arm for 32-bit device:
+2. Build this demo app with Bazel. The demo needs C++11. We configure the fat_apk_cpu flag to package support for 4 hardware variants. You may replace it with --config=android_arm64 on a 64-bit device and --config=android_arm for 32-bit device:
 
   ```shell
   bazel build -c opt --cxxopt='--std=c++11' --fat_apk_cpu=x86,x86_64,arm64-v8a,armeabi-v7a \
-- 
cgit v1.2.3


From 90dbbba70e1c617a36b1d71b650410d6d4d3cb02 Mon Sep 17 00:00:00 2001
From: Asim Shankar 
Date: Mon, 23 Jul 2018 17:29:42 -0700
Subject: [Java]: Release 1.10.0-rc0

PiperOrigin-RevId: 205753168
---
 tensorflow/java/maven/hadoop/pom.xml                | 2 +-
 tensorflow/java/maven/libtensorflow/pom.xml         | 2 +-
 tensorflow/java/maven/libtensorflow_jni/pom.xml     | 2 +-
 tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml | 2 +-
 tensorflow/java/maven/pom.xml                       | 2 +-
 tensorflow/java/maven/proto/pom.xml                 | 2 +-
 tensorflow/java/maven/spark-connector/pom.xml       | 2 +-
 tensorflow/java/maven/tensorflow/pom.xml            | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/java/maven/hadoop/pom.xml b/tensorflow/java/maven/hadoop/pom.xml
index 7391dfb965..2c2c4106cb 100644
--- a/tensorflow/java/maven/hadoop/pom.xml
+++ b/tensorflow/java/maven/hadoop/pom.xml
@@ -5,7 +5,7 @@
     org.tensorflow
     hadoop
     jar
-    1.9.0
+    1.10.0-rc0
     tensorflow-hadoop
     https://www.tensorflow.org
     TensorFlow TFRecord InputFormat/OutputFormat for Apache Hadoop
diff --git a/tensorflow/java/maven/libtensorflow/pom.xml b/tensorflow/java/maven/libtensorflow/pom.xml
index d44bdf8f81..5d4e04ecd3 100644
--- a/tensorflow/java/maven/libtensorflow/pom.xml
+++ b/tensorflow/java/maven/libtensorflow/pom.xml
@@ -6,7 +6,7 @@
   
     org.tensorflow
     parentpom
-    1.9.0
+    1.10.0-rc0
     ../
   
   libtensorflow
diff --git a/tensorflow/java/maven/libtensorflow_jni/pom.xml b/tensorflow/java/maven/libtensorflow_jni/pom.xml
index e8925c6fb1..e107904f7d 100644
--- a/tensorflow/java/maven/libtensorflow_jni/pom.xml
+++ b/tensorflow/java/maven/libtensorflow_jni/pom.xml
@@ -6,7 +6,7 @@
   
     org.tensorflow
     parentpom
-    1.9.0
+    1.10.0-rc0
     ../
   
   libtensorflow_jni
diff --git a/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml b/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
index 3bf4a2590c..b3c525233f 100644
--- a/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
+++ b/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
@@ -6,7 +6,7 @@
   
     org.tensorflow
     parentpom
-    1.9.0
+    1.10.0-rc0
     ../
   
   libtensorflow_jni_gpu
diff --git a/tensorflow/java/maven/pom.xml b/tensorflow/java/maven/pom.xml
index b96dcf2888..a2943a3172 100644
--- a/tensorflow/java/maven/pom.xml
+++ b/tensorflow/java/maven/pom.xml
@@ -6,7 +6,7 @@
   4.0.0
   org.tensorflow
   parentpom
-  1.9.0
+  1.10.0-rc0
   pom
 
   https://www.tensorflow.org
diff --git a/tensorflow/java/maven/proto/pom.xml b/tensorflow/java/maven/proto/pom.xml
index 5581d864d7..7080d81b7d 100644
--- a/tensorflow/java/maven/proto/pom.xml
+++ b/tensorflow/java/maven/proto/pom.xml
@@ -6,7 +6,7 @@
   
     org.tensorflow
     parentpom
-    1.9.0
+    1.10.0-rc0
     ../
   
   proto
diff --git a/tensorflow/java/maven/spark-connector/pom.xml b/tensorflow/java/maven/spark-connector/pom.xml
index 64956be02c..003d09a0b7 100644
--- a/tensorflow/java/maven/spark-connector/pom.xml
+++ b/tensorflow/java/maven/spark-connector/pom.xml
@@ -6,7 +6,7 @@
     org.tensorflow
     spark-connector_2.11
     jar
-    1.9.0
+    1.10.0-rc0
     spark-tensorflow-connector
     https://www.tensorflow.org
     TensorFlow TFRecord connector for Apache Spark DataFrames
diff --git a/tensorflow/java/maven/tensorflow/pom.xml b/tensorflow/java/maven/tensorflow/pom.xml
index 92e15aa2c7..b9affbf699 100644
--- a/tensorflow/java/maven/tensorflow/pom.xml
+++ b/tensorflow/java/maven/tensorflow/pom.xml
@@ -6,7 +6,7 @@
   
     org.tensorflow
     parentpom
-    1.9.0
+    1.10.0-rc0
     ../
   
   tensorflow
-- 
cgit v1.2.3


From 77455f98a956e0f1e381136856564d5c8773b4e7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 17:37:35 -0700
Subject: Add shape information to the "not a matrix" error message.

PiperOrigin-RevId: 205754132
---
 tensorflow/compiler/tf2xla/kernels/matmul_op.cc | 12 ++++++++----
 tensorflow/core/kernels/matmul_op.cc            | 12 ++++++++----
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
index 844080b8cf..aa45b02551 100644
--- a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
@@ -54,10 +54,14 @@ class MatMulOp : public XlaOpKernel {
     const TensorShape b_shape = ctx->InputShape(1);
 
     // Check that the dimensions of the two matrices are valid.
-    OP_REQUIRES(ctx, TensorShapeUtils::IsMatrix(a_shape),
-                errors::InvalidArgument("In[0] is not a matrix"));
-    OP_REQUIRES(ctx, TensorShapeUtils::IsMatrix(b_shape),
-                errors::InvalidArgument("In[1] is not a matrix"));
+    OP_REQUIRES(
+        ctx, TensorShapeUtils::IsMatrix(a_shape),
+        errors::InvalidArgument("In[0] is not a matrix. Instead it has shape ",
+                                a_shape.DebugString()));
+    OP_REQUIRES(
+        ctx, TensorShapeUtils::IsMatrix(b_shape),
+        errors::InvalidArgument("In[1] is not a matrix. Instead it has shape ",
+                                b_shape.DebugString()));
     int first_index = transpose_a_ ? 0 : 1;
     int second_index = transpose_b_ ? 1 : 0;
 
diff --git a/tensorflow/core/kernels/matmul_op.cc b/tensorflow/core/kernels/matmul_op.cc
index b596dbc782..80376c61aa 100644
--- a/tensorflow/core/kernels/matmul_op.cc
+++ b/tensorflow/core/kernels/matmul_op.cc
@@ -453,10 +453,14 @@ class MatMulOp : public OpKernel {
     const Tensor& b = ctx->input(1);
 
     // Check that the dimensions of the two matrices are valid.
-    OP_REQUIRES(ctx, TensorShapeUtils::IsMatrix(a.shape()),
-                errors::InvalidArgument("In[0] is not a matrix"));
-    OP_REQUIRES(ctx, TensorShapeUtils::IsMatrix(b.shape()),
-                errors::InvalidArgument("In[1] is not a matrix"));
+    OP_REQUIRES(
+        ctx, TensorShapeUtils::IsMatrix(a.shape()),
+        errors::InvalidArgument("In[0] is not a matrix. Instead it has shape ",
+                                a.shape().DebugString()));
+    OP_REQUIRES(
+        ctx, TensorShapeUtils::IsMatrix(b.shape()),
+        errors::InvalidArgument("In[1] is not a matrix. Instead it has shape ",
+                                b.shape().DebugString()));
     Eigen::array, 1> dim_pair;
     dim_pair[0].first = transpose_a_ ? 0 : 1;
     dim_pair[0].second = transpose_b_ ? 1 : 0;
-- 
cgit v1.2.3


From 86f63c717a354f342d1b714420a04c85434bb282 Mon Sep 17 00:00:00 2001
From: Smit Hinsu 
Date: Mon, 23 Jul 2018 17:42:19 -0700
Subject: Update example execution of configure script for installation from
 sources

PiperOrigin-RevId: 205754672
---
 tensorflow/docs_src/install/install_sources.md | 96 +++++++++++++++++++-------
 1 file changed, 70 insertions(+), 26 deletions(-)

diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index 4c09ba1a8b..31dcad64d4 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -223,46 +223,90 @@ will likely differ from our sample input:
 
     $ cd tensorflow  # cd to the top-level directory created
 $ ./configure
+You have bazel 0.15.0 installed.
 Please specify the location of python. [Default is /usr/bin/python]: /usr/bin/python2.7
+
+
 Found possible Python library paths:
   /usr/local/lib/python2.7/dist-packages
   /usr/lib/python2.7/dist-packages
 Please input the desired Python library path to use.  Default is [/usr/lib/python2.7/dist-packages]
 
-Using python library path: /usr/local/lib/python2.7/dist-packages
-Please specify optimization flags to use during compilation when bazel option "--config=opt" is specified [Default is -march=native]:
-Do you wish to use jemalloc as the malloc implementation? [Y/n]
-jemalloc enabled
-Do you wish to build TensorFlow with Google Cloud Platform support? [y/N]
-No Google Cloud Platform support will be enabled for TensorFlow
-Do you wish to build TensorFlow with Hadoop File System support? [y/N]
-No Hadoop File System support will be enabled for TensorFlow
-Do you wish to build TensorFlow with the XLA just-in-time compiler (experimental)? [y/N]
-No XLA support will be enabled for TensorFlow
-Do you wish to build TensorFlow with VERBS support? [y/N]
-No VERBS support will be enabled for TensorFlow
-Do you wish to build TensorFlow with OpenCL support? [y/N]
-No OpenCL support will be enabled for TensorFlow
-Do you wish to build TensorFlow with CUDA support? [y/N] Y
-CUDA support will be enabled for TensorFlow
-Do you want to use clang as CUDA compiler? [y/N]
-nvcc will be used as CUDA compiler
+Do you wish to build TensorFlow with jemalloc as malloc support? [Y/n]:
+jemalloc as malloc support will be enabled for TensorFlow.
+
+Do you wish to build TensorFlow with Google Cloud Platform support? [Y/n]:
+Google Cloud Platform support will be enabled for TensorFlow.
+
+Do you wish to build TensorFlow with Hadoop File System support? [Y/n]:
+Hadoop File System support will be enabled for TensorFlow.
+
+Do you wish to build TensorFlow with Amazon AWS Platform support? [Y/n]:
+Amazon AWS Platform support will be enabled for TensorFlow.
+
+Do you wish to build TensorFlow with Apache Kafka Platform support? [Y/n]:
+Apache Kafka Platform support will be enabled for TensorFlow.
+
+Do you wish to build TensorFlow with XLA JIT support? [y/N]:
+No XLA JIT support will be enabled for TensorFlow.
+
+Do you wish to build TensorFlow with GDR support? [y/N]:
+No GDR support will be enabled for TensorFlow.
+
+Do you wish to build TensorFlow with VERBS support? [y/N]:
+No VERBS support will be enabled for TensorFlow.
+
+Do you wish to build TensorFlow with OpenCL SYCL support? [y/N]:
+No OpenCL SYCL support will be enabled for TensorFlow.
+
+Do you wish to build TensorFlow with CUDA support? [y/N]: Y
+CUDA support will be enabled for TensorFlow.
+
 Please specify the CUDA SDK version you want to use. [Leave empty to default to CUDA 9.0]: 9.0
+
+
 Please specify the location where CUDA 9.0 toolkit is installed. Refer to README.md for more details. [Default is /usr/local/cuda]:
-Please specify which gcc should be used by nvcc as the host compiler. [Default is /usr/bin/gcc]:
-Please specify the cuDNN version you want to use. [Leave empty to default to cuDNN 7.0]: 7
+
+
+Please specify the cuDNN version you want to use. [Leave empty to default to cuDNN 7.0]: 7.0
+
+
 Please specify the location where cuDNN 7 library is installed. Refer to README.md for more details. [Default is /usr/local/cuda]:
-Please specify a list of comma-separated CUDA compute capabilities you want to build with.
+
+
+Do you wish to build TensorFlow with TensorRT support? [y/N]:
+No TensorRT support will be enabled for TensorFlow.
+
+Please specify the NCCL version you want to use. If NCLL 2.2 is not installed, then you can use version 1.3 that can be fetched automatically but it may have worse performance with multiple GPUs. [Default is 2.2]: 1.3
+
+
+Please specify a list of comma-separated Cuda compute capabilities you want to build with.
 You can find the compute capability of your device at: https://developer.nvidia.com/cuda-gpus.
-Please note that each additional compute capability significantly increases your build time and binary size.
+Please note that each additional compute capability significantly increases your
+build time and binary size. [Default is: 3.5,7.0] 6.1
+
+
+Do you want to use clang as CUDA compiler? [y/N]:
+nvcc will be used as CUDA compiler.
 
-Do you wish to build TensorFlow with MPI support? [y/N]
-MPI support will not be enabled for TensorFlow
+Please specify which gcc should be used by nvcc as the host compiler. [Default is /usr/bin/gcc]:
+
+
+Do you wish to build TensorFlow with MPI support? [y/N]:
+No MPI support will be enabled for TensorFlow.
+
+Please specify optimization flags to use during compilation when bazel option "--config=opt" is specified [Default is -march=native]:
+
+
+Would you like to interactively configure ./WORKSPACE for Android builds? [y/N]:
+Not configuring the WORKSPACE for Android builds.
+
+Preconfigured Bazel build configs. You can use any of the below by adding "--config=<>" to your build command. See tools/bazel.rc for more details.
+    --config=mkl            # Build with MKL support.
+    --config=monolithic     # Config for mostly static monolithic build.
 Configuration finished
 
    
 
-[Default is: "3.5,7.0"]: 6.0,7.0
-
 If you told `configure` to build for GPU support, then `configure` will create a
 canonical set of symbolic links to the CUDA libraries on your system. Therefore,
 every time you change the CUDA library paths, you must rerun the `configure`
-- 
cgit v1.2.3


From b553a232a7537ca23efb36d19b4d6f5198ff46d1 Mon Sep 17 00:00:00 2001
From: Smit Hinsu 
Date: Mon, 23 Jul 2018 17:44:16 -0700
Subject: Add larger inputs to conv2d benchmark for better coverage

PiperOrigin-RevId: 205754951
---
 tensorflow/python/ops/conv2d_benchmark.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/conv2d_benchmark.py b/tensorflow/python/ops/conv2d_benchmark.py
index aacdaa7ad0..28111c2730 100644
--- a/tensorflow/python/ops/conv2d_benchmark.py
+++ b/tensorflow/python/ops/conv2d_benchmark.py
@@ -175,7 +175,8 @@ class Conv2DBenchmark(test.Benchmark):
 
     data_types = [dtypes.float32, dtypes.float16]
     data_formats = ["NHWC", "NCHW"]
-    in_channels = list(range(3, 16))
+    in_channels = list(range(1, 10)) + list(range(10, 20, 2)) + list(
+        range(20, 33, 4))
     out_channels = [4, 16, 32]
     hw_strides = [[2, 2]]
     paddings = ["VALID", "SAME"]
-- 
cgit v1.2.3


From 6f6161a0110d99b2655efc9d933b753dadadbc38 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 17:45:23 -0700
Subject: Best Practices for writing custom operators

PiperOrigin-RevId: 205755115
---
 tensorflow/contrib/lite/g3doc/custom_operators.md | 44 +++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/tensorflow/contrib/lite/g3doc/custom_operators.md b/tensorflow/contrib/lite/g3doc/custom_operators.md
index 972e57f73e..f2fbcf64cf 100644
--- a/tensorflow/contrib/lite/g3doc/custom_operators.md
+++ b/tensorflow/contrib/lite/g3doc/custom_operators.md
@@ -89,3 +89,47 @@ builtins.AddCustom("Sin", Register_SIN());
 
 Note that a similar process as above can be followed for supporting for a set of
 operations instead of a single operator.
+
+## Best Practices for writing custom operators
+
+1.  Optimize memory allocations and de-allocations cautiously. It is more
+    efficient to allocate memory in Prepare() instead of Invoke(), and allocate
+    memory before a loop instead of in every iteration. Use temporary tensors
+    data rather than mallocing yourself (see item 2). Use pointers/references
+    instead of copying as much as possible.
+
+2.  If a data structure will persist during the entire operation, we advise
+    pre-allocating the memory using temporary tensors. You may need to use
+    OpData struct to reference the tensor indices in other functions. See
+    example in the
+    [kernel for convolution](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/kernels/conv.cc).
+    A sample code snippet is below
+
+    ```
+    auto* op_data = reinterpret_cast(node->user_data);
+    TfLiteIntArrayFree(node->temporaries);
+    node->temporaries = TfLiteIntArrayCreate(1);
+    node->temporaries->data[0] = op_data->temp_tensor_index;
+    TfLiteTensor* temp_tensor = &context->tensors[op_data->temp_tensor_index];
+    temp_tensor->type =  kTfLiteFloat32;
+    temp_tensor->allocation_type = kTfLiteArenaRw;
+    ```
+
+3.  If it doesn't cost too much wasted memory, prefer using a static fixed size
+    array (or in Resize() pre-allocated std::vector) rather than using a
+    dynamically allocating std::vector every iteration of execution.
+
+4.  Avoid instantiating standard library container templates that don't already
+    exist, because they affect binary size. For example, if you need a std::map
+    in your operation that doesn't exist in other kernels, using a std::vector
+    with direct indexing mapping could work while keeping the binary size small.
+    See what other kernels use to gain insight (or ask).
+
+5.  Check the pointer to the memory returned by malloc. If this pointer is
+    nullptr, no operations should be performed using that pointer. If you
+    malloc() in a function and have an error exit, deallocate memory before you
+    exit.
+
+6.  Use TF_LITE_ENSURE(context, condition) to check for a specific condition.
+    Your code must not leave memory hanging when TF_LITE_ENSURE is done, i.e.,
+    these should be done before any resources are allocated that will leak.
-- 
cgit v1.2.3


From dcf568a4e297bb70a74ed5665b924077b9cb650a Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Mon, 23 Jul 2018 17:49:04 -0700
Subject: Open source XLA GPU tests

PiperOrigin-RevId: 205755610
---
 tensorflow/compiler/xla/service/gpu/tests/BUILD    | 204 +++++++++++++++++++++
 .../xla/service/gpu/tests/gpu_codegen_test.cc      |  50 +++++
 .../xla/service/gpu/tests/gpu_codegen_test.h       |  42 +++++
 .../xla/service/gpu/tests/gpu_copy_test.cc         |  59 ++++++
 .../compiler/xla/service/gpu/tests/gpu_ftz_test.cc | 119 ++++++++++++
 .../xla/service/gpu/tests/gpu_fusion_test.cc       |  59 ++++++
 .../xla/service/gpu/tests/gpu_index_test.cc        | 147 +++++++++++++++
 .../service/gpu/tests/gpu_kernel_tiling_test.cc    | 177 ++++++++++++++++++
 .../compiler/xla/service/gpu/tests/gpu_ldg_test.cc | 141 ++++++++++++++
 .../xla/service/gpu/tests/gpu_noalias_test.cc      |  68 +++++++
 .../xla/service/gpu/tests/gpu_unrolling_test.cc    | 185 +++++++++++++++++++
 .../compiler/xla/service/gpu/tests/infeed_test.cc  | 121 ++++++++++++
 12 files changed, 1372 insertions(+)
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/BUILD
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_fusion_test.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_unrolling_test.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/infeed_test.cc

diff --git a/tensorflow/compiler/xla/service/gpu/tests/BUILD b/tensorflow/compiler/xla/service/gpu/tests/BUILD
new file mode 100644
index 0000000000..926262e2ad
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/BUILD
@@ -0,0 +1,204 @@
+# Description: GPU-specific XLA tests. For example, codegen tests that
+# verify the IR emitted.
+#
+# TODO(jlebar): None of these tests actually use the GPU, so they should not
+# need to run on machines with GPUs present.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = [":friends"])
+
+package_group(
+    name = "friends",
+    includes = [
+        "//tensorflow/compiler/xla:friends",
+    ],
+)
+
+# Filegroup used to collect source files for dependency checking.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+
+cc_library(
+    name = "gpu_codegen_test",
+    testonly = True,
+    srcs = ["gpu_codegen_test.cc"],
+    hdrs = ["gpu_codegen_test.h"],
+    tags = [
+        "requires-gpu-sm35",
+    ],
+    deps = [
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla/legacy_flags:debug_options_flags",
+        "//tensorflow/compiler/xla/service:gpu_plugin",
+        "//tensorflow/compiler/xla/service/gpu:gpu_executable",
+        "//tensorflow/compiler/xla/tests:filecheck",
+        "//tensorflow/compiler/xla/tests:llvm_irgen_test_base",
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "gpu_copy_test",
+    srcs = ["gpu_copy_test.cc"],
+    tags = [
+        "requires-gpu-sm35",
+    ],
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "gpu_ftz_test",
+    srcs = ["gpu_ftz_test.cc"],
+    tags = [
+        "requires-gpu-sm35",
+    ],
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "gpu_index_test",
+    srcs = ["gpu_index_test.cc"],
+    tags = [
+        "requires-gpu-sm35",
+    ],
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla:xla_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_module_config",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "gpu_infeed_test",
+    srcs = ["infeed_test.cc"],
+    tags = [
+        "requires-gpu-sm35",
+    ],
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:global_data",
+        "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/lib:arithmetic",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
+        "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:literal_test_util",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "gpu_kernel_tiling_test",
+    srcs = ["gpu_kernel_tiling_test.cc"],
+    tags = [
+        "requires-gpu-sm35",
+    ],
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_module_config",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "gpu_ldg_test",
+    srcs = ["gpu_ldg_test.cc"],
+    tags = ["requires-gpu-sm35"],
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "gpu_noalias_test",
+    srcs = ["gpu_noalias_test.cc"],
+    tags = [
+        "requires-gpu-sm35",
+    ],
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/compiler/xla:literal",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "gpu_fusion_test",
+    srcs = ["gpu_fusion_test.cc"],
+    tags = [
+        "requires-gpu-sm35",
+    ],
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/compiler/xla/service:hlo_module_config",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "gpu_unrolling_test",
+    srcs = ["gpu_unrolling_test.cc"],
+    tags = [
+        "requires-gpu-sm35",
+    ],
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/compiler/xla/service:hlo_module_config",
+        "//tensorflow/compiler/xla/service:hlo_parser",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.cc
new file mode 100644
index 0000000000..4b8415fe91
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.cc
@@ -0,0 +1,50 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+#include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h"
+#include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/gpu/gpu_executable.h"
+#include "tensorflow/compiler/xla/tests/filecheck.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace xla {
+namespace gpu {
+
+std::unique_ptr GpuCodegenTest::CreateNewModuleWithFTZ(bool ftz) {
+  HloModuleConfig config;
+  auto debug_options = legacy_flags::GetDebugOptionsFromFlags();
+  debug_options.set_xla_gpu_ftz(ftz);
+  debug_options.set_xla_gpu_max_kernel_unroll_factor(1);
+  // TODO(b/38354253): Change tests to use Parameters instead of Constants.
+  debug_options.add_xla_disable_hlo_passes("constant_folding");
+  config.set_debug_options(debug_options);
+
+  return MakeUnique(TestName(), config);
+}
+
+void GpuCodegenTest::CompileAndVerifyPtx(std::unique_ptr hlo_module,
+                                         const string& pattern) {
+  std::unique_ptr executable =
+      std::move(CompileToExecutable(std::move(hlo_module)).ValueOrDie());
+  string ptx_str =
+      std::string(static_cast(executable.get())->ptx());
+  StatusOr filecheck_result = RunFileCheck(ptx_str, pattern);
+  ASSERT_TRUE(filecheck_result.ok());
+  EXPECT_TRUE(filecheck_result.ValueOrDie());
+}
+
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h b/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h
new file mode 100644
index 0000000000..e4a3573bab
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h
@@ -0,0 +1,42 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_TESTS_GPU_CODEGEN_TEST_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_TESTS_GPU_CODEGEN_TEST_H_
+
+#include 
+
+#include "tensorflow/compiler/xla/tests/llvm_irgen_test_base.h"
+
+namespace xla {
+namespace gpu {
+
+// Tests that verify IR or PTX emitted by the GPU backend is as expected.
+class GpuCodegenTest : public LlvmIrGenTestBase {
+ protected:
+  // Like HloTestBase::CreateNewModule(), with a flag for configuring the ftz
+  // option.
+  std::unique_ptr CreateNewModuleWithFTZ(bool ftz);
+
+  // Compiles the given HLO module to PTX and verifies the PTX matches the given
+  // FileCheck pattern.  (See http://llvm.org/docs/CommandGuide/FileCheck.html).
+  void CompileAndVerifyPtx(std::unique_ptr hlo_module,
+                           const string& pattern);
+};
+
+}  // namespace gpu
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_TESTS_GPU_CODEGEN_TEST_H_
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc
new file mode 100644
index 0000000000..ce69e058e6
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_copy_test.cc
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include 
+#include 
+
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace gpu {
+
+class GpuCopyTest : public GpuCodegenTest {};
+
+// The GPU backend should not emit a copy kernel for the kCopy instruction in
+// this test. Instead, it should generate a CopyThunk which invokes cuMemcpy at
+// runtime.
+TEST_F(GpuCopyTest, UseMemcpy) {
+  HloComputation::Builder builder(TestName());
+
+  std::unique_ptr literal =
+      LiteralUtil::CreateR2({{1.0, 2.0}, {3.0, 4.0}});
+  HloInstruction* constant = builder.AddInstruction(
+      HloInstruction::CreateConstant(std::move(literal)));
+  builder.AddInstruction(HloInstruction::CreateUnary(
+      constant->shape(), HloOpcode::kCopy, constant));
+
+  std::unique_ptr computation = builder.Build();
+
+  auto hlo_module = CreateNewModule();
+  hlo_module->AddEntryComputation(std::move(computation));
+
+  // There should not be any kernel prefixed "copy".
+  CompileAndVerifyIr(std::move(hlo_module), "; CHECK-NOT: define void @_copy",
+                     /*match_optimized_ir=*/false);
+}
+
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc
new file mode 100644
index 0000000000..177b94934c
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_ftz_test.cc
@@ -0,0 +1,119 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+
+// Check that the ftz (flush denormals to zero) flag is reflected in PTX as
+// expected.
+
+namespace xla {
+namespace gpu {
+namespace {
+
+class GpuFtzTest : public GpuCodegenTest {
+ public:
+  explicit GpuFtzTest(bool ftz) : ftz_(ftz) {}
+
+  // Creates an HLO module that performs the given binary operation on some
+  // data.
+  std::unique_ptr CreateBinaryOpModule(HloOpcode op) {
+    HloComputation::Builder builder(TestName());
+
+    Shape param_shape = ShapeUtil::MakeShapeWithLayout(
+        F32, /*dimensions=*/{100, 100}, /*minor_to_major=*/{1, 0});
+    HloInstruction* x = builder.AddInstruction(HloInstruction::CreateParameter(
+        /* parameter_number=*/0, param_shape, "x"));
+    HloInstruction* y = builder.AddInstruction(HloInstruction::CreateParameter(
+        /* parameter_number=*/1, param_shape, "y"));
+    builder.AddInstruction(HloInstruction::CreateBinary(param_shape, op, x, y));
+
+    auto hlo_module = CreateNewModuleWithFTZ(ftz_);
+    hlo_module->AddEntryComputation(builder.Build());
+    return hlo_module;
+  }
+
+  // Creates an HLO module that performs the given unary operation on some data.
+  std::unique_ptr CreateUnaryOpModule(HloOpcode op) {
+    HloComputation::Builder builder(TestName());
+
+    Shape param_shape = ShapeUtil::MakeShapeWithLayout(
+        F32, /*dimensions=*/{100, 100}, /*minor_to_major=*/{1, 0});
+    HloInstruction* x = builder.AddInstruction(HloInstruction::CreateParameter(
+        /* parameter_number=*/0, param_shape, "x"));
+    builder.AddInstruction(HloInstruction::CreateUnary(param_shape, op, x));
+
+    auto hlo_module = CreateNewModuleWithFTZ(ftz_);
+    hlo_module->AddEntryComputation(builder.Build());
+    return hlo_module;
+  }
+
+  bool ftz_;
+};
+
+class GpuFtzEnabledTest : public GpuFtzTest {
+ public:
+  GpuFtzEnabledTest() : GpuFtzTest(/*ftz=*/true) {}
+};
+
+class GpuFtzDisabledTest : public GpuFtzTest {
+ public:
+  GpuFtzDisabledTest() : GpuFtzTest(/*ftz=*/false) {}
+};
+
+// Check that we emit mul.ftz.f32 when in ftz mode, and plain mul.f32 otherwise.
+TEST_F(GpuFtzEnabledTest, MultiplyFtz) {
+  CompileAndVerifyPtx(CreateBinaryOpModule(HloOpcode::kMultiply), R"(
+    CHECK-NOT: mul.f32
+    CHECK: mul.ftz.f32
+    CHECK-NOT: mul.f32
+  )");
+}
+TEST_F(GpuFtzDisabledTest, MultiplyFtz) {
+  CompileAndVerifyPtx(CreateBinaryOpModule(HloOpcode::kMultiply), R"(
+    CHECK-NOT: mul.ftz.f32
+    CHECK: mul.f32
+    CHECK-NOT: mul.ftz.f32
+  )");
+}
+
+// In NVPTX, exp(float) is implemented in libdevice, and consults __nvvm_reflect
+// to determine whether or not ftz is enabled.  The implementation uses two
+// calls to ex2.approx.  When ftz is on, we get two calls to the ftz version;
+// when ftz is off, we get one call to the ftz version and one call to the
+// regular version.
+TEST_F(GpuFtzEnabledTest, ExpFtz) {
+  CompileAndVerifyPtx(CreateUnaryOpModule(HloOpcode::kExp), R"(
+    CHECK-NOT: ex2.approx.f32
+    CHECK:     ex2.approx.ftz.f32
+    CHECK-NOT: ex2.approx.f32
+    CHECK:     ex2.approx.ftz.f32
+    CHECK-NOT: ex2.approx.f32
+    CHECK-NOT: ex2.approx.ftz.f32
+  )");
+}
+
+TEST_F(GpuFtzDisabledTest, ExpFtz) {
+  CompileAndVerifyPtx(CreateUnaryOpModule(HloOpcode::kExp), R"(
+    CHECK-NOT: ex2.approx.f32
+    CHECK-DAG: ex2.approx.ftz.f32
+    CHECK-DAG: ex2.approx.f32
+    CHECK-NOT: ex2.approx.f32
+    CHECK-NOT: ex2.approx.ftz.f32
+  )");
+}
+
+}  // namespace
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_fusion_test.cc
new file mode 100644
index 0000000000..674b436a8e
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_fusion_test.cc
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include 
+
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+#include "tensorflow/compiler/xla/service/hlo_module_config.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace gpu {
+namespace {
+
+class GpuFusionTest : public GpuCodegenTest {};
+
+TEST_F(GpuFusionTest, FusedReshape) {
+  const char* hlo_text = R"(
+    HloModule test_module
+
+    fused_computation {
+      p0.param_0 = f32[4,1,1]{2,1,0} parameter(0)
+      p1.param_1 = f32[4,1]{1,0} parameter(1)
+      reshape = f32[4,1]{1,0} reshape(p0.param_0)
+      ROOT add = f32[4,1] add(reshape, p1.param_1)
+    }
+
+    ENTRY BroadcastIntoAdd {
+      p0 = f32[4,1,1]{2,1,0} parameter(0)
+      p1 = f32[4,1]{1,0} parameter(1)
+      ROOT fusion = f32[4,1]{1,0} fusion(p0, p1), kind=kLoop,
+                                                  calls=fused_computation
+    }
+)";
+
+  CompileAndVerifyIr(hlo_text,
+                     R"(
+; CHECK-LABEL: @fusion
+; CHECK: fadd
+; CHECK: }
+      )");
+}
+
+}  // namespace
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc
new file mode 100644
index 0000000000..e5958165ef
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_index_test.cc
@@ -0,0 +1,147 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include 
+#include 
+
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_module_config.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/xla.pb.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace gpu {
+
+// This file tests the index expressions used to reference source tensors. When
+// the destination tensor and source tensor have compatible shapes, the linear
+// index is used to access the source tensor. Otherwise, dimensional indices
+// computed from the linear index are used to access the source tensor.
+
+class GpuIndexTest : public GpuCodegenTest {};
+
+TEST_F(GpuIndexTest, CompatibleUseLinearIndex) {
+  HloComputation::Builder builder(TestName());
+
+  auto param_shape = ShapeUtil::MakeShape(F32, {5, 7, 2});
+  HloInstruction* param_x = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, param_shape, "x"));
+  HloInstruction* param_y = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, param_shape, "y"));
+  builder.AddInstruction(HloInstruction::CreateBinary(
+      ShapeUtil::MakeShape(PRED, {5, 7, 2}), HloOpcode::kGe, param_x, param_y));
+
+  auto hlo_module = CreateNewModule();
+  hlo_module->AddEntryComputation(builder.Build());
+
+  // Check the optimized IR as the unoptimized IR contains dead udiv and urem.
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-NOT: udiv
+; CHECK-NOT: urem
+      )",
+                     /*match_optimized_ir=*/true);
+}
+
+TEST_F(GpuIndexTest, CompatibleUseLinearIndexWithReshape) {
+  HloModuleConfig config;
+  config.set_debug_options(HloTestBase::GetDebugOptionsForTest());
+  auto module = ParseHloString(R"(
+    HloModule test_module
+
+    ENTRY CompatibleUseLinearIndexWithReshape {
+      x = f32[5,7,2]{2,1,0} parameter(0)
+      y = f32[5,14]{1,0} parameter(1)
+      reshape = f32[5,7,2]{2,1,0} reshape(y)
+      ROOT gte = pred[5,7,2]{2,1,0} greater-than-or-equal-to(x, reshape)
+    })",
+                               config)
+                    .ValueOrDie();
+
+  // Check the optimized IR as the unoptimized IR contains dead udiv and urem.
+  CompileAndVerifyIr(std::move(module),
+                     R"(
+; CHECK-NOT: udiv
+; CHECK-NOT: urem
+      )",
+                     /*match_optimized_ir=*/true);
+}
+
+TEST_F(GpuIndexTest, CompatibleUseLinearIndexWithReshapeAndBroadcast) {
+  HloModuleConfig config;
+  config.set_debug_options(HloTestBase::GetDebugOptionsForTest());
+  auto module = ParseHloString(R"(
+    HloModule test_module
+
+    ENTRY CompatibleUseLinearIndexWithReshape {
+      x = f32[5,7,2]{2,1,0} parameter(0)
+      y = f32[14]{0} parameter(1)
+      reshape = f32[7,2]{1,0} reshape(y)
+      broadcast = f32[5,7,2]{2,1,0} broadcast(reshape), dimensions={1,2}
+      ROOT gte = pred[5,7,2]{2,1,0} greater-than-or-equal-to(x, broadcast)
+    })",
+                               config)
+                    .ValueOrDie();
+
+  // Check the optimized IR reuses the linear index by calculating modulo 14.
+  CompileAndVerifyIr(std::move(module),
+                     R"(
+; CHECK: %[[urem1:.*]] = urem i{{[0-9]*}} %[[linear_index:.*]], 14
+; CHECK: %[[bitcast:.*]] = bitcast i8 addrspace(1)* %[[alloc:.*]] to float addrspace(1)*
+; CHECK: %[[idx1:.*]] = zext i{{[0-9]*}} %[[urem1]] to i64
+; CHECK: getelementptr inbounds float, float addrspace(1)* %[[bitcast]], i64 %[[idx1]]
+      )",
+                     /*match_optimized_ir=*/true);
+}
+
+TEST_F(GpuIndexTest, CompatibleUseLinearIndexWithSizeOneDimensions) {
+  HloModuleConfig config;
+  auto debug_options = HloTestBase::GetDebugOptionsForTest();
+  debug_options.set_xla_gpu_max_kernel_unroll_factor(1);
+  config.set_debug_options(debug_options);
+
+  auto module = ParseHloString(R"(
+    HloModule  test_module
+
+    ENTRY CompatibleUseLinearIndexWithSizeOneDimensions  {
+      x = f32[1,1024,1,256]{3,2,1,0} parameter(0)
+      ROOT y = f16[1,1024,1,256]{2,3,1,0} convert(x)
+    })",
+                               config)
+                    .ValueOrDie();
+
+  // Check that the unoptimized IR reuses the linear index.
+  CompileAndVerifyIr(std::move(module),
+                     R"(
+; CHECK-LABEL: @fusion
+; CHECK: udiv i32 %[[linear_index:.*]], 262144
+; CHECK: %[[ld_addr:.*]] = getelementptr inbounds float, float* {{.*}}, i32 %[[linear_index]]
+; CHECK: load float, float* %[[ld_addr]]
+; CHECK: %[[st_addr:.*]] = getelementptr inbounds half, half* {{.*}}, i32 %[[linear_index]]
+; CHECK: store half {{.*}}, half* %[[st_addr]]
+      )",
+                     /*match_optimized_ir=*/false);
+}
+
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
new file mode 100644
index 0000000000..cca35316f0
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc
@@ -0,0 +1,177 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include 
+
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+#include "tensorflow/compiler/xla/service/hlo_module_config.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace gpu {
+namespace {
+
+class GpuKernelTilingTest : public GpuCodegenTest {
+ protected:
+  GpuKernelTilingTest() {
+    auto debug_options = HloTestBase::GetDebugOptionsForTest();
+    config_.set_debug_options(debug_options);
+    // Disable layout_assignment to use the preassigned layouts.
+    debug_options.add_xla_disable_hlo_passes("layout_assignment");
+  }
+  HloModuleConfig config_;
+};
+
+TEST_F(GpuKernelTilingTest, UnnestedTransposeWithProperDimensionsTiled) {
+  const char *const kHloString = R"(
+    HloModule unnested_transpose_1
+
+    ENTRY unnested_transpose_1 {
+      para0 = f16[32,3,64]{2,1,0} parameter(0)
+      ROOT copy1 = f16[32,3,64]{1,0,2} copy(para0)
+    })";
+
+  // Check that a call to llvm.nvvm.barrier0 is generated.
+  auto hlo_module = ParseHloString(kHloString, config_).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @copy
+; CHECK: tail call void @llvm.nvvm.barrier0()
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
+}
+
+TEST_F(GpuKernelTilingTest, UnnestedTransposeWithSmallDimensionsNotTiled) {
+  const char *const kHloString = R"(
+    HloModule unnested_transpose_2
+
+    ENTRY unnested_transpose_2 {
+      para0 = f16[2,3,64]{2,1,0} parameter(0)
+      ROOT copy1 = f16[2,3,64]{1,0,2} copy(para0)
+    })";
+
+  // Check that a call to llvm.nvvm.barrier0 is not generated.
+  auto hlo_module = ParseHloString(kHloString, config_).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @copy
+; CHECK-NOT: tail call void @llvm.nvvm.barrier0()
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+}
+
+TEST_F(GpuKernelTilingTest, SimpleFusionWithTransposeTiled) {
+  const char *const kHloString = R"(
+    HloModule multiple_output_fusion_1
+    fused_computation.1 {
+      param0 = f32[4,5,6,7,8]{4,3,2,1,0} parameter(0)
+      copy = f32[4,5,6,7,8]{2,1,4,3,0} copy(param0)
+      ROOT convert = f16[4,5,6,7,8]{2,1,4,3,0} convert(copy)
+    }
+
+    ENTRY copy_in_fusion_run_without_hlo_passes {
+      para0 = f32[4,5,6,7,8]{4,3,2,1,0} parameter(0)
+      ROOT fusion.1 = f16[4,5,6,7,8]{2,1,4,3,0} fusion(para0), kind=kLoop,
+        calls=fused_computation.1
+    })";
+
+  // Check that a call to llvm.nvvm.barrier0 is generated.
+  auto hlo_module = ParseHloString(kHloString, config_).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK: tail call void @llvm.nvvm.barrier0()
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
+}
+
+TEST_F(GpuKernelTilingTest, MultipleOutputFusionWithOnePossibleTransposeTiled) {
+  const char *const kHloString = R"(
+    HloModule multiple_output_fusion_1
+    fused_computation.1 {
+      param0 = f16[8,31,31,65]{3,2,1,0} parameter(0)
+      param1 = f16[8,31,31,65]{3,2,1,0} parameter(1)
+      copy0 = f16[8,31,31,65]{2,1,3,0} copy(param0)
+      copy1 = f16[8,31,31,65]{2,1,3,0} copy(param1)
+      ROOT tuple1 = (f16[8,31,31,65]{2,1,3,0}, f16[8,31,31,65]{2,1,3,0})
+        tuple(copy0, copy1)
+    }
+
+    ENTRY multiple_output_fusion_1 {
+      para0 = f16[8,31,31,65]{3,2,1,0} parameter(0)
+      para1 = f16[8,31,31,65]{3,2,1,0} parameter(1)
+      ROOT fusion.1 = (f16[8,31,31,65]{2,1,3,0}, f16[8,31,31,65]{2,1,3,0})
+        fusion(para0,para1), kind=kLoop, calls=fused_computation.1
+    })";
+
+  // Check that a call to llvm.nvvm.barrier0 is generated.
+  auto hlo_module = ParseHloString(kHloString, config_).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK: tail call void @llvm.nvvm.barrier0()
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+
+  // Check that the kernel runs correctly.
+  EXPECT_TRUE(RunAndCompareNoHloPasses(kHloString, ErrorSpec{0.0}));
+}
+
+TEST_F(GpuKernelTilingTest,
+       MultipleOutputFusionWithTwoPossibleTransposesNotTiled) {
+  const char *const kHloString = R"(
+    HloModule multiple_output_fusion_2
+    fused_computation.1 {
+      param0 = f16[8,31,31,65]{3,2,1,0} parameter(0)
+      param1 = f16[8,31,31,65]{1,3,2,0} parameter(1)
+      copy2 = f16[8,31,31,65]{2,1,3,0} copy(param0)
+      copy3 = f16[8,31,31,65]{2,1,3,0} copy(param1)
+      ROOT tuple1 = (f16[8,31,31,65]{2,1,3,0}, f16[8,31,31,65]{2,1,3,0})
+        tuple(copy2, copy3)
+    }
+
+    ENTRY multiple_output_fusion_2 {
+      para0 = f16[8,31,31,65]{3,2,1,0} parameter(0)
+      para1 = f16[8,31,31,65]{1,3,2,0} parameter(1)
+      ROOT fusion1 = (f16[8,31,31,65]{2,1,3,0}, f16[8,31,31,65]{2,1,3,0})
+        fusion(para0,para1), kind=kLoop, calls=fused_computation.1
+    })";
+
+  // Check that a call to llvm.nvvm.barrier0 is not generated.
+  auto hlo_module = ParseHloString(kHloString, config_).ValueOrDie();
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: define void @fusion
+; CHECK-NOT: tail call void @llvm.nvvm.barrier0()
+; CHECK: }
+)",
+                     /*match_optimized_ir=*/true);
+}
+
+}  // namespace
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc
new file mode 100644
index 0000000000..6c9ae7bada
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_ldg_test.cc
@@ -0,0 +1,141 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Tests that we emit ld.global.nc (the PTX instruction corresponding to CUDA's
+// __ldg builtin) for reads of buffers that don't change during a kernel's
+// execution.
+
+#include 
+#include 
+
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace gpu {
+
+class GpuLdgTest : public GpuCodegenTest {};
+
+// Parameters are never overwritten, so parameter reads should get ld.global.nc
+// reads.
+TEST_F(GpuLdgTest, LdgForParamRead) {
+  HloComputation::Builder builder(TestName());
+
+  auto shape = ShapeUtil::MakeShape(F32, {2, 2});
+  HloInstruction* param =
+      builder.AddInstruction(HloInstruction::CreateParameter(0, shape, "x"));
+  builder.AddInstruction(
+      HloInstruction::CreateBinary(shape, HloOpcode::kAdd, param, param));
+  std::unique_ptr computation = builder.Build();
+
+  auto hlo_module = CreateNewModule();
+  hlo_module->AddEntryComputation(std::move(computation));
+
+  CompileAndVerifyPtx(std::move(hlo_module), R"(
+    CHECK-NOT: ld.global.f32
+    CHECK: ld.global.nc.f32
+  )");
+}
+
+// Check that reading a buffer produced by a non-parameter HLO also results in
+// ld.global.nc, if that buffer isn't modified within the instruction that reads
+// it.
+TEST_F(GpuLdgTest, LdgForNonParamRead) {
+  HloComputation::Builder builder(TestName());
+
+  auto shape = ShapeUtil::MakeShape(F32, {2, 2});
+  HloInstruction* param =
+      builder.AddInstruction(HloInstruction::CreateParameter(0, shape, "x"));
+  HloInstruction* add = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape, HloOpcode::kAdd, param, param));
+  HloInstruction* square = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape, HloOpcode::kMultiply, add, add));
+  builder.AddInstruction(HloInstruction::CreateTuple({add, square}));
+  std::unique_ptr computation = builder.Build();
+
+  auto hlo_module = CreateNewModule();
+  hlo_module->AddEntryComputation(std::move(computation));
+
+  CompileAndVerifyPtx(std::move(hlo_module), R"(
+    CHECK: {
+    CHECK-NOT: ld.global.f32
+    CHECK: ld.global.nc.f32
+    CHECK: }
+  )");
+}
+
+// Check that reading a buffer that's modified in-place does not produce
+// ld.global.nc.
+//
+// We do this by creating a reduce that feeds into a sin.  We don't currently
+// fuse sin into reduce, and the sin is elementwise, so it reuses its input
+// buffer as its output.
+//
+// It seems like a fair bet that we won't start fusing sin into the output of
+// reduce in the foreseeable future.  But if that turns out to be wrong, I give
+// you, future reader, permission to delete this test.
+TEST_F(GpuLdgTest, NoLdgWhenSharingBuffer) {
+  auto hlo_module = CreateNewModule();
+  HloComputation::Builder builder(TestName());
+
+  HloComputation* reduce_computation;
+  {
+    auto embedded_builder = HloComputation::Builder("add");
+    auto lhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        0, ShapeUtil::MakeShape(F32, {}), "lhs"));
+    auto rhs = embedded_builder.AddInstruction(HloInstruction::CreateParameter(
+        1, ShapeUtil::MakeShape(F32, {}), "rhs"));
+    embedded_builder.AddInstruction(
+        HloInstruction::CreateBinary(lhs->shape(), HloOpcode::kAdd, lhs, rhs));
+    reduce_computation =
+        hlo_module->AddEmbeddedComputation(embedded_builder.Build());
+  }
+
+  auto param_shape = ShapeUtil::MakeShape(F32, {2, 2});
+  auto reduce_shape = ShapeUtil::MakeShape(F32, {2});
+  HloInstruction* param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, param_shape, "x"));
+  HloInstruction* reduce = builder.AddInstruction(HloInstruction::CreateReduce(
+      reduce_shape,
+      builder.AddInstruction(HloInstruction::CreateBinary(
+          param_shape, HloOpcode::kAdd, param, param)),
+      builder.AddInstruction(
+          HloInstruction::CreateConstant(LiteralUtil::CreateR0(0))),
+      {0}, reduce_computation));
+  builder.AddInstruction(
+      HloInstruction::CreateUnary(reduce_shape, HloOpcode::kSin, reduce));
+
+  std::unique_ptr computation = builder.Build();
+  hlo_module->AddEntryComputation(std::move(computation));
+
+  CompileAndVerifyPtx(std::move(hlo_module), R"(
+    CHECK-LABEL: .entry sin
+    CHECK: {
+    CHECK-NOT: ld.global.nc.f32
+    CHECK: ld.global.f32
+    CHECK: }
+  )");
+}
+
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc
new file mode 100644
index 0000000000..c42e5704a4
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_noalias_test.cc
@@ -0,0 +1,68 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include 
+#include 
+
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace gpu {
+
+class GpuNoAliasTest : public GpuCodegenTest {};
+
+TEST_F(GpuNoAliasTest, Concat) {
+  HloComputation::Builder builder(TestName());
+
+  auto param_shape = ShapeUtil::MakeShape(F32, {2, 2});
+  HloInstruction* param_x = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, param_shape, "x"));
+  HloInstruction* param_y = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, param_shape, "y"));
+  HloInstruction* concat =
+      builder.AddInstruction(HloInstruction::CreateConcatenate(
+          ShapeUtil::MakeShape(F32, {2, 4}), {param_x, param_y}, 1));
+  builder.AddInstruction(HloInstruction::CreateConcatenate(
+      ShapeUtil::MakeShape(F32, {2, 6}), {concat, param_x}, 1));
+
+  std::unique_ptr computation = builder.Build();
+
+  auto hlo_module = CreateNewModule();
+  hlo_module->AddEntryComputation(std::move(computation));
+
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK: %[[x_gep:.*]] = getelementptr inbounds [2 x [2 x float]], [2 x [2 x float]]* %x{{.*}}, i32 0
+; CHECK: load float, float* %[[x_gep]], {{.*}}, !noalias ![[param_noalias:.*]]
+; CHECK: %[[y_gep:.*]] = getelementptr inbounds [2 x [2 x float]], [2 x [2 x float]]* %y{{.*}}, i32 0
+; CHECK: load float, float* %[[y_gep]], {{.*}}, !noalias ![[param_noalias]]
+; CHECK: %[[result_ptr:.*]] = bitcast [2 x [6 x float]]* %fusion{{.*}} to float*
+; CHECK: %[[result_gep:.*]] = getelementptr inbounds float, float* %[[result_ptr]]
+; CHECK: store float {{.*}}, float* %[[result_gep]], !alias.scope ![[param_noalias]]
+; CHECK: ![[param_noalias]] = !{![[retval_buffer:.*]]}
+      )",
+                     /*match_optimized_ir=*/false);
+}
+
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_unrolling_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_unrolling_test.cc
new file mode 100644
index 0000000000..9622936306
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_unrolling_test.cc
@@ -0,0 +1,185 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include 
+
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+#include "tensorflow/compiler/xla/service/hlo_module_config.h"
+#include "tensorflow/compiler/xla/service/hlo_parser.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace gpu {
+namespace {
+
+class GpuUnrollingTest : public GpuCodegenTest {};
+
+const char *const kAddModule = R"(
+    HloModule test_module
+
+    fused_computation {
+      p0.param_0 = f32[2,2]{1,0} parameter(0)
+      p1.param_1 = f32[2,2]{1,0} parameter(1)
+      ROOT add = f32[2,2] add(p0.param_0, p1.param_1)
+    }
+
+    ENTRY BroadcastIntoAdd {
+      p0 = f32[2,2]{1,0} parameter(0)
+      p1 = f32[2,2]{1,0} parameter(1)
+      ROOT fusion = f32[2,2]{1,0} fusion(p0, p1), kind=kLoop,
+                                                  calls=fused_computation
+    })";
+
+TEST_F(GpuUnrollingTest, DoNotUnroll) {
+  HloModuleConfig config;
+  auto debug_options = HloTestBase::GetDebugOptionsForTest();
+  debug_options.set_xla_gpu_max_kernel_unroll_factor(1);
+  config.set_debug_options(debug_options);
+  auto hlo_module = ParseHloString(kAddModule, config).ValueOrDie();
+
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: @fusion
+; CHECK: fadd
+; CHECK-NOT: fadd
+; CHECK: }
+      )",
+                     /*match_optimized_ir=*/true);
+}
+
+TEST_F(GpuUnrollingTest, UnrollFourTimes) {
+  HloModuleConfig config;
+  auto debug_options = HloTestBase::GetDebugOptionsForTest();
+  // We request a factor of 8, but the computation works on 4 elements, limiting
+  // the maximum unroll factor.
+  debug_options.set_xla_gpu_max_kernel_unroll_factor(8);
+  config.set_debug_options(debug_options);
+  auto hlo_module = ParseHloString(kAddModule, config).ValueOrDie();
+
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: @fusion
+; CHECK: fadd
+; CHECK: fadd
+; CHECK: fadd
+; CHECK: fadd
+; CHECK-NOT: fadd
+; CHECK: }
+      )",
+                     /*match_optimized_ir=*/true);
+}
+
+TEST_F(GpuUnrollingTest, UnrollDefaultTimes) {
+  // The default unrolling factor is 4.
+  HloModuleConfig config;
+  config.set_debug_options(legacy_flags::GetDebugOptionsFromFlags());
+  auto hlo_module = ParseHloString(kAddModule, config).ValueOrDie();
+
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: @fusion
+; CHECK: load <4 x float>
+; CHECK: fadd
+; CHECK: fadd
+; CHECK: fadd
+; CHECK: fadd
+; CHECK-NOT: fadd
+; CHECK: store <4 x float>
+; CHECK: }
+      )",
+                     /*match_optimized_ir=*/true);
+}
+
+TEST_F(GpuUnrollingTest, UnrollUnfusedAdd) {
+  HloModuleConfig config;
+  auto debug_options = HloTestBase::GetDebugOptionsForTest();
+  debug_options.set_xla_gpu_max_kernel_unroll_factor(4);
+  config.set_debug_options(debug_options);
+
+  const char *const kUnfusedAddModule = R"(
+    HloModule test_module
+
+    ENTRY AddFunc {
+      p0 = f32[2,2]{1,0} parameter(0)
+      p1 = f32[2,2]{1,0} parameter(1)
+      ROOT add = f32[2,2]{1,0} add(p0, p1)
+    })";
+  auto hlo_module = ParseHloString(kUnfusedAddModule, config).ValueOrDie();
+
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: @add
+; CHECK: load <4 x float>
+; CHECK: fadd
+; CHECK: fadd
+; CHECK: fadd
+; CHECK: fadd
+; CHECK-NOT: fadd
+; CHECK: store <4 x float>
+; CHECK: }
+      )",
+                     /*match_optimized_ir=*/true);
+}
+
+TEST_F(GpuUnrollingTest, UnrollMultiOutputFusion) {
+  HloModuleConfig config;
+  auto debug_options = HloTestBase::GetDebugOptionsForTest();
+  debug_options.set_xla_gpu_max_kernel_unroll_factor(2);
+  config.set_debug_options(debug_options);
+
+  const char *const kMultiOutputFusionModule = R"(
+    HloModule test_module
+
+    fused_computation {
+      p0.param_0 = f32[2,2]{1,0} parameter(0)
+      p1.param_1 = f32[2,2]{1,0} parameter(1)
+      add = f32[2,2]{1,0} add(p0.param_0, p1.param_1)
+      mul = f32[2,2]{1,0} multiply(p0.param_0, p1.param_1)
+      ROOT tuple = (f32[2,2]{1,0}, f32[2,2]{1,0}) tuple(add, mul)
+    }
+
+    ENTRY BroadcastIntoAdd {
+      p0 = f32[2,2]{1,0} parameter(0)
+      p1 = f32[2,2]{1,0} parameter(1)
+      ROOT fusion = (f32[2,2]{1,0}, f32[2,2]{1,0}) fusion(p0, p1), kind=kLoop,
+                                                   calls=fused_computation
+    })";
+  auto hlo_module =
+      ParseHloString(kMultiOutputFusionModule, config).ValueOrDie();
+
+  CompileAndVerifyIr(std::move(hlo_module),
+                     R"(
+; CHECK-LABEL: @fusion
+; CHECK: load <2 x float>
+; CHECK: load <2 x float>
+; CHECK-NOT: load <2 x float>
+; CHECK: fadd
+; CHECK: fmul
+; CHECK: fadd
+; CHECK: fmul
+; CHECK: store <2 x float>
+; CHECK: store <2 x float>
+; CHECK-NOT: store <2 x float>
+; CHECK-NOT: fadd
+; CHECK-NOT: fmul
+; CHECK: }
+      )",
+                     /*match_optimized_ir=*/true);
+}
+
+}  // namespace
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/tests/infeed_test.cc b/tensorflow/compiler/xla/service/gpu/tests/infeed_test.cc
new file mode 100644
index 0000000000..ba5cd2d84d
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/infeed_test.cc
@@ -0,0 +1,121 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include 
+#include 
+
+#include "tensorflow/compiler/xla/client/global_data.h"
+#include "tensorflow/compiler/xla/client/lib/arithmetic.h"
+#include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
+#include "tensorflow/compiler/xla/literal.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/test_helpers.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/literal_test_util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/math/math_util.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+namespace {
+
+class InfeedTest : public ClientLibraryTestBase {
+ protected:
+  // Transfers the given literal to the infeed interface of the device, and
+  // check if the returned data from Infeed HLO is same as the literal.
+  void TestInfeedRoundTrip(const Literal& literal) {
+    // TODO(b/30481585) Explicitly reset the Infeed state so that the
+    // test is not affected by the state from the previous tests.
+    ASSERT_IS_OK(client_->TransferToInfeed(literal));
+    XlaBuilder builder(TestName());
+    Infeed(&builder, literal.shape());
+    if (ShapeUtil::IsTuple(literal.shape())) {
+      // TODO(b/30609564): Use ComputeAndCompareLiteral instead.
+      ComputeAndCompareTuple(&builder, literal, {});
+    } else {
+      ComputeAndCompareLiteral(&builder, literal, {});
+    }
+  }
+};
+
+TEST_F(InfeedTest, SingleInfeedR0Bool) {
+  TestInfeedRoundTrip(*LiteralUtil::CreateR0(true));
+}
+
+TEST_F(InfeedTest, SingleInfeedR1U32) {
+  TestInfeedRoundTrip(*LiteralUtil::CreateR1({1, 2, 3}));
+}
+
+TEST_F(InfeedTest, SingleInfeedR2F32) {
+  TestInfeedRoundTrip(*LiteralUtil::CreateR2F32Linspace(0.0, 1.0, 128, 64));
+}
+
+TEST_F(InfeedTest, SingleInfeedR3F32) {
+  TestInfeedRoundTrip(
+      *LiteralUtil::CreateR3({{{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}},
+                              {{1.1f, 2.1f, 3.1f}, {6.1f, 3.5f, 2.8f}}}));
+}
+
+TEST_F(InfeedTest, SingleInfeedR3F32DifferentLayout) {
+  const Layout r3_dim0minor = LayoutUtil::MakeLayout({0, 1, 2});
+  const Layout r3_dim0major = LayoutUtil::MakeLayout({2, 1, 0});
+
+  TestInfeedRoundTrip(*LiteralUtil::CreateR3WithLayout(
+      {{{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}},
+       {{1.1f, 2.1f, 3.1f}, {6.1f, 3.5f, 2.8f}}},
+      r3_dim0minor));
+
+  TestInfeedRoundTrip(*LiteralUtil::CreateR3WithLayout(
+      {{{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}},
+       {{1.1f, 2.1f, 3.1f}, {6.1f, 3.5f, 2.8f}}},
+      r3_dim0major));
+}
+
+TEST_F(InfeedTest, SingleInfeedR4S32) {
+  TestInfeedRoundTrip(*LiteralUtil::CreateR4(
+      {{{{1, -2}, {-4, 5}, {6, 7}}, {{8, 9}, {10, 11}, {12, 13}}},
+       {{{10, 3}, {7, -2}, {3, 6}}, {{2, 5}, {-11, 5}, {-2, -5}}}}));
+}
+
+// Tests that a large infeed can be handled.
+TEST_F(InfeedTest, LargeInfeed) {
+  Array4D array(80, 100, 8, 128);
+  array.FillIota(1.0f);
+  TestInfeedRoundTrip(*LiteralUtil::CreateR4FromArray4D(array));
+}
+
+TEST_F(InfeedTest, SingleInfeedTuple) {
+  TestInfeedRoundTrip(
+      *LiteralUtil::MakeTuple({LiteralUtil::CreateR1({1, 2, 3}).get(),
+                               LiteralUtil::CreateR0(false).get()}));
+}
+
+TEST_F(InfeedTest, SingleInfeedEmptyTuple) {
+  TestInfeedRoundTrip(*LiteralUtil::MakeTuple({}));
+}
+
+// Tests that a large tuple infeed can be handled.
+TEST_F(InfeedTest, SingleInfeedLargeTuple) {
+  Array4D array(40, 100, 8, 128);
+  array.FillIota(1.0f);
+  TestInfeedRoundTrip(*LiteralUtil::MakeTuple(
+      {LiteralUtil::CreateR4FromArray4D(array).get(),
+       LiteralUtil::CreateR0(5).get()}));
+}
+
+}  // namespace
+}  // namespace xla
-- 
cgit v1.2.3


From d58d099edb59ba22e35067d5538edd91fae00e74 Mon Sep 17 00:00:00 2001
From: Akshay Modi 
Date: Mon, 23 Jul 2018 18:02:53 -0700
Subject: Remove unnecessary thread pool and use the worker env's compute pool
 directly.

PiperOrigin-RevId: 205756865
---
 .../core/distributed_runtime/rpc/eager/grpc_eager_service_impl.cc    | 4 +---
 .../core/distributed_runtime/rpc/eager/grpc_eager_service_impl.h     | 5 ++---
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.cc b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.cc
index 52e06c263d..44e880de04 100644
--- a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.cc
@@ -27,9 +27,7 @@ namespace eager {
 
 GrpcEagerServiceImpl::GrpcEagerServiceImpl(
     const WorkerEnv* env, ::grpc::ServerBuilder* server_builder)
-    : local_impl_(env) {
-  request_handler_threadpool_ =
-      MakeUnique(env->env, "EagerServiceRequestHandler", 4);
+    : env_(env), local_impl_(env) {
   server_builder->RegisterService(&service_);
   cq_ = server_builder->AddCompletionQueue();
 }
diff --git a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.h b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.h
index 9a94026342..502f3ef529 100644
--- a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.h
+++ b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.h
@@ -45,7 +45,7 @@ class GrpcEagerServiceImpl : public AsyncServiceInterface {
  private:
 #define HANDLER(method)                                                        \
   void method##Handler(EagerCall* call) {   \
-    request_handler_threadpool_->Schedule([this, call]() {                     \
+    env_->compute_pool->Schedule([this, call]() {                              \
       call->SendResponse(                                                      \
           ToGrpcStatus(local_impl_.method(&call->request, &call->response)));  \
     });                                                                        \
@@ -64,6 +64,7 @@ class GrpcEagerServiceImpl : public AsyncServiceInterface {
   HANDLER(RegisterFunction);
 #undef HANDLER
 
+  const WorkerEnv* const env_;  // Not owned.
   EagerServiceImpl local_impl_;
 
   std::unique_ptr<::grpc::Alarm> shutdown_alarm_;
@@ -71,8 +72,6 @@ class GrpcEagerServiceImpl : public AsyncServiceInterface {
   std::unique_ptr<::grpc::ServerCompletionQueue> cq_;
   tensorflow::eager::grpc::EagerService::AsyncService service_;
 
-  std::unique_ptr request_handler_threadpool_;
-
   TF_DISALLOW_COPY_AND_ASSIGN(GrpcEagerServiceImpl);
 };
 
-- 
cgit v1.2.3


From efe370fcb367efd069c8166120858492dffa9a33 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay 
Date: Mon, 23 Jul 2018 18:51:53 -0700
Subject: Fix error when symbolic tensors are passed as input to Sequential
 model (when no input shape is specified.)

PiperOrigin-RevId: 205761788
---
 tensorflow/python/keras/engine/sequential.py       | 28 ++++++++++---
 tensorflow/python/keras/engine/sequential_test.py  | 46 ++++++++++++++++++++--
 tensorflow/python/keras/engine/training.py         |  8 +++-
 .../api/golden/tensorflow.keras.-sequential.pbtxt  |  4 ++
 .../tensorflow.keras.models.-sequential.pbtxt      |  4 ++
 5 files changed, 79 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py
index 371504a503..41cdfda660 100644
--- a/tensorflow/python/keras/engine/sequential.py
+++ b/tensorflow/python/keras/engine/sequential.py
@@ -213,13 +213,31 @@ class Sequential(Model):
       self.outputs = [self.layers[-1].output]
       self.build()
 
-  @checkpointable.no_automatic_dependency_tracking
   def build(self, input_shape=None):
-    if input_shape and not self.inputs:
-      batch_shape = tuple(input_shape)
+    self._set_inputs_and_outputs(input_shape=input_shape)
+
+  def symbolic_set_inputs(self, inputs):
+    self._set_inputs_and_outputs(tensor=inputs)
+
+  @checkpointable.no_automatic_dependency_tracking
+  def _set_inputs_and_outputs(self, input_shape=None, tensor=None):
+    """Set model's input and output specs based on the input received.
+
+    If `tensor` is provided, `input_shape` is not required.
+
+    Args:
+      input_shape: Optional shape of input.
+      tensor: Optional existing tensor to wrap into the `Input` layer.
+    """
+    if not self.inputs:
       dtype = K.floatx()
-      x = Input(
-          batch_shape=batch_shape, dtype=dtype, name=self.name + '_input')
+      if tensor is not None:
+        batch_shape = (None,) + tuple(tensor.get_shape().as_list()[1:])
+        x = Input(dtype=dtype, name=self.name + '_input', tensor=tensor)
+      elif input_shape is not None:
+        batch_shape = tuple(input_shape)
+        x = Input(
+            batch_shape=batch_shape, dtype=dtype, name=self.name + '_input')
       self.inputs = [x]
       for layer in self._layers:
         x = layer(x)
diff --git a/tensorflow/python/keras/engine/sequential_test.py b/tensorflow/python/keras/engine/sequential_test.py
index 0f54e29cee..4f4adca333 100644
--- a/tensorflow/python/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/engine/sequential_test.py
@@ -22,7 +22,6 @@ import numpy as np
 
 from tensorflow.python import keras
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.eager import context
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
@@ -104,9 +103,6 @@ class TestSequential(test.TestCase):
 
   @tf_test_util.run_in_graph_and_eager_modes
   def test_sequential_deferred_build_with_dataset_iterators(self):
-    if not context.executing_eagerly():
-      # TODO(psv/fchollet): Add support for this use case in graph mode.
-      return
     num_hidden = 5
     input_dim = 3
     num_classes = 2
@@ -136,6 +132,48 @@ class TestSequential(test.TestCase):
                      [None, num_classes])
     self.assertEqual(len(model.weights), 2 * 2)
 
+  def test_training_and_eval_methods_on_symbolic_tensors(self):
+    with self.test_session():
+
+      def create_model():
+        model = keras.Sequential()
+        model.add(keras.layers.Dense(10, activation='relu'))
+        model.add(keras.layers.Dense(4, activation='softmax'))
+
+        model.compile(
+            optimizer=rmsprop.RMSPropOptimizer(1e-3),
+            loss='categorical_crossentropy',
+            metrics=['accuracy'])
+        return model
+
+      inputs = keras.backend.zeros(shape=(10, 3))
+      targets = keras.backend.zeros(shape=(10, 4))
+
+      model = create_model()
+      model.fit(inputs, targets, epochs=10, steps_per_epoch=30)
+
+      model = create_model()
+      model.evaluate(inputs, targets, steps=2, verbose=0)
+
+      model = create_model()
+      model.predict(inputs, steps=2)
+
+      model = create_model()
+      model.train_on_batch(inputs, targets)
+
+      model = create_model()
+      model.test_on_batch(inputs, targets)
+
+      model = create_model()
+      model.fit(
+          inputs,
+          targets,
+          epochs=1,
+          steps_per_epoch=2,
+          verbose=0,
+          validation_data=(inputs, targets),
+          validation_steps=2)
+
   @tf_test_util.run_in_graph_and_eager_modes
   def test_invalid_use_cases(self):
     # Added objects must be layer instances
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index fbc2a11eda..1d4ab1fe37 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -991,10 +991,14 @@ class Model(Network):
         inputs = inputs[0]
 
       if tensor_util.is_tensor(inputs):
-        input_shape = (None,) + tuple(inputs.get_shape().as_list()[1:])
+        if context.executing_eagerly():
+          input_shape = (None,) + tuple(inputs.get_shape().as_list()[1:])
+          self.build(input_shape=input_shape)
+        else:
+          self.symbolic_set_inputs(inputs)
       else:
         input_shape = (None,) + inputs.shape[1:]
-      self.build(input_shape=input_shape)
+        self.build(input_shape=input_shape)
     elif context.executing_eagerly():
       self._eager_set_inputs(inputs)
     else:
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt
index 8295905975..65cfad77d1 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt
@@ -266,6 +266,10 @@ tf_class {
     name: "summary"
     argspec: "args=[\'self\', \'line_length\', \'positions\', \'print_fn\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "symbolic_set_inputs"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "test_on_batch"
     argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt
index 5211657414..6a83129f7d 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt
@@ -266,6 +266,10 @@ tf_class {
     name: "summary"
     argspec: "args=[\'self\', \'line_length\', \'positions\', \'print_fn\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "symbolic_set_inputs"
+    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "test_on_batch"
     argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-- 
cgit v1.2.3


From df7344f1933d932f03f472402068ff1883f0c011 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 19:08:40 -0700
Subject: Implementation of stack.

PiperOrigin-RevId: 205763219
---
 tensorflow/contrib/lite/build_def.bzl              | 408 +++++++++++----------
 tensorflow/contrib/lite/builtin_op_data.h          |   5 +
 .../contrib/lite/g3doc/tf_ops_compatibility.md     |  12 +
 tensorflow/contrib/lite/kernels/BUILD              |  15 +
 .../kernels/internal/reference/reference_ops.h     |  20 +
 tensorflow/contrib/lite/kernels/pack.cc            | 131 +++++++
 tensorflow/contrib/lite/kernels/pack_test.cc       | 119 ++++++
 tensorflow/contrib/lite/kernels/register.cc        |   2 +
 tensorflow/contrib/lite/model.cc                   |  10 +-
 .../contrib/lite/testing/generate_examples.py      |  38 ++
 .../propagate_array_data_types.cc                  |   8 +
 tensorflow/contrib/lite/toco/import_tensorflow.cc  |   1 +
 tensorflow/contrib/lite/toco/model.h               |   1 +
 tensorflow/contrib/lite/toco/tflite/operator.cc    |  22 ++
 .../contrib/lite/toco/tflite/operator_test.cc      |  10 +
 15 files changed, 599 insertions(+), 203 deletions(-)
 create mode 100644 tensorflow/contrib/lite/kernels/pack.cc
 create mode 100644 tensorflow/contrib/lite/kernels/pack_test.cc

diff --git a/tensorflow/contrib/lite/build_def.bzl b/tensorflow/contrib/lite/build_def.bzl
index 79f7455ad8..7c13f9011e 100644
--- a/tensorflow/contrib/lite/build_def.bzl
+++ b/tensorflow/contrib/lite/build_def.bzl
@@ -1,4 +1,5 @@
 """Generate Flatbuffer binary from json."""
+
 load(
     "//tensorflow:tensorflow.bzl",
     "tf_cc_test",
@@ -6,118 +7,120 @@ load(
 )
 
 def tflite_copts():
-  """Defines compile time flags."""
-  copts = [
-      "-DFARMHASH_NO_CXX_STRING",
-  ] + select({
-          str(Label("//tensorflow:android_arm64")): [
-              "-std=c++11",
-              "-O3",
-          ],
-          str(Label("//tensorflow:android_arm")): [
-              "-mfpu=neon",
-              "-mfloat-abi=softfp",
-              "-std=c++11",
-              "-O3",
-          ],
-          str(Label("//tensorflow:android_x86")): [
-              "-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK",
-          ],
-          str(Label("//tensorflow:ios_x86_64")): [
-              "-msse4.1",
-          ],
-          "//conditions:default": [],
-  }) + select({
-      str(Label("//tensorflow:with_default_optimizations")): [],
-      "//conditions:default": ["-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK"],
-  })
+    """Defines compile time flags."""
+    copts = [
+        "-DFARMHASH_NO_CXX_STRING",
+    ] + select({
+        str(Label("//tensorflow:android_arm64")): [
+            "-std=c++11",
+            "-O3",
+        ],
+        str(Label("//tensorflow:android_arm")): [
+            "-mfpu=neon",
+            "-mfloat-abi=softfp",
+            "-std=c++11",
+            "-O3",
+        ],
+        str(Label("//tensorflow:android_x86")): [
+            "-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK",
+        ],
+        str(Label("//tensorflow:ios_x86_64")): [
+            "-msse4.1",
+        ],
+        "//conditions:default": [],
+    }) + select({
+        str(Label("//tensorflow:with_default_optimizations")): [],
+        "//conditions:default": ["-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK"],
+    })
 
-  return copts
+    return copts
 
 LINKER_SCRIPT = "//tensorflow/contrib/lite/java/src/main/native:version_script.lds"
 
 def tflite_linkopts_unstripped():
-  """Defines linker flags to reduce size of TFLite binary.
+    """Defines linker flags to reduce size of TFLite binary.
 
-     These are useful when trying to investigate the relative size of the
-     symbols in TFLite.
+       These are useful when trying to investigate the relative size of the
+       symbols in TFLite.
 
-  Returns:
-     a select object with proper linkopts
-  """
-  return select({
-      "//tensorflow:android": [
-          "-Wl,--no-export-dynamic", # Only inc syms referenced by dynamic obj.
-          "-Wl,--exclude-libs,ALL",  # Exclude syms in all libs from auto export.
-          "-Wl,--gc-sections", # Eliminate unused code and data.
-          "-Wl,--as-needed", # Don't link unused libs.
-      ],
-      "//tensorflow/contrib/lite:mips": [],
-      "//tensorflow/contrib/lite:mips64": [],
-      "//conditions:default": [
-          "-Wl,--icf=all",  # Identical code folding.
-      ],
-  })
+    Returns:
+       a select object with proper linkopts
+    """
+    return select({
+        "//tensorflow:android": [
+            "-Wl,--no-export-dynamic",  # Only inc syms referenced by dynamic obj.
+            "-Wl,--exclude-libs,ALL",  # Exclude syms in all libs from auto export.
+            "-Wl,--gc-sections",  # Eliminate unused code and data.
+            "-Wl,--as-needed",  # Don't link unused libs.
+        ],
+        "//tensorflow/contrib/lite:mips": [],
+        "//tensorflow/contrib/lite:mips64": [],
+        "//conditions:default": [
+            "-Wl,--icf=all",  # Identical code folding.
+        ],
+    })
 
 def tflite_jni_linkopts_unstripped():
-  """Defines linker flags to reduce size of TFLite binary with JNI.
+    """Defines linker flags to reduce size of TFLite binary with JNI.
 
-     These are useful when trying to investigate the relative size of the
-     symbols in TFLite.
+       These are useful when trying to investigate the relative size of the
+       symbols in TFLite.
 
-  Returns:
-     a select object with proper linkopts
-  """
-  return select({
-      "//tensorflow:android": [
-          "-Wl,--gc-sections", # Eliminate unused code and data.
-          "-Wl,--as-needed", # Don't link unused libs.
-      ],
-      "//tensorflow/contrib/lite:mips": [],
-      "//tensorflow/contrib/lite:mips64": [],
-      "//conditions:default": [
-          "-Wl,--icf=all",  # Identical code folding.
-      ],
-  })
+    Returns:
+       a select object with proper linkopts
+    """
+    return select({
+        "//tensorflow:android": [
+            "-Wl,--gc-sections",  # Eliminate unused code and data.
+            "-Wl,--as-needed",  # Don't link unused libs.
+        ],
+        "//tensorflow/contrib/lite:mips": [],
+        "//tensorflow/contrib/lite:mips64": [],
+        "//conditions:default": [
+            "-Wl,--icf=all",  # Identical code folding.
+        ],
+    })
 
 def tflite_linkopts():
-  """Defines linker flags to reduce size of TFLite binary."""
-  return tflite_linkopts_unstripped() + select({
-      "//tensorflow:android": [
-          "-s",  # Omit symbol table.
-      ],
-      "//conditions:default": [],
-  })
+    """Defines linker flags to reduce size of TFLite binary."""
+    return tflite_linkopts_unstripped() + select({
+        "//tensorflow:android": [
+            "-s",  # Omit symbol table.
+        ],
+        "//conditions:default": [],
+    })
 
 def tflite_jni_linkopts():
-  """Defines linker flags to reduce size of TFLite binary with JNI."""
-  return tflite_jni_linkopts_unstripped() + select({
-      "//tensorflow:android": [
-          "-s",  # Omit symbol table.
-          "-latomic",  # Required for some uses of ISO C++11  in x86.
-      ],
-      "//conditions:default": [],
-  })
+    """Defines linker flags to reduce size of TFLite binary with JNI."""
+    return tflite_jni_linkopts_unstripped() + select({
+        "//tensorflow:android": [
+            "-s",  # Omit symbol table.
+            "-latomic",  # Required for some uses of ISO C++11  in x86.
+        ],
+        "//conditions:default": [],
+    })
 
-def tflite_jni_binary(name,
-                      copts=tflite_copts(),
-                      linkopts=tflite_jni_linkopts(),
-                      linkscript=LINKER_SCRIPT,
-                      linkshared=1,
-                      linkstatic=1,
-                      deps=[]):
-  """Builds a jni binary for TFLite."""
-  linkopts = linkopts + [
-      "-Wl,--version-script",  # Export only jni functions & classes.
-      "$(location {})".format(linkscript),
-  ]
-  native.cc_binary(
-      name=name,
-      copts=copts,
-      linkshared=linkshared,
-      linkstatic=linkstatic,
-      deps= deps + [linkscript],
-      linkopts=linkopts)
+def tflite_jni_binary(
+        name,
+        copts = tflite_copts(),
+        linkopts = tflite_jni_linkopts(),
+        linkscript = LINKER_SCRIPT,
+        linkshared = 1,
+        linkstatic = 1,
+        deps = []):
+    """Builds a jni binary for TFLite."""
+    linkopts = linkopts + [
+        "-Wl,--version-script",  # Export only jni functions & classes.
+        "$(location {})".format(linkscript),
+    ]
+    native.cc_binary(
+        name = name,
+        copts = copts,
+        linkshared = linkshared,
+        linkstatic = linkstatic,
+        deps = deps + [linkscript],
+        linkopts = linkopts,
+    )
 
 def tflite_cc_shared_object(name,
                             copts=tflite_copts(),
@@ -134,75 +137,75 @@ def tflite_cc_shared_object(name,
       deps=deps)
 
 def tf_to_tflite(name, src, options, out):
-  """Convert a frozen tensorflow graphdef to TF Lite's flatbuffer.
+    """Convert a frozen tensorflow graphdef to TF Lite's flatbuffer.
 
-  Args:
-    name: Name of rule.
-    src: name of the input graphdef file.
-    options: options passed to TOCO.
-    out: name of the output flatbuffer file.
-  """
+    Args:
+      name: Name of rule.
+      src: name of the input graphdef file.
+      options: options passed to TOCO.
+      out: name of the output flatbuffer file.
+    """
 
-  toco_cmdline = " ".join([
-      "//tensorflow/contrib/lite/toco:toco",
-      "--input_format=TENSORFLOW_GRAPHDEF",
-      "--output_format=TFLITE",
-      ("--input_file=$(location %s)" % src),
-      ("--output_file=$(location %s)" % out),
-  ] + options )
-  native.genrule(
-      name = name,
-      srcs=[src],
-      outs=[out],
-      cmd = toco_cmdline,
-      tools= ["//tensorflow/contrib/lite/toco:toco"],
-  )
+    toco_cmdline = " ".join([
+        "//tensorflow/contrib/lite/toco:toco",
+        "--input_format=TENSORFLOW_GRAPHDEF",
+        "--output_format=TFLITE",
+        ("--input_file=$(location %s)" % src),
+        ("--output_file=$(location %s)" % out),
+    ] + options)
+    native.genrule(
+        name = name,
+        srcs = [src],
+        outs = [out],
+        cmd = toco_cmdline,
+        tools = ["//tensorflow/contrib/lite/toco:toco"],
+    )
 
 def tflite_to_json(name, src, out):
-  """Convert a TF Lite flatbuffer to JSON.
+    """Convert a TF Lite flatbuffer to JSON.
 
-  Args:
-    name: Name of rule.
-    src: name of the input flatbuffer file.
-    out: name of the output JSON file.
-  """
+    Args:
+      name: Name of rule.
+      src: name of the input flatbuffer file.
+      out: name of the output JSON file.
+    """
 
-  flatc = "@flatbuffers//:flatc"
-  schema = "//tensorflow/contrib/lite/schema:schema.fbs"
-  native.genrule(
-      name = name,
-      srcs = [schema, src],
-      outs = [out],
-      cmd = ("TMP=`mktemp`; cp $(location %s) $${TMP}.bin &&"  +
-             "$(location %s) --raw-binary --strict-json -t" +
-             " -o /tmp $(location %s) -- $${TMP}.bin &&" +
-             "cp $${TMP}.json $(location %s)")
-            % (src, flatc, schema, out),
-      tools = [flatc],
-  )
+    flatc = "@flatbuffers//:flatc"
+    schema = "//tensorflow/contrib/lite/schema:schema.fbs"
+    native.genrule(
+        name = name,
+        srcs = [schema, src],
+        outs = [out],
+        cmd = ("TMP=`mktemp`; cp $(location %s) $${TMP}.bin &&" +
+               "$(location %s) --raw-binary --strict-json -t" +
+               " -o /tmp $(location %s) -- $${TMP}.bin &&" +
+               "cp $${TMP}.json $(location %s)") %
+              (src, flatc, schema, out),
+        tools = [flatc],
+    )
 
 def json_to_tflite(name, src, out):
-  """Convert a JSON file to TF Lite's flatbuffer.
+    """Convert a JSON file to TF Lite's flatbuffer.
 
-  Args:
-    name: Name of rule.
-    src: name of the input JSON file.
-    out: name of the output flatbuffer file.
-  """
+    Args:
+      name: Name of rule.
+      src: name of the input JSON file.
+      out: name of the output flatbuffer file.
+    """
 
-  flatc = "@flatbuffers//:flatc"
-  schema = "//tensorflow/contrib/lite/schema:schema_fbs"
-  native.genrule(
-      name = name,
-      srcs = [schema, src],
-      outs = [out],
-      cmd = ("TMP=`mktemp`; cp $(location %s) $${TMP}.json &&"  +
-             "$(location %s) --raw-binary --unknown-json --allow-non-utf8 -b" +
-             " -o /tmp $(location %s) $${TMP}.json &&" +
-             "cp $${TMP}.bin $(location %s)")
-      % (src, flatc, schema, out),
-      tools = [flatc],
-  )
+    flatc = "@flatbuffers//:flatc"
+    schema = "//tensorflow/contrib/lite/schema:schema_fbs"
+    native.genrule(
+        name = name,
+        srcs = [schema, src],
+        outs = [out],
+        cmd = ("TMP=`mktemp`; cp $(location %s) $${TMP}.json &&" +
+               "$(location %s) --raw-binary --unknown-json --allow-non-utf8 -b" +
+               " -o /tmp $(location %s) $${TMP}.json &&" +
+               "cp $${TMP}.bin $(location %s)") %
+              (src, flatc, schema, out),
+        tools = [flatc],
+    )
 
 # This is the master list of generated examples that will be made into tests. A
 # function called make_XXX_tests() must also appear in generate_examples.py.
@@ -245,6 +248,7 @@ def generated_test_models():
         "mul",
         "neg",
         "not_equal",
+        "pack",
         "pad",
         "padv2",
         "prelu",
@@ -279,58 +283,58 @@ def generated_test_models():
     ]
 
 def gen_zip_test(name, test_name, **kwargs):
-  """Generate a zipped-example test and its dependent zip files.
+    """Generate a zipped-example test and its dependent zip files.
 
-  Args:
-    name: Resulting cc_test target name
-    test_name: Test targets this model. Comes from the list above.
-    **kwargs: tf_cc_test kwargs.
-  """
-  gen_zipped_test_file(
-      name = "zip_%s" % test_name,
-      file = "%s.zip" % test_name,
-  )
-  tf_cc_test(name, **kwargs)
+    Args:
+      name: Resulting cc_test target name
+      test_name: Test targets this model. Comes from the list above.
+      **kwargs: tf_cc_test kwargs.
+    """
+    gen_zipped_test_file(
+        name = "zip_%s" % test_name,
+        file = "%s.zip" % test_name,
+    )
+    tf_cc_test(name, **kwargs)
 
 def gen_zipped_test_file(name, file):
-  """Generate a zip file of tests by using :generate_examples.
+    """Generate a zip file of tests by using :generate_examples.
 
-  Args:
-    name: Name of output. We will produce "`file`.files" as a target.
-    file: The name of one of the generated_examples targets, e.g. "transpose"
-  """
-  toco = "//tensorflow/contrib/lite/toco:toco"
-  native.genrule(
-      name = file + ".files",
-      cmd = ("$(locations :generate_examples) --toco $(locations %s) " % toco
-             + " --zip_to_output " + file + " $(@D)"),
-      outs = [file],
-      tools = [
-          ":generate_examples",
-          toco,
-      ],
-  )
+    Args:
+      name: Name of output. We will produce "`file`.files" as a target.
+      file: The name of one of the generated_examples targets, e.g. "transpose"
+    """
+    toco = "//tensorflow/contrib/lite/toco:toco"
+    native.genrule(
+        name = file + ".files",
+        cmd = ("$(locations :generate_examples) --toco $(locations %s) " % toco +
+               " --zip_to_output " + file + " $(@D)"),
+        outs = [file],
+        tools = [
+            ":generate_examples",
+            toco,
+        ],
+    )
 
-  native.filegroup(
-      name = name,
-      srcs = [file],
-  )
+    native.filegroup(
+        name = name,
+        srcs = [file],
+    )
 
 def gen_selected_ops(name, model):
-  """Generate the library that includes only used ops.
+    """Generate the library that includes only used ops.
 
-  Args:
-    name: Name of the generated library.
-    model: TFLite model to interpret.
-  """
-  out = name + "_registration.cc"
-  tool = "//tensorflow/contrib/lite/tools:generate_op_registrations"
-  tflite_path = "//tensorflow/contrib/lite"
-  native.genrule(
-      name = name,
-      srcs = [model],
-      outs = [out],
-      cmd = ("$(location %s) --input_model=$(location %s) --output_registration=$(location %s) --tflite_path=%s")
-      % (tool, model, out, tflite_path[2:]),
-      tools = [tool],
-  )
+    Args:
+      name: Name of the generated library.
+      model: TFLite model to interpret.
+    """
+    out = name + "_registration.cc"
+    tool = "//tensorflow/contrib/lite/tools:generate_op_registrations"
+    tflite_path = "//tensorflow/contrib/lite"
+    native.genrule(
+        name = name,
+        srcs = [model],
+        outs = [out],
+        cmd = ("$(location %s) --input_model=$(location %s) --output_registration=$(location %s) --tflite_path=%s") %
+              (tool, model, out, tflite_path[2:]),
+        tools = [tool],
+    )
diff --git a/tensorflow/contrib/lite/builtin_op_data.h b/tensorflow/contrib/lite/builtin_op_data.h
index a24aaad7dd..fd16aa1063 100644
--- a/tensorflow/contrib/lite/builtin_op_data.h
+++ b/tensorflow/contrib/lite/builtin_op_data.h
@@ -277,6 +277,11 @@ typedef struct {
   bool narrow_range;
 } TfLiteFakeQuantParams;
 
+typedef struct {
+  int values_count;
+  int axis;
+} TfLitePackParams;
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif  // __cplusplus
diff --git a/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md b/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md
index 49d00a66ba..967259b7a6 100644
--- a/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md
@@ -815,6 +815,18 @@ Outputs {
 }
 ```
 
+**PACK**
+
+```
+Inputs {
+  0: a list of tensors.
+  1: an integer.
+}
+Outputs {
+  0: A tensor of stacked tensors.
+}
+```
+
 And these are TensorFlow Lite operations that are present but not ready for
 custom models yet:
 
diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index 9549b4445d..c224132cae 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -176,6 +176,7 @@ cc_library(
         "mfcc.cc",
         "mul.cc",
         "neg.cc",
+        "pack.cc",
         "pad.cc",
         "pooling.cc",
         "pow.cc",
@@ -1156,6 +1157,20 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "pack_test",
+    size = "small",
+    srcs = ["pack_test.cc"],
+    tags = ["tflite_not_portable_ios"],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/contrib/lite:builtin_op_data",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index ef39be3f91..31a54c2b62 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -1860,6 +1860,26 @@ void Concatenation(int concat_dim, const Scalar* const* input_data,
   }
 }
 
+template 
+void Pack(int dim, const Scalar* const* input_data,
+          const Dims<4>* const* input_dims, int inputs_count,
+          Scalar* output_data, const Dims<4>& output_dims) {
+  TFLITE_DCHECK(IsPackedWithoutStrides(output_dims));
+  int outer_size = 1;
+  for (int i = dim + 1; i < 4; i++) {
+    outer_size *= output_dims.sizes[i];
+  }
+  Scalar* output_ptr = output_data;
+  const int copy_size = FlatSize(**input_dims) / outer_size;
+  for (int k = 0; k < outer_size; k++) {
+    for (int i = 0; i < inputs_count; ++i) {
+      memcpy(output_ptr, input_data[i] + k * copy_size,
+             copy_size * sizeof(Scalar));
+      output_ptr += copy_size;
+    }
+  }
+}
+
 // TODO(prabhumk): This is the same as the optimized implementation.
 // TODO(prabhumk): The quantized implementation of concatentation isn't fully
 // quantized as it takes scale as a floating point value. This should be fixed
diff --git a/tensorflow/contrib/lite/kernels/pack.cc b/tensorflow/contrib/lite/kernels/pack.cc
new file mode 100644
index 0000000000..bb3416f6a6
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/pack.cc
@@ -0,0 +1,131 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/lite/builtin_op_data.h"
+#include "tensorflow/contrib/lite/context.h"
+#include "tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/contrib/lite/kernels/internal/tensor.h"
+#include "tensorflow/contrib/lite/kernels/kernel_util.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace pack {
+namespace {
+
+constexpr int kOutputTensor = 0;
+
+// Op data for pack op.
+struct OpData {
+  int values_count;
+  int axis;
+};
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  auto* data = new OpData;
+  data->axis = 0;
+  return data;
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+  delete reinterpret_cast(buffer);
+}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  const OpData* data = reinterpret_cast(node->builtin_data);
+
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), data->values_count);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  const TfLiteTensor* input0 = GetInput(context, node, 0);
+  TF_LITE_ENSURE(context, NumDimensions(input0) < 4);
+  TF_LITE_ENSURE(context, NumDimensions(input0) >= data->axis);
+  // TODO(renjieliu): Support negative axis.
+  TF_LITE_ENSURE(context, data->axis >= 0);
+  if (input0->type != kTfLiteInt32 && input0->type != kTfLiteFloat32) {
+    context->ReportError(context,
+                         "Currently pack only supports int32 and float32.");
+    return kTfLiteError;
+  }
+  // Make sure all inputs have the same shape and type.
+  for (int i = 1; i < data->values_count; ++i) {
+    const TfLiteTensor* input = GetInput(context, node, i);
+    TF_LITE_ENSURE(context, HaveSameShapes(input0, input));
+    TF_LITE_ENSURE_EQ(context, input0->type, input->type);
+  }
+
+  // Resize output. rank R will become rank R + 1
+  const int dimension_size = NumDimensions(input0) + 1;
+  const TfLiteIntArray* input_shape = input0->dims;
+  TfLiteIntArray* output_shape = TfLiteIntArrayCreate(dimension_size);
+  int i = 0;
+  for (int index = 0; index < dimension_size; ++index) {
+    if (index == data->axis) {
+      output_shape->data[index] = data->values_count;
+    } else {
+      output_shape->data[index] = input_shape->data[i++];
+    }
+  }
+
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  TF_LITE_ENSURE_EQ(context, output->type, input0->type);
+
+  return context->ResizeTensor(context, output, output_shape);
+}
+
+template 
+void PackImpl(TfLiteContext* context, TfLiteNode* node, TfLiteTensor* output,
+              int values_count, int axis) {
+  VectorOfTensors all_inputs(*context, *node->inputs);
+  reference_ops::Pack(RemapDim(NumDimensions(output), axis),
+                         all_inputs.data(), all_inputs.dims(), values_count,
+                         GetTensorData(output), GetTensorDims(output));
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const OpData* data = reinterpret_cast(node->builtin_data);
+
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+  switch (output->type) {
+    case kTfLiteFloat32: {
+      PackImpl(context, node, output, data->values_count, data->axis);
+      break;
+    }
+    case kTfLiteInt32: {
+      PackImpl(context, node, output, data->values_count, data->axis);
+      break;
+    }
+    default: {
+      context->ReportError(context,
+                           "Currently pack only supports int32 and float32.");
+      return kTfLiteError;
+    }
+  }
+
+  return kTfLiteOk;
+}
+
+}  // namespace
+}  // namespace pack
+
+TfLiteRegistration* Register_PACK() {
+  static TfLiteRegistration r = {pack::Init, pack::Free, pack::Prepare,
+                                 pack::Eval};
+  return &r;
+}
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/pack_test.cc b/tensorflow/contrib/lite/kernels/pack_test.cc
new file mode 100644
index 0000000000..cb9fed69b1
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/pack_test.cc
@@ -0,0 +1,119 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include 
+#include "tensorflow/contrib/lite/interpreter.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/kernels/test_util.h"
+#include "tensorflow/contrib/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAre;
+
+template 
+class PackOpModel : public SingleOpModel {
+ public:
+  PackOpModel(const TensorData& input_template, int axis, int values_count) {
+    std::vector> all_input_shapes;
+    for (int i = 0; i < values_count; ++i) {
+      all_input_shapes.push_back(input_template.shape);
+      AddInput(input_template);
+    }
+    output_ = AddOutput({input_template.type, /*shape=*/{}, input_template.min,
+                         input_template.max});
+    SetBuiltinOp(BuiltinOperator_PACK, BuiltinOptions_PackOptions,
+                 CreatePackOptions(builder_, values_count, axis).Union());
+    BuildInterpreter(all_input_shapes);
+  }
+
+  void SetInput(int index, std::initializer_list data) {
+    PopulateTensor(index, data);
+  }
+
+  std::vector GetOutput() { return ExtractVector(output_); }
+  std::vector GetOutputShape() { return GetTensorShape(output_); }
+
+ private:
+  int output_;
+};
+
+TEST(PackOpTest, FloatThreeInputs) {
+  PackOpModel model({TensorType_FLOAT32, {2}}, 0, 3);
+  model.SetInput(0, {1, 4});
+  model.SetInput(1, {2, 5});
+  model.SetInput(2, {3, 6});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(3, 2));
+  EXPECT_THAT(model.GetOutput(), ElementsAre(1, 4, 2, 5, 3, 6));
+}
+
+TEST(PackOpTest, FloatThreeInputsDifferentAxis) {
+  PackOpModel model({TensorType_FLOAT32, {2}}, 1, 3);
+  model.SetInput(0, {1, 4});
+  model.SetInput(1, {2, 5});
+  model.SetInput(2, {3, 6});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(2, 3));
+  EXPECT_THAT(model.GetOutput(), ElementsAre(1, 2, 3, 4, 5, 6));
+}
+
+TEST(PackOpTest, FloatMultilDimensions) {
+  PackOpModel model({TensorType_FLOAT32, {2, 3}}, 1, 2);
+  model.SetInput(0, {1, 2, 3, 4, 5, 6});
+  model.SetInput(1, {7, 8, 9, 10, 11, 12});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(2, 2, 3));
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAre(1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12));
+}
+
+TEST(PackOpTest, IntThreeInputs) {
+  PackOpModel model({TensorType_INT32, {2}}, 0, 3);
+  model.SetInput(0, {1, 4});
+  model.SetInput(1, {2, 5});
+  model.SetInput(2, {3, 6});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(3, 2));
+  EXPECT_THAT(model.GetOutput(), ElementsAre(1, 4, 2, 5, 3, 6));
+}
+
+TEST(PackOpTest, IntThreeInputsDifferentAxis) {
+  PackOpModel model({TensorType_INT32, {2}}, 1, 3);
+  model.SetInput(0, {1, 4});
+  model.SetInput(1, {2, 5});
+  model.SetInput(2, {3, 6});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(2, 3));
+  EXPECT_THAT(model.GetOutput(), ElementsAre(1, 2, 3, 4, 5, 6));
+}
+
+TEST(PackOpTest, IntMultilDimensions) {
+  PackOpModel model({TensorType_INT32, {2, 3}}, 1, 2);
+  model.SetInput(0, {1, 2, 3, 4, 5, 6});
+  model.SetInput(1, {7, 8, 9, 10, 11, 12});
+  model.Invoke();
+  EXPECT_THAT(model.GetOutputShape(), ElementsAre(2, 2, 3));
+  EXPECT_THAT(model.GetOutput(),
+              ElementsAre(1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12));
+}
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc
index f0f2757277..0b70bed308 100644
--- a/tensorflow/contrib/lite/kernels/register.cc
+++ b/tensorflow/contrib/lite/kernels/register.cc
@@ -106,6 +106,7 @@ TfLiteRegistration* Register_RSQRT();
 TfLiteRegistration* Register_SHAPE();
 TfLiteRegistration* Register_POW();
 TfLiteRegistration* Register_FAKE_QUANT();
+TfLiteRegistration* Register_PACK();
 
 BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_RELU, Register_RELU());
@@ -195,6 +196,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_SHAPE, Register_SHAPE());
   AddBuiltin(BuiltinOperator_POW, Register_POW());
   AddBuiltin(BuiltinOperator_FAKE_QUANT, Register_FAKE_QUANT(), 1, 2);
+  AddBuiltin(BuiltinOperator_PACK, Register_PACK());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index d318591b49..ad9a7de39c 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -705,6 +705,15 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
       *builtin_data = static_cast(params);
       break;
     }
+    case BuiltinOperator_PACK: {
+      TfLitePackParams* params = MallocPOD();
+      if (auto* pack_params = op->builtin_options_as_PackOptions()) {
+        params->values_count = pack_params->values_count();
+        params->axis = pack_params->axis();
+      }
+      *builtin_data = reinterpret_cast(params);
+      break;
+    }
     case BuiltinOperator_DELEGATE: {
       // TODO(ycling): Revisit when supporting saving delegated models.
       error_reporter->Report("DELEGATE op shouldn't exist in model.");
@@ -763,7 +772,6 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_TOPK_V2:
     case BuiltinOperator_TRANSPOSE:
     case BuiltinOperator_POW:
-    case BuiltinOperator_PACK:
       break;
   }
   return kTfLiteOk;
diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index b3ccc65e85..41ece94237 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -2880,6 +2880,44 @@ def make_sparse_to_dense_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_pack_tests(zip_path):
+  """Make a set of tests to do stack."""
+
+  test_parameters = [{
+      "base_shape": [[3, 4, 3], [3, 4], [5]],
+      "num_tensors": [1, 2, 3, 4, 5, 6],
+      "axis": [0, 1, 2, 3],
+      "additional_shape": [1, 2, 3],
+  }]
+
+  def get_shape(parameters):
+    """Return a tweaked version of 'base_shape'."""
+    axis = parameters["axis"]
+    shape = parameters["base_shape"][:]
+    if axis < len(shape):
+      shape[axis] += parameters["additional_shape"]
+    return shape
+
+  def build_graph(parameters):
+    all_tensors = []
+    for n in range(0, parameters["num_tensors"]):
+      input_tensor = tf.placeholder(
+          dtype=tf.float32, name=("input%d" % n), shape=get_shape(parameters))
+      all_tensors.append(input_tensor)
+    out = tf.stack(all_tensors, parameters["axis"])
+    return all_tensors, [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    all_values = []
+    for _ in range(0, parameters["num_tensors"]):
+      input_values = create_tensor_data(np.float32, get_shape(parameters))
+      all_values.append(input_values)
+    return all_values, sess.run(
+        outputs, feed_dict=dict(zip(inputs, all_values)))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 # Toco binary path provided by the generate rule.
 bin_path = None
 
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc
index 3dda536ef7..9848d55c83 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -193,6 +193,14 @@ bool PropagateArrayDataTypes::Run(Model* model, std::size_t op_index) {
       SetDataTypeForAllOutputs(model, op, data_type);
       break;
     }
+    case OperatorType::kPack: {
+      const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
+      for (const auto& input : op->inputs) {
+        CHECK(data_type == model->GetArray(input).data_type);
+      }
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
+    }
     default: {
       // These operators produce outputs with the same type as their 1st input
       CHECK_GT(op->inputs.size(), 0);
diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc
index 8bb797fe0f..032c863945 100644
--- a/tensorflow/contrib/lite/toco/import_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc
@@ -1529,6 +1529,7 @@ tensorflow::Status ConvertPackOperator(
   for (int i = 0; i < num_inputs; ++i) {
     op->inputs.push_back(node.input(i));
   }
+  op->values_count = HasAttr(node, "N") ? GetIntAttr(node, "N") : num_inputs;
   op->axis = HasAttr(node, "axis") ? GetIntAttr(node, "axis") : 0;
   op->dtype = ConvertDataType(toco::GetDataTypeAttr(node, "T"));
   op->outputs.push_back(node.name());
diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h
index 6fe194516d..d629787939 100644
--- a/tensorflow/contrib/lite/toco/model.h
+++ b/tensorflow/contrib/lite/toco/model.h
@@ -1164,6 +1164,7 @@ struct TensorFlowRsqrtOperator : Operator {
 // TensorFlow equivalent: Pack
 struct PackOperator : Operator {
   PackOperator() : Operator(OperatorType::kPack) {}
+  int values_count;
   int axis = 0;
   ArrayDataType dtype = ArrayDataType::kNone;
 };
diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc
index 1a1c4b8944..4b2ef756cc 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator.cc
@@ -1013,6 +1013,26 @@ class ExpandDims
   int GetVersion(const Operator& op) const override { return 1; }
 };
 
+class Pack : public BuiltinOperator {
+ public:
+  using BuiltinOperator::BuiltinOperator;
+
+  flatbuffers::Offset WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreatePackOptions(*builder, op.values_count, op.axis);
+  }
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    op->values_count = options.values_count();
+    op->axis = options.axis();
+  }
+
+  int GetVersion(const Operator& op) const override { return 1; }
+};
+
 class Shape
     : public BuiltinOperator {
@@ -1256,6 +1276,8 @@ std::vector> BuildOperatorList() {
       new Shape(::tflite::BuiltinOperator_SHAPE, OperatorType::kShape));
   ops.emplace_back(new FakeQuant(::tflite::BuiltinOperator_FAKE_QUANT,
                                  OperatorType::kFakeQuant));
+  ops.emplace_back(
+      new Pack(::tflite::BuiltinOperator_PACK, OperatorType::kPack));
 
   // Custom Operators.
   ops.emplace_back(
diff --git a/tensorflow/contrib/lite/toco/tflite/operator_test.cc b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
index ff2d35b1f5..44de6fbf64 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
@@ -452,6 +452,16 @@ TEST_F(OperatorTest, BuiltinSparseToDense) {
   EXPECT_EQ(op.validate_indices, output_toco_op->validate_indices);
 }
 
+TEST_F(OperatorTest, BuiltinPack) {
+  PackOperator op;
+  op.values_count = 3;
+  op.axis = 1;
+  std::unique_ptr output_toco_op =
+      SerializeAndDeserialize(GetOperator("PACK", OperatorType::kPack), op);
+  EXPECT_EQ(op.values_count, output_toco_op->values_count);
+  EXPECT_EQ(op.axis, output_toco_op->axis);
+}
+
 TEST_F(OperatorTest, TensorFlowUnsupported) {
   TensorFlowUnsupportedOperator op;
   op.tensorflow_op = "MyCustomUnsupportedOp";
-- 
cgit v1.2.3


From 438eae1515d6b49e6d0578874657fb80459ef8e8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 19:48:48 -0700
Subject: Add Logical_or to schema.

PiperOrigin-RevId: 205765869
---
 tensorflow/contrib/lite/builtin_ops.h             |   1 +
 tensorflow/contrib/lite/model.cc                  |   1 +
 tensorflow/contrib/lite/nnapi_delegate.cc         |   1 +
 tensorflow/contrib/lite/schema/schema.fbs         |   5 +
 tensorflow/contrib/lite/schema/schema_generated.h | 124 ++++++++++++++++++++--
 5 files changed, 126 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h
index 558e547121..1ae73b9738 100644
--- a/tensorflow/contrib/lite/builtin_ops.h
+++ b/tensorflow/contrib/lite/builtin_ops.h
@@ -109,6 +109,7 @@ typedef enum {
   kTfLiteBuiltinReduceProd = 81,
   kTfLiteBuiltinReduceMax = 82,
   kTfLiteBuiltinPack = 83,
+  kTfLiteBuiltinLogicalOr = 84,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index ad9a7de39c..c6869feb16 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -772,6 +772,7 @@ TfLiteStatus ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_TOPK_V2:
     case BuiltinOperator_TRANSPOSE:
     case BuiltinOperator_POW:
+    case BuiltinOperator_LOGICAL_OR:
       break;
   }
   return kTfLiteOk;
diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc
index 659230e033..551e8ed320 100644
--- a/tensorflow/contrib/lite/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/nnapi_delegate.cc
@@ -622,6 +622,7 @@ TfLiteStatus AddOpsAndParams(
       case tflite::BuiltinOperator_POW:
       case tflite::BuiltinOperator_FAKE_QUANT:
       case tflite::BuiltinOperator_PACK:
+      case tflite::BuiltinOperator_LOGICAL_OR:
         logError("Op code %d is currently not delegated to NNAPI", builtin);
         return kTfLiteError;
         break;
diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs
index 0434199a08..a285bf9919 100644
--- a/tensorflow/contrib/lite/schema/schema.fbs
+++ b/tensorflow/contrib/lite/schema/schema.fbs
@@ -165,6 +165,7 @@ enum BuiltinOperator : byte {
   REDUCE_PROD = 81,
   REDUCE_MAX = 82,
   PACK = 83,
+  LOGICAL_OR = 84,
 }
 
 // Options for the builtin operators.
@@ -228,6 +229,7 @@ union BuiltinOptions {
   ArgMinOptions,
   FakeQuantOptions,
   PackOptions,
+  LogicalOrOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -544,6 +546,9 @@ table PackOptions {
   axis:int;
 }
 
+table LogicalOrOptions {
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h
index 9b84030938..8c1d6d6a36 100755
--- a/tensorflow/contrib/lite/schema/schema_generated.h
+++ b/tensorflow/contrib/lite/schema/schema_generated.h
@@ -208,6 +208,9 @@ struct FakeQuantOptionsT;
 struct PackOptions;
 struct PackOptionsT;
 
+struct LogicalOrOptions;
+struct LogicalOrOptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -357,11 +360,12 @@ enum BuiltinOperator {
   BuiltinOperator_REDUCE_PROD = 81,
   BuiltinOperator_REDUCE_MAX = 82,
   BuiltinOperator_PACK = 83,
+  BuiltinOperator_LOGICAL_OR = 84,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_PACK
+  BuiltinOperator_MAX = BuiltinOperator_LOGICAL_OR
 };
 
-inline BuiltinOperator (&EnumValuesBuiltinOperator())[83] {
+inline BuiltinOperator (&EnumValuesBuiltinOperator())[84] {
   static BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -445,7 +449,8 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[83] {
     BuiltinOperator_FAKE_QUANT,
     BuiltinOperator_REDUCE_PROD,
     BuiltinOperator_REDUCE_MAX,
-    BuiltinOperator_PACK
+    BuiltinOperator_PACK,
+    BuiltinOperator_LOGICAL_OR
   };
   return values;
 }
@@ -536,6 +541,7 @@ inline const char **EnumNamesBuiltinOperator() {
     "REDUCE_PROD",
     "REDUCE_MAX",
     "PACK",
+    "LOGICAL_OR",
     nullptr
   };
   return names;
@@ -607,11 +613,12 @@ enum BuiltinOptions {
   BuiltinOptions_ArgMinOptions = 57,
   BuiltinOptions_FakeQuantOptions = 58,
   BuiltinOptions_PackOptions = 59,
+  BuiltinOptions_LogicalOrOptions = 60,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_PackOptions
+  BuiltinOptions_MAX = BuiltinOptions_LogicalOrOptions
 };
 
-inline BuiltinOptions (&EnumValuesBuiltinOptions())[60] {
+inline BuiltinOptions (&EnumValuesBuiltinOptions())[61] {
   static BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -672,7 +679,8 @@ inline BuiltinOptions (&EnumValuesBuiltinOptions())[60] {
     BuiltinOptions_PowOptions,
     BuiltinOptions_ArgMinOptions,
     BuiltinOptions_FakeQuantOptions,
-    BuiltinOptions_PackOptions
+    BuiltinOptions_PackOptions,
+    BuiltinOptions_LogicalOrOptions
   };
   return values;
 }
@@ -739,6 +747,7 @@ inline const char **EnumNamesBuiltinOptions() {
     "ArgMinOptions",
     "FakeQuantOptions",
     "PackOptions",
+    "LogicalOrOptions",
     nullptr
   };
   return names;
@@ -989,6 +998,10 @@ template<> struct BuiltinOptionsTraits {
   static const BuiltinOptions enum_value = BuiltinOptions_PackOptions;
 };
 
+template<> struct BuiltinOptionsTraits {
+  static const BuiltinOptions enum_value = BuiltinOptions_LogicalOrOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -1492,6 +1505,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_PackOptions ?
       reinterpret_cast(value) : nullptr;
   }
+  LogicalOrOptionsT *AsLogicalOrOptions() {
+    return type == BuiltinOptions_LogicalOrOptions ?
+      reinterpret_cast(value) : nullptr;
+  }
+  const LogicalOrOptionsT *AsLogicalOrOptions() const {
+    return type == BuiltinOptions_LogicalOrOptions ?
+      reinterpret_cast(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -5391,6 +5412,46 @@ inline flatbuffers::Offset CreatePackOptions(
 
 flatbuffers::Offset CreatePackOptions(flatbuffers::FlatBufferBuilder &_fbb, const PackOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct LogicalOrOptionsT : public flatbuffers::NativeTable {
+  typedef LogicalOrOptions TableType;
+  LogicalOrOptionsT() {
+  }
+};
+
+struct LogicalOrOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef LogicalOrOptionsT NativeTableType;
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           verifier.EndTable();
+  }
+  LogicalOrOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(LogicalOrOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset Pack(flatbuffers::FlatBufferBuilder &_fbb, const LogicalOrOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct LogicalOrOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  explicit LogicalOrOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  LogicalOrOptionsBuilder &operator=(const LogicalOrOptionsBuilder &);
+  flatbuffers::Offset Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset CreateLogicalOrOptions(
+    flatbuffers::FlatBufferBuilder &_fbb) {
+  LogicalOrOptionsBuilder builder_(_fbb);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset CreateLogicalOrOptions(flatbuffers::FlatBufferBuilder &_fbb, const LogicalOrOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -5701,6 +5762,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const PackOptions *builtin_options_as_PackOptions() const {
     return builtin_options_type() == BuiltinOptions_PackOptions ? static_cast(builtin_options()) : nullptr;
   }
+  const LogicalOrOptions *builtin_options_as_LogicalOrOptions() const {
+    return builtin_options_type() == BuiltinOptions_LogicalOrOptions ? static_cast(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector *custom_options() const {
     return GetPointer *>(VT_CUSTOM_OPTIONS);
   }
@@ -5968,6 +6032,10 @@ template<> inline const PackOptions *Operator::builtin_options_as()
   return builtin_options_as_PackOptions();
 }
 
+template<> inline const LogicalOrOptions *Operator::builtin_options_as() const {
+  return builtin_options_as_LogicalOrOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -8060,6 +8128,29 @@ inline flatbuffers::Offset CreatePackOptions(flatbuffers::FlatBuffe
       _axis);
 }
 
+inline LogicalOrOptionsT *LogicalOrOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new LogicalOrOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void LogicalOrOptions::UnPackTo(LogicalOrOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+}
+
+inline flatbuffers::Offset LogicalOrOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const LogicalOrOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateLogicalOrOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset CreateLogicalOrOptions(flatbuffers::FlatBufferBuilder &_fbb, const LogicalOrOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const LogicalOrOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  return tflite::CreateLogicalOrOptions(
+      _fbb);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -8485,6 +8576,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_LogicalOrOptions: {
+      auto ptr = reinterpret_cast(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -8739,6 +8834,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_LogicalOrOptions: {
+      auto ptr = reinterpret_cast(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -8981,6 +9080,10 @@ inline flatbuffers::Offset BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast(value);
       return CreatePackOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_LogicalOrOptions: {
+      auto ptr = reinterpret_cast(value);
+      return CreateLogicalOrOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -9223,6 +9326,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new PackOptionsT(*reinterpret_cast(u.value));
       break;
     }
+    case BuiltinOptions_LogicalOrOptions: {
+      value = new LogicalOrOptionsT(*reinterpret_cast(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -9525,6 +9632,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_LogicalOrOptions: {
+      auto ptr = reinterpret_cast(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
-- 
cgit v1.2.3


From 7fe6d775bb4a3f1dcc6484cab3dae1563dac6b42 Mon Sep 17 00:00:00 2001
From: Dan Moldovan 
Date: Mon, 23 Jul 2018 20:56:42 -0700
Subject: Default nonempty list initializers to Python lists, because the idiom
 is widely used in lieu of a tuple, and almost never as an actual initialized
 list that is about to be added even more elements. This removes the
 possibility to create an initialized tensor list, and to cover for that this
 CL also introduces a special function for that purpose.

PiperOrigin-RevId: 205771027
---
 tensorflow/contrib/autograph/__init__.py           | 12 +--
 .../contrib/autograph/converters/lists_test.py     | 18 ++---
 .../autograph/examples/integration_tests/BUILD     | 12 ++-
 .../integration_tests/list_literals_test.py        | 41 ++++++++++
 .../contrib/autograph/lang/special_functions.py    | 37 +++++++++
 .../autograph/lang/special_functions_test.py       | 18 ++++-
 .../contrib/autograph/operators/data_structures.py | 91 +++++++++++++++++++---
 .../autograph/operators/data_structures_test.py    | 41 ++++++++++
 8 files changed, 244 insertions(+), 26 deletions(-)
 create mode 100644 tensorflow/contrib/autograph/examples/integration_tests/list_literals_test.py

diff --git a/tensorflow/contrib/autograph/__init__.py b/tensorflow/contrib/autograph/__init__.py
index 7821c98f1c..26e7a4a4d3 100644
--- a/tensorflow/contrib/autograph/__init__.py
+++ b/tensorflow/contrib/autograph/__init__.py
@@ -22,20 +22,21 @@ from __future__ import division
 from __future__ import print_function
 
 # TODO(mdan): Bring only the relevant symbols to the top level.
-from tensorflow.contrib.autograph import utils
 from tensorflow.contrib.autograph import operators
+from tensorflow.contrib.autograph import utils
+from tensorflow.contrib.autograph.core.errors import GraphConstructionError
+from tensorflow.contrib.autograph.core.errors import TfRuntimeError
+from tensorflow.contrib.autograph.core.errors import improved_errors
+from tensorflow.contrib.autograph.impl.api import RunMode
 from tensorflow.contrib.autograph.impl.api import convert
 from tensorflow.contrib.autograph.impl.api import converted_call
 from tensorflow.contrib.autograph.impl.api import do_not_convert
-from tensorflow.contrib.autograph.impl.api import RunMode
 from tensorflow.contrib.autograph.impl.api import to_code
-from tensorflow.contrib.autograph.core.errors import improved_errors
-from tensorflow.contrib.autograph.core.errors import GraphConstructionError
-from tensorflow.contrib.autograph.core.errors import TfRuntimeError
 from tensorflow.contrib.autograph.impl.api import to_graph
 from tensorflow.contrib.autograph.lang.directives import set_element_type
 from tensorflow.contrib.autograph.lang.directives import set_loop_options
 from tensorflow.contrib.autograph.lang.special_functions import stack
+from tensorflow.contrib.autograph.lang.special_functions import tensor_list
 from tensorflow.contrib.autograph.pyct.transformer import AutographParseError
 from tensorflow.python.util.all_util import remove_undocumented
 
@@ -57,6 +58,7 @@ _allowed_symbols = [
     'set_element_type',
     'set_loop_options',
     'stack',
+    'tensor_list',
     # Exceptions
     'AutographParseError',
     # Utilities: to be removed
diff --git a/tensorflow/contrib/autograph/converters/lists_test.py b/tensorflow/contrib/autograph/converters/lists_test.py
index 447a88bbe2..f906918ac0 100644
--- a/tensorflow/contrib/autograph/converters/lists_test.py
+++ b/tensorflow/contrib/autograph/converters/lists_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.contrib.autograph.converters import lists
 from tensorflow.contrib.autograph.core import converter_testing
 from tensorflow.contrib.autograph.lang import directives
+from tensorflow.contrib.autograph.lang import special_functions
 from tensorflow.contrib.autograph.pyct import anno
 from tensorflow.contrib.autograph.pyct import parser
 from tensorflow.python.framework import dtypes
@@ -52,20 +53,18 @@ class ListTest(converter_testing.TestCase):
       return [1, 2, 3]
 
     with self.converted(test_fn, lists, {}) as result:
-      with self.test_session() as sess:
-        tl = result.test_fn()
-        r = list_ops.tensor_list_stack(tl, dtypes.int32)
-        self.assertAllEqual(sess.run(r), [1, 2, 3])
+      self.assertAllEqual(result.test_fn(), [1, 2, 3])
 
   def test_list_append(self):
 
     def test_fn():
-      l = [1]
+      l = special_functions.tensor_list([1])
       l.append(2)
       l.append(3)
       return l
 
-    with self.converted(test_fn, lists, {}) as result:
+    ns = {'special_functions': special_functions}
+    with self.converted(test_fn, lists, ns) as result:
       with self.test_session() as sess:
         tl = result.test_fn()
         r = list_ops.tensor_list_stack(tl, dtypes.int32)
@@ -74,11 +73,12 @@ class ListTest(converter_testing.TestCase):
   def test_list_pop(self):
 
     def test_fn():
-      l = [1, 2, 3]
+      l = special_functions.tensor_list([1, 2, 3])
       s = l.pop()
       return s, l
 
-    node, ctx = self.prepare(test_fn, {})
+    ns = {'special_functions': special_functions}
+    node, ctx = self.prepare(test_fn, ns)
     def_, = anno.getanno(node.body[0].body[0].targets[0],
                          anno.Static.ORIG_DEFINITIONS)
     def_.directives[directives.set_element_type] = {
@@ -87,7 +87,7 @@ class ListTest(converter_testing.TestCase):
     }
     node = lists.transform(node, ctx)
 
-    with self.compiled(node, {}, dtypes.int32) as result:
+    with self.compiled(node, ns, dtypes.int32) as result:
       with self.test_session() as sess:
         ts, tl = result.test_fn()
         r = list_ops.tensor_list_stack(tl, dtypes.int32)
diff --git a/tensorflow/contrib/autograph/examples/integration_tests/BUILD b/tensorflow/contrib/autograph/examples/integration_tests/BUILD
index 1368ce244c..2a4a0f75e7 100644
--- a/tensorflow/contrib/autograph/examples/integration_tests/BUILD
+++ b/tensorflow/contrib/autograph/examples/integration_tests/BUILD
@@ -22,7 +22,17 @@ py_test(
         "keras_test.py",
     ],
     srcs_version = "PY2AND3",
-    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_test(
+    name = "list_literals_test",
+    srcs = [
+        "list_literals_test.py",
+    ],
+    srcs_version = "PY2AND3",
     deps = [
         "//tensorflow:tensorflow_py",
     ],
diff --git a/tensorflow/contrib/autograph/examples/integration_tests/list_literals_test.py b/tensorflow/contrib/autograph/examples/integration_tests/list_literals_test.py
new file mode 100644
index 0000000000..680b6dbaf0
--- /dev/null
+++ b/tensorflow/contrib/autograph/examples/integration_tests/list_literals_test.py
@@ -0,0 +1,41 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests of functions that use list literals."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+
+from tensorflow.contrib import autograph as ag
+
+
+def list_used_as_tuple():
+  return tf.constant([1, 2, 3])
+
+
+class ListLiteralsTest(tf.test.TestCase):
+
+  def test_basic(self):
+    converted = ag.to_graph(list_used_as_tuple)
+    result = converted()
+
+    with self.test_session() as sess:
+      self.assertAllEqual(sess.run(result), [1, 2, 3])
+
+
+if __name__ == '__main__':
+  tf.test.main()
diff --git a/tensorflow/contrib/autograph/lang/special_functions.py b/tensorflow/contrib/autograph/lang/special_functions.py
index 11135295a7..6149cbbd6c 100644
--- a/tensorflow/contrib/autograph/lang/special_functions.py
+++ b/tensorflow/contrib/autograph/lang/special_functions.py
@@ -26,6 +26,43 @@ from __future__ import print_function
 from tensorflow.contrib.autograph.operators import data_structures
 
 
+def tensor_list(elements,
+                element_dtype=None,
+                element_shape=None,
+                use_tensor_array=False):
+  """Creates an tensor list and populates it with the given elements.
+
+  This function provides a more uniform access to tensor lists and tensor
+  arrays, and allows optional initialization.
+
+  Note: this function is a simplified wrapper. If you need greater control,
+  it is recommended to use the underlying implementation directly.
+
+  Args:
+    elements: Iterable[tf.Tensor, ...], the elements to initially fill the list
+        with
+    element_dtype: Optional[tf.DType], data type for the elements in the list;
+        required if the list is empty
+    element_shape: Optional[tf.TensorShape], shape for the elements in the list;
+        required if the list is empty
+    use_tensor_array: bool, whether to use the more compatible but restrictive
+        tf.TensorArray implementation
+  Returns:
+    Union[tf.Tensor, tf.TensorArray], the new list.
+  Raises:
+    ValueError: for invalid arguments
+  """
+  if not (elements or (element_dtype and element_shape)):
+    raise ValueError(
+        'element_dtype and element_shape are required for empty lists')
+  if use_tensor_array:
+    return data_structures.tf_tensor_array_new(elements, element_dtype,
+                                               element_shape)
+  else:
+    return data_structures.tf_tensor_list_new(elements, element_dtype,
+                                              element_shape)
+
+
 def stack(list_or_tensor, element_dtype=None, strict=True):
   """Stacks the input, if it admits the notion of stacking.
 
diff --git a/tensorflow/contrib/autograph/lang/special_functions_test.py b/tensorflow/contrib/autograph/lang/special_functions_test.py
index a49cb64075..db492cc5c6 100644
--- a/tensorflow/contrib/autograph/lang/special_functions_test.py
+++ b/tensorflow/contrib/autograph/lang/special_functions_test.py
@@ -28,7 +28,23 @@ from tensorflow.python.platform import test
 
 class SpecialFunctionsTest(test.TestCase):
 
-  def test_basic(self):
+  def test_tensor_list_from_elements(self):
+    elements = [constant_op.constant([1, 2]), constant_op.constant([3, 4])]
+
+    l = special_functions.tensor_list(elements)
+    sl = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
+    with self.test_session() as sess:
+      self.assertAllEqual(sess.run(sl), [[1, 2], [3, 4]])
+
+  def test_tensor_list_array_from_elements(self):
+    elements = [constant_op.constant([1, 2]), constant_op.constant([3, 4])]
+
+    l = special_functions.tensor_list(elements, use_tensor_array=True)
+    sl = l.stack()
+    with self.test_session() as sess:
+      self.assertAllEqual(sess.run(sl), [[1, 2], [3, 4]])
+
+  def test_stack(self):
     self.assertEqual(special_functions.stack(1, strict=False), 1)
     self.assertListEqual(
         special_functions.stack([1, 2, 3], strict=False), [1, 2, 3])
diff --git a/tensorflow/contrib/autograph/operators/data_structures.py b/tensorflow/contrib/autograph/operators/data_structures.py
index 06d8727b0f..cc0a3c3544 100644
--- a/tensorflow/contrib/autograph/operators/data_structures.py
+++ b/tensorflow/contrib/autograph/operators/data_structures.py
@@ -28,7 +28,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import list_ops
 from tensorflow.python.ops import tensor_array_ops
-from tensorflow.python.ops import variables
 
 
 # TODO(mdan): Once control flow supports objects, repackage as a class.
@@ -48,29 +47,101 @@ def new_list(iterable=None):
   else:
     elements = ()
 
-  # TODO(mdan): Extend these criteria.
-  if any(isinstance(el, variables.Variable) for el in elements):
+  if elements:
+    # When the list contains elements, it is assumed to be a "Python" lvalue
+    # list.
     return _py_list_new(elements)
-  return _tf_tensor_list_new(elements)
+  return tf_tensor_list_new(elements)
 
 
-def _tf_tensor_list_new(elements):
+def tf_tensor_array_new(elements, element_dtype=None, element_shape=None):
   """Overload of new_list that stages a Tensor list creation."""
   elements = tuple(ops.convert_to_tensor(el) for el in elements)
+
+  all_dtypes = set(el.dtype for el in elements)
+  if len(all_dtypes) == 1:
+    inferred_dtype, = tuple(all_dtypes)
+    if element_dtype is not None and element_dtype != inferred_dtype:
+      raise ValueError(
+          'incompatible dtype; specified: {}, inferred from {}: {}'.format(
+              element_dtype, elements, inferred_dtype))
+  elif len(all_dtypes) > 1:
+    raise ValueError(
+        'TensorArray requires all elements to have the same dtype:'
+        ' {}'.format(elements))
+  else:
+    if element_dtype is None:
+      raise ValueError('dtype is required to create an empty TensorArray')
+
+  all_shapes = set(tuple(el.shape.as_list()) for el in elements)
+  if len(all_shapes) == 1:
+    inferred_shape, = tuple(all_shapes)
+    if element_shape is not None and element_shape != inferred_shape:
+      raise ValueError(
+          'incompatible shape; specified: {}, inferred from {}: {}'.format(
+              element_shape, elements, inferred_shape))
+  elif len(all_shapes) > 1:
+    raise ValueError(
+        'TensorArray requires all elements to have the same shape:'
+        ' {}'.format(elements))
+    # TODO(mdan): We may want to allow different shapes with infer_shape=False.
+  else:
+    inferred_shape = None
+
+  if element_dtype is None:
+    element_dtype = inferred_dtype
+  if element_shape is None:
+    element_shape = inferred_shape
+
+  l = tensor_array_ops.TensorArray(
+      dtype=element_dtype,
+      size=len(elements),
+      dynamic_size=True,
+      infer_shape=(element_shape is None),
+      element_shape=element_shape)
+  for i, el in enumerate(elements):
+    l = l.write(i, el)
+  return l
+
+
+def tf_tensor_list_new(elements, element_dtype=None, element_shape=None):
+  """Overload of new_list that stages a Tensor list creation."""
+  elements = tuple(ops.convert_to_tensor(el) for el in elements)
+
   all_dtypes = set(el.dtype for el in elements)
   if len(all_dtypes) == 1:
-    element_dtype = tuple(all_dtypes)[0]
+    inferred_dtype = tuple(all_dtypes)[0]
+    if element_dtype is not None and element_dtype != inferred_dtype:
+      raise ValueError(
+          'incompatible dtype; specified: {}, inferred from {}: {}'.format(
+              element_dtype, elements, inferred_dtype))
   else:
     # Heterogeneous lists are ok.
-    element_dtype = dtypes.variant
+    if element_dtype is not None:
+      raise ValueError(
+          'specified dtype {} is inconsistent with that of elements {}'.format(
+              element_dtype, elements))
+    inferred_dtype = dtypes.variant
 
-  # TODO(mdan): This may fail for elements of variable shapes.
   all_shapes = set(tuple(el.shape.as_list()) for el in elements)
   if len(all_shapes) == 1:
-    element_shape = array_ops.shape(elements[0])
+    inferred_shape = array_ops.shape(elements[0])
+    if element_shape is not None and element_shape != inferred_shape:
+      raise ValueError(
+          'incompatible shape; specified: {}, inferred from {}: {}'.format(
+              element_shape, elements, inferred_shape))
   else:
     # Heterogeneous lists are ok.
-    element_shape = constant_op.constant(-1)  # unknown shape, by convention
+    if element_shape is not None:
+      raise ValueError(
+          'specified shape {} is inconsistent with that of elements {}'.format(
+              element_shape, elements))
+    inferred_shape = constant_op.constant(-1)  # unknown shape, by convention
+
+  if element_dtype is None:
+    element_dtype = inferred_dtype
+  if element_shape is None:
+    element_shape = inferred_shape
 
   l = list_ops.empty_tensor_list(
       element_shape=element_shape, element_dtype=element_dtype)
diff --git a/tensorflow/contrib/autograph/operators/data_structures_test.py b/tensorflow/contrib/autograph/operators/data_structures_test.py
index 8bbb52d6c1..7ea11a839b 100644
--- a/tensorflow/contrib/autograph/operators/data_structures_test.py
+++ b/tensorflow/contrib/autograph/operators/data_structures_test.py
@@ -37,10 +37,51 @@ class ListTest(test.TestCase):
 
   def test_new_list_tensor(self):
     l = data_structures.new_list([3, 4, 5])
+    self.assertAllEqual(l, [3, 4, 5])
+
+  def test_tf_tensor_list_new(self):
+    l = data_structures.tf_tensor_list_new([3, 4, 5])
     t = list_ops.tensor_list_stack(l, element_dtype=dtypes.int32)
     with self.test_session() as sess:
       self.assertAllEqual(sess.run(t), [3, 4, 5])
 
+  def test_tf_tensor_list_new_illegal_input(self):
+    with self.assertRaises(ValueError):
+      data_structures.tf_tensor_list_new([3, 4.0])
+    # TODO(mdan): It might make more sense to type cast in this case.
+    with self.assertRaises(ValueError):
+      data_structures.tf_tensor_list_new([3, 4], element_dtype=dtypes.float32)
+    # Tensor lists do support heterogeneous lists.
+    self.assertIsNot(data_structures.tf_tensor_list_new([3, [4, 5]]), None)
+    with self.assertRaises(ValueError):
+      data_structures.tf_tensor_list_new([3, 4], element_shape=(2,))
+    with self.assertRaises(ValueError):
+      data_structures.tf_tensor_list_new([], element_shape=(2,))
+    with self.assertRaises(ValueError):
+      data_structures.tf_tensor_list_new([], element_dtype=dtypes.float32)
+
+  def test_tf_tensor_array_new(self):
+    l = data_structures.tf_tensor_array_new([3, 4, 5])
+    t = l.stack()
+    with self.test_session() as sess:
+      self.assertAllEqual(sess.run(t), [3, 4, 5])
+
+  def test_tf_tensor_array_new_illegal_input(self):
+    with self.assertRaises(ValueError):
+      data_structures.tf_tensor_array_new([3, 4.0])
+    with self.assertRaises(ValueError):
+      data_structures.tf_tensor_array_new([3, 4], element_dtype=dtypes.float32)
+    with self.assertRaises(ValueError):
+      data_structures.tf_tensor_array_new([3, [4, 5]])
+    with self.assertRaises(ValueError):
+      data_structures.tf_tensor_array_new([3, 4], element_shape=(2,))
+    with self.assertRaises(ValueError):
+      data_structures.tf_tensor_array_new([], element_shape=(2,))
+    # TAs can infer the shape.
+    self.assertIsNot(
+        data_structures.tf_tensor_array_new([], element_dtype=dtypes.float32),
+        None)
+
   def test_append_tensor_list(self):
     l = data_structures.new_list()
     x = constant_op.constant([1, 2, 3])
-- 
cgit v1.2.3


From e8836f85a1dd14bd1e1113c7fe9fc7037ebdaa76 Mon Sep 17 00:00:00 2001
From: adoda 
Date: Tue, 24 Jul 2018 12:43:16 +0800
Subject: fix typo

---
 tensorflow/contrib/lite/toco/tflite/export_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/toco/tflite/export_test.cc b/tensorflow/contrib/lite/toco/tflite/export_test.cc
index d1fdbcb8e9..a95937ba0f 100644
--- a/tensorflow/contrib/lite/toco/tflite/export_test.cc
+++ b/tensorflow/contrib/lite/toco/tflite/export_test.cc
@@ -262,7 +262,7 @@ TEST_F(VersionedOpExportTest, Export) {
   EXPECT_EQ(1, (*operators)[1]->opcode_index());
 }
 
-// TODO(ahentz): tests for tensors, inputs, outpus, opcodes and operators.
+// TODO(ahentz): tests for tensors, inputs, outputs, opcodes and operators.
 
 }  // namespace
 }  // namespace tflite
-- 
cgit v1.2.3


From fada40a564f9f22cc4372b33c90ccce592035a58 Mon Sep 17 00:00:00 2001
From: Bixia Zheng 
Date: Mon, 23 Jul 2018 21:51:03 -0700
Subject: [XLA:GPU] Add an operator<< to Thunk::Kind.

This allows the use of CHECK_EQ with Thunk::Kind values.

PiperOrigin-RevId: 205775065
---
 tensorflow/compiler/xla/service/gpu/BUILD          |  1 +
 .../xla/service/gpu/ir_emitter_unnested.cc         |  2 +-
 tensorflow/compiler/xla/service/gpu/thunk.cc       | 59 ++++++++++++++++++++++
 tensorflow/compiler/xla/service/gpu/thunk.h        |  4 +-
 4 files changed, 64 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/gpu/thunk.cc

diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index ca39797e81..06ff3d9bba 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -269,6 +269,7 @@ cc_library(
         "memset_thunk.cc",
         "outfeed_thunk.cc",
         "sequential_thunk.cc",
+        "thunk.cc",
         "thunk_schedule.cc",
         "tuple_thunk.cc",
         "while_thunk.cc",
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 4844dc92db..64a6baf66d 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -2778,7 +2778,7 @@ Status IrEmitterUnnested::EmitTargetElementLoopInThunk(
 Status IrEmitterUnnested::EmitTargetElementLoop(
     const HloInstruction& hlo,
     const llvm_ir::ElementGenerator& element_generator) {
-  CHECK(Thunk::Kind::kKernel == LastThunk()->kind());
+  CHECK_EQ(Thunk::Kind::kKernel, LastThunk()->kind());
   return EmitTargetElementLoopInThunk(hlo, element_generator,
                                       static_cast(LastThunk()));
 }
diff --git a/tensorflow/compiler/xla/service/gpu/thunk.cc b/tensorflow/compiler/xla/service/gpu/thunk.cc
new file mode 100644
index 0000000000..c78605cebb
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/thunk.cc
@@ -0,0 +1,59 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/gpu/thunk.h"
+
+namespace xla {
+namespace gpu {
+
+std::ostream& operator<<(std::ostream& os, Thunk::Kind kind) {
+  switch (kind) {
+    case Thunk::kConditional:
+      return os << "kConditional";
+    case Thunk::kConvolution:
+      return os << "kConvolution";
+    case Thunk::kCopy:
+      return os << "kCopy";
+    case Thunk::kCudnnBatchNormBackward:
+      return os << "kCudnnBatchNormBackward";
+    case Thunk::kCudnnBatchNormForwardInference:
+      return os << "kCudnnBatchNormForwardInference";
+    case Thunk::kCudnnBatchNormForwardTraining:
+      return os << "kCudnnBatchNormForwardTraining";
+    case Thunk::kFft:
+      return os << "kFft";
+    case Thunk::kGemm:
+      return os << "kGemm";
+    case Thunk::kInfeed:
+      return os << "kInfeed";
+    case Thunk::kKernel:
+      return os << "kKernel";
+    case Thunk::kMemset32BitValue:
+      return os << "kMemset32BitValue";
+    case Thunk::kMemzero:
+      return os << "kMemzero";
+    case Thunk::kOutfeed:
+      return os << "kOutfeed";
+    case Thunk::kSequential:
+      return os << "kSequential";
+    case Thunk::kTuple:
+      return os << "kTuple";
+    case Thunk::kWhile:
+      return os << "kWhile";
+  }
+}
+
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/thunk.h b/tensorflow/compiler/xla/service/gpu/thunk.h
index 99a1a0eae9..4df0bb005b 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk.h
+++ b/tensorflow/compiler/xla/service/gpu/thunk.h
@@ -41,7 +41,7 @@ class GpuExecutable;
 // This is thread-compatible.
 class Thunk {
  public:
-  enum class Kind {
+  enum Kind {
     kConditional,
     kConvolution,
     kCopy,
@@ -111,6 +111,8 @@ class Thunk {
 // A sequence of thunks.
 using ThunkSequence = std::vector>;
 
+std::ostream& operator<<(std::ostream& os, Thunk::Kind kind);
+
 }  // namespace gpu
 }  // namespace xla
 
-- 
cgit v1.2.3


From 12f51c2873354577dcea167823d5a4c4dee5dbce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Mon, 23 Jul 2018 22:29:55 -0700
Subject: Add LinearOperatorZeros.

PiperOrigin-RevId: 205777765
---
 tensorflow/python/kernel_tests/linalg/BUILD        |  18 +
 .../linalg/linear_operator_zeros_test.py           | 192 +++++++++
 tensorflow/python/ops/linalg/linalg.py             |   1 +
 .../python/ops/linalg/linear_operator_zeros.py     | 452 +++++++++++++++++++++
 ...nalg.-linear-operator-zeros.__metaclass__.pbtxt |  14 +
 .../tensorflow.linalg.-linear-operator-zeros.pbtxt | 130 ++++++
 .../tools/api/golden/tensorflow.linalg.pbtxt       |   4 +
 7 files changed, 811 insertions(+)
 create mode 100644 tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
 create mode 100644 tensorflow/python/ops/linalg/linear_operator_zeros.py
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-zeros.__metaclass__.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-zeros.pbtxt

diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index 487418e694..f4ec3e3996 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -234,3 +234,21 @@ cuda_py_test(
         "optonly",
     ],
 )
+
+cuda_py_test(
+    name = "linear_operator_zeros_test",
+    size = "medium",
+    srcs = ["linear_operator_zeros_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:linalg_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:random_ops",
+    ],
+    shard_count = 5,
+    tags = ["optonly"],  # Test is flaky without optimization.
+)
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
new file mode 100644
index 0000000000..8f60b55e0a
--- /dev/null
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_zeros_test.py
@@ -0,0 +1,192 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import random_seed
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
+from tensorflow.python.ops.linalg import linear_operator_test_util
+from tensorflow.python.platform import test
+
+
+random_seed.set_random_seed(23)
+rng = np.random.RandomState(2016)
+
+
+class LinearOperatorZerosTest(
+    linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
+  """Most tests done in the base class LinearOperatorDerivedClassTest."""
+
+  @property
+  def _tests_to_skip(self):
+    return ["log_abs_det", "solve", "solve_with_broadcast"]
+
+  @property
+  def _operator_build_infos(self):
+    build_info = linear_operator_test_util.OperatorBuildInfo
+    return [
+        build_info((1, 1)),
+        build_info((1, 3, 3)),
+        build_info((3, 4, 4)),
+        build_info((2, 1, 4, 4))]
+
+  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+    del use_placeholder
+    shape = list(build_info.shape)
+    assert shape[-1] == shape[-2]
+
+    batch_shape = shape[:-2]
+    num_rows = shape[-1]
+
+    operator = linalg_lib.LinearOperatorZeros(
+        num_rows, batch_shape=batch_shape, dtype=dtype)
+    matrix = array_ops.zeros(shape=shape, dtype=dtype)
+
+    return operator, matrix
+
+  def test_assert_positive_definite(self):
+    operator = linalg_lib.LinearOperatorZeros(num_rows=2)
+    with self.assertRaisesOpError("non-positive definite"):
+      operator.assert_positive_definite()
+
+  def test_assert_non_singular(self):
+    with self.assertRaisesOpError("non-invertible"):
+      operator = linalg_lib.LinearOperatorZeros(num_rows=2)
+      operator.assert_non_singular()
+
+  def test_assert_self_adjoint(self):
+    with self.test_session():
+      operator = linalg_lib.LinearOperatorZeros(num_rows=2)
+      operator.assert_self_adjoint().run()  # Should not fail
+
+  def test_non_scalar_num_rows_raises_static(self):
+    with self.assertRaisesRegexp(ValueError, "must be a 0-D Tensor"):
+      linalg_lib.LinearOperatorZeros(num_rows=[2])
+    with self.assertRaisesRegexp(ValueError, "must be a 0-D Tensor"):
+      linalg_lib.LinearOperatorZeros(num_rows=2, num_columns=[2])
+
+  def test_non_integer_num_rows_raises_static(self):
+    with self.assertRaisesRegexp(TypeError, "must be integer"):
+      linalg_lib.LinearOperatorZeros(num_rows=2.)
+    with self.assertRaisesRegexp(TypeError, "must be integer"):
+      linalg_lib.LinearOperatorZeros(num_rows=2, num_columns=2.)
+
+  def test_negative_num_rows_raises_static(self):
+    with self.assertRaisesRegexp(ValueError, "must be non-negative"):
+      linalg_lib.LinearOperatorZeros(num_rows=-2)
+    with self.assertRaisesRegexp(ValueError, "must be non-negative"):
+      linalg_lib.LinearOperatorZeros(num_rows=2, num_columns=-2)
+
+  def test_non_1d_batch_shape_raises_static(self):
+    with self.assertRaisesRegexp(ValueError, "must be a 1-D"):
+      linalg_lib.LinearOperatorZeros(num_rows=2, batch_shape=2)
+
+  def test_non_integer_batch_shape_raises_static(self):
+    with self.assertRaisesRegexp(TypeError, "must be integer"):
+      linalg_lib.LinearOperatorZeros(num_rows=2, batch_shape=[2.])
+
+  def test_negative_batch_shape_raises_static(self):
+    with self.assertRaisesRegexp(ValueError, "must be non-negative"):
+      linalg_lib.LinearOperatorZeros(num_rows=2, batch_shape=[-2])
+
+  def test_non_scalar_num_rows_raises_dynamic(self):
+    with self.test_session():
+      num_rows = array_ops.placeholder(dtypes.int32)
+      operator = linalg_lib.LinearOperatorZeros(
+          num_rows, assert_proper_shapes=True)
+      with self.assertRaisesOpError("must be a 0-D Tensor"):
+        operator.to_dense().eval(feed_dict={num_rows: [2]})
+
+  def test_negative_num_rows_raises_dynamic(self):
+    with self.test_session():
+      n = array_ops.placeholder(dtypes.int32)
+      operator = linalg_lib.LinearOperatorZeros(
+          num_rows=n, assert_proper_shapes=True)
+      with self.assertRaisesOpError("must be non-negative"):
+        operator.to_dense().eval(feed_dict={n: -2})
+
+      operator = linalg_lib.LinearOperatorZeros(
+          num_rows=2, num_columns=n, assert_proper_shapes=True)
+      with self.assertRaisesOpError("must be non-negative"):
+        operator.to_dense().eval(feed_dict={n: -2})
+
+  def test_non_1d_batch_shape_raises_dynamic(self):
+    with self.test_session():
+      batch_shape = array_ops.placeholder(dtypes.int32)
+      operator = linalg_lib.LinearOperatorZeros(
+          num_rows=2, batch_shape=batch_shape, assert_proper_shapes=True)
+      with self.assertRaisesOpError("must be a 1-D"):
+        operator.to_dense().eval(feed_dict={batch_shape: 2})
+
+  def test_negative_batch_shape_raises_dynamic(self):
+    with self.test_session():
+      batch_shape = array_ops.placeholder(dtypes.int32)
+      operator = linalg_lib.LinearOperatorZeros(
+          num_rows=2, batch_shape=batch_shape, assert_proper_shapes=True)
+      with self.assertRaisesOpError("must be non-negative"):
+        operator.to_dense().eval(feed_dict={batch_shape: [-2]})
+
+  def test_wrong_matrix_dimensions_raises_static(self):
+    operator = linalg_lib.LinearOperatorZeros(num_rows=2)
+    x = rng.randn(3, 3).astype(np.float32)
+    with self.assertRaisesRegexp(ValueError, "Dimensions.*not compatible"):
+      operator.matmul(x)
+
+  def test_wrong_matrix_dimensions_raises_dynamic(self):
+    num_rows = array_ops.placeholder(dtypes.int32)
+    x = array_ops.placeholder(dtypes.float32)
+
+    with self.test_session():
+      operator = linalg_lib.LinearOperatorZeros(
+          num_rows, assert_proper_shapes=True)
+      y = operator.matmul(x)
+      with self.assertRaisesOpError("Incompatible.*dimensions"):
+        y.eval(feed_dict={num_rows: 2, x: rng.rand(3, 3)})
+
+  def test_is_x_flags(self):
+    # The is_x flags are by default all True.
+    operator = linalg_lib.LinearOperatorZeros(num_rows=2)
+    self.assertFalse(operator.is_positive_definite)
+    self.assertFalse(operator.is_non_singular)
+    self.assertTrue(operator.is_self_adjoint)
+
+
+class LinearOperatorZerosNotSquareTest(
+    linear_operator_test_util.NonSquareLinearOperatorDerivedClassTest):
+
+  def _operator_and_matrix(self, build_info, dtype, use_placeholder):
+    del use_placeholder
+    shape = list(build_info.shape)
+
+    batch_shape = shape[:-2]
+    num_rows = shape[-2]
+    num_columns = shape[-1]
+
+    operator = linalg_lib.LinearOperatorZeros(
+        num_rows, num_columns, is_square=False, is_self_adjoint=False,
+        batch_shape=batch_shape, dtype=dtype)
+    matrix = array_ops.zeros(shape=shape, dtype=dtype)
+
+    return operator, matrix
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/linalg/linalg.py b/tensorflow/python/ops/linalg/linalg.py
index a7ba0bbe9c..c29b5033bb 100644
--- a/tensorflow/python/ops/linalg/linalg.py
+++ b/tensorflow/python/ops/linalg/linalg.py
@@ -31,6 +31,7 @@ from tensorflow.python.ops.linalg.linear_operator_identity import *
 from tensorflow.python.ops.linalg.linear_operator_kronecker import *
 from tensorflow.python.ops.linalg.linear_operator_low_rank_update import *
 from tensorflow.python.ops.linalg.linear_operator_lower_triangular import *
+from tensorflow.python.ops.linalg.linear_operator_zeros import *
 # pylint: enable=wildcard-import
 
 # Seal API.
diff --git a/tensorflow/python/ops/linalg/linear_operator_zeros.py b/tensorflow/python/ops/linalg/linear_operator_zeros.py
new file mode 100644
index 0000000000..b8a79c065b
--- /dev/null
+++ b/tensorflow/python/ops/linalg/linear_operator_zeros.py
@@ -0,0 +1,452 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""`LinearOperator` acting like a zero matrix."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg_impl as linalg
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_util
+from tensorflow.python.util.tf_export import tf_export
+
+__all__ = [
+    "LinearOperatorZeros",
+]
+
+
+@tf_export("linalg.LinearOperatorZeros")
+class LinearOperatorZeros(linear_operator.LinearOperator):
+  """`LinearOperator` acting like a [batch] zero matrix.
+
+  This operator acts like a [batch] zero matrix `A` with shape
+  `[B1,...,Bb, N, M]` for some `b >= 0`.  The first `b` indices index a
+  batch member.  For every batch index `(i1,...,ib)`, `A[i1,...,ib, : :]` is
+  an `N x M` matrix.  This matrix `A` is not materialized, but for
+  purposes of broadcasting this shape will be relevant.
+
+  `LinearOperatorZeros` is initialized with `num_rows`, and optionally
+  `num_columns, `batch_shape`, and `dtype` arguments.  If `num_columns` is
+  `None`, then this operator will be initialized as a square matrix. If
+  `batch_shape` is `None`, this operator efficiently passes through all
+  arguments.  If `batch_shape` is provided, broadcasting may occur, which will
+  require making copies.
+
+  ```python
+  # Create a 2 x 2 zero matrix.
+  operator = LinearOperatorZero(num_rows=2, dtype=tf.float32)
+
+  operator.to_dense()
+  ==> [[0., 0.]
+       [0., 0.]]
+
+  operator.shape
+  ==> [2, 2]
+
+  operator.determinant()
+  ==> 0.
+
+  x = ... Shape [2, 4] Tensor
+  operator.matmul(x)
+  ==> Shape [2, 4] Tensor, same as x.
+
+  # Create a 2-batch of 2x2 zero matrices
+  operator = LinearOperatorZeros(num_rows=2, batch_shape=[2])
+  operator.to_dense()
+  ==> [[[0., 0.]
+        [0., 0.]],
+       [[0., 0.]
+        [0., 0.]]]
+
+  # Here, even though the operator has a batch shape, the input is the same as
+  # the output, so x can be passed through without a copy.  The operator is able
+  # to detect that no broadcast is necessary because both x and the operator
+  # have statically defined shape.
+  x = ... Shape [2, 2, 3]
+  operator.matmul(x)
+  ==> Shape [2, 2, 3] Tensor, same as tf.zeros_like(x)
+
+  # Here the operator and x have different batch_shape, and are broadcast.
+  # This requires a copy, since the output is different size than the input.
+  x = ... Shape [1, 2, 3]
+  operator.matmul(x)
+  ==> Shape [2, 2, 3] Tensor, equal to tf.zeros_like([x, x])
+  ```
+
+  ### Shape compatibility
+
+  This operator acts on [batch] matrix with compatible shape.
+  `x` is a batch matrix with compatible shape for `matmul` and `solve` if
+
+  ```
+  operator.shape = [B1,...,Bb] + [N, M],  with b >= 0
+  x.shape =   [C1,...,Cc] + [M, R],
+  and [C1,...,Cc] broadcasts with [B1,...,Bb] to [D1,...,Dd]
+  ```
+
+  #### Matrix property hints
+
+  This `LinearOperator` is initialized with boolean flags of the form `is_X`,
+  for `X = non_singular, self_adjoint, positive_definite, square`.
+  These have the following meaning:
+
+  * If `is_X == True`, callers should expect the operator to have the
+    property `X`.  This is a promise that should be fulfilled, but is *not* a
+    runtime assert.  For example, finite floating point precision may result
+    in these promises being violated.
+  * If `is_X == False`, callers should expect the operator to not have `X`.
+  * If `is_X == None` (the default), callers should have no expectation either
+    way.
+  """
+
+  def __init__(self,
+               num_rows,
+               num_columns=None,
+               batch_shape=None,
+               dtype=None,
+               is_non_singular=False,
+               is_self_adjoint=True,
+               is_positive_definite=False,
+               is_square=True,
+               assert_proper_shapes=False,
+               name="LinearOperatorZeros"):
+    r"""Initialize a `LinearOperatorZeros`.
+
+    The `LinearOperatorZeros` is initialized with arguments defining `dtype`
+    and shape.
+
+    This operator is able to broadcast the leading (batch) dimensions, which
+    sometimes requires copying data.  If `batch_shape` is `None`, the operator
+    can take arguments of any batch shape without copying.  See examples.
+
+    Args:
+      num_rows:  Scalar non-negative integer `Tensor`.  Number of rows in the
+        corresponding zero matrix.
+      num_columns:  Scalar non-negative integer `Tensor`.  Number of columns in
+        the corresponding zero matrix. If `None`, defaults to the value of
+        `num_rows`.
+      batch_shape:  Optional `1-D` integer `Tensor`.  The shape of the leading
+        dimensions.  If `None`, this operator has no leading dimensions.
+      dtype:  Data type of the matrix that this operator represents.
+      is_non_singular:  Expect that this operator is non-singular.
+      is_self_adjoint:  Expect that this operator is equal to its hermitian
+        transpose.
+      is_positive_definite:  Expect that this operator is positive definite,
+        meaning the quadratic form `x^H A x` has positive real part for all
+        nonzero `x`.  Note that we do not require the operator to be
+        self-adjoint to be positive-definite.  See:
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
+      is_square:  Expect that this operator acts like square [batch] matrices.
+      assert_proper_shapes:  Python `bool`.  If `False`, only perform static
+        checks that initialization and method arguments have proper shape.
+        If `True`, and static checks are inconclusive, add asserts to the graph.
+      name: A name for this `LinearOperator`
+
+    Raises:
+      ValueError:  If `num_rows` is determined statically to be non-scalar, or
+        negative.
+      ValueError:  If `num_columns` is determined statically to be non-scalar,
+        or negative.
+      ValueError:  If `batch_shape` is determined statically to not be 1-D, or
+        negative.
+      ValueError:  If any of the following is not `True`:
+        `{is_self_adjoint, is_non_singular, is_positive_definite}`.
+    """
+    dtype = dtype or dtypes.float32
+    self._assert_proper_shapes = assert_proper_shapes
+
+    with ops.name_scope(name):
+      dtype = dtypes.as_dtype(dtype)
+      if not is_self_adjoint and is_square:
+        raise ValueError("A zero operator is always self adjoint.")
+      if is_non_singular:
+        raise ValueError("A zero operator is always singular.")
+      if is_positive_definite:
+        raise ValueError("A zero operator is always not positive-definite.")
+
+      super(LinearOperatorZeros, self).__init__(
+          dtype=dtype,
+          is_non_singular=is_non_singular,
+          is_self_adjoint=is_self_adjoint,
+          is_positive_definite=is_positive_definite,
+          is_square=is_square,
+          name=name)
+
+      self._num_rows = linear_operator_util.shape_tensor(
+          num_rows, name="num_rows")
+      self._num_rows_static = tensor_util.constant_value(self._num_rows)
+
+      if num_columns is None:
+        num_columns = num_rows
+
+      self._num_columns = linear_operator_util.shape_tensor(
+          num_columns, name="num_columns")
+      self._num_columns_static = tensor_util.constant_value(self._num_columns)
+
+      self._check_domain_range_possibly_add_asserts()
+
+      if (self._num_rows_static is not None and
+          self._num_columns_static is not None):
+        if is_square and self._num_rows_static != self._num_columns_static:
+          raise ValueError(
+              "LinearOperatorZeros initialized as is_square=True, but got "
+              "num_rows({}) != num_columns({})".format(
+                  self._num_rows_static,
+                  self._num_columns_static))
+
+      if batch_shape is None:
+        self._batch_shape_arg = None
+      else:
+        self._batch_shape_arg = linear_operator_util.shape_tensor(
+            batch_shape, name="batch_shape_arg")
+        self._batch_shape_static = tensor_util.constant_value(
+            self._batch_shape_arg)
+        self._check_batch_shape_possibly_add_asserts()
+
+  def _shape(self):
+    matrix_shape = tensor_shape.TensorShape((self._num_rows_static,
+                                             self._num_columns_static))
+    if self._batch_shape_arg is None:
+      return matrix_shape
+
+    batch_shape = tensor_shape.TensorShape(self._batch_shape_static)
+    return batch_shape.concatenate(matrix_shape)
+
+  def _shape_tensor(self):
+    matrix_shape = array_ops.stack((self._num_rows, self._num_columns), axis=0)
+    if self._batch_shape_arg is None:
+      return matrix_shape
+
+    return array_ops.concat((self._batch_shape_arg, matrix_shape), 0)
+
+  def _assert_non_singular(self):
+    raise errors.InvalidArgumentError(
+        node_def=None, op=None, message="Zero operators are always "
+        "non-invertible.")
+
+  def _assert_positive_definite(self):
+    raise errors.InvalidArgumentError(
+        node_def=None, op=None, message="Zero operators are always "
+        "non-positive definite.")
+
+  def _assert_self_adjoint(self):
+    return control_flow_ops.no_op("assert_self_adjoint")
+
+  def _possibly_broadcast_batch_shape(self, x):
+    """Return 'x', possibly after broadcasting the leading dimensions."""
+    # If we have no batch shape, our batch shape broadcasts with everything!
+    if self._batch_shape_arg is None:
+      return x
+
+    # Static attempt:
+    #   If we determine that no broadcast is necessary, pass x through
+    #   If we need a broadcast, add to an array of zeros.
+    #
+    # special_shape is the shape that, when broadcast with x's shape, will give
+    # the correct broadcast_shape.  Note that
+    #   We have already verified the second to last dimension of self.shape
+    #   matches x's shape in assert_compatible_matrix_dimensions.
+    #   Also, the final dimension of 'x' can have any shape.
+    #   Therefore, the final two dimensions of special_shape are 1's.
+    special_shape = self.batch_shape.concatenate([1, 1])
+    bshape = array_ops.broadcast_static_shape(x.get_shape(), special_shape)
+    if special_shape.is_fully_defined():
+      # bshape.is_fully_defined iff special_shape.is_fully_defined.
+      if bshape == x.get_shape():
+        return x
+      # Use the built in broadcasting of addition.
+      zeros = array_ops.zeros(shape=special_shape, dtype=self.dtype)
+      return x + zeros
+
+    # Dynamic broadcast:
+    #   Always add to an array of zeros, rather than using a "cond", since a
+    #   cond would require copying data from GPU --> CPU.
+    special_shape = array_ops.concat((self.batch_shape_tensor(), [1, 1]), 0)
+    zeros = array_ops.zeros(shape=special_shape, dtype=self.dtype)
+    return x + zeros
+
+  def _matmul(self, x, adjoint=False, adjoint_arg=False):
+    if self._assert_proper_shapes:
+      x = linalg.adjoint(x) if adjoint_arg else x
+      aps = linear_operator_util.assert_compatible_matrix_dimensions(self, x)
+      x = control_flow_ops.with_dependencies([aps], x)
+    if self.is_square:
+      # Note that adjoint has no effect since this matrix is self-adjoint.
+      if adjoint_arg:
+        output_shape = array_ops.concat([
+            array_ops.shape(x)[:-2],
+            [array_ops.shape(x)[-1], array_ops.shape(x)[-2]]], axis=0)
+      else:
+        output_shape = array_ops.shape(x)
+
+      return self._possibly_broadcast_batch_shape(
+          array_ops.zeros(shape=output_shape, dtype=x.dtype))
+
+    x_shape = array_ops.shape(x)
+    n = self._num_columns if adjoint else self._num_rows
+    m = x_shape[-2] if adjoint_arg else x_shape[-1]
+
+    output_shape = array_ops.concat([x_shape[:-2], [n, m]], axis=0)
+
+    zeros = array_ops.zeros(shape=output_shape, dtype=x.dtype)
+    return self._possibly_broadcast_batch_shape(zeros)
+
+  def _determinant(self):
+    if self.batch_shape.is_fully_defined():
+      return array_ops.zeros(shape=self.batch_shape, dtype=self.dtype)
+    else:
+      return array_ops.zeros(shape=self.batch_shape_tensor(), dtype=self.dtype)
+
+  def _trace(self):
+    # Get Tensor of all zeros of same shape as self.batch_shape.
+    if self.batch_shape.is_fully_defined():
+      return array_ops.zeros(shape=self.batch_shape, dtype=self.dtype)
+    else:
+      return array_ops.zeros(shape=self.batch_shape_tensor(), dtype=self.dtype)
+
+  def _diag_part(self):
+    return self._zeros_diag()
+
+  def add_to_tensor(self, mat, name="add_to_tensor"):
+    """Add matrix represented by this operator to `mat`.  Equiv to `I + mat`.
+
+    Args:
+      mat:  `Tensor` with same `dtype` and shape broadcastable to `self`.
+      name:  A name to give this `Op`.
+
+    Returns:
+      A `Tensor` with broadcast shape and same `dtype` as `self`.
+    """
+    return self._possibly_broadcast_batch_shape(mat)
+
+  def _check_domain_range_possibly_add_asserts(self):
+    """Static check of init arg `num_rows`, possibly add asserts."""
+    # Possibly add asserts.
+    if self._assert_proper_shapes:
+      self._num_rows = control_flow_ops.with_dependencies([
+          check_ops.assert_rank(
+              self._num_rows,
+              0,
+              message="Argument num_rows must be a 0-D Tensor."),
+          check_ops.assert_non_negative(
+              self._num_rows,
+              message="Argument num_rows must be non-negative."),
+      ], self._num_rows)
+      self._num_columns = control_flow_ops.with_dependencies([
+          check_ops.assert_rank(
+              self._num_columns,
+              0,
+              message="Argument num_columns must be a 0-D Tensor."),
+          check_ops.assert_non_negative(
+              self._num_columns,
+              message="Argument num_columns must be non-negative."),
+      ], self._num_columns)
+
+    # Static checks.
+    if not self._num_rows.dtype.is_integer:
+      raise TypeError("Argument num_rows must be integer type.  Found:"
+                      " %s" % self._num_rows)
+
+    if not self._num_columns.dtype.is_integer:
+      raise TypeError("Argument num_columns must be integer type.  Found:"
+                      " %s" % self._num_columns)
+
+    num_rows_static = self._num_rows_static
+    num_columns_static = self._num_columns_static
+
+    if num_rows_static is not None:
+      if num_rows_static.ndim != 0:
+        raise ValueError("Argument num_rows must be a 0-D Tensor.  Found:"
+                         " %s" % num_rows_static)
+
+      if num_rows_static < 0:
+        raise ValueError("Argument num_rows must be non-negative.  Found:"
+                         " %s" % num_rows_static)
+    if num_columns_static is not None:
+      if num_columns_static.ndim != 0:
+        raise ValueError("Argument num_columns must be a 0-D Tensor.  Found:"
+                         " %s" % num_columns_static)
+
+      if num_columns_static < 0:
+        raise ValueError("Argument num_columns must be non-negative.  Found:"
+                         " %s" % num_columns_static)
+
+  def _check_batch_shape_possibly_add_asserts(self):
+    """Static check of init arg `batch_shape`, possibly add asserts."""
+    if self._batch_shape_arg is None:
+      return
+
+    # Possibly add asserts
+    if self._assert_proper_shapes:
+      self._batch_shape_arg = control_flow_ops.with_dependencies([
+          check_ops.assert_rank(
+              self._batch_shape_arg,
+              1,
+              message="Argument batch_shape must be a 1-D Tensor."),
+          check_ops.assert_non_negative(
+              self._batch_shape_arg,
+              message="Argument batch_shape must be non-negative."),
+      ], self._batch_shape_arg)
+
+    # Static checks
+    if not self._batch_shape_arg.dtype.is_integer:
+      raise TypeError("Argument batch_shape must be integer type.  Found:"
+                      " %s" % self._batch_shape_arg)
+
+    if self._batch_shape_static is None:
+      return  # Cannot do any other static checks.
+
+    if self._batch_shape_static.ndim != 1:
+      raise ValueError("Argument batch_shape must be a 1-D Tensor.  Found:"
+                       " %s" % self._batch_shape_static)
+
+    if np.any(self._batch_shape_static < 0):
+      raise ValueError("Argument batch_shape must be non-negative.  Found:"
+                       "%s" % self._batch_shape_static)
+
+  def _min_matrix_dim(self):
+    """Minimum of domain/range dimension, if statically available, else None."""
+    domain_dim = self.domain_dimension.value
+    range_dim = self.range_dimension.value
+    if domain_dim is None or range_dim is None:
+      return None
+    return min(domain_dim, range_dim)
+
+  def _min_matrix_dim_tensor(self):
+    """Minimum of domain/range dimension, as a tensor."""
+    return math_ops.reduce_min(self.shape_tensor()[-2:])
+
+  def _zeros_diag(self):
+    """Returns the diagonal of this operator as all zeros."""
+    if self.shape.is_fully_defined():
+      d_shape = self.batch_shape.concatenate([self._min_matrix_dim()])
+    else:
+      d_shape = array_ops.concat(
+          [self.batch_shape_tensor(),
+           [self._min_matrix_dim_tensor()]], axis=0)
+
+    return array_ops.zeros(shape=d_shape, dtype=self.dtype)
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-zeros.__metaclass__.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-zeros.__metaclass__.pbtxt
new file mode 100644
index 0000000000..49ff85728f
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-zeros.__metaclass__.pbtxt
@@ -0,0 +1,14 @@
+path: "tensorflow.linalg.LinearOperatorZeros.__metaclass__"
+tf_class {
+  is_instance: ""
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "mro"
+  }
+  member_method {
+    name: "register"
+    argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-zeros.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-zeros.pbtxt
new file mode 100644
index 0000000000..a1b0e06b47
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-zeros.pbtxt
@@ -0,0 +1,130 @@
+path: "tensorflow.linalg.LinearOperatorZeros"
+tf_class {
+  is_instance: ""
+  is_instance: ""
+  is_instance: ""
+  member {
+    name: "batch_shape"
+    mtype: ""
+  }
+  member {
+    name: "domain_dimension"
+    mtype: ""
+  }
+  member {
+    name: "dtype"
+    mtype: ""
+  }
+  member {
+    name: "graph_parents"
+    mtype: ""
+  }
+  member {
+    name: "is_non_singular"
+    mtype: ""
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: ""
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: ""
+  }
+  member {
+    name: "is_square"
+    mtype: ""
+  }
+  member {
+    name: "name"
+    mtype: ""
+  }
+  member {
+    name: "range_dimension"
+    mtype: ""
+  }
+  member {
+    name: "shape"
+    mtype: ""
+  }
+  member {
+    name: "tensor_rank"
+    mtype: ""
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_rows\', \'num_columns\', \'batch_shape\', \'dtype\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'assert_proper_shapes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'False\', \'True\', \'False\', \'True\', \'False\', \'LinearOperatorZeros\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'mat\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt
index 3b5845f99a..d979116887 100644
--- a/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt
@@ -52,6 +52,10 @@ tf_module {
     name: "LinearOperatorScaledIdentity"
     mtype: ""
   }
+  member {
+    name: "LinearOperatorZeros"
+    mtype: ""
+  }
   member_method {
     name: "adjoint"
     argspec: "args=[\'matrix\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
cgit v1.2.3


From 37757856cded3a5608cfd218a0b25b41a148d995 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 00:46:17 -0700
Subject: Expose proto serialization publicly, to avoid code duplication in
 tensorflow_serving.

PiperOrigin-RevId: 205788702
---
 tensorflow/core/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 13e1b643d1..a960736295 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -663,6 +663,7 @@ cc_library(
         "lib/random/random_distributions.h",
         "lib/random/simple_philox.h",
         "lib/strings/numbers.h",
+        "lib/strings/proto_serialization.h",
         "lib/strings/str_util.h",
         "lib/strings/strcat.h",
         "lib/strings/stringprintf.h",
-- 
cgit v1.2.3


From fca1561b9d5932f940cf89e03128cf197547bed2 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar 
Date: Tue, 24 Jul 2018 00:52:24 -0700
Subject: BatchToSpaceND support quantization, so make the transformation know
 that.

PiperOrigin-RevId: 205789178
---
 tensorflow/contrib/lite/toco/graph_transformations/quantize.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
index 5be2757479..f6ce3b3ecb 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
@@ -50,6 +50,7 @@ bool SupportsQuantization(const Operator& op) {
          type == OperatorType::kSqueeze || type == OperatorType::kPad ||
          type == OperatorType::kPadV2 || type == OperatorType::kReshape ||
          type == OperatorType::kTanh || type == OperatorType::kMul ||
+         type == OperatorType::kBatchToSpaceND ||
          type == OperatorType::kSpaceToBatchND ||
          type == OperatorType::kSpaceToDepth ||
          type == OperatorType::kStridedSlice ||
-- 
cgit v1.2.3


From 33035bb79b6ecb408ef83cee5fc3e52ce058f39f Mon Sep 17 00:00:00 2001
From: Adrian Kuegel 
Date: Tue, 24 Jul 2018 01:13:09 -0700
Subject: Parallelize BitonicSort on GPU.

We now emit O(log^n) kernel thunks. Each thunk is responsible for looping over
the other dimensions, and then doing a comparison loop through the dimension
that should be sorted.

PiperOrigin-RevId: 205791397
---
 tensorflow/compiler/xla/service/gpu/ir_emitter.cc  |  12 ---
 tensorflow/compiler/xla/service/gpu/ir_emitter.h   |   1 -
 .../xla/service/gpu/ir_emitter_unnested.cc         |  48 ++++++++-
 tensorflow/compiler/xla/service/llvm_ir/BUILD      |   3 +
 .../compiler/xla/service/llvm_ir/sort_util.cc      | 120 +++++++--------------
 .../compiler/xla/service/llvm_ir/sort_util.h       |  12 ++-
 6 files changed, 97 insertions(+), 99 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
index 76180cf486..f95541cba4 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
@@ -37,7 +37,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/sort_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/service/name_uniquer.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -123,17 +122,6 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) {
   return Status::OK();
 }
 
-Status IrEmitter::HandleSort(HloInstruction* sort) {
-  auto values = sort->operand_count() > 1 ? sort->operand(1) : nullptr;
-  if (values != nullptr) {
-    // TODO(b/26783907): Also sort the values by their corresponding key.
-    return Unimplemented("Key/Value Sort is not implemented on GPU");
-  }
-  int dimension_to_sort = sort->dimensions(0);
-  return llvm_ir::EmitSortInPlace(dimension_to_sort, GetIrArray(*sort, *sort),
-                                  IrName(sort), &b_);
-}
-
 Status IrEmitter::HandleSend(HloInstruction*) {
   return Unimplemented("Send is not implemented on GPU");
 }
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
index 172d4a4e29..e89967a378 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
@@ -79,7 +79,6 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status HandleCrossReplicaSum(HloInstruction* crs) override;
   Status HandleInfeed(HloInstruction* infeed) override;
   Status HandleOutfeed(HloInstruction* outfeed) override;
-  Status HandleSort(HloInstruction* sort) override;
   Status HandleSend(HloInstruction* send) override;
   Status HandleSendDone(HloInstruction* send_done) override;
   Status HandleRecv(HloInstruction* recv) override;
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 64a6baf66d..b1038a3cc9 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -63,6 +63,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/sort_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/service/name_uniquer.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -71,6 +72,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/compiler/xla/window_util.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/logging.h"
@@ -2036,11 +2038,51 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) {
         /*mem_size=*/ShapeUtil::ByteSizeOf(sort->shape()), sort));
   }
 
-  thunks.push_back(
-      BuildKernelThunk(sort, /*implements_whole_instruction=*/false));
+  int64 dimension_to_sort = sort->dimensions(0);
+  int64 dimension_to_sort_bound = sort->shape().dimensions(dimension_to_sort);
+  int64 num_stages = tensorflow::Log2Ceiling(dimension_to_sort_bound);
+  auto index_type = b_.getInt64Ty();
+
+  // Naive C++ code for the outer loops:
+  //
+  // for (int64 stage = 0; stage < Log2Ceiling(dimension_to_sort_bound);
+  //     ++stage) {
+  //   int64 first_xor_mask = (1LL << (stage + 1)) - 1;
+  //   SortInPlace(first_xor_mask);
+  //   for (int64 mask = stage - 1; mask >= 0; --mask) {
+  //     int64 later_xor_mask = 1LL << mask;
+  //     SortInPlace(later_xor_mask);
+  //   }
+  // }
+  //
+  // This follows the algorithm described on Wikipedia:
+  // https://en.wikipedia.org/wiki/Bitonic_sorter
+
+  for (int64 stage = 0; stage < num_stages; ++stage) {
+    for (int64 mask = stage; mask >= 0; --mask) {
+      thunks.push_back(
+          BuildKernelThunk(sort, /*implements_whole_instruction=*/false));
+      LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
+          sort->shape(), ir_emitter_context_->device_description());
+      UpdateLaunchDimensions(launch_dimensions, thunks.back().get(),
+                             ir_emitter_context_->llvm_module());
+
+      llvm::Value* xor_mask;
+      if (mask == stage) {
+        xor_mask = llvm::ConstantInt::get(index_type, (1LL << (stage + 1)) - 1);
+      } else {
+        xor_mask = llvm::ConstantInt::get(index_type, 1LL << mask);
+      }
+
+      TF_RETURN_IF_ERROR(llvm_ir::EmitSortInPlace(
+          dimension_to_sort, GetIrArray(*sort, *sort), IrName(sort), xor_mask,
+          &b_, &launch_dimensions));
+    }
+  }
+
   thunk_sequence_->emplace_back(
       MakeUnique(std::move(thunks), sort));
-  return IrEmitter::HandleSort(sort);
+  return Status::OK();
 }
 
 Status IrEmitterUnnested::HandleTupleSelect(HloInstruction* tuple_select) {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index 462be543bc..0573304912 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -188,7 +188,10 @@ cc_library(
         ":ir_array",
         ":llvm_loop",
         ":llvm_util",
+        ":loop_emitter",
         "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla/service/gpu:parallel_loop_emitter",
+        "//tensorflow/compiler/xla/service/gpu:partition_assignment",
         "//tensorflow/core:lib",
         "@llvm//:core",
     ],
diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
index 585364458a..6f261c32f4 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.cc
@@ -19,12 +19,15 @@ limitations under the License.
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/Value.h"
 #include "tensorflow/compiler/xla/primitive_util.h"
+#include "tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h"
+#include "tensorflow/compiler/xla/service/gpu/partition_assignment.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/ir_array.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h"
 #include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/platform/types.h"
@@ -73,7 +76,9 @@ void EmitCompareLoop(int64 dimension_to_sort,
 }  // namespace
 
 Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
-                       tensorflow::StringPiece name, llvm::IRBuilder<>* b) {
+                       tensorflow::StringPiece name, llvm::Value* xor_mask,
+                       llvm::IRBuilder<>* b,
+                       const gpu::LaunchDimensions* launch_dimensions) {
   const Shape& keys_shape = keys_array.GetShape();
 
   // TODO(b/26783907): This case can probably be avoided with the Algebraic
@@ -83,11 +88,13 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
   }
 
   // Create loop nests which loop through the operand dimensions. The sort
-  // dimension is handled in three separate innermost loops which perform the
-  // sorting.
+  // dimension is handled in the innermost loop which performs the sorting.
   ForLoopNest loop_nest(name, b);
   IrArray::Index keys_index =
       loop_nest.EmitOperandArrayLoopNest(keys_array, dimension_to_sort, "keys");
+  if (loop_nest.GetInnerLoopBodyBasicBlock() != nullptr) {
+    SetToFirstInsertPoint(loop_nest.GetInnerLoopBodyBasicBlock(), b);
+  }
 
   // 'compare_keys_index' is the index of the element that 'keys_index' should
   // be compared to.
@@ -100,89 +107,42 @@ Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
     }
   }
 
-  // Create the sorting loops which do the sorting.
-  int64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort);
-  std::unique_ptr stages_loop = loop_nest.AddLoop(
-      /*start_index=*/0,
-      /*end_index=*/
-      tensorflow::Log2Ceiling64(dimension_to_sort_bound),
-      /*suffix=*/"sort_stages");
-  std::unique_ptr mask_loop = loop_nest.AddLoop(
-      /*suffix=*/"mask",
-      /*start_index=*/keys_index.GetConstantWithIndexType(0),
-      /*end_index=*/stages_loop->GetIndVarValue());
-  std::unique_ptr compare_loop = loop_nest.AddLoop(
-      /*start_index=*/0,
-      /*end_index=*/dimension_to_sort_bound,
-      /*suffix=*/"compare");
-
-  // Naive C++ code for the inner loops (without parallelization):
+  // Naive C++ code for the inner compare loop:
   //
-  // for (int64 stage = 0; stage < Log2Ceiling(dimension_to_sort_bound);
-  //     ++stage) {
-  //   int64 first_xor_mask = (1LL << (stage + 1)) - 1;
-  //   for (int64 i = 0; i < dimension_to_sort_bound; ++i) {
-  //     int64 j = i ^ first_xor_mask;
-  //     if (i < j && j < dimension_to_sort_bound) {
-  //       int64 min_key = std::min(keys[i], keys[j]);
-  //       keys[j] = std::max(keys[i], keys[j]);
-  //       keys[i] = min_key;
-  //     }
-  //   }
-  //   for (int64 mask = 0; mask < stage; ++mask) {
-  //     int64 later_xor_mask = (1LL << (stage - (mask + 1));
-  //     for (int64 i = 0; i < dimension_to_sort_bound; ++i) {
-  //       int64 j = i ^ later_xor_mask;
-  //       if (i < j && j < dimension_to_sort_bound) {
-  //         int64 min_key = std::min(keys[i], keys[j]);
-  //         keys[j] = std::max(keys[i], keys[j]);
-  //         keys[i] = min_key;
-  //       }
-  //     }
+  // for (int64 i = 0; i < dimension_to_sort_bound; ++i) {
+  //   int64 j = i ^ xor_mask;
+  //   if (i < j && j < dimension_to_sort_bound) {
+  //     int64 min_key = std::min(keys[i], keys[j]);
+  //     keys[j] = std::max(keys[i], keys[j]);
+  //     keys[i] = min_key;
   //   }
   // }
   //
   // This follows the algorithm described on Wikipedia:
   // https://en.wikipedia.org/wiki/Bitonic_sorter
 
-  SetToFirstInsertPoint(stages_loop->GetBodyBasicBlock(), b);
-  // The first xor mask of a stage is 2^(stage + 1) - 1.
-  auto first_xor_mask = b->CreateSub(
-      b->CreateShl(keys_index.GetConstantWithIndexType(1),
-                   b->CreateAdd(stages_loop->GetIndVarValue(),
-                                keys_index.GetConstantWithIndexType(1))),
-      keys_index.GetConstantWithIndexType(1));
-  std::unique_ptr first_compare_loop = ForLoop::EmitForLoop(
-      /*prefix=*/"first_compare",
-      /*start_index=*/keys_index.GetConstantWithIndexType(0),
-      /*end_index=*/
-      keys_index.GetConstantWithIndexType(dimension_to_sort_bound),
-      /*step=*/keys_index.GetConstantWithIndexType(1),
-      /*b=*/b);
-
-  SetToFirstInsertPoint(first_compare_loop->GetBodyBasicBlock(), b);
-  // 'first_compare_loop' iterates through the 'dimension_to_sort'.
-  keys_index[dimension_to_sort] = first_compare_loop->GetIndVarValue();
-  compare_keys_index[dimension_to_sort] =
-      b->CreateXor(first_compare_loop->GetIndVarValue(), first_xor_mask);
-  EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index, keys_array,
-                  b);
-
-  SetToFirstInsertPoint(compare_loop->GetPreheaderBasicBlock(), b);
-  // The later masks of a stage are 2^(stage - (mask_loop_ind_var + 1)).
-  auto later_xor_mask = b->CreateShl(
-      keys_index.GetConstantWithIndexType(1),
-      b->CreateSub(stages_loop->GetIndVarValue(),
-                   b->CreateAdd(mask_loop->GetIndVarValue(),
-                                keys_index.GetConstantWithIndexType(1))));
-
-  SetToFirstInsertPoint(compare_loop->GetBodyBasicBlock(), b);
-  // 'compare_loop' iterates through the 'dimension_to_sort'.
-  keys_index[dimension_to_sort] = compare_loop->GetIndVarValue();
-  compare_keys_index[dimension_to_sort] =
-      b->CreateXor(compare_loop->GetIndVarValue(), later_xor_mask);
-  EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index, keys_array,
-                  b);
+  int64 dimension_to_sort_bound =
+      keys_array.GetShape().dimensions(dimension_to_sort);
+  Shape compare_shape = ShapeUtil::MakeShape(keys_shape.element_type(),
+                                             {dimension_to_sort_bound});
+  auto compare_loop_body_emitter =
+      [&](const IrArray::Index& compare_index) -> Status {
+    keys_index[dimension_to_sort] = compare_index[0];
+    compare_keys_index[dimension_to_sort] =
+        b->CreateXor(compare_index[0], xor_mask);
+    EmitCompareLoop(dimension_to_sort, keys_index, compare_keys_index,
+                    keys_array, b);
+    return Status::OK();
+  };
+  if (launch_dimensions != nullptr) {
+    TF_RETURN_IF_ERROR(gpu::ParallelLoopEmitter(compare_loop_body_emitter,
+                                                compare_shape,
+                                                *launch_dimensions, b)
+                           .EmitLoop(name));
+  } else {
+    TF_RETURN_IF_ERROR(LoopEmitter(compare_loop_body_emitter, compare_shape, b)
+                           .EmitLoop(name));
+  }
 
   // Set the IR builder insert point to the exit basic block of the outer most
   // loop. This ensures later instructions are inserted after this loop nest.
diff --git a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
index d0f185e70b..e75f9b08fb 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/sort_util.h
@@ -16,6 +16,8 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_SORT_UTIL_H_
 #define TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_SORT_UTIL_H_
 
+#include "llvm/IR/Value.h"
+#include "tensorflow/compiler/xla/service/gpu/partition_assignment.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/ir_array.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
@@ -23,10 +25,14 @@ limitations under the License.
 
 namespace xla {
 namespace llvm_ir {
-// Emits llvm IR to sort the 'dimension_to_sort' dimension of 'keys_array' into
-// ascending order.
+// Emits llvm IR to do pairwise comparisons/swaps in the 'dimension_to_sort'
+// dimension of 'keys_array'. All other dimensions are kept as-is. This
+// implements the inner loop of BitonicSort. If 'launch_dimensions' is nullptr,
+// the inner compare loop will not be parallelized.
 Status EmitSortInPlace(int64 dimension_to_sort, const IrArray& keys_array,
-                       tensorflow::StringPiece name, llvm::IRBuilder<>* b);
+                       tensorflow::StringPiece name, llvm::Value* xor_mask,
+                       llvm::IRBuilder<>* b,
+                       const gpu::LaunchDimensions* launch_dimensions);
 }  // namespace llvm_ir
 }  // namespace xla
 
-- 
cgit v1.2.3


From c21078f527023e3074b63109fb768413f82a8f8f Mon Sep 17 00:00:00 2001
From: James Keeling 
Date: Tue, 24 Jul 2018 03:09:47 -0700
Subject: Add Python API functions to query kernels

This is part of the work to make available kernels easier to query at runtime.

PiperOrigin-RevId: 205802663
---
 tensorflow/python/BUILD                     | 28 ++++++++++++++++++
 tensorflow/python/framework/kernels.py      | 46 +++++++++++++++++++++++++++++
 tensorflow/python/framework/kernels_test.py | 41 +++++++++++++++++++++++++
 3 files changed, 115 insertions(+)
 create mode 100644 tensorflow/python/framework/kernels.py
 create mode 100644 tensorflow/python/framework/kernels_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 9c7f3b7b25..814239533c 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -96,6 +96,7 @@ py_library(
         ":image_ops",
         ":initializers_ns",
         ":io_ops",
+        ":kernels",
         ":layers",
         ":lib",
         ":list_ops",
@@ -789,6 +790,19 @@ py_library(
     ],
 )
 
+py_library(
+    name = "kernels",
+    srcs = [
+        "framework/kernels.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":pywrap_tensorflow",
+        ":util",
+        "//tensorflow/core:protos_all_py",
+    ],
+)
+
 py_library(
     name = "op_def_library",
     srcs = ["framework/op_def_library.py"],
@@ -1482,6 +1496,20 @@ py_test(
     ],
 )
 
+py_test(
+    name = "framework_kernels_test",
+    size = "small",
+    srcs = ["framework/kernels_test.py"],
+    main = "framework/kernels_test.py",
+    srcs_version = "PY2AND3",
+    deps = [
+        ":framework_test_lib",
+        ":kernels",
+        ":platform_test",
+        ":test_ops",
+    ],
+)
+
 tf_gen_op_wrapper_private_py(
     name = "array_ops_gen",
     visibility = [
diff --git a/tensorflow/python/framework/kernels.py b/tensorflow/python/framework/kernels.py
new file mode 100644
index 0000000000..f7641f3442
--- /dev/null
+++ b/tensorflow/python/framework/kernels.py
@@ -0,0 +1,46 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Functions for querying registered kernels."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.core.framework import kernel_def_pb2
+from tensorflow.python import pywrap_tensorflow as c_api
+from tensorflow.python.util import compat
+
+
+def get_all_registered_kernels():
+  """Returns a KernelList proto of all registered kernels.
+  """
+  buf = c_api.TF_GetAllRegisteredKernels()
+  data = c_api.TF_GetBuffer(buf)
+  kernel_list = kernel_def_pb2.KernelList()
+  kernel_list.ParseFromString(compat.as_bytes(data))
+  return kernel_list
+
+
+def get_registered_kernels_for_op(name):
+  """Returns a KernelList proto of registered kernels for a given op.
+
+  Args:
+    name: A string representing the name of the op whose kernels to retrieve.
+  """
+  buf = c_api.TF_GetRegisteredKernelsForOp(name)
+  data = c_api.TF_GetBuffer(buf)
+  kernel_list = kernel_def_pb2.KernelList()
+  kernel_list.ParseFromString(compat.as_bytes(data))
+  return kernel_list
diff --git a/tensorflow/python/framework/kernels_test.py b/tensorflow/python/framework/kernels_test.py
new file mode 100644
index 0000000000..c53500be73
--- /dev/null
+++ b/tensorflow/python/framework/kernels_test.py
@@ -0,0 +1,41 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for querying registered kernels."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import kernels
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import googletest
+
+
+class GetAllRegisteredKernelsTest(test_util.TensorFlowTestCase):
+
+  def testFindsAtLeastOneKernel(self):
+    kernel_list = kernels.get_all_registered_kernels()
+    self.assertGreater(len(kernel_list.kernel), 0)
+
+
+class GetRegisteredKernelsForOp(test_util.TensorFlowTestCase):
+
+  def testFindsAtLeastOneKernel(self):
+    kernel_list = kernels.get_registered_kernels_for_op("KernelLabel")
+    self.assertGreater(len(kernel_list.kernel), 0)
+    self.assertEqual(kernel_list.kernel[0].op, "KernelLabel")
+
+
+if __name__ == "__main__":
+  googletest.main()
-- 
cgit v1.2.3


From 226831aab92a395a26824a08caa9d43f0c3d604e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 07:40:12 -0700
Subject: Fix pack_test.

PiperOrigin-RevId: 205826660
---
 tensorflow/contrib/lite/kernels/pack_test.cc | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/pack_test.cc b/tensorflow/contrib/lite/kernels/pack_test.cc
index cb9fed69b1..485a50ad3a 100644
--- a/tensorflow/contrib/lite/kernels/pack_test.cc
+++ b/tensorflow/contrib/lite/kernels/pack_test.cc
@@ -22,6 +22,7 @@ namespace tflite {
 namespace {
 
 using ::testing::ElementsAre;
+using ::testing::ElementsAreArray;
 
 template 
 class PackOpModel : public SingleOpModel {
@@ -57,7 +58,7 @@ TEST(PackOpTest, FloatThreeInputs) {
   model.SetInput(2, {3, 6});
   model.Invoke();
   EXPECT_THAT(model.GetOutputShape(), ElementsAre(3, 2));
-  EXPECT_THAT(model.GetOutput(), ElementsAre(1, 4, 2, 5, 3, 6));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 4, 2, 5, 3, 6}));
 }
 
 TEST(PackOpTest, FloatThreeInputsDifferentAxis) {
@@ -67,7 +68,7 @@ TEST(PackOpTest, FloatThreeInputsDifferentAxis) {
   model.SetInput(2, {3, 6});
   model.Invoke();
   EXPECT_THAT(model.GetOutputShape(), ElementsAre(2, 3));
-  EXPECT_THAT(model.GetOutput(), ElementsAre(1, 2, 3, 4, 5, 6));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 4, 5, 6}));
 }
 
 TEST(PackOpTest, FloatMultilDimensions) {
@@ -77,7 +78,7 @@ TEST(PackOpTest, FloatMultilDimensions) {
   model.Invoke();
   EXPECT_THAT(model.GetOutputShape(), ElementsAre(2, 2, 3));
   EXPECT_THAT(model.GetOutput(),
-              ElementsAre(1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12));
+              ElementsAreArray({1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12}));
 }
 
 TEST(PackOpTest, IntThreeInputs) {
@@ -87,7 +88,7 @@ TEST(PackOpTest, IntThreeInputs) {
   model.SetInput(2, {3, 6});
   model.Invoke();
   EXPECT_THAT(model.GetOutputShape(), ElementsAre(3, 2));
-  EXPECT_THAT(model.GetOutput(), ElementsAre(1, 4, 2, 5, 3, 6));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 4, 2, 5, 3, 6}));
 }
 
 TEST(PackOpTest, IntThreeInputsDifferentAxis) {
@@ -97,7 +98,7 @@ TEST(PackOpTest, IntThreeInputsDifferentAxis) {
   model.SetInput(2, {3, 6});
   model.Invoke();
   EXPECT_THAT(model.GetOutputShape(), ElementsAre(2, 3));
-  EXPECT_THAT(model.GetOutput(), ElementsAre(1, 2, 3, 4, 5, 6));
+  EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 4, 5, 6}));
 }
 
 TEST(PackOpTest, IntMultilDimensions) {
@@ -107,7 +108,7 @@ TEST(PackOpTest, IntMultilDimensions) {
   model.Invoke();
   EXPECT_THAT(model.GetOutputShape(), ElementsAre(2, 2, 3));
   EXPECT_THAT(model.GetOutput(),
-              ElementsAre(1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12));
+              ElementsAreArray({1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12}));
 }
 }  // namespace
 }  // namespace tflite
-- 
cgit v1.2.3


From 1b33df1814e35015953c7cba392ba2a7387ce875 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Tue, 24 Jul 2018 08:29:15 -0700
Subject: [XLA:GPU] Don't lie about buffer alignment to LLVM

PiperOrigin-RevId: 205832336
---
 tensorflow/compiler/xla/service/gpu/BUILD          |  1 +
 .../compiler/xla/service/gpu/buffer_allocations.cc |  8 ++--
 .../compiler/xla/service/gpu/gpu_constants.cc      | 13 +++++-
 .../compiler/xla/service/gpu/gpu_constants.h       |  9 ++--
 .../xla/service/gpu/ir_emitter_unnested.cc         |  4 +-
 .../compiler/xla/service/gpu/nvptx_compiler.cc     |  2 +-
 tensorflow/compiler/xla/service/gpu/tests/BUILD    | 19 ++++++++
 .../xla/service/gpu/tests/gpu_alignment_test.cc    | 54 ++++++++++++++++++++++
 8 files changed, 99 insertions(+), 11 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/gpu/tests/gpu_alignment_test.cc

diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 06ff3d9bba..72aff197fc 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -36,6 +36,7 @@ cc_library(
     hdrs = ["gpu_constants.h"],
     deps = [
         "//tensorflow/compiler/xla:types",
+        "//tensorflow/core:framework",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc b/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc
index ab5149dcdb..b095d4cd73 100644
--- a/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc
+++ b/tensorflow/compiler/xla/service/gpu/buffer_allocations.cc
@@ -49,12 +49,12 @@ StatusOr> BufferAllocations::Builder::Build(
     if (registered_buffers_.count(i)) {
       se::DeviceMemoryBase address = FindOrDie(registered_buffers_, i);
       if (reinterpret_cast(address.opaque()) %
-              kCudaMallocAlignBytes !=
+              kEntryParameterAlignBytes !=
           0) {
         return InternalError(
             "Address of registered buffer %lld must be a multiple of %llx, but "
             "was %p",
-            i, kCudaMallocAlignBytes, address.opaque());
+            i, kEntryParameterAlignBytes, address.opaque());
       }
       buffer_allocations->SetBuffer(i, FindOrDie(registered_buffers_, i));
       continue;
@@ -71,12 +71,12 @@ StatusOr> BufferAllocations::Builder::Build(
         TF_ASSIGN_OR_RETURN(
             buffer, memory_allocator->Allocate(device_ordinal, buffer_size));
         if (reinterpret_cast(buffer.opaque()) %
-                kCudaMallocAlignBytes !=
+                kXlaAllocatedBufferAlignBytes !=
             0) {
           return InternalError(
               "Address returned by memory_allocator->Allocate must be a "
               "multiple of %llx, but was %p",
-              kCudaMallocAlignBytes, buffer.opaque());
+              kXlaAllocatedBufferAlignBytes, buffer.opaque());
         }
         // We do manual memory management within BufferAllocations.  Be sure not
         // to do a TF_RETURN_IF_ERROR between this line and the
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_constants.cc b/tensorflow/compiler/xla/service/gpu/gpu_constants.cc
index aa360c7f73..e6ddea6d25 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_constants.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_constants.cc
@@ -14,12 +14,21 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/xla/service/gpu/gpu_constants.h"
+#include "tensorflow/core/framework/allocator.h"
 
 namespace xla {
 namespace gpu {
 
-// http://docs.nvidia.com/cuda/cuda-c-programming-guide/#device-memory-accesses
-const int64 kCudaMallocAlignBytes = 256;
+// kEntryParameterAlignBytes is equal to EIGEN_MAX_ALIGN_BYTES, though including
+// Eigen headers here to get that symbol may not be a good idea.
+// EIGEN_MAX_ALIGN_BYTES may differ between CUDA-enabled builds vs CUDA-disabled
+// builds and we don't want the IR generated by XLA:GPU to depend on that.
+//
+// TODO(b/111767313): Consider raising EIGEN_MAX_ALIGN_BYTES if it helps.
+const int64 kEntryParameterAlignBytes = 16;
+
+const int64 kXlaAllocatedBufferAlignBytes =
+    tensorflow::Allocator::kAllocatorAlignment;
 
 }  // namespace gpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_constants.h b/tensorflow/compiler/xla/service/gpu/gpu_constants.h
index eb1ca4c6c9..925e6927b6 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_constants.h
+++ b/tensorflow/compiler/xla/service/gpu/gpu_constants.h
@@ -21,9 +21,12 @@ limitations under the License.
 namespace xla {
 namespace gpu {
 
-// Minimum alignment of cudaMalloc.  We require that buffers created by our
-// DeviceMemoryAllocator, and all input/output buffers, have this alignment.
-extern const int64 kCudaMallocAlignBytes;
+// Minimum alignment for buffers passed as incoming arguments by TensorFlow.
+extern const int64 kEntryParameterAlignBytes;
+
+// Minimum alignment for buffers allocated by XLA: the temp buffers and the live
+// out (result) buffers.
+extern const int64 kXlaAllocatedBufferAlignBytes;
 
 }  // namespace gpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index b1038a3cc9..1f31a7f36b 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -232,7 +232,9 @@ llvm::Function* IrEmitterUnnested::BuildKernelPrototype(
     kernel->addDereferenceableAttr(arg_no + 1, alloc->size());
     kernel->addParamAttr(
         arg_no, llvm::Attribute::get(context, llvm::Attribute::Alignment,
-                                     kCudaMallocAlignBytes));
+                                     alloc->is_entry_computation_parameter()
+                                         ? kEntryParameterAlignBytes
+                                         : kXlaAllocatedBufferAlignBytes));
 
     if (alloc->IsPreallocatedTempBuffer()) {
       fn_arg->setName("temp_buf");
diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
index ad29862d83..2eefadebcd 100644
--- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc
@@ -543,7 +543,7 @@ StatusOr> NVPTXCompiler::RunBackend(
       BufferAssigner::Run(module.get(), hlo_schedule->ConsumeHloOrdering(),
                           BufferSizeBytesFunction(),
                           /*color_alignment=*/[](LogicalBuffer::Color) {
-                            return kCudaMallocAlignBytes;
+                            return kXlaAllocatedBufferAlignBytes;
                           }));
   // BufferAssignment::Stats::ToString() and BufferAssignment::ToString()
   // include headers, so no need for us to print them ourselves.
diff --git a/tensorflow/compiler/xla/service/gpu/tests/BUILD b/tensorflow/compiler/xla/service/gpu/tests/BUILD
index 926262e2ad..686c3c16c9 100644
--- a/tensorflow/compiler/xla/service/gpu/tests/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/tests/BUILD
@@ -202,3 +202,22 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
+
+tf_cc_test(
+    name = "gpu_alignment_test",
+    testonly = True,
+    srcs = ["gpu_alignment_test.cc"],
+    tags = [
+        "requires-gpu-sm35",
+    ],
+    deps = [
+        ":gpu_codegen_test",
+        "//tensorflow/compiler/xla/service:gpu_plugin",
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
+        "//tensorflow/compiler/xla/service/llvm_ir:alias_analysis",
+        "//tensorflow/compiler/xla/tests:filecheck",
+        "//tensorflow/compiler/xla/tests:llvm_irgen_test_base",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_alignment_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_alignment_test.cc
new file mode 100644
index 0000000000..672c68e59b
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_alignment_test.cc
@@ -0,0 +1,54 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include 
+#include 
+
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
+#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/alias_analysis.h"
+#include "tensorflow/compiler/xla/tests/filecheck.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace gpu {
+namespace {
+
+class GpuAlignmentTest : public GpuCodegenTest {};
+
+TEST_F(GpuAlignmentTest, Test) {
+  const char* hlo_string = R"(
+HloModule GpuAlignmentTest
+
+ENTRY main {
+  zero = f32[] constant(0)
+  tok = token[] after-all()
+  a = f32[100] parameter(0)
+  b_tup = (f32[200], token[]) infeed(tok)
+  b = f32[200] get-tuple-element(b_tup), index=0
+  a_padded = f32[150] pad(a, zero), padding=0_50
+  b_sliced = f32[150] slice(b), slice={[0:150]}
+  ROOT c = f32[150] add(a_padded, b_sliced)
+}
+)";
+
+  CompileAndVerifyIr(hlo_string, R"(
+CHECK: @fusion(i8* align 64 dereferenceable(600) %alloc0, i8* align 16 dereferenceable(400) %alloc1, i8* align 64 dereferenceable(864) %temp_buf)
+)");
+}
+
+}  // namespace
+}  // namespace gpu
+}  // namespace xla
-- 
cgit v1.2.3


From 00f9eb9c589ad3c5ce5b4dde84763553c56ad0ee Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Tue, 24 Jul 2018 09:35:59 -0700
Subject: Improvement resource variable documentation.

Removes stale mentions of how to construct them from docstrings and adds a new
section to the programmer's guide explaining resource handles and resource
variable semantics.

PiperOrigin-RevId: 205842012
---
 tensorflow/contrib/eager/python/saver.py           |  4 +--
 .../base_api/api_def_ResourceScatterNdAdd.pbtxt    |  2 +-
 tensorflow/python/eager/backprop.py                |  8 ++---
 tensorflow/python/eager/function.py                | 27 +++++++-------
 tensorflow/python/ops/resource_variable_ops.py     | 17 ++++-----
 tensorflow/python/ops/variables.py                 | 41 +++++++++-------------
 6 files changed, 45 insertions(+), 54 deletions(-)

diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py
index fdaca90fd1..d709308647 100644
--- a/tensorflow/contrib/eager/python/saver.py
+++ b/tensorflow/contrib/eager/python/saver.py
@@ -125,8 +125,8 @@ class Saver(object):
 
     Args:
       var_list: The list of variables that will be saved and restored. Either a
-        list of `tfe.Variable` objects, or a dictionary mapping names to
-        `tfe.Variable` objects.
+        list of `tf.Variable` objects, or a dictionary mapping names to
+        `tf.Variable` objects.
 
     Raises:
       RuntimeError: if invoked when eager execution has not been enabled.
diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdAdd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdAdd.pbtxt
index 3b3a274df5..2b58969da2 100644
--- a/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdAdd.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdAdd.pbtxt
@@ -51,7 +51,7 @@ For example, say we want to update 4 scattered elements to a rank-1 tensor to
 8 elements. In Python, that update would look like this:
 
 ```python
-    ref = tfe.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+    ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8], use_resource=True)
     indices = tf.constant([[4], [3], [1] ,[7]])
     updates = tf.constant([9, 10, 11, 12])
     update = tf.scatter_nd_add(ref, indices, updates)
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 9e0bbce4a1..da8b93dba8 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -643,10 +643,10 @@ class GradientTape(object):
   Operations are recorded if they are executed within this context manager and
   at least one of their inputs is being "watched".
 
-  Trainable variables (created by `tf.contrib.eager.Variable` or
-  @{tf.get_variable}, trainable=True is default in both cases) are automatically
-  watched. Tensors can be manually watched by invoking the `watch` method on
-  this context manager.
+  Trainable variables (created by `tf.Variable` or @{tf.get_variable},
+  trainable=True is default in both cases) are automatically watched. Tensors
+  can be manually watched by invoking the `watch` method on this context
+  manager.
 
   For example, consider the function `y = x * x`. The gradient at `x = 3.0` can
   be computed as:
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index a6906f9efd..d283a85532 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -1077,7 +1077,7 @@ def defun(func=None, compiled=False):
   tf.enable_eager_execution()
 
   def fn():
-    x = tf.contrib.eager.Variable(0.0)
+    x = tf.Variable(0.0)
     x.assign_add(1.0)
     return x.read_value()
 
@@ -1094,19 +1094,18 @@ def defun(func=None, compiled=False):
   ```
 
   Finally, because each input signature is bound to a unique graph, if your
-  Python function constructs `tf.contrib.eager.Variable` objects, then each
-  graph constructed for that Python function will reference a unique set of
-  variables. To circumvent this problem, we recommend against compiling Python
-  functions that create `tf.contrib.eager.Variable` objects. Instead, Python
-  functions should either lexically close over `tf.contrib.eager.Variable`
-  objects or accept them as arguments, preferably encapsulated in an
-  object-oriented container. If you must create variables inside your Python
-  function and you want each graph generated for it to reference the same set of
-  variables, add logic to your Python function that ensures that variables are
-  only created the first time it is called and are reused for every subsequent
-  invocation; note that this is precisely what @{tf.keras.layers.Layer} objects
-  do, so we recommend using them to represent variable-bearing computations
-  whenever possible.
+  Python function constructs `tf.Variable` objects, then each graph constructed
+  for that Python function will reference a unique set of variables. To
+  circumvent this problem, we recommend against compiling Python functions that
+  create `tf.Variable` objects. Instead, Python functions should either
+  lexically close over `tf.Variable` objects or accept them as arguments,
+  preferably encapsulated in an object-oriented container. If you must create
+  variables inside your Python function and you want each graph generated for it
+  to reference the same set of variables, add logic to your Python function that
+  ensures that variables are only created the first time it is called and are
+  reused for every subsequent invocation; note that this is precisely what
+  @{tf.keras.layers.Layer} objects do, so we recommend using them to represent
+  variable-bearing computations whenever possible.
 
   Args:
     func: function to be compiled. If `func` is None, returns a
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index db071e3974..8b259b6b6b 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -196,15 +196,16 @@ class ResourceVariable(variables.RefVariable):
   the variable are fixed. The value can be changed using one of the assign
   methods.
 
-  Just like any `Tensor`, variables created with `ResourceVariable()` can be
-  used as inputs for other Ops in the graph. Additionally, all the operators
-  overloaded for the `Tensor` class are carried over to variables, so you can
-  also add nodes to the graph by just doing arithmetic on variables.
+  Just like any `Tensor`, variables created with
+  `tf.Variable(use_resource=True)` can be used as inputs for other Ops in the
+  graph. Additionally, all the operators overloaded for the `Tensor` class are
+  carried over to variables, so you can also add nodes to the graph by just
+  doing arithmetic on variables.
 
-  Unlike tf.Variable, a tf.ResourceVariable has well-defined semantics. Each
+  Unlike ref-based variable, a ResourceVariable has well-defined semantics. Each
   usage of a ResourceVariable in a TensorFlow graph adds a read_value operation
-  to the graph. The Tensors returned by a read_value operation are guaranteed
-  to see all modifications to the value of the variable which happen in any
+  to the graph. The Tensors returned by a read_value operation are guaranteed to
+  see all modifications to the value of the variable which happen in any
   operation on which the read_value depends on (either directly, indirectly, or
   via a control dependency) and guaranteed to not see any modification to the
   value of the variable from operations that depend on the read_value operation.
@@ -218,7 +219,7 @@ class ResourceVariable(variables.RefVariable):
   can cause tf.Variable and tf.ResourceVariable to behave differently:
 
   ```python
-  a = tf.ResourceVariable(1.0)
+  a = tf.Variable(1.0, use_resource=True)
   a.initializer.run()
 
   assign = a.assign(2.0)
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index d03d93beeb..fc00ce68ae 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -220,27 +220,31 @@ class Variable(six.with_metaclass(VariableMetaclass,
   various `Optimizer` classes use this collection as the default list of
   variables to optimize.
 
-  WARNING: tf.Variable objects have a non-intuitive memory model. A Variable is
-  represented internally as a mutable Tensor which can non-deterministically
-  alias other Tensors in a graph. The set of operations which consume a Variable
-  and can lead to aliasing is undetermined and can change across TensorFlow
-  versions. Avoid writing code which relies on the value of a Variable either
-  changing or not changing as other operations happen. For example, using
-  Variable objects or simple functions thereof as predicates in a `tf.cond` is
-  dangerous and error-prone:
+  WARNING: tf.Variable objects by default have a non-intuitive memory model. A
+  Variable is represented internally as a mutable Tensor which can
+  non-deterministically alias other Tensors in a graph. The set of operations
+  which consume a Variable and can lead to aliasing is undetermined and can
+  change across TensorFlow versions. Avoid writing code which relies on the
+  value of a Variable either changing or not changing as other operations
+  happen. For example, using Variable objects or simple functions thereof as
+  predicates in a `tf.cond` is dangerous and error-prone:
 
   ```
   v = tf.Variable(True)
   tf.cond(v, lambda: v.assign(False), my_false_fn)  # Note: this is broken.
   ```
 
-  Here replacing tf.Variable with tf.contrib.eager.Variable will fix any
-  nondeterminism issues.
+  Here replacing adding `use_resource=True` when constructing the variable will
+  fix any nondeterminism issues:
+  ```
+  v = tf.Variable(True, use_resource=True)
+  tf.cond(v, lambda: v.assign(False), my_false_fn)
+  ```
 
   To use the replacement for variables which does
   not have these issues:
 
-  * Replace `tf.Variable` with `tf.contrib.eager.Variable`;
+  * Add `use_resource=True` when constructing `tf.Variable`;
   * Call `tf.get_variable_scope().set_use_resource(True)` inside a
     `tf.variable_scope` before the `tf.get_variable()` call.
   """
@@ -869,19 +873,7 @@ class RefVariable(Variable):
       ValueError: If the initial value is not specified, or does not have a
         shape and `validate_shape` is `True`.
       RuntimeError: If eager execution is enabled.
-
-    @compatibility(eager)
-    `tf.Variable` is not compatible with eager execution.  Use
-    `tfe.Variable` instead which is compatible with both eager execution
-    and graph construction.  See [the TensorFlow Eager Execution
-    guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
-    for details on how variables work in eager execution.
-    @end_compatibility
     """
-    if context.executing_eagerly():
-      raise RuntimeError(
-          "tf.Variable not supported when eager execution is enabled. "
-          "Please use tf.contrib.eager.Variable instead")
     self._in_graph_mode = True
     if variable_def:
       # If variable_def is provided, recreates the variable from its fields.
@@ -992,8 +984,7 @@ class RefVariable(Variable):
       # Ensure that we weren't lifted into the eager context.
       if context.executing_eagerly():
         raise RuntimeError(
-            "tf.Variable not supported when eager execution is enabled. "
-            "Please use tf.contrib.eager.Variable instead")
+            "RefVariable not supported when eager execution is enabled. ")
       with ops.name_scope(name, "Variable", [] if init_from_fn else
                           [initial_value]) as name:
 
-- 
cgit v1.2.3


From f8bbd3ceb7e86b7595ba74a9a03cfc7c1be252a8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 09:38:26 -0700
Subject: A subsequent improvement to the creation of a config option to not
 link LGPL, including fix for the Android/Apple version of code (C++ macros-es
 fix)

PiperOrigin-RevId: 205842327
---
 tensorflow/tensorflow.bzl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 26970c8cb0..340d3f393c 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -244,6 +244,7 @@ def tf_copts(android_optimization_level_override="-O2", is_external=False):
             clean_dep("//tensorflow:windows"): get_win_copts(is_external),
             clean_dep("//tensorflow:windows_msvc"): get_win_copts(is_external),
             clean_dep("//tensorflow:ios"): ["-std=c++11"],
+            clean_dep("//tensorflow:no_lgpl_deps"): ["-D__TENSORFLOW_NO_LGPL_DEPS__", "-pthread"],
             "//conditions:default": ["-pthread"]
       }))
 
-- 
cgit v1.2.3


From 568727eed199dba04e37f500265b50f96fed455e Mon Sep 17 00:00:00 2001
From: Nick Felt 
Date: Tue, 24 Jul 2018 09:49:47 -0700
Subject: Add v2 summary support to Estimator.train() and MonitoredSession
 hooks

This change makes Estimator.train() support v2 summaries (tf.contrib.summary.*) out-of-the-box, to match the support for v1 summaries.  Estimator.train() will now handle the boilerplate necessary to initialize a file writer and enable summary writing every N steps, and will ensure that its own automatically exported summaries (for loss and global_step/sec) get written to the same underlying events file.

As part of this change, tf.train.SummarySaverHook, tf.train.CheckpointSaverHook, tf.train.StepCounterHook, and tf.train.ProfilerHook have also been adapted to write summaries using the v2 summary system (via a compatibility layer), instead of using FileWriterCache.

A couple additional smaller changes are:
- the 'session' parameter to FileWriter() can now be a callable returning a tf.Session instance.
- the introduction of tf.contrib.summary.record_summaries_if() which takes a boolean tensor for direct control of tf.contrib.summary.should_record_summaries().
- EstimatorSpec.train_op, besides a tf.Operation, is now allowed to be any Tensor-equivalent object rather than just a tf.Tensor.

PiperOrigin-RevId: 205843986
---
 .../contrib/summary/summary_ops_graph_test.py      |  20 +
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py |  17 +-
 tensorflow/core/kernels/summary_kernels.cc         |   2 +
 tensorflow/python/BUILD                            |   1 +
 tensorflow/python/estimator/estimator.py           |  24 +-
 tensorflow/python/estimator/estimator_test.py      | 260 ++++++++++-
 tensorflow/python/estimator/model_fn.py            |   3 +-
 tensorflow/python/estimator/training_test.py       |  10 +-
 tensorflow/python/ops/summary_ops_v2.py            |  68 ++-
 tensorflow/python/saved_model/builder_impl.py      |   5 +-
 .../python/summary/writer/event_file_writer_v2.py  |  71 +--
 tensorflow/python/summary/writer/writer.py         |   8 +-
 tensorflow/python/summary/writer/writer_test.py    |  54 ++-
 .../python/training/basic_session_run_hooks.py     | 182 +++++---
 .../training/basic_session_run_hooks_test.py       | 476 ++++++++++++++-------
 tensorflow/python/training/monitored_session.py    |  11 +-
 tensorflow/python/training/optimizer.py            |   6 +-
 17 files changed, 939 insertions(+), 279 deletions(-)

diff --git a/tensorflow/contrib/summary/summary_ops_graph_test.py b/tensorflow/contrib/summary/summary_ops_graph_test.py
index ae8336daaf..409fdf4583 100644
--- a/tensorflow/contrib/summary/summary_ops_graph_test.py
+++ b/tensorflow/contrib/summary/summary_ops_graph_test.py
@@ -228,6 +228,26 @@ class GraphFileTest(test_util.TensorFlowTestCase):
       sess.run(writer.flush())
       self.assertEqual(2, get_total())
 
+  def testSummaryOpsCollector(self):
+    summary_ops.scalar('x', 1.0, step=1)
+    with summary_ops.create_file_writer(self.get_temp_dir()).as_default():
+      s2 = summary_ops.scalar('x', 1.0, step=1)
+      collector1 = summary_ops._SummaryOpsCollector()
+      collector2 = summary_ops._SummaryOpsCollector()
+      with collector1.capture():
+        s3 = summary_ops.scalar('x', 1.0, step=1)
+        with collector2.capture():
+          s4 = summary_ops.scalar('x', 1.0, step=1)
+        s5 = summary_ops.scalar('x', 1.0, step=1)
+      s6 = summary_ops.scalar('x', 1.0, step=1)
+    summary_ops.scalar('six', 1.0, step=1)
+
+    # Ops defined outside summary writer context are ignored; ops defined inside
+    # SummaryOpsCollector capture context are stored to innermost such context.
+    self.assertItemsEqual([s2, s6], summary_ops.all_summary_ops())
+    self.assertItemsEqual([s3, s5], collector1.collected_ops)
+    self.assertItemsEqual([s4], collector2.collected_ops)
+
 
 class GraphDbTest(summary_test_util.SummaryDbTest):
 
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 42406db88a..1eb43ac7f7 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -1506,13 +1506,17 @@ class _OutfeedHostCall(object):
     _OutfeedHostCall.validate(host_calls)
     ret = {}
     for name, host_call in host_calls.items():
+      # Isolate host call summary ops from main graph.
+      summary_collector = contrib_summary._SummaryOpsCollector()  # pylint: disable=protected-access
       host_fn, tensors = host_call
       if isinstance(tensors, (tuple, list)):
-        ret[name] = host_fn(*tensors)
+        with summary_collector.capture():
+          ret[name] = host_fn(*tensors)
       else:
         # Must be dict.
         try:
-          ret[name] = host_fn(**tensors)
+          with summary_collector.capture():
+            ret[name] = host_fn(**tensors)
         except TypeError as e:
           logging.warning(
               'Exception while calling %s: %s. It is likely the tensors '
@@ -1627,11 +1631,14 @@ class _OutfeedHostCall(object):
           # dimension.
           dequeue_ops[i] = array_ops.concat(dequeue_ops[i], axis=0)
 
+        # Isolate host call summary ops from main graph.
+        summary_collector = contrib_summary._SummaryOpsCollector()  # pylint: disable=protected-access
         if self._tensor_keys[name] is not None:
           # The user-provided eval_metrics[1] is a dict.
           dequeue_ops = dict(zip(self._tensor_keys[name], dequeue_ops))
           try:
-            ret[name] = self._host_fns[name](**dequeue_ops)
+            with summary_collector.capture():
+              ret[name] = self._host_fns[name](**dequeue_ops)
           except TypeError as e:
             logging.warning(
                 'Exception while calling %s: %s. It is likely the tensors '
@@ -1639,8 +1646,8 @@ class _OutfeedHostCall(object):
                 'function\'s arguments', name, e, name)
             raise e
         else:
-          ret[name] = self._host_fns[name](*dequeue_ops)
-
+          with summary_collector.capture():
+            ret[name] = self._host_fns[name](*dequeue_ops)
     return ret
 
 
diff --git a/tensorflow/core/kernels/summary_kernels.cc b/tensorflow/core/kernels/summary_kernels.cc
index b287f0cc2f..b518c3cbf4 100644
--- a/tensorflow/core/kernels/summary_kernels.cc
+++ b/tensorflow/core/kernels/summary_kernels.cc
@@ -53,6 +53,7 @@ class CreateSummaryFileWriterOp : public OpKernel {
                                   max_queue, flush_millis, logdir,
                                   filename_suffix, ctx->env(), s);
                             }));
+    core::ScopedUnref unref(s);
   }
 };
 REGISTER_KERNEL_BUILDER(Name("CreateSummaryFileWriter").Device(DEVICE_CPU),
@@ -89,6 +90,7 @@ class CreateSummaryDbWriterOp : public OpKernel {
                   db, experiment_name, run_name, user_name, ctx->env(), s));
               return Status::OK();
             }));
+    core::ScopedUnref unref(s);
   }
 };
 REGISTER_KERNEL_BUILDER(Name("CreateSummaryDbWriter").Device(DEVICE_CPU),
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 814239533c..b5a0051c28 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2822,6 +2822,7 @@ py_library(
         ":framework_ops",
         ":math_ops",
         ":resource_variable_ops",
+        ":resources",
         ":smart_cond",
         ":summary_op_util",
         ":summary_ops_gen",
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 915ceeb98b..b7185e8966 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -46,6 +46,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import metrics as metrics_lib
 from tensorflow.python.ops import resources
+from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
@@ -65,6 +66,7 @@ from tensorflow.python.util import compat
 from tensorflow.python.util import compat_internal
 from tensorflow.python.util import function_utils
 from tensorflow.python.util import nest
+from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import estimator_export
 
 
@@ -1156,7 +1158,8 @@ class Estimator(object):
       Loss from training
     """
     worker_hooks = []
-    with ops.Graph().as_default() as g, g.device(self._device_fn):
+    with ops.Graph().as_default() as g, g.device(
+        self._device_fn), self._summary_writing_context():
       random_seed.set_random_seed(self._config.tf_random_seed)
       global_step_tensor = self._create_and_assert_global_step(g)
       training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
@@ -1190,7 +1193,7 @@ class Estimator(object):
     is_tpu_strategy = self._distribution.__class__.__name__ == 'TPUStrategy'
 
     worker_hooks = []
-    with ops.Graph().as_default() as g:
+    with ops.Graph().as_default() as g, self._summary_writing_context():
       with self._distribution.scope():
         random_seed.set_random_seed(self._config.tf_random_seed)
 
@@ -1519,6 +1522,23 @@ class Estimator(object):
                    (self._warm_start_settings,))
       warm_starting_util.warm_start(*self._warm_start_settings)
 
+  @tf_contextlib.contextmanager
+  def _summary_writing_context(self):
+    """Context manager for enabling V2 summary writing."""
+    # Avoid creating a file writer at all if no summary writing was requested.
+    if self._config.save_summary_steps <= 0:
+      yield
+      return
+    file_writer = summary_ops_v2.create_file_writer(
+        logdir=self._model_dir, filename_suffix='')
+    with file_writer.as_default():
+      # Create a boolean placeholder, default False, that SummarySaverHook can
+      # use to enable/disable V2 summary writing according to its own logic.
+      placeholder = array_ops.placeholder_with_default(False, shape=[])
+      training.SummarySaverHook._set_placeholder(placeholder)  # pylint: disable=protected-access
+      with summary_ops_v2.record_summaries_if(placeholder):
+        yield
+
 
 def create_per_tower_ready_op(scaffold):
   """Create a Scaffold.ready_op inside a tower."""
diff --git a/tensorflow/python/estimator/estimator_test.py b/tensorflow/python/estimator/estimator_test.py
index 8bc410ba0b..1dd45a07c2 100644
--- a/tensorflow/python/estimator/estimator_test.py
+++ b/tensorflow/python/estimator/estimator_test.py
@@ -22,6 +22,7 @@ import functools
 import glob
 import os
 import tempfile
+import time
 
 import numpy as np
 import six
@@ -29,6 +30,7 @@ import six
 from google.protobuf import text_format
 
 from tensorflow.core.protobuf import rewriter_config_pb2
+from tensorflow.core.util.event_pb2 import SessionLog
 from tensorflow.python.client import session
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.estimator import estimator
@@ -40,6 +42,7 @@ from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.framework import test_util
@@ -55,6 +58,7 @@ from tensorflow.python.ops import metrics as metrics_lib
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import string_ops
+from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.ops.losses import losses
@@ -85,13 +89,32 @@ def dummy_model_fn(features, labels, params):
   _, _, _ = features, labels, params
 
 
-def summaries_with_matching_keyword(keyword, dir_):
-  """Yields summary protos matching given keyword from event file."""
-
+def load_eventfile_contents(directory_path):
+  """Returns the contents of the singular event file in the given directory."""
   writer_cache.FileWriterCache.clear()
 
-  event_paths = glob.glob(os.path.join(dir_, 'events*'))
-  for event in summary_iterator.summary_iterator(event_paths[-1]):
+  # Get last Event written.
+  event_paths = glob.glob(os.path.join(directory_path, '*tfevent*'))
+  if len(event_paths) != 1:
+    raise AssertionError('Expected one eventfile, got %s' % str(event_paths))
+  return list(summary_iterator.summary_iterator(event_paths[0]))
+
+
+def make_summary_steps(eventlist):
+  """Returns dict of tags in eventlist mapped to steps where they're logged."""
+  tag_to_steps = {}
+  for event in eventlist:
+    if event.summary is not None:
+      for value in event.summary.value:
+        if value.tag not in tag_to_steps:
+          tag_to_steps[value.tag] = []
+        tag_to_steps[value.tag].append(event.step)
+  return tag_to_steps
+
+
+def summaries_with_matching_keyword(keyword, dir_):
+  """Yields summary protos matching given keyword from event file."""
+  for event in load_eventfile_contents(dir_):
     if event.summary is not None:
       for value in event.summary.value:
         if keyword in value.tag:
@@ -366,13 +389,51 @@ def dummy_input_fn():
           constant_op.constant([[1], [1]]))
 
 
+class StableGlobalStepEstimator(estimator.Estimator):
+  """Estimator subclass using a ResourceVariable global_step for testing."""
+  # TODO(nickfelt): remove after standard global_step is a ResourceVariable.
+
+  def _create_global_step(self, graph):
+    """Creates a stable ResourceVariable-based global step suitable for tests.
+
+    Args:
+      graph: The graph in which to create the global step.
+
+    Returns:
+      A global step `Tensor`.
+    """
+    with graph.as_default(), graph.name_scope(None):
+      return variable_scope.get_variable(
+          ops.GraphKeys.GLOBAL_STEP,
+          shape=[],
+          dtype=dtypes.int64,
+          initializer=init_ops.zeros_initializer(),
+          trainable=False,
+          collections=[
+              ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.GLOBAL_STEP
+          ],
+          # Use a ResourceVariable and set caching_device to make the read
+          # behavior deterministic and well-defined.
+          caching_device='cpu:0',
+          use_resource=True)
+
+
 def model_fn_global_step_incrementer(features, labels, mode):
   _, _ = features, labels
-  global_step = training.get_global_step()
   return model_fn_lib.EstimatorSpec(
       mode,
       loss=constant_op.constant(1.),
-      train_op=state_ops.assign_add(global_step, 1))
+      train_op=training.get_global_step().assign_add(1))
+
+
+def model_fn_with_v1_and_v2_summaries(features, labels, mode):
+  del features, labels
+  summary.scalar('foo-v1', 1.0)
+  summary_ops_v2.scalar('foo-v2', 2.0)
+  return model_fn_lib.EstimatorSpec(
+      mode,
+      loss=constant_op.constant(1.),
+      train_op=training.get_global_step().assign_add(1))
 
 
 def assert_features_op(expected_features, actual_features):
@@ -408,6 +469,25 @@ def _make_input_fn(features, labels):
   return _input_fn
 
 
+class RaiseOnceAtStepHook(session_run_hook.SessionRunHook):
+  """Hook that raises an Exception the first time it reaches step N."""
+
+  def __init__(self, n, ex):
+    self.n = n
+    self.ex = ex
+    self.raised = False
+
+  def before_run(self, run_context):
+    # Raise the first time we reach step N.
+    self.n -= 1
+    if 0 == self.n and not self.raised:
+      # Wait 1 sec so that event file names have different UNIX timestamps.
+      time.sleep(1.2)
+      self.raised = True
+      raise self.ex
+    return None
+
+
 class EstimatorTrainTest(test.TestCase):
 
   def test_callable_model_fn(self):
@@ -617,17 +697,171 @@ class EstimatorTrainTest(test.TestCase):
     self.assertEqual(
         5, estimator._load_global_step_from_checkpoint_dir(est.model_dir))
 
-  def test_loss_summary(self):
+  def test_summary_loss(self):
     est = estimator.Estimator(model_fn=model_fn_global_step_incrementer,
                               config=run_config.RunConfig(save_summary_steps=1))
     est.train(dummy_input_fn, steps=1)
+    events = load_eventfile_contents(est.model_dir)
+    self.assertEqual({'loss': [1]}, make_summary_steps(events))
 
-    # Make sure nothing is stuck in limbo.
-    writer_cache.FileWriterCache.clear()
+  def test_summary_user_defined_v1_and_v2(self):
+    est = StableGlobalStepEstimator(
+        model_fn=model_fn_with_v1_and_v2_summaries,
+        config=run_config.RunConfig(save_summary_steps=1))
+    est.train(dummy_input_fn, steps=1)
+    events = load_eventfile_contents(est.model_dir)
+    self.assertEqual(
+        {'foo-v1': [1], 'foo-v2': [0], 'loss': [1]},
+        make_summary_steps(events))
 
-    if check_eventfile_for_keyword('loss', est.model_dir):
-      return
-    self.fail('{} should be part of reported summaries.'.format('loss'))
+  def test_summary_writing_disabled(self):
+    est = StableGlobalStepEstimator(
+        model_fn=model_fn_with_v1_and_v2_summaries,
+        config=run_config.RunConfig(save_summary_steps=0))
+    est.train(dummy_input_fn, steps=1)
+    events = load_eventfile_contents(est.model_dir)
+    self.assertEqual({}, make_summary_steps(events))
+
+  def test_summary_saving_steps(self):
+    est = StableGlobalStepEstimator(
+        model_fn=model_fn_with_v1_and_v2_summaries,
+        config=run_config.RunConfig(save_summary_steps=2))
+    est.train(dummy_input_fn, steps=5)
+    events = load_eventfile_contents(est.model_dir)
+    self.assertEqual(
+        {'foo-v1': [1, 3, 5], 'foo-v2': [0, 2, 4], 'loss': [1, 3, 5]},
+        make_summary_steps(events))
+
+  def test_summary_additional_hook(self):
+    def model_fn_extra_summary_hook(features, labels, mode, config):
+      del features, labels
+      v1_op = summary.scalar('foo-v1', 1.0)
+      v2_op = summary_ops_v2.scalar('foo-v2', 2.0)
+      extra_hook = basic_session_run_hooks.SummarySaverHook(
+          output_dir=os.path.join(config.model_dir, 'extra'),
+          save_steps=3,
+          summary_op=control_flow_ops.with_dependencies([v2_op], v1_op))
+      return model_fn_lib.EstimatorSpec(
+          mode,
+          loss=constant_op.constant(1.),
+          train_op=training.get_global_step().assign_add(1),
+          training_hooks=[extra_hook])
+    est = StableGlobalStepEstimator(
+        model_fn=model_fn_extra_summary_hook,
+        config=run_config.RunConfig(save_summary_steps=2))
+    est.train(dummy_input_fn, steps=7)
+
+    events = load_eventfile_contents(est.model_dir)
+    self.assertEqual(
+        {'foo-v1': [1, 3, 5, 7], 'foo-v2': [0, 2, 4, 6], 'loss': [1, 3, 5, 7]},
+        make_summary_steps(events))
+    extra_dir = os.path.join(est.model_dir, 'extra')
+    extra_events = load_eventfile_contents(extra_dir)
+    self.assertEqual({'foo-v1': [1, 4, 7]}, make_summary_steps(extra_events))
+
+  def test_summary_user_defined_in_input_fn(self):
+    def input_fn_custom_summaries():
+      summary.scalar('foo-v1', 1.0)
+      summary_ops_v2.scalar('foo-v2', 2.0)
+      return ({'x': constant_op.constant([[1], [1]])},
+              constant_op.constant([[1], [1]]))
+    est = StableGlobalStepEstimator(
+        model_fn=model_fn_global_step_incrementer,
+        config=run_config.RunConfig(save_summary_steps=1))
+    est.train(input_fn_custom_summaries, steps=1)
+    events = load_eventfile_contents(est.model_dir)
+    self.assertEqual(
+        {'foo-v1': [1], 'foo-v2': [0], 'loss': [1]},
+        make_summary_steps(events))
+
+  def test_summary_with_warm_start(self):
+    est = StableGlobalStepEstimator(
+        model_fn=model_fn_with_v1_and_v2_summaries,
+        config=run_config.RunConfig(save_summary_steps=1))
+    est.train(dummy_input_fn, steps=5)
+    warm_started_est = StableGlobalStepEstimator(
+        model_fn=model_fn_with_v1_and_v2_summaries,
+        config=run_config.RunConfig(save_summary_steps=1),
+        warm_start_from=est.model_dir)
+    warm_started_est.train(dummy_input_fn, steps=3)
+    events = load_eventfile_contents(warm_started_est.model_dir)
+    self.assertEqual(
+        {'foo-v1': [1, 2, 3], 'foo-v2': [0, 1, 2], 'loss': [1, 2, 3]},
+        make_summary_steps(events))
+
+  def test_summary_with_error_and_auto_restart(self):
+    est = StableGlobalStepEstimator(
+        model_fn=model_fn_with_v1_and_v2_summaries,
+        config=run_config.RunConfig(
+            save_summary_steps=2, save_checkpoints_steps=5))
+    abort_hook = RaiseOnceAtStepHook(
+        7, errors_impl.AbortedError(None, None, 'Abort'))
+    est.train(dummy_input_fn, steps=10, hooks=[abort_hook])
+
+    # We expect two event files: one for the aborted run, and one post-restart.
+    event_paths = sorted(glob.glob(os.path.join(est.model_dir, '*tfevent*')))
+    self.assertEqual(2, len(event_paths))
+
+    # First file should have summaries up to the last checkpoint.
+    first_events = list(summary_iterator.summary_iterator(event_paths[0]))
+    first_summaries = make_summary_steps(first_events)
+    self.assertEqual([0, 2, 4], first_summaries['foo-v2'])
+    # The V1 summaries may or may not include step 5 (depending on the flush()
+    # sequence) so just check that at least 1 and 3 are there.
+    # TODO(nickfelt): ensure summaries *at* checkpoint step get flushed too.
+    self.assertEqual([1, 3], first_summaries['foo-v1'][:2])
+    self.assertEqual([1, 3], first_summaries['loss'][:2])
+
+    # Second file should pick up from global_step=5. Note that the 2 step save
+    # interval will reset at this step as well, so summaries logged at steps
+    # 2 and 4 continue not with 6, 8, ... but at steps 5, 7, ... instead.
+    second_events = list(summary_iterator.summary_iterator(event_paths[1]))
+    self.assertEqual(
+        {'foo-v1': [6, 8, 10], 'foo-v2': [5, 7, 9], 'loss': [6, 8, 10]},
+        make_summary_steps(second_events))
+    # Second file should contain a session START event at resumed global_step.
+    session_start_event = next(event for event in second_events
+                               if event.session_log.status == SessionLog.START)
+    self.assertEqual(5, session_start_event.step)
+
+  def test_summary_with_error_and_explicit_restart(self):
+    est = StableGlobalStepEstimator(
+        model_fn=model_fn_with_v1_and_v2_summaries,
+        config=run_config.RunConfig(
+            save_summary_steps=2, save_checkpoints_steps=5))
+    abort_hook = RaiseOnceAtStepHook(
+        7, errors_impl.UnknownError(None, None, 'Unknown failure'))
+    self.assertRaises(
+        errors_impl.UnknownError,
+        lambda: est.train(dummy_input_fn, max_steps=10, hooks=[abort_hook]))
+    # Explicitly retry after the error.
+    est.train(dummy_input_fn, max_steps=10, hooks=[abort_hook])
+
+    # We expect two event files: one for the failed run, and one post-restart.
+    event_paths = sorted(glob.glob(os.path.join(est.model_dir, '*tfevent*')))
+    self.assertEqual(2, len(event_paths))
+
+    # First file should have summaries up to the last checkpoint.
+    first_events = list(summary_iterator.summary_iterator(event_paths[0]))
+    first_summaries = make_summary_steps(first_events)
+    self.assertEqual([0, 2, 4], first_summaries['foo-v2'])
+    # The V1 summaries may or may not include step 5 (depending on the flush()
+    # sequence) so just check that at least 1 and 3 are there.
+    # TODO(nickfelt): ensure summaries *at* checkpoint step get flushed too.
+    self.assertEqual([1, 3], first_summaries['foo-v1'][:2])
+    self.assertEqual([1, 3], first_summaries['loss'][:2])
+
+    # Second file should pick up from global_step=5. Note that the 2 step save
+    # interval will reset at this step as well, so summaries logged at steps
+    # 2 and 4 continue not with 6, 8, ... but at steps 5, 7, ... instead.
+    second_events = list(summary_iterator.summary_iterator(event_paths[1]))
+    self.assertEqual(
+        {'foo-v1': [6, 8, 10], 'foo-v2': [5, 7, 9], 'loss': [6, 8, 10]},
+        make_summary_steps(second_events))
+    # Second file should contain a session START event at resumed global_step.
+    session_start_event = next(event for event in second_events
+                               if event.session_log.status == SessionLog.START)
+    self.assertEqual(5, session_start_event.step)
 
   def test_latest_checkpoint(self):
     est = estimator.Estimator(model_fn=model_fn_global_step_incrementer)
diff --git a/tensorflow/python/estimator/model_fn.py b/tensorflow/python/estimator/model_fn.py
index a9fd8f8e1a..b1b2f65edf 100644
--- a/tensorflow/python/estimator/model_fn.py
+++ b/tensorflow/python/estimator/model_fn.py
@@ -26,6 +26,7 @@ import six
 from tensorflow.python.estimator.export import export_output as export_output_lib
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import tag_constants
@@ -432,7 +433,7 @@ class _TPUEstimatorSpec(collections.namedtuple('TPUEstimatorSpec', [
 
 
 def _check_is_tensor_or_operation(x, name):
-  if not (isinstance(x, ops.Operation) or isinstance(x, ops.Tensor)):
+  if not (isinstance(x, ops.Operation) or tensor_util.is_tensor(x)):
     raise TypeError('{} must be Operation or Tensor, given: {}'.format(name, x))
 
 
diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py
index dc106c7d3b..121439a2cd 100644
--- a/tensorflow/python/estimator/training_test.py
+++ b/tensorflow/python/estimator/training_test.py
@@ -2059,7 +2059,7 @@ class TrainAndEvaluateIntegrationTest(test.TestCase):
 
   def _extract_loss_and_global_step(self, event_folder):
     """Returns the loss and global step in last event."""
-    event_paths = glob.glob(os.path.join(event_folder, 'events*'))
+    event_paths = sorted(glob.glob(os.path.join(event_folder, 'events*')))
 
     loss = None
     global_step_count = None
@@ -2139,10 +2139,12 @@ class TrainAndEvaluateIntegrationTest(test.TestCase):
     # Make sure nothing is stuck in limbo.
     writer_cache.FileWriterCache.clear()
 
-    # Examine the training events. Use a range to check global step to avoid
-    # flakyness due to global step race condition.
-    training_loss, _ = self._extract_loss_and_global_step(est.model_dir)
+    # Examine the training events.
+    training_loss, training_global_step = self._extract_loss_and_global_step(
+        est.model_dir)
     self.assertIsNotNone(training_loss)
+    # Training summaries are logged for steps 1 and 10, so we see final step.
+    self.assertEqual(max_steps, training_global_step)
 
     # Examine the eval events. The global step should be accurate.
     eval_loss, eval_global_step = self._extract_loss_and_global_step(
diff --git a/tensorflow/python/ops/summary_ops_v2.py b/tensorflow/python/ops/summary_ops_v2.py
index 00150fe688..669358d9db 100644
--- a/tensorflow/python/ops/summary_ops_v2.py
+++ b/tensorflow/python/ops/summary_ops_v2.py
@@ -37,6 +37,7 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_summary_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import resources
 from tensorflow.python.ops import summary_op_util
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import training_util
@@ -66,41 +67,39 @@ def should_record_summaries():
   return should_record_collection[0]
 
 
+@tf_contextlib.contextmanager
+def always_record_summaries():
+  """Sets the should_record_summaries Tensor to always true."""
+  with record_summaries_if(True):
+    yield
+
+
+@tf_contextlib.contextmanager
+def never_record_summaries():
+  """Sets the should_record_summaries Tensor to always false."""
+  with record_summaries_if(False):
+    yield
+
+
 # TODO(apassos) consider how to handle local step here.
 @tf_contextlib.contextmanager
 def record_summaries_every_n_global_steps(n, global_step=None):
   """Sets the should_record_summaries Tensor to true if global_step % n == 0."""
   if global_step is None:
     global_step = training_util.get_or_create_global_step()
-  collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
-  old = collection_ref[:]
-  try:
-    with ops.device("cpu:0"):
-      collection_ref[:] = [math_ops.equal(global_step % n, 0)]
-    yield
-  finally:
-    collection_ref[:] = old
-
-
-@tf_contextlib.contextmanager
-def always_record_summaries():
-  """Sets the should_record_summaries Tensor to always true."""
-  collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
-  old = collection_ref[:]
-  try:
-    collection_ref[:] = [True]
+  with ops.device("cpu:0"):
+    on_nth_global_step = math_ops.equal(global_step % n, 0)
+  with record_summaries_if(on_nth_global_step):
     yield
-  finally:
-    collection_ref[:] = old
 
 
 @tf_contextlib.contextmanager
-def never_record_summaries():
-  """Sets the should_record_summaries Tensor to always false."""
+def record_summaries_if(bool_value):
+  """Sets the should_record_summaries Tensor to the given boolean value."""
   collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
   old = collection_ref[:]
   try:
-    collection_ref[:] = [False]
+    collection_ref[:] = [bool_value]
     yield
   finally:
     collection_ref[:] = old
@@ -143,7 +142,6 @@ class SummaryWriter(object):
       finally:
         context.context().summary_writer_resource = old
 
-
   def init(self):
     """Operation to initialize the summary writer resource."""
     if self._resource is not None:
@@ -311,6 +309,9 @@ def _make_summary_writer(name, factory, **kwargs):
     # TODO(apassos): Consider doing this instead.
     #   ops.get_default_session().run(init_op)
     ops.add_to_collection(_SUMMARY_WRITER_INIT_COLLECTION_NAME, init_op)
+    # TODO(nickfelt): expose an actual op for this
+    is_initialized_op = constant_op.constant(True)
+    resources.register_resource(resource, init_op, is_initialized_op)
   return SummaryWriter(resource, init_op_fn)
 
 
@@ -325,6 +326,27 @@ def _nothing():
   return constant_op.constant(False)
 
 
+class _SummaryOpsCollector(object):
+  """Defines a context manager for isolating out a subset of summary ops.
+
+  Summary ops defined within this context will be accumulated within this
+  collector instead of being added to the graph-wide summary ops collection that
+  is returned by {@tf.contrib.summary.all_summary_ops}.
+  """
+
+  def __init__(self):
+    self.collected_ops = []
+
+  @tf_contextlib.contextmanager
+  def capture(self):
+    collection_ref = ops.get_collection_ref(ops.GraphKeys._SUMMARY_COLLECTION)  # pylint: disable=protected-access
+    original_ops = collection_ref[:]
+    collection_ref[:] = []
+    yield
+    self.collected_ops = collection_ref[:]
+    collection_ref[:] = original_ops
+
+
 def all_summary_ops():
   """Graph-mode only. Returns all summary ops.
 
diff --git a/tensorflow/python/saved_model/builder_impl.py b/tensorflow/python/saved_model/builder_impl.py
index e58be804c2..b67d0f2362 100644
--- a/tensorflow/python/saved_model/builder_impl.py
+++ b/tensorflow/python/saved_model/builder_impl.py
@@ -28,6 +28,7 @@ from tensorflow.core.protobuf import saved_model_pb2
 from tensorflow.core.protobuf import saver_pb2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging
@@ -178,10 +179,10 @@ class SavedModelBuilder(object):
         stored as a collection with key TRAIN_OP_KEY, but not executed.
 
     Raises:
-      TypeError if Train op is not of type `Operation`.
+      TypeError if Train op is not of type `Operation` or a Tensor.
     """
     if train_op is not None:
-      if (not isinstance(train_op, ops.Tensor) and
+      if (not tensor_util.is_tensor(train_op) and
           not isinstance(train_op, ops.Operation)):
         raise TypeError("train_op needs to be a Tensor or Op: %r" % train_op)
       ops.add_to_collection(constants.TRAIN_OP_KEY, train_op)
diff --git a/tensorflow/python/summary/writer/event_file_writer_v2.py b/tensorflow/python/summary/writer/event_file_writer_v2.py
index 5c66c0f7a8..262182d3b8 100644
--- a/tensorflow/python/summary/writer/event_file_writer_v2.py
+++ b/tensorflow/python/summary/writer/event_file_writer_v2.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.client import session as tf_session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -43,11 +44,11 @@ class EventFileWriterV2(object):
     """Creates an `EventFileWriterV2` and an event file to write to.
 
     On construction, this calls `tf.contrib.summary.create_file_writer` within
-    the graph from `session.graph` to look up a shared summary writer resource
-    for `logdir` if one exists, and create one if not. Creating the summary
+    the default graph, which finds and returns a shared summary writer resource
+    for `logdir` if one exists, and creates one if not. Creating the summary
     writer resource in turn creates a new event file in `logdir` to be filled
     with `Event` protocol buffers passed to `add_event`. Graph ops to control
-    this writer resource are added to `session.graph` during this init call;
+    this writer resource are added to the default graph during this init call;
     stateful methods on this class will call `session.run()` on these ops.
 
     Note that because the underlying resource is shared, it is possible that
@@ -61,38 +62,50 @@ class EventFileWriterV2(object):
     no effect.  See `tf.contrib.summary.create_file_writer` for details.
 
     Args:
-      session: A `tf.Session`. Session that will hold shared writer resource.
-        The writer ops will be added to session.graph during this init call.
+      session: A `tf.Session`, or a callable that provides one which will be
+        called on-demand. The session will hold the shared writer resource.
       logdir: A string. Directory where event file will be written.
       max_queue: Integer. Size of the queue for pending events and summaries.
       flush_secs: Number. How often, in seconds, to flush the
         pending events and summaries to disk.
       filename_suffix: A string. Every event file's name is suffixed with
         `filename_suffix`.
+
+    Raises:
+      ValueError: if `session` is not a `tf.Session` or a callable
     """
-    self._session = session
+    if isinstance(session, tf_session.SessionInterface):
+      self._session = lambda: session
+    elif callable(session):
+      self._session = session
+    else:
+      raise ValueError('session must be tf.Session or callable')
     self._logdir = logdir
+    self._initialized = False
     self._closed = False
     if not gfile.IsDirectory(self._logdir):
       gfile.MakeDirs(self._logdir)
 
-    with self._session.graph.as_default():
-      with ops.name_scope('filewriter'):
-        file_writer = summary_ops_v2.create_file_writer(
-            logdir=self._logdir,
-            max_queue=max_queue,
-            flush_millis=flush_secs * 1000,
-            filename_suffix=filename_suffix)
-        with summary_ops_v2.always_record_summaries(), file_writer.as_default():
-          self._event_placeholder = array_ops.placeholder_with_default(
-              constant_op.constant('unused', dtypes.string),
-              shape=[])
-          self._add_event_op = summary_ops_v2.import_event(
-              self._event_placeholder)
-        self._init_op = file_writer.init()
-        self._flush_op = file_writer.flush()
-        self._close_op = file_writer.close()
-      self._session.run(self._init_op)
+    with ops.name_scope('filewriter'):
+      file_writer = summary_ops_v2.create_file_writer(
+          logdir=self._logdir,
+          max_queue=max_queue,
+          flush_millis=flush_secs * 1000,
+          filename_suffix=filename_suffix)
+      with summary_ops_v2.always_record_summaries(), file_writer.as_default():
+        self._event_placeholder = array_ops.placeholder_with_default(
+            constant_op.constant('unused', dtypes.string),
+            shape=[])
+        self._add_event_op = summary_ops_v2.import_event(
+            self._event_placeholder)
+      self._init_op = file_writer.init()
+      self._flush_op = file_writer.flush()
+      self._close_op = file_writer.close()
+
+  def _init_if_needed(self):
+    if not self._initialized:
+      self._session().run(self._init_op)
+      self._initialized = True
 
   def get_logdir(self):
     """Returns the directory where event file will be written."""
@@ -108,7 +121,6 @@ class EventFileWriterV2(object):
     """
     if self._closed:
       self._closed = False
-      self._session.run(self._init_op)
 
   def add_event(self, event):
     """Adds an event to the event file.
@@ -117,8 +129,9 @@ class EventFileWriterV2(object):
       event: An `Event` protocol buffer.
     """
     if not self._closed:
+      self._init_if_needed()
       event_pb = event.SerializeToString()
-      self._session.run(
+      self._session().run(
           self._add_event_op, feed_dict={self._event_placeholder: event_pb})
 
   def flush(self):
@@ -127,7 +140,9 @@ class EventFileWriterV2(object):
     Call this method to make sure that all pending events have been written to
     disk.
     """
-    self._session.run(self._flush_op)
+    if not self._closed:
+      self._init_if_needed()
+      self._session().run(self._flush_op)
 
   def close(self):
     """Flushes the event file to disk and close the file.
@@ -135,6 +150,8 @@ class EventFileWriterV2(object):
     Call this method when you do not need the summary writer anymore.
     """
     if not self._closed:
+      self._init_if_needed()
       self.flush()
-      self._session.run(self._close_op)
+      self._session().run(self._close_op)
       self._closed = True
+      self._initialized = False
diff --git a/tensorflow/python/summary/writer/writer.py b/tensorflow/python/summary/writer/writer.py
index aca084fc91..2a967ae3a5 100644
--- a/tensorflow/python/summary/writer/writer.py
+++ b/tensorflow/python/summary/writer/writer.py
@@ -332,8 +332,11 @@ class FileWriter(SummaryToEventTransformer):
     the same shared resource name (which by default scoped to the logdir). If
     no such resource exists, one will be created using the remaining arguments
     to this constructor, but if one already exists those arguments are ignored.
-    In either case, ops will be added to `session.graph` to control the
+    In either case, ops will be added to the default graph to control the
     underlying file writer resource. See `tf.contrib.summary` for more details.
+    Instead of an actual `tf.Session`, this argument may also be a callable that
+    provides a `tf.Session` when invoked (e.g. `tf.get_default_session`), which
+    will be called on-demand when a session is needed.
 
     Args:
       logdir: A string. Directory where event file will be written.
@@ -344,7 +347,8 @@ class FileWriter(SummaryToEventTransformer):
       graph_def: DEPRECATED: Use the `graph` argument instead.
       filename_suffix: A string. Every event file's name is suffixed with
         `suffix`.
-      session: A `tf.Session` object. See details above.
+      session: A `tf.Session` object or a callable that provides `tf.Session`
+        objects. See details above.
 
     Raises:
       RuntimeError: If called with eager execution enabled.
diff --git a/tensorflow/python/summary/writer/writer_test.py b/tensorflow/python/summary/writer/writer_test.py
index dc990c2602..3380dea317 100644
--- a/tensorflow/python/summary/writer/writer_test.py
+++ b/tensorflow/python/summary/writer/writer_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for training_coordinator.py."""
+"""Tests for writer.py."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -574,6 +574,58 @@ class SessionBasedFileWriterTestCase(FileWriterTestCase):
     # No more files
     self.assertRaises(StopIteration, lambda: next(event_paths))
 
+  def testSesssionArgument_callableProvider(self):
+    logdir = self.get_temp_dir()
+    setup_writer = summary_ops_v2.create_file_writer(logdir=logdir)
+    with summary_ops_v2.always_record_summaries(), setup_writer.as_default():
+      summary1 = summary_ops_v2.scalar("one", 0.0, step=0)
+      summary2 = summary_ops_v2.scalar("two", 0.0, step=0)
+    sess1 = session.Session()
+    sess1.run(setup_writer.init())
+    sess1.run(summary1)
+    sess1.run(setup_writer.flush())
+    time.sleep(1.1)  # Ensure filename has a different timestamp
+    sess2 = session.Session()
+    sess2.run(setup_writer.init())
+    sess2.run(summary2)
+    sess2.run(setup_writer.flush())
+
+    # Using get_default_session as session provider should make this FileWriter
+    # send its summaries to the current default session's shared summary writer
+    # resource (initializing it as needed).
+    test_writer = writer.FileWriter(
+        session=ops.get_default_session, logdir=logdir)
+    with sess1.as_default():
+      test_writer.add_summary(self._createTaggedSummary("won"), 1)
+      test_writer.flush()
+    with sess2.as_default():
+      test_writer.add_summary(self._createTaggedSummary("too"), 1)
+      test_writer.flush()
+
+    event_paths = iter(sorted(glob.glob(os.path.join(logdir, "event*"))))
+
+    # First file should have tags "one", "won"
+    events = summary_iterator.summary_iterator(next(event_paths))
+    self.assertEqual("brain.Event:2", next(events).file_version)
+    self.assertEqual("one", next(events).summary.value[0].tag)
+    self.assertEqual("won", next(events).summary.value[0].tag)
+    self.assertRaises(StopIteration, lambda: next(events))
+
+    # Second file should have tags "two", "too"
+    events = summary_iterator.summary_iterator(next(event_paths))
+    self.assertEqual("brain.Event:2", next(events).file_version)
+    self.assertEqual("two", next(events).summary.value[0].tag)
+    self.assertEqual("too", next(events).summary.value[0].tag)
+    self.assertRaises(StopIteration, lambda: next(events))
+
+    # No more files
+    self.assertRaises(StopIteration, lambda: next(event_paths))
+
+  def testSessionArgument_notSessionOrCallable(self):
+    logdir = self.get_temp_dir()
+    self.assertRaises(
+        ValueError, lambda: writer.FileWriter(session=[], logdir=logdir))
+
 
 class FileWriterCacheTest(test.TestCase):
   """FileWriterCache tests."""
diff --git a/tensorflow/python/training/basic_session_run_hooks.py b/tensorflow/python/training/basic_session_run_hooks.py
index b0dd188db1..b8df7fe51b 100644
--- a/tensorflow/python/training/basic_session_run_hooks.py
+++ b/tensorflow/python/training/basic_session_run_hooks.py
@@ -31,12 +31,13 @@ from tensorflow.python.client import timeline
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.summary.writer import writer
 from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import training_util
 from tensorflow.python.training.session_run_hook import SessionRunArgs
-from tensorflow.python.training.summary_io import SummaryWriterCache
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -422,7 +423,9 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
     self._steps_per_run = steps_per_run
 
   def begin(self):
-    self._summary_writer = SummaryWriterCache.get(self._checkpoint_dir)
+    self._summary_writer = writer.FileWriter(
+        self._checkpoint_dir, session=ops.get_default_session,
+        filename_suffix="")
     self._global_step_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
     if self._global_step_tensor is None:
       raise RuntimeError(
@@ -431,10 +434,12 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
       l.begin()
 
   def after_create_session(self, session, coord):
+    del coord
+    # Ensure summary writer resource has been initialized.
+    session.run(summary_ops_v2.summary_writer_initializer_op())
     global_step = session.run(self._global_step_tensor)
-    # We do write graph and saver_def at the first call of before_run.
-    # We cannot do this in begin, since we let other hooks to change graph and
-    # add variables in begin. Graph is finalized after all begin calls.
+    # Write graph and saver_def once graph is finalized, which isn't true yet
+    # in begin() since later hooks can still change the graph.
     training_util.write_graph(
         ops.get_default_graph().as_graph_def(add_shapes=True),
         self._checkpoint_dir,
@@ -444,8 +449,9 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
     meta_graph_def = meta_graph.create_meta_graph_def(
         graph_def=graph.as_graph_def(add_shapes=True),
         saver_def=saver_def)
-    self._summary_writer.add_graph(graph)
-    self._summary_writer.add_meta_graph(meta_graph_def)
+    with ops.default_session(session):
+      self._summary_writer.add_graph(graph)
+      self._summary_writer.add_meta_graph(meta_graph_def)
     # The checkpoint saved here is the state at step "global_step".
     self._save(session, global_step)
     self._timer.update_last_triggered_step(global_step)
@@ -470,6 +476,8 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
       self._save(session, last_step)
     for l in self._listeners:
       l.end(session, last_step)
+    with ops.default_session(session):
+      self._summary_writer.flush()
 
   def _save(self, session, step):
     """Saves the latest checkpoint, returns should_stop."""
@@ -479,10 +487,12 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
       l.before_save(session, step)
 
     self._get_saver().save(session, self._save_path, global_step=step)
-    self._summary_writer.add_session_log(
-        SessionLog(
-            status=SessionLog.CHECKPOINT, checkpoint_path=self._save_path),
-        step)
+    with ops.default_session(session):
+      self._summary_writer.add_session_log(
+          SessionLog(
+              status=SessionLog.CHECKPOINT, checkpoint_path=self._save_path),
+          step)
+      self._summary_writer.flush()
 
     should_stop = False
     for l in self._listeners:
@@ -543,13 +553,23 @@ class StepCounterHook(session_run_hook.SessionRunHook):
 
   def begin(self):
     if self._summary_writer is None and self._output_dir:
-      self._summary_writer = SummaryWriterCache.get(self._output_dir)
+      self._summary_writer = writer.FileWriter(
+          self._output_dir, session=ops.get_default_session,
+          filename_suffix="")
     self._global_step_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
     if self._global_step_tensor is None:
       raise RuntimeError(
           "Global step should be created to use StepCounterHook.")
     self._summary_tag = training_util.get_global_step().op.name + "/sec"
 
+  def after_create_session(self, session, coord):
+    del coord
+    # Reset any stale state in case we're recovering from a previous error.
+    session.run(summary_ops_v2.summary_writer_initializer_op())
+    self._last_global_step = None
+    self._global_step_check_count = 0
+    self._timer.reset()
+
   def before_run(self, run_context):  # pylint: disable=unused-argument
     return SessionRunArgs(self._global_step_tensor)
 
@@ -562,8 +582,6 @@ class StepCounterHook(session_run_hook.SessionRunHook):
     logging.info("%s: %g", self._summary_tag, steps_per_sec)
 
   def after_run(self, run_context, run_values):
-    _ = run_context
-
     stale_global_step = run_values.results
     if self._timer.should_trigger_for_step(
         stale_global_step + self._steps_per_run):
@@ -573,7 +591,8 @@ class StepCounterHook(session_run_hook.SessionRunHook):
         elapsed_time, elapsed_steps = self._timer.update_last_triggered_step(
             global_step)
         if elapsed_time is not None:
-          self._log_and_record(elapsed_steps, elapsed_time, global_step)
+          with ops.default_session(run_context.session):
+            self._log_and_record(elapsed_steps, elapsed_time, global_step)
 
     # Check whether the global step has been increased. Here, we do not use the
     # timer.last_triggered_step as the timer might record a different global
@@ -599,6 +618,11 @@ class StepCounterHook(session_run_hook.SessionRunHook):
 
     self._last_global_step = stale_global_step
 
+  def end(self, session):
+    if self._summary_writer is not None:
+      with ops.default_session(session):
+        self._summary_writer.flush()
+
 
 @tf_export("train.NanLossDuringTrainingError")
 class NanLossDuringTrainingError(RuntimeError):
@@ -643,6 +667,25 @@ class NanTensorHook(session_run_hook.SessionRunHook):
 class SummarySaverHook(session_run_hook.SessionRunHook):
   """Saves summaries every N steps."""
 
+  _SUMMARY_PLACEHOLDER_COLLECTION = "_SUMMARY_SAVER_PLACEHOLDER"
+
+  @classmethod
+  def _set_placeholder(cls, placeholder):
+    """Sets a `tf.placeholder` to be fed by the first SummarySaverHook.
+
+    If a placeholder is provided, the first instance of SummarySaverHook in use
+    will feed it a boolean indicating whether summaries should be written,
+    according to the `save_steps` and `save_secs` parameters of that hook. This
+    makes the placeholder usable with `tf.contrib.summary.record_summaries_if`
+    to control `tf.contrib.summary` summary writing using the same schedule as
+    the `tf.summary` summary writing (which the hook controls directly).
+
+    Args:
+      placeholder: `tf.placeholder` for the first SummarySaverHook to feed
+    """
+    collection = ops.get_collection_ref(cls._SUMMARY_PLACEHOLDER_COLLECTION)
+    collection[:] = [placeholder]
+
   def __init__(self,
                save_steps=None,
                save_secs=None,
@@ -680,53 +723,82 @@ class SummarySaverHook(session_run_hook.SessionRunHook):
     self._scaffold = scaffold
     self._timer = SecondOrStepTimer(every_secs=save_secs,
                                     every_steps=save_steps)
+    self._placeholder = None
     # TODO(mdan): Throw an error if output_dir and summary_writer are None.
 
   def begin(self):
     if self._summary_writer is None and self._output_dir:
-      self._summary_writer = SummaryWriterCache.get(self._output_dir)
-    self._next_step = None
-    self._global_step_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
+      self._summary_writer = writer.FileWriter(
+          self._output_dir, filename_suffix="", session=ops.get_default_session)
+      # Designate the first SummarySaverHook to call begin() as the "primary"
+      # hook; it will control writing of v2 summaries via a placeholder bool.
+      collection = ops.get_collection_ref(self._SUMMARY_PLACEHOLDER_COLLECTION)
+      if collection:
+        self._placeholder = collection[0]
+        collection[:] = []
+    self._current_step = None
+    self._global_step_tensor = training_util.get_or_create_global_step()
     if self._global_step_tensor is None:
       raise RuntimeError(
           "Global step should be created to use SummarySaverHook.")
 
-  def before_run(self, run_context):  # pylint: disable=unused-argument
-    self._request_summary = (
-        self._next_step is None or
-        self._timer.should_trigger_for_step(self._next_step))
+  def after_create_session(self, session, coord):
+    del coord
+    # Reset any stale state in case we're recovering from a previous error.
+    session.run(summary_ops_v2.summary_writer_initializer_op())
+    self._current_step = None
+    self._timer.reset()
+
+  def before_run(self, run_context):
+    # For the first run, record a SessionLog.START at the pre-run global step.
+    if self._current_step is None:
+      self._current_step = run_context.session.run(self._global_step_tensor)
+      with ops.default_session(run_context.session):
+        self._summary_writer.add_session_log(
+            SessionLog(status=SessionLog.START), self._current_step)
     requests = {"global_step": self._global_step_tensor}
+    self._request_summary = self._timer.should_trigger_for_step(
+        self._current_step)
     if self._request_summary:
+      self._timer.update_last_triggered_step(self._current_step)
       if self._get_summary_op() is not None:
         requests["summary"] = self._get_summary_op()
-
-    return SessionRunArgs(requests)
+    feeds = {}
+    if self._placeholder is not None and self._request_summary:
+      feeds[self._placeholder] = self._request_summary
+    args = SessionRunArgs(fetches=requests, feed_dict=feeds)
+    return args
 
   def after_run(self, run_context, run_values):
-    _ = run_context
-    if not self._summary_writer:
-      return
-
+    # Collect any legacy v1 summaries to emit.
+    summaries_to_emit = []
+    if self._summary_writer and self._request_summary:
+      for summary in run_values.results.get("summary", []):
+        # Skip None results corresponding to V2 summary operations.
+        if summary is not None:
+          summaries_to_emit.append(summary)
+    # Heuristically estimate current step as possibly-stale value plus one.
     stale_global_step = run_values.results["global_step"]
-    global_step = stale_global_step + 1
-    if self._next_step is None or self._request_summary:
-      global_step = run_context.session.run(self._global_step_tensor)
-
-    if self._next_step is None:
-      self._summary_writer.add_session_log(
-          SessionLog(status=SessionLog.START), global_step)
-
-    if self._request_summary:
-      self._timer.update_last_triggered_step(global_step)
-      if "summary" in run_values.results:
-        for summary in run_values.results["summary"]:
-          self._summary_writer.add_summary(summary, global_step)
-
-    self._next_step = global_step + 1
+    self._current_step = stale_global_step + 1
+    # Read the actual post-run global step if we need better accuracy because
+    # 1) we will request summaries on the next run (based on estimate now) and
+    #    must ensure we record an accurate "last triggered step" value, or
+    # 2) we have legacy v1 summaries to emit using the post-run step value.
+    # Note: we could have dealt with (1) separately in before_run() but by doing
+    # it here we can consolidate the reads in case both (1) and (2) apply.
+    near_next_trigger = self._timer.should_trigger_for_step(self._current_step)
+    if near_next_trigger or summaries_to_emit:
+      self._current_step = run_context.session.run(self._global_step_tensor)
+    # Emit any legacy v1 summaries.
+    if summaries_to_emit:
+      with ops.default_session(run_context.session):
+        for summary in summaries_to_emit:
+          self._summary_writer.add_summary(summary, self._current_step)
 
   def end(self, session=None):
-    if self._summary_writer:
-      self._summary_writer.flush()
+    if self._summary_writer and session:
+      with ops.default_session(session):
+        self._summary_writer.flush()
 
   def _get_summary_op(self):
     """Fetches the summary op either from self._summary_op or self._scaffold.
@@ -893,19 +965,27 @@ class ProfilerHook(session_run_hook.SessionRunHook):
       show_memory: `bool`, if True, add object snapshot events to the trace
           showing the sizes and lifetimes of tensors.
     """
+    self._output_dir = output_dir
     self._output_file = os.path.join(output_dir, "timeline-{}.json")
-    self._file_writer = SummaryWriterCache.get(output_dir)
     self._show_dataflow = show_dataflow
     self._show_memory = show_memory
     self._timer = SecondOrStepTimer(
         every_secs=save_secs, every_steps=save_steps)
 
   def begin(self):
+    self._file_writer = writer.FileWriter(
+        self._output_dir, filename_suffix="", session=ops.get_default_session)
     self._next_step = None
     self._global_step_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
     if self._global_step_tensor is None:
       raise RuntimeError("Global step should be created to use ProfilerHook.")
 
+  def after_create_session(self, session, coord):
+    del coord
+    # Reset any stale state in case we're recovering from a previous error.
+    session.run(summary_ops_v2.summary_writer_initializer_op())
+    self._timer.reset()
+
   def before_run(self, run_context):
     self._request_summary = (
         self._next_step is None or
@@ -925,8 +1005,10 @@ class ProfilerHook(session_run_hook.SessionRunHook):
       self._save(global_step,
                  self._output_file.format(global_step),
                  run_values.run_metadata.step_stats)
-      self._file_writer.add_run_metadata(run_values.run_metadata,
-                                         "step_%d" % global_step)
+      with ops.default_session(run_context.session):
+        self._file_writer.add_run_metadata(run_values.run_metadata,
+                                           "step_%d" % global_step,
+                                           global_step=global_step)
 
     self._next_step = global_step + 1
 
@@ -938,6 +1020,10 @@ class ProfilerHook(session_run_hook.SessionRunHook):
           trace.generate_chrome_trace_format(
               show_dataflow=self._show_dataflow, show_memory=self._show_memory))
 
+  def end(self, session):
+    with ops.default_session(session):
+      self._file_writer.flush()
+
 
 def _as_graph_element(obj):
   """Retrieves Graph element."""
diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py
index b49a871a56..b89167f3c1 100644
--- a/tensorflow/python/training/basic_session_run_hooks_test.py
+++ b/tensorflow/python/training/basic_session_run_hooks_test.py
@@ -19,8 +19,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import glob
 import os.path
 import shutil
+import sys
 import tempfile
 import threading
 import time
@@ -28,6 +30,9 @@ import time
 from tensorflow.contrib.framework.python.framework import checkpoint_utils
 from tensorflow.contrib.framework.python.ops import variables
 from tensorflow.contrib.testing.python.framework import fake_summary_writer
+from tensorflow.core.framework import graph_pb2
+from tensorflow.core.protobuf import meta_graph_pb2
+from tensorflow.core.util.event_pb2 import SessionLog
 from tensorflow.python.client import session as session_lib
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
@@ -35,9 +40,12 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
@@ -45,13 +53,27 @@ from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.summary import summary as summary_lib
+from tensorflow.python.summary import summary_iterator
 from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import basic_session_run_hooks
 from tensorflow.python.training import monitored_session
+from tensorflow.python.training import saver
 from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import training_util
 
 
+def load_eventfile_contents(directory_path):
+  """Returns the contents of the singular event file in the given directory."""
+  writer_cache.FileWriterCache.clear()
+
+  # Get last Event written.
+  event_paths = glob.glob(os.path.join(directory_path, '*tfevent*'))
+  if len(event_paths) != 1:
+    raise AssertionError('Expected one eventfile, got %s' % str(event_paths))
+  result = list(summary_iterator.summary_iterator(event_paths[0]))
+  return result
+
+
 class MockCheckpointSaverListener(
     basic_session_run_hooks.CheckpointSaverListener):
 
@@ -717,11 +739,12 @@ class CheckpointSaverHookTest(test.TestCase):
                          checkpoint_utils.load_variable(self.model_dir,
                                                         self.global_step.name))
 
-  def test_summary_writer_defs(self):
-    fake_summary_writer.FakeSummaryWriter.install()
-    writer_cache.FileWriterCache.clear()
-    summary_writer = writer_cache.FileWriterCache.get(self.model_dir)
+  def _assertCheckpointEvent(self, event, step, checkpoint_path):
+    self.assertEqual(step, event.step)
+    self.assertEqual(SessionLog.CHECKPOINT, event.session_log.status)
+    self.assertEqual(checkpoint_path, event.session_log.checkpoint_path)
 
+  def test_summary_writer_defs(self):
     with self.graph.as_default():
       hook = basic_session_run_hooks.CheckpointSaverHook(
           self.model_dir, save_steps=2, scaffold=self.scaffold)
@@ -730,18 +753,40 @@ class CheckpointSaverHookTest(test.TestCase):
       with session_lib.Session() as sess:
         sess.run(self.scaffold.init_op)
         mon_sess = monitored_session._HookedSession(sess, [hook])
-        hook.after_create_session(sess, None)
-        mon_sess.run(self.train_op)
-      summary_writer.assert_summaries(
-          test_case=self,
-          expected_logdir=self.model_dir,
-          expected_added_meta_graphs=[
-              meta_graph.create_meta_graph_def(
-                  graph_def=self.graph.as_graph_def(add_shapes=True),
-                  saver_def=self.scaffold.saver.saver_def)
-          ])
-
-    fake_summary_writer.FakeSummaryWriter.uninstall()
+        hook.after_create_session(sess, None)  # Checkpoint saved at step 0.
+        expected_graph_def = self.graph.as_graph_def(add_shapes=True)
+        expected_meta_graph_def = meta_graph.create_meta_graph_def(
+            graph_def=expected_graph_def,
+            saver_def=self.scaffold.saver.saver_def)
+        mon_sess.run(self.train_op)  # No checkpoint saved at step 1.
+        mon_sess.run(self.train_op)  # Checkpoint saved at step 2.
+        mon_sess.run(self.train_op)  # No checkpoint saved at step 3.
+        hook.end(sess)    # Checkpoint saved at the last step (3)
+    events = iter(load_eventfile_contents(self.model_dir))
+    next(events)  # Skip version event that's always there.
+
+    # Graph.
+    event = next(events)
+    self.assertEqual(0, event.step)
+    actual_graph_def = graph_pb2.GraphDef()
+    actual_graph_def.ParseFromString(event.graph_def)
+    test_util.assert_equal_graph_def(actual_graph_def, expected_graph_def)
+
+    # Metagraph.
+    event = next(events)
+    self.assertEqual(0, event.step)
+    actual_meta_graph_def = meta_graph_pb2.MetaGraphDef()
+    actual_meta_graph_def.ParseFromString(event.meta_graph_def)
+    test_util.assert_meta_graph_protos_equal(
+        self, expected_meta_graph_def, actual_meta_graph_def)
+
+    # Checkpoints.
+    # Strip the "-step#" suffix off the latest checkpoint to get base path.
+    checkpoint_path = saver.latest_checkpoint(self.model_dir).rsplit('-', 1)[0]
+    self._assertCheckpointEvent(next(events), 0, checkpoint_path)
+    self._assertCheckpointEvent(next(events), 2, checkpoint_path)
+    self._assertCheckpointEvent(next(events), 3, checkpoint_path)
+    self.assertRaises(StopIteration, lambda: next(events))  # No more events.
 
   def test_save_checkpoint_before_first_train_step(self):
     with self.graph.as_default():
@@ -1102,167 +1147,305 @@ class StepCounterHookTest(test.TestCase):
         self.assertEqual('global_step/sec', summary_value.tag)
         self.assertGreater(summary_value.simple_value, 0)
 
+  def test_summary_writer(self):
+    with ops.Graph().as_default(), session_lib.Session() as sess:
+      variables.get_or_create_global_step()
+      train_op = training_util._increment_global_step(1)
+      hook = basic_session_run_hooks.StepCounterHook(
+          output_dir=self.log_dir, every_n_steps=10)
+      hook.begin()
+      sess.run(variables_lib.global_variables_initializer())
+      mon_sess = monitored_session._HookedSession(sess, [hook])
+      for _ in range(30):
+        mon_sess.run(train_op)
+      hook.end(sess)
+    events = iter(load_eventfile_contents(self.log_dir))
+    next(events)  # Skip version event that's always there.
+
+    event = next(events)
+    self.assertEqual(11, event.step)
+    self.assertEqual('global_step/sec', event.summary.value[0].tag)
+    self.assertLess(0, event.summary.value[0].simple_value)
 
-class SummarySaverHookTest(test.TestCase):
+    event = next(events)
+    self.assertEqual(21, event.step)
+    self.assertEqual('global_step/sec', event.summary.value[0].tag)
+    self.assertLess(0, event.summary.value[0].simple_value)
 
-  def setUp(self):
-    test.TestCase.setUp(self)
+    self.assertRaises(StopIteration, lambda: next(events))  # No more events.
 
-    self.log_dir = 'log/dir'
-    self.summary_writer = fake_summary_writer.FakeSummaryWriter(self.log_dir)
 
-    var = variables_lib.Variable(0.0)
-    tensor = state_ops.assign_add(var, 1.0)
-    tensor2 = tensor * 2
-    self.summary_op = summary_lib.scalar('my_summary', tensor)
-    self.summary_op2 = summary_lib.scalar('my_summary2', tensor2)
+class SummarySaverHookTest(test.TestCase):
 
-    variables.get_or_create_global_step()
-    self.train_op = training_util._increment_global_step(1)
+  def setUp(self):
+    test.TestCase.setUp(self)
+    self.logdir = self.get_temp_dir()
+    self._create_stable_global_step()
+
+  def _create_stable_global_step(self):
+    """Returns a new ResourceVariable global_step for deterministic tests."""
+    # TODO(nickfelt): remove after standard global_step is a ResourceVariable.
+    with ops.get_default_graph().name_scope(None):
+      return variable_scope.get_variable(
+          ops.GraphKeys.GLOBAL_STEP,
+          shape=[],
+          dtype=dtypes.int64,
+          initializer=init_ops.zeros_initializer(),
+          trainable=False,
+          collections=[
+              ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.GLOBAL_STEP
+          ],
+          # Use a ResourceVariable and set caching_device to make the read
+          # behavior deterministic and well-defined.
+          caching_device='cpu:0',
+          use_resource=True)
 
   def test_raise_when_scaffold_and_summary_op_both_missing(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook()
 
   def test_raise_when_scaffold_and_summary_op_both_present(self):
+    summary_op = summary_lib.merge_all()
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook(
-          scaffold=monitored_session.Scaffold(), summary_op=self.summary_op)
+          scaffold=monitored_session.Scaffold(), summary_op=summary_op)
 
-  def test_raise_in_both_secs_and_steps(self):
+  def test_raise_when_secs_and_steps_both_missing(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook(
-          save_secs=10, save_steps=20, summary_writer=self.summary_writer)
+          save_secs=None, save_steps=None, output_dir=self.logdir)
 
-  def test_raise_in_none_secs_and_steps(self):
+  def test_raise_when_secs_and_steps_both_present(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook(
-          save_secs=None, save_steps=None, summary_writer=self.summary_writer)
+          save_secs=10, save_steps=20, output_dir=self.logdir)
 
-  def test_save_steps(self):
-    hook = basic_session_run_hooks.SummarySaverHook(
-        save_steps=8,
-        summary_writer=self.summary_writer,
-        summary_op=self.summary_op)
+  def _makeHook(self, **kwargs):
+    kwargs['output_dir'] = self.logdir
+    kwargs['scaffold'] = monitored_session.Scaffold()
+    return basic_session_run_hooks.SummarySaverHook(**kwargs)
 
+  def _runForSteps(self, hook, steps, loop_body_fn=None):
+    train_op = training_util.get_global_step().assign_add(1)
     with self.test_session() as sess:
       hook.begin()
       sess.run(variables_lib.global_variables_initializer())
+      scaffold = hook._scaffold  # pylint: disable=protected-access
+      if scaffold is not None:
+        scaffold.finalize()
+        sess.run(scaffold.init_op)
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      for _ in range(30):
-        mon_sess.run(self.train_op)
+      for _ in range(steps):
+        mon_sess.run(train_op)
+        if loop_body_fn is not None:
+          loop_body_fn()
       hook.end(sess)
 
-    self.summary_writer.assert_summaries(
-        test_case=self,
-        expected_logdir=self.log_dir,
-        expected_summaries={
-            1: {
-                'my_summary': 1.0
-            },
-            9: {
-                'my_summary': 2.0
-            },
-            17: {
-                'my_summary': 3.0
-            },
-            25: {
-                'my_summary': 4.0
-            },
-        })
+  def _assertSessionEvent(self, event, step, session_status):
+    self.assertEqual(step, event.step)
+    self.assertEqual(session_status, event.session_log.status)
+
+  def _assertSummaryEvent(self, event, step, tag_value_list):
+    self.assertEqual(step, event.step)
+    tag_value_actual_list = [
+        (value.tag, value.simple_value) for value in event.summary.value
+    ]
+    self.assertItemsEqual(tag_value_list, tag_value_actual_list)
+
+  def test_no_summaries(self):
+    hook = self._makeHook(save_steps=1)
+    self._runForSteps(hook, 3)
+    events = iter(load_eventfile_contents(self.logdir))
+    next(events)  # Skip version event that's always there.
+    self._assertSessionEvent(next(events), 0, SessionLog.START)
+    self.assertRaises(StopIteration, lambda: next(events))
+
+  def test_basic_summaries(self):
+    summary_lib.scalar('foo-v1', 1.0)
+    with summary_ops_v2.create_file_writer(self.logdir).as_default():
+      with summary_ops_v2.always_record_summaries():
+        summary_ops_v2.scalar('foo-v2', 2.0)
+    hook = self._makeHook(save_steps=1)
+    self._runForSteps(hook, 3)
+    events = iter(load_eventfile_contents(self.logdir))
+    next(events)  # Skip version event that's always there.
+    self._assertSessionEvent(next(events), 0, SessionLog.START)
+
+    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 1, [('foo-v1', 1.0)])
+
+    self._assertSummaryEvent(next(events), 1, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 2, [('foo-v1', 1.0)])
+
+    self._assertSummaryEvent(next(events), 2, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 3, [('foo-v1', 1.0)])
+    self.assertRaises(StopIteration, lambda: next(events))
 
   def test_multiple_summaries(self):
-    hook = basic_session_run_hooks.SummarySaverHook(
-        save_steps=8,
-        summary_writer=self.summary_writer,
-        summary_op=[self.summary_op, self.summary_op2])
-
+    summary_lib.scalar('foo-v1', 1.0)
+    summary_lib.scalar('bar-v1', 10.0)
+    with summary_ops_v2.create_file_writer(self.logdir).as_default():
+      with summary_ops_v2.always_record_summaries():
+        foo = summary_ops_v2.scalar('foo-v2', 2.0)
+        # Ensure deterministic write order
+        with ops.control_dependencies([foo]):
+          summary_ops_v2.scalar('bar-v2', 20.0)
+    hook = self._makeHook(save_steps=1)
+    self._runForSteps(hook, 1)
+    events = iter(load_eventfile_contents(self.logdir))
+    next(events)  # Skip version event that's always there.
+    self._assertSessionEvent(next(events), 0, SessionLog.START)
+    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 0, [('bar-v2', 20.0)])
+    self._assertSummaryEvent(
+        next(events), 1, [('foo-v1', 1.0), ('bar-v1', 10.0)])
+    self.assertRaises(StopIteration, lambda: next(events))
+
+  def test_v2_summaries_only(self):
+    with summary_ops_v2.create_file_writer(self.logdir).as_default():
+      with summary_ops_v2.always_record_summaries():
+        summary_ops_v2.scalar('foo-v2', 2.0)
+    hook = self._makeHook(save_steps=1)
+    self._runForSteps(hook, 1)
+    events = iter(load_eventfile_contents(self.logdir))
+    next(events)  # Skip version event that's always there.
+    self._assertSessionEvent(next(events), 0, SessionLog.START)
+    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
+    self.assertRaises(StopIteration, lambda: next(events))
+
+  def test_v2_summaries_custom_file_writer(self):
+    other_dir = os.path.join(self.logdir, 'other')
+    other_writer = summary_ops_v2.create_file_writer(other_dir)
+    # SummarySaverHook only flushes the writer for logdir; this one needs to be
+    # manually flushed.
+    flush_op = other_writer.flush()
+    with summary_ops_v2.always_record_summaries():
+      with summary_ops_v2.create_file_writer(self.logdir).as_default():
+        summary_ops_v2.scalar('foo-v2', 2.0)
+      with other_writer.as_default():
+        summary_ops_v2.scalar('other-v2', 3.0)
+    hook = self._makeHook(save_steps=1)
+    self._runForSteps(hook, 1)
     with self.test_session() as sess:
-      hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
-      mon_sess = monitored_session._HookedSession(sess, [hook])
-      for _ in range(10):
-        mon_sess.run(self.train_op)
-      hook.end(sess)
+      sess.run(flush_op)
 
-    self.summary_writer.assert_summaries(
-        test_case=self,
-        expected_logdir=self.log_dir,
-        expected_summaries={
-            1: {
-                'my_summary': 1.0,
-                'my_summary2': 2.0
-            },
-            9: {
-                'my_summary': 2.0,
-                'my_summary2': 4.0
-            },
-        })
+    events = iter(load_eventfile_contents(self.logdir))
+    next(events)  # Skip version event that's always there.
+    self._assertSessionEvent(next(events), 0, SessionLog.START)
+    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
+    self.assertRaises(StopIteration, lambda: next(events))
 
-  def test_save_secs_saving_once_every_step(self):
-    hook = basic_session_run_hooks.SummarySaverHook(
-        save_secs=0.5,
-        summary_writer=self.summary_writer,
-        summary_op=self.summary_op)
+    events = iter(load_eventfile_contents(other_dir))
+    next(events)  # Skip version event that's always there.
+    self._assertSummaryEvent(next(events), 0, [('other-v2', 3.0)])
+    self.assertRaises(StopIteration, lambda: next(events))
 
-    with self.test_session() as sess:
-      hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
-      mon_sess = monitored_session._HookedSession(sess, [hook])
-      for _ in range(4):
-        mon_sess.run(self.train_op)
-        time.sleep(0.5)
-      hook.end(sess)
+  def test_save_steps(self):
+    summary_lib.scalar('foo-v1', 1.0)
+    placeholder = array_ops.placeholder_with_default(False, shape=[])
+    with summary_ops_v2.create_file_writer(self.logdir).as_default():
+      with summary_ops_v2.record_summaries_if(placeholder):
+        summary_ops_v2.scalar('foo-v2', 2.0)
 
-    self.summary_writer.assert_summaries(
-        test_case=self,
-        expected_logdir=self.log_dir,
-        expected_summaries={
-            1: {
-                'my_summary': 1.0
-            },
-            2: {
-                'my_summary': 2.0
-            },
-            3: {
-                'my_summary': 3.0
-            },
-            4: {
-                'my_summary': 4.0
-            },
-        })
+    basic_session_run_hooks.SummarySaverHook._set_placeholder(placeholder)
+    hook = self._makeHook(save_steps=8)
+    self._runForSteps(hook, 30)
+
+    events = load_eventfile_contents(self.logdir)
+    print('TEST SAVE STEPS EVENTS', str(events), file=sys.stderr)
+    events = iter(events)
+    next(events)  # Skip version event that's always there.
+    self._assertSessionEvent(next(events), 0, SessionLog.START)
+
+    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 1, [('foo-v1', 1.0)])
+
+    self._assertSummaryEvent(next(events), 8, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 9, [('foo-v1', 1.0)])
+
+    self._assertSummaryEvent(next(events), 16, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 17, [('foo-v1', 1.0)])
+
+    self._assertSummaryEvent(next(events), 24, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 25, [('foo-v1', 1.0)])
+    self.assertRaises(StopIteration, lambda: next(events))
 
   @test.mock.patch.object(time, 'time')
-  def test_save_secs_saving_once_every_three_steps(self, mock_time):
-    mock_time.return_value = 1484695987.209386
-    hook = basic_session_run_hooks.SummarySaverHook(
-        save_secs=9.,
-        summary_writer=self.summary_writer,
-        summary_op=self.summary_op)
+  def test_save_secs_saving_once_every_step(self, mock_time):
+    mock_time.return_value = 1000.0
+    summary_lib.scalar('foo-v1', 1.0)
+    placeholder = array_ops.placeholder_with_default(False, shape=[])
+    with summary_ops_v2.create_file_writer(self.logdir).as_default():
+      with summary_ops_v2.record_summaries_if(placeholder):
+        summary_ops_v2.scalar('foo-v2', 2.0)
 
-    with self.test_session() as sess:
-      hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
-      mon_sess = monitored_session._HookedSession(sess, [hook])
-      for _ in range(8):
-        mon_sess.run(self.train_op)
-        mock_time.return_value += 3.1
-      hook.end(sess)
+    basic_session_run_hooks.SummarySaverHook._set_placeholder(placeholder)
+    hook = self._makeHook(save_secs=0.5)
+    def fake_sleep():
+      mock_time.return_value += 0.5
+    self._runForSteps(hook, 4, fake_sleep)
+
+    events = iter(load_eventfile_contents(self.logdir))
+    next(events)  # Skip version event that's always there.
+    self._assertSessionEvent(next(events), 0, SessionLog.START)
+
+    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 1, [('foo-v1', 1.0)])
+
+    self._assertSummaryEvent(next(events), 1, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 2, [('foo-v1', 1.0)])
+
+    self._assertSummaryEvent(next(events), 2, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 3, [('foo-v1', 1.0)])
+
+    self._assertSummaryEvent(next(events), 3, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 4, [('foo-v1', 1.0)])
+    self.assertRaises(StopIteration, lambda: next(events))
+
+  @test.mock.patch.object(time, 'time')
+  def test_save_secs_saving_once_every_three_steps(self, mock_time):
+    mock_time.return_value = 1000.0
+    summary_lib.scalar('foo-v1', 1.0)
+    placeholder = array_ops.placeholder_with_default(False, shape=[])
+    with summary_ops_v2.create_file_writer(self.logdir).as_default():
+      with summary_ops_v2.record_summaries_if(placeholder):
+        summary_ops_v2.scalar('foo-v2', 2.0)
+
+    basic_session_run_hooks.SummarySaverHook._set_placeholder(placeholder)
+    hook = self._makeHook(save_secs=9)
+    def fake_sleep():
+      mock_time.return_value += 3.1
+    self._runForSteps(hook, 8, fake_sleep)
+
+    events = iter(load_eventfile_contents(self.logdir))
+    next(events)  # Skip version event that's always there.
+    self._assertSessionEvent(next(events), 0, SessionLog.START)
 
     # 24.8 seconds passed (3.1*8), it saves every 9 seconds starting from first:
-    self.summary_writer.assert_summaries(
+    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 1, [('foo-v1', 1.0)])
+
+    self._assertSummaryEvent(next(events), 3, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 4, [('foo-v1', 1.0)])
+
+    self._assertSummaryEvent(next(events), 6, [('foo-v2', 2.0)])
+    self._assertSummaryEvent(next(events), 7, [('foo-v1', 1.0)])
+    self.assertRaises(StopIteration, lambda: next(events))
+
+  def test_explicit_summary_writer_and_op(self):
+    summary_writer = fake_summary_writer.FakeSummaryWriter(self.logdir)
+    hook = basic_session_run_hooks.SummarySaverHook(
+        save_steps=1,
+        summary_writer=summary_writer,
+        summary_op=summary_lib.scalar('foo-v1', 1.0))
+    self._runForSteps(hook, 3)
+    summary_writer.assert_summaries(
         test_case=self,
-        expected_logdir=self.log_dir,
+        expected_logdir=self.logdir,
         expected_summaries={
-            1: {
-                'my_summary': 1.0
-            },
-            4: {
-                'my_summary': 2.0
-            },
-            7: {
-                'my_summary': 3.0
-            },
+            1: {'foo-v1': 1.0},
+            2: {'foo-v1': 1.0},
+            3: {'foo-v1': 1.0},
         })
 
 
@@ -1518,18 +1701,23 @@ class ProfilerHookTest(test.TestCase):
         sess.run(self.train_op)  # Saved.
         self.assertEqual(3, self._count_timeline_files())
 
-  def test_run_metadata_saves_in_first_step(self):
-    writer_cache.FileWriterCache.clear()
-    fake_summary_writer.FakeSummaryWriter.install()
-    fake_writer = writer_cache.FileWriterCache.get(self.output_dir)
+  def test_run_metadata_summary_saving(self):
     with self.graph.as_default():
       hook = basic_session_run_hooks.ProfilerHook(
-          save_secs=2, output_dir=self.output_dir)
+          save_steps=2, output_dir=self.output_dir)
       with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
         sess.run(self.train_op)  # Saved.
-        self.assertEqual(
-            list(fake_writer._added_run_metadata.keys()), ['step_1'])
-    fake_summary_writer.FakeSummaryWriter.uninstall()
+        sess.run(self.train_op)  # Not saved.
+        sess.run(self.train_op)  # Saved.
+    events = iter(load_eventfile_contents(self.output_dir))
+    next(events)  # Skip version event that's always there.
+    event = next(events)
+    self.assertEqual(1, event.step)
+    self.assertEqual('step_1', event.tagged_run_metadata.tag)
+    event = next(events)
+    self.assertEqual(3, event.step)
+    self.assertEqual('step_3', event.tagged_run_metadata.tag)
+    self.assertRaises(StopIteration, lambda: next(events))  # No more events.
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index 7b06bffa4b..8a4ca04b1e 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -31,6 +31,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import resources
+from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.summary import summary
@@ -204,13 +205,17 @@ class Scaffold(object):
           'local_init_op', ops.GraphKeys.LOCAL_INIT_OP,
           Scaffold.default_local_init_op)
     if self._summary_op is None:
+      def default_summary_op():
+        v1_op = summary.merge_all()
+        v2_ops = summary_ops_v2.all_summary_ops() or []
+        if v1_op is not None:
+          return control_flow_ops.with_dependencies(v2_ops, v1_op)
+        return control_flow_ops.group(v2_ops) if v2_ops else None
       self._summary_op = Scaffold.get_or_default('summary_op',
                                                  ops.GraphKeys.SUMMARY_OP,
-                                                 summary.merge_all)
-    # pylint: disable=g-long-lambda
+                                                 default_summary_op)
     if self._saver is None:
       self._saver = training_saver._get_saver_or_default()  # pylint: disable=protected-access
-    # pylint: enable=g-long-lambda
     self._saver.build()
 
     ops.get_default_graph().finalize()
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index f75db08059..b9d42b034e 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -611,10 +611,8 @@ class Optimizer(
             if isinstance(global_step, resource_variable_ops.ResourceVariable):
               # TODO(apassos): the implicit read in assign_add is slow; consider
               # making it less so.
-              apply_updates = resource_variable_ops.assign_add_variable_op(
-                  global_step.handle,
-                  ops.convert_to_tensor(1, dtype=global_step.dtype),
-                  name=name)
+              apply_updates = global_step.assign_add(
+                  1, name=name, read_value=False)
             else:
               apply_updates = state_ops.assign_add(global_step, 1, name=name)
 
-- 
cgit v1.2.3


From a45ffbd9b5c7d8fdaae6e41432f916639bdbe305 Mon Sep 17 00:00:00 2001
From: James Keeling 
Date: Tue, 24 Jul 2018 09:51:13 -0700
Subject: Allow TF_Delete* functions to accept nullptr

The TF_Delete* functions in TensorFlow's C API now safely do nothing when asked to delete a null pointer. This mirrors the behaviour of free in C and delete in C++.

PiperOrigin-RevId: 205844191
---
 tensorflow/c/c_api.cc      |  5 +++++
 tensorflow/c/c_api.h       |  1 +
 tensorflow/c/c_api_test.cc | 23 +++++++++++++++++++++++
 3 files changed, 29 insertions(+)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 1b937883c8..f516ce4f18 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -328,6 +328,7 @@ TF_Buffer* TF_NewBufferFromString(const void* proto, size_t proto_len) {
 }
 
 void TF_DeleteBuffer(TF_Buffer* buffer) {
+  if (buffer == nullptr) return;
   if (buffer->data_deallocator != nullptr) {
     (*buffer->data_deallocator)(const_cast(buffer->data),
                                 buffer->length);
@@ -357,6 +358,7 @@ void TF_CloseDeprecatedSession(TF_DeprecatedSession* s, TF_Status* status) {
 
 void TF_DeleteDeprecatedSession(TF_DeprecatedSession* s, TF_Status* status) {
   status->status = Status::OK();
+  if (s == nullptr) return;
   delete s->session;
   delete s;
 }
@@ -907,6 +909,7 @@ TF_Library* TF_LoadLibrary(const char* library_filename, TF_Status* status) {
 TF_Buffer TF_GetOpList(TF_Library* lib_handle) { return lib_handle->op_list; }
 
 void TF_DeleteLibraryHandle(TF_Library* lib_handle) {
+  if (lib_handle == nullptr) return;
   tensorflow::port::Free(const_cast(lib_handle->op_list.data));
   delete lib_handle;
 }
@@ -1854,6 +1857,7 @@ TF_Graph::TF_Graph()
 TF_Graph* TF_NewGraph() { return new TF_Graph; }
 
 void TF_DeleteGraph(TF_Graph* g) {
+  if (g == nullptr) return;
   g->mu.lock();
   g->delete_requested = true;
   const bool del = g->sessions.empty();
@@ -2529,6 +2533,7 @@ void TF_CloseSession(TF_Session* s, TF_Status* status) {
 
 void TF_DeleteSession(TF_Session* s, TF_Status* status) {
   status->status = Status::OK();
+  if (s == nullptr) return;
   TF_Graph* const graph = s->graph;
   if (graph != nullptr) {
     graph->mu.lock();
diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index c5035e0e41..c8ae6f2dd1 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -44,6 +44,7 @@ limitations under the License.
 // * size_t is used to represent byte sizes of objects that are
 //   materialized in the address space of the calling process.
 // * int is used as an index into arrays.
+// * Deletion functions are safe to call on nullptr.
 //
 // Questions left to address:
 // * Might at some point need a way for callers to provide their own Env.
diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc
index c470ab5649..e674b1623c 100644
--- a/tensorflow/c/c_api_test.cc
+++ b/tensorflow/c/c_api_test.cc
@@ -1426,6 +1426,29 @@ TEST(CAPI, SavedModelNullArgsAreValid) {
   TF_DeleteStatus(s);
 }
 
+TEST(CAPI, DeletingNullPointerIsSafe) {
+  TF_Status* status = TF_NewStatus();
+
+  TF_DeleteStatus(nullptr);
+  TF_DeleteBuffer(nullptr);
+  TF_DeleteTensor(nullptr);
+  TF_DeleteSessionOptions(nullptr);
+  TF_DeleteGraph(nullptr);
+  TF_DeleteImportGraphDefOptions(nullptr);
+  TF_DeleteImportGraphDefResults(nullptr);
+  TF_DeleteFunction(nullptr);
+  TF_DeleteSession(nullptr, status);
+  EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TF_DeletePRunHandle(nullptr);
+  TF_DeleteDeprecatedSession(nullptr, status);
+  EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
+  TF_DeleteDeviceList(nullptr);
+  TF_DeleteLibraryHandle(nullptr);
+  TF_DeleteApiDefMap(nullptr);
+
+  TF_DeleteStatus(status);
+}
+
 REGISTER_OP("TestOpWithNoGradient")
     .Input("x: T")
     .Output("y: T")
-- 
cgit v1.2.3


From 4883a912780ab5783db654bebc6fc3ac25b63d74 Mon Sep 17 00:00:00 2001
From: Allen Lavoie 
Date: Tue, 24 Jul 2018 11:09:33 -0700
Subject: Add support for custom mapping types to util.nest

Analagous to the existing support for custom collections.Sequence types. They need to be constructable with the same arguments as the base type for pack_sequence_as to work.

Leaves PyDict_* calls for dict subclasses, but adds more general (and likely much slower) fallbacks for instances of collections.Mapping which are not dict subclasses.

My hope is that this support will be enough so I can use a wrapper around dicts which does not inherit from dict in __setattr__ tracking (some tests failed without it). Inheriting from dict and properly shadowing a real dict seems impossible with CPython (since to shadow without synchronization issues, the wrapper needs to respond to updates to the original dict, but to work with e.g. {}.update(dict_subclass) the wrapper's C storage needs to also be updated).

PiperOrigin-RevId: 205858082
---
 tensorflow/python/BUILD             |   1 +
 tensorflow/python/util/nest.py      |  11 +-
 tensorflow/python/util/nest_test.py |  68 ++++++--
 tensorflow/python/util/util.cc      | 299 ++++++++++++++++++++++++++----------
 tensorflow/python/util/util.h       |   4 +-
 tensorflow/python/util/util.i       |   3 +
 6 files changed, 284 insertions(+), 102 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index b5a0051c28..a7c60f5450 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3408,6 +3408,7 @@ py_test(
         ":math_ops",
         ":util",
         "//third_party/py/numpy",
+        "@absl_py//absl/testing:parameterized",
     ],
 )
 
diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index d63f59a8c8..5aac559b9b 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -73,7 +73,7 @@ def _sequence_like(instance, args):
   Returns:
     `args` with the type of `instance`.
   """
-  if isinstance(instance, dict):
+  if isinstance(instance, (dict, _collections.Mapping)):
     # Pack dictionaries in a deterministic order by sorting the keys.
     # Notice this means that we ignore the original order of `OrderedDict`
     # instances. This is intentional, to avoid potential bugs caused by mixing
@@ -89,7 +89,7 @@ def _sequence_like(instance, args):
 
 
 def _yield_value(iterable):
-  if isinstance(iterable, dict):
+  if isinstance(iterable, (dict, _collections.Mapping)):
     # Iterate through dictionaries in a deterministic order by sorting the
     # keys. Notice this means that we ignore the original order of `OrderedDict`
     # instances. This is intentional, to avoid potential bugs caused by mixing
@@ -215,7 +215,7 @@ def flatten_dict_items(dictionary):
     ValueError: If any key and value have not the same structure, or if keys are
       not unique.
   """
-  if not isinstance(dictionary, dict):
+  if not isinstance(dictionary, (dict, _collections.Mapping)):
     raise TypeError("input must be a dictionary")
   flat_dictionary = {}
   for i, v in _six.iteritems(dictionary):
@@ -455,7 +455,7 @@ def assert_shallow_structure(shallow_tree, input_tree, check_types=True):
           "structure has length %s, while shallow structure has length %s."
           % (len(input_tree), len(shallow_tree)))
 
-    if check_types and isinstance(shallow_tree, dict):
+    if check_types and isinstance(shallow_tree, (dict, _collections.Mapping)):
       if set(input_tree) != set(shallow_tree):
         raise ValueError(
             "The two structures don't have the same keys. Input "
@@ -716,7 +716,7 @@ def yield_flat_paths(nest):
 
   # The _maybe_add_final_path_element function is used below in order to avoid
   # adding trailing slashes when the sub-element recursed into is a leaf.
-  if isinstance(nest, dict):
+  if isinstance(nest, (dict, _collections.Mapping)):
     for key in _sorted(nest):
       value = nest[key]
       for sub_path in yield_flat_paths(value):
@@ -760,3 +760,4 @@ def flatten_with_joined_string_paths(structure, separator="/"):
 
 
 _pywrap_tensorflow.RegisterSequenceClass(_collections.Sequence)
+_pywrap_tensorflow.RegisterMappingClass(_collections.Mapping)
diff --git a/tensorflow/python/util/nest_test.py b/tensorflow/python/util/nest_test.py
index 2f12b25354..26c6ea4b01 100644
--- a/tensorflow/python/util/nest_test.py
+++ b/tensorflow/python/util/nest_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import collections
 import time
 
+from absl.testing import parameterized
 import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
@@ -33,7 +34,22 @@ from tensorflow.python.platform import test
 from tensorflow.python.util import nest
 
 
-class NestTest(test.TestCase):
+class _CustomMapping(collections.Mapping):
+
+  def __init__(self, *args, **kwargs):
+    self._wrapped = dict(*args, **kwargs)
+
+  def __getitem__(self, key):
+    return self._wrapped[key]
+
+  def __iter__(self):
+    return iter(self._wrapped)
+
+  def __len__(self):
+    return len(self._wrapped)
+
+
+class NestTest(parameterized.TestCase, test.TestCase):
 
   PointXY = collections.namedtuple("Point", ["x", "y"])  # pylint: disable=invalid-name
 
@@ -72,26 +88,32 @@ class NestTest(test.TestCase):
     with self.assertRaises(ValueError):
       nest.pack_sequence_as([5, 6, [7, 8]], ["a", "b", "c"])
 
+  @parameterized.parameters({"mapping_type": collections.OrderedDict},
+                            {"mapping_type": _CustomMapping})
   @test_util.assert_no_new_pyobjects_executing_eagerly
-  def testFlattenDictOrder(self):
+  def testFlattenDictOrder(self, mapping_type):
     """`flatten` orders dicts by key, including OrderedDicts."""
-    ordered = collections.OrderedDict([("d", 3), ("b", 1), ("a", 0), ("c", 2)])
+    ordered = mapping_type([("d", 3), ("b", 1), ("a", 0), ("c", 2)])
     plain = {"d": 3, "b": 1, "a": 0, "c": 2}
     ordered_flat = nest.flatten(ordered)
     plain_flat = nest.flatten(plain)
     self.assertEqual([0, 1, 2, 3], ordered_flat)
     self.assertEqual([0, 1, 2, 3], plain_flat)
 
-  def testPackDictOrder(self):
+  @parameterized.parameters({"mapping_type": collections.OrderedDict},
+                            {"mapping_type": _CustomMapping})
+  def testPackDictOrder(self, mapping_type):
     """Packing orders dicts by key, including OrderedDicts."""
-    ordered = collections.OrderedDict([("d", 0), ("b", 0), ("a", 0), ("c", 0)])
+    custom = mapping_type([("d", 0), ("b", 0), ("a", 0), ("c", 0)])
     plain = {"d": 0, "b": 0, "a": 0, "c": 0}
     seq = [0, 1, 2, 3]
-    ordered_reconstruction = nest.pack_sequence_as(ordered, seq)
+    custom_reconstruction = nest.pack_sequence_as(custom, seq)
     plain_reconstruction = nest.pack_sequence_as(plain, seq)
+    self.assertIsInstance(custom_reconstruction, mapping_type)
+    self.assertIsInstance(plain_reconstruction, dict)
     self.assertEqual(
-        collections.OrderedDict([("d", 3), ("b", 1), ("a", 0), ("c", 2)]),
-        ordered_reconstruction)
+        mapping_type([("d", 3), ("b", 1), ("a", 0), ("c", 2)]),
+        custom_reconstruction)
     self.assertEqual({"d": 3, "b": 1, "a": 0, "c": 2}, plain_reconstruction)
 
   Abc = collections.namedtuple("A", ("b", "c"))  # pylint: disable=invalid-name
@@ -101,8 +123,10 @@ class NestTest(test.TestCase):
     # A nice messy mix of tuples, lists, dicts, and `OrderedDict`s.
     mess = [
         "z",
-        NestTest.Abc(3, 4),
-        {
+        NestTest.Abc(3, 4), {
+            "d": _CustomMapping({
+                41: 4
+            }),
             "c": [
                 1,
                 collections.OrderedDict([
@@ -111,17 +135,19 @@ class NestTest(test.TestCase):
                 ]),
             ],
             "b": 5
-        },
-        17
+        }, 17
     ]
 
     flattened = nest.flatten(mess)
-    self.assertEqual(flattened, ["z", 3, 4, 5, 1, 2, 3, 17])
+    self.assertEqual(flattened, ["z", 3, 4, 5, 1, 2, 3, 4, 17])
 
     structure_of_mess = [
         14,
         NestTest.Abc("a", True),
         {
+            "d": _CustomMapping({
+                41: 42
+            }),
             "c": [
                 0,
                 collections.OrderedDict([
@@ -142,6 +168,10 @@ class NestTest(test.TestCase):
     self.assertIsInstance(unflattened_ordered_dict, collections.OrderedDict)
     self.assertEqual(list(unflattened_ordered_dict.keys()), ["b", "a"])
 
+    unflattened_custom_mapping = unflattened[2]["d"]
+    self.assertIsInstance(unflattened_custom_mapping, _CustomMapping)
+    self.assertEqual(list(unflattened_custom_mapping.keys()), [41])
+
   def testFlatten_numpyIsNotFlattened(self):
     structure = np.array([1, 2, 3])
     flattened = nest.flatten(structure)
@@ -179,19 +209,23 @@ class NestTest(test.TestCase):
     self.assertFalse(nest.is_sequence(math_ops.tanh(ones)))
     self.assertFalse(nest.is_sequence(np.ones((4, 5))))
 
-  def testFlattenDictItems(self):
-    dictionary = {(4, 5, (6, 8)): ("a", "b", ("c", "d"))}
+  @parameterized.parameters({"mapping_type": _CustomMapping},
+                            {"mapping_type": dict})
+  def testFlattenDictItems(self, mapping_type):
+    dictionary = mapping_type({(4, 5, (6, 8)): ("a", "b", ("c", "d"))})
     flat = {4: "a", 5: "b", 6: "c", 8: "d"}
     self.assertEqual(nest.flatten_dict_items(dictionary), flat)
 
     with self.assertRaises(TypeError):
       nest.flatten_dict_items(4)
 
-    bad_dictionary = {(4, 5, (4, 8)): ("a", "b", ("c", "d"))}
+    bad_dictionary = mapping_type({(4, 5, (4, 8)): ("a", "b", ("c", "d"))})
     with self.assertRaisesRegexp(ValueError, "not unique"):
       nest.flatten_dict_items(bad_dictionary)
 
-    another_bad_dictionary = {(4, 5, (6, 8)): ("a", "b", ("c", ("d", "e")))}
+    another_bad_dictionary = mapping_type({
+        (4, 5, (6, 8)): ("a", "b", ("c", ("d", "e")))
+    })
     with self.assertRaisesRegexp(
         ValueError, "Key had [0-9]* elements, but value had [0-9]* elements"):
       nest.flatten_dict_items(another_bad_dictionary)
diff --git a/tensorflow/python/util/util.cc b/tensorflow/python/util/util.cc
index 366f8a0deb..f9e0b7e4d2 100644
--- a/tensorflow/python/util/util.cc
+++ b/tensorflow/python/util/util.cc
@@ -31,6 +31,8 @@ namespace {
 
 // Type object for collections.Sequence. This is set by RegisterSequenceClass.
 PyObject* CollectionsSequenceType = nullptr;
+// Type object for collections.Mapping, set by RegisterMappingClass.
+PyObject* CollectionsMappingType = nullptr;
 PyTypeObject* SparseTensorValueType = nullptr;
 
 const int kMaxItemsInCache = 1024;
@@ -45,6 +47,23 @@ bool IsString(PyObject* o) {
          PyUnicode_Check(o);
 }
 
+// Work around a writable-strings warning with Python 2's PyMapping_Keys macro,
+// and while we're at it give them consistent behavior by making sure the
+// returned value is a list.
+//
+// As with PyMapping_Keys, returns a new reference.
+PyObject* MappingKeys(PyObject* o) {
+#if PY_MAJOR_VERSION >= 3
+  return PyMapping_Keys(o);
+#else
+  static char key_method_name[] = "keys";
+  Safe_PyObjectPtr raw_result(PyObject_CallMethod(o, key_method_name, nullptr));
+  return PySequence_Fast(
+      raw_result.get(),
+      "The '.keys()' method of a custom mapping returned a non-sequence.");
+#endif
+}
+
 // Equivalent to Python's 'o.__class__.__name__'
 // Note that '__class__' attribute is set only in new-style classes.
 // A lot of tensorflow code uses __class__ without checks, so it seems like
@@ -85,6 +104,119 @@ string PyObjectToString(PyObject* o) {
   }
 }
 
+class CachedTypeCheck {
+ public:
+  explicit CachedTypeCheck(std::function ternary_predicate)
+      : ternary_predicate_(std::move(ternary_predicate)) {}
+
+  ~CachedTypeCheck() {
+    mutex_lock l(type_to_sequence_map_mu_);
+    for (const auto& pair : type_to_sequence_map_) {
+      Py_DECREF(pair.first);
+    }
+  }
+
+  // Caches successful executions of the one-argument (PyObject*) callable
+  // "ternary_predicate" based on the type of "o". -1 from the callable
+  // indicates an unsuccessful check (not cached), 0 indicates that "o"'s type
+  // does not match the predicate, and 1 indicates that it does. Used to avoid
+  // calling back into Python for expensive isinstance checks.
+  int CachedLookup(PyObject* o) {
+    // Try not to return to Python - see if the type has already been seen
+    // before.
+
+    auto* type = Py_TYPE(o);
+
+    {
+      mutex_lock l(type_to_sequence_map_mu_);
+      auto it = type_to_sequence_map_.find(type);
+      if (it != type_to_sequence_map_.end()) {
+        return it->second;
+      }
+    }
+
+    int check_result = ternary_predicate_(o);
+
+    if (check_result == -1) {
+      return -1;  // Type check error, not cached.
+    }
+
+    // NOTE: This is never decref'd as long as the object lives, which is likely
+    // forever, but we don't want the type to get deleted as long as it is in
+    // the map. This should not be too much of a leak, as there should only be a
+    // relatively small number of types in the map, and an even smaller number
+    // that are eligible for decref. As a precaution, we limit the size of the
+    // map to 1024.
+    {
+      mutex_lock l(type_to_sequence_map_mu_);
+      if (type_to_sequence_map_.size() < kMaxItemsInCache) {
+        Py_INCREF(type);
+        type_to_sequence_map_.insert({type, check_result});
+      }
+    }
+
+    return check_result;
+  }
+
+ private:
+  std::function ternary_predicate_;
+  mutex type_to_sequence_map_mu_;
+  std::unordered_map type_to_sequence_map_
+      GUARDED_BY(type_to_sequence_map_mu_);
+};
+
+// Returns 1 if `o` is considered a mapping for the purposes of Flatten().
+// Returns 0 otherwise.
+// Returns -1 if an error occurred.
+int IsMappingHelper(PyObject* o) {
+  static auto* const check_cache = new CachedTypeCheck([](PyObject* to_check) {
+    return PyObject_IsInstance(to_check, CollectionsMappingType);
+  });
+  if (PyDict_Check(o)) return true;
+  if (TF_PREDICT_FALSE(CollectionsMappingType == nullptr)) {
+    PyErr_SetString(
+        PyExc_RuntimeError,
+        tensorflow::strings::StrCat(
+            "collections.Mapping type has not been set. "
+            "Please call RegisterMappingClass before using this module")
+            .c_str());
+    return -1;
+  }
+  return check_cache->CachedLookup(o);
+}
+
+// Returns 1 if `o` is considered a sequence for the purposes of Flatten().
+// Returns 0 otherwise.
+// Returns -1 if an error occurred.
+int IsSequenceHelper(PyObject* o) {
+  static auto* const check_cache = new CachedTypeCheck([](PyObject* to_check) {
+    int is_instance = PyObject_IsInstance(to_check, CollectionsSequenceType);
+
+    // Don't cache a failed is_instance check.
+    if (is_instance == -1) return -1;
+
+    return static_cast(is_instance != 0 && !IsString(to_check));
+  });
+  // We treat dicts and other mappings as special cases of sequences.
+  if (IsMappingHelper(o)) return true;
+  if (PySet_Check(o) && !WarnedThatSetIsNotSequence) {
+    LOG(WARNING) << "Sets are not currently considered sequences, "
+                    "but this may change in the future, "
+                    "so consider avoiding using them.";
+    WarnedThatSetIsNotSequence = true;
+  }
+  if (TF_PREDICT_FALSE(CollectionsSequenceType == nullptr)) {
+    PyErr_SetString(
+        PyExc_RuntimeError,
+        tensorflow::strings::StrCat(
+            "collections.Sequence type has not been set. "
+            "Please call RegisterSequenceClass before using this module")
+            .c_str());
+    return -1;
+  }
+  return check_cache->CachedLookup(o);
+}
+
 // Implements the same idea as tensorflow.util.nest._yield_value
 // During construction we check if the iterable is a dictionary.
 // If so, we construct a sequence from its sorted keys that will be used
@@ -96,7 +228,12 @@ string PyObjectToString(PyObject* o) {
 // 'iterable' must not be modified while ValIterator is used.
 class ValIterator {
  public:
-  explicit ValIterator(PyObject* iterable) : dict_(nullptr), index_(0) {
+  explicit ValIterator(PyObject* iterable)
+      : dict_(nullptr),
+        mapping_(nullptr),
+        last_mapping_element_(nullptr),
+        seq_(nullptr),
+        index_(0) {
     if (PyDict_Check(iterable)) {
       dict_ = iterable;
       // PyDict_Keys returns a list, which can be used with
@@ -108,6 +245,10 @@ class ValIterator {
       // bugs caused by mixing ordered and plain dicts (e.g., flattening
       // a dict but using a corresponding `OrderedDict` to pack it back).
       PyList_Sort(seq_);
+    } else if (IsMappingHelper(iterable)) {
+      mapping_ = iterable;
+      seq_ = MappingKeys(iterable);
+      PyList_Sort(seq_);
     } else {
       seq_ = PySequence_Fast(iterable, "");
     }
@@ -122,7 +263,9 @@ class ValIterator {
     PyObject* element = nullptr;
     if (index_ < size_) {
       // Both PySequence_Fast_GET_ITEM and PyDict_GetItem return borrowed
-      // references.
+      // references. For general mappings, ValIterator keeps a reference to the
+      // last retrieved element (and decrefs it before producing the next
+      // element) to abstract away the borrowed/new difference.
       element = PySequence_Fast_GET_ITEM(seq_, index_);
       ++index_;
       if (dict_ != nullptr) {
@@ -132,85 +275,32 @@ class ValIterator {
                           "Dictionary was modified during iteration over it");
           return nullptr;
         }
+      } else if (mapping_ != nullptr) {
+        element = PyObject_GetItem(mapping_, element);
+        if (element == nullptr) {
+          PyErr_SetString(PyExc_RuntimeError,
+                          "Mapping was modified during iteration over it");
+          return nullptr;
+        }
+        last_mapping_element_.reset(element);
       }
     }
     return element;
   }
 
  private:
-  PyObject* seq_;
+  // Special casing for things that pass PyDict_Check (faster, no Python calls)
   PyObject* dict_;
+
+  // General mappings which have custom Python logic
+  PyObject* mapping_;
+  Safe_PyObjectPtr last_mapping_element_;
+
+  PyObject* seq_;
   Py_ssize_t size_;
   Py_ssize_t index_;
 };
 
-mutex g_type_to_sequence_map(LINKER_INITIALIZED);
-std::unordered_map* IsTypeSequenceMap() {
-  static auto* const m = new std::unordered_map;
-  return m;
-}
-
-// Returns 1 if `o` is considered a sequence for the purposes of Flatten().
-// Returns 0 otherwise.
-// Returns -1 if an error occurred.
-int IsSequenceHelper(PyObject* o) {
-  if (PyDict_Check(o)) return true;
-  if (PySet_Check(o) && !WarnedThatSetIsNotSequence) {
-    LOG(WARNING) << "Sets are not currently considered sequences, "
-                    "but this may change in the future, "
-                    "so consider avoiding using them.";
-    WarnedThatSetIsNotSequence = true;
-  }
-  if (TF_PREDICT_FALSE(CollectionsSequenceType == nullptr)) {
-    PyErr_SetString(
-        PyExc_RuntimeError,
-        tensorflow::strings::StrCat(
-            "collections.Sequence type has not been set. "
-            "Please call RegisterSequenceClass before using this module")
-            .c_str());
-    return -1;
-  }
-
-  // Try not to return to Python - see if the type has already been seen
-  // before.
-
-  auto* type_to_sequence_map = IsTypeSequenceMap();
-  auto* type = Py_TYPE(o);
-
-  {
-    mutex_lock l(g_type_to_sequence_map);
-    auto it = type_to_sequence_map->find(type);
-    if (it != type_to_sequence_map->end()) {
-      return it->second;
-    }
-  }
-
-  // NOTE: We explicitly release the g_type_to_sequence_map mutex,
-  // because PyObject_IsInstance() may release the GIL, allowing another thread
-  // concurrent entry to this function.
-  int is_instance = PyObject_IsInstance(o, CollectionsSequenceType);
-
-  // Don't cache a failed is_instance check.
-  if (is_instance == -1) return -1;
-
-  bool is_sequence = static_cast(is_instance != 0 && !IsString(o));
-
-  // NOTE: This is never decref'd, but we don't want the type to get deleted
-  // as long as it is in the map. This should not be too much of a
-  // leak, as there should only be a relatively small number of types in the
-  // map, and an even smaller number that are eligible for decref. As a
-  // precaution, we limit the size of the map to 1024.
-  {
-    mutex_lock l(g_type_to_sequence_map);
-    if (type_to_sequence_map->size() < kMaxItemsInCache) {
-      Py_INCREF(type);
-      type_to_sequence_map->insert({type, is_sequence});
-    }
-  }
-
-  return is_sequence;
-}
-
 bool IsSparseTensorValueType(PyObject* o) {
   if (TF_PREDICT_FALSE(SparseTensorValueType == nullptr)) {
     return false;
@@ -226,21 +316,35 @@ int IsSequenceForDataHelper(PyObject* o) {
 
 bool GetNextValuesForDict(PyObject* nested,
                           std::vector* next_values) {
-  std::vector result;
-
-  PyObject* keys = PyDict_Keys(nested);
-  if (PyList_Sort(keys) == -1) return false;
-  Py_ssize_t size = PyList_Size(keys);
+  Safe_PyObjectPtr keys(PyDict_Keys(nested));
+  if (PyList_Sort(keys.get()) == -1) return false;
+  Py_ssize_t size = PyList_Size(keys.get());
   for (Py_ssize_t i = 0; i < size; ++i) {
     // We know that key and item will not be deleted because nested owns
     // a reference to them and callers of flatten must not modify nested
     // while the method is running.
-    PyObject* key = PyList_GET_ITEM(keys, i);
+    PyObject* key = PyList_GET_ITEM(keys.get(), i);
     PyObject* item = PyDict_GetItem(nested, key);
     Py_INCREF(item);
     next_values->emplace_back(item);
   }
-  Py_DECREF(keys);
+  return true;
+}
+
+bool GetNextValuesForMapping(PyObject* nested,
+                             std::vector* next_values) {
+  Safe_PyObjectPtr keys(MappingKeys(nested));
+  if (keys.get() == nullptr) {
+    return false;
+  }
+  if (PyList_Sort(keys.get()) == -1) return false;
+  Py_ssize_t size = PyList_Size(keys.get());
+  for (Py_ssize_t i = 0; i < size; ++i) {
+    PyObject* key = PyList_GET_ITEM(keys.get(), i);
+    // Unlike PyDict_GetItem, PyObject_GetItem returns a new reference.
+    PyObject* item = PyObject_GetItem(nested, key);
+    next_values->emplace_back(item);
+  }
   return true;
 }
 
@@ -265,6 +369,9 @@ bool GetNextValues(PyObject* nested,
   if (PyDict_Check(nested)) {
     // if nested is dictionary, sort it by key and recurse on each value
     return GetNextValuesForDict(nested, next_values);
+  } else if (IsMappingHelper(nested)) {
+    // same treatment as dictionaries, but for custom mapping types
+    return GetNextValuesForMapping(nested, next_values);
   }
   // iterate and recurse
   return GetNextValuesForIterable(nested, next_values);
@@ -276,6 +383,9 @@ bool GetNextValuesForData(PyObject* nested,
   if (PyDict_Check(nested)) {
     // if nested is dictionary, sort it by key and recurse on each value
     return GetNextValuesForDict(nested, next_values);
+  } else if (IsMappingHelper(nested)) {
+    // same treatment as dictionaries, but for custom mapping types
+    return GetNextValuesForMapping(nested, next_values);
   } else if (IsSparseTensorValueType(nested)) {
     // if nested is a SparseTensorValue, just return itself as a single item
     Py_INCREF(nested);
@@ -320,8 +430,8 @@ bool FlattenHelper(
 // 'dict1' and 'dict2' are assumed to be Python dictionaries.
 void SetDifferentKeysError(PyObject* dict1, PyObject* dict2, string* error_msg,
                            bool* is_type_error) {
-  PyObject* k1 = PyDict_Keys(dict1);
-  PyObject* k2 = PyDict_Keys(dict2);
+  PyObject* k1 = MappingKeys(dict1);
+  PyObject* k2 = MappingKeys(dict2);
   *is_type_error = false;
   *error_msg = tensorflow::strings::StrCat(
       "The two dictionaries don't have the same set of keys. "
@@ -423,6 +533,24 @@ bool AssertSameStructureHelper(PyObject* o1, PyObject* o2, bool check_types,
           return true;
         }
       }
+    } else if (IsMappingHelper(o1)) {
+      // Fallback for custom mapping types. Instead of using PyDict methods
+      // which stay in C, we call iter(o1).
+      if (PyMapping_Size(o1) != PyMapping_Size(o2)) {
+        SetDifferentKeysError(o1, o2, error_msg, is_type_error);
+        return true;
+      }
+
+      Safe_PyObjectPtr iter(PyObject_GetIter(o1));
+      PyObject* key;
+      while ((key = PyIter_Next(iter.get())) != nullptr) {
+        if (!PyMapping_HasKey(o2, key)) {
+          SetDifferentKeysError(o1, o2, error_msg, is_type_error);
+          Py_DECREF(key);
+          return true;
+        }
+        Py_DECREF(key);
+      }
     }
   }
 
@@ -470,6 +598,19 @@ void RegisterSequenceClass(PyObject* sequence_class) {
   CollectionsSequenceType = sequence_class;
 }
 
+void RegisterMappingClass(PyObject* mapping_class) {
+  if (!PyType_Check(mapping_class)) {
+    PyErr_SetString(
+        PyExc_TypeError,
+        tensorflow::strings::StrCat(
+            "Expecting a class definition for `collections.Mapping`. Got ",
+            Py_TYPE(mapping_class)->tp_name)
+            .c_str());
+    return;
+  }
+  CollectionsMappingType = mapping_class;
+}
+
 void RegisterSparseTensorValueClass(PyObject* sparse_tensor_value_class) {
   if (!PyType_Check(sparse_tensor_value_class)) {
     PyErr_SetString(
diff --git a/tensorflow/python/util/util.h b/tensorflow/python/util/util.h
index 70efc10c9a..41dcc969f8 100644
--- a/tensorflow/python/util/util.h
+++ b/tensorflow/python/util/util.h
@@ -118,7 +118,9 @@ PyObject* Flatten(PyObject* nested);
 // the type from the module. This approach also requires some trigger from
 // Python so that we know that Python interpreter had been initialzied.
 void RegisterSequenceClass(PyObject* sequence_class);
-// Similar to the above function, except for the
+// Like RegisterSequenceClass, but for collections.Mapping.
+void RegisterMappingClass(PyObject* mapping_class);
+// Similar to the above functions, except for the
 // sparse_tensor.SparseTensorValue class.
 void RegisterSparseTensorValueClass(PyObject* sparse_tensor_value_class);
 
diff --git a/tensorflow/python/util/util.i b/tensorflow/python/util/util.i
index 9f3b11b982..6ad1484295 100644
--- a/tensorflow/python/util/util.i
+++ b/tensorflow/python/util/util.i
@@ -31,6 +31,9 @@ limitations under the License.
 %unignore tensorflow::swig::RegisterSequenceClass;
 %noexception tensorflow::swig::RegisterSequenceClass;
 
+%unignore tensorflow::swig::RegisterMappingClass;
+%noexception tensorflow::swig::RegisterMappingClass;
+
 %unignore tensorflow::swig::RegisterSparseTensorValueClass;
 %noexception tensorflow::swig::RegisterSparseTensorValueClass;
 
-- 
cgit v1.2.3


From 0527ba2f447fe0bc20152f393bcd672de0b59548 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan 
Date: Tue, 24 Jul 2018 11:10:08 -0700
Subject: warm start estimator from keras model during model to estimator
 conversion.

PiperOrigin-RevId: 205858208
---
 tensorflow/python/estimator/keras.py      | 112 ++++++++++++++-------
 tensorflow/python/estimator/keras_test.py | 155 ++++++++++++++++++++++++++++--
 2 files changed, 222 insertions(+), 45 deletions(-)

diff --git a/tensorflow/python/estimator/keras.py b/tensorflow/python/estimator/keras.py
index 076359b503..682be8e7cc 100644
--- a/tensorflow/python/estimator/keras.py
+++ b/tensorflow/python/estimator/keras.py
@@ -21,11 +21,14 @@ from __future__ import print_function
 
 import os
 import re
+import tempfile
+
 from tensorflow.python.client import session
 from tensorflow.python.estimator import estimator as estimator_lib
 from tensorflow.python.estimator import export as export_lib
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.estimator import run_config as run_config_lib
+from tensorflow.python.estimator.run_config import RunConfig
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
@@ -39,6 +42,7 @@ from tensorflow.python.keras.utils.generic_utils import CustomObjectScope
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import metrics as metrics_module
+from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.training import distribute as distribute_lib
@@ -426,29 +430,34 @@ def _create_keras_model_fn(keras_model, custom_objects=None):
   return model_fn
 
 
-def _save_first_checkpoint(keras_model, estimator, custom_objects,
-                           keras_weights):
+def _save_first_checkpoint(keras_model, custom_objects, config):
   """Save first checkpoint for the keras Estimator.
 
   Args:
     keras_model: an instance of compiled keras model.
-    estimator: keras estimator.
     custom_objects: Dictionary for custom objects.
-    keras_weights: A flat list of Numpy arrays for weights of given keras_model.
+    config: Estimator config.
 
   Returns:
-    The model_fn for a keras Estimator.
+    The path where keras model checkpoint is saved.
   """
+  # save checkpoint into subdirectory to allow warm start
+  keras_model_dir = os.path.join(config.model_dir, 'keras')
   # Load weights and save to checkpoint if there is no checkpoint
-  latest_path = saver_lib.latest_checkpoint(estimator.model_dir)
+  latest_path = saver_lib.latest_checkpoint(keras_model_dir)
   if not latest_path:
+    keras_weights = None
+    if _any_weight_initialized(keras_model):
+      keras_weights = keras_model.get_weights()
+    if not gfile.IsDirectory(keras_model_dir):
+      gfile.MakeDirs(keras_model_dir)
     with ops.Graph().as_default():
-      random_seed.set_random_seed(estimator.config.tf_random_seed)
+      random_seed.set_random_seed(config.tf_random_seed)
       training_util.create_global_step()
       model = _clone_and_build_model(model_fn_lib.ModeKeys.TRAIN, keras_model,
                                      custom_objects)
       # save to checkpoint
-      with session.Session(config=estimator._session_config) as sess:
+      with session.Session(config=config.session_config) as sess:
         if keras_weights:
           model.set_weights(keras_weights)
         # Make update ops and initialize all variables.
@@ -458,7 +467,46 @@ def _save_first_checkpoint(keras_model, estimator, custom_objects,
           K._initialize_variables(sess)
           # pylint: enable=protected-access
         saver = saver_lib.Saver()
-        saver.save(sess, os.path.join(estimator.model_dir, 'keras_model.ckpt'))
+        latest_path = os.path.join(keras_model_dir, 'keras_model.ckpt')
+        saver.save(sess, latest_path)
+  return latest_path
+
+
+def _maybe_overwrite_model_dir_and_session_config(config, model_dir):
+  """Overwrite estimator config by `model_dir` and `session_config` if needed.
+
+  Args:
+    config: Original estimator config.
+    model_dir: Estimator model checkpoint directory.
+
+  Returns:
+    Overwritten estimator config.
+
+  Raises:
+    ValueError: Model directory inconsistent between `model_dir` and `config`.
+  """
+
+  default_session_config = run_config_lib.get_default_session_config()
+  if isinstance(config, dict):
+    config = RunConfig(**config)
+  elif config is None:
+    config = RunConfig(session_config=default_session_config)
+  if config.session_config is None:
+    config = RunConfig.replace(config, session_config=default_session_config)
+
+  if model_dir is not None:
+    if (getattr(config, 'model_dir', None) is not None and
+        config.model_dir != model_dir):
+      raise ValueError(
+          "`model_dir` are set both in constructor and `RunConfig`, but with "
+          "different values. In constructor: '{}', in `RunConfig`: "
+          "'{}' ".format(model_dir, config.model_dir))
+    config = RunConfig.replace(config, model_dir=model_dir)
+  elif getattr(config, 'model_dir', None) is None:
+    model_dir = tempfile.mkdtemp()
+    config = RunConfig.replace(config, model_dir=model_dir)
+
+  return config
 
 
 def model_to_estimator(keras_model=None,
@@ -517,45 +565,39 @@ def model_to_estimator(keras_model=None,
         'Please compile the model with `model.compile()` '
         'before calling `model_to_estimator()`.')
 
-  if isinstance(config, dict):
-    config = run_config_lib.RunConfig(**config)
+  config = _maybe_overwrite_model_dir_and_session_config(config, model_dir)
 
   keras_model_fn = _create_keras_model_fn(keras_model, custom_objects)
-  estimator = estimator_lib.Estimator(
-      keras_model_fn, model_dir=model_dir, config=config)
-
-  # Check if we need to call get_weights:
   if _any_weight_initialized(keras_model):
-    keras_weights = keras_model.get_weights()
     # Warn if config passed to estimator tries to update GPUOptions. If a
     # session has already been created, the GPUOptions passed to the first
     # session sticks.
-    if estimator._session_config.HasField('gpu_options'):
+    if config.session_config.HasField('gpu_options'):
       logging.warning(
           'The Keras backend session has already been set. '
           'The _session_config passed to model_to_estimator will not be used.')
   else:
     # Pass the config into keras backend's default session.
-    sess = session.Session(config=estimator._session_config)
+    sess = session.Session(config=config.session_config)
     K.set_session(sess)
-    keras_weights = None
 
+  warm_start_path = None
   if keras_model._is_graph_network:
-    # TODO(yifeif): move checkpoint initialization to scaffold.init_fn
-    _save_first_checkpoint(keras_model,
-                           estimator,
-                           custom_objects,
-                           keras_weights)
+    warm_start_path = _save_first_checkpoint(keras_model, custom_objects,
+                                             config)
   elif keras_model.built:
-    logging.warning('You are creating an Estimator from a Keras model '
-                    'manually subclassed from `Model`, that was '
-                    'already called on some inputs (and thus already had '
-                    'weights). We are currently unable to preserve '
-                    'the model\'s state (its weights) '
-                    'as part of the estimator '
-                    'in this case. Be warned that the estimator '
-                    'has been created using '
-                    'a freshly initialized version of your model.\n'
-                    'Note that this doesn\'t affect the state of the '
-                    'model instance you passed as `keras_model` argument.')
+    logging.warning('You are creating an Estimator from a Keras model manually '
+                    'subclassed from `Model`, that was already called on some '
+                    'inputs (and thus already had weights). We are currently '
+                    'unable to preserve the model\'s state (its weights) as '
+                    'part of the estimator in this case. Be warned that the '
+                    'estimator has been created using a freshly initialized '
+                    'version of your model.\n'
+                    'Note that this doesn\'t affect the state of the model '
+                    'instance you passed as `keras_model` argument.')
+
+  estimator = estimator_lib.Estimator(keras_model_fn,
+                                      config=config,
+                                      warm_start_from=warm_start_path)
+
   return estimator
diff --git a/tensorflow/python/estimator/keras_test.py b/tensorflow/python/estimator/keras_test.py
index 7a3c5a9bf1..cf4ec7f4da 100644
--- a/tensorflow/python/estimator/keras_test.py
+++ b/tensorflow/python/estimator/keras_test.py
@@ -33,11 +33,13 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras import testing_utils
 from tensorflow.python.keras.optimizers import SGD
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.parsing_ops import gen_parsing_ops
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import rmsprop
+from tensorflow.python.training import session_run_hook
 
 
 try:
@@ -50,6 +52,8 @@ _TRAIN_SIZE = 200
 _INPUT_SIZE = (10,)
 _NUM_CLASS = 2
 
+_TMP_DIR = '/tmp'
+
 
 def simple_sequential_model():
   model = keras.models.Sequential()
@@ -167,6 +171,12 @@ def multi_inputs_multi_outputs_model():
   return model
 
 
+class MyHook(session_run_hook.SessionRunHook):
+
+  def begin(self):
+    _ = variable_scope.get_variable('temp', [1])
+
+
 class TestKerasEstimator(test_util.TensorFlowTestCase):
 
   def setUp(self):
@@ -203,6 +213,54 @@ class TestKerasEstimator(test_util.TensorFlowTestCase):
       writer_cache.FileWriterCache.clear()
       gfile.DeleteRecursively(self._config.model_dir)
 
+  # see b/109935364
+  @test_util.run_in_graph_and_eager_modes
+  def test_train_with_hooks(self):
+    for model_type in ['sequential', 'functional']:
+      keras_model, (_, _), (
+          _, _), train_input_fn, eval_input_fn = get_resource_for_simple_model(
+              model_type=model_type, is_evaluate=True)
+      keras_model.compile(
+          loss='categorical_crossentropy',
+          optimizer=rmsprop.RMSPropOptimizer(1e-3),
+          metrics=['mse', keras.metrics.categorical_accuracy])
+
+      my_hook = MyHook()
+      with self.test_session():
+        est_keras = keras_lib.model_to_estimator(
+            keras_model=keras_model, config=self._config)
+        before_eval_results = est_keras.evaluate(
+            input_fn=eval_input_fn, steps=1)
+        est_keras.train(input_fn=train_input_fn, hooks=[my_hook],
+                        steps=_TRAIN_SIZE / 16)
+        after_eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1)
+        self.assertLess(after_eval_results['loss'], before_eval_results['loss'])
+
+      writer_cache.FileWriterCache.clear()
+      gfile.DeleteRecursively(self._config.model_dir)
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_train_with_model_fit_and_hooks(self):
+    keras_model, (x_train, y_train), _, \
+      train_input_fn, eval_input_fn = get_resource_for_simple_model(
+          model_type='sequential', is_evaluate=True)
+
+    keras_model.compile(
+        loss='categorical_crossentropy',
+        optimizer=rmsprop.RMSPropOptimizer(1e-3),
+        metrics=['mse', keras.metrics.categorical_accuracy])
+    my_hook = MyHook()
+    with self.test_session():
+      keras_model.fit(x_train, y_train, epochs=1)
+
+      keras_est = keras_lib.model_to_estimator(
+          keras_model=keras_model, config=self._config)
+      before_eval_results = keras_est.evaluate(input_fn=eval_input_fn)
+      keras_est.train(input_fn=train_input_fn, hooks=[my_hook],
+                      steps=_TRAIN_SIZE / 16)
+      after_eval_results = keras_est.evaluate(input_fn=eval_input_fn, steps=1)
+      self.assertLess(after_eval_results['loss'], before_eval_results['loss'])
+
   @test_util.run_in_graph_and_eager_modes
   def test_train_with_tf_optimizer(self):
     for model_type in ['sequential', 'functional']:
@@ -473,27 +531,43 @@ class TestKerasEstimator(test_util.TensorFlowTestCase):
         est_keras.train(input_fn=invald_output_name_input_fn, steps=100)
 
   def test_custom_objects(self):
-    
+
     def relu6(x):
       return keras.backend.relu(x, max_value=6)
-    
+
     keras_model = simple_functional_model(activation=relu6)
     keras_model.compile(loss='categorical_crossentropy', optimizer='adam')
     custom_objects = {
         'relu6': relu6
     }
 
+    (x_train, y_train), _ = testing_utils.get_test_data(
+        train_samples=_TRAIN_SIZE,
+        test_samples=50,
+        input_shape=(10,),
+        num_classes=2)
+    y_train = keras.utils.to_categorical(y_train, 2)
+    input_name = keras_model.input_names[0]
+    output_name = keras_model.output_names[0]
+    train_input_fn = numpy_io.numpy_input_fn(
+        x=randomize_io_type(x_train, input_name),
+        y=randomize_io_type(y_train, output_name),
+        shuffle=False,
+        num_epochs=None,
+        batch_size=16)
     with self.assertRaisesRegexp(ValueError, 'relu6'):
       with self.test_session():
-        keras_lib.model_to_estimator(
+        est = keras_lib.model_to_estimator(
             keras_model=keras_model,
             model_dir=tempfile.mkdtemp(dir=self._base_dir))
+        est.train(input_fn=train_input_fn, steps=1)
 
     with self.test_session():
-      keras_lib.model_to_estimator(
+      est = keras_lib.model_to_estimator(
           keras_model=keras_model,
           model_dir=tempfile.mkdtemp(dir=self._base_dir),
           custom_objects=custom_objects)
+      est.train(input_fn=train_input_fn, steps=1)
 
   def test_tf_config(self):
     keras_model, (_, _), (_, _), _, _ = get_resource_for_simple_model()
@@ -530,12 +604,73 @@ class TestKerasEstimator(test_util.TensorFlowTestCase):
       gpu_options = config_pb2.GPUOptions(per_process_gpu_memory_fraction=0.3)
       sess_config = config_pb2.ConfigProto(gpu_options=gpu_options)
       self._config._session_config = sess_config
-      keras_lib.model_to_estimator(
-          keras_model=keras_model, config=self._config)
-      self.assertEqual(
-          keras.backend.get_session()
-          ._config.gpu_options.per_process_gpu_memory_fraction,
-          gpu_options.per_process_gpu_memory_fraction)
+      with self.test_session():
+        keras_lib.model_to_estimator(
+            keras_model=keras_model, config=self._config)
+        self.assertEqual(
+            keras.backend.get_session()
+            ._config.gpu_options.per_process_gpu_memory_fraction,
+            gpu_options.per_process_gpu_memory_fraction)
+
+  def test_with_empty_config(self):
+    keras_model, _, _, _, _ = get_resource_for_simple_model(
+        model_type='sequential', is_evaluate=True)
+    keras_model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        metrics=['mse', keras.metrics.categorical_accuracy])
+
+    with self.test_session():
+      est_keras = keras_lib.model_to_estimator(
+          keras_model=keras_model, model_dir=self._base_dir,
+          config=run_config_lib.RunConfig())
+      self.assertEqual(run_config_lib.get_default_session_config(),
+                       est_keras._session_config)
+      self.assertEqual(est_keras._session_config,
+                       est_keras._config.session_config)
+      self.assertEqual(self._base_dir, est_keras._config.model_dir)
+      self.assertEqual(self._base_dir, est_keras._model_dir)
+
+    with self.test_session():
+      est_keras = keras_lib.model_to_estimator(
+          keras_model=keras_model, model_dir=self._base_dir,
+          config=None)
+      self.assertEqual(run_config_lib.get_default_session_config(),
+                       est_keras._session_config)
+      self.assertEqual(est_keras._session_config,
+                       est_keras._config.session_config)
+      self.assertEqual(self._base_dir, est_keras._config.model_dir)
+      self.assertEqual(self._base_dir, est_keras._model_dir)
+
+  def test_with_empty_config_and_empty_model_dir(self):
+    keras_model, _, _, _, _ = get_resource_for_simple_model(
+        model_type='sequential', is_evaluate=True)
+    keras_model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        metrics=['mse', keras.metrics.categorical_accuracy])
+
+    with self.test_session():
+      with test.mock.patch.object(tempfile, 'mkdtemp', return_value=_TMP_DIR):
+        est_keras = keras_lib.model_to_estimator(
+            keras_model=keras_model,
+            config=run_config_lib.RunConfig())
+        self.assertEqual(est_keras._model_dir, _TMP_DIR)
+
+  def test_with_conflicting_model_dir_and_config(self):
+    keras_model, _, _, _, _ = get_resource_for_simple_model(
+        model_type='sequential', is_evaluate=True)
+    keras_model.compile(
+        loss='categorical_crossentropy',
+        optimizer='rmsprop',
+        metrics=['mse', keras.metrics.categorical_accuracy])
+
+    with self.test_session():
+      with self.assertRaisesRegexp(ValueError, '`model_dir` are set both in '
+                                   'constructor and `RunConfig`'):
+        keras_lib.model_to_estimator(
+            keras_model=keras_model, model_dir=self._base_dir,
+            config=run_config_lib.RunConfig(model_dir=_TMP_DIR))
 
   def test_pretrained_weights(self):
     keras_model, (_, _), (_, _), _, _ = get_resource_for_simple_model()
-- 
cgit v1.2.3


From aef000ed3c2863a5cc7ccb5bf1fb46116e7f4f02 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy 
Date: Tue, 24 Jul 2018 11:10:46 -0700
Subject: Build more cuda compute capabilities in cmake build.

Fixes #18652

PiperOrigin-RevId: 205858348
---
 tensorflow/contrib/cmake/CMakeLists.txt | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt
index 708618dcb0..6c93487e0d 100644
--- a/tensorflow/contrib/cmake/CMakeLists.txt
+++ b/tensorflow/contrib/cmake/CMakeLists.txt
@@ -394,16 +394,20 @@ if (tensorflow_ENABLE_GPU)
 
   # by default we assume compute cabability 3.5 and 5.2. If you change this change it in
   # CUDA_NVCC_FLAGS and cuda_config.h below
-  set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_30,code=\"sm_30,compute_30\";-gencode arch=compute_35,code=\"sm_35,compute_35\";-gencode arch=compute_52,code=\"sm_52,compute_52\")
+  set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_37,code=\"sm_37,compute_37\")
+  set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_52,code=\"sm_52,compute_52\")
+  set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_60,code=\"sm_60,compute_60\")
+  set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_61,code=\"sm_61,compute_61\")
+  set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-gencode arch=compute_70,code=\"sm_70,compute_70\")
   set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};--include-path ${PROJECT_BINARY_DIR}/$\{build_configuration\};--expt-relaxed-constexpr)
   set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};-ftz=true)  # Flush denormals to zero
   set(CUDA_INCLUDE ${CUDA_TOOLKIT_TARGET_DIR} ${CUDA_TOOLKIT_TARGET_DIR}/extras/CUPTI/include)
   include_directories(${CUDA_INCLUDE})
   if (WIN32)
-    add_definitions(-DGOOGLE_CUDA=1 -DTF_EXTRA_CUDA_CAPABILITIES=3.0,3.5,5.2)
+    add_definitions(-DGOOGLE_CUDA=1 -DTF_EXTRA_CUDA_CAPABILITIES=3.7,5.2,6.0,6.1,7.0)
   else (WIN32)
-    # Without these double quotes, cmake in Linux makes it "-DTF_EXTRA_CUDA_CAPABILITIES=3.0, -D3.5, -D5.2" for cc, which incurs build breaks
-    add_definitions(-DGOOGLE_CUDA=1 -D"TF_EXTRA_CUDA_CAPABILITIES=3.0,3.5,5.2")
+    # Without these double quotes, cmake in Linux makes it "-DTF_EXTRA_CUDA_CAPABILITIES=3.7, -D5.2, ..." for cc, which incurs build breaks
+    add_definitions(-DGOOGLE_CUDA=1 -D"TF_EXTRA_CUDA_CAPABILITIES=3.7,5.2,6.0,6.1,7.0")
   endif (WIN32)
 
   if (WIN32)
@@ -452,7 +456,7 @@ if (tensorflow_ENABLE_GPU)
   FILE(WRITE ${tensorflow_source_dir}/third_party/gpus/cuda/cuda_config.h
     "#ifndef CUDA_CUDA_CONFIG_H_\n"
     "#define CUDA_CUDA_CONFIG_H_\n"
-    "#define TF_CUDA_CAPABILITIES CudaVersion(\"3.0\"),CudaVersion(\"3.5\"),CudaVersion(\"5.2\")\n"
+    "#define TF_CUDA_CAPABILITIES CudaVersion(\"3.7\"),CudaVersion(\"5.2\"),CudaVersion(\"6.0\"),CudaVersion(\"6.1\"),CudaVersion(\"7.0\")\n"
     "#define TF_CUDA_VERSION \"64_${short_CUDA_VER}\"\n"
     "#define TF_CUDNN_VERSION \"64_${tensorflow_CUDNN_VERSION}\"\n"
     "#define TF_CUDA_TOOLKIT_PATH \"${CUDA_TOOLKIT_ROOT_DIR}\"\n"
-- 
cgit v1.2.3


From d53830cddfc74105e46a4bdb703cb1154a288f8f Mon Sep 17 00:00:00 2001
From: James Keeling 
Date: Tue, 24 Jul 2018 11:11:35 -0700
Subject: Update TF_ApiDefMapGet to return nullptr if there is an error.

Previously it would return an allocated buffer, even if there was an error and the buffer was not usable. This could cause memory leaks if the caller did not manually delete the buffer.

Because TF_DeleteBuffer has been updated to be safe to call on nullptr, it's still OK if callers attempt to delete this nullptr.

PiperOrigin-RevId: 205858542
---
 tensorflow/c/c_api.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index f516ce4f18..10bc8cdbee 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -2732,6 +2732,10 @@ TF_Buffer* TF_ApiDefMapGet(TF_ApiDefMap* api_def_map, const char* name,
 
   TF_Buffer* ret = TF_NewBuffer();
   status->status = MessageToBuffer(*api_def, ret);
+  if (!status->status.ok()) {
+    TF_DeleteBuffer(ret);
+    return nullptr;
+  }
   return ret;
 #endif  // __ANDROID__
 }
-- 
cgit v1.2.3


From 3acdbf8f904cf32e5d4d211934ee8d346aa48457 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein 
Date: Tue, 24 Jul 2018 11:13:40 -0700
Subject: [XLA] Document DynamicSlice and DynamicUpdateSlice semantics.

PiperOrigin-RevId: 205858924
---
 .../performance/xla/operation_semantics.md         | 26 ++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md
index d6fa8ab5f9..26a7b9e42c 100644
--- a/tensorflow/docs_src/performance/xla/operation_semantics.md
+++ b/tensorflow/docs_src/performance/xla/operation_semantics.md
@@ -791,8 +791,6 @@ DynamicSlice extracts a sub-array from the input array at dynamic
 `size_indices`, which specify the end point of exclusive slice intervals in each
 dimension: [start, start + size). The shape of `start_indices` must be rank ==
 1, with dimension size equal to the rank of `operand`.
-Note: handling of out-of-bounds slice indices (generated by incorrect runtime
-calculation of 'start_indices') is currently implementation-defined.
 
  `DynamicSlice(operand, start_indices, size_indices)` 
 
@@ -812,6 +810,17 @@ calculation of 'start_indices') is currently implementation-defined.
 :                 :                     : dimension to avoid wrapping modulo  :
 :                 :                     : dimension size.                     :
 
+The effective slice indices are computed by applying the following
+transformation for each index `i` in `[1, N)` before performing the slice:
+
+```
+start_indices[i] = clamp(start_indices[i], 0, operand.dimension_size[i] - size_indices[i])
+```
+
+This ensures that the extracted slice is always in-bounds with respect to the
+operand array. If the slice is in-bounds before the transformation is applied,
+the transformation has no effect.
+
 1-dimensional example:
 
 ```
@@ -847,8 +856,6 @@ The shape of `update` determines the shape of the sub-array of the result which
 is updated.
 The shape of `start_indices` must be rank == 1, with dimension size equal to
 the rank of `operand`.
-Note: handling of out-of-bounds slice indices (generated by incorrect runtime
-calculation of 'start_indices') is currently implementation-defined.
 
  `DynamicUpdateSlice(operand, update, start_indices)` 
 
@@ -866,6 +873,17 @@ calculation of 'start_indices') is currently implementation-defined.
 :                 :         : dimension. Value must be greater than or equal   :
 :                 :         : to zero.                                         :
 
+The effective slice indices are computed by applying the following
+transformation for each index `i` in `[1, N)` before performing the slice:
+
+```
+start_indices[i] = clamp(start_indices[i], 0, operand.dimension_size[i] - update.dimension_size[i])
+```
+
+This ensures that the updated slice is always in-bounds with respect to the
+operand array. If the slice is in-bounds before the transformation is applied,
+the transformation has no effect.
+
 1-dimensional example:
 
 ```
-- 
cgit v1.2.3


From 7d266d6116438b8f22cd0f2c3b66115c2c05da7d Mon Sep 17 00:00:00 2001
From: Justin Lebar 
Date: Tue, 24 Jul 2018 11:17:25 -0700
Subject: [XLA:GPU] Limit the maximum number of parameters to a fusion.

Very large fusions aren't usually useful on the GPU, and if they get
large enough, they can fail to compile (or successfully compile but fail
to launch!).

PiperOrigin-RevId: 205859710
---
 .../compiler/xla/service/gpu/instruction_fusion.cc | 49 +++++++++++++++++++++-
 .../compiler/xla/service/gpu/instruction_fusion.h  |  4 ++
 .../xla/service/gpu/instruction_fusion_test.cc     | 30 +++++++++++++
 3 files changed, 81 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
index 64ed3d748f..8abae43a5a 100644
--- a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc
@@ -183,8 +183,53 @@ bool GpuInstructionFusion::ShouldFuse(HloInstruction* consumer,
     return true;
   }
 
-  return IsFusile(*producer) && IsFusile(*consumer) &&
-         InstructionFusion::ShouldFuse(consumer, operand_index);
+  if (!IsFusile(*producer) || !IsFusile(*consumer) ||
+      !InstructionFusion::ShouldFuse(consumer, operand_index)) {
+    return false;
+  }
+
+  // Limit the maximum number of operands to a fusion.
+  //
+  // There's a limit to how many parameters we can pass to a CUDA kernel, but
+  // exactly what that limit is is hazy, as it depends on (among other things)
+  // how much GPU constant memory is in use for other purposes.
+  //
+  // Moreover, we don't even know at this point how many arguments the CUDA
+  // kernel for this fusion node will have: It depends on buffer assignment,
+  // where we will decide which of the fusion's operands live in XLA's big temp
+  // buffer versus in other allocations.
+  //
+  // As a heuristic, we simply cap the number of fusion operands at
+  // kMaxOperandsPerFusion.  This puts an upper bound on the number of
+  // parameters to the kernel, working around the correctness problem.
+  //
+  // This limit is also often good for performance.  In a fusion with many
+  // operands, each GPU thread likely has to do a lot of work, and so possibly
+  // uses a lot of registers, thus limiting occupancy.
+  //
+  // We put this check last because it's expensive to compute.
+
+  // The new fusion will have no more operands than
+  //   producer_operands + consumer_operands - 1
+  // (minus one because we're fusing the producer->consumer edge).  This fact
+  // may be enough to let us avoid having to compute the true total number of
+  // operands, taking into account the fact that producer and consumer may share
+  // operands.
+  if (producer->operand_count() + consumer->operand_count() - 1 >
+      kMaxOperandsPerFusion) {
+    tensorflow::gtl::FlatSet producer_operands(
+        producer->operands().begin(), producer->operands().end());
+    int64 new_num_operands =
+        producer->operand_count() +
+        c_count_if(consumer->operands(), [&](const HloInstruction* operand) {
+          return operand != producer && !producer_operands.count(operand);
+        });
+    if (new_num_operands > kMaxOperandsPerFusion) {
+      return false;
+    }
+  }
+
+  return true;
 }
 
 bool GpuInstructionFusion::ShouldFuseIntoMultiOutput(HloInstruction* consumer,
diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion.h b/tensorflow/compiler/xla/service/gpu/instruction_fusion.h
index f629d9ff2c..5ee1c004b6 100644
--- a/tensorflow/compiler/xla/service/gpu/instruction_fusion.h
+++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion.h
@@ -36,6 +36,10 @@ class GpuInstructionFusion : public InstructionFusion {
 
   HloInstruction::FusionKind ChooseKind(
       const HloInstruction* producer, const HloInstruction* consumer) override;
+
+  // Maximum number of operands allowed on a single fusion node.  Exposed
+  // publicly mainly for tests.
+  static constexpr int64 kMaxOperandsPerFusion = 64;
 };
 
 }  // namespace gpu
diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
index 98ba162cd9..229eb23f12 100644
--- a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
@@ -606,5 +606,35 @@ TEST_F(InstructionFusionTest, FuseScalarConstant) {
                       op::Parameter()));
 }
 
+// Check that we limit the number of operands to fusions we create.
+TEST_F(InstructionFusionTest, AvoidsLargeFusion) {
+  constexpr int64 kNumParams = 200;
+  ASSERT_GT(kNumParams, GpuInstructionFusion::kMaxOperandsPerFusion);
+
+  // Compute p0 + p1 + ... + pN.
+  HloComputation::Builder b(TestName());
+  Shape shape = ShapeUtil::MakeShape(F32, {10, 100});
+  auto param0 =
+      b.AddInstruction(HloInstruction::CreateParameter(0, shape, "p"));
+  auto sum = param0;
+  for (int64 i = 1; i < kNumParams; ++i) {
+    auto param =
+        b.AddInstruction(HloInstruction::CreateParameter(i, shape, "p"));
+    sum = b.AddInstruction(
+        HloInstruction::CreateBinary(shape, HloOpcode::kAdd, sum, param));
+  }
+  auto module = CreateNewModule();
+  auto computation = module->AddEntryComputation(b.Build());
+  EXPECT_TRUE(GpuInstructionFusion(/*may_duplicate=*/true)
+                  .Run(module.get())
+                  .ValueOrDie());
+  SCOPED_TRACE(module->ToString());
+  for (const HloInstruction* instr : computation->instructions()) {
+    EXPECT_LE(instr->operand_count(),
+              GpuInstructionFusion::kMaxOperandsPerFusion)
+        << instr->ToString();
+  }
+}
+
 }  // namespace gpu
 }  // namespace xla
-- 
cgit v1.2.3


From bd8ee2b6aba2f99d7ca9a7af12eda62db480f355 Mon Sep 17 00:00:00 2001
From: Alexandre Passos 
Date: Tue, 24 Jul 2018 11:17:33 -0700
Subject: Uses separate forward and backward graphs for tfe.defun backprop.

PiperOrigin-RevId: 205859733
---
 tensorflow/python/eager/function.py | 45 ++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index d283a85532..5e4f9e29da 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -470,37 +470,39 @@ class GraphModeFunction(object):
 
   def _construct_backprop_function(self):
     """Constructs the backprop function object for this function."""
-    with self._graph.as_default():
-      c_known_ops = set()
-      c_captured_tensors = set()
-
-      existing_op_len = len(self._graph.get_operations())
-      filtered_outputs = [x for x in self._python_returns if x is not None]
+    filtered_outputs = [x for x in self._python_returns if x is not None]
+    captures = {}
+    backwards_graph = CapturingGraph(captures)
+    backwards_graph._graph_key = self._graph._graph_key  # pylint: disable=protected-access
+    for collection in self._graph.collections:
+      backwards_graph.get_collection_ref(
+          collection)[:] = self._graph.get_collection(collection)
+    backwards_graph.seed = self._graph.seed
+    with backwards_graph.as_default():
       self._out_grad_placeholders = [
           graph_placeholder(x.dtype, x.shape) for x in filtered_outputs]
-      in_gradients = gradients_impl.gradients(
+      in_gradients = gradients_impl._GradientsHelper(  # pylint: disable=protected-access
           filtered_outputs,
           self._input_placeholders,
-          grad_ys=self._out_grad_placeholders)
-      for op in self._graph.get_operations()[existing_op_len:]:
-        if op.type in ["Variable", "VariableV2", "VarHandleOp"]:
-          raise ValueError("defun cannot capture variables created without "
-                           "using tf.get_variable. Op: %s" % op)
-        c_known_ops.add(op)
-        for i in op.inputs:
-          if i.op not in c_known_ops:
-            c_captured_tensors.add(i)
+          grad_ys=self._out_grad_placeholders,
+          src_graph=self._graph)
 
     backward_outputs = tuple(
         grad for grad in _flatten(in_gradients) if grad is not None)
     output_shapes = tuple(grad.shape for grad in backward_outputs)
 
-    captures = list(sorted(c_captured_tensors, key=lambda x: x.name))
+    ids = list(sorted(captures.keys()))
+    if ids:
+      extra_inputs, extra_placeholders = zip(*[captures[x] for x in ids])
+    else:
+      extra_inputs = []
+      extra_placeholders = []
+
     forward_name = _forward_name(self._func_name)
     self._forward_fdef = _EagerDefinedFunction(
         forward_name, self._graph, self._ops, self._input_placeholders,
-        filtered_outputs + captures, self._attrs)
-    all_inputs = self._out_grad_placeholders + captures
+        filtered_outputs + list(extra_inputs), self._attrs)
+    all_inputs = self._out_grad_placeholders + list(extra_placeholders)
     # Excluding input ops from the body as we do not intend to execute these
     # operations when the function is executed.
     all_ignored_ops = frozenset(x.op for x in all_inputs)
@@ -508,11 +510,12 @@ class GraphModeFunction(object):
     # means rerunning the function-defining code will always define the same
     # function, which is useful if we serialize this etc.
     function_def_ops = tuple(x
-                             for x in sorted(c_known_ops, key=lambda x: x.name)
+                             for x in sorted(backwards_graph.get_operations(),
+                                             key=lambda x: x.name)
                              if x not in all_ignored_ops)
     bname = _backward_name(self._func_name)
     self._backward_function = GraphModeFunction(
-        bname, all_inputs, [], self._graph, function_def_ops,
+        bname, all_inputs, [], backwards_graph, function_def_ops,
         backward_outputs, in_gradients, output_shapes, attrs=self._attrs)
 
   def _backprop_call(self, args):
-- 
cgit v1.2.3


From 9e2466b60cf81a92048d4a14237da198c8033dc4 Mon Sep 17 00:00:00 2001
From: Rohan Jain 
Date: Tue, 24 Jul 2018 11:25:43 -0700
Subject: Creating a MultiDeviceIterator that deterministically provides
 outputs to different iterators placed on devices.

PiperOrigin-RevId: 205861287
---
 .../contrib/data/kernels/prefetching_kernels.cc    | 463 +++++++++++++++++++++
 tensorflow/contrib/data/ops/dataset_ops.cc         |  74 ++++
 .../python/kernel_tests/prefetching_ops_test.py    | 151 +++++++
 .../contrib/data/python/ops/prefetching_ops.py     | 172 ++++++++
 4 files changed, 860 insertions(+)

diff --git a/tensorflow/contrib/data/kernels/prefetching_kernels.cc b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
index b3d464d716..6edc61b2c2 100644
--- a/tensorflow/contrib/data/kernels/prefetching_kernels.cc
+++ b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include 
 
 #include "tensorflow/core/common_runtime/process_function_library_runtime.h"
+#include "tensorflow/core/framework/dataset.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_op_kernel.h"
@@ -23,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
+namespace {
 
 struct BufferElement {
   // The producer sets `status` if getting the input element fails.
@@ -473,4 +475,465 @@ class IteratorGetDeviceOp : public OpKernel {
 REGISTER_KERNEL_BUILDER(Name("IteratorGetDevice").Device(DEVICE_CPU),
                         IteratorGetDeviceOp);
 
+Status VerifyTypesMatch(const DataTypeVector& expected,
+                        const DataTypeVector& received) {
+  if (expected.size() != received.size()) {
+    return errors::InvalidArgument(
+        "Number of components does not match: expected ", expected.size(),
+        " types but got ", received.size(), ".");
+  }
+  for (size_t i = 0; i < expected.size(); ++i) {
+    if (expected[i] != received[i]) {
+      return errors::InvalidArgument("Data type mismatch at component ", i,
+                                     ": expected ", DataTypeString(expected[i]),
+                                     " but got ", DataTypeString(received[i]),
+                                     ".");
+    }
+  }
+  return Status::OK();
+}
+
+Status VerifyShapesCompatible(const std::vector& expected,
+                              const std::vector& received) {
+  if (expected.size() != received.size()) {
+    return errors::InvalidArgument(
+        "Number of components does not match: expected ", expected.size(),
+        " shapes but got ", received.size(), ".");
+  }
+  for (size_t i = 0; i < expected.size(); ++i) {
+    if (!expected[i].IsCompatibleWith(received[i])) {
+      return errors::InvalidArgument("Incompatible shapes at component ", i,
+                                     ": expected ", expected[i].DebugString(),
+                                     " but got ", received[i].DebugString(),
+                                     ".");
+    }
+  }
+
+  return Status::OK();
+}
+
+string SanitizeThreadSuffix(string suffix) {
+  string clean;
+  for (int i = 0; i < suffix.size(); ++i) {
+    const char ch = suffix[i];
+    if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
+        (ch >= '0' && ch <= '9') || ch == '_' || ch == '-') {
+      clean += ch;
+    } else {
+      clean += '_';
+    }
+  }
+  return clean;
+}
+
+class MultiDeviceIterator : public ResourceBase {
+ public:
+  MultiDeviceIterator(const DataTypeVector& output_types,
+                      const std::vector& output_shapes,
+                      const std::vector& devices,
+                      std::unique_ptr flib_def,
+                      std::unique_ptr pflr,
+                      FunctionLibraryRuntime* lib)
+      : output_types_(output_types),
+        output_shapes_(output_shapes),
+        devices_(devices),
+        flib_def_(std::move(flib_def)),
+        pflr_(std::move(pflr)),
+        lib_(lib) {
+    buffer_.resize(devices_.size());
+  }
+
+  string DebugString() override {
+    return strings::StrCat("MultiDeviceIterator");
+  }
+
+  Status Init(std::unique_ptr iterator, int64* incarnation_id) {
+    mutex_lock l(mu_);
+    if (iterator) {
+      TF_RETURN_IF_ERROR(
+          VerifyTypesMatch(output_types_, iterator->output_dtypes()));
+      TF_RETURN_IF_ERROR(
+          VerifyShapesCompatible(output_shapes_, iterator->output_shapes()));
+    }
+    host_iterator_.reset(iterator.release());
+    incarnation_id_++;
+    *incarnation_id = incarnation_id_;
+    max_buffer_size_ = 0;
+    num_elements_ = 0;
+    buffer_.clear();
+    buffer_.resize(devices_.size());
+    return Status::OK();
+  }
+
+  Status GetNextFromShard(IteratorContext* ctx, int shard_num,
+                          int64 incarnation_id,
+                          std::vector* out_tensors,
+                          bool* end_of_sequence) {
+    // TODO(rohanj): This might potentially strand elements in other shards.
+    // Opportunity to do smarter locking semantics.
+    mutex_lock l(mu_);
+    // Make sure we're in the right incarnation.
+    if (incarnation_id != incarnation_id_) {
+      return errors::InvalidArgument(
+          "Current incarnation: ", incarnation_id_,
+          "; Supplied incarnation: ", incarnation_id);
+    }
+    // Then look it up in the buffer.
+    if (!buffer_[shard_num].empty()) {
+      const HostBufferElement& elem = buffer_[shard_num].front();
+      *out_tensors = elem.value;
+      *end_of_sequence = elem.end_of_sequence;
+      Status s = elem.status;
+      buffer_[shard_num].pop_front();
+      return s;
+    }
+    std::shared_ptr captured_iterator(host_iterator_);
+    if (captured_iterator) {
+      if (lib_ != nullptr) {
+        ctx->set_lib(lib_);
+      }
+      while (true) {
+        HostBufferElement elem;
+        elem.status =
+            captured_iterator->GetNext(ctx, &elem.value, &elem.end_of_sequence);
+        int buffer_index = num_elements_ % devices_.size();
+        num_elements_++;
+        if (buffer_index == shard_num) {
+          out_tensors->swap(elem.value);
+          *end_of_sequence = elem.end_of_sequence;
+          return elem.status;
+        } else {
+          buffer_[buffer_index].push_back(std::move(elem));
+          // TODO(rohanj): Put an upper bound to buffer size.
+          if (buffer_[buffer_index].size() > max_buffer_size_) {
+            max_buffer_size_ = buffer_[buffer_index].size();
+            VLOG(1) << "MultiDeviceIterator: Max buffer size increased to: "
+                    << max_buffer_size_;
+          }
+        }
+      }
+    } else {
+      return errors::FailedPrecondition("Iterator not initialized");
+    }
+    return Status::OK();
+  }
+
+  const DataTypeVector& output_types() const { return output_types_; }
+
+  const std::vector& output_shapes() const {
+    return output_shapes_;
+  }
+
+  std::shared_ptr function_library() {
+    tf_shared_lock l(mu_);
+    return lib_def_;
+  }
+
+ private:
+  struct HostBufferElement {
+    Status status;
+    bool end_of_sequence;
+    std::vector value;
+  };
+
+  mutex mu_;
+  const DataTypeVector output_types_;
+  const std::vector output_shapes_;
+  const std::vector devices_;
+  int64 num_elements_ GUARDED_BY(mu_) = 0;
+  int64 max_buffer_size_ GUARDED_BY(mu_) = 0;
+  int64 incarnation_id_ GUARDED_BY(mu_) = 0;
+  std::vector> buffer_ GUARDED_BY(mu_);
+  std::unique_ptr flib_def_;
+  std::unique_ptr pflr_;
+  FunctionLibraryRuntime* lib_ = nullptr;  // not owned.
+  std::shared_ptr host_iterator_;
+  std::shared_ptr lib_def_ GUARDED_BY(mu_);
+};
+
+// Just creates a MultiDeviceIterator and returns it.
+class MultiDeviceIteratorHandleOp : public OpKernel {
+ public:
+  explicit MultiDeviceIteratorHandleOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx), graph_def_version_(ctx->graph_def_version()) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("shared_name", &name_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("container", &container_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("devices", &devices_));
+  }
+
+  // The resource is deleted from the resource manager only when it is private
+  // to kernel.
+  ~MultiDeviceIteratorHandleOp() override {
+    if (resource_ != nullptr) {
+      resource_->Unref();
+      if (cinfo_.resource_is_private_to_kernel()) {
+        if (!cinfo_.resource_manager()
+                 ->template Delete(cinfo_.container(),
+                                                        cinfo_.name())
+                 .ok()) {
+          // Do nothing; the resource can have been deleted by session resets.
+        }
+      }
+    }
+  }
+
+  void Compute(OpKernelContext* context) override LOCKS_EXCLUDED(mu_) {
+    {
+      mutex_lock l(mu_);
+      if (resource_ == nullptr) {
+        FunctionLibraryRuntime* lib;
+        std::unique_ptr flib_def(nullptr);
+        std::unique_ptr pflr(nullptr);
+        OP_REQUIRES_OK(context, context->function_library()->Clone(
+                                    &flib_def, &pflr, &lib));
+        ResourceMgr* mgr = context->resource_manager();
+        OP_REQUIRES_OK(context, cinfo_.Init(mgr, def()));
+
+        MultiDeviceIterator* resource;
+        OP_REQUIRES_OK(
+            context,
+            mgr->LookupOrCreate(
+                cinfo_.container(), cinfo_.name(), &resource,
+                [this, lib, &flib_def, &pflr](MultiDeviceIterator** ret)
+                    EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+                      *ret = new MultiDeviceIterator(
+                          output_types_, output_shapes_, devices_,
+                          std::move(flib_def), std::move(pflr), lib);
+                      return Status::OK();
+                    }));
+
+        Status s = VerifyResource(resource);
+        if (TF_PREDICT_FALSE(!s.ok())) {
+          resource->Unref();
+          context->SetStatus(s);
+          return;
+        }
+
+        resource_ = resource;
+      }
+    }
+    OP_REQUIRES_OK(context, MakeResourceHandleToOutput(
+                                context, 0, cinfo_.container(), cinfo_.name(),
+                                MakeTypeIndex()));
+  }
+
+ private:
+  // During the first Compute(), resource is either created or looked up using
+  // shared_name. In the latter case, the resource found should be verified if
+  // it is compatible with this op's configuration. The verification may fail in
+  // cases such as two graphs asking queues of the same shared name to have
+  // inconsistent capacities.
+  Status VerifyResource(MultiDeviceIterator* resource) {
+    TF_RETURN_IF_ERROR(
+        VerifyTypesMatch(output_types_, resource->output_types()));
+    TF_RETURN_IF_ERROR(
+        VerifyShapesCompatible(output_shapes_, resource->output_shapes()));
+    return Status::OK();
+  }
+
+  mutex mu_;
+  ContainerInfo cinfo_;  // Written once under mu_ then constant afterwards.
+  MultiDeviceIterator* resource_ GUARDED_BY(mu_) = nullptr;
+  DataTypeVector output_types_;
+  std::vector output_shapes_;
+  const int graph_def_version_;
+  string name_;
+  string container_;
+  std::vector devices_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("MultiDeviceIterator").Device(DEVICE_CPU),
+                        MultiDeviceIteratorHandleOp);
+
+// Calls init on the MultiDeviceIterator.
+class MultiDeviceIteratorInitOp : public OpKernel {
+ public:
+  explicit MultiDeviceIteratorInitOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    DatasetBase* dataset;
+    OP_REQUIRES_OK(ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset));
+    MultiDeviceIterator* resource;
+    OP_REQUIRES_OK(ctx,
+                   LookupResource(ctx, HandleFromInput(ctx, 1), &resource));
+    core::ScopedUnref unref(resource);
+
+    IteratorContext iter_ctx = dataset::MakeIteratorContext(ctx);
+    std::unique_ptr iterator;
+    OP_REQUIRES_OK(ctx,
+                   dataset->MakeIterator(&iter_ctx, "Iterator", &iterator));
+    int64 incarnation_id;
+    OP_REQUIRES_OK(ctx, resource->Init(std::move(iterator), &incarnation_id));
+    Tensor tensor_incarnation_id(DT_INT64, TensorShape({}));
+    tensor_incarnation_id.scalar()() = incarnation_id;
+    OP_REQUIRES_OK(ctx,
+                   ctx->set_output("incarnation_id", tensor_incarnation_id));
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("MultiDeviceIteratorInit").Device(DEVICE_CPU),
+                        MultiDeviceIteratorInitOp);
+
+// Calls GetNextFromShard(shard) and returns a vector of Tensors as output.
+// TODO(rohanj): Implement using BackgroundWorker that Derek built?
+class MultiDeviceIteratorGetNextFromShardOp : public AsyncOpKernel {
+ public:
+  explicit MultiDeviceIteratorGetNextFromShardOp(OpKernelConstruction* ctx)
+      : AsyncOpKernel(ctx),
+        thread_pool_(new thread::ThreadPool(
+            ctx->env(), ThreadOptions(),
+            strings::StrCat("multi_device_iterator_get_next_thread_",
+                            SanitizeThreadSuffix(name())),
+            1 /* num_threads */, false /* low_latency_hint */)) {}
+
+  void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override {
+    const Tensor* tensor_shard_num;
+    OP_REQUIRES_OK(ctx, ctx->input("shard_num", &tensor_shard_num));
+    int32 shard_num = tensor_shard_num->scalar()();
+
+    const Tensor* tensor_incarnation_id;
+    OP_REQUIRES_OK(ctx, ctx->input("incarnation_id", &tensor_incarnation_id));
+    int64 incarnation_id = tensor_incarnation_id->scalar()();
+
+    MultiDeviceIterator* iterator;
+    OP_REQUIRES_OK(ctx,
+                   LookupResource(ctx, HandleFromInput(ctx, 0), &iterator));
+    thread_pool_->Schedule(std::bind(
+        [ctx, iterator, shard_num, incarnation_id](DoneCallback done) {
+          std::vector components;
+          bool end_of_sequence = false;
+
+          IteratorContext::Params params;
+          params.env = ctx->env();
+          params.runner = *(ctx->runner());
+          params.function_library = iterator->function_library();
+          DeviceBase* device = ctx->function_library()->device();
+          params.allocator_getter = [device](AllocatorAttributes attrs) {
+            return device->GetAllocator(attrs);
+          };
+          IteratorContext iter_ctx(std::move(params));
+
+          Status s =
+              iterator->GetNextFromShard(&iter_ctx, shard_num, incarnation_id,
+                                         &components, &end_of_sequence);
+          iterator->Unref();
+
+          if (!s.ok()) {
+            ctx->SetStatus(s);
+          } else if (end_of_sequence) {
+            ctx->SetStatus(errors::OutOfRange("End of sequence"));
+          } else {
+            for (int i = 0; i < components.size(); ++i) {
+              // TODO(mrry): Check that the shapes match the shape attrs.
+              ctx->set_output(i, components[i]);
+            }
+          }
+          done();
+        },
+        std::move(done)));
+  }
+
+ private:
+  std::unique_ptr thread_pool_;
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("MultiDeviceIteratorGetNextFromShard").Device(DEVICE_CPU),
+    MultiDeviceIteratorGetNextFromShardOp);
+
+class MultiDeviceIteratorToStringHandleOp : public OpKernel {
+ public:
+  explicit MultiDeviceIteratorToStringHandleOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& resource_handle_t = ctx->input(0);
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(resource_handle_t.shape()),
+                errors::InvalidArgument("resource_handle must be a scalar"));
+
+    // Validate that the handle corresponds to a real resource, and
+    // that it is an MultiDeviceIterator.
+    MultiDeviceIterator* resource;
+    OP_REQUIRES_OK(ctx,
+                   LookupResource(ctx, HandleFromInput(ctx, 0), &resource));
+    resource->Unref();
+
+    Tensor* string_handle_t;
+    OP_REQUIRES_OK(ctx,
+                   ctx->allocate_output(0, TensorShape({}), &string_handle_t));
+    string_handle_t->scalar()() =
+        resource_handle_t.scalar()().SerializeAsString();
+  }
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("MultiDeviceIteratorToStringHandle").Device(DEVICE_CPU),
+    MultiDeviceIteratorToStringHandleOp);
+
+class MultiDeviceIteratorFromStringHandleOp : public OpKernel {
+ public:
+  explicit MultiDeviceIteratorFromStringHandleOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+    OP_REQUIRES(
+        ctx,
+        output_types_.empty() || output_shapes_.empty() ||
+            output_types_.size() == output_shapes_.size(),
+        errors::InvalidArgument("If both 'output_types' and 'output_shapes' "
+                                "are set, they must have the same length."));
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& string_handle_t = ctx->input(0);
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(string_handle_t.shape()),
+                errors::InvalidArgument("string_handle must be a scalar"));
+
+    ResourceHandle resource_handle;
+    OP_REQUIRES(
+        ctx,
+        resource_handle.ParseFromString(string_handle_t.scalar()()),
+        errors::InvalidArgument(
+            "Could not parse string_handle as a valid ResourceHandle"));
+
+    OP_REQUIRES(
+        ctx, resource_handle.device() == ctx->device()->attributes().name(),
+        errors::InvalidArgument("Attempted create an iterator on device \"",
+                                ctx->device()->attributes().name(),
+                                "\" from handle defined on device \"",
+                                resource_handle.device(), "\""));
+
+    // Validate that the handle corresponds to a real resource, and
+    // that it is an MultiDeviceIterator.
+    MultiDeviceIterator* resource;
+    OP_REQUIRES_OK(ctx, LookupResource(ctx, resource_handle, &resource));
+    core::ScopedUnref unref_iterator(resource);
+    if (!output_types_.empty()) {
+      OP_REQUIRES_OK(ctx,
+                     VerifyTypesMatch(output_types_, resource->output_types()));
+    }
+    if (!output_shapes_.empty()) {
+      OP_REQUIRES_OK(ctx, VerifyShapesCompatible(output_shapes_,
+                                                 resource->output_shapes()));
+    }
+
+    Tensor* resource_handle_t;
+    OP_REQUIRES_OK(
+        ctx, ctx->allocate_output(0, TensorShape({}), &resource_handle_t));
+    resource_handle_t->scalar()() = resource_handle;
+  }
+
+ private:
+  DataTypeVector output_types_;
+  std::vector output_shapes_;
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("MultiDeviceIteratorFromStringHandle").Device(DEVICE_CPU),
+    MultiDeviceIteratorFromStringHandleOp);
+
+}  // anonymous namespace
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/data/ops/dataset_ops.cc b/tensorflow/contrib/data/ops/dataset_ops.cc
index b5c6f2e241..66a7c7fdcd 100644
--- a/tensorflow/contrib/data/ops/dataset_ops.cc
+++ b/tensorflow/contrib/data/ops/dataset_ops.cc
@@ -145,6 +145,80 @@ Resets the FunctionBufferingResource.
 function_buffer_resource: The FunctionBufferingResource handle.
 )doc");
 
+REGISTER_OP("MultiDeviceIterator")
+    .Output("handle: resource")
+    .Attr("devices: list(string) >= 1")
+    .Attr("shared_name: string")
+    .Attr("container: string")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .Doc(R"doc(
+Creates a MultiDeviceIterator resource.
+
+handle: Handle to the resource created.
+devices: A list of devices the iterator works across.
+shared_name: If non-empty, this resource will be shared under the given name
+  across multiple sessions.
+container: If non-empty, this resource is placed in the given container.
+  Otherwise, a default container is used.
+output_types: The type list for the return values.
+output_shapes: The list of shapes being produced.
+)doc");
+
+REGISTER_OP("MultiDeviceIteratorInit")
+    .Input("dataset: variant")
+    .Input("multi_device_iterator: resource")
+    .Output("incarnation_id: int64")
+    .Doc(R"doc(
+Initializes the multi device iterator with the given dataset.
+incarnation_id: An int64 indicating which incarnation of the MultiDeviceIterator
+  is running.
+dataset: Dataset to be iterated upon.
+multi_device_iterator: A MultiDeviceIteratorResource.
+)doc");
+
+REGISTER_OP("MultiDeviceIteratorGetNextFromShard")
+    .Input("multi_device_iterator: resource")
+    .Input("shard_num: int32")
+    .Input("incarnation_id: int64")
+    .Output("components: output_types")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .Doc(R"doc(
+Gets next element for the provided shard number.
+
+multi_device_iterator: A MultiDeviceIterator resource.
+shard_num: Integer representing which shard to fetch data for.
+incarnation_id: Which incarnation of the MultiDeviceIterator is running.
+components: Result of the get_next on the dataset.
+output_types: The type list for the return values.
+output_shapes: The list of shapes being produced.
+)doc");
+
+REGISTER_OP("MultiDeviceIteratorToStringHandle")
+    .Input("multi_device_iterator: resource")
+    .Output("string_handle: string")
+    .Doc(R"doc(
+Produces a string handle for the given MultiDeviceIterator.
+
+multi_device_iterator: A MultiDeviceIterator resource.
+string_handle: A string representing the resource.
+)doc");
+
+REGISTER_OP("MultiDeviceIteratorFromStringHandle")
+    .Input("string_handle: string")
+    .Output("multi_device_iterator: resource")
+    .Attr("output_types: list(type) >= 0 = []")
+    .Attr("output_shapes: list(shape) >= 0 = []")
+    .Doc(R"doc(
+Generates a MultiDeviceIterator resource from its provided string handle.
+
+string_handle: String representing the resource.
+multi_device_iterator: A MultiDeviceIterator resource.
+output_types: The type list for the return values.
+output_shapes: The list of shapes being produced.
+)doc");
+
 REGISTER_OP("ThreadPoolDataset")
     .Input("input_dataset: variant")
     .Input("thread_pool: resource")
diff --git a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
index 82543b1039..2da6131e8e 100644
--- a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import test
 
@@ -907,5 +908,155 @@ class CopyToDeviceTest(test.TestCase):
         sess.run(next_element)
 
 
+class MultiDeviceIteratorTest(test.TestCase):
+
+  def testBasic(self):
+    dataset = dataset_ops.Dataset.range(10)
+    multi_device_iterator = prefetching_ops.MultiDeviceIterator(
+        dataset, ["/cpu:1", "/cpu:2"])
+    elem_on_1, elem_on_2 = multi_device_iterator.get_next()
+
+    config = config_pb2.ConfigProto(device_count={"CPU": 3})
+    with self.test_session(config=config) as sess:
+      sess.run(multi_device_iterator.initializer)
+      for i in range(0, 10, 2):
+        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(elem_on_1)
+        sess.run(elem_on_2)
+
+  def testOneOnSameDevice(self):
+    with ops.device("/cpu:0"):
+      dataset = dataset_ops.Dataset.range(10)
+    multi_device_iterator = prefetching_ops.MultiDeviceIterator(
+        dataset, ["/cpu:0", "/cpu:1"])
+    elem_on_1, elem_on_2 = multi_device_iterator.get_next()
+
+    config = config_pb2.ConfigProto(device_count={"CPU": 2})
+    with self.test_session(config=config) as sess:
+      sess.run(multi_device_iterator.initializer)
+      for i in range(0, 10, 2):
+        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(elem_on_1)
+        sess.run(elem_on_2)
+
+  def testRepeatDevices(self):
+    with ops.device("/cpu:0"):
+      dataset = dataset_ops.Dataset.range(20)
+    multi_device_iterator = prefetching_ops.MultiDeviceIterator(
+        dataset, ["/cpu:1", "/cpu:2", "/cpu:1", "/cpu:2"])
+    elements = multi_device_iterator.get_next()
+    elem_on_1, elem_on_2, elem_on_3, elem_on_4 = elements
+
+    config = config_pb2.ConfigProto(device_count={"CPU": 3})
+    with self.test_session(config=config) as sess:
+      sess.run(multi_device_iterator.initializer)
+      for i in range(0, 20, 4):
+        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
+        self.assertEqual(i + 2, sess.run(elem_on_3))
+        self.assertEqual(i + 3, sess.run(elem_on_4))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(elem_on_1)
+        sess.run(elem_on_2)
+        sess.run(elem_on_3)
+        sess.run(elem_on_4)
+
+  def testNotFullyDivisible(self):
+    dataset = dataset_ops.Dataset.range(9)
+    multi_device_iterator = prefetching_ops.MultiDeviceIterator(
+        dataset, ["/cpu:1", "/cpu:2"])
+    elem_on_1, elem_on_2 = multi_device_iterator.get_next()
+
+    config = config_pb2.ConfigProto(device_count={"CPU": 3})
+    with self.test_session(config=config) as sess:
+      sess.run(multi_device_iterator.initializer)
+      for i in range(0, 8, 2):
+        self.assertEqual(i, sess.run(elem_on_1))
+        self.assertEqual(i + 1, sess.run(elem_on_2))
+      self.assertEqual(8, sess.run(elem_on_1))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(elem_on_1)
+        sess.run(elem_on_2)
+
+  def testUneven(self):
+    dataset = dataset_ops.Dataset.range(10)
+    multi_device_iterator = prefetching_ops.MultiDeviceIterator(
+        dataset, ["/cpu:1", "/cpu:2"])
+    elem_on_1, elem_on_2 = multi_device_iterator.get_next()
+
+    config = config_pb2.ConfigProto(device_count={"CPU": 3})
+    with self.test_session(config=config) as sess:
+      sess.run(multi_device_iterator.initializer)
+      for i in range(0, 10, 2):
+        self.assertEqual(i, sess.run(elem_on_1))
+      for i in range(0, 10, 2):
+        self.assertEqual(i + 1, sess.run(elem_on_2))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(elem_on_1)
+        sess.run(elem_on_2)
+
+  def testMultipleInitializations(self):
+    with ops.device("/cpu:0"):
+      epoch = array_ops.placeholder(dtypes.int64, shape=[])
+      dataset1 = dataset_ops.Dataset.from_tensors(epoch).repeat(1000)
+      dataset2 = dataset_ops.Dataset.range(1000)
+      dataset = dataset_ops.Dataset.zip((dataset1, dataset2))
+    multi_device_iterator = prefetching_ops.MultiDeviceIterator(
+        dataset, ["/cpu:1", "/cpu:2"], prefetch_buffer_size=4)
+    elem_on_1, elem_on_2 = multi_device_iterator.get_next()
+    init_op = multi_device_iterator.initializer
+
+    config = config_pb2.ConfigProto(device_count={"CPU": 3})
+    with self.test_session(config=config) as sess:
+      for i in range(1000):
+        sess.run(init_op, feed_dict={epoch: i})
+        self.assertEqual([(i, 0), (i, 1)], sess.run([elem_on_1, elem_on_2]))
+
+  def testBasicGpu(self):
+    if not test_util.is_gpu_available():
+      self.skipTest("No GPU available")
+
+    with compat.forward_compatibility_horizon(2018, 8, 4):
+      dataset = dataset_ops.Dataset.range(10)
+      multi_device_iterator = prefetching_ops.MultiDeviceIterator(
+          dataset, ["/cpu:1", "/gpu:0"])
+      elem_on_1, elem_on_2 = multi_device_iterator.get_next()
+
+      config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
+      with self.test_session(config=config) as sess:
+        sess.run(multi_device_iterator.initializer)
+        for i in range(0, 10, 2):
+          self.assertEqual(i, sess.run(elem_on_1))
+          self.assertEqual(i + 1, sess.run(elem_on_2))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(elem_on_1)
+          sess.run(elem_on_2)
+
+  def testUnevenGpu(self):
+    if not test_util.is_gpu_available():
+      self.skipTest("No GPU available")
+
+    with compat.forward_compatibility_horizon(2018, 8, 4):
+      dataset = dataset_ops.Dataset.range(10)
+      multi_device_iterator = prefetching_ops.MultiDeviceIterator(
+          dataset, ["/cpu:1", "/gpu:0"])
+      elem_on_1, elem_on_2 = multi_device_iterator.get_next()
+
+      config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1})
+      with self.test_session(config=config) as sess:
+        sess.run(multi_device_iterator.initializer)
+        for i in range(0, 10, 2):
+          self.assertEqual(i, sess.run(elem_on_1))
+        for i in range(0, 10, 2):
+          self.assertEqual(i + 1, sess.run(elem_on_2))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(elem_on_1)
+          sess.run(elem_on_2)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py
index 45abd6376c..0edd7c9fe9 100644
--- a/tensorflow/contrib/data/python/ops/prefetching_ops.py
+++ b/tensorflow/contrib/data/python/ops/prefetching_ops.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gen_dataset_ops as core_gen_dataset_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -523,3 +524,174 @@ class _CopyToDeviceDataset(dataset_ops.Dataset):
   @property
   def output_classes(self):
     return self._input_dataset.output_classes
+
+
+class _PerDeviceGenerator(dataset_ops.Dataset):
+  """A `dummy` generator dataset."""
+
+  def __init__(self, shard_num, multi_device_iterator_resource, incarnation_id,
+               source_device, target_device, output_shapes, output_types,
+               output_classes):
+    self._target_device = target_device
+    self._output_types = output_types
+    self._output_shapes = output_shapes
+    self._output_classes = output_classes
+    self._flat_output_shapes = nest.flatten(
+        sparse.as_dense_shapes(self._output_shapes, self._output_classes))
+    self._flat_output_types = nest.flatten(
+        sparse.as_dense_types(self._output_types, self._output_classes))
+
+    multi_device_iterator_string_handle = (
+        gen_dataset_ops.multi_device_iterator_to_string_handle(
+            multi_device_iterator_resource))
+
+    @function.Defun()
+    def _init_func():
+      return multi_device_iterator_string_handle
+
+    @function.Defun()
+    def _remote_init_func():
+      return functional_ops.remote_call(
+          target=source_device,
+          args=_init_func.captured_inputs,
+          Tout=[dtypes.string],
+          f=_init_func)
+
+    self._init_func = _remote_init_func
+    self._init_captured_args = _remote_init_func.captured_inputs
+
+    @function.Defun(dtypes.string)
+    def _next_func(string_handle):
+      multi_device_iterator = (
+          gen_dataset_ops.multi_device_iterator_from_string_handle(
+              string_handle=string_handle,
+              output_types=self._flat_output_types,
+              output_shapes=self._flat_output_shapes))
+      return gen_dataset_ops.multi_device_iterator_get_next_from_shard(
+          multi_device_iterator=multi_device_iterator,
+          shard_num=shard_num,
+          incarnation_id=incarnation_id,
+          output_types=self._flat_output_types,
+          output_shapes=self._flat_output_shapes)
+
+    @function.Defun(dtypes.string)
+    def _remote_next_func(string_handle):
+      return functional_ops.remote_call(
+          target=source_device,
+          args=[string_handle] + _next_func.captured_inputs,
+          Tout=self._flat_output_types,
+          f=_next_func)
+
+    self._next_func = _remote_next_func
+    self._next_captured_args = _remote_next_func.captured_inputs
+
+    @function.Defun(dtypes.string)
+    def _finalize_func(unused_string_handle):
+      return array_ops.constant(0, dtypes.int64)
+
+    @function.Defun(dtypes.string)
+    def _remote_finalize_func(string_handle):
+      return functional_ops.remote_call(
+          target=source_device,
+          args=[string_handle] + _finalize_func.captured_inputs,
+          Tout=[dtypes.int64],
+          f=_finalize_func)
+
+    self._finalize_func = _remote_finalize_func
+    self._finalize_captured_args = _remote_finalize_func.captured_inputs
+
+  def _as_variant_tensor(self):
+    with ops.device(self._target_device):
+      return core_gen_dataset_ops.generator_dataset(
+          self._init_captured_args,
+          self._next_captured_args,
+          self._finalize_captured_args,
+          init_func=self._init_func,
+          next_func=self._next_func,
+          finalize_func=self._finalize_func,
+          output_types=self._flat_output_types,
+          output_shapes=self._flat_output_shapes)
+
+  @property
+  def output_types(self):
+    return self._output_types
+
+  @property
+  def output_shapes(self):
+    return self._output_shapes
+
+  @property
+  def output_classes(self):
+    return self._output_classes
+
+
+class MultiDeviceIterator(object):
+  """An iterator over multiple devices."""
+
+  def __init__(self,
+               dataset,
+               devices,
+               prefetch_buffer_size=1,
+               source_device="/cpu:0"):
+    self._dataset = dataset
+    self._devices = devices
+    self._source_device = source_device
+    self._source_device_tensor = ops.convert_to_tensor(source_device)
+
+    self._flat_output_shapes = nest.flatten(
+        sparse.as_dense_shapes(self._dataset.output_shapes,
+                               self._dataset.output_classes))
+    self._flat_output_types = nest.flatten(
+        sparse.as_dense_types(self._dataset.output_types,
+                              self._dataset.output_classes))
+
+    # Create the MultiDeviceIterator.
+    with ops.device(self._source_device):
+      self._multi_device_iterator_resource = (
+          gen_dataset_ops.multi_device_iterator(
+              devices=self._devices,
+              shared_name="",
+              container="",
+              output_types=self._flat_output_types,
+              output_shapes=self._flat_output_shapes))
+
+      # The incarnation ID is used to ensure consistency between the per-device
+      # iterators and the multi-device iterator.
+      self._incarnation_id = gen_dataset_ops.multi_device_iterator_init(
+          self._dataset._as_variant_tensor(),  # pylint: disable=protected-access
+          self._multi_device_iterator_resource)
+
+    # TODO(rohanj): Explore the possibility of the MultiDeviceIterator to
+    # initialize the device side of the pipeline. This would allow the
+    # MultiDeviceIterator to choose, for example, to move some transformations
+    # into the device side from its input. It might be useful in rewriting.
+    # Create the per device iterators.
+    self._device_iterators = []
+    i = 0
+    for device in self._devices:
+      ds = _PerDeviceGenerator(
+          i, self._multi_device_iterator_resource, self._incarnation_id,
+          self._source_device_tensor, device, self._dataset.output_shapes,
+          self._dataset.output_types, self._dataset.output_classes)
+      ds = ds.prefetch(prefetch_buffer_size)
+      with ops.device(device):
+        self._device_iterators.append(ds.make_initializable_iterator())
+      i += 1
+
+    device_iterator_initializers = [
+        iterator.initializer for iterator in self._device_iterators
+    ]
+    self._initializer = control_flow_ops.group(*device_iterator_initializers)
+
+  def get_next(self):
+    result = []
+    i = 0
+    for device in self._devices:
+      with ops.device(device):
+        result.append(self._device_iterators[i].get_next())
+      i += 1
+    return result
+
+  @property
+  def initializer(self):
+    return self._initializer
-- 
cgit v1.2.3


From 3f74665d12d5adbc5dfb17d3616b9904af45b93a Mon Sep 17 00:00:00 2001
From: Billy Lamberta 
Date: Tue, 24 Jul 2018 11:31:56 -0700
Subject: Fix link

PiperOrigin-RevId: 205862584
---
 tensorflow/docs_src/guide/saved_model.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/docs_src/guide/saved_model.md b/tensorflow/docs_src/guide/saved_model.md
index acc3d3ca0b..717488e7cc 100644
--- a/tensorflow/docs_src/guide/saved_model.md
+++ b/tensorflow/docs_src/guide/saved_model.md
@@ -2,9 +2,8 @@
 
 The @{tf.train.Saver} class provides methods to save and restore models. The
 @{tf.saved_model.simple_save} function is an easy way to build a
-@{tf.saved_model$saved model} suitable for serving.
-[Estimators](@{$guide/estimators}) automatically save and restore
-variables in the `model_dir`.
+@{tf.saved_model$saved model} suitable for serving. [Estimators](./estimators)
+automatically save and restore variables in the `model_dir`.
 
 ## Save and restore variables
 
-- 
cgit v1.2.3


From badf913c0a2f83ca933b8fe73a29f7dd5d2bc5ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 11:45:30 -0700
Subject: New triangular solve algorithm.

PiperOrigin-RevId: 205865103
---
 tensorflow/compiler/tf2xla/lib/triangular_solve.cc | 828 ++++++++-------------
 tensorflow/compiler/tf2xla/lib/triangular_solve.h  |   6 -
 .../compiler/tf2xla/lib/triangular_solve_test.cc   |  64 +-
 tensorflow/compiler/xla/client/lib/numeric.cc      |  28 +
 tensorflow/compiler/xla/client/lib/numeric.h       |   9 +
 tensorflow/compiler/xla/client/lib/numeric_test.cc |  14 +
 6 files changed, 381 insertions(+), 568 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
index a2dd5a0d57..75c0ad7f7e 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
+++ b/tensorflow/compiler/tf2xla/lib/triangular_solve.cc
@@ -29,9 +29,307 @@ limitations under the License.
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/math/math_util.h"
 
 namespace tensorflow {
 
+// Get the diagonal blocks of the coefficient matrix
+xla::XlaOp DiagonalBlocks(xla::XlaOp a, int64 block_size) {
+  xla::XlaBuilder* builder = a.builder();
+  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr {
+    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(a));
+    int ndims = xla::ShapeUtil::Rank(shape);
+    int64 n = xla::ShapeUtil::GetDimension(shape, -1);
+    int64 num_blocks = n / block_size;
+
+    xla::XlaOp diag_blocks;
+
+    // If the coefficient matrix is exactly the block size, we just add a
+    // singleton dimension i.e. [..., n, n] -> [..., 1, n, n]
+    if (n == block_size) {
+      std::vector permutation(ndims);
+      std::iota(permutation.begin(), permutation.end(), 1);
+      permutation.insert(permutation.end() - 2, 0);
+      return Transpose(Broadcast(a, /*broadcast_sizes=*/{1}), permutation);
+    }
+
+    // We can grab entire blocks using gather
+    if (n > block_size) {
+      // Construct the starting indices of the diagonal blocks
+      auto gather_indices =
+          Transpose(Broadcast(Mul(Iota(builder, xla::S32, num_blocks),
+                                  xla::ConstantR0(builder, block_size)),
+                              /*broadcast_sizes=*/{2}),
+                    /*permutation=*/{1, 0});
+
+      // Gather the diagonal blocks
+      xla::GatherDimensionNumbers dim_numbers;
+      dim_numbers.add_output_window_dims(ndims - 1);
+      dim_numbers.add_output_window_dims(ndims);
+      dim_numbers.add_gather_dims_to_operand_dims(ndims - 2);
+      dim_numbers.add_gather_dims_to_operand_dims(ndims - 1);
+      dim_numbers.set_index_vector_dim(1);
+      diag_blocks = Gather(a, gather_indices, dim_numbers,
+                           /*window_bounds=*/{block_size, block_size});
+    }
+
+    // The last block might be smaller than the block size,
+    // so we will need to pad it
+    if (n % block_size != 0) {
+      // Pad with zeros
+      auto last_blocks =
+          SliceInMinorDims(a, {n - n % block_size, n - n % block_size}, {n, n});
+      xla::PaddingConfig config = xla::MakeNoPaddingConfig(ndims);
+      int64 padding = block_size - n % block_size;
+      config.mutable_dimensions(ndims - 1)->set_edge_padding_high(padding);
+      config.mutable_dimensions(ndims - 2)->set_edge_padding_high(padding);
+      last_blocks =
+          Pad(last_blocks, Zero(builder, shape.element_type()), config);
+
+      // Add a singleton dimension
+      // i.e. [..., block_size, block_size] -> [..., 1, block_size, block_size]
+      TF_ASSIGN_OR_RETURN(xla::Shape blocks_shape,
+                          builder->GetShape(last_blocks));
+      auto shape_dims = xla::AsInt64Slice(blocks_shape.dimensions());
+      auto last_blocks_dims = std::vector(ndims);
+      std::copy(shape_dims.begin(), shape_dims.end(), last_blocks_dims.begin());
+      last_blocks_dims.insert(last_blocks_dims.end() - 2, 1);
+      last_blocks = Reshape(last_blocks, last_blocks_dims);
+
+      // Concatenate with the other blocks if necessary
+      if (n > block_size) {
+        diag_blocks =
+            xla::ConcatInDim(builder, {diag_blocks, last_blocks}, ndims - 2);
+      } else {
+        diag_blocks = last_blocks;
+      }
+    }
+
+    return diag_blocks;
+  });
+}
+
+xla::XlaOp InvertDiagonalBlocks(xla::XlaOp diag_blocks, bool lower,
+                                bool transpose_a, bool conjugate_a) {
+  xla::XlaBuilder* builder = diag_blocks.builder();
+  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr {
+    // Input is a batch of square lower triangular square matrices. Its shape is
+    // (..., size, size). We resize this to (num_blocks, size, size).
+    TF_ASSIGN_OR_RETURN(xla::Shape shape, builder->GetShape(diag_blocks));
+    int64 block_size = xla::ShapeUtil::GetDimension(shape, -1);
+    int64 num_blocks = xla::ShapeUtil::ElementsIn(shape) /
+                       tensorflow::MathUtil::IPow(block_size, 2);
+    diag_blocks = Reshape(diag_blocks, {num_blocks, block_size, block_size});
+
+    // The input must be triangular because we rely on that when doing
+    // multiplications later on
+    diag_blocks = Triangle(diag_blocks, /*lower=*/lower);
+
+    // Rescale blocks to be unit triangular, but avoid dividing by
+    // zero (which can happen if the last block was padded) otherwise it will
+    // introduce nans which will propagate
+    auto diags = GetMatrixDiagonal(diag_blocks);
+    TF_ASSIGN_OR_RETURN(xla::Shape diags_shape, builder->GetShape(diags));
+    auto one = ScalarLike(diags, 1);
+    auto ones = Broadcast(one, xla::AsInt64Slice(diags_shape.dimensions()));
+    diags = Select(Eq(diags, Zero(builder, shape.element_type())), ones, diags);
+    auto scaled_diag_blocks = Div(diag_blocks, diags, {0, 2});
+
+    // We can now use the fact that for an upper triangular matrix
+    // [[L11, 0], [L21, L22]], given the inverses L11' and L22', we have
+    // L22' = -L22' * L21 * L11'. In our case, L21 is a vector and our blocks
+    // have been rescaled to be unit triangular, so L22 = L22' = 1.
+
+    // Initialize the output matrix with -1s on the diagonal. We use -1 instead
+    // of 1 because we cannot do matrix-vector multiplies with variable shapes
+    // inside of a loop, or do irregularly shaped in-place updates. Hence,
+    // L21 <- -L22 * L21 * L11 cannot be done naively. Instead, we update the
+    // entire row i.e. we calculate
+    // [L21 L22 0] <- -[L21 L22 0] @ diag_blocks([L11', -I, -I])
+    // which means [L21 L22 0] <- [-L21 * L11', L22, 0].
+    auto identity =
+        IdentityMatrix(builder, shape.element_type(), block_size, block_size);
+    auto neg_identity = -identity;
+
+    // The first or last  diagonal element should be set to 1 instead of -1
+    // though, since we never update it
+    auto pos_one = Reshape(One(builder, shape.element_type()), {1, 1});
+    auto start_index = (lower) ? 0 : block_size - 1;
+    auto output_block = DynamicUpdateSlice(
+        neg_identity, pos_one,
+        /*start_indices=*/xla::ConstantR1(builder, 2, start_index));
+
+    // Broadcast diag([1, -1, -1, ...]) to every block
+    xla::XlaOp output = Broadcast(output_block,
+                                  /*broadcast_sizes=*/{num_blocks});
+
+    // Now we construct a loop that performs matrix-vector multiplications
+    // inverting the blocks one row at a time
+    std::vector tuple_shapes = {
+        // The loop iteration counter is a scalar, incremented each iteration.
+        xla::ShapeUtil::MakeShape(xla::S32, {}),
+        // The output has the shape of A, with one row updated each iteration.
+        xla::ShapeUtil::MakeShape(shape.element_type(),
+                                  {num_blocks, block_size, block_size}),
+        // The input is a loop invariant.
+        xla::ShapeUtil::MakeShape(shape.element_type(),
+                                  {num_blocks, block_size, block_size})};
+    xla::Shape tuple_shape = xla::ShapeUtil::MakeTupleShape(tuple_shapes);
+
+    auto init_i = One(builder, xla::S32);
+    auto init = xla::Tuple(builder, {init_i, output, scaled_diag_blocks});
+
+    // Construct the loop condition function.
+    std::unique_ptr condb =
+        builder->CreateSubBuilder("InvertDiagCond");
+    {
+      auto i = GetTupleElement(
+          Parameter(condb.get(), 0, tuple_shape, "InvertDiagCondTuple"), 0);
+      Lt(i, xla::ConstantR0(condb.get(), block_size));
+    }
+    TF_ASSIGN_OR_RETURN(auto cond, condb->Build());
+
+    // Construct the loop body function.
+    std::unique_ptr bodyb =
+        builder->CreateSubBuilder("InvertDiagBody");
+    {
+      auto input_tuple =
+          Parameter(bodyb.get(), 0, tuple_shape, "InvertDiagBodyTuple");
+
+      auto i = GetTupleElement(input_tuple, 0);
+      auto body_out = GetTupleElement(input_tuple, 1);
+      auto body_input = GetTupleElement(input_tuple, 2);
+
+      auto zero = xla::ConstantR1(bodyb.get(), 1, 0);
+      auto j = (lower) ? i : ScalarLike(i, block_size - 1) - i;
+      auto start_indices =
+          xla::ConcatInDim(bodyb.get(), {zero, Reshape(j, {1}), zero}, 0);
+      auto input_row =
+          DynamicSlice(body_input, start_indices,
+                       /*slice_sizes=*/{num_blocks, 1, block_size});
+
+      // We want -L21 L11^{-1}
+      xla::DotDimensionNumbers dnums;
+      dnums.add_lhs_batch_dimensions(0);
+      dnums.add_rhs_batch_dimensions(0);
+      dnums.add_lhs_contracting_dimensions(2);
+      dnums.add_rhs_contracting_dimensions(1);
+      auto update = -DotGeneral(input_row, body_out, dnums);
+
+      body_out = DynamicUpdateSlice(body_out, update, start_indices);
+
+      auto next_i = i + ScalarLike(i, 1);
+      xla::Tuple(bodyb.get(), {next_i, body_out, body_input});
+    }
+    TF_ASSIGN_OR_RETURN(auto body, bodyb->Build());
+
+    // Construct the While loop and return the result,
+    // return while_loop(cond_fun, body_fun, init)[1]
+    auto invert_while = While(cond, body, init);
+    auto inv_diag_blocks = GetTupleElement(invert_while, 1);
+
+    // Undo the scaling
+    inv_diag_blocks = Div(inv_diag_blocks, diags,
+                          /*broadcast_dimensions=*/{0, 1});
+
+    // Reshape back to original batch major dimensions
+    return Reshape(inv_diag_blocks, xla::AsInt64Slice(shape.dimensions()));
+  });
+}
+
+xla::XlaOp SolveWithInvertedDiagonalBlocks(xla::XlaOp a, xla::XlaOp b,
+                                           xla::XlaOp inv_diag_blocks,
+                                           bool left_side, bool lower,
+                                           bool transpose_a, bool conjugate_a) {
+  xla::XlaBuilder* builder = a.builder();
+  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr {
+    TF_ASSIGN_OR_RETURN(xla::Shape blocks_shape,
+                        builder->GetShape(inv_diag_blocks));
+    TF_ASSIGN_OR_RETURN(xla::Shape b_shape, builder->GetShape(b));
+    int64 block_size = xla::ShapeUtil::GetDimension(blocks_shape, -1);
+
+    TF_ASSIGN_OR_RETURN(xla::Shape a_shape, builder->GetShape(a));
+    int64 ndims = xla::ShapeUtil::Rank(a_shape);
+    int64 n = xla::ShapeUtil::GetDimension(a_shape, -1);
+    int64 num_blocks = n / block_size + (n % block_size != 0);
+    int64 m_dim = (left_side) ? -1 : -2;
+    int64 m = xla::ShapeUtil::GetDimension(b_shape, m_dim);
+
+    // Initialize the solution
+    auto x = ZerosLike(b);
+
+    // This loop is unrolled for performance reasons, but it could be expressed
+    // rolled as well since the matrices are of the same size each iteration
+    for (int i = 0; i < num_blocks; i++) {
+      // High-level intuition: We have B[i] = L[i] @ X. Since L is upper
+      // triangular this means B[i] = L[i, :i + 1] @ X[:i + 1]. We can split
+      // this into two parts: B[i] = L[i, :i] @ X[:i] + L[i, i] @ X[i] which
+      // can be solved for X[i] as X[i] = inv(L[i, i]) @ B[i] - L[i, :i] @ X[:i]
+
+      // Decide whether we go from first block to last or vice versa
+      auto j = (left_side ^ lower ^ transpose_a) ? num_blocks - 1 - i : i;
+
+      // Get the size of the inverse blocks (the last one might be smaller)
+      int64 block = (n % block_size != 0 && j + 1 == num_blocks)
+                        ? n % block_size
+                        : block_size;
+      auto inv_block =
+          MaybeConjugate(Collapse(SliceInMinorDims(inv_diag_blocks, {j, 0, 0},
+                                                   {j + 1, block, block}),
+                                  /*dimensions=*/{ndims - 2, ndims - 1}),
+                         conjugate_a);
+
+      // Get the corresponding row of B
+      int64 k = std::min((j + 1) * block_size, n);
+      std::vector start = {j * block_size, 0};
+      std::vector end = {k, m};
+      if (!left_side) {
+        std::swap(start[0], start[1]);
+        std::swap(end[0], end[1]);
+      }
+      auto b_row = SliceInMinorDims(b, start, end);
+
+      xla::XlaOp remainder;
+      if (i == 0) {
+        remainder = b_row;
+      } else {
+        // This matrix multiply involves a lot of multiplying with zero (namely,
+        // X[i * block_size:] = 0), but this is faster than slicing...
+        end = {k, n};
+        if (!left_side) {
+          std::swap(end[0], end[1]);
+        }
+        if (transpose_a) {
+          std::swap(start[0], start[1]);
+          std::swap(end[0], end[1]);
+        }
+        auto a_row =
+            MaybeConjugate(SliceInMinorDims(a, start, end), conjugate_a);
+        if (left_side) {
+          remainder = b_row - BatchDot(a_row, x, transpose_a, false);
+        } else {
+          remainder = b_row - BatchDot(x, a_row, false, transpose_a);
+        }
+      }
+
+      xla::XlaOp x_update;
+      auto zero = Zero(builder, xla::S32);
+      auto start_index =
+          xla::ConstantR0WithType(builder, xla::S32, j * block_size);
+      std::vector update_starts = {start_index, zero};
+      if (left_side) {
+        x_update = BatchDot(inv_block, remainder, transpose_a, false);
+      } else {
+        x_update = BatchDot(remainder, inv_block, false, transpose_a);
+        std::swap(update_starts[0], update_starts[1]);
+      }
+      x = DynamicUpdateSliceInMinorDims(x, x_update, /*starts=*/update_starts);
+    }
+
+    return x;
+  });
+}
+
 xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side,
                            bool lower, bool transpose_a, bool conjugate_a,
                            int64 block_size) {
@@ -45,7 +343,7 @@ xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side,
           xla::ShapeUtil::HumanString(a_shape), " vs. ",
           xla::ShapeUtil::HumanString(b_shape));
     }
-    const int ndims = xla::ShapeUtil::Rank(a_shape);
+    const int64 ndims = xla::ShapeUtil::Rank(a_shape);
     if (ndims < 2) {
       return errors::InvalidArgument(
           "Arguments to TriangularSolve must have rank >= 2: ", ndims);
@@ -85,528 +383,18 @@ xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side,
           block_size);
     }
 
-    std::map base_computations;
-    auto get_base_triangular_solve =
-        [&](int k) -> xla::StatusOr {
-      xla::XlaComputation& computation = base_computations[k];
-      if (computation.IsNull()) {
-        std::unique_ptr sub = builder->CreateSubBuilder(
-            tensorflow::strings::StrCat("trsm_base_", k));
-
-        auto a_param = xla::Parameter(
-            sub.get(), 0,
-            xla::ShapeUtil::MakeShape(b_shape.element_type(),
-                                      ConcatVectors(batch_dimensions, {k, k})),
-            "a");
-
-        std::array b_lastd;
-        if (left_side) {
-          b_lastd = {k, n};
-        } else {
-          b_lastd = {m, k};
-        }
-        auto b_param = xla::Parameter(
-            sub.get(), 1,
-            xla::ShapeUtil::MakeShape(b_shape.element_type(),
-                                      ConcatVectors(batch_dimensions, b_lastd)),
-            "b");
-
-        // We use a left-looking or right-looking subroutine on the block
-        // diagonal in the lower=true cases, while falling back to a recursive
-        // call in others. The left-looking and right-looking subroutines are
-        // written with a While loop and so yields much faster compile times.
-        // Moreover, they can give higher performance on smaller (sub)problems.
-        if (left_side && lower) {
-          TriangularSolveLeftLooking(a_param, b_param, transpose_a,
-                                     conjugate_a);
-        } else if (!left_side && lower) {
-          TriangularSolveRightLooking(a_param, b_param, transpose_a,
-                                      conjugate_a);
-        } else {
-          TriangularSolve(a_param, b_param, left_side, lower, transpose_a,
-                          conjugate_a,
-                          /*block_size=*/1);
-        }
-
-        TF_ASSIGN_OR_RETURN(computation, sub->Build());
-      }
-      return &computation;
-    };
-
-    xla::XlaOp output = xla::ZerosLike(b);
-
-    // Right-looking blocked triangular solve.
-    // For an explanation of the algorithm, see the TRSM discussion in:
-    // Goto, Kazushige, and Robert Van De Geijn. "High-performance
-    // implementation of the level-3 BLAS." ACM Transactions on Mathematical
-    // Software (TOMS) 35.1 (2008): 4.
-
-    // In the code comments below, T = lambda x: np.swapaxes(x, -1, -2) if
-    // conjugate_a is False, or T = lambda x: np.conj(np.swapaxes(x, -1, -2)) if
-    // conjugate_a is True.
-
-    if (!left_side && lower == transpose_a) {
-      // for i in range(0, a.shape[-1], block_size):
-      for (int64 i = 0; i < n; i += block_size) {
-        int64 k = std::min(block_size, n - i);
-
-        // output[..., :, i:i+k] = triangular_solve(
-        //     a[..., i:i+k, i:i+k],
-        //     b[..., :, i:i+k] - np.matmul(output[..., :, :i],
-        //                                  a[..., :i, i:i+k]),
-        //     ..., block_size=1)
-        auto a_slice = SliceInMinorDims(a, {i, i}, {i + k, i + k});
-        auto b_slice = SliceInMinorDims(b, {0, i}, {m, i + k});
-
-        // Note that we multiply with the full output, since this is faster
-        // than slicing, and output[..., :, i:] = 0
-        xla::XlaOp a_prev;
-        if (lower) {
-          a_prev = SliceInMinorDims(a, {i, 0}, {i + k, n});
-        } else {
-          a_prev = SliceInMinorDims(a, {0, i}, {n, i + k});
-        }
-        auto prev_contribution = BatchDot(output, a_prev,
-                                          /*transpose_x=*/false,
-                                          /*transpose_y=*/transpose_a,
-                                          /*conjugate_x=*/false,
-                                          /*conjugate_y=*/conjugate_a);
-        auto to_solve = b_slice - prev_contribution;
-
-        xla::XlaOp update;
-        if (k > 1) {
-          TF_ASSIGN_OR_RETURN(xla::XlaComputation * solve,
-                              get_base_triangular_solve(k));
-          update = xla::Call(builder, *solve, {a_slice, to_solve});
-        } else {
-          auto a_slice_conj = MaybeConjugate(a_slice, conjugate_a);
-          update = to_solve / a_slice_conj;
-        }
-        output = UpdateSliceInMinorDims(output, update, {0, i});
-      }
-
-    } else if (left_side && lower != transpose_a) {
-      // for i in range(0, a.shape[-1], block_size):
-      for (int64 i = 0; i < m; i += block_size) {
-        int64 k = std::min(block_size, m - i);
-
-        // output[..., i:i+k, :] = triangular_solve(
-        //     a[..., i:i+k, i:i+k],
-        //     b[..., i:i+k, :] - np.matmul(a[..., i:i+k, :i],
-        //                                  output[..., :i, :]),
-        //     ..., block_size=1)
-        auto a_slice = SliceInMinorDims(a, {i, i}, {i + k, i + k});
-        auto b_slice = SliceInMinorDims(b, {i, 0}, {i + k, n});
-
-        xla::XlaOp a_prev;
-        if (lower) {
-          a_prev = SliceInMinorDims(a, {i, 0}, {i + k, m});
-        } else {
-          a_prev = SliceInMinorDims(a, {0, i}, {m, i + k});
-        }
-        auto prev_contribution = BatchDot(a_prev, output,
-                                          /*transpose_x=*/transpose_a,
-                                          /*transpose_y=*/false,
-                                          /*conjugate_x=*/conjugate_a,
-                                          /*conjugate_y=*/false);
-        auto to_solve = b_slice - prev_contribution;
-
-        xla::XlaOp update;
-        if (k > 1) {
-          TF_ASSIGN_OR_RETURN(xla::XlaComputation * solve,
-                              get_base_triangular_solve(k));
-          update = xla::Call(builder, *solve, {a_slice, to_solve});
-        } else {
-          auto a_slice_conj = MaybeConjugate(a_slice, conjugate_a);
-          update = to_solve / a_slice_conj;
-        }
-        output = UpdateSliceInMinorDims(output, update, {i, 0});
-      }
-    } else if (!left_side && lower != transpose_a) {
-      // for i in reversed(range(0, a.shape[-1], block_size)):
-      const int64 last_blk_ix =
-          xla::RoundUpToNearest(n, block_size) - block_size;
-      for (int64 i = last_blk_ix; i >= 0; i -= block_size) {
-        int64 k = std::min(block_size, n - i);
-
-        // output[..., :, i:i+k] = triangular_solve(
-        //     a[..., i:i+k, i:i+k],
-        //     b[..., :, i:i+k] - np.matmul(output[..., :, :i],
-        //                                  a[..., :i, i:i+k]),\
-        //     ..., block_size=1)
-        auto a_slice = SliceInMinorDims(a, {i, i}, {i + k, i + k});
-        auto b_slice = SliceInMinorDims(b, {0, i}, {m, i + k});
-
-        xla::XlaOp a_prev;
-        if (lower) {
-          a_prev = SliceInMinorDims(a, {0, i}, {n, i + k});
-        } else {
-          a_prev = SliceInMinorDims(a, {i, 0}, {i + k, n});
-        }
-        auto prev_contribution = BatchDot(output, a_prev,
-                                          /*transpose_x=*/false,
-                                          /*transpose_y=*/transpose_a,
-                                          /*conjugate_x=*/false,
-                                          /*conjugate_y=*/conjugate_a);
-        auto to_solve = b_slice - prev_contribution;
-
-        xla::XlaOp update;
-        if (k > 1) {
-          TF_ASSIGN_OR_RETURN(xla::XlaComputation * solve,
-                              get_base_triangular_solve(k));
-          update = xla::Call(builder, *solve, {a_slice, to_solve});
-        } else {
-          auto a_slice_conj = MaybeConjugate(a_slice, conjugate_a);
-          update = to_solve / a_slice_conj;
-        }
-        output = UpdateSliceInMinorDims(output, update, {0, i});
-      }
-    } else {  // left_side && lower == transpose_a
-      // for i in reversed(range(0, a.shape[-1], block_size)):
-      const int64 last_blk_ix =
-          xla::RoundUpToNearest(m, block_size) - block_size;
-      for (int64 i = last_blk_ix; i >= 0; i -= block_size) {
-        int64 k = std::min(block_size, m - i);
-
-        // output[..., i:i+k, :] = triangular_solve(
-        //     a[..., i:i+k, i:i+k],
-        //     b[..., i:i+k, :] - np.matmul(a[..., i:i+k, :i],
-        //                                  output[..., :i, :]),
-        //     ..., block_size=1)
-        auto a_slice = SliceInMinorDims(a, {i, i}, {i + k, i + k});
-        auto b_slice = SliceInMinorDims(b, {i, 0}, {i + k, n});
-
-        xla::XlaOp a_prev;
-        if (lower) {
-          a_prev = SliceInMinorDims(a, {0, i}, {m, i + k});
-        } else {
-          a_prev = SliceInMinorDims(a, {i, 0}, {i + k, m});
-        }
-        auto prev_contribution = BatchDot(a_prev, output,
-                                          /*transpose_x=*/transpose_a,
-                                          /*transpose_y=*/false,
-                                          /*conjugate_x=*/conjugate_a,
-                                          /*conjugate_y=*/false);
-        auto to_solve = b_slice - prev_contribution;
-
-        xla::XlaOp update;
-        if (k > 1) {
-          TF_ASSIGN_OR_RETURN(xla::XlaComputation * solve,
-                              get_base_triangular_solve(k));
-          update = xla::Call(builder, *solve, {a_slice, to_solve});
-        } else {
-          auto a_slice_conj = MaybeConjugate(a_slice, conjugate_a);
-          update = to_solve / a_slice_conj;
-        }
-        output = UpdateSliceInMinorDims(output, update, {i, 0});
-      }
-    }
-
-    return output;
-  });
-}
-
-xla::XlaOp TriangularSolveLeftLooking(xla::XlaOp a, xla::XlaOp b,
-                                      bool transpose_a, bool conjugate_a) {
-  xla::XlaBuilder* builder = a.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr {
-    TF_ASSIGN_OR_RETURN(xla::Shape a_shape, builder->GetShape(a));
-    TF_ASSIGN_OR_RETURN(xla::Shape b_shape, builder->GetShape(b));
-    const int64 m = xla::ShapeUtil::GetDimension(b_shape, -2);
-    const int64 n = xla::ShapeUtil::GetDimension(b_shape, -1);
-    const int64 ndims = xla::ShapeUtil::Rank(a_shape);
-
-    std::vector batch_dimensions;
-    int64 num_batches = 1;
-    for (int i = 0; i < ndims - 2; ++i) {
-      int64 a_size = a_shape.dimensions(i);
-      batch_dimensions.push_back(a_size);
-      num_batches = num_batches * a_size;
-    }
-
-    // Rescale the input to be unit triangular
-    auto diag = xla::GetMatrixDiagonal(a);
-    xla::XlaOp scaled_a;
-    std::vector broadcast_dimensions(ndims - 1);
-    std::iota(broadcast_dimensions.begin(), broadcast_dimensions.end(), 0);
-    if (transpose_a) {
-      scaled_a = Div(a, diag, broadcast_dimensions);
-    } else {
-      // Broadcast over the rows
-      broadcast_dimensions[ndims - 2] = ndims - 1;
-      scaled_a = Div(a, diag, broadcast_dimensions);
-    }
-
-    // The main computation is performed in a While loop.
-
-    // Allocate the output and set its first or last row,
-    // output = np.zeros_like(b)
-    // if transpose_a:
-    //   output[..., m-1:, :] = b[..., m-1:, :] / a[..., m-1:, m-1:]
-    // else:
-    //   output[..., :1, :] = b[..., :1, :] / a[..., :1, :1]
-    xla::XlaOp output = xla::ZerosLike(b);
-    {
-      auto i = transpose_a ? m - 1 : 0;
-      auto a_slice = SliceInMinorDims(scaled_a, {i, i}, {i + 1, i + 1});
-      auto b_slice = SliceInMinorDims(b, {i, 0}, {i + 1, n});
-      auto a_slice_conj = MaybeConjugate(a_slice, conjugate_a);
-      auto update = b_slice / a_slice_conj;
-      output = UpdateSliceInMinorDims(output, update, {i, 0});
-    }
-
-    // Construct the initial loop carry tuple,
-    // if transpose_a:
-    //   init = (m-2, output, a, b)
-    // else:
-    //   init = (1, output, a, b)
-    std::vector tuple_shapes = {
-        // The loop iteration counter is a scalar, incremented each iteration.
-        xla::ShapeUtil::MakeShape(xla::S32, {}),
-        // The output has the shape of b, with one row updated each iteration.
-        b_shape,
-        // The coefficient matrix a is a loop invariant.
-        a_shape,
-        // The right-hand-side matrix b is a loop invariant.
-        b_shape};
-    xla::Shape tuple_shape = xla::ShapeUtil::MakeTupleShape(tuple_shapes);
-    auto init_i = xla::ConstantR0(builder, transpose_a ? m - 2 : 1);
-    auto init = xla::Tuple(builder, {init_i, output, scaled_a, b});
-
-    // Construct the loop condition function,
-    // def cond_fun(loop_carry):
-    //   i, output, a, b = loop_carry
-    //   return i >= 0 if transpose_a else i < m
-    std::unique_ptr condb =
-        builder->CreateSubBuilder("TriangularSolveLeftLookingWhileCond");
-    {
-      auto i = xla::GetTupleElement(
-          xla::Parameter(condb.get(), 0, tuple_shape,
-                         "TriangularSolveLeftLookingWhileTuple"),
-          0);
-      if (transpose_a) {
-        xla::Ge(i, xla::ConstantR0(condb.get(), 0));
-      } else {
-        xla::Lt(i, xla::ConstantR0(condb.get(), m));
-      }
-    }
-    TF_ASSIGN_OR_RETURN(auto cond, condb->Build());
-
-    // Construct the loop body function,
-    // def body_fun(loop_carry):
-    //   i, output, a, b = loop_carry
-    //   if transpose_a:
-    //     a_row = np.swapaxes(a[..., i+1:, i:i+1], -1 -2)
-    //   else:
-    //     a_row = a[..., i:i+1, :i]
-    //   result_row = b[..., i:i+1, :] - np.matmul(a_row, output[..., :, :])
-    //   output[..., i:i+1, :] = result_row / a[..., i:i+1, i:i+1]
-    //   if transpose_a:
-    //     return (i - 1, output, a, b)
-    //   else:
-    //     return (i + 1, output, a, b)
-    // We have to do some extra FLOPs propagating zeros in the matrix multiply
-    // because we can't have the size of its arguments depend on the loop
-    // counter.
-    std::unique_ptr bodyb =
-        builder->CreateSubBuilder("TriangularSolveLeftLookingWhileBody");
-    {
-      auto input_tuple = xla::Parameter(bodyb.get(), 0, tuple_shape,
-                                        "TriangularSolveLeftLookingWhileTuple");
-
-      // i, output, a, b = loop_carry
-      auto i = xla::GetTupleElement(input_tuple, 0);
-      auto body_out = xla::GetTupleElement(input_tuple, 1);
-      auto body_a = xla::GetTupleElement(input_tuple, 2);
-      auto body_b = xla::GetTupleElement(input_tuple, 3);
-      auto zero = xla::ConstantR0(bodyb.get(), 0);
-
-      // We'd like to implement this:
-      //   if transpose_a:
-      //     a_row = T(a[..., i+1:, i:i+1])
-      //     result_row = (b[..., i:i+1, :]
-      //                   - np.matmul(a_row, body_out[..., i+1:, :]))
-      //   else:
-      //     result_row = (b[..., i:i+1, :]
-      //                   - np.matmul(a[..., i:i+1, :i], body_out[..., :i, :]))
-      // But since we can't have intermediate array sizes depend on the loop
-      // counter, we instead exploit the fact that we initialized the output to
-      // all zeros and use that as zero-padding (doing unnecessary FLOPs).
-      xla::XlaOp a_row;
-      if (transpose_a) {
-        a_row = DynamicSliceInMinorDims(body_a, {zero, i}, {m, 1});
-      } else {
-        a_row = DynamicSliceInMinorDims(body_a, {i, zero}, {1, m});
-      }
-      auto b_update = BatchDot(a_row, body_out,
-                               /*transpose_x=*/transpose_a,
-                               /*transpose_y=*/false,
-                               /*conjugate_x=*/conjugate_a,
-                               /*conjugate_y=*/false);
-      auto result_row_slice =
-          DynamicSliceInMinorDims(body_b, {i, zero}, {1, n});
-      auto result_row = result_row_slice - b_update;
-
-      // body_out[..., i:i+1, :] = result_row
-      body_out = DynamicUpdateSliceInMinorDims(body_out, result_row, {i, zero});
-
-      // if transpose_a:
-      //   return (i - 1, body_out, a, b)
-      // else:
-      //   return (i + 1, body_out, a, b)
-      auto next_i = xla::Add(
-          i, xla::ConstantR0(bodyb.get(), transpose_a ? -1 : 1));
-      xla::Tuple(bodyb.get(), {next_i, body_out, body_a, body_b});
-    }
-    TF_ASSIGN_OR_RETURN(auto body, bodyb->Build());
-
-    // Construct the While loop and return the result,
-    // return while_loop(cond_fun, body_fun, init)[1]
-    auto triangular_solve_left_looking_while = xla::While(cond, body, init);
-    output = xla::GetTupleElement(triangular_solve_left_looking_while, 1);
-    auto scaling = MaybeConjugate(diag, conjugate_a);
-    // Broadcast over the columns
-    broadcast_dimensions[ndims - 2] = ndims - 2;
-    return Div(output, scaling, broadcast_dimensions);
-  });
-}
-
-xla::XlaOp TriangularSolveRightLooking(xla::XlaOp a, xla::XlaOp b,
-                                       bool transpose_a, bool conjugate_a) {
-  xla::XlaBuilder* builder = a.builder();
-  return builder->ReportErrorOrReturn([&]() -> xla::StatusOr {
-    TF_ASSIGN_OR_RETURN(xla::Shape a_shape, builder->GetShape(a));
-    TF_ASSIGN_OR_RETURN(xla::Shape b_shape, builder->GetShape(b));
-    const int64 m = xla::ShapeUtil::GetDimension(b_shape, -2);
-    const int64 n = xla::ShapeUtil::GetDimension(b_shape, -1);
-    const int64 ndims = xla::ShapeUtil::Rank(a_shape);
-
-    std::vector batch_dimensions;
-    int64 num_batches = 1;
-    for (int i = 0; i < ndims - 2; ++i) {
-      int64 a_size = a_shape.dimensions(i);
-      batch_dimensions.push_back(a_size);
-      num_batches = num_batches * a_size;
-    }
+    // We find the diagonal blocks of the coefficient matrix
+    auto diag_blocks = DiagonalBlocks(a, block_size);
 
-    // Rescale the input to be unit triangular
-    auto diag = xla::GetMatrixDiagonal(a);
-    xla::XlaOp scaled_a;
-    std::vector broadcast_dimensions(ndims - 1);
-    std::iota(broadcast_dimensions.begin(), broadcast_dimensions.end(), 0);
-    if (transpose_a) {
-      // Broadcast over the rows
-      broadcast_dimensions[ndims - 2] = ndims - 1;
-      scaled_a = Div(a, diag, broadcast_dimensions);
-    } else {
-      scaled_a = Div(a, diag, broadcast_dimensions);
-    }
+    // We invert these blocks in parallel using batched matrix-vector products
+    auto inv_diag_blocks =
+        InvertDiagonalBlocks(diag_blocks, lower, transpose_a, conjugate_a);
 
-    // The main computation is performed in a While loop.
-    xla::XlaOp output = xla::ZerosLike(b);
+    // We now find the solution using GEMMs
+    auto x = SolveWithInvertedDiagonalBlocks(a, b, inv_diag_blocks, left_side,
+                                             lower, transpose_a, conjugate_a);
 
-    // Construct the initial loop carry tuple,
-    // if transpose_a:
-    //   init = (0, output, a, b)
-    // else:
-    //   init = (n-1, output, a, b)
-    std::vector tuple_shapes = {
-        // The loop iteration counter is a scalar, incremented each iteration.
-        xla::ShapeUtil::MakeShape(xla::S32, {}),
-        // The output has the shape of b, with one row updated each iteration.
-        b_shape,
-        // The coefficient matrix a is a loop invariant.
-        a_shape,
-        // The right-hand-side matrix b is a loop invariant.
-        b_shape};
-    xla::Shape tuple_shape = xla::ShapeUtil::MakeTupleShape(tuple_shapes);
-    auto init_i = xla::ConstantR0(builder, transpose_a ? 0 : n - 1);
-    auto init = xla::Tuple(builder, {init_i, output, scaled_a, b});
-
-    // Construct the loop condition function,
-    // def cond_fun(loop_carry):
-    //   i, output, a, b = loop_carry
-    //   return i < n if transpose_a else i >= 0
-    std::unique_ptr condb =
-        builder->CreateSubBuilder("TriangularSolveRightLookingWhileCond");
-    {
-      auto i = xla::GetTupleElement(
-          xla::Parameter(condb.get(), 0, tuple_shape,
-                         "TriangularSolveRightLookingWhileTuple"),
-          0);
-      if (transpose_a) {
-        xla::Lt(i, xla::ConstantR0(condb.get(), n));
-      } else {
-        xla::Ge(i, xla::ConstantR0(condb.get(), 0));
-      }
-    }
-    TF_ASSIGN_OR_RETURN(auto cond, condb->Build());
-
-    // Construct the loop body function,
-    // def body_fun(loop_carry):
-    //   i, output, a, b = loop_carry
-    //   if transpose_a:
-    //     a_row = np.swapaxes(a[..., :, i:i+1], -1, -2)
-    //   else:
-    //     a_row = a[..., :, i:i+1]
-    //   result_row = b[..., :, i:i+1] - np.matmul(output, a_row)
-    //   output[..., :, i:i+1] = result_row / a[..., i:i+1, i:i+1]
-    //   if transpose_a:
-    //     return (i - 1, output, a, b)
-    //   else:
-    //     return (i + 1, output, a, b)
-    // We have to do some extra FLOPs propagating zeros in the matrix multiply
-    // because we can't have the size of its arguments depend on the loop
-    // counter.
-    std::unique_ptr bodyb =
-        builder->CreateSubBuilder("TriangularSolveRightLookingWhileBody");
-    {
-      auto input_tuple = xla::Parameter(
-          bodyb.get(), 0, tuple_shape, "TriangularSolveRightLookingWhileTuple");
-
-      // i, output, a, b = loop_carry
-      auto i = xla::GetTupleElement(input_tuple, 0);
-      auto body_out = xla::GetTupleElement(input_tuple, 1);
-      auto body_a = xla::GetTupleElement(input_tuple, 2);
-      auto body_b = xla::GetTupleElement(input_tuple, 3);
-      auto zero = xla::ConstantR0(bodyb.get(), 0);
-
-      // result = b - np.matmul(output, a)
-      // result_row = result[..., :, i:i+1]
-      auto body_b_slice = DynamicSliceInMinorDims(body_b, {zero, i}, {m, 1});
-      xla::XlaOp a_slice;
-      if (transpose_a) {
-        a_slice = DynamicSliceInMinorDims(body_a, {i, zero}, {1, n});
-      } else {
-        a_slice = DynamicSliceInMinorDims(body_a, {zero, i}, {n, 1});
-      }
-      auto b_update = body_b_slice - BatchDot(body_out, a_slice,
-                                              /*transpose_x=*/false,
-                                              /*transpose_y=*/transpose_a,
-                                              /*conjugate_x=*/false,
-                                              /*conjugate_y=*/conjugate_a);
-
-      // body_out[..., :, i:i+1] = b_update
-      body_out = DynamicUpdateSliceInMinorDims(body_out, b_update, {zero, i});
-
-      // if transpose_a:
-      //   return (i + 1, body_out, a, b)
-      // else:
-      //   return (i - 1, body_out, a, b)
-      auto next_i = xla::Add(
-          i, xla::ConstantR0(bodyb.get(), transpose_a ? 1 : -1));
-      xla::Tuple(bodyb.get(), {next_i, body_out, body_a, body_b});
-    }
-    TF_ASSIGN_OR_RETURN(auto body, bodyb->Build());
-
-    // Construct the While loop and return the result,
-    // return while_loop(cond_fun, body_fun, init)[1]
-    auto triangular_solve_left_looking_while = xla::While(cond, body, init);
-    output = xla::GetTupleElement(triangular_solve_left_looking_while, 1);
-    auto scaling = MaybeConjugate(diag, conjugate_a);
-    // Broadcast over the rows
-    broadcast_dimensions[ndims - 2] = ndims - 1;
-    return Div(output, scaling, broadcast_dimensions);
+    return x;
   });
 }
 
diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve.h b/tensorflow/compiler/tf2xla/lib/triangular_solve.h
index 7eb9238014..2dce620ba8 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve.h
+++ b/tensorflow/compiler/tf2xla/lib/triangular_solve.h
@@ -61,12 +61,6 @@ xla::XlaOp TriangularSolve(xla::XlaOp a, xla::XlaOp b, bool left_side,
                            bool lower, bool transpose_a, bool conjugate_a,
                            int64 block_size = 128);
 
-xla::XlaOp TriangularSolveLeftLooking(xla::XlaOp a, xla::XlaOp b,
-                                      bool transpose_a, bool conjugate_a);
-
-xla::XlaOp TriangularSolveRightLooking(xla::XlaOp a, xla::XlaOp b,
-                                       bool transpose_a, bool conjugate_a);
-
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_COMPILER_TF2XLA_LIB_TRIANGULAR_SOLVE_H_
diff --git a/tensorflow/compiler/tf2xla/lib/triangular_solve_test.cc b/tensorflow/compiler/tf2xla/lib/triangular_solve_test.cc
index f1bff6037b..a29496dec4 100644
--- a/tensorflow/compiler/tf2xla/lib/triangular_solve_test.cc
+++ b/tensorflow/compiler/tf2xla/lib/triangular_solve_test.cc
@@ -207,6 +207,28 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerNotranspose) {
                              xla::ErrorSpec(1e-2, 1e-2));
 }
 
+XLA_TEST_F(TriangularSolveTest, SimpleLeftLowerNotransposeIrregularblock) {
+  xla::XlaBuilder builder(TestName());
+
+  xla::XlaOp a, b;
+  auto a_data = CreateR2Parameter(AValsLower(), 0, "a", &builder, &a);
+  auto b_data = CreateR2Parameter(BValsLeft(), 1, "b", &builder, &b);
+  TriangularSolve(a, b,
+                  /*left_side=*/true, /*lower=*/true,
+                  /*transpose_a=*/false, /*conjugate_a=*/false,
+                  /*block_size=*/3);
+
+  xla::Array2D expected({
+      {0.5, 1.0, 1.5},
+      {0.41666667, 0.33333333, 0.25},
+      {0.23148148, 0.18518519, 0.13888889},
+      {0.16835017, 0.13468013, 0.1010101},
+  });
+
+  ComputeAndCompareR2(&builder, expected, {a_data.get(), b_data.get()},
+                             xla::ErrorSpec(1e-2, 1e-2));
+}
+
 XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperTranspose) {
   xla::XlaBuilder builder(TestName());
 
@@ -307,47 +329,5 @@ XLA_TEST_F(TriangularSolveTest, SimpleLeftUpperTransposeNoconjugate) {
                                  xla::ErrorSpec(1e-2, 1e-2));
 }
 
-XLA_TEST_F(TriangularSolveLeftLookingTest, Simple) {
-  xla::XlaBuilder builder(TestName());
-
-  xla::XlaOp a, b;
-  auto a_data = CreateR2Parameter(AValsLower(), 0, "a", &builder, &a);
-  auto b_data = CreateR2Parameter(BValsLeft(), 1, "b", &builder, &b);
-  TriangularSolveLeftLooking(a, b,
-                             /*transpose_a=*/false,
-                             /*conjugate_a=*/false);
-
-  xla::Array2D expected({
-      {0.5, 1.0, 1.5},
-      {0.41666667, 0.33333333, 0.25},
-      {0.23148148, 0.18518519, 0.13888889},
-      {0.16835017, 0.13468013, 0.1010101},
-  });
-
-  ComputeAndCompareR2(&builder, expected, {a_data.get(), b_data.get()},
-                             xla::ErrorSpec(1e-2, 1e-2));
-}
-
-XLA_TEST_F(TriangularSolveLeftLookingTest, NonzeroUpperTriangle) {
-  xla::XlaBuilder builder(TestName());
-
-  xla::XlaOp a, b;
-  auto a_data = CreateR2Parameter(AValsFull(), 0, "a", &builder, &a);
-  auto b_data = CreateR2Parameter(BValsLeft(), 1, "b", &builder, &b);
-  TriangularSolveLeftLooking(a, b,
-                             /*transpose_a=*/false,
-                             /*conjugate_a=*/false);
-
-  xla::Array2D expected({
-      {0.5, 1.0, 1.5},
-      {0.41666667, 0.33333333, 0.25},
-      {0.23148148, 0.18518519, 0.13888889},
-      {0.16835017, 0.13468013, 0.1010101},
-  });
-
-  ComputeAndCompareR2(&builder, expected, {a_data.get(), b_data.get()},
-                             xla::ErrorSpec(1e-2, 1e-2));
-}
-
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/xla/client/lib/numeric.cc b/tensorflow/compiler/xla/client/lib/numeric.cc
index a6e460aa75..1c91237ae1 100644
--- a/tensorflow/compiler/xla/client/lib/numeric.cc
+++ b/tensorflow/compiler/xla/client/lib/numeric.cc
@@ -106,4 +106,32 @@ XlaOp GetMatrixDiagonal(XlaOp x) {
   });
 }
 
+XlaOp Triangle(XlaOp x, bool lower) {
+  XlaBuilder* builder = x.builder();
+  return builder->ReportErrorOrReturn([&]() -> StatusOr {
+    TF_ASSIGN_OR_RETURN(Shape shape, builder->GetShape(x));
+    const int64 n_dims = ShapeUtil::Rank(shape);
+    TF_RET_CHECK(n_dims >= 2);
+    const int64 m = shape.dimensions(n_dims - 2);
+    const int64 n = shape.dimensions(n_dims - 1);
+    tensorflow::gtl::ArraySlice major_dims(
+        AsInt64Slice(shape.dimensions()), /*pos=*/0, /*len=*/n_dims - 2);
+    auto a = Iota(builder, U32, n);
+    auto b = Iota(builder, U32, m);
+    xla::XlaOp indicator;
+    if (lower) {
+      indicator = Ge(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0});
+    } else {
+      indicator = Le(b, Broadcast(a, {m}), /*broadcast_dimensions=*/{0});
+    }
+    auto mask = Broadcast(indicator, major_dims);
+
+    return Select(mask, x, Zeros(builder, shape));
+  });
+}
+
+XlaOp UpperTriangle(XlaOp x) { return Triangle(x, false); }
+
+XlaOp LowerTriangle(XlaOp x) { return Triangle(x, true); }
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/numeric.h b/tensorflow/compiler/xla/client/lib/numeric.h
index e9037b722c..212f658313 100644
--- a/tensorflow/compiler/xla/client/lib/numeric.h
+++ b/tensorflow/compiler/xla/client/lib/numeric.h
@@ -34,6 +34,15 @@ XlaOp IdentityMatrix(XlaBuilder* builder, PrimitiveType type, int64 m, int64 n);
 // diagonal elements (i.e., with indices [..., i, i]).
 XlaOp GetMatrixDiagonal(XlaOp x);
 
+// Get the upper or lower triangle part of the last two dimensions
+XlaOp Triangle(XlaOp x, bool lower);
+
+// Get the upper triangle part of the last two dimensions
+XlaOp UpperTriangle(XlaOp x);
+
+// Get the lower triangle part of the last two dimensions
+XlaOp LowerTriangle(XlaOp x);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_NUMERIC_H_
diff --git a/tensorflow/compiler/xla/client/lib/numeric_test.cc b/tensorflow/compiler/xla/client/lib/numeric_test.cc
index 113d359197..f56cadc547 100644
--- a/tensorflow/compiler/xla/client/lib/numeric_test.cc
+++ b/tensorflow/compiler/xla/client/lib/numeric_test.cc
@@ -40,6 +40,20 @@ XLA_TEST_F(NumericTest, Iota) {
   ComputeAndCompareR1(&builder, {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, {});
 }
 
+XLA_TEST_F(NumericTest, Triangle) {
+  XlaBuilder builder(TestName());
+  Array3D input(2, 3, 4);
+  input.FillIota(0);
+
+  XlaOp a;
+  auto a_data = CreateR3Parameter(input, 0, "a", &builder, &a);
+  LowerTriangle(a);
+  Array3D expected({{{0, 0, 0, 0}, {4, 5, 0, 0}, {8, 9, 10, 0}},
+                           {{12, 0, 0, 0}, {16, 17, 0, 0}, {20, 21, 22, 0}}});
+
+  ComputeAndCompareR3(&builder, expected, {a_data.get()});
+}
+
 template 
 void NumericTest::TestMatrixDiagonal() {
   XlaBuilder builder("GetMatrixDiagonal");
-- 
cgit v1.2.3


From ff2aa1b59d4a111af094c0c7724e453eefe1f3b7 Mon Sep 17 00:00:00 2001
From: Billy Lamberta 
Date: Tue, 24 Jul 2018 11:52:23 -0700
Subject: Setup for TFLite subsite

PiperOrigin-RevId: 205866236
---
 tensorflow/contrib/lite/g3doc/README.md            |   4 +
 tensorflow/contrib/lite/g3doc/_book.yaml           |  58 +++
 tensorflow/contrib/lite/g3doc/_index.yaml          |  67 +++
 tensorflow/contrib/lite/g3doc/_project.yaml        |  10 +
 .../contrib/lite/g3doc/api_docs/python/_toc.yaml   |   6 +
 .../contrib/lite/g3doc/api_docs/python/index.md    |  10 +
 tensorflow/contrib/lite/g3doc/apis.md              |   3 +
 tensorflow/contrib/lite/g3doc/benchmarks.md        | 178 --------
 tensorflow/contrib/lite/g3doc/custom_operators.md  |   3 +
 tensorflow/contrib/lite/g3doc/demo_android.md      | 149 ++++++
 tensorflow/contrib/lite/g3doc/demo_ios.md          |  71 +++
 tensorflow/contrib/lite/g3doc/devguide.md          | 235 ++++++++++
 tensorflow/contrib/lite/g3doc/ios.md               |   3 +
 tensorflow/contrib/lite/g3doc/models.md            |   3 +
 tensorflow/contrib/lite/g3doc/ops_versioning.md    |   3 +
 tensorflow/contrib/lite/g3doc/overview.md          | 204 +++++++++
 tensorflow/contrib/lite/g3doc/performance.md       | 177 ++++++++
 tensorflow/contrib/lite/g3doc/rpi.md               |   3 +
 .../contrib/lite/g3doc/tf_ops_compatibility.md     |   3 +
 .../contrib/lite/g3doc/tfmobile/android_build.md   | 181 ++++++++
 tensorflow/contrib/lite/g3doc/tfmobile/index.md    | 284 ++++++++++++
 .../contrib/lite/g3doc/tfmobile/ios_build.md       | 110 +++++
 .../contrib/lite/g3doc/tfmobile/linking_libs.md    | 256 +++++++++++
 .../contrib/lite/g3doc/tfmobile/optimizing.md      | 504 +++++++++++++++++++++
 .../contrib/lite/g3doc/tfmobile/prepare_models.md  | 304 +++++++++++++
 tensorflow/docs_src/mobile/README.md               |   3 +
 tensorflow/docs_src/mobile/android_build.md        | 177 --------
 tensorflow/docs_src/mobile/index.md                |  33 --
 tensorflow/docs_src/mobile/ios_build.md            | 107 -----
 tensorflow/docs_src/mobile/leftnav_files           |  15 -
 tensorflow/docs_src/mobile/linking_libs.md         | 243 ----------
 tensorflow/docs_src/mobile/mobile_intro.md         | 248 ----------
 tensorflow/docs_src/mobile/optimizing.md           | 499 --------------------
 tensorflow/docs_src/mobile/prepare_models.md       | 301 ------------
 tensorflow/docs_src/mobile/tflite/demo_android.md  | 146 ------
 tensorflow/docs_src/mobile/tflite/demo_ios.md      |  68 ---
 tensorflow/docs_src/mobile/tflite/devguide.md      | 232 ----------
 tensorflow/docs_src/mobile/tflite/index.md         | 201 --------
 tensorflow/docs_src/mobile/tflite/performance.md   | 174 -------
 39 files changed, 2654 insertions(+), 2622 deletions(-)
 create mode 100644 tensorflow/contrib/lite/g3doc/README.md
 create mode 100644 tensorflow/contrib/lite/g3doc/_book.yaml
 create mode 100644 tensorflow/contrib/lite/g3doc/_index.yaml
 create mode 100644 tensorflow/contrib/lite/g3doc/_project.yaml
 create mode 100644 tensorflow/contrib/lite/g3doc/api_docs/python/_toc.yaml
 create mode 100644 tensorflow/contrib/lite/g3doc/api_docs/python/index.md
 delete mode 100644 tensorflow/contrib/lite/g3doc/benchmarks.md
 create mode 100644 tensorflow/contrib/lite/g3doc/demo_android.md
 create mode 100644 tensorflow/contrib/lite/g3doc/demo_ios.md
 create mode 100644 tensorflow/contrib/lite/g3doc/devguide.md
 create mode 100644 tensorflow/contrib/lite/g3doc/overview.md
 create mode 100644 tensorflow/contrib/lite/g3doc/performance.md
 create mode 100644 tensorflow/contrib/lite/g3doc/tfmobile/android_build.md
 create mode 100644 tensorflow/contrib/lite/g3doc/tfmobile/index.md
 create mode 100644 tensorflow/contrib/lite/g3doc/tfmobile/ios_build.md
 create mode 100644 tensorflow/contrib/lite/g3doc/tfmobile/linking_libs.md
 create mode 100644 tensorflow/contrib/lite/g3doc/tfmobile/optimizing.md
 create mode 100644 tensorflow/contrib/lite/g3doc/tfmobile/prepare_models.md
 create mode 100644 tensorflow/docs_src/mobile/README.md
 delete mode 100644 tensorflow/docs_src/mobile/android_build.md
 delete mode 100644 tensorflow/docs_src/mobile/index.md
 delete mode 100644 tensorflow/docs_src/mobile/ios_build.md
 delete mode 100644 tensorflow/docs_src/mobile/leftnav_files
 delete mode 100644 tensorflow/docs_src/mobile/linking_libs.md
 delete mode 100644 tensorflow/docs_src/mobile/mobile_intro.md
 delete mode 100644 tensorflow/docs_src/mobile/optimizing.md
 delete mode 100644 tensorflow/docs_src/mobile/prepare_models.md
 delete mode 100644 tensorflow/docs_src/mobile/tflite/demo_android.md
 delete mode 100644 tensorflow/docs_src/mobile/tflite/demo_ios.md
 delete mode 100644 tensorflow/docs_src/mobile/tflite/devguide.md
 delete mode 100644 tensorflow/docs_src/mobile/tflite/index.md
 delete mode 100644 tensorflow/docs_src/mobile/tflite/performance.md

diff --git a/tensorflow/contrib/lite/g3doc/README.md b/tensorflow/contrib/lite/g3doc/README.md
new file mode 100644
index 0000000000..e3db478481
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/README.md
@@ -0,0 +1,4 @@
+This is a *work-in-progress* TF Lite subsite for:
+https://www.tensorflow.org/mobile
+
+DO NOT PUBLISH
diff --git a/tensorflow/contrib/lite/g3doc/_book.yaml b/tensorflow/contrib/lite/g3doc/_book.yaml
new file mode 100644
index 0000000000..98abd5743b
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/_book.yaml
@@ -0,0 +1,58 @@
+upper_tabs:
+# Tabs left of dropdown menu
+- include: /_upper_tabs_left.yaml
+# Dropdown menu
+- name: Ecosystem
+  path: /ecosystem
+  is_default: True
+  menu:
+  - include: /ecosystem/_menu_toc.yaml
+  lower_tabs:
+    # Subsite tabs
+    other:
+    - name: Guide
+      contents:
+      - title: Overview
+        path: /mobile/overview
+      - title: Developer Guide
+        path: /mobile/devguide
+      - title: Android Demo App
+        path: /mobile/demo_android
+      - title: iOS Demo App
+        path: /mobile/demo_ios
+      - title: Performance
+        path: /mobile/performance
+      - break: True
+      - title: TensorFlow Lite APIs
+        path: /mobile/apis
+      - title: Custom operators
+        path: /mobile/custom_operators
+      - title: TensorFlow Lite Ops Versioning
+        path: /mobile/ops_versioning
+      - title: TensorFlow Lite Compatibility Guide
+        path: /mobile/tf_ops_compatibility
+      - title: List of Hosted Models
+        path: /mobile/models
+      - title: TensorFlow Lite for iOS
+        path: /mobile/ios
+      - title: TensorFlow Lite for Raspberry Pi
+        path: /mobile/rpi
+
+      - heading: TF Mobile
+        status: deprecated
+      - title: Overview
+        path: /mobile/tfmobile/
+      - title: Building TensorFlow on Android
+        path: /mobile/tfmobile/android_build
+      - title: Building TensorFlow on IOS
+        path: /mobile/tfmobile/ios_build
+      - title: Integrating TensorFlow libraries
+        path: /mobile/tfmobile/linking_libs
+      - title: Preparing models for mobile deployment
+        path: /mobile/tfmobile/prepare_models
+      - title: Optimizing for mobile
+        path: /mobile/tfmobile/optimizing
+
+    - name: API
+      contents:
+      - include: /mobile/api_docs/python/_toc.yaml
diff --git a/tensorflow/contrib/lite/g3doc/_index.yaml b/tensorflow/contrib/lite/g3doc/_index.yaml
new file mode 100644
index 0000000000..9119e49117
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/_index.yaml
@@ -0,0 +1,67 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+description: 
+landing_page:
+  rows:
+  - heading: TensorFlow Lite is a lightweight solution for mobile and embedded devices.
+    items:
+    - description: >
+        TensorFlow Lite is TensorFlow’s lightweight solution for mobile and
+        embedded devices. It enables on-device machine learning inference with
+        low latency and a small binary size. TensorFlow Lite also supports
+        hardware acceleration with the
+        Android Neural Networks API.
+      list:
+      - heading: Key point 1
+        description: >
+          [high-level overview]
+        icon:
+          icon_name: chevron_right
+          foreground: theme
+          background: grey
+      - heading: Key point 2
+        description: >
+          [high-level overview]
+        icon:
+          icon_name: chevron_right
+          foreground: theme
+          background: grey
+      - heading: Key point 3
+        description: >
+          [high-level overview]
+        icon:
+          icon_name: chevron_right
+          foreground: theme
+          background: grey
+    - code_block: |
+        
    +        $ toco --input_file=$(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \
+               --input_format=TENSORFLOW_GRAPHDEF \
+               --output_format=TFLITE \
+               --output_file=/tmp/mobilenet_v1_1.0_224.tflite \
+               --inference_type=FLOAT \
+               --input_type=FLOAT \
+               --input_arrays=input \
+               --output_arrays=MobilenetV1/Predictions/Reshape_1 \
+               --input_shapes=1,224,224,3
+        
    
+
+  - classname: devsite-landing-row-cards
+    items:
+    - heading: Using TensorFlow Lite on Android
+      image_path: /ecosystem/images/tf-logo-card-16x9.png
+      path: https://medium.com/tensorflow/using-tensorflow-lite-on-android-9bbc9cb7d69d
+      buttons:
+      - label: Read on TensorFlow blog
+        path: https://medium.com/tensorflow/using-tensorflow-lite-on-android-9bbc9cb7d69d
+    - heading: TensorFlow Lite at the Dev Summit
+      youtube_id: FAMfy7izB6A
+      buttons:
+      - label: Watch the video
+        path: https://www.youtube.com/watch?v=FAMfy7izB6A
+    - heading: TensorFlow Lite on GitHub
+      image_path: /ecosystem/images/github-card-16x9.png
+      path: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite
+      buttons:
+      - label: View on GitHub
+        path: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite
diff --git a/tensorflow/contrib/lite/g3doc/_project.yaml b/tensorflow/contrib/lite/g3doc/_project.yaml
new file mode 100644
index 0000000000..b39666516b
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/_project.yaml
@@ -0,0 +1,10 @@
+name: TensorFlow Lite
+breadcrumb_name: Mobile
+home_url: /mobile/
+parent_project_metadata_path: /_project.yaml
+description: >
+  TensorFlow Lite is a lightweight solution for mobile and embedded devices.
+use_site_branding: True
+hide_from_products_list: True
+content_license: cc3-apache2
+buganizer_id: 316308
diff --git a/tensorflow/contrib/lite/g3doc/api_docs/python/_toc.yaml b/tensorflow/contrib/lite/g3doc/api_docs/python/_toc.yaml
new file mode 100644
index 0000000000..1e1c44c692
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/api_docs/python/_toc.yaml
@@ -0,0 +1,6 @@
+# Automatically generated file; please do not edit
+toc:
+  - title: TensorFlow Lite
+    section:
+    - title: Overview
+      path: /mobile/api_docs/python/
diff --git a/tensorflow/contrib/lite/g3doc/api_docs/python/index.md b/tensorflow/contrib/lite/g3doc/api_docs/python/index.md
new file mode 100644
index 0000000000..70031a3c3d
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/api_docs/python/index.md
@@ -0,0 +1,10 @@
+Project: /mobile/_project.yaml
+Book: /mobile/_book.yaml
+page_type: reference
+
+
+
+
+# All symbols in TensorFlow Lite
+
+TEMP PAGE
diff --git a/tensorflow/contrib/lite/g3doc/apis.md b/tensorflow/contrib/lite/g3doc/apis.md
index e94a2cc44e..776803da8c 100644
--- a/tensorflow/contrib/lite/g3doc/apis.md
+++ b/tensorflow/contrib/lite/g3doc/apis.md
@@ -1,3 +1,6 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
 # TensorFlow Lite APIs
 
 TensorFlow Lite provides programming APIs in C++ and Java, and in both cases
diff --git a/tensorflow/contrib/lite/g3doc/benchmarks.md b/tensorflow/contrib/lite/g3doc/benchmarks.md
deleted file mode 100644
index 96536cba27..0000000000
--- a/tensorflow/contrib/lite/g3doc/benchmarks.md
+++ /dev/null
@@ -1,178 +0,0 @@
-# Performance Benchmark numbers
-
-This document contains the performance benchmark numbers for running a few well
-known models on some Android and iOS devices.
-
-The benchmark numbers were generated by running the [TFLite benchmark
-binary](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark)
-on Android and running the [iOS benchmark
-app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark/ios)
-on iOS.
-
-# Android benchmarks
-
-When running Android benchmarks, the CPU affinity is set to use big cores on the
-device to reduce variance (see
-[details](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark#reducing-variance-between-runs-on-android)).
-
-Models are assumed to have been downloaded from the link, unzipped and pushed to
-`/data/local/tmp/tflite_models` folder. The benchmark binary is built according
-to instructions listed
-[here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark#on-android)
-and is assumed to have been pushed to `/data/local/tmp`.
-
-The following command was used to run the benchmark:
-
-```
-adb shell taskset ${CPU_MASK} /data/local/tmp/benchmark_model \
-  --num_threads=1 \
-  --graph=/data/local/tmp/tflite_models/${GRAPH} \
-  --warmup_runs=1 \
-  --num_runs=50 \
-  --use_nnapi=false
-```
-
-where `${GRAPH}` is the name of model and `${CPU_MASK}` is the CPU affinity
-chosen according to the following table:
-
-Device | CPU_MASK |
--------| ----------
-Pixel 2 | f0 |
-Pixel xl | 0c |
-
-
-
-  
-    
-      
-      
-      
-    
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-
- 
    Model NameDevice Mean inference time (std dev)
    
-      Mobilenet_1.0_224(float)
-    Pixel 2 166.5 ms (2.6 ms)
    Pixel xl 122.9 ms (1.8 ms)  
    
-      Mobilenet_1.0_224 (quant)
-    Pixel 2 69.5 ms (0.9 ms)
    Pixel xl 78.9 ms (2.2 ms)  
    
-      NASNet mobile
-    Pixel 2 273.8 ms (3.5 ms)
    Pixel xl 210.8 ms (4.2 ms)
    
-      SqueezeNet
-    Pixel 2 234.0 ms (2.1 ms)
    Pixel xl 158.0 ms (2.1 ms)
    
-      Inception_ResNet_V2
-    Pixel 2 2846.0 ms (15.0 ms)
    Pixel xl 1973.0 ms (15.0 ms)  
    
-      Inception_V4
-    Pixel 2 3180.0 ms (11.7 ms)
    Pixel xl 2262.0 ms (21.0 ms)  
    
-
-# iOS benchmarks
-
-For running iOS benchmarks, the [benchmark
-app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark/ios)
-was modified to include the appropriate model and `benchmark_params.json` was
-modified  to set `num_threads` to 1.
-
-
-  
-    
-      
-      
-      
-    
-  
-  
-    
-    
-    
-  
-  
-    
-    
-    
-  
-  
-    
-    
-    
-  
-  
-    
-    
-    
-  
-  
-    
-    
-    
-  
-  
-    
-    
-    
-  
- 
    Model NameDevice Mean inference time (std dev)
    
-      Mobilenet_1.0_224(float)
-    iPhone 8 32.2 ms (0.8 ms)
    
-      Mobilenet_1.0_224 (quant)
-    iPhone 8 24.4 ms (0.8 ms)
    
-      NASNet mobile
-    iPhone 8 60.3 ms (0.6 ms)
    
-      SqueezeNet
-    iPhone 8 44.3 (0.7 ms)
    
-      Inception_ResNet_V2
-    iPhone 8562.4 ms (18.2 ms)
    
-      Inception_V4
-    iPhone 8 661.0 ms (29.2 ms)
    
diff --git a/tensorflow/contrib/lite/g3doc/custom_operators.md b/tensorflow/contrib/lite/g3doc/custom_operators.md
index f2fbcf64cf..2296f5a064 100644
--- a/tensorflow/contrib/lite/g3doc/custom_operators.md
+++ b/tensorflow/contrib/lite/g3doc/custom_operators.md
@@ -1,3 +1,6 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
 # How to use custom operators
 
 TensorFlow Lite currently supports a subset of TensorFlow operators. However, it
diff --git a/tensorflow/contrib/lite/g3doc/demo_android.md b/tensorflow/contrib/lite/g3doc/demo_android.md
new file mode 100644
index 0000000000..d79a2696b4
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/demo_android.md
@@ -0,0 +1,149 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# Android Demo App
+
+An example Android application using TensorFLow Lite is available
+[on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo).
+The demo is a sample camera app that classifies images continuously
+using either a quantized Mobilenet model or a floating point Inception-v3 model.
+To run the demo, a device running Android 5.0 ( API 21) or higher is required.
+
+In the demo app, inference is done using the TensorFlow Lite Java API. The demo
+app classifies frames in real-time, displaying the top most probable
+classifications. It also displays the time taken to detect the object.
+
+There are three ways to get the demo app to your device:
+
+* Download the [prebuilt binary APK](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
+* Use Android Studio to build the application.
+* Download the source code for TensorFlow Lite and the demo and build it using
+  bazel.
+
+
+## Download the pre-built binary
+
+The easiest way to try the demo is to download the
+[pre-built binary APK](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk)
+
+Once the APK is installed, click the app icon to start the program. The first
+time the app is opened, it asks for runtime permissions to access the device
+camera. The demo app opens the back-camera of the device and recognizes objects
+in the camera's field of view. At the bottom of the image (or at the left
+of the image if the device is in landscape mode), it displays top three objects
+classified and the classification latency.
+
+
+## Build in Android Studio with TensorFlow Lite AAR from JCenter
+
+Use Android Studio to try out changes in the project code and compile the demo
+app:
+
+* Install the latest version of
+  [Android Studio](https://developer.android.com/studio/index.html).
+* Make sure the Android SDK version is greater than 26 and NDK version is greater
+  than 14 (in the Android Studio settings).
+* Import the `tensorflow/contrib/lite/java/demo` directory as a new
+  Android Studio project.
+* Install all the Gradle extensions it requests.
+
+Now you can build and run the demo app. 
+
+The build process downloads the quantized [Mobilenet TensorFlow Lite model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip), and unzips it into the assets directory: `tensorflow/contrib/lite/java/demo/app/src/main/assets/`.
+
+Some additional details are available on the
+[TF Lite Android App page](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md).
+
+### Using other models
+
+To use a different model:
+* Download the floating point [Inception-v3 model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip).
+* Unzip and copy `inceptionv3_non_slim_2015.tflite` to the assets directory. 
+* Change the chosen classifier in [Camera2BasicFragment.java](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java)
    
+  from: `classifier = new ImageClassifierQuantizedMobileNet(getActivity());`
    
+  to: `classifier = new ImageClassifierFloatInception(getActivity());`.
+
+
+## Build TensorFlow Lite and the demo app from source
+
+### Clone the TensorFlow repo
+
+```sh
+git clone https://github.com/tensorflow/tensorflow
+```
+
+### Install Bazel
+
+If `bazel` is not installed on your system, see
+[Installing Bazel](https://bazel.build/versions/master/docs/install.html).
+
+Note: Bazel does not currently support Android builds on Windows. Windows users
+should download the
+[prebuilt binary](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
+
+### Install Android NDK and SDK
+
+The Android NDK is required to build the native (C/C++) TensorFlow Lite code. The
+current recommended version is *14b* and can be found on the
+[NDK Archives](https://developer.android.com/ndk/downloads/older_releases.html#ndk-14b-downloads)
+page.
+
+The Android SDK and build tools can be
+[downloaded separately](https://developer.android.com/tools/revisions/build-tools.html)
+or used as part of
+[Android Studio](https://developer.android.com/studio/index.html). To build the
+TensorFlow Lite Android demo, build tools require API >= 23 (but it will run on
+devices with API >= 21).
+
+In the root of the TensorFlow repository, update the `WORKSPACE` file with the
+`api_level` and location of the SDK and NDK. If you installed it with
+Android Studio, the SDK path can be found in the SDK manager. The default NDK
+path is:`{SDK path}/ndk-bundle.` For example:
+
+```
+android_sdk_repository (
+    name = "androidsdk",
+    api_level = 23,
+    build_tools_version = "23.0.2",
+    path = "/home/xxxx/android-sdk-linux/",
+)
+
+android_ndk_repository(
+    name = "androidndk",
+    path = "/home/xxxx/android-ndk-r10e/",
+    api_level = 19,
+)
+```
+
+Some additional details are available on the
+[TF Lite Android App page](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md).
+
+### Build the source code
+
+To build the demo app, run `bazel`:
+
+```
+bazel build --cxxopt=--std=c++11 //tensorflow/contrib/lite/java/demo/app/src/main:TfLiteCameraDemo
+```
+
+Caution: Because of an bazel bug, we only support building the Android demo app
+within a Python 2 environment.
+
+
+## About the demo
+
+The demo app is resizing each camera image frame (224 width * 224 height) to
+match the quantized MobileNets model (299 * 299 for Inception-v3). The resized
+image is converted—row by row—into a
+[ByteBuffer](https://developer.android.com/reference/java/nio/ByteBuffer.html).
+Its size is  1 * 224 * 224 * 3 bytes, where 1 is the number of images in a batch.
+224 * 224 (299 * 299) is the width and height of the image. 3 bytes represents
+the 3 colors of a pixel.
+
+This demo uses the TensorFlow Lite Java inference API
+for models which take a single input and provide a single output. This outputs a
+two-dimensional array, with the first dimension being the category index and the
+second dimension being the confidence of classification. Both models have 1001
+unique categories and the app sorts the probabilities of all the categories and
+displays the top three. The model file must be downloaded and bundled within the
+assets directory of the app.
diff --git a/tensorflow/contrib/lite/g3doc/demo_ios.md b/tensorflow/contrib/lite/g3doc/demo_ios.md
new file mode 100644
index 0000000000..a554898899
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/demo_ios.md
@@ -0,0 +1,71 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# iOS Demo App
+
+The TensorFlow Lite demo is a camera app that continuously classifies whatever
+it sees from your device's back camera, using a quantized MobileNet model. These
+instructions walk you through building and running the demo on an iOS device.
+
+## Prerequisites
+
+* You must have [Xcode](https://developer.apple.com/xcode/) installed and have a
+  valid Apple Developer ID, and have an iOS device set up and linked to your
+  developer account with all of the appropriate certificates. For these
+  instructions, we assume that you have already been able to build and deploy an
+  app to an iOS device with your current developer environment.
+
+* The demo app requires a camera and must be executed on a real iOS device. You
+  can build it and run with the iPhone Simulator but it won't have any camera
+  information to classify.
+
+* You don't need to build the entire TensorFlow library to run the demo, but you
+  will need to clone the TensorFlow repository if you haven't already:
+
+        git clone https://github.com/tensorflow/tensorflow
+
+* You'll also need the Xcode command-line tools:
+
+        xcode-select --install
+
+    If this is a new install, you will need to run the Xcode application once to
+    agree to the license before continuing.
+
+## Building the iOS Demo App
+
+1. Install CocoaPods if you don't have it:
+
+        sudo gem install cocoapods
+
+2. Download the model files used by the demo app (this is done from inside the
+   cloned directory):
+
+        sh tensorflow/contrib/lite/examples/ios/download_models.sh
+
+3. Install the pod to generate the workspace file:
+
+        cd tensorflow/contrib/lite/examples/ios/camera
+        pod install
+
+    If you have installed this pod before and that command doesn't work, try
+
+        pod update
+
+    At the end of this step you should have a file called 
+    `tflite_camera_example.xcworkspace`.
+
+4. Open the project in Xcode by typing this on the command line:
+
+        open tflite_camera_example.xcworkspace
+
+    This launches Xcode if it isn't open already and opens the
+    `tflite_camera_example` project.
+
+5. Build and run the app in Xcode.
+
+    Note that as mentioned earlier, you must already have a device set up and
+    linked to your Apple Developer account in order to deploy the app on a
+    device.
+
+You'll have to grant permissions for the app to use the device's camera. Point
+the camera at various objects and enjoy seeing how the model classifies things!
diff --git a/tensorflow/contrib/lite/g3doc/devguide.md b/tensorflow/contrib/lite/g3doc/devguide.md
new file mode 100644
index 0000000000..dc9cc98c08
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/devguide.md
@@ -0,0 +1,235 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# Developer Guide
+
+Using a TensorFlow Lite model in your mobile app requires multiple
+considerations: you must choose a pre-trained or custom model, convert the model
+to a TensorFLow Lite format, and finally, integrate the model in your app.
+
+## 1. Choose a model
+
+Depending on the use case, you can choose one of the popular open-sourced models,
+such as *InceptionV3* or *MobileNets*, and re-train these models with a custom
+data set or even build your own custom model.
+
+### Use a pre-trained model
+
+[MobileNets](https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html)
+is a family of mobile-first computer vision models for TensorFlow designed to
+effectively maximize accuracy, while taking into consideration the restricted
+resources for on-device or embedded applications. MobileNets are small,
+low-latency, low-power models parameterized to meet the resource constraints for
+a variety of uses. They can be used for classification, detection, embeddings, and
+segmentation—similar to other popular large scale models, such as
+[Inception](https://arxiv.org/pdf/1602.07261.pdf). Google provides 16 pre-trained
+[ImageNet](http://www.image-net.org/challenges/LSVRC/) classification checkpoints
+for MobileNets that can be used in mobile projects of all sizes.
+
+[Inception-v3](https://arxiv.org/abs/1512.00567) is an image recognition model
+that achieves fairly high accuracy recognizing general objects with 1000 classes,
+for example, "Zebra", "Dalmatian", and "Dishwasher". The model extracts general
+features from input images using a convolutional neural network and classifies
+them based on those features with fully-connected and softmax layers.
+
+[On Device Smart Reply](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html)
+is an on-device model that provides one-touch replies for incoming text messages
+by suggesting contextually relevant messages. The model is built specifically for
+memory constrained devices, such as watches and phones, and has been successfully
+used in Smart Replies on Android Wear. Currently, this model is Android-specific.
+
+These pre-trained models are [available for download](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/models.md)
+
+### Re-train Inception-V3 or MobileNet for a custom data set
+
+These pre-trained models were trained on the *ImageNet* data set which contains
+1000 predefined classes. If these classes are not sufficient for your use case,
+the model will need to be re-trained. This technique is called
+*transfer learning* and starts with a model that has been already trained on a
+problem, then retrains the model on a similar problem. Deep learning from
+scratch can take days, but transfer learning is fairly quick. In order to do
+this, you need to generate a custom data set labeled with the relevant classes.
+
+The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/)
+codelab walks through the re-training process step-by-step. The code supports
+both floating point and quantized inference.
+
+### Train a custom model
+
+A developer may choose to train a custom model using Tensorflow (see the
+[TensorFlow tutorials](../../tutorials/) for examples of building and training
+models). If you have already written a model, the first step is to export this
+to a `tf.GraphDef` file. This is required because some formats do not store the
+model structure outside the code, and we must communicate with other parts of the
+framework. See
+[Exporting the Inference Graph](https://github.com/tensorflow/models/blob/master/research/slim/README.md)
+to create .pb file for the custom model.
+
+TensorFlow Lite currently supports a subset of TensorFlow operators. Refer to the
+[TensorFlow Lite & TensorFlow Compatibility Guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md)
+for supported operators and their usage. This set of operators will continue to
+grow in future Tensorflow Lite releases.
+
+
+## 2. Convert the model format
+
+The model generated (or downloaded) in the previous step is a *standard*
+Tensorflow model and you should now have a .pb or .pbtxt `tf.GraphDef` file.
+Models generated with transfer learning (re-training) or custom models must be
+converted—but, we must first freeze the graph to convert the model to the
+Tensorflow Lite format. This process uses several model formats:
+
+* `tf.GraphDef` (.pb) —A protobuf that represents the TensorFlow training or
+  computation graph. It contains operators, tensors, and variables definitions.
+* *CheckPoint* (.ckpt) —Serialized variables from a TensorFlow graph. Since this
+  does not contain a graph structure, it cannot be interpreted by itself.
+* `FrozenGraphDef` —A subclass of `GraphDef` that does not contain
+  variables. A `GraphDef` can be converted to a `FrozenGraphDef` by taking a
+  CheckPoint and a `GraphDef`, and converting each variable into a constant
+  using the value retrieved from the CheckPoint.
+* `SavedModel` —A `GraphDef` and CheckPoint with a signature that labels
+  input and output arguments to a model. A `GraphDef` and CheckPoint can be
+  extracted from a `SavedModel`.
+* *TensorFlow Lite model* (.tflite) —A serialized
+  [FlatBuffer](https://google.github.io/flatbuffers/) that contains TensorFlow
+  Lite operators and tensors for the TensorFlow Lite interpreter, similar to a
+  `FrozenGraphDef`.
+
+### Freeze Graph
+
+To use the `GraphDef` .pb file with TensorFlow Lite, you must have checkpoints
+that contain trained weight parameters. The .pb file only contains the structure
+of the graph. The process of merging the checkpoint values with the graph
+structure is called *freezing the graph*.
+
+You should have a checkpoints folder or download them for a pre-trained model
+(for example,
+[MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md)).
+
+To freeze the graph, use the following command (changing the arguments):
+
+```
+freeze_graph --input_graph=/tmp/mobilenet_v1_224.pb \
+  --input_checkpoint=/tmp/checkpoints/mobilenet-10202.ckpt \
+  --input_binary=true \
+  --output_graph=/tmp/frozen_mobilenet_v1_224.pb \
+  --output_node_names=MobileNetV1/Predictions/Reshape_1
+```
+
+The `input_binary` flag must be enabled so the protobuf is read and written in
+a binary format. Set the `input_graph` and `input_checkpoint` files.
+
+The `output_node_names` may not be obvious outside of the code that built the
+model. The easiest way to find them is to visualize the graph, either with
+[TensorBoard](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/#3)
+or `graphviz`.
+
+The frozen `GraphDef` is now ready for conversion to the `FlatBuffer` format
+(.tflite) for use on Android or iOS devices. For Android, the Tensorflow
+Optimizing Converter tool supports both float and quantized models. To convert
+the frozen `GraphDef` to the .tflite format:
+
+```
+toco --input_file=$(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \
+  --input_format=TENSORFLOW_GRAPHDEF \
+  --output_format=TFLITE \
+  --output_file=/tmp/mobilenet_v1_1.0_224.tflite \
+  --inference_type=FLOAT \
+  --input_type=FLOAT \
+  --input_arrays=input \
+  --output_arrays=MobilenetV1/Predictions/Reshape_1 \
+  --input_shapes=1,224,224,3
+```
+
+The `input_file` argument should reference the frozen `GraphDef` file
+containing the model architecture. The [frozen_graph.pb](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz)
+file used here is available for download. `output_file` is where the TensorFlow
+Lite model will get generated. The `input_type` and `inference_type`
+arguments should be set to `FLOAT`, unless converting a
+quantized model.
+Setting the `input_array`, `output_array`, and `input_shape` arguments are not as
+straightforward. The easiest way to find these values is to explore the graph
+using Tensorboard. Reuse the arguments for specifying the output nodes for
+inference in the `freeze_graph` step.
+
+It is also possible to use the Tensorflow Optimizing Converter with protobufs
+from either Python or from the command line (see the 
+[toco_from_protos.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/python/toco_from_protos.py)
+example). This allows you to integrate the conversion step into the model design
+workflow, ensuring the model is easily convertible to a mobile inference graph.
+For example:
+
+```python
+import tensorflow as tf
+
+img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3))
+val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.])
+out = tf.identity(val, name="out")
+
+with tf.Session() as sess:
+  tflite_model = tf.contrib.lite.toco_convert(sess.graph_def, [img], [out])
+  open("converteds_model.tflite", "wb").write(tflite_model)
+```
+
+For usage, see the Tensorflow Optimizing Converter
+[command-line examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md).
+
+Refer to the
+[Ops compatibility guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md)
+for troubleshooting help, and if that doesn't help, please
+[file an issue](https://github.com/tensorflow/tensorflow/issues).
+
+The [development repo](https://github.com/tensorflow/tensorflow) contains a tool
+to visualize TensorFlow Lite models after conversion. To build the
+[visualize.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/tools/visualize.py)
+tool:
+
+```sh
+bazel run tensorflow/contrib/lite/tools:visualize -- model.tflite model_viz.html
+```
+
+This generates an interactive HTML page listing subgraphs, operations, and a
+graph visualization.
+
+
+## 3. Use the TensorFlow Lite model for inference in a mobile app
+
+After completing the prior steps, you should now have a `.tflite` model file.
+
+### Android
+
+Since Android apps are written in Java and the core TensorFlow library is in C++,
+a JNI library is provided as an interface. This is only meant for inference—it
+provides the ability to load a graph, set up inputs, and run the model to
+calculate outputs.
+
+The open source Android demo app uses the JNI interface and is available
+[on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app).
+You can also download a
+[prebuilt APK](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
+See the Android demo guide for details.
+
+The Android mobile guide has instructions for
+installing TensorFlow on Android and setting up `bazel` and Android Studio.
+
+### iOS
+
+To integrate a TensorFlow model in an iOS app, see the
+[TensorFlow Lite for iOS](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/ios.md)
+guide and iOS demo guide.
+
+#### Core ML support
+
+Core ML is a machine learning framework used in Apple products. In addition to
+using Tensorflow Lite models directly in your applications, you can convert
+trained Tensorflow models to the
+[CoreML](https://developer.apple.com/machine-learning/) format for use on Apple
+devices. To use the converter, refer to the
+[Tensorflow-CoreML converter documentation](https://github.com/tf-coreml/tf-coreml).
+
+### Raspberry Pi
+
+Compile Tensorflow Lite for a Raspberry Pi by following the
+[RPi build instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/rpi.md)
+This compiles a static library file (`.a`) used to build your app. There are
+plans for Python bindings and a demo app.
diff --git a/tensorflow/contrib/lite/g3doc/ios.md b/tensorflow/contrib/lite/g3doc/ios.md
index e0358a444d..d78d373ccf 100644
--- a/tensorflow/contrib/lite/g3doc/ios.md
+++ b/tensorflow/contrib/lite/g3doc/ios.md
@@ -1,3 +1,6 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
 # TensorFlow Lite for iOS
 
 ## Building
diff --git a/tensorflow/contrib/lite/g3doc/models.md b/tensorflow/contrib/lite/g3doc/models.md
index 4e7d33a1b6..3292aece0e 100644
--- a/tensorflow/contrib/lite/g3doc/models.md
+++ b/tensorflow/contrib/lite/g3doc/models.md
@@ -1,3 +1,6 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
 # List of Hosted Models
 
 ## Image classification (Float Models)
diff --git a/tensorflow/contrib/lite/g3doc/ops_versioning.md b/tensorflow/contrib/lite/g3doc/ops_versioning.md
index bd2f797e6c..b06f4fd3b8 100644
--- a/tensorflow/contrib/lite/g3doc/ops_versioning.md
+++ b/tensorflow/contrib/lite/g3doc/ops_versioning.md
@@ -1,3 +1,6 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
 # TensorFlow Lite Ops Versioning
 
 This document describes TensorFlow Lite's op versioning schema. Op
diff --git a/tensorflow/contrib/lite/g3doc/overview.md b/tensorflow/contrib/lite/g3doc/overview.md
new file mode 100644
index 0000000000..be60d7941a
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/overview.md
@@ -0,0 +1,204 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# Introduction to TensorFlow Lite
+
+TensorFlow Lite is TensorFlow’s lightweight solution for mobile and embedded
+devices. It enables on-device machine learning inference with low latency and a
+small binary size. TensorFlow Lite also supports hardware acceleration with the
+[Android Neural Networks
+API](https://developer.android.com/ndk/guides/neuralnetworks/index.html).
+
+TensorFlow Lite uses many techniques for achieving low latency such as
+optimizing the kernels for mobile apps, pre-fused activations, and quantized
+kernels that allow smaller and faster (fixed-point math) models.
+
+Most of our TensorFlow Lite documentation is [on
+GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite)
+for the time being.
+
+## What does TensorFlow Lite contain?
+
+TensorFlow Lite supports a set of core operators, both quantized and
+float, which have been tuned for mobile platforms. They incorporate pre-fused
+activations and biases to further enhance performance and quantized
+accuracy. Additionally, TensorFlow Lite also supports using custom operations in
+models.
+
+TensorFlow Lite defines a new model file format, based on
+[FlatBuffers](https://google.github.io/flatbuffers/). FlatBuffers is an
+open-sourced, efficient cross platform serialization library. It is similar to
+[protocol buffers](https://developers.google.com/protocol-buffers/?hl=en), but
+the primary difference is that FlatBuffers does not need a parsing/unpacking
+step to a secondary representation before you can access data, often coupled
+with per-object memory allocation. Also, the code footprint of FlatBuffers is an
+order of magnitude smaller than protocol buffers.
+
+TensorFlow Lite has a new mobile-optimized interpreter, which has the key goals
+of keeping apps lean and fast. The interpreter uses a static graph ordering and
+a custom (less-dynamic) memory allocator to ensure minimal load, initialization,
+and execution latency.
+
+TensorFlow Lite provides an interface to leverage hardware acceleration, if
+available on the device. It does so via the
+[Android Neural Networks API](https://developer.android.com/ndk/guides/neuralnetworks/index.html),
+available on Android 8.1 (API level 27) and higher.
+
+## Why do we need a new mobile-specific library?
+
+Machine Learning is changing the computing paradigm, and we see an emerging
+trend of new use cases on mobile and embedded devices. Consumer expectations are
+also trending toward natural, human-like interactions with their devices, driven
+by the camera and voice interaction models.
+
+There are several factors which are fueling interest in this domain:
+
+- Innovation at the silicon layer is enabling new possibilities for hardware
+  acceleration, and frameworks such as the Android Neural Networks API make it
+  easy to leverage these.
+
+- Recent advances in real-time computer-vision and spoken language understanding
+  have led to mobile-optimized benchmark models being open sourced
+  (e.g. MobileNets, SqueezeNet).
+
+- Widely-available smart appliances create new possibilities for
+  on-device intelligence.
+
+- Interest in stronger user data privacy paradigms where user data does not need
+  to leave the mobile device.
+
+- Ability to serve ‘offline’ use cases, where the device does not need to be
+  connected to a network.
+
+We believe the next wave of machine learning applications will have significant
+processing on mobile and embedded devices.
+
+## TensorFlow Lite highlights
+
+TensorFlow Lite provides:
+
+- A set of core operators, both quantized and float, many of which have been
+  tuned for mobile platforms.  These can be used to create and run custom
+  models.  Developers can also write their own custom operators and use them in
+  models.
+
+- A new [FlatBuffers](https://google.github.io/flatbuffers/)-based
+  model file format.
+
+- On-device interpreter with kernels optimized for faster execution on mobile.
+
+- TensorFlow converter to convert TensorFlow-trained models to the TensorFlow
+  Lite format.
+
+- Smaller in size: TensorFlow Lite is smaller than 300KB when all supported
+  operators are linked and less than 200KB when using only the operators needed
+  for supporting InceptionV3 and Mobilenet.
+
+- **Pre-tested models:**
+
+    All of the following models are guaranteed to work out of the box:
+
+    - Inception V3, a popular model for detecting the dominant objects
+      present in an image.
+
+    - [MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md),
+      a family of mobile-first computer vision models designed to effectively
+      maximize accuracy while being mindful of the restricted resources for an
+      on-device or embedded application. They are small, low-latency, low-power
+      models parameterized to meet the resource constraints of a variety of use
+      cases. They can be built upon for classification, detection, embeddings
+      and segmentation. MobileNet models are smaller but [lower in
+      accuracy](https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html)
+      than Inception V3.
+
+    - On Device Smart Reply, an on-device model which provides one-touch
+      replies for an incoming text message by suggesting contextually relevant
+      messages. The model was built specifically for memory constrained devices
+      such as watches & phones and it has been successfully used to surface
+      [Smart Replies on Android
+      Wear](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html)
+      to all first-party and third-party apps.
+
+    Also see the complete list of
+    [TensorFlow Lite's supported models](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/models.md),
+    including the model sizes, performance numbers, and downloadable model files.
+
+- Quantized versions of the MobileNet model, which runs faster than the
+  non-quantized (float) version on CPU.
+
+- New Android demo app to illustrate the use of TensorFlow Lite with a quantized
+  MobileNet model for object classification.
+
+- Java and C++ API support
+
+
+## Getting Started
+
+We recommend you try out TensorFlow Lite with the pre-tested models indicated
+above. If you have an existing model, you will need to test whether your model
+is compatible with both the converter and the supported operator set.  To test
+your model, see the
+[documentation on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite).
+
+### Retrain Inception-V3 or MobileNet for a custom data set
+
+The pre-trained models mentioned above have been trained on the ImageNet data
+set, which consists of 1000 predefined classes. If those classes are not
+relevant or useful for your use case, you will need to retrain those
+models. This technique is called transfer learning, which starts with a model
+that has been already trained on a problem and will then be retrained on a
+similar problem. Deep learning from scratch can take days, but transfer learning
+can be done fairly quickly. In order to do this, you'll need to generate your
+custom data set labeled with the relevant classes.
+
+The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/)
+codelab walks through this process step-by-step. The retraining code supports
+retraining for both floating point and quantized inference.
+
+## TensorFlow Lite Architecture
+
+The following diagram shows the architectural design of TensorFlow Lite:
+
+TensorFlow Lite architecture diagram
+
+Starting with a trained TensorFlow model on disk, you'll convert that model to
+the TensorFlow Lite file format (`.tflite`) using the TensorFlow Lite
+Converter. Then you can use that converted file in your mobile application.
+
+Deploying the TensorFlow Lite model file uses:
+
+- Java API: A convenience wrapper around the C++ API on Android.
+
+- C++ API: Loads the TensorFlow Lite Model File and invokes the Interpreter. The
+  same library is available on both Android and iOS.
+
+- Interpreter: Executes the model using a set of kernels. The interpreter
+  supports selective kernel loading; without kernels it is only 100KB, and 300KB
+  with all the kernels loaded. This is a significant reduction from the 1.5M
+  required by TensorFlow Mobile.
+
+- On select Android devices, the Interpreter will use the Android Neural
+  Networks API for hardware acceleration, or default to CPU execution if none
+  are available.
+
+You can also implement custom kernels using the C++ API that can be used by the
+Interpreter.
+
+## Future Work
+
+In future releases, TensorFlow Lite will support more models and built-in
+operators, contain performance improvements for both fixed point and floating
+point models, improvements to the tools to enable easier developer workflows and
+support for other smaller devices and more. As we continue development, we hope
+that TensorFlow Lite will greatly simplify the developer experience of targeting
+a model for small devices.
+
+Future plans include using specialized machine learning hardware to get the best
+possible performance for a particular model on a particular device.
+
+## Next Steps
+
+The TensorFlow Lite [GitHub repository](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite).
+contains additional docs, code samples, and demo applications.
diff --git a/tensorflow/contrib/lite/g3doc/performance.md b/tensorflow/contrib/lite/g3doc/performance.md
new file mode 100644
index 0000000000..613e9f97c3
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/performance.md
@@ -0,0 +1,177 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# Performance
+
+This document lists TensorFlow Lite performance benchmarks when running well
+known models on some Android and iOS devices.
+
+These performance benchmark numbers were generated with the
+[Android TFLite benchmark binary](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark)
+and the [iOS benchmark app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark/ios).
+
+# Android performance benchmarks
+
+For Android benchmarks, the CPU affinity is set to use big cores on the device to
+reduce variance (see [details](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark#reducing-variance-between-runs-on-android)).
+
+It assumes that models were download and unzipped to the
+`/data/local/tmp/tflite_models` directory. The benchmark binary is built
+using [these instructions](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark#on-android)
+and assumed in the `/data/local/tmp` directory.
+
+To run the benchmark:
+
+```
+adb shell taskset ${CPU_MASK} /data/local/tmp/benchmark_model \
+  --num_threads=1 \
+  --graph=/data/local/tmp/tflite_models/${GRAPH} \
+  --warmup_runs=1 \
+  --num_runs=50 \
+  --use_nnapi=false
+```
+
+Here, `${GRAPH}` is the name of model and `${CPU_MASK}` is the CPU affinity
+chosen according to the following table:
+
+Device | CPU_MASK |
+-------| ----------
+Pixel 2 | f0 |
+Pixel xl | 0c |
+
+
+
+  
+    
+      
+      
+      
+    
+  
+  
+    
+    
+    
+  
+   
+     
+     
+  
+  
+    
+    
+    
+  
+   
+     
+     
+  
+  
+    
+    
+    
+  
+   
+     
+     
+  
+  
+    
+    
+    
+  
+   
+     
+     
+  
+  
+    
+    
+    
+  
+   
+     
+     
+  
+  
+    
+    
+    
+  
+   
+     
+     
+  
+
+ 
    Model NameDevice Mean inference time (std dev)
    
+      Mobilenet_1.0_224(float)
+    Pixel 2 166.5 ms (2.6 ms)
    Pixel xl 122.9 ms (1.8 ms)  
    
+      Mobilenet_1.0_224 (quant)
+    Pixel 2 69.5 ms (0.9 ms)
    Pixel xl 78.9 ms (2.2 ms)  
    
+      NASNet mobile
+    Pixel 2 273.8 ms (3.5 ms)
    Pixel xl 210.8 ms (4.2 ms)
    
+      SqueezeNet
+    Pixel 2 234.0 ms (2.1 ms)
    Pixel xl 158.0 ms (2.1 ms)
    
+      Inception_ResNet_V2
+    Pixel 2 2846.0 ms (15.0 ms)
    Pixel xl 1973.0 ms (15.0 ms)  
    
+      Inception_V4
+    Pixel 2 3180.0 ms (11.7 ms)
    Pixel xl 2262.0 ms (21.0 ms)  
    
+
+# iOS benchmarks
+
+To run iOS benchmarks, the [benchmark
+app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark/ios)
+was modified to include the appropriate model and `benchmark_params.json` was
+modified  to set `num_threads` to 1.
+
+
+  
+    
+      
+      
+      
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+ 
    Model NameDevice Mean inference time (std dev)
    
+      Mobilenet_1.0_224(float)
+    iPhone 8 32.2 ms (0.8 ms)
    
+      Mobilenet_1.0_224 (quant)
+    iPhone 8 24.4 ms (0.8 ms)
    
+      NASNet mobile
+    iPhone 8 60.3 ms (0.6 ms)
    
+      SqueezeNet
+    iPhone 8 44.3 (0.7 ms)
    
+      Inception_ResNet_V2
+    iPhone 8562.4 ms (18.2 ms)
    
+      Inception_V4
+    iPhone 8 661.0 ms (29.2 ms)
    
diff --git a/tensorflow/contrib/lite/g3doc/rpi.md b/tensorflow/contrib/lite/g3doc/rpi.md
index ab50789307..cdc9172d87 100644
--- a/tensorflow/contrib/lite/g3doc/rpi.md
+++ b/tensorflow/contrib/lite/g3doc/rpi.md
@@ -1,3 +1,6 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
 # TensorFlow Lite for Raspberry Pi
 
 ## Cross compiling
diff --git a/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md b/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md
index 967259b7a6..0e8f4339fc 100644
--- a/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md
@@ -1,3 +1,6 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
 # TensorFlow Lite & TensorFlow Compatibility Guide
 
 TensorFlow Lite supports a number of TensorFlow operations used in common
diff --git a/tensorflow/contrib/lite/g3doc/tfmobile/android_build.md b/tensorflow/contrib/lite/g3doc/tfmobile/android_build.md
new file mode 100644
index 0000000000..76e16fc9db
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/tfmobile/android_build.md
@@ -0,0 +1,181 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# Building TensorFlow on Android
+
+To get you started working with TensorFlow on Android, we'll walk through two
+ways to build our TensorFlow mobile demos and deploying them on an Android
+device. The first is Android Studio, which lets you build and deploy in an
+IDE. The second is building with Bazel and deploying with ADB on the command
+line.
+
+Why choose one or the other of these methods?
+
+The simplest way to use TensorFlow on Android is to use Android Studio. If you
+aren't planning to customize your TensorFlow build at all, or if you want to use
+Android Studio's editor and other features to build an app and just want to add
+TensorFlow to it, we recommend using Android Studio.
+
+If you are using custom ops, or have some other reason to build TensorFlow from
+scratch, scroll down and see our instructions
+for [building the demo with Bazel](#build_the_demo_using_bazel).
+
+## Build the demo using Android Studio
+
+**Prerequisites**
+
+If you haven't already, do the following two things:
+
+- Install [Android Studio](https://developer.android.com/studio/index.html),
+  following the instructions on their website.
+
+- Clone the TensorFlow repository from GitHub:
+
+        git clone https://github.com/tensorflow/tensorflow
+
+**Building**
+
+1. Open Android Studio, and from the Welcome screen, select **Open an existing
+   Android Studio project**.
+
+2. From the **Open File or Project** window that appears, navigate to and select
+    the `tensorflow/examples/android` directory from wherever you cloned the
+    TensorFlow GitHub repo.  Click OK.
+
+    If it asks you to do a Gradle Sync, click OK.
+
+    You may also need to install various platforms and tools, if you get
+    errors like "Failed to find target with hash string 'android-23' and similar.
+
+3. Open the `build.gradle` file (you can go to **1:Project** in the side panel
+    and find it under the **Gradle Scripts** zippy under **Android**). Look for
+    the `nativeBuildSystem` variable and set it to `none` if it isn't already:
+
+        // set to 'bazel', 'cmake', 'makefile', 'none'
+        def nativeBuildSystem = 'none'
+
+4. Click the *Run* button (the green arrow) or select *Run > Run 'android'* from the
+    top menu. You may need to rebuild the project using *Build > Rebuild Project*.
+
+    If it asks you to use Instant Run, click **Proceed Without Instant Run**.
+
+    Also, you need to have an Android device plugged in with developer options
+    enabled at this
+    point. See [here](https://developer.android.com/studio/run/device.html) for
+    more details on setting up developer devices.
+
+This installs three apps on your phone that are all part of the TensorFlow
+Demo. See [Android Sample Apps](#android_sample_apps) for more information about
+them.
+
+## Adding TensorFlow to your apps using Android Studio
+
+To add TensorFlow to your own apps on Android, the simplest way is to add the
+following lines to your Gradle build file:
+
+    allprojects {
+        repositories {
+            jcenter()
+        }
+	}
+
+    dependencies {
+        compile 'org.tensorflow:tensorflow-android:+'
+    }
+
+This automatically downloads the latest stable version of TensorFlow as an AAR
+and installs it in your project.
+
+## Build the demo using Bazel
+
+Another way to use TensorFlow on Android is to build an APK
+using [Bazel](https://bazel.build/) and load it onto your device
+using [ADB](https://developer.android.com/studio/command-line/adb.html). This
+requires some knowledge of build systems and Android developer tools, but we'll
+guide you through the basics here.
+
+- First, follow our instructions for
+  installing from sources.
+  This will also guide you through installing Bazel and cloning the
+  TensorFlow code.
+
+- Download the Android [SDK](https://developer.android.com/studio/index.html)
+  and [NDK](https://developer.android.com/ndk/downloads/index.html) if you do
+  not already have them. You need at least version 12b of the NDK, and 23 of the
+  SDK.
+
+- In your copy of the TensorFlow source, update the
+  [WORKSPACE](https://github.com/tensorflow/tensorflow/blob/master/WORKSPACE)
+  file with the location of your SDK and NDK, where it says <PATH_TO_NDK>
+  and <PATH_TO_SDK>.
+
+- Run Bazel to build the demo APK:
+
+        bazel build -c opt //tensorflow/examples/android:tensorflow_demo
+
+- Use [ADB](https://developer.android.com/studio/command-line/adb.html#move) to
+  install the APK onto your device:
+
+        adb install -r bazel-bin/tensorflow/examples/android/tensorflow_demo.apk
+
+Note: In general when compiling for Android with Bazel you need
+`--config=android` on the Bazel command line, though in this case this
+particular example is Android-only, so you don't need it here.
+
+This installs three apps on your phone that are all part of the TensorFlow
+Demo. See [Android Sample Apps](#android_sample_apps) for more information about
+them.
+
+## Android Sample Apps
+
+The
+[Android example code](https://www.tensorflow.org/code/tensorflow/examples/android/) is
+a single project that builds and installs three sample apps which all use the
+same underlying code. The sample apps all take video input from a phone's
+camera:
+
+- **TF Classify** uses the Inception v3 model to label the objects it’s pointed
+  at with classes from Imagenet. There are only 1,000 categories in Imagenet,
+  which misses most everyday objects and includes many things you’re unlikely to
+  encounter often in real life, so the results can often be quite amusing. For
+  example there’s no ‘person’ category, so instead it will often guess things it
+  does know that are often associated with pictures of people, like a seat belt
+  or an oxygen mask. If you do want to customize this example to recognize
+  objects you care about, you can use
+  the
+  [TensorFlow for Poets codelab](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/index.html#0) as
+  an example for how to train a model based on your own data.
+
+- **TF Detect** uses a multibox model to try to draw bounding boxes around the
+  locations of people in the camera. These boxes are annotated with the
+  confidence for each detection result. Results will not be perfect, as this
+  kind of object detection is still an active research topic.  The demo also
+  includes optical tracking for when objects move between frames, which runs
+  more frequently than the TensorFlow inference. This improves the user
+  experience since the apparent frame rate is faster, but it also gives the
+  ability to estimate which boxes refer to the same object between frames, which
+  is important for counting objects over time.
+
+- **TF Stylize** implements a real-time style transfer algorithm on the camera
+  feed. You can select which styles to use and mix between them using the
+  palette at the bottom of the screen, and also switch out the resolution of the
+  processing to go higher or lower rez.
+
+When you build and install the demo, you'll see three app icons on your phone,
+one for each of the demos. Tapping on them should open up the app and let you
+explore what they do. You can enable profiling statistics on-screen by tapping
+the volume up button while they’re running.
+
+### Android Inference Library
+
+Because Android apps need to be written in Java, and core TensorFlow is in C++,
+TensorFlow has a JNI library to interface between the two. Its interface is aimed
+only at inference, so it provides the ability to load a graph, set up inputs,
+and run the model to calculate particular outputs. You can see the full
+documentation for the minimal set of methods in
+[TensorFlowInferenceInterface.java](https://www.tensorflow.org/code/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java)
+
+The demos applications use this interface, so they’re a good place to look for
+example usage. You can download prebuilt binary jars
+at
+[ci.tensorflow.org](https://ci.tensorflow.org/view/Nightly/job/nightly-android/).
diff --git a/tensorflow/contrib/lite/g3doc/tfmobile/index.md b/tensorflow/contrib/lite/g3doc/tfmobile/index.md
new file mode 100644
index 0000000000..bd047bfcec
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/tfmobile/index.md
@@ -0,0 +1,284 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# Overview
+
+TensorFlow was designed to be a good deep learning solution for mobile
+platforms. Currently we have two solutions for deploying machine learning
+applications on mobile and embedded devices: TensorFlow for Mobile and
+TensorFlow Lite.
+
+## TensorFlow Lite versus TensorFlow Mobile
+
+Here are a few of the differences between the two:
+
+- TensorFlow Lite is an evolution of TensorFlow Mobile.  In most cases, apps
+  developed with TensorFlow Lite will have a smaller binary size, fewer
+  dependencies, and better performance.
+
+- TensorFlow Lite is in developer preview, so not all use cases are covered yet.
+  We expect you to use TensorFlow Mobile to cover production cases.
+
+- TensorFlow Lite supports only a limited set of operators, so not all models
+  will work on it by default. TensorFlow for Mobile has a fuller set of
+  supported functionality.
+
+TensorFlow Lite provides better performance and a small binary size on mobile
+platforms as well as the ability to leverage hardware acceleration if available
+on their platforms. In addition, it has many fewer dependencies so it can be
+built and hosted on simpler, more constrained device scenarios. TensorFlow Lite
+also allows targeting accelerators through the [Neural Networks
+API](https://developer.android.com/ndk/guides/neuralnetworks/index.html).
+
+TensorFlow Lite currently has coverage for a limited set of operators. While
+TensorFlow for Mobile supports only a constrained set of ops by default, in
+principle if you use an arbitrary operator in TensorFlow, it can be customized
+to build that kernel. Thus use cases which are not currently supported by
+TensorFlow Lite should continue to use TensorFlow for Mobile. As TensorFlow Lite
+evolves, it will gain additional operators, and the decision will be easier to
+make.
+
+
+## Introduction to TensorFlow Mobile
+
+TensorFlow was designed from the ground up to be a good deep learning solution
+for mobile platforms like Android and iOS. This mobile guide should help you
+understand how machine learning can work on mobile platforms and how to
+integrate TensorFlow into your mobile apps effectively and efficiently.
+
+## About this Guide
+
+This guide is aimed at developers who have a TensorFlow model that’s
+successfully working in a desktop environment, who want to integrate it into
+a mobile application, and cannot use TensorFlow Lite. Here are the
+main challenges you’ll face during that process:
+
+- Understanding how to use Tensorflow for mobile.
+- Building TensorFlow for your platform.
+- Integrating the TensorFlow library into your application.
+- Preparing your model file for mobile deployment.
+- Optimizing for latency, RAM usage, model file size, and binary size.
+
+## Common use cases for mobile machine learning
+
+**Why run TensorFlow on mobile?**
+
+Traditionally, deep learning has been associated with data centers and giant
+clusters of high-powered GPU machines. However, it can be very expensive and
+time-consuming to send all of the data a device has access to across a network
+connection. Running on mobile makes it possible to deliver very interactive
+applications in a way that’s not possible when you have to wait for a network
+round trip.
+
+Here are some common use cases for on-device deep learning:
+
+### Speech Recognition
+
+There are a lot of interesting applications that can be built with a
+speech-driven interface, and many of these require on-device processing. Most of
+the time a user isn’t giving commands, and so streaming audio continuously to a
+remote server would be a waste of bandwidth, since it would mostly be silence or
+background noises. To solve this problem it’s common to have a small neural
+network running on-device
+[listening out for a particular keyword](../tutorials/sequences/audio_recognition).
+Once that keyword has been spotted, the rest of the
+conversation can be transmitted over to the server for further processing if
+more computing power is needed.
+
+### Image Recognition
+
+It can be very useful for a mobile app to be able to make sense of a camera
+image. If your users are taking photos, recognizing what’s in them can help your
+camera apps apply appropriate filters, or label the photos so they’re easily
+findable. It’s important for embedded applications too, since you can use image
+sensors to detect all sorts of interesting conditions, whether it’s spotting
+endangered animals in the wild
+or
+[reporting how late your train is running](https://svds.com/tensorflow-image-recognition-raspberry-pi/).
+
+TensorFlow comes with several examples of recognizing the types of objects
+inside images along with a variety of different pre-trained models, and they can
+all be run on mobile devices. You can try out
+our
+[Tensorflow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/index.html#0) and
+[Tensorflow for Poets 2: Optimize for Mobile](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/index.html#0) codelabs to
+see how to take a pretrained model and run some very fast and lightweight
+training to teach it to recognize specific objects, and then optimize it to
+run on mobile.
+
+### Object Localization
+
+Sometimes it’s important to know where objects are in an image as well as what
+they are. There are lots of augmented reality use cases that could benefit a
+mobile app, such as guiding users to the right component when offering them
+help fixing their wireless network or providing informative overlays on top of
+landscape features. Embedded applications often need to count objects that are
+passing by them, whether it’s pests in a field of crops, or people, cars and
+bikes going past a street lamp.
+
+TensorFlow offers a pretrained model for drawing bounding boxes around people
+detected in images, together with tracking code to follow them over time. The
+tracking is especially important for applications where you’re trying to count
+how many objects are present over time, since it gives you a good idea when a
+new object enters or leaves the scene. We have some sample code for this
+available for Android [on
+GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android),
+and also a [more general object detection
+model](https://github.com/tensorflow/models/tree/master/research/object_detection/README.md)
+available as well.
+
+### Gesture Recognition
+
+It can be useful to be able to control applications with hand or other
+gestures, either recognized from images or through analyzing accelerometer
+sensor data. Creating those models is beyond the scope of this guide, but
+TensorFlow is an effective way of deploying them.
+
+### Optical Character Recognition
+
+Google Translate’s live camera view is a great example of how effective
+interactive on-device detection of text can be.
+
+
    
+  
+
    
+
+There are multiple steps involved in recognizing text in images. You first have
+to identify the areas where the text is present, which is a variation on the
+object localization problem, and can be solved with similar techniques. Once you
+have an area of text, you then need to interpret it as letters, and then use a
+language model to help guess what words they represent. The simplest way to
+estimate what letters are present is to segment the line of text into individual
+letters, and then apply a simple neural network to the bounding box of each. You
+can get good results with the kind of models used for MNIST, which you can find
+in TensorFlow’s tutorials, though you may want a higher-resolution input.  A
+more advanced alternative is to use an LSTM model to process a whole line of
+text at once, with the model itself handling the segmentation into different
+characters.
+
+### Translation
+
+Translating from one language to another quickly and accurately, even if you
+don’t have a network connection, is an important use case. Deep networks are
+very effective at this sort of task, and you can find descriptions of a lot of
+different models in the literature. Often these are sequence-to-sequence
+recurrent models where you’re able to run a single graph to do the whole
+translation, without needing to run separate parsing stages.
+
+### Text Classification
+
+If you want to suggest relevant prompts to users based on what they’re typing or
+reading, it can be very useful to understand the meaning of the text. This is
+where text classification comes in. Text classification is an umbrella term
+that covers everything from sentiment analysis to topic discovery. You’re likely
+to have your own categories or labels that you want to apply, so the best place
+to start is with an example
+like
+[Skip-Thoughts](https://github.com/tensorflow/models/tree/master/research/skip_thoughts/),
+and then train on your own examples.
+
+### Voice Synthesis
+
+A synthesized voice can be a great way of giving users feedback or aiding
+accessibility, and recent advances such as
+[WaveNet](https://deepmind.com/blog/wavenet-generative-model-raw-audio/) show
+that deep learning can offer very natural-sounding speech.
+
+## Mobile machine learning and the cloud
+
+These examples of use cases give an idea of how on-device networks can
+complement cloud services. Cloud has a great deal of computing power in a
+controlled environment, but running on devices can offer higher interactivity.
+In situations where the cloud is unavailable, or your cloud capacity is limited,
+you can provide an offline experience, or reduce cloud workload by processing
+easy cases on device.
+
+Doing on-device computation can also signal when it's time to switch to working
+on the cloud. A good example of this is hotword detection in speech. Since
+devices are able to constantly listen out for the keywords, this then triggers a
+lot of traffic to cloud-based speech recognition once one is recognized. Without
+the on-device component, the whole application wouldn’t be feasible, and this
+pattern exists across several other applications as well. Recognizing that some
+sensor input is interesting enough for further processing makes a lot of
+interesting products possible.
+
+## What hardware and software should you have?
+
+TensorFlow runs on Ubuntu Linux, Windows 10, and OS X. For a list of all
+supported operating systems and instructions to install TensorFlow, see
+Installing Tensorflow.
+
+Note that some of the sample code we provide for mobile TensorFlow requires you
+to compile TensorFlow from source, so you’ll need more than just `pip install`
+to work through all the sample code.
+
+To try out the mobile examples, you’ll need a device set up for development,
+using
+either [Android Studio](https://developer.android.com/studio/install.html),
+or [XCode](https://developer.apple.com/xcode/) if you're developing for iOS.
+
+## What should you do before you get started?
+
+Before thinking about how to get your solution on mobile:
+
+1. Determine whether your problem is solvable by mobile machine learning
+2. Create a labelled dataset to define your problem
+3. Pick an effective model for the problem
+
+We'll discuss these in more detail below.
+
+### Is your problem solvable by mobile machine learning?
+
+Once you have an idea of the problem you want to solve, you need to make a plan
+of how to build your solution. The most important first step is making sure that
+your problem is actually solvable, and the best way to do that is to mock it up
+using humans in the loop.
+
+For example, if you want to drive a robot toy car using voice commands, try
+recording some audio from the device and listen back to it to see if you can
+make sense of what’s being said. Often you’ll find there are problems in the
+capture process, such as the motor drowning out speech or not being able to hear
+at a distance, and you should tackle these problems before investing in the
+modeling process.
+
+Another example would be giving photos taken from your app to people see if they
+can classify what’s in them, in the way you’re looking for. If they can’t do
+that (for example, trying to estimate calories in food from photos may be
+impossible because all white soups look the same), then you’ll need to redesign
+your experience to cope with that. A good rule of thumb is that if a human can’t
+handle the task then it will be difficult to train a computer to do better.
+
+### Create a labelled dataset
+
+After you’ve solved any fundamental issues with your use case, you need to
+create a labeled dataset to define what problem you’re trying to solve. This
+step is extremely important, more than picking which model to use. You want it
+to be as representative as possible of your actual use case, since the model
+will only be effective at the task you teach it. It’s also worth investing in
+tools to make labeling the data as efficient and accurate as possible. For
+example, if you’re able to switch from having to click a button on a web
+interface to simple keyboard shortcuts, you may be able to speed up the
+generation process a lot. You should also start by doing the initial labeling
+yourself, so you can learn about the difficulties and likely errors, and
+possibly change your labeling or data capture process to avoid them. Once you
+and your team are able to consistently label examples (that is once you
+generally agree on the same labels for most examples), you can then try and
+capture your knowledge in a manual and teach external raters how to run the same
+process.
+
+### Pick an effective model
+
+The next step is to pick an effective model to use. You might be able to avoid
+training a model from scratch if someone else has already implemented a model
+similar to what you need; we have a repository of models implemented in
+TensorFlow [on GitHub](https://github.com/tensorflow/models) that you can look
+through. Lean towards the simplest model you can find, and try to get started as
+soon as you have even a small amount of labelled data, since you’ll get the best
+results when you’re able to iterate quickly. The shorter the time it takes to
+try training a model and running it in its real application, the better overall
+results you’ll see. It’s common for an algorithm to get great training accuracy
+numbers but then fail to be useful within a real application because there’s a
+mismatch between the dataset and real usage. Prototype end-to-end usage as soon
+as possible to create a consistent user experience.
diff --git a/tensorflow/contrib/lite/g3doc/tfmobile/ios_build.md b/tensorflow/contrib/lite/g3doc/tfmobile/ios_build.md
new file mode 100644
index 0000000000..6223707892
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/tfmobile/ios_build.md
@@ -0,0 +1,110 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# Building TensorFlow on iOS
+
+## Using CocoaPods
+
+The simplest way to get started with TensorFlow on iOS is using the CocoaPods
+package management system. You can add the `TensorFlow-experimental` pod to your
+Podfile, which installs a universal binary framework. This makes it easy to get
+started but has the disadvantage of being hard to customize, which is important
+in case you want to shrink your binary size. If you do need the ability to
+customize your libraries, see later sections on how to do that.
+
+## Creating your own app
+
+If you'd like to add TensorFlow capabilities to your own app, do the following:
+
+- Create your own app or load your already-created app in XCode.
+
+- Add a file named Podfile at the project root directory with the following content:
+
+        target 'YourProjectName'
+        pod 'TensorFlow-experimental'
+
+- Run `pod install` to download and install the `TensorFlow-experimental` pod.
+
+- Open `YourProjectName.xcworkspace` and add your code.
+
+- In your app's **Build Settings**, make sure to add `$(inherited)` to the
+  **Other Linker Flags**, and **Header Search Paths** sections.
+
+## Running the Samples
+
+You'll need Xcode 7.3 or later to run our iOS samples.
+
+There are currently three examples: simple, benchmark, and camera. For now, you
+can download the sample code by cloning the main tensorflow repository (we are
+planning to make the samples available as a separate repository later).
+
+From the root of the tensorflow folder, download [Inception
+v1](https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip),
+and extract the label and graph files into the data folders inside both the
+simple and camera examples using these steps:
+
+    mkdir -p ~/graphs
+    curl -o ~/graphs/inception5h.zip \
+     https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip \
+     && unzip ~/graphs/inception5h.zip -d ~/graphs/inception5h
+    cp ~/graphs/inception5h/* tensorflow/examples/ios/benchmark/data/
+    cp ~/graphs/inception5h/* tensorflow/examples/ios/camera/data/
+    cp ~/graphs/inception5h/* tensorflow/examples/ios/simple/data/
+
+Change into one of the sample directories, download the
+[Tensorflow-experimental](https://cocoapods.org/pods/TensorFlow-experimental)
+pod, and open the Xcode workspace. Note that installing the pod can take a long
+time since it is big (~450MB). If you want to run the simple example, then:
+
+    cd tensorflow/examples/ios/simple
+    pod install
+    open tf_simple_example.xcworkspace   # note .xcworkspace, not .xcodeproj
+                                         # this is created by pod install
+
+Run the simple app in the XCode simulator. You should see a single-screen app
+with a **Run Model** button. Tap that, and you should see some debug output
+appear below indicating that the example Grace Hopper image in directory data
+has been analyzed, with a military uniform recognized.
+
+Run the other samples using the same process. The camera example requires a real
+device connected. Once you build and run that, you should get a live camera view
+that you can point at objects to get real-time recognition results.
+
+### iOS Example details
+
+There are three demo applications for iOS, all defined in Xcode projects inside
+[tensorflow/examples/ios](https://www.tensorflow.org/code/tensorflow/examples/ios/).
+
+- **Simple**: This is a minimal example showing how to load and run a TensorFlow
+  model in as few lines as possible. It just consists of a single view with a
+  button that executes the model loading and inference when its pressed.
+
+- **Camera**: This is very similar to the Android TF Classify demo. It loads
+  Inception v3 and outputs its best label estimate for what’s in the live camera
+  view. As with the Android version, you can train your own custom model using
+  TensorFlow for Poets and drop it into this example with minimal code changes.
+
+- **Benchmark**: is quite close to Simple, but it runs the graph repeatedly and
+  outputs similar statistics to the benchmark tool on Android.
+
+
+### Troubleshooting
+
+- Make sure you use the TensorFlow-experimental pod (and not TensorFlow).
+
+- The TensorFlow-experimental pod is current about ~450MB. The reason it is so
+  big is because we are bundling multiple platforms, and the pod includes all
+  TensorFlow functionality (e.g. operations). The final app size after build is
+  substantially smaller though (~25MB). Working with the complete pod is
+  convenient during development, but see below section on how you can build your
+  own custom TensorFlow library to reduce the size.
+
+## Building the TensorFlow iOS libraries from source
+
+While Cocoapods is the quickest and easiest way of getting started, you sometimes
+need more flexibility to determine which parts of TensorFlow your app should be
+shipped with. For such cases, you can build the iOS libraries from the
+sources. [This
+guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/ios#building-the-tensorflow-ios-libraries-from-source)
+contains detailed instructions on how to do that.
+
diff --git a/tensorflow/contrib/lite/g3doc/tfmobile/linking_libs.md b/tensorflow/contrib/lite/g3doc/tfmobile/linking_libs.md
new file mode 100644
index 0000000000..4c2071ed05
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/tfmobile/linking_libs.md
@@ -0,0 +1,256 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# Integrating TensorFlow libraries
+
+Once you have made some progress on a model that addresses the problem you’re
+trying to solve, it’s important to test it out inside your application
+immediately. There are often unexpected differences between your training data
+and what users actually encounter in the real world, and getting a clear picture
+of the gap as soon as possible improves the product experience.
+
+This page talks about how to integrate the TensorFlow libraries into your own
+mobile applications, once you have already successfully built and deployed the
+TensorFlow mobile demo apps.
+
+## Linking the library
+
+After you've managed to build the examples, you'll probably want to call
+TensorFlow from one of your existing applications. The very easiest way to do
+this is to use the Pod installation steps described in
+Building TensorFlow on iOS, but if you want to build
+TensorFlow from source (for example to customize which operators are included)
+you'll need to break out TensorFlow as a framework, include the right header
+files, and link against the built libraries and dependencies.
+
+### Android
+
+For Android, you just need to link in a Java library contained in a JAR file
+called `libandroid_tensorflow_inference_java.jar`. There are three ways to
+include this functionality in your program:
+
+1. Include the jcenter AAR which contains it, as in this
+ [example app](https://github.com/googlecodelabs/tensorflow-for-poets-2/blob/master/android/tfmobile/build.gradle#L59-L65)
+
+2. Download the nightly precompiled version from
+[ci.tensorflow.org](http://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/).
+
+3. Build the JAR file yourself using the instructions [in our Android GitHub repo](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/android)
+
+### iOS
+
+Pulling in the TensorFlow libraries on iOS is a little more complicated. Here is
+a checklist of what you’ll need to do to your iOS app:
+
+- Link against tensorflow/contrib/makefile/gen/lib/libtensorflow-core.a, usually
+  by adding `-L/your/path/tensorflow/contrib/makefile/gen/lib/` and
+  `-ltensorflow-core` to your linker flags.
+
+- Link against the generated protobuf libraries by adding
+  `-L/your/path/tensorflow/contrib/makefile/gen/protobuf_ios/lib` and
+  `-lprotobuf` and `-lprotobuf-lite` to your command line.
+
+- For the include paths, you need the root of your TensorFlow source folder as
+  the first entry, followed by
+  `tensorflow/contrib/makefile/downloads/protobuf/src`,
+  `tensorflow/contrib/makefile/downloads`,
+  `tensorflow/contrib/makefile/downloads/eigen`, and
+  `tensorflow/contrib/makefile/gen/proto`.
+
+- Make sure your binary is built with `-force_load` (or the equivalent on your
+  platform), aimed at the TensorFlow library to ensure that it’s linked
+  correctly. More detail on why this is necessary can be found in the next
+  section, [Global constructor magic](#global_constructor_magic). On Linux-like
+  platforms, you’ll need different flags, more like
+  `-Wl,--allow-multiple-definition -Wl,--whole-archive`.
+
+You’ll also need to link in the Accelerator framework, since this is used to
+speed up some of the operations.
+
+## Global constructor magic
+
+One of the subtlest problems you may run up against is the “No session factory
+registered for the given session options” error when trying to call TensorFlow
+from your own application. To understand why this is happening and how to fix
+it, you need to know a bit about the architecture of TensorFlow.
+
+The framework is designed to be very modular, with a thin core and a large
+number of specific objects that are independent and can be mixed and matched as
+needed. To enable this, the coding pattern in C++ had to let modules easily
+notify the framework about the services they offer, without requiring a central
+list that has to be updated separately from each implementation. It also had to
+allow separate libraries to add their own implementations without needing a
+recompile of the core.
+
+To achieve this capability, TensorFlow uses a registration pattern in a lot of
+places. In the code, it looks like this:
+
+```
+class MulKernel : OpKernel {
+	Status Compute(OpKernelContext* context) { … }
+};
+REGISTER_KERNEL(MulKernel, “Mul”);
+```
+
+This would be in a standalone `.cc` file linked into your application, either
+as part of the main set of kernels or as a separate custom library. The magic
+part is that the `REGISTER_KERNEL()` macro is able to inform the core of
+TensorFlow that it has an implementation of the Mul operation, so that it can be
+called in any graphs that require it.
+
+From a programming point of view, this setup is very convenient. The
+implementation and registration code live in the same file, and adding new
+implementations is as simple as compiling and linking it in. The difficult part
+comes from the way that the `REGISTER_KERNEL()` macro is implemented. C++
+doesn’t offer a good mechanism for doing this sort of registration, so we have
+to resort to some tricky code. Under the hood, the macro is implemented so that
+it produces something like this:
+
+```
+class RegisterMul {
+	public:
+		RegisterMul() {
+			global_kernel_registry()->Register(“Mul”, [](){
+				return new MulKernel()
+			});
+	}
+};
+RegisterMul g_register_mul;
+```
+
+This sets up a class `RegisterMul` with a constructor that tells the global
+kernel registry what function to call when somebody asks it how to create a
+“Mul” kernel. Then there’s a global object of that class, and so the constructor
+should be called at the start of any program.
+
+While this may sound sensible, the unfortunate part is that the global object
+that’s defined is not used by any other code, so linkers not designed with this
+in mind will decide that it can be deleted. As a result, the constructor is
+never called, and the class is never registered. All sorts of modules use this
+pattern in TensorFlow, and it happens that `Session` implementations are the
+first to be looked for when the code is run, which is why it shows up as the
+characteristic error when this problem occurs.
+
+The solution is to force the linker to not strip any code from the library, even
+if it believes it’s unused. On iOS, this step can be accomplished with the
+`-force_load` flag, specifying a library path, and on Linux you need
+`--whole-archive`. These persuade the linker to not be as aggressive about
+stripping, and should retain the globals.
+
+The actual implementation of the various `REGISTER_*` macros is a bit more
+complicated in practice, but they all suffer the same underlying problem. If
+you’re interested in how they work, [op_kernel.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_kernel.h#L1091)
+is a good place to start investigating.
+
+## Protobuf problems
+
+TensorFlow relies on
+the [Protocol Buffer](https://developers.google.com/protocol-buffers/) library,
+commonly known as protobuf. This library takes definitions of data structures
+and produces serialization and access code for them in a variety of
+languages. The tricky part is that this generated code needs to be linked
+against shared libraries for the exact same version of the framework that was
+used for the generator. This can be an issue when `protoc`, the tool used to
+generate the code, is from a different version of protobuf than the libraries in
+the standard linking and include paths. For example, you might be using a copy
+of `protoc` that was built locally in `~/projects/protobuf-3.0.1.a`, but you have
+libraries installed at `/usr/local/lib` and `/usr/local/include` that are from
+3.0.0.
+
+The symptoms of this issue are errors during the compilation or linking phases
+with protobufs. Usually, the build tools take care of this, but if you’re using
+the makefile, make sure you’re building the protobuf library locally and using
+it, as shown in [this Makefile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/makefile/Makefile#L18).
+
+Another situation that can cause problems is when protobuf headers and source
+files need to be generated as part of the build process. This process makes
+building more complex, since the first phase has to be a pass over the protobuf
+definitions to create all the needed code files, and only after that can you go
+ahead and do a build of the library code.
+
+### Multiple versions of protobufs in the same app
+
+Protobufs generate headers that are needed as part of the C++ interface to the
+overall TensorFlow library. This complicates using the library as a standalone
+framework.
+
+If your application is already using version 1 of the protocol buffers library,
+you may have trouble integrating TensorFlow because it requires version 2. If
+you just try to link both versions into the same binary, you’ll see linking
+errors because some of the symbols clash. To solve this particular problem, we
+have an experimental script at [rename_protobuf.sh](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/makefile/rename_protobuf.sh).
+
+You need to run this as part of the makefile build, after you’ve downloaded all
+the dependencies:
+
+```
+tensorflow/contrib/makefile/download_dependencies.sh
+tensorflow/contrib/makefile/rename_protobuf.sh
+```
+
+## Calling the TensorFlow API
+
+Once you have the framework available, you then need to call into it. The usual
+pattern is that you first load your model, which represents a preset set of
+numeric computations, and then you run inputs through that model (for example,
+images from a camera) and receive outputs (for example, predicted labels).
+
+On Android, we provide the Java Inference Library that is focused on just this
+use case, while on iOS and Raspberry Pi you call directly into the C++ API.
+
+### Android
+
+Here’s what a typical Inference Library sequence looks like on Android:
+
+```
+// Load the model from disk.
+TensorFlowInferenceInterface inferenceInterface =
+new TensorFlowInferenceInterface(assetManager, modelFilename);
+
+// Copy the input data into TensorFlow.
+inferenceInterface.feed(inputName, floatValues, 1, inputSize, inputSize, 3);
+
+// Run the inference call.
+inferenceInterface.run(outputNames, logStats);
+
+// Copy the output Tensor back into the output array.
+inferenceInterface.fetch(outputName, outputs);
+```
+
+You can find the source of this code in the [Android examples](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/android/src/org/tensorflow/demo/TensorFlowImageClassifier.java#L107).
+
+### iOS and Raspberry Pi
+
+Here’s the equivalent code for iOS and Raspberry Pi:
+
+```
+// Load the model.
+PortableReadFileToProto(file_path, &tensorflow_graph);
+
+// Create a session from the model.
+tensorflow::Status s = session->Create(tensorflow_graph);
+if (!s.ok()) {
+    LOG(FATAL) << "Could not create TensorFlow Graph: " << s;
+}
+
+// Run the model.
+std::string input_layer = "input";
+std::string output_layer = "output";
+std::vector outputs;
+tensorflow::Status run_status = session->Run({\{input_layer, image_tensor}},
+                               {output_layer}, {}, &outputs);
+if (!run_status.ok()) {
+    LOG(FATAL) << "Running model failed: " << run_status;
+}
+
+// Access the output data.
+tensorflow::Tensor* output = &outputs[0];
+```
+
+This is all based on the
+[iOS sample code](https://www.tensorflow.org/code/tensorflow/examples/ios/simple/RunModelViewController.mm),
+but there’s nothing iOS-specific; the same code should be usable on any platform
+that supports C++.
+
+You can also find specific examples for Raspberry Pi
+[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/pi_examples/label_image/label_image.cc).
diff --git a/tensorflow/contrib/lite/g3doc/tfmobile/optimizing.md b/tensorflow/contrib/lite/g3doc/tfmobile/optimizing.md
new file mode 100644
index 0000000000..a0192c3541
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/tfmobile/optimizing.md
@@ -0,0 +1,504 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# Optimizing for mobile
+
+There are some special issues that you have to deal with when you’re trying to
+ship on mobile or embedded devices, and you’ll need to think about these as
+you’re developing your model.
+
+These issues are:
+
+- Model and Binary Size
+- App speed and model loading speed
+- Performance and threading
+
+We'll discuss a few of these below.
+
+## What are the minimum device requirements for TensorFlow?
+
+You need at least one megabyte of program memory and several megabytes of RAM to
+run the base TensorFlow runtime, so it’s not suitable for DSPs or
+microcontrollers. Other than those, the biggest constraint is usually the
+calculation speed of the device, and whether you can run the model you need for
+your application with a low enough latency. You can use the benchmarking tools
+in [How to Profile your Model](#how_to_profile_your_model) to get an idea of how
+many FLOPs are required for a model, and then use that to make rule-of-thumb
+estimates of how fast they will run on different devices. For example, a modern
+smartphone might be able to run 10 GFLOPs per second, so the best you could hope
+for from a 5 GFLOP model is two frames per second, though you may do worse
+depending on what the exact computation patterns are.
+
+This model dependence means that it’s possible to run TensorFlow even on very
+old or constrained phones, as long as you optimize your network to fit within
+the latency budget and possibly within limited RAM too. For memory usage, you
+mostly need to make sure that the intermediate buffers that TensorFlow creates
+aren’t too large, which you can examine in the benchmark output too.
+
+## Speed
+
+One of the highest priorities of most model deployments is figuring out how to
+run the inference fast enough to give a good user experience. The first place to
+start is by looking at the total number of floating point operations that are
+required to execute the graph. You can get a very rough estimate of this by
+using the `benchmark_model` tool:
+
+    bazel build -c opt tensorflow/tools/benchmark:benchmark_model && \
+    bazel-bin/tensorflow/tools/benchmark/benchmark_model \
+    --graph=/tmp/inception_graph.pb --input_layer="Mul:0" \
+    --input_layer_shape="1,299,299,3" --input_layer_type="float" \
+    --output_layer="softmax:0" --show_run_order=false --show_time=false \
+    --show_memory=false --show_summary=true --show_flops=true --logtostderr
+
+This should show you an estimate of how many operations are needed to run the
+graph. You can then use that information to figure out how feasible your model
+is to run on the devices you’re targeting. For an example, a high-end phone from
+2016 might be able to do 20 billion FLOPs per second, so the best speed you
+could hope for from a model that requires 10 billion FLOPs is around 500ms. On a
+device like the Raspberry Pi 3 that can do about 5 billion FLOPs, you may only
+get one inference every two seconds.
+
+Having this estimate helps you plan for what you’ll be able to realistically
+achieve on a device. If the model is using too many ops, then there are a lot of
+opportunities to optimize the architecture to reduce that number.
+
+Advanced techniques include [SqueezeNet](https://arxiv.org/abs/1602.07360)
+and [MobileNet](https://arxiv.org/abs/1704.04861), which are architectures
+designed to produce models for mobile -- lean and fast but with a small accuracy
+cost.  You can also just look at alternative models, even older ones, which may
+be smaller. For example, Inception v1 only has around 7 million parameters,
+compared to Inception v3’s 24 million, and requires only 3 billion FLOPs rather
+than 9 billion for v3.
+
+## Model Size
+
+Models that run on a device need to be stored somewhere on the device, and very
+large neural networks can be hundreds of megabytes. Most users are reluctant to
+download very large app bundles from app stores, so you want to make your model
+as small as possible. Furthermore, smaller neural networks can persist in and
+out of a mobile device's memory faster.
+
+To understand how large your network will be on disk, start by looking at the
+size on disk of your `GraphDef` file after you’ve run `freeze_graph` and
+`strip_unused_nodes` on it (see Preparing models for
+more details on these tools), since then it should only contain
+inference-related nodes. To double-check that your results are as expected, run
+the `summarize_graph` tool to see how many parameters are in constants:
+
+    bazel build tensorflow/tools/graph_transforms:summarize_graph && \
+    bazel-bin/tensorflow/tools/graph_transforms/summarize_graph \
+    --in_graph=/tmp/tensorflow_inception_graph.pb
+
+That command should give you output that looks something like this:
+
+    No inputs spotted.
+    Found 1 possible outputs: (name=softmax, op=Softmax)
+    Found 23885411 (23.89M) const parameters, 0 (0) variable parameters,
+    and 99 control_edges
+    Op types used: 489 Const, 99 CheckNumerics, 99 Identity, 94
+    BatchNormWithGlobalNormalization, 94 Conv2D, 94 Relu, 11 Concat, 9 AvgPool,
+    5 MaxPool, 1 Sub, 1 Softmax, 1 ResizeBilinear, 1 Reshape, 1 Mul, 1 MatMul,
+    1 ExpandDims, 1 DecodeJpeg, 1 Cast, 1 BiasAdd
+
+The important part for our current purposes is the number of const
+parameters. In most models these will be stored as 32-bit floats to start, so if
+you multiply the number of const parameters by four, you should get something
+that’s close to the size of the file on disk. You can often get away with only
+eight-bits per parameter with very little loss of accuracy in the final result,
+so if your file size is too large you can try using
+quantize_weights
+to transform the parameters down.
+
+    bazel build tensorflow/tools/graph_transforms:transform_graph && \
+    bazel-bin/tensorflow/tools/graph_transforms/transform_graph \
+    --in_graph=/tmp/tensorflow_inception_optimized.pb \
+    --out_graph=/tmp/tensorflow_inception_quantized.pb \
+    --inputs='Mul:0' --outputs='softmax:0' --transforms='quantize_weights'
+
+If you look at the resulting file size, you should see that it’s about a quarter
+of the original at 23MB.
+
+Another transform is `round_weights`, which doesn't make the file smaller, but it
+makes the file compressible to about the same size as when `quantize_weights` is
+used. This is particularly useful for mobile development, taking advantage of
+the fact that app bundles are compressed before they’re downloaded by consumers.
+
+The original file does not compress well with standard algorithms, because the
+bit patterns of even very similar numbers can be very different. The
+`round_weights` transform keeps the weight parameters stored as floats, but
+rounds them to a set number of step values. This means there are a lot more
+repeated byte patterns in the stored model, and so compression can often bring
+the size down dramatically, in many cases to near the size it would be if they
+were stored as eight bit.
+
+Another advantage of `round_weights` is that the framework doesn’t have to
+allocate a temporary buffer to unpack the parameters into, as we have to when
+we just use `quantize_weights`. This saves a little bit of latency (though the
+results should be cached so it’s only costly on the first run) and makes it
+possible to use memory mapping, as described later.
+
+## Binary Size
+
+One of the biggest differences between mobile and server development is the
+importance of binary size. On desktop machines it’s not unusual to have
+executables that are hundreds of megabytes on disk, but for mobile and embedded
+apps it’s vital to keep the binary as small as possible so that user downloads
+are easy. As mentioned above, TensorFlow only includes a subset of op
+implementations by default, but this still results in a 12 MB final
+executable. To reduce this, you can set up the library to only include the
+implementations of the ops that you actually need, based on automatically
+analyzing your model. To use it:
+
+- Run `tools/print_required_ops/print_selective_registration_header.py` on your
+  model to produce a header file that only enables the ops it uses.
+
+- Place the `ops_to_register.h` file somewhere that the compiler can find
+  it. This can be in the root of your TensorFlow source folder.
+
+- Build TensorFlow with `SELECTIVE_REGISTRATION` defined, for example by passing
+  in `--copts=”-DSELECTIVE_REGISTRATION”` to your Bazel build command.
+
+This process recompiles the library so that only the needed ops and types are
+included, which can dramatically reduce the executable size. For example, with
+Inception v3, the new size is only 1.5MB.
+
+## How to Profile your Model
+
+Once you have an idea of what your device's peak performance range is, it’s
+worth looking at its actual current performance. Using a standalone TensorFlow
+benchmark, rather than running it inside a larger app, helps isolate just the
+Tensorflow contribution to the
+latency. The
+[tensorflow/tools/benchmark](https://www.tensorflow.org/code/tensorflow/tools/benchmark/) tool
+is designed to help you do this. To run it on Inception v3 on your desktop
+machine, build this benchmark model:
+
+    bazel build -c opt tensorflow/tools/benchmark:benchmark_model && \
+    bazel-bin/tensorflow/tools/benchmark/benchmark_model \
+    --graph=/tmp/tensorflow_inception_graph.pb --input_layer="Mul" \
+    --input_layer_shape="1,299,299,3" --input_layer_type="float" \
+    --output_layer="softmax:0" --show_run_order=false --show_time=false \
+    --show_memory=false --show_summary=true --show_flops=true --logtostderr
+
+You should see output that looks something like this:
+
+
    +============================== Top by Computation Time ==============================
+[node
+ type]  [start]  [first] [avg ms]     [%]  [cdf%]  [mem KB]  [Name]
+Conv2D   22.859   14.212   13.700  4.972%  4.972%  3871.488  conv_4/Conv2D
+Conv2D    8.116    8.964   11.315  4.106%  9.078%  5531.904  conv_2/Conv2D
+Conv2D   62.066   16.504    7.274  2.640% 11.717%   443.904  mixed_3/conv/Conv2D
+Conv2D    2.530    6.226    4.939  1.792% 13.510%  2765.952  conv_1/Conv2D
+Conv2D   55.585    4.605    4.665  1.693% 15.203%   313.600  mixed_2/tower/conv_1/Conv2D
+Conv2D  127.114    5.469    4.630  1.680% 16.883%    81.920  mixed_10/conv/Conv2D
+Conv2D   47.391    6.994    4.588  1.665% 18.548%   313.600  mixed_1/tower/conv_1/Conv2D
+Conv2D   39.463    7.878    4.336  1.574% 20.122%   313.600  mixed/tower/conv_1/Conv2D
+Conv2D  127.113    4.192    3.894  1.413% 21.535%   114.688  mixed_10/tower_1/conv/Conv2D
+Conv2D   70.188    5.205    3.626  1.316% 22.850%   221.952  mixed_4/conv/Conv2D
+
+============================== Summary by node type ==============================
+[Node type]  [count]  [avg ms]    [avg %]    [cdf %]  [mem KB]
+Conv2D            94   244.899    88.952%    88.952% 35869.953
+BiasAdd           95     9.664     3.510%    92.462% 35873.984
+AvgPool            9     7.990     2.902%    95.364%  7493.504
+Relu              94     5.727     2.080%    97.444% 35869.953
+MaxPool            5     3.485     1.266%    98.710%  3358.848
+Const            192     1.727     0.627%    99.337%     0.000
+Concat            11     1.081     0.393%    99.730%  9892.096
+MatMul             1     0.665     0.242%    99.971%     4.032
+Softmax            1     0.040     0.015%    99.986%     4.032
+<>                 1     0.032     0.012%    99.997%     0.000
+Reshape            1     0.007     0.003%   100.000%     0.000
+
+Timings (microseconds): count=50 first=330849 curr=274803 min=232354 max=415352 avg=275563 std=44193
+Memory (bytes): count=50 curr=128366400(all same)
+514 nodes defined 504 nodes observed
+
    
+
+This is the summary view, which is enabled by the show_summary flag. To
+interpret it, the first table is a list of the nodes that took the most time, in
+order by how long they took. From left to right, the columns are:
+
+- Node type, what kind of operation this was.
+
+- Start time of the op, showing where it falls in the sequence of operations.
+
+- First time in milliseconds. This is how long the operation took on the first
+  run of the benchmark, since by default 20 runs are executed to get more
+  reliable statistics. The first time is useful to spot which ops are doing
+  expensive calculations on the first run, and then caching the results.
+
+- Average time for the operation across all runs, in milliseconds.
+
+- What percentage of the total time for one run the op took. This is useful to
+  understand where the hotspots are.
+
+- The cumulative total time of this and the previous ops in the table. This is
+  handy for understanding what the distribution of work is across the layers, to
+  see if just a few of the nodes are taking up most of the time.
+  
+- The amount of memory consumed by outputs of this type of op.
+
+- Name of the node.
+
+The second table is similar, but instead of breaking down the timings by
+particular named nodes, it groups them by the kind of op. This is very useful to
+understand which op implementations you might want to optimize or eliminate from
+your graph. The table is arranged with the most costly operations at the start,
+and only shows the top ten entries, with a placeholder for other nodes. The
+columns from left to right are:
+
+- Type of the nodes being analyzed.
+
+- Accumulated average time taken by all nodes of this type, in milliseconds.
+
+- What percentage of the total time was taken by this type of operation.
+
+- Cumulative time taken by this and op types higher in the table, so you can
+  understand the distribution of the workload.
+
+-  How much memory the outputs of this op type took up.
+
+Both of these tables are set up so that you can easily copy and paste their
+results into spreadsheet documents, since they are output with tabs as
+separators between the columns. The summary by node type can be the most useful
+when looking for optimization opportunities, since it’s a pointer to the code
+that’s taking the most time. In this case, you can see that the Conv2D ops are
+almost 90% of the execution time. This is a sign that the graph is pretty
+optimal, since convolutions and matrix multiplies are expected to be the bulk of
+a neural network’s computing workload.
+
+As a rule of thumb, it’s more worrying if you see a lot of other operations
+taking up more than a small fraction of the time. For neural networks, the ops
+that don’t involve large matrix multiplications should usually be dwarfed by the
+ones that do, so if you see a lot of time going into those it’s a sign that
+either your network is non-optimally constructed, or the code implementing those
+ops is not as optimized as it could
+be. [Performance bugs](https://github.com/tensorflow/tensorflow/issues) or
+patches are always welcome if you do encounter this situation, especially if
+they include an attached model exhibiting this behavior and the command line
+used to run the benchmark tool on it.
+
+The run above was on your desktop, but the tool also works on Android, which is
+where it’s most useful for mobile development. Here’s an example command line to
+run it on a 64-bit ARM device:
+
+    bazel build -c opt --config=android_arm64 \
+    tensorflow/tools/benchmark:benchmark_model
+    adb push bazel-bin/tensorflow/tools/benchmark/benchmark_model /data/local/tmp
+    adb push /tmp/tensorflow_inception_graph.pb /data/local/tmp/
+    adb shell '/data/local/tmp/benchmark_model \
+    --graph=/data/local/tmp/tensorflow_inception_graph.pb --input_layer="Mul" \
+    --input_layer_shape="1,299,299,3" --input_layer_type="float" \
+    --output_layer="softmax:0" --show_run_order=false --show_time=false \
+    --show_memory=false --show_summary=true'
+
+You can interpret the results in exactly the same way as the desktop version
+above. If you have any trouble figuring out what the right input and output
+names and types are, take a look at the
+Preparing models
+page for details about detecting these for your model, and look at the
+`summarize_graph` tool which may give you
+helpful information.
+
+There isn’t good support for command line tools on iOS, so instead there’s a
+separate example
+at
+[tensorflow/examples/ios/benchmark](https://www.tensorflow.org/code/tensorflow/examples/ios/benchmark) that
+packages the same functionality inside a standalone app. This outputs the
+statistics to both the screen of the device and the debug log. If you want
+on-screen statistics for the Android example apps, you can turn them on by
+pressing the volume-up button.
+
+## Profiling within your own app
+
+The output you see from the benchmark tool is generated from modules that are
+included as part of the standard TensorFlow runtime, which means you have access
+to them within your own applications too. You can see an example of how to do
+that [here](https://www.tensorflow.org/code/tensorflow/examples/ios/benchmark/BenchmarkViewController.mm?l=139).
+
+The basic steps are:
+
+1. Create a StatSummarizer object:
+
+        tensorflow::StatSummarizer stat_summarizer(tensorflow_graph);
+
+2. Set up the options:
+
+        tensorflow::RunOptions run_options;
+        run_options.set_trace_level(tensorflow::RunOptions::FULL_TRACE);
+        tensorflow::RunMetadata run_metadata;
+
+3. Run the graph:
+
+        run_status = session->Run(run_options, inputs, output_layer_names, {},
+                                  output_layers, &run_metadata);
+
+4. Calculate the results and print them out:
+
+        assert(run_metadata.has_step_stats());
+        const tensorflow::StepStats& step_stats = run_metadata.step_stats();
+        stat_summarizer->ProcessStepStats(step_stats);
+        stat_summarizer->PrintStepStats();
+
+## Visualizing Models
+
+The most effective way to speed up your code is by altering your model so it
+does less work. To do that, you need to understand what your model is doing, and
+visualizing it is a good first step. To get a high-level overview of your graph,
+use [TensorBoard](https://github.com/tensorflow/tensorboard).
+
+## Threading
+
+The desktop version of TensorFlow has a sophisticated threading model, and will
+try to run multiple operations in parallel if it can. In our terminology this is
+called “inter-op parallelism” (though to avoid confusion with “intra-op”, you
+could think of it as “between-op” instead), and can be set by specifying
+`inter_op_parallelism_threads` in the session options.
+
+By default, mobile devices run operations serially; that is,
+`inter_op_parallelism_threads` is set to 1. Mobile processors usually have few
+cores and a small cache, so running multiple operations accessing disjoint parts
+of memory usually doesn’t help performance. “Intra-op parallelism” (or
+“within-op”) can be very helpful though, especially for computation-bound
+operations like convolutions where different threads can feed off the same small
+set of memory.
+
+On mobile, how many threads an op will use is set to the number of cores by
+default, or 2 when the number of cores can't be determined. You can override the
+default number of threads that ops are using by setting
+`intra_op_parallelism_threads` in the session options.  It’s a good idea to
+reduce the default if your app has its own threads doing heavy processing, so
+that they don’t interfere with each other.
+
+To see more details on session options, look at [ConfigProto](https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto).
+
+## Retrain with mobile data
+
+The biggest cause of accuracy problems when running models on mobile apps is
+unrepresentative training data. For example, most of the Imagenet photos are
+well-framed so that the object is in the center of the picture, well-lit, and
+shot with a normal lens. Photos from mobile devices are often poorly framed,
+badly lit, and can have fisheye distortions, especially selfies.
+
+The solution is to expand your training set with data actually captured from
+your application. This step can involve extra work, since you’ll have to label
+the examples yourself, but even if you just use it to expand your original
+training data, it can help the training set dramatically. Improving the training
+set by doing this, and by fixing other quality issues like duplicates or badly
+labeled examples is the single best way to improve accuracy. It’s usually a
+bigger help than altering your model architecture or using different techniques.
+
+## Reducing model loading time and/or memory footprint
+
+Most operating systems allow you to load a file using memory mapping, rather
+than going through the usual I/O APIs. Instead of allocating an area of memory
+on the heap and then copying bytes from disk into it, you simply tell the
+operating system to make the entire contents of a file appear directly in
+memory. This has several advantages:
+
+* Speeds loading
+* Reduces paging (increases performance)
+* Does not count towards RAM budget for your app
+
+TensorFlow has support for memory mapping the weights that form the bulk of most
+model files. Because of limitations in the `ProtoBuf` serialization format, we
+have to make a few changes to our model loading and processing code. The
+way memory mapping works is that we have a single file where the first part is a
+normal `GraphDef` serialized into the protocol buffer wire format, but then the
+weights are appended in a form that can be directly mapped.
+
+To create this file, run the
+`tensorflow/contrib/util:convert_graphdef_memmapped_format` tool. This takes in
+a `GraphDef` file that’s been run through `freeze_graph` and converts it to the
+format that has the weights appended at the end. Since that file’s no longer a
+standard `GraphDef` protobuf, you then need to make some changes to the loading
+code. You can see an example of this in
+the
+[iOS Camera demo app](https://www.tensorflow.org/code/tensorflow/examples/ios/camera/tensorflow_utils.mm?l=147),
+in the `LoadMemoryMappedModel()` function.
+
+The same code (with the Objective C calls for getting the filenames substituted)
+can be used on other platforms too. Because we’re using memory mapping, we need
+to start by creating a special TensorFlow environment object that’s set up with
+the file we’ll be using:
+
+    std::unique_ptr memmapped_env;
+    memmapped_env->reset(
+          new tensorflow::MemmappedEnv(tensorflow::Env::Default()));
+    tensorflow::Status mmap_status =
+          (memmapped_env->get())->InitializeFromFile(file_path);
+
+You then need to pass in this environment to subsequent calls, like this one for
+loading the graph:
+
+    tensorflow::GraphDef tensorflow_graph;
+    tensorflow::Status load_graph_status = ReadBinaryProto(
+        memmapped_env->get(),
+        tensorflow::MemmappedFileSystem::kMemmappedPackageDefaultGraphDef,
+        &tensorflow_graph);
+
+You also need to create the session with a pointer to the environment you’ve
+created:
+
+    tensorflow::SessionOptions options;
+    options.config.mutable_graph_options()
+        ->mutable_optimizer_options()
+        ->set_opt_level(::tensorflow::OptimizerOptions::L0);
+    options.env = memmapped_env->get();
+
+    tensorflow::Session* session_pointer = nullptr;
+    tensorflow::Status session_status =
+        tensorflow::NewSession(options, &session_pointer);
+
+One thing to notice here is that we’re also disabling automatic optimizations,
+since in some cases these will fold constant sub-trees, and so create copies of
+tensor values that we don’t want and use up more RAM.
+
+Once you’ve gone through these steps, you can use the session and graph as
+normal, and you should see a reduction in loading time and memory usage.
+
+## Protecting model files from easy copying
+
+By default, your models will be stored in the standard serialized protobuf
+format on disk. In theory this means that anybody can copy your model, which you
+may not want. However, in practice, most models are so application-specific and
+obfuscated by optimizations that the risk is similar to that of competitors
+disassembling and reusing your code, but if you do want to make it tougher for
+casual users to access your files it is possible to take some basic steps.
+
+Most of our examples use
+the
+[ReadBinaryProto()](https://www.tensorflow.org/code/tensorflow/core/platform/env.cc?q=core/platform/env.cc&l=409) convenience
+call to load a `GraphDef` from disk. This does require an unencrypted protobuf on
+disk. Luckily though, the implementation of the call is pretty straightforward
+and it should be easy to write an equivalent that can decrypt in memory. Here's
+some code that shows how you can read and decrypt a protobuf using your own
+decryption routine:
+
+    Status ReadEncryptedProto(Env* env, const string& fname,
+                              ::tensorflow::protobuf::MessageLite* proto) {
+      string data;
+      TF_RETURN_IF_ERROR(ReadFileToString(env, fname, &data));
+
+      DecryptData(&data);  // Your own function here.
+
+      if (!proto->ParseFromString(&data)) {
+        TF_RETURN_IF_ERROR(stream->status());
+        return errors::DataLoss("Can't parse ", fname, " as binary proto");
+      }
+      return Status::OK();
+    }
+
+To use this you’d need to define the DecryptData() function yourself. It could
+be as simple as something like:
+
+    void DecryptData(string* data) {
+      for (int i = 0; i < data.size(); ++i) {
+        data[i] = data[i] ^ 0x23;
+      }
+    }
+
+You may want something more complex, but exactly what you’ll need is outside the
+current scope here.
diff --git a/tensorflow/contrib/lite/g3doc/tfmobile/prepare_models.md b/tensorflow/contrib/lite/g3doc/tfmobile/prepare_models.md
new file mode 100644
index 0000000000..6b4e4a92bd
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/tfmobile/prepare_models.md
@@ -0,0 +1,304 @@
+book_path: /mobile/_book.yaml
+project_path: /mobile/_project.yaml
+
+# Preparing models for mobile deployment
+
+The requirements for storing model information during training are very
+different from when you want to release it as part of a mobile app. This section
+covers the tools involved in converting from a training model to something
+releasable in production.
+
+## What is up with all the different saved file formats?
+
+You may find yourself getting very confused by all the different ways that
+TensorFlow can save out graphs. To help, here’s a rundown of some of the
+different components, and what they are used for. The objects are mostly defined
+and serialized as protocol buffers:
+
+- [NodeDef](https://www.tensorflow.org/code/tensorflow/core/framework/node_def.proto):
+  Defines a single operation in a model. It has a unique name, a list of the
+  names of other nodes it pulls inputs from, the operation type it implements
+  (for example `Add`, or `Mul`), and any attributes that are needed to control
+  that operation. This is the basic unit of computation for TensorFlow, and all
+  work is done by iterating through a network of these nodes, applying each one
+  in turn. One particular operation type that’s worth knowing about is `Const`,
+  since this holds information about a constant. This may be a single, scalar
+  number or string, but it can also hold an entire multi-dimensional tensor
+  array. The values for a `Const` are stored inside the `NodeDef`, and so large
+  constants can take up a lot of room when serialized.
+
+- [Checkpoint](https://www.tensorflow.org/code/tensorflow/core/util/tensor_bundle/tensor_bundle.h). Another
+  way of storing values for a model is by using `Variable` ops. Unlike `Const`
+  ops, these don’t store their content as part of the `NodeDef`, so they take up
+  very little space within the `GraphDef` file. Instead their values are held in
+  RAM while a computation is running, and then saved out to disk as checkpoint
+  files periodically. This typically happens as a neural network is being
+  trained and weights are updated, so it’s a time-critical operation, and it may
+  happen in a distributed fashion across many workers, so the file format has to
+  be both fast and flexible. They are stored as multiple checkpoint files,
+  together with metadata files that describe what’s contained within the
+  checkpoints. When you’re referring to a checkpoint in the API (for example
+  when passing a filename in as a command line argument), you’ll use the common
+  prefix for a set of related files. If you had these files:
+
+        /tmp/model/model-chkpt-1000.data-00000-of-00002
+        /tmp/model/model-chkpt-1000.data-00001-of-00002
+        /tmp/model/model-chkpt-1000.index
+        /tmp/model/model-chkpt-1000.meta
+
+    You would refer to them as `/tmp/model/chkpt-1000`.
+
+- [GraphDef](https://www.tensorflow.org/code/tensorflow/core/framework/graph.proto):
+  Has a list of `NodeDefs`, which together define the computational graph to
+  execute. During training, some of these nodes will be `Variables`, and so if
+  you want to have a complete graph you can run, including the weights, you’ll
+  need to call a restore operation to pull those values from
+  checkpoints. Because checkpoint loading has to be flexible to deal with all of
+  the training requirements, this can be tricky to implement on mobile and
+  embedded devices, especially those with no proper file system available like
+  iOS. This is where
+  the
+  [`freeze_graph.py`](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py) script
+  comes in handy. As mentioned above, `Const` ops store their values as part of
+  the `NodeDef`, so if all the `Variable` weights are converted to `Const` nodes,
+  then we only need a single `GraphDef` file to hold the model architecture and
+  the weights. Freezing the graph handles the process of loading the
+  checkpoints, and then converts all Variables to Consts. You can then load the
+  resulting file in a single call, without having to restore variable values
+  from checkpoints. One thing to watch out for with `GraphDef` files is that
+  sometimes they’re stored in text format for easy inspection. These versions
+  usually have a ‘.pbtxt’ filename suffix, whereas the binary files end with
+  ‘.pb’.
+
+- [FunctionDefLibrary](https://www.tensorflow.org/code/tensorflow/core/framework/function.proto):
+  This appears in `GraphDef`, and is effectively a set of sub-graphs, each with
+  information about their input and output nodes. Each sub-graph can then be
+  used as an op in the main graph, allowing easy instantiation of different
+  nodes, in a similar way to how functions encapsulate code in other languages.
+
+- [MetaGraphDef](https://www.tensorflow.org/code/tensorflow/core/protobuf/meta_graph.proto):
+  A plain `GraphDef` only has information about the network of computations, but
+  doesn’t have any extra information about the model or how it can be
+  used. `MetaGraphDef` contains a `GraphDef` defining the computation part of
+  the model, but also includes information like ‘signatures’, which are
+  suggestions about which inputs and outputs you may want to call the model
+  with, data on how and where any checkpoint files are saved, and convenience
+  tags for grouping ops together for ease of use.
+
+- [SavedModel](https://www.tensorflow.org/code/tensorflow/core/protobuf/saved_model.proto):
+  It’s common to want to have different versions of a graph that rely on a
+  common set of variable checkpoints. For example, you might need a GPU and a
+  CPU version of the same graph, but keep the same weights for both. You might
+  also need some extra files (like label names) as part of your
+  model. The
+  [SavedModel](https://www.tensorflow.org/code/tensorflow/python/saved_model/README.md) format
+  addresses these needs by letting you save multiple versions of the same graph
+  without duplicating variables, and also storing asset files in the same
+  bundle. Under the hood, it uses `MetaGraphDef` and checkpoint files, along
+  with extra metadata files. It’s the format that you’ll want to use if you’re
+  deploying a web API using TensorFlow Serving, for example.
+
+## How do you get a model you can use on mobile?
+
+In most situations, training a model with TensorFlow will give you a folder
+containing a `GraphDef` file (usually ending with the `.pb` or `.pbtxt` extension) and
+a set of checkpoint files. What you need for mobile or embedded deployment is a
+single `GraphDef` file that’s been ‘frozen’, or had its variables converted into
+inline constants so everything’s in one file.  To handle the conversion, you’ll
+need the `freeze_graph.py` script, that’s held in
+[`tensorflow/python/tools/freeze_graph.py`](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py). You’ll run it like this:
+
+    bazel build tensorflow/python/tools:freeze_graph
+    bazel-bin/tensorflow/python/tools/freeze_graph \
+    --input_graph=/tmp/model/my_graph.pb \
+    --input_checkpoint=/tmp/model/model.ckpt-1000 \
+    --output_graph=/tmp/frozen_graph.pb \
+    --output_node_names=output_node \
+
+The `input_graph` argument should point to the `GraphDef` file that holds your
+model architecture. It’s possible that your `GraphDef` has been stored in a text
+format on disk, in which case it’s likely to end in `.pbtxt` instead of `.pb`,
+and you should add an extra `--input_binary=false` flag to the command.
+
+The `input_checkpoint` should be the most recent saved checkpoint. As mentioned
+in the checkpoint section, you need to give the common prefix to the set of
+checkpoints here, rather than a full filename.
+
+`output_graph` defines where the resulting frozen `GraphDef` will be
+saved. Because it’s likely to contain a lot of weight values that take up a
+large amount of space in text format, it’s always saved as a binary protobuf.
+
+`output_node_names` is a list of the names of the nodes that you want to extract
+the results of your graph from. This is needed because the freezing process
+needs to understand which parts of the graph are actually needed, and which are
+artifacts of the training process, like summarization ops. Only ops that
+contribute to calculating the given output nodes will be kept. If you know how
+your graph is going to be used, these should just be the names of the nodes you
+pass into `Session::Run()` as your fetch targets. The easiest way to find the
+node names is to inspect the Node objects while building your graph in python.
+Inspecting your graph in TensorBoard is another simple way.  You can get some
+suggestions on likely outputs by running the [`summarize_graph` tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/graph_transforms/README.md#inspecting-graphs).
+
+Because the output format for TensorFlow has changed over time, there are a
+variety of other less commonly used flags available too, like `input_saver`, but
+hopefully you shouldn’t need these on graphs trained with modern versions of the
+framework.
+
+## Using the Graph Transform Tool
+
+A lot of the things you need to do to efficiently run a model on device are
+available through the [Graph Transform
+Tool](https://www.tensorflow.org/code/tensorflow/tools/graph_transforms/README.md). This
+command-line tool takes an input `GraphDef` file, applies the set of rewriting
+rules you request, and then writes out the result as a `GraphDef`. See the
+documentation for more information on how to build and run this tool.
+
+### Removing training-only nodes
+
+TensorFlow `GraphDefs` produced by the training code contain all of the
+computation that’s needed for back-propagation and updates of weights, as well
+as the queuing and decoding of inputs, and the saving out of checkpoints. All of
+these nodes are no longer needed during inference, and some of the operations
+like checkpoint saving aren’t even supported on mobile platforms. To create a
+model file that you can load on devices you need to delete those unneeded
+operations by running the `strip_unused_nodes` rule in the Graph Transform Tool.
+
+The trickiest part of this process is figuring out the names of the nodes you
+want to use as inputs and outputs during inference.  You'll need these anyway
+once you start to run inference, but you also need them here so that the
+transform can calculate which nodes are not needed on the inference-only
+path. These may not be obvious from the training code. The easiest way to
+determine the node name is to explore the graph with TensorBoard.
+
+Remember that mobile applications typically gather their data from sensors and
+have it as arrays in memory, whereas training typically involves loading and
+decoding representations of the data stored on disk. In the case of Inception v3
+for example, there’s a `DecodeJpeg` op at the start of the graph that’s designed
+to take JPEG-encoded data from a file retrieved from disk and turn it into an
+arbitrary-sized image. After that there’s a `BilinearResize` op to scale it to
+the expected size, followed by a couple of other ops that convert the byte data
+into float and scale the value magnitudes it in the way the rest of the graph
+expects. A typical mobile app will skip most of these steps because it’s getting
+its input directly from a live camera, so the input node you will actually
+supply will be the output of the `Mul` node in this case.
+
+
+
+You’ll need to do a similar process of inspection to figure out the correct
+output nodes.
+
+If you’ve just been given a frozen `GraphDef` file, and are not sure about the
+contents, try using the `summarize_graph` tool to print out information
+about the inputs and outputs it finds from the graph structure. Here’s an
+example with the original Inception v3 file:
+
+    bazel run tensorflow/tools/graph_transforms:summarize_graph --
+    --in_graph=tensorflow_inception_graph.pb
+
+Once you have an idea of what the input and output nodes are, you can feed them
+into the graph transform tool as the `--input_names` and `--output_names`
+arguments, and call the `strip_unused_nodes` transform, like this:
+
+    bazel run tensorflow/tools/graph_transforms:transform_graph --
+    --in_graph=tensorflow_inception_graph.pb
+    --out_graph=optimized_inception_graph.pb --inputs='Mul' --outputs='softmax'
+    --transforms='
+      strip_unused_nodes(type=float, shape="1,299,299,3")
+      fold_constants(ignore_errors=true)
+      fold_batch_norms
+      fold_old_batch_norms'
+
+One thing to look out for here is that you need to specify the size and type
+that you want your inputs to be. This is because any values that you’re going to
+be passing in as inputs to inference need to be fed to special `Placeholder` op
+nodes, and the transform may need to create them if they don’t already exist. In
+the case of Inception v3 for example, a `Placeholder` node replaces the old
+`Mul` node that used to output the resized and rescaled image array, since we’re
+going to be doing that processing ourselves before we call TensorFlow. It keeps
+the original name though, which is why we always feed in inputs to `Mul` when we
+run a session with our modified Inception graph.
+
+After you’ve run this process, you’ll have a graph that only contains the actual
+nodes you need to run your prediction process. This is the point where it
+becomes useful to run metrics on the graph, so it’s worth running
+`summarize_graph` again to understand what’s in your model.
+
+## What ops should you include on mobile?
+
+There are hundreds of operations available in TensorFlow, and each one has
+multiple implementations for different data types. On mobile platforms, the size
+of the executable binary that’s produced after compilation is important, because
+app download bundles need to be as small as possible for the best user
+experience. If all of the ops and data types are compiled into the TensorFlow
+library then the total size of the compiled library can be tens of megabytes, so
+by default only a subset of ops and data types are included.
+
+That means that if you load a model file that’s been trained on a desktop
+machine, you may see the error “No OpKernel was registered to support Op” when
+you load it on mobile. The first thing to try is to make sure you’ve stripped
+out any training-only nodes, since the error will occur at load time even if the
+op is never executed. If you’re still hitting the same problem once that’s done,
+you’ll need to look at adding the op to your built library.
+
+The criteria for including ops and types fall into several categories:
+
+- Are they only useful in back-propagation, for gradients? Since mobile is
+  focused on inference, we don’t include these.
+
+- Are they useful mainly for other training needs, such as checkpoint saving?
+  These we leave out.
+
+- Do they rely on frameworks that aren’t always available on mobile, such as
+  libjpeg? To avoid extra dependencies we don’t include ops like `DecodeJpeg`.
+
+- Are there types that aren’t commonly used? We don’t include boolean variants
+  of ops for example, since we don’t see much use of them in typical inference
+  graphs.
+
+These ops are trimmed by default to optimize for inference on mobile, but it is
+possible to alter some build files to change the default.  After alternating the
+build files, you will need to recompile TensorFlow.  See below for more details
+on how to do this, and also see optimizing binary size
+for more on reducing your binary size.
+
+### Locate the implementation
+
+Operations are broken into two parts. The first is the op definition, which
+declares the signature of the operation, which inputs, outputs, and attributes
+it has. These take up very little space, and so all are included by default. The
+implementations of the op computations are done in kernels, which live in the
+`tensorflow/core/kernels` folder. You need to compile the C++ file containing
+the kernel implementation of the op you need into the library. To figure out
+which file that is, you can search for the operation name in the source
+files.
+
+[Here’s an example search in github](https://github.com/search?utf8=%E2%9C%93&q=repo%3Atensorflow%2Ftensorflow+extension%3Acc+path%3Atensorflow%2Fcore%2Fkernels+REGISTER+Mul&type=Code&ref=searchresults).
+
+You’ll see that this search is looking for the `Mul` op implementation, and it
+finds it in `tensorflow/core/kernels/cwise_op_mul_1.cc`. You need to look for
+macros beginning with `REGISTER`, with the op name you care about as one of the
+string arguments.
+
+In this case, the implementations are actually broken up across multiple `.cc`
+files, so you’d need to include all of them in your build. If you’re more
+comfortable using the command line for code search, here’s a grep command that
+also locates the right files if you run it from the root of your TensorFlow
+repository:
+
+`grep 'REGISTER.*"Mul"' tensorflow/core/kernels/*.cc`
+
+### Add the implementation to the build
+
+If you’re using Bazel, and building for Android, you’ll want to add the files
+you’ve found to
+the
+[`android_extended_ops_group1`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3565) or
+[`android_extended_ops_group2`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3632) targets. You
+may also need to include any .cc files they depend on in there. If the build
+complains about missing header files, add the .h’s that are needed into
+the
+[`android_extended_ops`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3525) target.
+
+If you’re using a makefile targeting iOS, Raspberry Pi, etc, go to
+[`tensorflow/contrib/makefile/tf_op_files.txt`](https://www.tensorflow.org/code/tensorflow/contrib/makefile/tf_op_files.txt) and
+add the right implementation files there.
diff --git a/tensorflow/docs_src/mobile/README.md b/tensorflow/docs_src/mobile/README.md
new file mode 100644
index 0000000000..ecf4267265
--- /dev/null
+++ b/tensorflow/docs_src/mobile/README.md
@@ -0,0 +1,3 @@
+# TF Lite subsite
+
+This subsite directory lives in [tensorflow/contrib/lite/g3doc](../../contrib/lite/g3doc/).
diff --git a/tensorflow/docs_src/mobile/android_build.md b/tensorflow/docs_src/mobile/android_build.md
deleted file mode 100644
index f4b07db459..0000000000
--- a/tensorflow/docs_src/mobile/android_build.md
+++ /dev/null
@@ -1,177 +0,0 @@
-# Building TensorFlow on Android
-
-To get you started working with TensorFlow on Android, we'll walk through two
-ways to build our TensorFlow mobile demos and deploying them on an Android
-device. The first is Android Studio, which lets you build and deploy in an
-IDE. The second is building with Bazel and deploying with ADB on the command
-line.
-
-Why choose one or the other of these methods?
-
-The simplest way to use TensorFlow on Android is to use Android Studio. If you
-aren't planning to customize your TensorFlow build at all, or if you want to use
-Android Studio's editor and other features to build an app and just want to add
-TensorFlow to it, we recommend using Android Studio.
-
-If you are using custom ops, or have some other reason to build TensorFlow from
-scratch, scroll down and see our instructions
-for [building the demo with Bazel](#build_the_demo_using_bazel).
-
-## Build the demo using Android Studio
-
-**Prerequisites**
-
-If you haven't already, do the following two things:
-
-- Install [Android Studio](https://developer.android.com/studio/index.html),
-  following the instructions on their website.
-
-- Clone the TensorFlow repository from GitHub:
-
-        git clone https://github.com/tensorflow/tensorflow
-
-**Building**
-
-1. Open Android Studio, and from the Welcome screen, select **Open an existing
-   Android Studio project**.
-
-2. From the **Open File or Project** window that appears, navigate to and select
-    the `tensorflow/examples/android` directory from wherever you cloned the
-    TensorFlow GitHub repo.  Click OK.
-
-    If it asks you to do a Gradle Sync, click OK.
-
-    You may also need to install various platforms and tools, if you get
-    errors like "Failed to find target with hash string 'android-23' and similar.
-
-3. Open the `build.gradle` file (you can go to **1:Project** in the side panel
-    and find it under the **Gradle Scripts** zippy under **Android**). Look for
-    the `nativeBuildSystem` variable and set it to `none` if it isn't already:
-
-        // set to 'bazel', 'cmake', 'makefile', 'none'
-        def nativeBuildSystem = 'none'
-
-4. Click the *Run* button (the green arrow) or select *Run > Run 'android'* from the
-    top menu. You may need to rebuild the project using *Build > Rebuild Project*.
-
-    If it asks you to use Instant Run, click **Proceed Without Instant Run**.
-
-    Also, you need to have an Android device plugged in with developer options
-    enabled at this
-    point. See [here](https://developer.android.com/studio/run/device.html) for
-    more details on setting up developer devices.
-
-This installs three apps on your phone that are all part of the TensorFlow
-Demo. See [Android Sample Apps](#android_sample_apps) for more information about
-them.
-
-## Adding TensorFlow to your apps using Android Studio
-
-To add TensorFlow to your own apps on Android, the simplest way is to add the
-following lines to your Gradle build file:
-
-    allprojects {
-        repositories {
-            jcenter()
-        }
-	}
-
-    dependencies {
-        compile 'org.tensorflow:tensorflow-android:+'
-    }
-
-This automatically downloads the latest stable version of TensorFlow as an AAR
-and installs it in your project.
-
-## Build the demo using Bazel
-
-Another way to use TensorFlow on Android is to build an APK
-using [Bazel](https://bazel.build/) and load it onto your device
-using [ADB](https://developer.android.com/studio/command-line/adb.html). This
-requires some knowledge of build systems and Android developer tools, but we'll
-guide you through the basics here.
-
-- First, follow our instructions for @{$install/install_sources$installing from sources}.
-  This will also guide you through installing Bazel and cloning the
-  TensorFlow code.
-
-- Download the Android [SDK](https://developer.android.com/studio/index.html)
-  and [NDK](https://developer.android.com/ndk/downloads/index.html) if you do
-  not already have them. You need at least version 12b of the NDK, and 23 of the
-  SDK.
-
-- In your copy of the TensorFlow source, update the
-  [WORKSPACE](https://github.com/tensorflow/tensorflow/blob/master/WORKSPACE)
-  file with the location of your SDK and NDK, where it says <PATH_TO_NDK>
-  and <PATH_TO_SDK>.
-
-- Run Bazel to build the demo APK:
-
-        bazel build -c opt //tensorflow/examples/android:tensorflow_demo
-
-- Use [ADB](https://developer.android.com/studio/command-line/adb.html#move) to
-  install the APK onto your device:
-
-        adb install -r bazel-bin/tensorflow/examples/android/tensorflow_demo.apk
-
-Note: In general when compiling for Android with Bazel you need
-`--config=android` on the Bazel command line, though in this case this
-particular example is Android-only, so you don't need it here.
-
-This installs three apps on your phone that are all part of the TensorFlow
-Demo. See [Android Sample Apps](#android_sample_apps) for more information about
-them.
-
-## Android Sample Apps
-
-The
-[Android example code](https://www.tensorflow.org/code/tensorflow/examples/android/) is
-a single project that builds and installs three sample apps which all use the
-same underlying code. The sample apps all take video input from a phone's
-camera:
-
-- **TF Classify** uses the Inception v3 model to label the objects it’s pointed
-  at with classes from Imagenet. There are only 1,000 categories in Imagenet,
-  which misses most everyday objects and includes many things you’re unlikely to
-  encounter often in real life, so the results can often be quite amusing. For
-  example there’s no ‘person’ category, so instead it will often guess things it
-  does know that are often associated with pictures of people, like a seat belt
-  or an oxygen mask. If you do want to customize this example to recognize
-  objects you care about, you can use
-  the
-  [TensorFlow for Poets codelab](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/index.html#0) as
-  an example for how to train a model based on your own data.
-
-- **TF Detect** uses a multibox model to try to draw bounding boxes around the
-  locations of people in the camera. These boxes are annotated with the
-  confidence for each detection result. Results will not be perfect, as this
-  kind of object detection is still an active research topic.  The demo also
-  includes optical tracking for when objects move between frames, which runs
-  more frequently than the TensorFlow inference. This improves the user
-  experience since the apparent frame rate is faster, but it also gives the
-  ability to estimate which boxes refer to the same object between frames, which
-  is important for counting objects over time.
-
-- **TF Stylize** implements a real-time style transfer algorithm on the camera
-  feed. You can select which styles to use and mix between them using the
-  palette at the bottom of the screen, and also switch out the resolution of the
-  processing to go higher or lower rez.
-
-When you build and install the demo, you'll see three app icons on your phone,
-one for each of the demos. Tapping on them should open up the app and let you
-explore what they do. You can enable profiling statistics on-screen by tapping
-the volume up button while they’re running.
-
-### Android Inference Library
-
-Because Android apps need to be written in Java, and core TensorFlow is in C++,
-TensorFlow has a JNI library to interface between the two. Its interface is aimed
-only at inference, so it provides the ability to load a graph, set up inputs,
-and run the model to calculate particular outputs. You can see the full
-documentation for the minimal set of methods in
-[TensorFlowInferenceInterface.java](https://www.tensorflow.org/code/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java)
-
-The demos applications use this interface, so they’re a good place to look for
-example usage. You can download prebuilt binary jars
-at
-[ci.tensorflow.org](https://ci.tensorflow.org/view/Nightly/job/nightly-android/).
diff --git a/tensorflow/docs_src/mobile/index.md b/tensorflow/docs_src/mobile/index.md
deleted file mode 100644
index 6032fcad02..0000000000
--- a/tensorflow/docs_src/mobile/index.md
+++ /dev/null
@@ -1,33 +0,0 @@
-# Overview
-
-TensorFlow was designed to be a good deep learning solution for mobile
-platforms. Currently we have two solutions for deploying machine learning
-applications on mobile and embedded devices:
-@{$mobile/mobile_intro$TensorFlow for Mobile} and @{$mobile/tflite$TensorFlow Lite}.
-
-## TensorFlow Lite versus TensorFlow Mobile
-
-Here are a few of the differences between the two:
-
-- TensorFlow Lite is an evolution of TensorFlow Mobile.  In most cases, apps
-  developed with TensorFlow Lite will have a smaller binary size, fewer
-  dependencies, and better performance.
-
-- TensorFlow Lite supports only a limited set of operators, so not all models
-  will work on it by default. TensorFlow for Mobile has a fuller set of
-  supported functionality.
-
-TensorFlow Lite provides better performance and a small binary size on mobile
-platforms as well as the ability to leverage hardware acceleration if available
-on their platforms. In addition, it has many fewer dependencies so it can be
-built and hosted on simpler, more constrained device scenarios. TensorFlow Lite
-also allows targeting accelerators through the [Neural Networks
-API](https://developer.android.com/ndk/guides/neuralnetworks/index.html).
-
-TensorFlow Lite currently has coverage for a limited set of operators. While
-TensorFlow for Mobile supports only a constrained set of ops by default, in
-principle if you use an arbitrary operator in TensorFlow, it can be customized
-to build that kernel. Thus use cases which are not currently supported by
-TensorFlow Lite should continue to use TensorFlow for Mobile. As TensorFlow Lite
-evolves, it will gain additional operators, and the decision will be easier to
-make.
diff --git a/tensorflow/docs_src/mobile/ios_build.md b/tensorflow/docs_src/mobile/ios_build.md
deleted file mode 100644
index 4c84a1214a..0000000000
--- a/tensorflow/docs_src/mobile/ios_build.md
+++ /dev/null
@@ -1,107 +0,0 @@
-# Building TensorFlow on iOS
-
-## Using CocoaPods
-
-The simplest way to get started with TensorFlow on iOS is using the CocoaPods
-package management system. You can add the `TensorFlow-experimental` pod to your
-Podfile, which installs a universal binary framework. This makes it easy to get
-started but has the disadvantage of being hard to customize, which is important
-in case you want to shrink your binary size. If you do need the ability to
-customize your libraries, see later sections on how to do that.
-
-## Creating your own app
-
-If you'd like to add TensorFlow capabilities to your own app, do the following:
-
-- Create your own app or load your already-created app in XCode.
-
-- Add a file named Podfile at the project root directory with the following content:
-
-        target 'YourProjectName'
-        pod 'TensorFlow-experimental'
-
-- Run `pod install` to download and install the `TensorFlow-experimental` pod.
-
-- Open `YourProjectName.xcworkspace` and add your code.
-
-- In your app's **Build Settings**, make sure to add `$(inherited)` to the
-  **Other Linker Flags**, and **Header Search Paths** sections.
-
-## Running the Samples
-
-You'll need Xcode 7.3 or later to run our iOS samples.
-
-There are currently three examples: simple, benchmark, and camera. For now, you
-can download the sample code by cloning the main tensorflow repository (we are
-planning to make the samples available as a separate repository later).
-
-From the root of the tensorflow folder, download [Inception
-v1](https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip),
-and extract the label and graph files into the data folders inside both the
-simple and camera examples using these steps:
-
-    mkdir -p ~/graphs
-    curl -o ~/graphs/inception5h.zip \
-     https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip \
-     && unzip ~/graphs/inception5h.zip -d ~/graphs/inception5h
-    cp ~/graphs/inception5h/* tensorflow/examples/ios/benchmark/data/
-    cp ~/graphs/inception5h/* tensorflow/examples/ios/camera/data/
-    cp ~/graphs/inception5h/* tensorflow/examples/ios/simple/data/
-
-Change into one of the sample directories, download the
-[Tensorflow-experimental](https://cocoapods.org/pods/TensorFlow-experimental)
-pod, and open the Xcode workspace. Note that installing the pod can take a long
-time since it is big (~450MB). If you want to run the simple example, then:
-
-    cd tensorflow/examples/ios/simple
-    pod install
-    open tf_simple_example.xcworkspace   # note .xcworkspace, not .xcodeproj
-                                         # this is created by pod install
-
-Run the simple app in the XCode simulator. You should see a single-screen app
-with a **Run Model** button. Tap that, and you should see some debug output
-appear below indicating that the example Grace Hopper image in directory data
-has been analyzed, with a military uniform recognized.
-
-Run the other samples using the same process. The camera example requires a real
-device connected. Once you build and run that, you should get a live camera view
-that you can point at objects to get real-time recognition results.
-
-### iOS Example details
-
-There are three demo applications for iOS, all defined in Xcode projects inside
-[tensorflow/examples/ios](https://www.tensorflow.org/code/tensorflow/examples/ios/).
-
-- **Simple**: This is a minimal example showing how to load and run a TensorFlow
-  model in as few lines as possible. It just consists of a single view with a
-  button that executes the model loading and inference when its pressed.
-
-- **Camera**: This is very similar to the Android TF Classify demo. It loads
-  Inception v3 and outputs its best label estimate for what’s in the live camera
-  view. As with the Android version, you can train your own custom model using
-  TensorFlow for Poets and drop it into this example with minimal code changes.
-
-- **Benchmark**: is quite close to Simple, but it runs the graph repeatedly and
-  outputs similar statistics to the benchmark tool on Android.
-
-
-### Troubleshooting
-
-- Make sure you use the TensorFlow-experimental pod (and not TensorFlow).
-
-- The TensorFlow-experimental pod is current about ~450MB. The reason it is so
-  big is because we are bundling multiple platforms, and the pod includes all
-  TensorFlow functionality (e.g. operations). The final app size after build is
-  substantially smaller though (~25MB). Working with the complete pod is
-  convenient during development, but see below section on how you can build your
-  own custom TensorFlow library to reduce the size.
-
-## Building the TensorFlow iOS libraries from source
-
-While Cocoapods is the quickest and easiest way of getting started, you sometimes
-need more flexibility to determine which parts of TensorFlow your app should be
-shipped with. For such cases, you can build the iOS libraries from the
-sources. [This
-guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/ios#building-the-tensorflow-ios-libraries-from-source)
-contains detailed instructions on how to do that.
-
diff --git a/tensorflow/docs_src/mobile/leftnav_files b/tensorflow/docs_src/mobile/leftnav_files
deleted file mode 100644
index 97340ef7e1..0000000000
--- a/tensorflow/docs_src/mobile/leftnav_files
+++ /dev/null
@@ -1,15 +0,0 @@
-index.md
-### TensorFlow Lite
-tflite/index.md
-tflite/devguide.md
-tflite/demo_android.md
-tflite/demo_ios.md
-tflite/performance.md
->>>
-### TensorFlow Mobile
-mobile_intro.md
-android_build.md
-ios_build.md
-linking_libs.md
-prepare_models.md
-optimizing.md
diff --git a/tensorflow/docs_src/mobile/linking_libs.md b/tensorflow/docs_src/mobile/linking_libs.md
deleted file mode 100644
index efef5dd0da..0000000000
--- a/tensorflow/docs_src/mobile/linking_libs.md
+++ /dev/null
@@ -1,243 +0,0 @@
-# Integrating TensorFlow libraries
-
-Once you have made some progress on a model that addresses the problem you’re
-trying to solve, it’s important to test it out inside your application
-immediately. There are often unexpected differences between your training data
-and what users actually encounter in the real world, and getting a clear picture
-of the gap as soon as possible improves the product experience.
-
-This page talks about how to integrate the TensorFlow libraries into your own
-mobile applications, once you have already successfully built and deployed the
-TensorFlow mobile demo apps.
-
-## Linking the library
-
-After you've managed to build the examples, you'll probably want to call
-TensorFlow from one of your existing applications. The very easiest way to do
-this is to use the Pod installation steps described
-@{$mobile/ios_build#using_cocoapods$here}, but if you want to build TensorFlow
-from source (for example to customize which operators are included) you'll need
-to break out TensorFlow as a framework, include the right header files, and link
-against the built libraries and dependencies.
-
-### Android
-
-For Android, you just need to link in a Java library contained in a JAR file
-called `libandroid_tensorflow_inference_java.jar`. There are three ways to
-include this functionality in your program:
-
-1. Include the jcenter AAR which contains it, as in this
- [example app](https://github.com/googlecodelabs/tensorflow-for-poets-2/blob/master/android/tfmobile/build.gradle#L59-L65)
-
-2. Download the nightly precompiled version from
-[ci.tensorflow.org](http://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/).
-
-3. Build the JAR file yourself using the instructions [in our Android GitHub repo](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/android)
-
-### iOS
-
-Pulling in the TensorFlow libraries on iOS is a little more complicated. Here is
-a checklist of what you’ll need to do to your iOS app:
-
-- Link against tensorflow/contrib/makefile/gen/lib/libtensorflow-core.a, usually
-  by adding `-L/your/path/tensorflow/contrib/makefile/gen/lib/` and
-  `-ltensorflow-core` to your linker flags.
-
-- Link against the generated protobuf libraries by adding
-  `-L/your/path/tensorflow/contrib/makefile/gen/protobuf_ios/lib` and
-  `-lprotobuf` and `-lprotobuf-lite` to your command line.
-
-- For the include paths, you need the root of your TensorFlow source folder as
-  the first entry, followed by
-  `tensorflow/contrib/makefile/downloads/protobuf/src`,
-  `tensorflow/contrib/makefile/downloads`,
-  `tensorflow/contrib/makefile/downloads/eigen`, and
-  `tensorflow/contrib/makefile/gen/proto`.
-
-- Make sure your binary is built with `-force_load` (or the equivalent on your
-  platform), aimed at the TensorFlow library to ensure that it’s linked
-  correctly. More detail on why this is necessary can be found in the next
-  section, [Global constructor magic](#global_constructor_magic). On Linux-like
-  platforms, you’ll need different flags, more like
-  `-Wl,--allow-multiple-definition -Wl,--whole-archive`.
-
-You’ll also need to link in the Accelerator framework, since this is used to
-speed up some of the operations.
-
-## Global constructor magic
-
-One of the subtlest problems you may run up against is the “No session factory
-registered for the given session options” error when trying to call TensorFlow
-from your own application. To understand why this is happening and how to fix
-it, you need to know a bit about the architecture of TensorFlow.
-
-The framework is designed to be very modular, with a thin core and a large
-number of specific objects that are independent and can be mixed and matched as
-needed. To enable this, the coding pattern in C++ had to let modules easily
-notify the framework about the services they offer, without requiring a central
-list that has to be updated separately from each implementation. It also had to
-allow separate libraries to add their own implementations without needing a
-recompile of the core.
-
-To achieve this capability, TensorFlow uses a registration pattern in a lot of
-places. In the code, it looks like this:
-
-    class MulKernel : OpKernel {
-      Status Compute(OpKernelContext* context) { … }
-    };
-    REGISTER_KERNEL(MulKernel, “Mul”);
-
-This would be in a standalone `.cc` file linked into your application, either
-as part of the main set of kernels or as a separate custom library. The magic
-part is that the `REGISTER_KERNEL()` macro is able to inform the core of
-TensorFlow that it has an implementation of the Mul operation, so that it can be
-called in any graphs that require it.
-
-From a programming point of view, this setup is very convenient. The
-implementation and registration code live in the same file, and adding new
-implementations is as simple as compiling and linking it in. The difficult part
-comes from the way that the `REGISTER_KERNEL()` macro is implemented. C++
-doesn’t offer a good mechanism for doing this sort of registration, so we have
-to resort to some tricky code. Under the hood, the macro is implemented so that
-it produces something like this:
-
-    class RegisterMul {
-     public:
-      RegisterMul() {
-        global_kernel_registry()->Register(“Mul”, [](){
-          return new MulKernel()
-        });
-      }
-    };
-    RegisterMul g_register_mul;
-
-This sets up a class `RegisterMul` with a constructor that tells the global
-kernel registry what function to call when somebody asks it how to create a
-“Mul” kernel. Then there’s a global object of that class, and so the constructor
-should be called at the start of any program.
-
-While this may sound sensible, the unfortunate part is that the global object
-that’s defined is not used by any other code, so linkers not designed with this
-in mind will decide that it can be deleted. As a result, the constructor is
-never called, and the class is never registered. All sorts of modules use this
-pattern in TensorFlow, and it happens that `Session` implementations are the
-first to be looked for when the code is run, which is why it shows up as the
-characteristic error when this problem occurs.
-
-The solution is to force the linker to not strip any code from the library, even
-if it believes it’s unused. On iOS, this step can be accomplished with the
-`-force_load` flag, specifying a library path, and on Linux you need
-`--whole-archive`. These persuade the linker to not be as aggressive about
-stripping, and should retain the globals.
-
-The actual implementation of the various `REGISTER_*` macros is a bit more
-complicated in practice, but they all suffer the same underlying problem. If
-you’re interested in how they work, [op_kernel.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_kernel.h#L1091)
-is a good place to start investigating.
-
-## Protobuf problems
-
-TensorFlow relies on
-the [Protocol Buffer](https://developers.google.com/protocol-buffers/) library,
-commonly known as protobuf. This library takes definitions of data structures
-and produces serialization and access code for them in a variety of
-languages. The tricky part is that this generated code needs to be linked
-against shared libraries for the exact same version of the framework that was
-used for the generator. This can be an issue when `protoc`, the tool used to
-generate the code, is from a different version of protobuf than the libraries in
-the standard linking and include paths. For example, you might be using a copy
-of `protoc` that was built locally in `~/projects/protobuf-3.0.1.a`, but you have
-libraries installed at `/usr/local/lib` and `/usr/local/include` that are from
-3.0.0.
-
-The symptoms of this issue are errors during the compilation or linking phases
-with protobufs. Usually, the build tools take care of this, but if you’re using
-the makefile, make sure you’re building the protobuf library locally and using
-it, as shown in [this Makefile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/makefile/Makefile#L18).
-
-Another situation that can cause problems is when protobuf headers and source
-files need to be generated as part of the build process. This process makes
-building more complex, since the first phase has to be a pass over the protobuf
-definitions to create all the needed code files, and only after that can you go
-ahead and do a build of the library code.
-
-### Multiple versions of protobufs in the same app
-
-Protobufs generate headers that are needed as part of the C++ interface to the
-overall TensorFlow library. This complicates using the library as a standalone
-framework.
-
-If your application is already using version 1 of the protocol buffers library,
-you may have trouble integrating TensorFlow because it requires version 2. If
-you just try to link both versions into the same binary, you’ll see linking
-errors because some of the symbols clash. To solve this particular problem, we
-have an experimental script at [rename_protobuf.sh](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/makefile/rename_protobuf.sh).
-
-You need to run this as part of the makefile build, after you’ve downloaded all
-the dependencies:
-
-    tensorflow/contrib/makefile/download_dependencies.sh
-    tensorflow/contrib/makefile/rename_protobuf.sh
-
-## Calling the TensorFlow API
-
-Once you have the framework available, you then need to call into it. The usual
-pattern is that you first load your model, which represents a preset set of
-numeric computations, and then you run inputs through that model (for example,
-images from a camera) and receive outputs (for example, predicted labels).
-
-On Android, we provide the Java Inference Library that is focused on just this
-use case, while on iOS and Raspberry Pi you call directly into the C++ API.
-
-### Android
-
-Here’s what a typical Inference Library sequence looks like on Android:
-
-    // Load the model from disk.
-    TensorFlowInferenceInterface inferenceInterface =
-    new TensorFlowInferenceInterface(assetManager, modelFilename);
-
-    // Copy the input data into TensorFlow.
-    inferenceInterface.feed(inputName, floatValues, 1, inputSize, inputSize, 3);
-
-    // Run the inference call.
-    inferenceInterface.run(outputNames, logStats);
-
-    // Copy the output Tensor back into the output array.
-    inferenceInterface.fetch(outputName, outputs);
-
-You can find the source of this code in the [Android examples](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/android/src/org/tensorflow/demo/TensorFlowImageClassifier.java#L107).
-
-### iOS and Raspberry Pi
-
-Here’s the equivalent code for iOS and Raspberry Pi:
-
-    // Load the model.
-    PortableReadFileToProto(file_path, &tensorflow_graph);
-
-    // Create a session from the model.
-    tensorflow::Status s = session->Create(tensorflow_graph);
-    if (!s.ok()) {
-      LOG(FATAL) << "Could not create TensorFlow Graph: " << s;
-    }
-
-    // Run the model.
-    std::string input_layer = "input";
-    std::string output_layer = "output";
-    std::vector outputs;
-    tensorflow::Status run_status = session->Run({{input_layer, image_tensor}},
-                               {output_layer}, {}, &outputs);
-    if (!run_status.ok()) {
-      LOG(FATAL) << "Running model failed: " << run_status;
-    }
-
-    // Access the output data.
-    tensorflow::Tensor* output = &outputs[0];
-
-This is all based on the
-[iOS sample code](https://www.tensorflow.org/code/tensorflow/examples/ios/simple/RunModelViewController.mm),
-but there’s nothing iOS-specific; the same code should be usable on any platform
-that supports C++.
-
-You can also find specific examples for Raspberry Pi
-[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/pi_examples/label_image/label_image.cc).
diff --git a/tensorflow/docs_src/mobile/mobile_intro.md b/tensorflow/docs_src/mobile/mobile_intro.md
deleted file mode 100644
index baad443308..0000000000
--- a/tensorflow/docs_src/mobile/mobile_intro.md
+++ /dev/null
@@ -1,248 +0,0 @@
-# Introduction to TensorFlow Mobile
-
-TensorFlow was designed from the ground up to be a good deep learning solution
-for mobile platforms like Android and iOS. This mobile guide should help you
-understand how machine learning can work on mobile platforms and how to
-integrate TensorFlow into your mobile apps effectively and efficiently.
-
-## About this Guide
-
-This guide is aimed at developers who have a TensorFlow model that’s
-successfully working in a desktop environment, who want to integrate it into
-a mobile application, and cannot use TensorFlow Lite. Here are the
-main challenges you’ll face during that process:
-
-- Understanding how to use Tensorflow for mobile.
-- Building TensorFlow for your platform.
-- Integrating the TensorFlow library into your application.
-- Preparing your model file for mobile deployment.
-- Optimizing for latency, RAM usage, model file size, and binary size.
-
-## Common use cases for mobile machine learning
-
-**Why run TensorFlow on mobile?**
-
-Traditionally, deep learning has been associated with data centers and giant
-clusters of high-powered GPU machines. However, it can be very expensive and
-time-consuming to send all of the data a device has access to across a network
-connection. Running on mobile makes it possible to deliver very interactive
-applications in a way that’s not possible when you have to wait for a network
-round trip.
-
-Here are some common use cases for on-device deep learning:
-
-### Speech Recognition
-
-There are a lot of interesting applications that can be built with a
-speech-driven interface, and many of these require on-device processing. Most of
-the time a user isn’t giving commands, and so streaming audio continuously to a
-remote server would be a waste of bandwidth, since it would mostly be silence or
-background noises. To solve this problem it’s common to have a small neural
-network running on-device
-[listening out for a particular keyword](../tutorials/sequences/audio_recognition).
-Once that keyword has been spotted, the rest of the
-conversation can be transmitted over to the server for further processing if
-more computing power is needed.
-
-### Image Recognition
-
-It can be very useful for a mobile app to be able to make sense of a camera
-image. If your users are taking photos, recognizing what’s in them can help your
-camera apps apply appropriate filters, or label the photos so they’re easily
-findable. It’s important for embedded applications too, since you can use image
-sensors to detect all sorts of interesting conditions, whether it’s spotting
-endangered animals in the wild
-or
-[reporting how late your train is running](https://svds.com/tensorflow-image-recognition-raspberry-pi/).
-
-TensorFlow comes with several examples of recognizing the types of objects
-inside images along with a variety of different pre-trained models, and they can
-all be run on mobile devices. You can try out
-our
-[Tensorflow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/index.html#0) and
-[Tensorflow for Poets 2: Optimize for Mobile](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/index.html#0) codelabs to
-see how to take a pretrained model and run some very fast and lightweight
-training to teach it to recognize specific objects, and then optimize it to
-run on mobile.
-
-### Object Localization
-
-Sometimes it’s important to know where objects are in an image as well as what
-they are. There are lots of augmented reality use cases that could benefit a
-mobile app, such as guiding users to the right component when offering them
-help fixing their wireless network or providing informative overlays on top of
-landscape features. Embedded applications often need to count objects that are
-passing by them, whether it’s pests in a field of crops, or people, cars and
-bikes going past a street lamp.
-
-TensorFlow offers a pretrained model for drawing bounding boxes around people
-detected in images, together with tracking code to follow them over time. The
-tracking is especially important for applications where you’re trying to count
-how many objects are present over time, since it gives you a good idea when a
-new object enters or leaves the scene. We have some sample code for this
-available for Android [on
-GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android),
-and also a [more general object detection
-model](https://github.com/tensorflow/models/tree/master/research/object_detection/README.md)
-available as well.
-
-### Gesture Recognition
-
-It can be useful to be able to control applications with hand or other
-gestures, either recognized from images or through analyzing accelerometer
-sensor data. Creating those models is beyond the scope of this guide, but
-TensorFlow is an effective way of deploying them.
-
-### Optical Character Recognition
-
-Google Translate’s live camera view is a great example of how effective
-interactive on-device detection of text can be.
-
-
    
-  
-
    
-
-There are multiple steps involved in recognizing text in images. You first have
-to identify the areas where the text is present, which is a variation on the
-object localization problem, and can be solved with similar techniques. Once you
-have an area of text, you then need to interpret it as letters, and then use a
-language model to help guess what words they represent. The simplest way to
-estimate what letters are present is to segment the line of text into individual
-letters, and then apply a simple neural network to the bounding box of each. You
-can get good results with the kind of models used for MNIST, which you can find
-in TensorFlow’s tutorials, though you may want a higher-resolution input.  A
-more advanced alternative is to use an LSTM model to process a whole line of
-text at once, with the model itself handling the segmentation into different
-characters.
-
-### Translation
-
-Translating from one language to another quickly and accurately, even if you
-don’t have a network connection, is an important use case. Deep networks are
-very effective at this sort of task, and you can find descriptions of a lot of
-different models in the literature. Often these are sequence-to-sequence
-recurrent models where you’re able to run a single graph to do the whole
-translation, without needing to run separate parsing stages.
-
-### Text Classification
-
-If you want to suggest relevant prompts to users based on what they’re typing or
-reading, it can be very useful to understand the meaning of the text. This is
-where text classification comes in. Text classification is an umbrella term
-that covers everything from sentiment analysis to topic discovery. You’re likely
-to have your own categories or labels that you want to apply, so the best place
-to start is with an example
-like
-[Skip-Thoughts](https://github.com/tensorflow/models/tree/master/research/skip_thoughts/),
-and then train on your own examples.
-
-### Voice Synthesis
-
-A synthesized voice can be a great way of giving users feedback or aiding
-accessibility, and recent advances such as
-[WaveNet](https://deepmind.com/blog/wavenet-generative-model-raw-audio/) show
-that deep learning can offer very natural-sounding speech.
-
-## Mobile machine learning and the cloud
-
-These examples of use cases give an idea of how on-device networks can
-complement cloud services. Cloud has a great deal of computing power in a
-controlled environment, but running on devices can offer higher interactivity.
-In situations where the cloud is unavailable, or your cloud capacity is limited,
-you can provide an offline experience, or reduce cloud workload by processing
-easy cases on device.
-
-Doing on-device computation can also signal when it's time to switch to working
-on the cloud. A good example of this is hotword detection in speech. Since
-devices are able to constantly listen out for the keywords, this then triggers a
-lot of traffic to cloud-based speech recognition once one is recognized. Without
-the on-device component, the whole application wouldn’t be feasible, and this
-pattern exists across several other applications as well. Recognizing that some
-sensor input is interesting enough for further processing makes a lot of
-interesting products possible.
-
-## What hardware and software should you have?
-
-TensorFlow runs on Ubuntu Linux, Windows 10, and OS X. For a list of all
-supported operating systems and instructions to install TensorFlow, see
-@{$install$Installing Tensorflow}.
-
-Note that some of the sample code we provide for mobile TensorFlow requires you
-to compile TensorFlow from source, so you’ll need more than just `pip install`
-to work through all the sample code.
-
-To try out the mobile examples, you’ll need a device set up for development,
-using
-either [Android Studio](https://developer.android.com/studio/install.html),
-or [XCode](https://developer.apple.com/xcode/) if you're developing for iOS.
-
-## What should you do before you get started?
-
-Before thinking about how to get your solution on mobile:
-
-1. Determine whether your problem is solvable by mobile machine learning
-2. Create a labelled dataset to define your problem
-3. Pick an effective model for the problem
-
-We'll discuss these in more detail below.
-
-### Is your problem solvable by mobile machine learning?
-
-Once you have an idea of the problem you want to solve, you need to make a plan
-of how to build your solution. The most important first step is making sure that
-your problem is actually solvable, and the best way to do that is to mock it up
-using humans in the loop.
-
-For example, if you want to drive a robot toy car using voice commands, try
-recording some audio from the device and listen back to it to see if you can
-make sense of what’s being said. Often you’ll find there are problems in the
-capture process, such as the motor drowning out speech or not being able to hear
-at a distance, and you should tackle these problems before investing in the
-modeling process.
-
-Another example would be giving photos taken from your app to people see if they
-can classify what’s in them, in the way you’re looking for. If they can’t do
-that (for example, trying to estimate calories in food from photos may be
-impossible because all white soups look the same), then you’ll need to redesign
-your experience to cope with that. A good rule of thumb is that if a human can’t
-handle the task then it will be difficult to train a computer to do better.
-
-### Create a labelled dataset
-
-After you’ve solved any fundamental issues with your use case, you need to
-create a labeled dataset to define what problem you’re trying to solve. This
-step is extremely important, more than picking which model to use. You want it
-to be as representative as possible of your actual use case, since the model
-will only be effective at the task you teach it. It’s also worth investing in
-tools to make labeling the data as efficient and accurate as possible. For
-example, if you’re able to switch from having to click a button on a web
-interface to simple keyboard shortcuts, you may be able to speed up the
-generation process a lot. You should also start by doing the initial labeling
-yourself, so you can learn about the difficulties and likely errors, and
-possibly change your labeling or data capture process to avoid them. Once you
-and your team are able to consistently label examples (that is once you
-generally agree on the same labels for most examples), you can then try and
-capture your knowledge in a manual and teach external raters how to run the same
-process.
-
-### Pick an effective model
-
-The next step is to pick an effective model to use. You might be able to avoid
-training a model from scratch if someone else has already implemented a model
-similar to what you need; we have a repository of models implemented in
-TensorFlow [on GitHub](https://github.com/tensorflow/models) that you can look
-through. Lean towards the simplest model you can find, and try to get started as
-soon as you have even a small amount of labelled data, since you’ll get the best
-results when you’re able to iterate quickly. The shorter the time it takes to
-try training a model and running it in its real application, the better overall
-results you’ll see. It’s common for an algorithm to get great training accuracy
-numbers but then fail to be useful within a real application because there’s a
-mismatch between the dataset and real usage. Prototype end-to-end usage as soon
-as possible to create a consistent user experience.
-
-## Next Steps
-
-We suggest you get started by building one of our demos for
-@{$mobile/android_build$Android} or @{$mobile/ios_build$iOS}.
diff --git a/tensorflow/docs_src/mobile/optimizing.md b/tensorflow/docs_src/mobile/optimizing.md
deleted file mode 100644
index 778e4d3a62..0000000000
--- a/tensorflow/docs_src/mobile/optimizing.md
+++ /dev/null
@@ -1,499 +0,0 @@
-# Optimizing for mobile
-
-There are some special issues that you have to deal with when you’re trying to
-ship on mobile or embedded devices, and you’ll need to think about these as
-you’re developing your model.
-
-These issues are:
-
-- Model and Binary Size
-- App speed and model loading speed
-- Performance and threading
-
-We'll discuss a few of these below.
-
-## What are the minimum device requirements for TensorFlow?
-
-You need at least one megabyte of program memory and several megabytes of RAM to
-run the base TensorFlow runtime, so it’s not suitable for DSPs or
-microcontrollers. Other than those, the biggest constraint is usually the
-calculation speed of the device, and whether you can run the model you need for
-your application with a low enough latency. You can use the benchmarking tools
-in [How to Profile your Model](#how_to_profile_your_model) to get an idea of how
-many FLOPs are required for a model, and then use that to make rule-of-thumb
-estimates of how fast they will run on different devices. For example, a modern
-smartphone might be able to run 10 GFLOPs per second, so the best you could hope
-for from a 5 GFLOP model is two frames per second, though you may do worse
-depending on what the exact computation patterns are.
-
-This model dependence means that it’s possible to run TensorFlow even on very
-old or constrained phones, as long as you optimize your network to fit within
-the latency budget and possibly within limited RAM too. For memory usage, you
-mostly need to make sure that the intermediate buffers that TensorFlow creates
-aren’t too large, which you can examine in the benchmark output too.
-
-## Speed
-
-One of the highest priorities of most model deployments is figuring out how to
-run the inference fast enough to give a good user experience. The first place to
-start is by looking at the total number of floating point operations that are
-required to execute the graph. You can get a very rough estimate of this by
-using the `benchmark_model` tool:
-
-    bazel build -c opt tensorflow/tools/benchmark:benchmark_model && \
-    bazel-bin/tensorflow/tools/benchmark/benchmark_model \
-    --graph=/tmp/inception_graph.pb --input_layer="Mul:0" \
-    --input_layer_shape="1,299,299,3" --input_layer_type="float" \
-    --output_layer="softmax:0" --show_run_order=false --show_time=false \
-    --show_memory=false --show_summary=true --show_flops=true --logtostderr
-
-This should show you an estimate of how many operations are needed to run the
-graph. You can then use that information to figure out how feasible your model
-is to run on the devices you’re targeting. For an example, a high-end phone from
-2016 might be able to do 20 billion FLOPs per second, so the best speed you
-could hope for from a model that requires 10 billion FLOPs is around 500ms. On a
-device like the Raspberry Pi 3 that can do about 5 billion FLOPs, you may only
-get one inference every two seconds.
-
-Having this estimate helps you plan for what you’ll be able to realistically
-achieve on a device. If the model is using too many ops, then there are a lot of
-opportunities to optimize the architecture to reduce that number.
-
-Advanced techniques include [SqueezeNet](https://arxiv.org/abs/1602.07360)
-and [MobileNet](https://arxiv.org/abs/1704.04861), which are architectures
-designed to produce models for mobile -- lean and fast but with a small accuracy
-cost.  You can also just look at alternative models, even older ones, which may
-be smaller. For example, Inception v1 only has around 7 million parameters,
-compared to Inception v3’s 24 million, and requires only 3 billion FLOPs rather
-than 9 billion for v3.
-
-## Model Size
-
-Models that run on a device need to be stored somewhere on the device, and very
-large neural networks can be hundreds of megabytes. Most users are reluctant to
-download very large app bundles from app stores, so you want to make your model
-as small as possible. Furthermore, smaller neural networks can persist in and
-out of a mobile device's memory faster.
-
-To understand how large your network will be on disk, start by looking at the
-size on disk of your `GraphDef` file after you’ve run `freeze_graph` and
-`strip_unused_nodes` on it (see @{$mobile/prepare_models$Preparing models} for
-more details on these tools), since then it should only contain
-inference-related nodes. To double-check that your results are as expected, run
-the `summarize_graph` tool to see how many parameters are in constants:
-
-    bazel build tensorflow/tools/graph_transforms:summarize_graph && \
-    bazel-bin/tensorflow/tools/graph_transforms/summarize_graph \
-    --in_graph=/tmp/tensorflow_inception_graph.pb
-
-That command should give you output that looks something like this:
-
-    No inputs spotted.
-    Found 1 possible outputs: (name=softmax, op=Softmax)
-    Found 23885411 (23.89M) const parameters, 0 (0) variable parameters,
-    and 99 control_edges
-    Op types used: 489 Const, 99 CheckNumerics, 99 Identity, 94
-    BatchNormWithGlobalNormalization, 94 Conv2D, 94 Relu, 11 Concat, 9 AvgPool,
-    5 MaxPool, 1 Sub, 1 Softmax, 1 ResizeBilinear, 1 Reshape, 1 Mul, 1 MatMul,
-    1 ExpandDims, 1 DecodeJpeg, 1 Cast, 1 BiasAdd
-
-The important part for our current purposes is the number of const
-parameters. In most models these will be stored as 32-bit floats to start, so if
-you multiply the number of const parameters by four, you should get something
-that’s close to the size of the file on disk. You can often get away with only
-eight-bits per parameter with very little loss of accuracy in the final result,
-so if your file size is too large you can try using
-@{$performance/quantization$quantize_weights} to transform the parameters down.
-
-    bazel build tensorflow/tools/graph_transforms:transform_graph && \
-    bazel-bin/tensorflow/tools/graph_transforms/transform_graph \
-    --in_graph=/tmp/tensorflow_inception_optimized.pb \
-    --out_graph=/tmp/tensorflow_inception_quantized.pb \
-    --inputs='Mul:0' --outputs='softmax:0' --transforms='quantize_weights'
-
-If you look at the resulting file size, you should see that it’s about a quarter
-of the original at 23MB.
-
-Another transform is `round_weights`, which doesn't make the file smaller, but it
-makes the file compressible to about the same size as when `quantize_weights` is
-used. This is particularly useful for mobile development, taking advantage of
-the fact that app bundles are compressed before they’re downloaded by consumers.
-
-The original file does not compress well with standard algorithms, because the
-bit patterns of even very similar numbers can be very different. The
-`round_weights` transform keeps the weight parameters stored as floats, but
-rounds them to a set number of step values. This means there are a lot more
-repeated byte patterns in the stored model, and so compression can often bring
-the size down dramatically, in many cases to near the size it would be if they
-were stored as eight bit.
-
-Another advantage of `round_weights` is that the framework doesn’t have to
-allocate a temporary buffer to unpack the parameters into, as we have to when
-we just use `quantize_weights`. This saves a little bit of latency (though the
-results should be cached so it’s only costly on the first run) and makes it
-possible to use memory mapping, as described later.
-
-## Binary Size
-
-One of the biggest differences between mobile and server development is the
-importance of binary size. On desktop machines it’s not unusual to have
-executables that are hundreds of megabytes on disk, but for mobile and embedded
-apps it’s vital to keep the binary as small as possible so that user downloads
-are easy. As mentioned above, TensorFlow only includes a subset of op
-implementations by default, but this still results in a 12 MB final
-executable. To reduce this, you can set up the library to only include the
-implementations of the ops that you actually need, based on automatically
-analyzing your model. To use it:
-
-- Run `tools/print_required_ops/print_selective_registration_header.py` on your
-  model to produce a header file that only enables the ops it uses.
-
-- Place the `ops_to_register.h` file somewhere that the compiler can find
-  it. This can be in the root of your TensorFlow source folder.
-
-- Build TensorFlow with `SELECTIVE_REGISTRATION` defined, for example by passing
-  in `--copts=”-DSELECTIVE_REGISTRATION”` to your Bazel build command.
-
-This process recompiles the library so that only the needed ops and types are
-included, which can dramatically reduce the executable size. For example, with
-Inception v3, the new size is only 1.5MB.
-
-## How to Profile your Model
-
-Once you have an idea of what your device's peak performance range is, it’s
-worth looking at its actual current performance. Using a standalone TensorFlow
-benchmark, rather than running it inside a larger app, helps isolate just the
-Tensorflow contribution to the
-latency. The
-[tensorflow/tools/benchmark](https://www.tensorflow.org/code/tensorflow/tools/benchmark/) tool
-is designed to help you do this. To run it on Inception v3 on your desktop
-machine, build this benchmark model:
-
-    bazel build -c opt tensorflow/tools/benchmark:benchmark_model && \
-    bazel-bin/tensorflow/tools/benchmark/benchmark_model \
-    --graph=/tmp/tensorflow_inception_graph.pb --input_layer="Mul" \
-    --input_layer_shape="1,299,299,3" --input_layer_type="float" \
-    --output_layer="softmax:0" --show_run_order=false --show_time=false \
-    --show_memory=false --show_summary=true --show_flops=true --logtostderr
-
-You should see output that looks something like this:
-
-
    -============================== Top by Computation Time ==============================
-[node
- type]  [start]  [first] [avg ms]     [%]  [cdf%]  [mem KB]  [Name]
-Conv2D   22.859   14.212   13.700  4.972%  4.972%  3871.488  conv_4/Conv2D
-Conv2D    8.116    8.964   11.315  4.106%  9.078%  5531.904  conv_2/Conv2D
-Conv2D   62.066   16.504    7.274  2.640% 11.717%   443.904  mixed_3/conv/Conv2D
-Conv2D    2.530    6.226    4.939  1.792% 13.510%  2765.952  conv_1/Conv2D
-Conv2D   55.585    4.605    4.665  1.693% 15.203%   313.600  mixed_2/tower/conv_1/Conv2D
-Conv2D  127.114    5.469    4.630  1.680% 16.883%    81.920  mixed_10/conv/Conv2D
-Conv2D   47.391    6.994    4.588  1.665% 18.548%   313.600  mixed_1/tower/conv_1/Conv2D
-Conv2D   39.463    7.878    4.336  1.574% 20.122%   313.600  mixed/tower/conv_1/Conv2D
-Conv2D  127.113    4.192    3.894  1.413% 21.535%   114.688  mixed_10/tower_1/conv/Conv2D
-Conv2D   70.188    5.205    3.626  1.316% 22.850%   221.952  mixed_4/conv/Conv2D
-
-============================== Summary by node type ==============================
-[Node type]  [count]  [avg ms]    [avg %]    [cdf %]  [mem KB]
-Conv2D            94   244.899    88.952%    88.952% 35869.953
-BiasAdd           95     9.664     3.510%    92.462% 35873.984
-AvgPool            9     7.990     2.902%    95.364%  7493.504
-Relu              94     5.727     2.080%    97.444% 35869.953
-MaxPool            5     3.485     1.266%    98.710%  3358.848
-Const            192     1.727     0.627%    99.337%     0.000
-Concat            11     1.081     0.393%    99.730%  9892.096
-MatMul             1     0.665     0.242%    99.971%     4.032
-Softmax            1     0.040     0.015%    99.986%     4.032
-<>                 1     0.032     0.012%    99.997%     0.000
-Reshape            1     0.007     0.003%   100.000%     0.000
-
-Timings (microseconds): count=50 first=330849 curr=274803 min=232354 max=415352 avg=275563 std=44193
-Memory (bytes): count=50 curr=128366400(all same)
-514 nodes defined 504 nodes observed
-
    
-
-This is the summary view, which is enabled by the show_summary flag. To
-interpret it, the first table is a list of the nodes that took the most time, in
-order by how long they took. From left to right, the columns are:
-
-- Node type, what kind of operation this was.
-
-- Start time of the op, showing where it falls in the sequence of operations.
-
-- First time in milliseconds. This is how long the operation took on the first
-  run of the benchmark, since by default 20 runs are executed to get more
-  reliable statistics. The first time is useful to spot which ops are doing
-  expensive calculations on the first run, and then caching the results.
-
-- Average time for the operation across all runs, in milliseconds.
-
-- What percentage of the total time for one run the op took. This is useful to
-  understand where the hotspots are.
-
-- The cumulative total time of this and the previous ops in the table. This is
-  handy for understanding what the distribution of work is across the layers, to
-  see if just a few of the nodes are taking up most of the time.
-  
-- The amount of memory consumed by outputs of this type of op.
-
-- Name of the node.
-
-The second table is similar, but instead of breaking down the timings by
-particular named nodes, it groups them by the kind of op. This is very useful to
-understand which op implementations you might want to optimize or eliminate from
-your graph. The table is arranged with the most costly operations at the start,
-and only shows the top ten entries, with a placeholder for other nodes. The
-columns from left to right are:
-
-- Type of the nodes being analyzed.
-
-- Accumulated average time taken by all nodes of this type, in milliseconds.
-
-- What percentage of the total time was taken by this type of operation.
-
-- Cumulative time taken by this and op types higher in the table, so you can
-  understand the distribution of the workload.
-
--  How much memory the outputs of this op type took up.
-
-Both of these tables are set up so that you can easily copy and paste their
-results into spreadsheet documents, since they are output with tabs as
-separators between the columns. The summary by node type can be the most useful
-when looking for optimization opportunities, since it’s a pointer to the code
-that’s taking the most time. In this case, you can see that the Conv2D ops are
-almost 90% of the execution time. This is a sign that the graph is pretty
-optimal, since convolutions and matrix multiplies are expected to be the bulk of
-a neural network’s computing workload.
-
-As a rule of thumb, it’s more worrying if you see a lot of other operations
-taking up more than a small fraction of the time. For neural networks, the ops
-that don’t involve large matrix multiplications should usually be dwarfed by the
-ones that do, so if you see a lot of time going into those it’s a sign that
-either your network is non-optimally constructed, or the code implementing those
-ops is not as optimized as it could
-be. [Performance bugs](https://github.com/tensorflow/tensorflow/issues) or
-patches are always welcome if you do encounter this situation, especially if
-they include an attached model exhibiting this behavior and the command line
-used to run the benchmark tool on it.
-
-The run above was on your desktop, but the tool also works on Android, which is
-where it’s most useful for mobile development. Here’s an example command line to
-run it on a 64-bit ARM device:
-
-    bazel build -c opt --config=android_arm64 \
-    tensorflow/tools/benchmark:benchmark_model
-    adb push bazel-bin/tensorflow/tools/benchmark/benchmark_model /data/local/tmp
-    adb push /tmp/tensorflow_inception_graph.pb /data/local/tmp/
-    adb shell '/data/local/tmp/benchmark_model \
-    --graph=/data/local/tmp/tensorflow_inception_graph.pb --input_layer="Mul" \
-    --input_layer_shape="1,299,299,3" --input_layer_type="float" \
-    --output_layer="softmax:0" --show_run_order=false --show_time=false \
-    --show_memory=false --show_summary=true'
-
-You can interpret the results in exactly the same way as the desktop version
-above. If you have any trouble figuring out what the right input and output
-names and types are, take a look at the @{$mobile/prepare_models$Preparing models}
-page for details about detecting these for your model, and look at the
-`summarize_graph` tool which may give you
-helpful information.
-
-There isn’t good support for command line tools on iOS, so instead there’s a
-separate example
-at
-[tensorflow/examples/ios/benchmark](https://www.tensorflow.org/code/tensorflow/examples/ios/benchmark) that
-packages the same functionality inside a standalone app. This outputs the
-statistics to both the screen of the device and the debug log. If you want
-on-screen statistics for the Android example apps, you can turn them on by
-pressing the volume-up button.
-
-## Profiling within your own app
-
-The output you see from the benchmark tool is generated from modules that are
-included as part of the standard TensorFlow runtime, which means you have access
-to them within your own applications too. You can see an example of how to do
-that [here](https://www.tensorflow.org/code/tensorflow/examples/ios/benchmark/BenchmarkViewController.mm?l=139).
-
-The basic steps are:
-
-1. Create a StatSummarizer object:
-
-        tensorflow::StatSummarizer stat_summarizer(tensorflow_graph);
-
-2. Set up the options:
-
-        tensorflow::RunOptions run_options;
-        run_options.set_trace_level(tensorflow::RunOptions::FULL_TRACE);
-        tensorflow::RunMetadata run_metadata;
-
-3. Run the graph:
-
-        run_status = session->Run(run_options, inputs, output_layer_names, {},
-                                  output_layers, &run_metadata);
-
-4. Calculate the results and print them out:
-
-        assert(run_metadata.has_step_stats());
-        const tensorflow::StepStats& step_stats = run_metadata.step_stats();
-        stat_summarizer->ProcessStepStats(step_stats);
-        stat_summarizer->PrintStepStats();
-
-## Visualizing Models
-
-The most effective way to speed up your code is by altering your model so it
-does less work. To do that, you need to understand what your model is doing, and
-visualizing it is a good first step. To get a high-level overview of your graph,
-use [TensorBoard](https://github.com/tensorflow/tensorboard).
-
-## Threading
-
-The desktop version of TensorFlow has a sophisticated threading model, and will
-try to run multiple operations in parallel if it can. In our terminology this is
-called “inter-op parallelism” (though to avoid confusion with “intra-op”, you
-could think of it as “between-op” instead), and can be set by specifying
-`inter_op_parallelism_threads` in the session options.
-
-By default, mobile devices run operations serially; that is,
-`inter_op_parallelism_threads` is set to 1. Mobile processors usually have few
-cores and a small cache, so running multiple operations accessing disjoint parts
-of memory usually doesn’t help performance. “Intra-op parallelism” (or
-“within-op”) can be very helpful though, especially for computation-bound
-operations like convolutions where different threads can feed off the same small
-set of memory.
-
-On mobile, how many threads an op will use is set to the number of cores by
-default, or 2 when the number of cores can't be determined. You can override the
-default number of threads that ops are using by setting
-`intra_op_parallelism_threads` in the session options.  It’s a good idea to
-reduce the default if your app has its own threads doing heavy processing, so
-that they don’t interfere with each other.
-
-To see more details on session options, look at [ConfigProto](https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto).
-
-## Retrain with mobile data
-
-The biggest cause of accuracy problems when running models on mobile apps is
-unrepresentative training data. For example, most of the Imagenet photos are
-well-framed so that the object is in the center of the picture, well-lit, and
-shot with a normal lens. Photos from mobile devices are often poorly framed,
-badly lit, and can have fisheye distortions, especially selfies.
-
-The solution is to expand your training set with data actually captured from
-your application. This step can involve extra work, since you’ll have to label
-the examples yourself, but even if you just use it to expand your original
-training data, it can help the training set dramatically. Improving the training
-set by doing this, and by fixing other quality issues like duplicates or badly
-labeled examples is the single best way to improve accuracy. It’s usually a
-bigger help than altering your model architecture or using different techniques.
-
-## Reducing model loading time and/or memory footprint
-
-Most operating systems allow you to load a file using memory mapping, rather
-than going through the usual I/O APIs. Instead of allocating an area of memory
-on the heap and then copying bytes from disk into it, you simply tell the
-operating system to make the entire contents of a file appear directly in
-memory. This has several advantages:
-
-* Speeds loading
-* Reduces paging (increases performance)
-* Does not count towards RAM budget for your app
-
-TensorFlow has support for memory mapping the weights that form the bulk of most
-model files. Because of limitations in the `ProtoBuf` serialization format, we
-have to make a few changes to our model loading and processing code. The
-way memory mapping works is that we have a single file where the first part is a
-normal `GraphDef` serialized into the protocol buffer wire format, but then the
-weights are appended in a form that can be directly mapped.
-
-To create this file, run the
-`tensorflow/contrib/util:convert_graphdef_memmapped_format` tool. This takes in
-a `GraphDef` file that’s been run through `freeze_graph` and converts it to the
-format that has the weights appended at the end. Since that file’s no longer a
-standard `GraphDef` protobuf, you then need to make some changes to the loading
-code. You can see an example of this in
-the
-[iOS Camera demo app](https://www.tensorflow.org/code/tensorflow/examples/ios/camera/tensorflow_utils.mm?l=147),
-in the `LoadMemoryMappedModel()` function.
-
-The same code (with the Objective C calls for getting the filenames substituted)
-can be used on other platforms too. Because we’re using memory mapping, we need
-to start by creating a special TensorFlow environment object that’s set up with
-the file we’ll be using:
-
-    std::unique_ptr memmapped_env;
-    memmapped_env->reset(
-          new tensorflow::MemmappedEnv(tensorflow::Env::Default()));
-    tensorflow::Status mmap_status =
-          (memmapped_env->get())->InitializeFromFile(file_path);
-
-You then need to pass in this environment to subsequent calls, like this one for
-loading the graph:
-
-    tensorflow::GraphDef tensorflow_graph;
-    tensorflow::Status load_graph_status = ReadBinaryProto(
-        memmapped_env->get(),
-        tensorflow::MemmappedFileSystem::kMemmappedPackageDefaultGraphDef,
-        &tensorflow_graph);
-
-You also need to create the session with a pointer to the environment you’ve
-created:
-
-    tensorflow::SessionOptions options;
-    options.config.mutable_graph_options()
-        ->mutable_optimizer_options()
-        ->set_opt_level(::tensorflow::OptimizerOptions::L0);
-    options.env = memmapped_env->get();
-
-    tensorflow::Session* session_pointer = nullptr;
-    tensorflow::Status session_status =
-        tensorflow::NewSession(options, &session_pointer);
-
-One thing to notice here is that we’re also disabling automatic optimizations,
-since in some cases these will fold constant sub-trees, and so create copies of
-tensor values that we don’t want and use up more RAM.
-
-Once you’ve gone through these steps, you can use the session and graph as
-normal, and you should see a reduction in loading time and memory usage.
-
-## Protecting model files from easy copying
-
-By default, your models will be stored in the standard serialized protobuf
-format on disk. In theory this means that anybody can copy your model, which you
-may not want. However, in practice, most models are so application-specific and
-obfuscated by optimizations that the risk is similar to that of competitors
-disassembling and reusing your code, but if you do want to make it tougher for
-casual users to access your files it is possible to take some basic steps.
-
-Most of our examples use
-the
-[ReadBinaryProto()](https://www.tensorflow.org/code/tensorflow/core/platform/env.cc?q=core/platform/env.cc&l=409) convenience
-call to load a `GraphDef` from disk. This does require an unencrypted protobuf on
-disk. Luckily though, the implementation of the call is pretty straightforward
-and it should be easy to write an equivalent that can decrypt in memory. Here's
-some code that shows how you can read and decrypt a protobuf using your own
-decryption routine:
-
-    Status ReadEncryptedProto(Env* env, const string& fname,
-                              ::tensorflow::protobuf::MessageLite* proto) {
-      string data;
-      TF_RETURN_IF_ERROR(ReadFileToString(env, fname, &data));
-
-      DecryptData(&data);  // Your own function here.
-
-      if (!proto->ParseFromString(&data)) {
-        TF_RETURN_IF_ERROR(stream->status());
-        return errors::DataLoss("Can't parse ", fname, " as binary proto");
-      }
-      return Status::OK();
-    }
-
-To use this you’d need to define the DecryptData() function yourself. It could
-be as simple as something like:
-
-    void DecryptData(string* data) {
-      for (int i = 0; i < data.size(); ++i) {
-        data[i] = data[i] ^ 0x23;
-      }
-    }
-
-You may want something more complex, but exactly what you’ll need is outside the
-current scope here.
diff --git a/tensorflow/docs_src/mobile/prepare_models.md b/tensorflow/docs_src/mobile/prepare_models.md
deleted file mode 100644
index 2b84dbb973..0000000000
--- a/tensorflow/docs_src/mobile/prepare_models.md
+++ /dev/null
@@ -1,301 +0,0 @@
-# Preparing models for mobile deployment
-
-The requirements for storing model information during training are very
-different from when you want to release it as part of a mobile app. This section
-covers the tools involved in converting from a training model to something
-releasable in production.
-
-## What is up with all the different saved file formats?
-
-You may find yourself getting very confused by all the different ways that
-TensorFlow can save out graphs. To help, here’s a rundown of some of the
-different components, and what they are used for. The objects are mostly defined
-and serialized as protocol buffers:
-
-- [NodeDef](https://www.tensorflow.org/code/tensorflow/core/framework/node_def.proto):
-  Defines a single operation in a model. It has a unique name, a list of the
-  names of other nodes it pulls inputs from, the operation type it implements
-  (for example `Add`, or `Mul`), and any attributes that are needed to control
-  that operation. This is the basic unit of computation for TensorFlow, and all
-  work is done by iterating through a network of these nodes, applying each one
-  in turn. One particular operation type that’s worth knowing about is `Const`,
-  since this holds information about a constant. This may be a single, scalar
-  number or string, but it can also hold an entire multi-dimensional tensor
-  array. The values for a `Const` are stored inside the `NodeDef`, and so large
-  constants can take up a lot of room when serialized.
-
-- [Checkpoint](https://www.tensorflow.org/code/tensorflow/core/util/tensor_bundle/tensor_bundle.h). Another
-  way of storing values for a model is by using `Variable` ops. Unlike `Const`
-  ops, these don’t store their content as part of the `NodeDef`, so they take up
-  very little space within the `GraphDef` file. Instead their values are held in
-  RAM while a computation is running, and then saved out to disk as checkpoint
-  files periodically. This typically happens as a neural network is being
-  trained and weights are updated, so it’s a time-critical operation, and it may
-  happen in a distributed fashion across many workers, so the file format has to
-  be both fast and flexible. They are stored as multiple checkpoint files,
-  together with metadata files that describe what’s contained within the
-  checkpoints. When you’re referring to a checkpoint in the API (for example
-  when passing a filename in as a command line argument), you’ll use the common
-  prefix for a set of related files. If you had these files:
-
-        /tmp/model/model-chkpt-1000.data-00000-of-00002
-        /tmp/model/model-chkpt-1000.data-00001-of-00002
-        /tmp/model/model-chkpt-1000.index
-        /tmp/model/model-chkpt-1000.meta
-
-    You would refer to them as `/tmp/model/chkpt-1000`.
-
-- [GraphDef](https://www.tensorflow.org/code/tensorflow/core/framework/graph.proto):
-  Has a list of `NodeDefs`, which together define the computational graph to
-  execute. During training, some of these nodes will be `Variables`, and so if
-  you want to have a complete graph you can run, including the weights, you’ll
-  need to call a restore operation to pull those values from
-  checkpoints. Because checkpoint loading has to be flexible to deal with all of
-  the training requirements, this can be tricky to implement on mobile and
-  embedded devices, especially those with no proper file system available like
-  iOS. This is where
-  the
-  [`freeze_graph.py`](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py) script
-  comes in handy. As mentioned above, `Const` ops store their values as part of
-  the `NodeDef`, so if all the `Variable` weights are converted to `Const` nodes,
-  then we only need a single `GraphDef` file to hold the model architecture and
-  the weights. Freezing the graph handles the process of loading the
-  checkpoints, and then converts all Variables to Consts. You can then load the
-  resulting file in a single call, without having to restore variable values
-  from checkpoints. One thing to watch out for with `GraphDef` files is that
-  sometimes they’re stored in text format for easy inspection. These versions
-  usually have a ‘.pbtxt’ filename suffix, whereas the binary files end with
-  ‘.pb’.
-
-- [FunctionDefLibrary](https://www.tensorflow.org/code/tensorflow/core/framework/function.proto):
-  This appears in `GraphDef`, and is effectively a set of sub-graphs, each with
-  information about their input and output nodes. Each sub-graph can then be
-  used as an op in the main graph, allowing easy instantiation of different
-  nodes, in a similar way to how functions encapsulate code in other languages.
-
-- [MetaGraphDef](https://www.tensorflow.org/code/tensorflow/core/protobuf/meta_graph.proto):
-  A plain `GraphDef` only has information about the network of computations, but
-  doesn’t have any extra information about the model or how it can be
-  used. `MetaGraphDef` contains a `GraphDef` defining the computation part of
-  the model, but also includes information like ‘signatures’, which are
-  suggestions about which inputs and outputs you may want to call the model
-  with, data on how and where any checkpoint files are saved, and convenience
-  tags for grouping ops together for ease of use.
-
-- [SavedModel](https://www.tensorflow.org/code/tensorflow/core/protobuf/saved_model.proto):
-  It’s common to want to have different versions of a graph that rely on a
-  common set of variable checkpoints. For example, you might need a GPU and a
-  CPU version of the same graph, but keep the same weights for both. You might
-  also need some extra files (like label names) as part of your
-  model. The
-  [SavedModel](https://www.tensorflow.org/code/tensorflow/python/saved_model/README.md) format
-  addresses these needs by letting you save multiple versions of the same graph
-  without duplicating variables, and also storing asset files in the same
-  bundle. Under the hood, it uses `MetaGraphDef` and checkpoint files, along
-  with extra metadata files. It’s the format that you’ll want to use if you’re
-  deploying a web API using TensorFlow Serving, for example.
-
-## How do you get a model you can use on mobile?
-
-In most situations, training a model with TensorFlow will give you a folder
-containing a `GraphDef` file (usually ending with the `.pb` or `.pbtxt` extension) and
-a set of checkpoint files. What you need for mobile or embedded deployment is a
-single `GraphDef` file that’s been ‘frozen’, or had its variables converted into
-inline constants so everything’s in one file.  To handle the conversion, you’ll
-need the `freeze_graph.py` script, that’s held in
-[`tensorflow/python/tools/freeze_graph.py`](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py). You’ll run it like this:
-
-    bazel build tensorflow/python/tools:freeze_graph
-    bazel-bin/tensorflow/python/tools/freeze_graph \
-    --input_graph=/tmp/model/my_graph.pb \
-    --input_checkpoint=/tmp/model/model.ckpt-1000 \
-    --output_graph=/tmp/frozen_graph.pb \
-    --output_node_names=output_node \
-
-The `input_graph` argument should point to the `GraphDef` file that holds your
-model architecture. It’s possible that your `GraphDef` has been stored in a text
-format on disk, in which case it’s likely to end in `.pbtxt` instead of `.pb`,
-and you should add an extra `--input_binary=false` flag to the command.
-
-The `input_checkpoint` should be the most recent saved checkpoint. As mentioned
-in the checkpoint section, you need to give the common prefix to the set of
-checkpoints here, rather than a full filename.
-
-`output_graph` defines where the resulting frozen `GraphDef` will be
-saved. Because it’s likely to contain a lot of weight values that take up a
-large amount of space in text format, it’s always saved as a binary protobuf.
-
-`output_node_names` is a list of the names of the nodes that you want to extract
-the results of your graph from. This is needed because the freezing process
-needs to understand which parts of the graph are actually needed, and which are
-artifacts of the training process, like summarization ops. Only ops that
-contribute to calculating the given output nodes will be kept. If you know how
-your graph is going to be used, these should just be the names of the nodes you
-pass into `Session::Run()` as your fetch targets. The easiest way to find the
-node names is to inspect the Node objects while building your graph in python.
-Inspecting your graph in TensorBoard is another simple way.  You can get some
-suggestions on likely outputs by running the [`summarize_graph` tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/graph_transforms/README.md#inspecting-graphs).
-
-Because the output format for TensorFlow has changed over time, there are a
-variety of other less commonly used flags available too, like `input_saver`, but
-hopefully you shouldn’t need these on graphs trained with modern versions of the
-framework.
-
-## Using the Graph Transform Tool
-
-A lot of the things you need to do to efficiently run a model on device are
-available through the [Graph Transform
-Tool](https://www.tensorflow.org/code/tensorflow/tools/graph_transforms/README.md). This
-command-line tool takes an input `GraphDef` file, applies the set of rewriting
-rules you request, and then writes out the result as a `GraphDef`. See the
-documentation for more information on how to build and run this tool.
-
-### Removing training-only nodes
-
-TensorFlow `GraphDefs` produced by the training code contain all of the
-computation that’s needed for back-propagation and updates of weights, as well
-as the queuing and decoding of inputs, and the saving out of checkpoints. All of
-these nodes are no longer needed during inference, and some of the operations
-like checkpoint saving aren’t even supported on mobile platforms. To create a
-model file that you can load on devices you need to delete those unneeded
-operations by running the `strip_unused_nodes` rule in the Graph Transform Tool.
-
-The trickiest part of this process is figuring out the names of the nodes you
-want to use as inputs and outputs during inference.  You'll need these anyway
-once you start to run inference, but you also need them here so that the
-transform can calculate which nodes are not needed on the inference-only
-path. These may not be obvious from the training code. The easiest way to
-determine the node name is to explore the graph with TensorBoard.
-
-Remember that mobile applications typically gather their data from sensors and
-have it as arrays in memory, whereas training typically involves loading and
-decoding representations of the data stored on disk. In the case of Inception v3
-for example, there’s a `DecodeJpeg` op at the start of the graph that’s designed
-to take JPEG-encoded data from a file retrieved from disk and turn it into an
-arbitrary-sized image. After that there’s a `BilinearResize` op to scale it to
-the expected size, followed by a couple of other ops that convert the byte data
-into float and scale the value magnitudes it in the way the rest of the graph
-expects. A typical mobile app will skip most of these steps because it’s getting
-its input directly from a live camera, so the input node you will actually
-supply will be the output of the `Mul` node in this case.
-
-
-
-You’ll need to do a similar process of inspection to figure out the correct
-output nodes.
-
-If you’ve just been given a frozen `GraphDef` file, and are not sure about the
-contents, try using the `summarize_graph` tool to print out information
-about the inputs and outputs it finds from the graph structure. Here’s an
-example with the original Inception v3 file:
-
-    bazel run tensorflow/tools/graph_transforms:summarize_graph --
-    --in_graph=tensorflow_inception_graph.pb
-
-Once you have an idea of what the input and output nodes are, you can feed them
-into the graph transform tool as the `--input_names` and `--output_names`
-arguments, and call the `strip_unused_nodes` transform, like this:
-
-    bazel run tensorflow/tools/graph_transforms:transform_graph --
-    --in_graph=tensorflow_inception_graph.pb
-    --out_graph=optimized_inception_graph.pb --inputs='Mul' --outputs='softmax'
-    --transforms='
-      strip_unused_nodes(type=float, shape="1,299,299,3")
-      fold_constants(ignore_errors=true)
-      fold_batch_norms
-      fold_old_batch_norms'
-
-One thing to look out for here is that you need to specify the size and type
-that you want your inputs to be. This is because any values that you’re going to
-be passing in as inputs to inference need to be fed to special `Placeholder` op
-nodes, and the transform may need to create them if they don’t already exist. In
-the case of Inception v3 for example, a `Placeholder` node replaces the old
-`Mul` node that used to output the resized and rescaled image array, since we’re
-going to be doing that processing ourselves before we call TensorFlow. It keeps
-the original name though, which is why we always feed in inputs to `Mul` when we
-run a session with our modified Inception graph.
-
-After you’ve run this process, you’ll have a graph that only contains the actual
-nodes you need to run your prediction process. This is the point where it
-becomes useful to run metrics on the graph, so it’s worth running
-`summarize_graph` again to understand what’s in your model.
-
-## What ops should you include on mobile?
-
-There are hundreds of operations available in TensorFlow, and each one has
-multiple implementations for different data types. On mobile platforms, the size
-of the executable binary that’s produced after compilation is important, because
-app download bundles need to be as small as possible for the best user
-experience. If all of the ops and data types are compiled into the TensorFlow
-library then the total size of the compiled library can be tens of megabytes, so
-by default only a subset of ops and data types are included.
-
-That means that if you load a model file that’s been trained on a desktop
-machine, you may see the error “No OpKernel was registered to support Op” when
-you load it on mobile. The first thing to try is to make sure you’ve stripped
-out any training-only nodes, since the error will occur at load time even if the
-op is never executed. If you’re still hitting the same problem once that’s done,
-you’ll need to look at adding the op to your built library.
-
-The criteria for including ops and types fall into several categories:
-
-- Are they only useful in back-propagation, for gradients? Since mobile is
-  focused on inference, we don’t include these.
-
-- Are they useful mainly for other training needs, such as checkpoint saving?
-  These we leave out.
-
-- Do they rely on frameworks that aren’t always available on mobile, such as
-  libjpeg? To avoid extra dependencies we don’t include ops like `DecodeJpeg`.
-
-- Are there types that aren’t commonly used? We don’t include boolean variants
-  of ops for example, since we don’t see much use of them in typical inference
-  graphs.
-
-These ops are trimmed by default to optimize for inference on mobile, but it is
-possible to alter some build files to change the default.  After alternating the
-build files, you will need to recompile TensorFlow.  See below for more details
-on how to do this, and also see @{$mobile/optimizing#binary_size$Optimizing} for
-more on reducing your binary size.
-
-### Locate the implementation
-
-Operations are broken into two parts. The first is the op definition, which
-declares the signature of the operation, which inputs, outputs, and attributes
-it has. These take up very little space, and so all are included by default. The
-implementations of the op computations are done in kernels, which live in the
-`tensorflow/core/kernels` folder. You need to compile the C++ file containing
-the kernel implementation of the op you need into the library. To figure out
-which file that is, you can search for the operation name in the source
-files.
-
-[Here’s an example search in github](https://github.com/search?utf8=%E2%9C%93&q=repo%3Atensorflow%2Ftensorflow+extension%3Acc+path%3Atensorflow%2Fcore%2Fkernels+REGISTER+Mul&type=Code&ref=searchresults).
-
-You’ll see that this search is looking for the `Mul` op implementation, and it
-finds it in `tensorflow/core/kernels/cwise_op_mul_1.cc`. You need to look for
-macros beginning with `REGISTER`, with the op name you care about as one of the
-string arguments.
-
-In this case, the implementations are actually broken up across multiple `.cc`
-files, so you’d need to include all of them in your build. If you’re more
-comfortable using the command line for code search, here’s a grep command that
-also locates the right files if you run it from the root of your TensorFlow
-repository:
-
-`grep 'REGISTER.*"Mul"' tensorflow/core/kernels/*.cc`
-
-### Add the implementation to the build
-
-If you’re using Bazel, and building for Android, you’ll want to add the files
-you’ve found to
-the
-[`android_extended_ops_group1`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3565) or
-[`android_extended_ops_group2`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3632) targets. You
-may also need to include any .cc files they depend on in there. If the build
-complains about missing header files, add the .h’s that are needed into
-the
-[`android_extended_ops`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3525) target.
-
-If you’re using a makefile targeting iOS, Raspberry Pi, etc, go to
-[`tensorflow/contrib/makefile/tf_op_files.txt`](https://www.tensorflow.org/code/tensorflow/contrib/makefile/tf_op_files.txt) and
-add the right implementation files there.
diff --git a/tensorflow/docs_src/mobile/tflite/demo_android.md b/tensorflow/docs_src/mobile/tflite/demo_android.md
deleted file mode 100644
index fdf0bcf3c1..0000000000
--- a/tensorflow/docs_src/mobile/tflite/demo_android.md
+++ /dev/null
@@ -1,146 +0,0 @@
-# Android Demo App
-
-An example Android application using TensorFLow Lite is available
-[on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo).
-The demo is a sample camera app that classifies images continuously
-using either a quantized Mobilenet model or a floating point Inception-v3 model.
-To run the demo, a device running Android 5.0 ( API 21) or higher is required.
-
-In the demo app, inference is done using the TensorFlow Lite Java API. The demo
-app classifies frames in real-time, displaying the top most probable
-classifications. It also displays the time taken to detect the object.
-
-There are three ways to get the demo app to your device:
-
-* Download the [prebuilt binary APK](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
-* Use Android Studio to build the application.
-* Download the source code for TensorFlow Lite and the demo and build it using
-  bazel.
-
-
-## Download the pre-built binary
-
-The easiest way to try the demo is to download the
-[pre-built binary APK](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk)
-
-Once the APK is installed, click the app icon to start the program. The first
-time the app is opened, it asks for runtime permissions to access the device
-camera. The demo app opens the back-camera of the device and recognizes objects
-in the camera's field of view. At the bottom of the image (or at the left
-of the image if the device is in landscape mode), it displays top three objects
-classified and the classification latency.
-
-
-## Build in Android Studio with TensorFlow Lite AAR from JCenter
-
-Use Android Studio to try out changes in the project code and compile the demo
-app:
-
-* Install the latest version of
-  [Android Studio](https://developer.android.com/studio/index.html).
-* Make sure the Android SDK version is greater than 26 and NDK version is greater
-  than 14 (in the Android Studio settings).
-* Import the `tensorflow/contrib/lite/java/demo` directory as a new
-  Android Studio project.
-* Install all the Gradle extensions it requests.
-
-Now you can build and run the demo app. 
-
-The build process downloads the quantized [Mobilenet TensorFlow Lite model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip), and unzips it into the assets directory: `tensorflow/contrib/lite/java/demo/app/src/main/assets/`.
-
-Some additional details are available on the
-[TF Lite Android App page](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md).
-
-### Using other models
-
-To use a different model:
-* Download the floating point [Inception-v3 model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip).
-* Unzip and copy `inceptionv3_non_slim_2015.tflite` to the assets directory. 
-* Change the chosen classifier in [Camera2BasicFragment.java](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java)
    
-  from: `classifier = new ImageClassifierQuantizedMobileNet(getActivity());`
    
-  to: `classifier = new ImageClassifierFloatInception(getActivity());`.
-
-
-## Build TensorFlow Lite and the demo app from source
-
-### Clone the TensorFlow repo
-
-```sh
-git clone https://github.com/tensorflow/tensorflow
-```
-
-### Install Bazel
-
-If `bazel` is not installed on your system, see
-[Installing Bazel](https://bazel.build/versions/master/docs/install.html).
-
-Note: Bazel does not currently support Android builds on Windows. Windows users
-should download the
-[prebuilt binary](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
-
-### Install Android NDK and SDK
-
-The Android NDK is required to build the native (C/C++) TensorFlow Lite code. The
-current recommended version is *14b* and can be found on the
-[NDK Archives](https://developer.android.com/ndk/downloads/older_releases.html#ndk-14b-downloads)
-page.
-
-The Android SDK and build tools can be
-[downloaded separately](https://developer.android.com/tools/revisions/build-tools.html)
-or used as part of
-[Android Studio](https://developer.android.com/studio/index.html). To build the
-TensorFlow Lite Android demo, build tools require API >= 23 (but it will run on
-devices with API >= 21).
-
-In the root of the TensorFlow repository, update the `WORKSPACE` file with the
-`api_level` and location of the SDK and NDK. If you installed it with
-Android Studio, the SDK path can be found in the SDK manager. The default NDK
-path is:`{SDK path}/ndk-bundle.` For example:
-
-```
-android_sdk_repository (
-    name = "androidsdk",
-    api_level = 23,
-    build_tools_version = "23.0.2",
-    path = "/home/xxxx/android-sdk-linux/",
-)
-
-android_ndk_repository(
-    name = "androidndk",
-    path = "/home/xxxx/android-ndk-r10e/",
-    api_level = 19,
-)
-```
-
-Some additional details are available on the
-[TF Lite Android App page](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md).
-
-### Build the source code
-
-To build the demo app, run `bazel`:
-
-```
-bazel build --cxxopt=--std=c++11 //tensorflow/contrib/lite/java/demo/app/src/main:TfLiteCameraDemo
-```
-
-Caution: Because of an bazel bug, we only support building the Android demo app
-within a Python 2 environment.
-
-
-## About the demo
-
-The demo app is resizing each camera image frame (224 width * 224 height) to
-match the quantized MobileNets model (299 * 299 for Inception-v3). The resized
-image is converted—row by row—into a
-[ByteBuffer](https://developer.android.com/reference/java/nio/ByteBuffer.html).
-Its size is  1 * 224 * 224 * 3 bytes, where 1 is the number of images in a batch.
-224 * 224 (299 * 299) is the width and height of the image. 3 bytes represents
-the 3 colors of a pixel.
-
-This demo uses the TensorFlow Lite Java inference API
-for models which take a single input and provide a single output. This outputs a
-two-dimensional array, with the first dimension being the category index and the
-second dimension being the confidence of classification. Both models have 1001
-unique categories and the app sorts the probabilities of all the categories and
-displays the top three. The model file must be downloaded and bundled within the
-assets directory of the app.
diff --git a/tensorflow/docs_src/mobile/tflite/demo_ios.md b/tensorflow/docs_src/mobile/tflite/demo_ios.md
deleted file mode 100644
index 3be21da89f..0000000000
--- a/tensorflow/docs_src/mobile/tflite/demo_ios.md
+++ /dev/null
@@ -1,68 +0,0 @@
-# iOS Demo App
-
-The TensorFlow Lite demo is a camera app that continuously classifies whatever
-it sees from your device's back camera, using a quantized MobileNet model. These
-instructions walk you through building and running the demo on an iOS device.
-
-## Prerequisites
-
-* You must have [Xcode](https://developer.apple.com/xcode/) installed and have a
-  valid Apple Developer ID, and have an iOS device set up and linked to your
-  developer account with all of the appropriate certificates. For these
-  instructions, we assume that you have already been able to build and deploy an
-  app to an iOS device with your current developer environment.
-
-* The demo app requires a camera and must be executed on a real iOS device. You
-  can build it and run with the iPhone Simulator but it won't have any camera
-  information to classify.
-
-* You don't need to build the entire TensorFlow library to run the demo, but you
-  will need to clone the TensorFlow repository if you haven't already:
-
-        git clone https://github.com/tensorflow/tensorflow
-
-* You'll also need the Xcode command-line tools:
-
-        xcode-select --install
-
-    If this is a new install, you will need to run the Xcode application once to
-    agree to the license before continuing.
-
-## Building the iOS Demo App
-
-1. Install CocoaPods if you don't have it:
-
-        sudo gem install cocoapods
-
-2. Download the model files used by the demo app (this is done from inside the
-   cloned directory):
-
-        sh tensorflow/contrib/lite/examples/ios/download_models.sh
-
-3. Install the pod to generate the workspace file:
-
-        cd tensorflow/contrib/lite/examples/ios/camera
-        pod install
-
-    If you have installed this pod before and that command doesn't work, try
-
-        pod update
-
-    At the end of this step you should have a file called 
-    `tflite_camera_example.xcworkspace`.
-
-4. Open the project in Xcode by typing this on the command line:
-
-        open tflite_camera_example.xcworkspace
-
-    This launches Xcode if it isn't open already and opens the
-    `tflite_camera_example` project.
-
-5. Build and run the app in Xcode.
-
-    Note that as mentioned earlier, you must already have a device set up and
-    linked to your Apple Developer account in order to deploy the app on a
-    device.
-
-You'll have to grant permissions for the app to use the device's camera. Point
-the camera at various objects and enjoy seeing how the model classifies things!
diff --git a/tensorflow/docs_src/mobile/tflite/devguide.md b/tensorflow/docs_src/mobile/tflite/devguide.md
deleted file mode 100644
index b168d6c183..0000000000
--- a/tensorflow/docs_src/mobile/tflite/devguide.md
+++ /dev/null
@@ -1,232 +0,0 @@
-# Developer Guide
-
-Using a TensorFlow Lite model in your mobile app requires multiple
-considerations: you must choose a pre-trained or custom model, convert the model
-to a TensorFLow Lite format, and finally, integrate the model in your app.
-
-## 1. Choose a model
-
-Depending on the use case, you can choose one of the popular open-sourced models,
-such as *InceptionV3* or *MobileNets*, and re-train these models with a custom
-data set or even build your own custom model.
-
-### Use a pre-trained model
-
-[MobileNets](https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html)
-is a family of mobile-first computer vision models for TensorFlow designed to
-effectively maximize accuracy, while taking into consideration the restricted
-resources for on-device or embedded applications. MobileNets are small,
-low-latency, low-power models parameterized to meet the resource constraints for
-a variety of uses. They can be used for classification, detection, embeddings, and
-segmentation—similar to other popular large scale models, such as
-[Inception](https://arxiv.org/pdf/1602.07261.pdf). Google provides 16 pre-trained
-[ImageNet](http://www.image-net.org/challenges/LSVRC/) classification checkpoints
-for MobileNets that can be used in mobile projects of all sizes.
-
-[Inception-v3](https://arxiv.org/abs/1512.00567) is an image recognition model
-that achieves fairly high accuracy recognizing general objects with 1000 classes,
-for example, "Zebra", "Dalmatian", and "Dishwasher". The model extracts general
-features from input images using a convolutional neural network and classifies
-them based on those features with fully-connected and softmax layers.
-
-[On Device Smart Reply](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html)
-is an on-device model that provides one-touch replies for incoming text messages
-by suggesting contextually relevant messages. The model is built specifically for
-memory constrained devices, such as watches and phones, and has been successfully
-used in Smart Replies on Android Wear. Currently, this model is Android-specific.
-
-These pre-trained models are [available for download](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/models.md)
-
-### Re-train Inception-V3 or MobileNet for a custom data set
-
-These pre-trained models were trained on the *ImageNet* data set which contains
-1000 predefined classes. If these classes are not sufficient for your use case,
-the model will need to be re-trained. This technique is called
-*transfer learning* and starts with a model that has been already trained on a
-problem, then retrains the model on a similar problem. Deep learning from
-scratch can take days, but transfer learning is fairly quick. In order to do
-this, you need to generate a custom data set labeled with the relevant classes.
-
-The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/)
-codelab walks through the re-training process step-by-step. The code supports
-both floating point and quantized inference.
-
-### Train a custom model
-
-A developer may choose to train a custom model using Tensorflow (see the
-[TensorFlow tutorials](../../tutorials/) for examples of building and training
-models). If you have already written a model, the first step is to export this
-to a @{tf.GraphDef} file. This is required because some formats do not store the
-model structure outside the code, and we must communicate with other parts of the
-framework. See
-[Exporting the Inference Graph](https://github.com/tensorflow/models/blob/master/research/slim/README.md)
-to create .pb file for the custom model.
-
-TensorFlow Lite currently supports a subset of TensorFlow operators. Refer to the
-[TensorFlow Lite & TensorFlow Compatibility Guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md)
-for supported operators and their usage. This set of operators will continue to
-grow in future Tensorflow Lite releases.
-
-
-## 2. Convert the model format
-
-The model generated (or downloaded) in the previous step is a *standard*
-Tensorflow model and you should now have a .pb or .pbtxt @{tf.GraphDef} file.
-Models generated with transfer learning (re-training) or custom models must be
-converted—but, we must first freeze the graph to convert the model to the
-Tensorflow Lite format. This process uses several model formats:
-
-* @{tf.GraphDef} (.pb) —A protobuf that represents the TensorFlow training or
-  computation graph. It contains operators, tensors, and variables definitions.
-* *CheckPoint* (.ckpt) —Serialized variables from a TensorFlow graph. Since this
-  does not contain a graph structure, it cannot be interpreted by itself.
-* `FrozenGraphDef` —A subclass of `GraphDef` that does not contain
-  variables. A `GraphDef` can be converted to a `FrozenGraphDef` by taking a
-  CheckPoint and a `GraphDef`, and converting each variable into a constant
-  using the value retrieved from the CheckPoint.
-* `SavedModel` —A `GraphDef` and CheckPoint with a signature that labels
-  input and output arguments to a model. A `GraphDef` and CheckPoint can be
-  extracted from a `SavedModel`.
-* *TensorFlow Lite model* (.tflite) —A serialized
-  [FlatBuffer](https://google.github.io/flatbuffers/) that contains TensorFlow
-  Lite operators and tensors for the TensorFlow Lite interpreter, similar to a
-  `FrozenGraphDef`.
-
-### Freeze Graph
-
-To use the `GraphDef` .pb file with TensorFlow Lite, you must have checkpoints
-that contain trained weight parameters. The .pb file only contains the structure
-of the graph. The process of merging the checkpoint values with the graph
-structure is called *freezing the graph*.
-
-You should have a checkpoints folder or download them for a pre-trained model
-(for example,
-[MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md)).
-
-To freeze the graph, use the following command (changing the arguments):
-
-```
-freeze_graph --input_graph=/tmp/mobilenet_v1_224.pb \
-  --input_checkpoint=/tmp/checkpoints/mobilenet-10202.ckpt \
-  --input_binary=true \
-  --output_graph=/tmp/frozen_mobilenet_v1_224.pb \
-  --output_node_names=MobileNetV1/Predictions/Reshape_1
-```
-
-The `input_binary` flag must be enabled so the protobuf is read and written in
-a binary format. Set the `input_graph` and `input_checkpoint` files.
-
-The `output_node_names` may not be obvious outside of the code that built the
-model. The easiest way to find them is to visualize the graph, either with
-[TensorBoard](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/#3)
-or `graphviz`.
-
-The frozen `GraphDef` is now ready for conversion to the `FlatBuffer` format
-(.tflite) for use on Android or iOS devices. For Android, the Tensorflow
-Optimizing Converter tool supports both float and quantized models. To convert
-the frozen `GraphDef` to the .tflite format:
-
-```
-toco --input_file=$(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \
-  --input_format=TENSORFLOW_GRAPHDEF \
-  --output_format=TFLITE \
-  --output_file=/tmp/mobilenet_v1_1.0_224.tflite \
-  --inference_type=FLOAT \
-  --input_type=FLOAT \
-  --input_arrays=input \
-  --output_arrays=MobilenetV1/Predictions/Reshape_1 \
-  --input_shapes=1,224,224,3
-```
-
-The `input_file` argument should reference the frozen `GraphDef` file
-containing the model architecture. The [frozen_graph.pb](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz)
-file used here is available for download. `output_file` is where the TensorFlow
-Lite model will get generated. The `input_type` and `inference_type`
-arguments should be set to `FLOAT`, unless converting a
-@{$performance/quantization$quantized model}. Setting the `input_array`,
-`output_array`, and `input_shape` arguments are not as straightforward. The
-easiest way to find these values is to explore the graph using Tensorboard. Reuse
-the arguments for specifying the output nodes for inference in the
-`freeze_graph` step.
-
-It is also possible to use the Tensorflow Optimizing Converter with protobufs
-from either Python or from the command line (see the 
-[toco_from_protos.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/python/toco_from_protos.py)
-example). This allows you to integrate the conversion step into the model design
-workflow, ensuring the model is easily convertible to a mobile inference graph.
-For example:
-
-```python
-import tensorflow as tf
-
-img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3))
-val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.])
-out = tf.identity(val, name="out")
-
-with tf.Session() as sess:
-  tflite_model = tf.contrib.lite.toco_convert(sess.graph_def, [img], [out])
-  open("converteds_model.tflite", "wb").write(tflite_model)
-```
-
-For usage, see the Tensorflow Optimizing Converter
-[command-line examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md).
-
-Refer to the
-[Ops compatibility guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md)
-for troubleshooting help, and if that doesn't help, please
-[file an issue](https://github.com/tensorflow/tensorflow/issues).
-
-The [development repo](https://github.com/tensorflow/tensorflow) contains a tool
-to visualize TensorFlow Lite models after conversion. To build the
-[visualize.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/tools/visualize.py)
-tool:
-
-```sh
-bazel run tensorflow/contrib/lite/tools:visualize -- model.tflite model_viz.html
-```
-
-This generates an interactive HTML page listing subgraphs, operations, and a
-graph visualization.
-
-
-## 3. Use the TensorFlow Lite model for inference in a mobile app
-
-After completing the prior steps, you should now have a `.tflite` model file.
-
-### Android
-
-Since Android apps are written in Java and the core TensorFlow library is in C++,
-a JNI library is provided as an interface. This is only meant for inference—it
-provides the ability to load a graph, set up inputs, and run the model to
-calculate outputs.
-
-The open source Android demo app uses the JNI interface and is available
-[on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app).
-You can also download a
-[prebuilt APK](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
-See the @{$tflite/demo_android} guide for details.
-
-The @{$mobile/android_build} guide has instructions for installing TensorFlow on
-Android and setting up `bazel` and Android Studio.
-
-### iOS
-
-To integrate a TensorFlow model in an iOS app, see the
-[TensorFlow Lite for iOS](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/ios.md)
-guide and @{$tflite/demo_ios} guide.
-
-#### Core ML support
-
-Core ML is a machine learning framework used in Apple products. In addition to
-using Tensorflow Lite models directly in your applications, you can convert
-trained Tensorflow models to the
-[CoreML](https://developer.apple.com/machine-learning/) format for use on Apple
-devices. To use the converter, refer to the
-[Tensorflow-CoreML converter documentation](https://github.com/tf-coreml/tf-coreml).
-
-### Raspberry Pi
-
-Compile Tensorflow Lite for a Raspberry Pi by following the
-[RPi build instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/rpi.md)
-This compiles a static library file (`.a`) used to build your app. There are
-plans for Python bindings and a demo app.
diff --git a/tensorflow/docs_src/mobile/tflite/index.md b/tensorflow/docs_src/mobile/tflite/index.md
deleted file mode 100644
index cc4af2a875..0000000000
--- a/tensorflow/docs_src/mobile/tflite/index.md
+++ /dev/null
@@ -1,201 +0,0 @@
-# Introduction to TensorFlow Lite
-
-TensorFlow Lite is TensorFlow’s lightweight solution for mobile and embedded
-devices. It enables on-device machine learning inference with low latency and a
-small binary size. TensorFlow Lite also supports hardware acceleration with the
-[Android Neural Networks
-API](https://developer.android.com/ndk/guides/neuralnetworks/index.html).
-
-TensorFlow Lite uses many techniques for achieving low latency such as
-optimizing the kernels for mobile apps, pre-fused activations, and quantized
-kernels that allow smaller and faster (fixed-point math) models.
-
-Most of our TensorFlow Lite documentation is [on
-GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite)
-for the time being.
-
-## What does TensorFlow Lite contain?
-
-TensorFlow Lite supports a set of core operators, both quantized and
-float, which have been tuned for mobile platforms. They incorporate pre-fused
-activations and biases to further enhance performance and quantized
-accuracy. Additionally, TensorFlow Lite also supports using custom operations in
-models.
-
-TensorFlow Lite defines a new model file format, based on
-[FlatBuffers](https://google.github.io/flatbuffers/). FlatBuffers is an
-open-sourced, efficient cross platform serialization library. It is similar to
-[protocol buffers](https://developers.google.com/protocol-buffers/?hl=en), but
-the primary difference is that FlatBuffers does not need a parsing/unpacking
-step to a secondary representation before you can access data, often coupled
-with per-object memory allocation. Also, the code footprint of FlatBuffers is an
-order of magnitude smaller than protocol buffers.
-
-TensorFlow Lite has a new mobile-optimized interpreter, which has the key goals
-of keeping apps lean and fast. The interpreter uses a static graph ordering and
-a custom (less-dynamic) memory allocator to ensure minimal load, initialization,
-and execution latency.
-
-TensorFlow Lite provides an interface to leverage hardware acceleration, if
-available on the device. It does so via the
-[Android Neural Networks API](https://developer.android.com/ndk/guides/neuralnetworks/index.html),
-available on Android 8.1 (API level 27) and higher.
-
-## Why do we need a new mobile-specific library?
-
-Machine Learning is changing the computing paradigm, and we see an emerging
-trend of new use cases on mobile and embedded devices. Consumer expectations are
-also trending toward natural, human-like interactions with their devices, driven
-by the camera and voice interaction models.
-
-There are several factors which are fueling interest in this domain:
-
-- Innovation at the silicon layer is enabling new possibilities for hardware
-  acceleration, and frameworks such as the Android Neural Networks API make it
-  easy to leverage these.
-
-- Recent advances in real-time computer-vision and spoken language understanding
-  have led to mobile-optimized benchmark models being open sourced
-  (e.g. MobileNets, SqueezeNet).
-
-- Widely-available smart appliances create new possibilities for
-  on-device intelligence.
-
-- Interest in stronger user data privacy paradigms where user data does not need
-  to leave the mobile device.
-
-- Ability to serve ‘offline’ use cases, where the device does not need to be
-  connected to a network.
-
-We believe the next wave of machine learning applications will have significant
-processing on mobile and embedded devices.
-
-## TensorFlow Lite highlights
-
-TensorFlow Lite provides:
-
-- A set of core operators, both quantized and float, many of which have been
-  tuned for mobile platforms.  These can be used to create and run custom
-  models.  Developers can also write their own custom operators and use them in
-  models.
-
-- A new [FlatBuffers](https://google.github.io/flatbuffers/)-based
-  model file format.
-
-- On-device interpreter with kernels optimized for faster execution on mobile.
-
-- TensorFlow converter to convert TensorFlow-trained models to the TensorFlow
-  Lite format.
-
-- Smaller in size: TensorFlow Lite is smaller than 300KB when all supported
-  operators are linked and less than 200KB when using only the operators needed
-  for supporting InceptionV3 and Mobilenet.
-
-- **Pre-tested models:**
-
-    All of the following models are guaranteed to work out of the box:
-
-    - Inception V3, a popular model for detecting the dominant objects
-      present in an image.
-
-    - [MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md),
-      a family of mobile-first computer vision models designed to effectively
-      maximize accuracy while being mindful of the restricted resources for an
-      on-device or embedded application. They are small, low-latency, low-power
-      models parameterized to meet the resource constraints of a variety of use
-      cases. They can be built upon for classification, detection, embeddings
-      and segmentation. MobileNet models are smaller but [lower in
-      accuracy](https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html)
-      than Inception V3.
-
-    - On Device Smart Reply, an on-device model which provides one-touch
-      replies for an incoming text message by suggesting contextually relevant
-      messages. The model was built specifically for memory constrained devices
-      such as watches & phones and it has been successfully used to surface
-      [Smart Replies on Android
-      Wear](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html)
-      to all first-party and third-party apps.
-
-    Also see the complete list of
-    [TensorFlow Lite's supported models](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/models.md),
-    including the model sizes, performance numbers, and downloadable model files.
-
-- Quantized versions of the MobileNet model, which runs faster than the
-  non-quantized (float) version on CPU.
-
-- New Android demo app to illustrate the use of TensorFlow Lite with a quantized
-  MobileNet model for object classification.
-
-- Java and C++ API support
-
-
-## Getting Started
-
-We recommend you try out TensorFlow Lite with the pre-tested models indicated
-above. If you have an existing model, you will need to test whether your model
-is compatible with both the converter and the supported operator set.  To test
-your model, see the
-[documentation on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite).
-
-### Retrain Inception-V3 or MobileNet for a custom data set
-
-The pre-trained models mentioned above have been trained on the ImageNet data
-set, which consists of 1000 predefined classes. If those classes are not
-relevant or useful for your use case, you will need to retrain those
-models. This technique is called transfer learning, which starts with a model
-that has been already trained on a problem and will then be retrained on a
-similar problem. Deep learning from scratch can take days, but transfer learning
-can be done fairly quickly. In order to do this, you'll need to generate your
-custom data set labeled with the relevant classes.
-
-The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/)
-codelab walks through this process step-by-step. The retraining code supports
-retraining for both floating point and quantized inference.
-
-## TensorFlow Lite Architecture
-
-The following diagram shows the architectural design of TensorFlow Lite:
-
-TensorFlow Lite architecture diagram
-
-Starting with a trained TensorFlow model on disk, you'll convert that model to
-the TensorFlow Lite file format (`.tflite`) using the TensorFlow Lite
-Converter. Then you can use that converted file in your mobile application.
-
-Deploying the TensorFlow Lite model file uses:
-
-- Java API: A convenience wrapper around the C++ API on Android.
-
-- C++ API: Loads the TensorFlow Lite Model File and invokes the Interpreter. The
-  same library is available on both Android and iOS.
-
-- Interpreter: Executes the model using a set of kernels. The interpreter
-  supports selective kernel loading; without kernels it is only 100KB, and 300KB
-  with all the kernels loaded. This is a significant reduction from the 1.5M
-  required by TensorFlow Mobile.
-
-- On select Android devices, the Interpreter will use the Android Neural
-  Networks API for hardware acceleration, or default to CPU execution if none
-  are available.
-
-You can also implement custom kernels using the C++ API that can be used by the
-Interpreter.
-
-## Future Work
-
-In future releases, TensorFlow Lite will support more models and built-in
-operators, contain performance improvements for both fixed point and floating
-point models, improvements to the tools to enable easier developer workflows and
-support for other smaller devices and more. As we continue development, we hope
-that TensorFlow Lite will greatly simplify the developer experience of targeting
-a model for small devices.
-
-Future plans include using specialized machine learning hardware to get the best
-possible performance for a particular model on a particular device.
-
-## Next Steps
-
-The TensorFlow Lite [GitHub repository](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite).
-contains additional docs, code samples, and demo applications.
diff --git a/tensorflow/docs_src/mobile/tflite/performance.md b/tensorflow/docs_src/mobile/tflite/performance.md
deleted file mode 100644
index 79bacaaa1b..0000000000
--- a/tensorflow/docs_src/mobile/tflite/performance.md
+++ /dev/null
@@ -1,174 +0,0 @@
-# Performance
-
-This document lists TensorFlow Lite performance benchmarks when running well
-known models on some Android and iOS devices.
-
-These performance benchmark numbers were generated with the
-[Android TFLite benchmark binary](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark)
-and the [iOS benchmark app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark/ios).
-
-# Android performance benchmarks
-
-For Android benchmarks, the CPU affinity is set to use big cores on the device to
-reduce variance (see [details](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark#reducing-variance-between-runs-on-android)).
-
-It assumes that models were download and unzipped to the
-`/data/local/tmp/tflite_models` directory. The benchmark binary is built
-using [these instructions](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark#on-android)
-and assumed in the `/data/local/tmp` directory.
-
-To run the benchmark:
-
-```
-adb shell taskset ${CPU_MASK} /data/local/tmp/benchmark_model \
-  --num_threads=1 \
-  --graph=/data/local/tmp/tflite_models/${GRAPH} \
-  --warmup_runs=1 \
-  --num_runs=50 \
-  --use_nnapi=false
-```
-
-Here, `${GRAPH}` is the name of model and `${CPU_MASK}` is the CPU affinity
-chosen according to the following table:
-
-Device | CPU_MASK |
--------| ----------
-Pixel 2 | f0 |
-Pixel xl | 0c |
-
-
-
-  
-    
-      
-      
-      
-    
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-  
-    
-    
-    
-  
-   
-     
-     
-  
-
- 
    Model NameDevice Mean inference time (std dev)
    
-      Mobilenet_1.0_224(float)
-    Pixel 2 166.5 ms (2.6 ms)
    Pixel xl 122.9 ms (1.8 ms)  
    
-      Mobilenet_1.0_224 (quant)
-    Pixel 2 69.5 ms (0.9 ms)
    Pixel xl 78.9 ms (2.2 ms)  
    
-      NASNet mobile
-    Pixel 2 273.8 ms (3.5 ms)
    Pixel xl 210.8 ms (4.2 ms)
    
-      SqueezeNet
-    Pixel 2 234.0 ms (2.1 ms)
    Pixel xl 158.0 ms (2.1 ms)
    
-      Inception_ResNet_V2
-    Pixel 2 2846.0 ms (15.0 ms)
    Pixel xl 1973.0 ms (15.0 ms)  
    
-      Inception_V4
-    Pixel 2 3180.0 ms (11.7 ms)
    Pixel xl 2262.0 ms (21.0 ms)  
    
-
-# iOS benchmarks
-
-To run iOS benchmarks, the [benchmark
-app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tools/benchmark/ios)
-was modified to include the appropriate model and `benchmark_params.json` was
-modified  to set `num_threads` to 1.
-
-
-  
-    
-      
-      
-      
-    
-  
-  
-    
-    
-    
-  
-  
-    
-    
-    
-  
-  
-    
-    
-    
-  
-  
-    
-    
-    
-  
-  
-    
-    
-    
-  
-  
-    
-    
-    
-  
- 
    Model NameDevice Mean inference time (std dev)
    
-      Mobilenet_1.0_224(float)
-    iPhone 8 32.2 ms (0.8 ms)
    
-      Mobilenet_1.0_224 (quant)
-    iPhone 8 24.4 ms (0.8 ms)
    
-      NASNet mobile
-    iPhone 8 60.3 ms (0.6 ms)
    
-      SqueezeNet
-    iPhone 8 44.3 (0.7 ms)
    
-      Inception_ResNet_V2
-    iPhone 8562.4 ms (18.2 ms)
    
-      Inception_V4
-    iPhone 8 661.0 ms (29.2 ms)
    
-- 
cgit v1.2.3


From eb1ee5a245327b509dfb929d2dbbef90cd3299a5 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Tue, 24 Jul 2018 11:52:30 -0700
Subject: [TF:XLA] Bump open source llvm revision to r337814

PiperOrigin-RevId: 205866253
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 3b7674e397..062beb9348 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -485,11 +485,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "llvm",
       urls = [
-          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/62b518b75a780a3bc75982cbe54b0e7bc262aa6e.tar.gz",
-	  "https://github.com/llvm-mirror/llvm/archive/62b518b75a780a3bc75982cbe54b0e7bc262aa6e.tar.gz",
+          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/35ffbe6bcf3b755f30633d834534a892b4c5fb29.tar.gz",
+	  "https://github.com/llvm-mirror/llvm/archive/35ffbe6bcf3b755f30633d834534a892b4c5fb29.tar.gz",
       ],
-      sha256 = "51ab0edcf7dde0207f5cf141aec16b14fcac5290112cdf1ea671a2757f719f8b",
-      strip_prefix = "llvm-62b518b75a780a3bc75982cbe54b0e7bc262aa6e",
+      sha256 = "42b3924b56339bb953b587f3e55788c8fefa51068756e6ac2ee4aed9c187bbb8",
+      strip_prefix = "llvm-35ffbe6bcf3b755f30633d834534a892b4c5fb29",
       build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"),
   )
 
-- 
cgit v1.2.3


From a9489a0d05c89ab6b2cb94dda95e9ff911ad2058 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay 
Date: Tue, 24 Jul 2018 12:47:50 -0700
Subject: Add binary accuracy metric class. Remove the need for decorators in
 metric subclasses.

PiperOrigin-RevId: 205875214
---
 tensorflow/python/keras/metrics.py                 | 175 +++++++++++++++------
 tensorflow/python/keras/metrics_test.py            |  95 ++++++++++-
 .../api/golden/tensorflow.keras.metrics.pbtxt      |   2 +-
 3 files changed, 223 insertions(+), 49 deletions(-)

diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py
index 72e15763cb..7d8b1fec45 100644
--- a/tensorflow/python/keras/metrics.py
+++ b/tensorflow/python/keras/metrics.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 
 from abc import ABCMeta
 from abc import abstractmethod
+
+import types
 import six
 
 from tensorflow.python.eager import context
@@ -58,7 +60,14 @@ from tensorflow.python.util import tf_decorator
 from tensorflow.python.util.tf_export import tf_export
 
 
-def update_state(update_state_fn):
+def check_is_tensor_or_operation(x, name):
+  """Raises type error if the given input is not a tensor or operation."""
+  if not (isinstance(x, ops.Tensor) or isinstance(x, ops.Operation)):
+    raise TypeError('{0} must be a Tensor or Operation, given: {1}'.format(
+        name, x))
+
+
+def update_state_wrapper(update_state_fn):
   """Decorator to wrap metric `update_state()` with `defun()`, `add_update()`.
 
   Args:
@@ -70,7 +79,7 @@ def update_state(update_state_fn):
       executed to update the metric state with the given inputs.
   """
 
-  def decorated(*args, **kwargs):
+  def decorated(metric_obj, *args, **kwargs):
     """Decorated function with `defun()` and `add_update()`."""
 
     # Converting update_state_fn() into a graph function, so that
@@ -79,14 +88,15 @@ def update_state(update_state_fn):
     defuned_update_state_fn = function.defun(update_state_fn)
     update_op = defuned_update_state_fn(*args, **kwargs)
     if update_op is not None:  # update_op will be None in eager execution.
-      metric_obj = args[0]
       metric_obj.add_update(update_op, inputs=True)
+      check_is_tensor_or_operation(
+          update_op, 'Metric {0}\'s update'.format(metric_obj.name))
     return update_op
 
   return tf_decorator.make_decorator(update_state_fn, decorated)
 
 
-def result(result_fn):
+def result_wrapper(result_fn):
   """Decorator to wrap metric `result()` function in `merge_call()`.
 
   Result computation is an idempotent operation that simply calculates the
@@ -104,26 +114,29 @@ def result(result_fn):
     The metric result tensor.
   """
 
-  def decorated(*args):
+  def decorated(metric_obj, *args):
     """Decorated function with merge_call."""
     tower_context = distribute_lib.get_tower_context()
     if tower_context is None:  # if in cross tower context already
-      return result_fn()
-
-    # TODO(psv): Test distribution of metrics using different distribution
-    # strategies.
-
-    # Creating a wrapper for merge_fn. merge_call invokes the given merge_fn
-    # with distribution object as the first parameter. We create a wrapper here
-    # so that the result function need not have that parameter.
-    def merge_fn_wrapper(distribution, merge_fn, *args):
-      # We will get `PerDevice` merge function. Taking the first one as all are
-      # identical copies of the function that we had passed below.
-      return distribution.unwrap(merge_fn)[0](*args)
-
-    # Wrapping result in merge_call. merge_call is used when we want to leave
-    # tower mode and compute a value in cross tower mode.
-    return tower_context.merge_call(merge_fn_wrapper, result_fn, *args)
+      result_t = result_fn(*args)
+    else:
+      # TODO(psv): Test distribution of metrics using different distribution
+      # strategies.
+
+      # Creating a wrapper for merge_fn. merge_call invokes the given merge_fn
+      # with distribution object as the first parameter. We create a wrapper
+      # here so that the result function need not have that parameter.
+      def merge_fn_wrapper(distribution, merge_fn, *args):
+        # We will get `PerDevice` merge function. Taking the first one as all
+        # are identical copies of the function that we had passed below.
+        return distribution.unwrap(merge_fn)[0](*args)
+
+      # Wrapping result in merge_call. merge_call is used when we want to leave
+      # tower mode and compute a value in cross tower mode.
+      result_t = tower_context.merge_call(merge_fn_wrapper, result_fn, *args)
+    check_is_tensor_or_operation(result_t,
+                                 'Metric {0}\'s result'.format(metric_obj.name))
+    return result_t
 
   return tf_decorator.make_decorator(result_fn, decorated)
 
@@ -246,24 +259,19 @@ class Metric(Layer):
   * `__init__()`: All state variables should be created in this method by
     calling `self.add_weight()` like: `self.var = self.add_weight(...)`
   * `update_state()`: Has all updates to the state variables like:
-    self.var.assign_add(...). Please decorate the function with:
-    @update_state: Converts `update_state()` into a graph function, so that
-    we can return a single op that performs all of the variable updates and
-    adds the update op to the metric layer.
+    self.var.assign_add(...).
   * `result()`: Computes and returns a value for the metric
-    from the state variables. Please decorate the function with:
-    @result: Wraps `result()` in a distribution strategy merge_call().
+    from the state variables.
 
   Example subclass implementation:
 
   ```
   class BinaryTruePositives(Metric):
-    def __init__(self, name='binary-true-positives', dtype=dtypes.float64):
+    def __init__(self, name='binary-true-positives', dtype=None):
       super(BinaryTruePositives, self).__init__(name=name, dtype=dtype)
       self.true_positives = self.add_weight(
           'true_positives', initializer=init_ops.zeros_initializer)
 
-    @update_state
     def update_state(self, y_true, y_pred, sample_weight=None):
       y_true = math_ops.cast(y_true, dtypes.bool)
       y_pred = math_ops.cast(y_pred, dtypes.bool)
@@ -278,17 +286,24 @@ class Metric(Layer):
         values = math_ops.multiply(values, sample_weight)
       state_ops.assign_add(self.true_positives, math_ops.reduce_sum(values))
 
-    @result
     def result(self):
       return array_ops.identity(self.true_positives)
   ```
   """
   __metaclass__ = ABCMeta
 
-  def __init__(self, name=None, dtype=dtypes.float64):
+  def __init__(self, name=None, dtype=None):
     super(Metric, self).__init__(name=name, dtype=dtype)
     self.stateful = True  # All metric layers are stateful.
     self.built = True
+    self._dtype = K.floatx() if dtype is None else dtypes.as_dtype(dtype).name
+
+  def __new__(cls, *args, **kwargs):
+    obj = super(Metric, cls).__new__(cls, *args, **kwargs)
+    obj.update_state = types.MethodType(
+        update_state_wrapper(obj.update_state), obj)
+    obj.result = types.MethodType(result_wrapper(obj.result), obj)
+    return obj
 
   def __call__(self, *args, **kwargs):
     """Accumulates statistics and then computes metric result value.
@@ -301,9 +316,9 @@ class Metric(Layer):
     Returns:
       The metric value tensor.
     """
-    update_op = self.update_state(*args, **kwargs)
+    update_op = self.update_state(*args, **kwargs)  # pylint: disable=not-callable
     with ops.control_dependencies([update_op]):
-      return self.result()
+      return self.result()  # pylint: disable=not-callable
 
   def reset_states(self):
     """Resets all of the metric state variables.
@@ -318,10 +333,8 @@ class Metric(Layer):
   def update_state(self, *args, **kwargs):
     """Accumulates statistics for the metric.
 
-    Please decorate the function with:
-    @update_state: Converts `update_state()` into a graph function, so that
-    we can return a single op that performs all of the variable updates
-      This means:
+    Note: This function is executed as a graph function in graph mode.
+    This means:
       a) Operations on the same resource are executed in textual order.
          This should make it easier to do things like add the updated
          value of a variable to another, for example.
@@ -343,9 +356,6 @@ class Metric(Layer):
 
     Result computation is an idempotent operation that simply calculates the
     metric value using the state variables.
-
-    Please decorate the function with:
-    @result: Wraps `result()` in a distribution strategy merge_call().
     """
     NotImplementedError('Must be implemented in subclasses.')
 
@@ -380,7 +390,13 @@ class Mean(Metric):
   Use `sample_weight` of 0 to mask values.
   """
 
-  def __init__(self, name='mean', dtype=dtypes.float64):
+  def __init__(self, name='mean', dtype=None):
+    """Creates a `Mean` instance.
+
+    Args:
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+    """
     super(Mean, self).__init__(name=name, dtype=dtype)
     # Create new state variables
     self.total = self.add_weight(
@@ -388,7 +404,6 @@ class Mean(Metric):
     self.count = self.add_weight(
         'count', initializer=init_ops.zeros_initializer)
 
-  @update_state
   def update_state(self, values, sample_weight=None):
     """Accumulates statistics for computing the mean.
 
@@ -418,14 +433,84 @@ class Mean(Metric):
     state_ops.assign_add(self.total, values)
     state_ops.assign_add(self.count, num_values)
 
-  @result
   def result(self):
     return _safe_div(self.total, self.count)
 
 
+class MeanMetricWrapper(Mean):
+  """Wraps a stateless metric function with the Mean metric."""
+
+  def __init__(self, fn, name=None, dtype=None, **kwargs):
+    """Creates a `MeanMetricWrapper` instance.
+
+    Args:
+      fn: The metric function to wrap, with signature
+        `fn(y_true, y_pred, **kwargs)`.
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+      **kwargs: The keyword arguments that are passed on to `fn`.
+    """
+    super(MeanMetricWrapper, self).__init__(name=name, dtype=dtype)
+    self._fn = fn
+    self._fn_kwargs = kwargs
+
+  def update_state(self, y_true, y_pred, sample_weight=None):
+    """Accumulates metric statistics.
+
+    `y_true` and `y_pred` should have the same shape.
+
+    Args:
+      y_true: The ground truth values.
+      y_pred: The predicted values.
+      sample_weight: Optional weighting of each example. Defaults to 1. Can be
+        a `Tensor` whose rank is either 0, or the same rank as `y_true`,
+        and must be broadcastable to `y_true`.
+    """
+    y_true = math_ops.cast(y_true, self._dtype)
+    y_pred = math_ops.cast(y_pred, self._dtype)
+    y_pred, y_true, sample_weight = _squeeze_or_expand_dimensions(
+        y_pred, y_true, sample_weight)
+
+    matches = self._fn(y_true, y_pred, **self._fn_kwargs)
+    super(MeanMetricWrapper, self).update_state(
+        matches, sample_weight=sample_weight)
+
+  def get_config(self):
+    config = self._fn_kwargs
+    base_config = super(MeanMetricWrapper, self).get_config()
+    return dict(list(base_config.items()) + list(config.items()))
+
+
+class BinaryAccuracy(MeanMetricWrapper):
+  """Calculates how often predictions matches labels.
+
+  This metric creates two local variables, `total` and `count` that are used to
+  compute the frequency with which `y_pred` matches `y_true`. This frequency is
+  ultimately returned as `binary accuracy`: an idempotent operation that simply
+  divides `total` by `count`.
+
+  If `sample_weight` is `None`, weights default to 1.
+  Use `sample_weight` of 0 to mask values.
+  """
+
+  def __init__(self, name='binary-accuracy', dtype=None, threshold=0.5):
+    """Creates a `BinaryAccuracy` instance.
+
+    Args:
+      name: (Optional) string name of the metric instance.
+      dtype: (Optional) data type of the metric result.
+      threshold: (Optional) Float representing the threshold for deciding
+      whether prediction values are 1 or 0.
+    """
+    super(BinaryAccuracy, self).__init__(
+        binary_accuracy, name, dtype=dtype, threshold=threshold)
+
+
 @tf_export('keras.metrics.binary_accuracy')
-def binary_accuracy(y_true, y_pred):
-  return K.mean(math_ops.equal(y_true, math_ops.round(y_pred)), axis=-1)
+def binary_accuracy(y_true, y_pred, threshold=0.5):
+  threshold = math_ops.cast(threshold, y_pred.dtype)
+  y_pred = math_ops.cast(y_pred > threshold, y_pred.dtype)
+  return K.mean(math_ops.equal(y_true, y_pred), axis=-1)
 
 
 @tf_export('keras.metrics.categorical_accuracy')
diff --git a/tensorflow/python/keras/metrics_test.py b/tensorflow/python/keras/metrics_test.py
index 6d8269f34d..d583379708 100644
--- a/tensorflow/python/keras/metrics_test.py
+++ b/tensorflow/python/keras/metrics_test.py
@@ -196,7 +196,7 @@ class KerasMetricsTest(test.TestCase):
     # check config
     self.assertEqual(m.name, 'my_mean')
     self.assertTrue(m.stateful)
-    self.assertEqual(m.dtype, dtypes.float64)
+    self.assertEqual(m.dtype, dtypes.float32)
     self.assertEqual(len(m.variables), 2)
     self.evaluate(variables.global_variables_initializer())
 
@@ -212,7 +212,7 @@ class KerasMetricsTest(test.TestCase):
     # check update_state() and result() + state accumulation + tensor input
     update_op = m.update_state(ops.convert_n_to_tensor([1, 5]))
     self.evaluate(update_op)
-    self.assertEqual(self.evaluate(m.result()), 106 / 3)
+    self.assertAlmostEqual(self.evaluate(m.result()), 106 / 3, 2)
     self.assertEqual(self.evaluate(m.total), 106)  # 100 + 1 + 5
     self.assertEqual(self.evaluate(m.count), 3)
 
@@ -223,7 +223,8 @@ class KerasMetricsTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
   def test_mean_with_sample_weight(self):
-    m = metrics.Mean()
+    m = metrics.Mean(dtype=dtypes.float64)
+    self.assertEqual(m.dtype, dtypes.float64)
     self.evaluate(variables.global_variables_initializer())
 
     # check scalar weight
@@ -308,6 +309,94 @@ class KerasMetricsTest(test.TestCase):
     self.assertEqual(200., self.evaluate(restore_mean.result()))
     self.assertEqual(3, self.evaluate(restore_mean.count))
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_binary_accuracy(self):
+    acc_obj = metrics.BinaryAccuracy(name='my acc')
+
+    # check config
+    self.assertEqual(acc_obj.name, 'my acc')
+    self.assertTrue(acc_obj.stateful)
+    self.assertEqual(len(acc_obj.variables), 2)
+    self.assertEqual(acc_obj.dtype, dtypes.float32)
+    self.evaluate(variables.global_variables_initializer())
+
+    # verify that correct value is returned
+    update_op = acc_obj.update_state([[1], [0]], [[1], [0]])
+    self.evaluate(update_op)
+    result = self.evaluate(acc_obj.result())
+    self.assertEqual(result, 1)  # 2/2
+
+    # check y_pred squeeze
+    update_op = acc_obj.update_state([[1], [1]], [[[1]], [[0]]])
+    self.evaluate(update_op)
+    result = self.evaluate(acc_obj.result())
+    self.assertAlmostEqual(result, 0.75, 2)  # 3/4
+
+    # check y_true squeeze
+    result_t = acc_obj([[[1]], [[1]]], [[1], [0]])
+    result = self.evaluate(result_t)
+    self.assertAlmostEqual(result, 0.67, 2)  # 4/6
+
+    # check with sample_weight
+    result_t = acc_obj([[1], [1]], [[1], [0]], [[0.5], [0.2]])
+    result = self.evaluate(result_t)
+    self.assertAlmostEqual(result, 0.67, 2)  # 4.5/6.7
+
+    # check incompatible shapes
+    with self.assertRaisesRegexp(ValueError,
+                                 r'Shapes \(1,\) and \(2,\) are incompatible'):
+      acc_obj.update_state([1, 1], [1])
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_binary_accuracy_threshold(self):
+    acc_obj = metrics.BinaryAccuracy(threshold=0.7)
+    self.evaluate(variables.global_variables_initializer())
+    result_t = acc_obj([[1], [1], [0], [0]], [[0.9], [0.6], [0.4], [0.8]])
+    result = self.evaluate(result_t)
+    self.assertAlmostEqual(result, 0.5, 2)
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_invalid_result(self):
+
+    class InvalidResult(metrics.Metric):
+
+      def __init__(self, name='invalid-result', dtype=dtypes.float64):
+        super(InvalidResult, self).__init__(name=name, dtype=dtype)
+
+      def update_state(self, *args, **kwargs):
+        pass
+
+      def result(self):
+        return 1
+
+    invalid_result_obj = InvalidResult()
+    with self.assertRaisesRegexp(
+        TypeError,
+        'Metric invalid-result\'s result must be a Tensor or Operation, given:'
+    ):
+      invalid_result_obj.result()
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_invalid_update(self):
+
+    class InvalidUpdate(metrics.Metric):
+
+      def __init__(self, name='invalid-update', dtype=dtypes.float64):
+        super(InvalidUpdate, self).__init__(name=name, dtype=dtype)
+
+      def update_state(self, *args, **kwargs):
+        return [1]
+
+      def result(self):
+        pass
+
+    invalid_update_obj = InvalidUpdate()
+    with self.assertRaisesRegexp(
+        TypeError,
+        'Metric invalid-update\'s update must be a Tensor or Operation, given:'
+    ):
+      invalid_update_obj.update_state()
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.metrics.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.metrics.pbtxt
index a97a9b5758..73b577da37 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.metrics.pbtxt
@@ -22,7 +22,7 @@ tf_module {
   }
   member_method {
     name: "binary_accuracy"
-    argspec: "args=[\'y_true\', \'y_pred\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'y_true\', \'y_pred\', \'threshold\'], varargs=None, keywords=None, defaults=[\'0.5\'], "
   }
   member_method {
     name: "binary_crossentropy"
-- 
cgit v1.2.3


From 26e531ee70f0b7efbb8f1452a40d5e926b7f38c0 Mon Sep 17 00:00:00 2001
From: Nick Felt 
Date: Tue, 24 Jul 2018 12:50:46 -0700
Subject: Automated rollback of commit 568727eed199dba04e37f500265b50f96fed455e

PiperOrigin-RevId: 205875586
---
 .../contrib/summary/summary_ops_graph_test.py      |  20 -
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py |  17 +-
 tensorflow/core/kernels/summary_kernels.cc         |   2 -
 tensorflow/python/BUILD                            |   1 -
 tensorflow/python/estimator/estimator.py           |  24 +-
 tensorflow/python/estimator/estimator_test.py      | 260 +----------
 tensorflow/python/estimator/model_fn.py            |   3 +-
 tensorflow/python/estimator/training_test.py       |  10 +-
 tensorflow/python/ops/summary_ops_v2.py            |  68 +--
 tensorflow/python/saved_model/builder_impl.py      |   5 +-
 .../python/summary/writer/event_file_writer_v2.py  |  71 ++-
 tensorflow/python/summary/writer/writer.py         |   8 +-
 tensorflow/python/summary/writer/writer_test.py    |  54 +--
 .../python/training/basic_session_run_hooks.py     | 182 +++-----
 .../training/basic_session_run_hooks_test.py       | 476 +++++++--------------
 tensorflow/python/training/monitored_session.py    |  11 +-
 tensorflow/python/training/optimizer.py            |   6 +-
 17 files changed, 279 insertions(+), 939 deletions(-)

diff --git a/tensorflow/contrib/summary/summary_ops_graph_test.py b/tensorflow/contrib/summary/summary_ops_graph_test.py
index 409fdf4583..ae8336daaf 100644
--- a/tensorflow/contrib/summary/summary_ops_graph_test.py
+++ b/tensorflow/contrib/summary/summary_ops_graph_test.py
@@ -228,26 +228,6 @@ class GraphFileTest(test_util.TensorFlowTestCase):
       sess.run(writer.flush())
       self.assertEqual(2, get_total())
 
-  def testSummaryOpsCollector(self):
-    summary_ops.scalar('x', 1.0, step=1)
-    with summary_ops.create_file_writer(self.get_temp_dir()).as_default():
-      s2 = summary_ops.scalar('x', 1.0, step=1)
-      collector1 = summary_ops._SummaryOpsCollector()
-      collector2 = summary_ops._SummaryOpsCollector()
-      with collector1.capture():
-        s3 = summary_ops.scalar('x', 1.0, step=1)
-        with collector2.capture():
-          s4 = summary_ops.scalar('x', 1.0, step=1)
-        s5 = summary_ops.scalar('x', 1.0, step=1)
-      s6 = summary_ops.scalar('x', 1.0, step=1)
-    summary_ops.scalar('six', 1.0, step=1)
-
-    # Ops defined outside summary writer context are ignored; ops defined inside
-    # SummaryOpsCollector capture context are stored to innermost such context.
-    self.assertItemsEqual([s2, s6], summary_ops.all_summary_ops())
-    self.assertItemsEqual([s3, s5], collector1.collected_ops)
-    self.assertItemsEqual([s4], collector2.collected_ops)
-
 
 class GraphDbTest(summary_test_util.SummaryDbTest):
 
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 1eb43ac7f7..42406db88a 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -1506,17 +1506,13 @@ class _OutfeedHostCall(object):
     _OutfeedHostCall.validate(host_calls)
     ret = {}
     for name, host_call in host_calls.items():
-      # Isolate host call summary ops from main graph.
-      summary_collector = contrib_summary._SummaryOpsCollector()  # pylint: disable=protected-access
       host_fn, tensors = host_call
       if isinstance(tensors, (tuple, list)):
-        with summary_collector.capture():
-          ret[name] = host_fn(*tensors)
+        ret[name] = host_fn(*tensors)
       else:
         # Must be dict.
         try:
-          with summary_collector.capture():
-            ret[name] = host_fn(**tensors)
+          ret[name] = host_fn(**tensors)
         except TypeError as e:
           logging.warning(
               'Exception while calling %s: %s. It is likely the tensors '
@@ -1631,14 +1627,11 @@ class _OutfeedHostCall(object):
           # dimension.
           dequeue_ops[i] = array_ops.concat(dequeue_ops[i], axis=0)
 
-        # Isolate host call summary ops from main graph.
-        summary_collector = contrib_summary._SummaryOpsCollector()  # pylint: disable=protected-access
         if self._tensor_keys[name] is not None:
           # The user-provided eval_metrics[1] is a dict.
           dequeue_ops = dict(zip(self._tensor_keys[name], dequeue_ops))
           try:
-            with summary_collector.capture():
-              ret[name] = self._host_fns[name](**dequeue_ops)
+            ret[name] = self._host_fns[name](**dequeue_ops)
           except TypeError as e:
             logging.warning(
                 'Exception while calling %s: %s. It is likely the tensors '
@@ -1646,8 +1639,8 @@ class _OutfeedHostCall(object):
                 'function\'s arguments', name, e, name)
             raise e
         else:
-          with summary_collector.capture():
-            ret[name] = self._host_fns[name](*dequeue_ops)
+          ret[name] = self._host_fns[name](*dequeue_ops)
+
     return ret
 
 
diff --git a/tensorflow/core/kernels/summary_kernels.cc b/tensorflow/core/kernels/summary_kernels.cc
index b518c3cbf4..b287f0cc2f 100644
--- a/tensorflow/core/kernels/summary_kernels.cc
+++ b/tensorflow/core/kernels/summary_kernels.cc
@@ -53,7 +53,6 @@ class CreateSummaryFileWriterOp : public OpKernel {
                                   max_queue, flush_millis, logdir,
                                   filename_suffix, ctx->env(), s);
                             }));
-    core::ScopedUnref unref(s);
   }
 };
 REGISTER_KERNEL_BUILDER(Name("CreateSummaryFileWriter").Device(DEVICE_CPU),
@@ -90,7 +89,6 @@ class CreateSummaryDbWriterOp : public OpKernel {
                   db, experiment_name, run_name, user_name, ctx->env(), s));
               return Status::OK();
             }));
-    core::ScopedUnref unref(s);
   }
 };
 REGISTER_KERNEL_BUILDER(Name("CreateSummaryDbWriter").Device(DEVICE_CPU),
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index a7c60f5450..b5876c3457 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2822,7 +2822,6 @@ py_library(
         ":framework_ops",
         ":math_ops",
         ":resource_variable_ops",
-        ":resources",
         ":smart_cond",
         ":summary_op_util",
         ":summary_ops_gen",
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index b7185e8966..915ceeb98b 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -46,7 +46,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import metrics as metrics_lib
 from tensorflow.python.ops import resources
-from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
@@ -66,7 +65,6 @@ from tensorflow.python.util import compat
 from tensorflow.python.util import compat_internal
 from tensorflow.python.util import function_utils
 from tensorflow.python.util import nest
-from tensorflow.python.util import tf_contextlib
 from tensorflow.python.util.tf_export import estimator_export
 
 
@@ -1158,8 +1156,7 @@ class Estimator(object):
       Loss from training
     """
     worker_hooks = []
-    with ops.Graph().as_default() as g, g.device(
-        self._device_fn), self._summary_writing_context():
+    with ops.Graph().as_default() as g, g.device(self._device_fn):
       random_seed.set_random_seed(self._config.tf_random_seed)
       global_step_tensor = self._create_and_assert_global_step(g)
       training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
@@ -1193,7 +1190,7 @@ class Estimator(object):
     is_tpu_strategy = self._distribution.__class__.__name__ == 'TPUStrategy'
 
     worker_hooks = []
-    with ops.Graph().as_default() as g, self._summary_writing_context():
+    with ops.Graph().as_default() as g:
       with self._distribution.scope():
         random_seed.set_random_seed(self._config.tf_random_seed)
 
@@ -1522,23 +1519,6 @@ class Estimator(object):
                    (self._warm_start_settings,))
       warm_starting_util.warm_start(*self._warm_start_settings)
 
-  @tf_contextlib.contextmanager
-  def _summary_writing_context(self):
-    """Context manager for enabling V2 summary writing."""
-    # Avoid creating a file writer at all if no summary writing was requested.
-    if self._config.save_summary_steps <= 0:
-      yield
-      return
-    file_writer = summary_ops_v2.create_file_writer(
-        logdir=self._model_dir, filename_suffix='')
-    with file_writer.as_default():
-      # Create a boolean placeholder, default False, that SummarySaverHook can
-      # use to enable/disable V2 summary writing according to its own logic.
-      placeholder = array_ops.placeholder_with_default(False, shape=[])
-      training.SummarySaverHook._set_placeholder(placeholder)  # pylint: disable=protected-access
-      with summary_ops_v2.record_summaries_if(placeholder):
-        yield
-
 
 def create_per_tower_ready_op(scaffold):
   """Create a Scaffold.ready_op inside a tower."""
diff --git a/tensorflow/python/estimator/estimator_test.py b/tensorflow/python/estimator/estimator_test.py
index 1dd45a07c2..8bc410ba0b 100644
--- a/tensorflow/python/estimator/estimator_test.py
+++ b/tensorflow/python/estimator/estimator_test.py
@@ -22,7 +22,6 @@ import functools
 import glob
 import os
 import tempfile
-import time
 
 import numpy as np
 import six
@@ -30,7 +29,6 @@ import six
 from google.protobuf import text_format
 
 from tensorflow.core.protobuf import rewriter_config_pb2
-from tensorflow.core.util.event_pb2 import SessionLog
 from tensorflow.python.client import session
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.estimator import estimator
@@ -42,7 +40,6 @@ from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.framework import test_util
@@ -58,7 +55,6 @@ from tensorflow.python.ops import metrics as metrics_lib
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import string_ops
-from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.ops.losses import losses
@@ -89,32 +85,13 @@ def dummy_model_fn(features, labels, params):
   _, _, _ = features, labels, params
 
 
-def load_eventfile_contents(directory_path):
-  """Returns the contents of the singular event file in the given directory."""
-  writer_cache.FileWriterCache.clear()
-
-  # Get last Event written.
-  event_paths = glob.glob(os.path.join(directory_path, '*tfevent*'))
-  if len(event_paths) != 1:
-    raise AssertionError('Expected one eventfile, got %s' % str(event_paths))
-  return list(summary_iterator.summary_iterator(event_paths[0]))
-
-
-def make_summary_steps(eventlist):
-  """Returns dict of tags in eventlist mapped to steps where they're logged."""
-  tag_to_steps = {}
-  for event in eventlist:
-    if event.summary is not None:
-      for value in event.summary.value:
-        if value.tag not in tag_to_steps:
-          tag_to_steps[value.tag] = []
-        tag_to_steps[value.tag].append(event.step)
-  return tag_to_steps
-
-
 def summaries_with_matching_keyword(keyword, dir_):
   """Yields summary protos matching given keyword from event file."""
-  for event in load_eventfile_contents(dir_):
+
+  writer_cache.FileWriterCache.clear()
+
+  event_paths = glob.glob(os.path.join(dir_, 'events*'))
+  for event in summary_iterator.summary_iterator(event_paths[-1]):
     if event.summary is not None:
       for value in event.summary.value:
         if keyword in value.tag:
@@ -389,51 +366,13 @@ def dummy_input_fn():
           constant_op.constant([[1], [1]]))
 
 
-class StableGlobalStepEstimator(estimator.Estimator):
-  """Estimator subclass using a ResourceVariable global_step for testing."""
-  # TODO(nickfelt): remove after standard global_step is a ResourceVariable.
-
-  def _create_global_step(self, graph):
-    """Creates a stable ResourceVariable-based global step suitable for tests.
-
-    Args:
-      graph: The graph in which to create the global step.
-
-    Returns:
-      A global step `Tensor`.
-    """
-    with graph.as_default(), graph.name_scope(None):
-      return variable_scope.get_variable(
-          ops.GraphKeys.GLOBAL_STEP,
-          shape=[],
-          dtype=dtypes.int64,
-          initializer=init_ops.zeros_initializer(),
-          trainable=False,
-          collections=[
-              ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.GLOBAL_STEP
-          ],
-          # Use a ResourceVariable and set caching_device to make the read
-          # behavior deterministic and well-defined.
-          caching_device='cpu:0',
-          use_resource=True)
-
-
 def model_fn_global_step_incrementer(features, labels, mode):
   _, _ = features, labels
+  global_step = training.get_global_step()
   return model_fn_lib.EstimatorSpec(
       mode,
       loss=constant_op.constant(1.),
-      train_op=training.get_global_step().assign_add(1))
-
-
-def model_fn_with_v1_and_v2_summaries(features, labels, mode):
-  del features, labels
-  summary.scalar('foo-v1', 1.0)
-  summary_ops_v2.scalar('foo-v2', 2.0)
-  return model_fn_lib.EstimatorSpec(
-      mode,
-      loss=constant_op.constant(1.),
-      train_op=training.get_global_step().assign_add(1))
+      train_op=state_ops.assign_add(global_step, 1))
 
 
 def assert_features_op(expected_features, actual_features):
@@ -469,25 +408,6 @@ def _make_input_fn(features, labels):
   return _input_fn
 
 
-class RaiseOnceAtStepHook(session_run_hook.SessionRunHook):
-  """Hook that raises an Exception the first time it reaches step N."""
-
-  def __init__(self, n, ex):
-    self.n = n
-    self.ex = ex
-    self.raised = False
-
-  def before_run(self, run_context):
-    # Raise the first time we reach step N.
-    self.n -= 1
-    if 0 == self.n and not self.raised:
-      # Wait 1 sec so that event file names have different UNIX timestamps.
-      time.sleep(1.2)
-      self.raised = True
-      raise self.ex
-    return None
-
-
 class EstimatorTrainTest(test.TestCase):
 
   def test_callable_model_fn(self):
@@ -697,171 +617,17 @@ class EstimatorTrainTest(test.TestCase):
     self.assertEqual(
         5, estimator._load_global_step_from_checkpoint_dir(est.model_dir))
 
-  def test_summary_loss(self):
+  def test_loss_summary(self):
     est = estimator.Estimator(model_fn=model_fn_global_step_incrementer,
                               config=run_config.RunConfig(save_summary_steps=1))
     est.train(dummy_input_fn, steps=1)
-    events = load_eventfile_contents(est.model_dir)
-    self.assertEqual({'loss': [1]}, make_summary_steps(events))
 
-  def test_summary_user_defined_v1_and_v2(self):
-    est = StableGlobalStepEstimator(
-        model_fn=model_fn_with_v1_and_v2_summaries,
-        config=run_config.RunConfig(save_summary_steps=1))
-    est.train(dummy_input_fn, steps=1)
-    events = load_eventfile_contents(est.model_dir)
-    self.assertEqual(
-        {'foo-v1': [1], 'foo-v2': [0], 'loss': [1]},
-        make_summary_steps(events))
-
-  def test_summary_writing_disabled(self):
-    est = StableGlobalStepEstimator(
-        model_fn=model_fn_with_v1_and_v2_summaries,
-        config=run_config.RunConfig(save_summary_steps=0))
-    est.train(dummy_input_fn, steps=1)
-    events = load_eventfile_contents(est.model_dir)
-    self.assertEqual({}, make_summary_steps(events))
-
-  def test_summary_saving_steps(self):
-    est = StableGlobalStepEstimator(
-        model_fn=model_fn_with_v1_and_v2_summaries,
-        config=run_config.RunConfig(save_summary_steps=2))
-    est.train(dummy_input_fn, steps=5)
-    events = load_eventfile_contents(est.model_dir)
-    self.assertEqual(
-        {'foo-v1': [1, 3, 5], 'foo-v2': [0, 2, 4], 'loss': [1, 3, 5]},
-        make_summary_steps(events))
-
-  def test_summary_additional_hook(self):
-    def model_fn_extra_summary_hook(features, labels, mode, config):
-      del features, labels
-      v1_op = summary.scalar('foo-v1', 1.0)
-      v2_op = summary_ops_v2.scalar('foo-v2', 2.0)
-      extra_hook = basic_session_run_hooks.SummarySaverHook(
-          output_dir=os.path.join(config.model_dir, 'extra'),
-          save_steps=3,
-          summary_op=control_flow_ops.with_dependencies([v2_op], v1_op))
-      return model_fn_lib.EstimatorSpec(
-          mode,
-          loss=constant_op.constant(1.),
-          train_op=training.get_global_step().assign_add(1),
-          training_hooks=[extra_hook])
-    est = StableGlobalStepEstimator(
-        model_fn=model_fn_extra_summary_hook,
-        config=run_config.RunConfig(save_summary_steps=2))
-    est.train(dummy_input_fn, steps=7)
-
-    events = load_eventfile_contents(est.model_dir)
-    self.assertEqual(
-        {'foo-v1': [1, 3, 5, 7], 'foo-v2': [0, 2, 4, 6], 'loss': [1, 3, 5, 7]},
-        make_summary_steps(events))
-    extra_dir = os.path.join(est.model_dir, 'extra')
-    extra_events = load_eventfile_contents(extra_dir)
-    self.assertEqual({'foo-v1': [1, 4, 7]}, make_summary_steps(extra_events))
-
-  def test_summary_user_defined_in_input_fn(self):
-    def input_fn_custom_summaries():
-      summary.scalar('foo-v1', 1.0)
-      summary_ops_v2.scalar('foo-v2', 2.0)
-      return ({'x': constant_op.constant([[1], [1]])},
-              constant_op.constant([[1], [1]]))
-    est = StableGlobalStepEstimator(
-        model_fn=model_fn_global_step_incrementer,
-        config=run_config.RunConfig(save_summary_steps=1))
-    est.train(input_fn_custom_summaries, steps=1)
-    events = load_eventfile_contents(est.model_dir)
-    self.assertEqual(
-        {'foo-v1': [1], 'foo-v2': [0], 'loss': [1]},
-        make_summary_steps(events))
+    # Make sure nothing is stuck in limbo.
+    writer_cache.FileWriterCache.clear()
 
-  def test_summary_with_warm_start(self):
-    est = StableGlobalStepEstimator(
-        model_fn=model_fn_with_v1_and_v2_summaries,
-        config=run_config.RunConfig(save_summary_steps=1))
-    est.train(dummy_input_fn, steps=5)
-    warm_started_est = StableGlobalStepEstimator(
-        model_fn=model_fn_with_v1_and_v2_summaries,
-        config=run_config.RunConfig(save_summary_steps=1),
-        warm_start_from=est.model_dir)
-    warm_started_est.train(dummy_input_fn, steps=3)
-    events = load_eventfile_contents(warm_started_est.model_dir)
-    self.assertEqual(
-        {'foo-v1': [1, 2, 3], 'foo-v2': [0, 1, 2], 'loss': [1, 2, 3]},
-        make_summary_steps(events))
-
-  def test_summary_with_error_and_auto_restart(self):
-    est = StableGlobalStepEstimator(
-        model_fn=model_fn_with_v1_and_v2_summaries,
-        config=run_config.RunConfig(
-            save_summary_steps=2, save_checkpoints_steps=5))
-    abort_hook = RaiseOnceAtStepHook(
-        7, errors_impl.AbortedError(None, None, 'Abort'))
-    est.train(dummy_input_fn, steps=10, hooks=[abort_hook])
-
-    # We expect two event files: one for the aborted run, and one post-restart.
-    event_paths = sorted(glob.glob(os.path.join(est.model_dir, '*tfevent*')))
-    self.assertEqual(2, len(event_paths))
-
-    # First file should have summaries up to the last checkpoint.
-    first_events = list(summary_iterator.summary_iterator(event_paths[0]))
-    first_summaries = make_summary_steps(first_events)
-    self.assertEqual([0, 2, 4], first_summaries['foo-v2'])
-    # The V1 summaries may or may not include step 5 (depending on the flush()
-    # sequence) so just check that at least 1 and 3 are there.
-    # TODO(nickfelt): ensure summaries *at* checkpoint step get flushed too.
-    self.assertEqual([1, 3], first_summaries['foo-v1'][:2])
-    self.assertEqual([1, 3], first_summaries['loss'][:2])
-
-    # Second file should pick up from global_step=5. Note that the 2 step save
-    # interval will reset at this step as well, so summaries logged at steps
-    # 2 and 4 continue not with 6, 8, ... but at steps 5, 7, ... instead.
-    second_events = list(summary_iterator.summary_iterator(event_paths[1]))
-    self.assertEqual(
-        {'foo-v1': [6, 8, 10], 'foo-v2': [5, 7, 9], 'loss': [6, 8, 10]},
-        make_summary_steps(second_events))
-    # Second file should contain a session START event at resumed global_step.
-    session_start_event = next(event for event in second_events
-                               if event.session_log.status == SessionLog.START)
-    self.assertEqual(5, session_start_event.step)
-
-  def test_summary_with_error_and_explicit_restart(self):
-    est = StableGlobalStepEstimator(
-        model_fn=model_fn_with_v1_and_v2_summaries,
-        config=run_config.RunConfig(
-            save_summary_steps=2, save_checkpoints_steps=5))
-    abort_hook = RaiseOnceAtStepHook(
-        7, errors_impl.UnknownError(None, None, 'Unknown failure'))
-    self.assertRaises(
-        errors_impl.UnknownError,
-        lambda: est.train(dummy_input_fn, max_steps=10, hooks=[abort_hook]))
-    # Explicitly retry after the error.
-    est.train(dummy_input_fn, max_steps=10, hooks=[abort_hook])
-
-    # We expect two event files: one for the failed run, and one post-restart.
-    event_paths = sorted(glob.glob(os.path.join(est.model_dir, '*tfevent*')))
-    self.assertEqual(2, len(event_paths))
-
-    # First file should have summaries up to the last checkpoint.
-    first_events = list(summary_iterator.summary_iterator(event_paths[0]))
-    first_summaries = make_summary_steps(first_events)
-    self.assertEqual([0, 2, 4], first_summaries['foo-v2'])
-    # The V1 summaries may or may not include step 5 (depending on the flush()
-    # sequence) so just check that at least 1 and 3 are there.
-    # TODO(nickfelt): ensure summaries *at* checkpoint step get flushed too.
-    self.assertEqual([1, 3], first_summaries['foo-v1'][:2])
-    self.assertEqual([1, 3], first_summaries['loss'][:2])
-
-    # Second file should pick up from global_step=5. Note that the 2 step save
-    # interval will reset at this step as well, so summaries logged at steps
-    # 2 and 4 continue not with 6, 8, ... but at steps 5, 7, ... instead.
-    second_events = list(summary_iterator.summary_iterator(event_paths[1]))
-    self.assertEqual(
-        {'foo-v1': [6, 8, 10], 'foo-v2': [5, 7, 9], 'loss': [6, 8, 10]},
-        make_summary_steps(second_events))
-    # Second file should contain a session START event at resumed global_step.
-    session_start_event = next(event for event in second_events
-                               if event.session_log.status == SessionLog.START)
-    self.assertEqual(5, session_start_event.step)
+    if check_eventfile_for_keyword('loss', est.model_dir):
+      return
+    self.fail('{} should be part of reported summaries.'.format('loss'))
 
   def test_latest_checkpoint(self):
     est = estimator.Estimator(model_fn=model_fn_global_step_incrementer)
diff --git a/tensorflow/python/estimator/model_fn.py b/tensorflow/python/estimator/model_fn.py
index b1b2f65edf..a9fd8f8e1a 100644
--- a/tensorflow/python/estimator/model_fn.py
+++ b/tensorflow/python/estimator/model_fn.py
@@ -26,7 +26,6 @@ import six
 from tensorflow.python.estimator.export import export_output as export_output_lib
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import tag_constants
@@ -433,7 +432,7 @@ class _TPUEstimatorSpec(collections.namedtuple('TPUEstimatorSpec', [
 
 
 def _check_is_tensor_or_operation(x, name):
-  if not (isinstance(x, ops.Operation) or tensor_util.is_tensor(x)):
+  if not (isinstance(x, ops.Operation) or isinstance(x, ops.Tensor)):
     raise TypeError('{} must be Operation or Tensor, given: {}'.format(name, x))
 
 
diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py
index 121439a2cd..dc106c7d3b 100644
--- a/tensorflow/python/estimator/training_test.py
+++ b/tensorflow/python/estimator/training_test.py
@@ -2059,7 +2059,7 @@ class TrainAndEvaluateIntegrationTest(test.TestCase):
 
   def _extract_loss_and_global_step(self, event_folder):
     """Returns the loss and global step in last event."""
-    event_paths = sorted(glob.glob(os.path.join(event_folder, 'events*')))
+    event_paths = glob.glob(os.path.join(event_folder, 'events*'))
 
     loss = None
     global_step_count = None
@@ -2139,12 +2139,10 @@ class TrainAndEvaluateIntegrationTest(test.TestCase):
     # Make sure nothing is stuck in limbo.
     writer_cache.FileWriterCache.clear()
 
-    # Examine the training events.
-    training_loss, training_global_step = self._extract_loss_and_global_step(
-        est.model_dir)
+    # Examine the training events. Use a range to check global step to avoid
+    # flakyness due to global step race condition.
+    training_loss, _ = self._extract_loss_and_global_step(est.model_dir)
     self.assertIsNotNone(training_loss)
-    # Training summaries are logged for steps 1 and 10, so we see final step.
-    self.assertEqual(max_steps, training_global_step)
 
     # Examine the eval events. The global step should be accurate.
     eval_loss, eval_global_step = self._extract_loss_and_global_step(
diff --git a/tensorflow/python/ops/summary_ops_v2.py b/tensorflow/python/ops/summary_ops_v2.py
index 669358d9db..00150fe688 100644
--- a/tensorflow/python/ops/summary_ops_v2.py
+++ b/tensorflow/python/ops/summary_ops_v2.py
@@ -37,7 +37,6 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_summary_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.ops import resources
 from tensorflow.python.ops import summary_op_util
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import training_util
@@ -67,39 +66,41 @@ def should_record_summaries():
   return should_record_collection[0]
 
 
-@tf_contextlib.contextmanager
-def always_record_summaries():
-  """Sets the should_record_summaries Tensor to always true."""
-  with record_summaries_if(True):
-    yield
-
-
-@tf_contextlib.contextmanager
-def never_record_summaries():
-  """Sets the should_record_summaries Tensor to always false."""
-  with record_summaries_if(False):
-    yield
-
-
 # TODO(apassos) consider how to handle local step here.
 @tf_contextlib.contextmanager
 def record_summaries_every_n_global_steps(n, global_step=None):
   """Sets the should_record_summaries Tensor to true if global_step % n == 0."""
   if global_step is None:
     global_step = training_util.get_or_create_global_step()
-  with ops.device("cpu:0"):
-    on_nth_global_step = math_ops.equal(global_step % n, 0)
-  with record_summaries_if(on_nth_global_step):
+  collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
+  old = collection_ref[:]
+  try:
+    with ops.device("cpu:0"):
+      collection_ref[:] = [math_ops.equal(global_step % n, 0)]
+    yield
+  finally:
+    collection_ref[:] = old
+
+
+@tf_contextlib.contextmanager
+def always_record_summaries():
+  """Sets the should_record_summaries Tensor to always true."""
+  collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
+  old = collection_ref[:]
+  try:
+    collection_ref[:] = [True]
     yield
+  finally:
+    collection_ref[:] = old
 
 
 @tf_contextlib.contextmanager
-def record_summaries_if(bool_value):
-  """Sets the should_record_summaries Tensor to the given boolean value."""
+def never_record_summaries():
+  """Sets the should_record_summaries Tensor to always false."""
   collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
   old = collection_ref[:]
   try:
-    collection_ref[:] = [bool_value]
+    collection_ref[:] = [False]
     yield
   finally:
     collection_ref[:] = old
@@ -142,6 +143,7 @@ class SummaryWriter(object):
       finally:
         context.context().summary_writer_resource = old
 
+
   def init(self):
     """Operation to initialize the summary writer resource."""
     if self._resource is not None:
@@ -309,9 +311,6 @@ def _make_summary_writer(name, factory, **kwargs):
     # TODO(apassos): Consider doing this instead.
     #   ops.get_default_session().run(init_op)
     ops.add_to_collection(_SUMMARY_WRITER_INIT_COLLECTION_NAME, init_op)
-    # TODO(nickfelt): expose an actual op for this
-    is_initialized_op = constant_op.constant(True)
-    resources.register_resource(resource, init_op, is_initialized_op)
   return SummaryWriter(resource, init_op_fn)
 
 
@@ -326,27 +325,6 @@ def _nothing():
   return constant_op.constant(False)
 
 
-class _SummaryOpsCollector(object):
-  """Defines a context manager for isolating out a subset of summary ops.
-
-  Summary ops defined within this context will be accumulated within this
-  collector instead of being added to the graph-wide summary ops collection that
-  is returned by {@tf.contrib.summary.all_summary_ops}.
-  """
-
-  def __init__(self):
-    self.collected_ops = []
-
-  @tf_contextlib.contextmanager
-  def capture(self):
-    collection_ref = ops.get_collection_ref(ops.GraphKeys._SUMMARY_COLLECTION)  # pylint: disable=protected-access
-    original_ops = collection_ref[:]
-    collection_ref[:] = []
-    yield
-    self.collected_ops = collection_ref[:]
-    collection_ref[:] = original_ops
-
-
 def all_summary_ops():
   """Graph-mode only. Returns all summary ops.
 
diff --git a/tensorflow/python/saved_model/builder_impl.py b/tensorflow/python/saved_model/builder_impl.py
index b67d0f2362..e58be804c2 100644
--- a/tensorflow/python/saved_model/builder_impl.py
+++ b/tensorflow/python/saved_model/builder_impl.py
@@ -28,7 +28,6 @@ from tensorflow.core.protobuf import saved_model_pb2
 from tensorflow.core.protobuf import saver_pb2
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_util
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging
@@ -179,10 +178,10 @@ class SavedModelBuilder(object):
         stored as a collection with key TRAIN_OP_KEY, but not executed.
 
     Raises:
-      TypeError if Train op is not of type `Operation` or a Tensor.
+      TypeError if Train op is not of type `Operation`.
     """
     if train_op is not None:
-      if (not tensor_util.is_tensor(train_op) and
+      if (not isinstance(train_op, ops.Tensor) and
           not isinstance(train_op, ops.Operation)):
         raise TypeError("train_op needs to be a Tensor or Op: %r" % train_op)
       ops.add_to_collection(constants.TRAIN_OP_KEY, train_op)
diff --git a/tensorflow/python/summary/writer/event_file_writer_v2.py b/tensorflow/python/summary/writer/event_file_writer_v2.py
index 262182d3b8..5c66c0f7a8 100644
--- a/tensorflow/python/summary/writer/event_file_writer_v2.py
+++ b/tensorflow/python/summary/writer/event_file_writer_v2.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.client import session as tf_session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -44,11 +43,11 @@ class EventFileWriterV2(object):
     """Creates an `EventFileWriterV2` and an event file to write to.
 
     On construction, this calls `tf.contrib.summary.create_file_writer` within
-    the default graph, which finds and returns a shared summary writer resource
-    for `logdir` if one exists, and creates one if not. Creating the summary
+    the graph from `session.graph` to look up a shared summary writer resource
+    for `logdir` if one exists, and create one if not. Creating the summary
     writer resource in turn creates a new event file in `logdir` to be filled
     with `Event` protocol buffers passed to `add_event`. Graph ops to control
-    this writer resource are added to the default graph during this init call;
+    this writer resource are added to `session.graph` during this init call;
     stateful methods on this class will call `session.run()` on these ops.
 
     Note that because the underlying resource is shared, it is possible that
@@ -62,50 +61,38 @@ class EventFileWriterV2(object):
     no effect.  See `tf.contrib.summary.create_file_writer` for details.
 
     Args:
-      session: A `tf.Session`, or a callable that provides one which will be
-        called on-demand. The session will hold the shared writer resource.
+      session: A `tf.Session`. Session that will hold shared writer resource.
+        The writer ops will be added to session.graph during this init call.
       logdir: A string. Directory where event file will be written.
       max_queue: Integer. Size of the queue for pending events and summaries.
       flush_secs: Number. How often, in seconds, to flush the
         pending events and summaries to disk.
       filename_suffix: A string. Every event file's name is suffixed with
         `filename_suffix`.
-
-    Raises:
-      ValueError: if `session` is not a `tf.Session` or a callable
     """
-    if isinstance(session, tf_session.SessionInterface):
-      self._session = lambda: session
-    elif callable(session):
-      self._session = session
-    else:
-      raise ValueError('session must be tf.Session or callable')
+    self._session = session
     self._logdir = logdir
-    self._initialized = False
     self._closed = False
     if not gfile.IsDirectory(self._logdir):
       gfile.MakeDirs(self._logdir)
 
-    with ops.name_scope('filewriter'):
-      file_writer = summary_ops_v2.create_file_writer(
-          logdir=self._logdir,
-          max_queue=max_queue,
-          flush_millis=flush_secs * 1000,
-          filename_suffix=filename_suffix)
-      with summary_ops_v2.always_record_summaries(), file_writer.as_default():
-        self._event_placeholder = array_ops.placeholder_with_default(
-            constant_op.constant('unused', dtypes.string),
-            shape=[])
-        self._add_event_op = summary_ops_v2.import_event(
-            self._event_placeholder)
-      self._init_op = file_writer.init()
-      self._flush_op = file_writer.flush()
-      self._close_op = file_writer.close()
-
-  def _init_if_needed(self):
-    if not self._initialized:
-      self._session().run(self._init_op)
-      self._initialized = True
+    with self._session.graph.as_default():
+      with ops.name_scope('filewriter'):
+        file_writer = summary_ops_v2.create_file_writer(
+            logdir=self._logdir,
+            max_queue=max_queue,
+            flush_millis=flush_secs * 1000,
+            filename_suffix=filename_suffix)
+        with summary_ops_v2.always_record_summaries(), file_writer.as_default():
+          self._event_placeholder = array_ops.placeholder_with_default(
+              constant_op.constant('unused', dtypes.string),
+              shape=[])
+          self._add_event_op = summary_ops_v2.import_event(
+              self._event_placeholder)
+        self._init_op = file_writer.init()
+        self._flush_op = file_writer.flush()
+        self._close_op = file_writer.close()
+      self._session.run(self._init_op)
 
   def get_logdir(self):
     """Returns the directory where event file will be written."""
@@ -121,6 +108,7 @@ class EventFileWriterV2(object):
     """
     if self._closed:
       self._closed = False
+      self._session.run(self._init_op)
 
   def add_event(self, event):
     """Adds an event to the event file.
@@ -129,9 +117,8 @@ class EventFileWriterV2(object):
       event: An `Event` protocol buffer.
     """
     if not self._closed:
-      self._init_if_needed()
       event_pb = event.SerializeToString()
-      self._session().run(
+      self._session.run(
           self._add_event_op, feed_dict={self._event_placeholder: event_pb})
 
   def flush(self):
@@ -140,9 +127,7 @@ class EventFileWriterV2(object):
     Call this method to make sure that all pending events have been written to
     disk.
     """
-    if not self._closed:
-      self._init_if_needed()
-      self._session().run(self._flush_op)
+    self._session.run(self._flush_op)
 
   def close(self):
     """Flushes the event file to disk and close the file.
@@ -150,8 +135,6 @@ class EventFileWriterV2(object):
     Call this method when you do not need the summary writer anymore.
     """
     if not self._closed:
-      self._init_if_needed()
       self.flush()
-      self._session().run(self._close_op)
+      self._session.run(self._close_op)
       self._closed = True
-      self._initialized = False
diff --git a/tensorflow/python/summary/writer/writer.py b/tensorflow/python/summary/writer/writer.py
index 2a967ae3a5..aca084fc91 100644
--- a/tensorflow/python/summary/writer/writer.py
+++ b/tensorflow/python/summary/writer/writer.py
@@ -332,11 +332,8 @@ class FileWriter(SummaryToEventTransformer):
     the same shared resource name (which by default scoped to the logdir). If
     no such resource exists, one will be created using the remaining arguments
     to this constructor, but if one already exists those arguments are ignored.
-    In either case, ops will be added to the default graph to control the
+    In either case, ops will be added to `session.graph` to control the
     underlying file writer resource. See `tf.contrib.summary` for more details.
-    Instead of an actual `tf.Session`, this argument may also be a callable that
-    provides a `tf.Session` when invoked (e.g. `tf.get_default_session`), which
-    will be called on-demand when a session is needed.
 
     Args:
       logdir: A string. Directory where event file will be written.
@@ -347,8 +344,7 @@ class FileWriter(SummaryToEventTransformer):
       graph_def: DEPRECATED: Use the `graph` argument instead.
       filename_suffix: A string. Every event file's name is suffixed with
         `suffix`.
-      session: A `tf.Session` object or a callable that provides `tf.Session`
-        objects. See details above.
+      session: A `tf.Session` object. See details above.
 
     Raises:
       RuntimeError: If called with eager execution enabled.
diff --git a/tensorflow/python/summary/writer/writer_test.py b/tensorflow/python/summary/writer/writer_test.py
index 3380dea317..dc990c2602 100644
--- a/tensorflow/python/summary/writer/writer_test.py
+++ b/tensorflow/python/summary/writer/writer_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for writer.py."""
+"""Tests for training_coordinator.py."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -574,58 +574,6 @@ class SessionBasedFileWriterTestCase(FileWriterTestCase):
     # No more files
     self.assertRaises(StopIteration, lambda: next(event_paths))
 
-  def testSesssionArgument_callableProvider(self):
-    logdir = self.get_temp_dir()
-    setup_writer = summary_ops_v2.create_file_writer(logdir=logdir)
-    with summary_ops_v2.always_record_summaries(), setup_writer.as_default():
-      summary1 = summary_ops_v2.scalar("one", 0.0, step=0)
-      summary2 = summary_ops_v2.scalar("two", 0.0, step=0)
-    sess1 = session.Session()
-    sess1.run(setup_writer.init())
-    sess1.run(summary1)
-    sess1.run(setup_writer.flush())
-    time.sleep(1.1)  # Ensure filename has a different timestamp
-    sess2 = session.Session()
-    sess2.run(setup_writer.init())
-    sess2.run(summary2)
-    sess2.run(setup_writer.flush())
-
-    # Using get_default_session as session provider should make this FileWriter
-    # send its summaries to the current default session's shared summary writer
-    # resource (initializing it as needed).
-    test_writer = writer.FileWriter(
-        session=ops.get_default_session, logdir=logdir)
-    with sess1.as_default():
-      test_writer.add_summary(self._createTaggedSummary("won"), 1)
-      test_writer.flush()
-    with sess2.as_default():
-      test_writer.add_summary(self._createTaggedSummary("too"), 1)
-      test_writer.flush()
-
-    event_paths = iter(sorted(glob.glob(os.path.join(logdir, "event*"))))
-
-    # First file should have tags "one", "won"
-    events = summary_iterator.summary_iterator(next(event_paths))
-    self.assertEqual("brain.Event:2", next(events).file_version)
-    self.assertEqual("one", next(events).summary.value[0].tag)
-    self.assertEqual("won", next(events).summary.value[0].tag)
-    self.assertRaises(StopIteration, lambda: next(events))
-
-    # Second file should have tags "two", "too"
-    events = summary_iterator.summary_iterator(next(event_paths))
-    self.assertEqual("brain.Event:2", next(events).file_version)
-    self.assertEqual("two", next(events).summary.value[0].tag)
-    self.assertEqual("too", next(events).summary.value[0].tag)
-    self.assertRaises(StopIteration, lambda: next(events))
-
-    # No more files
-    self.assertRaises(StopIteration, lambda: next(event_paths))
-
-  def testSessionArgument_notSessionOrCallable(self):
-    logdir = self.get_temp_dir()
-    self.assertRaises(
-        ValueError, lambda: writer.FileWriter(session=[], logdir=logdir))
-
 
 class FileWriterCacheTest(test.TestCase):
   """FileWriterCache tests."""
diff --git a/tensorflow/python/training/basic_session_run_hooks.py b/tensorflow/python/training/basic_session_run_hooks.py
index b8df7fe51b..b0dd188db1 100644
--- a/tensorflow/python/training/basic_session_run_hooks.py
+++ b/tensorflow/python/training/basic_session_run_hooks.py
@@ -31,13 +31,12 @@ from tensorflow.python.client import timeline
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
-from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.summary.writer import writer
 from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import training_util
 from tensorflow.python.training.session_run_hook import SessionRunArgs
+from tensorflow.python.training.summary_io import SummaryWriterCache
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -423,9 +422,7 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
     self._steps_per_run = steps_per_run
 
   def begin(self):
-    self._summary_writer = writer.FileWriter(
-        self._checkpoint_dir, session=ops.get_default_session,
-        filename_suffix="")
+    self._summary_writer = SummaryWriterCache.get(self._checkpoint_dir)
     self._global_step_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
     if self._global_step_tensor is None:
       raise RuntimeError(
@@ -434,12 +431,10 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
       l.begin()
 
   def after_create_session(self, session, coord):
-    del coord
-    # Ensure summary writer resource has been initialized.
-    session.run(summary_ops_v2.summary_writer_initializer_op())
     global_step = session.run(self._global_step_tensor)
-    # Write graph and saver_def once graph is finalized, which isn't true yet
-    # in begin() since later hooks can still change the graph.
+    # We do write graph and saver_def at the first call of before_run.
+    # We cannot do this in begin, since we let other hooks to change graph and
+    # add variables in begin. Graph is finalized after all begin calls.
     training_util.write_graph(
         ops.get_default_graph().as_graph_def(add_shapes=True),
         self._checkpoint_dir,
@@ -449,9 +444,8 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
     meta_graph_def = meta_graph.create_meta_graph_def(
         graph_def=graph.as_graph_def(add_shapes=True),
         saver_def=saver_def)
-    with ops.default_session(session):
-      self._summary_writer.add_graph(graph)
-      self._summary_writer.add_meta_graph(meta_graph_def)
+    self._summary_writer.add_graph(graph)
+    self._summary_writer.add_meta_graph(meta_graph_def)
     # The checkpoint saved here is the state at step "global_step".
     self._save(session, global_step)
     self._timer.update_last_triggered_step(global_step)
@@ -476,8 +470,6 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
       self._save(session, last_step)
     for l in self._listeners:
       l.end(session, last_step)
-    with ops.default_session(session):
-      self._summary_writer.flush()
 
   def _save(self, session, step):
     """Saves the latest checkpoint, returns should_stop."""
@@ -487,12 +479,10 @@ class CheckpointSaverHook(session_run_hook.SessionRunHook):
       l.before_save(session, step)
 
     self._get_saver().save(session, self._save_path, global_step=step)
-    with ops.default_session(session):
-      self._summary_writer.add_session_log(
-          SessionLog(
-              status=SessionLog.CHECKPOINT, checkpoint_path=self._save_path),
-          step)
-      self._summary_writer.flush()
+    self._summary_writer.add_session_log(
+        SessionLog(
+            status=SessionLog.CHECKPOINT, checkpoint_path=self._save_path),
+        step)
 
     should_stop = False
     for l in self._listeners:
@@ -553,23 +543,13 @@ class StepCounterHook(session_run_hook.SessionRunHook):
 
   def begin(self):
     if self._summary_writer is None and self._output_dir:
-      self._summary_writer = writer.FileWriter(
-          self._output_dir, session=ops.get_default_session,
-          filename_suffix="")
+      self._summary_writer = SummaryWriterCache.get(self._output_dir)
     self._global_step_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
     if self._global_step_tensor is None:
       raise RuntimeError(
           "Global step should be created to use StepCounterHook.")
     self._summary_tag = training_util.get_global_step().op.name + "/sec"
 
-  def after_create_session(self, session, coord):
-    del coord
-    # Reset any stale state in case we're recovering from a previous error.
-    session.run(summary_ops_v2.summary_writer_initializer_op())
-    self._last_global_step = None
-    self._global_step_check_count = 0
-    self._timer.reset()
-
   def before_run(self, run_context):  # pylint: disable=unused-argument
     return SessionRunArgs(self._global_step_tensor)
 
@@ -582,6 +562,8 @@ class StepCounterHook(session_run_hook.SessionRunHook):
     logging.info("%s: %g", self._summary_tag, steps_per_sec)
 
   def after_run(self, run_context, run_values):
+    _ = run_context
+
     stale_global_step = run_values.results
     if self._timer.should_trigger_for_step(
         stale_global_step + self._steps_per_run):
@@ -591,8 +573,7 @@ class StepCounterHook(session_run_hook.SessionRunHook):
         elapsed_time, elapsed_steps = self._timer.update_last_triggered_step(
             global_step)
         if elapsed_time is not None:
-          with ops.default_session(run_context.session):
-            self._log_and_record(elapsed_steps, elapsed_time, global_step)
+          self._log_and_record(elapsed_steps, elapsed_time, global_step)
 
     # Check whether the global step has been increased. Here, we do not use the
     # timer.last_triggered_step as the timer might record a different global
@@ -618,11 +599,6 @@ class StepCounterHook(session_run_hook.SessionRunHook):
 
     self._last_global_step = stale_global_step
 
-  def end(self, session):
-    if self._summary_writer is not None:
-      with ops.default_session(session):
-        self._summary_writer.flush()
-
 
 @tf_export("train.NanLossDuringTrainingError")
 class NanLossDuringTrainingError(RuntimeError):
@@ -667,25 +643,6 @@ class NanTensorHook(session_run_hook.SessionRunHook):
 class SummarySaverHook(session_run_hook.SessionRunHook):
   """Saves summaries every N steps."""
 
-  _SUMMARY_PLACEHOLDER_COLLECTION = "_SUMMARY_SAVER_PLACEHOLDER"
-
-  @classmethod
-  def _set_placeholder(cls, placeholder):
-    """Sets a `tf.placeholder` to be fed by the first SummarySaverHook.
-
-    If a placeholder is provided, the first instance of SummarySaverHook in use
-    will feed it a boolean indicating whether summaries should be written,
-    according to the `save_steps` and `save_secs` parameters of that hook. This
-    makes the placeholder usable with `tf.contrib.summary.record_summaries_if`
-    to control `tf.contrib.summary` summary writing using the same schedule as
-    the `tf.summary` summary writing (which the hook controls directly).
-
-    Args:
-      placeholder: `tf.placeholder` for the first SummarySaverHook to feed
-    """
-    collection = ops.get_collection_ref(cls._SUMMARY_PLACEHOLDER_COLLECTION)
-    collection[:] = [placeholder]
-
   def __init__(self,
                save_steps=None,
                save_secs=None,
@@ -723,82 +680,53 @@ class SummarySaverHook(session_run_hook.SessionRunHook):
     self._scaffold = scaffold
     self._timer = SecondOrStepTimer(every_secs=save_secs,
                                     every_steps=save_steps)
-    self._placeholder = None
     # TODO(mdan): Throw an error if output_dir and summary_writer are None.
 
   def begin(self):
     if self._summary_writer is None and self._output_dir:
-      self._summary_writer = writer.FileWriter(
-          self._output_dir, filename_suffix="", session=ops.get_default_session)
-      # Designate the first SummarySaverHook to call begin() as the "primary"
-      # hook; it will control writing of v2 summaries via a placeholder bool.
-      collection = ops.get_collection_ref(self._SUMMARY_PLACEHOLDER_COLLECTION)
-      if collection:
-        self._placeholder = collection[0]
-        collection[:] = []
-    self._current_step = None
-    self._global_step_tensor = training_util.get_or_create_global_step()
+      self._summary_writer = SummaryWriterCache.get(self._output_dir)
+    self._next_step = None
+    self._global_step_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
     if self._global_step_tensor is None:
       raise RuntimeError(
           "Global step should be created to use SummarySaverHook.")
 
-  def after_create_session(self, session, coord):
-    del coord
-    # Reset any stale state in case we're recovering from a previous error.
-    session.run(summary_ops_v2.summary_writer_initializer_op())
-    self._current_step = None
-    self._timer.reset()
-
-  def before_run(self, run_context):
-    # For the first run, record a SessionLog.START at the pre-run global step.
-    if self._current_step is None:
-      self._current_step = run_context.session.run(self._global_step_tensor)
-      with ops.default_session(run_context.session):
-        self._summary_writer.add_session_log(
-            SessionLog(status=SessionLog.START), self._current_step)
+  def before_run(self, run_context):  # pylint: disable=unused-argument
+    self._request_summary = (
+        self._next_step is None or
+        self._timer.should_trigger_for_step(self._next_step))
     requests = {"global_step": self._global_step_tensor}
-    self._request_summary = self._timer.should_trigger_for_step(
-        self._current_step)
     if self._request_summary:
-      self._timer.update_last_triggered_step(self._current_step)
       if self._get_summary_op() is not None:
         requests["summary"] = self._get_summary_op()
-    feeds = {}
-    if self._placeholder is not None and self._request_summary:
-      feeds[self._placeholder] = self._request_summary
-    args = SessionRunArgs(fetches=requests, feed_dict=feeds)
-    return args
+
+    return SessionRunArgs(requests)
 
   def after_run(self, run_context, run_values):
-    # Collect any legacy v1 summaries to emit.
-    summaries_to_emit = []
-    if self._summary_writer and self._request_summary:
-      for summary in run_values.results.get("summary", []):
-        # Skip None results corresponding to V2 summary operations.
-        if summary is not None:
-          summaries_to_emit.append(summary)
-    # Heuristically estimate current step as possibly-stale value plus one.
+    _ = run_context
+    if not self._summary_writer:
+      return
+
     stale_global_step = run_values.results["global_step"]
-    self._current_step = stale_global_step + 1
-    # Read the actual post-run global step if we need better accuracy because
-    # 1) we will request summaries on the next run (based on estimate now) and
-    #    must ensure we record an accurate "last triggered step" value, or
-    # 2) we have legacy v1 summaries to emit using the post-run step value.
-    # Note: we could have dealt with (1) separately in before_run() but by doing
-    # it here we can consolidate the reads in case both (1) and (2) apply.
-    near_next_trigger = self._timer.should_trigger_for_step(self._current_step)
-    if near_next_trigger or summaries_to_emit:
-      self._current_step = run_context.session.run(self._global_step_tensor)
-    # Emit any legacy v1 summaries.
-    if summaries_to_emit:
-      with ops.default_session(run_context.session):
-        for summary in summaries_to_emit:
-          self._summary_writer.add_summary(summary, self._current_step)
+    global_step = stale_global_step + 1
+    if self._next_step is None or self._request_summary:
+      global_step = run_context.session.run(self._global_step_tensor)
+
+    if self._next_step is None:
+      self._summary_writer.add_session_log(
+          SessionLog(status=SessionLog.START), global_step)
+
+    if self._request_summary:
+      self._timer.update_last_triggered_step(global_step)
+      if "summary" in run_values.results:
+        for summary in run_values.results["summary"]:
+          self._summary_writer.add_summary(summary, global_step)
+
+    self._next_step = global_step + 1
 
   def end(self, session=None):
-    if self._summary_writer and session:
-      with ops.default_session(session):
-        self._summary_writer.flush()
+    if self._summary_writer:
+      self._summary_writer.flush()
 
   def _get_summary_op(self):
     """Fetches the summary op either from self._summary_op or self._scaffold.
@@ -965,27 +893,19 @@ class ProfilerHook(session_run_hook.SessionRunHook):
       show_memory: `bool`, if True, add object snapshot events to the trace
           showing the sizes and lifetimes of tensors.
     """
-    self._output_dir = output_dir
     self._output_file = os.path.join(output_dir, "timeline-{}.json")
+    self._file_writer = SummaryWriterCache.get(output_dir)
     self._show_dataflow = show_dataflow
     self._show_memory = show_memory
     self._timer = SecondOrStepTimer(
         every_secs=save_secs, every_steps=save_steps)
 
   def begin(self):
-    self._file_writer = writer.FileWriter(
-        self._output_dir, filename_suffix="", session=ops.get_default_session)
     self._next_step = None
     self._global_step_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
     if self._global_step_tensor is None:
       raise RuntimeError("Global step should be created to use ProfilerHook.")
 
-  def after_create_session(self, session, coord):
-    del coord
-    # Reset any stale state in case we're recovering from a previous error.
-    session.run(summary_ops_v2.summary_writer_initializer_op())
-    self._timer.reset()
-
   def before_run(self, run_context):
     self._request_summary = (
         self._next_step is None or
@@ -1005,10 +925,8 @@ class ProfilerHook(session_run_hook.SessionRunHook):
       self._save(global_step,
                  self._output_file.format(global_step),
                  run_values.run_metadata.step_stats)
-      with ops.default_session(run_context.session):
-        self._file_writer.add_run_metadata(run_values.run_metadata,
-                                           "step_%d" % global_step,
-                                           global_step=global_step)
+      self._file_writer.add_run_metadata(run_values.run_metadata,
+                                         "step_%d" % global_step)
 
     self._next_step = global_step + 1
 
@@ -1020,10 +938,6 @@ class ProfilerHook(session_run_hook.SessionRunHook):
           trace.generate_chrome_trace_format(
               show_dataflow=self._show_dataflow, show_memory=self._show_memory))
 
-  def end(self, session):
-    with ops.default_session(session):
-      self._file_writer.flush()
-
 
 def _as_graph_element(obj):
   """Retrieves Graph element."""
diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py
index b89167f3c1..b49a871a56 100644
--- a/tensorflow/python/training/basic_session_run_hooks_test.py
+++ b/tensorflow/python/training/basic_session_run_hooks_test.py
@@ -19,10 +19,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import glob
 import os.path
 import shutil
-import sys
 import tempfile
 import threading
 import time
@@ -30,9 +28,6 @@ import time
 from tensorflow.contrib.framework.python.framework import checkpoint_utils
 from tensorflow.contrib.framework.python.ops import variables
 from tensorflow.contrib.testing.python.framework import fake_summary_writer
-from tensorflow.core.framework import graph_pb2
-from tensorflow.core.protobuf import meta_graph_pb2
-from tensorflow.core.util.event_pb2 import SessionLog
 from tensorflow.python.client import session as session_lib
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
@@ -40,12 +35,9 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import state_ops
-from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
@@ -53,27 +45,13 @@ from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.summary import summary as summary_lib
-from tensorflow.python.summary import summary_iterator
 from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import basic_session_run_hooks
 from tensorflow.python.training import monitored_session
-from tensorflow.python.training import saver
 from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import training_util
 
 
-def load_eventfile_contents(directory_path):
-  """Returns the contents of the singular event file in the given directory."""
-  writer_cache.FileWriterCache.clear()
-
-  # Get last Event written.
-  event_paths = glob.glob(os.path.join(directory_path, '*tfevent*'))
-  if len(event_paths) != 1:
-    raise AssertionError('Expected one eventfile, got %s' % str(event_paths))
-  result = list(summary_iterator.summary_iterator(event_paths[0]))
-  return result
-
-
 class MockCheckpointSaverListener(
     basic_session_run_hooks.CheckpointSaverListener):
 
@@ -739,12 +717,11 @@ class CheckpointSaverHookTest(test.TestCase):
                          checkpoint_utils.load_variable(self.model_dir,
                                                         self.global_step.name))
 
-  def _assertCheckpointEvent(self, event, step, checkpoint_path):
-    self.assertEqual(step, event.step)
-    self.assertEqual(SessionLog.CHECKPOINT, event.session_log.status)
-    self.assertEqual(checkpoint_path, event.session_log.checkpoint_path)
-
   def test_summary_writer_defs(self):
+    fake_summary_writer.FakeSummaryWriter.install()
+    writer_cache.FileWriterCache.clear()
+    summary_writer = writer_cache.FileWriterCache.get(self.model_dir)
+
     with self.graph.as_default():
       hook = basic_session_run_hooks.CheckpointSaverHook(
           self.model_dir, save_steps=2, scaffold=self.scaffold)
@@ -753,40 +730,18 @@ class CheckpointSaverHookTest(test.TestCase):
       with session_lib.Session() as sess:
         sess.run(self.scaffold.init_op)
         mon_sess = monitored_session._HookedSession(sess, [hook])
-        hook.after_create_session(sess, None)  # Checkpoint saved at step 0.
-        expected_graph_def = self.graph.as_graph_def(add_shapes=True)
-        expected_meta_graph_def = meta_graph.create_meta_graph_def(
-            graph_def=expected_graph_def,
-            saver_def=self.scaffold.saver.saver_def)
-        mon_sess.run(self.train_op)  # No checkpoint saved at step 1.
-        mon_sess.run(self.train_op)  # Checkpoint saved at step 2.
-        mon_sess.run(self.train_op)  # No checkpoint saved at step 3.
-        hook.end(sess)    # Checkpoint saved at the last step (3)
-    events = iter(load_eventfile_contents(self.model_dir))
-    next(events)  # Skip version event that's always there.
-
-    # Graph.
-    event = next(events)
-    self.assertEqual(0, event.step)
-    actual_graph_def = graph_pb2.GraphDef()
-    actual_graph_def.ParseFromString(event.graph_def)
-    test_util.assert_equal_graph_def(actual_graph_def, expected_graph_def)
-
-    # Metagraph.
-    event = next(events)
-    self.assertEqual(0, event.step)
-    actual_meta_graph_def = meta_graph_pb2.MetaGraphDef()
-    actual_meta_graph_def.ParseFromString(event.meta_graph_def)
-    test_util.assert_meta_graph_protos_equal(
-        self, expected_meta_graph_def, actual_meta_graph_def)
-
-    # Checkpoints.
-    # Strip the "-step#" suffix off the latest checkpoint to get base path.
-    checkpoint_path = saver.latest_checkpoint(self.model_dir).rsplit('-', 1)[0]
-    self._assertCheckpointEvent(next(events), 0, checkpoint_path)
-    self._assertCheckpointEvent(next(events), 2, checkpoint_path)
-    self._assertCheckpointEvent(next(events), 3, checkpoint_path)
-    self.assertRaises(StopIteration, lambda: next(events))  # No more events.
+        hook.after_create_session(sess, None)
+        mon_sess.run(self.train_op)
+      summary_writer.assert_summaries(
+          test_case=self,
+          expected_logdir=self.model_dir,
+          expected_added_meta_graphs=[
+              meta_graph.create_meta_graph_def(
+                  graph_def=self.graph.as_graph_def(add_shapes=True),
+                  saver_def=self.scaffold.saver.saver_def)
+          ])
+
+    fake_summary_writer.FakeSummaryWriter.uninstall()
 
   def test_save_checkpoint_before_first_train_step(self):
     with self.graph.as_default():
@@ -1147,305 +1102,167 @@ class StepCounterHookTest(test.TestCase):
         self.assertEqual('global_step/sec', summary_value.tag)
         self.assertGreater(summary_value.simple_value, 0)
 
-  def test_summary_writer(self):
-    with ops.Graph().as_default(), session_lib.Session() as sess:
-      variables.get_or_create_global_step()
-      train_op = training_util._increment_global_step(1)
-      hook = basic_session_run_hooks.StepCounterHook(
-          output_dir=self.log_dir, every_n_steps=10)
-      hook.begin()
-      sess.run(variables_lib.global_variables_initializer())
-      mon_sess = monitored_session._HookedSession(sess, [hook])
-      for _ in range(30):
-        mon_sess.run(train_op)
-      hook.end(sess)
-    events = iter(load_eventfile_contents(self.log_dir))
-    next(events)  # Skip version event that's always there.
-
-    event = next(events)
-    self.assertEqual(11, event.step)
-    self.assertEqual('global_step/sec', event.summary.value[0].tag)
-    self.assertLess(0, event.summary.value[0].simple_value)
-
-    event = next(events)
-    self.assertEqual(21, event.step)
-    self.assertEqual('global_step/sec', event.summary.value[0].tag)
-    self.assertLess(0, event.summary.value[0].simple_value)
-
-    self.assertRaises(StopIteration, lambda: next(events))  # No more events.
-
 
 class SummarySaverHookTest(test.TestCase):
 
   def setUp(self):
     test.TestCase.setUp(self)
-    self.logdir = self.get_temp_dir()
-    self._create_stable_global_step()
-
-  def _create_stable_global_step(self):
-    """Returns a new ResourceVariable global_step for deterministic tests."""
-    # TODO(nickfelt): remove after standard global_step is a ResourceVariable.
-    with ops.get_default_graph().name_scope(None):
-      return variable_scope.get_variable(
-          ops.GraphKeys.GLOBAL_STEP,
-          shape=[],
-          dtype=dtypes.int64,
-          initializer=init_ops.zeros_initializer(),
-          trainable=False,
-          collections=[
-              ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.GLOBAL_STEP
-          ],
-          # Use a ResourceVariable and set caching_device to make the read
-          # behavior deterministic and well-defined.
-          caching_device='cpu:0',
-          use_resource=True)
+
+    self.log_dir = 'log/dir'
+    self.summary_writer = fake_summary_writer.FakeSummaryWriter(self.log_dir)
+
+    var = variables_lib.Variable(0.0)
+    tensor = state_ops.assign_add(var, 1.0)
+    tensor2 = tensor * 2
+    self.summary_op = summary_lib.scalar('my_summary', tensor)
+    self.summary_op2 = summary_lib.scalar('my_summary2', tensor2)
+
+    variables.get_or_create_global_step()
+    self.train_op = training_util._increment_global_step(1)
 
   def test_raise_when_scaffold_and_summary_op_both_missing(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook()
 
   def test_raise_when_scaffold_and_summary_op_both_present(self):
-    summary_op = summary_lib.merge_all()
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook(
-          scaffold=monitored_session.Scaffold(), summary_op=summary_op)
+          scaffold=monitored_session.Scaffold(), summary_op=self.summary_op)
 
-  def test_raise_when_secs_and_steps_both_missing(self):
+  def test_raise_in_both_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook(
-          save_secs=None, save_steps=None, output_dir=self.logdir)
+          save_secs=10, save_steps=20, summary_writer=self.summary_writer)
 
-  def test_raise_when_secs_and_steps_both_present(self):
+  def test_raise_in_none_secs_and_steps(self):
     with self.assertRaises(ValueError):
       basic_session_run_hooks.SummarySaverHook(
-          save_secs=10, save_steps=20, output_dir=self.logdir)
+          save_secs=None, save_steps=None, summary_writer=self.summary_writer)
 
-  def _makeHook(self, **kwargs):
-    kwargs['output_dir'] = self.logdir
-    kwargs['scaffold'] = monitored_session.Scaffold()
-    return basic_session_run_hooks.SummarySaverHook(**kwargs)
+  def test_save_steps(self):
+    hook = basic_session_run_hooks.SummarySaverHook(
+        save_steps=8,
+        summary_writer=self.summary_writer,
+        summary_op=self.summary_op)
 
-  def _runForSteps(self, hook, steps, loop_body_fn=None):
-    train_op = training_util.get_global_step().assign_add(1)
     with self.test_session() as sess:
       hook.begin()
       sess.run(variables_lib.global_variables_initializer())
-      scaffold = hook._scaffold  # pylint: disable=protected-access
-      if scaffold is not None:
-        scaffold.finalize()
-        sess.run(scaffold.init_op)
       mon_sess = monitored_session._HookedSession(sess, [hook])
-      for _ in range(steps):
-        mon_sess.run(train_op)
-        if loop_body_fn is not None:
-          loop_body_fn()
+      for _ in range(30):
+        mon_sess.run(self.train_op)
       hook.end(sess)
 
-  def _assertSessionEvent(self, event, step, session_status):
-    self.assertEqual(step, event.step)
-    self.assertEqual(session_status, event.session_log.status)
-
-  def _assertSummaryEvent(self, event, step, tag_value_list):
-    self.assertEqual(step, event.step)
-    tag_value_actual_list = [
-        (value.tag, value.simple_value) for value in event.summary.value
-    ]
-    self.assertItemsEqual(tag_value_list, tag_value_actual_list)
-
-  def test_no_summaries(self):
-    hook = self._makeHook(save_steps=1)
-    self._runForSteps(hook, 3)
-    events = iter(load_eventfile_contents(self.logdir))
-    next(events)  # Skip version event that's always there.
-    self._assertSessionEvent(next(events), 0, SessionLog.START)
-    self.assertRaises(StopIteration, lambda: next(events))
-
-  def test_basic_summaries(self):
-    summary_lib.scalar('foo-v1', 1.0)
-    with summary_ops_v2.create_file_writer(self.logdir).as_default():
-      with summary_ops_v2.always_record_summaries():
-        summary_ops_v2.scalar('foo-v2', 2.0)
-    hook = self._makeHook(save_steps=1)
-    self._runForSteps(hook, 3)
-    events = iter(load_eventfile_contents(self.logdir))
-    next(events)  # Skip version event that's always there.
-    self._assertSessionEvent(next(events), 0, SessionLog.START)
-
-    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 1, [('foo-v1', 1.0)])
-
-    self._assertSummaryEvent(next(events), 1, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 2, [('foo-v1', 1.0)])
-
-    self._assertSummaryEvent(next(events), 2, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 3, [('foo-v1', 1.0)])
-    self.assertRaises(StopIteration, lambda: next(events))
+    self.summary_writer.assert_summaries(
+        test_case=self,
+        expected_logdir=self.log_dir,
+        expected_summaries={
+            1: {
+                'my_summary': 1.0
+            },
+            9: {
+                'my_summary': 2.0
+            },
+            17: {
+                'my_summary': 3.0
+            },
+            25: {
+                'my_summary': 4.0
+            },
+        })
 
   def test_multiple_summaries(self):
-    summary_lib.scalar('foo-v1', 1.0)
-    summary_lib.scalar('bar-v1', 10.0)
-    with summary_ops_v2.create_file_writer(self.logdir).as_default():
-      with summary_ops_v2.always_record_summaries():
-        foo = summary_ops_v2.scalar('foo-v2', 2.0)
-        # Ensure deterministic write order
-        with ops.control_dependencies([foo]):
-          summary_ops_v2.scalar('bar-v2', 20.0)
-    hook = self._makeHook(save_steps=1)
-    self._runForSteps(hook, 1)
-    events = iter(load_eventfile_contents(self.logdir))
-    next(events)  # Skip version event that's always there.
-    self._assertSessionEvent(next(events), 0, SessionLog.START)
-    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 0, [('bar-v2', 20.0)])
-    self._assertSummaryEvent(
-        next(events), 1, [('foo-v1', 1.0), ('bar-v1', 10.0)])
-    self.assertRaises(StopIteration, lambda: next(events))
-
-  def test_v2_summaries_only(self):
-    with summary_ops_v2.create_file_writer(self.logdir).as_default():
-      with summary_ops_v2.always_record_summaries():
-        summary_ops_v2.scalar('foo-v2', 2.0)
-    hook = self._makeHook(save_steps=1)
-    self._runForSteps(hook, 1)
-    events = iter(load_eventfile_contents(self.logdir))
-    next(events)  # Skip version event that's always there.
-    self._assertSessionEvent(next(events), 0, SessionLog.START)
-    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
-    self.assertRaises(StopIteration, lambda: next(events))
-
-  def test_v2_summaries_custom_file_writer(self):
-    other_dir = os.path.join(self.logdir, 'other')
-    other_writer = summary_ops_v2.create_file_writer(other_dir)
-    # SummarySaverHook only flushes the writer for logdir; this one needs to be
-    # manually flushed.
-    flush_op = other_writer.flush()
-    with summary_ops_v2.always_record_summaries():
-      with summary_ops_v2.create_file_writer(self.logdir).as_default():
-        summary_ops_v2.scalar('foo-v2', 2.0)
-      with other_writer.as_default():
-        summary_ops_v2.scalar('other-v2', 3.0)
-    hook = self._makeHook(save_steps=1)
-    self._runForSteps(hook, 1)
-    with self.test_session() as sess:
-      sess.run(flush_op)
-
-    events = iter(load_eventfile_contents(self.logdir))
-    next(events)  # Skip version event that's always there.
-    self._assertSessionEvent(next(events), 0, SessionLog.START)
-    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
-    self.assertRaises(StopIteration, lambda: next(events))
-
-    events = iter(load_eventfile_contents(other_dir))
-    next(events)  # Skip version event that's always there.
-    self._assertSummaryEvent(next(events), 0, [('other-v2', 3.0)])
-    self.assertRaises(StopIteration, lambda: next(events))
-
-  def test_save_steps(self):
-    summary_lib.scalar('foo-v1', 1.0)
-    placeholder = array_ops.placeholder_with_default(False, shape=[])
-    with summary_ops_v2.create_file_writer(self.logdir).as_default():
-      with summary_ops_v2.record_summaries_if(placeholder):
-        summary_ops_v2.scalar('foo-v2', 2.0)
-
-    basic_session_run_hooks.SummarySaverHook._set_placeholder(placeholder)
-    hook = self._makeHook(save_steps=8)
-    self._runForSteps(hook, 30)
-
-    events = load_eventfile_contents(self.logdir)
-    print('TEST SAVE STEPS EVENTS', str(events), file=sys.stderr)
-    events = iter(events)
-    next(events)  # Skip version event that's always there.
-    self._assertSessionEvent(next(events), 0, SessionLog.START)
-
-    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 1, [('foo-v1', 1.0)])
-
-    self._assertSummaryEvent(next(events), 8, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 9, [('foo-v1', 1.0)])
-
-    self._assertSummaryEvent(next(events), 16, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 17, [('foo-v1', 1.0)])
-
-    self._assertSummaryEvent(next(events), 24, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 25, [('foo-v1', 1.0)])
-    self.assertRaises(StopIteration, lambda: next(events))
-
-  @test.mock.patch.object(time, 'time')
-  def test_save_secs_saving_once_every_step(self, mock_time):
-    mock_time.return_value = 1000.0
-    summary_lib.scalar('foo-v1', 1.0)
-    placeholder = array_ops.placeholder_with_default(False, shape=[])
-    with summary_ops_v2.create_file_writer(self.logdir).as_default():
-      with summary_ops_v2.record_summaries_if(placeholder):
-        summary_ops_v2.scalar('foo-v2', 2.0)
-
-    basic_session_run_hooks.SummarySaverHook._set_placeholder(placeholder)
-    hook = self._makeHook(save_secs=0.5)
-    def fake_sleep():
-      mock_time.return_value += 0.5
-    self._runForSteps(hook, 4, fake_sleep)
+    hook = basic_session_run_hooks.SummarySaverHook(
+        save_steps=8,
+        summary_writer=self.summary_writer,
+        summary_op=[self.summary_op, self.summary_op2])
 
-    events = iter(load_eventfile_contents(self.logdir))
-    next(events)  # Skip version event that's always there.
-    self._assertSessionEvent(next(events), 0, SessionLog.START)
+    with self.test_session() as sess:
+      hook.begin()
+      sess.run(variables_lib.global_variables_initializer())
+      mon_sess = monitored_session._HookedSession(sess, [hook])
+      for _ in range(10):
+        mon_sess.run(self.train_op)
+      hook.end(sess)
 
-    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 1, [('foo-v1', 1.0)])
+    self.summary_writer.assert_summaries(
+        test_case=self,
+        expected_logdir=self.log_dir,
+        expected_summaries={
+            1: {
+                'my_summary': 1.0,
+                'my_summary2': 2.0
+            },
+            9: {
+                'my_summary': 2.0,
+                'my_summary2': 4.0
+            },
+        })
 
-    self._assertSummaryEvent(next(events), 1, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 2, [('foo-v1', 1.0)])
+  def test_save_secs_saving_once_every_step(self):
+    hook = basic_session_run_hooks.SummarySaverHook(
+        save_secs=0.5,
+        summary_writer=self.summary_writer,
+        summary_op=self.summary_op)
 
-    self._assertSummaryEvent(next(events), 2, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 3, [('foo-v1', 1.0)])
+    with self.test_session() as sess:
+      hook.begin()
+      sess.run(variables_lib.global_variables_initializer())
+      mon_sess = monitored_session._HookedSession(sess, [hook])
+      for _ in range(4):
+        mon_sess.run(self.train_op)
+        time.sleep(0.5)
+      hook.end(sess)
 
-    self._assertSummaryEvent(next(events), 3, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 4, [('foo-v1', 1.0)])
-    self.assertRaises(StopIteration, lambda: next(events))
+    self.summary_writer.assert_summaries(
+        test_case=self,
+        expected_logdir=self.log_dir,
+        expected_summaries={
+            1: {
+                'my_summary': 1.0
+            },
+            2: {
+                'my_summary': 2.0
+            },
+            3: {
+                'my_summary': 3.0
+            },
+            4: {
+                'my_summary': 4.0
+            },
+        })
 
   @test.mock.patch.object(time, 'time')
   def test_save_secs_saving_once_every_three_steps(self, mock_time):
-    mock_time.return_value = 1000.0
-    summary_lib.scalar('foo-v1', 1.0)
-    placeholder = array_ops.placeholder_with_default(False, shape=[])
-    with summary_ops_v2.create_file_writer(self.logdir).as_default():
-      with summary_ops_v2.record_summaries_if(placeholder):
-        summary_ops_v2.scalar('foo-v2', 2.0)
-
-    basic_session_run_hooks.SummarySaverHook._set_placeholder(placeholder)
-    hook = self._makeHook(save_secs=9)
-    def fake_sleep():
-      mock_time.return_value += 3.1
-    self._runForSteps(hook, 8, fake_sleep)
-
-    events = iter(load_eventfile_contents(self.logdir))
-    next(events)  # Skip version event that's always there.
-    self._assertSessionEvent(next(events), 0, SessionLog.START)
-
-    # 24.8 seconds passed (3.1*8), it saves every 9 seconds starting from first:
-    self._assertSummaryEvent(next(events), 0, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 1, [('foo-v1', 1.0)])
-
-    self._assertSummaryEvent(next(events), 3, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 4, [('foo-v1', 1.0)])
+    mock_time.return_value = 1484695987.209386
+    hook = basic_session_run_hooks.SummarySaverHook(
+        save_secs=9.,
+        summary_writer=self.summary_writer,
+        summary_op=self.summary_op)
 
-    self._assertSummaryEvent(next(events), 6, [('foo-v2', 2.0)])
-    self._assertSummaryEvent(next(events), 7, [('foo-v1', 1.0)])
-    self.assertRaises(StopIteration, lambda: next(events))
+    with self.test_session() as sess:
+      hook.begin()
+      sess.run(variables_lib.global_variables_initializer())
+      mon_sess = monitored_session._HookedSession(sess, [hook])
+      for _ in range(8):
+        mon_sess.run(self.train_op)
+        mock_time.return_value += 3.1
+      hook.end(sess)
 
-  def test_explicit_summary_writer_and_op(self):
-    summary_writer = fake_summary_writer.FakeSummaryWriter(self.logdir)
-    hook = basic_session_run_hooks.SummarySaverHook(
-        save_steps=1,
-        summary_writer=summary_writer,
-        summary_op=summary_lib.scalar('foo-v1', 1.0))
-    self._runForSteps(hook, 3)
-    summary_writer.assert_summaries(
+    # 24.8 seconds passed (3.1*8), it saves every 9 seconds starting from first:
+    self.summary_writer.assert_summaries(
         test_case=self,
-        expected_logdir=self.logdir,
+        expected_logdir=self.log_dir,
         expected_summaries={
-            1: {'foo-v1': 1.0},
-            2: {'foo-v1': 1.0},
-            3: {'foo-v1': 1.0},
+            1: {
+                'my_summary': 1.0
+            },
+            4: {
+                'my_summary': 2.0
+            },
+            7: {
+                'my_summary': 3.0
+            },
         })
 
 
@@ -1701,23 +1518,18 @@ class ProfilerHookTest(test.TestCase):
         sess.run(self.train_op)  # Saved.
         self.assertEqual(3, self._count_timeline_files())
 
-  def test_run_metadata_summary_saving(self):
+  def test_run_metadata_saves_in_first_step(self):
+    writer_cache.FileWriterCache.clear()
+    fake_summary_writer.FakeSummaryWriter.install()
+    fake_writer = writer_cache.FileWriterCache.get(self.output_dir)
     with self.graph.as_default():
       hook = basic_session_run_hooks.ProfilerHook(
-          save_steps=2, output_dir=self.output_dir)
+          save_secs=2, output_dir=self.output_dir)
       with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
         sess.run(self.train_op)  # Saved.
-        sess.run(self.train_op)  # Not saved.
-        sess.run(self.train_op)  # Saved.
-    events = iter(load_eventfile_contents(self.output_dir))
-    next(events)  # Skip version event that's always there.
-    event = next(events)
-    self.assertEqual(1, event.step)
-    self.assertEqual('step_1', event.tagged_run_metadata.tag)
-    event = next(events)
-    self.assertEqual(3, event.step)
-    self.assertEqual('step_3', event.tagged_run_metadata.tag)
-    self.assertRaises(StopIteration, lambda: next(events))  # No more events.
+        self.assertEqual(
+            list(fake_writer._added_run_metadata.keys()), ['step_1'])
+    fake_summary_writer.FakeSummaryWriter.uninstall()
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index 8a4ca04b1e..7b06bffa4b 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -31,7 +31,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import resources
-from tensorflow.python.ops import summary_ops_v2
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.summary import summary
@@ -205,17 +204,13 @@ class Scaffold(object):
           'local_init_op', ops.GraphKeys.LOCAL_INIT_OP,
           Scaffold.default_local_init_op)
     if self._summary_op is None:
-      def default_summary_op():
-        v1_op = summary.merge_all()
-        v2_ops = summary_ops_v2.all_summary_ops() or []
-        if v1_op is not None:
-          return control_flow_ops.with_dependencies(v2_ops, v1_op)
-        return control_flow_ops.group(v2_ops) if v2_ops else None
       self._summary_op = Scaffold.get_or_default('summary_op',
                                                  ops.GraphKeys.SUMMARY_OP,
-                                                 default_summary_op)
+                                                 summary.merge_all)
+    # pylint: disable=g-long-lambda
     if self._saver is None:
       self._saver = training_saver._get_saver_or_default()  # pylint: disable=protected-access
+    # pylint: enable=g-long-lambda
     self._saver.build()
 
     ops.get_default_graph().finalize()
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index b9d42b034e..f75db08059 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -611,8 +611,10 @@ class Optimizer(
             if isinstance(global_step, resource_variable_ops.ResourceVariable):
               # TODO(apassos): the implicit read in assign_add is slow; consider
               # making it less so.
-              apply_updates = global_step.assign_add(
-                  1, name=name, read_value=False)
+              apply_updates = resource_variable_ops.assign_add_variable_op(
+                  global_step.handle,
+                  ops.convert_to_tensor(1, dtype=global_step.dtype),
+                  name=name)
             else:
               apply_updates = state_ops.assign_add(global_step, 1, name=name)
 
-- 
cgit v1.2.3


From 3ca47448added403f065b43395f38c538c1602f9 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne 
Date: Tue, 24 Jul 2018 13:00:41 -0700
Subject: Automated rollback of commit 09c4c387913c86247121589caa7fb2e85351fa58

PiperOrigin-RevId: 205877002
---
 tensorflow/core/BUILD                              | 11 -----
 tensorflow/core/common_runtime/gpu/gpu_device.cc   | 51 +---------------------
 tensorflow/core/common_runtime/gpu/gpu_device.h    | 11 -----
 .../gpu/gpu_device_kernel_check.cu.cc              | 37 ----------------
 .../common_runtime/gpu/gpu_device_kernel_check.h   | 32 --------------
 5 files changed, 1 insertion(+), 141 deletions(-)
 delete mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
 delete mode 100644 tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index a960736295..84555b60da 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -90,7 +90,6 @@ load(
     "tf_genrule_cmd_append_to_srcs",
     "tf_opts_nortti_if_android",
     "tf_features_nomodules_if_android",
-    "tf_gpu_kernel_library",
 )
 load("//tensorflow:tensorflow.bzl", "tf_cc_test_mkl")
 load("//tensorflow:tensorflow.bzl", "tf_cc_test_gpu")
@@ -2950,15 +2949,6 @@ cc_library(
     ],
 )
 
-tf_gpu_kernel_library(
-    name = "gpu_device_kernel_check",
-    srcs = ["common_runtime/gpu/gpu_device_kernel_check.cu.cc"],
-    hdrs = ["common_runtime/gpu/gpu_device_kernel_check.h"],
-    deps = [
-        "//tensorflow/core:stream_executor",
-    ],
-)
-
 GPU_RUNTIME_HEADERS = [
     "common_runtime/gpu/cuda_host_allocator.h",
     "common_runtime/gpu/gpu_bfc_allocator.h",
@@ -2997,7 +2987,6 @@ tf_cuda_library(
         ":core_cpu_lib",
         ":framework",
         ":framework_internal",
-        ":gpu_device_kernel_check",
         ":gpu_id_impl",
         ":gpu_init_impl",
         ":gpu_lib",
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index fbe158c777..3292ef2f62 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -31,7 +31,6 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/common_runtime/device_factory.h"
-#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id_manager.h"
@@ -378,7 +377,7 @@ Status BaseGPUDevice::Init(const SessionOptions& options) {
     }
   }
 
-  return CheckGPU();
+  return Status::OK();
 }
 
 bool BaseGPUDevice::RequiresRecordingAccessedTensors() const {
@@ -895,54 +894,6 @@ Allocator* BaseGPUDevice::GetScopedAllocator(AllocatorAttributes attr,
   return gpu_allocator_;
 }
 
-Status BaseGPUDevice::CheckGPU() {
-  se::Stream* stream = tensorflow_gpu_device_info()->stream;
-  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
-  Tensor device_tensor(gpu_allocator_, DT_FLOAT, {});
-  if (!device_tensor.IsInitialized()) {
-    return errors::ResourceExhausted("Failed to allocate ", sizeof(float),
-                                     " bytes on the GPU for initialization "
-                                     "checks");
-  }
-  float* val_dev = device_tensor.scalar().data();
-  const cudaStream_t cu_stream = *reinterpret_cast(
-      stream->implementation()->GpuStreamMemberHack());
-  {
-    se::cuda::ScopedActivateExecutorContext scoped_activation{stream->parent()};
-    run_test_kernel(val_dev, cu_stream);
-    // We have to use the CUDA runtime function cudaPeekAtLastError here,
-    // because 'stream' does not provide a way to check if a kernel launch
-    // succeeds. Calling 'stream->BlockHostUntilDone()', which internally calls
-    // 'cuCtxSynchronize()', does not catch all kernel launch errors.
-    cudaError_t cuda_error = cudaPeekAtLastError();
-    if (cuda_error == cudaSuccess) {
-      cuda_error = cudaDeviceSynchronize();
-    }
-    TF_RETURN_IF_ERROR(CudaErrorToStatus(cuda_error, *stream));
-  }
-
-  float val_host = 0.;
-  stream->ThenMemcpy(&val_host, se::DeviceMemoryBase(val_dev, sizeof(float)),
-                     sizeof(float));
-  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
-  if (val_host != 12345.) {
-    return errors::Internal(
-        "GPU kernel for initialization returned wrong value: ", val_host);
-  }
-  return Status::OK();
-}
-
-Status BaseGPUDevice::CudaErrorToStatus(cudaError_t cuda_error,
-                                        const se::Stream& stream) {
-  if (cuda_error != cudaSuccess) {
-    return errors::Internal(
-        "Failed to run GPU kernel for the initialization check. Received "
-        "error ",
-        cudaGetErrorName(cuda_error), " after running GPU kernel.");
-  }
-  return Status::OK();
-}
-
 const int BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength = 1000;
 const int BaseGPUDeviceFactory::InterconnectMap::kStreamExecutorStrength = 1;
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index d02901a7ae..56d03d7a8c 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -26,7 +26,6 @@ limitations under the License.
 #include 
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "cuda/include/cuda_runtime_api.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_id.h"
@@ -116,12 +115,6 @@ class BaseGPUDevice : public LocalDevice {
   se::StreamExecutor* executor_;  // not owned
   std::unique_ptr scoped_allocator_mgr_;
 
-  // Returns a Status corresponding to a cudaError_t. The CUDA error must have
-  // been obtained from a CUDA kernel launch used to check if the GPU is
-  // initialized properly.
-  virtual Status CudaErrorToStatus(cudaError_t cuda_error,
-                                   const se::Stream& stream);
-
  private:
   struct StreamGroup {
     se::Stream* compute = nullptr;
@@ -158,10 +151,6 @@ class BaseGPUDevice : public LocalDevice {
   Status MaybeCopyTensorToGPU(const AllocatorAttributes& alloc_attrs,
                               const Tensor& from, Tensor* to,
                               StatusCallback done);
-
-  // Checks that the GPU is capable of doing work, by running a test kernel on
-  // it.
-  Status CheckGPU();
 };
 
 class BaseGPUDeviceFactory : public DeviceFactory {
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
deleted file mode 100644
index 017565195b..0000000000
--- a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.cu.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#if GOOGLE_CUDA
-
-#include "tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h"
-#include "tensorflow/stream_executor/cuda/cuda_activation.h"
-
-namespace {
-__global__ void test_kernel(float* val) {
-  if (blockIdx.x == 0 && threadIdx.x == 0) {
-    (*val) = 12345.;
-  }
-}
-}  // namespace
-
-namespace tensorflow {
-
-void run_test_kernel(float* val, cudaStream_t cu_stream) {
-  test_kernel<<<1, 1, 0, cu_stream>>>(val);
-}
-
-}  // namespace tensorflow
-
-#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h b/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
deleted file mode 100644
index 064fb7a49f..0000000000
--- a/tensorflow/core/common_runtime/gpu/gpu_device_kernel_check.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-
-#if GOOGLE_CUDA
-
-#include "tensorflow/core/platform/stream_executor.h"
-
-namespace tensorflow {
-
-// Runs a GPU kernel to test that it functions correctly. Sets 'val' to 12345.
-void run_test_kernel(float* val, cudaStream_t cu_stream);
-
-}  // namespace tensorflow
-
-#endif  // GOOGLE_CUDA
-
-#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_KERNEL_CHECK_H_
-- 
cgit v1.2.3


From bb384118db531a7951735dcdc809b5735bc02a76 Mon Sep 17 00:00:00 2001
From: Yifei Feng 
Date: Tue, 24 Jul 2018 13:12:54 -0700
Subject: Upgrade bazel to 0.15.0.

PiperOrigin-RevId: 205878953
---
 WORKSPACE                                                      | 2 +-
 configure.py                                                   | 2 +-
 tensorflow/tools/ci_build/ci_sanity.sh                         | 2 +-
 tensorflow/tools/ci_build/install/install_bazel.sh             | 2 +-
 tensorflow/tools/ci_build/install/install_bazel_from_source.sh | 2 +-
 tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh      | 8 ++++----
 tensorflow/tools/docker/Dockerfile.devel                       | 2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu                   | 2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7      | 2 +-
 9 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index e7cf23a159..17961829a6 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -18,7 +18,7 @@ closure_repositories()
 # files, in case the parsing of those build files depends on the bazel
 # version we require here.
 load("//tensorflow:version_check.bzl", "check_bazel_version_at_least")
-check_bazel_version_at_least("0.13.0")
+check_bazel_version_at_least("0.15.0")
 
 load("//tensorflow:workspace.bzl", "tf_workspace")
 
diff --git a/configure.py b/configure.py
index 1df7bc736f..f97bf8a668 100644
--- a/configure.py
+++ b/configure.py
@@ -1451,7 +1451,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.13.0')
+  check_bazel_version('0.15.0')
 
   reset_tf_configure_bazelrc(args.workspace)
   cleanup_makefile()
diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index db37edf809..866fe95d2b 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -354,7 +354,7 @@ do_external_licenses_check(){
 
   # Whitelist
   echo ${EXTRA_LICENSE_FILE}
-  grep -e "@bazel_tools//src" -e "@bazel_tools//tools/" -e "@com_google_absl//" -e "//external" -e "@local" -e "@com_github_googlecloudplatform_google_cloud_cpp//" -v ${EXTRA_LICENSES_FILE} > temp.txt
+  grep -e "@bazel_tools//src" -e "@bazel_tools//tools/" -e "@com_google_absl//" -e "//external" -e "@local" -e "@com_github_googlecloudplatform_google_cloud_cpp//" -e "@embedded_jdk//" -v ${EXTRA_LICENSES_FILE} > temp.txt
   mv temp.txt ${EXTRA_LICENSES_FILE}
 
 
diff --git a/tensorflow/tools/ci_build/install/install_bazel.sh b/tensorflow/tools/ci_build/install/install_bazel.sh
index adbff8f6ef..e284401b8a 100755
--- a/tensorflow/tools/ci_build/install/install_bazel.sh
+++ b/tensorflow/tools/ci_build/install/install_bazel.sh
@@ -15,7 +15,7 @@
 # ==============================================================================
 
 # Select bazel version.
-BAZEL_VERSION="0.14.1"
+BAZEL_VERSION="0.15.0"
 
 set +e
 local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}')
diff --git a/tensorflow/tools/ci_build/install/install_bazel_from_source.sh b/tensorflow/tools/ci_build/install/install_bazel_from_source.sh
index 9d24b3e421..87be81577d 100755
--- a/tensorflow/tools/ci_build/install/install_bazel_from_source.sh
+++ b/tensorflow/tools/ci_build/install/install_bazel_from_source.sh
@@ -18,7 +18,7 @@
 # It will compile bazel from source and install it in /usr/local/bin
 
 # Select bazel version.
-BAZEL_VERSION="0.14.1"
+BAZEL_VERSION="0.15.0"
 
 set +e
 local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}')
diff --git a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
index c03cbd9c66..0482cf619a 100644
--- a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
+++ b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
@@ -33,10 +33,10 @@ function set_remote_cache_options {
   echo "build --tls_enabled=true" >> "${TMP_BAZELRC}"
   echo "build --remote_timeout=3600" >> "${TMP_BAZELRC}"
   echo "build --auth_enabled=true" >> "${TMP_BAZELRC}"
-  echo "build --spawn_strategy=remote" >> "${TMP_BAZELRC}"
-  echo "build --strategy=Javac=remote" >> "${TMP_BAZELRC}"
-  echo "build --strategy=Closure=remote" >> "${TMP_BAZELRC}"
-  echo "build --genrule_strategy=remote" >> "${TMP_BAZELRC}"
+  echo "build --spawn_strategy=standalone" >> "${TMP_BAZELRC}"
+  echo "build --strategy=Javac=standalone" >> "${TMP_BAZELRC}"
+  echo "build --strategy=Closure=standalone" >> "${TMP_BAZELRC}"
+  echo "build --genrule_strategy=standalone" >> "${TMP_BAZELRC}"
   echo "build --google_credentials=$GOOGLE_CLOUD_CREDENTIAL" >> "${TMP_BAZELRC}"
 }
 
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index fd94d64268..f7fe4119da 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -63,7 +63,7 @@ RUN echo "startup --batch" >>/etc/bazel.bazelrc
 RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
     >>/etc/bazel.bazelrc
 # Install the most recent bazel release.
-ENV BAZEL_VERSION 0.14.1
+ENV BAZEL_VERSION 0.15.0
 WORKDIR /
 RUN mkdir /bazel && \
     cd /bazel && \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index a5560e459c..340f96df48 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -79,7 +79,7 @@ RUN echo "startup --batch" >>/etc/bazel.bazelrc
 RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
     >>/etc/bazel.bazelrc
 # Install the most recent bazel release.
-ENV BAZEL_VERSION 0.14.1
+ENV BAZEL_VERSION 0.15.0
 WORKDIR /
 RUN mkdir /bazel && \
     cd /bazel && \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
index 3bedc8cf34..30bc2d2806 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
@@ -4,7 +4,7 @@ LABEL maintainer="Gunhan Gulsoy "
 
 # It is possible to override these for releases.
 ARG TF_BRANCH=master
-ARG BAZEL_VERSION=0.5.4
+ARG BAZEL_VERSION=0.15.0
 ARG TF_AVAILABLE_CPUS=32
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-- 
cgit v1.2.3


From e2f8d4a8bdfc4e3970cacc89a6b184297205a1cc Mon Sep 17 00:00:00 2001
From: Smit Hinsu 
Date: Tue, 24 Jul 2018 13:16:31 -0700
Subject: Add data format as a parameter in ConvParameters to support NHWC
 format

PiperOrigin-RevId: 205879506
---
 .../kernels/fused_conv2d_bias_activation_op.cc     |  3 ++
 .../fused_conv/kernels/fused_conv_ops_gpu.h        | 12 ++---
 tensorflow/core/kernels/conv_grad_filter_ops.cc    |  1 +
 tensorflow/core/kernels/conv_grad_input_ops.cc     |  1 +
 tensorflow/core/kernels/conv_grad_ops_3d.cc        |  2 +
 tensorflow/core/kernels/conv_ops.cc                |  1 +
 tensorflow/core/kernels/conv_ops_3d.cc             |  1 +
 tensorflow/core/kernels/conv_ops_gpu.h             | 20 ++++---
 tensorflow/core/kernels/conv_ops_test.cc           | 62 +++++++++++-----------
 9 files changed, 60 insertions(+), 43 deletions(-)

diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
index 4554a3d89a..0ccb4583ab 100644
--- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
+++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
@@ -443,6 +443,8 @@ void LaunchFusedConv2DBiasActivationOp::
                                          : dnn::DataLayout::kBatchDepthYX;
   constexpr auto filter_layout = is_int8x4 ? dnn::FilterLayout::kOutputInputYX4
                                            : dnn::FilterLayout::kOutputInputYX;
+  constexpr auto compute_data_format =
+      is_int8x4 ? FORMAT_NCHW_VECT_C : FORMAT_NCHW;
 
   dnn::BatchDescriptor conv_input_desc;
   conv_input_desc.set_count(batch_size)
@@ -529,6 +531,7 @@ void LaunchFusedConv2DBiasActivationOp::
       batch_size,
       conv_input_depth,
       {{conv_input_rows, conv_input_cols}},
+      compute_data_format,
       output_depth,
       {{filter_rows, filter_cols}},
       // TODO(yangzihao): Add support for arbitrary dilations for fused conv.
diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h b/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h
index ba52697679..b9c131a2e9 100644
--- a/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h
+++ b/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h
@@ -29,13 +29,13 @@ namespace tensorflow {
 class FusedConvParameters : public ConvParameters {
  public:
   FusedConvParameters(int64 batch, int64 in_depths, const SpatialArray& in,
-                      int64 out_depths, const SpatialArray& filter,
-                      const SpatialArray& dilation, const SpatialArray& stride,
-                      const SpatialArray& padding, DataType dtype,
-                      int device_id, bool has_side_input,
+                      TensorFormat data_format, int64 out_depths,
+                      const SpatialArray& filter, const SpatialArray& dilation,
+                      const SpatialArray& stride, const SpatialArray& padding,
+                      DataType dtype, int device_id, bool has_side_input,
                       ActivationMode activation_mode)
-      : ConvParameters(batch, in_depths, in, out_depths, filter, dilation,
-                       stride, padding, dtype, device_id),
+      : ConvParameters(batch, in_depths, in, data_format, out_depths, filter,
+                       dilation, stride, padding, dtype, device_id),
         activation_mode_(activation_mode),
         has_side_input_(has_side_input) {
     hash_code_ = Hash64Combine(hash_code_, has_side_input);
diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc
index aca75176a5..63b1bcda43 100644
--- a/tensorflow/core/kernels/conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc
@@ -909,6 +909,7 @@ void LaunchConv2DBackpropFilterOp::operator()(
       dims.in_depth,                       // in_depths
       {{input_desc.height(),               // in_rows
         input_desc.width()}},              // in_cols
+      FORMAT_NCHW,                         // compute_data_format
       dims.out_depth,                      // out_depths
       {{dims.spatial_dims[0].filter_size,  // filter_rows
         dims.spatial_dims[1].filter_size,  // filter_cols
diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc
index 63a775afa8..d664a11e73 100644
--- a/tensorflow/core/kernels/conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_input_ops.cc
@@ -957,6 +957,7 @@ void LaunchConv2DBackpropInputOp::operator()(
       dims.in_depth,                       // in_depths
       {{input_desc.height(),               // in_rows
         input_desc.width()}},              // in_cols
+      FORMAT_NCHW,                         // compute_data_format
       dims.out_depth,                      // out_depths
       {{dims.spatial_dims[0].filter_size,  // filter_rows
         dims.spatial_dims[1].filter_size,  // filter_cols
diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc
index 980b1063de..15f1bf9aba 100644
--- a/tensorflow/core/kernels/conv_grad_ops_3d.cc
+++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc
@@ -716,6 +716,7 @@ class Conv3DBackpropInputOp : public OpKernel {
         batch,
         in_depth,
         {{input_size[0], input_size[1], input_size[2]}},
+        FORMAT_NCHW,
         out_depth,
         {{filter_size[0], filter_size[1], filter_size[2]}},
         {{dilations[0], dilations[1], dilations[2]}},
@@ -1112,6 +1113,7 @@ class Conv3DBackpropFilterOp : public OpKernel {
         batch,
         in_depth,
         {{input_size[0], input_size[1], input_size[2]}},
+        FORMAT_NCHW,
         out_depth,
         {{filter_size[0], filter_size[1], filter_size[2]}},
         {{dilations[0], dilations[1], dilations[2]}},
diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc
index 3b9886eece..ef692418d6 100644
--- a/tensorflow/core/kernels/conv_ops.cc
+++ b/tensorflow/core/kernels/conv_ops.cc
@@ -713,6 +713,7 @@ void LaunchConv2DOp::operator()(
       in_depths,         // in_depths
       {{in_rows,         // in_rows
         in_cols}},       // in_cols
+      FORMAT_NCHW,       // compute_data_format
       out_depths,        // out_depths
       {{patch_rows,      // filter_rows
         patch_cols,      // filter_cols
diff --git a/tensorflow/core/kernels/conv_ops_3d.cc b/tensorflow/core/kernels/conv_ops_3d.cc
index 9ec16be67d..a1eed4e68c 100644
--- a/tensorflow/core/kernels/conv_ops_3d.cc
+++ b/tensorflow/core/kernels/conv_ops_3d.cc
@@ -415,6 +415,7 @@ struct LaunchConvOp {
         in_batch,
         in_depth,
         {{in_planes, in_rows, in_cols}},
+        FORMAT_NCHW,
         out_depth,
         {{filter_planes, filter_rows, filter_cols}},
         {{dilations[0], dilations[1], dilations[2]}},
diff --git a/tensorflow/core/kernels/conv_ops_gpu.h b/tensorflow/core/kernels/conv_ops_gpu.h
index d2c8020bb6..afc611f277 100644
--- a/tensorflow/core/kernels/conv_ops_gpu.h
+++ b/tensorflow/core/kernels/conv_ops_gpu.h
@@ -85,13 +85,15 @@ class ConvParameters {
  public:
   using SpatialArray = gtl::InlinedVector;
   ConvParameters(int64 batch, int64 in_depths, const SpatialArray& in,
-                 int64 out_depths, const SpatialArray& filter,
-                 const SpatialArray& dilation, const SpatialArray& stride,
-                 const SpatialArray& padding, DataType dtype, int device_id)
+                 TensorFormat data_format, int64 out_depths,
+                 const SpatialArray& filter, const SpatialArray& dilation,
+                 const SpatialArray& stride, const SpatialArray& padding,
+                 DataType dtype, int device_id)
       : batch_(batch),
         in_depths_(in_depths),
         out_depths_(out_depths),
         in_(in),
+        data_format_(data_format),
         filter_(filter),
         dilation_(dilation),
         stride_(stride),
@@ -101,6 +103,7 @@ class ConvParameters {
     hash_code_ = batch;
     hash_code_ = Hash64Combine(hash_code_, in_depths);
     for (int64 val : in) hash_code_ = Hash64Combine(hash_code_, val);
+    hash_code_ = Hash64Combine(hash_code_, data_format);
     hash_code_ = Hash64Combine(hash_code_, out_depths);
     for (int64 val : filter) hash_code_ = Hash64Combine(hash_code_, val);
     for (int64 val : dilation) hash_code_ = Hash64Combine(hash_code_, val);
@@ -123,6 +126,7 @@ class ConvParameters {
     return strings::StrCat(
         batch_, ", ", in_depths_, ", ",
         "(", str_util::Join(in_, ", "), "), ",
+        ::tensorflow::ToString(data_format_), ", ",
         out_depths_, ", ",
         "(", str_util::Join(filter_, ", "), "), ",
         "(", str_util::Join(dilation_, ", "), "), ",
@@ -148,12 +152,13 @@ class ConvParameters {
 
  protected:
   using ParameterDataType =
-      std::tuple;
+      std::tuple;
 
   ParameterDataType get_data_as_tuple() const {
-    return std::make_tuple(batch_, in_depths_, in_, out_depths_, filter_,
-                           dilation_, stride_, padding_, dtype_, device_id_);
+    return std::make_tuple(batch_, in_depths_, in_, data_format_, out_depths_,
+                           filter_, dilation_, stride_, padding_, dtype_,
+                           device_id_);
   }
 
   uint64 hash_code_;
@@ -178,6 +183,7 @@ class ConvParameters {
   int64 in_depths_;
   int64 out_depths_;
   SpatialArray in_;
+  TensorFormat data_format_;
   SpatialArray filter_;
   SpatialArray dilation_;
   SpatialArray stride_;
diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index 4f9a96ce17..c281153795 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -44,41 +44,43 @@ struct ConvParametersPeer {
 
 TEST(ConvParameters, WinogradNonfusedAlgoSize) {
   ConvParametersPeer conv_params_small = {{
-      1,         // batch
-      32,        // in_depths
-      {{300,     // in_rows
-        300}},   // in_cols
-      128,       // out_depths
-      {{3,       // filter_rows
-        3}},     // filter_cols
-      {{1,       // dilation_rows
-        1}},     // dilation_cols
-      {{1,       // stride_rows
-        1}},     // stride_cols
-      {{0,       // padding_rows
-        0}},     // padding_cols
-      DT_FLOAT,  // tensor datatype
-      0,         // device_id
+      1,            // batch
+      32,           // in_depths
+      {{300,        // in_rows
+        300}},      // in_cols
+      FORMAT_NCHW,  // compute_data_format
+      128,          // out_depths
+      {{3,          // filter_rows
+        3}},        // filter_cols
+      {{1,          // dilation_rows
+        1}},        // dilation_cols
+      {{1,          // stride_rows
+        1}},        // stride_cols
+      {{0,          // padding_rows
+        0}},        // padding_cols
+      DT_FLOAT,     // tensor datatype
+      0,            // device_id
   }};
   EXPECT_TRUE(
       conv_params_small.ShouldIncludeWinogradNonfusedAlgoPreCudnn7());
 
   ConvParametersPeer conv_params_large = {{
-      1,         // batch
-      128,       // in_depths
-      {{300,     // in_rows
-        300}},   // in_cols
-      768,       // out_depths
-      {{3,       // filter_rows
-        3}},     // filter_cols
-      {{1,       // dilation_rows
-        1}},     // dilation_cols
-      {{1,       // stride_rows
-        1}},     // stride_cols
-      {{0,       // padding_rows
-        0}},     // padding_cols
-      DT_FLOAT,  // tensor datatype
-      0,         // device_id
+      1,            // batch
+      128,          // in_depths
+      {{300,        // in_rows
+        300}},      // in_cols
+      FORMAT_NCHW,  // compute_data_format
+      768,          // out_depths
+      {{3,          // filter_rows
+        3}},        // filter_cols
+      {{1,          // dilation_rows
+        1}},        // dilation_cols
+      {{1,          // stride_rows
+        1}},        // stride_cols
+      {{0,          // padding_rows
+        0}},        // padding_cols
+      DT_FLOAT,     // tensor datatype
+      0,            // device_id
   }};
   EXPECT_FALSE(
       conv_params_large.ShouldIncludeWinogradNonfusedAlgoPreCudnn7());
-- 
cgit v1.2.3


From d6d95beac43b9f85c57e8302711db53a48b96e65 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Tue, 24 Jul 2018 13:28:26 -0700
Subject: Remove BufferAllocation::is_reusable() and introduce is_readonly();
 NFC

 - Instead of remembering is_reusable_, remember BufferAllocation::is_tuple_ and
   compute is_reusable() from is_tuple() and is_thread_local().

 - Introduce is_readonly() which tells us whether an allocation holds readonly
   data.  In the future this will return true for constant buffer allocations
   but today only entry parameters are readonly.

is_reusable() is about lifetime whereas is_readonly() is about write access.  In
particular, we sometimes "re-use" readonly allocations e.g. when the init value
of a while loop is an entry parameter and the while body is readonly.

PiperOrigin-RevId: 205881338
---
 .../compiler/xla/service/buffer_assignment.cc      | 51 +++++++++-------------
 .../compiler/xla/service/buffer_assignment.h       | 41 ++++++++++-------
 tensorflow/compiler/xla/service/hlo.proto          |  2 +-
 .../compiler/xla/service/llvm_ir/alias_analysis.cc |  3 +-
 4 files changed, 48 insertions(+), 49 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index 783e3f7e73..bcca9f46d3 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -270,7 +270,7 @@ BufferAllocationProto BufferAllocation::ToProto() const {
   proto.set_index(index_);
   proto.set_size(size_);
   proto.set_is_thread_local(is_thread_local_);
-  proto.set_is_reusable(is_reusable_);
+  proto.set_is_tuple(is_tuple_);
   proto.set_color(color_.value());
   if (is_entry_computation_parameter_) {
     proto.set_is_entry_computation_parameter(true);
@@ -491,20 +491,16 @@ BufferAssignment::GetUniqueTopLevelOutputSlice() const {
 }
 
 BufferAllocation* BufferAssignment::NewEmptyAllocation(
-    int64 size, bool is_thread_local, bool is_reusable,
-    LogicalBuffer::Color color) {
+    int64 size, LogicalBuffer::Color color) {
   BufferAllocation::Index index = allocations_.size();
-  allocations_.emplace_back(index, size, is_thread_local, is_reusable, color);
+  allocations_.emplace_back(index, size, color);
   BufferAllocation* allocation = &allocations_.back();
   return allocation;
 }
 
 BufferAllocation* BufferAssignment::NewAllocation(const LogicalBuffer& buffer,
-                                                  int64 size,
-                                                  bool is_thread_local,
-                                                  bool is_reusable) {
-  BufferAllocation* allocation =
-      NewEmptyAllocation(size, is_thread_local, is_reusable, buffer.color());
+                                                  int64 size) {
+  BufferAllocation* allocation = NewEmptyAllocation(size, buffer.color());
   AddAssignment(allocation, buffer, /*offset=*/0, size);
   allocation->peak_buffers_.push_back(&buffer);
   return allocation;
@@ -517,7 +513,8 @@ void BufferAssignment::AddAssignment(BufferAllocation* allocation,
   CHECK_EQ(0, allocation_index_for_buffer_.count(&buffer))
       << "LogicalBuffer " << buffer << " already has an allocation.";
   CHECK(allocation->is_reusable() || allocation->assigned_buffers().empty())
-      << "Non-reusable allocation already assigned a buffer";
+      << "Non-reusable allocation already assigned a buffer: "
+      << allocation->ToString();
 
   TF_CHECK_OK(points_to_analysis().VerifyBuffer(buffer));
 
@@ -751,8 +748,8 @@ bool BufferAssigner::MaybeAssignBuffer(BufferAllocation* allocation,
     return false;
   }
 
-  if (allocation->is_entry_computation_parameter()) {
-    VLOG(4) << "Can't assign: allocation holds parameter";
+  if (allocation->is_readonly()) {
+    VLOG(4) << "Can't assign: allocation is readonly";
     return false;
   }
 
@@ -923,9 +920,7 @@ Status BufferAssigner::AssignBuffersForComputation(
       // computations do not need special allocations because they live inside
       // callers.
       BufferAllocation* allocation =
-          assignment->NewAllocation(*buffer, buffer_size,
-                                    /*is_thread_local=*/false,
-                                    /*is_reusable=*/false);
+          assignment->NewAllocation(*buffer, buffer_size);
       allocation->set_entry_computation_parameter(
           instruction->parameter_number(), buffer->index());
       VLOG(3) << "New allocation #" << allocation->index()
@@ -934,20 +929,18 @@ Status BufferAssigner::AssignBuffersForComputation(
     }
 
     if (is_thread_local) {
-      // We do not reuse thread-local buffers for now, because they are
-      // dynamically allocated and their lifetimes are hard to compute.
-      BufferAllocation* allocation = assignment->NewAllocation(
-          *buffer, buffer_size, is_thread_local, /*is_reusable=*/false);
+      BufferAllocation* allocation =
+          assignment->NewAllocation(*buffer, buffer_size);
+      allocation->set_is_thread_local(true);
       VLOG(3) << "New allocation #" << allocation->index()
               << " for thread-local: " << *buffer;
       continue;
     }
 
     if (ShapeUtil::IsTuple(buffer->shape())) {
-      // TODO(b/34669761): Don't reuse tuple buffers because the GPU backend
-      // assumes longer buffer liveness than indicated by the analysis.
-      BufferAllocation* allocation = assignment->NewAllocation(
-          *buffer, buffer_size, is_thread_local, /*is_reusable=*/false);
+      BufferAllocation* allocation =
+          assignment->NewAllocation(*buffer, buffer_size);
+      allocation->set_is_tuple(true);
       VLOG(3) << "New allocation #" << allocation->index()
               << " for tuple-shaped buffer: " << *buffer;
       continue;
@@ -1030,8 +1023,8 @@ Status BufferAssigner::AssignBuffersForComputation(
     }
 
     if (!assignment->HasAllocation(*buffer)) {
-      BufferAllocation* allocation = assignment->NewAllocation(
-          *buffer, buffer_size, is_thread_local, /*is_reusable=*/true);
+      BufferAllocation* allocation =
+          assignment->NewAllocation(*buffer, buffer_size);
       allocation_indices.push_back(allocation->index());
       VLOG(3) << "New allocation #" << allocation->index()
               << " for: " << *buffer;
@@ -1227,8 +1220,8 @@ void BufferAssigner::AssignBuffersFromHeapSimulator(
         result.fragmentation_size;
   }
 
-  BufferAllocation* allocation = assignment->NewEmptyAllocation(
-      result.heap_size, /*is_thread_local=*/false, /*is_reusable=*/true, color);
+  BufferAllocation* allocation =
+      assignment->NewEmptyAllocation(result.heap_size, color);
   for (const auto& buffer_chunk : result.chunk_map) {
     // TODO(lauj) Remove this down_cast after downstream users of
     // BufferAllocation::assigned_buffers() are updated to use BufferValue.
@@ -1584,9 +1577,7 @@ void BufferAssigner::AssignColocatedBufferSets(
         // allocations for each colocated buffer set. When liveness has
         // module-level scope, we can allow buffers to be shared across
         // computations (in some cases).
-        allocation = assignment->NewAllocation(*buffer, buffer_size,
-                                               /*is_thread_local=*/false,
-                                               /*is_reusable=*/true);
+        allocation = assignment->NewAllocation(*buffer, buffer_size);
         if (entry_parameter_number >= 0) {
           // This colocated buffer set contains an entry parameter and other
           // logical buffers which use the parameter as read-only in a while
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.h b/tensorflow/compiler/xla/service/buffer_assignment.h
index ad0b0bf7c2..8844b6e3ba 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.h
+++ b/tensorflow/compiler/xla/service/buffer_assignment.h
@@ -58,13 +58,8 @@ class BufferAllocation {
   // contiguously and can be used as array indexes.
   using Index = int64;
 
-  BufferAllocation(Index index, int64 size, bool is_thread_local,
-                   bool is_reusable, LogicalBuffer::Color color)
-      : index_(index),
-        size_(size),
-        is_thread_local_(is_thread_local),
-        is_reusable_(is_reusable),
-        color_(color) {}
+  BufferAllocation(Index index, int64 size, LogicalBuffer::Color color)
+      : index_(index), size_(size), color_(color) {}
   ~BufferAllocation() {}
 
   // Returns the index of this allocation.
@@ -74,9 +69,26 @@ class BufferAllocation {
   // inside of a map or reduce computation. Such allocations need to be thread
   // local.
   bool is_thread_local() const { return is_thread_local_; }
+  void set_is_thread_local(bool is_thread_local) {
+    is_thread_local_ = is_thread_local;
+  }
 
   // Whether this allocation can be used by more than one logical buffer.
-  bool is_reusable() const { return is_reusable_; }
+  bool is_reusable() const {
+    // We do not reuse thread-local buffers for now, because they are
+    // dynamically allocated and their lifetimes are hard to compute.
+    //
+    // TODO(b/34669761): Don't reuse tuple buffers because the GPU backend
+    // assumes longer buffer liveness than indicated by the analysis.
+    return !is_thread_local() && !is_tuple();
+  }
+
+  // Whether this allocation is readonly i.e. backed by memory we cannot write
+  // to.
+  bool is_readonly() const { return is_entry_computation_parameter(); }
+
+  bool is_tuple() const { return is_tuple_; }
+  void set_is_tuple(bool is_tuple) { is_tuple_ = is_tuple; }
 
   // Whether this allocation holds a LogicalBuffer from a parameter of the entry
   // computation. These buffers have lifetimes which may be longer than the
@@ -256,10 +268,10 @@ class BufferAllocation {
   int64 size_;
 
   // Whether this buffer needs to be thread-local.
-  bool is_thread_local_;
+  bool is_thread_local_ = false;
 
-  // Whether this buffer is usable by more than one logical buffer.
-  bool is_reusable_;
+  // Whether this buffer holds a tuple.
+  bool is_tuple_ = false;
 
   // Color of the allocation.
   LogicalBuffer::Color color_;
@@ -426,14 +438,11 @@ class BufferAssignment {
 
   // Creates and returns a new BufferAllocation, with no assigned
   // LogicalBuffers. Ownership is maintained internally.
-  BufferAllocation* NewEmptyAllocation(int64 size, bool is_thread_local,
-                                       bool is_reusable,
-                                       LogicalBuffer::Color color);
+  BufferAllocation* NewEmptyAllocation(int64 size, LogicalBuffer::Color color);
 
   // Helper that calls NewEmptyAllocation and AddAssignment in one call,
   // creating an allocation containing a single LogicalBuffer.
-  BufferAllocation* NewAllocation(const LogicalBuffer& buffer, int64 size,
-                                  bool is_thread_local, bool is_reusable);
+  BufferAllocation* NewAllocation(const LogicalBuffer& buffer, int64 size);
 
   // Adds a LogicalBuffer to the set assigned to the given allocation.
   void AddAssignment(BufferAllocation* allocation, const LogicalBuffer& buffer,
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index 87abc0e74f..50d7f1823c 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -244,7 +244,7 @@ message BufferAllocationProto {
   int64 index = 1;
   int64 size = 2;
   bool is_thread_local = 3;
-  bool is_reusable = 4;
+  bool is_tuple = 11;
   bool is_entry_computation_parameter = 5;
   int64 parameter_number = 6;
   repeated int64 parameter_shape_index = 10;
diff --git a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
index 93a8c130e1..e5370eca56 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
@@ -28,8 +28,7 @@ namespace llvm_ir {
 // Sentry allocation used to represent parameters of the entry computation in
 // alias_scope_metadata_ and noalias_metadata_.
 static const BufferAllocation* kParameterAllocation = new BufferAllocation(
-    /*index=*/-1, /*size=*/0, /*is_thread_local=*/false, /*is_reusable=*/false,
-    LogicalBuffer::Color(0));
+    /*index=*/-1, /*size=*/0, LogicalBuffer::Color(0));
 
 void AliasAnalysis::AddAliasingInformationToIrArray(const HloInstruction& hlo,
                                                     llvm_ir::IrArray* array,
-- 
cgit v1.2.3


From e25386e18a2bea60886daa3157dfb3a32781d863 Mon Sep 17 00:00:00 2001
From: Nupur Garg 
Date: Tue, 24 Jul 2018 13:33:37 -0700
Subject: Remove functions from TFLite public Python API.

PiperOrigin-RevId: 205882419
---
 tensorflow/contrib/lite/python/lite.py | 61 ++++++++++++++++------------------
 1 file changed, 29 insertions(+), 32 deletions(-)

diff --git a/tensorflow/contrib/lite/python/lite.py b/tensorflow/contrib/lite/python/lite.py
index 29a1487c1f..2f9b9d469a 100644
--- a/tensorflow/contrib/lite/python/lite.py
+++ b/tensorflow/contrib/lite/python/lite.py
@@ -40,24 +40,23 @@ from google.protobuf import text_format as _text_format
 from google.protobuf.message import DecodeError
 from tensorflow.contrib.lite.python import lite_constants as constants
 from tensorflow.contrib.lite.python.convert import build_toco_convert_protos  # pylint: disable=unused-import
-from tensorflow.contrib.lite.python.convert import tensor_name
+from tensorflow.contrib.lite.python.convert import tensor_name as _tensor_name
 from tensorflow.contrib.lite.python.convert import toco_convert
 from tensorflow.contrib.lite.python.convert import toco_convert_protos  # pylint: disable=unused-import
-from tensorflow.contrib.lite.python.convert_saved_model import freeze_saved_model
-from tensorflow.contrib.lite.python.convert_saved_model import get_tensors_from_tensor_names
-from tensorflow.contrib.lite.python.convert_saved_model import set_tensor_shapes
+from tensorflow.contrib.lite.python.convert_saved_model import freeze_saved_model as _freeze_saved_model
+from tensorflow.contrib.lite.python.convert_saved_model import get_tensors_from_tensor_names as _get_tensors_from_tensor_names
+from tensorflow.contrib.lite.python.convert_saved_model import set_tensor_shapes as _set_tensor_shapes
 from tensorflow.contrib.lite.python.interpreter import Interpreter  # pylint: disable=unused-import
 from tensorflow.contrib.lite.python.op_hint import convert_op_hints_to_stubs  # pylint: disable=unused-import
 from tensorflow.contrib.lite.python.op_hint import OpHint  # pylint: disable=unused-import
 from tensorflow.core.framework import graph_pb2 as _graph_pb2
 from tensorflow.python import keras as _keras
 from tensorflow.python.client import session as _session
-from tensorflow.python.framework import graph_util as tf_graph_util
-from tensorflow.python.framework.importer import import_graph_def
-from tensorflow.python.ops.variables import global_variables_initializer
-from tensorflow.python.saved_model import signature_constants
-from tensorflow.python.saved_model import tag_constants
-# from tensorflow.python.util.all_util import remove_undocumented
+from tensorflow.python.framework import graph_util as _tf_graph_util
+from tensorflow.python.framework.importer import import_graph_def as _import_graph_def
+from tensorflow.python.ops.variables import global_variables_initializer as _global_variables_initializer
+from tensorflow.python.saved_model import signature_constants as _signature_constants
+from tensorflow.python.saved_model import tag_constants as _tag_constants
 
 
 class TocoConverter(object):
@@ -196,7 +195,7 @@ class TocoConverter(object):
         input_arrays or output_arrays contains an invalid tensor name.
     """
     with _session.Session() as sess:
-      sess.run(global_variables_initializer())
+      sess.run(_global_variables_initializer())
 
       # Read GraphDef from file.
       graph_def = _graph_pb2.GraphDef()
@@ -218,12 +217,12 @@ class TocoConverter(object):
           raise ValueError(
               "Unable to parse input file '{}'.".format(graph_def_file))
       sess.graph.as_default()
-      import_graph_def(graph_def, name="")
+      _import_graph_def(graph_def, name="")
 
       # Get input and output tensors.
-      input_tensors = get_tensors_from_tensor_names(sess.graph, input_arrays)
-      output_tensors = get_tensors_from_tensor_names(sess.graph, output_arrays)
-      set_tensor_shapes(input_tensors, input_shapes)
+      input_tensors = _get_tensors_from_tensor_names(sess.graph, input_arrays)
+      output_tensors = _get_tensors_from_tensor_names(sess.graph, output_arrays)
+      _set_tensor_shapes(input_tensors, input_shapes)
 
       # Check if graph is frozen.
       if not _is_frozen_graph(sess):
@@ -261,12 +260,12 @@ class TocoConverter(object):
       TocoConverter class.
     """
     if tag_set is None:
-      tag_set = set([tag_constants.SERVING])
+      tag_set = set([_tag_constants.SERVING])
     if signature_key is None:
-      signature_key = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
+      signature_key = _signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
 
-    result = freeze_saved_model(saved_model_dir, input_arrays, input_shapes,
-                                output_arrays, tag_set, signature_key)
+    result = _freeze_saved_model(saved_model_dir, input_arrays, input_shapes,
+                                 output_arrays, tag_set, signature_key)
     return cls(
         graph_def=result[0], input_tensors=result[1], output_tensors=result[2])
 
@@ -299,15 +298,15 @@ class TocoConverter(object):
 
     # Get input and output tensors.
     if input_arrays:
-      input_tensors = get_tensors_from_tensor_names(sess.graph, input_arrays)
+      input_tensors = _get_tensors_from_tensor_names(sess.graph, input_arrays)
     else:
       input_tensors = keras_model.inputs
 
     if output_arrays:
-      output_tensors = get_tensors_from_tensor_names(sess.graph, output_arrays)
+      output_tensors = _get_tensors_from_tensor_names(sess.graph, output_arrays)
     else:
       output_tensors = keras_model.outputs
-    set_tensor_shapes(input_tensors, input_shapes)
+    _set_tensor_shapes(input_tensors, input_shapes)
 
     graph_def = _freeze_graph(sess, output_tensors)
     return cls(graph_def, input_tensors, output_tensors)
@@ -328,12 +327,12 @@ class TocoConverter(object):
     for tensor in self._input_tensors:
       if not tensor.get_shape():
         raise ValueError("Provide an input shape for input array '{0}'.".format(
-            tensor_name(tensor)))
+            _tensor_name(tensor)))
       shape = tensor.get_shape().as_list()
       if None in shape[1:]:
         raise ValueError(
             "None is only supported in the 1st dimension. Tensor '{0}' has "
-            "invalid shape '{1}'.".format(tensor_name(tensor), shape))
+            "invalid shape '{1}'.".format(_tensor_name(tensor), shape))
       elif shape[0] is None:
         self._set_batch_size(batch_size=1)
 
@@ -343,7 +342,7 @@ class TocoConverter(object):
       quantized_stats = []
       invalid_stats = []
       for tensor in self._input_tensors:
-        name = tensor_name(tensor)
+        name = _tensor_name(tensor)
         if name in self.quantized_input_stats:
           quantized_stats.append(self.quantized_input_stats[name])
         else:
@@ -381,7 +380,7 @@ class TocoConverter(object):
     Returns:
       List of strings.
     """
-    return [tensor_name(tensor) for tensor in self._input_tensors]
+    return [_tensor_name(tensor) for tensor in self._input_tensors]
 
   def _set_batch_size(self, batch_size):
     """Sets the first dimension of the input tensor to `batch_size`.
@@ -428,11 +427,9 @@ def _freeze_graph(sess, output_tensors):
     Frozen GraphDef.
   """
   if not _is_frozen_graph(sess):
-    sess.run(global_variables_initializer())
-    output_arrays = [tensor_name(tensor) for tensor in output_tensors]
-    return tf_graph_util.convert_variables_to_constants(sess, sess.graph_def,
-                                                        output_arrays)
+    sess.run(_global_variables_initializer())
+    output_arrays = [_tensor_name(tensor) for tensor in output_tensors]
+    return _tf_graph_util.convert_variables_to_constants(
+        sess, sess.graph_def, output_arrays)
   else:
     return sess.graph_def
-
-# remove_undocumented(__name__)
-- 
cgit v1.2.3


From 6b2ae9b3da572d2f8e1eccdf2922efce43cbecd4 Mon Sep 17 00:00:00 2001
From: Youlong Cheng 
Date: Tue, 24 Jul 2018 13:44:46 -0700
Subject: PUBLIC: Enable eval and predict on multi-hosts in broadcast mode.

PiperOrigin-RevId: 205884309
---
 tensorflow/contrib/tpu/python/tpu/tpu_context.py   | 4 ++--
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
index 2cb68f74a0..a9cf54f77d 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
@@ -594,7 +594,7 @@ class _InternalTPUContext(object):
         raise ValueError(
             'eval batch size {} must be divisible by number of replicas {}'
             .format(self._eval_batch_size, num_replicas))
-      if num_hosts > 1:
+      if num_hosts > 1 and not self.is_input_broadcast_with_iterators():
         raise ValueError(
             'TPUEstimator.evaluate should be running on single TPU worker. '
             'got {}.'.format(num_hosts))
@@ -609,7 +609,7 @@ class _InternalTPUContext(object):
         raise ValueError(
             'predict batch size {} must be divisible by number of replicas {}'
             .format(self._predict_batch_size, num_replicas))
-      if num_hosts > 1:
+      if num_hosts > 1 and not self.is_input_broadcast_with_iterators():
         raise ValueError(
             'TPUEstimator.predict should be running on single TPU worker. '
             'got {}.'.format(num_hosts))
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 42406db88a..2c7e7d84c0 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -1764,7 +1764,8 @@ class TPUEstimator(estimator_lib.Estimator):
   Current limitations:
   --------------------
 
-  1. TPU evaluation only works on a single host (one TPU worker).
+  1. TPU evaluation only works on a single host (one TPU worker) except
+     BROADCAST mode.
 
   2. `input_fn` for evaluation should **NOT** raise an end-of-input exception
      (`OutOfRangeError` or `StopIteration`). And all evaluation steps and all
-- 
cgit v1.2.3


From ee0bd6ef450b388fadea63b31b65b13bd12f17d6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 13:50:22 -0700
Subject: Automated rollback of commit 0ea6847c892497afdd20c1150fee1e532612ca17

PiperOrigin-RevId: 205885304
---
 tensorflow/compiler/jit/xla_compilation_cache.cc   |  18 +++-
 tensorflow/compiler/jit/xla_device_context.cc      | 117 ++++++++++++++-------
 tensorflow/compiler/jit/xla_device_context.h       |   5 +-
 tensorflow/compiler/jit/xla_tensor.cc              |   4 +-
 tensorflow/compiler/xla/service/executable.cc      |  13 ++-
 tensorflow/compiler/xla/service/hlo_runner.cc      |   9 +-
 .../xla/tests/local_client_execute_test.cc         |   4 +
 .../compiler/xla/tests/local_client_test_base.cc   |  14 ++-
 .../compiler/xla/tests/xla_hlo_profile_test.cc     |   1 +
 .../stream_executor/host/host_gpu_executor.cc      |   2 +-
 tensorflow/stream_executor/stream.cc               |   6 ++
 11 files changed, 143 insertions(+), 50 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_compilation_cache.cc b/tensorflow/compiler/jit/xla_compilation_cache.cc
index 7ed609c437..54a41a4daa 100644
--- a/tensorflow/compiler/jit/xla_compilation_cache.cc
+++ b/tensorflow/compiler/jit/xla_compilation_cache.cc
@@ -40,7 +40,23 @@ namespace tensorflow {
 XlaCompilationCache::XlaCompilationCache(xla::LocalClient* client,
                                          DeviceType device_type)
     : client_(client), device_type_(std::move(device_type)) {}
-XlaCompilationCache::~XlaCompilationCache() = default;
+XlaCompilationCache::~XlaCompilationCache() {
+  // Ensure any use of our programs have completed by waiting for all stream
+  // executors to complete.
+  for (auto* executor : client_->backend().stream_executors()) {
+    bool ok = executor->SynchronizeAllActivity();
+    if (!ok) {
+      LOG(ERROR) << "Error synchronizing activity while waiting for all "
+                    "programs to complete";
+    }
+  }
+  // TODO(b/110813685): Think about the program ownership model. Programs are
+  // currently owned by the compilation cache which means we must wait for
+  // program completion in the destructor. There are multiple compilation caches
+  // around, which complicates things a little. Perhaps having programs be
+  // shared_ptrs (an invasive change) would make the model easier to reason
+  // about?
+}
 
 string XlaCompilationCache::DebugString() {
   return "XLA JIT compilation cache";
diff --git a/tensorflow/compiler/jit/xla_device_context.cc b/tensorflow/compiler/jit/xla_device_context.cc
index 04778c0090..8cf198239c 100644
--- a/tensorflow/compiler/jit/xla_device_context.cc
+++ b/tensorflow/compiler/jit/xla_device_context.cc
@@ -74,43 +74,64 @@ Status XlaTransferManager::TransferLiteralToDevice(
   xla::Shape xla_shape;
   TF_RETURN_IF_ERROR(TensorShapeToXLAShape(host_tensor.dtype(),
                                            host_tensor.shape(), &xla_shape));
-  xla::BorrowingLiteral literal(
+  // Create a reference to hold onto host_tensor until after the literal has
+  // been transferred. Also make sure the literal exists until the function
+  // asynchronously completes, as it will be wrapped in an xla::LiteralSlice.
+  TensorReference ref(host_tensor);
+  auto literal = std::make_shared(
       static_cast(DMAHelper::base(&host_tensor)), xla_shape);
 
   XlaTensor* xla_tensor = XlaTensor::FromTensor(device_tensor);
   const xla::ShapedBuffer& shaped_buffer = xla_tensor->shaped_buffer();
-  VLOG(1) << "Transfer to device as literal: " << literal.ToString() << " "
+  VLOG(1) << "Transfer to device as literal: " << literal->ToString() << " "
           << shaped_buffer.ToString();
-  TF_RETURN_IF_ERROR(transfer_manager_->TransferLiteralToDevice(
-      host_to_device_stream_, literal, shaped_buffer));
+  if (UseMultipleStreams()) {
+    // Initially wait for the compute stream so that memory allocations are
+    // synchronized.
+    host_to_device_stream_->ThenWaitFor(stream_);
+  }
+  TF_RETURN_IF_ERROR(transfer_manager_->TransferLiteralToDeviceAsync(
+      host_to_device_stream_, *literal, shaped_buffer));
   if (UseMultipleStreams()) {
     se::Event event(stream_->parent());
     TF_RET_CHECK(event.Init()) << "Event failed to initialize!";
     host_to_device_stream_->ThenRecordEvent(&event);
     xla_tensor->SetDefinedOn(host_to_device_stream_, std::move(event));
   }
+  // Unref the host tensor, and capture the literal shared_ptr too so it goes
+  // out of scope when the lambda completes.
+  host_to_device_stream_->ThenDoHostCallback([ref, literal]() { ref.Unref(); });
   return Status::OK();
 }
 
-Status XlaTransferManager::TransferLiteralFromDevice(
-    Tensor* host_tensor, const Tensor& device_tensor) const {
+void XlaTransferManager::TransferLiteralFromDevice(
+    Tensor* host_tensor, const Tensor& device_tensor,
+    const StatusCallback& done) const {
   const xla::ShapedBuffer& shaped_buffer =
       XlaTensor::FromTensor(&device_tensor)->shaped_buffer();
 
-  TF_ASSIGN_OR_RETURN(std::unique_ptr literal,
-                      transfer_manager_->TransferLiteralFromDevice(
-                          device_to_host_stream_, shaped_buffer));
-  VLOG(1) << "Transfer from device as literal: " << literal->ToString() << " "
-          << shaped_buffer.ToString();
-  Tensor tensor;
-  TF_RETURN_IF_ERROR(
-      LiteralToHostTensor(*literal, host_tensor->dtype(), &tensor));
-  // Reshape the tensor back to its declared shape.
-  if (!host_tensor->CopyFrom(tensor, device_tensor.shape())) {
-    return errors::Internal(
-        "Tensor::CopyFrom failed when copying from XLA device to CPU");
-  }
-  return Status::OK();
+  TensorReference ref(device_tensor);
+  transfer_manager_->TransferLiteralFromDevice(
+      device_to_host_stream_, shaped_buffer,
+      [=, &shaped_buffer](
+          xla::StatusOr > literal_or) {
+        ref.Unref();
+        done([&]() -> Status {
+          TF_ASSIGN_OR_RETURN(auto literal, std::move(literal_or));
+          VLOG(1) << "Transfer from device as literal: " << literal->ToString()
+                  << " " << shaped_buffer.ToString();
+          Tensor tensor;
+          TF_RETURN_IF_ERROR(
+              LiteralToHostTensor(*literal, host_tensor->dtype(), &tensor));
+          // Reshape the tensor back to its declared shape.
+          Status status;
+          if (!host_tensor->CopyFrom(tensor, device_tensor.shape())) {
+            status = errors::Internal(
+                "Tensor::CopyFrom failed when copying from XLA device to CPU");
+          }
+          return status;
+        }());
+      });
 }
 
 void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
@@ -163,6 +184,12 @@ void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
       return;
     }
     status = TransferLiteralToDevice(reshaped_cpu_tensor, device_tensor);
+    if (status.ok()) {
+      xla_tensor->set_host_tensor(*cpu_tensor);
+      host_to_device_stream_->ThenDoHostCallback(
+          [done]() { done(Status::OK()); });
+      return;
+    }
   } else {
     se::DeviceMemoryBase dev_dst_ptr =
         XlaTensor::DeviceMemoryFromTensor(*device_tensor);
@@ -212,7 +239,8 @@ void XlaTransferManager::CopyDeviceTensorToCPU(const Tensor* device_tensor,
 
   Status status;
   if (transfer_as_literal_) {
-    status = TransferLiteralFromDevice(cpu_tensor, *device_tensor);
+    TransferLiteralFromDevice(cpu_tensor, *device_tensor, done);
+    return;
   } else {
     device_to_host_stream_->ThenMemcpy(dst_ptr, dev_src_ptr, total_bytes);
     // TODO(hpucha): Make this asynchronous.
@@ -234,15 +262,15 @@ void XlaTransferManager::CopyDeviceTensorToDevice(const Tensor& src_tensor,
           << reinterpret_cast(src_tensor.tensor_data().data())
           << " "
           << reinterpret_cast(dst_tensor->tensor_data().data());
-  // TODO(phawkins): replace this code with an asynchronous implementation.
-  auto body = [&]() {
+  // Perform memory allocation now, and enqueue the device-to-device transfer.
+  Status status = [&]() -> Status {
     if (src_tensor.NumElements() == 0) {
       return Status::OK();
     }
     // TODO(jmolloy): We co-opt the device_to_host stream for device to device
     // transfers; perhaps we should have a dedicated device to device stream? or
     // one per device?
-    auto device_to_device_stream = device_to_host_stream_;
+    auto device_to_device_stream = stream_;
     XlaTensor* xla_src = XlaTensor::FromTensor(&src_tensor);
     XlaTensor* xla_dst = XlaTensor::FromTensor(dst_tensor);
     CHECK(xla_src && xla_dst)
@@ -254,29 +282,40 @@ void XlaTransferManager::CopyDeviceTensorToDevice(const Tensor& src_tensor,
       TF_RETURN_IF_ERROR(
           xla_dst->AllocateShapedBuffer(src_tensor.dtype(), shape, client_,
                                         stream_->parent()->device_ordinal()));
+      if (stream_ != device_to_device_stream) {
+        // Initially wait for the compute stream so that memory allocations are
+        // synchronized.
+        device_to_device_stream->ThenWaitFor(stream_);
+      }
     }
 
     if (se::Event* event =
             xla_src->GetDefinitionEvent(device_to_device_stream)) {
       device_to_device_stream->ThenWaitFor(event);
       xla_src->SetDefinedOn(device_to_device_stream);
-      TF_RETURN_IF_ERROR(device_to_device_stream->BlockHostUntilDone());
     }
-    TF_RETURN_IF_ERROR(
-        xla_dst->shaped_buffer().buffers().ForEachMutableElementWithStatus(
-            [&](const xla::ShapeIndex& index, se::DeviceMemoryBase* buffer) {
-              const se::DeviceMemoryBase& from_buffer =
-                  xla_src->shaped_buffer().buffers().element(index);
-              CHECK_EQ(buffer->size(), from_buffer.size());
-              if (!stream_->parent()->SynchronousMemcpy(buffer, from_buffer,
-                                                        buffer->size())) {
-                return errors::Internal("Device to device memcpy failed");
-              }
-              return Status::OK();
-            }));
+
+    auto from_iter = xla_src->shaped_buffer().buffers().begin();
+    auto to_iter = xla_dst->shaped_buffer().buffers().begin();
+    for (auto end_iter = xla_src->shaped_buffer().buffers().end();
+         from_iter != end_iter; ++from_iter, ++to_iter) {
+      device_to_device_stream->ThenMemcpyD2D(
+          &to_iter->second, from_iter->second, to_iter->second.size());
+    }
+
+    if (UseMultipleStreams()) {
+      se::Event event(stream_->parent());
+      CHECK(event.Init());
+      device_to_device_stream->ThenRecordEvent(&event);
+      xla_dst->SetDefinedOn(device_to_device_stream, std::move(event));
+    }
     return Status::OK();
-  };
-  done(body());
+  }();
+  if (!status.ok()) {
+    return done(status);
+  } else {
+    stream_->ThenDoHostCallback([=]() { done(Status::OK()); });
+  }
 }
 
 XlaDeviceContext::XlaDeviceContext(
diff --git a/tensorflow/compiler/jit/xla_device_context.h b/tensorflow/compiler/jit/xla_device_context.h
index c726495f96..912f8d779e 100644
--- a/tensorflow/compiler/jit/xla_device_context.h
+++ b/tensorflow/compiler/jit/xla_device_context.h
@@ -66,8 +66,9 @@ class XlaTransferManager {
  private:
   Status TransferLiteralToDevice(const Tensor& host_tensor,
                                  Tensor* device_tensor) const;
-  Status TransferLiteralFromDevice(Tensor* host_tensor,
-                                   const Tensor& device_tensor) const;
+  void TransferLiteralFromDevice(Tensor* host_tensor,
+                                 const Tensor& device_tensor,
+                                 const StatusCallback& done) const;
   bool UseMultipleStreams() const { return stream_ != host_to_device_stream_; }
 
   // The main compute stream of the device, used to synchronize the transfer
diff --git a/tensorflow/compiler/jit/xla_tensor.cc b/tensorflow/compiler/jit/xla_tensor.cc
index 5dff187fff..d777dfa5a3 100644
--- a/tensorflow/compiler/jit/xla_tensor.cc
+++ b/tensorflow/compiler/jit/xla_tensor.cc
@@ -92,10 +92,8 @@ se::Event* XlaTensor::GetDefinitionEvent(se::Stream* stream) {
 
 void XlaTensor::SetDefinedOn(se::Stream* stream, se::Event event) {
   mutex_lock lock(mu_);
-  CHECK(!definition_event_.has_value())
-      << "SetDefinedOn must only be called once!";
   definition_event_ = std::move(event);
-  streams_defined_on_.push_back(stream);
+  streams_defined_on_ = {stream};
 }
 
 void XlaTensor::SetDefinedOn(se::Stream* stream) {
diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc
index 7cf2746947..fd75847d0c 100644
--- a/tensorflow/compiler/xla/service/executable.cc
+++ b/tensorflow/compiler/xla/service/executable.cc
@@ -82,7 +82,18 @@ StatusOr Executable::ExecuteOnStreamWrapper(
 
   StatusOr return_value =
       ExecuteOnStream(run_options, arguments, profile_ptr.get());
-  TF_RETURN_IF_ERROR(return_value.status());
+  if (!return_value.status().ok()) {
+    if (profile != nullptr) {
+      // Ensure the ThenStartTimer call has completed before we destroy timer.
+      // We already have a failure status to return, so just log this if it
+      // fails.
+      Status status = stream->BlockHostUntilDone();
+      if (!status.ok()) {
+        LOG(ERROR) << "Failed to BlockHostUntilDone: " << status;
+      }
+    }
+    return return_value.status();
+  }
 
   if (profile != nullptr) {
     VLOG(1) << "enqueueing 'stop timer' and blocking host until done...";
diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc
index 4f0569f405..b2725e2918 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.cc
+++ b/tensorflow/compiler/xla/service/hlo_runner.cc
@@ -180,8 +180,12 @@ StatusOr HloRunner::ExecuteWithDeviceBuffers(
 
   TF_ASSIGN_OR_RETURN(std::unique_ptr executable,
                       CreateExecutable(std::move(module), run_hlo_passes));
-  return executable->ExecuteOnStreamWrapper(&service_run_options,
-                                            /*profile=*/profile, arguments);
+  TF_ASSIGN_OR_RETURN(
+      ScopedShapedBuffer retval,
+      executable->ExecuteOnStreamWrapper(&service_run_options,
+                                         /*profile=*/profile, arguments));
+  TF_RETURN_IF_ERROR(stream.BlockHostUntilDone());
+  return std::move(retval);
 }
 
 StatusOr HloRunner::ExecuteWithDeviceBuffers(
@@ -309,6 +313,7 @@ StatusOr>> HloRunner::ExecuteReplicated(
 
   std::vector> exec_results;
   for (int64 i = 0; i < options.num_replicas; ++i) {
+    TF_RETURN_IF_ERROR(streams[i]->BlockHostUntilDone());
     TF_ASSIGN_OR_RETURN(std::unique_ptr literal,
                         backend().transfer_manager()->TransferLiteralFromDevice(
                             streams[i].get(), results[i]));
diff --git a/tensorflow/compiler/xla/tests/local_client_execute_test.cc b/tensorflow/compiler/xla/tests/local_client_execute_test.cc
index 2f4d197ae6..5c3498c84c 100644
--- a/tensorflow/compiler/xla/tests/local_client_execute_test.cc
+++ b/tensorflow/compiler/xla/tests/local_client_execute_test.cc
@@ -772,6 +772,10 @@ XLA_TEST_F(LocalClientExecuteTest, CompileExecutable) {
   ScopedShapedBuffer result =
       executable->Run({&x_array}, DefaultExecutableRunOptions())
           .ConsumeValueOrDie();
+  ASSERT_IS_OK(local_client_->mutable_backend()
+                   ->BorrowStream(0)
+                   .ValueOrDie()
+                   ->BlockHostUntilDone());
 
   LiteralTestUtil::ExpectR1Near(
       {2.0f, 4.0f, 6.0f}, *ShapedBufferToLiteral(result), error_spec_);
diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.cc b/tensorflow/compiler/xla/tests/local_client_test_base.cc
index 88797a7d0a..c31ba0e713 100644
--- a/tensorflow/compiler/xla/tests/local_client_test_base.cc
+++ b/tensorflow/compiler/xla/tests/local_client_test_base.cc
@@ -189,7 +189,19 @@ StatusOr LocalClientTestBase::ExecuteLocally(
   TF_ASSIGN_OR_RETURN(
       std::unique_ptr executable,
       local_client_->Compile(computation, argument_layouts, build_options));
-  return executable->Run(arguments, run_options);
+  TF_ASSIGN_OR_RETURN(auto ret, executable->Run(arguments, run_options));
+
+  auto device_ordinal =
+      build_options.device_ordinal() == -1 ? 0 : build_options.device_ordinal();
+  auto* stream = run_options.stream();
+  if (!stream) {
+    stream = local_client_->mutable_backend()
+                 ->BorrowStream(device_ordinal)
+                 .ValueOrDie()
+                 .get();
+  }
+  TF_RETURN_IF_ERROR(stream->BlockHostUntilDone());
+  return std::move(ret);
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
index 4d4dd62a3f..c000ff4dc8 100644
--- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
+++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
@@ -172,6 +172,7 @@ void ExecuteAndFetchProfile(string* profile_output, LocalClient* client,
       auto execution_result,
       executable->ExecuteOnStream(&run_options, {&lhs_arg, &rhs_arg},
                                   &hlo_execution_profile));
+  TF_ASSERT_OK(stream_ptr->BlockHostUntilDone());
   (void)execution_result;
 
   *profile_output =
diff --git a/tensorflow/stream_executor/host/host_gpu_executor.cc b/tensorflow/stream_executor/host/host_gpu_executor.cc
index 3cd97b3cf1..8adf739b17 100644
--- a/tensorflow/stream_executor/host/host_gpu_executor.cc
+++ b/tensorflow/stream_executor/host/host_gpu_executor.cc
@@ -93,7 +93,7 @@ bool HostExecutor::MemcpyDeviceToDevice(Stream *stream,
   // the nature of the HostExecutor) memcpy  on the stream (HostStream)
   // associated with the HostExecutor.
   AsHostStream(stream)->EnqueueTask(
-      [src_mem, dst_mem, size]() { memcpy(src_mem, dst_mem, size); });
+      [src_mem, dst_mem, size]() { memcpy(dst_mem, src_mem, size); });
   return true;
 }
 
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index ca1b8e28e6..2c495c99e1 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -268,6 +268,12 @@ Stream::~Stream() {
   VLOG_CALL();
 
   temporary_memory_manager_.ForceDeallocateAll();
+  // Ensure the stream is completed.
+  auto status = BlockHostUntilDone();
+  if (!status.ok()) {
+    LOG(WARNING) << "Error blocking host until done in stream destructor: "
+                 << status;
+  }
 
   if (allocated_) {
     parent_->DeallocateStream(this);
-- 
cgit v1.2.3


From 57d051e7b156313c0beef6eb1fd9e6ca955a568a Mon Sep 17 00:00:00 2001
From: Akshay Modi 
Date: Tue, 24 Jul 2018 13:50:42 -0700
Subject: Don't cache zero tensors in graph at all

PiperOrigin-RevId: 205885372
---
 tensorflow/python/eager/backprop.py      | 16 ++++++++----
 tensorflow/python/eager/backprop_test.py | 43 +++++++++++++-------------------
 2 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index da8b93dba8..c59ad09bf1 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -599,15 +599,18 @@ def _fast_fill(value, shape, dtype):
 
 
 def _zeros(shape, dtype):
-  """Wraps array_ops.zeros to cache last zero for a given shape and dtype."""
-  device = context.context().device_name
+  """Helper to return (possibly cached) zero tensors in eager mode."""
   if dtype == dtypes.variant:
     # TODO(apassos): need to save enough information about variant tensors to do
     # a zeros
     return None
-  # pylint: disable=protected-access
-  cache_key = shape, dtype, device, context.context()._eager_context.mode
-  # pylint: enable=protected-access
+
+  ctx = context.context()
+  if not ctx.executing_eagerly():
+    return array_ops.zeros(shape, dtype)
+
+  device = ctx.device_name
+  cache_key = shape, dtype, device
   cached = _zeros_cache.get(cache_key)
   if cached is None:
     cached = _fast_fill(0, shape, dtype)
@@ -616,6 +619,9 @@ def _zeros(shape, dtype):
 
 
 def _ones(shape, dtype):
+  if not context.context().executing_eagerly():
+    return array_ops.ones(shape, dtype)
+
   if shape == ():  # pylint: disable=g-explicit-bool-comparison
     return constant_op.constant(1, dtype=dtype)
   return _fast_fill(1, shape, dtype)
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 95a3a8b629..3d3f54b9c4 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -925,32 +925,23 @@ class BackpropTest(test.TestCase):
         'did you forget to return a value from fn?'):
       val_and_grads_fn(x, y)
 
-  def testZerosCacheDoesntLeakAcrossModes(self):
-    with ops.Graph().as_default():
-      t = random_ops.random_normal(shape=[100, 2])
-      x = random_ops.random_normal(shape=[100, 4])
-      dy = random_ops.random_normal(shape=[100, 4])
-      with backprop.GradientTape() as gradient_tape:
-        gradient_tape.watch(x)
-        x1, _ = array_ops.split(x, num_or_size_splits=2, axis=1)
-        y1 = x1 ** 2.
-        y = array_ops.concat([y1, t], axis=1)
-
-      dx = gradient_tape.gradient(y, x, output_gradients=dy)
-      with self.test_session() as sess:
-        sess.run(variables.global_variables_initializer())
-        sess.run(dx)
-
-    t = random_ops.random_normal(shape=[100, 2])
-    x = random_ops.random_normal(shape=[100, 4])
-    dy = random_ops.random_normal(shape=[100, 4])
-    with backprop.GradientTape() as gradient_tape:
-      gradient_tape.watch(x)
-      x1, _ = array_ops.split(x, num_or_size_splits=2, axis=1)
-      y1 = x1 ** 2.
-      y = array_ops.concat([y1, t], axis=1)
-
-    dx = gradient_tape.gradient(y, x, output_gradients=dy)
+  def testZerosCacheDoesntLeakAcrossGraphs(self):
+    with context.graph_mode():
+      def get_grad():
+        with ops.Graph().as_default(), self.test_session():
+          t = constant_op.constant(1, dtype=dtypes.float32, shape=(10, 4))
+          x = constant_op.constant(2, dtype=dtypes.float32, shape=(10, 4))
+          with backprop.GradientTape() as gt:
+            tape.watch(x)
+            x1, _ = array_ops.split(x, num_or_size_splits=2, axis=1)
+            y1 = x1**2
+            y = array_ops.concat([y1, t], axis=1)
+          return self.evaluate(gt.gradient(y, x))
+
+      grad1 = get_grad()
+      grad2 = get_grad()
+
+      self.assertAllEqual(grad1, grad2)
 
 
 if __name__ == '__main__':
-- 
cgit v1.2.3


From 76e8f7b7fdf89b131e0406022129d5dde6b89e40 Mon Sep 17 00:00:00 2001
From: Anna R 
Date: Tue, 24 Jul 2018 14:02:04 -0700
Subject: Initial API compatibility script for TF2.0. I am pretty much reusing
 1.0 conversion script but passing V2 data. Also, remove code from
 tf_update.py which is also in ast_edits.py.

PiperOrigin-RevId: 205887317
---
 tensorflow/tools/compatibility/BUILD               |  57 +++
 tensorflow/tools/compatibility/renames_v2.py       | 134 ++++++
 .../compatibility/testdata/test_file_v1_10.py      |  34 ++
 tensorflow/tools/compatibility/tf_upgrade.py       | 486 +--------------------
 tensorflow/tools/compatibility/tf_upgrade_test.py  |   5 +-
 tensorflow/tools/compatibility/tf_upgrade_v2.py    | 115 +++++
 .../tools/compatibility/tf_upgrade_v2_test.py      |  83 ++++
 tensorflow/tools/compatibility/update/BUILD        |  15 +
 .../update/generate_v2_renames_map.py              | 103 +++++
 9 files changed, 547 insertions(+), 485 deletions(-)
 create mode 100644 tensorflow/tools/compatibility/renames_v2.py
 create mode 100644 tensorflow/tools/compatibility/testdata/test_file_v1_10.py
 create mode 100644 tensorflow/tools/compatibility/tf_upgrade_v2.py
 create mode 100644 tensorflow/tools/compatibility/tf_upgrade_v2_test.py
 create mode 100644 tensorflow/tools/compatibility/update/BUILD
 create mode 100644 tensorflow/tools/compatibility/update/generate_v2_renames_map.py

diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index b7bfb29aae..55792c51fe 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -8,10 +8,17 @@ load(
     "tf_cc_test",  # @unused
 )
 
+py_library(
+    name = "ast_edits",
+    srcs = ["ast_edits.py"],
+    srcs_version = "PY2AND3",
+)
+
 py_binary(
     name = "tf_upgrade",
     srcs = ["tf_upgrade.py"],
     srcs_version = "PY2AND3",
+    deps = [":ast_edits"],
 )
 
 py_test(
@@ -26,6 +33,28 @@ py_test(
     ],
 )
 
+py_binary(
+    name = "tf_upgrade_v2",
+    srcs = [
+        "renames_v2.py",
+        "tf_upgrade_v2.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [":ast_edits"],
+)
+
+py_test(
+    name = "tf_upgrade_v2_test",
+    srcs = ["tf_upgrade_v2_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":tf_upgrade_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "@six_archive//:six",
+    ],
+)
+
 # Keep for reference, this test will succeed in 0.11 but fail in 1.0
 # py_test(
 #     name = "test_file_v0_11",
@@ -62,9 +91,37 @@ py_test(
     ],
 )
 
+genrule(
+    name = "generate_upgraded_file_v2",
+    testonly = 1,
+    srcs = ["testdata/test_file_v1_10.py"],
+    outs = [
+        "test_file_v2_0.py",
+        "report_v2.txt",
+    ],
+    cmd = ("$(location :tf_upgrade_v2)" +
+           " --infile $(location testdata/test_file_v1_10.py)" +
+           " --outfile $(location test_file_v2_0.py)" +
+           " --reportfile $(location report_v2.txt)"),
+    tools = [":tf_upgrade_v2"],
+)
+
+py_test(
+    name = "test_file_v2_0",
+    size = "small",
+    srcs = ["test_file_v2_0.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
 exports_files(
     [
+        "ast_edits.py",
         "tf_upgrade.py",
+        "renames_v2.py",
         "testdata/test_file_v0_11.py",
+        "testdata/test_file_v1_10.py",
     ],
 )
diff --git a/tensorflow/tools/compatibility/renames_v2.py b/tensorflow/tools/compatibility/renames_v2.py
new file mode 100644
index 0000000000..216aa41b60
--- /dev/null
+++ b/tensorflow/tools/compatibility/renames_v2.py
@@ -0,0 +1,134 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=line-too-long
+"""List of renames to apply when converting from TF 1.0 to TF 2.0.
+
+THIS FILE IS AUTOGENERATED: To update, please run:
+  bazel build tensorflow/tools/compatibility/update:generate_v2_renames_map
+  bazel-bin/tensorflow/tools/compatibility/update/generate_v2_renames_map
+This file should be updated whenever endpoints are deprecated.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+renames = {
+    'tf.acos': 'tf.math.acos',
+    'tf.acosh': 'tf.math.acosh',
+    'tf.add': 'tf.math.add',
+    'tf.as_string': 'tf.dtypes.as_string',
+    'tf.asin': 'tf.math.asin',
+    'tf.asinh': 'tf.math.asinh',
+    'tf.atan': 'tf.math.atan',
+    'tf.atan2': 'tf.math.atan2',
+    'tf.atanh': 'tf.math.atanh',
+    'tf.batch_to_space_nd': 'tf.manip.batch_to_space_nd',
+    'tf.betainc': 'tf.math.betainc',
+    'tf.ceil': 'tf.math.ceil',
+    'tf.check_numerics': 'tf.debugging.check_numerics',
+    'tf.cholesky': 'tf.linalg.cholesky',
+    'tf.cos': 'tf.math.cos',
+    'tf.cosh': 'tf.math.cosh',
+    'tf.cross': 'tf.linalg.cross',
+    'tf.decode_base64': 'tf.io.decode_base64',
+    'tf.decode_compressed': 'tf.io.decode_compressed',
+    'tf.decode_json_example': 'tf.io.decode_json_example',
+    'tf.decode_raw': 'tf.io.decode_raw',
+    'tf.dequantize': 'tf.quantization.dequantize',
+    'tf.diag': 'tf.linalg.tensor_diag',
+    'tf.diag_part': 'tf.linalg.tensor_diag_part',
+    'tf.digamma': 'tf.math.digamma',
+    'tf.encode_base64': 'tf.io.encode_base64',
+    'tf.equal': 'tf.math.equal',
+    'tf.erfc': 'tf.math.erfc',
+    'tf.exp': 'tf.math.exp',
+    'tf.expm1': 'tf.math.expm1',
+    'tf.extract_image_patches': 'tf.image.extract_image_patches',
+    'tf.fake_quant_with_min_max_args': 'tf.quantization.fake_quant_with_min_max_args',
+    'tf.fake_quant_with_min_max_args_gradient': 'tf.quantization.fake_quant_with_min_max_args_gradient',
+    'tf.fake_quant_with_min_max_vars': 'tf.quantization.fake_quant_with_min_max_vars',
+    'tf.fake_quant_with_min_max_vars_gradient': 'tf.quantization.fake_quant_with_min_max_vars_gradient',
+    'tf.fake_quant_with_min_max_vars_per_channel': 'tf.quantization.fake_quant_with_min_max_vars_per_channel',
+    'tf.fake_quant_with_min_max_vars_per_channel_gradient': 'tf.quantization.fake_quant_with_min_max_vars_per_channel_gradient',
+    'tf.fft': 'tf.spectral.fft',
+    'tf.floor': 'tf.math.floor',
+    'tf.gather_nd': 'tf.manip.gather_nd',
+    'tf.greater': 'tf.math.greater',
+    'tf.greater_equal': 'tf.math.greater_equal',
+    'tf.ifft': 'tf.spectral.ifft',
+    'tf.igamma': 'tf.math.igamma',
+    'tf.igammac': 'tf.math.igammac',
+    'tf.invert_permutation': 'tf.math.invert_permutation',
+    'tf.is_finite': 'tf.debugging.is_finite',
+    'tf.is_inf': 'tf.debugging.is_inf',
+    'tf.is_nan': 'tf.debugging.is_nan',
+    'tf.less': 'tf.math.less',
+    'tf.less_equal': 'tf.math.less_equal',
+    'tf.lgamma': 'tf.math.lgamma',
+    'tf.log': 'tf.math.log',
+    'tf.log1p': 'tf.math.log1p',
+    'tf.logical_and': 'tf.math.logical_and',
+    'tf.logical_not': 'tf.math.logical_not',
+    'tf.logical_or': 'tf.math.logical_or',
+    'tf.matching_files': 'tf.io.matching_files',
+    'tf.matrix_band_part': 'tf.linalg.band_part',
+    'tf.matrix_determinant': 'tf.linalg.det',
+    'tf.matrix_diag': 'tf.linalg.diag',
+    'tf.matrix_diag_part': 'tf.linalg.diag_part',
+    'tf.matrix_inverse': 'tf.linalg.inv',
+    'tf.matrix_set_diag': 'tf.linalg.set_diag',
+    'tf.matrix_solve': 'tf.linalg.solve',
+    'tf.matrix_triangular_solve': 'tf.linalg.triangular_solve',
+    'tf.maximum': 'tf.math.maximum',
+    'tf.minimum': 'tf.math.minimum',
+    'tf.not_equal': 'tf.math.not_equal',
+    'tf.parse_tensor': 'tf.io.parse_tensor',
+    'tf.polygamma': 'tf.math.polygamma',
+    'tf.qr': 'tf.linalg.qr',
+    'tf.quantized_concat': 'tf.quantization.quantized_concat',
+    'tf.read_file': 'tf.io.read_file',
+    'tf.reciprocal': 'tf.math.reciprocal',
+    'tf.regex_replace': 'tf.strings.regex_replace',
+    'tf.reshape': 'tf.manip.reshape',
+    'tf.reverse': 'tf.manip.reverse',
+    'tf.reverse_v2': 'tf.manip.reverse',
+    'tf.rint': 'tf.math.rint',
+    'tf.rsqrt': 'tf.math.rsqrt',
+    'tf.scatter_nd': 'tf.manip.scatter_nd',
+    'tf.segment_max': 'tf.math.segment_max',
+    'tf.segment_mean': 'tf.math.segment_mean',
+    'tf.segment_min': 'tf.math.segment_min',
+    'tf.segment_prod': 'tf.math.segment_prod',
+    'tf.segment_sum': 'tf.math.segment_sum',
+    'tf.sin': 'tf.math.sin',
+    'tf.sinh': 'tf.math.sinh',
+    'tf.space_to_batch_nd': 'tf.manip.space_to_batch_nd',
+    'tf.squared_difference': 'tf.math.squared_difference',
+    'tf.string_join': 'tf.strings.join',
+    'tf.string_strip': 'tf.strings.strip',
+    'tf.string_to_hash_bucket': 'tf.strings.to_hash_bucket',
+    'tf.string_to_hash_bucket_fast': 'tf.strings.to_hash_bucket_fast',
+    'tf.string_to_hash_bucket_strong': 'tf.strings.to_hash_bucket_strong',
+    'tf.string_to_number': 'tf.strings.to_number',
+    'tf.substr': 'tf.strings.substr',
+    'tf.tan': 'tf.math.tan',
+    'tf.tile': 'tf.manip.tile',
+    'tf.unsorted_segment_max': 'tf.math.unsorted_segment_max',
+    'tf.unsorted_segment_min': 'tf.math.unsorted_segment_min',
+    'tf.unsorted_segment_prod': 'tf.math.unsorted_segment_prod',
+    'tf.unsorted_segment_sum': 'tf.math.unsorted_segment_sum',
+    'tf.write_file': 'tf.io.write_file',
+    'tf.zeta': 'tf.math.zeta'
+}
diff --git a/tensorflow/tools/compatibility/testdata/test_file_v1_10.py b/tensorflow/tools/compatibility/testdata/test_file_v1_10.py
new file mode 100644
index 0000000000..a49035a1a0
--- /dev/null
+++ b/tensorflow/tools/compatibility/testdata/test_file_v1_10.py
@@ -0,0 +1,34 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tf upgrader."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import tensorflow as tf
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test as test_lib
+
+
+class TestUpgrade(test_util.TensorFlowTestCase):
+  """Test various APIs that have been changed in 2.0."""
+
+  def testRenames(self):
+    with self.test_session():
+      self.assertAllClose(1.04719755, tf.acos(0.5).eval())
+      self.assertAllClose(0.5, tf.rsqrt(4.0).eval())
+
+if __name__ == "__main__":
+  test_lib.main()
diff --git a/tensorflow/tools/compatibility/tf_upgrade.py b/tensorflow/tools/compatibility/tf_upgrade.py
index 1f8833582a..96705b1a4c 100644
--- a/tensorflow/tools/compatibility/tf_upgrade.py
+++ b/tensorflow/tools/compatibility/tf_upgrade.py
@@ -19,491 +19,11 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
-import ast
-import collections
-import os
-import shutil
-import sys
-import tempfile
-import traceback
 
+from tensorflow.tools.compatibility import ast_edits
 
-class APIChangeSpec(object):
-  """This class defines the transformations that need to happen.
 
-  This class must provide the following fields:
-
-  * `function_keyword_renames`: maps function names to a map of old -> new
-    argument names
-  * `function_renames`: maps function names to new function names
-  * `change_to_function`: a set of function names that have changed (for
-    notifications)
-  * `function_reorders`: maps functions whose argument order has changed to the
-    list of arguments in the new order
-  * `function_handle`: maps function names to custom handlers for the function
-
-  For an example, see `TFAPIChangeSpec`.
-  """
-
-
-class _FileEditTuple(
-    collections.namedtuple("_FileEditTuple",
-                           ["comment", "line", "start", "old", "new"])):
-  """Each edit that is recorded by a _FileEditRecorder.
-
-  Fields:
-    comment: A description of the edit and why it was made.
-    line: The line number in the file where the edit occurs (1-indexed).
-    start: The line number in the file where the edit occurs (0-indexed).
-    old: text string to remove (this must match what was in file).
-    new: text string to add in place of `old`.
-  """
-
-  __slots__ = ()
-
-
-class _FileEditRecorder(object):
-  """Record changes that need to be done to the file."""
-
-  def __init__(self, filename):
-    # all edits are lists of chars
-    self._filename = filename
-
-    self._line_to_edit = collections.defaultdict(list)
-    self._errors = []
-
-  def process(self, text):
-    """Process a list of strings, each corresponding to the recorded changes.
-
-    Args:
-      text: A list of lines of text (assumed to contain newlines)
-    Returns:
-      A tuple of the modified text and a textual description of what is done.
-    Raises:
-      ValueError: if substitution source location does not have expected text.
-    """
-
-    change_report = ""
-
-    # Iterate of each line
-    for line, edits in self._line_to_edit.items():
-      offset = 0
-      # sort by column so that edits are processed in order in order to make
-      # indexing adjustments cumulative for changes that change the string
-      # length
-      edits.sort(key=lambda x: x.start)
-
-      # Extract each line to a list of characters, because mutable lists
-      # are editable, unlike immutable strings.
-      char_array = list(text[line - 1])
-
-      # Record a description of the change
-      change_report += "%r Line %d\n" % (self._filename, line)
-      change_report += "-" * 80 + "\n\n"
-      for e in edits:
-        change_report += "%s\n" % e.comment
-      change_report += "\n    Old: %s" % (text[line - 1])
-
-      # Make underscore buffers for underlining where in the line the edit was
-      change_list = [" "] * len(text[line - 1])
-      change_list_new = [" "] * len(text[line - 1])
-
-      # Iterate for each edit
-      for e in edits:
-        # Create effective start, end by accounting for change in length due
-        # to previous edits
-        start_eff = e.start + offset
-        end_eff = start_eff + len(e.old)
-
-        # Make sure the edit is changing what it should be changing
-        old_actual = "".join(char_array[start_eff:end_eff])
-        if old_actual != e.old:
-          raise ValueError("Expected text %r but got %r" %
-                           ("".join(e.old), "".join(old_actual)))
-        # Make the edit
-        char_array[start_eff:end_eff] = list(e.new)
-
-        # Create the underline highlighting of the before and after
-        change_list[e.start:e.start + len(e.old)] = "~" * len(e.old)
-        change_list_new[start_eff:end_eff] = "~" * len(e.new)
-
-        # Keep track of how to generate effective ranges
-        offset += len(e.new) - len(e.old)
-
-      # Finish the report comment
-      change_report += "         %s\n" % "".join(change_list)
-      text[line - 1] = "".join(char_array)
-      change_report += "    New: %s" % (text[line - 1])
-      change_report += "         %s\n\n" % "".join(change_list_new)
-    return "".join(text), change_report, self._errors
-
-  def add(self, comment, line, start, old, new, error=None):
-    """Add a new change that is needed.
-
-    Args:
-      comment: A description of what was changed
-      line: Line number (1 indexed)
-      start: Column offset (0 indexed)
-      old: old text
-      new: new text
-      error: this "edit" is something that cannot be fixed automatically
-    Returns:
-      None
-    """
-
-    self._line_to_edit[line].append(
-        _FileEditTuple(comment, line, start, old, new))
-    if error:
-      self._errors.append("%s:%d: %s" % (self._filename, line, error))
-
-
-class _ASTCallVisitor(ast.NodeVisitor):
-  """AST Visitor that processes function calls.
-
-  Updates function calls from old API version to new API version using a given
-  change spec.
-  """
-
-  def __init__(self, filename, lines, api_change_spec):
-    self._filename = filename
-    self._file_edit = _FileEditRecorder(filename)
-    self._lines = lines
-    self._api_change_spec = api_change_spec
-
-  def process(self, lines):
-    return self._file_edit.process(lines)
-
-  def generic_visit(self, node):
-    ast.NodeVisitor.generic_visit(self, node)
-
-  def _rename_functions(self, node, full_name):
-    function_renames = self._api_change_spec.function_renames
-    try:
-      new_name = function_renames[full_name]
-      self._file_edit.add("Renamed function %r to %r" % (full_name, new_name),
-                          node.lineno, node.col_offset, full_name, new_name)
-    except KeyError:
-      pass
-
-  def _get_attribute_full_path(self, node):
-    """Traverse an attribute to generate a full name e.g. tf.foo.bar.
-
-    Args:
-      node: A Node of type Attribute.
-
-    Returns:
-      a '.'-delimited full-name or None if the tree was not a simple form.
-      i.e. `foo()+b).bar` returns None, while `a.b.c` would return "a.b.c".
-    """
-    curr = node
-    items = []
-    while not isinstance(curr, ast.Name):
-      if not isinstance(curr, ast.Attribute):
-        return None
-      items.append(curr.attr)
-      curr = curr.value
-    items.append(curr.id)
-    return ".".join(reversed(items))
-
-  def _find_true_position(self, node):
-    """Return correct line number and column offset for a given node.
-
-    This is necessary mainly because ListComp's location reporting reports
-    the next token after the list comprehension list opening.
-
-    Args:
-      node: Node for which we wish to know the lineno and col_offset
-    """
-    import re
-    find_open = re.compile("^\s*(\\[).*$")
-    find_string_chars = re.compile("['\"]")
-
-    if isinstance(node, ast.ListComp):
-      # Strangely, ast.ListComp returns the col_offset of the first token
-      # after the '[' token which appears to be a bug. Workaround by
-      # explicitly finding the real start of the list comprehension.
-      line = node.lineno
-      col = node.col_offset
-      # loop over lines
-      while 1:
-        # Reverse the text to and regular expression search for whitespace
-        text = self._lines[line - 1]
-        reversed_preceding_text = text[:col][::-1]
-        # First find if a [ can be found with only whitespace between it and
-        # col.
-        m = find_open.match(reversed_preceding_text)
-        if m:
-          new_col_offset = col - m.start(1) - 1
-          return line, new_col_offset
-        else:
-          if (reversed_preceding_text == "" or
-              reversed_preceding_text.isspace()):
-            line = line - 1
-            prev_line = self._lines[line - 1]
-            # TODO(aselle):
-            # this is poor comment detection, but it is good enough for
-            # cases where the comment does not contain string literal starting/
-            # ending characters. If ast gave us start and end locations of the
-            # ast nodes rather than just start, we could use string literal
-            # node ranges to filter out spurious #'s that appear in string
-            # literals.
-            comment_start = prev_line.find("#")
-            if comment_start == -1:
-              col = len(prev_line) - 1
-            elif find_string_chars.search(prev_line[comment_start:]) is None:
-              col = comment_start
-            else:
-              return None, None
-          else:
-            return None, None
-    # Most other nodes return proper locations (with notably does not), but
-    # it is not possible to use that in an argument.
-    return node.lineno, node.col_offset
-
-  def visit_Call(self, node):  # pylint: disable=invalid-name
-    """Handle visiting a call node in the AST.
-
-    Args:
-      node: Current Node
-    """
-
-    # Find a simple attribute name path e.g. "tf.foo.bar"
-    full_name = self._get_attribute_full_path(node.func)
-
-    # Make sure the func is marked as being part of a call
-    node.func.is_function_for_call = True
-
-    if full_name:
-      # Call special handlers
-      function_handles = self._api_change_spec.function_handle
-      if full_name in function_handles:
-        function_handles[full_name](self._file_edit, node)
-
-      # Examine any non-keyword argument and make it into a keyword argument
-      # if reordering required.
-      function_reorders = self._api_change_spec.function_reorders
-      function_keyword_renames = (
-          self._api_change_spec.function_keyword_renames)
-
-      if full_name in function_reorders:
-        reordered = function_reorders[full_name]
-        for idx, arg in enumerate(node.args):
-          lineno, col_offset = self._find_true_position(arg)
-          if lineno is None or col_offset is None:
-            self._file_edit.add(
-                "Failed to add keyword %r to reordered function %r" %
-                (reordered[idx], full_name),
-                arg.lineno,
-                arg.col_offset,
-                "",
-                "",
-                error="A necessary keyword argument failed to be inserted.")
-          else:
-            keyword_arg = reordered[idx]
-            if (full_name in function_keyword_renames and
-                keyword_arg in function_keyword_renames[full_name]):
-              keyword_arg = function_keyword_renames[full_name][keyword_arg]
-            self._file_edit.add("Added keyword %r to reordered function %r" %
-                                (reordered[idx], full_name), lineno, col_offset,
-                                "", keyword_arg + "=")
-
-      # Examine each keyword argument and convert it to the final renamed form
-      renamed_keywords = ({} if full_name not in function_keyword_renames else
-                          function_keyword_renames[full_name])
-      for keyword in node.keywords:
-        argkey = keyword.arg
-        argval = keyword.value
-
-        if argkey in renamed_keywords:
-          argval_lineno, argval_col_offset = self._find_true_position(argval)
-          if argval_lineno is not None and argval_col_offset is not None:
-            # TODO(aselle): We should scan backward to find the start of the
-            # keyword key. Unfortunately ast does not give you the location of
-            # keyword keys, so we are forced to infer it from the keyword arg
-            # value.
-            key_start = argval_col_offset - len(argkey) - 1
-            key_end = key_start + len(argkey) + 1
-            if (self._lines[argval_lineno - 1][key_start:key_end] == argkey +
-                "="):
-              self._file_edit.add("Renamed keyword argument from %r to %r" %
-                                  (argkey,
-                                   renamed_keywords[argkey]), argval_lineno,
-                                  argval_col_offset - len(argkey) - 1,
-                                  argkey + "=", renamed_keywords[argkey] + "=")
-              continue
-          self._file_edit.add(
-              "Failed to rename keyword argument from %r to %r" %
-              (argkey, renamed_keywords[argkey]),
-              argval.lineno,
-              argval.col_offset - len(argkey) - 1,
-              "",
-              "",
-              error="Failed to find keyword lexographically. Fix manually.")
-
-    ast.NodeVisitor.generic_visit(self, node)
-
-  def visit_Attribute(self, node):  # pylint: disable=invalid-name
-    """Handle bare Attributes i.e. [tf.foo, tf.bar].
-
-    Args:
-      node: Node that is of type ast.Attribute
-    """
-    full_name = self._get_attribute_full_path(node)
-    if full_name:
-      self._rename_functions(node, full_name)
-    if full_name in self._api_change_spec.change_to_function:
-      if not hasattr(node, "is_function_for_call"):
-        new_text = full_name + "()"
-        self._file_edit.add("Changed %r to %r" % (full_name, new_text),
-                            node.lineno, node.col_offset, full_name, new_text)
-
-    ast.NodeVisitor.generic_visit(self, node)
-
-
-class ASTCodeUpgrader(object):
-  """Handles upgrading a set of Python files using a given API change spec."""
-
-  def __init__(self, api_change_spec):
-    if not isinstance(api_change_spec, APIChangeSpec):
-      raise TypeError("Must pass APIChangeSpec to ASTCodeUpgrader, got %s" %
-                      type(api_change_spec))
-    self._api_change_spec = api_change_spec
-
-  def process_file(self, in_filename, out_filename):
-    """Process the given python file for incompatible changes.
-
-    Args:
-      in_filename: filename to parse
-      out_filename: output file to write to
-    Returns:
-      A tuple representing number of files processed, log of actions, errors
-    """
-
-    # Write to a temporary file, just in case we are doing an implace modify.
-    with open(in_filename, "r") as in_file, \
-        tempfile.NamedTemporaryFile("w", delete=False) as temp_file:
-      ret = self.process_opened_file(in_filename, in_file, out_filename,
-                                     temp_file)
-
-    shutil.move(temp_file.name, out_filename)
-    return ret
-
-  # Broad exceptions are required here because ast throws whatever it wants.
-  # pylint: disable=broad-except
-  def process_opened_file(self, in_filename, in_file, out_filename, out_file):
-    """Process the given python file for incompatible changes.
-
-    This function is split out to facilitate StringIO testing from
-    tf_upgrade_test.py.
-
-    Args:
-      in_filename: filename to parse
-      in_file: opened file (or StringIO)
-      out_filename: output file to write to
-      out_file: opened file (or StringIO)
-    Returns:
-      A tuple representing number of files processed, log of actions, errors
-    """
-    process_errors = []
-    text = "-" * 80 + "\n"
-    text += "Processing file %r\n outputting to %r\n" % (in_filename,
-                                                         out_filename)
-    text += "-" * 80 + "\n\n"
-
-    parsed_ast = None
-    lines = in_file.readlines()
-    try:
-      parsed_ast = ast.parse("".join(lines))
-    except Exception:
-      text += "Failed to parse %r\n\n" % in_filename
-      text += traceback.format_exc()
-    if parsed_ast:
-      visitor = _ASTCallVisitor(in_filename, lines, self._api_change_spec)
-      visitor.visit(parsed_ast)
-      out_text, new_text, process_errors = visitor.process(lines)
-      text += new_text
-      if out_file:
-        out_file.write(out_text)
-    text += "\n"
-    return 1, text, process_errors
-
-  # pylint: enable=broad-except
-
-  def process_tree(self, root_directory, output_root_directory,
-                   copy_other_files):
-    """Processes upgrades on an entire tree of python files in place.
-
-    Note that only Python files. If you have custom code in other languages,
-    you will need to manually upgrade those.
-
-    Args:
-      root_directory: Directory to walk and process.
-      output_root_directory: Directory to use as base.
-      copy_other_files: Copy files that are not touched by this converter.
-
-    Returns:
-      A tuple of files processed, the report string ofr all files, and errors
-    """
-
-    # make sure output directory doesn't exist
-    if output_root_directory and os.path.exists(output_root_directory):
-      print("Output directory %r must not already exist." %
-            (output_root_directory))
-      sys.exit(1)
-
-    # make sure output directory does not overlap with root_directory
-    norm_root = os.path.split(os.path.normpath(root_directory))
-    norm_output = os.path.split(os.path.normpath(output_root_directory))
-    if norm_root == norm_output:
-      print("Output directory %r same as input directory %r" %
-            (root_directory, output_root_directory))
-      sys.exit(1)
-
-    # Collect list of files to process (we do this to correctly handle if the
-    # user puts the output directory in some sub directory of the input dir)
-    files_to_process = []
-    files_to_copy = []
-    for dir_name, _, file_list in os.walk(root_directory):
-      py_files = [f for f in file_list if f.endswith(".py")]
-      copy_files = [f for f in file_list if not f.endswith(".py")]
-      for filename in py_files:
-        fullpath = os.path.join(dir_name, filename)
-        fullpath_output = os.path.join(output_root_directory,
-                                       os.path.relpath(fullpath,
-                                                       root_directory))
-        files_to_process.append((fullpath, fullpath_output))
-      if copy_other_files:
-        for filename in copy_files:
-          fullpath = os.path.join(dir_name, filename)
-          fullpath_output = os.path.join(output_root_directory,
-                                         os.path.relpath(
-                                             fullpath, root_directory))
-          files_to_copy.append((fullpath, fullpath_output))
-
-    file_count = 0
-    tree_errors = []
-    report = ""
-    report += ("=" * 80) + "\n"
-    report += "Input tree: %r\n" % root_directory
-    report += ("=" * 80) + "\n"
-
-    for input_path, output_path in files_to_process:
-      output_directory = os.path.dirname(output_path)
-      if not os.path.isdir(output_directory):
-        os.makedirs(output_directory)
-      file_count += 1
-      _, l_report, l_errors = self.process_file(input_path, output_path)
-      tree_errors += l_errors
-      report += l_report
-    for input_path, output_path in files_to_copy:
-      output_directory = os.path.dirname(output_path)
-      if not os.path.isdir(output_directory):
-        os.makedirs(output_directory)
-      shutil.copy(input_path, output_path)
-    return file_count, report, tree_errors
-
-
-class TFAPIChangeSpec(APIChangeSpec):
+class TFAPIChangeSpec(ast_edits.APIChangeSpec):
   """List of maps that describe what changed in the API."""
 
   def __init__(self):
@@ -718,7 +238,7 @@ Simple usage:
       default="report.txt")
   args = parser.parse_args()
 
-  upgrade = ASTCodeUpgrader(TFAPIChangeSpec())
+  upgrade = ast_edits.ASTCodeUpgrader(TFAPIChangeSpec())
   report_text = None
   report_filename = args.report_filename
   files_processed = 0
diff --git a/tensorflow/tools/compatibility/tf_upgrade_test.py b/tensorflow/tools/compatibility/tf_upgrade_test.py
index 3d02eacba6..66325ea2ad 100644
--- a/tensorflow/tools/compatibility/tf_upgrade_test.py
+++ b/tensorflow/tools/compatibility/tf_upgrade_test.py
@@ -22,6 +22,7 @@ import tempfile
 import six
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test as test_lib
+from tensorflow.tools.compatibility import ast_edits
 from tensorflow.tools.compatibility import tf_upgrade
 
 
@@ -36,7 +37,7 @@ class TestUpgrade(test_util.TensorFlowTestCase):
   def _upgrade(self, old_file_text):
     in_file = six.StringIO(old_file_text)
     out_file = six.StringIO()
-    upgrader = tf_upgrade.ASTCodeUpgrader(tf_upgrade.TFAPIChangeSpec())
+    upgrader = ast_edits.ASTCodeUpgrader(tf_upgrade.TFAPIChangeSpec())
     count, report, errors = (
         upgrader.process_opened_file("test.py", in_file,
                                      "test_out.py", out_file))
@@ -139,7 +140,7 @@ class TestUpgradeFiles(test_util.TensorFlowTestCase):
     upgraded = "tf.multiply(a, b)\n"
     temp_file.write(original)
     temp_file.close()
-    upgrader = tf_upgrade.ASTCodeUpgrader(tf_upgrade.TFAPIChangeSpec())
+    upgrader = ast_edits.ASTCodeUpgrader(tf_upgrade.TFAPIChangeSpec())
     upgrader.process_file(temp_file.name, temp_file.name)
     self.assertAllEqual(open(temp_file.name).read(), upgraded)
     os.unlink(temp_file.name)
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2.py b/tensorflow/tools/compatibility/tf_upgrade_v2.py
new file mode 100644
index 0000000000..9702430a12
--- /dev/null
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2.py
@@ -0,0 +1,115 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Upgrader for Python scripts from 1.* TensorFlow to 2.0 TensorFlow."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+
+from tensorflow.tools.compatibility import ast_edits
+from tensorflow.tools.compatibility import renames_v2
+
+
+class TFAPIChangeSpec(ast_edits.APIChangeSpec):
+  """List of maps that describe what changed in the API."""
+
+  def __init__(self):
+    # Maps from a function name to a dictionary that describes how to
+    # map from an old argument keyword to the new argument keyword.
+    self.function_keyword_renames = {}
+
+    # Mapping from function to the new name of the function
+    self.function_renames = renames_v2.renames
+
+    # Variables that should be changed to functions.
+    self.change_to_function = {}
+
+    # Functions that were reordered should be changed to the new keyword args
+    # for safety, if positional arguments are used. If you have reversed the
+    # positional arguments yourself, this could do the wrong thing.
+    self.function_reorders = {}
+
+    # Specially handled functions.
+    self.function_handle = {}
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.RawDescriptionHelpFormatter,
+      description="""Convert a TensorFlow Python file to 2.0
+
+Simple usage:
+  tf_convert_v2.py --infile foo.py --outfile bar.py
+  tf_convert_v2.py --intree ~/code/old --outtree ~/code/new
+""")
+  parser.add_argument(
+      "--infile",
+      dest="input_file",
+      help="If converting a single file, the name of the file "
+      "to convert")
+  parser.add_argument(
+      "--outfile",
+      dest="output_file",
+      help="If converting a single file, the output filename.")
+  parser.add_argument(
+      "--intree",
+      dest="input_tree",
+      help="If converting a whole tree of files, the directory "
+      "to read from (relative or absolute).")
+  parser.add_argument(
+      "--outtree",
+      dest="output_tree",
+      help="If converting a whole tree of files, the output "
+      "directory (relative or absolute).")
+  parser.add_argument(
+      "--copyotherfiles",
+      dest="copy_other_files",
+      help=("If converting a whole tree of files, whether to "
+            "copy the other files."),
+      type=bool,
+      default=False)
+  parser.add_argument(
+      "--reportfile",
+      dest="report_filename",
+      help=("The name of the file where the report log is "
+            "stored."
+            "(default: %(default)s)"),
+      default="report.txt")
+  args = parser.parse_args()
+
+  upgrade = ast_edits.ASTCodeUpgrader(TFAPIChangeSpec())
+  report_text = None
+  report_filename = args.report_filename
+  files_processed = 0
+  if args.input_file:
+    files_processed, report_text, errors = upgrade.process_file(
+        args.input_file, args.output_file)
+    files_processed = 1
+  elif args.input_tree:
+    files_processed, report_text, errors = upgrade.process_tree(
+        args.input_tree, args.output_tree, args.copy_other_files)
+  else:
+    parser.print_help()
+  if report_text:
+    open(report_filename, "w").write(report_text)
+    print("TensorFlow 2.0 Upgrade Script")
+    print("-----------------------------")
+    print("Converted %d files\n" % files_processed)
+    print("Detected %d errors that require attention" % len(errors))
+    print("-" * 80)
+    print("\n".join(errors))
+    print("\nMake sure to read the detailed log %r\n" % report_filename)
diff --git a/tensorflow/tools/compatibility/tf_upgrade_v2_test.py b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
new file mode 100644
index 0000000000..57ac04de06
--- /dev/null
+++ b/tensorflow/tools/compatibility/tf_upgrade_v2_test.py
@@ -0,0 +1,83 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tf 2.0 upgrader."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import os
+import tempfile
+import six
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test as test_lib
+from tensorflow.tools.compatibility import ast_edits
+from tensorflow.tools.compatibility import tf_upgrade_v2
+
+
+class TestUpgrade(test_util.TensorFlowTestCase):
+  """Test various APIs that have been changed in 2.0.
+
+  We also test whether a converted file is executable. test_file_v1_10.py
+  aims to exhaustively test that API changes are convertible and actually
+  work when run with current TensorFlow.
+  """
+
+  def _upgrade(self, old_file_text):
+    in_file = six.StringIO(old_file_text)
+    out_file = six.StringIO()
+    upgrader = ast_edits.ASTCodeUpgrader(tf_upgrade_v2.TFAPIChangeSpec())
+    count, report, errors = (
+        upgrader.process_opened_file("test.py", in_file,
+                                     "test_out.py", out_file))
+    return count, report, errors, out_file.getvalue()
+
+  def testParseError(self):
+    _, report, unused_errors, unused_new_text = self._upgrade(
+        "import tensorflow as tf\na + \n")
+    self.assertTrue(report.find("Failed to parse") != -1)
+
+  def testReport(self):
+    text = "tf.acos(a)\n"
+    _, report, unused_errors, unused_new_text = self._upgrade(text)
+    # This is not a complete test, but it is a sanity test that a report
+    # is generating information.
+    self.assertTrue(report.find("Renamed function `tf.acos` to `tf.math.acos`"))
+
+  def testRename(self):
+    text = "tf.acos(a)\n"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, "tf.math.acos(a)\n")
+    text = "tf.rsqrt(tf.log(3.8))\n"
+    _, unused_report, unused_errors, new_text = self._upgrade(text)
+    self.assertEqual(new_text, "tf.math.rsqrt(tf.math.log(3.8))\n")
+
+
+class TestUpgradeFiles(test_util.TensorFlowTestCase):
+
+  def testInplace(self):
+    """Check to make sure we don't have a file system race."""
+    temp_file = tempfile.NamedTemporaryFile("w", delete=False)
+    original = "tf.acos(a, b)\n"
+    upgraded = "tf.math.acos(a, b)\n"
+    temp_file.write(original)
+    temp_file.close()
+    upgrader = ast_edits.ASTCodeUpgrader(tf_upgrade_v2.TFAPIChangeSpec())
+    upgrader.process_file(temp_file.name, temp_file.name)
+    self.assertAllEqual(open(temp_file.name).read(), upgraded)
+    os.unlink(temp_file.name)
+
+
+if __name__ == "__main__":
+  test_lib.main()
diff --git a/tensorflow/tools/compatibility/update/BUILD b/tensorflow/tools/compatibility/update/BUILD
new file mode 100644
index 0000000000..feb37c902e
--- /dev/null
+++ b/tensorflow/tools/compatibility/update/BUILD
@@ -0,0 +1,15 @@
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:private"])
+
+py_binary(
+    name = "generate_v2_renames_map",
+    srcs = ["generate_v2_renames_map.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/python:lib",
+        "//tensorflow/tools/common:public_api",
+        "//tensorflow/tools/common:traverse",
+    ],
+)
diff --git a/tensorflow/tools/compatibility/update/generate_v2_renames_map.py b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
new file mode 100644
index 0000000000..567eceb0b6
--- /dev/null
+++ b/tensorflow/tools/compatibility/update/generate_v2_renames_map.py
@@ -0,0 +1,103 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=line-too-long
+"""Script for updating tensorflow/tools/compatibility/renames_v2.py.
+
+To update renames_v2.py, run:
+  bazel build tensorflow/tools/compatibility/update:generate_v2_renames_map
+  bazel-bin/tensorflow/tools/compatibility/update/generate_v2_renames_map
+"""
+# pylint: enable=line-too-long
+
+import tensorflow as tf
+
+from tensorflow.python.lib.io import file_io
+from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_export
+from tensorflow.tools.common import public_api
+from tensorflow.tools.common import traverse
+
+
+_OUTPUT_FILE_PATH = 'third_party/tensorflow/tools/compatibility/renames_v2.py'
+_FILE_HEADER = """# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=line-too-long
+\"\"\"List of renames to apply when converting from TF 1.0 to TF 2.0.
+
+THIS FILE IS AUTOGENERATED: To update, please run:
+  bazel build tensorflow/tools/compatibility/update:generate_v2_renames_map
+  bazel-bin/tensorflow/tools/compatibility/update/generate_v2_renames_map
+This file should be updated whenever endpoints are deprecated.
+\"\"\"
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+"""
+
+
+def update_renames_v2(output_file_path):
+  """Writes a Python dictionary mapping deprecated to canonical API names.
+
+  Args:
+    output_file_path: File path to write output to. Any existing contents
+      would be replaced.
+  """
+  # Set of rename lines to write to output file in the form:
+  #   'tf.deprecated_name': 'tf.canonical_name'
+  rename_line_set = set()
+  # _tf_api_names attribute name
+  tensorflow_api_attr = tf_export.API_ATTRS[tf_export.TENSORFLOW_API_NAME].names
+
+  def visit(unused_path, unused_parent, children):
+    """Visitor that collects rename strings to add to rename_line_set."""
+    for child in children:
+      _, attr = tf_decorator.unwrap(child[1])
+      if not hasattr(attr, '__dict__'):
+        continue
+      api_names = attr.__dict__.get(tensorflow_api_attr, [])
+      deprecated_api_names = attr.__dict__.get('_tf_deprecated_api_names', [])
+      canonical_name = tf_export.get_canonical_name(
+          api_names, deprecated_api_names)
+      for name in deprecated_api_names:
+        rename_line_set.add('    \'tf.%s\': \'tf.%s\'' % (name, canonical_name))
+
+  visitor = public_api.PublicAPIVisitor(visit)
+  visitor.do_not_descend_map['tf'].append('contrib')
+  traverse.traverse(tf, visitor)
+
+  renames_file_text = '%srenames = {\n%s\n}\n' % (
+      _FILE_HEADER, ',\n'.join(sorted(rename_line_set)))
+  file_io.write_string_to_file(output_file_path, renames_file_text)
+
+
+def main(unused_argv):
+  update_renames_v2(_OUTPUT_FILE_PATH)
+
+
+if __name__ == '__main__':
+  tf.app.run(main=main)
-- 
cgit v1.2.3


From e9398c43cf470a7388df7d20baf6dd10a3b42edb Mon Sep 17 00:00:00 2001
From: Akshay Modi 
Date: Tue, 24 Jul 2018 14:10:01 -0700
Subject: Push tensors from client to workers.

At times, a server cannot open a reverse connection to the client. This is
required when using the _Send/_Recv ops and the client needs to send a tensor
to the server (tensors are pulled). Instead, this adds a way to push the
tensors directly from the client.

Currently, pushing tensors always happens in sync mode.

PiperOrigin-RevId: 205888825
---
 tensorflow/core/common_runtime/eager/context.cc    |  19 +++-
 tensorflow/core/common_runtime/eager/context.h     |   9 +-
 tensorflow/core/common_runtime/eager/execute.cc    | 122 ++++++++++++++++-----
 .../core/distributed_runtime/eager/eager_client.h  |   1 +
 .../eager/eager_service_impl.cc                    |  36 +++++-
 .../distributed_runtime/eager/eager_service_impl.h |   3 +
 .../eager/eager_service_impl_test.cc               |  79 ++++++++++++-
 .../eager/remote_execute_node.h                    |  19 ++--
 .../rpc/eager/grpc_eager_client.cc                 |   1 +
 .../rpc/eager/grpc_eager_service.cc                |  14 ++-
 .../rpc/eager/grpc_eager_service.h                 |  15 +++
 .../rpc/eager/grpc_eager_service_impl.cc           |   1 +
 .../rpc/eager/grpc_eager_service_impl.h            |   1 +
 tensorflow/core/protobuf/eager_service.proto       |  23 ++++
 tensorflow/python/eager/context.py                 |   5 +
 tensorflow/python/framework/ops.py                 |   5 +-
 16 files changed, 300 insertions(+), 53 deletions(-)

diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 1c5e9a2a31..5e0f0a45f8 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -17,8 +17,20 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/process_util.h"
 #include "tensorflow/core/lib/core/blocking_counter.h"
+#include "tensorflow/core/util/env_var.h"
 
 namespace tensorflow {
+namespace {
+
+bool ReadBoolFromEnvVar(StringPiece env_var_name, bool default_val) {
+  bool val;
+  if (ReadBoolFromEnvVar(env_var_name, default_val, &val).ok()) {
+    return val;
+  }
+  return default_val;
+}
+
+}  // namespace
 
 EagerContext::EagerContext(const SessionOptions& opts,
                            ContextDevicePlacementPolicy default_policy,
@@ -34,7 +46,8 @@ EagerContext::EagerContext(const SessionOptions& opts,
           local_device_manager_.get(), opts.env, TF_GRAPH_DEF_VERSION,
           &func_lib_def_, {}, thread_pool_.get())),
       log_device_placement_(opts.config.log_device_placement()),
-      async_default_(async) {
+      async_default_(async),
+      use_send_tensor_rpc_(false) {
   InitDeviceMapAndAsync();
   if (opts.config.inter_op_parallelism_threads() > 0) {
     runner_ = [this](std::function closure) {
@@ -66,7 +79,9 @@ EagerContext::EagerContext(
       remote_device_manager_(std::move(remote_device_manager)),
       server_(std::move(server)),
       remote_eager_workers_(std::move(remote_eager_workers)),
-      remote_contexts_(remote_contexts) {
+      remote_contexts_(remote_contexts),
+      use_send_tensor_rpc_(
+          ReadBoolFromEnvVar("TF_EAGER_REMOTE_USE_SEND_TENSOR_RPC", false)) {
   InitDeviceMapAndAsync();
 }
 #endif
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index d0563280bf..4a180e074d 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -182,6 +182,11 @@ class EagerContext {
 #ifndef __ANDROID__
   Status GetClientAndContextID(Device* device, eager::EagerClient** client,
                                uint64* context_id);
+
+  // If true, then tensors should be shipped across processes via the
+  // EagerService.SendTensor RPC. If false, _Send/_Recv ops should be used
+  // instead (which in-turn use WorkerService.RecvTensor RPCs.
+  bool UseSendTensorRPC() { return use_send_tensor_rpc_; }
 #endif
  private:
   void InitDeviceMapAndAsync();
@@ -239,16 +244,18 @@ class EagerContext {
 
   const std::unique_ptr remote_device_manager_;
 
+#ifndef __ANDROID__
   // The server_ is not const since we release it when the context is destroyed.
   // Therefore the server_ object is not marked as const (even though it should
   // be).
-#ifndef __ANDROID__
   std::unique_ptr server_;
   const std::unique_ptr remote_eager_workers_;
 
   const gtl::FlatMap remote_contexts_;
   gtl::FlatMap>
       device_to_client_cache_;
+
+  const bool use_send_tensor_rpc_;
 #endif
 };
 
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 27d0cd611f..7ea78b63d9 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -585,6 +585,87 @@ Status EagerLocalExecute(EagerOperation* op,
   return status;
 }
 
+std::function GetRemoteTensorDestructor(
+    EagerContext* ctx, eager::EagerClient* eager_client, uint64 context_id,
+    uint64 op_id, int output_num) {
+  return [ctx, eager_client, context_id, op_id, output_num]() {
+    std::unique_ptr request(new eager::EnqueueRequest);
+    request->set_context_id(context_id);
+
+    auto* handle_to_decref = request->add_queue()->mutable_handle_to_decref();
+    handle_to_decref->set_op_id(op_id);
+    handle_to_decref->set_output_num(output_num);
+
+    if (ctx->Async()) {
+      tensorflow::uint64 id = ctx->NextId();
+      auto* node =
+          new eager::RemoteExecuteNode(id, std::move(request), eager_client);
+      ctx->ExecutorAdd(node);
+    } else {
+      eager::EnqueueRequest* actual_request = request.release();
+      eager::EnqueueResponse* response = new eager::EnqueueResponse;
+      eager_client->EnqueueAsync(
+          actual_request, response,
+          [actual_request, response](const tensorflow::Status& s) {
+            delete actual_request;
+            delete response;
+          });
+    }
+
+    return tensorflow::Status::OK();
+  };
+}
+
+// When !ctx->UseSendTensorRPC(), then tensors are shipped between remote
+// devices by the receiver invoking the WorkerService.RecvTensor RPC *on the
+// sender* (Rendezvous::RecvAsync() invoked by the _Recv kernel).
+//
+// However, in some configurations the node that has the tensor to be copied
+// isn't running a server (WorkerService RPC interface). For such cases,
+// this function enables sending tensors using the EagerService.SendTensor RPC
+// *on the receiver*.
+Status EagerRemoteSendTensor(EagerContext* ctx, TensorHandle* h,
+                             Device* recv_device, TensorHandle** result) {
+  eager::EagerClient* eager_client;
+  uint64 context_id;
+  TF_RETURN_IF_ERROR(
+      ctx->GetClientAndContextID(recv_device, &eager_client, &context_id));
+
+  eager::SendTensorRequest request;
+  eager::SendTensorResponse response;
+
+  request.set_context_id(context_id);
+  request.set_op_id(ctx->NextId());
+  request.set_device_name(recv_device->name());
+
+  const Tensor* tensor;
+  TF_RETURN_IF_ERROR(h->Tensor(&tensor));
+  tensor->AsProtoTensorContent(request.add_tensors());
+
+  const tensorflow::uint64 id = request.op_id();
+
+  // TODO(nareshmodi): support making this call async.
+  Notification n;
+  Status status;
+  eager_client->SendTensorAsync(&request, &response,
+                                [&n, &status](const Status& s) {
+                                  status = s;
+                                  n.Notify();
+                                });
+  n.WaitForNotification();
+  if (!status.ok()) return status;
+
+  std::function destructor =
+      GetRemoteTensorDestructor(ctx, eager_client, context_id, id, 0);
+
+  *result = new TensorHandle(id, /*output_num=*/0, /*remote_shape_node_id=*/0,
+                             tensor->dtype(), std::move(destructor),
+                             recv_device, recv_device, ctx);
+  (*result)->SetRemoteShape(MakeUnique(tensor->shape()));
+
+  return Status::OK();
+}
+
 Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
                           int* num_retvals) {
 #ifdef __ANDROID__
@@ -598,10 +679,12 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
   TF_RETURN_IF_ERROR(
       ctx->GetClientAndContextID(op->Device(), &eager_client, &context_id));
 
-  eager::EnqueueRequest request;
+  std::unique_ptr request(new eager::EnqueueRequest);
   eager::EnqueueResponse response;
 
-  auto* remote_op = request.add_queue()->mutable_operation();
+  request->set_context_id(context_id);
+
+  auto* remote_op = request->add_queue()->mutable_operation();
 
   for (int i = 0; i < op->Inputs().size(); i++) {
     tensorflow::Device* input_device;
@@ -631,8 +714,6 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
   op->Attrs().FillAttrValueMap(remote_op->mutable_attrs());
   remote_op->set_device(op->Device()->name());
 
-  request.set_context_id(context_id);
-
   DataTypeVector output_dtypes;
   TF_RETURN_IF_ERROR(GetOutputDTypes(op, &output_dtypes));
 
@@ -654,32 +735,11 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
   for (int i = 0; i < *num_retvals; i++) {
     // TODO(nareshmodi): Change the callback to instead add the decref to a list
     // of pending decrefs that we can send as a batch with the next execute.
-    std::function callback = [ctx, eager_client, context_id, id, i]() {
-      eager::EnqueueRequest request;
-      request.set_context_id(context_id);
-
-      auto* handle_to_decref = request.add_queue()->mutable_handle_to_decref();
-      handle_to_decref->set_op_id(id);
-      handle_to_decref->set_output_num(i);
-
-      if (ctx->Async()) {
-        tensorflow::uint64 id = ctx->NextId();
-        auto* node = new eager::RemoteExecuteNode(id, request, eager_client);
-        ctx->ExecutorAdd(node);
-      } else {
-        Notification n;
-        eager::EnqueueResponse response;
-        eager_client->EnqueueAsync(
-            &request, &response,
-            [&n](const tensorflow::Status& s) { n.Notify(); });
-        n.WaitForNotification();
-      }
-
-      return tensorflow::Status::OK();
-    };
+    std::function destructor =
+        GetRemoteTensorDestructor(ctx, eager_client, context_id, id, i);
 
     retvals[i] = new TensorHandle(remote_op->id(), i, remote_node_id,
-                                  output_dtypes[i], std::move(callback),
+                                  output_dtypes[i], std::move(destructor),
                                   op_device, op_device, op->EagerContext());
   }
 
@@ -693,7 +753,7 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
     }
     // Unable to capture via std::move, so bind instead.
     auto* node = new eager::RemoteExecuteNode(
-        remote_node_id, request, eager_client, op->Inputs(),
+        remote_node_id, std::move(request), eager_client, op->Inputs(),
         std::bind(
             [](const gtl::InlinedVector& retvals,
                const Status& status, const eager::EnqueueResponse& response) {
@@ -710,7 +770,7 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals,
   } else {
     Notification n;
     Status status;
-    eager_client->EnqueueAsync(&request, &response,
+    eager_client->EnqueueAsync(request.get(), &response,
                                [&n, &status](const Status& s) {
                                  status = s;
                                  n.Notify();
@@ -939,6 +999,8 @@ Status EagerCopyToDevice(TensorHandle* h, EagerContext* ctx,
 
   if (sender_is_local && recver_is_local) {
     return LocalEagerCopyToDevice(h, ctx, recv_device, result);
+  } else if (ctx->UseSendTensorRPC() && sender_is_local && !recver_is_local) {
+    return EagerRemoteSendTensor(ctx, h, recv_device, result);
   } else {
     string wire_id = GetUniqueWireID();
 
diff --git a/tensorflow/core/distributed_runtime/eager/eager_client.h b/tensorflow/core/distributed_runtime/eager/eager_client.h
index 9ba8c8d80c..707f3234b9 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_client.h
+++ b/tensorflow/core/distributed_runtime/eager/eager_client.h
@@ -39,6 +39,7 @@ class EagerClient {
   CLIENT_METHOD(KeepAlive);
   CLIENT_METHOD(CloseContext);
   CLIENT_METHOD(RegisterFunction);
+  CLIENT_METHOD(SendTensor);
 
 #undef CLIENT_METHOD
 };
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
index 466e779fab..916c8720f0 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc
@@ -81,10 +81,11 @@ Status GetNumRetvals(tensorflow::EagerContext* context, const string& op_name,
 
 Status EagerServiceImpl::CreateContext(const CreateContextRequest* request,
                                        CreateContextResponse* response) {
-  //make sure env_ , env_->rendezvous_mgr available
+  // make sure env_ , env_->rendezvous_mgr available
   if (env_ == nullptr || env_->rendezvous_mgr == nullptr) {
-    return tensorflow::errors::Internal("invalid eager env_ or env_->rendezvous_mgr.");
-  } 
+    return tensorflow::errors::Internal(
+        "invalid eager env_ or env_->rendezvous_mgr.");
+  }
   std::vector devices;
 
   TF_RETURN_IF_ERROR(tensorflow::DeviceFactory::AddDevices(
@@ -266,6 +267,35 @@ Status EagerServiceImpl::RegisterFunction(
   return context->Context()->AddFunctionDef(request->function_def());
 }
 
+Status EagerServiceImpl::SendTensor(const SendTensorRequest* request,
+                                    SendTensorResponse* response) {
+  ServerContext* context = nullptr;
+  TF_RETURN_IF_ERROR(GetServerContext(request->context_id(), &context));
+  core::ScopedUnref context_unref(context);
+
+  tensorflow::gtl::InlinedVector tensors;
+  for (const auto& tensor_proto : request->tensors()) {
+    Tensor tensor;
+    if (!tensor.FromProto(tensor_proto)) {
+      return errors::InvalidArgument("Unable to parse tensor proto");
+    }
+
+    TensorHandle* tensor_handle =
+        new TensorHandle(tensor, nullptr, nullptr, nullptr);
+
+    TensorHandle* copied_handle = nullptr;
+    TF_RETURN_IF_ERROR(EagerCopyToDevice(tensor_handle, context->Context(),
+                                         request->device_name().c_str(),
+                                         &copied_handle));
+    tensors.push_back(copied_handle);
+    tensor_handle->Unref();
+  }
+
+  context->AddOperationOutputs(tensors, request->op_id());
+
+  return Status::OK();
+}
+
 tensorflow::Status EagerServiceImpl::GetServerContext(
     uint64 context_id, ServerContext** server_context) {
   mutex_lock l(contexts_mu_);
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.h b/tensorflow/core/distributed_runtime/eager/eager_service_impl.h
index b0e4aa84b9..718b4e2457 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.h
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.h
@@ -62,6 +62,9 @@ class EagerServiceImpl {
   Status RegisterFunction(const RegisterFunctionRequest* request,
                           RegisterFunctionResponse* response);
 
+  Status SendTensor(const SendTensorRequest* request,
+                    SendTensorResponse* response);
+
  protected:
   // This is the server-side execution context. All state regarding execution of
   // a client's ops is held in this server-side context (all generated tensors,
diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
index b98386ba86..d1f2a6da8f 100644
--- a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
+++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc
@@ -84,7 +84,7 @@ class EagerServiceImplTest : public ::testing::Test {
   std::unique_ptr device_mgr_;
 };
 
-void SetTensorProto(AttrValue* val) {
+void SetTensorProto(TensorProto* tensor_proto) {
   int64_t dims[] = {2, 2};
   float data[] = {1.0f, 2.0f, 3.0f, 4.0f};
   TF_Tensor* t = TF_AllocateTensor(
@@ -92,7 +92,7 @@ void SetTensorProto(AttrValue* val) {
   memcpy(TF_TensorData(t), &data[0], TF_TensorByteSize(t));
   tensorflow::Tensor tensor;
   TF_ASSERT_OK(tensorflow::TF_TensorToTensor(t, &tensor));
-  tensor.AsProtoTensorContent(val->mutable_tensor());
+  tensor.AsProtoTensorContent(tensor_proto);
   TF_DeleteTensor(t);
 }
 
@@ -175,7 +175,7 @@ TEST_F(EagerServiceImplTest, BasicTest) {
   val.set_type(tensorflow::DataType::DT_FLOAT);
   const_attrs.insert({"dtype", val});
   val.Clear();
-  SetTensorProto(&val);
+  SetTensorProto(val.mutable_tensor());
   const_attrs.insert({"value", val});
 
   AddOperationToEnqueueRequest(1, "Const", {}, const_attrs,
@@ -260,7 +260,7 @@ TEST_F(EagerServiceImplTest, BasicFunctionTest) {
   const_attrs.insert({"dtype", val});
   val.Clear();
 
-  SetTensorProto(&val);
+  SetTensorProto(val.mutable_tensor());
   const_attrs.insert({"value", val});
 
   AddOperationToEnqueueRequest(1, "Const", {}, const_attrs,
@@ -294,6 +294,77 @@ TEST_F(EagerServiceImplTest, BasicFunctionTest) {
                                                &close_context_response));
 }
 
+// Test creates a context and attempts to send a tensor (using the RPC), and
+// then use the tensor.
+TEST_F(EagerServiceImplTest, SendTensorTest) {
+  TestEagerServiceImpl eager_service_impl(&worker_env_);
+
+  CreateContextRequest request;
+  request.mutable_server_def()->set_job_name("localhost");
+  request.mutable_server_def()->set_task_index(0);
+  request.set_rendezvous_id(random::New64());
+  CreateContextResponse response;
+
+  TF_ASSERT_OK(eager_service_impl.CreateContext(&request, &response));
+
+  uint64 context_id = response.context_id();
+
+  SendTensorRequest send_tensor_request;
+  send_tensor_request.set_context_id(context_id);
+  send_tensor_request.set_op_id(1);
+  SetTensorProto(send_tensor_request.add_tensors());
+  SendTensorResponse send_tensor_response;
+
+  TF_ASSERT_OK(eager_service_impl.SendTensor(&send_tensor_request,
+                                             &send_tensor_response));
+
+  EnqueueRequest remote_enqueue_request;
+  remote_enqueue_request.set_context_id(context_id);
+  EnqueueResponse remote_enqueue_response;
+
+  std::unordered_map attrs;
+  AttrValue val;
+  val.Clear();
+  val.set_type(tensorflow::DataType::DT_FLOAT);
+  attrs.insert({"T", val});
+  val.Clear();
+  val.set_b(false);
+  attrs.insert({"transpose_a", val});
+  attrs.insert({"transpose_b", val});
+
+  AddOperationToEnqueueRequest(2, "MatMul", {{1, 0}, {1, 0}}, attrs,
+                               "/job:localhost/replica:0/task:0/device:CPU:0",
+                               &remote_enqueue_request);
+
+  TF_ASSERT_OK(eager_service_impl.Enqueue(&remote_enqueue_request,
+                                          &remote_enqueue_response));
+
+  const tensorflow::Tensor* t = nullptr;
+  tensorflow::TensorHandle* tensor_handle;
+  TF_ASSERT_OK(eager_service_impl.GetTensorHandle(
+      response.context_id(), RemoteTensorHandleInternal(2, 0), &tensor_handle));
+  TF_ASSERT_OK(tensor_handle->Tensor(&t));
+
+  Device* device = nullptr;
+  TF_ASSERT_OK(tensor_handle->Device(&device));
+  EXPECT_NE(device, nullptr);
+  EXPECT_EQ(device->name(), "/job:localhost/replica:0/task:0/device:CPU:0");
+
+  auto actual = t->flat();
+  EXPECT_EQ(4, actual.size());
+
+  EXPECT_EQ(7, actual(0));
+  EXPECT_EQ(10, actual(1));
+  EXPECT_EQ(15, actual(2));
+  EXPECT_EQ(22, actual(3));
+
+  CloseContextRequest close_context_request;
+  close_context_request.set_context_id(context_id);
+  CloseContextResponse close_context_response;
+  TF_ASSERT_OK(eager_service_impl.CloseContext(&close_context_request,
+                                               &close_context_response));
+}
+
 }  // namespace
 }  // namespace eager
 }  // namespace tensorflow
diff --git a/tensorflow/core/distributed_runtime/eager/remote_execute_node.h b/tensorflow/core/distributed_runtime/eager/remote_execute_node.h
index 28b68c3b88..0e3a68c4d8 100644
--- a/tensorflow/core/distributed_runtime/eager/remote_execute_node.h
+++ b/tensorflow/core/distributed_runtime/eager/remote_execute_node.h
@@ -29,8 +29,8 @@ namespace eager {
 class RemoteExecuteNode : public tensorflow::EagerNode {
  public:
   RemoteExecuteNode(
-      tensorflow::uint64 id, const tensorflow::eager::EnqueueRequest& request,
-      tensorflow::eager::EagerClient* eager_client,
+      tensorflow::uint64 id, std::unique_ptr request,
+      EagerClient* eager_client,
       const gtl::InlinedVector& inputs,
       std::function
           done_callback)
@@ -45,8 +45,8 @@ class RemoteExecuteNode : public tensorflow::EagerNode {
   }
 
   RemoteExecuteNode(tensorflow::uint64 id,
-                    const tensorflow::eager::EnqueueRequest& request,
-                    tensorflow::eager::EagerClient* eager_client)
+                    std::unique_ptr request,
+                    EagerClient* eager_client)
       : tensorflow::EagerNode(id),
         request_(std::move(request)),
         eager_client_(eager_client) {}
@@ -58,10 +58,10 @@ class RemoteExecuteNode : public tensorflow::EagerNode {
   }
 
   tensorflow::Status Run() override {
-    tensorflow::eager::EnqueueResponse response;
-    tensorflow::Status status;
+    EnqueueResponse response;
+    Status status;
     Notification n;
-    eager_client_->EnqueueAsync(&request_, &response,
+    eager_client_->EnqueueAsync(request_.get(), &response,
                                 [&n, &status](const tensorflow::Status& s) {
                                   status.Update(s);
                                   n.Notify();
@@ -76,9 +76,8 @@ class RemoteExecuteNode : public tensorflow::EagerNode {
   }
 
  private:
-  EnqueueRequest request_;
-  tensorflow::eager::EagerClient*
-      eager_client_;  // Not owned, and must outlive the RemoteExecuteNode.
+  std::unique_ptr request_;
+  EagerClient* eager_client_;  // Not owned, and must outlive this node.
 
   // This is required to ensure that the tensor handles stay alive across the
   // execution.
diff --git a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc
index b23466037f..181422118c 100644
--- a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc
+++ b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc
@@ -49,6 +49,7 @@ class GrpcEagerClient : public EagerClient {
   CLIENT_METHOD(KeepAlive);
   CLIENT_METHOD(CloseContext);
   CLIENT_METHOD(RegisterFunction);
+  CLIENT_METHOD(SendTensor);
 
 #undef CLIENT_METHOD
 
diff --git a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service.cc b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service.cc
index 39ab6856c5..ab3aa3fd1d 100644
--- a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service.cc
+++ b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service.cc
@@ -36,6 +36,7 @@ static const char* grpcEagerService_method_names[] = {
     "/tensorflow.eager.EagerService/KeepAlive",
     "/tensorflow.eager.EagerService/CloseContext",
     "/tensorflow.eager.EagerService/RegisterFunction",
+    "/tensorflow.eager.EagerService/SendTensor",
 };
 
 std::unique_ptr EagerService::NewStub(
@@ -62,7 +63,9 @@ EagerService::Stub::Stub(
                               ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_RegisterFunction_(grpcEagerService_method_names[5],
                                   ::grpc::internal::RpcMethod::NORMAL_RPC,
-                                  channel) {}
+                                  channel),
+      rpcmethod_SendTensor_(grpcEagerService_method_names[6],
+                            ::grpc::internal::RpcMethod::NORMAL_RPC, channel) {}
 
 ::grpc::Status EagerService::Stub::CreateContext(
     ::grpc::ClientContext* context, const CreateContextRequest& request,
@@ -106,8 +109,15 @@ EagerService::Stub::Stub(
       channel_.get(), rpcmethod_RegisterFunction_, context, request, response);
 }
 
+::grpc::Status EagerService::Stub::SendTensor(::grpc::ClientContext* context,
+                                              const SendTensorRequest& request,
+                                              SendTensorResponse* response) {
+  return ::grpc::internal::BlockingUnaryCall(
+      channel_.get(), rpcmethod_SendTensor_, context, request, response);
+}
+
 EagerService::AsyncService::AsyncService() {
-  for (int i = 0; i < 6; ++i) {
+  for (int i = 0; i < 7; ++i) {
     AddMethod(new ::grpc::internal::RpcServiceMethod(
         grpcEagerService_method_names[i],
         ::grpc::internal::RpcMethod::NORMAL_RPC, nullptr));
diff --git a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service.h b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service.h
index 66458186ad..521e0ac4fa 100644
--- a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service.h
+++ b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service.h
@@ -69,6 +69,9 @@ class EagerService final {
     virtual ::grpc::Status RegisterFunction(
         ::grpc::ClientContext* context, const RegisterFunctionRequest& request,
         RegisterFunctionResponse* response) = 0;
+    virtual ::grpc::Status SendTensor(::grpc::ClientContext* context,
+                                      const SendTensorRequest& request,
+                                      SendTensorResponse* response) = 0;
   };
   class Stub final : public StubInterface {
    public:
@@ -91,6 +94,9 @@ class EagerService final {
     ::grpc::Status RegisterFunction(
         ::grpc::ClientContext* context, const RegisterFunctionRequest& request,
         RegisterFunctionResponse* response) override;
+    ::grpc::Status SendTensor(::grpc::ClientContext* context,
+                              const SendTensorRequest& request,
+                              SendTensorResponse* response) override;
 
    private:
     std::shared_ptr< ::grpc::ChannelInterface> channel_;
@@ -100,6 +106,7 @@ class EagerService final {
     const ::grpc::internal::RpcMethod rpcmethod_KeepAlive_;
     const ::grpc::internal::RpcMethod rpcmethod_CloseContext_;
     const ::grpc::internal::RpcMethod rpcmethod_RegisterFunction_;
+    const ::grpc::internal::RpcMethod rpcmethod_SendTensor_;
   };
   static std::unique_ptr NewStub(
       const std::shared_ptr< ::grpc::ChannelInterface>& channel,
@@ -157,6 +164,14 @@ class EagerService final {
       ::grpc::Service::RequestAsyncUnary(5, context, request, response,
                                          new_call_cq, notification_cq, tag);
     }
+    void RequestSendTensor(
+        ::grpc::ServerContext* context, SendTensorRequest* request,
+        ::grpc::ServerAsyncResponseWriter* response,
+        ::grpc::CompletionQueue* new_call_cq,
+        ::grpc::ServerCompletionQueue* notification_cq, void* tag) {
+      ::grpc::Service::RequestAsyncUnary(6, context, request, response,
+                                         new_call_cq, notification_cq, tag);
+    }
   };
 };
 
diff --git a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.cc b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.cc
index 44e880de04..f511674e1f 100644
--- a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.cc
@@ -48,6 +48,7 @@ void GrpcEagerServiceImpl::HandleRPCsLoop() {
   ENQUEUE_REQUEST(KeepAlive);
   ENQUEUE_REQUEST(CloseContext);
   ENQUEUE_REQUEST(RegisterFunction);
+  ENQUEUE_REQUEST(SendTensor);
 #undef ENQUEUE_REQUEST
 
   void* tag;  // Matches the operation started against this cq_.
diff --git a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.h b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.h
index 502f3ef529..537e9043bd 100644
--- a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.h
+++ b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_service_impl.h
@@ -62,6 +62,7 @@ class GrpcEagerServiceImpl : public AsyncServiceInterface {
   HANDLER(KeepAlive);
   HANDLER(CloseContext);
   HANDLER(RegisterFunction);
+  HANDLER(SendTensor);
 #undef HANDLER
 
   const WorkerEnv* const env_;  // Not owned.
diff --git a/tensorflow/core/protobuf/eager_service.proto b/tensorflow/core/protobuf/eager_service.proto
index 5b05a1b3ee..63ba4eb173 100644
--- a/tensorflow/core/protobuf/eager_service.proto
+++ b/tensorflow/core/protobuf/eager_service.proto
@@ -8,6 +8,7 @@ import "tensorflow/core/framework/function.proto";
 import "tensorflow/core/framework/versions.proto";
 import "tensorflow/core/protobuf/tensorflow_server.proto";
 import "tensorflow/core/framework/tensor_shape.proto";
+import "tensorflow/core/framework/tensor.proto";
 
 message RemoteTensorHandle {
   // The ID of the operation that produced this tensor.
@@ -128,6 +129,24 @@ message RegisterFunctionRequest {
 message RegisterFunctionResponse {
 }
 
+message SendTensorRequest {
+  fixed64 context_id = 1;
+
+  // All remote tensors are identified by . To mimic this
+  // situation when directly sending tensors, we include an "artificial" op ID
+  // (which would have corresponded to the _Recv op when not using SendTensor).
+  int64 op_id = 2;
+  // The index within the repeated field is the output number that will help
+  // uniquely identify (along with the above op_id) the particular tensor.
+  repeated TensorProto tensors = 3;
+
+  // The device on which the tensors should be resident.
+  string device_name = 4;
+}
+
+message SendTensorResponse {
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 //
 // Eager Service defines a TensorFlow service that executes operations eagerly
@@ -174,4 +193,8 @@ service EagerService {
   // Takes a FunctionDef and makes it enqueable on the remote worker.
   rpc RegisterFunction(RegisterFunctionRequest)
       returns (RegisterFunctionResponse);
+
+  // An RPC to push tensors to the server. At times, certain environments don't
+  // allow the server to connect back to the client.
+  rpc SendTensor(SendTensorRequest) returns (SendTensorResponse);
 }
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 85b9491903..495a674526 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -177,6 +177,11 @@ class Context(object):
         - tf.contrib.eager.SYNC: executes each operation synchronously.
         - tf.contrib.eager.ASYNC: executes each operation asynchronously. These
           operations may return "non-ready" handles.
+      server_def: (Optional.) A tensorflow::ServerDef proto.
+        Enables execution on remote devices. GrpcServers need to be started by
+        creating an identical server_def to this, and setting the appropriate
+        task_indexes, so that the servers can communicate. It will then be
+        possible to execute operations on remote devices.
 
     Raises:
      ValueError: If execution_mode is not valid.
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index b813cd6c06..6a5c44e4d9 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5237,7 +5237,10 @@ def enable_eager_execution(config=None,
      to this function.
   """
   return enable_eager_execution_internal(
-      config, device_policy, execution_mode, None)
+      config=config,
+      device_policy=device_policy,
+      execution_mode=execution_mode,
+      server_def=None)
 
 
 def enable_eager_execution_internal(config=None,
-- 
cgit v1.2.3


From 2ddf7fab12e38099a4a527237bc8b893ed4c21d0 Mon Sep 17 00:00:00 2001
From: Sanjoy Das 
Date: Tue, 24 Jul 2018 14:22:08 -0700
Subject: Assign buffers for constants

(Still disabled on all backends by default)

PiperOrigin-RevId: 205890903
---
 .../compiler/xla/service/buffer_assignment.cc      |  46 +++++---
 .../compiler/xla/service/buffer_assignment.h       |  28 ++++-
 .../compiler/xla/service/buffer_assignment_test.cc | 128 ++++++++++++++++++---
 tensorflow/compiler/xla/service/hlo.proto          |   1 +
 4 files changed, 171 insertions(+), 32 deletions(-)

diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc
index bcca9f46d3..b4c7cf0dd8 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment.cc
@@ -279,6 +279,7 @@ BufferAllocationProto BufferAllocation::ToProto() const {
     }
     proto.set_parameter_number(parameter_number_);
   }
+  proto.set_is_constant(is_constant_);
   proto.set_maybe_live_out(maybe_live_out_);
   for (const auto& buffer_offset_size : assigned_buffers_) {
     BufferAllocationProto::Assigned* proto_assigned = proto.add_assigned();
@@ -304,6 +305,9 @@ string BufferAllocation::ToString() const {
     StrAppend(&output, ", parameter ", parameter_number(), " at ShapeIndex ",
               param_shape_index().ToString());
   }
+  if (is_constant()) {
+    StrAppend(&output, ", constant");
+  }
   if (is_thread_local()) {
     StrAppend(&output, ", thread-local");
   }
@@ -606,6 +610,10 @@ Status BufferAssignment::ComputeSummaryStats() {
       stats_.parameter_allocation_count++;
       stats_.parameter_allocation_bytes += allocation.size();
     }
+    if (allocation.is_constant()) {
+      stats_.constant_allocation_count++;
+      stats_.constant_allocation_bytes += allocation.size();
+    }
     if (allocation.maybe_live_out()) {
       stats_.maybe_live_out_allocation_count++;
       stats_.maybe_live_out_allocation_bytes += allocation.size();
@@ -642,6 +650,8 @@ string BufferAssignment::Stats::ToString() const {
   Appendf(&s, "BufferAssignment stats:\n");
   Appendf(&s, "             parameter allocation: %10s\n",
           HumanReadableNumBytes(parameter_allocation_bytes).c_str());
+  Appendf(&s, "              constant allocation: %10s\n",
+          HumanReadableNumBytes(constant_allocation_bytes).c_str());
   Appendf(&s, "        maybe_live_out allocation: %10s\n",
           HumanReadableNumBytes(maybe_live_out_allocation_bytes).c_str());
   Appendf(&s, "     preallocated temp allocation: %10s\n",
@@ -719,8 +729,10 @@ StatusOr> BufferAssigner::Run(
     const HloModule* module, std::unique_ptr hlo_ordering,
     LogicalBuffer::SizeFunction buffer_size,
     LogicalBuffer::AlignmentFunction color_alignment,
-    bool allow_input_output_aliasing, BufferLiveness::Colorer colorer) {
-  BufferAssigner assigner(allow_input_output_aliasing, std::move(colorer));
+    bool allow_input_output_aliasing, bool allocate_buffers_for_constants,
+    BufferLiveness::Colorer colorer) {
+  BufferAssigner assigner(allow_input_output_aliasing,
+                          allocate_buffers_for_constants, std::move(colorer));
   return assigner.CreateAssignment(module, std::move(hlo_ordering),
                                    std::move(buffer_size),
                                    std::move(color_alignment));
@@ -902,15 +914,19 @@ Status BufferAssigner::AssignBuffersForComputation(
     TF_RET_CHECK(!assignment->HasAllocation(*buffer));
 
     const HloInstruction* instruction = buffer->instruction();
+    const int64 buffer_size = assignment->buffer_size_(*buffer);
+
     if (instruction->opcode() == HloOpcode::kConstant) {
-      // No BufferAllocations for constants.
-      // TODO(b/32248867): For consistency, constants should get allocations.
-      VLOG(3) << "Skipping constant: " << *buffer;
+      if (allocate_buffers_for_constants_) {
+        BufferAllocation* allocation =
+            assignment->NewAllocation(*buffer, buffer_size);
+        allocation->set_constant(true);
+        VLOG(3) << "New allocation #" << allocation->index() << " for constant "
+                << *buffer;
+      }
       continue;
     }
 
-    const int64 buffer_size = assignment->buffer_size_(*buffer);
-
     const bool is_entry_parameter =
         instruction->opcode() == HloOpcode::kParameter &&
         computation == computation->parent()->entry_computation();
@@ -1078,6 +1094,7 @@ Status BufferAssigner::AssignBuffersWithSequentialOrdering(
       VLOG(2) << "Simulating heap for color " << color;
       int64 alignment = assignment->color_alignment_(color);
       HeapSimulator::Options options;
+      options.alloc_constants = allocate_buffers_for_constants_;
       BufferValueFlatSet buffer_value_set =
           ToBufferValueFlatSet(single_colored_set.second);
       options.buffers_to_assign = &buffer_value_set;
@@ -1559,6 +1576,7 @@ void BufferAssigner::AssignColocatedBufferSets(
     // param in 'colocated_buffer_set'.
     int64 entry_parameter_number = -1;
     const ShapeIndex* entry_parameter_shape_idx = nullptr;
+    bool is_constant = false;
     for (const LogicalBuffer* buffer : colocated_buffer_set) {
       const HloInstruction* instruction = buffer->instruction();
       const HloComputation* computation = instruction->parent();
@@ -1566,10 +1584,14 @@ void BufferAssigner::AssignColocatedBufferSets(
           computation == computation->parent()->entry_computation()) {
         entry_parameter_number = instruction->parameter_number();
         entry_parameter_shape_idx = &buffer->index();
-        break;
+      } else if (instruction->opcode() == HloOpcode::kConstant) {
+        is_constant = true;
       }
     }
 
+    CHECK(!is_constant || entry_parameter_number == -1)
+        << "Copy insertion should have inserted copies to prevent this.";
+
     for (const LogicalBuffer* buffer : colocated_buffer_set) {
       const int64 buffer_size = assignment->buffer_size_(*buffer);
       if (allocation == nullptr) {
@@ -1579,14 +1601,12 @@ void BufferAssigner::AssignColocatedBufferSets(
         // computations (in some cases).
         allocation = assignment->NewAllocation(*buffer, buffer_size);
         if (entry_parameter_number >= 0) {
-          // This colocated buffer set contains an entry parameter and other
-          // logical buffers which use the parameter as read-only in a while
-          // body computation (which updates in place).
-          // Set 'entry_computation_parameter' to indicate that it contains
-          // an entry parameter, and to prevent reuse in MaybeAssignBuffer.
           allocation->set_entry_computation_parameter(
               entry_parameter_number, *entry_parameter_shape_idx);
         }
+        if (is_constant) {
+          allocation->set_constant(true);
+        }
         colocated_allocations->insert(allocation->index());
       } else {
         CHECK_EQ(buffer_size, allocation->size())
diff --git a/tensorflow/compiler/xla/service/buffer_assignment.h b/tensorflow/compiler/xla/service/buffer_assignment.h
index 8844b6e3ba..4fcf1fc73d 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.h
+++ b/tensorflow/compiler/xla/service/buffer_assignment.h
@@ -85,7 +85,9 @@ class BufferAllocation {
 
   // Whether this allocation is readonly i.e. backed by memory we cannot write
   // to.
-  bool is_readonly() const { return is_entry_computation_parameter(); }
+  bool is_readonly() const {
+    return is_entry_computation_parameter() || is_constant();
+  }
 
   bool is_tuple() const { return is_tuple_; }
   void set_is_tuple(bool is_tuple) { is_tuple_ = is_tuple; }
@@ -96,6 +98,13 @@ class BufferAllocation {
   bool is_entry_computation_parameter() const {
     return is_entry_computation_parameter_;
   }
+
+  // Whether this allocation holds a constant.  On the CPU and GPU backends
+  // constant allocations are not allocated dynamically, instead we resolve
+  // references to these buffer allocations to a global in the readonly section
+  // of the binary.
+  bool is_constant() const { return is_constant_; }
+
   // If this allocation holds a Buffer from a parameter of the entry
   // computation, this methods returns the parameter number. CHECKs otherwise.
   int64 parameter_number() const {
@@ -201,7 +210,9 @@ class BufferAllocation {
            // of the computation.
            !maybe_live_out() &&
            // Thread-local buffers are allocated using `alloca`s.
-           !is_thread_local();
+           !is_thread_local() &&
+           // Constant buffers are allocated as global values.
+           !is_constant();
   }
 
   // Add a heap trace which was used to assign slices to logical buffers in this
@@ -257,6 +268,8 @@ class BufferAllocation {
     parameter_number_ = parameter_number;
     param_shape_index_ = std::move(param_shape_index);
   }
+
+  void set_constant(bool is_constant) { is_constant_ = is_constant; }
   void set_maybe_live_out(bool value) { maybe_live_out_ = value; }
   void set_index(Index index) { index_ = index; }
   void set_size(int64 size) { size_ = size; }
@@ -295,6 +308,9 @@ class BufferAllocation {
   // might not actually escape.
   bool maybe_live_out_ = false;
 
+  // See comment on the is_constant() accessor.
+  bool is_constant_ = false;
+
   // Mapping from the set of buffers assigned to this allocation to their
   // logical offsets and sizes.
   tensorflow::gtl::FlatMap assigned_buffers_;
@@ -410,6 +426,8 @@ class BufferAssignment {
   struct Stats {
     int64 parameter_allocation_count = 0;
     int64 parameter_allocation_bytes = 0;
+    int64 constant_allocation_count = 0;
+    int64 constant_allocation_bytes = 0;
     int64 maybe_live_out_allocation_count = 0;
     int64 maybe_live_out_allocation_bytes = 0;
     int64 preallocated_temp_allocation_count = 0;
@@ -502,12 +520,15 @@ class BufferAssigner {
       LogicalBuffer::SizeFunction buffer_size,
       LogicalBuffer::AlignmentFunction color_alignment,
       bool allow_input_output_aliasing = false,
+      bool allocate_buffers_for_constants = false,
       BufferLiveness::Colorer colorer = BufferLiveness::DefaultColorer());
 
  private:
   BufferAssigner(bool allow_input_output_aliasing,
+                 bool allocate_buffers_for_constants,
                  BufferLiveness::Colorer colorer)
       : allow_input_output_aliasing_(allow_input_output_aliasing),
+        allocate_buffers_for_constants_(allocate_buffers_for_constants),
         colorer_(colorer) {}
   virtual ~BufferAssigner() = default;
 
@@ -604,6 +625,9 @@ class BufferAssigner {
   // buffers can be shared if their sizes match.
   bool allow_input_output_aliasing_;
 
+  // If true, allocate buffers for constant instructions.
+  bool allocate_buffers_for_constants_;
+
   // Functor used to assign colors to newly allocated logical buffers.
   BufferLiveness::Colorer colorer_;
 
diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
index bfd20921e2..dea855d39a 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
@@ -89,7 +89,20 @@ class BufferAssignmentTest : public HloTestBase {
     return BufferAssigner::Run(
                module, xla::MakeUnique(module),
                backend().compiler()->BufferSizeBytesFunction(),
-               [alignment](LogicalBuffer::Color) { return alignment; })
+               [alignment](LogicalBuffer::Color) { return alignment; },
+               /*allow_input_output_aliasing=*/false,
+               /*allocate_buffers_for_constants=*/true)
+        .ConsumeValueOrDie();
+  }
+
+  std::unique_ptr RunBufferAssignmentNoBuffersForConstants(
+      HloModule* module, int64 alignment = 1) {
+    return BufferAssigner::Run(
+               module, xla::MakeUnique(module),
+               backend().compiler()->BufferSizeBytesFunction(),
+               [alignment](LogicalBuffer::Color) { return alignment; },
+               /*allow_input_output_aliasing=*/false,
+               /*allocate_buffers_for_constants=*/false)
         .ConsumeValueOrDie();
   }
 
@@ -98,8 +111,9 @@ class BufferAssignmentTest : public HloTestBase {
     return BufferAssigner::Run(
                module, xla::MakeUnique(module),
                backend().compiler()->BufferSizeBytesFunction(),
-               [alignment](LogicalBuffer::Color) { return alignment; }, false,
-               std::move(colorer))
+               [alignment](LogicalBuffer::Color) { return alignment; },
+               /*allow_input_output_aliasing=*/false,
+               /*allocate_buffers_for_constants=*/true, std::move(colorer))
         .ConsumeValueOrDie();
   }
 
@@ -115,7 +129,9 @@ class BufferAssignmentTest : public HloTestBase {
                module,
                xla::MakeUnique(module, module_sequence),
                backend().compiler()->BufferSizeBytesFunction(),
-               [alignment](LogicalBuffer::Color) { return alignment; })
+               [alignment](LogicalBuffer::Color) { return alignment; },
+               /*allow_input_output_aliasing=*/false,
+               /*allocate_buffers_for_constants=*/true)
         .ConsumeValueOrDie();
   }
 
@@ -294,9 +310,15 @@ TEST_F(BufferAssignmentTest, ScalarConstant) {
   auto module = CreateNewModule();
   module->AddEntryComputation(builder.Build());
 
-  auto buffers = RunBufferAssignment(module.get());
-  // Check that the constant does not have a buffer assigned.
-  EXPECT_FALSE(buffers->HasTopLevelAllocation(const0));
+  {
+    auto buffers = RunBufferAssignment(module.get());
+    EXPECT_TRUE(buffers->HasTopLevelAllocation(const0));
+  }
+
+  {
+    auto buffers = RunBufferAssignmentNoBuffersForConstants(module.get());
+    EXPECT_FALSE(buffers->HasTopLevelAllocation(const0));
+  }
 }
 
 TEST_F(BufferAssignmentTest, BufferForConst) {
@@ -312,12 +334,18 @@ TEST_F(BufferAssignmentTest, BufferForConst) {
   auto module = CreateNewModule();
   module->AddEntryComputation(builder.Build());
 
-  auto buffers = RunBufferAssignment(module.get());
-  // The two constant nodes have no buffers assigned.
-  EXPECT_FALSE(buffers->HasTopLevelAllocation(const0));
-  EXPECT_FALSE(buffers->HasTopLevelAllocation(const1));
-  // The add node has an output buffer.
-  GetAssignedOutputAllocation(*buffers, add);
+  {
+    auto buffers = RunBufferAssignment(module.get());
+    EXPECT_TRUE(buffers->HasTopLevelAllocation(const0));
+    EXPECT_TRUE(buffers->HasTopLevelAllocation(const1));
+    GetAssignedOutputAllocation(*buffers, add);
+  }
+  {
+    auto buffers = RunBufferAssignmentNoBuffersForConstants(module.get());
+    EXPECT_FALSE(buffers->HasTopLevelAllocation(const0));
+    EXPECT_FALSE(buffers->HasTopLevelAllocation(const1));
+    GetAssignedOutputAllocation(*buffers, add);
+  }
 }
 
 TEST_F(BufferAssignmentTest, HasAllocationAt) {
@@ -1196,7 +1224,7 @@ TEST_F(BufferAssignmentTest, ElementOfNestedTupleParameterAsOutput) {
 
 // TODO(b/32248867): Enable when buffer assignment gives allocations to
 // constants.
-TEST_F(BufferAssignmentTest, DISABLED_TupleConstantAsOutput) {
+TEST_F(BufferAssignmentTest, TupleConstantAsOutput) {
   // Test that a tuple constant which is forwarded to the computation output
   // is properly handled.
   auto builder = HloComputation::Builder(TestName());
@@ -1644,6 +1672,66 @@ TEST_F(BufferAssignmentTest, PeakBuffersWhile) {
       nonbcast_buffer->instruction() == condition->parameter_instruction(0));
 }
 
+TEST_F(BufferAssignmentTest, ConstantBuffersAreNotReused) {
+  const char* hlo_text = R"(
+HloModule Module
+
+True {
+  ROOT x.0.1 = f32[] parameter(0)
+}
+
+False {
+  x.0.0 = f32[] parameter(0)
+  ROOT copy.1 = f32[] copy(x.0.0)
+}
+
+ENTRY main {
+  pred.1.0 = pred[] parameter(0)
+  constant.1.1 = f32[] constant(56)
+  copy.2 = f32[] copy(constant.1.1)
+  constant.1.2 = f32[] constant(12)
+  ROOT conditional.1.3 = f32[] conditional(pred.1.0, copy.2, constant.1.2),
+      true_computation=True, false_computation=False
+}
+)";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module,
+                          ParseHloString(hlo_text));
+
+  HloInstruction* constant_1 =
+      module->entry_computation()->GetInstructionWithName("constant.1.1");
+  HloInstruction* constant_2 =
+      module->entry_computation()->GetInstructionWithName("constant.1.2");
+
+  auto buffers = RunBufferAssignment(module.get());
+
+  {
+    const BufferAllocation& allocation_for_const_1 =
+        GetTopLevelAllocation(*buffers, constant_1);
+    EXPECT_TRUE(allocation_for_const_1.is_constant());
+    for (const auto& buffer_offset_pair :
+         allocation_for_const_1.assigned_buffers()) {
+      EXPECT_NE(buffer_offset_pair.first->instruction()->opcode(),
+                HloOpcode::kCopy);
+      EXPECT_NE(buffer_offset_pair.first->instruction()->opcode(),
+                HloOpcode::kConditional);
+    }
+  }
+
+  {
+    const BufferAllocation& allocation_for_const_2 =
+        GetTopLevelAllocation(*buffers, constant_2);
+    EXPECT_TRUE(allocation_for_const_2.is_constant());
+    for (const auto& buffer_offset_pair :
+         allocation_for_const_2.assigned_buffers()) {
+      EXPECT_NE(buffer_offset_pair.first->instruction()->opcode(),
+                HloOpcode::kCopy);
+      EXPECT_NE(buffer_offset_pair.first->instruction()->opcode(),
+                HloOpcode::kConditional);
+    }
+  }
+}
+
 class WhileBufferAssignmentTest : public HloTestBase {
  protected:
   std::unique_ptr BuildWhileConditionComputation(
@@ -1683,7 +1771,9 @@ class WhileBufferAssignmentTest : public HloTestBase {
     return BufferAssigner::Run(
                module, xla::MakeUnique(module, sequence),
                ByteSizeOf,
-               [alignment](LogicalBuffer::Color) { return alignment; })
+               [alignment](LogicalBuffer::Color) { return alignment; },
+               /*allow_input_output_aliasing=*/false,
+               /*allocate_buffers_for_constants=*/true)
         .ConsumeValueOrDie();
   }
 
@@ -1927,7 +2017,9 @@ TEST_F(WhileBufferAssignmentTest, ColocatedBuffers) {
           module.get(),
           xla::MakeUnique(module.get(), sequence),
           backend().compiler()->BufferSizeBytesFunction(),
-          [](LogicalBuffer::Color) { return 1; }));
+          [](LogicalBuffer::Color) { return 1; },
+          /*allow_input_output_aliasing=*/false,
+          /*allocate_buffers_for_constants=*/true));
 
   // The result tuple elements must be assigned with different buffers.
   TF_ASSERT_OK_AND_ASSIGN(auto slice0, assignment->GetUniqueSlice(tuple, {0}));
@@ -2181,7 +2273,9 @@ TEST_F(WhileBufferAssignmentTest, WhileLoopsInterferingResultRange) {
       BufferAssigner::Run(
           module.get(),
           xla::MakeUnique(module.get(), sequence),
-          ByteSizeOf, [](LogicalBuffer::Color) { return 1; })
+          ByteSizeOf, [](LogicalBuffer::Color) { return 1; },
+          /*allow_input_output_aliasing=*/false,
+          /*allocate_buffers_for_constants=*/true)
           .ConsumeValueOrDie();
 
   EXPECT_TRUE(BuffersDistinct({while0}, {while1}, *assignment));
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index 50d7f1823c..63a8a813cd 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -246,6 +246,7 @@ message BufferAllocationProto {
   bool is_thread_local = 3;
   bool is_tuple = 11;
   bool is_entry_computation_parameter = 5;
+  bool is_constant = 12;
   int64 parameter_number = 6;
   repeated int64 parameter_shape_index = 10;
   bool maybe_live_out = 7;
-- 
cgit v1.2.3


From 62696ff0ede36f05b945d6465c709c5f586d63ed Mon Sep 17 00:00:00 2001
From: Igor Ganichev 
Date: Tue, 24 Jul 2018 14:24:59 -0700
Subject: Use metric's dtype to represent matches

Before this change, even when the actual metric is using float32,
we used float64 to represent boolean matches.

PiperOrigin-RevId: 205891351
---
 tensorflow/contrib/eager/python/metrics_impl.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index efa6ba0626..6efafccd6b 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -291,8 +291,6 @@ class Metric(checkpointable.CheckpointableBase):
 
 class Mean(Metric):
   """Computes the (weighted) mean of the given values."""
-  # TODO(josh11b): Maybe have a dtype argument that defaults to tf.float64?
-  # Or defaults to type of the input if it is tf.float32, else tf.float64?
 
   def __init__(self, name=None, dtype=dtypes.float64,
                use_global_variables=False):
@@ -377,7 +375,7 @@ class Accuracy(Mean):
         array_ops.shape(labels), array_ops.shape(predictions),
         message="Shapes of labels and predictions are unequal")
     matches = math_ops.equal(labels, predictions)
-    matches = math_ops.cast(matches, dtypes.float64)
+    matches = math_ops.cast(matches, self.dtype)
     super(Accuracy, self).call(matches, weights=weights)
     if weights is None:
       return labels, predictions
@@ -421,7 +419,7 @@ class CategoricalAccuracy(Mean):
     labels = math_ops.argmax(labels, axis=-1)
     predictions = math_ops.argmax(predictions, axis=-1)
     matches = math_ops.equal(labels, predictions)
-    matches = math_ops.cast(matches, dtypes.float64)
+    matches = math_ops.cast(matches, self.dtype)
     super(CategoricalAccuracy, self).call(matches, weights=weights)
     if weights is None:
       return labels, predictions
@@ -472,7 +470,7 @@ class BinaryAccuracy(Mean):
     predictions = ops.convert_to_tensor(predictions)
     predictions = predictions > self.threshold
     matches = math_ops.equal(labels, predictions)
-    matches = math_ops.cast(matches, dtypes.float64)
+    matches = math_ops.cast(matches, self.dtype)
     super(BinaryAccuracy, self).call(matches, weights=weights)
     if weights is None:
       return labels, predictions
@@ -520,7 +518,7 @@ class SparseAccuracy(Mean):
     predictions = math_ops.argmax(predictions, axis=-1)
     labels = math_ops.cast(labels, dtypes.int64)
     matches = math_ops.equal(labels, predictions)
-    matches = math_ops.cast(matches, dtypes.float64)
+    matches = math_ops.cast(matches, self.dtype)
     super(SparseAccuracy, self).call(matches, weights=weights)
     if weights is None:
       return labels, predictions
-- 
cgit v1.2.3


From b6b6d39ade422e3fe545fe24f1b06ab0bfd72efc Mon Sep 17 00:00:00 2001
From: Russell Power 
Date: Tue, 24 Jul 2018 14:33:23 -0700
Subject: Disable wait timeout for TPUEstimator.

If the main loop triggers an error, this will raise immediately.  Fixes testing timeouts.

PiperOrigin-RevId: 205892842
---
 tensorflow/contrib/tpu/python/tpu/error_handling.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/error_handling.py b/tensorflow/contrib/tpu/python/tpu/error_handling.py
index 182cac6f0f..14659fe68f 100644
--- a/tensorflow/contrib/tpu/python/tpu/error_handling.py
+++ b/tensorflow/contrib/tpu/python/tpu/error_handling.py
@@ -101,7 +101,7 @@ class ErrorRendezvous(object):
     except Exception as e:  # pylint: disable=broad-except
       self.record_error(source, e, session)
 
-  def raise_errors(self, timeout_sec=5):
+  def raise_errors(self, timeout_sec=0):
     """Wait for up to `timeout` seconds for all error sources to finish.
 
     Preferentially raise "interesting" errors (errors not in the
-- 
cgit v1.2.3


From e4cfe8a009a8e136ace5d74eb1bebf1ef00ea344 Mon Sep 17 00:00:00 2001
From: Asim Shankar 
Date: Tue, 24 Jul 2018 14:47:22 -0700
Subject: eager: Add a benchmark to contrast subclassed and functional keras
 models.

This is a trivial, not very representative model which likely accentuates
overheads. May revisit the model in the future, for now just setting up
the plumbing for the benchmark.

PiperOrigin-RevId: 205895247
---
 tensorflow/python/eager/BUILD              |  1 +
 tensorflow/python/eager/benchmarks_test.py | 60 ++++++++++++++++++++++++++----
 2 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 6ede8e4f4d..58b287fe4b 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -322,6 +322,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:pywrap_tensorflow",
         "//tensorflow/python:random_ops",
+        "//tensorflow/python/keras",
     ],
 )
 
diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 3aad4a114a..afc4bf0066 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -31,6 +31,7 @@ import numpy as np
 import six
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
+from tensorflow.python import keras
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import backprop  # pylint: disable=unused-import
 from tensorflow.python.eager import context
@@ -70,6 +71,25 @@ def c_tfe_py_fastpath_execute(a,
     six.raise_from(core._status_to_exception(e.code, message), None)
 
 
+class SubclassedKerasModel(keras.Model):
+
+  def __init__(self):
+    super(SubclassedKerasModel, self).__init__()
+    self.layer = keras.layers.Dense(
+        10, kernel_initializer="ones", bias_initializer="zeros")
+
+  def call(self, x):
+    return self.layer(x)
+
+
+def make_keras_model():
+  x = keras.Input(shape=(10,))
+  y = keras.layers.Dense(
+      10, kernel_initializer="ones", bias_initializer="zeros")(
+          x)
+  return keras.Model(inputs=x, outputs=y)
+
+
 class MicroBenchmarks(test.Benchmark):
 
   def __init__(self):
@@ -115,6 +135,7 @@ class MicroBenchmarks(test.Benchmark):
 
     def func():
       ops.EagerTensor(value, context=handle, device=device, dtype=dtype)
+
     self._run(func, 30000)
 
   def benchmark_create_float_tensor_from_list_CPU(self):
@@ -211,8 +232,8 @@ class MicroBenchmarks(test.Benchmark):
     inputs = [m]
 
     def f():
-      pywrap_tensorflow.TFE_Py_Execute(
-          ctx_handle, None, "Identity", inputs, attrs, 1)
+      pywrap_tensorflow.TFE_Py_Execute(ctx_handle, None, "Identity", inputs,
+                                       attrs, 1)
 
     self._run(f, 30000)
 
@@ -234,14 +255,13 @@ class MicroBenchmarks(test.Benchmark):
     def f():
       with backprop.GradientTape():
         pass
+
     self._run(f, 30000)
 
   def benchmark_tf_gradient_function_no_op(self):
     with context.device(CPU):
       m = gen_array_ops.identity(self._m_2)
-      self._run(
-          lambda: backprop.gradients_function(lambda x: x, [0])(m),
-          30000)
+      self._run(lambda: backprop.gradients_function(lambda x: x, [0])(m), 30000)
 
   def _benchmark_np_matmul(self, m, transpose_b, num_iters):
     a = m.cpu().numpy()
@@ -255,6 +275,7 @@ class MicroBenchmarks(test.Benchmark):
     self._run(func, num_iters, execution_mode=execution_mode)
 
   def _benchmark_gen_math_ops_matmul(self, m, transpose_b, num_iters):
+
     def func():
       gen_math_ops.mat_mul(m, m, transpose_b=transpose_b)
 
@@ -276,9 +297,10 @@ class MicroBenchmarks(test.Benchmark):
     device = context.context().device_name
     attrs = ("transpose_a", False, "transpose_b", transpose_b, "T",
              m.dtype.as_datatype_enum)
+
     def func():
-      pywrap_tensorflow.TFE_Py_Execute(ctx_handle, device, "MatMul",
-                                       inputs, attrs, 1)
+      pywrap_tensorflow.TFE_Py_Execute(ctx_handle, device, "MatMul", inputs,
+                                       attrs, 1)
 
     self._run(func, num_iters)
 
@@ -542,6 +564,30 @@ class MicroBenchmarks(test.Benchmark):
       self._benchmark_read_variable_with_tape(
           m, num_iters=self._num_iters_2_by_2)
 
+  def benchmark_keras_model_subclassed(self):
+    model = SubclassedKerasModel()
+    data = random_ops.random_uniform((10, 10))
+
+    func = lambda: model(data)
+    # First call is more expensive (creates variables etc.), discount that.
+    func()
+
+    # The whole point of this test is to contrast subclassing with
+    # the functional style of keras model building, so validate that
+    # the models are equivalent.
+    assert np.equal(func(), make_keras_model()(data)).all()
+
+    self._run(func, 30000)
+
+  def benchmark_keras_model_functional(self):
+    model = make_keras_model()
+    data = random_ops.random_uniform((10, 10))
+    func = lambda: model(data)
+    # Symmetry with benchmark_keras_model_subclassed
+    func()
+    assert np.equal(func(), SubclassedKerasModel()(data)).all()
+    self._run(func, 30000)
+
 
 if __name__ == "__main__":
   test.main()
-- 
cgit v1.2.3


From 779b789cc02ba1466da46158359c3132ef04c3ab Mon Sep 17 00:00:00 2001
From: Nick Felt 
Date: Tue, 24 Jul 2018 14:48:52 -0700
Subject: Update tensorboard dependency to 1.10.x

PiperOrigin-RevId: 205895470
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 2e278aa60b..1f4c3d47bf 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -55,7 +55,7 @@ REQUIRED_PACKAGES = [
     'six >= 1.10.0',
     'protobuf >= 3.6.0',
     'setuptools <= 39.1.0',
-    'tensorboard >= 1.8.0, < 1.9.0',
+    'tensorboard >= 1.10.0, < 1.11.0',
     'termcolor >= 1.1.0',
 ]
 
-- 
cgit v1.2.3


From d09afb711610b88f394d318622e862fcd327f440 Mon Sep 17 00:00:00 2001
From: Xuechen Li 
Date: Tue, 24 Jul 2018 15:03:21 -0700
Subject: Add dataset specific parameters in config file.

PiperOrigin-RevId: 205898175
---
 .../contrib/eager/python/examples/revnet/config.py | 23 ++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/revnet/config.py b/tensorflow/contrib/eager/python/examples/revnet/config.py
index e108686b66..821a4878c1 100644
--- a/tensorflow/contrib/eager/python/examples/revnet/config.py
+++ b/tensorflow/contrib/eager/python/examples/revnet/config.py
@@ -33,7 +33,8 @@ def get_hparams_cifar_38():
   """RevNet-38 configurations for CIFAR-10/CIFAR-100."""
 
   config = tf.contrib.training.HParams()
-  # Hyperparameters from the RevNet paper
+  config.add_hparam("num_train_images", 50000)
+  config.add_hparam("num_eval_images", 10000)
   config.add_hparam("init_filters", 32)
   config.add_hparam("init_kernel", 3)
   config.add_hparam("init_stride", 1)
@@ -67,7 +68,8 @@ def get_hparams_cifar_38():
   config.add_hparam("div255", True)
   # This is imprecise, when training with validation set,
   # we only have 40k images in training data
-  config.add_hparam("iters_per_epoch", 50000 // config.batch_size)
+  config.add_hparam("iters_per_epoch",
+                    config.num_train_images // config.batch_size)
   config.add_hparam("epochs", config.max_train_iter // config.iters_per_epoch)
 
   # Customized TPU hyperparameters due to differing batch size caused by
@@ -76,7 +78,8 @@ def get_hparams_cifar_38():
   # https://cloud.google.com/tpu/docs/troubleshooting
   config.add_hparam("tpu_batch_size", 1024)
   config.add_hparam("tpu_eval_batch_size", 1024)
-  config.add_hparam("tpu_iters_per_epoch", 50000 // config.tpu_batch_size)
+  config.add_hparam("tpu_iters_per_epoch",
+                    config.num_train_images // config.tpu_batch_size)
   config.add_hparam("tpu_epochs",
                     config.max_train_iter // config.tpu_iters_per_epoch)
 
@@ -109,6 +112,8 @@ def get_hparams_imagenet_56():
   config = tf.contrib.training.HParams()
   config.add_hparam("n_classes", 1000)
   config.add_hparam("dataset", "ImageNet")
+  config.add_hparam("num_train_images", 1281167)
+  config.add_hparam("num_eval_images", 50000)
   config.add_hparam("init_filters", 128)
   config.add_hparam("init_kernel", 7)
   config.add_hparam("init_stride", 2)
@@ -126,6 +131,9 @@ def get_hparams_imagenet_56():
   else:
     config.add_hparam("input_shape", (224, 224, 3))
     config.add_hparam("data_format", "channels_last")
+  # Due to bottleneck residual blocks
+  filters = [f * 4 for f in config.filters]
+  config.filters = filters
 
   # Training details
   config.add_hparam("weight_decay", 1e-4)
@@ -140,11 +148,9 @@ def get_hparams_imagenet_56():
   config.add_hparam("dtype", tf.float32)
   config.add_hparam("eval_batch_size", 256)
   config.add_hparam("div255", True)
-  config.add_hparam("iters_per_epoch", 1281167 // config.batch_size)
+  config.add_hparam("iters_per_epoch",
+                    config.num_train_images // config.batch_size)
   config.add_hparam("epochs", config.max_train_iter // config.iters_per_epoch)
-  # Due to bottleneck residual blocks
-  filters = [f * 4 for f in config.filters]
-  config.filters = filters
 
   # Customized TPU hyperparameters due to differing batch size caused by
   # TPU architecture specifics
@@ -152,7 +158,8 @@ def get_hparams_imagenet_56():
   # https://cloud.google.com/tpu/docs/troubleshooting
   config.add_hparam("tpu_batch_size", 1024)
   config.add_hparam("tpu_eval_batch_size", 1024)
-  config.add_hparam("tpu_iters_per_epoch", 1281167 // config.tpu_batch_size)
+  config.add_hparam("tpu_iters_per_epoch",
+                    config.num_train_images // config.tpu_batch_size)
   config.add_hparam("tpu_epochs",
                     config.max_train_iter // config.tpu_iters_per_epoch)
 
-- 
cgit v1.2.3


From eabda97225faf53ec528621299f5b6c57a7847b0 Mon Sep 17 00:00:00 2001
From: Allen Lavoie 
Date: Tue, 24 Jul 2018 15:03:46 -0700
Subject: Dictionary tracking for tf.keras.Model attribute assignment

Does not inherit from dict (and so won't pass isinstance checks). I've written a small tome about why in a comment on the class definition. This seems not to break anyone, but if it does we can add Mapping to the problematic isinstance checks (as I've done for TF's nest util and Session fetching); ideally custom mappings would be supported everywhere dicts are anyway.

PiperOrigin-RevId: 205898305
---
 tensorflow/python/client/session.py                |   3 +-
 tensorflow/python/estimator/keras.py               |   5 +-
 tensorflow/python/framework/test_util.py           |   5 +-
 .../training/checkpointable/data_structures.py     | 189 ++++++++++++++++++++-
 .../checkpointable/data_structures_test.py         | 119 +++++++++++++
 .../training/checkpointable/tracking_test.py       |  37 +++-
 tensorflow/python/training/checkpointable/util.py  |  38 +++--
 tensorflow/python/util/util.cc                     |   9 +-
 8 files changed, 380 insertions(+), 25 deletions(-)

diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 8ede6ab54c..180bb74d00 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import functools
 import re
 import threading
@@ -243,7 +244,7 @@ class _FetchMapper(object):
     elif isinstance(fetch, (list, tuple)):
       # NOTE(touts): This is also the code path for namedtuples.
       return _ListFetchMapper(fetch)
-    elif isinstance(fetch, dict):
+    elif isinstance(fetch, collections.Mapping):
       return _DictFetchMapper(fetch)
     else:
       # Look for a handler in the registered expansions.
diff --git a/tensorflow/python/estimator/keras.py b/tensorflow/python/estimator/keras.py
index 682be8e7cc..70517ae278 100644
--- a/tensorflow/python/estimator/keras.py
+++ b/tensorflow/python/estimator/keras.py
@@ -184,7 +184,7 @@ def _in_place_subclassed_model_reset(model):
   # Replace layers on the model with fresh layers
   layers_to_names = {value: key for key, value in attributes_cache.items()}
   original_layers = model._layers[:]
-  model._layers = []
+  model._layers = data_structures.NoDependency([])
   for layer in original_layers:  # We preserve layer order.
     config = layer.get_config()
     # This will not work for nested subclassed models used as layers.
@@ -232,7 +232,8 @@ def _in_place_subclassed_model_reset(model):
       ]
       for name in attributes_to_cache:
         attributes_cache[name] = getattr(model, name)
-  model._original_attributes_cache = attributes_cache
+  model._original_attributes_cache = data_structures.NoDependency(
+      attributes_cache)
   # Reset built state
   model.built = False
   model.inputs = None
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 2bc2a189fa..d7e7a2c111 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -19,6 +19,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections
 import contextlib
 import gc
 import itertools
@@ -1227,8 +1228,8 @@ class TensorFlowTestCase(googletest.TestCase):
       a = a._asdict()
     if hasattr(b, "_asdict"):
       b = b._asdict()
-    a_is_dict = isinstance(a, dict)
-    if a_is_dict != isinstance(b, dict):
+    a_is_dict = isinstance(a, collections.Mapping)
+    if a_is_dict != isinstance(b, collections.Mapping):
       raise ValueError("Can't compare dict to non-dict, a%s vs b%s. %s" %
                        (path_str, path_str, msg))
     if a_is_dict:
diff --git a/tensorflow/python/training/checkpointable/data_structures.py b/tensorflow/python/training/checkpointable/data_structures.py
index 019d43f09c..507cda8734 100644
--- a/tensorflow/python/training/checkpointable/data_structures.py
+++ b/tensorflow/python/training/checkpointable/data_structures.py
@@ -57,6 +57,8 @@ def _wrap_or_unwrap(value):
     return value.value
   if isinstance(value, base.CheckpointableBase):
     return value  # Skip conversion for already checkpointable objects.
+  elif isinstance(value, dict):
+    return _DictWrapper(value)
   elif isinstance(value, list):
     return _ListWrapper(value)
   else:
@@ -438,12 +440,15 @@ class Mapping(CheckpointableDataStructure, collections.Mapping):
   def __init__(self, *args, **kwargs):
     """Construct a new sequence. Arguments are passed to `dict()`."""
     super(Mapping, self).__init__()
-    self._storage = dict(*args, **kwargs)
+    self._storage = self._make_storage(*args, **kwargs)
     self._storage.update(
         {key: self._track_value(
             value, name=self._name_element(key))
          for key, value in self._storage.items()})
 
+  def _make_storage(self, *args, **kwargs):
+    return dict(*args, **kwargs)
+
   def _name_element(self, key):
     if not isinstance(key, six.string_types):
       raise TypeError(
@@ -476,3 +481,185 @@ class Mapping(CheckpointableDataStructure, collections.Mapping):
 
   def __iter__(self):
     return iter(self._storage)
+
+
+# Unlike _ListWrapper, having _DictWrapper inherit from dict and pass isinstance
+# checks seems infeasible. CPython will not call Python methods/properties on
+# dictionary subclasses when running e.g. {}.update(dict_subclass), and instead
+# collects elements directly from dict_subclass's C structs. So subclassing dict
+# implies that the storage has to be "self" (i.e. the C structs for the object
+# must be updated correctly), but we also need that storage to be the wrapped
+# dictionary to avoid synchronization bugs (un-tracked external modifications
+# should still show up when the dict is accessed through the wrapper). Monkey
+# patching all of the "wrapped" dict's methods instead of creating a wrapper
+# object is an option, but not a very attractive one (replacing methods without
+# creating reference cycles is difficult, and then dicts would need to be
+# special cased everywhere as being checkpointable).
+class _DictWrapper(Mapping, collections.MutableMapping):
+  """Wraps built-in dicts to support restore-on-create for variables.
+
+  _DictWrapper is to Mapping as _ListWrapper is to List. Unlike Mapping,
+  _DictWrapper allows non-string keys and values and arbitrary mutations (delete
+  keys, reassign values). Like _ListWrapper, these mutations mean that
+  _DictWrapper will raise an exception on save.
+  """
+
+  def __new__(cls, *args):
+    if len(args) == 1 and isinstance(args[0], dict):
+      return super(_DictWrapper, cls).__new__(cls)
+    else:
+      # Allow construction from a sequence, e.g. for nest.pack_sequence_as. In
+      # this case there's nothing to wrap, so we make a normal dictionary. Also
+      # allows constructing empty instances of the _DictWrapper type, as Session
+      # is wont to do (and again there's nothing to wrap, so a normal dictionary
+      # makes more sense).
+      return dict(*args)
+
+  def __init__(self, wrapped_dict):
+    self._non_string_key = False
+    self._non_append_mutation = False
+    self._external_modification = False
+    super(_DictWrapper, self).__init__(wrapped_dict)
+    self._update_snapshot()
+
+  def _make_storage(self, wrapped_dict):
+    """Re-use the wrapped dict for storage (to force them to be in sync)."""
+    return wrapped_dict
+
+  @property
+  def _checkpoint_dependencies(self):
+    """Check that the object is saveable before listing its dependencies."""
+    self._check_external_modification()
+    if self._non_string_key:
+      raise ValueError(
+          "Unable to save the object %s (a dictionary wrapper constructed "
+          "automatically on attribute assignment). The wrapped dictionary "
+          "contains a non-string key which maps to a checkpointable object or "
+          "mutable data structure.\n\nIf you don't need this dictionary "
+          "checkpointed, wrap it in a tf.contrib.checkpoint.NoDependency "
+          "object; it will be automatically un-wrapped and subsequently "
+          "ignored." % (self,))
+    if self._non_append_mutation:
+      raise ValueError(
+          "Unable to save the object %s (a dictionary wrapper constructed "
+          "automatically on attribute assignment). A key mapping to a "
+          "checkpointable object was overwritten or deleted, which would "
+          "cause problems for restoration.\n\nIf you don't need this "
+          "dictionary checkpointed, wrap it in a "
+          "tf.contrib.checkpoint.NoDependency object; it will be automatically "
+          "un-wrapped and subsequently ignored." % (self,))
+    if self._external_modification:
+      raise ValueError(
+          "Unable to save the object %s (a dictionary wrapper constructed "
+          "automatically on attribute assignment). The wrapped dictionary was "
+          "modified outside the wrapper (its final value was %s, its value "
+          "when a checkpoint dependency was added was %s), which breaks "
+          "restoration on object creation.\n\nIf you don't need this "
+          "dictionary checkpointed, wrap it in a "
+          "tf.contrib.checkpoint.NoDependency object; it will be automatically "
+          "un-wrapped and subsequently ignored." % (
+              self, self, self._last_wrapped_dict_snapshot))
+    assert not self._dirty  # Any reason for dirtiness should have an exception.
+    return super(_DictWrapper, self)._checkpoint_dependencies
+
+  @property
+  def _dirty(self):
+    """Check if there has already been a mutation which prevents saving."""
+    return (self._external_modification
+            or self._non_append_mutation
+            or self._non_string_key)
+
+  def _check_external_modification(self):
+    """Checks for any changes to the wrapped dict not through the wrapper."""
+    if self._dirty:
+      return
+    if self != self._last_wrapped_dict_snapshot:
+      self._external_modification = True
+      self._last_wrapped_dict_snapshot = None
+
+  def _update_snapshot(self):
+    """Acknowledges tracked changes to the wrapped dict."""
+    if self._dirty:
+      return
+    self._last_wrapped_dict_snapshot = dict(self)
+
+  def _track_value(self, value, name):
+    """Allows storage of non-checkpointable objects."""
+    if isinstance(name, six.string_types):
+      string_key = True
+    else:
+      name = "-non_string_key"
+      string_key = False
+    try:
+      no_dependency = isinstance(value, NoDependency)
+      value = super(_DictWrapper, self)._track_value(value=value, name=name)
+      if not (string_key or no_dependency):
+        # A non-string key maps to a checkpointable value. This data structure
+        # is not saveable.
+        self._non_string_key = True
+      return value
+    except ValueError:
+      # Even if this value isn't checkpointable, we need to make sure
+      # NoDependency objects get unwrapped.
+      return sticky_attribute_assignment(
+          checkpointable=self, value=value, name=name)
+
+  def _name_element(self, key):
+    """Don't throw errors for non-string keys."""
+    if isinstance(key, six.string_types):
+      return super(_DictWrapper, self)._name_element(key)
+    else:
+      return key
+
+  def __setitem__(self, key, value):
+    """Allow any modifications, but possibly mark the wrapper as unsaveable."""
+    self._check_external_modification()
+    no_dep = isinstance(value, NoDependency)
+    if isinstance(key, six.string_types):
+      existing_dependency = self._lookup_dependency(key)
+      value = self._track_value(value, name=key)
+    else:
+      value = _wrap_or_unwrap(value)
+      existing_dependency = None
+      if not no_dep and isinstance(value, base.CheckpointableBase):
+        # Non-string keys are OK as long as we have no reason to add a
+        # dependency on the value (either because the value is not
+        # checkpointable, or because it was wrapped in a NoDependency object).
+        self._non_string_key = True
+    current_value = self._storage.setdefault(key, value)
+    if current_value is not value:
+      if ((not no_dep and isinstance(value, base.CheckpointableBase))
+          # We don't want to just check that the existing object is
+          # checkpointable, since it may have been wrapped in a NoDependency
+          # object.
+          or existing_dependency is not None):
+        # A checkpointable object was replaced under the same key; this means
+        # that restoring would be error-prone, so we'll throw an exception on
+        # save.
+        self._non_append_mutation = True
+      self._storage[key] = value
+
+    self._update_snapshot()
+
+  def __delitem__(self, key):
+    self._check_external_modification()
+    existing_value = self[key]
+    if isinstance(existing_value, base.CheckpointableBase):
+      # Deleting tracked checkpointable values means restoring is problematic,
+      # so we'll throw an exception on save.
+      self._non_append_mutation = True
+    del self._storage[key]
+    self._update_snapshot()
+
+  def __repr__(self):
+    return "DictWrapper(%s)" % (repr(self._storage),)
+
+  def __hash__(self):
+    raise TypeError("unhashable type: 'DictWrapper'")
+
+  def __eq__(self, other):
+    return self._storage == getattr(other, "_storage", other)
+
+  def update(self, *args, **kwargs):
+    for key, value in dict(*args, **kwargs).items():
+      self[key] = value
diff --git a/tensorflow/python/training/checkpointable/data_structures_test.py b/tensorflow/python/training/checkpointable/data_structures_test.py
index 7bee00a927..472b7c32b4 100644
--- a/tensorflow/python/training/checkpointable/data_structures_test.py
+++ b/tensorflow/python/training/checkpointable/data_structures_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.training.checkpointable import data_structures
 from tensorflow.python.training.checkpointable import tracking
+from tensorflow.python.training.checkpointable import util
 
 
 class HasList(training.Model):
@@ -303,6 +304,124 @@ class MappingTests(test.TestCase):
                         data_structures.Mapping()])
     self.assertEqual(2, len(has_mappings))
     self.assertNotIn(data_structures.Mapping(), has_mappings)
+    # In contrast to Mapping, dict wrappers are not hashable
+    a = tracking.Checkpointable()
+    a.d = {}
+    self.assertEqual({}, a.d)
+    self.assertFalse({} != a.d)  # pylint: disable=g-explicit-bool-comparison
+    self.assertNotEqual({1: 2}, a.d)
+    with self.assertRaisesRegexp(TypeError, "unhashable"):
+      set([a.d])
+
+  def testDictWrapperBadKeys(self):
+    a = tracking.Checkpointable()
+    a.d = {}
+    a.d[1] = data_structures.List()
+    model = training.Model()
+    model.sub = a
+    save_path = os.path.join(self.get_temp_dir(), "ckpt")
+    with self.assertRaisesRegexp(ValueError, "non-string key"):
+      model.save_weights(save_path)
+
+  def testDictWrapperNoDependency(self):
+    a = tracking.Checkpointable()
+    a.d = data_structures.NoDependency({})
+    a.d[1] = [3]
+    self.assertEqual([a], util.list_objects(a))
+    model = training.Model()
+    model.sub = a
+    save_path = os.path.join(self.get_temp_dir(), "ckpt")
+    model.save_weights(save_path)
+    model.load_weights(save_path)
+
+  def testNonStringKeyNotCheckpointableValue(self):
+    a = tracking.Checkpointable()
+    a.d = {}
+    a.d["a"] = [3]
+    a.d[1] = data_structures.NoDependency([3])
+    self.assertEqual([a, a.d, a.d["a"]], util.list_objects(a))
+    model = training.Model()
+    model.sub = a
+    save_path = os.path.join(self.get_temp_dir(), "ckpt")
+    model.save_weights(save_path)
+    model.load_weights(save_path)
+
+  def testNonAppendNotCheckpointable(self):
+    # Non-append mutations (deleting or overwriting values) are OK when the
+    # values aren't tracked.
+    a = tracking.Checkpointable()
+    a.d = {}
+    a.d["a"] = [3]
+    a.d[1] = 3
+    a.d[1] = 2
+    self.assertEqual(2, a.d[1])
+    del a.d[1]
+    a.d[2] = data_structures.NoDependency(tracking.Checkpointable())
+    second = tracking.Checkpointable()
+    a.d[2] = data_structures.NoDependency(second)
+    self.assertIs(second, a.d[2])
+    self.assertEqual([a, a.d, a.d["a"]], util.list_objects(a))
+    model = training.Model()
+    model.sub = a
+    save_path = os.path.join(self.get_temp_dir(), "ckpt")
+    model.save_weights(save_path)
+    model.load_weights(save_path)
+
+  def testDelNoSave(self):
+    model = training.Model()
+    model.d = {}
+    model.d["a"] = []
+    del model.d["a"]
+    save_path = os.path.join(self.get_temp_dir(), "ckpt")
+    with self.assertRaisesRegexp(ValueError, "overwritten or deleted"):
+      model.save_weights(save_path)
+
+  def testPopNoSave(self):
+    model = training.Model()
+    model.d = {}
+    model.d["a"] = []
+    model.d.pop("a")
+    save_path = os.path.join(self.get_temp_dir(), "ckpt")
+    with self.assertRaisesRegexp(ValueError, "overwritten or deleted"):
+      model.save_weights(save_path)
+
+  def testExternalModificationNoSave(self):
+    model = training.Model()
+    external_reference = {}
+    model.d = external_reference
+    external_reference["a"] = []
+    save_path = os.path.join(self.get_temp_dir(), "ckpt")
+    with self.assertRaisesRegexp(ValueError, "modified outside the wrapper"):
+      model.save_weights(save_path)
+
+  def testOverwriteNoSave(self):
+    model = training.Model()
+    model.d = {}
+    model.d["a"] = {}
+    model.d["a"] = {}
+    save_path = os.path.join(self.get_temp_dir(), "ckpt")
+    with self.assertRaisesRegexp(ValueError, "overwritten or deleted"):
+      model.save_weights(save_path)
+
+  def testIter(self):
+    model = training.Model()
+    model.d = {1: 3}
+    model.d[1] = 3
+    self.assertEqual([1], list(model.d))
+    new_dict = {}
+    # This update() is super tricky. If the dict wrapper subclasses dict,
+    # CPython will access its storage directly instead of calling any
+    # methods/properties on the object. So the options are either not to
+    # subclass dict (in which case update will call normal iter methods, but the
+    # object won't pass isinstance checks) or to subclass dict and keep that
+    # storage updated (no shadowing all its methods like _ListWrapper).
+    new_dict.update(model.d)
+    self.assertEqual({1: 3}, new_dict)
+
+  def testConstructableFromSequence(self):
+    result = data_structures._DictWrapper([(1, 2), (3, 4)])
+    self.assertIsInstance(result, dict)
+    self.assertEqual({1: 2, 3: 4}, result)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/training/checkpointable/tracking_test.py b/tensorflow/python/training/checkpointable/tracking_test.py
index 96da0d6e47..f8d17cd417 100644
--- a/tensorflow/python/training/checkpointable/tracking_test.py
+++ b/tensorflow/python/training/checkpointable/tracking_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 import os
 
 import numpy
+import six
 
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras.engine import training
@@ -143,6 +144,29 @@ class InterfaceTests(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "A list element was replaced"):
       checkpoint.save(os.path.join(self.get_temp_dir(), "ckpt"))
 
+  @test_util.run_in_graph_and_eager_modes
+  def testDictionariesBasic(self):
+    a = training.Model()
+    b = training.Model()
+    a.attribute = {"b": b}
+    c = training.Model()
+    a.attribute["c"] = []
+    a.attribute["c"].append(c)
+    a_deps = util.list_objects(a)
+    self.assertIn(b, a_deps)
+    self.assertIn(c, a_deps)
+    self.assertIs(b, a.attribute["b"])
+    six.assertCountEqual(
+        self,
+        ["b", "c"],
+        [dep.name for dep in a.attribute._checkpoint_dependencies])
+    self.assertEqual([b, c], a.layers)
+    self.assertEqual([b, c], a.attribute.layers)
+    self.assertEqual([c], a.attribute["c"].layers)
+    checkpoint = util.Checkpoint(a=a)
+    save_path = checkpoint.save(os.path.join(self.get_temp_dir(), "ckpt"))
+    checkpoint.restore(save_path).assert_consumed()
+
   @test_util.run_in_graph_and_eager_modes
   def testNoDepList(self):
     a = training.Model()
@@ -159,12 +183,13 @@ class InterfaceTests(test.TestCase):
   @test_util.run_in_graph_and_eager_modes
   def testAssertions(self):
     a = tracking.Checkpointable()
-    a.l = [numpy.zeros([2, 2])]
-    self.assertAllEqual([numpy.zeros([2, 2])], a.l)
-    self.assertAllClose([numpy.zeros([2, 2])], a.l)
-    nest.map_structure(self.assertAllClose, a.l, [numpy.zeros([2, 2])])
-    a.tensors = [array_ops.ones([2, 2]), array_ops.zeros([3, 3])]
-    self.assertAllClose([numpy.ones([2, 2]), numpy.zeros([3, 3])],
+    a.l = {"k": [numpy.zeros([2, 2])]}
+    self.assertAllEqual(nest.flatten({"k": [numpy.zeros([2, 2])]}),
+                        nest.flatten(a.l))
+    self.assertAllClose({"k": [numpy.zeros([2, 2])]}, a.l)
+    nest.map_structure(self.assertAllClose, a.l, {"k": [numpy.zeros([2, 2])]})
+    a.tensors = {"k": [array_ops.ones([2, 2]), array_ops.zeros([3, 3])]}
+    self.assertAllClose({"k": [numpy.ones([2, 2]), numpy.zeros([3, 3])]},
                         self.evaluate(a.tensors))
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/training/checkpointable/util.py b/tensorflow/python/training/checkpointable/util.py
index 686232fe27..5d26a817d4 100644
--- a/tensorflow/python/training/checkpointable/util.py
+++ b/tensorflow/python/training/checkpointable/util.py
@@ -361,24 +361,42 @@ class _ObjectIdentityWeakKeyDictionary(_ObjectIdentityDictionary):
         yield unwrapped
 
 
-class _ObjectIdentityWeakSet(collections.MutableSet):
-  """Like weakref.WeakSet, but compares objects with "is"."""
+class _ObjectIdentitySet(collections.MutableSet):
+  """Like the built-in set, but compares objects with "is"."""
 
-  def __init__(self):
-    self._storage = set()
+  def __init__(self, *args):
+    self._storage = set([self._wrap_key(obj) for obj in list(*args)])
+
+  def _wrap_key(self, key):
+    return _ObjectIdentityWrapper(key)
 
   def __contains__(self, key):
-    return _WeakObjectIdentityWrapper(key) in self._storage
+    return self._wrap_key(key) in self._storage
 
   def discard(self, key):
-    self._storage.discard(_WeakObjectIdentityWrapper(key))
+    self._storage.discard(self._wrap_key(key))
 
   def add(self, key):
-    self._storage.add(_WeakObjectIdentityWrapper(key))
+    self._storage.add(self._wrap_key(key))
+
+  def __len__(self):
+    return len(self._storage)
+
+  def __iter__(self):
+    keys = list(self._storage)
+    for key in keys:
+      yield key.unwrapped
+
+
+class _ObjectIdentityWeakSet(_ObjectIdentitySet):
+  """Like weakref.WeakSet, but compares objects with "is"."""
+
+  def _wrap_key(self, key):
+    return _WeakObjectIdentityWrapper(key)
 
   def __len__(self):
     # Iterate, discarding old weak refs
-    return len(list(self))
+    return len([_ for _ in self])
 
   def __iter__(self):
     keys = list(self._storage)
@@ -857,8 +875,8 @@ class CheckpointLoadStatus(_LoadStatus):
     for checkpointable_object in list_objects(self._root_checkpointable):
       self._checkpoint.all_python_objects.add(checkpointable_object)
     unused_python_objects = (
-        set(self._checkpoint.all_python_objects)
-        - set(self._checkpoint.object_by_proto_id.values()))
+        _ObjectIdentitySet(self._checkpoint.all_python_objects)
+        - _ObjectIdentitySet(self._checkpoint.object_by_proto_id.values()))
     if unused_python_objects:
       raise AssertionError(
           ("Some Python objects were not bound to checkpointed values, likely "
diff --git a/tensorflow/python/util/util.cc b/tensorflow/python/util/util.cc
index f9e0b7e4d2..ad85a44f8d 100644
--- a/tensorflow/python/util/util.cc
+++ b/tensorflow/python/util/util.cc
@@ -506,9 +506,12 @@ bool AssertSameStructureHelper(PyObject* o1, PyObject* o2, bool check_types,
       }
     } else if (type1 != type2
                /* If both sequences are list types, don't complain. This allows
-               one to be a list subclass (e.g. _ListWrapper used for automatic
-               dependency tracking.) */
-               && !(PyList_Check(o1) && PyList_Check(o2))) {
+                  one to be a list subclass (e.g. _ListWrapper used for
+                  automatic dependency tracking.) */
+               && !(PyList_Check(o1) && PyList_Check(o2))
+               /* Two mapping types will also compare equal, making _DictWrapper
+                  and dict compare equal. */
+               && !(IsMappingHelper(o1) && IsMappingHelper(o2))) {
       *is_type_error = true;
       *error_msg = tensorflow::strings::StrCat(
           "The two namedtuples don't have the same sequence type. "
-- 
cgit v1.2.3


From 74a75900faf88d7ce4e05f4bebd2b872abdf16a9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 15:23:12 -0700
Subject: Adding defun

PiperOrigin-RevId: 205901720
---
 .../examples/generative_examples/dcgan.ipynb       | 24 ++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
index 54cc4dc5da..44ff43a111 100644
--- a/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
+++ b/tensorflow/contrib/eager/python/examples/generative_examples/dcgan.ipynb
@@ -29,7 +29,7 @@
       "source": [
         "This notebook demonstrates how to generate images of handwritten digits using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). To do so, we use Deep Convolutional Generative Adverserial Networks ([DCGAN](https://arxiv.org/pdf/1511.06434.pdf)).\n",
         "\n",
-        "This model takes about 40 seconds per epoch to train on a single Tesla K80 on Colab, as of July 2018.\n",
+        "This model takes about ~30 seconds per epoch (using tf.contrib.eager.defun to create graph functions) to train on a single Tesla K80 on Colab, as of July 2018.\n",
         "\n",
         "Below is the output generated after training the generator and discriminator models for 150 epochs.\n",
         "\n",
@@ -203,7 +203,7 @@
         "## Write the generator and discriminator models\n",
         "\n",
         "* **Generator** \n",
-        "  * It is responsible for **creating the convincing images good enough to fool the discriminator**.\n",
+        "  * It is responsible for **creating convincing images that are good enough to fool the discriminator**.\n",
         "  * It consists of Conv2DTranspose (Upsampling) layers. We start with a fully connected layer and upsample the image 2 times so as to reach the desired image size (mnist image size) which is (28, 28, 1). \n",
         "  * We use **leaky relu** activation except for the **last layer** which uses **tanh** activation.\n",
         "  \n",
@@ -314,6 +314,26 @@
         "discriminator = Discriminator()"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "k1HpMSLImuRi"
+      },
+      "outputs": [],
+      "source": [
+        "# Defun gives 10 secs/epoch performance boost\n",
+        "generator.call = tf.contrib.eager.defun(generator.call)\n",
+        "discriminator.call = tf.contrib.eager.defun(discriminator.call)"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
-- 
cgit v1.2.3


From ba4ccc83adc397936ac01f80dd04ee8b2686c929 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling 
Date: Tue, 24 Jul 2018 15:26:39 -0700
Subject: Improve TFLite Python error handling.

When `InterpreterBuilder` fails, now it fails silently and later
user sees "Interpreter was not initialized.". There's no way to
get the error message and troubleshoot. This fixes the issue and
displays the error message.

PiperOrigin-RevId: 205902280
---
 .../interpreter_wrapper/interpreter_wrapper.cc     | 48 +++++++++++++++-------
 .../interpreter_wrapper/interpreter_wrapper.h      | 18 +++++++-
 2 files changed, 50 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
index f97919363b..9ab05f3068 100644
--- a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
+++ b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
@@ -108,7 +108,9 @@ std::unique_ptr CreateInterpreter(
   ImportNumpy();
 
   std::unique_ptr interpreter;
-  tflite::InterpreterBuilder(*model, resolver)(&interpreter);
+  if (tflite::InterpreterBuilder(*model, resolver)(&interpreter) != kTfLiteOk) {
+    return nullptr;
+  }
   return interpreter;
 }
 
@@ -182,13 +184,37 @@ PyObject* PyTupleFromQuantizationParam(const TfLiteQuantizationParams& param) {
 
 }  // namespace
 
+InterpreterWrapper* InterpreterWrapper::CreateInterpreterWrapper(
+    std::unique_ptr model,
+    std::unique_ptr error_reporter,
+    std::string* error_msg) {
+  if (!model) {
+    *error_msg = error_reporter->message();
+    return nullptr;
+  }
+
+  auto resolver = absl::make_unique();
+  auto interpreter = CreateInterpreter(model.get(), *resolver);
+  if (!interpreter) {
+    *error_msg = error_reporter->message();
+    return nullptr;
+  }
+
+  InterpreterWrapper* wrapper =
+      new InterpreterWrapper(std::move(model), std::move(error_reporter),
+                             std::move(resolver), std::move(interpreter));
+  return wrapper;
+}
+
 InterpreterWrapper::InterpreterWrapper(
     std::unique_ptr model,
-    std::unique_ptr error_reporter)
+    std::unique_ptr error_reporter,
+    std::unique_ptr resolver,
+    std::unique_ptr interpreter)
     : model_(std::move(model)),
       error_reporter_(std::move(error_reporter)),
-      resolver_(absl::make_unique()),
-      interpreter_(CreateInterpreter(model_.get(), *resolver_)) {}
+      resolver_(std::move(resolver)),
+      interpreter_(std::move(interpreter)) {}
 
 InterpreterWrapper::~InterpreterWrapper() {}
 
@@ -421,11 +447,8 @@ InterpreterWrapper* InterpreterWrapper::CreateWrapperCPPFromFile(
   std::unique_ptr error_reporter(new PythonErrorReporter);
   std::unique_ptr model =
       tflite::FlatBufferModel::BuildFromFile(model_path, error_reporter.get());
-  if (!model) {
-    *error_msg = error_reporter->message();
-    return nullptr;
-  }
-  return new InterpreterWrapper(std::move(model), std::move(error_reporter));
+  return CreateInterpreterWrapper(std::move(model), std::move(error_reporter),
+                                  error_msg);
 }
 
 InterpreterWrapper* InterpreterWrapper::CreateWrapperCPPFromBuffer(
@@ -439,11 +462,8 @@ InterpreterWrapper* InterpreterWrapper::CreateWrapperCPPFromBuffer(
   std::unique_ptr model =
       tflite::FlatBufferModel::BuildFromBuffer(buf, length,
                                                error_reporter.get());
-  if (!model) {
-    *error_msg = error_reporter->message();
-    return nullptr;
-  }
-  return new InterpreterWrapper(std::move(model), std::move(error_reporter));
+  return CreateInterpreterWrapper(std::move(model), std::move(error_reporter),
+                                  error_msg);
 }
 
 PyObject* InterpreterWrapper::ResetVariableTensorsToZero() {
diff --git a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h
index 556ec7117a..3e03751da4 100644
--- a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h
+++ b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h
@@ -69,14 +69,28 @@ class InterpreterWrapper {
   PyObject* tensor(PyObject* base_object, int i);
 
  private:
-  InterpreterWrapper(std::unique_ptr model,
-                     std::unique_ptr error_reporter);
+  // Helper function to construct an `InterpreterWrapper` object.
+  // It only returns InterpreterWrapper if it can construct an `Interpreter`.
+  // Otherwise it returns `nullptr`.
+  static InterpreterWrapper* CreateInterpreterWrapper(
+      std::unique_ptr model,
+      std::unique_ptr error_reporter,
+      std::string* error_msg);
+
+  InterpreterWrapper(
+      std::unique_ptr model,
+      std::unique_ptr error_reporter,
+      std::unique_ptr resolver,
+      std::unique_ptr interpreter);
 
   // InterpreterWrapper is not copyable or assignable. We avoid the use of
   // InterpreterWrapper() = delete here for SWIG compatibility.
   InterpreterWrapper();
   InterpreterWrapper(const InterpreterWrapper& rhs);
 
+  // The public functions which creates `InterpreterWrapper` should ensure all
+  // these member variables are initialized successfully. Otherwise it should
+  // report the error and return `nullptr`.
   const std::unique_ptr model_;
   const std::unique_ptr error_reporter_;
   const std::unique_ptr resolver_;
-- 
cgit v1.2.3


From 74d4bd78363b75538ce3aa7ec5ec20678da69c28 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 15:32:45 -0700
Subject: Internal Change

PiperOrigin-RevId: 205903203
---
 tensorflow/contrib/lite/testing/BUILD                | 11 ++---------
 tensorflow/contrib/lite/testing/generate_testspec.cc | 12 +++++++-----
 tensorflow/contrib/lite/testing/generate_testspec.h  |  2 ++
 tensorflow/contrib/lite/testing/join.h               |  3 ++-
 tensorflow/contrib/lite/testing/test_runner.h        |  2 +-
 tensorflow/contrib/lite/testing/tf_driver.cc         |  4 ++--
 tensorflow/contrib/lite/testing/tflite_diff_flags.h  |  2 ++
 7 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index 4c37bcb3c9..6c7f494e9b 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -112,7 +112,6 @@ cc_library(
 cc_test(
     name = "message_test",
     srcs = ["message_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":message",
         "@com_google_googletest//:gtest_main",
@@ -132,7 +131,6 @@ cc_test(
     name = "split_test",
     size = "small",
     srcs = ["split_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":split",
         "@com_google_googletest//:gtest_main",
@@ -142,13 +140,13 @@ cc_test(
 cc_library(
     name = "join",
     hdrs = ["join.h"],
+    deps = ["//tensorflow/contrib/lite:string"],
 )
 
 cc_test(
     name = "join_test",
     size = "small",
     srcs = ["join_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":join",
         "@com_google_googletest//:gtest_main",
@@ -174,7 +172,6 @@ cc_test(
     srcs = ["tflite_driver_test.cc"],
     data = ["//tensorflow/contrib/lite:testdata/multi_add.bin"],
     tags = [
-        "no_oss",
         "tflite_not_portable_android",
         "tflite_not_portable_ios",
     ],
@@ -196,7 +193,6 @@ cc_library(
 cc_test(
     name = "tokenize_test",
     srcs = ["tokenize_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":tokenize",
         "@com_google_googletest//:gtest_main",
@@ -219,7 +215,6 @@ cc_library(
 cc_test(
     name = "test_runner_test",
     srcs = ["test_runner_test.cc"],
-    tags = ["no_oss"],
     deps = [
         ":test_runner",
         "@com_google_googletest//:gtest_main",
@@ -258,7 +253,6 @@ cc_test(
     srcs = ["tf_driver_test.cc"],
     data = ["//tensorflow/contrib/lite:testdata/multi_add.pb"],
     tags = [
-        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
@@ -275,6 +269,7 @@ cc_library(
         ":join",
         ":split",
         ":tf_driver",
+        "//tensorflow/contrib/lite:string",
         "//tensorflow/core:framework",
     ],
 )
@@ -284,7 +279,6 @@ cc_test(
     size = "small",
     srcs = ["generate_testspec_test.cc"],
     tags = [
-        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
@@ -341,7 +335,6 @@ tf_cc_test(
     ],
     tags = [
         "no_cuda_on_cpu_tap",
-        "no_oss",
         "tflite_not_portable",
     ],
     deps = [
diff --git a/tensorflow/contrib/lite/testing/generate_testspec.cc b/tensorflow/contrib/lite/testing/generate_testspec.cc
index c1092e4d25..f29c188e6c 100644
--- a/tensorflow/contrib/lite/testing/generate_testspec.cc
+++ b/tensorflow/contrib/lite/testing/generate_testspec.cc
@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include 
+
 #include "tensorflow/contrib/lite/testing/generate_testspec.h"
 #include "tensorflow/contrib/lite/testing/join.h"
 #include "tensorflow/contrib/lite/testing/split.h"
@@ -88,13 +90,13 @@ bool GenerateTestSpecFromTensorflowModel(
   TfDriver runner(input_layer, input_layer_type, input_layer_shape,
                   output_layer);
   if (!runner.IsValid()) {
-    cerr << runner.GetErrorMessage() << endl;
+    std::cerr << runner.GetErrorMessage() << std::endl;
     return false;
   }
 
   runner.LoadModel(tensorflow_model_path);
   if (!runner.IsValid()) {
-    cerr << runner.GetErrorMessage() << endl;
+    std::cerr << runner.GetErrorMessage() << std::endl;
     return false;
   }
 
@@ -118,14 +120,14 @@ bool GenerateTestSpecFromTensorflowModel(
     for (int j = 0; j < input_values.size(); j++) {
       runner.SetInput(j, input_values[j]);
       if (!runner.IsValid()) {
-        cerr << runner.GetErrorMessage() << endl;
+        std::cerr << runner.GetErrorMessage() << std::endl;
         return false;
       }
     }
 
     runner.Invoke();
     if (!runner.IsValid()) {
-      cerr << runner.GetErrorMessage() << endl;
+      std::cerr << runner.GetErrorMessage() << std::endl;
       return false;
     }
 
@@ -137,7 +139,7 @@ bool GenerateTestSpecFromTensorflowModel(
     for (int j = 0; j < output_layer.size(); j++) {
       stream << "  output: \"" << runner.ReadOutput(j) << "\"\n";
       if (!runner.IsValid()) {
-        cerr << runner.GetErrorMessage() << endl;
+        std::cerr << runner.GetErrorMessage() << std::endl;
         return false;
       }
     }
diff --git a/tensorflow/contrib/lite/testing/generate_testspec.h b/tensorflow/contrib/lite/testing/generate_testspec.h
index bfaf5e7ec8..b3d0db31c0 100644
--- a/tensorflow/contrib/lite/testing/generate_testspec.h
+++ b/tensorflow/contrib/lite/testing/generate_testspec.h
@@ -19,6 +19,8 @@ limitations under the License.
 #include 
 #include 
 
+#include "tensorflow/contrib/lite/string.h"
+
 namespace tflite {
 namespace testing {
 
diff --git a/tensorflow/contrib/lite/testing/join.h b/tensorflow/contrib/lite/testing/join.h
index 1edee01cf9..4be19ad756 100644
--- a/tensorflow/contrib/lite/testing/join.h
+++ b/tensorflow/contrib/lite/testing/join.h
@@ -17,7 +17,8 @@ limitations under the License.
 
 #include 
 #include 
-#include 
+
+#include "tensorflow/contrib/lite/string.h"
 
 namespace tflite {
 namespace testing {
diff --git a/tensorflow/contrib/lite/testing/test_runner.h b/tensorflow/contrib/lite/testing/test_runner.h
index 96ab6be54e..fac7d01aab 100644
--- a/tensorflow/contrib/lite/testing/test_runner.h
+++ b/tensorflow/contrib/lite/testing/test_runner.h
@@ -90,7 +90,7 @@ class TestRunner {
 
   // Invalidate the test runner, preventing it from executing any further.
   void Invalidate(const string& error_message) {
-    cerr << error_message << std::endl;
+    std::cerr << error_message << std::endl;
     error_message_ = error_message;
   }
   bool IsValid() const { return error_message_.empty(); }
diff --git a/tensorflow/contrib/lite/testing/tf_driver.cc b/tensorflow/contrib/lite/testing/tf_driver.cc
index 3b27f6f3da..d6a6ff8f56 100644
--- a/tensorflow/contrib/lite/testing/tf_driver.cc
+++ b/tensorflow/contrib/lite/testing/tf_driver.cc
@@ -28,8 +28,8 @@ namespace {
 
 tensorflow::Tensor CreateTensor(const tensorflow::DataType type,
                                 const std::vector& dim) {
-  tensorflow::TensorShape shape{gtl::ArraySlice{
-      reinterpret_cast(dim.data()), dim.size()}};
+  tensorflow::TensorShape shape{tensorflow::gtl::ArraySlice{
+      reinterpret_cast(dim.data()), dim.size()}};
   return {type, shape};
 }
 
diff --git a/tensorflow/contrib/lite/testing/tflite_diff_flags.h b/tensorflow/contrib/lite/testing/tflite_diff_flags.h
index 7a57e8d3fb..695c2a3de6 100644
--- a/tensorflow/contrib/lite/testing/tflite_diff_flags.h
+++ b/tensorflow/contrib/lite/testing/tflite_diff_flags.h
@@ -15,6 +15,8 @@ limitations under the License.
 #ifndef TENSORFLOW_CONTRIB_LITE_TESTING_TFLITE_DIFF_FLAGS_H_
 #define TENSORFLOW_CONTRIB_LITE_TESTING_TFLITE_DIFF_FLAGS_H_
 
+#include 
+
 #include "tensorflow/contrib/lite/testing/split.h"
 #include "tensorflow/contrib/lite/testing/tflite_diff_util.h"
 #include "tensorflow/core/util/command_line_flags.h"
-- 
cgit v1.2.3


From 3aeaae05f94ea19b60bf9220697e058673a11e85 Mon Sep 17 00:00:00 2001
From: Allen Lavoie 
Date: Tue, 24 Jul 2018 15:35:29 -0700
Subject: Fix //tensorflow/python/eager:memory_test in python 3

Just a tiny xrange issue. Was running a bunch of tests and noticed this failure.

PiperOrigin-RevId: 205903619
---
 tensorflow/python/eager/BUILD          | 1 +
 tensorflow/python/eager/memory_test.py | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 58b287fe4b..32a8452f62 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -405,6 +405,7 @@ cuda_py_test(
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_test_lib",
+        "@six_archive//:six",
     ],
     tags = [
         "optonly",  # The test is too slow in non-opt mode
diff --git a/tensorflow/python/eager/memory_test.py b/tensorflow/python/eager/memory_test.py
index 74c6cbdd31..a1a59d511f 100644
--- a/tensorflow/python/eager/memory_test.py
+++ b/tensorflow/python/eager/memory_test.py
@@ -24,6 +24,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import six
+
 from tensorflow.python import keras
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
@@ -63,7 +65,7 @@ class MemoryTest(test.TestCase):
 
       initial = memory_profiler.memory_usage(-1)[0]
 
-      for _ in xrange(num_iters):
+      for _ in six.moves.range(num_iters):
         f()
 
       increase = memory_profiler.memory_usage(-1)[0] - initial
-- 
cgit v1.2.3


From da5126b19faabade710d50a1d600dfa4c931e89b Mon Sep 17 00:00:00 2001
From: Shivani Agrawal 
Date: Tue, 24 Jul 2018 15:55:08 -0700
Subject: quick fix on typo.

PiperOrigin-RevId: 205906547
---
 tensorflow/core/kernels/data/stats_dataset_ops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/data/stats_dataset_ops.cc b/tensorflow/core/kernels/data/stats_dataset_ops.cc
index 754c32b6ca..58ec3d4495 100644
--- a/tensorflow/core/kernels/data/stats_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/stats_dataset_ops.cc
@@ -390,7 +390,7 @@ class FeatureStatsDatasetOp : public UnaryDatasetOpKernel {
 
         for (const auto& feature_list :
              example.feature_lists().feature_list()) {
-          stats_aggregator->IncrementCounter("feature_lists_count", "reainer",
+          stats_aggregator->IncrementCounter("feature_lists_count", "trainer",
                                              1);
           for (const auto& feature : feature_list.second.feature()) {
             feature_values_list_size_sum += AddStatsFeatureValues(feature);
-- 
cgit v1.2.3


From 1d35ecc2e25edeadcf416e11f2a11f00db58af61 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan 
Date: Tue, 24 Jul 2018 16:09:27 -0700
Subject: Correct natural exp decay documentation.

PiperOrigin-RevId: 205908932
---
 tensorflow/python/training/learning_rate_decay.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/training/learning_rate_decay.py b/tensorflow/python/training/learning_rate_decay.py
index 51190264e8..fd195a7965 100644
--- a/tensorflow/python/training/learning_rate_decay.py
+++ b/tensorflow/python/training/learning_rate_decay.py
@@ -356,7 +356,15 @@ def natural_exp_decay(learning_rate,
   The function returns the decayed learning rate.  It is computed as:
 
   ```python
-  decayed_learning_rate = learning_rate * exp(-decay_rate * global_step)
+  decayed_learning_rate = learning_rate * exp(-decay_rate * global_step /
+  decay_step)
+  ```
+
+  or, if `staircase` is `True`, as:
+
+  ```python
+  decayed_learning_rate = learning_rate * exp(-decay_rate * floor(global_step /
+  decay_step))
   ```
 
   Example: decay exponentially with a base of 0.96:
@@ -365,8 +373,10 @@ def natural_exp_decay(learning_rate,
   ...
   global_step = tf.Variable(0, trainable=False)
   learning_rate = 0.1
+  decay_steps = 5
   k = 0.5
-  learning_rate = tf.train.exponential_time_decay(learning_rate, global_step, k)
+  learning_rate = tf.train.natural_exp_decay(learning_rate, global_step,
+                                             decay_steps, k)
 
   # Passing global_step to minimize() will increment it at each step.
   learning_step = (
-- 
cgit v1.2.3


From 20a3f82b92cec4e31eea8f59f609961cbc3a1454 Mon Sep 17 00:00:00 2001
From: Michael Kuperstein 
Date: Tue, 24 Jul 2018 16:33:33 -0700
Subject: [XLA] Fix up table formatting in Map semantics.

PiperOrigin-RevId: 205912552
---
 tensorflow/docs_src/performance/xla/operation_semantics.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md
index 26a7b9e42c..fe9afc4ecb 100644
--- a/tensorflow/docs_src/performance/xla/operation_semantics.md
+++ b/tensorflow/docs_src/performance/xla/operation_semantics.md
@@ -1334,7 +1334,7 @@ See also
 | Arguments         | Type                   | Semantics                      |
 | ----------------- | ---------------------- | ------------------------------ |
 | `operands`        | sequence of N `XlaOp`s | N arrays of types T_0..T_{N-1} |
-| `computation`     | `XlaComputation`        | computation of type `T_0, T_1, |
+| `computation`     | `XlaComputation`       | computation of type `T_0, T_1, |
 :                   :                        : ..., T_{N + M -1} -> S` with N :
 :                   :                        : parameters of type T and M of  :
 :                   :                        : arbitrary type                 :
-- 
cgit v1.2.3


From 55c5f1289845649ee6e13bd90bc51e113f74c143 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 16:38:02 -0700
Subject: Tabularized the VLOGs printing per-op execution times to make it
 easier to see what ops take much time.

PiperOrigin-RevId: 205913222
---
 tensorflow/core/grappler/costs/virtual_scheduler.cc | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index 7f68272950..6a1b0aebfa 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/util/device_name_utils.h"
 
@@ -858,8 +859,9 @@ Costs VirtualScheduler::Summary() const {
     const auto& memory_cost = op_cost_pair.second.memory_time.count();
     const bool is_op_cost_accurate = !op_cost_pair.second.inaccurate;
     if (cost) {  // Skip printing out zero-cost ops.
-      VLOG(1) << " + " << op << " : " << (is_op_cost_accurate ? "" : "~")
-              << cost << " / " << compute_cost << " / " << memory_cost;
+      VLOG(1) << strings::Printf(" + %30s : %c %10ld / %10ld / %10ld",
+                                 op.c_str(), (is_op_cost_accurate ? ' ' : '~'),
+                                 cost, compute_cost, memory_cost);
     }
   }
 
@@ -934,9 +936,11 @@ Costs VirtualScheduler::Summary() const {
                                : 0.0;
       if (cost || mem_usage_percent > 1.0) {
         // Print out only non-zero cost ops or ops with > 1% memory usage.
-        VLOG(1) << " + " << op << " : " << (is_op_cost_accurate ? "" : "~")
-                << cost << " / " << compute_cost << " / " << memory_cost << " ("
-                << strings::HumanReadableNumBytes(op_mem_usage) << " ["
+        VLOG(1) << strings::Printf(" + %30s : %c %10ld / %10ld / %10ld",
+                                   op.c_str(),
+                                   (is_op_cost_accurate ? ' ' : '~'), cost,
+                                   compute_cost, memory_cost)
+                << " (" << strings::HumanReadableNumBytes(op_mem_usage) << " ["
                 << mem_usage_percent << "%] "
                 << (persisent_ops.count(op) > 0 ? ": persistent op)" : ")");
       }
-- 
cgit v1.2.3


From 4c161d7306eb934232e3fe65de2c31c3bb7cf875 Mon Sep 17 00:00:00 2001
From: Zhenyu Tan 
Date: Tue, 24 Jul 2018 16:50:28 -0700
Subject: use parameterized test in rmsprop

PiperOrigin-RevId: 205914985
---
 tensorflow/contrib/optimizer_v2/BUILD              |   1 +
 tensorflow/contrib/optimizer_v2/rmsprop_test.py    | 718 ++++++++++-----------
 tensorflow/python/framework/test_util.py           |  73 +++
 .../python/keras/layers/cudnn_recurrent_test.py    |   4 +-
 tensorflow/python/keras/testing_utils.py           |  73 ---
 5 files changed, 435 insertions(+), 434 deletions(-)

diff --git a/tensorflow/contrib/optimizer_v2/BUILD b/tensorflow/contrib/optimizer_v2/BUILD
index 5225ecc14f..3ba3ee29ec 100644
--- a/tensorflow/contrib/optimizer_v2/BUILD
+++ b/tensorflow/contrib/optimizer_v2/BUILD
@@ -193,6 +193,7 @@ cuda_py_test(
     srcs = ["rmsprop_test.py"],
     additional_deps = [
         ":training",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:embedding_ops",
         "//tensorflow/python:framework",
diff --git a/tensorflow/contrib/optimizer_v2/rmsprop_test.py b/tensorflow/contrib/optimizer_v2/rmsprop_test.py
index ed68f6afbf..dc23ef241a 100644
--- a/tensorflow/contrib/optimizer_v2/rmsprop_test.py
+++ b/tensorflow/contrib/optimizer_v2/rmsprop_test.py
@@ -19,15 +19,16 @@ from __future__ import division
 from __future__ import print_function
 
 import copy
-import itertools
 import math
 
+from absl.testing import parameterized
 import numpy as np
 
 from tensorflow.contrib.optimizer_v2 import rmsprop
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -48,13 +49,8 @@ _TEST_PARAM_VALUES = [
     [0.5, 0.95, 0.9, 1e-5, True, False],
 ]
 
-_TESTPARAMS = [
-    [data_type] + values
-    for data_type, values in itertools.product(_DATA_TYPES, _TEST_PARAM_VALUES)
-]
-
 
-class RMSPropOptimizerTest(test.TestCase):
+class RMSPropOptimizerTest(test.TestCase, parameterized.TestCase):
 
   def _rmsprop_update_numpy(self, var, g, mg, rms, mom, lr, decay, momentum,
                             epsilon, centered):
@@ -87,362 +83,366 @@ class RMSPropOptimizerTest(test.TestCase):
       var_t[gindex] = var[gindex] - mom_t[gindex]
     return var_t, mg_t, rms_t, mom_t
 
-  def testDense(self):
-    # TODO(yori): Use ParameterizedTest when available
-    for (dtype, learning_rate, decay, momentum,
-         epsilon, centered, use_resource) in _TESTPARAMS:
-      with self.test_session(use_gpu=True):
-        # Initialize variables for numpy implementation.
-        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
-        grads0_np = np.array([0.1, 0.2], dtype=dtype.as_numpy_dtype)
-        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
-        grads1_np = np.array([0.01, 0.2], dtype=dtype.as_numpy_dtype)
-
-        if use_resource:
-          var0 = resource_variable_ops.ResourceVariable(var0_np)
-          var1 = resource_variable_ops.ResourceVariable(var1_np)
-        else:
-          var0 = variables.Variable(var0_np)
-          var1 = variables.Variable(var1_np)
-        grads0 = constant_op.constant(grads0_np)
-        grads1 = constant_op.constant(grads1_np)
-        opt = rmsprop.RMSPropOptimizer(
-            learning_rate=learning_rate,
-            decay=decay,
-            momentum=momentum,
-            epsilon=epsilon,
-            centered=centered)
-
-        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        variables.global_variables_initializer().run()
-
-        mg0 = opt.get_slot(var0, "mg")
-        self.assertEqual(mg0 is not None, centered)
-        mg1 = opt.get_slot(var1, "mg")
-        self.assertEqual(mg1 is not None, centered)
-        rms0 = opt.get_slot(var0, "rms")
-        self.assertTrue(rms0 is not None)
-        rms1 = opt.get_slot(var1, "rms")
-        self.assertTrue(rms1 is not None)
-        mom0 = opt.get_slot(var0, "momentum")
-        self.assertTrue(mom0 is not None)
-        mom1 = opt.get_slot(var1, "momentum")
-        self.assertTrue(mom1 is not None)
-
-        mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
-        mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
-        rms0_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
-        rms1_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
-        mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
-        mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
-
-        # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
-
-        # Run 4 steps of RMSProp
-        for _ in range(1, 5):
-          update.run()
-
-          var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
-              var0_np, grads0_np, mg0_np, rms0_np, mom0_np, learning_rate,
-              decay, momentum, epsilon, centered)
-          var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy(
-              var1_np, grads1_np, mg1_np, rms1_np, mom1_np, learning_rate,
-              decay, momentum, epsilon, centered)
-
-          # Validate updated params
-          if centered:
-            self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
-            self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
-          self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
-          self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
-          self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
-          self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
-
-  def testMinimizeSparseResourceVariable(self):
-    for dtype in [dtypes.float32, dtypes.float64]:
-      with self.test_session():
-        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
-        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
-        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
-        loss = pred * pred
-        sgd_op = rmsprop.RMSPropOptimizer(
-            learning_rate=1.0,
-            decay=0.0,
-            momentum=0.0,
-            epsilon=0.0,
-            centered=False).minimize(loss)
-        variables.global_variables_initializer().run()
-        # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
-        # Run 1 step of sgd
-        sgd_op.run()
-        # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[0., 1.]], var0.eval(), atol=0.01)
-
-  def testMinimizeSparseResourceVariableCentered(self):
-    for dtype in [dtypes.float32, dtypes.float64]:
-      with self.test_session():
-        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
-        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
-        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
-        loss = pred * pred
-        sgd_op = rmsprop.RMSPropOptimizer(
-            learning_rate=1.0,
-            decay=0.0,
-            momentum=0.0,
-            epsilon=1.0,
-            centered=True).minimize(loss)
-        variables.global_variables_initializer().run()
-        # Fetch params to validate initial values
-        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
-        # Run 1 step of sgd
-        sgd_op.run()
-        # Validate updated params
-        self.assertAllCloseAccordingToType(
-            [[-111, -138]], var0.eval(), atol=0.01)
-
-  def testSparse(self):
-    # TODO(yori): Use ParameterizedTest when available
-    for (dtype, learning_rate, decay,
-         momentum, epsilon, centered, _) in _TESTPARAMS:
-      with self.test_session(use_gpu=True):
-        # Initialize variables for numpy implementation.
-        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
-        grads0_np = np.array([0.1], dtype=dtype.as_numpy_dtype)
-        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
-        grads1_np = np.array([0.01], dtype=dtype.as_numpy_dtype)
-
+  @parameterized.named_parameters(
+      *test_util.generate_combinations_with_testcase_name(
+          dtype=_DATA_TYPES, param_value=_TEST_PARAM_VALUES))
+  def testDense(self, dtype, param_value):
+    (learning_rate, decay, momentum, epsilon, centered, use_resource) = tuple(
+        param_value)
+    with self.test_session(use_gpu=True):
+      # Initialize variables for numpy implementation.
+      var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+      grads0_np = np.array([0.1, 0.2], dtype=dtype.as_numpy_dtype)
+      var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+      grads1_np = np.array([0.01, 0.2], dtype=dtype.as_numpy_dtype)
+
+      if use_resource:
+        var0 = resource_variable_ops.ResourceVariable(var0_np)
+        var1 = resource_variable_ops.ResourceVariable(var1_np)
+      else:
         var0 = variables.Variable(var0_np)
         var1 = variables.Variable(var1_np)
-        grads0_np_indices = np.array([0], dtype=np.int32)
-        grads0 = ops.IndexedSlices(
-            constant_op.constant(grads0_np),
-            constant_op.constant(grads0_np_indices), constant_op.constant([1]))
-        grads1_np_indices = np.array([1], dtype=np.int32)
-        grads1 = ops.IndexedSlices(
-            constant_op.constant(grads1_np),
-            constant_op.constant(grads1_np_indices), constant_op.constant([1]))
-        opt = rmsprop.RMSPropOptimizer(
-            learning_rate=learning_rate,
-            decay=decay,
-            momentum=momentum,
-            epsilon=epsilon,
-            centered=centered)
-        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        variables.global_variables_initializer().run()
-
-        mg0 = opt.get_slot(var0, "mg")
-        self.assertEqual(mg0 is not None, centered)
-        mg1 = opt.get_slot(var1, "mg")
-        self.assertEqual(mg1 is not None, centered)
-        rms0 = opt.get_slot(var0, "rms")
-        self.assertTrue(rms0 is not None)
-        rms1 = opt.get_slot(var1, "rms")
-        self.assertTrue(rms1 is not None)
-        mom0 = opt.get_slot(var0, "momentum")
-        self.assertTrue(mom0 is not None)
-        mom1 = opt.get_slot(var1, "momentum")
-        self.assertTrue(mom1 is not None)
-
-        mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
-        mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
-        rms0_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
-        rms1_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
-        mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
-        mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
-
-        # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
-
-        # Run 4 steps of RMSProp
-        for _ in range(1, 5):
-          update.run()
-
-          var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy(
-              var0_np, grads0_np_indices, grads0_np, mg0_np, rms0_np, mom0_np,
-              learning_rate, decay, momentum, epsilon, centered)
-          var1_np, mg1_np, rms1_np, mom1_np = self._sparse_rmsprop_update_numpy(
-              var1_np, grads1_np_indices, grads1_np, mg1_np, rms1_np, mom1_np,
-              learning_rate, decay, momentum, epsilon, centered)
-
-          # Validate updated params
-          if centered:
-            self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
-            self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
-          self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
-          self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
-          self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
-          self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
-          self.assertAllCloseAccordingToType(var0_np, var0.eval())
-          self.assertAllCloseAccordingToType(var1_np, var1.eval())
-
-  def testWithoutMomentum(self):
-    for dtype in [dtypes.half, dtypes.float32]:
-      with self.test_session(use_gpu=True):
-        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
-        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
-        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
-        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
-        opt = rmsprop.RMSPropOptimizer(
-            learning_rate=2.0, decay=0.9, momentum=0.0, epsilon=1.0)
-        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        variables.global_variables_initializer().run()
-
-        rms0 = opt.get_slot(var0, "rms")
-        self.assertTrue(rms0 is not None)
-        rms1 = opt.get_slot(var1, "rms")
-        self.assertTrue(rms1 is not None)
-        mom0 = opt.get_slot(var0, "momentum")
-        self.assertTrue(mom0 is not None)
-        mom1 = opt.get_slot(var1, "momentum")
-        self.assertTrue(mom1 is not None)
-
-        # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
-        # Step 1: the rms accumulators where 1. So we should see a normal
-        # update: v -= grad * learning_rate
-        update.run()
-        # Check the root mean square accumulators.
-        self.assertAllCloseAccordingToType(
-            np.array([0.901, 0.901]), rms0.eval())
-        self.assertAllCloseAccordingToType(
-            np.array([0.90001, 0.90001]), rms1.eval())
-        # Check the parameters.
-        self.assertAllCloseAccordingToType(
-            np.array([
-                1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)),
-                2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0))
-            ]), var0.eval())
-        self.assertAllCloseAccordingToType(
-            np.array([
-                3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)),
-                4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0))
-            ]), var1.eval())
-        # Step 2: the root mean square accumulators contain the previous update.
-        update.run()
-        # Check the rms accumulators.
-        self.assertAllCloseAccordingToType(
-            np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval())
-        self.assertAllCloseAccordingToType(
-            np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval())
-        # Check the parameters.
-        self.assertAllCloseAccordingToType(
-            np.array([
-                1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)) -
-                (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1.0)),
-                2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)) -
-                (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1.0))
-            ]), var0.eval())
-        self.assertAllCloseAccordingToType(
-            np.array([
-                3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) -
-                (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0)),
-                4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) -
-                (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0))
-            ]), var1.eval())
-
-  def testWithMomentum(self):
-    for dtype in [dtypes.half, dtypes.float32]:
-      with self.test_session(use_gpu=True):
-        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
-        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
-        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
-        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
-
-        opt = rmsprop.RMSPropOptimizer(
-            learning_rate=2.0, decay=0.9, momentum=0.5, epsilon=1e-5)
-        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
-        variables.global_variables_initializer().run()
-
-        rms0 = opt.get_slot(var0, "rms")
-        self.assertTrue(rms0 is not None)
-        rms1 = opt.get_slot(var1, "rms")
-        self.assertTrue(rms1 is not None)
-        mom0 = opt.get_slot(var0, "momentum")
-        self.assertTrue(mom0 is not None)
-        mom1 = opt.get_slot(var1, "momentum")
-        self.assertTrue(mom1 is not None)
-
-        # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
-        # Step 1: rms = 1, mom = 0. So we should see a normal
-        # update: v -= grad * learning_rate
+      grads0 = constant_op.constant(grads0_np)
+      grads1 = constant_op.constant(grads1_np)
+      opt = rmsprop.RMSPropOptimizer(
+          learning_rate=learning_rate,
+          decay=decay,
+          momentum=momentum,
+          epsilon=epsilon,
+          centered=centered)
+
+      update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+      variables.global_variables_initializer().run()
+
+      mg0 = opt.get_slot(var0, "mg")
+      self.assertEqual(mg0 is not None, centered)
+      mg1 = opt.get_slot(var1, "mg")
+      self.assertEqual(mg1 is not None, centered)
+      rms0 = opt.get_slot(var0, "rms")
+      self.assertIsNotNone(rms0)
+      rms1 = opt.get_slot(var1, "rms")
+      self.assertIsNotNone(rms1)
+      mom0 = opt.get_slot(var0, "momentum")
+      self.assertIsNotNone(mom0)
+      mom1 = opt.get_slot(var1, "momentum")
+      self.assertIsNotNone(mom1)
+
+      mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+      mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+      rms0_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
+      rms1_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
+      mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+      mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+
+      # Fetch params to validate initial values
+      self.assertAllClose([1.0, 2.0], var0.eval())
+      self.assertAllClose([3.0, 4.0], var1.eval())
+
+      # Run 4 steps of RMSProp
+      for _ in range(4):
         update.run()
-        # Check the root mean square accumulators.
-        self.assertAllCloseAccordingToType(
-            np.array([0.901, 0.901]), rms0.eval())
-        self.assertAllCloseAccordingToType(
-            np.array([0.90001, 0.90001]), rms1.eval())
-        # Check the momentum accumulators
-        self.assertAllCloseAccordingToType(
-            np.array([(0.1 * 2.0 / math.sqrt(0.901 + 1e-5)),
-                      (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))]), mom0.eval())
-        self.assertAllCloseAccordingToType(
-            np.array([(0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)),
-                      (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))]), mom1.eval())
-
-        # Check that the parameters.
-        self.assertAllCloseAccordingToType(
-            np.array([
-                1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)),
-                2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))
-            ]), var0.eval())
-        self.assertAllCloseAccordingToType(
-            np.array([
-                3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)),
-                4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))
-            ]), var1.eval())
-
-        # Step 2: the root mean square accumulators contain the previous update.
+
+        var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
+            var0_np, grads0_np, mg0_np, rms0_np, mom0_np, learning_rate,
+            decay, momentum, epsilon, centered)
+        var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy(
+            var1_np, grads1_np, mg1_np, rms1_np, mom1_np, learning_rate,
+            decay, momentum, epsilon, centered)
+
+        # Validate updated params
+        if centered:
+          self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
+          self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
+        self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
+        self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
+        self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
+        self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
+        self.assertAllCloseAccordingToType(var0_np, var0.eval())
+        self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  @parameterized.parameters([dtypes.float32, dtypes.float64])
+  def testMinimizeSparseResourceVariable(self, dtype):
+    with self.test_session():
+      var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+      x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+      pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+      loss = pred * pred
+      sgd_op = rmsprop.RMSPropOptimizer(
+          learning_rate=1.0,
+          decay=0.0,
+          momentum=0.0,
+          epsilon=0.0,
+          centered=False).minimize(loss)
+      variables.global_variables_initializer().run()
+      # Fetch params to validate initial values
+      self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+      # Run 1 step of sgd
+      sgd_op.run()
+      # Validate updated params
+      self.assertAllCloseAccordingToType(
+          [[0., 1.]], var0.eval(), atol=0.01)
+
+  @parameterized.parameters([dtypes.float32, dtypes.float64])
+  def testMinimizeSparseResourceVariableCentered(self, dtype):
+    with self.test_session():
+      var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+      x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+      pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+      loss = pred * pred
+      sgd_op = rmsprop.RMSPropOptimizer(
+          learning_rate=1.0,
+          decay=0.0,
+          momentum=0.0,
+          epsilon=1.0,
+          centered=True).minimize(loss)
+      variables.global_variables_initializer().run()
+      # Fetch params to validate initial values
+      self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+      # Run 1 step of sgd
+      sgd_op.run()
+      # Validate updated params
+      self.assertAllCloseAccordingToType(
+          [[-111, -138]], var0.eval(), atol=0.01)
+
+  @parameterized.named_parameters(
+      *test_util.generate_combinations_with_testcase_name(
+          dtype=_DATA_TYPES, param_value=_TEST_PARAM_VALUES))
+  def testSparse(self, dtype, param_value):
+    (learning_rate, decay, momentum, epsilon, centered, _) = tuple(
+        param_value)
+    with self.test_session(use_gpu=True):
+      # Initialize variables for numpy implementation.
+      var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+      grads0_np = np.array([0.1], dtype=dtype.as_numpy_dtype)
+      var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+      grads1_np = np.array([0.01], dtype=dtype.as_numpy_dtype)
+
+      var0 = variables.Variable(var0_np)
+      var1 = variables.Variable(var1_np)
+      grads0_np_indices = np.array([0], dtype=np.int32)
+      grads0 = ops.IndexedSlices(
+          constant_op.constant(grads0_np),
+          constant_op.constant(grads0_np_indices), constant_op.constant([1]))
+      grads1_np_indices = np.array([1], dtype=np.int32)
+      grads1 = ops.IndexedSlices(
+          constant_op.constant(grads1_np),
+          constant_op.constant(grads1_np_indices), constant_op.constant([1]))
+      opt = rmsprop.RMSPropOptimizer(
+          learning_rate=learning_rate,
+          decay=decay,
+          momentum=momentum,
+          epsilon=epsilon,
+          centered=centered)
+      update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+      variables.global_variables_initializer().run()
+
+      mg0 = opt.get_slot(var0, "mg")
+      self.assertEqual(mg0 is not None, centered)
+      mg1 = opt.get_slot(var1, "mg")
+      self.assertEqual(mg1 is not None, centered)
+      rms0 = opt.get_slot(var0, "rms")
+      self.assertIsNotNone(rms0)
+      rms1 = opt.get_slot(var1, "rms")
+      self.assertIsNotNone(rms1)
+      mom0 = opt.get_slot(var0, "momentum")
+      self.assertIsNotNone(mom0)
+      mom1 = opt.get_slot(var1, "momentum")
+      self.assertIsNotNone(mom1)
+
+      mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+      mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+      rms0_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
+      rms1_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
+      mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+      mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+
+      # Fetch params to validate initial values
+      self.assertAllClose([1.0, 2.0], var0.eval())
+      self.assertAllClose([3.0, 4.0], var1.eval())
+
+      # Run 4 steps of RMSProp
+      for _ in range(4):
         update.run()
-        # Check the rms accumulators.
-        self.assertAllCloseAccordingToType(
-            np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval())
-        self.assertAllCloseAccordingToType(
-            np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval())
-        self.assertAllCloseAccordingToType(
-            np.array([
-                0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
-                (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5)),
-                0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
-                (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5))
-            ]), mom0.eval())
-        self.assertAllCloseAccordingToType(
-            np.array([
-                0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
-                (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5)),
-                0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
-                (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5))
-            ]), mom1.eval())
-
-        # Check the parameters.
-        self.assertAllCloseAccordingToType(
-            np.array([
-                1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) -
-                (0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
-                 (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5))),
-                2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) -
-                (0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
-                 (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5)))
-            ]), var0.eval())
-
-        self.assertAllCloseAccordingToType(
-            np.array([
-                3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) -
-                (0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
-                 (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5))),
-                4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) -
-                (0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
-                 (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5)))
-            ]), var1.eval())
+
+        var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy(
+            var0_np, grads0_np_indices, grads0_np, mg0_np, rms0_np, mom0_np,
+            learning_rate, decay, momentum, epsilon, centered)
+        var1_np, mg1_np, rms1_np, mom1_np = self._sparse_rmsprop_update_numpy(
+            var1_np, grads1_np_indices, grads1_np, mg1_np, rms1_np, mom1_np,
+            learning_rate, decay, momentum, epsilon, centered)
+
+        # Validate updated params
+        if centered:
+          self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
+          self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
+        self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
+        self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
+        self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
+        self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
+        self.assertAllCloseAccordingToType(var0_np, var0.eval())
+        self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  @parameterized.parameters(_DATA_TYPES)
+  def testWithoutMomentum(self, dtype):
+    with self.test_session(use_gpu=True):
+      var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+      var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+      grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+      grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+      opt = rmsprop.RMSPropOptimizer(
+          learning_rate=2.0, decay=0.9, momentum=0.0, epsilon=1.0)
+      update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+      variables.global_variables_initializer().run()
+
+      rms0 = opt.get_slot(var0, "rms")
+      self.assertIsNotNone(rms0)
+      rms1 = opt.get_slot(var1, "rms")
+      self.assertIsNotNone(rms1)
+      mom0 = opt.get_slot(var0, "momentum")
+      self.assertIsNotNone(mom0)
+      mom1 = opt.get_slot(var1, "momentum")
+      self.assertIsNotNone(mom1)
+
+      # Fetch params to validate initial values
+      self.assertAllClose([1.0, 2.0], var0.eval())
+      self.assertAllClose([3.0, 4.0], var1.eval())
+      # Step 1: the rms accumulators where 1. So we should see a normal
+      # update: v -= grad * learning_rate
+      update.run()
+      # Check the root mean square accumulators.
+      self.assertAllCloseAccordingToType(
+          np.array([0.901, 0.901]), rms0.eval())
+      self.assertAllCloseAccordingToType(
+          np.array([0.90001, 0.90001]), rms1.eval())
+      # Check the parameters.
+      self.assertAllCloseAccordingToType(
+          np.array([
+              1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)),
+              2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0))
+          ]), var0.eval())
+      self.assertAllCloseAccordingToType(
+          np.array([
+              3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)),
+              4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0))
+          ]), var1.eval())
+      # Step 2: the root mean square accumulators contain the previous update.
+      update.run()
+      # Check the rms accumulators.
+      self.assertAllCloseAccordingToType(
+          np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval())
+      self.assertAllCloseAccordingToType(
+          np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval())
+      # Check the parameters.
+      self.assertAllCloseAccordingToType(
+          np.array([
+              1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)) -
+              (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1.0)),
+              2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)) -
+              (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1.0))
+          ]), var0.eval())
+      self.assertAllCloseAccordingToType(
+          np.array([
+              3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) -
+              (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0)),
+              4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) -
+              (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0))
+          ]), var1.eval())
+
+  @parameterized.parameters(_DATA_TYPES)
+  def testWithMomentum(self, dtype):
+    with self.test_session(use_gpu=True):
+      var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+      var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+      grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+      grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+
+      opt = rmsprop.RMSPropOptimizer(
+          learning_rate=2.0, decay=0.9, momentum=0.5, epsilon=1e-5)
+      update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+      variables.global_variables_initializer().run()
+
+      rms0 = opt.get_slot(var0, "rms")
+      self.assertIsNotNone(rms0)
+      rms1 = opt.get_slot(var1, "rms")
+      self.assertIsNotNone(rms1)
+      mom0 = opt.get_slot(var0, "momentum")
+      self.assertIsNotNone(mom0)
+      mom1 = opt.get_slot(var1, "momentum")
+      self.assertIsNotNone(mom1)
+
+      # Fetch params to validate initial values
+      self.assertAllClose([1.0, 2.0], var0.eval())
+      self.assertAllClose([3.0, 4.0], var1.eval())
+      # Step 1: rms = 1, mom = 0. So we should see a normal
+      # update: v -= grad * learning_rate
+      update.run()
+      # Check the root mean square accumulators.
+      self.assertAllCloseAccordingToType(
+          np.array([0.901, 0.901]), rms0.eval())
+      self.assertAllCloseAccordingToType(
+          np.array([0.90001, 0.90001]), rms1.eval())
+      # Check the momentum accumulators
+      self.assertAllCloseAccordingToType(
+          np.array([(0.1 * 2.0 / math.sqrt(0.901 + 1e-5)),
+                    (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))]), mom0.eval())
+      self.assertAllCloseAccordingToType(
+          np.array([(0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)),
+                    (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))]), mom1.eval())
+
+      # Check that the parameters.
+      self.assertAllCloseAccordingToType(
+          np.array([
+              1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)),
+              2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))
+          ]), var0.eval())
+      self.assertAllCloseAccordingToType(
+          np.array([
+              3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)),
+              4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))
+          ]), var1.eval())
+
+      # Step 2: the root mean square accumulators contain the previous update.
+      update.run()
+      # Check the rms accumulators.
+      self.assertAllCloseAccordingToType(
+          np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval())
+      self.assertAllCloseAccordingToType(
+          np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval())
+      self.assertAllCloseAccordingToType(
+          np.array([
+              0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
+              (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5)),
+              0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
+              (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5))
+          ]), mom0.eval())
+      self.assertAllCloseAccordingToType(
+          np.array([
+              0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
+              (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5)),
+              0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
+              (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5))
+          ]), mom1.eval())
+
+      # Check the parameters.
+      self.assertAllCloseAccordingToType(
+          np.array([
+              1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) -
+              (0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
+               (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5))),
+              2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) -
+              (0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
+               (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5)))
+          ]), var0.eval())
+
+      self.assertAllCloseAccordingToType(
+          np.array([
+              3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) -
+              (0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
+               (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5))),
+              4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) -
+              (0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
+               (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5)))
+          ]), var1.eval())
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index d7e7a2c111..fc47b1cca5 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -20,6 +20,7 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
+from collections import OrderedDict
 import contextlib
 import gc
 import itertools
@@ -572,6 +573,78 @@ def assert_no_garbage_created(f):
   return decorator
 
 
+def _combine_named_parameters(**kwargs):
+  """Generate combinations based on its keyword arguments.
+
+  Two sets of returned combinations can be concatenated using +.  Their product
+  can be computed using `times()`.
+
+  Args:
+    **kwargs: keyword arguments of form `option=[possibilities, ...]`
+         or `option=the_only_possibility`.
+
+  Returns:
+    a list of dictionaries for each combination. Keys in the dictionaries are
+    the keyword argument names.  Each key has one value - one of the
+    corresponding keyword argument values.
+  """
+  if not kwargs:
+    return [OrderedDict()]
+
+  sort_by_key = lambda k: k[0][0]
+  kwargs = OrderedDict(sorted(kwargs.items(), key=sort_by_key))
+  first = list(kwargs.items())[0]
+
+  rest = dict(list(kwargs.items())[1:])
+  rest_combined = _combine_named_parameters(**rest)
+
+  key = first[0]
+  values = first[1]
+  if not isinstance(values, list):
+    values = [values]
+
+  combinations = [
+      OrderedDict(sorted(list(combined.items()) + [(key, v)], key=sort_by_key))
+      for v in values
+      for combined in rest_combined
+  ]
+  return combinations
+
+
+def generate_combinations_with_testcase_name(**kwargs):
+  """Generate combinations based on its keyword arguments using combine().
+
+  This function calls combine() and appends a testcase name to the list of
+  dictionaries returned. The 'testcase_name' key is a required for named
+  parameterized tests.
+
+  Args:
+    **kwargs: keyword arguments of form `option=[possibilities, ...]`
+         or `option=the_only_possibility`.
+
+  Returns:
+    a list of dictionaries for each combination. Keys in the dictionaries are
+    the keyword argument names.  Each key has one value - one of the
+    corresponding keyword argument values.
+  """
+  combinations = _combine_named_parameters(**kwargs)
+  named_combinations = []
+  for combination in combinations:
+    assert isinstance(combination, OrderedDict)
+    name = "".join([
+        "_{}_{}".format(
+            "".join(filter(str.isalnum, key)),
+            "".join(filter(str.isalnum, str(value))))
+        for key, value in combination.items()
+    ])
+    named_combinations.append(
+        OrderedDict(
+            list(combination.items()) + [("testcase_name",
+                                          "_test{}".format(name))]))
+
+  return named_combinations
+
+
 def run_all_in_graph_and_eager_modes(cls):
   """Execute all test methods in the given class with and without eager."""
   base_decorator = run_in_graph_and_eager_modes
diff --git a/tensorflow/python/keras/layers/cudnn_recurrent_test.py b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
index 8fd970239f..2ed0aa8f26 100644
--- a/tensorflow/python/keras/layers/cudnn_recurrent_test.py
+++ b/tensorflow/python/keras/layers/cudnn_recurrent_test.py
@@ -220,7 +220,7 @@ class CuDNNTest(test.TestCase, parameterized.TestCase):
         self.assertNotEqual(out4.max(), out5.max())
 
   @parameterized.named_parameters(
-      *testing_utils.generate_combinations_with_testcase_name(
+      *test_util.generate_combinations_with_testcase_name(
           rnn_type=['LSTM', 'GRU'], to_cudnn=[True, False],
           bidirectional=[True, False], implementation=[1, 2],
           model_nest_level=[1, 2], model_type=['seq', 'func']))
@@ -301,7 +301,7 @@ class CuDNNTest(test.TestCase, parameterized.TestCase):
     os.remove(fname)
 
   @parameterized.named_parameters(
-      *testing_utils.generate_combinations_with_testcase_name(
+      *test_util.generate_combinations_with_testcase_name(
           rnn_type=['LSTM', 'GRU'], to_cudnn=[True, False]))
   def test_load_weights_between_noncudnn_rnn_time_distributed(self, rnn_type,
                                                               to_cudnn):
diff --git a/tensorflow/python/keras/testing_utils.py b/tensorflow/python/keras/testing_utils.py
index 17aba7d86c..6e8ee06ff5 100644
--- a/tensorflow/python/keras/testing_utils.py
+++ b/tensorflow/python/keras/testing_utils.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from collections import OrderedDict
 import numpy as np
 
 from tensorflow.python import keras
@@ -185,75 +184,3 @@ def layer_test(layer_cls, kwargs=None, input_shape=None, input_dtype=None,
   # for further checks in the caller function
   return actual_output
 
-
-def _combine_named_parameters(**kwargs):
-  """Generate combinations based on its keyword arguments.
-
-  Two sets of returned combinations can be concatenated using +.  Their product
-  can be computed using `times()`.
-
-  Args:
-    **kwargs: keyword arguments of form `option=[possibilities, ...]`
-         or `option=the_only_possibility`.
-
-  Returns:
-    a list of dictionaries for each combination. Keys in the dictionaries are
-    the keyword argument names.  Each key has one value - one of the
-    corresponding keyword argument values.
-  """
-  if not kwargs:
-    return [OrderedDict()]
-
-  sort_by_key = lambda k: k[0][0]
-  kwargs = OrderedDict(sorted(kwargs.items(), key=sort_by_key))
-  first = list(kwargs.items())[0]
-
-  rest = dict(list(kwargs.items())[1:])
-  rest_combined = _combine_named_parameters(**rest)
-
-  key = first[0]
-  values = first[1]
-  if not isinstance(values, list):
-    values = [values]
-
-  combinations = [
-      OrderedDict(sorted(list(combined.items()) + [(key, v)], key=sort_by_key))
-      for v in values
-      for combined in rest_combined
-  ]
-  return combinations
-
-
-def generate_combinations_with_testcase_name(**kwargs):
-  """Generate combinations based on its keyword arguments using combine().
-
-  This function calls combine() and appends a testcase name to the list of
-  dictionaries returned. The 'testcase_name' key is a required for named
-  parameterized tests.
-
-  Args:
-    **kwargs: keyword arguments of form `option=[possibilities, ...]`
-         or `option=the_only_possibility`.
-
-  Returns:
-    a list of dictionaries for each combination. Keys in the dictionaries are
-    the keyword argument names.  Each key has one value - one of the
-    corresponding keyword argument values.
-  """
-  combinations = _combine_named_parameters(**kwargs)
-  named_combinations = []
-  for combination in combinations:
-    assert isinstance(combination, OrderedDict)
-    name = ''.join([
-        '_{}_{}'.format(
-            ''.join(filter(str.isalnum, key)),
-            ''.join(filter(str.isalnum, str(value))))
-        for key, value in combination.items()
-    ])
-    named_combinations.append(
-        OrderedDict(
-            list(combination.items()) + [('testcase_name',
-                                          '_test{}'.format(name))]))
-
-  return named_combinations
-
-- 
cgit v1.2.3


From 0cf2c612e5e6ff8c5026011e8186056801def747 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 17:07:15 -0700
Subject: Keras ReLU Consolidation

Consolidate functionality of ThresholdedReLU and LeakyReLU layers into ReLU
layer

PiperOrigin-RevId: 205917439
---
 tensorflow/python/keras/activations.py             | 22 +++++---
 tensorflow/python/keras/backend.py                 | 32 ++++++++++--
 tensorflow/python/keras/backend_test.py            | 61 ++++++++++++++++++++++
 .../python/keras/layers/advanced_activations.py    | 37 ++++++++++---
 .../keras/layers/advanced_activations_test.py      |  8 +++
 .../api/golden/tensorflow.keras.activations.pbtxt  |  2 +-
 .../api/golden/tensorflow.keras.backend.pbtxt      |  2 +-
 .../golden/tensorflow.keras.layers.-re-l-u.pbtxt   |  2 +-
 8 files changed, 144 insertions(+), 22 deletions(-)

diff --git a/tensorflow/python/keras/activations.py b/tensorflow/python/keras/activations.py
index f608dea430..99645de736 100644
--- a/tensorflow/python/keras/activations.py
+++ b/tensorflow/python/keras/activations.py
@@ -128,20 +128,26 @@ def softsign(x):
 
 
 @tf_export('keras.activations.relu')
-def relu(x, alpha=0., max_value=None):
+def relu(x, alpha=0., max_value=None, threshold=0):
   """Rectified Linear Unit.
 
+  With default values, it returns element-wise `max(x, 0)`.
+
+  Otherwise, it follows:
+  `f(x) = max_value` for `x >= max_value`,
+  `f(x) = x` for `threshold <= x < max_value`,
+  `f(x) = alpha * (x - threshold)` otherwise.
+
   Arguments:
-      x: Input tensor.
-      alpha: Slope of the negative part. Defaults to zero.
-      max_value: Maximum value for the output.
+      x: A tensor or variable.
+      alpha: A scalar, slope of negative section (default=`0.`).
+      max_value: float. Saturation threshold.
+      threshold: float. Threshold value for thresholded activation.
 
   Returns:
-      The (leaky) rectified linear unit activation: `x` if `x > 0`,
-        `alpha * x` if `x < 0`. If `max_value` is defined, the result
-        is truncated to this value.
+      A tensor.
   """
-  return K.relu(x, alpha=alpha, max_value=max_value)
+  return K.relu(x, alpha=alpha, max_value=max_value, threshold=threshold)
 
 
 @tf_export('keras.activations.tanh')
diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py
index 333f927d2f..38794f1612 100644
--- a/tensorflow/python/keras/backend.py
+++ b/tensorflow/python/keras/backend.py
@@ -3372,26 +3372,48 @@ def in_test_phase(x, alt, training=None):
 
 
 @tf_export('keras.backend.relu')
-def relu(x, alpha=0., max_value=None):
+def relu(x, alpha=0., max_value=None, threshold=0):
   """Rectified linear unit.
 
   With default values, it returns element-wise `max(x, 0)`.
 
+  Otherwise, it follows:
+  `f(x) = max_value` for `x >= max_value`,
+  `f(x) = x` for `threshold <= x < max_value`,
+  `f(x) = alpha * (x - threshold)` otherwise.
+
   Arguments:
       x: A tensor or variable.
       alpha: A scalar, slope of negative section (default=`0.`).
-      max_value: Saturation threshold.
+      max_value: float. Saturation threshold.
+      threshold: float. Threshold value for thresholded activation.
 
   Returns:
       A tensor.
   """
+  clip_max = max_value is not None
+
   if alpha != 0.:
-    negative_part = nn.relu(-x)
-  x = nn.relu(x)
-  if max_value is not None:
+    if threshold != 0:
+      negative_part = nn.relu(-x + threshold)
+    else:
+      negative_part = nn.relu(-x)
+
+  if threshold != 0:
+    # computes x for x > threshold else 0
+    x = x * math_ops.cast(math_ops.greater(x, threshold), floatx())
+  elif max_value == 6:
+    # if no threshold, then can use nn.relu6 native TF op for performance
+    x = nn.relu6(x)
+    clip_max = False
+  else:
+    x = nn.relu(x)
+
+  if clip_max:
     max_value = _to_tensor(max_value, x.dtype.base_dtype)
     zero = _to_tensor(0., x.dtype.base_dtype)
     x = clip_ops.clip_by_value(x, zero, max_value)
+
   if alpha != 0.:
     alpha = _to_tensor(alpha, x.dtype.base_dtype)
     x -= alpha * negative_part
diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py
index 36478ea089..40e7910061 100644
--- a/tensorflow/python/keras/backend_test.py
+++ b/tensorflow/python/keras/backend_test.py
@@ -23,6 +23,7 @@ import scipy.sparse
 
 from tensorflow.python import keras
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -490,6 +491,66 @@ class BackendLinearAlgebraTest(test.TestCase):
                                        input_shape_a=(4, 7),
                                        input_shape_b=(4, 7))
 
+  def test_relu(self):
+    x = ops.convert_to_tensor([[-4, 0], [2, 7]], 'float32')
+    with self.test_session():
+      # standard relu
+      relu_op = keras.backend.relu(x)
+      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
+
+      # alpha
+      relu_op = keras.backend.relu(x, alpha=0.5)
+      self.assertAllClose(keras.backend.eval(relu_op), [[-2, 0], [2, 7]])
+
+      # max_value < some elements
+      relu_op = keras.backend.relu(x, max_value=5)
+      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 5]])
+
+      # nn.relu6 used
+      relu_op = keras.backend.relu(x, max_value=6)
+      self.assertTrue('Relu6' in relu_op.name)  # uses tf.nn.relu6
+      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 6]])
+
+      # max value > 6
+      relu_op = keras.backend.relu(x, max_value=10)
+      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
+
+      # max value is float
+      relu_op = keras.backend.relu(x, max_value=4.3)
+      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 4.3]])
+
+      # max value == 0
+      relu_op = keras.backend.relu(x, max_value=0)
+      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 0]])
+
+      # alpha and max_value
+      relu_op = keras.backend.relu(x, alpha=0.25, max_value=3)
+      self.assertAllClose(keras.backend.eval(relu_op), [[-1, 0], [2, 3]])
+
+      # threshold
+      relu_op = keras.backend.relu(x, threshold=3)
+      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 7]])
+
+      # threshold is float
+      relu_op = keras.backend.relu(x, threshold=1.5)
+      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [2, 7]])
+
+      # threshold is negative
+      relu_op = keras.backend.relu(x, threshold=-5)
+      self.assertAllClose(keras.backend.eval(relu_op), [[-4, 0], [2, 7]])
+
+      # threshold and max_value
+      relu_op = keras.backend.relu(x, threshold=3, max_value=5)
+      self.assertAllClose(keras.backend.eval(relu_op), [[0, 0], [0, 5]])
+
+      # threshold and alpha
+      relu_op = keras.backend.relu(x, alpha=0.25, threshold=4)
+      self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 7]])
+
+      # threshold, alpha, and max_value
+      relu_op = keras.backend.relu(x, alpha=0.25, threshold=4, max_value=5)
+      self.assertAllClose(keras.backend.eval(relu_op), [[-2, -1], [-0.5, 5]])
+
 
 class BackendShapeOpsTest(test.TestCase):
 
diff --git a/tensorflow/python/keras/layers/advanced_activations.py b/tensorflow/python/keras/layers/advanced_activations.py
index eba10da6f3..61ab69c16f 100644
--- a/tensorflow/python/keras/layers/advanced_activations.py
+++ b/tensorflow/python/keras/layers/advanced_activations.py
@@ -284,6 +284,13 @@ class Softmax(Layer):
 class ReLU(Layer):
   """Rectified Linear Unit activation function.
 
+  With default values, it returns element-wise `max(x, 0)`.
+
+  Otherwise, it follows:
+  `f(x) = max_value` for `x >= max_value`,
+  `f(x) = x` for `threshold <= x < max_value`,
+  `f(x) = negative_slope * (x - threshold)` otherwise.
+
   Input shape:
       Arbitrary. Use the keyword argument `input_shape`
       (tuple of integers, does not include the samples axis)
@@ -294,21 +301,39 @@ class ReLU(Layer):
 
   Arguments:
       max_value: float >= 0. Maximum activation value.
+      negative_slope: float >= 0. Negative slope coefficient.
+      threshold: float. Threshold value for thresholded activation.
   """
 
-  def __init__(self, max_value=None, **kwargs):
+  def __init__(self, max_value=None, negative_slope=0, threshold=0, **kwargs):
     super(ReLU, self).__init__(**kwargs)
-    self.support_masking = True
-    self.max_value = K.cast_to_floatx(max_value)
-    if self.max_value < 0.:
+    if max_value is not None and max_value < 0.:
       raise ValueError('max_value of Relu layer '
                        'cannot be negative value: ' + str(max_value))
+    if negative_slope < 0.:
+      raise ValueError('negative_slope of Relu layer '
+                       'cannot be negative value: ' + str(negative_slope))
+
+    self.support_masking = True
+    self.max_value = K.cast_to_floatx(max_value)
+    self.negative_slope = K.cast_to_floatx(negative_slope)
+    self.threshold = K.cast_to_floatx(threshold)
 
   def call(self, inputs):
-    return activations.relu(inputs, max_value=self.max_value)
+    # alpha is used for leaky relu slope in activations instead of
+    # negative_slope.
+    return activations.relu(
+        inputs,
+        alpha=self.negative_slope,
+        max_value=self.max_value,
+        threshold=self.threshold)
 
   def get_config(self):
-    config = {'max_value': self.max_value}
+    config = {
+        'max_value': self.max_value,
+        'negative_slope': self.negative_slope,
+        'threshold': self.threshold
+    }
     base_config = super(ReLU, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 
diff --git a/tensorflow/python/keras/layers/advanced_activations_test.py b/tensorflow/python/keras/layers/advanced_activations_test.py
index 9e1f15b1bc..53c1baa2bb 100644
--- a/tensorflow/python/keras/layers/advanced_activations_test.py
+++ b/tensorflow/python/keras/layers/advanced_activations_test.py
@@ -75,6 +75,14 @@ class AdvancedActivationsTest(test.TestCase):
         testing_utils.layer_test(keras.layers.ReLU,
                                  kwargs={'max_value': -10},
                                  input_shape=(2, 3, 4))
+    with self.assertRaisesRegexp(
+        ValueError,
+        'negative_slope of Relu layer cannot be negative value: -2'):
+      with self.test_session():
+        testing_utils.layer_test(
+            keras.layers.ReLU,
+            kwargs={'negative_slope': -2},
+            input_shape=(2, 3, 4))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.activations.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.activations.pbtxt
index 2cd83baf65..2e9de9ebb2 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.activations.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.activations.pbtxt
@@ -22,7 +22,7 @@ tf_module {
   }
   member_method {
     name: "relu"
-    argspec: "args=[\'x\', \'alpha\', \'max_value\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], "
+    argspec: "args=[\'x\', \'alpha\', \'max_value\', \'threshold\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\', \'0\'], "
   }
   member_method {
     name: "selu"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt
index fddac63b78..126ce8db6a 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt
@@ -366,7 +366,7 @@ tf_module {
   }
   member_method {
     name: "relu"
-    argspec: "args=[\'x\', \'alpha\', \'max_value\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\'], "
+    argspec: "args=[\'x\', \'alpha\', \'max_value\', \'threshold\'], varargs=None, keywords=None, defaults=[\'0.0\', \'None\', \'0\'], "
   }
   member_method {
     name: "repeat"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-re-l-u.pbtxt
index c00fa79adf..4d3de58bd1 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-re-l-u.pbtxt
@@ -82,7 +82,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'max_value\'], varargs=None, keywords=kwargs, defaults=[\'None\'], "
+    argspec: "args=[\'self\', \'max_value\', \'negative_slope\', \'threshold\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'0\', \'0\'], "
   }
   member_method {
     name: "add_loss"
-- 
cgit v1.2.3


From b3d6fd048c09acca275576873d69a49f54efcafd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 17:17:56 -0700
Subject: Added QuantizedMatMul to OpLevelCostEstimator.

PiperOrigin-RevId: 205918819
---
 .../core/grappler/costs/op_level_cost_estimator.cc | 135 +++++++++++----------
 1 file changed, 71 insertions(+), 64 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index d34eecd009..5b303f6ccb 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -65,6 +65,7 @@ constexpr char kAvgPool[] = "AvgPool";
 constexpr char kAvgPoolGrad[] = "AvgPoolGrad";
 constexpr char kFusedBatchNorm[] = "FusedBatchNorm";
 constexpr char kFusedBatchNormGrad[] = "FusedBatchNormGrad";
+constexpr char kQuantizedMatMulV2[] = "QuantizedMatMulV2";
 
 static const Costs::Duration kMinComputeTime(1);
 
@@ -226,6 +227,7 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
       {kMatMul, wrap(&OpLevelCostEstimator::PredictMatMul)},
       {kSparseMatMul, wrap(&OpLevelCostEstimator::PredictMatMul)},
       {kBatchMatMul, wrap(&OpLevelCostEstimator::PredictBatchMatMul)},
+      {kQuantizedMatMulV2, wrap(&OpLevelCostEstimator::PredictMatMul)},
 
       {kNoOp, wrap(&OpLevelCostEstimator::PredictNoOp)},
       {kGuaranteeConst, wrap(&OpLevelCostEstimator::PredictNoOp)},
@@ -268,67 +270,70 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
       EIGEN_COST(scalar_product_op) + EIGEN_COST(scalar_max_op) +
       EIGEN_COST(scalar_min_op) + EIGEN_COST(scalar_round_op);
 
-  elementwise_ops_ = {// Unary ops alphabetically sorted
-                      {"Acos", EIGEN_COST(scalar_acos_op)},
-                      {"Asin", EIGEN_COST(scalar_asin_op)},
-                      {"Atan", EIGEN_COST(scalar_atan_op)},
-                      {"Atan2", EIGEN_COST(scalar_quotient_op) +
-                                    EIGEN_COST(scalar_atan_op)},
-                      {"Ceil", EIGEN_COST(scalar_ceil_op)},
-                      {"Cos", EIGEN_COST(scalar_cos_op)},
-                      {"Dequantize", EIGEN_COST(scalar_product_op)},
-                      {"Erf", 1},
-                      {"Erfc", 1},
-                      {"Exp", EIGEN_COST(scalar_exp_op)},
-                      {"Expm1", EIGEN_COST(scalar_expm1_op)},
-                      {"Floor", EIGEN_COST(scalar_floor_op)},
-                      {"Inv", EIGEN_COST(scalar_inverse_op)},
-                      {"InvGrad", 1},
-                      {"Lgamma", 1},
-                      {"Log", EIGEN_COST(scalar_log_op)},
-                      {"Log1p", EIGEN_COST(scalar_log1p_op)},
-                      {"Neg", EIGEN_COST(scalar_opposite_op)},
-                      {"QuantizeV2", quantize_v2_cost},
-                      {"Reciprocal", EIGEN_COST(scalar_inverse_op)},
-                      {"Rint", 1},
-                      {"Round", EIGEN_COST(scalar_round_op)},
-                      {"Rsqrt", EIGEN_COST(scalar_rsqrt_op)},
-                      {"Sqrt", EIGEN_COST(scalar_sqrt_op)},
-                      {"Square", EIGEN_COST(scalar_square_op)},
-                      {"Tanh", EIGEN_COST(scalar_tanh_op)},
-                      {"Relu", EIGEN_COST(scalar_max_op)},
-                      {"Sigmoid", EIGEN_COST(scalar_sigmoid_op)},
-                      {"Sign", EIGEN_COST(scalar_sign_op)},
-                      {"Sin", EIGEN_COST(scalar_sin_op)},
-                      {"Tan", EIGEN_COST(scalar_tan_op)},
-                      // Binary ops alphabetically sorted
-                      {"Add", EIGEN_COST(scalar_sum_op)},
-                      {"ApproximateEqual", 1},
-                      {"BiasAdd", EIGEN_COST(scalar_sum_op)},
-                      {"Div", EIGEN_COST(scalar_quotient_op)},
-                      {"Equal", 1},
-                      {"FloorDiv", EIGEN_COST(scalar_quotient_op)},
-                      {"FloorMod", EIGEN_COST(scalar_mod_op)},
-                      {"Greater", 1},
-                      {"GreaterEqual", 1},
-                      {"Less", 1},
-                      {"LessEqual", 1},
-                      {"LogicalAnd", EIGEN_COST(scalar_boolean_and_op)},
-                      {"LogicalNot", 1},
-                      {"LogicalOr", EIGEN_COST(scalar_boolean_or_op)},
-                      {"Maximum", EIGEN_COST(scalar_max_op)},
-                      {"Minimum", EIGEN_COST(scalar_min_op)},
-                      {"Mod", EIGEN_COST(scalar_mod_op)},
-                      {"Mul", EIGEN_COST(scalar_product_op)},
-                      {"NotEqual", 1},
-                      {"QuantizedAdd", EIGEN_COST(scalar_sum_op)},
-                      {"QuantizedMul", EIGEN_COST(scalar_product_op)},
-                      {"RealDiv", EIGEN_COST(scalar_quotient_op)},
-                      {"ReluGrad", EIGEN_COST(scalar_max_op)},
-                      {"SquareDifference", 1},
-                      {"Sub", EIGEN_COST(scalar_difference_op)},
-                      {"TruncateDiv", EIGEN_COST(scalar_quotient_op)},
-                      {"TruncateMod", EIGEN_COST(scalar_mod_op)}};
+  elementwise_ops_ = {
+      // Unary ops alphabetically sorted
+      {"Acos", EIGEN_COST(scalar_acos_op)},
+      {"Asin", EIGEN_COST(scalar_asin_op)},
+      {"Atan", EIGEN_COST(scalar_atan_op)},
+      {"Atan2", EIGEN_COST(scalar_quotient_op) +
+                    EIGEN_COST(scalar_atan_op)},
+      {"Ceil", EIGEN_COST(scalar_ceil_op)},
+      {"Cos", EIGEN_COST(scalar_cos_op)},
+      {"Dequantize", EIGEN_COST(scalar_product_op)},
+      {"Erf", 1},
+      {"Erfc", 1},
+      {"Exp", EIGEN_COST(scalar_exp_op)},
+      {"Expm1", EIGEN_COST(scalar_expm1_op)},
+      {"Floor", EIGEN_COST(scalar_floor_op)},
+      {"Inv", EIGEN_COST(scalar_inverse_op)},
+      {"InvGrad", 1},
+      {"Lgamma", 1},
+      {"Log", EIGEN_COST(scalar_log_op)},
+      {"Log1p", EIGEN_COST(scalar_log1p_op)},
+      {"Neg", EIGEN_COST(scalar_opposite_op)},
+      {"QuantizeV2", quantize_v2_cost},
+      {"Reciprocal", EIGEN_COST(scalar_inverse_op)},
+      {"Rint", 1},
+      {"Round", EIGEN_COST(scalar_round_op)},
+      {"Rsqrt", EIGEN_COST(scalar_rsqrt_op)},
+      {"Sqrt", EIGEN_COST(scalar_sqrt_op)},
+      {"Square", EIGEN_COST(scalar_square_op)},
+      {"Tanh", EIGEN_COST(scalar_tanh_op)},
+      {"Relu", EIGEN_COST(scalar_max_op)},
+      {"Sigmoid", EIGEN_COST(scalar_sigmoid_op)},
+      {"QuantizedSigmoid", EIGEN_COST(scalar_sigmoid_op)},
+      {"Sign", EIGEN_COST(scalar_sign_op)},
+      {"Sin", EIGEN_COST(scalar_sin_op)},
+      {"Tan", EIGEN_COST(scalar_tan_op)},
+      // Binary ops alphabetically sorted
+      {"Add", EIGEN_COST(scalar_sum_op)},
+      {"ApproximateEqual", 1},
+      {"BiasAdd", EIGEN_COST(scalar_sum_op)},
+      {"QuantizedBiasAdd", EIGEN_COST(scalar_sum_op)},
+      {"Div", EIGEN_COST(scalar_quotient_op)},
+      {"Equal", 1},
+      {"FloorDiv", EIGEN_COST(scalar_quotient_op)},
+      {"FloorMod", EIGEN_COST(scalar_mod_op)},
+      {"Greater", 1},
+      {"GreaterEqual", 1},
+      {"Less", 1},
+      {"LessEqual", 1},
+      {"LogicalAnd", EIGEN_COST(scalar_boolean_and_op)},
+      {"LogicalNot", 1},
+      {"LogicalOr", EIGEN_COST(scalar_boolean_or_op)},
+      {"Maximum", EIGEN_COST(scalar_max_op)},
+      {"Minimum", EIGEN_COST(scalar_min_op)},
+      {"Mod", EIGEN_COST(scalar_mod_op)},
+      {"Mul", EIGEN_COST(scalar_product_op)},
+      {"NotEqual", 1},
+      {"QuantizedAdd", EIGEN_COST(scalar_sum_op)},
+      {"QuantizedMul", EIGEN_COST(scalar_product_op)},
+      {"RealDiv", EIGEN_COST(scalar_quotient_op)},
+      {"ReluGrad", EIGEN_COST(scalar_max_op)},
+      {"SquareDifference", 1},
+      {"Sub", EIGEN_COST(scalar_difference_op)},
+      {"TruncateDiv", EIGEN_COST(scalar_quotient_op)},
+      {"TruncateMod", EIGEN_COST(scalar_mod_op)}};
 
 #undef EIGEN_COST
 
@@ -675,7 +680,7 @@ int64 OpLevelCostEstimator::CountMatMulOperations(
   }
 
   ops = m_dim * n_dim * k_dim * 2;
-  VLOG(1) << "Operations for Matmul" << ops;
+  VLOG(1) << "Operations for Matmul: " << ops;
 
   if (mat_mul != nullptr) {
     mat_mul->m = m_dim;
@@ -972,8 +977,10 @@ int64 OpLevelCostEstimator::CalculateTensorElementCount(
 
 int64 OpLevelCostEstimator::CalculateTensorSize(
     const OpInfo::TensorProperties& tensor, bool* found_unknown_shapes) const {
-  return CalculateTensorElementCount(tensor, found_unknown_shapes) *
-         DataTypeSize(BaseType(tensor.dtype()));
+  int64 count = CalculateTensorElementCount(tensor, found_unknown_shapes);
+  int size = DataTypeSize(BaseType(tensor.dtype()));
+  VLOG(2) << "Count: " << count << " DataTypeSize: " << size;
+  return count * size;
 }
 
 int64 OpLevelCostEstimator::CalculateInputSize(
-- 
cgit v1.2.3


From d2f3441c0c3d552293f5c64784261e267653484b Mon Sep 17 00:00:00 2001
From: Raymond Yuan 
Date: Tue, 24 Jul 2018 18:00:08 -0700
Subject: Added support for build on subclass models

PiperOrigin-RevId: 205923522
---
 tensorflow/python/keras/engine/base_layer.py      |  10 +-
 tensorflow/python/keras/engine/training.py        |  91 ++++++++
 tensorflow/python/keras/model_subclassing_test.py | 270 +++++++++++++++++++++-
 3 files changed, 363 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py
index e02792208b..b41f6ee03b 100644
--- a/tensorflow/python/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/engine/base_layer.py
@@ -723,9 +723,17 @@ class Layer(checkpointable.CheckpointableBase):
             self._dtype = input_list[0].dtype.base_dtype.name
           except AttributeError:
             pass
+
         if all(hasattr(x, 'shape') for x in input_list):
           input_shapes = nest.map_structure(lambda x: x.shape, inputs)
-        self.build(input_shapes)
+
+        if (not hasattr(self, '_is_graph_network') or
+            self.__class__.__name__ == 'Sequential'):
+          # Only if self is a layer or an instance of a sequential model do we
+          # need to build it.
+          self.build(input_shapes)
+        # We must set self.built since user defined build functions are not
+        # constrained to set self.built.
         self.built = True
 
       # Check input assumptions set after layer building, e.g. input shape.
diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py
index 1d4ab1fe37..4df739254b 100644
--- a/tensorflow/python/keras/engine/training.py
+++ b/tensorflow/python/keras/engine/training.py
@@ -27,6 +27,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras import backend as K
 from tensorflow.python.keras import losses
@@ -43,6 +44,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import optimizer as tf_optimizer_module
 from tensorflow.python.training.checkpointable import base as checkpointable
+from tensorflow.python.util import tf_inspect
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -560,6 +562,95 @@ class Model(Network):
     trainable_weights = self.trainable_weights
     self._collected_trainable_weights = trainable_weights
 
+  def build(self, input_shape):
+    """Build the model based on input shapes received.
+
+    This is to be used for subclassed models, which do not know at instantiation
+    time what their inputs look like.
+
+    Args:
+     input_shape: Single tuple, TensorShape, or list of shapes, where shapes
+         are tuples, integers, or TensorShapes.
+
+    Raises:
+      ValueError:
+        1. In case of invalid user-provided data (not of type tuple,
+           list, or TensorShape).
+        2. If the model requires call arguments that are agnostic
+           to the input shapes (positional or kwarg in call signature).
+        3. If not all layers were properly built.
+        4. If float type inputs are not supported within the layers.
+
+      In each of these cases, the user should build their model by calling it
+      on real tensor data.
+    """
+    if self._is_graph_network:
+      self.built = True
+      return
+
+    # If subclass network
+    if input_shape is None:
+      raise ValueError('Input shape must be defined when calling build on a '
+                       'model subclass network.')
+    valid_types = (tuple, list, tensor_shape.TensorShape)
+    if not isinstance(input_shape, valid_types):
+      raise ValueError('Specified input shape is not one of the valid types. '
+                       'Please specify a batch input shape of type tuple or '
+                       'list of input shapes. User provided '
+                       'input type: {}'.format(type(input_shape)))
+
+    def _generate_dummy_data_from_shape(shape):
+      if isinstance(shape, tensor_shape.TensorShape):
+        shape = shape.as_list()
+
+      # Replace Nones in input shape with dummy `1` value
+      shape = [x.value if isinstance(x, tensor_shape.Dimension) else x
+               for x in shape]
+      shape = [1 if x is None else x for x in shape]
+      return array_ops.ones(shape, dtype=K.floatx())
+
+    if input_shape and not self.inputs:
+      if isinstance(input_shape, list):
+        # List of input shapes
+        x = [_generate_dummy_data_from_shape(shape) for shape in input_shape]
+      else:
+        x = _generate_dummy_data_from_shape(input_shape)
+
+      kwargs = {}
+      num_call_args = len(tf_inspect.getargspec(self.call).args)
+      if self._expects_training_arg and num_call_args == 3:
+        # Has call signature of call(self, input, training)
+        kwargs['training'] = False
+      elif num_call_args > 2:
+        # Has invalid call signature of call(self, input, *args, **kwargs)
+        raise ValueError('Currently, you cannot build your model if it has '
+                         'positional or keyword arguments that are not '
+                         'inputs to the model, but are required for its '
+                         '`call` method. Instead, in order to instantiate '
+                         'and build your model, `call` your model on real '
+                         'tensor data with all expected call arguments.')
+
+      try:
+        self.call(x, **kwargs)
+      except (errors.InvalidArgumentError, TypeError):
+        raise ValueError('You cannot build your model by calling `build` '
+                         'if your layers do not support float type inputs. '
+                         'Instead, in order to instantiate and build your '
+                         'model, `call` your model on real tensor data (of '
+                         'the correct dtype).')
+
+    if self._layers:
+      self._track_layers(self._layers)
+    if self.layers:
+      for layer in self.layers:
+        if not layer.built:
+          raise ValueError('Layer: {} was not built in your model. Calling '
+                           '`build` manually on a subclassed model is only '
+                           'allowed for models with a static topology. '
+                           'In this case, you can build your model by '
+                           'calling it on real tensor data.'.format(layer))
+    self.built = True
+
   def _check_trainable_weights_consistency(self):
     """Check trainable weights count consistency.
 
diff --git a/tensorflow/python/keras/model_subclassing_test.py b/tensorflow/python/keras/model_subclassing_test.py
index 3ac4852eff..5fbc191e78 100644
--- a/tensorflow/python/keras/model_subclassing_test.py
+++ b/tensorflow/python/keras/model_subclassing_test.py
@@ -29,6 +29,8 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import test
 from tensorflow.python.training.checkpointable import data_structures
@@ -65,6 +67,22 @@ class SimpleTestModel(keras.Model):
     return self.dense2(x)
 
 
+class SimpleConvTestModel(keras.Model):
+
+  def __init__(self, num_classes=10):
+    super(SimpleConvTestModel, self).__init__(name='test_model')
+    self.num_classes = num_classes
+
+    self.conv1 = keras.layers.Conv2D(32, (3, 3), activation='relu')
+    self.flatten = keras.layers.Flatten()
+    self.dense1 = keras.layers.Dense(num_classes, activation='softmax')
+
+  def call(self, x):
+    x = self.conv1(x)
+    x = self.flatten(x)
+    return self.dense1(x)
+
+
 class MultiIOTestModel(keras.Model):
 
   def __init__(self, use_bn=False, use_dp=False, num_classes=(2, 3)):
@@ -173,6 +191,213 @@ def get_nested_model_3(input_dim, num_classes):
 
 class ModelSubclassingTest(test.TestCase):
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_invalid_input_shape_build(self):
+    num_classes = 2
+    input_dim = 50
+
+    model = SimpleTestModel(num_classes=num_classes,
+                            use_dp=True,
+                            use_bn=True)
+
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    with self.assertRaisesRegexp(
+        ValueError, 'input shape is not one of the valid types'):
+      model.build(input_shape=tensor_shape.Dimension(input_dim))
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_embed_dtype_with_subclass_build(self):
+    class Embedding(keras.layers.Layer):
+      """An Embedding layer."""
+
+      def __init__(self, vocab_size, embedding_dim, **kwargs):
+        super(Embedding, self).__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.embedding_dim = embedding_dim
+
+      def build(self, _):
+        self.embedding = self.add_variable(
+            'embedding_kernel',
+            shape=[self.vocab_size, self.embedding_dim],
+            dtype=np.float32,
+            initializer=init_ops.random_uniform_initializer(-0.1, 0.1),
+            trainable=True)
+
+      def call(self, x):
+        return embedding_ops.embedding_lookup(self.embedding, x)
+
+    class EmbedModel(keras.Model):
+
+      def __init__(self, vocab_size, embed_size):
+        super(EmbedModel, self).__init__()
+        self.embed1 = Embedding(vocab_size, embed_size)
+
+      def call(self, inputs):
+        return self.embed1(inputs)
+
+    model = EmbedModel(100, 20)
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    with self.assertRaisesRegexp(
+        ValueError, 'if your layers do not support float type inputs'):
+      model.build(input_shape=(35, 20))
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_single_time_step_rnn_build(self):
+    dim = 4
+    timesteps = 1
+    batch_input_shape = (None, timesteps, dim)
+    units = 3
+
+    class SimpleRNNModel(keras.Model):
+
+      def __init__(self):
+        super(SimpleRNNModel, self).__init__()
+        self.lstm = keras.layers.LSTM(units)
+
+      def call(self, inputs):
+        return self.lstm(inputs)
+
+    model = SimpleRNNModel()
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    model.build(batch_input_shape)
+    self.assertTrue(model.weights, ('Model should have weights now that it '
+                                    'has been properly built.'))
+    self.assertTrue(model.built, 'Model should be built after calling `build`.')
+    model(array_ops.ones((32, timesteps, dim)))
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_single_io_subclass_build(self):
+    num_classes = 2
+    input_dim = 50
+    batch_size = None
+
+    model = SimpleTestModel(num_classes=num_classes,
+                            use_dp=True,
+                            use_bn=True)
+
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    model.build(input_shape=(batch_size, input_dim))
+    self.assertTrue(model.weights, ('Model should have weights now that it '
+                                    'has been properly built.'))
+    self.assertTrue(model.built, 'Model should be built after calling `build`.')
+    model(array_ops.ones((32, input_dim)))
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_single_io_dimension_subclass_build(self):
+    num_classes = 2
+    input_dim = tensor_shape.Dimension(50)
+    batch_size = tensor_shape.Dimension(None)
+
+    model = SimpleTestModel(num_classes=num_classes,
+                            use_dp=True,
+                            use_bn=True)
+
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    model.build(input_shape=(batch_size, input_dim))
+    self.assertTrue(model.weights, ('Model should have weights now that it '
+                                    'has been properly built.'))
+    self.assertTrue(model.built, 'Model should be built after calling `build`.')
+    model(array_ops.ones((32, input_dim)))
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_multidim_io_subclass_build(self):
+    num_classes = 10
+    # Input size, e.g. image
+    batch_size = 32
+    input_shape = (32, 32, 3)
+
+    model = SimpleConvTestModel(num_classes)
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    batch_input_shape = (batch_size,) + input_shape
+    model.build(input_shape=batch_input_shape)
+    self.assertTrue(model.weights, ('Model should have weights now that it '
+                                    'has been properly built.'))
+    self.assertTrue(model.built, 'Model should be built after calling `build`.')
+
+    model(array_ops.ones(batch_input_shape))
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_tensorshape_io_subclass_build(self):
+    num_classes = 10
+    # Input size, e.g. image
+    batch_size = None
+    input_shape = (32, 32, 3)
+
+    model = SimpleConvTestModel(num_classes)
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    model.build(
+        input_shape=tensor_shape.TensorShape((batch_size,) + input_shape))
+    self.assertTrue(model.weights, ('Model should have weights now that it '
+                                    'has been properly built.'))
+    self.assertTrue(model.built, 'Model should be built after calling `build`.')
+
+    model(array_ops.ones((32,) + input_shape))
+
+  def test_subclass_save_model(self):
+    num_classes = 10
+    # Input size, e.g. image
+    batch_size = None
+    input_shape = (32, 32, 3)
+
+    model = SimpleConvTestModel(num_classes)
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    model.build(
+        input_shape=tensor_shape.TensorShape((batch_size,) + input_shape))
+    self.assertTrue(model.weights, ('Model should have weights now that it '
+                                    'has been properly built.'))
+    self.assertTrue(model.built, 'Model should be built after calling `build`.')
+    weights = model.get_weights()
+
+    tf_format_name = os.path.join(self.get_temp_dir(), 'ckpt')
+    model.save_weights(tf_format_name)
+    if h5py is not None:
+      hdf5_format_name = os.path.join(self.get_temp_dir(), 'weights.h5')
+      model.save_weights(hdf5_format_name)
+
+    model = SimpleConvTestModel(num_classes)
+    model.build(
+        input_shape=tensor_shape.TensorShape((batch_size,) + input_shape))
+    if h5py is not None:
+      model.load_weights(hdf5_format_name)
+      self.assertAllClose(weights, model.get_weights())
+    model.load_weights(tf_format_name)
+    self.assertAllClose(weights, model.get_weights())
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_multi_io_subclass_build(self):
+    batch_size = None
+    num_samples = 1000
+    input_dim = 50
+    model = MultiIOTestModel()
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    batch_input_shape = tensor_shape.TensorShape((batch_size, input_dim))
+    model.build(
+        input_shape=[batch_input_shape, batch_input_shape])
+    self.assertTrue(model.weights, ('Model should have weights now that it '
+                                    'has been properly built.'))
+    self.assertTrue(model.built, 'Model should be built after calling `build`.')
+    x1 = array_ops.ones((num_samples, input_dim))
+    x2 = array_ops.ones((num_samples, input_dim))
+    model([x1, x2])
+
   @test_util.run_in_graph_and_eager_modes
   def test_single_io_workflow_with_np_arrays(self):
     num_classes = 2
@@ -750,6 +975,16 @@ class CustomCallModel(keras.Model):
       return combined
 
 
+class TrainingNoDefaultModel(keras.Model):
+
+  def __init__(self):
+    super(TrainingNoDefaultModel, self).__init__()
+    self.dense1 = keras.layers.Dense(1)
+
+  def call(self, x, training):
+    return self.dense1(x)
+
+
 class CustomCallSignatureTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes
@@ -766,6 +1001,32 @@ class CustomCallSignatureTests(test.TestCase):
     output = model(first, second=second, training=False)
     self.assertAllClose(expected_output, self.evaluate(output))
 
+  @test_util.run_in_graph_and_eager_modes
+  def test_training_args_call_build(self):
+    input_dim = 2
+
+    model = TrainingNoDefaultModel()
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    model.build((None, input_dim))
+    self.assertTrue(model.weights, ('Model should have weights now that it '
+                                    'has been properly built.'))
+    self.assertTrue(model.built, 'Model should be built after calling `build`.')
+
+  @test_util.run_in_graph_and_eager_modes
+  def test_custom_call_kwargs_and_build(self):
+    first_input_shape = (2, 3)
+    second_input_shape = (2, 5)
+
+    model = CustomCallModel()
+    self.assertFalse(model.built, 'Model should not have been built')
+    self.assertFalse(model.weights, ('Model should have no weights since it '
+                                     'has not been built.'))
+    with self.assertRaisesRegexp(
+        ValueError, 'cannot build your model if it has positional'):
+      model.build(input_shape=[first_input_shape, second_input_shape])
+
   @test_util.run_in_graph_and_eager_modes
   def test_inputs_in_signature(self):
 
@@ -829,14 +1090,9 @@ class CustomCallSignatureTests(test.TestCase):
 
   def test_training_no_default(self):
 
-    class TrainingNoDefault(keras.Model):
-
-      def call(self, x, training):
-        return x
-
     with context.graph_mode():
-      model = TrainingNoDefault()
-      arg = array_ops.ones([])
+      model = TrainingNoDefaultModel()
+      arg = array_ops.ones([1, 1])
       model(arg, True)
       six.assertCountEqual(self, [arg], model.inputs)
 
-- 
cgit v1.2.3


From a5a4bb31c43bfef94bf8998ce07766f1597a7a02 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" 
Date: Tue, 24 Jul 2018 18:03:21 -0700
Subject: Internal change.

PiperOrigin-RevId: 205923892
---
 tensorflow/python/keras/BUILD        | 1 -
 tensorflow/python/kernel_tests/BUILD | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index a495d48545..df409d2aa5 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -732,7 +732,6 @@ py_test(
     size = "medium",
     srcs = ["preprocessing/image_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["nomsan"],  # TODO(b/110990716) reenable
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index db2e7e2c2a..adf97569ab 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2845,7 +2845,6 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
     ],
     shard_count = 20,
-    tags = ["nomsan"],  # TODO(b/110990716) reenable
 )
 
 cuda_py_test(
-- 
cgit v1.2.3